diff --git a/README.md b/README.md index 1674064..a938451 100644 --- a/README.md +++ b/README.md @@ -697,8 +697,10 @@ into `failures.json` (see "Backup layout"). `backup-metadata` fetches ML data in requests of up to 200 files. When a request fails, the error is logged, each of its files is written with the reason in an -`mlDataError` field instead of `mlData`, and the dump goes on. The exit code is -non-zero if any ML data request failed. +`mlDataError` field instead of `mlData`, and the dump goes on. A file whose ML +data does not decrypt, gunzip or parse is logged and written the same way, with +why in `mlDataError`. The exit code is non-zero if any file was written with +`mlDataError`. `helper fix-missing-thumbnails` regenerates thumbnails for JPEG images only, because the bundled decoder (`jpeg-js`) decodes only JPEG. A non-JPEG image @@ -763,8 +765,12 @@ A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as for has no `mlData`. The backup waits for the library's ML data fetch to finish before it writes the JSON files. If that fetch fails, each file whose ML data is not in the cache gets the reason in `mlDataError` instead and counts as failed, -and the next run fetches it again. The JSON files are rewritten on every run, so -ML data that arrived since the last run appears. +and the next run fetches it again. The same goes for a file with no ML data in +the cache whose ML data the server sent but quak could not decrypt, gunzip or +parse. Its `mlDataError` says why, and `failures.json` records it with an error +starting `ML data: `. quak keeps that reason in memory only and caches nothing +for the file, so the next run asks for its ML data again. The JSON files are +rewritten on every run, so ML data that arrived since the last run appears. A file's JSON holds, as `imageMetadata`, what `backup-metadata --exif` records from its original, or from a live photo's image: `format`, `width` and `height` @@ -978,6 +984,12 @@ resolves once that refresh (including its cache write), the ML data fetch and the precache fetches already running have all finished, so the cache directory can then be removed. +`lib.status().mlDataErrors` lists, as `{ fileID, error }`, each file whose ML +data the last ML data fetch got but could not decrypt, gunzip or parse; the +`fetchMLData` event with status `"done"` carries the same list. Such ML data is +not cached, so the next ML data fetch asks for it again. The reasons are kept in +memory only, never in the cache. + ### Default reads vs. fresh reads Default reads — `lib.albums`, `lib.photos`, `lib.timeline` — answer diff --git a/TODO.md b/TODO.md index c2dbf45..ac3dfd6 100644 --- a/TODO.md +++ b/TODO.md @@ -17,8 +17,7 @@ pre-1.0 # Next Step -Three bugs are open: https://git.eeqj.de/sneak/quak/issues/198, -https://git.eeqj.de/sneak/quak/issues/199 and +Two bugs are open: https://git.eeqj.de/sneak/quak/issues/199 and https://git.eeqj.de/sneak/quak/issues/205. The check of quak against a live Ente account, https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak has, so it waits on him. @@ -28,6 +27,16 @@ declares one. # Completed Steps +- 2026-10-16: ML data the server sends that does not decrypt, gunzip or parse is + no longer taken for no ML data (issue 198). `fetchMLDataBatch` returns why for + each such file beside the payloads. The library keeps those reasons from its + last ML data fetch in memory, not in the cache, and gives them in + `status().mlDataErrors` and in the `fetchMLData` event with status `"done"`. + `quak backup` writes the reason as `mlDataError` in the file's JSON when no ML + data is cached for it, and records the file in `failures.json` with an error + starting `ML data: `; `backup-metadata` writes `mlDataError` too. Both exit 1. + Tests serve ML data encrypted under a key that is not the file's. + - 2026-10-16: The `~/bin/quak` that `make install` builds retries a refused connection and a name that does not resolve, and the backup records either as transient (issue 206). Bun's `fetch` reports them with codes of its own, diff --git a/src/backup.ts b/src/backup.ts index 1c44dd4..c20ff56 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -203,6 +203,9 @@ export interface BackupLibrary { // A file's cached ML data, as `lib.mldata.forFile()` returns it, or // undefined when none is cached. mlData(fileID: number): Promise; + // Why the ML data the last ML data fetch got for a file did not decrypt, + // gunzip or parse, or undefined when it did or the fetch got none. + mlDataError(fileID: number): unknown; } type FailureClass = "transient" | "permanent" | "unknown"; @@ -813,19 +816,18 @@ const runLockedBackup = async ( // Phase 2: rebuild the derived views from the model. Sidecars first, for // every present original (this repairs stale ones), each with the file's // ML data once an ML data fetch has completed, and its image metadata. - // When the fetch fails, a file with no cached ML data gets the reason - // instead and is recorded as failed. The next run fetches its ML data - // again because none is cached. + // When the fetch got a file's ML data but could not decrypt, gunzip or + // parse it, or when the fetch failed, a file with no cached ML data gets + // the reason instead and is recorded as failed. The next run fetches its + // ML data again because none is cached. if (includeOriginals) { let mlDataFailure: unknown; - let mlDataError: string | undefined; try { log("Fetching ML data..."); await lib.fetchMLData(); } catch (err) { mlDataFailure = err; - mlDataError = describeError(err); - log(`FAILED ML data: ${mlDataError}`); + log(`FAILED ML data: ${describeError(err)}`); } for (const file of distinct.values()) { if (lock.lost) throw lock.lost; @@ -842,13 +844,19 @@ const runLockedBackup = async ( storedThisRun.has(file.id), ); const mlData = await lib.mlData(file.id); - if (mlData === undefined && mlDataError !== undefined) { - writeSidecar(path, file, { mlDataError }, image); + const failure = lib.mlDataError(file.id) ?? mlDataFailure; + if (mlData === undefined && failure !== undefined) { + writeSidecar( + path, + file, + { mlDataError: describeError(failure) }, + image, + ); recordFailure( file.id, file.metadata.title, collectionName.get(file.collectionID) ?? "", - new Error("ML data", { cause: mlDataFailure }), + new Error("ML data", { cause: failure }), ); } else { writeSidecar(path, file, { mlData }, image); diff --git a/src/cli-commands.ts b/src/cli-commands.ts index d545cf1..8dac711 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -394,11 +394,11 @@ export const backupMetadataCommand = async ( // Refresh first so the dump holds current account state, not what the // cache last held; a failed refresh throws. await lib.fresh(); - const { failedMLBatches } = await runMetadataBackup(lib, client, dir, { + const { mlDataErrors } = await runMetadataBackup(lib, client, dir, { exif: opts.exif || opts.all, onProgress: (msg) => ctx.stderr.write(msg + "\n"), }); - return failedMLBatches > 0 ? 1 : 0; + return mlDataErrors > 0 ? 1 : 0; } finally { await lib.close(); } diff --git a/src/client.ts b/src/client.ts index bcb3dac..2f5ea8d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -373,13 +373,17 @@ export class Client { } // Fetch machine-learning data (face detections + CLIP embeddings) for up - // to a batch of files, each decrypted with its own key. One request; the - // library batches at `MLDATA_BATCH_SIZE` and schedules each batch through - // its metadata request pool. + // to a batch of files, each decrypted with its own key, and for each file + // whose ML data did not decrypt, gunzip or parse, why not. One request; + // the library batches at `MLDATA_BATCH_SIZE` and schedules each batch + // through its metadata request pool. async fetchMLData(args: { fileIDs: number[]; fileKeys: Map; - }): Promise> { + }): Promise<{ + payloads: Map; + errors: Map; + }> { this.assertLoggedIn(); return fetchMLDataBatch(this.api, args.fileIDs, args.fileKeys); } diff --git a/src/library/index.ts b/src/library/index.ts index 95d8e1d..2370c56 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -89,6 +89,7 @@ export { } from "./content.js"; export { type MLDataAPI, type SimilarResult } from "./mlsearch.js"; import type { CollectionsPage, FilesPage } from "../client.js"; +import { describeError } from "../errors.js"; import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js"; import type { Collection, EnteFile } from "../model/types.js"; import { runBackup, type BackupOptions, type BackupResult } from "../backup.js"; @@ -138,12 +139,16 @@ export interface LibraryClient { sinceTime: number; }): Promise; // Fetch ML data (face detections + CLIP embeddings) for up to a batch of - // files. Optional: a client without it simply disables ML fetching, leaving - // the metadata refresh untouched. + // files, and for each file whose ML data did not decrypt, gunzip or parse, + // why not. Optional: a client without it simply disables ML fetching, + // leaving the metadata refresh untouched. fetchMLData?(args: { fileIDs: number[]; fileKeys: Map; - }): Promise>; + }): Promise<{ + payloads: Map; + errors: Map; + }>; // The byte source for the on-disk content cache. Optional so a mock client // that only serves metadata still satisfies the interface; when absent (and // no explicit `contentSource` is passed to `open`) the content cache is @@ -154,9 +159,11 @@ export interface LibraryClient { // A progress event for one unit of background work. A metadata "refresh" or an // ML "fetchMLData" pass each fire "started" before their network work and then // exactly one of "done" or "failed"; "failed" carries the error message and -// an ML "done" reports how many payloads it stored. The precache fills -// ("precacheThumbnails"/"precacheOriginals", #48) fire "started"/"done" around -// each sweep that has work, "done" reporting the count newly cached. +// an ML "done" reports how many payloads it stored and, in `mlDataErrors`, +// each file whose ML data did not decrypt, gunzip or parse, with the reason. +// The precache fills ("precacheThumbnails"/"precacheOriginals", #48) fire +// "started"/"done" around each sweep that has work, "done" reporting the count +// newly cached. export interface RefreshEvent { operation: | "refresh" @@ -166,6 +173,7 @@ export interface RefreshEvent { status: "started" | "done" | "failed"; error?: string; fetched?: number; + mlDataErrors?: { fileID: number; error: string }[]; } export type RefreshProgressCallback = (event: RefreshEvent) => void; @@ -225,6 +233,10 @@ export interface LibraryStatus { lastMLFetchAt?: number; // The most recent ML fetch pass's error, set only while it failed. lastMLError?: string; + // Each file whose ML data the most recent ML fetch pass got but could not + // decrypt, gunzip or parse, with the reason. Such ML data is not cached, + // so the next pass asks for it again. + mlDataErrors: { fileID: number; error: string }[]; // ML payloads stored on disk and CLIP embeddings in the index; undefined // when ML fetching is disabled. mlStored?: number; @@ -292,6 +304,10 @@ export class Library { private lastError?: string; private lastMLFetchAt?: number; private lastMLError?: string; + // For each file whose ML data the most recent ML fetch pass got but could + // not decrypt, gunzip or parse, why not, by fileID. Kept in RAM only, + // never in the cache. + private readonly mlDataErrors = new Map(); // The plain-record projection as of the last refresh, and the GUI change // subscribers. A refresh that alters the projection notifies each with the // delta; `lastRecords` is kept current every refresh so a subscriber that @@ -542,6 +558,7 @@ export class Library { lastError: this.lastError, lastMLFetchAt: this.lastMLFetchAt, lastMLError: this.lastMLError, + mlDataErrors: this.describeMLDataErrors(), mlStored: ml?.stored, mlIndexed: ml?.indexed, originalsUsedBytes: originals?.usedBytes, @@ -610,6 +627,7 @@ export class Library { contentConcurrency: this.pools.content.concurrency, fetchMLData: () => this.fetchMLDataNow(), mlData: (fileID) => this.mldata.forFile({ fileID }), + mlDataError: (fileID) => this.mlDataErrors.get(fileID), }, { ...opts, downloadDirectory }, ); @@ -836,8 +854,10 @@ export class Library { // One ML fetch pass: fetch, decrypt and store the ML data for every file // the store knows about that is not cached (or whose `updationTime` has - // advanced), through the metadata pool, and update the CLIP index. A - // failure is reported through `status()` and `onProgress`, then thrown. + // advanced), through the metadata pool, and update the CLIP index. For a + // file whose ML data does not decrypt, gunzip or parse, it keeps the + // reason in `mlDataErrors` instead. A failure is reported through + // `status()` and `onProgress`, then thrown. private async runMLFetch(): Promise { const mldata = this.mlStore; // Bind so the call keeps the client as its receiver when invoked @@ -847,6 +867,9 @@ export class Library { const files = this.uniqueFiles(); const needed = mldata.neededFor(files); + // A file whose ML data did not decode in the pass before is still + // among `needed`, so this pass gives its reason anew. + this.mlDataErrors.clear(); if (needed.length === 0) return; this.emit({ operation: "fetchMLData", status: "started" }); @@ -862,10 +885,13 @@ export class Library { for (let i = 0; i < needed.length; i += MLDATA_BATCH_SIZE) { if (this.closed) break; const batch = needed.slice(i, i + MLDATA_BATCH_SIZE); - const payloads = await this.pools.metadata.run( + const { payloads, errors } = await this.pools.metadata.run( () => fetchMLData({ fileIDs: batch, fileKeys }), { priority: "background" }, ); + for (const [id, error] of errors) { + this.mlDataErrors.set(id, error); + } stored += (await mldata.storeFetched(payloads, updation)) .stored; } @@ -876,6 +902,7 @@ export class Library { operation: "fetchMLData", status: "done", fetched: stored, + mlDataErrors: this.describeMLDataErrors(), }); } catch (err) { const error = err instanceof Error ? err.message : String(err); @@ -885,6 +912,14 @@ export class Library { } } + // `mlDataErrors` as `status()` and the ML "done" event give it. + private describeMLDataErrors(): { fileID: number; error: string }[] { + return [...this.mlDataErrors].map(([fileID, err]) => ({ + fileID, + error: describeError(err), + })); + } + // The distinct files the store holds, one entry per fileID (a file in // several collections shares its ML data), each carrying the key and the // newest `updationTime` seen across its memberships. diff --git a/src/metadata-backup.ts b/src/metadata-backup.ts index d453bf8..2f65f3f 100644 --- a/src/metadata-backup.ts +++ b/src/metadata-backup.ts @@ -2,6 +2,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import * as jpeg from "jpeg-js"; import type { Client } from "./client.js"; +import { describeError } from "./errors.js"; import { readExifTags } from "./exif.js"; import type { Library, Photo } from "./library/index.js"; import { sanitizeFileName } from "./filename.js"; @@ -91,14 +92,15 @@ const extractExif = async ( // of plain JSON: account, per-collection, and per-file records including the // private and public magic metadata and (by default) the ML data. Collections // and files are enumerated from the library's cache, which the caller refreshes -// first. Returns how many ML data requests failed; their files are still +// first. Returns how many files have no ML data because their ML data request +// failed or their ML data did not decrypt, gunzip or parse; they are still // written, with `mlDataError` in place of `mlData`. export const runMetadataBackup = async ( lib: Library, client: Client, outDir: string, opts?: MetadataBackupOptions, -): Promise<{ failedMLBatches: number }> => { +): Promise<{ mlDataErrors: number }> => { const log = opts?.onProgress ?? (() => {}); const wantExif = opts?.exif ?? false; @@ -164,24 +166,28 @@ export const runMetadataBackup = async ( } // One failed request (retries exhausted) must not end the dump: its files - // get the reason in `mlDataError` and the other batches go on. + // get the reason in `mlDataError` and the other batches go on. So does a + // file whose ML data does not decrypt, gunzip or parse. log("Fetching ML data (face detections, CLIP embeddings)..."); const mlDataMap = new Map(); const mlDataErrors = new Map(); - let failedMLBatches = 0; const fileIDs = [...fileKeys.keys()]; for (let i = 0; i < fileIDs.length; i += MLDATA_BATCH_SIZE) { const batch = fileIDs.slice(i, i + MLDATA_BATCH_SIZE); try { - const result = await fetchMLDataBatch( + const { payloads, errors } = await fetchMLDataBatch( client.getApiClient(), batch, fileKeys, ); - for (const [id, payload] of result) mlDataMap.set(id, payload); + for (const [id, payload] of payloads) mlDataMap.set(id, payload); + for (const [id, err] of errors) { + const reason = describeError(err); + log(`ML data of file ${id} could not be read: ${reason}`); + mlDataErrors.set(id, reason); + } } catch (err) { const reason = err instanceof Error ? err.message : String(err); - failedMLBatches++; log( `ML data request for ${batch.length} file(s) failed: ${reason}`, ); @@ -229,5 +235,5 @@ export const runMetadataBackup = async ( } log("Metadata backup complete."); - return { failedMLBatches }; + return { mlDataErrors: mlDataErrors.size }; }; diff --git a/src/mldata-fetch.ts b/src/mldata-fetch.ts index e5a254d..bee4a84 100644 --- a/src/mldata-fetch.ts +++ b/src/mldata-fetch.ts @@ -28,43 +28,43 @@ interface RawRemoteFileData { updatedAt?: number; } -// Decrypt one entry with its file key and gunzip the JSON payload. Returns -// undefined when the key is unknown or the entry does not decrypt/parse, so one -// corrupt file never fails a whole batch. -const decodeEntry = ( - entry: RawRemoteFileData, - key: Uint8Array | undefined, -): MLData | undefined => { - if (!key) return undefined; - try { - const decrypted = decryptBlob( - fromBase64(entry.encryptedData), - fromBase64(entry.decryptionHeader), - key, - ); - const json = gunzipSync(Buffer.from(decrypted)).toString("utf-8"); - return JSON.parse(json) as MLData; - } catch { - return undefined; - } +// Decrypt one entry with its file key and gunzip the JSON payload. Throws why +// when the entry does not decrypt, gunzip or parse. +const decodeEntry = (entry: RawRemoteFileData, key: Uint8Array): MLData => { + const decrypted = decryptBlob( + fromBase64(entry.encryptedData), + fromBase64(entry.decryptionHeader), + key, + ); + const json = gunzipSync(Buffer.from(decrypted)).toString("utf-8"); + return JSON.parse(json) as MLData; }; // Fetch ML data for up to `MLDATA_BATCH_SIZE` ids in a single request. This is // the unit the request pools schedule; callers with more ids split them into -// batches and submit each batch to the pool. +// batches and submit each batch to the pool. Returns the payload of each file +// whose entry decodes and, in `errors`, why not for each file whose entry does +// not, so one corrupt file never fails a whole batch. An entry for a file +// whose key is not known, which quak did not ask for, is skipped. export const fetchMLDataBatch = async ( api: ApiClient, fileIDs: number[], fileKeys: Map, -): Promise> => { +): Promise<{ payloads: Map; errors: Map }> => { const { data } = await api.postJSON<{ data: RawRemoteFileData[] }>( "/files/data/fetch", { type: "mldata", fileIDs }, ); - const result = new Map(); + const payloads = new Map(); + const errors = new Map(); for (const entry of data ?? []) { - const payload = decodeEntry(entry, fileKeys.get(entry.fileID)); - if (payload) result.set(entry.fileID, payload); + const key = fileKeys.get(entry.fileID); + if (!key) continue; + try { + payloads.set(entry.fileID, decodeEntry(entry, key)); + } catch (err) { + errors.set(entry.fileID, err); + } } - return result; + return { payloads, errors }; }; diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index 8636a8f..ab35eca 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -877,6 +877,56 @@ describe("quak backup", () => { ]); }); + // The server sends beach.jpg's ML data encrypted under a key that is not + // the file's, so quak cannot decrypt it. The file's JSON has the reason + // in place of the ML data, and the file goes into `failures.json` with + // it, so the exit code is 1. The other two files have no ML data, which + // is no failure. + it("records a file whose ML data does not decrypt in its JSON and in failures.json, and exits 1", async () => { + const otherKey = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const mlData = encryptStreamBlob( + gzipSync(JSON.stringify(ML_DATA)), + otherKey, + ); + const fetch: typeof globalThis.fetch = async (input, init) => { + if (new URL(String(input)).pathname !== "/files/data/fetch") { + return server(input, init); + } + const data = [ + { + fileID: BEACH.id, + encryptedData: toBase64(mlData.ciphertext), + decryptionHeader: toBase64(mlData.header), + }, + ]; + return new Response(JSON.stringify({ data }), { + headers: { "content-type": "application/json" }, + }); + }; + const dir = mkdtempSync(join(root, "backup-")); + + const { code, stderr } = await backup(dir, {}, fetch); + + expect(code).toBe(1); + const reason = "secretstream chunk authentication failed"; + const beachJSON = readJSON(join(dir, DAY, "2024-03-01.100.json")); + expect(beachJSON).not.toHaveProperty("mlData"); + expect(beachJSON.mlDataError).toBe(reason); + const failures = readJSON(join(dir, "failures.json")); + expect(Object.keys(failures.files)).toEqual(["100"]); + expect(failures.files["100"]).toMatchObject({ + fileID: 100, + title: "beach.jpg", + error: `ML data: ${reason}`, + }); + expect(stderr).toContain(`beach.jpg (id 100): ML data: ${reason}\n`); + for (const fileID of [200, 300]) { + const json = readJSON(join(dir, DAY, `2024-03-01.${fileID}.json`)); + expect(json).not.toHaveProperty("mlData"); + expect(json).not.toHaveProperty("mlDataError"); + } + }); + // Something changed beach.jpg on disk after it was backed up. A run with // --verify hashes every stored original, finds that one no longer matches // the hash Ente recorded, says so, and downloads it again; it put back the diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 09bedc6..fb4147d 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -938,13 +938,14 @@ const ML_PAYLOAD = { face: { faces: [] }, clip: { embedding: [0.5, 0.25] } }; class MLClient extends MockClient { mlError?: string; - async fetchMLData(args: { - fileIDs: number[]; - }): Promise> { + async fetchMLData(args: { fileIDs: number[] }): Promise<{ + payloads: Map; + errors: Map; + }> { if (this.mlError) throw new Error(this.mlError); - const result = new Map(); - if (args.fileIDs.includes(100)) result.set(100, ML_PAYLOAD); - return result; + const payloads = new Map(); + if (args.fileIDs.includes(100)) payloads.set(100, ML_PAYLOAD); + return { payloads, errors: new Map() }; } } @@ -1517,6 +1518,7 @@ describe("backup album folders", () => { contentConcurrency: 5, fetchMLData: async () => {}, mlData: async () => undefined, + mlDataError: () => undefined, whoami: () => ({ email: "backup@example.com", userID: USER_ID }), }); diff --git a/test/cli/metadata-backup.test.ts b/test/cli/metadata-backup.test.ts index 5bff19c..0902e44 100644 --- a/test/cli/metadata-backup.test.ts +++ b/test/cli/metadata-backup.test.ts @@ -489,15 +489,15 @@ describe("quak backup-metadata", () => { }); }); -describe("quak backup-metadata when an ML data request fails", () => { - // Run the CLI command against the mock and return its exit code, stderr - // and output directory. - const runCommand = async (failMLDataFor?: number) => { +describe("quak backup-metadata when a file's ML data cannot be read", () => { + // Run the CLI command against `fetch`, the mock unless given, and return + // its exit code, stderr and output directory. + const runCommand = async (fetch = buildMetaFetch(mock)) => { const client = await Client.login({ email: TEST_EMAIL, password: TEST_PASSWORD, apiOptions: { - fetch: buildMetaFetch(mock, failMLDataFor), + fetch, retry: { sleep: async () => {} }, }, }); @@ -518,7 +518,9 @@ describe("quak backup-metadata when an ML data request fails", () => { }; it("writes every file, marks the failed batch's files, and exits 1", async () => { - const { code, stderr, outDir } = await runCommand(200); + const { code, stderr, outDir } = await runCommand( + buildMetaFetch(mock, 200), + ); expect(code).toBe(1); expect(stderr).toContain("ML data request for 1 file(s) failed"); @@ -543,6 +545,49 @@ describe("quak backup-metadata when an ML data request fails", () => { expect(failed.mlDataError).toContain("500"); }); + // The server sends ML data for diagram.png encrypted under a key that is + // not the file's, so quak cannot decrypt it. + it("writes the reason for ML data that does not decrypt, and exits 1", async () => { + const otherKey = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptBlob( + new Uint8Array(gzipSync(Buffer.from("{}"))), + otherKey, + ); + const { code, stderr, outDir } = await runCommand( + buildMetaFetch({ + ...mock, + encryptedMLData: { + ...mock.encryptedMLData, + 200: { + encryptedData: toBase64(ciphertext), + decryptionHeader: toBase64(header), + }, + }, + }), + ); + + expect(code).toBe(1); + const reason = "secretstream chunk authentication failed"; + expect(stderr).toContain( + `ML data of file 200 could not be read: ${reason}\n`, + ); + const failed = JSON.parse( + readFileSync( + join(outDir, "collections", "20-__Work", "200.json"), + "utf-8", + ), + ); + expect(failed.mlData).toBeUndefined(); + expect(failed.mlDataError).toBe(reason); + const ok = JSON.parse( + readFileSync( + join(outDir, "collections", "10-Vacation", "100.json"), + "utf-8", + ), + ); + expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]); + }); + it("exits 0 when every ML data request succeeds", async () => { const { code } = await runCommand(); expect(code).toBe(0); diff --git a/test/library/mldata.test.ts b/test/library/mldata.test.ts index 3de1cdc..186824f 100644 --- a/test/library/mldata.test.ts +++ b/test/library/mldata.test.ts @@ -12,9 +12,10 @@ * * 2. `Library` wiring: after each refresh the library fetches ML data through * the metadata pool for every known file not yet cached, is incremental on - * later refreshes, and refetches a file whose `updationTime` advanced. - * Opening a cache directory written by another account starts empty, - * its ML data included (issue #104). + * later refreshes, and refetches a file whose `updationTime` advanced. A + * file whose ML data did not decode is reported with the reason, and + * asked for again. Opening a cache directory written by another account + * starts empty, its ML data included (issue #104). * * Embedding values are chosen to be exactly representable as float32 so the * round-trip through `clip.f32` compares equal. @@ -26,7 +27,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { MLDataStore } from "../../src/library/mldata.js"; -import { Library } from "../../src/library/index.js"; +import { Library, type RefreshEvent } from "../../src/library/index.js"; import type { CollectionsPage, FilesPage } from "../../src/client.js"; import type { MLData } from "../../src/mldata-fetch.js"; import type { Collection, EnteFile } from "../../src/model/types.js"; @@ -256,12 +257,14 @@ const file = ( // A mock client that serves scripted collection/file pages and per-file ML // payloads, recording every ML fetch request so incremental behaviour is -// provable. +// provable. A file in `mlErrorByFile` has ML data that did not decode, for +// the reason given there. class MLMockClient { userID = USER_ID; collectionsQueue: CollectionsPage[] = []; filesByCollection = new Map(); mlByFile = new Map(); + mlErrorByFile = new Map(); mlFetchCalls: number[][] = []; whoami(): { email: string; userID: number } { @@ -298,14 +301,20 @@ class MLMockClient { async fetchMLData(args: { fileIDs: number[]; fileKeys: Map; - }): Promise> { + }): Promise<{ + payloads: Map; + errors: Map; + }> { this.mlFetchCalls.push([...args.fileIDs]); - const result = new Map(); + const payloads = new Map(); + const errors = new Map(); for (const id of args.fileIDs) { const p = this.mlByFile.get(id); - if (p) result.set(id, p); + if (p) payloads.set(id, p); + const error = this.mlErrorByFile.get(id); + if (error) errors.set(id, error); } - return result; + return { payloads, errors }; } filesFor(collectionID: number, ...pages: FilesPage[]): void { @@ -449,6 +458,74 @@ describe("Library ML-data fetch on refresh", () => { } }); + it("reports a file whose ML data did not decode through status() and onProgress, and asks for it again", async () => { + const client = new MLMockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90), file(1002, 1, 95)], + deleted: [], + cursor: 95, + }); + client.mlByFile.set(1001, payload([0.5, 0.25, 0.75])); + client.mlErrorByFile.set( + 1002, + new Error("secretstream chunk authentication failed"), + ); + const events: RefreshEvent[] = []; + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: Infinity, + onProgress: (event) => events.push(event), + }); + try { + await vi.waitFor( + () => expect(lib.status().lastMLFetchAt).toBeGreaterThan(0), + { timeout: 2000, interval: 5 }, + ); + const reasons = [ + { + fileID: 1002, + error: "secretstream chunk authentication failed", + }, + ]; + expect(lib.status().mlDataErrors).toEqual(reasons); + expect(lib.status().mlStored).toBe(1); + expect(events.filter((e) => e.operation === "fetchMLData")).toEqual( + [ + { operation: "fetchMLData", status: "started" }, + { + operation: "fetchMLData", + status: "done", + fetched: 1, + mlDataErrors: reasons, + }, + ], + ); + + // Nothing was cached for 1002, so the pass a refresh starts asks + // for it again, and once its ML data decodes the reason is gone. + client.mlErrorByFile.clear(); + client.mlByFile.set(1002, payload([1, -2, 0.5])); + await lib.fresh(); + await vi.waitFor( + () => { + expect(lib.status().mlStored).toBe(2); + expect(lib.status().mlDataErrors).toEqual([]); + }, + { timeout: 2000, interval: 5 }, + ); + expect(client.mlFetchCalls.at(-1)).toEqual([1002]); + } finally { + await lib.close(); + } + }); + it("close() resolves only after a running ML data fetch has stored its payloads", async () => { const client = new MLMockClient(); client.collectionsQueue.push({