diff --git a/README.md b/README.md index 1674064..5727cb6 100644 --- a/README.md +++ b/README.md @@ -697,8 +697,10 @@ into `failures.json` (see "Backup layout"). `backup-metadata` fetches ML data in requests of up to 200 files. When a request fails, the error is logged, each of its files is written with the reason in an -`mlDataError` field instead of `mlData`, and the dump goes on. The exit code is -non-zero if any ML data request failed. +`mlDataError` field instead of `mlData`, and the dump goes on. A file whose ML +data does not decrypt, gunzip or parse is logged and written the same way, with +why in `mlDataError`. The exit code is non-zero if any file was written with +`mlDataError`. `helper fix-missing-thumbnails` regenerates thumbnails for JPEG images only, because the bundled decoder (`jpeg-js`) decodes only JPEG. A non-JPEG image @@ -763,8 +765,15 @@ A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as for has no `mlData`. The backup waits for the library's ML data fetch to finish before it writes the JSON files. If that fetch fails, each file whose ML data is not in the cache gets the reason in `mlDataError` instead and counts as failed, -and the next run fetches it again. The JSON files are rewritten on every run, so -ML data that arrived since the last run appears. +and the next run fetches it again. The same goes for a file with no ML data in +the cache whose ML data the server sent but quak could not decrypt, gunzip or +parse. Its `mlDataError` says why, and `failures.json` records it with an error +starting `ML data: `. quak keeps that reason in memory only and caches nothing +for the file, so the next run asks for its ML data again. An ML data fetch that +fails, such as one the library's refresh starts while the backup writes the JSON +files, leaves the reasons as the last fetch that succeeded left them. The JSON +files are rewritten on every run, so ML data that arrived since the last run +appears. A file's JSON holds, as `imageMetadata`, what `backup-metadata --exif` records from its original, or from a live photo's image: `format`, `width` and `height` @@ -978,6 +987,14 @@ resolves once that refresh (including its cache write), the ML data fetch and the precache fetches already running have all finished, so the cache directory can then be removed. +`lib.status().mlDataErrors` lists, as `{ fileID, error }`, each file whose ML +data the last ML data fetch that succeeded got but could not decrypt, gunzip or +parse; the `fetchMLData` event with status `"done"` carries the same list. A +fetch that is still running leaves the list as it is until it ends, and a fetch +that fails leaves it as it was. Such ML data is not cached, so the next ML data +fetch asks for it again. The reasons are kept in memory only, never in the +cache. + ### Default reads vs. fresh reads Default reads — `lib.albums`, `lib.photos`, `lib.timeline` — answer diff --git a/TODO.md b/TODO.md index c2dbf45..c688fa1 100644 --- a/TODO.md +++ b/TODO.md @@ -17,8 +17,7 @@ pre-1.0 # Next Step -Three bugs are open: https://git.eeqj.de/sneak/quak/issues/198, -https://git.eeqj.de/sneak/quak/issues/199 and +Two bugs are open: https://git.eeqj.de/sneak/quak/issues/199 and https://git.eeqj.de/sneak/quak/issues/205. The check of quak against a live Ente account, https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak has, so it waits on him. @@ -28,6 +27,19 @@ declares one. # Completed Steps +- 2026-10-16: ML data the server sends that does not decrypt, gunzip or parse is + no longer taken for no ML data (issue 198). `fetchMLDataBatch` returns why for + each such file beside the payloads; ML data that parses to something other + than a JSON object, such as `null`, is one of them. The library keeps the + reasons from its last ML data fetch that succeeded in memory, not in the + cache, and gives them in `status().mlDataErrors` and in the `fetchMLData` + event with status `"done"`; a fetch still running or one that fails does not + change them. `quak backup` writes the reason as `mlDataError` in the file's + JSON when no ML data is cached for it, and records the file in `failures.json` + with an error starting `ML data: `; `backup-metadata` writes `mlDataError` + too. Both exit 1. Tests serve ML data encrypted under a key that is not the + file's. + - 2026-10-16: The `~/bin/quak` that `make install` builds retries a refused connection and a name that does not resolve, and the backup records either as transient (issue 206). Bun's `fetch` reports them with codes of its own, diff --git a/src/backup.ts b/src/backup.ts index 1c44dd4..f2aceaa 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -203,6 +203,11 @@ export interface BackupLibrary { // A file's cached ML data, as `lib.mldata.forFile()` returns it, or // undefined when none is cached. mlData(fileID: number): Promise; + // Why the ML data the last ML data fetch that succeeded got for a file did + // not decrypt, gunzip or parse, or undefined when it did or the fetch got + // none. A fetch still running does not change it. A fetch that fails + // leaves it as it was. + mlDataError(fileID: number): unknown; } type FailureClass = "transient" | "permanent" | "unknown"; @@ -813,19 +818,18 @@ const runLockedBackup = async ( // Phase 2: rebuild the derived views from the model. Sidecars first, for // every present original (this repairs stale ones), each with the file's // ML data once an ML data fetch has completed, and its image metadata. - // When the fetch fails, a file with no cached ML data gets the reason - // instead and is recorded as failed. The next run fetches its ML data - // again because none is cached. + // When the fetch got a file's ML data but could not decrypt, gunzip or + // parse it, or when the fetch failed, a file with no cached ML data gets + // the reason instead and is recorded as failed. The next run fetches its + // ML data again because none is cached. if (includeOriginals) { let mlDataFailure: unknown; - let mlDataError: string | undefined; try { log("Fetching ML data..."); await lib.fetchMLData(); } catch (err) { mlDataFailure = err; - mlDataError = describeError(err); - log(`FAILED ML data: ${mlDataError}`); + log(`FAILED ML data: ${describeError(err)}`); } for (const file of distinct.values()) { if (lock.lost) throw lock.lost; @@ -842,13 +846,19 @@ const runLockedBackup = async ( storedThisRun.has(file.id), ); const mlData = await lib.mlData(file.id); - if (mlData === undefined && mlDataError !== undefined) { - writeSidecar(path, file, { mlDataError }, image); + const failure = lib.mlDataError(file.id) ?? mlDataFailure; + if (mlData === undefined && failure !== undefined) { + writeSidecar( + path, + file, + { mlDataError: describeError(failure) }, + image, + ); recordFailure( file.id, file.metadata.title, collectionName.get(file.collectionID) ?? "", - new Error("ML data", { cause: mlDataFailure }), + new Error("ML data", { cause: failure }), ); } else { writeSidecar(path, file, { mlData }, image); diff --git a/src/cli-commands.ts b/src/cli-commands.ts index d545cf1..a6d8b8a 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -394,11 +394,11 @@ export const backupMetadataCommand = async ( // Refresh first so the dump holds current account state, not what the // cache last held; a failed refresh throws. await lib.fresh(); - const { failedMLBatches } = await runMetadataBackup(lib, client, dir, { + const result = await runMetadataBackup(lib, client, dir, { exif: opts.exif || opts.all, onProgress: (msg) => ctx.stderr.write(msg + "\n"), }); - return failedMLBatches > 0 ? 1 : 0; + return result.filesWithMLDataError > 0 ? 1 : 0; } finally { await lib.close(); } diff --git a/src/client.ts b/src/client.ts index bcb3dac..2f5ea8d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -373,13 +373,17 @@ export class Client { } // Fetch machine-learning data (face detections + CLIP embeddings) for up - // to a batch of files, each decrypted with its own key. One request; the - // library batches at `MLDATA_BATCH_SIZE` and schedules each batch through - // its metadata request pool. + // to a batch of files, each decrypted with its own key, and for each file + // whose ML data did not decrypt, gunzip or parse, why not. One request; + // the library batches at `MLDATA_BATCH_SIZE` and schedules each batch + // through its metadata request pool. async fetchMLData(args: { fileIDs: number[]; fileKeys: Map; - }): Promise> { + }): Promise<{ + payloads: Map; + errors: Map; + }> { this.assertLoggedIn(); return fetchMLDataBatch(this.api, args.fileIDs, args.fileKeys); } diff --git a/src/library/index.ts b/src/library/index.ts index 95d8e1d..7efdd7d 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -89,6 +89,7 @@ export { } from "./content.js"; export { type MLDataAPI, type SimilarResult } from "./mlsearch.js"; import type { CollectionsPage, FilesPage } from "../client.js"; +import { describeError } from "../errors.js"; import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js"; import type { Collection, EnteFile } from "../model/types.js"; import { runBackup, type BackupOptions, type BackupResult } from "../backup.js"; @@ -138,12 +139,16 @@ export interface LibraryClient { sinceTime: number; }): Promise; // Fetch ML data (face detections + CLIP embeddings) for up to a batch of - // files. Optional: a client without it simply disables ML fetching, leaving - // the metadata refresh untouched. + // files, and for each file whose ML data did not decrypt, gunzip or parse, + // why not. Optional: a client without it simply disables ML fetching, + // leaving the metadata refresh untouched. fetchMLData?(args: { fileIDs: number[]; fileKeys: Map; - }): Promise>; + }): Promise<{ + payloads: Map; + errors: Map; + }>; // The byte source for the on-disk content cache. Optional so a mock client // that only serves metadata still satisfies the interface; when absent (and // no explicit `contentSource` is passed to `open`) the content cache is @@ -154,9 +159,11 @@ export interface LibraryClient { // A progress event for one unit of background work. A metadata "refresh" or an // ML "fetchMLData" pass each fire "started" before their network work and then // exactly one of "done" or "failed"; "failed" carries the error message and -// an ML "done" reports how many payloads it stored. The precache fills -// ("precacheThumbnails"/"precacheOriginals", #48) fire "started"/"done" around -// each sweep that has work, "done" reporting the count newly cached. +// an ML "done" reports how many payloads it stored and, in `mlDataErrors`, +// each file whose ML data did not decrypt, gunzip or parse, with the reason. +// The precache fills ("precacheThumbnails"/"precacheOriginals", #48) fire +// "started"/"done" around each sweep that has work, "done" reporting the count +// newly cached. export interface RefreshEvent { operation: | "refresh" @@ -166,6 +173,7 @@ export interface RefreshEvent { status: "started" | "done" | "failed"; error?: string; fetched?: number; + mlDataErrors?: { fileID: number; error: string }[]; } export type RefreshProgressCallback = (event: RefreshEvent) => void; @@ -225,6 +233,11 @@ export interface LibraryStatus { lastMLFetchAt?: number; // The most recent ML fetch pass's error, set only while it failed. lastMLError?: string; + // Each file whose ML data the last ML fetch pass that succeeded got but + // could not decrypt, gunzip or parse, with the reason. A pass still + // running leaves it as it is. A pass that fails leaves it as it was. Such + // ML data is not cached, so the next pass asks for it again. + mlDataErrors: { fileID: number; error: string }[]; // ML payloads stored on disk and CLIP embeddings in the index; undefined // when ML fetching is disabled. mlStored?: number; @@ -292,6 +305,12 @@ export class Library { private lastError?: string; private lastMLFetchAt?: number; private lastMLError?: string; + // For each file whose ML data the last ML fetch pass that succeeded got + // but could not decrypt, gunzip or parse, why not, by fileID. A pass + // gathers its own and puts them here when it ends without failing, so + // `status()` and `backup()` never see the list of a pass that is still + // running or that failed. Kept in RAM only, never in the cache. + private mlDataErrors = new Map(); // The plain-record projection as of the last refresh, and the GUI change // subscribers. A refresh that alters the projection notifies each with the // delta; `lastRecords` is kept current every refresh so a subscriber that @@ -542,6 +561,7 @@ export class Library { lastError: this.lastError, lastMLFetchAt: this.lastMLFetchAt, lastMLError: this.lastMLError, + mlDataErrors: this.describeMLDataErrors(), mlStored: ml?.stored, mlIndexed: ml?.indexed, originalsUsedBytes: originals?.usedBytes, @@ -610,6 +630,7 @@ export class Library { contentConcurrency: this.pools.content.concurrency, fetchMLData: () => this.fetchMLDataNow(), mlData: (fileID) => this.mldata.forFile({ fileID }), + mlDataError: (fileID) => this.mlDataErrors.get(fileID), }, { ...opts, downloadDirectory }, ); @@ -836,8 +857,11 @@ export class Library { // One ML fetch pass: fetch, decrypt and store the ML data for every file // the store knows about that is not cached (or whose `updationTime` has - // advanced), through the metadata pool, and update the CLIP index. A - // failure is reported through `status()` and `onProgress`, then thrown. + // advanced), through the metadata pool, and update the CLIP index. For a + // file whose ML data does not decrypt, gunzip or parse, it gathers the + // reason instead, and puts the reasons it gathered in `mlDataErrors` when + // it ends without failing. A failure leaves `mlDataErrors` as it was and + // is reported through `status()` and `onProgress`, then thrown. private async runMLFetch(): Promise { const mldata = this.mlStore; // Bind so the call keeps the client as its receiver when invoked @@ -847,7 +871,14 @@ export class Library { const files = this.uniqueFiles(); const needed = mldata.neededFor(files); - if (needed.length === 0) return; + // This pass's reasons, put in `this.mlDataErrors` when it ends without + // failing. A file whose ML data did not decode in the pass before is + // still among `needed`, so this pass gives its reason anew. + const mlDataErrors = new Map(); + if (needed.length === 0) { + this.mlDataErrors = mlDataErrors; + return; + } this.emit({ operation: "fetchMLData", status: "started" }); try { @@ -862,20 +893,25 @@ export class Library { for (let i = 0; i < needed.length; i += MLDATA_BATCH_SIZE) { if (this.closed) break; const batch = needed.slice(i, i + MLDATA_BATCH_SIZE); - const payloads = await this.pools.metadata.run( + const { payloads, errors } = await this.pools.metadata.run( () => fetchMLData({ fileIDs: batch, fileKeys }), { priority: "background" }, ); + for (const [id, error] of errors) { + mlDataErrors.set(id, error); + } stored += (await mldata.storeFetched(payloads, updation)) .stored; } + this.mlDataErrors = mlDataErrors; this.lastMLFetchAt = Date.now(); this.lastMLError = undefined; this.emit({ operation: "fetchMLData", status: "done", fetched: stored, + mlDataErrors: this.describeMLDataErrors(), }); } catch (err) { const error = err instanceof Error ? err.message : String(err); @@ -885,6 +921,14 @@ export class Library { } } + // `mlDataErrors` as `status()` and the ML "done" event give it. + private describeMLDataErrors(): { fileID: number; error: string }[] { + return [...this.mlDataErrors].map(([fileID, err]) => ({ + fileID, + error: describeError(err), + })); + } + // The distinct files the store holds, one entry per fileID (a file in // several collections shares its ML data), each carrying the key and the // newest `updationTime` seen across its memberships. diff --git a/src/metadata-backup.ts b/src/metadata-backup.ts index d453bf8..79fc4ed 100644 --- a/src/metadata-backup.ts +++ b/src/metadata-backup.ts @@ -2,6 +2,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import * as jpeg from "jpeg-js"; import type { Client } from "./client.js"; +import { describeError } from "./errors.js"; import { readExifTags } from "./exif.js"; import type { Library, Photo } from "./library/index.js"; import { sanitizeFileName } from "./filename.js"; @@ -91,14 +92,15 @@ const extractExif = async ( // of plain JSON: account, per-collection, and per-file records including the // private and public magic metadata and (by default) the ML data. Collections // and files are enumerated from the library's cache, which the caller refreshes -// first. Returns how many ML data requests failed; their files are still +// first. Returns how many files have no ML data because their ML data request +// failed or their ML data did not decrypt, gunzip or parse; they are still // written, with `mlDataError` in place of `mlData`. export const runMetadataBackup = async ( lib: Library, client: Client, outDir: string, opts?: MetadataBackupOptions, -): Promise<{ failedMLBatches: number }> => { +): Promise<{ filesWithMLDataError: number }> => { const log = opts?.onProgress ?? (() => {}); const wantExif = opts?.exif ?? false; @@ -164,24 +166,28 @@ export const runMetadataBackup = async ( } // One failed request (retries exhausted) must not end the dump: its files - // get the reason in `mlDataError` and the other batches go on. + // get the reason in `mlDataError` and the other batches go on. So does a + // file whose ML data does not decrypt, gunzip or parse. log("Fetching ML data (face detections, CLIP embeddings)..."); const mlDataMap = new Map(); const mlDataErrors = new Map(); - let failedMLBatches = 0; const fileIDs = [...fileKeys.keys()]; for (let i = 0; i < fileIDs.length; i += MLDATA_BATCH_SIZE) { const batch = fileIDs.slice(i, i + MLDATA_BATCH_SIZE); try { - const result = await fetchMLDataBatch( + const { payloads, errors } = await fetchMLDataBatch( client.getApiClient(), batch, fileKeys, ); - for (const [id, payload] of result) mlDataMap.set(id, payload); + for (const [id, payload] of payloads) mlDataMap.set(id, payload); + for (const [id, err] of errors) { + const reason = describeError(err); + log(`ML data of file ${id} could not be read: ${reason}`); + mlDataErrors.set(id, reason); + } } catch (err) { const reason = err instanceof Error ? err.message : String(err); - failedMLBatches++; log( `ML data request for ${batch.length} file(s) failed: ${reason}`, ); @@ -229,5 +235,5 @@ export const runMetadataBackup = async ( } log("Metadata backup complete."); - return { failedMLBatches }; + return { filesWithMLDataError: mlDataErrors.size }; }; diff --git a/src/mldata-fetch.ts b/src/mldata-fetch.ts index e5a254d..7cca0c0 100644 --- a/src/mldata-fetch.ts +++ b/src/mldata-fetch.ts @@ -28,43 +28,52 @@ interface RawRemoteFileData { updatedAt?: number; } -// Decrypt one entry with its file key and gunzip the JSON payload. Returns -// undefined when the key is unknown or the entry does not decrypt/parse, so one -// corrupt file never fails a whole batch. -const decodeEntry = ( - entry: RawRemoteFileData, - key: Uint8Array | undefined, -): MLData | undefined => { - if (!key) return undefined; - try { - const decrypted = decryptBlob( - fromBase64(entry.encryptedData), - fromBase64(entry.decryptionHeader), - key, - ); - const json = gunzipSync(Buffer.from(decrypted)).toString("utf-8"); - return JSON.parse(json) as MLData; - } catch { - return undefined; +// Decrypt one entry with its file key and gunzip the JSON payload. Throws why +// when the entry does not decrypt, gunzip or parse, or parses to something +// other than a JSON object, such as `null`, a number or an array. +const decodeEntry = (entry: RawRemoteFileData, key: Uint8Array): MLData => { + const decrypted = decryptBlob( + fromBase64(entry.encryptedData), + fromBase64(entry.decryptionHeader), + key, + ); + const json = gunzipSync(Buffer.from(decrypted)).toString("utf-8"); + const payload: unknown = JSON.parse(json); + if ( + typeof payload !== "object" || + payload === null || + Array.isArray(payload) + ) { + throw new Error("not a JSON object"); } + return payload as MLData; }; // Fetch ML data for up to `MLDATA_BATCH_SIZE` ids in a single request. This is // the unit the request pools schedule; callers with more ids split them into -// batches and submit each batch to the pool. +// batches and submit each batch to the pool. Returns the payload of each file +// whose entry decodes and, in `errors`, why not for each file whose entry does +// not, so one corrupt file never fails a whole batch. An entry for a file +// whose key is not known, which quak did not ask for, is skipped. export const fetchMLDataBatch = async ( api: ApiClient, fileIDs: number[], fileKeys: Map, -): Promise> => { +): Promise<{ payloads: Map; errors: Map }> => { const { data } = await api.postJSON<{ data: RawRemoteFileData[] }>( "/files/data/fetch", { type: "mldata", fileIDs }, ); - const result = new Map(); + const payloads = new Map(); + const errors = new Map(); for (const entry of data ?? []) { - const payload = decodeEntry(entry, fileKeys.get(entry.fileID)); - if (payload) result.set(entry.fileID, payload); + const key = fileKeys.get(entry.fileID); + if (!key) continue; + try { + payloads.set(entry.fileID, decodeEntry(entry, key)); + } catch (err) { + errors.set(entry.fileID, err); + } } - return result; + return { payloads, errors }; }; diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index 8636a8f..80714ae 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -9,7 +9,7 @@ * data this file encrypts with libsodium, as Ente's clients encrypt it. So * quak's own code fetches each original, decrypts it, checks it against the * hash its uploader recorded and unpacks a live photo, and quak's own retries - * repeat a failed request. Two tests run `lib.backup()` instead, as a program + * repeat a failed request. Three tests run `lib.backup()` instead, as a program * using quak as a library does. * * The account, user 42, has three albums: @@ -877,6 +877,133 @@ describe("quak backup", () => { ]); }); + // The server sends beach.jpg's ML data encrypted under a key that is not + // the file's, so quak cannot decrypt it. The file's JSON has the reason + // in place of the ML data, and the file goes into `failures.json` with + // it, so the exit code is 1. The other two files have no ML data, which + // is no failure. + it("records a file whose ML data does not decrypt in its JSON and in failures.json, and exits 1", async () => { + const otherKey = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const mlData = encryptStreamBlob( + gzipSync(JSON.stringify(ML_DATA)), + otherKey, + ); + const fetch: typeof globalThis.fetch = async (input, init) => { + if (new URL(String(input)).pathname !== "/files/data/fetch") { + return server(input, init); + } + const data = [ + { + fileID: BEACH.id, + encryptedData: toBase64(mlData.ciphertext), + decryptionHeader: toBase64(mlData.header), + }, + ]; + return new Response(JSON.stringify({ data }), { + headers: { "content-type": "application/json" }, + }); + }; + const dir = mkdtempSync(join(root, "backup-")); + + const { code, stderr } = await backup(dir, {}, fetch); + + expect(code).toBe(1); + const reason = "secretstream chunk authentication failed"; + const beachJSON = readJSON(join(dir, DAY, "2024-03-01.100.json")); + expect(beachJSON).not.toHaveProperty("mlData"); + expect(beachJSON.mlDataError).toBe(reason); + const failures = readJSON(join(dir, "failures.json")); + expect(Object.keys(failures.files)).toEqual(["100"]); + expect(failures.files["100"]).toMatchObject({ + fileID: 100, + title: "beach.jpg", + error: `ML data: ${reason}`, + }); + expect(stderr).toContain(`beach.jpg (id 100): ML data: ${reason}\n`); + for (const fileID of [200, 300]) { + const json = readJSON(join(dir, DAY, `2024-03-01.${fileID}.json`)); + expect(json).not.toHaveProperty("mlData"); + expect(json).not.toHaveProperty("mlDataError"); + } + }); + + // A program using quak as a library backs the account up, and another ML + // data fetch, as one the library's refresh starts, fails while the backup + // writes the JSON files. The backup's own fetch got beach.jpg's ML data + // under a key that is not the file's; the fetch that fails leaves that + // reason in place, so beach.jpg's JSON and `failures.json` still give it. + it("keeps the reason in beach.jpg's JSON and in failures.json when an ML data fetch fails while lib.backup() writes the JSON files", async () => { + const otherKey = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const mlData = encryptStreamBlob( + gzipSync(JSON.stringify(ML_DATA)), + otherKey, + ); + let mlDataFetchFails = false; + const fetch: typeof globalThis.fetch = async (input, init) => { + if (new URL(String(input)).pathname !== "/files/data/fetch") { + return server(input, init); + } + if (mlDataFetchFails) throw new Error("the connection was reset"); + const data = [ + { + fileID: BEACH.id, + encryptedData: toBase64(mlData.ciphertext), + decryptionHeader: toBase64(mlData.header), + }, + ]; + return new Response(JSON.stringify({ data }), { + headers: { "content-type": "application/json" }, + }); + }; + const dir = mkdtempSync(join(root, "backup-")); + const lib = await Library.open({ + client: loadSession(join(sessionDir, "session.json"), { + fetch, + retry: { ...UNATTENDED_RETRY_OPTIONS, sleep: async () => {} }, + })!, + cacheDirectory: `${dir}-cache`, + downloadDirectory: dir, + refreshIntervalSeconds: Infinity, + precacheThumbnails: false, + precacheOriginals: false, + }); + try { + // The backup reads a file's cached ML data just before it writes + // that file's JSON. The first time, a refresh starts an ML data + // fetch whose request fails, and the backup goes on once that + // fetch has failed. + const forFile = lib.mldata.forFile.bind(lib.mldata); + vi.spyOn(lib.mldata, "forFile").mockImplementationOnce( + async (args) => { + mlDataFetchFails = true; + await lib.fresh(); + await vi.waitFor( + () => expect(lib.status().lastMLError).toBeDefined(), + { timeout: 2000, interval: 5 }, + ); + return forFile(args); + }, + ); + + const result = await lib.backup(); + + expect(result.failed).toBe(1); + const reason = "secretstream chunk authentication failed"; + const beachJSON = readJSON(join(dir, DAY, "2024-03-01.100.json")); + expect(beachJSON).not.toHaveProperty("mlData"); + expect(beachJSON.mlDataError).toBe(reason); + const failures = readJSON(join(dir, "failures.json")); + expect(Object.keys(failures.files)).toEqual(["100"]); + expect(failures.files["100"]).toMatchObject({ + fileID: 100, + title: "beach.jpg", + error: `ML data: ${reason}`, + }); + } finally { + await lib.close(); + } + }); + // Something changed beach.jpg on disk after it was backed up. A run with // --verify hashes every stored original, finds that one no longer matches // the hash Ente recorded, says so, and downloads it again; it put back the diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 09bedc6..fb4147d 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -938,13 +938,14 @@ const ML_PAYLOAD = { face: { faces: [] }, clip: { embedding: [0.5, 0.25] } }; class MLClient extends MockClient { mlError?: string; - async fetchMLData(args: { - fileIDs: number[]; - }): Promise> { + async fetchMLData(args: { fileIDs: number[] }): Promise<{ + payloads: Map; + errors: Map; + }> { if (this.mlError) throw new Error(this.mlError); - const result = new Map(); - if (args.fileIDs.includes(100)) result.set(100, ML_PAYLOAD); - return result; + const payloads = new Map(); + if (args.fileIDs.includes(100)) payloads.set(100, ML_PAYLOAD); + return { payloads, errors: new Map() }; } } @@ -1517,6 +1518,7 @@ describe("backup album folders", () => { contentConcurrency: 5, fetchMLData: async () => {}, mlData: async () => undefined, + mlDataError: () => undefined, whoami: () => ({ email: "backup@example.com", userID: USER_ID }), }); diff --git a/test/cli/metadata-backup.test.ts b/test/cli/metadata-backup.test.ts index 5bff19c..0902e44 100644 --- a/test/cli/metadata-backup.test.ts +++ b/test/cli/metadata-backup.test.ts @@ -489,15 +489,15 @@ describe("quak backup-metadata", () => { }); }); -describe("quak backup-metadata when an ML data request fails", () => { - // Run the CLI command against the mock and return its exit code, stderr - // and output directory. - const runCommand = async (failMLDataFor?: number) => { +describe("quak backup-metadata when a file's ML data cannot be read", () => { + // Run the CLI command against `fetch`, the mock unless given, and return + // its exit code, stderr and output directory. + const runCommand = async (fetch = buildMetaFetch(mock)) => { const client = await Client.login({ email: TEST_EMAIL, password: TEST_PASSWORD, apiOptions: { - fetch: buildMetaFetch(mock, failMLDataFor), + fetch, retry: { sleep: async () => {} }, }, }); @@ -518,7 +518,9 @@ describe("quak backup-metadata when an ML data request fails", () => { }; it("writes every file, marks the failed batch's files, and exits 1", async () => { - const { code, stderr, outDir } = await runCommand(200); + const { code, stderr, outDir } = await runCommand( + buildMetaFetch(mock, 200), + ); expect(code).toBe(1); expect(stderr).toContain("ML data request for 1 file(s) failed"); @@ -543,6 +545,49 @@ describe("quak backup-metadata when an ML data request fails", () => { expect(failed.mlDataError).toContain("500"); }); + // The server sends ML data for diagram.png encrypted under a key that is + // not the file's, so quak cannot decrypt it. + it("writes the reason for ML data that does not decrypt, and exits 1", async () => { + const otherKey = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptBlob( + new Uint8Array(gzipSync(Buffer.from("{}"))), + otherKey, + ); + const { code, stderr, outDir } = await runCommand( + buildMetaFetch({ + ...mock, + encryptedMLData: { + ...mock.encryptedMLData, + 200: { + encryptedData: toBase64(ciphertext), + decryptionHeader: toBase64(header), + }, + }, + }), + ); + + expect(code).toBe(1); + const reason = "secretstream chunk authentication failed"; + expect(stderr).toContain( + `ML data of file 200 could not be read: ${reason}\n`, + ); + const failed = JSON.parse( + readFileSync( + join(outDir, "collections", "20-__Work", "200.json"), + "utf-8", + ), + ); + expect(failed.mlData).toBeUndefined(); + expect(failed.mlDataError).toBe(reason); + const ok = JSON.parse( + readFileSync( + join(outDir, "collections", "10-Vacation", "100.json"), + "utf-8", + ), + ); + expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]); + }); + it("exits 0 when every ML data request succeeds", async () => { const { code } = await runCommand(); expect(code).toBe(0); diff --git a/test/library/mldata.test.ts b/test/library/mldata.test.ts index 3de1cdc..78ad662 100644 --- a/test/library/mldata.test.ts +++ b/test/library/mldata.test.ts @@ -12,9 +12,11 @@ * * 2. `Library` wiring: after each refresh the library fetches ML data through * the metadata pool for every known file not yet cached, is incremental on - * later refreshes, and refetches a file whose `updationTime` advanced. - * Opening a cache directory written by another account starts empty, - * its ML data included (issue #104). + * later refreshes, and refetches a file whose `updationTime` advanced. A + * file whose ML data did not decode is reported with the reason, which + * stays listed until the next pass ends, and asked for again. Opening a + * cache directory written by another account starts empty, its ML data + * included (issue #104). * * Embedding values are chosen to be exactly representable as float32 so the * round-trip through `clip.f32` compares equal. @@ -26,7 +28,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { MLDataStore } from "../../src/library/mldata.js"; -import { Library } from "../../src/library/index.js"; +import { Library, type RefreshEvent } from "../../src/library/index.js"; import type { CollectionsPage, FilesPage } from "../../src/client.js"; import type { MLData } from "../../src/mldata-fetch.js"; import type { Collection, EnteFile } from "../../src/model/types.js"; @@ -256,12 +258,14 @@ const file = ( // A mock client that serves scripted collection/file pages and per-file ML // payloads, recording every ML fetch request so incremental behaviour is -// provable. +// provable. A file in `mlErrorByFile` has ML data that did not decode, for +// the reason given there. class MLMockClient { userID = USER_ID; collectionsQueue: CollectionsPage[] = []; filesByCollection = new Map(); mlByFile = new Map(); + mlErrorByFile = new Map(); mlFetchCalls: number[][] = []; whoami(): { email: string; userID: number } { @@ -298,14 +302,20 @@ class MLMockClient { async fetchMLData(args: { fileIDs: number[]; fileKeys: Map; - }): Promise> { + }): Promise<{ + payloads: Map; + errors: Map; + }> { this.mlFetchCalls.push([...args.fileIDs]); - const result = new Map(); + const payloads = new Map(); + const errors = new Map(); for (const id of args.fileIDs) { const p = this.mlByFile.get(id); - if (p) result.set(id, p); + if (p) payloads.set(id, p); + const error = this.mlErrorByFile.get(id); + if (error) errors.set(id, error); } - return result; + return { payloads, errors }; } filesFor(collectionID: number, ...pages: FilesPage[]): void { @@ -449,6 +459,137 @@ describe("Library ML-data fetch on refresh", () => { } }); + it("reports a file whose ML data did not decode through status() and onProgress, and asks for it again", async () => { + const client = new MLMockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90), file(1002, 1, 95)], + deleted: [], + cursor: 95, + }); + client.mlByFile.set(1001, payload([0.5, 0.25, 0.75])); + client.mlErrorByFile.set( + 1002, + new Error("secretstream chunk authentication failed"), + ); + const events: RefreshEvent[] = []; + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: Infinity, + onProgress: (event) => events.push(event), + }); + try { + await vi.waitFor( + () => expect(lib.status().lastMLFetchAt).toBeGreaterThan(0), + { timeout: 2000, interval: 5 }, + ); + const reasons = [ + { + fileID: 1002, + error: "secretstream chunk authentication failed", + }, + ]; + expect(lib.status().mlDataErrors).toEqual(reasons); + expect(lib.status().mlStored).toBe(1); + expect(events.filter((e) => e.operation === "fetchMLData")).toEqual( + [ + { operation: "fetchMLData", status: "started" }, + { + operation: "fetchMLData", + status: "done", + fetched: 1, + mlDataErrors: reasons, + }, + ], + ); + + // Nothing was cached for 1002, so the pass a refresh starts asks + // for it again, and once its ML data decodes the reason is gone. + client.mlErrorByFile.clear(); + client.mlByFile.set(1002, payload([1, -2, 0.5])); + await lib.fresh(); + await vi.waitFor( + () => { + expect(lib.status().mlStored).toBe(2); + expect(lib.status().mlDataErrors).toEqual([]); + }, + { timeout: 2000, interval: 5 }, + ); + expect(client.mlFetchCalls.at(-1)).toEqual([1002]); + } finally { + await lib.close(); + } + }); + + it("still lists the last pass's reasons while the next pass runs", async () => { + const client = new MLMockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1002, 1, 95)], + deleted: [], + cursor: 95, + }); + client.mlErrorByFile.set( + 1002, + new Error("secretstream chunk authentication failed"), + ); + const reasons = [ + { fileID: 1002, error: "secretstream chunk authentication failed" }, + ]; + let release!: () => void; + const held = new Promise((r) => (release = r)); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: Infinity, + }); + try { + await vi.waitFor( + () => expect(lib.status().lastMLFetchAt).toBeGreaterThan(0), + { timeout: 2000, interval: 5 }, + ); + expect(lib.status().mlDataErrors).toEqual(reasons); + + // Hold the next ML data fetch open until the test releases it. + let fetchStarted!: () => void; + const started = new Promise((r) => (fetchStarted = r)); + const realFetch = client.fetchMLData.bind(client); + client.fetchMLData = async (args) => { + fetchStarted(); + await held; + return realFetch(args); + }; + client.mlErrorByFile.clear(); + client.mlByFile.set(1002, payload([1, -2, 0.5])); + + // The pass a refresh starts asks for 1002 again. While it is + // held, the reason from the pass before is still listed; once it + // ends, its own empty list takes the place of that one. + await lib.fresh(); + await started; + expect(lib.status().mlDataErrors).toEqual(reasons); + release(); + await vi.waitFor( + () => expect(lib.status().mlDataErrors).toEqual([]), + { timeout: 2000, interval: 5 }, + ); + } finally { + release(); + await lib.close(); + } + }); + it("close() resolves only after a running ML data fetch has stored its payloads", async () => { const client = new MLMockClient(); client.collectionsQueue.push({ diff --git a/test/mldata-fetch/mldata-fetch.test.ts b/test/mldata-fetch/mldata-fetch.test.ts new file mode 100644 index 0000000..b2e4b0e --- /dev/null +++ b/test/mldata-fetch/mldata-fetch.test.ts @@ -0,0 +1,54 @@ +// `fetchMLDataBatch`, against a server that sends one file's ML data. +// +// ML data that decrypts and gunzips but is not a JSON object is reported with +// the reason, as ML data that does not parse is, and is never returned as a +// payload: the library would fail to store it. + +import { gzipSync } from "node:zlib"; +import sodium from "libsodium-wrappers-sumo"; +import { beforeAll, describe, expect, it } from "vitest"; + +import { ApiClient } from "../../src/api/client.js"; +import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js"; +import { fetchMLDataBatch } from "../../src/mldata-fetch.js"; + +beforeAll(async () => { + await init(); + await sodium.ready; +}); + +// Fetch the ML data of file 1 from a server that sends `json`, gzipped and +// encrypted with the file's key. +const fetchServing = async (json: string) => { + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptBlob( + new Uint8Array(gzipSync(json)), + key, + ); + const data = [ + { + fileID: 1, + encryptedData: toBase64(ciphertext), + decryptionHeader: toBase64(header), + }, + ]; + const api = new ApiClient({ + fetch: async () => + new Response(JSON.stringify({ data }), { + headers: { "content-type": "application/json" }, + }), + }); + return fetchMLDataBatch(api, [1], new Map([[1, key]])); +}; + +describe("fetchMLDataBatch", () => { + it.each(["null", "5", "[]"])( + "gives a reason for ML data that is %s", + async (json) => { + const { payloads, errors } = await fetchServing(json); + + expect(payloads.size).toBe(0); + expect(errors.get(1)).toEqual(new Error("not a JSON object")); + }, + ); +});