quak backup writes each file's ML data into its JSON (closes #163)

lib.backup() now waits for an ML data fetch, joining the one its refresh
started or starting one, before it writes the per-file JSON, and each
file's JSON carries the cached payload as mlData. When the fetch fails,
each file with no cached ML data gets mlDataError and an entry in
failures.json, so the result counts it as failed and quak backup exits 1;
the next run fetches again.

Judgement call: the wait comes after the originals are downloaded, so the fetch runs alongside the downloads.
Judgement call: a failed ML fetch is recorded per file in failures.json, which is how the exit code goes non-zero without changing src/cli-commands.ts.

Model: opus-5-5

src/backup.ts

@@ -3,14 +3,15 @@
 // `lib.backup()` waits for a completed refresh of the library (a failed one
 // fails the backup before any file is touched), then, for every file in scope,
 // puts its original at its save path under `downloadDirectory`, as
-// `Photo.download()` does, and rebuilds the derived views (per-file sidecars,
-// per-collection symlink trees, per-collection JSON) from the model. The
-// on-disk layout:
+// `Photo.download()` does, waits for an ML data fetch, and rebuilds the derived
+// views (per-file sidecars, per-collection symlink trees, per-collection JSON)
+// from the model. The on-disk layout:
 //
 //     <downloadDirectory>/
 //         YYYY/YYYY-MM/YYYY-MM-DD/
 //             YYYY-MM-DD.<fileID>.<ext>    the decrypted bytes (the save path)
-//             YYYY-MM-DD.<fileID>.json     per-file metadata sidecar
+//             YYYY-MM-DD.<fileID>.json     per-file metadata sidecar, with
+//                                          the file's ML data
 //         collections/<name>/<title>       symlink to the original
 //         collections/<name>.json          per-collection metadata
 //         failures.json                    durable ledger of unresolved failures
@@ -29,13 +30,15 @@
 // directories of albums that no longer exist.
 //
 // Resilience (issue #8): no per-file condition aborts the run. A failed
-// download or a failed symlink is caught, recorded in `failures.json` with a
-// classification, a running attempt count, and the last-tried time, and the run
-// continues. `result.failed` — and thus the CLI's exit code — stays non-zero
-// while any failure remains unresolved and clears once every one succeeds. Each
-// run reconciles the ledger against the files it attempted, so an entry for a
-// file that has since left the library (deleted) or this run's scope is dropped
-// rather than counted forever, which would poison a scheduled backup's exit code.
+// download, a failed symlink, or ML data missing because the ML data fetch
+// failed is caught, recorded in `failures.json` with a classification, a
+// running attempt count, and the last-tried time, and the run continues.
+// `result.failed` — and thus the CLI's exit code — stays non-zero while any
+// failure remains unresolved and clears once every one succeeds. Each run
+// reconciles the ledger against the files it attempted, so an entry for a file
+// that has since left the library (deleted) or this run's scope is dropped
+// rather than counted forever, which would poison a scheduled backup's exit
+// code.
 
 import {
     lstatSync,
@@ -60,6 +63,7 @@ import {
     storedAtSavePath,
 } from "./library/content.js";
 import { representative } from "./library/records.js";
+import type { MLData } from "./mldata-fetch.js";
 import type { Collection, EnteFile } from "./model/types.js";
 
 export type ProgressCallback = (message: string) => void;
@@ -117,6 +121,13 @@ export interface BackupLibrary {
         destination: string,
     ): Promise<{ path: string; videoPath?: string }>;
     thumbnail(fileID: number): Promise<{ path: string }>;
+    // Wait for an ML data fetch to complete, joining one already running or
+    // starting one. Rejects with the reason when it fails; resolves at once
+    // when the library cannot fetch ML data.
+    fetchMLData(): Promise<void>;
+    // A file's cached ML data, as `lib.mldata.forFile()` returns it, or
+    // undefined when none is cached.
+    mlData(fileID: number): Promise<MLData | undefined>;
 }
 
 type FailureClass = "transient" | "permanent" | "unknown";
@@ -340,7 +351,13 @@ const saveLedger = (path: string, ledger: Map<number, FailureEntry>): void => {
     );
 };
 
-const writeSidecar = (path: string, file: EnteFile): void => {
+// The file's JSON: its basic fields, its magic metadata, and its ML data, or
+// the reason the ML data is missing.
+const writeSidecar = (
+    path: string,
+    file: EnteFile,
+    ml: { mlData?: MLData; mlDataError?: string },
+): void => {
     const meta: Record<string, unknown> = {
         id: file.id,
         collectionID: file.collectionID,
@@ -349,6 +366,8 @@ const writeSidecar = (path: string, file: EnteFile): void => {
     };
     if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
     if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
+    if (ml.mlData) meta.mlData = ml.mlData;
+    if (ml.mlDataError) meta.mlDataError = ml.mlDataError;
     writeFileSync(path, JSON.stringify(meta, null, 2));
 };
 
@@ -497,12 +516,37 @@ export const runBackup = async (
     }
 
     // Phase 2: rebuild the derived views from the model. Sidecars first, for
-    // every present original (this repairs stale ones).
+    // every present original (this repairs stale ones), each with the file's
+    // ML data once an ML data fetch has completed. When the fetch fails, a
+    // file with no cached ML data gets the reason instead and is recorded as
+    // failed. The next run fetches its ML data again because none is cached.
     if (includeOriginals) {
+        let mlDataError: string | undefined;
+        try {
+            log("Fetching ML data...");
+            await lib.fetchMLData();
+        } catch (err) {
+            mlDataError = errorMessage(err);
+            log(`FAILED ML data: ${mlDataError}`);
+        }
         for (const file of distinct.values()) {
-            if (storedAtSavePath(downloadDirectory, file) !== undefined) {
-                const path = savePath(downloadDirectory, file);
-                writeSidecar(withExtension(path, ".json"), file);
+            if (storedAtSavePath(downloadDirectory, file) === undefined) {
+                continue;
+            }
+            const path = withExtension(
+                savePath(downloadDirectory, file),
+                ".json",
+            );
+            const mlData = await lib.mlData(file.id);
+            if (mlData === undefined && mlDataError !== undefined) {
+                writeSidecar(path, file, { mlDataError });
+                recordFailure(
+                    file,
+                    collectionName.get(file.collectionID) ?? "",
+                    new Error(`ML data: ${mlDataError}`),
+                );
+            } else {
+                writeSidecar(path, file, { mlData });
             }
         }
     }