Fetch, store, and index per-file ML data and CLIP embeddings (closes #49)

Ente's per-file ML data (face detections + CLIP embeddings) is fetched,
decrypted, and cached under cacheDirectory/mldata/, never in
metadata.json. A new mldata-fetch module holds the decrypt+gunzip, reused
by the metadata backup and the library. MLDataStore writes one payload
file per fileID by rename (present means complete) and derives clip.json
+ clip.f32 (embeddings packed as one Float32Array, one-read load). On open
the index is rebuilt whenever it disagrees with the payloads in either
direction — an id it names is gone, or an embedding-bearing payload on
disk is absent from it — so a crash between writing payloads and rewriting
the index self-heals instead of dropping those embeddings. Appended to as
payloads arrive; fetched.json records each file's fetch-time updationTime.

Model: opus-4-8

src/library/mldata.ts

@@ -258,9 +258,11 @@ export class MLDataStore {
         }
     }
 
-    // Load the packed index if it is present and consistent with the payloads:
-    // its ids must all still be present and its vector file must be exactly the
-    // size the id count and embedding length imply. Returns whether it loaded.
+    // Load the packed index if it is present and agrees with the payloads in
+    // both directions: every id it names must still be present, its vector file
+    // must be exactly the size the id count and embedding length imply, and no
+    // embedding-bearing payload on disk may be missing from it. Returns whether
+    // it loaded.
     private async tryLoadIndex(): Promise<boolean> {
         let metaRaw: string;
         try {
@@ -281,6 +283,21 @@ export class MLDataStore {
             return false;
         if (meta.fileIDs.some((id) => !this.present.has(id))) return false;
 
+        // The reverse must hold too. A payload carrying an embedding but absent
+        // from the index means the index is stale — realistically the process
+        // died after storeFetched renamed the payloads into place but before it
+        // rewrote clip.json/clip.f32. Loading such an index as "consistent"
+        // would drop those embeddings for good (neededFor sees the payloads
+        // present and never refetches), so treat it as a disagreement and
+        // rebuild. Only present ids the index omits are read; a payload
+        // legitimately without an embedding stays out and forces no rebuild.
+        const indexed = new Set(meta.fileIDs);
+        for (const id of this.present) {
+            if (indexed.has(id)) continue;
+            const payload = await this.readPayload(id);
+            if (payload && clipEmbedding(payload)) return false;
+        }
+
         let bytes: Buffer;
         try {
             bytes = await readFile(join(this.dir, CLIP_VECTORS));

test/library/mldata.test.ts

@@ -122,6 +122,30 @@ describe("MLDataStore", () => {
         expect(reopened.getIndex().fileIDs).toEqual([100]);
     });
 
+    it("rebuilds the index when a payload on disk is missing from it", async () => {
+        const store = await MLDataStore.open(dir);
+        await store.storeFetched(
+            new Map([[100, payload([0.5, 0.25, 0.75])]]),
+            new Map([[100, 10]]),
+        );
+
+        // A crash between storeFetched renaming a payload into place and
+        // rewriting the index leaves the payload complete on disk but absent
+        // from clip.json. Write a second payload directly to reproduce that
+        // torn state without touching the index.
+        writeFileSync(
+            join(dir, "200.json"),
+            JSON.stringify(payload([1, -2, 0.5])),
+        );
+
+        // Reopening self-heals with no manual delete: the index is rebuilt from
+        // the payloads to include the orphaned embedding.
+        const reopened = await MLDataStore.open(dir);
+        const index = reopened.getIndex();
+        expect(index.fileIDs).toEqual([100, 200]);
+        expect([...index.embeddings]).toEqual([0.5, 0.25, 0.75, 1, -2, 0.5]);
+    });
+
     it("appends new payloads and overwrites a refetched file in place", async () => {
         const store = await MLDataStore.open(dir);
         await store.storeFetched(