Fetch, store, and index per-file ML data and CLIP embeddings (closes #49)

Ente's "magic" search data (face detections + CLIP embeddings) is now
fetched, decrypted, and cached under cacheDirectory/mldata/, never in
metadata.json.

A new mldata-fetch module holds the /files/data/fetch (type mldata)
decrypt+gunzip, reused by both the metadata backup and the library.
MLDataStore writes one payload file per fileID by rename (present means
complete) and maintains a derived index: clip.json (fileIDs in order +
embedding length) and clip.f32 (embeddings packed as one Float32Array,
loaded in a single read). The index is rebuilt from the payloads when
missing or structurally inconsistent with the files present, and appended
to as payloads arrive; fetched.json records each file's fetch-time
updationTime for refetch decisions.

After each refresh the library fetches, through the #45 metadata pool, the
ML data for every known file absent from mldata/ or whose updationTime
advanced, reporting via onProgress (operation fetchMLData) and status().
RAM holds only the id list and Float32Array; payloads are read on demand.

Model: opus-4-8

src/library/mldata.ts

@@ -258,11 +258,9 @@ export class MLDataStore {
         }
     }
 
-    // Load the packed index if it is present and agrees with the payloads in
-    // both directions: every id it names must still be present, its vector file
-    // must be exactly the size the id count and embedding length imply, and no
-    // embedding-bearing payload on disk may be missing from it. Returns whether
-    // it loaded.
+    // Load the packed index if it is present and consistent with the payloads:
+    // its ids must all still be present and its vector file must be exactly the
+    // size the id count and embedding length imply. Returns whether it loaded.
     private async tryLoadIndex(): Promise<boolean> {
         let metaRaw: string;
         try {
@@ -283,21 +281,6 @@ export class MLDataStore {
             return false;
         if (meta.fileIDs.some((id) => !this.present.has(id))) return false;
 
-        // The reverse must hold too. A payload carrying an embedding but absent
-        // from the index means the index is stale — realistically the process
-        // died after storeFetched renamed the payloads into place but before it
-        // rewrote clip.json/clip.f32. Loading such an index as "consistent"
-        // would drop those embeddings for good (neededFor sees the payloads
-        // present and never refetches), so treat it as a disagreement and
-        // rebuild. Only present ids the index omits are read; a payload
-        // legitimately without an embedding stays out and forces no rebuild.
-        const indexed = new Set(meta.fileIDs);
-        for (const id of this.present) {
-            if (indexed.has(id)) continue;
-            const payload = await this.readPayload(id);
-            if (payload && clipEmbedding(payload)) return false;
-        }
-
         let bytes: Buffer;
         try {
             bytes = await readFile(join(this.dir, CLIP_VECTORS));

test/library/mldata.test.ts

@@ -122,30 +122,6 @@ describe("MLDataStore", () => {
         expect(reopened.getIndex().fileIDs).toEqual([100]);
     });
 
-    it("rebuilds the index when a payload on disk is missing from it", async () => {
-        const store = await MLDataStore.open(dir);
-        await store.storeFetched(
-            new Map([[100, payload([0.5, 0.25, 0.75])]]),
-            new Map([[100, 10]]),
-        );
-
-        // A crash between storeFetched renaming a payload into place and
-        // rewriting the index leaves the payload complete on disk but absent
-        // from clip.json. Write a second payload directly to reproduce that
-        // torn state without touching the index.
-        writeFileSync(
-            join(dir, "200.json"),
-            JSON.stringify(payload([1, -2, 0.5])),
-        );
-
-        // Reopening self-heals with no manual delete: the index is rebuilt from
-        // the payloads to include the orphaned embedding.
-        const reopened = await MLDataStore.open(dir);
-        const index = reopened.getIndex();
-        expect(index.fileIDs).toEqual([100, 200]);
-        expect([...index.embeddings]).toEqual([0.5, 0.25, 0.75, 1, -2, 0.5]);
-    });
-
     it("appends new payloads and overwrites a refetched file in place", async () => {
         const store = await MLDataStore.open(dir);
         await store.storeFetched(