From 72ea8dcb015de0c5e517863cabd3407114e26050 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 22 Sep 2026 12:52:50 +0200 Subject: [PATCH] On-disk JSON metadata store for the local cache (closes #41) Adds the metadata.json store: loads whole into RAM with id-lookup Maps, rewrites whole through the exported fsync atomic writer (temp, fsync, rename, dir fsync); a missing, unparseable, or wrong-schema file loads as empty (it is a cache); directory 0700, file 0600. No lock file, no public sync(). Model: opus-4-8 --- src/library/store.ts | 188 ++++++++++++++++++++++++++++ test/library/store.test.ts | 246 +++++++++++++++++++++++++++++++++++++ 2 files changed, 434 insertions(+) create mode 100644 src/library/store.ts create mode 100644 test/library/store.test.ts diff --git a/src/library/store.ts b/src/library/store.ts new file mode 100644 index 0000000..3174c1f --- /dev/null +++ b/src/library/store.ts @@ -0,0 +1,188 @@ +// On-disk JSON metadata store for the local cache. +// +// The store keeps one `metadata.json` file holding the account's server +// state: the user id, a schema version, the cursor for the incremental +// collections listing, and the decrypted collection and file records. The +// whole file is read into RAM on load and rewritten as a whole on save; there +// is no partial update and no lock file. A separate refresh unit populates the +// store from the server — this module only stores what it is given. +// +// The file is a cache, so it is never trusted to exist or to be intact: a +// missing or unreadable file loads as an empty store rather than an error, and +// the refresh unit then repopulates it. + +import { mkdir, chmod, readFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +import { writeAtomic } from "../download/index.js"; +import type { Collection, EnteFile, Microseconds } from "../model/types.js"; + +// Bumped only when the on-disk shape changes incompatibly. A file written +// under a different version is discarded on load (see `load`): re-fetching +// from the server is always safe and cheaper than migrating a cache. +export const METADATA_SCHEMA_VERSION = 1; + +// Directory and file modes match `session.json`: the records hold decrypted +// key material, so on a shared machine only the owner may read them. +const DIR_MODE = 0o700; +const FILE_MODE = 0o600; + +// On-disk shapes. They mirror the in-memory model exactly except for the +// binary `key`, which JSON cannot hold and which is stored as base64. +type StoredCollection = Omit & { key: string }; +type StoredFile = Omit & { key: string }; + +interface StoredMetadata { + schemaVersion: number; + userID: number; + collectionsSinceTime: Microseconds; + collections: StoredCollection[]; + files: StoredFile[]; +} + +const encodeKey = (key: Uint8Array): string => + Buffer.from(key).toString("base64"); + +const decodeKey = (encoded: string): Uint8Array => + new Uint8Array(Buffer.from(encoded, "base64")); + +// A file membership is identified by the pair (collectionID, fileID): the same +// underlying file can belong to several collections, each a distinct record +// with its own key. +const fileKey = (collectionID: number, fileID: number): string => + `${collectionID}:${fileID}`; + +export class MetadataStore { + readonly path: string; + readonly schemaVersion = METADATA_SCHEMA_VERSION; + userID = 0; + collectionsSinceTime: Microseconds = 0; + + private readonly collections = new Map(); + private readonly files = new Map(); + + private constructor(path: string) { + this.path = path; + } + + // Load the store at `path`. A missing file, an unreadable one, unparseable + // contents, or a mismatched schema version all yield an empty store bound + // to that path — never a thrown error, because the file is only a cache. + static async load(path: string): Promise { + const store = new MetadataStore(path); + + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch { + return store; + } + + try { + const parsed = JSON.parse(raw) as StoredMetadata; + if (parsed.schemaVersion !== METADATA_SCHEMA_VERSION) { + return store; + } + store.userID = parsed.userID ?? 0; + store.collectionsSinceTime = parsed.collectionsSinceTime ?? 0; + for (const stored of parsed.collections ?? []) { + const collection: Collection = { + ...stored, + key: decodeKey(stored.key), + }; + store.collections.set(collection.id, collection); + } + for (const stored of parsed.files ?? []) { + const file: EnteFile = { + ...stored, + key: decodeKey(stored.key), + }; + store.files.set(fileKey(file.collectionID, file.id), file); + } + } catch { + // Any corruption discards the partial result: a half-read cache is + // worse than an empty one, since the refresh unit will rebuild it. + return new MetadataStore(path); + } + + return store; + } + + // Rewrite the whole file. The directory is created 0700 and the file left + // 0600; the write itself is the download layer's durable atomic writer + // (temp file, fsync, rename, dir fsync), so a reader never sees a partial + // file and a crash cannot leave a truncated one. There is no lock file and + // no `sync()` beyond the writer's own fsyncs. + async save(): Promise { + const model: StoredMetadata = { + schemaVersion: METADATA_SCHEMA_VERSION, + userID: this.userID, + collectionsSinceTime: this.collectionsSinceTime, + collections: [...this.collections.values()].map((c) => ({ + ...c, + key: encodeKey(c.key), + })), + files: [...this.files.values()].map((f) => ({ + ...f, + key: encodeKey(f.key), + })), + }; + + const dir = dirname(this.path); + // chmod after mkdir so the mode is 0700 even when the directory + // already existed with a looser mode; mkdir alone would not tighten + // an existing directory. + await mkdir(dir, { recursive: true, mode: DIR_MODE }); + await chmod(dir, DIR_MODE); + + const payload = new TextEncoder().encode( + JSON.stringify(model, null, 2), + ); + await writeAtomic(this.path, payload); + // The atomic writer's temp file inherits the default mode; tighten the + // renamed file to 0600. The 0700 directory already keeps other users + // out during the brief window before this runs. + await chmod(this.path, FILE_MODE); + } + + getCollection(id: number): Collection | undefined { + return this.collections.get(id); + } + + listCollections(): Collection[] { + return [...this.collections.values()]; + } + + putCollection(collection: Collection): void { + this.collections.set(collection.id, collection); + } + + // Removing a collection also drops its file memberships: a file record is + // only meaningful as part of a collection the cache still knows about. + deleteCollection(id: number): void { + this.collections.delete(id); + for (const [key, file] of this.files) { + if (file.collectionID === id) { + this.files.delete(key); + } + } + } + + getFile(collectionID: number, fileID: number): EnteFile | undefined { + return this.files.get(fileKey(collectionID, fileID)); + } + + listFiles(collectionID: number): EnteFile[] { + return [...this.files.values()].filter( + (f) => f.collectionID === collectionID, + ); + } + + putFile(file: EnteFile): void { + this.files.set(fileKey(file.collectionID, file.id), file); + } + + deleteFile(collectionID: number, fileID: number): void { + this.files.delete(fileKey(collectionID, fileID)); + } +} diff --git a/test/library/store.test.ts b/test/library/store.test.ts new file mode 100644 index 0000000..21212b3 --- /dev/null +++ b/test/library/store.test.ts @@ -0,0 +1,246 @@ +/** + * Tests for the on-disk JSON metadata store (`MetadataStore`). + * + * The store is the local cache the library keeps of the account's server + * state: one `metadata.json` file holding the user id, a schema version, the + * cursor for the incremental collections listing, and the decrypted + * collection and file records. The whole file is read into RAM on load and + * rewritten as a whole on save. A separate refresh unit (issue #42) is what + * populates it; this unit only stores. + * + * Four contracts are load-bearing and each is exercised below: + * + * 1. **Round-trip fidelity.** Everything put into the store — including the + * binary decryption keys, which JSON cannot hold directly and which the + * store base64-encodes — comes back byte-for-byte after a save and a fresh + * load. A cache that quietly dropped or mangled a field would hand the + * caller wrong keys or stale metadata. + * + * 2. **A missing or corrupt file loads as an empty store, never an error.** + * The file is only a cache: if it is absent (first run) or unreadable + * (interrupted write on an older build, disk corruption, hand-editing), + * the right answer is to start empty and let the refresh unit repopulate, + * not to crash the whole library. + * + * 3. **Writes are atomic and durable.** The store reuses the same + * fsync-before-rename atomic writer the download layer uses, so a reader + * never sees a half-written file and a crash cannot leave a truncated one. + * The observable consequence tested here is that a save leaves exactly the + * destination file behind — no temporary sibling — and that overwriting an + * existing store preserves a complete, re-loadable file. + * + * 4. **Permissions match `session.json`.** The directory is `0700` and the + * file is `0600`, because the records contain decrypted key material and + * must not be readable by other users on a shared machine. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + mkdtempSync, + rmSync, + readdirSync, + statSync, + writeFileSync, + mkdirSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + MetadataStore, + METADATA_SCHEMA_VERSION, +} from "../../src/library/store.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +// A representative decrypted collection, including a binary key and all three +// magic-metadata layers, so the round-trip test proves every field survives. +const sampleCollection = (): Collection => ({ + id: 12345, + ownerID: 42, + key: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]), + name: "Holiday 2026", + type: "album", + updationTime: 1_700_000_000_000_000, + isShared: true, + magicMetadata: { visibility: 0 }, + pubMagicMetadata: { subType: 0, coverID: 999 }, + sharedMagicMetadata: { note: "shared with a friend" }, +}); + +// A representative decrypted file membership: metadata, both blob headers, a +// binary key, a content hash, and file/thumbnail sizes. +const sampleFile = (): EnteFile => ({ + id: 67890, + collectionID: 12345, + ownerID: 42, + key: new Uint8Array([9, 8, 7, 6, 5, 4, 3, 2, 1]), + metadata: { + title: "IMG_0001.jpg", + fileType: "image", + creationTime: 1_699_000_000_000_000, + modificationTime: 1_699_000_500_000_000, + latitude: 52.52, + longitude: 13.405, + hash: "sha256:deadbeef", + }, + magicMetadata: { editedName: "sunset" }, + pubMagicMetadata: { editedTime: 1_699_000_600_000_000 }, + file: { decryptionHeader: "ZmlsZUhlYWRlcg==", size: 4_194_304 }, + thumbnail: { decryptionHeader: "dGh1bWJIZWFkZXI=", size: 8192 }, + updationTime: 1_700_000_100_000_000, +}); + +describe("MetadataStore", () => { + let dir: string; + let path: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "quak-store-")); + // Deliberately nest the store one level below the temp dir so save() + // has to create its own directory and set its mode. + path = join(dir, "cache", "metadata.json"); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("round-trips the whole model, keys and all", async () => { + const store = await MetadataStore.load(path); + store.userID = 42; + store.collectionsSinceTime = 1_700_000_000_000_000; + store.putCollection(sampleCollection()); + store.putFile(sampleFile()); + await store.save(); + + const reloaded = await MetadataStore.load(path); + expect(reloaded.userID).toBe(42); + expect(reloaded.collectionsSinceTime).toBe(1_700_000_000_000_000); + + // The binary key must come back as the exact bytes, not a base64 + // string or a plain object of numbered keys. + const collection = reloaded.getCollection(12345); + expect(collection).toEqual(sampleCollection()); + expect(collection?.key).toBeInstanceOf(Uint8Array); + + const file = reloaded.getFile(12345, 67890); + expect(file).toEqual(sampleFile()); + expect(file?.key).toBeInstanceOf(Uint8Array); + + expect(reloaded.listCollections()).toHaveLength(1); + expect(reloaded.listFiles(12345)).toHaveLength(1); + }); + + it("writes the declared schema version", async () => { + const store = await MetadataStore.load(path); + await store.save(); + const reloaded = await MetadataStore.load(path); + expect(reloaded.schemaVersion).toBe(METADATA_SCHEMA_VERSION); + }); + + it("loads an empty store when the file is missing", async () => { + const store = await MetadataStore.load(path); + expect(store.userID).toBe(0); + expect(store.listCollections()).toEqual([]); + expect(store.getCollection(1)).toBeUndefined(); + }); + + it("loads an empty store when the file is corrupt", async () => { + mkdirSync(join(dir, "cache"), { recursive: true }); + writeFileSync(path, "{ this is not valid json ]["); + const store = await MetadataStore.load(path); + expect(store.listCollections()).toEqual([]); + expect(store.listFiles(12345)).toEqual([]); + }); + + it("loads an empty store when the schema version does not match", async () => { + // A cache written by a future build with an incompatible schema is + // discarded rather than misread; the refresh unit repopulates it. + mkdirSync(join(dir, "cache"), { recursive: true }); + writeFileSync( + path, + JSON.stringify({ + schemaVersion: METADATA_SCHEMA_VERSION + 1, + userID: 42, + collectionsSinceTime: 0, + collections: [], + files: [], + }), + ); + const store = await MetadataStore.load(path); + expect(store.userID).toBe(0); + expect(store.listCollections()).toEqual([]); + }); + + it("creates the directory 0700 and the file 0600", async () => { + const store = await MetadataStore.load(path); + store.putCollection(sampleCollection()); + await store.save(); + + // Directory 0700, file 0600: on a shared machine the decrypted keys + // in this file must be readable only by their owner. Mask to the + // permission bits; the file-type bits are not part of the assertion. + expect(statSync(join(dir, "cache")).mode & 0o777).toBe(0o700); + expect(statSync(path).mode & 0o777).toBe(0o600); + }); + + it("leaves exactly the destination behind, with no temp sibling", async () => { + const store = await MetadataStore.load(path); + store.putCollection(sampleCollection()); + await store.save(); + // The atomic writer stages a temporary file and renames it into + // place; on success nothing temporary is left in the directory. + expect(readdirSync(join(dir, "cache"))).toEqual(["metadata.json"]); + }); + + it("overwrites an existing store atomically and stays re-loadable", async () => { + const first = await MetadataStore.load(path); + first.userID = 1; + first.putCollection(sampleCollection()); + await first.save(); + + const second = await MetadataStore.load(path); + second.userID = 2; + second.deleteCollection(12345); + await second.save(); + + const reloaded = await MetadataStore.load(path); + expect(reloaded.userID).toBe(2); + expect(reloaded.getCollection(12345)).toBeUndefined(); + expect(readdirSync(join(dir, "cache"))).toEqual(["metadata.json"]); + }); + + it("deletes a collection together with its file memberships", async () => { + const store = await MetadataStore.load(path); + store.putCollection(sampleCollection()); + store.putFile(sampleFile()); + store.deleteCollection(12345); + expect(store.getCollection(12345)).toBeUndefined(); + expect(store.getFile(12345, 67890)).toBeUndefined(); + expect(store.listFiles(12345)).toEqual([]); + }); + + it("scopes file records to their collection membership", async () => { + // The same underlying file can be a member of two collections, each a + // separate record with its own key. Storing one must not touch the + // other, and lookups are per membership. + const store = await MetadataStore.load(path); + const inA = sampleFile(); + const inB: EnteFile = { + ...sampleFile(), + collectionID: 55555, + key: new Uint8Array([100, 101, 102]), + }; + store.putFile(inA); + store.putFile(inB); + + expect(store.getFile(12345, 67890)?.key).toEqual(inA.key); + expect(store.getFile(55555, 67890)?.key).toEqual(inB.key); + expect(store.listFiles(12345)).toHaveLength(1); + expect(store.listFiles(55555)).toHaveLength(1); + + store.deleteFile(12345, 67890); + expect(store.getFile(12345, 67890)).toBeUndefined(); + expect(store.getFile(55555, 67890)?.key).toEqual(inB.key); + }); +});