From 893cc95ab18954ddbb87024a55ad2048b1fdd7a4 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 22 Sep 2026 10:20:13 +0000 Subject: [PATCH] feat(library/store): on-disk JSON metadata store for the local cache Add `MetadataStore` (src/library/store.ts): one metadata.json holding the user id, schema version, collections cursor, and decrypted collection and file records. Loaded whole into RAM with Maps for id lookups; rewritten whole through the download layer's fsync atomic writer (temp, fsync, rename, dir fsync). Binary keys are base64-encoded on disk. A missing, unparseable, or wrong-schema file loads as an empty store, because the file is only a cache the refresh unit repopulates. Directory 0700, file 0600, matching session.json. No lock file; no sync() beyond the writer. This unit only stores; issue 42's refresh unit populates it. (closes #41) Model: opus-4-8 --- TODO.md | 7 ++ src/library/store.ts | 188 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 src/library/store.ts diff --git a/TODO.md b/TODO.md index ccd4e66..212e598 100644 --- a/TODO.md +++ b/TODO.md @@ -18,6 +18,13 @@ Update the README API reference section to match the current implementation. # Completed Steps +- 2026-09-22: Added the on-disk JSON metadata store (`src/library/store.ts`, + issue 41), phase 2 of the cache/API work. One `metadata.json` holds the user + id, schema version, collections cursor, and the decrypted collection and file + records, loaded whole into RAM with `Map`s for id lookups and rewritten whole + through the download layer's fsync atomic writer. Missing, corrupt, or + wrong-schema files load empty; the directory is 0700 and the file 0600. The + store only stores — the refresh unit (issue 42) populates it. - 2026-09-22: Carried file size, thumbnail size, and the deletion flag through `decryptFile` (issue 37, foundation for the cache/API design). Live files now populate `file.size`/`thumbnail.size` from the server's `info` (left diff --git a/src/library/store.ts b/src/library/store.ts new file mode 100644 index 0000000..3174c1f --- /dev/null +++ b/src/library/store.ts @@ -0,0 +1,188 @@ +// On-disk JSON metadata store for the local cache. +// +// The store keeps one `metadata.json` file holding the account's server +// state: the user id, a schema version, the cursor for the incremental +// collections listing, and the decrypted collection and file records. The +// whole file is read into RAM on load and rewritten as a whole on save; there +// is no partial update and no lock file. A separate refresh unit populates the +// store from the server — this module only stores what it is given. +// +// The file is a cache, so it is never trusted to exist or to be intact: a +// missing or unreadable file loads as an empty store rather than an error, and +// the refresh unit then repopulates it. + +import { mkdir, chmod, readFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +import { writeAtomic } from "../download/index.js"; +import type { Collection, EnteFile, Microseconds } from "../model/types.js"; + +// Bumped only when the on-disk shape changes incompatibly. A file written +// under a different version is discarded on load (see `load`): re-fetching +// from the server is always safe and cheaper than migrating a cache. +export const METADATA_SCHEMA_VERSION = 1; + +// Directory and file modes match `session.json`: the records hold decrypted +// key material, so on a shared machine only the owner may read them. +const DIR_MODE = 0o700; +const FILE_MODE = 0o600; + +// On-disk shapes. They mirror the in-memory model exactly except for the +// binary `key`, which JSON cannot hold and which is stored as base64. +type StoredCollection = Omit & { key: string }; +type StoredFile = Omit & { key: string }; + +interface StoredMetadata { + schemaVersion: number; + userID: number; + collectionsSinceTime: Microseconds; + collections: StoredCollection[]; + files: StoredFile[]; +} + +const encodeKey = (key: Uint8Array): string => + Buffer.from(key).toString("base64"); + +const decodeKey = (encoded: string): Uint8Array => + new Uint8Array(Buffer.from(encoded, "base64")); + +// A file membership is identified by the pair (collectionID, fileID): the same +// underlying file can belong to several collections, each a distinct record +// with its own key. +const fileKey = (collectionID: number, fileID: number): string => + `${collectionID}:${fileID}`; + +export class MetadataStore { + readonly path: string; + readonly schemaVersion = METADATA_SCHEMA_VERSION; + userID = 0; + collectionsSinceTime: Microseconds = 0; + + private readonly collections = new Map(); + private readonly files = new Map(); + + private constructor(path: string) { + this.path = path; + } + + // Load the store at `path`. A missing file, an unreadable one, unparseable + // contents, or a mismatched schema version all yield an empty store bound + // to that path — never a thrown error, because the file is only a cache. + static async load(path: string): Promise { + const store = new MetadataStore(path); + + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch { + return store; + } + + try { + const parsed = JSON.parse(raw) as StoredMetadata; + if (parsed.schemaVersion !== METADATA_SCHEMA_VERSION) { + return store; + } + store.userID = parsed.userID ?? 0; + store.collectionsSinceTime = parsed.collectionsSinceTime ?? 0; + for (const stored of parsed.collections ?? []) { + const collection: Collection = { + ...stored, + key: decodeKey(stored.key), + }; + store.collections.set(collection.id, collection); + } + for (const stored of parsed.files ?? []) { + const file: EnteFile = { + ...stored, + key: decodeKey(stored.key), + }; + store.files.set(fileKey(file.collectionID, file.id), file); + } + } catch { + // Any corruption discards the partial result: a half-read cache is + // worse than an empty one, since the refresh unit will rebuild it. + return new MetadataStore(path); + } + + return store; + } + + // Rewrite the whole file. The directory is created 0700 and the file left + // 0600; the write itself is the download layer's durable atomic writer + // (temp file, fsync, rename, dir fsync), so a reader never sees a partial + // file and a crash cannot leave a truncated one. There is no lock file and + // no `sync()` beyond the writer's own fsyncs. + async save(): Promise { + const model: StoredMetadata = { + schemaVersion: METADATA_SCHEMA_VERSION, + userID: this.userID, + collectionsSinceTime: this.collectionsSinceTime, + collections: [...this.collections.values()].map((c) => ({ + ...c, + key: encodeKey(c.key), + })), + files: [...this.files.values()].map((f) => ({ + ...f, + key: encodeKey(f.key), + })), + }; + + const dir = dirname(this.path); + // chmod after mkdir so the mode is 0700 even when the directory + // already existed with a looser mode; mkdir alone would not tighten + // an existing directory. + await mkdir(dir, { recursive: true, mode: DIR_MODE }); + await chmod(dir, DIR_MODE); + + const payload = new TextEncoder().encode( + JSON.stringify(model, null, 2), + ); + await writeAtomic(this.path, payload); + // The atomic writer's temp file inherits the default mode; tighten the + // renamed file to 0600. The 0700 directory already keeps other users + // out during the brief window before this runs. + await chmod(this.path, FILE_MODE); + } + + getCollection(id: number): Collection | undefined { + return this.collections.get(id); + } + + listCollections(): Collection[] { + return [...this.collections.values()]; + } + + putCollection(collection: Collection): void { + this.collections.set(collection.id, collection); + } + + // Removing a collection also drops its file memberships: a file record is + // only meaningful as part of a collection the cache still knows about. + deleteCollection(id: number): void { + this.collections.delete(id); + for (const [key, file] of this.files) { + if (file.collectionID === id) { + this.files.delete(key); + } + } + } + + getFile(collectionID: number, fileID: number): EnteFile | undefined { + return this.files.get(fileKey(collectionID, fileID)); + } + + listFiles(collectionID: number): EnteFile[] { + return [...this.files.values()].filter( + (f) => f.collectionID === collectionID, + ); + } + + putFile(file: EnteFile): void { + this.files.set(fileKey(file.collectionID, file.id), file); + } + + deleteFile(collectionID: number, fileID: number): void { + this.files.delete(fileKey(collectionID, fileID)); + } +}