quak/src/library/store.ts

// On-disk JSON metadata store for the local cache.
//
// The store keeps one `metadata.json` file holding the account's server
// state: the user id, a schema version, the cursor for the incremental
// collections listing, the decrypted collection and file records, and the
// collections and files the refresh could not decrypt. The whole file is read
// into RAM on load and rewritten as a whole on save; there is no partial
// update and no lock file. A separate refresh unit populates the store from
// the server — this module only stores what it is given.
//
// The file is a cache, so it is never trusted to exist or to be intact: a
// missing or unreadable file loads as an empty store rather than an error, and
// the refresh unit then repopulates it.

import { mkdir, chmod, readFile } from "node:fs/promises";
import { dirname } from "node:path";

import { writeAtomic } from "../download/index.js";
import type { Collection, EnteFile, Microseconds } from "../model/types.js";
import { representative } from "./records.js";

// Bumped only when the on-disk shape changes incompatibly. A file written
// under a different version is discarded on load (see `load`): re-fetching
// from the server is always safe and cheaper than migrating a cache.
export const METADATA_SCHEMA_VERSION = 1;

// Directory and file modes match `session.json`: the records hold decrypted
// key material, so on a shared machine only the owner may read them.
const DIR_MODE = 0o700;
const FILE_MODE = 0o600;

// On-disk shapes. They mirror the in-memory model exactly except for the
// binary `key`, which JSON cannot hold and which is stored as base64.
type StoredCollection = Omit<Collection, "key"> & { key: string };
type StoredFile = Omit<EnteFile, "key"> & { key: string };

// A collection, or with `fileID` a file in it, that the last refresh to fetch
// it could not decrypt or parse, and why. It is kept until a later refresh
// fetches it again, because a refresh fetches only what changed.
export interface Undecryptable {
    collectionID: number;
    fileID?: number;
    error: string;
}

interface StoredMetadata {
    schemaVersion: number;
    userID: number;
    collectionsSinceTime: Microseconds;
    collections: StoredCollection[];
    files: StoredFile[];
    undecryptable: Undecryptable[];
}

const encodeKey = (key: Uint8Array): string =>
    Buffer.from(key).toString("base64");

const decodeKey = (encoded: string): Uint8Array =>
    new Uint8Array(Buffer.from(encoded, "base64"));

// A file membership is identified by the pair (collectionID, fileID): the same
// underlying file can belong to several collections, each a distinct record
// with its own key.
const fileKey = (collectionID: number, fileID: number): string =>
    `${collectionID}:${fileID}`;

export class MetadataStore {
    readonly path: string;
    readonly schemaVersion = METADATA_SCHEMA_VERSION;
    userID = 0;
    collectionsSinceTime: Microseconds = 0;

    // True when `load` populated this store from a valid existing file; false
    // on a first run or a missing/corrupt/wrong-version file that loaded empty.
    // `Library.open` reads it to decide whether the first refresh may run in
    // the background (an existing copy already serves reads) or must be awaited.
    loadedFromDisk = false;

    private readonly collections = new Map<number, Collection>();
    private readonly files = new Map<string, EnteFile>();
    private undecryptable: Undecryptable[] = [];

    private constructor(path: string) {
        this.path = path;
    }

    // Load the store at `path`. A missing file, an unreadable one, unparseable
    // contents, or a mismatched schema version all yield an empty store bound
    // to that path — never a thrown error, because the file is only a cache.
    static async load(path: string): Promise<MetadataStore> {
        const store = new MetadataStore(path);

        let raw: string;
        try {
            raw = await readFile(path, "utf8");
        } catch {
            return store;
        }

        try {
            const parsed = JSON.parse(raw) as StoredMetadata;
            if (parsed.schemaVersion !== METADATA_SCHEMA_VERSION) {
                return store;
            }
            store.loadedFromDisk = true;
            store.userID = parsed.userID ?? 0;
            store.collectionsSinceTime = parsed.collectionsSinceTime ?? 0;
            for (const stored of parsed.collections ?? []) {
                const collection: Collection = {
                    ...stored,
                    key: decodeKey(stored.key),
                };
                store.collections.set(collection.id, collection);
            }
            for (const stored of parsed.files ?? []) {
                const file: EnteFile = {
                    ...stored,
                    key: decodeKey(stored.key),
                };
                store.files.set(fileKey(file.collectionID, file.id), file);
            }
            for (const record of parsed.undecryptable) {
                if (typeof record.collectionID !== "number") {
                    throw new Error("damaged record of what did not decrypt");
                }
                store.undecryptable.push(record);
            }
        } catch {
            // Any corruption discards the partial result: a half-read cache is
            // worse than an empty one, since the refresh unit will rebuild it.
            return new MetadataStore(path);
        }

        return store;
    }

    // Rewrite the whole file. The directory is created 0700 and the file left
    // 0600; the write itself is the download layer's durable atomic writer
    // (temp file, fsync, rename, dir fsync), so a reader never sees a partial
    // file and a crash cannot leave a truncated one. There is no lock file and
    // no `sync()` beyond the writer's own fsyncs.
    async save(): Promise<void> {
        const model: StoredMetadata = {
            schemaVersion: METADATA_SCHEMA_VERSION,
            userID: this.userID,
            collectionsSinceTime: this.collectionsSinceTime,
            collections: [...this.collections.values()].map((c) => ({
                ...c,
                key: encodeKey(c.key),
            })),
            files: [...this.files.values()].map((f) => ({
                ...f,
                key: encodeKey(f.key),
            })),
            undecryptable: this.undecryptable,
        };

        const dir = dirname(this.path);
        // chmod after mkdir so the mode is 0700 even when the directory
        // already existed with a looser mode; mkdir alone would not tighten
        // an existing directory.
        await mkdir(dir, { recursive: true, mode: DIR_MODE });
        await chmod(dir, DIR_MODE);

        const payload = new TextEncoder().encode(
            JSON.stringify(model, null, 2),
        );
        await writeAtomic(this.path, payload);
        // The atomic writer's temp file inherits the default mode; tighten the
        // renamed file to 0600. The 0700 directory already keeps other users
        // out during the brief window before this runs.
        await chmod(this.path, FILE_MODE);
    }

    getCollection(id: number): Collection | undefined {
        return this.collections.get(id);
    }

    listCollections(): Collection[] {
        return [...this.collections.values()];
    }

    putCollection(collection: Collection): void {
        this.collections.set(collection.id, collection);
    }

    // Removing a collection also drops its file memberships, and any record
    // that it or one of its files could not be decrypted: a file record is
    // only meaningful as part of a collection the cache still knows about.
    deleteCollection(id: number): void {
        this.collections.delete(id);
        for (const [key, file] of this.files) {
            if (file.collectionID === id) {
                this.files.delete(key);
            }
        }
        this.undecryptable = this.undecryptable.filter(
            (record) => record.collectionID !== id,
        );
    }

    getFile(collectionID: number, fileID: number): EnteFile | undefined {
        return this.files.get(fileKey(collectionID, fileID));
    }

    // The membership of a file its record is read from (`representative`), or
    // undefined. Any membership could fetch the bytes, but the content cache
    // resolves a fileID to a file this way so that it dates the save path
    // from the same membership as `photo.savePath`.
    getFileByID(fileID: number): EnteFile | undefined {
        const memberships: EnteFile[] = [];
        for (const file of this.files.values()) {
            if (file.id === fileID) memberships.push(file);
        }
        return memberships.length > 0 ? representative(memberships) : undefined;
    }

    listFiles(collectionID: number): EnteFile[] {
        return [...this.files.values()].filter(
            (f) => f.collectionID === collectionID,
        );
    }

    putFile(file: EnteFile): void {
        this.files.set(fileKey(file.collectionID, file.id), file);
    }

    deleteFile(collectionID: number, fileID: number): void {
        this.files.delete(fileKey(collectionID, fileID));
    }

    listUndecryptable(): Undecryptable[] {
        return [...this.undecryptable];
    }

    // Replaces any earlier record of the same collection or file.
    putUndecryptable(record: Undecryptable): void {
        this.deleteUndecryptable(record.collectionID, record.fileID);
        this.undecryptable.push(record);
    }

    // Forgets the record of the collection, or with `fileID` of that file in
    // it. Returns whether there was one.
    deleteUndecryptable(collectionID: number, fileID?: number): boolean {
        const before = this.undecryptable.length;
        this.undecryptable = this.undecryptable.filter(
            (record) =>
                record.collectionID !== collectionID ||
                record.fileID !== fileID,
        );
        return this.undecryptable.length < before;
    }
}