/** * Tests for the `quak backup` logic, now built on the library API (issue #51). * * `lib.backup({ downloadDirectory })` refreshes the library, fetches each * pending file's original through the content cache/pools, and materialises the * unchanged on-disk layout: * * / * originals/ * . the decrypted bytes ("present means complete") * .json per-file metadata sidecar (rebuilt each run) * collections/ * / symlink into ../originals (rebuilt each run) * <name>.json per-collection metadata (rebuilt each run) * failures.json durable ledger of unresolved failures * * The properties that distinguish backup from a naive download loop, and that * these tests lock down: * * 1. Present-means-complete: an original already on disk is not re-fetched, so * runs are idempotent and interrupted runs resume. * 2. Per-file resilience: a download failure or a symlink failure is recorded * and the run continues (issue #8); the derived symlink/JSON views are * rebuilt from the model every run. * 3. A durable `failures.json` records each unresolved failure's classification, * attempt count, and last-tried time; the exit code (result.failed) is * non-zero while any failure remains and clears once every one is resolved. * * The cache and download layers are covered elsewhere (content.test.ts, * download tests); here a mock library client and a stand-in content source * drive the backup logic with no crypto and no network. */ import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, writeFileSync, } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { Library } from "../../src/library/index.js"; import type { ContentSource } from "../../src/library/content.js"; import type { CollectionsPage, FilesPage } from "../../src/client.js"; import type { Collection, EnteFile } from "../../src/model/types.js"; const USER_ID = 42; // Decrypted-byte length each stub original writes, keyed by fileID. const SIZE_BY_ID: Record<number, number> = { 100: 3000, 101: 2000, 200: 1500 }; const collection = (id: number, name: string): Collection => ({ id, ownerID: USER_ID, key: new Uint8Array([id & 0xff]), name, type: "album", updationTime: 1, isShared: false, }); const file = (id: number, collectionID: number, title: string): EnteFile => ({ id, collectionID, ownerID: USER_ID, key: new Uint8Array([id & 0xff]), metadata: { title, fileType: "image", creationTime: 1, modificationTime: 1, }, file: { decryptionHeader: "aGVhZGVy" }, thumbnail: { decryptionHeader: "dGh1bWI=" }, updationTime: 1, }); // A metadata-only client: two albums, three files, served once. No ML. class MockClient { private served = false; whoami(): { email: string; userID: number } { return { email: "backup@example.com", userID: USER_ID }; } async collectionsSince(): Promise<CollectionsPage> { if (this.served) return { collections: [], deleted: [], cursor: 1 }; this.served = true; return { collections: [collection(1, "Vacation"), collection(2, "Work")], deleted: [], cursor: 1, }; } async filesSince(args: { collectionID: number }): Promise<FilesPage> { const files = args.collectionID === 1 ? [file(100, 1, "beach.jpg"), file(101, 1, "sunset.jpg")] : args.collectionID === 2 ? [file(200, 2, "diagram.png")] : []; return { files, deleted: [], cursor: 1 }; } } // A content source that writes byte buffers of the expected length and can be // told to fail one fileID's original, to exercise per-file resilience. interface StubSource extends ContentSource { failID?: number; originalCalls: number; } const stubSource = (): StubSource => { const s: StubSource = { originalCalls: 0, original: async ({ file: f, destination }) => { s.originalCalls++; if (s.failID === f.id) throw new Error("HTTP 500 from server"); const size = SIZE_BY_ID[f.id] ?? 10; writeFileSync(destination, Buffer.alloc(size)); return { bytesWritten: size }; }, thumbnail: async ({ destination }) => { writeFileSync(destination, Buffer.alloc(5)); return { bytesWritten: 5 }; }, }; return s; }; let root: string; const openLibrary = (source: ContentSource): Promise<Library> => Library.open({ client: new MockClient(), cacheDirectory: join(root, "cache"), contentSource: source, refreshIntervalSeconds: 3600, }); const readLedger = ( outDir: string, ): { files: Record<string, Record<string, unknown>> } => JSON.parse(readFileSync(join(outDir, "failures.json"), "utf-8")); // Write a durable ledger holding one prior failure, to exercise pruning of // entries the current run cannot resolve. const seedLedger = (outDir: string, fileID: number, title: string): void => { mkdirSync(outDir, { recursive: true }); writeFileSync( join(outDir, "failures.json"), JSON.stringify({ version: 1, files: { [String(fileID)]: { fileID, title, classification: "transient", attempts: 1, lastTriedAt: Date.now(), error: "HTTP 500 from server", }, }, }), ); }; beforeEach(() => { root = mkdtempSync(join(tmpdir(), "quak-backup-test-")); }); afterEach(() => { if (root && existsSync(root)) rmSync(root, { recursive: true, force: true }); }); describe("lib.backup", () => { it("throws before any network when no downloadDirectory is given", async () => { const source = stubSource(); const lib = await openLibrary(source); await expect(lib.backup()).rejects.toThrow(/downloadDirectory/i); expect(source.originalCalls).toBe(0); lib.close(); }); it("writes the expected on-disk layout for every file", async () => { const source = stubSource(); const lib = await openLibrary(source); const outDir = join(root, "backup"); const result = await lib.backup({ downloadDirectory: outDir }); expect(result.totalFiles).toBe(3); expect(result.downloaded).toBe(3); expect(result.skipped).toBe(0); expect(result.failed).toBe(0); expect(result.errors).toEqual([]); // Originals under originals/<fileID>.<ext>. expect(readFileSync(join(outDir, "originals", "100.jpg")).length).toBe( 3000, ); expect(readFileSync(join(outDir, "originals", "101.jpg")).length).toBe( 2000, ); expect(readFileSync(join(outDir, "originals", "200.png")).length).toBe( 1500, ); // Per-file metadata sidecar. const sidecar = JSON.parse( readFileSync(join(outDir, "originals", "100.json"), "utf-8"), ); expect(sidecar.id).toBe(100); expect(sidecar.metadata.title).toBe("beach.jpg"); // Collection dirs contain symlinks into ../originals. const beach = join(outDir, "collections", "Vacation", "beach.jpg"); expect(lstatSync(beach).isSymbolicLink()).toBe(true); expect(readlinkSync(beach)).toContain("originals"); expect(readFileSync(beach).length).toBe(3000); // Per-collection metadata JSON. const vacation = JSON.parse( readFileSync(join(outDir, "collections", "Vacation.json"), "utf-8"), ); expect(vacation.name).toBe("Vacation"); expect(vacation.files.length).toBe(2); expect(vacation.files[0].metadata.title).toBeDefined(); // A clean run leaves no failure ledger behind. expect(existsSync(join(outDir, "failures.json"))).toBe(false); lib.close(); }); it("is an idempotent no-op when every original is already present", async () => { const source = stubSource(); const lib = await openLibrary(source); const outDir = join(root, "backup"); const first = await lib.backup({ downloadDirectory: outDir }); expect(first.downloaded).toBe(3); const callsAfterFirst = source.originalCalls; const second = await lib.backup({ downloadDirectory: outDir }); expect(second.downloaded).toBe(0); expect(second.skipped).toBe(3); expect(second.failed).toBe(0); // A present original is neither fetched nor copied again. expect(source.originalCalls).toBe(callsAfterFirst); lib.close(); }); it("continues past a download failure and records it in failures.json", async () => { const source = stubSource(); source.failID = 101; const lib = await openLibrary(source); const outDir = join(root, "backup"); const result = await lib.backup({ downloadDirectory: outDir }); expect(result.totalFiles).toBe(3); expect(result.downloaded).toBe(2); expect(result.failed).toBe(1); expect(result.errors.length).toBe(1); expect(result.errors[0]!.fileID).toBe(101); expect(result.errors[0]!.title).toBe("sunset.jpg"); // The two good files are on disk; the failed one is not. expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true); expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(false); expect( existsSync(join(outDir, "collections", "Vacation", "sunset.jpg")), ).toBe(false); // Durable ledger with classification, attempts, last-tried. const ledger = readLedger(outDir); const entry = ledger.files["101"]!; expect(entry.attempts).toBe(1); expect(entry.classification).toBeDefined(); expect(typeof entry.lastTriedAt).toBe("number"); lib.close(); }); it("increments the attempt count across runs and clears the ledger once resolved", async () => { const source = stubSource(); source.failID = 101; const lib = await openLibrary(source); const outDir = join(root, "backup"); const r1 = await lib.backup({ downloadDirectory: outDir }); expect(r1.failed).toBe(1); expect(readLedger(outDir).files["101"]!.attempts).toBe(1); // Second run: the two good files are present, only 101 is retried. const r2 = await lib.backup({ downloadDirectory: outDir }); expect(r2.failed).toBe(1); expect(r2.skipped).toBe(2); expect(readLedger(outDir).files["101"]!.attempts).toBe(2); // Resume with a healthy source: 101 downloads, the rest are skipped. source.failID = undefined; const r3 = await lib.backup({ downloadDirectory: outDir }); expect(r3.failed).toBe(0); expect(r3.skipped).toBe(2); expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(true); // A ledger with no remaining failures is removed. expect(existsSync(join(outDir, "failures.json"))).toBe(false); lib.close(); }); it("does not abort when a symlink cannot be created (issue #8)", async () => { const source = stubSource(); const lib = await openLibrary(source); const outDir = join(root, "backup"); // Occupy beach.jpg's symlink path with a directory so symlink creation // fails for that one file. mkdirSync(join(outDir, "collections", "Vacation", "beach.jpg"), { recursive: true, }); const result = await lib.backup({ downloadDirectory: outDir }); // Every original still downloads despite the symlink failure. expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true); expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); // The other symlinks are still built. expect( lstatSync( join(outDir, "collections", "Vacation", "sunset.jpg"), ).isSymbolicLink(), ).toBe(true); expect( lstatSync( join(outDir, "collections", "Work", "diagram.png"), ).isSymbolicLink(), ).toBe(true); // The symlink failure is recorded, not thrown. expect(result.failed).toBeGreaterThanOrEqual(1); const err = result.errors.find((e) => e.fileID === 100); expect(err).toBeDefined(); expect(err!.collection).toBe("Vacation"); expect(readLedger(outDir).files["100"]).toBeDefined(); lib.close(); }); it("rebuilds a stale sidecar and a missing symlink on a later run", async () => { const source = stubSource(); const lib = await openLibrary(source); const outDir = join(root, "backup"); await lib.backup({ downloadDirectory: outDir }); // Corrupt a sidecar and delete a symlink between runs. writeFileSync(join(outDir, "originals", "100.json"), "not json"); rmSync(join(outDir, "collections", "Vacation", "beach.jpg")); const result = await lib.backup({ downloadDirectory: outDir }); expect(result.failed).toBe(0); // The derived views are repaired from the model. const sidecar = JSON.parse( readFileSync(join(outDir, "originals", "100.json"), "utf-8"), ); expect(sidecar.metadata.title).toBe("beach.jpg"); expect( lstatSync( join(outDir, "collections", "Vacation", "beach.jpg"), ).isSymbolicLink(), ).toBe(true); lib.close(); }); it("backs up only the named albums when onlyAlbumNames is given", async () => { const source = stubSource(); const lib = await openLibrary(source); const outDir = join(root, "backup"); const result = await lib.backup({ downloadDirectory: outDir, onlyAlbumNames: ["Work"], }); expect(result.totalFiles).toBe(1); expect(result.downloaded).toBe(1); expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(false); expect(existsSync(join(outDir, "collections", "Work.json"))).toBe(true); expect(existsSync(join(outDir, "collections", "Vacation.json"))).toBe( false, ); lib.close(); }); it("prunes a ledger entry for a file no longer in the library and exits zero", async () => { const source = stubSource(); const lib = await openLibrary(source); const outDir = join(root, "backup"); // A prior failure for a file that has since left the library (deleted // from the account). This run has no way to resolve it, so it must not // keep the exit code non-zero forever. seedLedger(outDir, 999, "gone.jpg"); const result = await lib.backup({ downloadDirectory: outDir }); // Everything still present is backed up cleanly, and the stale entry is // dropped rather than counted. expect(result.downloaded).toBe(3); expect(result.failed).toBe(0); expect(existsSync(join(outDir, "failures.json"))).toBe(false); lib.close(); }); it("prunes an out-of-scope ledger entry on a scoped run and exits zero", async () => { const source = stubSource(); const lib = await openLibrary(source); const outDir = join(root, "backup"); // A prior failure for a Vacation file; this run is scoped to Work and // never attempts it, so it must not poison the scoped run's exit code. seedLedger(outDir, 100, "beach.jpg"); const result = await lib.backup({ downloadDirectory: outDir, onlyAlbumNames: ["Work"], }); expect(result.totalFiles).toBe(1); expect(result.failed).toBe(0); expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); expect(existsSync(join(outDir, "failures.json"))).toBe(false); lib.close(); }); it("also stores thumbnails when includeThumbnails is set", async () => { const source = stubSource(); const lib = await openLibrary(source); const outDir = join(root, "backup"); await lib.backup({ downloadDirectory: outDir, includeThumbnails: true, }); expect(existsSync(join(outDir, "thumbnails", "100.jpg"))).toBe(true); expect(existsSync(join(outDir, "thumbnails", "200.jpg"))).toBe(true); lib.close(); }); });