quak backup --verify re-hashes stored originals and downloads again any that do not match (closes #168)

`--verify`, or `lib.backup({ verify: true })`, hashes each original already
at its save path as the download check does, streamed, a live photo as
`<imageHash>:<videoHash>`. A mismatch is logged, removed and put back in the
same run, and what is put back is hashed too, since a copy from the content
cache is not checked; one that still does not match, or a failed fetch, goes
into `failures.json`. A file with no recorded hash counts as unchecked. The
result, the summary and `--json` gain `verified`, `mismatched` and
`unchecked`.

Judgement call: a stored original that cannot be read for hashing is recorded as failed and left in place.
Judgement call: a copy put back that still does not match stays at its save path and is not counted as downloaded.
Judgement call: the summary prints the three counts only with `--verify`.

Model: opus-5-5

test/cli/backup.test.ts

@@ -62,6 +62,7 @@ import { HEIC_WITH_EXIF } from "../exif-heic.js";
 import { JPEG_WITH_EXIF } from "../exif-jpeg.js";
 import {
     asLivePhoto,
+    blake2b,
     cdnSource,
     IMAGE,
     livePhotoHash,
@@ -1271,6 +1272,181 @@ describe("image metadata in each file's JSON", () => {
     });
 });
 
+// With `verify`, each original already stored is hashed, and one that does not
+// match the content hash its metadata records is downloaded again.
+describe("backup with verify", () => {
+    // MockClient's files, each recording the hash of what `stubSource` writes
+    // for it, except diagram.png (200), which records none.
+    class HashedClient extends MockClient {
+        override async filesSince(args: {
+            collectionID: number;
+        }): Promise<FilesPage> {
+            const page = await super.filesSince(args);
+            const files = page.files.map((f) =>
+                f.id === 200
+                    ? f
+                    : {
+                          ...f,
+                          metadata: {
+                              ...f.metadata,
+                              hash: blake2b(Buffer.alloc(SIZE_BY_ID[f.id]!)),
+                          },
+                      },
+            );
+            return { ...page, files };
+        }
+    }
+
+    // A backup of the account in `root/backup`, the library that made it, and
+    // the source it fetched from.
+    const backedUp = async (): Promise<{
+        lib: Library;
+        source: StubSource;
+        outDir: string;
+    }> => {
+        const source = stubSource();
+        const lib = await openLibrary(source, new HashedClient());
+        const outDir = join(root, "backup");
+        await lib.backup({ downloadDirectory: outDir });
+        return { lib, source, outDir };
+    };
+
+    // What `stubSource` writes for beach.jpg (100), and other bytes of the
+    // same length.
+    const good = Buffer.alloc(SIZE_BY_ID[100]!);
+    const corrupt = Buffer.alloc(SIZE_BY_ID[100]!, 1);
+
+    it("leaves an original that matches its hash, and one with no hash, as they are", async () => {
+        const { lib, source, outDir } = await backedUp();
+        const calls = source.originalCalls;
+
+        const result = await lib.backup({
+            downloadDirectory: outDir,
+            verify: true,
+        });
+
+        expect(result).toMatchObject({
+            downloaded: 0,
+            skipped: 3,
+            verified: 2,
+            mismatched: 0,
+            unchecked: 1,
+            failed: 0,
+        });
+        expect(source.originalCalls).toBe(calls);
+        await lib.close();
+    });
+
+    it("downloads again an original that does not match its hash", async () => {
+        const { lib, outDir } = await backedUp();
+        writeFileSync(saved(outDir, "100.jpg"), corrupt);
+        const log: string[] = [];
+
+        const result = await lib.backup({
+            downloadDirectory: outDir,
+            verify: true,
+            onProgress: (msg) => log.push(msg),
+        });
+
+        expect(result).toMatchObject({
+            downloaded: 1,
+            skipped: 2,
+            verified: 1,
+            mismatched: 1,
+            unchecked: 1,
+            failed: 0,
+        });
+        expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(good);
+        expect(log.filter((msg) => msg.startsWith("MISMATCH"))).toEqual([
+            "MISMATCH original beach.jpg (100): its bytes do not match its content hash",
+        ]);
+        await lib.close();
+    });
+
+    it("records a failed download in failures.json, with the original removed", async () => {
+        const { lib, source, outDir } = await backedUp();
+        writeFileSync(saved(outDir, "100.jpg"), corrupt);
+        source.failID = 100;
+
+        const result = await lib.backup({
+            downloadDirectory: outDir,
+            verify: true,
+        });
+
+        expect(result).toMatchObject({
+            downloaded: 0,
+            mismatched: 1,
+            failed: 1,
+        });
+        expect(Object.keys(readLedger(outDir).files)).toEqual(["100"]);
+        expect(existsSync(saved(outDir, "100.jpg"))).toBe(false);
+        await lib.close();
+    });
+
+    it("records in failures.json an original the cache puts back with the same bad bytes", async () => {
+        const lib = await openLibrary(stubSource(), new HashedClient());
+        const outDir = join(root, "backup");
+        // The cache holds a bad copy, and a backup copies an original the
+        // cache holds to its save path.
+        const cached = await lib.photos.byID({ fileID: 100 })!.original();
+        writeFileSync(cached.path, corrupt);
+        await lib.backup({ downloadDirectory: outDir });
+
+        const result = await lib.backup({
+            downloadDirectory: outDir,
+            verify: true,
+        });
+
+        expect(result).toMatchObject({
+            downloaded: 0,
+            mismatched: 1,
+            failed: 1,
+        });
+        expect(result.errors.map((e) => e.error)).toEqual([
+            "the original put back does not match its content hash either",
+        ]);
+        expect(Object.keys(readLedger(outDir).files)).toEqual(["100"]);
+        expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(corrupt);
+        await lib.close();
+    });
+
+    it("hashes nothing without verify", async () => {
+        const { lib, outDir } = await backedUp();
+        writeFileSync(saved(outDir, "100.jpg"), corrupt);
+
+        const result = await lib.backup({ downloadDirectory: outDir });
+
+        expect(result).toMatchObject({
+            downloaded: 0,
+            skipped: 3,
+            verified: 0,
+            mismatched: 0,
+            unchecked: 0,
+            failed: 0,
+        });
+        expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(corrupt);
+        await lib.close();
+    });
+
+    // Root ignores file permissions, so this fails when run as root. The
+    // `test` phase of the `Dockerfile` runs as the `node` user.
+    it("records an original it cannot read as failed, and leaves it", async () => {
+        const { lib, outDir } = await backedUp();
+        const original = saved(outDir, "100.jpg");
+        chmodSync(original, 0o000);
+
+        const result = await lib
+            .backup({ downloadDirectory: outDir, verify: true })
+            .finally(() => chmodSync(original, 0o600));
+
+        expect(result).toMatchObject({ downloaded: 0, verified: 1, failed: 1 });
+        expect(result.errors.map((e) => e.fileID)).toEqual([100]);
+        expect(result.errors[0]!.error).toMatch(/EACCES/);
+        expect(readFileSync(original)).toEqual(good);
+        await lib.close();
+    });
+});
+
 // Every entry under collections/, one level of directories deep, with each
 // symlink's target.
 const tree = (outDir: string): string[] => {
@@ -1766,6 +1942,64 @@ describe("backup of live photos", () => {
         },
     );
 
+    it("verifies a live photo's image and video together, and downloads it again when one does not match", async () => {
+        const { file: live, body } = await asLivePhoto(
+            file(500, 10, "IMG_0500.HEIC"),
+        );
+        const lib = await open([live], new Map([[500, body]]));
+        const outDir = join(root, "backup");
+        await lib.backup({ downloadDirectory: outDir });
+
+        const good = await lib.backup({
+            downloadDirectory: outDir,
+            verify: true,
+        });
+        expect(good).toMatchObject({ skipped: 1, verified: 1, mismatched: 0 });
+
+        const video = saved(outDir, "500.mov");
+        writeFileSync(video, "another few seconds of video");
+        const bad = await lib.backup({
+            downloadDirectory: outDir,
+            verify: true,
+        });
+
+        expect(bad).toMatchObject({
+            downloaded: 1,
+            verified: 0,
+            mismatched: 1,
+            failed: 0,
+        });
+        expect(readFileSync(saved(outDir, "500.heic"))).toEqual(
+            Buffer.from(IMAGE),
+        );
+        expect(readFileSync(video)).toEqual(Buffer.from(VIDEO));
+        expect(tree(outDir)).toEqual(linked);
+        await lib.close();
+    });
+
+    it("removes both files of a live photo that does not match its hash when downloading it again fails", async () => {
+        const { file: live, body } = await asLivePhoto(
+            file(500, 10, "IMG_0500.HEIC"),
+        );
+        const bodies = new Map([[500, body]]);
+        const lib = await open([live], bodies);
+        const outDir = join(root, "backup");
+        await lib.backup({ downloadDirectory: outDir });
+        writeFileSync(saved(outDir, "500.mov"), "another few seconds of video");
+        bodies.delete(500);
+
+        const result = await lib.backup({
+            downloadDirectory: outDir,
+            verify: true,
+        });
+
+        expect(result).toMatchObject({ mismatched: 1, failed: 1 });
+        expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]);
+        expect(existsSync(saved(outDir, "500.heic"))).toBe(false);
+        expect(existsSync(saved(outDir, "500.mov"))).toBe(false);
+        await lib.close();
+    });
+
     it("serves a live photo the backup stored to a library reading the backup", async () => {
         const { file: live, body } = await asLivePhoto(
             file(500, 10, "IMG_0500.HEIC"),