quak backup --verify re-hashes stored originals and downloads again any that do not match (closes #168)

`--verify`, or `lib.backup({ verify: true })`, hashes each original already
at its save path as the download check does, streamed, a live photo as
`<imageHash>:<videoHash>`. A mismatch is logged, removed and put back in the
same run, and what is put back is hashed too, since a copy from the content
cache is not checked; one that still does not match, or a failed fetch, goes
into `failures.json`. A file with no recorded hash counts as unchecked. The
result, the summary and `--json` gain `verified`, `mismatched` and
`unchecked`.

Judgement call: a stored original that cannot be read for hashing is recorded as failed and left in place.
Judgement call: a copy put back that still does not match stays at its save path and is not counted as downloaded.
Judgement call: the summary prints the three counts only with `--verify`.

Model: opus-5-5

src/backup.ts

@@ -38,6 +38,11 @@
 // why they could not be read), so that a run does not read every stored
 // original again; a sidecar without them gets them read from the original.
 //
+// With `verify`, each original already at its save path is hashed as the
+// download check hashes it, and one that does not match the content hash its
+// metadata records is removed and fetched again in the same run. What is put
+// back is hashed too, and recorded as failed if it still does not match.
+//
 // Resilience (issue #8): no per-file condition aborts the run. A failed
 // download, a failed symlink, or ML data missing because the ML data fetch
 // failed is caught, recorded in `failures.json` with a classification, a
@@ -50,6 +55,7 @@
 // code.
 
 import {
+    createReadStream,
     lstatSync,
     mkdirSync,
     readdirSync,
@@ -65,6 +71,12 @@ import { readFile } from "node:fs/promises";
 import { dirname, extname, join, relative, resolve } from "node:path";
 import lockfile from "proper-lockfile";
 
+import {
+    chunkHashFinal,
+    chunkHashInit,
+    chunkHashUpdate,
+    init,
+} from "./crypto/index.js";
 import { removeLeftoverTempFiles } from "./download/index.js";
 import { sanitizeFileName, withExtension } from "./filename.js";
 import {
@@ -92,6 +104,9 @@ export interface BackupOptions {
     includeThumbnails?: boolean;
     // Restrict the backup to albums with these names; others are left untouched.
     onlyAlbumNames?: string[];
+    // Hash each original already at its save path, and fetch again any whose
+    // bytes do not match the content hash its metadata records. Default false.
+    verify?: boolean;
     onProgress?: ProgressCallback;
     // The caller already holds the lock in `downloadDirectory`, taken with
     // `lockBackupDirectory`, and releases it itself, so the backup does not
@@ -113,6 +128,13 @@ export interface BackupResult {
     downloaded: number;
     // Originals already at their save path and left untouched.
     skipped: number;
+    // With `verify`, the originals already at their save path whose hash
+    // matched, those whose hash did not (each removed and fetched again), and
+    // those whose metadata records no hash (left as they are). All three are
+    // zero without `verify`.
+    verified: number;
+    mismatched: number;
+    unchecked: number;
     // Files with an unresolved failure after this run (the ledger size); the
     // CLI exits non-zero while this is above zero. A file can be both
     // downloaded and failed if its bytes landed but its symlink did not.
@@ -369,6 +391,26 @@ const saveLedger = (path: string, ledger: Map<number, FailureEntry>): void => {
     );
 };
 
+// The content hash of the original stored at `stored`, computed as the download
+// check computes it: over each file's bytes, read in chunks, and for a live
+// photo `<imageHash>:<videoHash>`.
+const storedHash = async (stored: {
+    path: string;
+    videoPath?: string;
+}): Promise<string> => {
+    await init();
+    const hashFile = async (path: string): Promise<string> => {
+        const state = chunkHashInit();
+        for await (const chunk of createReadStream(path)) {
+            chunkHashUpdate(state, chunk as Buffer);
+        }
+        return chunkHashFinal(state);
+    };
+    const hash = await hashFile(stored.path);
+    if (stored.videoPath === undefined) return hash;
+    return `${hash}:${await hashFile(stored.videoPath)}`;
+};
+
 // A file's EXIF, XMP and dimensions as its JSON holds them: what
 // `extractImageMetadata` found in its original, or why the original could not
 // be read.
@@ -468,6 +510,7 @@ const runLockedBackup = async (
     const includeThumbnails = opts.includeThumbnails ?? false;
     const log = opts.onProgress ?? (() => {});
     const only = opts.onlyAlbumNames ? new Set(opts.onlyAlbumNames) : undefined;
+    const verify = opts.verify ?? false;
 
     log("Refreshing library...");
     await lib.refresh();
@@ -526,6 +569,9 @@ const runLockedBackup = async (
     const storedThisRun = new Set<number>();
     let downloaded = 0;
     let skipped = 0;
+    let verified = 0;
+    let mismatched = 0;
+    let unchecked = 0;
 
     const recordFailure = (
         file: EnteFile,
@@ -557,10 +603,47 @@ const runLockedBackup = async (
 
     // Phase 1: get the bytes. Put each pending original at its save path
     // through the content cache/pools, as `Photo.download()` does, and fetch
-    // the optional thumbnails; a present file is left as is.
+    // the optional thumbnails; a present file is left as is. With `verify`, a
+    // present original is hashed first, and one that does not match the hash
+    // its metadata records is removed and fetched like a missing one. One
+    // that cannot be read is recorded as failed and left where it is.
     if (includeOriginals) {
         for (const [fileID, file] of distinct) {
-            if (storedAtSavePath(downloadDirectory, file) !== undefined) {
+            let stored = storedAtSavePath(downloadDirectory, file);
+            let mismatch = false;
+            if (stored !== undefined && verify) {
+                try {
+                    if (file.metadata.hash === undefined) {
+                        unchecked++;
+                    } else if (
+                        (await storedHash(stored)) === file.metadata.hash
+                    ) {
+                        verified++;
+                    } else {
+                        log(
+                            `MISMATCH original ${file.metadata.title} (${fileID}): its bytes do not match its content hash`,
+                        );
+                        mismatched++;
+                        mismatch = true;
+                        rmSync(stored.path);
+                        if (stored.videoPath !== undefined) {
+                            rmSync(stored.videoPath);
+                        }
+                        stored = undefined;
+                    }
+                } catch (err) {
+                    log(
+                        `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
+                    );
+                    recordFailure(
+                        file,
+                        collectionName.get(file.collectionID) ?? "",
+                        err,
+                    );
+                    continue;
+                }
+            }
+            if (stored !== undefined) {
                 skipped++;
                 continue;
             }
@@ -569,10 +652,24 @@ const runLockedBackup = async (
                 // A fetched original is written straight to its save path (a
                 // live photo beside it); only one that was already cached
                 // elsewhere is copied.
-                await placeOriginal(downloadDirectory, file, (dest) =>
-                    lib.original(fileID, dest),
+                const placed = await placeOriginal(
+                    downloadDirectory,
+                    file,
+                    (dest) => lib.original(fileID, dest),
                 );
                 storedThisRun.add(fileID);
+                // A copy from the cache is not checked as a download is and can
+                // hold the same bad bytes, so what is put back after a
+                // mismatch is hashed too. A bad copy stays where it is and the
+                // file fails.
+                if (
+                    mismatch &&
+                    (await storedHash(placed)) !== file.metadata.hash
+                ) {
+                    throw new Error(
+                        "the original put back does not match its content hash either",
+                    );
+                }
                 downloaded++;
             } catch (err) {
                 log(
@@ -735,6 +832,9 @@ const runLockedBackup = async (
         totalFiles: distinct.size,
         downloaded,
         skipped,
+        verified,
+        mismatched,
+        unchecked,
         failed: ledger.size,
         errors,
     };

src/cli-commands.ts

@@ -402,7 +402,7 @@ export const backupMetadataCommand = async (
 export const backupCommand = async (
     ctx: CliContext,
     dir: string,
-    opts: { json?: boolean },
+    opts: { json?: boolean; verify?: boolean },
 ): Promise<number> => {
     await init();
     const client = requireSession(ctx);
@@ -433,6 +433,7 @@ export const backupCommand = async (
             const result = await lib.backup({
                 downloadDirectory: dir,
                 lockHeld: true,
+                verify: opts.verify,
                 onProgress: (msg) => {
                     if (!opts.json) ctx.stderr.write(msg + "\n");
                 },
@@ -445,6 +446,11 @@ export const backupCommand = async (
                 ctx.stderr.write(`  Total files: ${result.totalFiles}\n`);
                 ctx.stderr.write(`  Downloaded:  ${result.downloaded}\n`);
                 ctx.stderr.write(`  Skipped:     ${result.skipped}\n`);
+                if (opts.verify) {
+                    ctx.stderr.write(`  Verified:    ${result.verified}\n`);
+                    ctx.stderr.write(`  Mismatched:  ${result.mismatched}\n`);
+                    ctx.stderr.write(`  Unchecked:   ${result.unchecked}\n`);
+                }
                 ctx.stderr.write(`  Failed:      ${result.failed}\n`);
                 if (result.errors.length > 0) {
                     ctx.stderr.write("\nFailed files:\n");