--verify keeps a mismatched original until a matching copy replaces it (closes #199)

With --verify, an original whose hash does not match is no longer removed
before its replacement is downloaded. replaceOriginal, beside placeOriginal
in src/library/content.ts, has the content cache's new fetchOriginal write
the replacement to a temporary file beside the original, hashes it, and
renames it over the original only when it matches; a live photo's image and
video are both hashed before either is renamed. A failed or mismatching
download leaves the old bytes and album links, removes the temporary files,
and lands in failures.json. The replacement is always downloaded, never
copied from the cache. The leftover-file cleanup also takes a live photo's
replacement files, .quak-<pid>-<random>.<ext>.

Model: opus-5-5

src/backup.ts

@@ -41,8 +41,9 @@
 //
 // With `verify`, each original already at its save path is hashed as the
 // download check hashes it, and one that does not match the content hash its
-// metadata records is removed and fetched again in the same run. What is put
-// back is hashed too, and recorded as failed if it still does not match.
+// metadata records is replaced once a matching copy is downloaded, in the same
+// run. Until then it stays as it is: a download that fails, or that does not
+// match either, leaves it in place and is recorded as failed.
 //
 // Resilience (issue #8): no per-file condition aborts the run. A failed
 // download, a failed symlink, or ML data missing because the ML data fetch
@@ -61,7 +62,6 @@
 // code.
 
 import {
-    createReadStream,
     lstatSync,
     mkdirSync,
     readdirSync,
@@ -77,12 +77,6 @@ import { readFile } from "node:fs/promises";
 import { dirname, extname, join, relative, resolve } from "node:path";
 import lockfile from "proper-lockfile";
 
-import {
-    chunkHashFinal,
-    chunkHashInit,
-    chunkHashUpdate,
-    init,
-} from "./crypto/index.js";
 import { removeLeftoverTempFiles } from "./download/index.js";
 import { describeError } from "./errors.js";
 import {
@@ -94,8 +88,10 @@ import {
 import {
     copyAtomic,
     placeOriginal,
+    replaceOriginal,
     savePath,
     storedAtSavePath,
+    storedHash,
 } from "./library/content.js";
 import { representative } from "./library/records.js";
 import type { Undecryptable } from "./library/store.js";
@@ -152,9 +148,9 @@ export interface BackupResult {
     // Originals already at their save path and left untouched.
     skipped: number;
     // With `verify`, the originals already at their save path whose hash
-    // matched, those whose hash did not (each removed and fetched again), and
-    // those whose metadata records no hash (left as they are). All three are
-    // zero without `verify`.
+    // matched, those whose hash did not (each replaced once a matching copy is
+    // downloaded), and those whose metadata records no hash (left as they
+    // are). All three are zero without `verify`.
     verified: number;
     mismatched: number;
     unchecked: number;
@@ -192,6 +188,15 @@ export interface BackupLibrary {
         fileID: number,
         destination: string,
     ): Promise<{ path: string; videoPath?: string }>;
+    // Fetch an original's bytes to `destination` through the content pool,
+    // a live photo as its image and its video beside it, even when the cache
+    // or the library's save path already holds it, and return where they
+    // landed. With `verify`, an original whose hash does not match is replaced
+    // with what this fetches.
+    fetchOriginal(
+        fileID: number,
+        destination: string,
+    ): Promise<{ path: string; videoPath?: string }>;
     thumbnail(fileID: number): Promise<{ path: string }>;
     // How many originals the backup works on at once: the limit of the
     // library's content pool.
@@ -444,26 +449,6 @@ const saveLedger = (path: string, ledger: Map<number, FailureEntry>): void => {
     );
 };
 
-// The content hash of the original stored at `stored`, computed as the download
-// check computes it: over each file's bytes, read in chunks, and for a live
-// photo `<imageHash>:<videoHash>`.
-const storedHash = async (stored: {
-    path: string;
-    videoPath?: string;
-}): Promise<string> => {
-    await init();
-    const hashFile = async (path: string): Promise<string> => {
-        const state = chunkHashInit();
-        for await (const chunk of createReadStream(path)) {
-            chunkHashUpdate(state, chunk as Buffer);
-        }
-        return chunkHashFinal(state);
-    };
-    const hash = await hashFile(stored.path);
-    if (stored.videoPath === undefined) return hash;
-    return `${hash}:${await hashFile(stored.videoPath)}`;
-};
-
 // A file's EXIF, XMP and dimensions as its JSON holds them: what
 // `extractImageMetadata` found in its original, or why the original could not
 // be read.
@@ -685,11 +670,13 @@ const runLockedBackup = async (
     // through the content cache/pools, as `Photo.download()` does, and fetch
     // the optional thumbnails; a present file is left as is. With `verify`, a
     // present original is hashed first, and one that does not match the hash
-    // its metadata records is removed and fetched like a missing one. One
-    // that cannot be read is recorded as failed and left where it is.
-    // `storeOriginal` does this for one original, and records its failure.
+    // its metadata records is downloaded again and replaced by
+    // `replaceOriginal`, which leaves it as it was when that download fails
+    // or does not match either. One that cannot be read is recorded as failed
+    // and left where it is. `storeOriginal` does this for one original, and
+    // records its failure.
     const storeOriginal = async (file: EnteFile): Promise<void> => {
-        let stored = storedAtSavePath(downloadDirectory, file);
+        const stored = storedAtSavePath(downloadDirectory, file);
         let mismatch = false;
         if (stored !== undefined && verify) {
             try {
@@ -703,11 +690,6 @@ const runLockedBackup = async (
                     );
                     mismatched++;
                     mismatch = true;
-                    rmSync(stored.path);
-                    if (stored.videoPath !== undefined) {
-                        rmSync(stored.videoPath);
-                    }
-                    stored = undefined;
                 }
             } catch (err) {
                 log(
@@ -722,29 +704,25 @@ const runLockedBackup = async (
                 return;
             }
         }
-        if (stored !== undefined) {
+        if (stored !== undefined && !mismatch) {
             skipped++;
             return;
         }
         try {
             log(`Fetching original ${file.metadata.title} (${file.id})...`);
-            // A fetched original is written straight to its save path (a
-            // live photo beside it); only one that was already cached
-            // elsewhere is copied.
-            const placed = await placeOriginal(
-                downloadDirectory,
-                file,
-                (dest) => lib.original(file.id, dest),
-            );
-            storedThisRun.add(file.id);
-            // A copy from the cache is not checked as a download is and can
-            // hold the same bad bytes, so what is put back after a mismatch
-            // is hashed too. A bad copy stays where it is and the file fails.
-            if (mismatch && (await storedHash(placed)) !== file.metadata.hash) {
-                throw new Error(
-                    "the original put back does not match its content hash either",
+            if (mismatch) {
+                await replaceOriginal(downloadDirectory, file, (dest) =>
+                    lib.fetchOriginal(file.id, dest),
+                );
+            } else {
+                // A fetched original is written straight to its save path (a
+                // live photo beside it); only one that was already cached
+                // elsewhere is copied.
+                await placeOriginal(downloadDirectory, file, (dest) =>
+                    lib.original(file.id, dest),
                 );
             }
+            storedThisRun.add(file.id);
             downloaded++;
         } catch (err) {
             log(