--verify keeps a mismatched original until a matching copy replaces it (closes #199)

With --verify, an original whose hash does not match is no longer removed
before its replacement is downloaded. replaceOriginal, beside placeOriginal
in src/library/content.ts, has the content cache's new fetchOriginal write
the replacement to a temporary file beside the original, hashes it, and
renames it over the original only when it matches; a live photo's image and
video are both hashed before either is renamed. A failed or mismatching
download leaves the old bytes and album links, removes the temporary files,
and lands in failures.json. The replacement is always downloaded, never
copied from the cache. The leftover-file cleanup also takes a live photo's
replacement files, .quak-<pid>-<random>.<ext>.

Model: opus-5-5

src/library/content.ts

@@ -13,6 +13,8 @@
 // originals, the thumbnail pool for thumbnails. The pool limits concurrency,
 // orders on-demand work ahead of background, and dedups by key so a fileID
 // requested twice while the first is still in flight downloads once.
+// `fetchOriginal` passes no key, so it is not shared with a fetch of the same
+// file already in flight.
 //
 // Integrity. The reused streaming decrypt is the enforced guarantee: every
 // chunk is authenticated and the writer renames the file into place only once
@@ -27,9 +29,10 @@
 // It also names each original's save path under the download directory
 // (`savePath`), where `Photo.download()` and `lib.backup()` put it. A copy in
 // the cache does not count as saved there, but is copied there rather than
-// fetched again.
+// fetched again. An original stored there whose bytes do not match its content
+// hash is replaced only by a download that does (`replaceOriginal`).
 
-import { existsSync, readFileSync, statSync } from "node:fs";
+import { createReadStream, existsSync, readFileSync, statSync } from "node:fs";
 import {
     chmod,
     copyFile,
@@ -44,6 +47,12 @@ import {
 import { basename, dirname, extname, join } from "node:path";
 
 import type { ApiClient } from "../api/client.js";
+import {
+    chunkHashFinal,
+    chunkHashInit,
+    chunkHashUpdate,
+    init,
+} from "../crypto/index.js";
 import {
     downloadFile,
     downloadThumbnail,
@@ -349,6 +358,26 @@ export const storedAtSavePath = (
     return storedOriginal(dirname(path), basename(path, extname(path)), file);
 };
 
+// The content hash of the original stored at `stored`, computed as the download
+// check computes it: over each file's bytes, read in chunks, and for a live
+// photo `<imageHash>:<videoHash>`.
+export const storedHash = async (stored: {
+    path: string;
+    videoPath?: string;
+}): Promise<string> => {
+    await init();
+    const hashFile = async (path: string): Promise<string> => {
+        const state = chunkHashInit();
+        for await (const chunk of createReadStream(path)) {
+            chunkHashUpdate(state, chunk as Buffer);
+        }
+        return chunkHashFinal(state);
+    };
+    const hash = await hashFile(stored.path);
+    if (stored.videoPath === undefined) return hash;
+    return `${hash}:${await hashFile(stored.videoPath)}`;
+};
+
 // Copy bytes into `dest` via a temp file in the same directory plus rename, so
 // `dest` appears only once it is whole ("present means complete"). As in the
 // download writer, the temp file is fsynced before the rename and the directory
@@ -405,6 +434,62 @@ export const placeOriginal = async (
     return { path, videoPath };
 };
 
+// Replace the original of `file` stored at its save path under `root` with a
+// new copy, keeping the stored one until the copy is complete and its hash,
+// computed as `storedHash` computes it, is the content hash the file's metadata
+// records. `get` is given a temporary file beside the save path and writes the
+// copy there, a live photo as its image and its video beside that file, each
+// with its own extension. Only a copy that matches is renamed over the stored
+// one, and a live photo's image and video are both hashed before either is
+// renamed. The image is renamed first: should the run stop, or the video's
+// rename fail, before the video is renamed, the new image is left beside the
+// old video, and a later backup with `verify` replaces the video if the video
+// was the part that did not match. When `get` fails or the copy does not match,
+// the temporary files are removed and the stored original stays as it was.
+export const replaceOriginal = async (
+    root: string,
+    file: EnteFile,
+    get: (dest: string) => Promise<{ path: string; videoPath?: string }>,
+): Promise<{ path: string; videoPath?: string }> => {
+    const dest = savePath(root, file);
+    const dir = dirname(dest);
+    // Named as the download writer names its temporary files; a live photo's
+    // image and video get this name with their own extensions. The next
+    // backup removes any that a killed run leaves (`removeLeftoverTempFiles`).
+    const tmp = join(
+        dir,
+        `.quak-${process.pid}-${Math.random().toString(36).slice(2)}.tmp`,
+    );
+    let got: { path: string; videoPath?: string } | undefined;
+    try {
+        got = await get(tmp);
+        if ((await storedHash(got)) !== file.metadata.hash) {
+            throw new Error(
+                "the downloaded original does not match its content hash either",
+            );
+        }
+        if (got.videoPath === undefined) {
+            await rename(got.path, dest);
+            await fsyncPath(dir);
+            return { path: dest };
+        }
+        const path = withExtension(dest, extname(got.path));
+        const videoPath = withExtension(dest, extname(got.videoPath));
+        await rename(got.path, path);
+        await rename(got.videoPath, videoPath);
+        const name = basename(dest, extname(dest));
+        await writeLivePhotoJSON(dir, name, { path, videoPath });
+        return { path, videoPath };
+    } finally {
+        // Whatever was not renamed into place. A failure to remove it must
+        // not replace the error that explains what went wrong.
+        for (const path of [tmp, got?.path, got?.videoPath]) {
+            if (path === undefined) continue;
+            await rm(path, { force: true }).catch(() => undefined);
+        }
+    }
+};
+
 export class ContentCache implements PhotoContent, ThumbnailsAPI {
     private readonly pools: RequestPools;
     private readonly source: ContentSource;
@@ -543,6 +628,35 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
         };
     }
 
+    // Fetch an original through the content pool to `destination`, a live
+    // photo as its image and its video beside it, and return where it landed.
+    // Unlike `backupOriginal`, it fetches even when the cache or the save path
+    // already holds the original, and it records nothing: what it writes is
+    // the caller's. It is not shared with a fetch of the same file already
+    // under way, which writes somewhere else.
+    async fetchOriginal(
+        fileID: number,
+        destination: string,
+    ): Promise<{ path: string; videoPath?: string }> {
+        const file = this.getFile(fileID);
+        if (!file) throw new Error(`content cache: unknown file ${fileID}`);
+        return this.pools.content.run(
+            async () => {
+                const fetched = await this.fetchInto(
+                    file,
+                    destination,
+                    "original",
+                    undefined,
+                );
+                for (const path of [fetched.path, fetched.videoPath]) {
+                    if (path !== undefined) await chmod(path, FILE_MODE);
+                }
+                return fetched;
+            },
+            { priority: "on-demand" },
+        );
+    }
+
     async ensure(args: EnsureOptions): Promise<EnsureResult[]> {
         return this.ensureThumbnails(args);
     }

src/library/index.ts

@@ -606,6 +606,8 @@ export class Library {
                 undecryptable: () => this.store.listUndecryptable(),
                 original: (fileID, destination) =>
                     cache!.backupOriginal(fileID, destination),
+                fetchOriginal: (fileID, destination) =>
+                    cache!.fetchOriginal(fileID, destination),
                 thumbnail: (fileID) => cache!.thumbnail(fileID),
                 contentConcurrency: this.pools.content.concurrency,
                 fetchMLData: () => this.fetchMLDataNow(),