From 73846b32fc54920a48e9bcbfe5f463728acfd035 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 14:57:36 +0000 Subject: [PATCH] quak backup --verify re-hashes stored originals and downloads again any that do not match (closes #168) `--verify`, or `lib.backup({ verify: true })`, hashes each original already at its save path as the download check does, streamed, a live photo as `:`. A mismatch is logged, removed and fetched again in the same run; a failed fetch goes into `failures.json`. A file with no recorded hash counts as unchecked. The result, the summary and `--json` gain `verified`, `mismatched` and `unchecked`. Judgement call: a stored original that cannot be read for hashing is recorded as failed and left in place. Judgement call: the summary prints the three counts only with `--verify`. Judgement call: the README's `BackupOptions` list does not name `verify`, to stay clear of https://git.eeqj.de/sneak/quak/pulls/178's edit of that paragraph; Backup layout documents it. Model: opus-5-5 --- README.md | 20 +++- TODO.md | 9 ++ bin/quak.ts | 6 +- src/backup.ts | 87 +++++++++++++++- src/cli-commands.ts | 8 +- test/cli/backup.test.ts | 207 ++++++++++++++++++++++++++++++++++++++ test/cli/commands.test.ts | 58 ++++++++++- test/live-photo.ts | 3 +- 8 files changed, 391 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ac4a043..d8ae843 100644 --- a/README.md +++ b/README.md @@ -549,7 +549,7 @@ quak collections [--json] list all collections quak files --collection [--json] list files in a collection quak get [--out path] [--collection] download and decrypt a file quak get-thumb [--out] [--collection] download and decrypt a thumbnail -quak backup [--json] full incremental backup +quak backup [--json] [--verify] full incremental backup quak backup-metadata [--exif] dump the metadata quak keeps as JSON quak helper list-missing-thumbnails [--json] find files with missing thumbnails quak helper fix-missing-thumbnails [--file ids] [--json] generate + upload missing thumbnails @@ -582,6 +582,9 @@ reads no tag from is recorded, base64, as `exifRaw`, with the reason in `exifError`. `collections`, `files`, `backup`, `helper list-missing-thumbnails` and `helper fix-missing-thumbnails` take `--json` for machine-readable output. +`backup --verify` also hashes the originals already in the backup and downloads +again any that do not match the content hash Ente records (see "Backup layout"). + `backup-metadata` fetches ML data in requests of up to 200 files. When a request fails, the error is logged, each of its files is written with the reason in an `mlDataError` field instead of `mlData`, and the dump goes on. The exit code is @@ -697,6 +700,21 @@ if any files failed. `quak backup` opens its library with the thumbnail and originals precache off, so the only file content it fetches is the originals the backup stores. +With `--verify`, or `lib.backup({ verify: true })`, a run also hashes each +original already at its save path the way a download is checked (see "On-disk +cache layout" below): its bytes, read in chunks, or a live photo's image and +video, joined as `:`. An original that matches the content +hash its metadata records is left as it is. One that does not is logged on one +line naming the file, deleted (a live photo's image and video both), and +downloaded again in the same run like a missing one; if that download fails, the +file goes into `failures.json`. A file whose metadata records no hash is left as +it is and counted as unchecked. A stored original that cannot be read is left as +it is and counts as failed. The summary and `--json` add the counts `verified`, +`mismatched` and `unchecked`, all of originals that were already stored; one +first downloaded in this run is in none of them. A mismatch that was downloaded +again does not make the exit code non-zero. Without `--verify` nothing is +hashed, the summary is unchanged, and the three counts are 0 in `--json`. + Each original is written to a temporary file in the same directory, synced to disk, and renamed into place, so an original is either complete or absent, even after a power cut. A downloaded original's temporary file is named diff --git a/TODO.md b/TODO.md index cbd2391..8f83fef 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,15 @@ declares one. # Completed Steps +- 2026-10-06: `quak backup --verify` and `lib.backup({ verify: true })` hash + each original already at its save path as the download check does, streamed, a + live photo as `:` (issue 168). One that does not match + the content hash its metadata records is logged, removed (both files of a live + photo) and downloaded again in the same run; a failed download goes into + `failures.json`. One with no recorded hash is left alone. The result, `--json` + and the summary gain `verified`, `mismatched` and `unchecked`. Without + `--verify` nothing is hashed. + - 2026-10-06: `quak backup` writes each original's EXIF, XMP and dimensions into the file's JSON as `imageMetadata`, what `backup-metadata --exif` records (issue 167): for a live photo from its image, for a video nothing, and `{}` diff --git a/bin/quak.ts b/bin/quak.ts index f974348..77a1a95 100644 --- a/bin/quak.ts +++ b/bin/quak.ts @@ -130,9 +130,13 @@ program ) .argument("", "Output directory") .option("--json", "Print result as JSON instead of human-readable summary") + .option( + "--verify", + "Re-hash stored originals and download again any that do not match", + ) // A backup usually runs from cron with nobody watching, so every request // it makes retries for longer than the other commands' requests do. - .action((dir: string, opts: { json?: boolean }) => + .action((dir: string, opts: { json?: boolean; verify?: boolean }) => run( backupCommand( { diff --git a/src/backup.ts b/src/backup.ts index 4da6aa4..d73ac52 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -35,6 +35,10 @@ // why they could not be read), so that a run does not read every stored // original again; a sidecar without them gets them read from the original. // +// With `verify`, each original already at its save path is hashed as the +// download check hashes it, and one that does not match the content hash its +// metadata records is removed and fetched again in the same run. +// // Resilience (issue #8): no per-file condition aborts the run. A failed // download, a failed symlink, or ML data missing because the ML data fetch // failed is caught, recorded in `failures.json` with a classification, a @@ -47,6 +51,7 @@ // code. import { + createReadStream, lstatSync, mkdirSync, readdirSync, @@ -61,6 +66,12 @@ import { import { readFile } from "node:fs/promises"; import { dirname, extname, join, relative, resolve } from "node:path"; +import { + chunkHashFinal, + chunkHashInit, + chunkHashUpdate, + init, +} from "./crypto/index.js"; import { removeLeftoverTempFiles } from "./download/index.js"; import { sanitizeFileName, withExtension } from "./filename.js"; import { @@ -88,6 +99,9 @@ export interface BackupOptions { includeThumbnails?: boolean; // Restrict the backup to albums with these names; others are left untouched. onlyAlbumNames?: string[]; + // Hash each original already at its save path, and fetch again any whose + // bytes do not match the content hash its metadata records. Default false. + verify?: boolean; onProgress?: ProgressCallback; } @@ -105,6 +119,13 @@ export interface BackupResult { downloaded: number; // Originals already at their save path and left untouched. skipped: number; + // With `verify`, the originals already at their save path whose hash + // matched, those whose hash did not (each removed and fetched again), and + // those whose metadata records no hash (left as they are). All three are + // zero without `verify`. + verified: number; + mismatched: number; + unchecked: number; // Files with an unresolved failure after this run (the ledger size); the // CLI exits non-zero while this is above zero. A file can be both // downloaded and failed if its bytes landed but its symlink did not. @@ -361,6 +382,26 @@ const saveLedger = (path: string, ledger: Map): void => { ); }; +// The content hash of the original stored at `stored`, computed as the download +// check computes it: over each file's bytes, read in chunks, and for a live +// photo `:`. +const storedHash = async (stored: { + path: string; + videoPath?: string; +}): Promise => { + await init(); + const hashFile = async (path: string): Promise => { + const state = chunkHashInit(); + for await (const chunk of createReadStream(path)) { + chunkHashUpdate(state, chunk as Buffer); + } + return chunkHashFinal(state); + }; + const hash = await hashFile(stored.path); + if (stored.videoPath === undefined) return hash; + return `${hash}:${await hashFile(stored.videoPath)}`; +}; + // A file's EXIF, XMP and dimensions as its JSON holds them: what // `extractImageMetadata` found in its original, or why the original could not // be read. @@ -465,6 +506,7 @@ export const runBackup = async ( const includeThumbnails = opts.includeThumbnails ?? false; const log = opts.onProgress ?? (() => {}); const only = opts.onlyAlbumNames ? new Set(opts.onlyAlbumNames) : undefined; + const verify = opts.verify ?? false; log("Refreshing library..."); await lib.refresh(); @@ -523,6 +565,9 @@ export const runBackup = async ( const storedThisRun = new Set(); let downloaded = 0; let skipped = 0; + let verified = 0; + let mismatched = 0; + let unchecked = 0; const recordFailure = ( file: EnteFile, @@ -554,10 +599,45 @@ export const runBackup = async ( // Phase 1: get the bytes. Put each pending original at its save path // through the content cache/pools, as `Photo.download()` does, and fetch - // the optional thumbnails; a present file is left as is. + // the optional thumbnails; a present file is left as is. With `verify`, a + // present original is hashed first, and one that does not match the hash + // its metadata records is removed and fetched like a missing one. One + // that cannot be read is recorded as failed and left where it is. if (includeOriginals) { for (const [fileID, file] of distinct) { - if (storedAtSavePath(downloadDirectory, file) !== undefined) { + let stored = storedAtSavePath(downloadDirectory, file); + if (stored !== undefined && verify) { + try { + if (file.metadata.hash === undefined) { + unchecked++; + } else if ( + (await storedHash(stored)) === file.metadata.hash + ) { + verified++; + } else { + log( + `MISMATCH original ${file.metadata.title} (${fileID}): its bytes do not match its content hash`, + ); + mismatched++; + rmSync(stored.path); + if (stored.videoPath !== undefined) { + rmSync(stored.videoPath); + } + stored = undefined; + } + } catch (err) { + log( + `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`, + ); + recordFailure( + file, + collectionName.get(file.collectionID) ?? "", + err, + ); + continue; + } + } + if (stored !== undefined) { skipped++; continue; } @@ -732,6 +812,9 @@ export const runBackup = async ( totalFiles: distinct.size, downloaded, skipped, + verified, + mismatched, + unchecked, failed: ledger.size, errors, }; diff --git a/src/cli-commands.ts b/src/cli-commands.ts index 30aa4ee..87baeaf 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -401,7 +401,7 @@ export const backupMetadataCommand = async ( export const backupCommand = async ( ctx: CliContext, dir: string, - opts: { json?: boolean }, + opts: { json?: boolean; verify?: boolean }, ): Promise => { await init(); const client = requireSession(ctx); @@ -420,6 +420,7 @@ export const backupCommand = async ( try { const result = await lib.backup({ downloadDirectory: dir, + verify: opts.verify, onProgress: (msg) => { if (!opts.json) ctx.stderr.write(msg + "\n"); }, @@ -432,6 +433,11 @@ export const backupCommand = async ( ctx.stderr.write(` Total files: ${result.totalFiles}\n`); ctx.stderr.write(` Downloaded: ${result.downloaded}\n`); ctx.stderr.write(` Skipped: ${result.skipped}\n`); + if (opts.verify) { + ctx.stderr.write(` Verified: ${result.verified}\n`); + ctx.stderr.write(` Mismatched: ${result.mismatched}\n`); + ctx.stderr.write(` Unchecked: ${result.unchecked}\n`); + } ctx.stderr.write(` Failed: ${result.failed}\n`); if (result.errors.length > 0) { ctx.stderr.write("\nFailed files:\n"); diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index b30ad5c..84cc8f3 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -61,6 +61,7 @@ import { HEIC_WITH_EXIF } from "../exif-heic.js"; import { JPEG_WITH_EXIF } from "../exif-jpeg.js"; import { asLivePhoto, + blake2b, cdnSource, IMAGE, livePhotoHash, @@ -1196,6 +1197,154 @@ describe("image metadata in each file's JSON", () => { }); }); +// With `verify`, each original already stored is hashed, and one that does not +// match the content hash its metadata records is downloaded again. +describe("backup with verify", () => { + // MockClient's files, each recording the hash of what `stubSource` writes + // for it, except diagram.png (200), which records none. + class HashedClient extends MockClient { + override async filesSince(args: { + collectionID: number; + }): Promise { + const page = await super.filesSince(args); + const files = page.files.map((f) => + f.id === 200 + ? f + : { + ...f, + metadata: { + ...f.metadata, + hash: blake2b(Buffer.alloc(SIZE_BY_ID[f.id]!)), + }, + }, + ); + return { ...page, files }; + } + } + + // A backup of the account in `root/backup`, the library that made it, and + // the source it fetched from. + const backedUp = async (): Promise<{ + lib: Library; + source: StubSource; + outDir: string; + }> => { + const source = stubSource(); + const lib = await openLibrary(source, new HashedClient()); + const outDir = join(root, "backup"); + await lib.backup({ downloadDirectory: outDir }); + return { lib, source, outDir }; + }; + + // What `stubSource` writes for beach.jpg (100), and other bytes of the + // same length. + const good = Buffer.alloc(SIZE_BY_ID[100]!); + const corrupt = Buffer.alloc(SIZE_BY_ID[100]!, 1); + + it("leaves an original that matches its hash, and one with no hash, as they are", async () => { + const { lib, source, outDir } = await backedUp(); + const calls = source.originalCalls; + + const result = await lib.backup({ + downloadDirectory: outDir, + verify: true, + }); + + expect(result).toMatchObject({ + downloaded: 0, + skipped: 3, + verified: 2, + mismatched: 0, + unchecked: 1, + failed: 0, + }); + expect(source.originalCalls).toBe(calls); + await lib.close(); + }); + + it("downloads again an original that does not match its hash", async () => { + const { lib, outDir } = await backedUp(); + writeFileSync(saved(outDir, "100.jpg"), corrupt); + const log: string[] = []; + + const result = await lib.backup({ + downloadDirectory: outDir, + verify: true, + onProgress: (msg) => log.push(msg), + }); + + expect(result).toMatchObject({ + downloaded: 1, + skipped: 2, + verified: 1, + mismatched: 1, + unchecked: 1, + failed: 0, + }); + expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(good); + expect(log.filter((msg) => msg.startsWith("MISMATCH"))).toEqual([ + "MISMATCH original beach.jpg (100): its bytes do not match its content hash", + ]); + await lib.close(); + }); + + it("records a failed download in failures.json, with the original removed", async () => { + const { lib, source, outDir } = await backedUp(); + writeFileSync(saved(outDir, "100.jpg"), corrupt); + source.failID = 100; + + const result = await lib.backup({ + downloadDirectory: outDir, + verify: true, + }); + + expect(result).toMatchObject({ + downloaded: 0, + mismatched: 1, + failed: 1, + }); + expect(Object.keys(readLedger(outDir).files)).toEqual(["100"]); + expect(existsSync(saved(outDir, "100.jpg"))).toBe(false); + await lib.close(); + }); + + it("hashes nothing without verify", async () => { + const { lib, outDir } = await backedUp(); + writeFileSync(saved(outDir, "100.jpg"), corrupt); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ + downloaded: 0, + skipped: 3, + verified: 0, + mismatched: 0, + unchecked: 0, + failed: 0, + }); + expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(corrupt); + await lib.close(); + }); + + // Root ignores file permissions, so this fails when run as root. The + // `test` phase of the `Dockerfile` runs as the `node` user. + it("records an original it cannot read as failed, and leaves it", async () => { + const { lib, outDir } = await backedUp(); + const original = saved(outDir, "100.jpg"); + chmodSync(original, 0o000); + + const result = await lib + .backup({ downloadDirectory: outDir, verify: true }) + .finally(() => chmodSync(original, 0o600)); + + expect(result).toMatchObject({ downloaded: 0, verified: 1, failed: 1 }); + expect(result.errors.map((e) => e.fileID)).toEqual([100]); + expect(result.errors[0]!.error).toMatch(/EACCES/); + expect(readFileSync(original)).toEqual(good); + await lib.close(); + }); +}); + // Every entry under collections/, one level of directories deep, with each // symlink's target. const tree = (outDir: string): string[] => { @@ -1691,6 +1840,64 @@ describe("backup of live photos", () => { }, ); + it("verifies a live photo's image and video together, and downloads it again when one does not match", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + const outDir = join(root, "backup"); + await lib.backup({ downloadDirectory: outDir }); + + const good = await lib.backup({ + downloadDirectory: outDir, + verify: true, + }); + expect(good).toMatchObject({ skipped: 1, verified: 1, mismatched: 0 }); + + const video = saved(outDir, "500.mov"); + writeFileSync(video, "another few seconds of video"); + const bad = await lib.backup({ + downloadDirectory: outDir, + verify: true, + }); + + expect(bad).toMatchObject({ + downloaded: 1, + verified: 0, + mismatched: 1, + failed: 0, + }); + expect(readFileSync(saved(outDir, "500.heic"))).toEqual( + Buffer.from(IMAGE), + ); + expect(readFileSync(video)).toEqual(Buffer.from(VIDEO)); + expect(tree(outDir)).toEqual(linked); + await lib.close(); + }); + + it("removes both files of a live photo that does not match its hash when downloading it again fails", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const bodies = new Map([[500, body]]); + const lib = await open([live], bodies); + const outDir = join(root, "backup"); + await lib.backup({ downloadDirectory: outDir }); + writeFileSync(saved(outDir, "500.mov"), "another few seconds of video"); + bodies.delete(500); + + const result = await lib.backup({ + downloadDirectory: outDir, + verify: true, + }); + + expect(result).toMatchObject({ mismatched: 1, failed: 1 }); + expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]); + expect(existsSync(saved(outDir, "500.heic"))).toBe(false); + expect(existsSync(saved(outDir, "500.mov"))).toBe(false); + await lib.close(); + }); + it("serves a live photo the backup stored to a library reading the backup", async () => { const { file: live, body } = await asLivePhoto( file(500, 10, "IMG_0500.HEIC"), diff --git a/test/cli/commands.test.ts b/test/cli/commands.test.ts index 633cc35..60bf353 100644 --- a/test/cli/commands.test.ts +++ b/test/cli/commands.test.ts @@ -52,13 +52,14 @@ import { import { run } from "../../src/cli-run.js"; import { loadSession } from "../../src/cli-session.js"; import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js"; -import type { ContentSource } from "../../src/library/content.js"; +import { savePath, type ContentSource } from "../../src/library/content.js"; import type { Collection, EnteFile } from "../../src/model/types.js"; import { init, toBase64 } from "../../src/crypto/index.js"; import { defaultCacheDirectory } from "../../src/library/index.js"; import { HEIC_WITH_EXIF } from "../exif-heic.js"; import { asLivePhoto, + blake2b, cdnSource, IMAGE, livePhotoHash, @@ -742,6 +743,61 @@ describe("backup", () => { expect(stderr.text).toBe("Starting backup...\n"); }); + it("--verify downloads again an original that does not match its hash, prints the counts, and exits 0", async () => { + // Each file records the hash of the original the fake writes for it. + const client = { + ...fakeClient(), + filesSince: async (args: { collectionID: number }) => ({ + files: (FILES[args.collectionID] ?? []).map((f) => ({ + ...f, + metadata: { + ...f.metadata, + hash: blake2b(Buffer.alloc(7, f.id & 0xff)), + }, + })), + deleted: [], + cursor: 1, + }), + } as unknown as Client; + const ctx = context(client); + const dir = join(root, "backup"); + expect(await backupCommand(ctx, dir, {})).toBe(0); + writeFileSync(savePath(dir, FILES[1]![0]!), "corrupt"); + + expect(await backupCommand(ctx, dir, { verify: true })).toBe(0); + + expect(stderr.text).toContain( + "MISMATCH original beach.jpg (100): its bytes do not match its content hash\n", + ); + expect(stderr.text).toContain( + " Downloaded: 1\n" + + " Skipped: 2\n" + + " Verified: 2\n" + + " Mismatched: 1\n" + + " Unchecked: 0\n" + + " Failed: 0\n", + ); + }); + + it("--verify --json adds the verified, mismatched and unchecked counts", async () => { + const dir = join(root, "backup"); + expect(await backupCommand(context(), dir, {})).toBe(0); + + const code = await backupCommand(context(), dir, { + verify: true, + json: true, + }); + + expect(code).toBe(0); + expect(JSON.parse(stdout.text)).toMatchObject({ + skipped: 3, + verified: 0, + mismatched: 0, + unchecked: 3, + failed: 0, + }); + }); + it("exits 1 and lists each file when the ML data fetch fails", async () => { const client = { ...fakeClient(), diff --git a/test/live-photo.ts b/test/live-photo.ts index 1e3f17e..a966ff0 100644 --- a/test/live-photo.ts +++ b/test/live-photo.ts @@ -30,7 +30,8 @@ export const livePhotoZip = ( }, ): Uint8Array => zipSync(entries); -const blake2b = (bytes: Uint8Array): string => +// The content hash Ente's clients record for an original's bytes. +export const blake2b = (bytes: Uint8Array): string => createHash("blake2b512").update(bytes).digest("base64"); // The hash Ente's clients record for a live photo: the unkeyed BLAKE2b-512 of