From cb981ea75d29211e2332827bfc6a39d72c03f8f6 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 23 Sep 2026 06:16:10 +0000 Subject: [PATCH] Write each backed-up original once and skip the precache (closes #106) An original fetched for a backup is now written by the download writer straight into the backup's originals/, and the content cache records it there instead of keeping its own copy. One the cache already held is still copied. `quak backup` opens its library with the thumbnail and originals precache off, as the one-shot commands do. Model: opus-5-5 --- README.md | 48 ++++++++++++++++++++++----------------- TODO.md | 7 ++++++ src/backup.ts | 12 ++++++---- src/cli-commands.ts | 4 ++++ src/library/content.ts | 24 +++++++++++++++++--- src/library/index.ts | 3 ++- test/cli/backup.test.ts | 30 +++++++++++++++++++++++- test/cli/commands.test.ts | 11 +++++++++ 8 files changed, 109 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 73db1a7..51bf3ef 100644 --- a/README.md +++ b/README.md @@ -510,22 +510,25 @@ symlink you put there stays, and a directory that still holds one after its symlinks are removed stays too, with its JSON. Each file is downloaded exactly once regardless of how many collections it -appears in. On subsequent runs, existing originals are skipped. If a download -fails, the error is logged and the backup continues with the next file. The exit -code is non-zero if any files failed. +appears in, and written once: straight into `originals/`, with no copy left in +the cache. An original the cache already held is copied from there instead. On +subsequent runs, existing originals are skipped. If a download fails, the error +is logged and the backup continues with the next file. The exit code is non-zero +if any files failed. `quak backup` opens its library with the thumbnail and +originals precache off, so it fetches only what the backup stores. -Each original is copied to a temporary file named -`.quak-backup-.--.tmp` in the same directory, synced -to disk, and renamed into place, so an original is either complete or absent, -even after a power cut. A run that is killed can leave one of these temporary -files behind; the next backup deletes those whose process is no longer running. -Downloads and the content cache use the same scheme with -`.quak--.tmp` names, and opening a library deletes those in the -cache whose process is no longer running, so a download another process has in -progress in the same cache is left alone. The rename replaces whatever was at -the destination rather than writing through it: a symlink there is replaced, not -followed, and the new file has the temporary file's permissions, not those of -the file it replaced. +Each original is written to a temporary file in the same directory, synced to +disk, and renamed into place, so an original is either complete or absent, even +after a power cut. A downloaded original's temporary file is named +`.quak--.tmp`, one copied from the cache +`.quak-backup-.--.tmp`. A run that is killed can leave +one of these temporary files behind; the next backup deletes those whose process +is no longer running. The content cache uses the same scheme, and opening a +library deletes the temporary files in the cache whose process is no longer +running, so a download another process has in progress in the same cache is left +alone. The rename replaces whatever was at the destination rather than writing +through it: a symlink there is replaced, not followed, and the new file has the +temporary file's permissions, not those of the file it replaced. ## TODO @@ -667,12 +670,15 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange` default limit 20). quak bundles no text encoder, so `searchByEmbedding` takes a query vector the caller produced elsewhere. - `await lib.backup(opts?)` → `BackupResult`. It refreshes, fetches every - in-scope original (and, with `includeThumbnails`, thumbnails) through the - content cache, and rebuilds the on-disk backup tree with a durable failure - ledger. `BackupOptions`: `downloadDirectory` (falls back to the one `open()` - was given), `includeOriginals` (default `true`), `includeThumbnails` (default - `false`), `onlyAlbumNames`, and `onProgress`. See Backup layout above for the - tree it writes. + in-scope original not already in the backup (and, with `includeThumbnails`, + thumbnails) through the content cache, and rebuilds the on-disk backup tree + with a durable failure ledger. A fetched original is written straight into the + backup's `originals/` and not into the cache, which then counts it as present; + one the cache already held is copied from there. `BackupOptions`: + `downloadDirectory` (falls back to the one `open()` was given), + `includeOriginals` (default `true`), `includeThumbnails` (default `false`), + `onlyAlbumNames`, and `onProgress`. See Backup layout above for the tree it + writes. ### Request pools diff --git a/TODO.md b/TODO.md index 111fcf0..afaa444 100644 --- a/TODO.md +++ b/TODO.md @@ -18,6 +18,13 @@ Tag v1.0.0. # Completed Steps +- 2026-09-23: `quak backup` writes each original once and no longer fills the + cache (issue 106). An original fetched for a backup is written by the download + writer straight into the backup's `originals/`, and the content cache records + it there instead of keeping its own copy; one the cache already held is still + copied. `quak backup` opens its library with the thumbnail and originals + precache off. + - 2026-09-23: `backup-metadata`, `helper list-missing-thumbnails` and `helper fix-missing-thumbnails` refresh before they answer (issue 100). Each awaits `lib.fresh()` before reading, so a file added since the cache was diff --git a/src/backup.ts b/src/backup.ts index e20dce1..019619c 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -4,7 +4,8 @@ // fails the backup before any file is touched), then, for every file in scope, // gets its original bytes onto disk under `downloadDirectory` and rebuilds the // derived views (per-file sidecars, per-collection symlink trees, -// per-collection JSON) from the model. The on-disk layout is the historical one, unchanged: +// per-collection JSON) from the model. The on-disk layout is the historical +// one, unchanged: // // / // originals/. the decrypted bytes @@ -96,8 +97,9 @@ export interface BackupLibrary { listCollections(): Collection[]; listFiles(collectionID: number): EnteFile[]; // Get an original's bytes onto disk through the content cache/pools, - // returning where they landed (the cache, or a prior backup). - original(fileID: number): Promise<{ path: string }>; + // returning where they landed: `destination` when they were fetched now, + // otherwise wherever they already were (the cache, or a prior backup). + original(fileID: number, destination: string): Promise<{ path: string }>; thumbnail(fileID: number): Promise<{ path: string }>; } @@ -424,7 +426,9 @@ export const runBackup = async ( } try { log(`Fetching original ${file.metadata.title} (${fileID})...`); - const { path } = await lib.original(fileID); + // A fetched original is written straight to `dest`; only one + // that was already cached elsewhere is copied. + const { path } = await lib.original(fileID, dest); await copyAtomic(path, dest); downloaded++; } catch (err) { diff --git a/src/cli-commands.ts b/src/cli-commands.ts index a78526f..041b656 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -380,10 +380,14 @@ export const backupCommand = async ( if (!client) return 1; ctx.stderr.write("Starting backup...\n"); + // The precache is off: the backup fetches what it needs, and must not + // also fill the cache with every thumbnail and the recent originals. const lib = await Library.open({ client, downloadDirectory: dir, cacheDirectory: ctx.cacheDir, + precacheThumbnails: false, + precacheOriginals: false, }); try { const result = await lib.backup({ diff --git a/src/library/content.ts b/src/library/content.ts index ee0691e..fce4c9b 100644 --- a/src/library/content.ts +++ b/src/library/content.ts @@ -321,6 +321,23 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { return this.get(fileID, "thumbnail", "on-demand", opts?.onProgress); } + // Get an original for a backup. One not present anywhere is written + // straight to `destination` and recorded there, so no second copy lands + // in the cache; one already present is returned where it is. + async backupOriginal( + fileID: number, + destination: string, + ): Promise { + const result = await this.acquire( + fileID, + "original", + "on-demand", + undefined, + { destination }, + ); + return { path: result.path, bytes: result.bytes }; + } + async ensure(args: EnsureOptions): Promise { return this.ensureThumbnails(args); } @@ -422,13 +439,14 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { // The core: return the cached path if present, else fetch through the pool, // store, and return it. `cached` distinguishes a present hit (no network, - // no download event) from a fresh fetch. + // no download event) from a fresh fetch. A fetched original is stored at + // `opts.destination` when given, instead of in `originalsDir`. private async acquire( fileID: number, kind: Kind, priority: Priority, signal: AbortSignal | undefined, - opts?: { onByte?: ProgressCallback }, + opts?: { onByte?: ProgressCallback; destination?: string }, ): Promise<{ path: string; bytes: number; cached: boolean }> { const file = this.getFile(fileID); if (!file) throw new Error(`content cache: unknown file ${fileID}`); @@ -469,7 +487,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { kind === "original" ? this.originalsDir : this.thumbnailsDir; const dest = kind === "original" - ? join(dir, originalName(file)) + ? (opts?.destination ?? join(dir, originalName(file))) : join(dir, `${fileID}${THUMBNAIL_EXT}`); const pool = kind === "original" ? this.pools.content : this.pools.thumbnails; diff --git a/src/library/index.ts b/src/library/index.ts index f6561f4..a7d7865 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -571,7 +571,8 @@ export class Library { refresh: () => this.refreshNow(), listCollections: () => this.store.listCollections(), listFiles: (id) => this.store.listFiles(id), - original: (fileID) => cache!.original(fileID), + original: (fileID, destination) => + cache!.backupOriginal(fileID, destination), thumbnail: (fileID) => cache!.thumbnail(fileID), }, { ...opts, downloadDirectory }, diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index ea3717e..7bcf3bd 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -562,11 +562,39 @@ describe("lib.backup", () => { lib.close(); }); - it("fsyncs a copied original before the rename and its directory after", async () => { + it("fetches each original once and writes it only into the backup", async () => { + // A backup of a 500 GB account must write 500 GB, not a copy in the + // cache as well: an original fetched for the backup goes straight + // into its originals/, and the cache records it there. + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result.downloaded).toBe(3); + expect(source.originalCalls).toBe(3); + expect(readdirSync(join(root, "cache", "originals"))).toEqual([]); + const stored = readdirSync(join(outDir, "originals")).filter( + (name) => !name.endsWith(".json"), + ); + expect(stored.sort()).toEqual(["100.jpg", "101.jpg", "200.png"]); + // The cache counts the backup's copy as present: reading the + // original afterwards fetches nothing and answers with that copy. + const read = await lib.photos.byID({ fileID: 100 })!.original(); + expect(read.path).toBe(join(outDir, "originals", "100.jpg")); + expect(source.originalCalls).toBe(3); + await lib.close(); + }); + + it("fsyncs an original copied from the cache before the rename and its directory after", async () => { const lib = await openLibrary(stubSource()); const outDir = join(root, "backup"); const originals = join(outDir, "originals"); const dest = join(originals, "100.jpg"); + // Only an original already in the cache is copied into the backup; + // one fetched for the backup is written there by the download writer. + await lib.photos.byID({ fileID: 100 })!.original(); fsEvents.length = 0; await lib.backup({ downloadDirectory: outDir }); diff --git a/test/cli/commands.test.ts b/test/cli/commands.test.ts index 3e4a1ba..3f33a5b 100644 --- a/test/cli/commands.test.ts +++ b/test/cli/commands.test.ts @@ -13,6 +13,7 @@ import { existsSync, mkdtempSync, + readdirSync, readFileSync, rmSync, statSync, @@ -427,6 +428,16 @@ describe("backup", () => { expect(stdout.text).toBe(""); }); + // The backup opens its library with the precache off: it fetches the + // originals it needs into the backup, and must not also fetch every + // thumbnail in the account, or keep originals, in the per-user cache. + it("leaves nothing in the cache's originals and thumbnails", async () => { + const dir = join(root, "backup"); + expect(await backupCommand(context(), dir, {})).toBe(0); + expect(readdirSync(join(root, "cache", "thumbnails"))).toEqual([]); + expect(readdirSync(join(root, "cache", "originals"))).toEqual([]); + }); + it("exits 1 and lists the file when one download fails", async () => { const ctx = context(fakeClient({ failID: 101 })); expect(await backupCommand(ctx, join(root, "backup"), {})).toBe(1); -- 2.54.0