diff --git a/README.md b/README.md index bc74537..422e5d5 100644 --- a/README.md +++ b/README.md @@ -1096,6 +1096,12 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange` Each pool orders on-demand work ahead of background/precache work and dedups in-flight fetches by key, and an idle pool never lends its slots to a busy one. +`lib.backup()`, and so `quak backup`, works on up to the content pool's limit of +originals at once, 5 unless the library was opened with other `pools`. The limit +covers everything it does for one original, hashing it for `verify` included, so +however large the account is, no more originals than that are open at once. With +`includeThumbnails`, it fetches the thumbnails one at a time. + ### On-disk cache layout Under `cacheDirectory`: diff --git a/TODO.md b/TODO.md index 5f555ea..3c991de 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,14 @@ declares one. # Completed Steps +- 2026-10-14: `lib.backup()`, and so `quak backup`, works on up to the content + pool's limit of originals at once, 5 by default, instead of one at a time + (issue 182). That many workers take files from one shared list, each doing all + the work for one original, its `--verify` hashing included, so no more + originals than that are open at once. `BackupLibrary` says the limit as + `contentConcurrency`. A test holds each download at the fake Ente server to + count those in flight. + - 2026-10-14: A backup whose lock is lost no longer dies with a stack trace (issue 184). `lockBackupDirectory` gives proper-lockfile an `onCompromised` that records why the lock was lost instead of throwing from its timer, and diff --git a/src/backup.ts b/src/backup.ts index bd76d2f..bbb7d7c 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -6,9 +6,9 @@ // file. It waits for a completed refresh of the library (a failed one fails the // backup before any file is touched), then, for every file in scope, puts its // original at its save path under `downloadDirectory`, as `Photo.download()` -// does, waits for an ML data fetch, and rebuilds the derived views (per-file -// sidecars, per-collection symlink trees, per-collection JSON) from the model. -// The on-disk layout: +// does, up to the content pool's limit of them at once, waits for an ML data +// fetch, and rebuilds the derived views (per-file sidecars, per-collection +// symlink trees, per-collection JSON) from the model. The on-disk layout: // // / // YYYY/YYYY-MM/YYYY-MM-DD/ @@ -181,6 +181,9 @@ export interface BackupLibrary { destination: string, ): Promise<{ path: string; videoPath?: string }>; thumbnail(fileID: number): Promise<{ path: string }>; + // How many originals the backup works on at once: the limit of the + // library's content pool. + contentConcurrency: number; // Wait for an ML data fetch to complete, joining one already running or // starting one. Rejects with the reason when it fails; resolves at once // when the library cannot fetch ML data. @@ -654,75 +657,32 @@ const runLockedBackup = async ( // present original is hashed first, and one that does not match the hash // its metadata records is removed and fetched like a missing one. One // that cannot be read is recorded as failed and left where it is. - if (includeOriginals) { - for (const [fileID, file] of distinct) { - if (lock.lost) throw lock.lost; - let stored = storedAtSavePath(downloadDirectory, file); - let mismatch = false; - if (stored !== undefined && verify) { - try { - if (file.metadata.hash === undefined) { - unchecked++; - } else if ( - (await storedHash(stored)) === file.metadata.hash - ) { - verified++; - } else { - log( - `MISMATCH original ${file.metadata.title} (${fileID}): its bytes do not match its content hash`, - ); - mismatched++; - mismatch = true; - rmSync(stored.path); - if (stored.videoPath !== undefined) { - rmSync(stored.videoPath); - } - stored = undefined; - } - } catch (err) { - log( - `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`, - ); - recordFailure( - file.id, - file.metadata.title, - collectionName.get(file.collectionID) ?? "", - err, - ); - continue; - } - } - if (stored !== undefined) { - skipped++; - continue; - } + // `storeOriginal` does this for one original, and records its failure + // rather than throwing. + const storeOriginal = async (file: EnteFile): Promise => { + let stored = storedAtSavePath(downloadDirectory, file); + let mismatch = false; + if (stored !== undefined && verify) { try { - log(`Fetching original ${file.metadata.title} (${fileID})...`); - // A fetched original is written straight to its save path (a - // live photo beside it); only one that was already cached - // elsewhere is copied. - const placed = await placeOriginal( - downloadDirectory, - file, - (dest) => lib.original(fileID, dest), - ); - storedThisRun.add(fileID); - // A copy from the cache is not checked as a download is and can - // hold the same bad bytes, so what is put back after a - // mismatch is hashed too. A bad copy stays where it is and the - // file fails. - if ( - mismatch && - (await storedHash(placed)) !== file.metadata.hash - ) { - throw new Error( - "the original put back does not match its content hash either", + if (file.metadata.hash === undefined) { + unchecked++; + } else if ((await storedHash(stored)) === file.metadata.hash) { + verified++; + } else { + log( + `MISMATCH original ${file.metadata.title} (${file.id}): its bytes do not match its content hash`, ); + mismatched++; + mismatch = true; + rmSync(stored.path); + if (stored.videoPath !== undefined) { + rmSync(stored.videoPath); + } + stored = undefined; } - downloaded++; } catch (err) { log( - `FAILED original ${file.metadata.title}: ${errorMessage(err)}`, + `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`, ); recordFailure( file.id, @@ -730,8 +690,56 @@ const runLockedBackup = async ( collectionName.get(file.collectionID) ?? "", err, ); + return; } } + if (stored !== undefined) { + skipped++; + return; + } + try { + log(`Fetching original ${file.metadata.title} (${file.id})...`); + // A fetched original is written straight to its save path (a + // live photo beside it); only one that was already cached + // elsewhere is copied. + const placed = await placeOriginal( + downloadDirectory, + file, + (dest) => lib.original(file.id, dest), + ); + storedThisRun.add(file.id); + // A copy from the cache is not checked as a download is and can + // hold the same bad bytes, so what is put back after a mismatch + // is hashed too. A bad copy stays where it is and the file fails. + if (mismatch && (await storedHash(placed)) !== file.metadata.hash) { + throw new Error( + "the original put back does not match its content hash either", + ); + } + downloaded++; + } catch (err) { + log(`FAILED original ${file.metadata.title}: ${errorMessage(err)}`); + recordFailure( + file.id, + file.metadata.title, + collectionName.get(file.collectionID) ?? "", + err, + ); + } + }; + + // Up to the content pool's limit of originals are worked on at once, + // their `verify` hashing included, so however large the account is, no + // more than that many are open. Each worker takes the next file from the + // iterator they all share until none is left. + if (includeOriginals) { + const pending = distinct.values(); + const worker = async (): Promise => { + for (const file of pending) await storeOriginal(file); + }; + await Promise.all( + Array.from({ length: lib.contentConcurrency }, worker), + ); } if (includeThumbnails) { diff --git a/src/library/index.ts b/src/library/index.ts index eb36410..95d8e1d 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -607,6 +607,7 @@ export class Library { original: (fileID, destination) => cache!.backupOriginal(fileID, destination), thumbnail: (fileID) => cache!.thumbnail(fileID), + contentConcurrency: this.pools.content.concurrency, fetchMLData: () => this.fetchMLDataNow(), mlData: (fileID) => this.mldata.forFile({ fileID }), }, diff --git a/test/cli/backup-concurrency.test.ts b/test/cli/backup-concurrency.test.ts index fe8caa4..964bff5 100644 --- a/test/cli/backup-concurrency.test.ts +++ b/test/cli/backup-concurrency.test.ts @@ -208,7 +208,8 @@ const holdingServer = (failing?: number) => { // Let the held downloads go one at a time, the first to arrive first, each // once `limit` of them wait, or all the downloads still to come when there // are fewer. Each of the eight photos is downloaded once. With a backup that -// downloads fewer than `limit` at once, this fails waiting for them. +// downloads fewer than `limit` at once, this fails waiting for them. Each wait +// allows five seconds, for a loaded machine's disk to sync the files written. const releaseOneAtATime = async ( held: { release: () => void }[], limit: number, @@ -216,7 +217,7 @@ const releaseOneAtATime = async ( for (let left = PHOTO_IDS.length; left > 0; left--) { await vi.waitFor( () => expect(held).toHaveLength(Math.min(limit, left)), - { interval: 5 }, + { interval: 5, timeout: 5000 }, ); held.shift()!.release(); } @@ -340,6 +341,7 @@ describe("lib.backup()", () => { }); await vi.waitFor(() => expect(holding.held).toHaveLength(5), { interval: 5, + timeout: 5000, }); expect(mismatches()).toHaveLength(5); diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index 886a1fc..0ec1fd7 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -704,9 +704,9 @@ describe("a backup whose lock is lost", () => { } }; - // beach.jpg, whose download was under way, is stored; no other file is - // started. `quak backup` prints why as one line, with no stack trace, and - // exits 1. + // The backup downloads up to five originals at once, so all three are + // under way when the lock is lost, and beach.jpg is stored. `quak backup` + // prints why as one line, with no stack trace, and exits 1. it("stops quak backup before its next file when backup.lock is removed, and exits 1", async () => { const result = backup(dir, {}, fetch); await heldRequested; @@ -720,9 +720,12 @@ describe("a backup whose lock is lost", () => { "Starting backup...\n" + "Refreshing library...\n" + "Fetching original beach.jpg (100)...\n" + + "Fetching original IMG_0002.HEIC (300)...\n" + + "Fetching original toast.mp4 (200)...\n" + + "Fetching ML data...\n" + `quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'\n`, ); - expect(downloads).toEqual([BEACH.id]); + expect(downloads.sort()).toEqual([100, 200, 300]); expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual( Buffer.from(JPEG_WITH_EXIF), ); @@ -798,7 +801,7 @@ describe("a backup whose lock is lost", () => { await expect(result).rejects.toThrow( `lost the lock on ${dir}: Unable to update lock within the stale threshold`, ); - expect(downloads).toEqual([BEACH.id]); + expect(downloads.sort()).toEqual([100, 200, 300]); expect(existsSync(lockPath)).toBe(true); } finally { await lib.close(); @@ -823,7 +826,7 @@ describe("a backup whose lock is lost", () => { const lost = `lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'`; await expect(result).rejects.toThrow(lost); expect(lock.lost?.message).toBe(lost); - expect(downloads).toEqual([BEACH.id]); + expect(downloads.sort()).toEqual([100, 200, 300]); } finally { await lib.close(); await lock.release(); diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 065a2c5..7330b72 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -1491,6 +1491,7 @@ describe("backup album folders", () => { thumbnail: async () => { throw new Error("no thumbnails in this stand-in"); }, + contentConcurrency: 5, fetchMLData: async () => {}, mlData: async () => undefined, whoami: () => ({ email: "backup@example.com", userID: USER_ID }),