diff --git a/README.md b/README.md index 5654819..d8bb308 100644 --- a/README.md +++ b/README.md @@ -1053,6 +1053,12 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange` Each pool orders on-demand work ahead of background/precache work and dedups in-flight fetches by key, and an idle pool never lends its slots to a busy one. +`lib.backup()`, and so `quak backup`, works on up to the content pool's limit of +originals at once, 5 unless the library was opened with other `pools`. The limit +covers everything it does for one original, hashing it for `verify` included, so +however large the account is, no more originals than that are open at once. With +`includeThumbnails`, it fetches the thumbnails one at a time. + ### On-disk cache layout Under `cacheDirectory`: diff --git a/TODO.md b/TODO.md index 25a6bfb..e8db7a2 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,14 @@ declares one. # Completed Steps +- 2026-10-14: `lib.backup()`, and so `quak backup`, works on up to the content + pool's limit of originals at once, 5 by default, instead of one at a time + (issue 182). That many workers take files from one shared list, each doing all + the work for one original, its `--verify` hashing included, so no more + originals than that are open at once. `BackupLibrary` says the limit as + `contentConcurrency`. A test holds each download at the fake Ente server to + count those in flight. + - 2026-10-14: `quak backup` no longer asks the server for changes every 3 seconds while it runs (issue 183). It opens its library with `refreshIntervalSeconds: Infinity`, which the library now takes as no diff --git a/src/backup.ts b/src/backup.ts index 184d466..b0cf52d 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -5,9 +5,9 @@ // lock. It waits for a completed refresh of the library (a failed one fails the // backup before any file is touched), then, for every file in scope, puts its // original at its save path under `downloadDirectory`, as `Photo.download()` -// does, waits for an ML data fetch, and rebuilds the derived views (per-file -// sidecars, per-collection symlink trees, per-collection JSON) from the model. -// The on-disk layout: +// does, up to the content pool's limit of them at once, waits for an ML data +// fetch, and rebuilds the derived views (per-file sidecars, per-collection +// symlink trees, per-collection JSON) from the model. The on-disk layout: // // / // YYYY/YYYY-MM/YYYY-MM-DD/ @@ -161,6 +161,9 @@ export interface BackupLibrary { destination: string, ): Promise<{ path: string; videoPath?: string }>; thumbnail(fileID: number): Promise<{ path: string }>; + // How many originals the backup works on at once: the limit of the + // library's content pool. + contentConcurrency: number; // Wait for an ML data fetch to complete, joining one already running or // starting one. Rejects with the reason when it fails; resolves at once // when the library cannot fetch ML data. @@ -607,81 +610,87 @@ const runLockedBackup = async ( // present original is hashed first, and one that does not match the hash // its metadata records is removed and fetched like a missing one. One // that cannot be read is recorded as failed and left where it is. - if (includeOriginals) { - for (const [fileID, file] of distinct) { - let stored = storedAtSavePath(downloadDirectory, file); - let mismatch = false; - if (stored !== undefined && verify) { - try { - if (file.metadata.hash === undefined) { - unchecked++; - } else if ( - (await storedHash(stored)) === file.metadata.hash - ) { - verified++; - } else { - log( - `MISMATCH original ${file.metadata.title} (${fileID}): its bytes do not match its content hash`, - ); - mismatched++; - mismatch = true; - rmSync(stored.path); - if (stored.videoPath !== undefined) { - rmSync(stored.videoPath); - } - stored = undefined; - } - } catch (err) { - log( - `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`, - ); - recordFailure( - file, - collectionName.get(file.collectionID) ?? "", - err, - ); - continue; - } - } - if (stored !== undefined) { - skipped++; - continue; - } + // `storeOriginal` does this for one original, and records its failure + // rather than throwing. + const storeOriginal = async (file: EnteFile): Promise => { + let stored = storedAtSavePath(downloadDirectory, file); + let mismatch = false; + if (stored !== undefined && verify) { try { - log(`Fetching original ${file.metadata.title} (${fileID})...`); - // A fetched original is written straight to its save path (a - // live photo beside it); only one that was already cached - // elsewhere is copied. - const placed = await placeOriginal( - downloadDirectory, - file, - (dest) => lib.original(fileID, dest), - ); - storedThisRun.add(fileID); - // A copy from the cache is not checked as a download is and can - // hold the same bad bytes, so what is put back after a - // mismatch is hashed too. A bad copy stays where it is and the - // file fails. - if ( - mismatch && - (await storedHash(placed)) !== file.metadata.hash - ) { - throw new Error( - "the original put back does not match its content hash either", + if (file.metadata.hash === undefined) { + unchecked++; + } else if ((await storedHash(stored)) === file.metadata.hash) { + verified++; + } else { + log( + `MISMATCH original ${file.metadata.title} (${file.id}): its bytes do not match its content hash`, ); + mismatched++; + mismatch = true; + rmSync(stored.path); + if (stored.videoPath !== undefined) { + rmSync(stored.videoPath); + } + stored = undefined; } - downloaded++; } catch (err) { log( - `FAILED original ${file.metadata.title}: ${errorMessage(err)}`, + `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`, ); recordFailure( file, collectionName.get(file.collectionID) ?? "", err, ); + return; } } + if (stored !== undefined) { + skipped++; + return; + } + try { + log(`Fetching original ${file.metadata.title} (${file.id})...`); + // A fetched original is written straight to its save path (a + // live photo beside it); only one that was already cached + // elsewhere is copied. + const placed = await placeOriginal( + downloadDirectory, + file, + (dest) => lib.original(file.id, dest), + ); + storedThisRun.add(file.id); + // A copy from the cache is not checked as a download is and can + // hold the same bad bytes, so what is put back after a mismatch + // is hashed too. A bad copy stays where it is and the file fails. + if (mismatch && (await storedHash(placed)) !== file.metadata.hash) { + throw new Error( + "the original put back does not match its content hash either", + ); + } + downloaded++; + } catch (err) { + log(`FAILED original ${file.metadata.title}: ${errorMessage(err)}`); + recordFailure( + file, + collectionName.get(file.collectionID) ?? "", + err, + ); + } + }; + + // Up to the content pool's limit of originals are worked on at once, + // their `verify` hashing included, so however large the account is, no + // more than that many are open. Each worker takes the next file from the + // iterator they all share until none is left. + if (includeOriginals) { + const pending = distinct.values(); + const worker = async (): Promise => { + for (const file of pending) await storeOriginal(file); + }; + await Promise.all( + Array.from({ length: lib.contentConcurrency }, worker), + ); } if (includeThumbnails) { diff --git a/src/library/index.ts b/src/library/index.ts index 080020b..d44b3b8 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -604,6 +604,7 @@ export class Library { original: (fileID, destination) => cache!.backupOriginal(fileID, destination), thumbnail: (fileID) => cache!.thumbnail(fileID), + contentConcurrency: this.pools.content.concurrency, fetchMLData: () => this.fetchMLDataNow(), mlData: (fileID) => this.mldata.forFile({ fileID }), }, diff --git a/test/cli/backup-concurrency.test.ts b/test/cli/backup-concurrency.test.ts index fe8caa4..964bff5 100644 --- a/test/cli/backup-concurrency.test.ts +++ b/test/cli/backup-concurrency.test.ts @@ -208,7 +208,8 @@ const holdingServer = (failing?: number) => { // Let the held downloads go one at a time, the first to arrive first, each // once `limit` of them wait, or all the downloads still to come when there // are fewer. Each of the eight photos is downloaded once. With a backup that -// downloads fewer than `limit` at once, this fails waiting for them. +// downloads fewer than `limit` at once, this fails waiting for them. Each wait +// allows five seconds, for a loaded machine's disk to sync the files written. const releaseOneAtATime = async ( held: { release: () => void }[], limit: number, @@ -216,7 +217,7 @@ const releaseOneAtATime = async ( for (let left = PHOTO_IDS.length; left > 0; left--) { await vi.waitFor( () => expect(held).toHaveLength(Math.min(limit, left)), - { interval: 5 }, + { interval: 5, timeout: 5000 }, ); held.shift()!.release(); } @@ -340,6 +341,7 @@ describe("lib.backup()", () => { }); await vi.waitFor(() => expect(holding.held).toHaveLength(5), { interval: 5, + timeout: 5000, }); expect(mismatches()).toHaveLength(5); diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 977a584..439dc5a 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -1490,6 +1490,7 @@ describe("backup album folders", () => { thumbnail: async () => { throw new Error("no thumbnails in this stand-in"); }, + contentConcurrency: 5, fetchMLData: async () => {}, mlData: async () => undefined, whoami: () => ({ email: "backup@example.com", userID: USER_ID }),