README.md

@@ -1096,6 +1096,12 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange`
 Each pool orders on-demand work ahead of background/precache work and dedups
 in-flight fetches by key, and an idle pool never lends its slots to a busy one.
 
+`lib.backup()`, and so `quak backup`, works on up to the content pool's limit of
+originals at once, 5 unless the library was opened with other `pools`. The limit
+covers everything it does for one original, hashing it for `verify` included, so
+however large the account is, no more originals than that are open at once. With
+`includeThumbnails`, it fetches the thumbnails one at a time.
+
 ### On-disk cache layout
 
 Under `cacheDirectory`:

TODO.md

@@ -17,18 +17,23 @@ pre-1.0
 
 # Next Step
 
-`quak backup` downloading one original at a time,
+None: no implementation work is open. The check of quak against a live Ente
-https://git.eeqj.de/sneak/quak/issues/182, is fixed in
+account, https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak
-https://git.eeqj.de/sneak/quak/pulls/192, which waits on review. No other
+has, so it waits on him.
-implementation work is open. The check of quak against a live Ente account,
-https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak has, so
-it waits on him.
 
 Tagging and releases are decided by sneak alone, and happen only when he
 declares one.
 
 # Completed Steps
 
+- 2026-10-14: `lib.backup()`, and so `quak backup`, works on up to the content
+  pool's limit of originals at once, 5 by default, instead of one at a time
+  (issue 182). That many workers take files from one shared list, each doing all
+  the work for one original, its `--verify` hashing included, so no more
+  originals than that are open at once. `BackupLibrary` says the limit as
+  `contentConcurrency`. A test holds each download at the fake Ente server to
+  count those in flight.
+
 - 2026-10-14: Next Step said the cache design, issue 36, waits on sneak's
   review, though that issue closed on 2026-10-03 (issue 194). It now names the
   issues open on the tracker.

src/backup.ts

@@ -6,9 +6,9 @@
 // file. It waits for a completed refresh of the library (a failed one fails the
 // backup before any file is touched), then, for every file in scope, puts its
 // original at its save path under `downloadDirectory`, as `Photo.download()`
-// does, waits for an ML data fetch, and rebuilds the derived views (per-file
+// does, up to the content pool's limit of them at once, waits for an ML data
-// sidecars, per-collection symlink trees, per-collection JSON) from the model.
+// fetch, and rebuilds the derived views (per-file sidecars, per-collection
-// The on-disk layout:
+// symlink trees, per-collection JSON) from the model. The on-disk layout:
 //
 //     <downloadDirectory>/
 //         YYYY/YYYY-MM/YYYY-MM-DD/
@@ -181,6 +181,9 @@ export interface BackupLibrary {
         destination: string,
     ): Promise<{ path: string; videoPath?: string }>;
     thumbnail(fileID: number): Promise<{ path: string }>;
+    // How many originals the backup works on at once: the limit of the
+    // library's content pool.
+    contentConcurrency: number;
     // Wait for an ML data fetch to complete, joining one already running or
     // starting one. Rejects with the reason when it fails; resolves at once
     // when the library cannot fetch ML data.
@@ -654,75 +657,32 @@ const runLockedBackup = async (
     // present original is hashed first, and one that does not match the hash
     // its metadata records is removed and fetched like a missing one. One
     // that cannot be read is recorded as failed and left where it is.
-    if (includeOriginals) {
+    // `storeOriginal` does this for one original, and records its failure
-        for (const [fileID, file] of distinct) {
+    // rather than throwing.
-            if (lock.lost) throw lock.lost;
+    const storeOriginal = async (file: EnteFile): Promise<void> => {
-            let stored = storedAtSavePath(downloadDirectory, file);
+        let stored = storedAtSavePath(downloadDirectory, file);
-            let mismatch = false;
+        let mismatch = false;
-            if (stored !== undefined && verify) {
+        if (stored !== undefined && verify) {
-                try {
-                    if (file.metadata.hash === undefined) {
-                        unchecked++;
-                    } else if (
-                        (await storedHash(stored)) === file.metadata.hash
-                    ) {
-                        verified++;
-                    } else {
-                        log(
-                            `MISMATCH original ${file.metadata.title} (${fileID}): its bytes do not match its content hash`,
-                        );
-                        mismatched++;
-                        mismatch = true;
-                        rmSync(stored.path);
-                        if (stored.videoPath !== undefined) {
-                            rmSync(stored.videoPath);
-                        }
-                        stored = undefined;
-                    }
-                } catch (err) {
-                    log(
-                        `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
-                    );
-                    recordFailure(
-                        file.id,
-                        file.metadata.title,
-                        collectionName.get(file.collectionID) ?? "",
-                        err,
-                    );
-                    continue;
-                }
-            }
-            if (stored !== undefined) {
-                skipped++;
-                continue;
-            }
             try {
-                log(`Fetching original ${file.metadata.title} (${fileID})...`);
+                if (file.metadata.hash === undefined) {
-                // A fetched original is written straight to its save path (a
+                    unchecked++;
-                // live photo beside it); only one that was already cached
+                } else if ((await storedHash(stored)) === file.metadata.hash) {
-                // elsewhere is copied.
+                    verified++;
-                const placed = await placeOriginal(
+                } else {
-                    downloadDirectory,
+                    log(
-                    file,
+                        `MISMATCH original ${file.metadata.title} (${file.id}): its bytes do not match its content hash`,
-                    (dest) => lib.original(fileID, dest),
-                );
-                storedThisRun.add(fileID);
-                // A copy from the cache is not checked as a download is and can
-                // hold the same bad bytes, so what is put back after a
-                // mismatch is hashed too. A bad copy stays where it is and the
-                // file fails.
-                if (
-                    mismatch &&
-                    (await storedHash(placed)) !== file.metadata.hash
-                ) {
-                    throw new Error(
-                        "the original put back does not match its content hash either",
                     );
+                    mismatched++;
+                    mismatch = true;
+                    rmSync(stored.path);
+                    if (stored.videoPath !== undefined) {
+                        rmSync(stored.videoPath);
+                    }
+                    stored = undefined;
                 }
-                downloaded++;
             } catch (err) {
                 log(
-                    `FAILED original ${file.metadata.title}: ${errorMessage(err)}`,
+                    `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
                 );
                 recordFailure(
                     file.id,
@@ -730,8 +690,56 @@ const runLockedBackup = async (
                     collectionName.get(file.collectionID) ?? "",
                     err,
                 );
+                return;
             }
         }
+        if (stored !== undefined) {
+            skipped++;
+            return;
+        }
+        try {
+            log(`Fetching original ${file.metadata.title} (${file.id})...`);
+            // A fetched original is written straight to its save path (a
+            // live photo beside it); only one that was already cached
+            // elsewhere is copied.
+            const placed = await placeOriginal(
+                downloadDirectory,
+                file,
+                (dest) => lib.original(file.id, dest),
+            );
+            storedThisRun.add(file.id);
+            // A copy from the cache is not checked as a download is and can
+            // hold the same bad bytes, so what is put back after a mismatch
+            // is hashed too. A bad copy stays where it is and the file fails.
+            if (mismatch && (await storedHash(placed)) !== file.metadata.hash) {
+                throw new Error(
+                    "the original put back does not match its content hash either",
+                );
+            }
+            downloaded++;
+        } catch (err) {
+            log(`FAILED original ${file.metadata.title}: ${errorMessage(err)}`);
+            recordFailure(
+                file.id,
+                file.metadata.title,
+                collectionName.get(file.collectionID) ?? "",
+                err,
+            );
+        }
+    };
+
+    // Up to the content pool's limit of originals are worked on at once,
+    // their `verify` hashing included, so however large the account is, no
+    // more than that many are open. Each worker takes the next file from the
+    // iterator they all share until none is left.
+    if (includeOriginals) {
+        const pending = distinct.values();
+        const worker = async (): Promise<void> => {
+            for (const file of pending) await storeOriginal(file);
+        };
+        await Promise.all(
+            Array.from({ length: lib.contentConcurrency }, worker),
+        );
     }
 
     if (includeThumbnails) {

src/library/index.ts

@@ -607,6 +607,7 @@ export class Library {
                 original: (fileID, destination) =>
                     cache!.backupOriginal(fileID, destination),
                 thumbnail: (fileID) => cache!.thumbnail(fileID),
+                contentConcurrency: this.pools.content.concurrency,
                 fetchMLData: () => this.fetchMLDataNow(),
                 mlData: (fileID) => this.mldata.forFile({ fileID }),
             },

test/cli/backup-concurrency.test.ts

@@ -208,7 +208,8 @@ const holdingServer = (failing?: number) => {
 // Let the held downloads go one at a time, the first to arrive first, each
 // once `limit` of them wait, or all the downloads still to come when there
 // are fewer. Each of the eight photos is downloaded once. With a backup that
-// downloads fewer than `limit` at once, this fails waiting for them.
+// downloads fewer than `limit` at once, this fails waiting for them. Each wait
+// allows five seconds, for a loaded machine's disk to sync the files written.
 const releaseOneAtATime = async (
     held: { release: () => void }[],
     limit: number,
@@ -216,7 +217,7 @@ const releaseOneAtATime = async (
     for (let left = PHOTO_IDS.length; left > 0; left--) {
         await vi.waitFor(
             () => expect(held).toHaveLength(Math.min(limit, left)),
-            { interval: 5 },
+            { interval: 5, timeout: 5000 },
         );
         held.shift()!.release();
     }
@@ -340,6 +341,7 @@ describe("lib.backup()", () => {
             });
             await vi.waitFor(() => expect(holding.held).toHaveLength(5), {
                 interval: 5,
+                timeout: 5000,
             });
             expect(mismatches()).toHaveLength(5);
 

test/cli/backup-end-to-end.test.ts

@@ -704,9 +704,9 @@ describe("a backup whose lock is lost", () => {
         }
     };
 
-    // beach.jpg, whose download was under way, is stored; no other file is
+    // The backup downloads up to five originals at once, so all three are
-    // started. `quak backup` prints why as one line, with no stack trace, and
+    // under way when the lock is lost, and beach.jpg is stored. `quak backup`
-    // exits 1.
+    // prints why as one line, with no stack trace, and exits 1.
     it("stops quak backup before its next file when backup.lock is removed, and exits 1", async () => {
         const result = backup(dir, {}, fetch);
         await heldRequested;
@@ -720,9 +720,12 @@ describe("a backup whose lock is lost", () => {
             "Starting backup...\n" +
                 "Refreshing library...\n" +
                 "Fetching original beach.jpg (100)...\n" +
+                "Fetching original IMG_0002.HEIC (300)...\n" +
+                "Fetching original toast.mp4 (200)...\n" +
+                "Fetching ML data...\n" +
                 `quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'\n`,
         );
-        expect(downloads).toEqual([BEACH.id]);
+        expect(downloads.sort()).toEqual([100, 200, 300]);
         expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual(
             Buffer.from(JPEG_WITH_EXIF),
         );
@@ -798,7 +801,7 @@ describe("a backup whose lock is lost", () => {
             await expect(result).rejects.toThrow(
                 `lost the lock on ${dir}: Unable to update lock within the stale threshold`,
             );
-            expect(downloads).toEqual([BEACH.id]);
+            expect(downloads.sort()).toEqual([100, 200, 300]);
             expect(existsSync(lockPath)).toBe(true);
         } finally {
             await lib.close();
@@ -823,7 +826,7 @@ describe("a backup whose lock is lost", () => {
             const lost = `lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'`;
             await expect(result).rejects.toThrow(lost);
             expect(lock.lost?.message).toBe(lost);
-            expect(downloads).toEqual([BEACH.id]);
+            expect(downloads.sort()).toEqual([100, 200, 300]);
         } finally {
             await lib.close();
             await lock.release();

test/cli/backup.test.ts

@@ -1491,6 +1491,7 @@ describe("backup album folders", () => {
         thumbnail: async () => {
             throw new Error("no thumbnails in this stand-in");
         },
+        contentConcurrency: 5,
         fetchMLData: async () => {},
         mlData: async () => undefined,
         whoami: () => ({ email: "backup@example.com", userID: USER_ID }),