quak backup works on up to the content pool's limit of originals at once (closes #182)

lib.backup() ran its originals one at a time, so the content pool's five
slots, or the contentConcurrency of the pools a library was opened with,
went unused. The work for one original, its verify hashing included, is
now storeOriginal, and that many workers take files from one iterator
over the backup's files, so no more originals than the limit are open at
once. BackupLibrary gains contentConcurrency, which the library sets from
its content pool. The README's "Request pools" says so. The tests of a
backup whose lock is lost now expect all three originals downloaded,
since all three are under way at once.

Model: opus-5-5

test/cli/backup-concurrency.test.ts

@@ -208,7 +208,8 @@ const holdingServer = (failing?: number) => {
 // Let the held downloads go one at a time, the first to arrive first, each
 // once `limit` of them wait, or all the downloads still to come when there
 // are fewer. Each of the eight photos is downloaded once. With a backup that
-// downloads fewer than `limit` at once, this fails waiting for them.
+// downloads fewer than `limit` at once, this fails waiting for them. Each wait
+// allows five seconds, for a loaded machine's disk to sync the files written.
 const releaseOneAtATime = async (
     held: { release: () => void }[],
     limit: number,
@@ -216,7 +217,7 @@ const releaseOneAtATime = async (
     for (let left = PHOTO_IDS.length; left > 0; left--) {
         await vi.waitFor(
             () => expect(held).toHaveLength(Math.min(limit, left)),
-            { interval: 5 },
+            { interval: 5, timeout: 5000 },
         );
         held.shift()!.release();
     }
@@ -340,6 +341,7 @@ describe("lib.backup()", () => {
             });
             await vi.waitFor(() => expect(holding.held).toHaveLength(5), {
                 interval: 5,
+                timeout: 5000,
             });
             expect(mismatches()).toHaveLength(5);
 

test/cli/backup-end-to-end.test.ts

@@ -704,9 +704,9 @@ describe("a backup whose lock is lost", () => {
         }
     };
 
-    // beach.jpg, whose download was under way, is stored; no other file is
-    // started. `quak backup` prints why as one line, with no stack trace, and
-    // exits 1.
+    // The backup downloads up to five originals at once, so all three are
+    // under way when the lock is lost, and beach.jpg is stored. `quak backup`
+    // prints why as one line, with no stack trace, and exits 1.
     it("stops quak backup before its next file when backup.lock is removed, and exits 1", async () => {
         const result = backup(dir, {}, fetch);
         await heldRequested;
@@ -720,9 +720,12 @@ describe("a backup whose lock is lost", () => {
             "Starting backup...\n" +
                 "Refreshing library...\n" +
                 "Fetching original beach.jpg (100)...\n" +
+                "Fetching original IMG_0002.HEIC (300)...\n" +
+                "Fetching original toast.mp4 (200)...\n" +
+                "Fetching ML data...\n" +
                 `quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'\n`,
         );
-        expect(downloads).toEqual([BEACH.id]);
+        expect(downloads.sort()).toEqual([100, 200, 300]);
         expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual(
             Buffer.from(JPEG_WITH_EXIF),
         );
@@ -798,7 +801,7 @@ describe("a backup whose lock is lost", () => {
             await expect(result).rejects.toThrow(
                 `lost the lock on ${dir}: Unable to update lock within the stale threshold`,
             );
-            expect(downloads).toEqual([BEACH.id]);
+            expect(downloads.sort()).toEqual([100, 200, 300]);
             expect(existsSync(lockPath)).toBe(true);
         } finally {
             await lib.close();
@@ -823,7 +826,7 @@ describe("a backup whose lock is lost", () => {
             const lost = `lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'`;
             await expect(result).rejects.toThrow(lost);
             expect(lock.lost?.message).toBe(lost);
-            expect(downloads).toEqual([BEACH.id]);
+            expect(downloads.sort()).toEqual([100, 200, 300]);
         } finally {
             await lib.close();
             await lock.release();

test/cli/backup.test.ts

@@ -1491,6 +1491,7 @@ describe("backup album folders", () => {
         thumbnail: async () => {
             throw new Error("no thumbnails in this stand-in");
         },
+        contentConcurrency: 5,
         fetchMLData: async () => {},
         mlData: async () => undefined,
         whoami: () => ({ email: "backup@example.com", userID: USER_ID }),