quak backup works on up to the content pool's limit of originals at once (closes #182) #192

Merged
clawbot merged 10 commits from issue-182-parallel-backup into next 2026-10-14 09:07:00 +02:00
Showing only changes of commit 9f6fa1990f - Show all commits
+63
View File
@@ -396,6 +396,69 @@ describe("lib.backup()", () => {
}
});
// A throw from `onProgress` fails the backup: `lib.backup()` rejects with
// it. It rejects only once it has finished with every original it was
// working on, so nothing is written into the backup after it has ended
// and removed its lock. Here, with a content pool of eight, all eight
// downloads wait at once. Photo 103's is let go first and answered 404,
// and the line reporting that failure throws while the other seven wait.
it("when onProgress throws, rejects only once the originals it was working on are in place", async () => {
const dir = mkdtempSync(join(root, "backup-"));
const holding = holdingServer(103);
const lib = await openLibrary(
dir,
holding.fetch,
new RequestPools({ contentConcurrency: 8 }),
);
const lines: string[] = [];
try {
const backup = lib.backup({
onProgress: (line) => {
lines.push(line);
if (line.startsWith("FAILED")) {
throw new Error("the progress output is closed");
}
},
});
// The photos whose originals are in place, and whether the lock
// is there, at the moment `lib.backup()` rejects.
let storedWhenRejected: number[] = [];
let lockedWhenRejected = true;
const rejection = backup.catch((err: unknown) => {
storedWhenRejected = PHOTO_IDS.filter((id) =>
existsSync(savedAt(dir, id)),
);
lockedWhenRejected = existsSync(join(dir, "backup.lock"));
return err;
});
await vi.waitFor(() => expect(holding.held).toHaveLength(8), {
interval: 5,
timeout: 5000,
});
const at103 = holding.held.findIndex((h) => h.id === 103);
holding.held.splice(at103, 1)[0]!.release();
await vi.waitFor(
() =>
expect(
lines.some((line) => line.startsWith("FAILED")),
).toBe(true),
{ interval: 5, timeout: 5000 },
);
for (const download of holding.held) download.release();
expect(await rejection).toEqual(
new Error("the progress output is closed"),
);
expect(storedWhenRejected).toEqual(
PHOTO_IDS.filter((id) => id !== 103),
);
expect(lockedWhenRejected).toBe(false);
} finally {
await lib.close();
}
});
// Working on several originals at once changes only how long a backup
// takes. The same two runs, with a content pool of one, which backs up
// one original at a time, and with the default pools, end the same way: