diff --git a/README.md b/README.md index bc74537..12fd670 100644 --- a/README.md +++ b/README.md @@ -779,9 +779,9 @@ A running backup loses its lock when someone removes `backup.lock`, or when another backup takes it over because the running one left it untouched for 10 seconds, as a stopped or stalled process does. The backup finds this out the next time it goes to touch the lock, which it does every 5 seconds, and then -stops before it starts another file. The original it is downloading at that -moment is still stored. It leaves `failures.json` and `backup.lock` as they are, -since another backup may be using them. `quak backup` prints +stops before it starts another file. The originals it is downloading at that +moment are still stored. It leaves `failures.json` and `backup.lock` as they +are, since another backup may be using them. `quak backup` prints `quak: lost the lock on : ` as one line and exits 1, and `lib.backup()` rejects with `lost the lock on : `. @@ -1096,6 +1096,11 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange` Each pool orders on-demand work ahead of background/precache work and dedups in-flight fetches by key, and an idle pool never lends its slots to a busy one. +`lib.backup()`, and so `quak backup`, works on up to the content pool's limit of +originals at once, 5 unless the library was opened with other `pools`. The limit +covers everything it does for one original, hashing it for `verify` included, so +however large the account is, no more originals than that are open at once. + ### On-disk cache layout Under `cacheDirectory`: diff --git a/TODO.md b/TODO.md index c231724..1aa7f9b 100644 --- a/TODO.md +++ b/TODO.md @@ -17,18 +17,26 @@ pre-1.0 # Next Step -`quak backup` downloading one original at a time, -https://git.eeqj.de/sneak/quak/issues/182, is fixed in -https://git.eeqj.de/sneak/quak/pulls/192, which waits on review. No other -implementation work is open. The check of quak against a live Ente account, -https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak has, so -it waits on him. +None: no implementation work is open. The check of quak against a live Ente +account, https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak +has, so it waits on him. Tagging and releases are decided by sneak alone, and happen only when he declares one. # Completed Steps +- 2026-10-14: `lib.backup()`, and so `quak backup`, works on up to the content + pool's limit of originals at once, 5 by default, instead of one at a time + (issue 182). That many workers take files from one shared list, each doing all + the work for one original, its `--verify` hashing included, so no more + originals than that are open at once. `BackupLibrary` says the limit as + `contentConcurrency`. A test holds each download at the fake Ente server to + count those in flight. Once a worker has thrown, as one does when `onProgress` + throws, or the lock is lost, no worker takes another file; the backup fails + once the originals under way are in place, and a lost lock stops it before it + fetches ML data. + - 2026-10-14: Next Step said the cache design, issue 36, waits on sneak's review, though that issue closed on 2026-10-03 (issue 194). It now names the issues open on the tracker. diff --git a/src/backup.ts b/src/backup.ts index bd76d2f..488ff30 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -6,9 +6,9 @@ // file. It waits for a completed refresh of the library (a failed one fails the // backup before any file is touched), then, for every file in scope, puts its // original at its save path under `downloadDirectory`, as `Photo.download()` -// does, waits for an ML data fetch, and rebuilds the derived views (per-file -// sidecars, per-collection symlink trees, per-collection JSON) from the model. -// The on-disk layout: +// does, up to the content pool's limit of them at once, waits for an ML data +// fetch, and rebuilds the derived views (per-file sidecars, per-collection +// symlink trees, per-collection JSON) from the model. The on-disk layout: // // / // YYYY/YYYY-MM/YYYY-MM-DD/ @@ -181,6 +181,9 @@ export interface BackupLibrary { destination: string, ): Promise<{ path: string; videoPath?: string }>; thumbnail(fileID: number): Promise<{ path: string }>; + // How many originals the backup works on at once: the limit of the + // library's content pool. + contentConcurrency: number; // Wait for an ML data fetch to complete, joining one already running or // starting one. Rejects with the reason when it fails; resolves at once // when the library cannot fetch ML data. @@ -654,75 +657,31 @@ const runLockedBackup = async ( // present original is hashed first, and one that does not match the hash // its metadata records is removed and fetched like a missing one. One // that cannot be read is recorded as failed and left where it is. - if (includeOriginals) { - for (const [fileID, file] of distinct) { - if (lock.lost) throw lock.lost; - let stored = storedAtSavePath(downloadDirectory, file); - let mismatch = false; - if (stored !== undefined && verify) { - try { - if (file.metadata.hash === undefined) { - unchecked++; - } else if ( - (await storedHash(stored)) === file.metadata.hash - ) { - verified++; - } else { - log( - `MISMATCH original ${file.metadata.title} (${fileID}): its bytes do not match its content hash`, - ); - mismatched++; - mismatch = true; - rmSync(stored.path); - if (stored.videoPath !== undefined) { - rmSync(stored.videoPath); - } - stored = undefined; - } - } catch (err) { - log( - `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`, - ); - recordFailure( - file.id, - file.metadata.title, - collectionName.get(file.collectionID) ?? "", - err, - ); - continue; - } - } - if (stored !== undefined) { - skipped++; - continue; - } + // `storeOriginal` does this for one original, and records its failure. + const storeOriginal = async (file: EnteFile): Promise => { + let stored = storedAtSavePath(downloadDirectory, file); + let mismatch = false; + if (stored !== undefined && verify) { try { - log(`Fetching original ${file.metadata.title} (${fileID})...`); - // A fetched original is written straight to its save path (a - // live photo beside it); only one that was already cached - // elsewhere is copied. - const placed = await placeOriginal( - downloadDirectory, - file, - (dest) => lib.original(fileID, dest), - ); - storedThisRun.add(fileID); - // A copy from the cache is not checked as a download is and can - // hold the same bad bytes, so what is put back after a - // mismatch is hashed too. A bad copy stays where it is and the - // file fails. - if ( - mismatch && - (await storedHash(placed)) !== file.metadata.hash - ) { - throw new Error( - "the original put back does not match its content hash either", + if (file.metadata.hash === undefined) { + unchecked++; + } else if ((await storedHash(stored)) === file.metadata.hash) { + verified++; + } else { + log( + `MISMATCH original ${file.metadata.title} (${file.id}): its bytes do not match its content hash`, ); + mismatched++; + mismatch = true; + rmSync(stored.path); + if (stored.videoPath !== undefined) { + rmSync(stored.videoPath); + } + stored = undefined; } - downloaded++; } catch (err) { log( - `FAILED original ${file.metadata.title}: ${errorMessage(err)}`, + `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`, ); recordFailure( file.id, @@ -730,8 +689,74 @@ const runLockedBackup = async ( collectionName.get(file.collectionID) ?? "", err, ); + return; } } + if (stored !== undefined) { + skipped++; + return; + } + try { + log(`Fetching original ${file.metadata.title} (${file.id})...`); + // A fetched original is written straight to its save path (a + // live photo beside it); only one that was already cached + // elsewhere is copied. + const placed = await placeOriginal( + downloadDirectory, + file, + (dest) => lib.original(file.id, dest), + ); + storedThisRun.add(file.id); + // A copy from the cache is not checked as a download is and can + // hold the same bad bytes, so what is put back after a mismatch + // is hashed too. A bad copy stays where it is and the file fails. + if (mismatch && (await storedHash(placed)) !== file.metadata.hash) { + throw new Error( + "the original put back does not match its content hash either", + ); + } + downloaded++; + } catch (err) { + log(`FAILED original ${file.metadata.title}: ${errorMessage(err)}`); + recordFailure( + file.id, + file.metadata.title, + collectionName.get(file.collectionID) ?? "", + err, + ); + } + }; + + // Up to the content pool's limit of originals are worked on at once, + // their `verify` hashing included, so however large the account is, no + // more than that many are open. Each worker takes the next file from the + // iterator they all share until none is left. Once a worker has thrown, + // as one does when `onProgress` throws, or the lock is lost, no worker + // takes another file. The backup fails only once every worker has + // stopped, so none writes into the backup after it has ended. It checks + // the lock then too, so it does not go on to the thumbnails or the ML + // data fetch without it. + if (includeOriginals) { + const pending = distinct.values(); + let workerThrew = false; + const worker = async (): Promise => { + for (const file of pending) { + if (workerThrew || lock.lost) return; + try { + await storeOriginal(file); + } catch (err) { + workerThrew = true; + throw err; + } + } + }; + const workers = await Promise.allSettled( + Array.from({ length: lib.contentConcurrency }, worker), + ); + for (const w of workers) { + if (w.status === "rejected") throw w.reason; + } + if (lock.lost) throw lock.lost; } if (includeThumbnails) { @@ -960,6 +985,14 @@ export const runBackup = async ( "open the library with one)", ); } + // As `BoundedPool` does. A limit below 1, or one that is not a number, + // would back up no original and end with no error. + const { contentConcurrency } = lib; + if (!Number.isInteger(contentConcurrency) || contentConcurrency < 1) { + throw new RangeError( + `contentConcurrency must be a positive integer, got ${contentConcurrency}`, + ); + } if (opts.lock) { return runLockedBackup(lib, opts, downloadDirectory, opts.lock); } diff --git a/src/library/index.ts b/src/library/index.ts index eb36410..95d8e1d 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -607,6 +607,7 @@ export class Library { original: (fileID, destination) => cache!.backupOriginal(fileID, destination), thumbnail: (fileID) => cache!.thumbnail(fileID), + contentConcurrency: this.pools.content.concurrency, fetchMLData: () => this.fetchMLDataNow(), mlData: (fileID) => this.mldata.forFile({ fileID }), }, diff --git a/test/cli/backup-concurrency.test.ts b/test/cli/backup-concurrency.test.ts new file mode 100644 index 0000000..f3c476c --- /dev/null +++ b/test/cli/backup-concurrency.test.ts @@ -0,0 +1,608 @@ +/** + * How many originals `lib.backup()` works on at once. + * + * The backup puts up to the content pool's limit of originals at their save + * paths at the same time: 5 for a library opened with the default + * `RequestPools`, as `quak backup` opens it, or the `contentConcurrency` of the + * pools a library was opened with. Everything the backup does for one original + * counts against that limit, hashing it for `verify` included, so however large + * the account is, the backup has no more than that many originals open at once. + * + * As in `backup-end-to-end.test.ts`, the real `Client` and `Library` run + * against `buildMetaFetch` from `test/ente-server.ts`, a fake of the Ente HTTP + * API serving data this file encrypts with libsodium. Here a test can also hold + * each original's download until it lets it go, to see how many the backup has + * in flight at once. + * + * The account, user 42, has one album of its own, Holiday, of eight photos, + * all taken at noon on 1 March 2024, each with an original of its own. + */ + +import { + existsSync, + lstatSync, + mkdtempSync, + readdirSync, + readFileSync, + readlinkSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import sodium from "libsodium-wrappers-sumo"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +import type { BackupOptions } from "../../src/backup.js"; +import { Client, type ClientSnapshot } from "../../src/client.js"; +import { init, toBase64 } from "../../src/crypto/index.js"; +import { Library } from "../../src/library/index.js"; +import { RequestPools } from "../../src/library/pools.js"; +import { + buildAccount, + buildMetaFetch, + encryptSecretbox, + encryptStreamBlob, +} from "../ente-server.js"; +import { blake2b } from "../live-photo.js"; + +const EMAIL = "backup@example.com"; +const PASSWORD = "backup-password"; +// The fake server logs every account in as user 42. +const USER_ID = 42; + +// Noon on 1 March 2024 in the time zone of the machine running the test, in +// microseconds, as Ente records times. +const TAKEN = new Date(2024, 2, 1, 12).getTime() * 1000; + +// The eight photos' IDs. Photo `id` is titled `photo-.jpg`, and its +// original is a few bytes naming it. +const PHOTO_IDS = [101, 102, 103, 104, 105, 106, 107, 108]; +const originalOf = (id: number): Uint8Array => + new TextEncoder().encode(`the photo ${id}`); + +// Where a backup to `dir` saves photo `id`'s original. +const savedAt = (dir: string, id: number): string => + join(dir, "2024/2024-03/2024-03-01", `2024-03-01.${id}.jpg`); + +// The fake server, the session it logs the account in with, and the +// directory every backup below is made in. +let server: typeof globalThis.fetch; +let session: ClientSnapshot; +let root: string; + +// The fake server, holding the account encrypted as Ente's clients upload it. +const buildServer = async () => { + const account = await buildAccount(PASSWORD); + + // The album as `/collections/v2` lists it: its key encrypted with the + // account's master key, its name with its own key. + const albumKey = sodium.crypto_secretbox_keygen(); + const encryptedAlbumKey = encryptSecretbox(albumKey, account.masterKey); + const encryptedName = encryptSecretbox( + new TextEncoder().encode("Holiday"), + albumKey, + ); + const album = { + id: 10, + owner: { id: USER_ID }, + encryptedKey: toBase64(encryptedAlbumKey.ciphertext), + keyDecryptionNonce: toBase64(encryptedAlbumKey.nonce), + encryptedName: toBase64(encryptedName.ciphertext), + nameDecryptionNonce: toBase64(encryptedName.nonce), + type: "album", + updationTime: TAKEN, + }; + + // Each photo as the album's `/collections/v2/diff` lists it: its key + // encrypted with the album's key, and its metadata, holding the hash of + // its original, encrypted with its own key. Its original, encrypted with + // its own key too, is what the files CDN serves for it. + const ciphertexts: Record = {}; + const photos = PHOTO_IDS.map((id) => { + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const original = encryptStreamBlob(originalOf(id), key); + ciphertexts[id] = original.ciphertext; + const encryptedKey = encryptSecretbox(key, albumKey); + const metadata = encryptStreamBlob( + new TextEncoder().encode( + JSON.stringify({ + title: `photo-${id}.jpg`, + fileType: 0, + hash: blake2b(originalOf(id)), + creationTime: TAKEN, + modificationTime: TAKEN, + }), + ), + key, + ); + return { + id, + collectionID: 10, + ownerID: USER_ID, + encryptedKey: toBase64(encryptedKey.ciphertext), + keyDecryptionNonce: toBase64(encryptedKey.nonce), + metadata: { + encryptedData: toBase64(metadata.ciphertext), + decryptionHeader: toBase64(metadata.header), + }, + file: { decryptionHeader: toBase64(original.header) }, + thumbnail: { + decryptionHeader: toBase64(sodium.randombytes_buf(24)), + }, + updationTime: TAKEN, + }; + }); + + return buildMetaFetch({ + verifier: account.verifier, + srpAttributes: account.srpAttributes, + keyAttributes: account.keyAttributes, + encryptedToken: account.encryptedToken, + collections: [album], + filesByCollection: { 10: photos }, + encryptedMLData: {}, + fileCiphertexts: ciphertexts, + }); +}; + +beforeAll(async () => { + await init(); + await sodium.ready; + root = mkdtempSync(join(tmpdir(), "quak-backup-concurrency-")); + server = await buildServer(); + const client = await Client.login({ + email: EMAIL, + password: PASSWORD, + apiOptions: { fetch: server }, + }); + session = client.toJSON(); +}); + +afterAll(() => { + rmSync(root, { recursive: true, force: true }); +}); + +// The ID of the photo whose original a request downloads, or undefined for any +// other request. quak downloads an original from +// `https://files.ente.io/?fileID=`. +const downloadOf = (input: RequestInfo | URL): number | undefined => { + const url = new URL(String(input)); + return url.hostname === "files.ente.io" + ? Number(url.searchParams.get("fileID")) + : undefined; +}; + +// The fake server, except that it answers 404 to every download of photo +// `failing`'s original. +const failingFor = + (failing: number): typeof globalThis.fetch => + async (input, init) => + downloadOf(input) === failing + ? new Response("not found", { status: 404 }) + : server(input, init); + +// The fake server, holding each download of an original until the test lets +// it go. `held` lists the downloads waiting, the first to arrive first; +// `most()` is the largest number that ever waited at once. A held download of +// photo `failing`'s original is answered 404 once it is let go. +const holdingServer = (failing?: number) => { + const held: { id: number; release: () => void }[] = []; + let most = 0; + const fetch: typeof globalThis.fetch = async (input, init) => { + const id = downloadOf(input); + if (id !== undefined) { + await new Promise((release) => { + held.push({ id, release }); + most = Math.max(most, held.length); + }); + if (id === failing) { + return new Response("not found", { status: 404 }); + } + } + return server(input, init); + }; + return { fetch, held, most: () => most }; +}; + +// Let the held downloads go one at a time, the first to arrive first, each +// once `limit` of them wait, or all the downloads still to come when there +// are fewer. Each of the eight photos is downloaded once. With a backup that +// downloads fewer than `limit` at once, this fails waiting for them. Each wait +// allows five seconds, for a loaded machine's disk to sync the files written. +const releaseOneAtATime = async ( + held: { release: () => void }[], + limit: number, +): Promise => { + for (let left = PHOTO_IDS.length; left > 0; left--) { + await vi.waitFor( + () => expect(held).toHaveLength(Math.min(limit, left)), + { interval: 5, timeout: 5000 }, + ); + held.shift()!.release(); + } +}; + +// A library of the account opened as `quak backup` opens one, with no +// precache and no background refresh, sending its requests through `fetch`. +// It backs up to `dir` and keeps its cache beside it. Its request pools are +// `pools`, or by default the library's own. +const openLibrary = ( + dir: string, + fetch: typeof globalThis.fetch, + pools?: RequestPools, +): Promise => + Library.open({ + client: Client.fromJSON(session, { fetch }), + downloadDirectory: dir, + cacheDirectory: `${dir}-cache`, + refreshIntervalSeconds: Infinity, + precacheThumbnails: false, + precacheOriginals: false, + pools, + }); + +// Every entry under `dir`, relative to it, sorted, a symlink as +// `name -> target`. +const listTree = (dir: string): string[] => + readdirSync(dir, { recursive: true }) + .sort() + .map((name) => { + const path = join(dir, name); + return lstatSync(path).isSymbolicLink() + ? `${name} -> ${readlinkSync(path)}` + : name; + }); + +const expectStored = (dir: string, ids: number[]): void => { + for (const id of ids) { + expect(readFileSync(savedAt(dir, id))).toEqual( + Buffer.from(originalOf(id)), + ); + } +}; + +describe("lib.backup()", () => { + // A library opened with the default pools, as `quak backup` opens one, + // downloads up to five originals at once. As soon as one finishes, the + // backup starts on the next photo, so five stay in flight until fewer + // than five are left. + it("downloads five originals at once, and never more, with the default pools", async () => { + const dir = mkdtempSync(join(root, "backup-")); + const holding = holdingServer(); + const lib = await openLibrary(dir, holding.fetch); + try { + const backup = lib.backup(); + await releaseOneAtATime(holding.held, 5); + const result = await backup; + + expect(holding.most()).toBe(5); + expect(result).toMatchObject({ + totalFiles: 8, + downloaded: 8, + failed: 0, + }); + expectStored(dir, PHOTO_IDS); + } finally { + await lib.close(); + } + }); + + // The limit is the content pool's, so a library opened with pools of + // another `contentConcurrency` backs up that many at once. + it("downloads as many originals at once as the content pool of the pools it was opened with allows", async () => { + const dir = mkdtempSync(join(root, "backup-")); + const holding = holdingServer(); + const lib = await openLibrary( + dir, + holding.fetch, + new RequestPools({ contentConcurrency: 2 }), + ); + try { + const backup = lib.backup(); + await releaseOneAtATime(holding.held, 2); + const result = await backup; + + expect(holding.most()).toBe(2); + expect(result).toMatchObject({ downloaded: 8, failed: 0 }); + expectStored(dir, PHOTO_IDS); + } finally { + await lib.close(); + } + }); + + // With `verify`, hashing an original already stored is part of the work + // on it, and counts against the limit too: a large backup never has more + // than five originals open for hashing. Here every stored original has + // changed on disk, so each is hashed, found not to match, and downloaded + // again. While five downloads wait, the backup has hashed only those five + // photos' originals; it opens the next one once a download finishes. + it("with verify, hashes no more originals ahead of their downloads than the limit", async () => { + const dir = mkdtempSync(join(root, "backup-")); + const first = await openLibrary(dir, server); + try { + await first.backup(); + } finally { + await first.close(); + } + for (const id of PHOTO_IDS) { + writeFileSync(savedAt(dir, id), "not the photo any more"); + } + + const holding = holdingServer(); + const lib = await openLibrary(dir, holding.fetch); + const lines: string[] = []; + const mismatches = () => + lines.filter((line) => line.startsWith("MISMATCH")); + try { + const backup = lib.backup({ + verify: true, + onProgress: (line) => lines.push(line), + }); + await vi.waitFor(() => expect(holding.held).toHaveLength(5), { + interval: 5, + timeout: 5000, + }); + expect(mismatches()).toHaveLength(5); + + await releaseOneAtATime(holding.held, 5); + const result = await backup; + + expect(holding.most()).toBe(5); + expect(mismatches()).toHaveLength(8); + expect(result).toMatchObject({ + downloaded: 8, + verified: 0, + mismatched: 8, + failed: 0, + }); + expectStored(dir, PHOTO_IDS); + } finally { + await lib.close(); + } + }); + + // The server answers 404 for photo 103's original. A 404 is not retried, + // so that download fails at once, while other downloads are still in + // flight. The backup records the failure and goes on: the downloads + // beside it finish, the photos after it are downloaded, and only photo + // 103 is missing from the backup and in failures.json. + it("goes on with the other originals when one fails", async () => { + const dir = mkdtempSync(join(root, "backup-")); + const holding = holdingServer(103); + const lib = await openLibrary(dir, holding.fetch); + try { + const backup = lib.backup(); + await releaseOneAtATime(holding.held, 5); + const result = await backup; + + expect(result).toMatchObject({ + totalFiles: 8, + downloaded: 7, + failed: 1, + }); + expect(result.errors.map((e) => e.fileID)).toEqual([103]); + expectStored( + dir, + PHOTO_IDS.filter((id) => id !== 103), + ); + expect(existsSync(savedAt(dir, 103))).toBe(false); + const failures = JSON.parse( + readFileSync(join(dir, "failures.json"), "utf-8"), + ); + expect(Object.keys(failures.files)).toEqual(["103"]); + } finally { + await lib.close(); + } + }); + + // A throw from `onProgress` fails the backup: `lib.backup()` rejects with + // it. From then on the backup starts no further original. It finishes the + // ones it is working on, and rejects only once they are in place, so + // nothing is written into the backup after it has ended and removed its + // lock. Here the first five downloads wait. Photo 103's is let go first + // and answered 404, and the line reporting that failure throws while the + // other four wait. Only those five downloads are held: a backup that went + // on would store the other three photos. + it("when onProgress throws, starts no further original, and rejects once the originals it was working on are in place", async () => { + const dir = mkdtempSync(join(root, "backup-")); + const downloads: number[] = []; + const held: { id: number; release: () => void }[] = []; + const fetch: typeof globalThis.fetch = async (input, init) => { + const id = downloadOf(input); + if (id !== undefined) { + downloads.push(id); + if (downloads.length <= 5) { + await new Promise((release) => + held.push({ id, release }), + ); + } + if (id === 103) { + return new Response("not found", { status: 404 }); + } + } + return server(input, init); + }; + const lib = await openLibrary(dir, fetch); + const lines: string[] = []; + try { + const backup = lib.backup({ + onProgress: (line) => { + lines.push(line); + if (line.startsWith("FAILED")) { + throw new Error("the progress output is closed"); + } + }, + }); + // The photos whose originals are in place, and whether the lock + // is there, at the moment `lib.backup()` rejects. + let storedWhenRejected: number[] = []; + let lockedWhenRejected = true; + const rejection = backup.catch((err: unknown) => { + storedWhenRejected = PHOTO_IDS.filter((id) => + existsSync(savedAt(dir, id)), + ); + lockedWhenRejected = existsSync(join(dir, "backup.lock")); + return err; + }); + + await vi.waitFor(() => expect(held).toHaveLength(5), { + interval: 5, + timeout: 5000, + }); + const at103 = held.findIndex((h) => h.id === 103); + held.splice(at103, 1)[0]!.release(); + await vi.waitFor( + () => + expect( + lines.some((line) => line.startsWith("FAILED")), + ).toBe(true), + { interval: 5, timeout: 5000 }, + ); + for (const download of held) download.release(); + + expect(await rejection).toEqual( + new Error("the progress output is closed"), + ); + expect(downloads.sort()).toEqual([101, 102, 103, 104, 105]); + expect(storedWhenRejected).toEqual([101, 102, 104, 105]); + expect(lockedWhenRejected).toBe(false); + } finally { + await lib.close(); + } + }); + + // Once the backup's lock is lost, another backup may be writing to the + // directory, so the backup starts no further original. It finishes the + // ones it is working on, and `lib.backup()` rejects once they are all in + // place. Here `backup.lock` is removed while the first five downloads + // wait, and proper-lockfile's timer finds it gone. As in + // `backup-end-to-end.test.ts`, only `setTimeout` and `clearTimeout` are + // faked, so the test can fire that timer without waiting 5 seconds. Only + // the first five downloads are held: a backup that went on would store + // the other three photos. + it("when its lock is lost, starts no further original, and rejects once the originals it was working on are in place", async () => { + const dir = mkdtempSync(join(root, "backup-")); + const lockPath = join(dir, "backup.lock"); + const downloads: number[] = []; + const held: (() => void)[] = []; + const fetch: typeof globalThis.fetch = async (input, init) => { + const id = downloadOf(input); + if (id !== undefined) { + downloads.push(id); + if (downloads.length <= 5) { + await new Promise((release) => held.push(release)); + } + } + return server(input, init); + }; + const lib = await openLibrary(dir, fetch); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const backup = lib.backup(); + // The photos whose originals are in place at the moment + // `lib.backup()` rejects. + let storedWhenRejected: number[] = []; + const rejection = backup.catch((err: unknown) => { + storedWhenRejected = PHOTO_IDS.filter((id) => + existsSync(savedAt(dir, id)), + ); + return err; + }); + + await vi.waitFor(() => expect(held).toHaveLength(5), { + interval: 5, + timeout: 5000, + }); + rmSync(lockPath, { recursive: true }); + // Fire proper-lockfile's timer, and wait for the look at the lock + // on disk that it starts to finish: Node lists a file system call + // in progress as an `FSReqCallback`. + vi.advanceTimersByTime(5_000); + while (process.getActiveResourcesInfo().includes("FSReqCallback")) { + await new Promise((resolve) => setImmediate(resolve)); + } + for (const release of held) release(); + + expect(await rejection).toEqual( + new Error( + `lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'`, + ), + ); + expect(downloads.sort()).toEqual([101, 102, 103, 104, 105]); + expect(storedWhenRejected).toEqual([101, 102, 103, 104, 105]); + } finally { + vi.useRealTimers(); + await lib.close(); + } + }); + + // Working on several originals at once changes only how long a backup + // takes. The same two runs, with a content pool of one, which backs up + // one original at a time, and with the default pools, end the same way: + // the same result, the same progress lines (in another order), the same + // failures.json, and the same tree. The first run stores every photo but + // 103, whose download is answered 404. Then two stored originals change + // on disk, and a run with `verify` puts them back and tries 103 again. + it("ends as a backup of one original at a time does", async () => { + const twoRuns = async (pools: RequestPools) => { + const dir = mkdtempSync(join(root, "backup-")); + const lines: string[] = []; + const backup = async (opts: BackupOptions) => { + const lib = await openLibrary(dir, failingFor(103), pools); + try { + return await lib.backup({ + ...opts, + onProgress: (line) => lines.push(line), + }); + } finally { + await lib.close(); + } + }; + const first = await backup({}); + writeFileSync(savedAt(dir, 101), "not the photo any more"); + writeFileSync(savedAt(dir, 105), "not the photo any more"); + const second = await backup({ verify: true }); + + // Each entry of failures.json without the time it was last + // tried, which differs from run to run. + const { files } = JSON.parse( + readFileSync(join(dir, "failures.json"), "utf-8"), + ) as { files: Record }; + for (const entry of Object.values(files)) delete entry.lastTriedAt; + + return { + first, + second, + lines: lines.sort(), + failures: files, + tree: listTree(dir), + }; + }; + + const oneAtATime = await twoRuns( + new RequestPools({ contentConcurrency: 1 }), + ); + const fiveAtATime = await twoRuns(new RequestPools()); + + expect(oneAtATime.first).toMatchObject({ + totalFiles: 8, + downloaded: 7, + skipped: 0, + failed: 1, + }); + expect(oneAtATime.second).toMatchObject({ + totalFiles: 8, + downloaded: 2, + skipped: 5, + verified: 5, + mismatched: 2, + unchecked: 0, + failed: 1, + }); + expect(oneAtATime.failures).toEqual({ + "103": expect.objectContaining({ fileID: 103, attempts: 2 }), + }); + expect(fiveAtATime).toEqual(oneAtATime); + }); +}); diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index 886a1fc..65bf5b6 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -704,10 +704,11 @@ describe("a backup whose lock is lost", () => { } }; - // beach.jpg, whose download was under way, is stored; no other file is - // started. `quak backup` prints why as one line, with no stack trace, and - // exits 1. - it("stops quak backup before its next file when backup.lock is removed, and exits 1", async () => { + // The backup downloads up to five originals at once, so it has started + // all three when the lock is lost, and beach.jpg is stored. The backup + // then stops before it fetches ML data. `quak backup` prints why as one + // line, with no stack trace, and exits 1. + it("stops quak backup before it fetches ML data when backup.lock is removed, and exits 1", async () => { const result = backup(dir, {}, fetch); await heldRequested; rmSync(lockPath, { recursive: true }); @@ -720,9 +721,11 @@ describe("a backup whose lock is lost", () => { "Starting backup...\n" + "Refreshing library...\n" + "Fetching original beach.jpg (100)...\n" + + "Fetching original IMG_0002.HEIC (300)...\n" + + "Fetching original toast.mp4 (200)...\n" + `quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'\n`, ); - expect(downloads).toEqual([BEACH.id]); + expect(downloads.sort()).toEqual([100, 200, 300]); expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual( Buffer.from(JPEG_WITH_EXIF), ); @@ -798,7 +801,7 @@ describe("a backup whose lock is lost", () => { await expect(result).rejects.toThrow( `lost the lock on ${dir}: Unable to update lock within the stale threshold`, ); - expect(downloads).toEqual([BEACH.id]); + expect(downloads.sort()).toEqual([100, 200, 300]); expect(existsSync(lockPath)).toBe(true); } finally { await lib.close(); @@ -823,7 +826,7 @@ describe("a backup whose lock is lost", () => { const lost = `lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'`; await expect(result).rejects.toThrow(lost); expect(lock.lost?.message).toBe(lost); - expect(downloads).toEqual([BEACH.id]); + expect(downloads.sort()).toEqual([100, 200, 300]); } finally { await lib.close(); await lock.release(); diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 065a2c5..1938a4b 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -909,6 +909,27 @@ describe("the backup lock", () => { expect(existsSync(lockPath(outDir))).toBe(false); await lib.close(); }); + + // `runBackup` takes a library object from its caller. With a + // `contentConcurrency` below 1, or one that is not a number, it would + // back up no original and report no error. It rejects any that is not a + // positive integer, as `BoundedPool` does, before it takes the lock, + // which would create the download directory, and before it asks the + // library for anything: the object here has nothing else. + it("rejects a library whose contentConcurrency is not a positive integer, before taking the lock", async () => { + const outDir = join(root, "backup"); + for (const contentConcurrency of [undefined, 0, -1, 1.5, NaN]) { + const lib = { contentConcurrency } as unknown as BackupLibrary; + + await expect( + runBackup(lib, { downloadDirectory: outDir }), + ).rejects.toThrow( + `contentConcurrency must be a positive integer, got ${contentConcurrency}`, + ); + + expect(existsSync(outDir)).toBe(false); + } + }); }); // Also serves ML data: file 100 has `ML_PAYLOAD`, the other two have none. @@ -1491,6 +1512,7 @@ describe("backup album folders", () => { thumbnail: async () => { throw new Error("no thumbnails in this stand-in"); }, + contentConcurrency: 5, fetchMLData: async () => {}, mlData: async () => undefined, whoami: () => ({ email: "backup@example.com", userID: USER_ID }),