quak backup works on up to the content pool's limit of originals at once (closes #182) #192
@@ -779,9 +779,9 @@ A running backup loses its lock when someone removes `backup.lock`, or when
|
||||
another backup takes it over because the running one left it untouched for 10
|
||||
seconds, as a stopped or stalled process does. The backup finds this out the
|
||||
next time it goes to touch the lock, which it does every 5 seconds, and then
|
||||
stops before it starts another file. The original it is downloading at that
|
||||
moment is still stored. It leaves `failures.json` and `backup.lock` as they are,
|
||||
since another backup may be using them. `quak backup` prints
|
||||
stops before it starts another file. The originals it is downloading at that
|
||||
moment are still stored. It leaves `failures.json` and `backup.lock` as they
|
||||
are, since another backup may be using them. `quak backup` prints
|
||||
`quak: lost the lock on <dir>: <reason>` as one line and exits 1, and
|
||||
`lib.backup()` rejects with `lost the lock on <dir>: <reason>`.
|
||||
|
||||
@@ -1096,6 +1096,11 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange`
|
||||
Each pool orders on-demand work ahead of background/precache work and dedups
|
||||
in-flight fetches by key, and an idle pool never lends its slots to a busy one.
|
||||
|
||||
`lib.backup()`, and so `quak backup`, works on up to the content pool's limit of
|
||||
originals at once, 5 unless the library was opened with other `pools`. The limit
|
||||
covers everything it does for one original, hashing it for `verify` included, so
|
||||
however large the account is, no more originals than that are open at once.
|
||||
|
||||
### On-disk cache layout
|
||||
|
||||
Under `cacheDirectory`:
|
||||
|
||||
@@ -17,18 +17,26 @@ pre-1.0
|
||||
|
||||
# Next Step
|
||||
|
||||
`quak backup` downloading one original at a time,
|
||||
https://git.eeqj.de/sneak/quak/issues/182, is fixed in
|
||||
https://git.eeqj.de/sneak/quak/pulls/192, which waits on review. No other
|
||||
implementation work is open. The check of quak against a live Ente account,
|
||||
https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak has, so
|
||||
it waits on him.
|
||||
None: no implementation work is open. The check of quak against a live Ente
|
||||
account, https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak
|
||||
has, so it waits on him.
|
||||
|
||||
Tagging and releases are decided by sneak alone, and happen only when he
|
||||
declares one.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-14: `lib.backup()`, and so `quak backup`, works on up to the content
|
||||
pool's limit of originals at once, 5 by default, instead of one at a time
|
||||
(issue 182). That many workers take files from one shared list, each doing all
|
||||
the work for one original, its `--verify` hashing included, so no more
|
||||
originals than that are open at once. `BackupLibrary` says the limit as
|
||||
`contentConcurrency`. A test holds each download at the fake Ente server to
|
||||
count those in flight. Once a worker has thrown, as one does when `onProgress`
|
||||
throws, or the lock is lost, no worker takes another file; the backup fails
|
||||
once the originals under way are in place, and a lost lock stops it before it
|
||||
fetches ML data.
|
||||
|
||||
- 2026-10-14: Next Step said the cache design, issue 36, waits on sneak's
|
||||
review, though that issue closed on 2026-10-03 (issue 194). It now names the
|
||||
issues open on the tracker.
|
||||
|
||||
+100
-67
@@ -6,9 +6,9 @@
|
||||
// file. It waits for a completed refresh of the library (a failed one fails the
|
||||
// backup before any file is touched), then, for every file in scope, puts its
|
||||
// original at its save path under `downloadDirectory`, as `Photo.download()`
|
||||
// does, waits for an ML data fetch, and rebuilds the derived views (per-file
|
||||
// sidecars, per-collection symlink trees, per-collection JSON) from the model.
|
||||
// The on-disk layout:
|
||||
// does, up to the content pool's limit of them at once, waits for an ML data
|
||||
// fetch, and rebuilds the derived views (per-file sidecars, per-collection
|
||||
// symlink trees, per-collection JSON) from the model. The on-disk layout:
|
||||
//
|
||||
// <downloadDirectory>/
|
||||
// YYYY/YYYY-MM/YYYY-MM-DD/
|
||||
@@ -181,6 +181,9 @@ export interface BackupLibrary {
|
||||
destination: string,
|
||||
): Promise<{ path: string; videoPath?: string }>;
|
||||
thumbnail(fileID: number): Promise<{ path: string }>;
|
||||
// How many originals the backup works on at once: the limit of the
|
||||
// library's content pool.
|
||||
contentConcurrency: number;
|
||||
// Wait for an ML data fetch to complete, joining one already running or
|
||||
// starting one. Rejects with the reason when it fails; resolves at once
|
||||
// when the library cannot fetch ML data.
|
||||
@@ -654,75 +657,31 @@ const runLockedBackup = async (
|
||||
// present original is hashed first, and one that does not match the hash
|
||||
// its metadata records is removed and fetched like a missing one. One
|
||||
// that cannot be read is recorded as failed and left where it is.
|
||||
if (includeOriginals) {
|
||||
for (const [fileID, file] of distinct) {
|
||||
if (lock.lost) throw lock.lost;
|
||||
let stored = storedAtSavePath(downloadDirectory, file);
|
||||
let mismatch = false;
|
||||
if (stored !== undefined && verify) {
|
||||
try {
|
||||
if (file.metadata.hash === undefined) {
|
||||
unchecked++;
|
||||
} else if (
|
||||
(await storedHash(stored)) === file.metadata.hash
|
||||
) {
|
||||
verified++;
|
||||
} else {
|
||||
log(
|
||||
`MISMATCH original ${file.metadata.title} (${fileID}): its bytes do not match its content hash`,
|
||||
);
|
||||
mismatched++;
|
||||
mismatch = true;
|
||||
rmSync(stored.path);
|
||||
if (stored.videoPath !== undefined) {
|
||||
rmSync(stored.videoPath);
|
||||
}
|
||||
stored = undefined;
|
||||
}
|
||||
} catch (err) {
|
||||
log(
|
||||
`FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
|
||||
);
|
||||
recordFailure(
|
||||
file.id,
|
||||
file.metadata.title,
|
||||
collectionName.get(file.collectionID) ?? "",
|
||||
err,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (stored !== undefined) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
// `storeOriginal` does this for one original, and records its failure.
|
||||
const storeOriginal = async (file: EnteFile): Promise<void> => {
|
||||
let stored = storedAtSavePath(downloadDirectory, file);
|
||||
let mismatch = false;
|
||||
if (stored !== undefined && verify) {
|
||||
try {
|
||||
log(`Fetching original ${file.metadata.title} (${fileID})...`);
|
||||
// A fetched original is written straight to its save path (a
|
||||
// live photo beside it); only one that was already cached
|
||||
// elsewhere is copied.
|
||||
const placed = await placeOriginal(
|
||||
downloadDirectory,
|
||||
file,
|
||||
(dest) => lib.original(fileID, dest),
|
||||
);
|
||||
storedThisRun.add(fileID);
|
||||
// A copy from the cache is not checked as a download is and can
|
||||
// hold the same bad bytes, so what is put back after a
|
||||
// mismatch is hashed too. A bad copy stays where it is and the
|
||||
// file fails.
|
||||
if (
|
||||
mismatch &&
|
||||
(await storedHash(placed)) !== file.metadata.hash
|
||||
) {
|
||||
throw new Error(
|
||||
"the original put back does not match its content hash either",
|
||||
if (file.metadata.hash === undefined) {
|
||||
unchecked++;
|
||||
} else if ((await storedHash(stored)) === file.metadata.hash) {
|
||||
verified++;
|
||||
} else {
|
||||
log(
|
||||
`MISMATCH original ${file.metadata.title} (${file.id}): its bytes do not match its content hash`,
|
||||
);
|
||||
mismatched++;
|
||||
mismatch = true;
|
||||
rmSync(stored.path);
|
||||
if (stored.videoPath !== undefined) {
|
||||
rmSync(stored.videoPath);
|
||||
}
|
||||
stored = undefined;
|
||||
}
|
||||
downloaded++;
|
||||
} catch (err) {
|
||||
log(
|
||||
`FAILED original ${file.metadata.title}: ${errorMessage(err)}`,
|
||||
`FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
|
||||
);
|
||||
recordFailure(
|
||||
file.id,
|
||||
@@ -730,8 +689,74 @@ const runLockedBackup = async (
|
||||
collectionName.get(file.collectionID) ?? "",
|
||||
err,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (stored !== undefined) {
|
||||
skipped++;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
log(`Fetching original ${file.metadata.title} (${file.id})...`);
|
||||
// A fetched original is written straight to its save path (a
|
||||
// live photo beside it); only one that was already cached
|
||||
// elsewhere is copied.
|
||||
const placed = await placeOriginal(
|
||||
downloadDirectory,
|
||||
file,
|
||||
(dest) => lib.original(file.id, dest),
|
||||
);
|
||||
storedThisRun.add(file.id);
|
||||
// A copy from the cache is not checked as a download is and can
|
||||
// hold the same bad bytes, so what is put back after a mismatch
|
||||
// is hashed too. A bad copy stays where it is and the file fails.
|
||||
if (mismatch && (await storedHash(placed)) !== file.metadata.hash) {
|
||||
throw new Error(
|
||||
"the original put back does not match its content hash either",
|
||||
);
|
||||
}
|
||||
downloaded++;
|
||||
} catch (err) {
|
||||
log(`FAILED original ${file.metadata.title}: ${errorMessage(err)}`);
|
||||
recordFailure(
|
||||
file.id,
|
||||
file.metadata.title,
|
||||
collectionName.get(file.collectionID) ?? "",
|
||||
err,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Up to the content pool's limit of originals are worked on at once,
|
||||
// their `verify` hashing included, so however large the account is, no
|
||||
// more than that many are open. Each worker takes the next file from the
|
||||
// iterator they all share until none is left. Once a worker has thrown,
|
||||
// as one does when `onProgress` throws, or the lock is lost, no worker
|
||||
// takes another file. The backup fails only once every worker has
|
||||
// stopped, so none writes into the backup after it has ended. It checks
|
||||
// the lock then too, so it does not go on to the thumbnails or the ML
|
||||
// data fetch without it.
|
||||
if (includeOriginals) {
|
||||
const pending = distinct.values();
|
||||
let workerThrew = false;
|
||||
const worker = async (): Promise<void> => {
|
||||
for (const file of pending) {
|
||||
if (workerThrew || lock.lost) return;
|
||||
try {
|
||||
await storeOriginal(file);
|
||||
} catch (err) {
|
||||
workerThrew = true;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
};
|
||||
const workers = await Promise.allSettled(
|
||||
Array.from({ length: lib.contentConcurrency }, worker),
|
||||
);
|
||||
for (const w of workers) {
|
||||
if (w.status === "rejected") throw w.reason;
|
||||
}
|
||||
if (lock.lost) throw lock.lost;
|
||||
}
|
||||
|
||||
if (includeThumbnails) {
|
||||
@@ -960,6 +985,14 @@ export const runBackup = async (
|
||||
"open the library with one)",
|
||||
);
|
||||
}
|
||||
// As `BoundedPool` does. A limit below 1, or one that is not a number,
|
||||
// would back up no original and end with no error.
|
||||
const { contentConcurrency } = lib;
|
||||
if (!Number.isInteger(contentConcurrency) || contentConcurrency < 1) {
|
||||
throw new RangeError(
|
||||
`contentConcurrency must be a positive integer, got ${contentConcurrency}`,
|
||||
);
|
||||
}
|
||||
if (opts.lock) {
|
||||
return runLockedBackup(lib, opts, downloadDirectory, opts.lock);
|
||||
}
|
||||
|
||||
@@ -607,6 +607,7 @@ export class Library {
|
||||
original: (fileID, destination) =>
|
||||
cache!.backupOriginal(fileID, destination),
|
||||
thumbnail: (fileID) => cache!.thumbnail(fileID),
|
||||
contentConcurrency: this.pools.content.concurrency,
|
||||
fetchMLData: () => this.fetchMLDataNow(),
|
||||
mlData: (fileID) => this.mldata.forFile({ fileID }),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,608 @@
|
||||
/**
|
||||
* How many originals `lib.backup()` works on at once.
|
||||
*
|
||||
* The backup puts up to the content pool's limit of originals at their save
|
||||
* paths at the same time: 5 for a library opened with the default
|
||||
* `RequestPools`, as `quak backup` opens it, or the `contentConcurrency` of the
|
||||
* pools a library was opened with. Everything the backup does for one original
|
||||
* counts against that limit, hashing it for `verify` included, so however large
|
||||
* the account is, the backup has no more than that many originals open at once.
|
||||
*
|
||||
* As in `backup-end-to-end.test.ts`, the real `Client` and `Library` run
|
||||
* against `buildMetaFetch` from `test/ente-server.ts`, a fake of the Ente HTTP
|
||||
* API serving data this file encrypts with libsodium. Here a test can also hold
|
||||
* each original's download until it lets it go, to see how many the backup has
|
||||
* in flight at once.
|
||||
*
|
||||
* The account, user 42, has one album of its own, Holiday, of eight photos,
|
||||
* all taken at noon on 1 March 2024, each with an original of its own.
|
||||
*/
|
||||
|
||||
import {
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
readlinkSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import sodium from "libsodium-wrappers-sumo";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { BackupOptions } from "../../src/backup.js";
|
||||
import { Client, type ClientSnapshot } from "../../src/client.js";
|
||||
import { init, toBase64 } from "../../src/crypto/index.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
import { RequestPools } from "../../src/library/pools.js";
|
||||
import {
|
||||
buildAccount,
|
||||
buildMetaFetch,
|
||||
encryptSecretbox,
|
||||
encryptStreamBlob,
|
||||
} from "../ente-server.js";
|
||||
import { blake2b } from "../live-photo.js";
|
||||
|
||||
const EMAIL = "backup@example.com";
|
||||
const PASSWORD = "backup-password";
|
||||
// The fake server logs every account in as user 42.
|
||||
const USER_ID = 42;
|
||||
|
||||
// Noon on 1 March 2024 in the time zone of the machine running the test, in
|
||||
// microseconds, as Ente records times.
|
||||
const TAKEN = new Date(2024, 2, 1, 12).getTime() * 1000;
|
||||
|
||||
// The eight photos' IDs. Photo `id` is titled `photo-<id>.jpg`, and its
|
||||
// original is a few bytes naming it.
|
||||
const PHOTO_IDS = [101, 102, 103, 104, 105, 106, 107, 108];
|
||||
const originalOf = (id: number): Uint8Array =>
|
||||
new TextEncoder().encode(`the photo ${id}`);
|
||||
|
||||
// Where a backup to `dir` saves photo `id`'s original.
|
||||
const savedAt = (dir: string, id: number): string =>
|
||||
join(dir, "2024/2024-03/2024-03-01", `2024-03-01.${id}.jpg`);
|
||||
|
||||
// The fake server, the session it logs the account in with, and the
|
||||
// directory every backup below is made in.
|
||||
let server: typeof globalThis.fetch;
|
||||
let session: ClientSnapshot;
|
||||
let root: string;
|
||||
|
||||
// The fake server, holding the account encrypted as Ente's clients upload it.
|
||||
const buildServer = async () => {
|
||||
const account = await buildAccount(PASSWORD);
|
||||
|
||||
// The album as `/collections/v2` lists it: its key encrypted with the
|
||||
// account's master key, its name with its own key.
|
||||
const albumKey = sodium.crypto_secretbox_keygen();
|
||||
const encryptedAlbumKey = encryptSecretbox(albumKey, account.masterKey);
|
||||
const encryptedName = encryptSecretbox(
|
||||
new TextEncoder().encode("Holiday"),
|
||||
albumKey,
|
||||
);
|
||||
const album = {
|
||||
id: 10,
|
||||
owner: { id: USER_ID },
|
||||
encryptedKey: toBase64(encryptedAlbumKey.ciphertext),
|
||||
keyDecryptionNonce: toBase64(encryptedAlbumKey.nonce),
|
||||
encryptedName: toBase64(encryptedName.ciphertext),
|
||||
nameDecryptionNonce: toBase64(encryptedName.nonce),
|
||||
type: "album",
|
||||
updationTime: TAKEN,
|
||||
};
|
||||
|
||||
// Each photo as the album's `/collections/v2/diff` lists it: its key
|
||||
// encrypted with the album's key, and its metadata, holding the hash of
|
||||
// its original, encrypted with its own key. Its original, encrypted with
|
||||
// its own key too, is what the files CDN serves for it.
|
||||
const ciphertexts: Record<number, Uint8Array> = {};
|
||||
const photos = PHOTO_IDS.map((id) => {
|
||||
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
const original = encryptStreamBlob(originalOf(id), key);
|
||||
ciphertexts[id] = original.ciphertext;
|
||||
const encryptedKey = encryptSecretbox(key, albumKey);
|
||||
const metadata = encryptStreamBlob(
|
||||
new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
title: `photo-${id}.jpg`,
|
||||
fileType: 0,
|
||||
hash: blake2b(originalOf(id)),
|
||||
creationTime: TAKEN,
|
||||
modificationTime: TAKEN,
|
||||
}),
|
||||
),
|
||||
key,
|
||||
);
|
||||
return {
|
||||
id,
|
||||
collectionID: 10,
|
||||
ownerID: USER_ID,
|
||||
encryptedKey: toBase64(encryptedKey.ciphertext),
|
||||
keyDecryptionNonce: toBase64(encryptedKey.nonce),
|
||||
metadata: {
|
||||
encryptedData: toBase64(metadata.ciphertext),
|
||||
decryptionHeader: toBase64(metadata.header),
|
||||
},
|
||||
file: { decryptionHeader: toBase64(original.header) },
|
||||
thumbnail: {
|
||||
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
||||
},
|
||||
updationTime: TAKEN,
|
||||
};
|
||||
});
|
||||
|
||||
return buildMetaFetch({
|
||||
verifier: account.verifier,
|
||||
srpAttributes: account.srpAttributes,
|
||||
keyAttributes: account.keyAttributes,
|
||||
encryptedToken: account.encryptedToken,
|
||||
collections: [album],
|
||||
filesByCollection: { 10: photos },
|
||||
encryptedMLData: {},
|
||||
fileCiphertexts: ciphertexts,
|
||||
});
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
await init();
|
||||
await sodium.ready;
|
||||
root = mkdtempSync(join(tmpdir(), "quak-backup-concurrency-"));
|
||||
server = await buildServer();
|
||||
const client = await Client.login({
|
||||
email: EMAIL,
|
||||
password: PASSWORD,
|
||||
apiOptions: { fetch: server },
|
||||
});
|
||||
session = client.toJSON();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// The ID of the photo whose original a request downloads, or undefined for any
|
||||
// other request. quak downloads an original from
|
||||
// `https://files.ente.io/?fileID=<id>`.
|
||||
const downloadOf = (input: RequestInfo | URL): number | undefined => {
|
||||
const url = new URL(String(input));
|
||||
return url.hostname === "files.ente.io"
|
||||
? Number(url.searchParams.get("fileID"))
|
||||
: undefined;
|
||||
};
|
||||
|
||||
// The fake server, except that it answers 404 to every download of photo
|
||||
// `failing`'s original.
|
||||
const failingFor =
|
||||
(failing: number): typeof globalThis.fetch =>
|
||||
async (input, init) =>
|
||||
downloadOf(input) === failing
|
||||
? new Response("not found", { status: 404 })
|
||||
: server(input, init);
|
||||
|
||||
// The fake server, holding each download of an original until the test lets
|
||||
// it go. `held` lists the downloads waiting, the first to arrive first;
|
||||
// `most()` is the largest number that ever waited at once. A held download of
|
||||
// photo `failing`'s original is answered 404 once it is let go.
|
||||
const holdingServer = (failing?: number) => {
|
||||
const held: { id: number; release: () => void }[] = [];
|
||||
let most = 0;
|
||||
const fetch: typeof globalThis.fetch = async (input, init) => {
|
||||
const id = downloadOf(input);
|
||||
if (id !== undefined) {
|
||||
await new Promise<void>((release) => {
|
||||
held.push({ id, release });
|
||||
most = Math.max(most, held.length);
|
||||
});
|
||||
if (id === failing) {
|
||||
return new Response("not found", { status: 404 });
|
||||
}
|
||||
}
|
||||
return server(input, init);
|
||||
};
|
||||
return { fetch, held, most: () => most };
|
||||
};
|
||||
|
||||
// Let the held downloads go one at a time, the first to arrive first, each
|
||||
// once `limit` of them wait, or all the downloads still to come when there
|
||||
// are fewer. Each of the eight photos is downloaded once. With a backup that
|
||||
// downloads fewer than `limit` at once, this fails waiting for them. Each wait
|
||||
// allows five seconds, for a loaded machine's disk to sync the files written.
|
||||
const releaseOneAtATime = async (
|
||||
held: { release: () => void }[],
|
||||
limit: number,
|
||||
): Promise<void> => {
|
||||
for (let left = PHOTO_IDS.length; left > 0; left--) {
|
||||
await vi.waitFor(
|
||||
() => expect(held).toHaveLength(Math.min(limit, left)),
|
||||
{ interval: 5, timeout: 5000 },
|
||||
);
|
||||
held.shift()!.release();
|
||||
}
|
||||
};
|
||||
|
||||
// A library of the account opened as `quak backup` opens one, with no
|
||||
// precache and no background refresh, sending its requests through `fetch`.
|
||||
// It backs up to `dir` and keeps its cache beside it. Its request pools are
|
||||
// `pools`, or by default the library's own.
|
||||
const openLibrary = (
|
||||
dir: string,
|
||||
fetch: typeof globalThis.fetch,
|
||||
pools?: RequestPools,
|
||||
): Promise<Library> =>
|
||||
Library.open({
|
||||
client: Client.fromJSON(session, { fetch }),
|
||||
downloadDirectory: dir,
|
||||
cacheDirectory: `${dir}-cache`,
|
||||
refreshIntervalSeconds: Infinity,
|
||||
precacheThumbnails: false,
|
||||
precacheOriginals: false,
|
||||
pools,
|
||||
});
|
||||
|
||||
// Every entry under `dir`, relative to it, sorted, a symlink as
|
||||
// `name -> target`.
|
||||
const listTree = (dir: string): string[] =>
|
||||
readdirSync(dir, { recursive: true })
|
||||
.sort()
|
||||
.map((name) => {
|
||||
const path = join(dir, name);
|
||||
return lstatSync(path).isSymbolicLink()
|
||||
? `${name} -> ${readlinkSync(path)}`
|
||||
: name;
|
||||
});
|
||||
|
||||
const expectStored = (dir: string, ids: number[]): void => {
|
||||
for (const id of ids) {
|
||||
expect(readFileSync(savedAt(dir, id))).toEqual(
|
||||
Buffer.from(originalOf(id)),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
describe("lib.backup()", () => {
|
||||
// A library opened with the default pools, as `quak backup` opens one,
|
||||
// downloads up to five originals at once. As soon as one finishes, the
|
||||
// backup starts on the next photo, so five stay in flight until fewer
|
||||
// than five are left.
|
||||
it("downloads five originals at once, and never more, with the default pools", async () => {
|
||||
const dir = mkdtempSync(join(root, "backup-"));
|
||||
const holding = holdingServer();
|
||||
const lib = await openLibrary(dir, holding.fetch);
|
||||
try {
|
||||
const backup = lib.backup();
|
||||
await releaseOneAtATime(holding.held, 5);
|
||||
const result = await backup;
|
||||
|
||||
expect(holding.most()).toBe(5);
|
||||
expect(result).toMatchObject({
|
||||
totalFiles: 8,
|
||||
downloaded: 8,
|
||||
failed: 0,
|
||||
});
|
||||
expectStored(dir, PHOTO_IDS);
|
||||
} finally {
|
||||
await lib.close();
|
||||
}
|
||||
});
|
||||
|
||||
// The limit is the content pool's, so a library opened with pools of
|
||||
// another `contentConcurrency` backs up that many at once.
|
||||
it("downloads as many originals at once as the content pool of the pools it was opened with allows", async () => {
|
||||
const dir = mkdtempSync(join(root, "backup-"));
|
||||
const holding = holdingServer();
|
||||
const lib = await openLibrary(
|
||||
dir,
|
||||
holding.fetch,
|
||||
new RequestPools({ contentConcurrency: 2 }),
|
||||
);
|
||||
try {
|
||||
const backup = lib.backup();
|
||||
await releaseOneAtATime(holding.held, 2);
|
||||
const result = await backup;
|
||||
|
||||
expect(holding.most()).toBe(2);
|
||||
expect(result).toMatchObject({ downloaded: 8, failed: 0 });
|
||||
expectStored(dir, PHOTO_IDS);
|
||||
} finally {
|
||||
await lib.close();
|
||||
}
|
||||
});
|
||||
|
||||
// With `verify`, hashing an original already stored is part of the work
|
||||
// on it, and counts against the limit too: a large backup never has more
|
||||
// than five originals open for hashing. Here every stored original has
|
||||
// changed on disk, so each is hashed, found not to match, and downloaded
|
||||
// again. While five downloads wait, the backup has hashed only those five
|
||||
// photos' originals; it opens the next one once a download finishes.
|
||||
it("with verify, hashes no more originals ahead of their downloads than the limit", async () => {
|
||||
const dir = mkdtempSync(join(root, "backup-"));
|
||||
const first = await openLibrary(dir, server);
|
||||
try {
|
||||
await first.backup();
|
||||
} finally {
|
||||
await first.close();
|
||||
}
|
||||
for (const id of PHOTO_IDS) {
|
||||
writeFileSync(savedAt(dir, id), "not the photo any more");
|
||||
}
|
||||
|
||||
const holding = holdingServer();
|
||||
const lib = await openLibrary(dir, holding.fetch);
|
||||
const lines: string[] = [];
|
||||
const mismatches = () =>
|
||||
lines.filter((line) => line.startsWith("MISMATCH"));
|
||||
try {
|
||||
const backup = lib.backup({
|
||||
verify: true,
|
||||
onProgress: (line) => lines.push(line),
|
||||
});
|
||||
await vi.waitFor(() => expect(holding.held).toHaveLength(5), {
|
||||
interval: 5,
|
||||
timeout: 5000,
|
||||
});
|
||||
expect(mismatches()).toHaveLength(5);
|
||||
|
||||
await releaseOneAtATime(holding.held, 5);
|
||||
const result = await backup;
|
||||
|
||||
expect(holding.most()).toBe(5);
|
||||
expect(mismatches()).toHaveLength(8);
|
||||
expect(result).toMatchObject({
|
||||
downloaded: 8,
|
||||
verified: 0,
|
||||
mismatched: 8,
|
||||
failed: 0,
|
||||
});
|
||||
expectStored(dir, PHOTO_IDS);
|
||||
} finally {
|
||||
await lib.close();
|
||||
}
|
||||
});
|
||||
|
||||
// The server answers 404 for photo 103's original. A 404 is not retried,
|
||||
// so that download fails at once, while other downloads are still in
|
||||
// flight. The backup records the failure and goes on: the downloads
|
||||
// beside it finish, the photos after it are downloaded, and only photo
|
||||
// 103 is missing from the backup and in failures.json.
|
||||
it("goes on with the other originals when one fails", async () => {
|
||||
const dir = mkdtempSync(join(root, "backup-"));
|
||||
const holding = holdingServer(103);
|
||||
const lib = await openLibrary(dir, holding.fetch);
|
||||
try {
|
||||
const backup = lib.backup();
|
||||
await releaseOneAtATime(holding.held, 5);
|
||||
const result = await backup;
|
||||
|
||||
expect(result).toMatchObject({
|
||||
totalFiles: 8,
|
||||
downloaded: 7,
|
||||
failed: 1,
|
||||
});
|
||||
expect(result.errors.map((e) => e.fileID)).toEqual([103]);
|
||||
expectStored(
|
||||
dir,
|
||||
PHOTO_IDS.filter((id) => id !== 103),
|
||||
);
|
||||
expect(existsSync(savedAt(dir, 103))).toBe(false);
|
||||
const failures = JSON.parse(
|
||||
readFileSync(join(dir, "failures.json"), "utf-8"),
|
||||
);
|
||||
expect(Object.keys(failures.files)).toEqual(["103"]);
|
||||
} finally {
|
||||
await lib.close();
|
||||
}
|
||||
});
|
||||
|
||||
// A throw from `onProgress` fails the backup: `lib.backup()` rejects with
|
||||
// it. From then on the backup starts no further original. It finishes the
|
||||
// ones it is working on, and rejects only once they are in place, so
|
||||
// nothing is written into the backup after it has ended and removed its
|
||||
// lock. Here the first five downloads wait. Photo 103's is let go first
|
||||
// and answered 404, and the line reporting that failure throws while the
|
||||
// other four wait. Only those five downloads are held: a backup that went
|
||||
// on would store the other three photos.
|
||||
it("when onProgress throws, starts no further original, and rejects once the originals it was working on are in place", async () => {
|
||||
const dir = mkdtempSync(join(root, "backup-"));
|
||||
const downloads: number[] = [];
|
||||
const held: { id: number; release: () => void }[] = [];
|
||||
const fetch: typeof globalThis.fetch = async (input, init) => {
|
||||
const id = downloadOf(input);
|
||||
if (id !== undefined) {
|
||||
downloads.push(id);
|
||||
if (downloads.length <= 5) {
|
||||
await new Promise<void>((release) =>
|
||||
held.push({ id, release }),
|
||||
);
|
||||
}
|
||||
if (id === 103) {
|
||||
return new Response("not found", { status: 404 });
|
||||
}
|
||||
}
|
||||
return server(input, init);
|
||||
};
|
||||
const lib = await openLibrary(dir, fetch);
|
||||
const lines: string[] = [];
|
||||
try {
|
||||
const backup = lib.backup({
|
||||
onProgress: (line) => {
|
||||
lines.push(line);
|
||||
if (line.startsWith("FAILED")) {
|
||||
throw new Error("the progress output is closed");
|
||||
}
|
||||
},
|
||||
});
|
||||
// The photos whose originals are in place, and whether the lock
|
||||
// is there, at the moment `lib.backup()` rejects.
|
||||
let storedWhenRejected: number[] = [];
|
||||
let lockedWhenRejected = true;
|
||||
const rejection = backup.catch((err: unknown) => {
|
||||
storedWhenRejected = PHOTO_IDS.filter((id) =>
|
||||
existsSync(savedAt(dir, id)),
|
||||
);
|
||||
lockedWhenRejected = existsSync(join(dir, "backup.lock"));
|
||||
return err;
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(held).toHaveLength(5), {
|
||||
interval: 5,
|
||||
timeout: 5000,
|
||||
});
|
||||
const at103 = held.findIndex((h) => h.id === 103);
|
||||
held.splice(at103, 1)[0]!.release();
|
||||
await vi.waitFor(
|
||||
() =>
|
||||
expect(
|
||||
lines.some((line) => line.startsWith("FAILED")),
|
||||
).toBe(true),
|
||||
{ interval: 5, timeout: 5000 },
|
||||
);
|
||||
for (const download of held) download.release();
|
||||
|
||||
expect(await rejection).toEqual(
|
||||
new Error("the progress output is closed"),
|
||||
);
|
||||
expect(downloads.sort()).toEqual([101, 102, 103, 104, 105]);
|
||||
expect(storedWhenRejected).toEqual([101, 102, 104, 105]);
|
||||
expect(lockedWhenRejected).toBe(false);
|
||||
} finally {
|
||||
await lib.close();
|
||||
}
|
||||
});
|
||||
|
||||
// Once the backup's lock is lost, another backup may be writing to the
|
||||
// directory, so the backup starts no further original. It finishes the
|
||||
// ones it is working on, and `lib.backup()` rejects once they are all in
|
||||
// place. Here `backup.lock` is removed while the first five downloads
|
||||
// wait, and proper-lockfile's timer finds it gone. As in
|
||||
// `backup-end-to-end.test.ts`, only `setTimeout` and `clearTimeout` are
|
||||
// faked, so the test can fire that timer without waiting 5 seconds. Only
|
||||
// the first five downloads are held: a backup that went on would store
|
||||
// the other three photos.
|
||||
it("when its lock is lost, starts no further original, and rejects once the originals it was working on are in place", async () => {
|
||||
const dir = mkdtempSync(join(root, "backup-"));
|
||||
const lockPath = join(dir, "backup.lock");
|
||||
const downloads: number[] = [];
|
||||
const held: (() => void)[] = [];
|
||||
const fetch: typeof globalThis.fetch = async (input, init) => {
|
||||
const id = downloadOf(input);
|
||||
if (id !== undefined) {
|
||||
downloads.push(id);
|
||||
if (downloads.length <= 5) {
|
||||
await new Promise<void>((release) => held.push(release));
|
||||
}
|
||||
}
|
||||
return server(input, init);
|
||||
};
|
||||
const lib = await openLibrary(dir, fetch);
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
const backup = lib.backup();
|
||||
// The photos whose originals are in place at the moment
|
||||
// `lib.backup()` rejects.
|
||||
let storedWhenRejected: number[] = [];
|
||||
const rejection = backup.catch((err: unknown) => {
|
||||
storedWhenRejected = PHOTO_IDS.filter((id) =>
|
||||
existsSync(savedAt(dir, id)),
|
||||
);
|
||||
return err;
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(held).toHaveLength(5), {
|
||||
interval: 5,
|
||||
timeout: 5000,
|
||||
});
|
||||
rmSync(lockPath, { recursive: true });
|
||||
// Fire proper-lockfile's timer, and wait for the look at the lock
|
||||
// on disk that it starts to finish: Node lists a file system call
|
||||
// in progress as an `FSReqCallback`.
|
||||
vi.advanceTimersByTime(5_000);
|
||||
while (process.getActiveResourcesInfo().includes("FSReqCallback")) {
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
for (const release of held) release();
|
||||
|
||||
expect(await rejection).toEqual(
|
||||
new Error(
|
||||
`lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'`,
|
||||
),
|
||||
);
|
||||
expect(downloads.sort()).toEqual([101, 102, 103, 104, 105]);
|
||||
expect(storedWhenRejected).toEqual([101, 102, 103, 104, 105]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
await lib.close();
|
||||
}
|
||||
});
|
||||
|
||||
// Working on several originals at once changes only how long a backup
|
||||
// takes. The same two runs, with a content pool of one, which backs up
|
||||
// one original at a time, and with the default pools, end the same way:
|
||||
// the same result, the same progress lines (in another order), the same
|
||||
// failures.json, and the same tree. The first run stores every photo but
|
||||
// 103, whose download is answered 404. Then two stored originals change
|
||||
// on disk, and a run with `verify` puts them back and tries 103 again.
|
||||
it("ends as a backup of one original at a time does", async () => {
|
||||
const twoRuns = async (pools: RequestPools) => {
|
||||
const dir = mkdtempSync(join(root, "backup-"));
|
||||
const lines: string[] = [];
|
||||
const backup = async (opts: BackupOptions) => {
|
||||
const lib = await openLibrary(dir, failingFor(103), pools);
|
||||
try {
|
||||
return await lib.backup({
|
||||
...opts,
|
||||
onProgress: (line) => lines.push(line),
|
||||
});
|
||||
} finally {
|
||||
await lib.close();
|
||||
}
|
||||
};
|
||||
const first = await backup({});
|
||||
writeFileSync(savedAt(dir, 101), "not the photo any more");
|
||||
writeFileSync(savedAt(dir, 105), "not the photo any more");
|
||||
const second = await backup({ verify: true });
|
||||
|
||||
// Each entry of failures.json without the time it was last
|
||||
// tried, which differs from run to run.
|
||||
const { files } = JSON.parse(
|
||||
readFileSync(join(dir, "failures.json"), "utf-8"),
|
||||
) as { files: Record<string, { lastTriedAt?: number }> };
|
||||
for (const entry of Object.values(files)) delete entry.lastTriedAt;
|
||||
|
||||
return {
|
||||
first,
|
||||
second,
|
||||
lines: lines.sort(),
|
||||
failures: files,
|
||||
tree: listTree(dir),
|
||||
};
|
||||
};
|
||||
|
||||
const oneAtATime = await twoRuns(
|
||||
new RequestPools({ contentConcurrency: 1 }),
|
||||
);
|
||||
const fiveAtATime = await twoRuns(new RequestPools());
|
||||
|
||||
expect(oneAtATime.first).toMatchObject({
|
||||
totalFiles: 8,
|
||||
downloaded: 7,
|
||||
skipped: 0,
|
||||
failed: 1,
|
||||
});
|
||||
expect(oneAtATime.second).toMatchObject({
|
||||
totalFiles: 8,
|
||||
downloaded: 2,
|
||||
skipped: 5,
|
||||
verified: 5,
|
||||
mismatched: 2,
|
||||
unchecked: 0,
|
||||
failed: 1,
|
||||
});
|
||||
expect(oneAtATime.failures).toEqual({
|
||||
"103": expect.objectContaining({ fileID: 103, attempts: 2 }),
|
||||
});
|
||||
expect(fiveAtATime).toEqual(oneAtATime);
|
||||
});
|
||||
});
|
||||
@@ -704,10 +704,11 @@ describe("a backup whose lock is lost", () => {
|
||||
}
|
||||
};
|
||||
|
||||
// beach.jpg, whose download was under way, is stored; no other file is
|
||||
// started. `quak backup` prints why as one line, with no stack trace, and
|
||||
// exits 1.
|
||||
it("stops quak backup before its next file when backup.lock is removed, and exits 1", async () => {
|
||||
// The backup downloads up to five originals at once, so it has started
|
||||
// all three when the lock is lost, and beach.jpg is stored. The backup
|
||||
// then stops before it fetches ML data. `quak backup` prints why as one
|
||||
// line, with no stack trace, and exits 1.
|
||||
it("stops quak backup before it fetches ML data when backup.lock is removed, and exits 1", async () => {
|
||||
const result = backup(dir, {}, fetch);
|
||||
await heldRequested;
|
||||
rmSync(lockPath, { recursive: true });
|
||||
@@ -720,9 +721,11 @@ describe("a backup whose lock is lost", () => {
|
||||
"Starting backup...\n" +
|
||||
"Refreshing library...\n" +
|
||||
"Fetching original beach.jpg (100)...\n" +
|
||||
"Fetching original IMG_0002.HEIC (300)...\n" +
|
||||
"Fetching original toast.mp4 (200)...\n" +
|
||||
`quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'\n`,
|
||||
);
|
||||
expect(downloads).toEqual([BEACH.id]);
|
||||
expect(downloads.sort()).toEqual([100, 200, 300]);
|
||||
expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual(
|
||||
Buffer.from(JPEG_WITH_EXIF),
|
||||
);
|
||||
@@ -798,7 +801,7 @@ describe("a backup whose lock is lost", () => {
|
||||
await expect(result).rejects.toThrow(
|
||||
`lost the lock on ${dir}: Unable to update lock within the stale threshold`,
|
||||
);
|
||||
expect(downloads).toEqual([BEACH.id]);
|
||||
expect(downloads.sort()).toEqual([100, 200, 300]);
|
||||
expect(existsSync(lockPath)).toBe(true);
|
||||
} finally {
|
||||
await lib.close();
|
||||
@@ -823,7 +826,7 @@ describe("a backup whose lock is lost", () => {
|
||||
const lost = `lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'`;
|
||||
await expect(result).rejects.toThrow(lost);
|
||||
expect(lock.lost?.message).toBe(lost);
|
||||
expect(downloads).toEqual([BEACH.id]);
|
||||
expect(downloads.sort()).toEqual([100, 200, 300]);
|
||||
} finally {
|
||||
await lib.close();
|
||||
await lock.release();
|
||||
|
||||
@@ -909,6 +909,27 @@ describe("the backup lock", () => {
|
||||
expect(existsSync(lockPath(outDir))).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
// `runBackup` takes a library object from its caller. With a
|
||||
// `contentConcurrency` below 1, or one that is not a number, it would
|
||||
// back up no original and report no error. It rejects any that is not a
|
||||
// positive integer, as `BoundedPool` does, before it takes the lock,
|
||||
// which would create the download directory, and before it asks the
|
||||
// library for anything: the object here has nothing else.
|
||||
it("rejects a library whose contentConcurrency is not a positive integer, before taking the lock", async () => {
|
||||
const outDir = join(root, "backup");
|
||||
for (const contentConcurrency of [undefined, 0, -1, 1.5, NaN]) {
|
||||
const lib = { contentConcurrency } as unknown as BackupLibrary;
|
||||
|
||||
await expect(
|
||||
runBackup(lib, { downloadDirectory: outDir }),
|
||||
).rejects.toThrow(
|
||||
`contentConcurrency must be a positive integer, got ${contentConcurrency}`,
|
||||
);
|
||||
|
||||
expect(existsSync(outDir)).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Also serves ML data: file 100 has `ML_PAYLOAD`, the other two have none.
|
||||
@@ -1491,6 +1512,7 @@ describe("backup album folders", () => {
|
||||
thumbnail: async () => {
|
||||
throw new Error("no thumbnails in this stand-in");
|
||||
},
|
||||
contentConcurrency: 5,
|
||||
fetchMLData: async () => {},
|
||||
mlData: async () => undefined,
|
||||
whoami: () => ({ email: "backup@example.com", userID: USER_ID }),
|
||||
|
||||
Reference in New Issue
Block a user