diff --git a/README.md b/README.md index fe40038..5654819 100644 --- a/README.md +++ b/README.md @@ -788,7 +788,8 @@ subsequent runs, existing originals are skipped. If a download fails, the error is logged and the backup continues with the next file. The exit code is non-zero if any files failed. `quak backup` opens its library with the thumbnail and originals precache off, so the only file content it fetches is the originals the -backup stores. +backup stores. It also opens it with no background timer, so it asks the server +for changes only as it starts, not again while it downloads. With `--verify`, or `lib.backup({ verify: true })`, a run also hashes each original already at its save path the way a download is checked (see "On-disk @@ -869,7 +870,7 @@ background, so an unreachable server does not block opening. | `client` | required | the account client (a `Client`, or any `LibraryClient`) | | `cacheDirectory` | `quak/` under the per-user cache directory | where `metadata.json` and the content cache live | | `downloadDirectory` | `photos` in the working directory | root of the save paths; an original stored there counts as cached | -| `refreshIntervalSeconds` | `3` | background refresh cadence | +| `refreshIntervalSeconds` | `3` | background refresh cadence; `Infinity` for no background timer | | `precacheThumbnails` | `true` | prefetch every thumbnail, newest first | | `precacheOriginals` | `true` | prefetch the favorites album and the latest-window originals | | `precacheOriginalsDays` | `7` | length in days of that latest window | diff --git a/TODO.md b/TODO.md index 87de288..25a6bfb 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,13 @@ declares one. # Completed Steps +- 2026-10-14: `quak backup` no longer asks the server for changes every 3 + seconds while it runs (issue 183). It opens its library with + `refreshIntervalSeconds: Infinity`, which the library now takes as no + background timer, so the backup refreshes only as it starts and no timed + refresh starts another ML data pass during the run. A test holds a backup's + download open while two hours pass and counts its `/collections/v2` requests. + - 2026-10-14: `test/cli/backup-end-to-end.test.ts` runs `quak backup` as cron does (issue 186): `backupCommand` and `run`, the real `loadSession` reading a session saved after logging in, and every request answered by a fake of the diff --git a/src/cli-commands.ts b/src/cli-commands.ts index 5f8b0b6..154c180 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -427,10 +427,13 @@ export const backupCommand = async ( try { // The precache is off: the backup fetches what it needs, and must not // also fill the cache with every thumbnail and the recent originals. + // No refresh starts on a timer: the backup lists its files once, from + // the refresh it waits for, and can run for hours. const lib = await Library.open({ client, downloadDirectory: dir, cacheDirectory: ctx.cacheDir, + refreshIntervalSeconds: Infinity, precacheThumbnails: false, precacheOriginals: false, }); diff --git a/src/library/index.ts b/src/library/index.ts index 489dd88..080020b 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -179,6 +179,9 @@ export interface LibraryOptions { // directory at open. The content cache treats an original already stored // at its save path as present. downloadDirectory?: string; + // How long after one refresh ends the background timer starts the next. + // `Infinity` starts none: the library then refreshes only as it opens and + // when `fresh()` or `backup()` asks. refreshIntervalSeconds?: number; onProgress?: RefreshProgressCallback; // The bounded request pools (issue #45), shared by the ML-data fetch (the @@ -627,7 +630,8 @@ export class Library { } private scheduleNext(): void { - if (this.closed) return; + // `setTimeout` would turn an infinite delay into 1 ms. + if (this.closed || this.intervalMs === Infinity) return; this.timer = setTimeout(() => { void this.runRefresh().then(() => this.scheduleNext()); }, this.intervalMs); diff --git a/test/cli/commands.test.ts b/test/cli/commands.test.ts index 09189a5..241e259 100644 --- a/test/cli/commands.test.ts +++ b/test/cli/commands.test.ts @@ -25,6 +25,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { PassThrough } from "node:stream"; import * as jpegJs from "jpeg-js"; +import sodium from "libsodium-wrappers-sumo"; import { describe, it, @@ -55,7 +56,7 @@ import { loadSession } from "../../src/cli-session.js"; import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js"; import { savePath, type ContentSource } from "../../src/library/content.js"; import type { Collection, EnteFile } from "../../src/model/types.js"; -import { init, toBase64 } from "../../src/crypto/index.js"; +import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js"; import { defaultCacheDirectory } from "../../src/library/index.js"; import { HEIC_WITH_EXIF } from "../exif-heic.js"; import { @@ -903,6 +904,146 @@ describe("backup", () => { // Opening the library would have made its cache directory. expect(existsSync(join(root, "cache"))).toBe(false); }); + + // A backup runs for as long as its downloads take, hours on a large + // account. It lists its files from the refresh it waits for as it starts, + // so it must not ask the server for changes again on a timer while it + // runs: each refresh is a request, and each also starts a pass that asks + // again for the ML data of every file Ente holds none for. + // + // A saved session is read back by `loadSession`, its `fetch` a fake of the + // Ente HTTP API serving one album with one photo, encrypted as the server + // serves them. The fake holds the photo's download open while the clock + // moves on by two hours, then serves it. Only `setTimeout` and + // `clearTimeout` are faked: the library's refresh timer runs on them. + it("asks the server for changes only as it starts, however long it runs", async () => { + const masterKey = new Uint8Array(32).fill(1); + const albumKey = new Uint8Array(32).fill(2); + const photoKey = new Uint8Array(32).fill(3); + saveSession(join(root, "session"), { + email: "cli@example.com", + userID: USER_ID, + token: "token", + masterKey: toBase64(masterKey), + secretKey: toBase64(new Uint8Array(32)), + publicKey: toBase64(new Uint8Array(32)), + }); + // A secretbox as the server sends one: ciphertext and nonce, base64. + const box = (plaintext: Uint8Array, key: Uint8Array) => { + const nonce = sodium.randombytes_buf( + sodium.crypto_secretbox_NONCEBYTES, + ); + return { + ciphertext: toBase64( + sodium.crypto_secretbox_easy(plaintext, nonce, key), + ), + nonce: toBase64(nonce), + }; + }; + const albumKeyBox = box(albumKey, masterKey); + const albumName = box(new TextEncoder().encode("Vacation"), albumKey); + const photoKeyBox = box(photoKey, albumKey); + const metadata = encryptBlob( + new TextEncoder().encode( + JSON.stringify({ + title: "beach.jpg", + fileType: 0, + creationTime: 1_700_000_000_000_000, + modificationTime: 1_700_000_000_000_000, + }), + ), + photoKey, + ); + const content = encryptBlob( + new TextEncoder().encode("the photo's bytes"), + photoKey, + ); + const album = { + id: 1, + owner: { id: USER_ID }, + encryptedKey: albumKeyBox.ciphertext, + keyDecryptionNonce: albumKeyBox.nonce, + encryptedName: albumName.ciphertext, + nameDecryptionNonce: albumName.nonce, + type: "album", + updationTime: 1, + }; + const photo = { + id: 100, + collectionID: 1, + ownerID: USER_ID, + encryptedKey: photoKeyBox.ciphertext, + keyDecryptionNonce: photoKeyBox.nonce, + metadata: { + encryptedData: toBase64(metadata.ciphertext), + decryptionHeader: toBase64(metadata.header), + }, + file: { decryptionHeader: toBase64(content.header) }, + // The backup fetches no thumbnail. + thumbnail: { decryptionHeader: "" }, + updationTime: 1, + }; + + // Each refresh starts with `collectionsSince`, which sends one + // `/collections/v2` request. + let refreshes = 0; + let downloadRequested!: () => void; + const requested = new Promise((resolve) => { + downloadRequested = resolve; + }); + let releaseDownload!: () => void; + const released = new Promise((resolve) => { + releaseDownload = resolve; + }); + const json = (body: unknown): Response => + new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + }); + const fetch = async (url: string | URL): Promise => { + const { hostname, pathname } = new URL(String(url)); + if (hostname === "files.ente.io") { + downloadRequested(); + await released; + return new Response(content.ciphertext); + } + if (pathname === "/collections/v2") { + refreshes++; + return json({ collections: [album] }); + } + if (pathname === "/collections/v2/diff") { + return json({ diff: [photo], hasMore: false }); + } + // Ente holds no ML data for the photo. + if (pathname === "/files/data/fetch") return json({ data: [] }); + return new Response("not found", { status: 404 }); + }; + const ctx = { + ...context(), + // The download's idle deadline is set past the two hours, or quak + // would give up the held download as stalled. + loadSession: (path: string) => + loadSession(path, { + fetch: fetch as typeof globalThis.fetch, + downloadTimeoutMs: 3 * 3_600_000, + }), + }; + + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const backup = backupCommand(ctx, join(root, "backup"), {}); + await requested; + vi.advanceTimersByTime(2 * 3_600_000); + releaseDownload(); + expect(await backup).toBe(0); + } finally { + vi.useRealTimers(); + } + + // Two refreshes, both before the download: the library opens onto the + // empty cache with one, and the backup waits for the other before it + // lists its files. + expect(refreshes).toBe(2); + }); }); describe("helper list-missing-thumbnails", () => {