From 288825b11ae4b14453e130740d0ba250a07e3952 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 14 Oct 2026 04:10:14 +0200 Subject: [PATCH] quak backup no longer refreshes its library on a timer (closes #183) The backup opened its library with the default 3-second refresh interval, so for its whole run it asked the server for changes every few seconds, and each refresh started another ML data pass. It now opens it with refreshIntervalSeconds: Infinity. The library takes Infinity as no background timer, since setTimeout would turn it into 1 ms. A refresh interval of 3600, as the read commands use, would still refresh once an hour in a backup that runs for hours. Model: opus-5-5 --- README.md | 5 +- TODO.md | 7 ++ src/cli-commands.ts | 3 + src/library/index.ts | 6 +- test/cli/commands.test.ts | 143 +++++++++++++++++++++++++++++++++++++- 5 files changed, 160 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fe40038..5654819 100644 --- a/README.md +++ b/README.md @@ -788,7 +788,8 @@ subsequent runs, existing originals are skipped. If a download fails, the error is logged and the backup continues with the next file. The exit code is non-zero if any files failed. `quak backup` opens its library with the thumbnail and originals precache off, so the only file content it fetches is the originals the -backup stores. +backup stores. It also opens it with no background timer, so it asks the server +for changes only as it starts, not again while it downloads. With `--verify`, or `lib.backup({ verify: true })`, a run also hashes each original already at its save path the way a download is checked (see "On-disk @@ -869,7 +870,7 @@ background, so an unreachable server does not block opening. | `client` | required | the account client (a `Client`, or any `LibraryClient`) | | `cacheDirectory` | `quak/` under the per-user cache directory | where `metadata.json` and the content cache live | | `downloadDirectory` | `photos` in the working directory | root of the save paths; an original stored there counts as cached | -| `refreshIntervalSeconds` | `3` | background refresh cadence | +| `refreshIntervalSeconds` | `3` | background refresh cadence; `Infinity` for no background timer | | `precacheThumbnails` | `true` | prefetch every thumbnail, newest first | | `precacheOriginals` | `true` | prefetch the favorites album and the latest-window originals | | `precacheOriginalsDays` | `7` | length in days of that latest window | diff --git a/TODO.md b/TODO.md index 87de288..25a6bfb 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,13 @@ declares one. # Completed Steps +- 2026-10-14: `quak backup` no longer asks the server for changes every 3 + seconds while it runs (issue 183). It opens its library with + `refreshIntervalSeconds: Infinity`, which the library now takes as no + background timer, so the backup refreshes only as it starts and no timed + refresh starts another ML data pass during the run. A test holds a backup's + download open while two hours pass and counts its `/collections/v2` requests. + - 2026-10-14: `test/cli/backup-end-to-end.test.ts` runs `quak backup` as cron does (issue 186): `backupCommand` and `run`, the real `loadSession` reading a session saved after logging in, and every request answered by a fake of the diff --git a/src/cli-commands.ts b/src/cli-commands.ts index 5f8b0b6..154c180 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -427,10 +427,13 @@ export const backupCommand = async ( try { // The precache is off: the backup fetches what it needs, and must not // also fill the cache with every thumbnail and the recent originals. + // No refresh starts on a timer: the backup lists its files once, from + // the refresh it waits for, and can run for hours. const lib = await Library.open({ client, downloadDirectory: dir, cacheDirectory: ctx.cacheDir, + refreshIntervalSeconds: Infinity, precacheThumbnails: false, precacheOriginals: false, }); diff --git a/src/library/index.ts b/src/library/index.ts index 489dd88..080020b 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -179,6 +179,9 @@ export interface LibraryOptions { // directory at open. The content cache treats an original already stored // at its save path as present. downloadDirectory?: string; + // How long after one refresh ends the background timer starts the next. + // `Infinity` starts none: the library then refreshes only as it opens and + // when `fresh()` or `backup()` asks. refreshIntervalSeconds?: number; onProgress?: RefreshProgressCallback; // The bounded request pools (issue #45), shared by the ML-data fetch (the @@ -627,7 +630,8 @@ export class Library { } private scheduleNext(): void { - if (this.closed) return; + // `setTimeout` would turn an infinite delay into 1 ms. + if (this.closed || this.intervalMs === Infinity) return; this.timer = setTimeout(() => { void this.runRefresh().then(() => this.scheduleNext()); }, this.intervalMs); diff --git a/test/cli/commands.test.ts b/test/cli/commands.test.ts index 09189a5..241e259 100644 --- a/test/cli/commands.test.ts +++ b/test/cli/commands.test.ts @@ -25,6 +25,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { PassThrough } from "node:stream"; import * as jpegJs from "jpeg-js"; +import sodium from "libsodium-wrappers-sumo"; import { describe, it, @@ -55,7 +56,7 @@ import { loadSession } from "../../src/cli-session.js"; import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js"; import { savePath, type ContentSource } from "../../src/library/content.js"; import type { Collection, EnteFile } from "../../src/model/types.js"; -import { init, toBase64 } from "../../src/crypto/index.js"; +import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js"; import { defaultCacheDirectory } from "../../src/library/index.js"; import { HEIC_WITH_EXIF } from "../exif-heic.js"; import { @@ -903,6 +904,146 @@ describe("backup", () => { // Opening the library would have made its cache directory. expect(existsSync(join(root, "cache"))).toBe(false); }); + + // A backup runs for as long as its downloads take, hours on a large + // account. It lists its files from the refresh it waits for as it starts, + // so it must not ask the server for changes again on a timer while it + // runs: each refresh is a request, and each also starts a pass that asks + // again for the ML data of every file Ente holds none for. + // + // A saved session is read back by `loadSession`, its `fetch` a fake of the + // Ente HTTP API serving one album with one photo, encrypted as the server + // serves them. The fake holds the photo's download open while the clock + // moves on by two hours, then serves it. Only `setTimeout` and + // `clearTimeout` are faked: the library's refresh timer runs on them. + it("asks the server for changes only as it starts, however long it runs", async () => { + const masterKey = new Uint8Array(32).fill(1); + const albumKey = new Uint8Array(32).fill(2); + const photoKey = new Uint8Array(32).fill(3); + saveSession(join(root, "session"), { + email: "cli@example.com", + userID: USER_ID, + token: "token", + masterKey: toBase64(masterKey), + secretKey: toBase64(new Uint8Array(32)), + publicKey: toBase64(new Uint8Array(32)), + }); + // A secretbox as the server sends one: ciphertext and nonce, base64. + const box = (plaintext: Uint8Array, key: Uint8Array) => { + const nonce = sodium.randombytes_buf( + sodium.crypto_secretbox_NONCEBYTES, + ); + return { + ciphertext: toBase64( + sodium.crypto_secretbox_easy(plaintext, nonce, key), + ), + nonce: toBase64(nonce), + }; + }; + const albumKeyBox = box(albumKey, masterKey); + const albumName = box(new TextEncoder().encode("Vacation"), albumKey); + const photoKeyBox = box(photoKey, albumKey); + const metadata = encryptBlob( + new TextEncoder().encode( + JSON.stringify({ + title: "beach.jpg", + fileType: 0, + creationTime: 1_700_000_000_000_000, + modificationTime: 1_700_000_000_000_000, + }), + ), + photoKey, + ); + const content = encryptBlob( + new TextEncoder().encode("the photo's bytes"), + photoKey, + ); + const album = { + id: 1, + owner: { id: USER_ID }, + encryptedKey: albumKeyBox.ciphertext, + keyDecryptionNonce: albumKeyBox.nonce, + encryptedName: albumName.ciphertext, + nameDecryptionNonce: albumName.nonce, + type: "album", + updationTime: 1, + }; + const photo = { + id: 100, + collectionID: 1, + ownerID: USER_ID, + encryptedKey: photoKeyBox.ciphertext, + keyDecryptionNonce: photoKeyBox.nonce, + metadata: { + encryptedData: toBase64(metadata.ciphertext), + decryptionHeader: toBase64(metadata.header), + }, + file: { decryptionHeader: toBase64(content.header) }, + // The backup fetches no thumbnail. + thumbnail: { decryptionHeader: "" }, + updationTime: 1, + }; + + // Each refresh starts with `collectionsSince`, which sends one + // `/collections/v2` request. + let refreshes = 0; + let downloadRequested!: () => void; + const requested = new Promise((resolve) => { + downloadRequested = resolve; + }); + let releaseDownload!: () => void; + const released = new Promise((resolve) => { + releaseDownload = resolve; + }); + const json = (body: unknown): Response => + new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + }); + const fetch = async (url: string | URL): Promise => { + const { hostname, pathname } = new URL(String(url)); + if (hostname === "files.ente.io") { + downloadRequested(); + await released; + return new Response(content.ciphertext); + } + if (pathname === "/collections/v2") { + refreshes++; + return json({ collections: [album] }); + } + if (pathname === "/collections/v2/diff") { + return json({ diff: [photo], hasMore: false }); + } + // Ente holds no ML data for the photo. + if (pathname === "/files/data/fetch") return json({ data: [] }); + return new Response("not found", { status: 404 }); + }; + const ctx = { + ...context(), + // The download's idle deadline is set past the two hours, or quak + // would give up the held download as stalled. + loadSession: (path: string) => + loadSession(path, { + fetch: fetch as typeof globalThis.fetch, + downloadTimeoutMs: 3 * 3_600_000, + }), + }; + + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const backup = backupCommand(ctx, join(root, "backup"), {}); + await requested; + vi.advanceTimersByTime(2 * 3_600_000); + releaseDownload(); + expect(await backup).toBe(0); + } finally { + vi.useRealTimers(); + } + + // Two refreshes, both before the download: the library opens onto the + // empty cache with one, and the backup waits for the other before it + // lists its files. + expect(refreshes).toBe(2); + }); }); describe("helper list-missing-thumbnails", () => {