quak backup no longer refreshes its library on a timer (closes #183) #190
@@ -788,7 +788,8 @@ subsequent runs, existing originals are skipped. If a download fails, the error
|
||||
is logged and the backup continues with the next file. The exit code is non-zero
|
||||
if any files failed. `quak backup` opens its library with the thumbnail and
|
||||
originals precache off, so the only file content it fetches is the originals the
|
||||
backup stores.
|
||||
backup stores. It also opens it with no background timer, so it asks the server
|
||||
for changes only as it starts, not again while it downloads.
|
||||
|
||||
With `--verify`, or `lib.backup({ verify: true })`, a run also hashes each
|
||||
original already at its save path the way a download is checked (see "On-disk
|
||||
@@ -869,7 +870,7 @@ background, so an unreachable server does not block opening.
|
||||
| `client` | required | the account client (a `Client`, or any `LibraryClient`) |
|
||||
| `cacheDirectory` | `quak/<userID>` under the per-user cache directory | where `metadata.json` and the content cache live |
|
||||
| `downloadDirectory` | `photos` in the working directory | root of the save paths; an original stored there counts as cached |
|
||||
| `refreshIntervalSeconds` | `3` | background refresh cadence |
|
||||
| `refreshIntervalSeconds` | `3` | background refresh cadence; `Infinity` for no background timer |
|
||||
| `precacheThumbnails` | `true` | prefetch every thumbnail, newest first |
|
||||
| `precacheOriginals` | `true` | prefetch the favorites album and the latest-window originals |
|
||||
| `precacheOriginalsDays` | `7` | length in days of that latest window |
|
||||
|
||||
@@ -25,6 +25,13 @@ declares one.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-14: `quak backup` no longer asks the server for changes every 3
|
||||
seconds while it runs (issue 183). It opens its library with
|
||||
`refreshIntervalSeconds: Infinity`, which the library now takes as no
|
||||
background timer, so the backup refreshes only as it starts and no timed
|
||||
refresh starts another ML data pass during the run. A test holds a backup's
|
||||
download open while two hours pass and counts its `/collections/v2` requests.
|
||||
|
||||
- 2026-10-14: `test/cli/backup-end-to-end.test.ts` runs `quak backup` as cron
|
||||
does (issue 186): `backupCommand` and `run`, the real `loadSession` reading a
|
||||
session saved after logging in, and every request answered by a fake of the
|
||||
|
||||
@@ -427,10 +427,13 @@ export const backupCommand = async (
|
||||
try {
|
||||
// The precache is off: the backup fetches what it needs, and must not
|
||||
// also fill the cache with every thumbnail and the recent originals.
|
||||
// No refresh starts on a timer: the backup lists its files once, from
|
||||
// the refresh it waits for, and can run for hours.
|
||||
const lib = await Library.open({
|
||||
client,
|
||||
downloadDirectory: dir,
|
||||
cacheDirectory: ctx.cacheDir,
|
||||
refreshIntervalSeconds: Infinity,
|
||||
precacheThumbnails: false,
|
||||
precacheOriginals: false,
|
||||
});
|
||||
|
||||
@@ -179,6 +179,9 @@ export interface LibraryOptions {
|
||||
// directory at open. The content cache treats an original already stored
|
||||
// at its save path as present.
|
||||
downloadDirectory?: string;
|
||||
// How long after one refresh ends the background timer starts the next.
|
||||
// `Infinity` starts none: the library then refreshes only as it opens and
|
||||
// when `fresh()` or `backup()` asks.
|
||||
refreshIntervalSeconds?: number;
|
||||
onProgress?: RefreshProgressCallback;
|
||||
// The bounded request pools (issue #45), shared by the ML-data fetch (the
|
||||
@@ -627,7 +630,8 @@ export class Library {
|
||||
}
|
||||
|
||||
private scheduleNext(): void {
|
||||
if (this.closed) return;
|
||||
// `setTimeout` would turn an infinite delay into 1 ms.
|
||||
if (this.closed || this.intervalMs === Infinity) return;
|
||||
this.timer = setTimeout(() => {
|
||||
void this.runRefresh().then(() => this.scheduleNext());
|
||||
}, this.intervalMs);
|
||||
|
||||
+142
-1
@@ -25,6 +25,7 @@ import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { PassThrough } from "node:stream";
|
||||
import * as jpegJs from "jpeg-js";
|
||||
import sodium from "libsodium-wrappers-sumo";
|
||||
import {
|
||||
describe,
|
||||
it,
|
||||
@@ -55,7 +56,7 @@ import { loadSession } from "../../src/cli-session.js";
|
||||
import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js";
|
||||
import { savePath, type ContentSource } from "../../src/library/content.js";
|
||||
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||
import { init, toBase64 } from "../../src/crypto/index.js";
|
||||
import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js";
|
||||
import { defaultCacheDirectory } from "../../src/library/index.js";
|
||||
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
||||
import {
|
||||
@@ -903,6 +904,146 @@ describe("backup", () => {
|
||||
// Opening the library would have made its cache directory.
|
||||
expect(existsSync(join(root, "cache"))).toBe(false);
|
||||
});
|
||||
|
||||
// A backup runs for as long as its downloads take, hours on a large
|
||||
// account. It lists its files from the refresh it waits for as it starts,
|
||||
// so it must not ask the server for changes again on a timer while it
|
||||
// runs: each refresh is a request, and each also starts a pass that asks
|
||||
// again for the ML data of every file Ente holds none for.
|
||||
//
|
||||
// A saved session is read back by `loadSession`, its `fetch` a fake of the
|
||||
// Ente HTTP API serving one album with one photo, encrypted as the server
|
||||
// serves them. The fake holds the photo's download open while the clock
|
||||
// moves on by two hours, then serves it. Only `setTimeout` and
|
||||
// `clearTimeout` are faked: the library's refresh timer runs on them.
|
||||
it("asks the server for changes only as it starts, however long it runs", async () => {
|
||||
const masterKey = new Uint8Array(32).fill(1);
|
||||
const albumKey = new Uint8Array(32).fill(2);
|
||||
const photoKey = new Uint8Array(32).fill(3);
|
||||
saveSession(join(root, "session"), {
|
||||
email: "cli@example.com",
|
||||
userID: USER_ID,
|
||||
token: "token",
|
||||
masterKey: toBase64(masterKey),
|
||||
secretKey: toBase64(new Uint8Array(32)),
|
||||
publicKey: toBase64(new Uint8Array(32)),
|
||||
});
|
||||
// A secretbox as the server sends one: ciphertext and nonce, base64.
|
||||
const box = (plaintext: Uint8Array, key: Uint8Array) => {
|
||||
const nonce = sodium.randombytes_buf(
|
||||
sodium.crypto_secretbox_NONCEBYTES,
|
||||
);
|
||||
return {
|
||||
ciphertext: toBase64(
|
||||
sodium.crypto_secretbox_easy(plaintext, nonce, key),
|
||||
),
|
||||
nonce: toBase64(nonce),
|
||||
};
|
||||
};
|
||||
const albumKeyBox = box(albumKey, masterKey);
|
||||
const albumName = box(new TextEncoder().encode("Vacation"), albumKey);
|
||||
const photoKeyBox = box(photoKey, albumKey);
|
||||
const metadata = encryptBlob(
|
||||
new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
title: "beach.jpg",
|
||||
fileType: 0,
|
||||
creationTime: 1_700_000_000_000_000,
|
||||
modificationTime: 1_700_000_000_000_000,
|
||||
}),
|
||||
),
|
||||
photoKey,
|
||||
);
|
||||
const content = encryptBlob(
|
||||
new TextEncoder().encode("the photo's bytes"),
|
||||
photoKey,
|
||||
);
|
||||
const album = {
|
||||
id: 1,
|
||||
owner: { id: USER_ID },
|
||||
encryptedKey: albumKeyBox.ciphertext,
|
||||
keyDecryptionNonce: albumKeyBox.nonce,
|
||||
encryptedName: albumName.ciphertext,
|
||||
nameDecryptionNonce: albumName.nonce,
|
||||
type: "album",
|
||||
updationTime: 1,
|
||||
};
|
||||
const photo = {
|
||||
id: 100,
|
||||
collectionID: 1,
|
||||
ownerID: USER_ID,
|
||||
encryptedKey: photoKeyBox.ciphertext,
|
||||
keyDecryptionNonce: photoKeyBox.nonce,
|
||||
metadata: {
|
||||
encryptedData: toBase64(metadata.ciphertext),
|
||||
decryptionHeader: toBase64(metadata.header),
|
||||
},
|
||||
file: { decryptionHeader: toBase64(content.header) },
|
||||
// The backup fetches no thumbnail.
|
||||
thumbnail: { decryptionHeader: "" },
|
||||
updationTime: 1,
|
||||
};
|
||||
|
||||
// Each refresh starts with `collectionsSince`, which sends one
|
||||
// `/collections/v2` request.
|
||||
let refreshes = 0;
|
||||
let downloadRequested!: () => void;
|
||||
const requested = new Promise<void>((resolve) => {
|
||||
downloadRequested = resolve;
|
||||
});
|
||||
let releaseDownload!: () => void;
|
||||
const released = new Promise<void>((resolve) => {
|
||||
releaseDownload = resolve;
|
||||
});
|
||||
const json = (body: unknown): Response =>
|
||||
new Response(JSON.stringify(body), {
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
const fetch = async (url: string | URL): Promise<Response> => {
|
||||
const { hostname, pathname } = new URL(String(url));
|
||||
if (hostname === "files.ente.io") {
|
||||
downloadRequested();
|
||||
await released;
|
||||
return new Response(content.ciphertext);
|
||||
}
|
||||
if (pathname === "/collections/v2") {
|
||||
refreshes++;
|
||||
return json({ collections: [album] });
|
||||
}
|
||||
if (pathname === "/collections/v2/diff") {
|
||||
return json({ diff: [photo], hasMore: false });
|
||||
}
|
||||
// Ente holds no ML data for the photo.
|
||||
if (pathname === "/files/data/fetch") return json({ data: [] });
|
||||
return new Response("not found", { status: 404 });
|
||||
};
|
||||
const ctx = {
|
||||
...context(),
|
||||
// The download's idle deadline is set past the two hours, or quak
|
||||
// would give up the held download as stalled.
|
||||
loadSession: (path: string) =>
|
||||
loadSession(path, {
|
||||
fetch: fetch as typeof globalThis.fetch,
|
||||
downloadTimeoutMs: 3 * 3_600_000,
|
||||
}),
|
||||
};
|
||||
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
const backup = backupCommand(ctx, join(root, "backup"), {});
|
||||
await requested;
|
||||
vi.advanceTimersByTime(2 * 3_600_000);
|
||||
releaseDownload();
|
||||
expect(await backup).toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
|
||||
// Two refreshes, both before the download: the library opens onto the
|
||||
// empty cache with one, and the backup waits for the other before it
|
||||
// lists its files.
|
||||
expect(refreshes).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("helper list-missing-thumbnails", () => {
|
||||
|
||||
Reference in New Issue
Block a user