quak backup no longer refreshes its library on a timer (closes #183) #190
@@ -788,7 +788,8 @@ subsequent runs, existing originals are skipped. If a download fails, the error
|
|||||||
is logged and the backup continues with the next file. The exit code is non-zero
|
is logged and the backup continues with the next file. The exit code is non-zero
|
||||||
if any files failed. `quak backup` opens its library with the thumbnail and
|
if any files failed. `quak backup` opens its library with the thumbnail and
|
||||||
originals precache off, so the only file content it fetches is the originals the
|
originals precache off, so the only file content it fetches is the originals the
|
||||||
backup stores.
|
backup stores. It also opens it with no background timer, so it asks the server
|
||||||
|
for changes only as it starts, not again while it downloads.
|
||||||
|
|
||||||
With `--verify`, or `lib.backup({ verify: true })`, a run also hashes each
|
With `--verify`, or `lib.backup({ verify: true })`, a run also hashes each
|
||||||
original already at its save path the way a download is checked (see "On-disk
|
original already at its save path the way a download is checked (see "On-disk
|
||||||
@@ -869,7 +870,7 @@ background, so an unreachable server does not block opening.
|
|||||||
| `client` | required | the account client (a `Client`, or any `LibraryClient`) |
|
| `client` | required | the account client (a `Client`, or any `LibraryClient`) |
|
||||||
| `cacheDirectory` | `quak/<userID>` under the per-user cache directory | where `metadata.json` and the content cache live |
|
| `cacheDirectory` | `quak/<userID>` under the per-user cache directory | where `metadata.json` and the content cache live |
|
||||||
| `downloadDirectory` | `photos` in the working directory | root of the save paths; an original stored there counts as cached |
|
| `downloadDirectory` | `photos` in the working directory | root of the save paths; an original stored there counts as cached |
|
||||||
| `refreshIntervalSeconds` | `3` | background refresh cadence |
|
| `refreshIntervalSeconds` | `3` | background refresh cadence; `Infinity` for no background timer |
|
||||||
| `precacheThumbnails` | `true` | prefetch every thumbnail, newest first |
|
| `precacheThumbnails` | `true` | prefetch every thumbnail, newest first |
|
||||||
| `precacheOriginals` | `true` | prefetch the favorites album and the latest-window originals |
|
| `precacheOriginals` | `true` | prefetch the favorites album and the latest-window originals |
|
||||||
| `precacheOriginalsDays` | `7` | length in days of that latest window |
|
| `precacheOriginalsDays` | `7` | length in days of that latest window |
|
||||||
|
|||||||
@@ -25,6 +25,13 @@ declares one.
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-14: `quak backup` no longer asks the server for changes every 3
|
||||||
|
seconds while it runs (issue 183). It opens its library with
|
||||||
|
`refreshIntervalSeconds: Infinity`, which the library now takes as no
|
||||||
|
background timer, so the backup refreshes only as it starts and no timed
|
||||||
|
refresh starts another ML data pass during the run. A test holds a backup's
|
||||||
|
download open while two hours pass and counts its `/collections/v2` requests.
|
||||||
|
|
||||||
- 2026-10-14: `test/cli/backup-end-to-end.test.ts` runs `quak backup` as cron
|
- 2026-10-14: `test/cli/backup-end-to-end.test.ts` runs `quak backup` as cron
|
||||||
does (issue 186): `backupCommand` and `run`, the real `loadSession` reading a
|
does (issue 186): `backupCommand` and `run`, the real `loadSession` reading a
|
||||||
session saved after logging in, and every request answered by a fake of the
|
session saved after logging in, and every request answered by a fake of the
|
||||||
|
|||||||
@@ -427,10 +427,13 @@ export const backupCommand = async (
|
|||||||
try {
|
try {
|
||||||
// The precache is off: the backup fetches what it needs, and must not
|
// The precache is off: the backup fetches what it needs, and must not
|
||||||
// also fill the cache with every thumbnail and the recent originals.
|
// also fill the cache with every thumbnail and the recent originals.
|
||||||
|
// No refresh starts on a timer: the backup lists its files once, from
|
||||||
|
// the refresh it waits for, and can run for hours.
|
||||||
const lib = await Library.open({
|
const lib = await Library.open({
|
||||||
client,
|
client,
|
||||||
downloadDirectory: dir,
|
downloadDirectory: dir,
|
||||||
cacheDirectory: ctx.cacheDir,
|
cacheDirectory: ctx.cacheDir,
|
||||||
|
refreshIntervalSeconds: Infinity,
|
||||||
precacheThumbnails: false,
|
precacheThumbnails: false,
|
||||||
precacheOriginals: false,
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -179,6 +179,9 @@ export interface LibraryOptions {
|
|||||||
// directory at open. The content cache treats an original already stored
|
// directory at open. The content cache treats an original already stored
|
||||||
// at its save path as present.
|
// at its save path as present.
|
||||||
downloadDirectory?: string;
|
downloadDirectory?: string;
|
||||||
|
// How long after one refresh ends the background timer starts the next.
|
||||||
|
// `Infinity` starts none: the library then refreshes only as it opens and
|
||||||
|
// when `fresh()` or `backup()` asks.
|
||||||
refreshIntervalSeconds?: number;
|
refreshIntervalSeconds?: number;
|
||||||
onProgress?: RefreshProgressCallback;
|
onProgress?: RefreshProgressCallback;
|
||||||
// The bounded request pools (issue #45), shared by the ML-data fetch (the
|
// The bounded request pools (issue #45), shared by the ML-data fetch (the
|
||||||
@@ -627,7 +630,8 @@ export class Library {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private scheduleNext(): void {
|
private scheduleNext(): void {
|
||||||
if (this.closed) return;
|
// `setTimeout` would turn an infinite delay into 1 ms.
|
||||||
|
if (this.closed || this.intervalMs === Infinity) return;
|
||||||
this.timer = setTimeout(() => {
|
this.timer = setTimeout(() => {
|
||||||
void this.runRefresh().then(() => this.scheduleNext());
|
void this.runRefresh().then(() => this.scheduleNext());
|
||||||
}, this.intervalMs);
|
}, this.intervalMs);
|
||||||
|
|||||||
+142
-1
@@ -25,6 +25,7 @@ import { join } from "node:path";
|
|||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { PassThrough } from "node:stream";
|
import { PassThrough } from "node:stream";
|
||||||
import * as jpegJs from "jpeg-js";
|
import * as jpegJs from "jpeg-js";
|
||||||
|
import sodium from "libsodium-wrappers-sumo";
|
||||||
import {
|
import {
|
||||||
describe,
|
describe,
|
||||||
it,
|
it,
|
||||||
@@ -55,7 +56,7 @@ import { loadSession } from "../../src/cli-session.js";
|
|||||||
import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js";
|
import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js";
|
||||||
import { savePath, type ContentSource } from "../../src/library/content.js";
|
import { savePath, type ContentSource } from "../../src/library/content.js";
|
||||||
import type { Collection, EnteFile } from "../../src/model/types.js";
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||||
import { init, toBase64 } from "../../src/crypto/index.js";
|
import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js";
|
||||||
import { defaultCacheDirectory } from "../../src/library/index.js";
|
import { defaultCacheDirectory } from "../../src/library/index.js";
|
||||||
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
||||||
import {
|
import {
|
||||||
@@ -903,6 +904,146 @@ describe("backup", () => {
|
|||||||
// Opening the library would have made its cache directory.
|
// Opening the library would have made its cache directory.
|
||||||
expect(existsSync(join(root, "cache"))).toBe(false);
|
expect(existsSync(join(root, "cache"))).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// A backup runs for as long as its downloads take, hours on a large
|
||||||
|
// account. It lists its files from the refresh it waits for as it starts,
|
||||||
|
// so it must not ask the server for changes again on a timer while it
|
||||||
|
// runs: each refresh is a request, and each also starts a pass that asks
|
||||||
|
// again for the ML data of every file Ente holds none for.
|
||||||
|
//
|
||||||
|
// A saved session is read back by `loadSession`, its `fetch` a fake of the
|
||||||
|
// Ente HTTP API serving one album with one photo, encrypted as the server
|
||||||
|
// serves them. The fake holds the photo's download open while the clock
|
||||||
|
// moves on by two hours, then serves it. Only `setTimeout` and
|
||||||
|
// `clearTimeout` are faked: the library's refresh timer runs on them.
|
||||||
|
it("asks the server for changes only as it starts, however long it runs", async () => {
|
||||||
|
const masterKey = new Uint8Array(32).fill(1);
|
||||||
|
const albumKey = new Uint8Array(32).fill(2);
|
||||||
|
const photoKey = new Uint8Array(32).fill(3);
|
||||||
|
saveSession(join(root, "session"), {
|
||||||
|
email: "cli@example.com",
|
||||||
|
userID: USER_ID,
|
||||||
|
token: "token",
|
||||||
|
masterKey: toBase64(masterKey),
|
||||||
|
secretKey: toBase64(new Uint8Array(32)),
|
||||||
|
publicKey: toBase64(new Uint8Array(32)),
|
||||||
|
});
|
||||||
|
// A secretbox as the server sends one: ciphertext and nonce, base64.
|
||||||
|
const box = (plaintext: Uint8Array, key: Uint8Array) => {
|
||||||
|
const nonce = sodium.randombytes_buf(
|
||||||
|
sodium.crypto_secretbox_NONCEBYTES,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
ciphertext: toBase64(
|
||||||
|
sodium.crypto_secretbox_easy(plaintext, nonce, key),
|
||||||
|
),
|
||||||
|
nonce: toBase64(nonce),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
const albumKeyBox = box(albumKey, masterKey);
|
||||||
|
const albumName = box(new TextEncoder().encode("Vacation"), albumKey);
|
||||||
|
const photoKeyBox = box(photoKey, albumKey);
|
||||||
|
const metadata = encryptBlob(
|
||||||
|
new TextEncoder().encode(
|
||||||
|
JSON.stringify({
|
||||||
|
title: "beach.jpg",
|
||||||
|
fileType: 0,
|
||||||
|
creationTime: 1_700_000_000_000_000,
|
||||||
|
modificationTime: 1_700_000_000_000_000,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
photoKey,
|
||||||
|
);
|
||||||
|
const content = encryptBlob(
|
||||||
|
new TextEncoder().encode("the photo's bytes"),
|
||||||
|
photoKey,
|
||||||
|
);
|
||||||
|
const album = {
|
||||||
|
id: 1,
|
||||||
|
owner: { id: USER_ID },
|
||||||
|
encryptedKey: albumKeyBox.ciphertext,
|
||||||
|
keyDecryptionNonce: albumKeyBox.nonce,
|
||||||
|
encryptedName: albumName.ciphertext,
|
||||||
|
nameDecryptionNonce: albumName.nonce,
|
||||||
|
type: "album",
|
||||||
|
updationTime: 1,
|
||||||
|
};
|
||||||
|
const photo = {
|
||||||
|
id: 100,
|
||||||
|
collectionID: 1,
|
||||||
|
ownerID: USER_ID,
|
||||||
|
encryptedKey: photoKeyBox.ciphertext,
|
||||||
|
keyDecryptionNonce: photoKeyBox.nonce,
|
||||||
|
metadata: {
|
||||||
|
encryptedData: toBase64(metadata.ciphertext),
|
||||||
|
decryptionHeader: toBase64(metadata.header),
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: toBase64(content.header) },
|
||||||
|
// The backup fetches no thumbnail.
|
||||||
|
thumbnail: { decryptionHeader: "" },
|
||||||
|
updationTime: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Each refresh starts with `collectionsSince`, which sends one
|
||||||
|
// `/collections/v2` request.
|
||||||
|
let refreshes = 0;
|
||||||
|
let downloadRequested!: () => void;
|
||||||
|
const requested = new Promise<void>((resolve) => {
|
||||||
|
downloadRequested = resolve;
|
||||||
|
});
|
||||||
|
let releaseDownload!: () => void;
|
||||||
|
const released = new Promise<void>((resolve) => {
|
||||||
|
releaseDownload = resolve;
|
||||||
|
});
|
||||||
|
const json = (body: unknown): Response =>
|
||||||
|
new Response(JSON.stringify(body), {
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
});
|
||||||
|
const fetch = async (url: string | URL): Promise<Response> => {
|
||||||
|
const { hostname, pathname } = new URL(String(url));
|
||||||
|
if (hostname === "files.ente.io") {
|
||||||
|
downloadRequested();
|
||||||
|
await released;
|
||||||
|
return new Response(content.ciphertext);
|
||||||
|
}
|
||||||
|
if (pathname === "/collections/v2") {
|
||||||
|
refreshes++;
|
||||||
|
return json({ collections: [album] });
|
||||||
|
}
|
||||||
|
if (pathname === "/collections/v2/diff") {
|
||||||
|
return json({ diff: [photo], hasMore: false });
|
||||||
|
}
|
||||||
|
// Ente holds no ML data for the photo.
|
||||||
|
if (pathname === "/files/data/fetch") return json({ data: [] });
|
||||||
|
return new Response("not found", { status: 404 });
|
||||||
|
};
|
||||||
|
const ctx = {
|
||||||
|
...context(),
|
||||||
|
// The download's idle deadline is set past the two hours, or quak
|
||||||
|
// would give up the held download as stalled.
|
||||||
|
loadSession: (path: string) =>
|
||||||
|
loadSession(path, {
|
||||||
|
fetch: fetch as typeof globalThis.fetch,
|
||||||
|
downloadTimeoutMs: 3 * 3_600_000,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||||
|
try {
|
||||||
|
const backup = backupCommand(ctx, join(root, "backup"), {});
|
||||||
|
await requested;
|
||||||
|
vi.advanceTimersByTime(2 * 3_600_000);
|
||||||
|
releaseDownload();
|
||||||
|
expect(await backup).toBe(0);
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two refreshes, both before the download: the library opens onto the
|
||||||
|
// empty cache with one, and the backup waits for the other before it
|
||||||
|
// lists its files.
|
||||||
|
expect(refreshes).toBe(2);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("helper list-missing-thumbnails", () => {
|
describe("helper list-missing-thumbnails", () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user