Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8bc542f1e5 | ||
|
|
c465496682 | ||
|
|
0a756be5de | ||
|
|
a0a97f1019 |
@@ -803,8 +803,7 @@ subsequent runs, existing originals are skipped. If a download fails, the error
|
|||||||
is logged and the backup continues with the next file. The exit code is non-zero
|
is logged and the backup continues with the next file. The exit code is non-zero
|
||||||
if any files failed. `quak backup` opens its library with the thumbnail and
|
if any files failed. `quak backup` opens its library with the thumbnail and
|
||||||
originals precache off, so the only file content it fetches is the originals the
|
originals precache off, so the only file content it fetches is the originals the
|
||||||
backup stores. It also opens it with no background timer, so it asks the server
|
backup stores.
|
||||||
for changes only as it starts, not again while it downloads.
|
|
||||||
|
|
||||||
With `--verify`, or `lib.backup({ verify: true })`, a run also hashes each
|
With `--verify`, or `lib.backup({ verify: true })`, a run also hashes each
|
||||||
original already at its save path the way a download is checked (see "On-disk
|
original already at its save path the way a download is checked (see "On-disk
|
||||||
@@ -885,7 +884,7 @@ background, so an unreachable server does not block opening.
|
|||||||
| `client` | required | the account client (a `Client`, or any `LibraryClient`) |
|
| `client` | required | the account client (a `Client`, or any `LibraryClient`) |
|
||||||
| `cacheDirectory` | `quak/<userID>` under the per-user cache directory | where `metadata.json` and the content cache live |
|
| `cacheDirectory` | `quak/<userID>` under the per-user cache directory | where `metadata.json` and the content cache live |
|
||||||
| `downloadDirectory` | `photos` in the working directory | root of the save paths; an original stored there counts as cached |
|
| `downloadDirectory` | `photos` in the working directory | root of the save paths; an original stored there counts as cached |
|
||||||
| `refreshIntervalSeconds` | `3` | background refresh cadence; `Infinity` for no background timer |
|
| `refreshIntervalSeconds` | `3` | background refresh cadence |
|
||||||
| `precacheThumbnails` | `true` | prefetch every thumbnail, newest first |
|
| `precacheThumbnails` | `true` | prefetch every thumbnail, newest first |
|
||||||
| `precacheOriginals` | `true` | prefetch the favorites album and the latest-window originals |
|
| `precacheOriginals` | `true` | prefetch the favorites album and the latest-window originals |
|
||||||
| `precacheOriginalsDays` | `7` | length in days of that latest window |
|
| `precacheOriginalsDays` | `7` | length in days of that latest window |
|
||||||
|
|||||||
@@ -36,13 +36,6 @@ declares one.
|
|||||||
remain. A file whose copy in another album decrypts is backed up as usual. The
|
remain. A file whose copy in another album decrypts is backed up as usual. The
|
||||||
read commands answer from what decrypted.
|
read commands answer from what decrypted.
|
||||||
|
|
||||||
- 2026-10-14: `quak backup` no longer asks the server for changes every 3
|
|
||||||
seconds while it runs (issue 183). It opens its library with
|
|
||||||
`refreshIntervalSeconds: Infinity`, which the library now takes as no
|
|
||||||
background timer, so the backup refreshes only as it starts and no timed
|
|
||||||
refresh starts another ML data pass during the run. A test holds a backup's
|
|
||||||
download open while two hours pass and counts its `/collections/v2` requests.
|
|
||||||
|
|
||||||
- 2026-10-14: `test/cli/backup-end-to-end.test.ts` runs `quak backup` as cron
|
- 2026-10-14: `test/cli/backup-end-to-end.test.ts` runs `quak backup` as cron
|
||||||
does (issue 186): `backupCommand` and `run`, the real `loadSession` reading a
|
does (issue 186): `backupCommand` and `run`, the real `loadSession` reading a
|
||||||
session saved after logging in, and every request answered by a fake of the
|
session saved after logging in, and every request answered by a fake of the
|
||||||
|
|||||||
@@ -427,13 +427,10 @@ export const backupCommand = async (
|
|||||||
try {
|
try {
|
||||||
// The precache is off: the backup fetches what it needs, and must not
|
// The precache is off: the backup fetches what it needs, and must not
|
||||||
// also fill the cache with every thumbnail and the recent originals.
|
// also fill the cache with every thumbnail and the recent originals.
|
||||||
// No refresh starts on a timer: the backup lists its files once, from
|
|
||||||
// the refresh it waits for, and can run for hours.
|
|
||||||
const lib = await Library.open({
|
const lib = await Library.open({
|
||||||
client,
|
client,
|
||||||
downloadDirectory: dir,
|
downloadDirectory: dir,
|
||||||
cacheDirectory: ctx.cacheDir,
|
cacheDirectory: ctx.cacheDir,
|
||||||
refreshIntervalSeconds: Infinity,
|
|
||||||
precacheThumbnails: false,
|
precacheThumbnails: false,
|
||||||
precacheOriginals: false,
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -179,9 +179,6 @@ export interface LibraryOptions {
|
|||||||
// directory at open. The content cache treats an original already stored
|
// directory at open. The content cache treats an original already stored
|
||||||
// at its save path as present.
|
// at its save path as present.
|
||||||
downloadDirectory?: string;
|
downloadDirectory?: string;
|
||||||
// How long after one refresh ends the background timer starts the next.
|
|
||||||
// `Infinity` starts none: the library then refreshes only as it opens and
|
|
||||||
// when `fresh()` or `backup()` asks.
|
|
||||||
refreshIntervalSeconds?: number;
|
refreshIntervalSeconds?: number;
|
||||||
onProgress?: RefreshProgressCallback;
|
onProgress?: RefreshProgressCallback;
|
||||||
// The bounded request pools (issue #45), shared by the ML-data fetch (the
|
// The bounded request pools (issue #45), shared by the ML-data fetch (the
|
||||||
@@ -631,8 +628,7 @@ export class Library {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private scheduleNext(): void {
|
private scheduleNext(): void {
|
||||||
// `setTimeout` would turn an infinite delay into 1 ms.
|
if (this.closed) return;
|
||||||
if (this.closed || this.intervalMs === Infinity) return;
|
|
||||||
this.timer = setTimeout(() => {
|
this.timer = setTimeout(() => {
|
||||||
void this.runRefresh().then(() => this.scheduleNext());
|
void this.runRefresh().then(() => this.scheduleNext());
|
||||||
}, this.intervalMs);
|
}, this.intervalMs);
|
||||||
|
|||||||
@@ -23,13 +23,7 @@
|
|||||||
* key replaced by bytes no key opens, or its metadata text that is not JSON.
|
* key replaced by bytes no key opens, or its metadata text that is not JSON.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||||
existsSync,
|
|
||||||
mkdtempSync,
|
|
||||||
readdirSync,
|
|
||||||
readFileSync,
|
|
||||||
rmSync,
|
|
||||||
} from "node:fs";
|
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import sodium from "libsodium-wrappers-sumo";
|
import sodium from "libsodium-wrappers-sumo";
|
||||||
@@ -259,18 +253,6 @@ const brokenAccount = (): FakeEnte => {
|
|||||||
return ente;
|
return ente;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Album 1, "Trip", holds file 10, and album 2, "Family", holds file 20. Both
|
|
||||||
// files decrypt, but album 2's own key does not, so the album is the only
|
|
||||||
// record left out. All of it as of time 1.
|
|
||||||
const brokenAlbumAccount = (): FakeEnte => {
|
|
||||||
const ente = new FakeEnte();
|
|
||||||
ente.putAlbum(1, "Trip", 1);
|
|
||||||
ente.putFile(1, 10, 1);
|
|
||||||
ente.putAlbum(2, "Family", 1, true);
|
|
||||||
ente.putFile(2, 20, 1);
|
|
||||||
return ente;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Collects everything written to it.
|
// Collects everything written to it.
|
||||||
class Output {
|
class Output {
|
||||||
text = "";
|
text = "";
|
||||||
@@ -339,12 +321,6 @@ const failures = (): Record<string, { error: string; attempts: number }> => {
|
|||||||
const stored = (id: number): boolean =>
|
const stored = (id: number): boolean =>
|
||||||
existsSync(join(backupDir(), DAY, `2026-03-01.${id}.jpg`));
|
existsSync(join(backupDir(), DAY, `2026-03-01.${id}.jpg`));
|
||||||
|
|
||||||
// The IDs of the files that album `name`'s JSON lists.
|
|
||||||
const albumFiles = (name: string): number[] =>
|
|
||||||
JSON.parse(
|
|
||||||
readFileSync(join(backupDir(), "collections", `${name}.json`), "utf-8"),
|
|
||||||
).files.map((f: { id: number }) => f.id);
|
|
||||||
|
|
||||||
describe("quak backup with records it cannot decrypt", () => {
|
describe("quak backup with records it cannot decrypt", () => {
|
||||||
it("backs up every other file, records each file it cannot decrypt in failures.json, names each such album, and exits 1", async () => {
|
it("backs up every other file, records each file it cannot decrypt in failures.json, names each such album, and exits 1", async () => {
|
||||||
expect(await backup(brokenAccount())).toBe(1);
|
expect(await backup(brokenAccount())).toBe(1);
|
||||||
@@ -422,29 +398,6 @@ describe("quak backup with records it cannot decrypt", () => {
|
|||||||
expect(stderr.text).not.toContain("FAILED");
|
expect(stderr.text).not.toContain("FAILED");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("exits 1 when the only record it cannot decrypt is an album, and 0 once the album decrypts", async () => {
|
|
||||||
const ente = brokenAlbumAccount();
|
|
||||||
|
|
||||||
// No file is in `failures.json`: the album alone keeps the exit code
|
|
||||||
// at 1.
|
|
||||||
expect(await backup(ente)).toBe(1);
|
|
||||||
expect(failures()).toEqual({});
|
|
||||||
|
|
||||||
ente.putAlbum(2, "Family", 2);
|
|
||||||
expect(await backup(ente)).toBe(0);
|
|
||||||
expect(stored(20)).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("exits 1 when the only record it cannot decrypt is an album, and 0 once the album is deleted", async () => {
|
|
||||||
const ente = brokenAlbumAccount();
|
|
||||||
|
|
||||||
expect(await backup(ente)).toBe(1);
|
|
||||||
expect(failures()).toEqual({});
|
|
||||||
|
|
||||||
ente.deleteAlbum(2, 2);
|
|
||||||
expect(await backup(ente)).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("backs up a file that does not decrypt in one album but does in another, and does not report it", async () => {
|
it("backs up a file that does not decrypt in one album but does in another, and does not report it", async () => {
|
||||||
// File 30 is in albums 1 and 3, each holding its key sealed under the
|
// File 30 is in albums 1 and 3, each holding its key sealed under the
|
||||||
// album's own key. In album 1 that is corrupt; in album 3 it is not,
|
// album's own key. In album 1 that is corrupt; in album 3 it is not,
|
||||||
@@ -459,13 +412,6 @@ describe("quak backup with records it cannot decrypt", () => {
|
|||||||
|
|
||||||
expect(stored(30)).toBe(true);
|
expect(stored(30)).toBe(true);
|
||||||
expect(failures()).toEqual({});
|
expect(failures()).toEqual({});
|
||||||
// Album 1 has no copy of file 30 that decrypts, so it neither links
|
|
||||||
// the original nor lists the file; album 3 lists it.
|
|
||||||
expect(readdirSync(join(backupDir(), "collections", "Trip"))).toEqual(
|
|
||||||
[],
|
|
||||||
);
|
|
||||||
expect(albumFiles("Trip")).toEqual([]);
|
|
||||||
expect(albumFiles("Favorites")).toEqual([30]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("backs up that file in a run limited to the album where it does not decrypt", async () => {
|
it("backs up that file in a run limited to the album where it does not decrypt", async () => {
|
||||||
|
|||||||
+1
-142
@@ -25,7 +25,6 @@ import { join } from "node:path";
|
|||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { PassThrough } from "node:stream";
|
import { PassThrough } from "node:stream";
|
||||||
import * as jpegJs from "jpeg-js";
|
import * as jpegJs from "jpeg-js";
|
||||||
import sodium from "libsodium-wrappers-sumo";
|
|
||||||
import {
|
import {
|
||||||
describe,
|
describe,
|
||||||
it,
|
it,
|
||||||
@@ -56,7 +55,7 @@ import { loadSession } from "../../src/cli-session.js";
|
|||||||
import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js";
|
import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js";
|
||||||
import { savePath, type ContentSource } from "../../src/library/content.js";
|
import { savePath, type ContentSource } from "../../src/library/content.js";
|
||||||
import type { Collection, EnteFile } from "../../src/model/types.js";
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||||
import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js";
|
import { init, toBase64 } from "../../src/crypto/index.js";
|
||||||
import { defaultCacheDirectory } from "../../src/library/index.js";
|
import { defaultCacheDirectory } from "../../src/library/index.js";
|
||||||
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
||||||
import {
|
import {
|
||||||
@@ -904,146 +903,6 @@ describe("backup", () => {
|
|||||||
// Opening the library would have made its cache directory.
|
// Opening the library would have made its cache directory.
|
||||||
expect(existsSync(join(root, "cache"))).toBe(false);
|
expect(existsSync(join(root, "cache"))).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
// A backup runs for as long as its downloads take, hours on a large
|
|
||||||
// account. It lists its files from the refresh it waits for as it starts,
|
|
||||||
// so it must not ask the server for changes again on a timer while it
|
|
||||||
// runs: each refresh is a request, and each also starts a pass that asks
|
|
||||||
// again for the ML data of every file Ente holds none for.
|
|
||||||
//
|
|
||||||
// A saved session is read back by `loadSession`, its `fetch` a fake of the
|
|
||||||
// Ente HTTP API serving one album with one photo, encrypted as the server
|
|
||||||
// serves them. The fake holds the photo's download open while the clock
|
|
||||||
// moves on by two hours, then serves it. Only `setTimeout` and
|
|
||||||
// `clearTimeout` are faked: the library's refresh timer runs on them.
|
|
||||||
it("asks the server for changes only as it starts, however long it runs", async () => {
|
|
||||||
const masterKey = new Uint8Array(32).fill(1);
|
|
||||||
const albumKey = new Uint8Array(32).fill(2);
|
|
||||||
const photoKey = new Uint8Array(32).fill(3);
|
|
||||||
saveSession(join(root, "session"), {
|
|
||||||
email: "cli@example.com",
|
|
||||||
userID: USER_ID,
|
|
||||||
token: "token",
|
|
||||||
masterKey: toBase64(masterKey),
|
|
||||||
secretKey: toBase64(new Uint8Array(32)),
|
|
||||||
publicKey: toBase64(new Uint8Array(32)),
|
|
||||||
});
|
|
||||||
// A secretbox as the server sends one: ciphertext and nonce, base64.
|
|
||||||
const box = (plaintext: Uint8Array, key: Uint8Array) => {
|
|
||||||
const nonce = sodium.randombytes_buf(
|
|
||||||
sodium.crypto_secretbox_NONCEBYTES,
|
|
||||||
);
|
|
||||||
return {
|
|
||||||
ciphertext: toBase64(
|
|
||||||
sodium.crypto_secretbox_easy(plaintext, nonce, key),
|
|
||||||
),
|
|
||||||
nonce: toBase64(nonce),
|
|
||||||
};
|
|
||||||
};
|
|
||||||
const albumKeyBox = box(albumKey, masterKey);
|
|
||||||
const albumName = box(new TextEncoder().encode("Vacation"), albumKey);
|
|
||||||
const photoKeyBox = box(photoKey, albumKey);
|
|
||||||
const metadata = encryptBlob(
|
|
||||||
new TextEncoder().encode(
|
|
||||||
JSON.stringify({
|
|
||||||
title: "beach.jpg",
|
|
||||||
fileType: 0,
|
|
||||||
creationTime: 1_700_000_000_000_000,
|
|
||||||
modificationTime: 1_700_000_000_000_000,
|
|
||||||
}),
|
|
||||||
),
|
|
||||||
photoKey,
|
|
||||||
);
|
|
||||||
const content = encryptBlob(
|
|
||||||
new TextEncoder().encode("the photo's bytes"),
|
|
||||||
photoKey,
|
|
||||||
);
|
|
||||||
const album = {
|
|
||||||
id: 1,
|
|
||||||
owner: { id: USER_ID },
|
|
||||||
encryptedKey: albumKeyBox.ciphertext,
|
|
||||||
keyDecryptionNonce: albumKeyBox.nonce,
|
|
||||||
encryptedName: albumName.ciphertext,
|
|
||||||
nameDecryptionNonce: albumName.nonce,
|
|
||||||
type: "album",
|
|
||||||
updationTime: 1,
|
|
||||||
};
|
|
||||||
const photo = {
|
|
||||||
id: 100,
|
|
||||||
collectionID: 1,
|
|
||||||
ownerID: USER_ID,
|
|
||||||
encryptedKey: photoKeyBox.ciphertext,
|
|
||||||
keyDecryptionNonce: photoKeyBox.nonce,
|
|
||||||
metadata: {
|
|
||||||
encryptedData: toBase64(metadata.ciphertext),
|
|
||||||
decryptionHeader: toBase64(metadata.header),
|
|
||||||
},
|
|
||||||
file: { decryptionHeader: toBase64(content.header) },
|
|
||||||
// The backup fetches no thumbnail.
|
|
||||||
thumbnail: { decryptionHeader: "" },
|
|
||||||
updationTime: 1,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Each refresh starts with `collectionsSince`, which sends one
|
|
||||||
// `/collections/v2` request.
|
|
||||||
let refreshes = 0;
|
|
||||||
let downloadRequested!: () => void;
|
|
||||||
const requested = new Promise<void>((resolve) => {
|
|
||||||
downloadRequested = resolve;
|
|
||||||
});
|
|
||||||
let releaseDownload!: () => void;
|
|
||||||
const released = new Promise<void>((resolve) => {
|
|
||||||
releaseDownload = resolve;
|
|
||||||
});
|
|
||||||
const json = (body: unknown): Response =>
|
|
||||||
new Response(JSON.stringify(body), {
|
|
||||||
headers: { "content-type": "application/json" },
|
|
||||||
});
|
|
||||||
const fetch = async (url: string | URL): Promise<Response> => {
|
|
||||||
const { hostname, pathname } = new URL(String(url));
|
|
||||||
if (hostname === "files.ente.io") {
|
|
||||||
downloadRequested();
|
|
||||||
await released;
|
|
||||||
return new Response(content.ciphertext);
|
|
||||||
}
|
|
||||||
if (pathname === "/collections/v2") {
|
|
||||||
refreshes++;
|
|
||||||
return json({ collections: [album] });
|
|
||||||
}
|
|
||||||
if (pathname === "/collections/v2/diff") {
|
|
||||||
return json({ diff: [photo], hasMore: false });
|
|
||||||
}
|
|
||||||
// Ente holds no ML data for the photo.
|
|
||||||
if (pathname === "/files/data/fetch") return json({ data: [] });
|
|
||||||
return new Response("not found", { status: 404 });
|
|
||||||
};
|
|
||||||
const ctx = {
|
|
||||||
...context(),
|
|
||||||
// The download's idle deadline is set past the two hours, or quak
|
|
||||||
// would give up the held download as stalled.
|
|
||||||
loadSession: (path: string) =>
|
|
||||||
loadSession(path, {
|
|
||||||
fetch: fetch as typeof globalThis.fetch,
|
|
||||||
downloadTimeoutMs: 3 * 3_600_000,
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
|
|
||||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
|
||||||
try {
|
|
||||||
const backup = backupCommand(ctx, join(root, "backup"), {});
|
|
||||||
await requested;
|
|
||||||
vi.advanceTimersByTime(2 * 3_600_000);
|
|
||||||
releaseDownload();
|
|
||||||
expect(await backup).toBe(0);
|
|
||||||
} finally {
|
|
||||||
vi.useRealTimers();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Two refreshes, both before the download: the library opens onto the
|
|
||||||
// empty cache with one, and the backup waits for the other before it
|
|
||||||
// lists its files.
|
|
||||||
expect(refreshes).toBe(2);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("helper list-missing-thumbnails", () => {
|
describe("helper list-missing-thumbnails", () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user