sneak
cfb3d31047
A run of lib.backup() limited to "Trip" must store a file whose copy in Trip does not decrypt when its copy in an album the run skips does. Today the run neither stores it nor records it as failed. Model: opus-5-5
463 lines
18 KiB
TypeScript
463 lines
18 KiB
TypeScript
/**
|
|
* `quak backup` over an account holding an album or a file quak cannot decrypt
|
|
* (issue 181).
|
|
*
|
|
* The server sends every album and file encrypted, and a refresh decrypts each
|
|
* one it fetches. A record whose key no key of the account opens, or whose
|
|
* metadata is not JSON, must not fail the refresh: a refresh that fails commits
|
|
* nothing, so `quak backup` would store nothing, on every run, until the
|
|
* server's copy changed. Instead the refresh leaves that record out, and the
|
|
* library keeps a list of what it left out in its cache. It has to: the next
|
|
* refresh asks the server only for what changed since the last one, so it will
|
|
* not see the record again.
|
|
*
|
|
* The backup stores every other file. It records each file it could not
|
|
* decrypt in `failures.json`, prints one line for each album it could not
|
|
* decrypt, and exits 1 while any remain. Once the server sends a copy that
|
|
* decrypts, or deletes the record, the next run drops it and can exit 0. The
|
|
* read commands answer from what did decrypt.
|
|
*
|
|
* These tests run the commands with a real saved session over a stand-in Ente
|
|
* server, which serves albums, files and originals encrypted with libsodium as
|
|
* the real server does. A record is broken the way a corrupt one arrives: its
|
|
* key replaced by bytes no key opens, or its metadata text that is not JSON.
|
|
*/
|
|
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import sodium from "libsodium-wrappers-sumo";
|
|
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
|
|
|
import {
|
|
backupCommand,
|
|
collectionsCommand,
|
|
filesCommand,
|
|
saveSession,
|
|
type CliContext,
|
|
} from "../../src/cli-commands.js";
|
|
import { loadSession } from "../../src/cli-session.js";
|
|
import { Client, type ClientSnapshot } from "../../src/client.js";
|
|
import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js";
|
|
import { Library } from "../../src/library/index.js";
|
|
|
|
const USER_ID = 42;
|
|
|
|
// Every file is taken at noon local time on 2026-03-01, in microseconds as Ente
|
|
// stores times, so its original is saved in the folder `DAY` whatever the time
|
|
// zone of the machine running the tests.
|
|
const TAKEN = new Date(2026, 2, 1, 12).getTime() * 1000;
|
|
const DAY = join("2026", "2026-03", "2026-03-01");
|
|
|
|
type Row = { id: number; updationTime: number } & Record<string, unknown>;
|
|
|
|
// `plaintext` sealed under `key` with a fresh nonce, both base64, as Ente
|
|
// sends a key or an album name.
|
|
const secretbox = (
|
|
plaintext: Uint8Array,
|
|
key: Uint8Array,
|
|
): { ciphertext: string; nonce: string } => {
|
|
const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
return {
|
|
ciphertext: toBase64(
|
|
sodium.crypto_secretbox_easy(plaintext, nonce, key),
|
|
),
|
|
nonce: toBase64(nonce),
|
|
};
|
|
};
|
|
|
|
// What a corrupt row carries in place of its sealed key: random bytes of the
|
|
// same length, which no key opens.
|
|
const corruptKey = (): string =>
|
|
toBase64(
|
|
sodium.randombytes_buf(
|
|
sodium.crypto_secretbox_KEYBYTES + sodium.crypto_secretbox_MACBYTES,
|
|
),
|
|
);
|
|
|
|
const json = (body: unknown): Response =>
|
|
new Response(JSON.stringify(body), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
});
|
|
|
|
// A stand-in Ente server for one account. It holds each album, and each file in
|
|
// an album, as the row `/collections/v2` and `/collections/v2/diff` serve, and
|
|
// answers a request with the rows whose `updationTime` is past the request's
|
|
// `sinceTime`, as the real server does. A change to a file also moves its
|
|
// album's `updationTime`, which is what tells a client to list the album's
|
|
// files again. Originals are served by file ID, as the files CDN serves them.
|
|
class FakeEnte {
|
|
private readonly masterKey = sodium.crypto_secretbox_keygen();
|
|
private readonly keyPair = sodium.crypto_box_keypair();
|
|
private readonly albums = new Map<number, Row>();
|
|
private readonly albumKeys = new Map<number, Uint8Array>();
|
|
// Each album's file rows, by album ID and then file ID.
|
|
private readonly files = new Map<number, Map<number, Row>>();
|
|
// Each file's key and its original as served, by file ID. A file in two
|
|
// albums is one original under one key, sealed under each album's key.
|
|
private readonly originals = new Map<
|
|
number,
|
|
{ key: Uint8Array; header: string; body: Uint8Array }
|
|
>();
|
|
|
|
// The session `quak login` would have saved for the account.
|
|
snapshot(): ClientSnapshot {
|
|
return {
|
|
email: "user@example.com",
|
|
userID: USER_ID,
|
|
token: "token",
|
|
masterKey: toBase64(this.masterKey),
|
|
secretKey: toBase64(this.keyPair.privateKey),
|
|
publicKey: toBase64(this.keyPair.publicKey),
|
|
};
|
|
}
|
|
|
|
// Album `id`, named `name`, as of time `t`. With `corrupt`, its key cannot
|
|
// be decrypted. An album keeps its key from one call to the next, so its
|
|
// files' rows still decrypt once it is put back without `corrupt`.
|
|
putAlbum(id: number, name: string, t: number, corrupt = false): void {
|
|
const key = this.albumKeys.get(id) ?? sodium.crypto_secretbox_keygen();
|
|
this.albumKeys.set(id, key);
|
|
const sealedKey = secretbox(key, this.masterKey);
|
|
const sealedName = secretbox(new TextEncoder().encode(name), key);
|
|
this.albums.set(id, {
|
|
id,
|
|
owner: { id: USER_ID },
|
|
encryptedKey: corrupt ? corruptKey() : sealedKey.ciphertext,
|
|
keyDecryptionNonce: sealedKey.nonce,
|
|
encryptedName: sealedName.ciphertext,
|
|
nameDecryptionNonce: sealedName.nonce,
|
|
type: "album",
|
|
updationTime: t,
|
|
});
|
|
if (!this.files.has(id)) this.files.set(id, new Map());
|
|
}
|
|
|
|
// File `id`, titled `photo-<id>.jpg`, in album `albumID` as of time `t`.
|
|
// With `corruptKey`, its key cannot be decrypted. With `metadata`, its
|
|
// metadata decrypts to that text instead of the JSON a client uploads.
|
|
putFile(
|
|
albumID: number,
|
|
id: number,
|
|
t: number,
|
|
opts: { corruptKey?: boolean; metadata?: string } = {},
|
|
): void {
|
|
const original = this.originalOf(id);
|
|
const sealedKey = secretbox(original.key, this.albumKeys.get(albumID)!);
|
|
const metadata =
|
|
opts.metadata ??
|
|
JSON.stringify({
|
|
title: `photo-${id}.jpg`,
|
|
fileType: 0,
|
|
creationTime: TAKEN,
|
|
modificationTime: TAKEN,
|
|
});
|
|
const sealedMetadata = encryptBlob(
|
|
new TextEncoder().encode(metadata),
|
|
original.key,
|
|
);
|
|
this.files.get(albumID)!.set(id, {
|
|
id,
|
|
collectionID: albumID,
|
|
ownerID: USER_ID,
|
|
encryptedKey: opts.corruptKey ? corruptKey() : sealedKey.ciphertext,
|
|
keyDecryptionNonce: sealedKey.nonce,
|
|
metadata: {
|
|
encryptedData: toBase64(sealedMetadata.ciphertext),
|
|
decryptionHeader: toBase64(sealedMetadata.header),
|
|
},
|
|
file: { decryptionHeader: original.header },
|
|
thumbnail: { decryptionHeader: original.header },
|
|
updationTime: t,
|
|
});
|
|
this.moveAlbum(albumID, t);
|
|
}
|
|
|
|
// A deleted record stays on the server as a tombstone holding only its
|
|
// ID, its time and `isDeleted`.
|
|
deleteFile(albumID: number, id: number, t: number): void {
|
|
this.files
|
|
.get(albumID)!
|
|
.set(id, { id, updationTime: t, isDeleted: true });
|
|
this.moveAlbum(albumID, t);
|
|
}
|
|
|
|
deleteAlbum(id: number, t: number): void {
|
|
this.albums.set(id, { id, updationTime: t, isDeleted: true });
|
|
}
|
|
|
|
// The `fetch` the saved session's client is given.
|
|
readonly fetch = (async (input: string | URL | Request) => {
|
|
const url = new URL(input instanceof Request ? input.url : input);
|
|
const since = Number(url.searchParams.get("sinceTime"));
|
|
const changed = (rows: Iterable<Row>): Row[] =>
|
|
[...rows].filter((row) => row.updationTime > since);
|
|
switch (url.pathname) {
|
|
case "/collections/v2":
|
|
return json({ collections: changed(this.albums.values()) });
|
|
case "/collections/v2/diff": {
|
|
const albumID = Number(url.searchParams.get("collectionID"));
|
|
const rows = this.files.get(albumID)?.values() ?? [];
|
|
return json({ diff: changed(rows), hasMore: false });
|
|
}
|
|
case "/files/data/fetch":
|
|
// No file has ML data.
|
|
return json({ data: [] });
|
|
default: {
|
|
const fileID = Number(url.searchParams.get("fileID"));
|
|
const original = this.originals.get(fileID);
|
|
return original
|
|
? new Response(original.body)
|
|
: new Response("not found", { status: 404 });
|
|
}
|
|
}
|
|
}) as typeof globalThis.fetch;
|
|
|
|
private originalOf(id: number): {
|
|
key: Uint8Array;
|
|
header: string;
|
|
body: Uint8Array;
|
|
} {
|
|
const known = this.originals.get(id);
|
|
if (known) return known;
|
|
const key = sodium.crypto_secretbox_keygen();
|
|
const { header, ciphertext } = encryptBlob(
|
|
new TextEncoder().encode(`original of ${id}`),
|
|
key,
|
|
);
|
|
const original = { key, header: toBase64(header), body: ciphertext };
|
|
this.originals.set(id, original);
|
|
return original;
|
|
}
|
|
|
|
private moveAlbum(albumID: number, t: number): void {
|
|
this.albums.set(albumID, {
|
|
...this.albums.get(albumID)!,
|
|
updationTime: t,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Album 1, "Trip", holds file 10, which decrypts, file 11, whose key does not,
|
|
// and file 12, whose metadata is not JSON. Album 2, "Family", holds file 20,
|
|
// but the album's own key does not decrypt. All of it as of time 1.
|
|
const brokenAccount = (): FakeEnte => {
|
|
const ente = new FakeEnte();
|
|
ente.putAlbum(1, "Trip", 1);
|
|
ente.putFile(1, 10, 1);
|
|
ente.putFile(1, 11, 1, { corruptKey: true });
|
|
ente.putFile(1, 12, 1, { metadata: "not json" });
|
|
ente.putAlbum(2, "Family", 1, true);
|
|
ente.putFile(2, 20, 1);
|
|
return ente;
|
|
};
|
|
|
|
// Collects everything written to it.
|
|
class Output {
|
|
text = "";
|
|
write(text: string): void {
|
|
this.text += text;
|
|
}
|
|
}
|
|
|
|
let root: string;
|
|
let stdout: Output;
|
|
let stderr: Output;
|
|
|
|
beforeAll(async () => {
|
|
await init();
|
|
await sodium.ready;
|
|
});
|
|
|
|
beforeEach(() => {
|
|
root = mkdtempSync(join(tmpdir(), "quak-undecryptable-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(root, { recursive: true, force: true });
|
|
});
|
|
|
|
// A context for one command, with fresh output, whose saved session talks to
|
|
// `ente`. Every run shares one cache directory, as runs of `quak` do.
|
|
const context = (ente: FakeEnte): CliContext => {
|
|
stdout = new Output();
|
|
stderr = new Output();
|
|
saveSession(join(root, "session"), ente.snapshot());
|
|
return {
|
|
stdout,
|
|
stderr,
|
|
sessionDir: join(root, "session"),
|
|
cacheDir: join(root, "cache"),
|
|
loadSession: (path) =>
|
|
loadSession(path, { fetch: ente.fetch, retry: { attempts: 1 } }),
|
|
login: async () => {
|
|
throw new Error("login not expected");
|
|
},
|
|
prompt: async () => {
|
|
throw new Error("prompt not expected");
|
|
},
|
|
promptSecret: async () => {
|
|
throw new Error("prompt not expected");
|
|
},
|
|
};
|
|
};
|
|
|
|
const backupDir = (): string => join(root, "backup");
|
|
|
|
// Runs `quak backup` into the backup directory and returns its exit code.
|
|
const backup = (ente: FakeEnte, opts: { json?: boolean } = {}) =>
|
|
backupCommand(context(ente), backupDir(), opts);
|
|
|
|
// The backup's `failures.json` entries by file ID, or {} when it has none.
|
|
const failures = (): Record<string, { error: string; attempts: number }> => {
|
|
const path = join(backupDir(), "failures.json");
|
|
return existsSync(path)
|
|
? JSON.parse(readFileSync(path, "utf-8")).files
|
|
: {};
|
|
};
|
|
|
|
// Whether file `id`'s original is at its save path in the backup.
|
|
const stored = (id: number): boolean =>
|
|
existsSync(join(backupDir(), DAY, `2026-03-01.${id}.jpg`));
|
|
|
|
describe("quak backup with records it cannot decrypt", () => {
|
|
it("backs up every other file, records each file it cannot decrypt in failures.json, names each such album, and exits 1", async () => {
|
|
expect(await backup(brokenAccount())).toBe(1);
|
|
|
|
expect(stored(10)).toBe(true);
|
|
// Without its album's key, file 20 cannot even be listed.
|
|
expect(stored(20)).toBe(false);
|
|
expect(failures()).toEqual({
|
|
"11": expect.objectContaining({
|
|
fileID: 11,
|
|
error: expect.stringMatching(
|
|
/^could not be decrypted: .*wrong secret key/,
|
|
),
|
|
}),
|
|
"12": expect.objectContaining({
|
|
fileID: 12,
|
|
error: expect.stringMatching(/^could not be decrypted: .*JSON/),
|
|
}),
|
|
});
|
|
expect(stderr.text).toMatch(
|
|
/^FAILED album 2: could not be decrypted: .*wrong secret key/m,
|
|
);
|
|
expect(stderr.text).toContain(" Failed: 2\n");
|
|
});
|
|
|
|
it("names each album it cannot decrypt in the --json result", async () => {
|
|
expect(await backup(brokenAccount(), { json: true })).toBe(1);
|
|
|
|
expect(JSON.parse(stdout.text)).toMatchObject({
|
|
downloaded: 1,
|
|
failed: 2,
|
|
undecryptableAlbums: [2],
|
|
});
|
|
});
|
|
|
|
it("still reports them on the next run, though the server does not send them again", async () => {
|
|
const ente = brokenAccount();
|
|
await backup(ente);
|
|
|
|
// Nothing changed on the server, so this run's refresh gets no rows at
|
|
// all. What it reports, it remembered from the first run.
|
|
expect(await backup(ente)).toBe(1);
|
|
|
|
expect(Object.keys(failures())).toEqual(["11", "12"]);
|
|
expect(failures()["11"]!.attempts).toBe(2);
|
|
expect(stderr.text).toMatch(/^FAILED album 2: could not be decrypted/m);
|
|
});
|
|
|
|
it("drops each once the server sends a copy that decrypts, and then exits 0", async () => {
|
|
const ente = brokenAccount();
|
|
await backup(ente);
|
|
|
|
// At time 2 files 11 and 12 and album 2 change on the server, and now
|
|
// each decrypts. Album 2 brings file 20 with it.
|
|
ente.putFile(1, 11, 2);
|
|
ente.putFile(1, 12, 2);
|
|
ente.putAlbum(2, "Family", 2);
|
|
expect(await backup(ente)).toBe(0);
|
|
|
|
expect([11, 12, 20].map(stored)).toEqual([true, true, true]);
|
|
expect(existsSync(join(backupDir(), "failures.json"))).toBe(false);
|
|
expect(stderr.text).not.toContain("FAILED");
|
|
});
|
|
|
|
it("drops each once the server deletes it, and then exits 0", async () => {
|
|
const ente = brokenAccount();
|
|
await backup(ente);
|
|
|
|
ente.deleteFile(1, 11, 2);
|
|
ente.deleteFile(1, 12, 2);
|
|
ente.deleteAlbum(2, 2);
|
|
expect(await backup(ente)).toBe(0);
|
|
|
|
expect(existsSync(join(backupDir(), "failures.json"))).toBe(false);
|
|
expect(stderr.text).not.toContain("FAILED");
|
|
});
|
|
|
|
it("backs up a file that does not decrypt in one album but does in another, and does not report it", async () => {
|
|
// File 30 is in albums 1 and 3, each holding its key sealed under the
|
|
// album's own key. In album 1 that is corrupt; in album 3 it is not,
|
|
// and one copy that decrypts is all the backup needs.
|
|
const ente = new FakeEnte();
|
|
ente.putAlbum(1, "Trip", 1);
|
|
ente.putFile(1, 30, 1, { corruptKey: true });
|
|
ente.putAlbum(3, "Favorites", 1);
|
|
ente.putFile(3, 30, 1);
|
|
|
|
expect(await backup(ente)).toBe(0);
|
|
|
|
expect(stored(30)).toBe(true);
|
|
expect(failures()).toEqual({});
|
|
});
|
|
|
|
it("backs up that file in a run limited to the album where it does not decrypt", async () => {
|
|
// The same account, backed up by `lib.backup()` limited to album 1,
|
|
// "Trip", as `quak backup` cannot be. File 30 is in Trip, so the run
|
|
// must store it, from its copy in album 3 that the run otherwise
|
|
// skips.
|
|
const ente = new FakeEnte();
|
|
ente.putAlbum(1, "Trip", 1);
|
|
ente.putFile(1, 30, 1, { corruptKey: true });
|
|
ente.putAlbum(3, "Favorites", 1);
|
|
ente.putFile(3, 30, 1);
|
|
const lib = await Library.open({
|
|
client: Client.fromJSON(ente.snapshot(), {
|
|
fetch: ente.fetch,
|
|
retry: { attempts: 1 },
|
|
}),
|
|
downloadDirectory: backupDir(),
|
|
cacheDirectory: join(root, "cache"),
|
|
precacheThumbnails: false,
|
|
precacheOriginals: false,
|
|
});
|
|
try {
|
|
const result = await lib.backup({
|
|
downloadDirectory: backupDir(),
|
|
onlyAlbumNames: ["Trip"],
|
|
});
|
|
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
|
|
expect(stored(30)).toBe(true);
|
|
expect(failures()).toEqual({});
|
|
});
|
|
});
|
|
|
|
describe("the read commands with records quak cannot decrypt", () => {
|
|
it("quak collections and quak files answer from what decrypts", async () => {
|
|
const ente = brokenAccount();
|
|
|
|
expect(await collectionsCommand(context(ente), {})).toBe(0);
|
|
expect(stdout.text).toBe("1\talbum\tTrip\n");
|
|
|
|
expect(await filesCommand(context(ente), { collection: "1" })).toBe(0);
|
|
expect(stdout.text).toBe("10\timage\tphoto-10.jpg\n");
|
|
});
|
|
});
|