sneak
349e364109
collectionsSince and filesSince put a row that does not decrypt or parse in an undecryptable list with the reason instead of throwing. The refresh leaves it out, an album with its files, and the store keeps the list in metadata.json until the server sends a copy that decrypts or deletes it, since the next refresh asks only for changes. lib.backup() records each such file in failures.json as "could not be decrypted: <reason>", unless a copy in another album decrypted, logs one line per such album and lists their IDs in undecryptableAlbums; quak backup exits 1 while any remain. Tests use a stand-in Ente server with real encryption. Model: opus-5-5
428 lines
16 KiB
TypeScript
428 lines
16 KiB
TypeScript
/**
|
|
* `quak backup` over an account holding an album or a file quak cannot decrypt
|
|
* (issue 181).
|
|
*
|
|
* The server sends every album and file encrypted, and a refresh decrypts each
|
|
* one it fetches. A record whose key no key of the account opens, or whose
|
|
* metadata is not JSON, must not fail the refresh: a refresh that fails commits
|
|
* nothing, so `quak backup` would store nothing, on every run, until the
|
|
* server's copy changed. Instead the refresh leaves that record out, and the
|
|
* library keeps a list of what it left out in its cache. It has to: the next
|
|
* refresh asks the server only for what changed since the last one, so it will
|
|
* not see the record again.
|
|
*
|
|
* The backup stores every other file. It records each file it could not
|
|
* decrypt in `failures.json`, prints one line for each album it could not
|
|
* decrypt, and exits 1 while any remain. Once the server sends a copy that
|
|
* decrypts, or deletes the record, the next run drops it and can exit 0. The
|
|
* read commands answer from what did decrypt.
|
|
*
|
|
* These tests run the commands with a real saved session over a stand-in Ente
|
|
* server, which serves albums, files and originals encrypted with libsodium as
|
|
* the real server does. A record is broken the way a corrupt one arrives: its
|
|
* key replaced by bytes no key opens, or its metadata text that is not JSON.
|
|
*/
|
|
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import sodium from "libsodium-wrappers-sumo";
|
|
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
|
|
|
import {
|
|
backupCommand,
|
|
collectionsCommand,
|
|
filesCommand,
|
|
saveSession,
|
|
type CliContext,
|
|
} from "../../src/cli-commands.js";
|
|
import { loadSession } from "../../src/cli-session.js";
|
|
import type { ClientSnapshot } from "../../src/client.js";
|
|
import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js";
|
|
|
|
const USER_ID = 42;
|
|
|
|
// Every file is taken at noon local time on 2026-03-01, in microseconds as Ente
|
|
// stores times, so its original is saved in the folder `DAY` whatever the time
|
|
// zone of the machine running the tests.
|
|
const TAKEN = new Date(2026, 2, 1, 12).getTime() * 1000;
|
|
const DAY = join("2026", "2026-03", "2026-03-01");
|
|
|
|
type Row = { id: number; updationTime: number } & Record<string, unknown>;
|
|
|
|
// `plaintext` sealed under `key` with a fresh nonce, both base64, as Ente
|
|
// sends a key or an album name.
|
|
const secretbox = (
|
|
plaintext: Uint8Array,
|
|
key: Uint8Array,
|
|
): { ciphertext: string; nonce: string } => {
|
|
const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
|
return {
|
|
ciphertext: toBase64(
|
|
sodium.crypto_secretbox_easy(plaintext, nonce, key),
|
|
),
|
|
nonce: toBase64(nonce),
|
|
};
|
|
};
|
|
|
|
// What a corrupt row carries in place of its sealed key: random bytes of the
|
|
// same length, which no key opens.
|
|
const corruptKey = (): string =>
|
|
toBase64(
|
|
sodium.randombytes_buf(
|
|
sodium.crypto_secretbox_KEYBYTES + sodium.crypto_secretbox_MACBYTES,
|
|
),
|
|
);
|
|
|
|
const json = (body: unknown): Response =>
|
|
new Response(JSON.stringify(body), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
});
|
|
|
|
// A stand-in Ente server for one account. It holds each album, and each file in
|
|
// an album, as the row `/collections/v2` and `/collections/v2/diff` serve, and
|
|
// answers a request with the rows whose `updationTime` is past the request's
|
|
// `sinceTime`, as the real server does. A change to a file also moves its
|
|
// album's `updationTime`, which is what tells a client to list the album's
|
|
// files again. Originals are served by file ID, as the files CDN serves them.
|
|
class FakeEnte {
|
|
private readonly masterKey = sodium.crypto_secretbox_keygen();
|
|
private readonly keyPair = sodium.crypto_box_keypair();
|
|
private readonly albums = new Map<number, Row>();
|
|
private readonly albumKeys = new Map<number, Uint8Array>();
|
|
// Each album's file rows, by album ID and then file ID.
|
|
private readonly files = new Map<number, Map<number, Row>>();
|
|
// Each file's key and its original as served, by file ID. A file in two
|
|
// albums is one original under one key, sealed under each album's key.
|
|
private readonly originals = new Map<
|
|
number,
|
|
{ key: Uint8Array; header: string; body: Uint8Array }
|
|
>();
|
|
|
|
// The session `quak login` would have saved for the account.
|
|
snapshot(): ClientSnapshot {
|
|
return {
|
|
email: "user@example.com",
|
|
userID: USER_ID,
|
|
token: "token",
|
|
masterKey: toBase64(this.masterKey),
|
|
secretKey: toBase64(this.keyPair.privateKey),
|
|
publicKey: toBase64(this.keyPair.publicKey),
|
|
};
|
|
}
|
|
|
|
// Album `id`, named `name`, as of time `t`. With `corrupt`, its key cannot
|
|
// be decrypted. An album keeps its key from one call to the next, so its
|
|
// files' rows still decrypt once it is put back without `corrupt`.
|
|
putAlbum(id: number, name: string, t: number, corrupt = false): void {
|
|
const key = this.albumKeys.get(id) ?? sodium.crypto_secretbox_keygen();
|
|
this.albumKeys.set(id, key);
|
|
const sealedKey = secretbox(key, this.masterKey);
|
|
const sealedName = secretbox(new TextEncoder().encode(name), key);
|
|
this.albums.set(id, {
|
|
id,
|
|
owner: { id: USER_ID },
|
|
encryptedKey: corrupt ? corruptKey() : sealedKey.ciphertext,
|
|
keyDecryptionNonce: sealedKey.nonce,
|
|
encryptedName: sealedName.ciphertext,
|
|
nameDecryptionNonce: sealedName.nonce,
|
|
type: "album",
|
|
updationTime: t,
|
|
});
|
|
if (!this.files.has(id)) this.files.set(id, new Map());
|
|
}
|
|
|
|
// File `id`, titled `photo-<id>.jpg`, in album `albumID` as of time `t`.
|
|
// With `corruptKey`, its key cannot be decrypted. With `metadata`, its
|
|
// metadata decrypts to that text instead of the JSON a client uploads.
|
|
putFile(
|
|
albumID: number,
|
|
id: number,
|
|
t: number,
|
|
opts: { corruptKey?: boolean; metadata?: string } = {},
|
|
): void {
|
|
const original = this.originalOf(id);
|
|
const sealedKey = secretbox(original.key, this.albumKeys.get(albumID)!);
|
|
const metadata =
|
|
opts.metadata ??
|
|
JSON.stringify({
|
|
title: `photo-${id}.jpg`,
|
|
fileType: 0,
|
|
creationTime: TAKEN,
|
|
modificationTime: TAKEN,
|
|
});
|
|
const sealedMetadata = encryptBlob(
|
|
new TextEncoder().encode(metadata),
|
|
original.key,
|
|
);
|
|
this.files.get(albumID)!.set(id, {
|
|
id,
|
|
collectionID: albumID,
|
|
ownerID: USER_ID,
|
|
encryptedKey: opts.corruptKey ? corruptKey() : sealedKey.ciphertext,
|
|
keyDecryptionNonce: sealedKey.nonce,
|
|
metadata: {
|
|
encryptedData: toBase64(sealedMetadata.ciphertext),
|
|
decryptionHeader: toBase64(sealedMetadata.header),
|
|
},
|
|
file: { decryptionHeader: original.header },
|
|
thumbnail: { decryptionHeader: original.header },
|
|
updationTime: t,
|
|
});
|
|
this.moveAlbum(albumID, t);
|
|
}
|
|
|
|
// A deleted record stays on the server as a tombstone holding only its
|
|
// ID, its time and `isDeleted`.
|
|
deleteFile(albumID: number, id: number, t: number): void {
|
|
this.files
|
|
.get(albumID)!
|
|
.set(id, { id, updationTime: t, isDeleted: true });
|
|
this.moveAlbum(albumID, t);
|
|
}
|
|
|
|
deleteAlbum(id: number, t: number): void {
|
|
this.albums.set(id, { id, updationTime: t, isDeleted: true });
|
|
}
|
|
|
|
// The `fetch` the saved session's client is given.
|
|
readonly fetch = (async (input: string | URL | Request) => {
|
|
const url = new URL(input instanceof Request ? input.url : input);
|
|
const since = Number(url.searchParams.get("sinceTime"));
|
|
const changed = (rows: Iterable<Row>): Row[] =>
|
|
[...rows].filter((row) => row.updationTime > since);
|
|
switch (url.pathname) {
|
|
case "/collections/v2":
|
|
return json({ collections: changed(this.albums.values()) });
|
|
case "/collections/v2/diff": {
|
|
const albumID = Number(url.searchParams.get("collectionID"));
|
|
const rows = this.files.get(albumID)?.values() ?? [];
|
|
return json({ diff: changed(rows), hasMore: false });
|
|
}
|
|
case "/files/data/fetch":
|
|
// No file has ML data.
|
|
return json({ data: [] });
|
|
default: {
|
|
const fileID = Number(url.searchParams.get("fileID"));
|
|
const original = this.originals.get(fileID);
|
|
return original
|
|
? new Response(original.body)
|
|
: new Response("not found", { status: 404 });
|
|
}
|
|
}
|
|
}) as typeof globalThis.fetch;
|
|
|
|
private originalOf(id: number): {
|
|
key: Uint8Array;
|
|
header: string;
|
|
body: Uint8Array;
|
|
} {
|
|
const known = this.originals.get(id);
|
|
if (known) return known;
|
|
const key = sodium.crypto_secretbox_keygen();
|
|
const { header, ciphertext } = encryptBlob(
|
|
new TextEncoder().encode(`original of ${id}`),
|
|
key,
|
|
);
|
|
const original = { key, header: toBase64(header), body: ciphertext };
|
|
this.originals.set(id, original);
|
|
return original;
|
|
}
|
|
|
|
private moveAlbum(albumID: number, t: number): void {
|
|
this.albums.set(albumID, {
|
|
...this.albums.get(albumID)!,
|
|
updationTime: t,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Album 1, "Trip", holds file 10, which decrypts, file 11, whose key does not,
|
|
// and file 12, whose metadata is not JSON. Album 2, "Family", holds file 20,
|
|
// but the album's own key does not decrypt. All of it as of time 1.
|
|
const brokenAccount = (): FakeEnte => {
|
|
const ente = new FakeEnte();
|
|
ente.putAlbum(1, "Trip", 1);
|
|
ente.putFile(1, 10, 1);
|
|
ente.putFile(1, 11, 1, { corruptKey: true });
|
|
ente.putFile(1, 12, 1, { metadata: "not json" });
|
|
ente.putAlbum(2, "Family", 1, true);
|
|
ente.putFile(2, 20, 1);
|
|
return ente;
|
|
};
|
|
|
|
// Collects everything written to it.
|
|
class Output {
|
|
text = "";
|
|
write(text: string): void {
|
|
this.text += text;
|
|
}
|
|
}
|
|
|
|
let root: string;
|
|
let stdout: Output;
|
|
let stderr: Output;
|
|
|
|
beforeAll(async () => {
|
|
await init();
|
|
await sodium.ready;
|
|
});
|
|
|
|
beforeEach(() => {
|
|
root = mkdtempSync(join(tmpdir(), "quak-undecryptable-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(root, { recursive: true, force: true });
|
|
});
|
|
|
|
// A context for one command, with fresh output, whose saved session talks to
|
|
// `ente`. Every run shares one cache directory, as runs of `quak` do.
|
|
const context = (ente: FakeEnte): CliContext => {
|
|
stdout = new Output();
|
|
stderr = new Output();
|
|
saveSession(join(root, "session"), ente.snapshot());
|
|
return {
|
|
stdout,
|
|
stderr,
|
|
sessionDir: join(root, "session"),
|
|
cacheDir: join(root, "cache"),
|
|
loadSession: (path) =>
|
|
loadSession(path, { fetch: ente.fetch, retry: { attempts: 1 } }),
|
|
login: async () => {
|
|
throw new Error("login not expected");
|
|
},
|
|
prompt: async () => {
|
|
throw new Error("prompt not expected");
|
|
},
|
|
promptSecret: async () => {
|
|
throw new Error("prompt not expected");
|
|
},
|
|
};
|
|
};
|
|
|
|
const backupDir = (): string => join(root, "backup");
|
|
|
|
// Runs `quak backup` into the backup directory and returns its exit code.
|
|
const backup = (ente: FakeEnte, opts: { json?: boolean } = {}) =>
|
|
backupCommand(context(ente), backupDir(), opts);
|
|
|
|
// The backup's `failures.json` entries by file ID, or {} when it has none.
|
|
const failures = (): Record<string, { error: string; attempts: number }> => {
|
|
const path = join(backupDir(), "failures.json");
|
|
return existsSync(path)
|
|
? JSON.parse(readFileSync(path, "utf-8")).files
|
|
: {};
|
|
};
|
|
|
|
// Whether file `id`'s original is at its save path in the backup.
|
|
const stored = (id: number): boolean =>
|
|
existsSync(join(backupDir(), DAY, `2026-03-01.${id}.jpg`));
|
|
|
|
describe("quak backup with records it cannot decrypt", () => {
|
|
it("backs up every other file, records each file it cannot decrypt in failures.json, names each such album, and exits 1", async () => {
|
|
expect(await backup(brokenAccount())).toBe(1);
|
|
|
|
expect(stored(10)).toBe(true);
|
|
// Without its album's key, file 20 cannot even be listed.
|
|
expect(stored(20)).toBe(false);
|
|
expect(failures()).toEqual({
|
|
"11": expect.objectContaining({
|
|
fileID: 11,
|
|
error: expect.stringMatching(
|
|
/^could not be decrypted: .*wrong secret key/,
|
|
),
|
|
}),
|
|
"12": expect.objectContaining({
|
|
fileID: 12,
|
|
error: expect.stringMatching(/^could not be decrypted: .*JSON/),
|
|
}),
|
|
});
|
|
expect(stderr.text).toMatch(
|
|
/^FAILED album 2: could not be decrypted: .*wrong secret key/m,
|
|
);
|
|
expect(stderr.text).toContain(" Failed: 2\n");
|
|
});
|
|
|
|
it("names each album it cannot decrypt in the --json result", async () => {
|
|
expect(await backup(brokenAccount(), { json: true })).toBe(1);
|
|
|
|
expect(JSON.parse(stdout.text)).toMatchObject({
|
|
downloaded: 1,
|
|
failed: 2,
|
|
undecryptableAlbums: [2],
|
|
});
|
|
});
|
|
|
|
it("still reports them on the next run, though the server does not send them again", async () => {
|
|
const ente = brokenAccount();
|
|
await backup(ente);
|
|
|
|
// Nothing changed on the server, so this run's refresh gets no rows at
|
|
// all. What it reports, it remembered from the first run.
|
|
expect(await backup(ente)).toBe(1);
|
|
|
|
expect(Object.keys(failures())).toEqual(["11", "12"]);
|
|
expect(failures()["11"]!.attempts).toBe(2);
|
|
expect(stderr.text).toMatch(/^FAILED album 2: could not be decrypted/m);
|
|
});
|
|
|
|
it("drops each once the server sends a copy that decrypts, and then exits 0", async () => {
|
|
const ente = brokenAccount();
|
|
await backup(ente);
|
|
|
|
// At time 2 files 11 and 12 and album 2 change on the server, and now
|
|
// each decrypts. Album 2 brings file 20 with it.
|
|
ente.putFile(1, 11, 2);
|
|
ente.putFile(1, 12, 2);
|
|
ente.putAlbum(2, "Family", 2);
|
|
expect(await backup(ente)).toBe(0);
|
|
|
|
expect([11, 12, 20].map(stored)).toEqual([true, true, true]);
|
|
expect(existsSync(join(backupDir(), "failures.json"))).toBe(false);
|
|
expect(stderr.text).not.toContain("FAILED");
|
|
});
|
|
|
|
it("drops each once the server deletes it, and then exits 0", async () => {
|
|
const ente = brokenAccount();
|
|
await backup(ente);
|
|
|
|
ente.deleteFile(1, 11, 2);
|
|
ente.deleteFile(1, 12, 2);
|
|
ente.deleteAlbum(2, 2);
|
|
expect(await backup(ente)).toBe(0);
|
|
|
|
expect(existsSync(join(backupDir(), "failures.json"))).toBe(false);
|
|
expect(stderr.text).not.toContain("FAILED");
|
|
});
|
|
|
|
it("backs up a file that does not decrypt in one album but does in another, and does not report it", async () => {
|
|
// File 30 is in albums 1 and 3, each holding its key sealed under the
|
|
// album's own key. In album 1 that is corrupt; in album 3 it is not,
|
|
// and one copy that decrypts is all the backup needs.
|
|
const ente = new FakeEnte();
|
|
ente.putAlbum(1, "Trip", 1);
|
|
ente.putFile(1, 30, 1, { corruptKey: true });
|
|
ente.putAlbum(3, "Favorites", 1);
|
|
ente.putFile(3, 30, 1);
|
|
|
|
expect(await backup(ente)).toBe(0);
|
|
|
|
expect(stored(30)).toBe(true);
|
|
expect(failures()).toEqual({});
|
|
});
|
|
});
|
|
|
|
describe("the read commands with records quak cannot decrypt", () => {
|
|
it("quak collections and quak files answer from what decrypts", async () => {
|
|
const ente = brokenAccount();
|
|
|
|
expect(await collectionsCommand(context(ente), {})).toBe(0);
|
|
expect(stdout.text).toBe("1\talbum\tTrip\n");
|
|
|
|
expect(await filesCommand(context(ente), { collection: "1" })).toBe(0);
|
|
expect(stdout.text).toBe("10\timage\tphoto-10.jpg\n");
|
|
});
|
|
});
|