sneak
767545f4ab
check / check (push) Waiting to run
The onProgress test now runs eight photos with a pool of five and holds only the first five downloads, so a backup that went on after the throw would download and store the other three. The end-to-end test of a removed backup.lock expects the backup to stop after its three originals, before it fetches ML data. Model: opus-5-5
836 lines
32 KiB
TypeScript
836 lines
32 KiB
TypeScript
/**
|
|
* `quak backup`, end to end.
|
|
*
|
|
* Each test runs the command as cron runs it: `backupCommand` reads the
|
|
* session file that logging in saved, opens the library on the real `Client`
|
|
* and backs the whole account up into a directory, and `run` turns the outcome
|
|
* into the exit code, as `bin/quak.ts` does. Only the server is a stand-in:
|
|
* `buildMetaFetch` from `test/ente-server.ts` answers the Ente HTTP API from
|
|
* data this file encrypts with libsodium, as Ente's clients encrypt it. So
|
|
* quak's own code fetches each original, decrypts it, checks it against the
|
|
* hash its uploader recorded and unpacks a live photo, and quak's own retries
|
|
* repeat a failed request. Two tests run `lib.backup()` instead, as a program
|
|
* using quak as a library does.
|
|
*
|
|
* The account, user 42, has three albums:
|
|
*
|
|
* Trips its own: beach.jpg
|
|
* Family its own: beach.jpg and the live photo IMG_0002.HEIC
|
|
* Friend's Wedding shared with it by user 99: toast.mp4
|
|
*
|
|
* beach.jpg is one file in two albums. It holds EXIF (a Canon EOS R5), and
|
|
* Ente has ML data for it. All three files were taken at noon on 1 March 2024.
|
|
*/
|
|
|
|
import {
|
|
existsSync,
|
|
lstatSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readdirSync,
|
|
readFileSync,
|
|
readlinkSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { Writable } from "node:stream";
|
|
import { gzipSync } from "node:zlib";
|
|
import sodium from "libsodium-wrappers-sumo";
|
|
import {
|
|
afterAll,
|
|
afterEach,
|
|
beforeAll,
|
|
beforeEach,
|
|
describe,
|
|
expect,
|
|
it,
|
|
vi,
|
|
} from "vitest";
|
|
|
|
import {
|
|
backupCommand,
|
|
saveSession,
|
|
type CliContext,
|
|
} from "../../src/cli-commands.js";
|
|
import { run } from "../../src/cli-run.js";
|
|
import { loadSession } from "../../src/cli-session.js";
|
|
import { Client } from "../../src/client.js";
|
|
import { init, toBase64 } from "../../src/crypto/index.js";
|
|
import { Library, lockBackupDirectory } from "../../src/library/index.js";
|
|
import { UNATTENDED_RETRY_OPTIONS } from "../../src/retry.js";
|
|
import {
|
|
buildAccount,
|
|
buildMetaFetch,
|
|
encryptSecretbox,
|
|
encryptStreamBlob,
|
|
} from "../ente-server.js";
|
|
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
|
import { JPEG_WITH_EXIF } from "../exif-jpeg.js";
|
|
import { blake2b, livePhotoHash, livePhotoZip, VIDEO } from "../live-photo.js";
|
|
|
|
const EMAIL = "backup@example.com";
|
|
const PASSWORD = "backup-password";
|
|
// The fake server logs every account in as user 42.
|
|
const USER_ID = 42;
|
|
const FRIEND_ID = 99;
|
|
|
|
// Noon on 1 March 2024 in the time zone of the machine running the test, the
|
|
// one quak dates save paths in, in microseconds, as Ente records times.
|
|
const TAKEN = new Date(2024, 2, 1, 12).getTime() * 1000;
|
|
// The date folder every original is saved in.
|
|
const DAY = "2024/2024-03/2024-03-01";
|
|
|
|
const TOAST = new TextEncoder().encode("a few seconds of the wedding toast");
|
|
|
|
// Ente's ML data for beach.jpg: a face it found, and the photo's CLIP
|
|
// embedding.
|
|
const ML_DATA = {
|
|
face: { faces: [{ faceID: "100_1", score: 0.98 }] },
|
|
clip: { embedding: [0.5, 0.6, 0.7] },
|
|
};
|
|
|
|
// Each file's ID, its owner, its original as uploaded, and the metadata its
|
|
// uploader recorded. `fileType` is Ente's number for the kind: 0 an image, 1 a
|
|
// video, 2 a live photo, whose original is a ZIP of its image and its video.
|
|
// `hash` is the content hash a download must match.
|
|
const BEACH = {
|
|
id: 100,
|
|
ownerID: USER_ID,
|
|
original: JPEG_WITH_EXIF,
|
|
metadata: {
|
|
title: "beach.jpg",
|
|
fileType: 0,
|
|
hash: blake2b(JPEG_WITH_EXIF),
|
|
},
|
|
};
|
|
const TOAST_VIDEO = {
|
|
id: 200,
|
|
ownerID: FRIEND_ID,
|
|
original: TOAST,
|
|
metadata: { title: "toast.mp4", fileType: 1, hash: blake2b(TOAST) },
|
|
};
|
|
const LIVE_PHOTO = {
|
|
id: 300,
|
|
ownerID: USER_ID,
|
|
original: livePhotoZip({
|
|
"image.heic": HEIC_WITH_EXIF,
|
|
"video.mov": VIDEO,
|
|
}),
|
|
metadata: {
|
|
title: "IMG_0002.HEIC",
|
|
fileType: 2,
|
|
hash: livePhotoHash(HEIC_WITH_EXIF, VIDEO),
|
|
},
|
|
};
|
|
|
|
// The fake Ente server, as the `fetch` quak sends its requests through.
|
|
let server: typeof globalThis.fetch;
|
|
let root: string;
|
|
let sessionDir: string;
|
|
|
|
// The fake server, holding the account encrypted as Ente's clients upload it.
|
|
const buildServer = async () => {
|
|
const account = await buildAccount(PASSWORD);
|
|
|
|
// An album as `/collections/v2` lists it, its name encrypted with its own
|
|
// key. An album of the account's own carries that key encrypted with the
|
|
// master key; an album shared with it carries the key sealed to its public
|
|
// key, and no nonce.
|
|
const album = (id: number, name: string, ownerID: number) => {
|
|
const key = sodium.crypto_secretbox_keygen();
|
|
const encryptedName = encryptSecretbox(
|
|
new TextEncoder().encode(name),
|
|
key,
|
|
);
|
|
let keyFields: { encryptedKey: string; keyDecryptionNonce?: string };
|
|
if (ownerID === USER_ID) {
|
|
const encryptedKey = encryptSecretbox(key, account.masterKey);
|
|
keyFields = {
|
|
encryptedKey: toBase64(encryptedKey.ciphertext),
|
|
keyDecryptionNonce: toBase64(encryptedKey.nonce),
|
|
};
|
|
} else {
|
|
const sealedKey = sodium.crypto_box_seal(key, account.publicKey);
|
|
keyFields = { encryptedKey: toBase64(sealedKey) };
|
|
}
|
|
const row = {
|
|
id,
|
|
owner: { id: ownerID },
|
|
...keyFields,
|
|
encryptedName: toBase64(encryptedName.ciphertext),
|
|
nameDecryptionNonce: toBase64(encryptedName.nonce),
|
|
type: "album",
|
|
updationTime: TAKEN,
|
|
};
|
|
return { key, row };
|
|
};
|
|
const trips = album(10, "Trips", USER_ID);
|
|
const family = album(20, "Family", USER_ID);
|
|
const wedding = album(30, "Friend's Wedding", FRIEND_ID);
|
|
|
|
// Each file's original is encrypted with the file's own key, as one
|
|
// secretstream, which the server serves whole.
|
|
const encrypted = (file: typeof BEACH) => {
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
return { file, key, content: encryptStreamBlob(file.original, key) };
|
|
};
|
|
const beach = encrypted(BEACH);
|
|
const toast = encrypted(TOAST_VIDEO);
|
|
const livePhoto = encrypted(LIVE_PHOTO);
|
|
|
|
// A file as an album's `/collections/v2/diff` lists it: its key encrypted
|
|
// with the album's key, its metadata encrypted with its own key, and the
|
|
// header that starts the decryption of its original.
|
|
const inAlbum = (
|
|
{ file, key, content }: ReturnType<typeof encrypted>,
|
|
collectionID: number,
|
|
albumKey: Uint8Array,
|
|
) => {
|
|
const encryptedKey = encryptSecretbox(key, albumKey);
|
|
const metadata = encryptStreamBlob(
|
|
new TextEncoder().encode(
|
|
JSON.stringify({
|
|
...file.metadata,
|
|
creationTime: TAKEN,
|
|
modificationTime: TAKEN,
|
|
}),
|
|
),
|
|
key,
|
|
);
|
|
return {
|
|
id: file.id,
|
|
collectionID,
|
|
ownerID: file.ownerID,
|
|
encryptedKey: toBase64(encryptedKey.ciphertext),
|
|
keyDecryptionNonce: toBase64(encryptedKey.nonce),
|
|
metadata: {
|
|
encryptedData: toBase64(metadata.ciphertext),
|
|
decryptionHeader: toBase64(metadata.header),
|
|
},
|
|
file: { decryptionHeader: toBase64(content.header) },
|
|
thumbnail: {
|
|
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
|
},
|
|
updationTime: TAKEN,
|
|
};
|
|
};
|
|
|
|
// ML data is gzipped JSON, encrypted with the file's key.
|
|
const mlData = encryptStreamBlob(
|
|
gzipSync(JSON.stringify(ML_DATA)),
|
|
beach.key,
|
|
);
|
|
|
|
return buildMetaFetch({
|
|
verifier: account.verifier,
|
|
srpAttributes: account.srpAttributes,
|
|
keyAttributes: account.keyAttributes,
|
|
encryptedToken: account.encryptedToken,
|
|
collections: [trips.row, family.row, wedding.row],
|
|
filesByCollection: {
|
|
10: [inAlbum(beach, 10, trips.key)],
|
|
20: [
|
|
inAlbum(beach, 20, family.key),
|
|
inAlbum(livePhoto, 20, family.key),
|
|
],
|
|
30: [inAlbum(toast, 30, wedding.key)],
|
|
},
|
|
encryptedMLData: {
|
|
100: {
|
|
encryptedData: toBase64(mlData.ciphertext),
|
|
decryptionHeader: toBase64(mlData.header),
|
|
},
|
|
},
|
|
fileCiphertexts: {
|
|
100: beach.content.ciphertext,
|
|
200: toast.content.ciphertext,
|
|
300: livePhoto.content.ciphertext,
|
|
},
|
|
});
|
|
};
|
|
|
|
beforeAll(async () => {
|
|
await init();
|
|
await sodium.ready;
|
|
root = mkdtempSync(join(tmpdir(), "quak-backup-end-to-end-"));
|
|
server = await buildServer();
|
|
|
|
// Log in once, as `quak login` does: the session file it saves is what
|
|
// every backup below reads.
|
|
sessionDir = join(root, "session");
|
|
const client = await Client.login({
|
|
email: EMAIL,
|
|
password: PASSWORD,
|
|
apiOptions: { fetch: server },
|
|
});
|
|
saveSession(sessionDir, client.toJSON());
|
|
});
|
|
|
|
afterAll(() => {
|
|
rmSync(root, { recursive: true, force: true });
|
|
});
|
|
|
|
// Run `quak backup <dir>` as `bin/quak.ts` runs it, sending its requests
|
|
// through `fetch`, and return its exit code and everything it printed on
|
|
// stderr. Its requests are retried as those of `quak backup` are, except that
|
|
// a retry does not wait. Its cache is beside `dir`, so each run of one
|
|
// directory finds the cache the run before it left, as cron's runs find the
|
|
// user's.
|
|
const backup = async (
|
|
dir: string,
|
|
opts: { verify?: boolean } = {},
|
|
fetch = server,
|
|
): Promise<{ code: number; stderr: string }> => {
|
|
let printed = "";
|
|
const stderr = new Writable({
|
|
write(chunk: Buffer, _encoding, done) {
|
|
printed += chunk.toString();
|
|
done();
|
|
},
|
|
});
|
|
const stdout = new Writable({
|
|
write(_chunk, _encoding, done) {
|
|
done();
|
|
},
|
|
});
|
|
// A backup never prompts: any of these ends the run with exit 1.
|
|
const prompted = async (): Promise<never> => {
|
|
throw new Error("quak backup prompted");
|
|
};
|
|
const ctx: CliContext = {
|
|
stdout,
|
|
stderr,
|
|
sessionDir,
|
|
cacheDir: `${dir}-cache`,
|
|
loadSession: (path) =>
|
|
loadSession(path, {
|
|
fetch,
|
|
retry: { ...UNATTENDED_RETRY_OPTIONS, sleep: async () => {} },
|
|
}),
|
|
login: prompted,
|
|
prompt: prompted,
|
|
promptSecret: prompted,
|
|
};
|
|
const code = await new Promise<number>((resolve) => {
|
|
void run(backupCommand(ctx, dir, opts), stdout, stderr, resolve);
|
|
});
|
|
return { code, stderr: printed };
|
|
};
|
|
|
|
// The ID of the file whose original a request downloads, or undefined for any
|
|
// other request. quak downloads an original from
|
|
// `https://files.ente.io/?fileID=<id>`.
|
|
const downloadOf = (input: RequestInfo | URL): number | undefined => {
|
|
const url = new URL(String(input));
|
|
return url.hostname === "files.ente.io"
|
|
? Number(url.searchParams.get("fileID"))
|
|
: undefined;
|
|
};
|
|
|
|
// The fake server, pushing onto `downloads` the ID of each file whose original
|
|
// is downloaded.
|
|
const recordingDownloads =
|
|
(downloads: number[]): typeof globalThis.fetch =>
|
|
async (input, init) => {
|
|
const fileID = downloadOf(input);
|
|
if (fileID !== undefined) downloads.push(fileID);
|
|
return server(input, init);
|
|
};
|
|
|
|
// Every entry under `dir`, relative to it, sorted, a symlink as
|
|
// `name -> target`.
|
|
const listTree = (dir: string): string[] =>
|
|
readdirSync(dir, { recursive: true })
|
|
.sort()
|
|
.map((name) => {
|
|
const path = join(dir, name);
|
|
return lstatSync(path).isSymbolicLink()
|
|
? `${name} -> ${readlinkSync(path)}`
|
|
: name;
|
|
});
|
|
|
|
const readJSON = (path: string) => JSON.parse(readFileSync(path, "utf-8"));
|
|
|
|
describe("the first backup of the account", () => {
|
|
let dir: string;
|
|
let code: number;
|
|
const downloads: number[] = [];
|
|
|
|
beforeAll(async () => {
|
|
dir = mkdtempSync(join(root, "backup-"));
|
|
({ code } = await backup(dir, {}, recordingDownloads(downloads)));
|
|
});
|
|
|
|
it("exits 0, having downloaded each file once", () => {
|
|
expect(code).toBe(0);
|
|
// beach.jpg is in two albums but is downloaded, and stored, once.
|
|
expect(downloads.sort()).toEqual([100, 200, 300]);
|
|
});
|
|
|
|
// The tree "Backup layout" in the README describes. Each original is at
|
|
// its save path, `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>`, with
|
|
// the extension of its title; the live photo is its image and its video,
|
|
// each with the extension it had inside the ZIP, and a `.livephoto.json`
|
|
// naming the two. Beside each original is its JSON. Each album is a
|
|
// folder of links named after the files' titles, with a JSON file beside
|
|
// it. No file failed, so there is no `failures.json`, and the run removed
|
|
// its lock, `backup.lock`, as it ended.
|
|
it("lays the backup out as the README describes", () => {
|
|
expect(listTree(dir)).toEqual([
|
|
"2024",
|
|
"2024/2024-03",
|
|
"2024/2024-03/2024-03-01",
|
|
"2024/2024-03/2024-03-01/2024-03-01.100.jpg",
|
|
"2024/2024-03/2024-03-01/2024-03-01.100.json",
|
|
"2024/2024-03/2024-03-01/2024-03-01.200.json",
|
|
"2024/2024-03/2024-03-01/2024-03-01.200.mp4",
|
|
"2024/2024-03/2024-03-01/2024-03-01.300.heic",
|
|
"2024/2024-03/2024-03-01/2024-03-01.300.json",
|
|
"2024/2024-03/2024-03-01/2024-03-01.300.livephoto.json",
|
|
"2024/2024-03/2024-03-01/2024-03-01.300.mov",
|
|
"account.json",
|
|
"collections",
|
|
"collections/Family",
|
|
"collections/Family.json",
|
|
"collections/Family/IMG_0002.heic -> ../../2024/2024-03/2024-03-01/2024-03-01.300.heic",
|
|
"collections/Family/IMG_0002.mov -> ../../2024/2024-03/2024-03-01/2024-03-01.300.mov",
|
|
"collections/Family/beach.jpg -> ../../2024/2024-03/2024-03-01/2024-03-01.100.jpg",
|
|
"collections/Friend's Wedding",
|
|
"collections/Friend's Wedding.json",
|
|
"collections/Friend's Wedding/toast.mp4 -> ../../2024/2024-03/2024-03-01/2024-03-01.200.mp4",
|
|
"collections/Trips",
|
|
"collections/Trips.json",
|
|
"collections/Trips/beach.jpg -> ../../2024/2024-03/2024-03-01/2024-03-01.100.jpg",
|
|
]);
|
|
});
|
|
|
|
it("stores each original decrypted, a live photo as its image and its video", () => {
|
|
const day = join(dir, DAY);
|
|
expect(readFileSync(join(day, "2024-03-01.100.jpg"))).toEqual(
|
|
Buffer.from(JPEG_WITH_EXIF),
|
|
);
|
|
expect(readFileSync(join(day, "2024-03-01.200.mp4"))).toEqual(
|
|
Buffer.from(TOAST),
|
|
);
|
|
expect(readFileSync(join(day, "2024-03-01.300.heic"))).toEqual(
|
|
Buffer.from(HEIC_WITH_EXIF),
|
|
);
|
|
expect(readFileSync(join(day, "2024-03-01.300.mov"))).toEqual(
|
|
Buffer.from(VIDEO),
|
|
);
|
|
expect(readJSON(join(day, "2024-03-01.300.livephoto.json"))).toEqual({
|
|
image: "2024-03-01.300.heic",
|
|
video: "2024-03-01.300.mov",
|
|
});
|
|
});
|
|
|
|
// A file's JSON holds what Ente records for it, decrypted, and what quak
|
|
// read from its original: its ML data when Ente has any, and, for an
|
|
// image, its EXIF. A video gets no image metadata.
|
|
it("writes each file's JSON with its ML data and image metadata", () => {
|
|
const day = join(dir, DAY);
|
|
|
|
const beach = readJSON(join(day, "2024-03-01.100.json"));
|
|
expect(beach).toMatchObject({
|
|
id: 100,
|
|
ownerID: USER_ID,
|
|
metadata: {
|
|
title: "beach.jpg",
|
|
fileType: "image",
|
|
creationTime: TAKEN,
|
|
hash: BEACH.metadata.hash,
|
|
},
|
|
updationTime: TAKEN,
|
|
});
|
|
expect(beach.mlData).toEqual(ML_DATA);
|
|
expect(beach.imageMetadata.exif.Make.description).toBe("Canon");
|
|
expect(beach.imageMetadata.exif.Model.description).toBe("EOS R5");
|
|
|
|
const toast = readJSON(join(day, "2024-03-01.200.json"));
|
|
expect(toast).toMatchObject({
|
|
id: 200,
|
|
ownerID: FRIEND_ID,
|
|
metadata: { title: "toast.mp4", fileType: "video" },
|
|
});
|
|
expect(toast).not.toHaveProperty("mlData");
|
|
expect(toast).not.toHaveProperty("imageMetadata");
|
|
expect(toast).not.toHaveProperty("imageMetadataError");
|
|
|
|
// A live photo's image metadata is read from its image.
|
|
const livePhoto = readJSON(join(day, "2024-03-01.300.json"));
|
|
expect(livePhoto).toMatchObject({
|
|
id: 300,
|
|
ownerID: USER_ID,
|
|
metadata: {
|
|
title: "IMG_0002.HEIC",
|
|
fileType: "livePhoto",
|
|
hash: LIVE_PHOTO.metadata.hash,
|
|
},
|
|
});
|
|
expect(livePhoto.imageMetadata.exif.Make.description).toBe("Canon");
|
|
});
|
|
|
|
it("writes each album's JSON and the account's", () => {
|
|
const albumJSON = (name: string) =>
|
|
readJSON(join(dir, "collections", `${name}.json`));
|
|
const fileIDs = (album: { files: { id: number }[] }) =>
|
|
album.files.map((f) => f.id);
|
|
|
|
const trips = albumJSON("Trips");
|
|
expect(trips).toMatchObject({
|
|
id: 10,
|
|
name: "Trips",
|
|
type: "album",
|
|
ownerID: USER_ID,
|
|
isShared: false,
|
|
updationTime: TAKEN,
|
|
});
|
|
expect(fileIDs(trips)).toEqual([100]);
|
|
|
|
const family = albumJSON("Family");
|
|
expect(family).toMatchObject({ id: 20, isShared: false });
|
|
expect(fileIDs(family)).toEqual([100, 300]);
|
|
|
|
const wedding = albumJSON("Friend's Wedding");
|
|
expect(wedding).toMatchObject({
|
|
id: 30,
|
|
ownerID: FRIEND_ID,
|
|
isShared: true,
|
|
});
|
|
expect(fileIDs(wedding)).toEqual([200]);
|
|
|
|
expect(readJSON(join(dir, "account.json"))).toEqual({
|
|
email: EMAIL,
|
|
userID: USER_ID,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("quak backup", () => {
|
|
// An original already at its save path is complete, so a later run, such
|
|
// as the next night's from cron, only rewrites the JSON files and the
|
|
// album links.
|
|
it("downloads nothing on a second run, and exits 0", async () => {
|
|
const dir = mkdtempSync(join(root, "backup-"));
|
|
expect((await backup(dir)).code).toBe(0);
|
|
const firstTree = listTree(dir);
|
|
|
|
const downloads: number[] = [];
|
|
const { code } = await backup(dir, {}, recordingDownloads(downloads));
|
|
expect(code).toBe(0);
|
|
expect(downloads).toEqual([]);
|
|
expect(listTree(dir)).toEqual(firstTree);
|
|
});
|
|
|
|
// The files CDN is briefly unavailable for toast.mp4: its first two
|
|
// downloads answer 503. A 5xx is worth repeating, so quak tries again,
|
|
// here without waiting, and the third download succeeds.
|
|
it("stores a file whose download fails twice with 503 and then succeeds, and exits 0", async () => {
|
|
let attempts = 0;
|
|
const fetch: typeof globalThis.fetch = async (input, init) => {
|
|
if (downloadOf(input) === 200 && ++attempts <= 2) {
|
|
return new Response("unavailable", { status: 503 });
|
|
}
|
|
return server(input, init);
|
|
};
|
|
const dir = mkdtempSync(join(root, "backup-"));
|
|
|
|
const { code } = await backup(dir, {}, fetch);
|
|
|
|
expect(code).toBe(0);
|
|
expect(attempts).toBe(3);
|
|
expect(readFileSync(join(dir, DAY, "2024-03-01.200.mp4"))).toEqual(
|
|
Buffer.from(TOAST),
|
|
);
|
|
expect(existsSync(join(dir, "failures.json"))).toBe(false);
|
|
});
|
|
|
|
// The server answers 404 for toast.mp4 on every try. A 404 is an answer,
|
|
// not a hiccup, so quak does not repeat it: the file goes into
|
|
// `failures.json` with the kind of failure, how many times it has been
|
|
// tried and why, the backup goes on with the other files, and the exit
|
|
// code is 1, so the next run tries again.
|
|
it("records a file whose download always answers 404 in failures.json, stores the rest, and exits 1", async () => {
|
|
let attempts = 0;
|
|
const fetch: typeof globalThis.fetch = async (input, init) => {
|
|
if (downloadOf(input) === 200) {
|
|
attempts++;
|
|
return new Response("not found", { status: 404 });
|
|
}
|
|
return server(input, init);
|
|
};
|
|
const dir = mkdtempSync(join(root, "backup-"));
|
|
|
|
const { code, stderr } = await backup(dir, {}, fetch);
|
|
|
|
expect(code).toBe(1);
|
|
expect(attempts).toBe(1);
|
|
const failures = readJSON(join(dir, "failures.json"));
|
|
expect(Object.keys(failures.files)).toEqual(["200"]);
|
|
expect(failures.files["200"]).toMatchObject({
|
|
fileID: 200,
|
|
title: "toast.mp4",
|
|
classification: "permanent",
|
|
attempts: 1,
|
|
});
|
|
expect(failures.files["200"].error).toContain("404");
|
|
expect(stderr).toContain("[Friend's Wedding] toast.mp4 (id 200): ");
|
|
|
|
// The rest of the account is backed up; the video is not, and its
|
|
// album folder has no link to it.
|
|
expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual(
|
|
Buffer.from(JPEG_WITH_EXIF),
|
|
);
|
|
expect(readFileSync(join(dir, DAY, "2024-03-01.300.heic"))).toEqual(
|
|
Buffer.from(HEIC_WITH_EXIF),
|
|
);
|
|
expect(existsSync(join(dir, DAY, "2024-03-01.200.mp4"))).toBe(false);
|
|
expect(
|
|
readdirSync(join(dir, "collections", "Friend's Wedding")),
|
|
).toEqual([]);
|
|
});
|
|
|
|
// The server no longer accepts the saved session's token, as after
|
|
// `quak logout` elsewhere, and answers every request with 401. The run
|
|
// stops at the refresh it starts with, says to log in again, and exits 3.
|
|
// It took its lock first, so every request was sent while the lock was
|
|
// there, and it removed the lock as it ended: the directory is left as it
|
|
// was, empty.
|
|
it("exits 3 when the refresh is answered with 401, writing nothing but the lock", async () => {
|
|
const dir = mkdtempSync(join(root, "backup-"));
|
|
const lockAtEachRequest: boolean[] = [];
|
|
const fetch: typeof globalThis.fetch = async () => {
|
|
lockAtEachRequest.push(existsSync(join(dir, "backup.lock")));
|
|
return new Response(null, { status: 401 });
|
|
};
|
|
|
|
const { code, stderr } = await backup(dir, {}, fetch);
|
|
|
|
expect(code).toBe(3);
|
|
expect(stderr).toBe(
|
|
"Starting backup...\n" +
|
|
"Refreshing library...\n" +
|
|
`quak: the saved session is no longer valid; run "quak login"\n`,
|
|
);
|
|
expect(lockAtEachRequest).not.toHaveLength(0);
|
|
expect(lockAtEachRequest).not.toContain(false);
|
|
expect(readdirSync(dir)).toEqual([]);
|
|
});
|
|
|
|
// Something changed beach.jpg on disk after it was backed up. A run with
|
|
// --verify hashes every stored original, finds that one no longer matches
|
|
// the hash Ente recorded, says so, and downloads it again; it put back the
|
|
// right bytes, so the exit code is 0.
|
|
it("--verify puts back an original that changed on disk, and exits 0", async () => {
|
|
const dir = mkdtempSync(join(root, "backup-"));
|
|
expect((await backup(dir)).code).toBe(0);
|
|
const beach = join(dir, DAY, "2024-03-01.100.jpg");
|
|
writeFileSync(beach, "not the photo any more");
|
|
|
|
const downloads: number[] = [];
|
|
const { code, stderr } = await backup(
|
|
dir,
|
|
{ verify: true },
|
|
recordingDownloads(downloads),
|
|
);
|
|
|
|
expect(code).toBe(0);
|
|
expect(downloads).toEqual([100]);
|
|
expect(readFileSync(beach)).toEqual(Buffer.from(JPEG_WITH_EXIF));
|
|
expect(stderr).toContain(
|
|
"MISMATCH original beach.jpg (100): its bytes do not match its content hash",
|
|
);
|
|
expect(stderr).toContain(" Verified: 2\n Mismatched: 1\n");
|
|
});
|
|
});
|
|
|
|
// `backup.lock` is a directory that proper-lockfile touches every 5 seconds,
|
|
// from a timer, while the backup runs. Each time, it first looks at the lock
|
|
// on disk. The lock is lost when it is gone, or when another backup has made
|
|
// its own in its place, as one does with a lock nobody touched for 10
|
|
// seconds. Another backup may then be writing to the directory, so the backup
|
|
// stops before it starts another file.
|
|
//
|
|
// The fake server holds one file's download open until the test lets it go,
|
|
// so the lock can be lost while the backup runs: beach.jpg's, the first file
|
|
// backed up, unless the test names another. Only `setTimeout` and
|
|
// `clearTimeout` are faked, so a test can fire proper-lockfile's timer without
|
|
// waiting 5 seconds.
|
|
describe("a backup whose lock is lost", () => {
|
|
let dir: string;
|
|
let lockPath: string;
|
|
let downloads: number[];
|
|
let heldFile: number;
|
|
let heldRequested: Promise<void>;
|
|
let releaseHeld: () => void;
|
|
let fetch: typeof globalThis.fetch;
|
|
|
|
beforeEach(() => {
|
|
dir = mkdtempSync(join(root, "backup-"));
|
|
lockPath = join(dir, "backup.lock");
|
|
downloads = [];
|
|
heldFile = BEACH.id;
|
|
let requested!: () => void;
|
|
heldRequested = new Promise((resolve) => {
|
|
requested = resolve;
|
|
});
|
|
const released = new Promise<void>((resolve) => {
|
|
releaseHeld = resolve;
|
|
});
|
|
const recording = recordingDownloads(downloads);
|
|
fetch = async (input, init) => {
|
|
if (downloadOf(input) === heldFile) {
|
|
requested();
|
|
await released;
|
|
}
|
|
return recording(input, init);
|
|
};
|
|
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
// Fire proper-lockfile's timer, and wait for the look at the lock on disk
|
|
// that it starts to finish: Node lists a file system call in progress as
|
|
// an `FSReqCallback`.
|
|
const fireLockTimer = async (): Promise<void> => {
|
|
vi.advanceTimersByTime(5_000);
|
|
while (process.getActiveResourcesInfo().includes("FSReqCallback")) {
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
}
|
|
};
|
|
|
|
// The backup downloads up to five originals at once, so all three are
|
|
// under way when the lock is lost, and beach.jpg is stored. The backup
|
|
// then stops before it fetches ML data. `quak backup` prints why as one
|
|
// line, with no stack trace, and exits 1.
|
|
it("stops quak backup before it fetches ML data when backup.lock is removed, and exits 1", async () => {
|
|
const result = backup(dir, {}, fetch);
|
|
await heldRequested;
|
|
rmSync(lockPath, { recursive: true });
|
|
await fireLockTimer();
|
|
releaseHeld();
|
|
const { code, stderr } = await result;
|
|
|
|
expect(code).toBe(1);
|
|
expect(stderr).toBe(
|
|
"Starting backup...\n" +
|
|
"Refreshing library...\n" +
|
|
"Fetching original beach.jpg (100)...\n" +
|
|
"Fetching original IMG_0002.HEIC (300)...\n" +
|
|
"Fetching original toast.mp4 (200)...\n" +
|
|
`quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'\n`,
|
|
);
|
|
expect(downloads.sort()).toEqual([100, 200, 300]);
|
|
expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual(
|
|
Buffer.from(JPEG_WITH_EXIF),
|
|
);
|
|
});
|
|
|
|
// Here the lock is lost while toast.mp4, the last original, downloads.
|
|
// All three originals are stored, beside what the backup wrote before it
|
|
// started them, but no file's JSON and no album is written.
|
|
it("stops quak backup before any file's JSON when the lock is lost during the last original", async () => {
|
|
heldFile = TOAST_VIDEO.id;
|
|
const result = backup(dir, {}, fetch);
|
|
await heldRequested;
|
|
rmSync(lockPath, { recursive: true });
|
|
await fireLockTimer();
|
|
releaseHeld();
|
|
const { code, stderr } = await result;
|
|
|
|
expect(code).toBe(1);
|
|
expect(stderr).toContain(`quak: lost the lock on ${dir}: `);
|
|
expect(listTree(dir)).toEqual([
|
|
"2024",
|
|
"2024/2024-03",
|
|
"2024/2024-03/2024-03-01",
|
|
"2024/2024-03/2024-03-01/2024-03-01.100.jpg",
|
|
"2024/2024-03/2024-03-01/2024-03-01.200.mp4",
|
|
"2024/2024-03/2024-03-01/2024-03-01.300.heic",
|
|
"2024/2024-03/2024-03-01/2024-03-01.300.livephoto.json",
|
|
"2024/2024-03/2024-03-01/2024-03-01.300.mov",
|
|
"account.json",
|
|
"collections",
|
|
]);
|
|
});
|
|
|
|
// Nothing touches the lock but quak, which finds it as it left it each
|
|
// time it looks, so the backup runs to the end and removes its lock.
|
|
it("lets quak backup run to the end while its lock stays", async () => {
|
|
const result = backup(dir, {}, fetch);
|
|
await heldRequested;
|
|
await fireLockTimer();
|
|
await fireLockTimer();
|
|
releaseHeld();
|
|
const { code } = await result;
|
|
|
|
expect(code).toBe(0);
|
|
expect(downloads.sort()).toEqual([100, 200, 300]);
|
|
expect(existsSync(lockPath)).toBe(false);
|
|
});
|
|
|
|
// The library a program using quak opens to back `dir` up.
|
|
const openLibrary = () =>
|
|
Library.open({
|
|
client: loadSession(join(sessionDir, "session.json"), { fetch })!,
|
|
cacheDirectory: `${dir}-cache`,
|
|
downloadDirectory: dir,
|
|
refreshIntervalSeconds: Infinity,
|
|
precacheThumbnails: false,
|
|
precacheOriginals: false,
|
|
});
|
|
|
|
// A program using quak as a library sees the same: `lib.backup()` rejects
|
|
// with the reason. Here another backup has made its own lock in place of
|
|
// quak's, and that lock is left where it is.
|
|
it("rejects lib.backup() when another backup has taken the lock over", async () => {
|
|
const lib = await openLibrary();
|
|
try {
|
|
const result = lib.backup();
|
|
await heldRequested;
|
|
rmSync(lockPath, { recursive: true });
|
|
mkdirSync(lockPath);
|
|
await fireLockTimer();
|
|
releaseHeld();
|
|
|
|
await expect(result).rejects.toThrow(
|
|
`lost the lock on ${dir}: Unable to update lock within the stale threshold`,
|
|
);
|
|
expect(downloads.sort()).toEqual([100, 200, 300]);
|
|
expect(existsSync(lockPath)).toBe(true);
|
|
} finally {
|
|
await lib.close();
|
|
}
|
|
});
|
|
|
|
// A program can take the lock before it opens its library, as `quak
|
|
// backup` does: `lockBackupDirectory` returns the lock, which the program
|
|
// passes to `lib.backup()` and releases once the backup ends. The backup
|
|
// checks it as it checks a lock it took, and the lock's `lost` says why it
|
|
// was lost.
|
|
it("rejects lib.backup() given its caller's lock once that lock is lost", async () => {
|
|
const lock = await lockBackupDirectory(dir);
|
|
const lib = await openLibrary();
|
|
try {
|
|
const result = lib.backup({ lock });
|
|
await heldRequested;
|
|
rmSync(lockPath, { recursive: true });
|
|
await fireLockTimer();
|
|
releaseHeld();
|
|
|
|
const lost = `lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'`;
|
|
await expect(result).rejects.toThrow(lost);
|
|
expect(lock.lost?.message).toBe(lost);
|
|
expect(downloads.sort()).toEqual([100, 200, 300]);
|
|
} finally {
|
|
await lib.close();
|
|
await lock.release();
|
|
}
|
|
});
|
|
});
|