clawbot
bb3b5cd2da
check / check (push) Waiting to run
A file or album whose key, metadata or name does not decrypt used to throw out of the refresh, so quak backup stopped before storing anything, every run. Client.collectionsSince and filesSince now return such a row in undecryptable with the reason, the refresh leaves it out, and the store keeps the list in metadata.json until a later refresh decrypts or deletes the record. lib.backup() records each such file in failures.json unless a copy in another album decrypted, in which case it stores the original from that copy. Each such album is listed in BackupResult.undecryptableAlbums and printed as one FAILED album line. quak backup exits 1 while either remains. An album or file that no longer decrypts loses its album folder, JSON and links; its original stays. Model: opus-5-5
2033 lines
75 KiB
TypeScript
2033 lines
75 KiB
TypeScript
/**
|
|
* Tests for the `quak backup` logic, now built on the library API (issue #51).
|
|
*
|
|
* `lib.backup({ downloadDirectory })` refreshes the library, puts each pending
|
|
* file's original at its save path through the content cache/pools, and
|
|
* materialises the on-disk layout:
|
|
*
|
|
* <downloadDirectory>/
|
|
* YYYY/YYYY-MM/YYYY-MM-DD/
|
|
* YYYY-MM-DD.<fileID>.<ext> the decrypted bytes ("present means complete")
|
|
* YYYY-MM-DD.<fileID>.json per-file metadata sidecar (rebuilt each run)
|
|
* collections/
|
|
* <name>/<title> symlink to the original (rebuilt each run)
|
|
* <name>.json per-collection metadata (rebuilt each run)
|
|
* account.json the account's email and user ID
|
|
* failures.json durable ledger of unresolved failures
|
|
*
|
|
* The properties that distinguish backup from a naive download loop, and that
|
|
* these tests lock down:
|
|
*
|
|
* 1. Present-means-complete: an original already on disk is not re-fetched, so
|
|
* runs are idempotent and interrupted runs resume.
|
|
* 2. Per-file resilience: a download failure or a symlink failure is recorded
|
|
* and the run continues (issue #8); the derived symlink/JSON views are
|
|
* rebuilt from the model every run.
|
|
* 3. A durable `failures.json` records each unresolved failure's classification,
|
|
* attempt count, and last-tried time; the exit code (result.failed) is
|
|
* non-zero while any failure remains and clears once every one is resolved.
|
|
*
|
|
* The cache and download layers are covered elsewhere (content.test.ts,
|
|
* download tests); here a mock library client and a stand-in content source
|
|
* drive the backup logic with no crypto and no network.
|
|
*/
|
|
|
|
import {
|
|
chmodSync,
|
|
existsSync,
|
|
lstatSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readdirSync,
|
|
readFileSync,
|
|
readlinkSync,
|
|
rmSync,
|
|
symlinkSync,
|
|
utimesSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { spawnSync } from "node:child_process";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
|
|
|
import { runBackup, type BackupLibrary } from "../../src/backup.js";
|
|
import { Library } from "../../src/library/index.js";
|
|
import type { ContentSource } from "../../src/library/content.js";
|
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
|
import { extractImageMetadata } from "../../src/metadata-backup.js";
|
|
import type { MLData } from "../../src/mldata-fetch.js";
|
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
|
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
|
import { JPEG_WITH_EXIF } from "../exif-jpeg.js";
|
|
import {
|
|
asLivePhoto,
|
|
blake2b,
|
|
cdnSource,
|
|
IMAGE,
|
|
livePhotoHash,
|
|
livePhotoZip,
|
|
VIDEO,
|
|
} from "../live-photo.js";
|
|
|
|
// `open` and `rename` are wrapped to record, in order, every fsync and rename,
|
|
// so a test can pin the sequence "fsync the temp file, rename, fsync the
|
|
// directory" that makes a copied original survive a power cut. `vi.hoisted`
|
|
// because `vi.mock` factories run before module-level constants exist.
|
|
const fsEvents = vi.hoisted(() => [] as string[]);
|
|
|
|
vi.mock("node:fs/promises", async (importOriginal) => {
|
|
const actual = await importOriginal<typeof import("node:fs/promises")>();
|
|
return {
|
|
...actual,
|
|
open: async (
|
|
...args: Parameters<typeof actual.open>
|
|
): Promise<Awaited<ReturnType<typeof actual.open>>> => {
|
|
const handle = await actual.open(...args);
|
|
const realSync = handle.sync.bind(handle);
|
|
handle.sync = async (): Promise<void> => {
|
|
fsEvents.push(`sync:${String(args[0])}`);
|
|
await realSync();
|
|
};
|
|
return handle;
|
|
},
|
|
rename: async (from: string, to: string): Promise<void> => {
|
|
fsEvents.push(`rename:${to}`);
|
|
await actual.rename(from, to);
|
|
},
|
|
};
|
|
});
|
|
|
|
const USER_ID = 42;
|
|
|
|
// Decrypted-byte length each stub original writes, keyed by fileID.
|
|
const SIZE_BY_ID: Record<number, number> = { 100: 3000, 101: 2000, 200: 1500 };
|
|
|
|
// Every file is taken at noon local time on 2026-03-01, in microseconds as Ente
|
|
// stores times, so the machine's time zone cannot move it to another day. Its
|
|
// original is saved in the folder `DAY`, named `2026-03-01.<fileID>.<ext>`.
|
|
const TAKEN = new Date(2026, 2, 1, 12).getTime() * 1000;
|
|
const DAY = join("2026", "2026-03", "2026-03-01");
|
|
|
|
// Where the backup in `outDir` saves `name` (`<fileID>.<ext>`), and the target
|
|
// of an album folder's symlink to it.
|
|
const saved = (outDir: string, name: string): string =>
|
|
join(outDir, DAY, `2026-03-01.${name}`);
|
|
const linkTo = (name: string): string => `../../${DAY}/2026-03-01.${name}`;
|
|
|
|
const collection = (id: number, name: string): Collection => ({
|
|
id,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
name,
|
|
type: "album",
|
|
updationTime: 1,
|
|
isShared: false,
|
|
});
|
|
|
|
const file = (id: number, collectionID: number, title: string): EnteFile => ({
|
|
id,
|
|
collectionID,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
metadata: {
|
|
title,
|
|
fileType: "image",
|
|
creationTime: TAKEN,
|
|
modificationTime: 1,
|
|
},
|
|
file: { decryptionHeader: "aGVhZGVy" },
|
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
|
updationTime: 1,
|
|
});
|
|
|
|
// A metadata-only client: two albums, three files, served once. No ML.
|
|
class MockClient {
|
|
private served = false;
|
|
whoami(): { email: string; userID: number } {
|
|
return { email: "backup@example.com", userID: USER_ID };
|
|
}
|
|
async collectionsSince(): Promise<CollectionsPage> {
|
|
if (this.served) return { collections: [], deleted: [], cursor: 1 };
|
|
this.served = true;
|
|
return {
|
|
collections: [collection(1, "Vacation"), collection(2, "Work")],
|
|
deleted: [],
|
|
cursor: 1,
|
|
};
|
|
}
|
|
async filesSince(args: { collectionID: number }): Promise<FilesPage> {
|
|
const files =
|
|
args.collectionID === 1
|
|
? [file(100, 1, "beach.jpg"), file(101, 1, "sunset.jpg")]
|
|
: args.collectionID === 2
|
|
? [file(200, 2, "diagram.png")]
|
|
: [];
|
|
return { files, deleted: [], cursor: 1 };
|
|
}
|
|
}
|
|
|
|
// A server that names an album and a file so as to climb out of the backup
|
|
// directory.
|
|
class HostileClient extends MockClient {
|
|
override async collectionsSince(): Promise<CollectionsPage> {
|
|
const page = await super.collectionsSince();
|
|
return {
|
|
...page,
|
|
collections: page.collections.length
|
|
? [collection(3, "../escape")]
|
|
: [],
|
|
};
|
|
}
|
|
override async filesSince(args: {
|
|
collectionID: number;
|
|
}): Promise<FilesPage> {
|
|
const files =
|
|
args.collectionID === 3
|
|
? [file(300, 3, "../../.ssh/authorized_keys")]
|
|
: [];
|
|
return { files, deleted: [], cursor: 1 };
|
|
}
|
|
}
|
|
|
|
// A content source that writes byte buffers of the expected length and can be
|
|
// told to fail one fileID's original, to exercise per-file resilience.
|
|
interface StubSource extends ContentSource {
|
|
failID?: number;
|
|
failThumbID?: number;
|
|
originalCalls: number;
|
|
}
|
|
|
|
const stubSource = (): StubSource => {
|
|
const s: StubSource = {
|
|
originalCalls: 0,
|
|
original: async ({ file: f, destination }) => {
|
|
s.originalCalls++;
|
|
if (s.failID === f.id) throw new Error("HTTP 500 from server");
|
|
const size = SIZE_BY_ID[f.id] ?? 10;
|
|
writeFileSync(destination, Buffer.alloc(size));
|
|
return { bytesWritten: size };
|
|
},
|
|
thumbnail: async ({ file: f, destination }) => {
|
|
if (s.failThumbID === f.id) throw new Error("HTTP 500 from server");
|
|
writeFileSync(destination, Buffer.alloc(5));
|
|
return { bytesWritten: 5 };
|
|
},
|
|
};
|
|
return s;
|
|
};
|
|
|
|
let root: string;
|
|
|
|
const openLibrary = (
|
|
source: ContentSource,
|
|
client: MockClient = new MockClient(),
|
|
): Promise<Library> =>
|
|
Library.open({
|
|
client,
|
|
cacheDirectory: join(root, "cache"),
|
|
contentSource: source,
|
|
refreshIntervalSeconds: 3600,
|
|
// These tests count exact fetches; the background precache (#48) would
|
|
// add its own, so it is off here (it is covered in precache.test.ts).
|
|
precacheThumbnails: false,
|
|
precacheOriginals: false,
|
|
});
|
|
|
|
const readLedger = (
|
|
outDir: string,
|
|
): { files: Record<string, Record<string, unknown>> } =>
|
|
JSON.parse(readFileSync(join(outDir, "failures.json"), "utf-8"));
|
|
|
|
// Write a durable ledger holding one prior failure, to exercise pruning of
|
|
// entries the current run cannot resolve.
|
|
const seedLedger = (outDir: string, fileID: number, title: string): void => {
|
|
mkdirSync(outDir, { recursive: true });
|
|
writeFileSync(
|
|
join(outDir, "failures.json"),
|
|
JSON.stringify({
|
|
version: 1,
|
|
files: {
|
|
[String(fileID)]: {
|
|
fileID,
|
|
title,
|
|
classification: "transient",
|
|
attempts: 1,
|
|
lastTriedAt: Date.now(),
|
|
error: "HTTP 500 from server",
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
};
|
|
|
|
beforeEach(() => {
|
|
root = mkdtempSync(join(tmpdir(), "quak-backup-test-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
if (root && existsSync(root))
|
|
rmSync(root, { recursive: true, force: true });
|
|
});
|
|
|
|
describe("lib.backup", () => {
|
|
it("throws before any network when no downloadDirectory is given", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
// The library always has a download directory, so only an empty one
|
|
// given to backup() reaches runBackup as none.
|
|
await expect(lib.backup({ downloadDirectory: "" })).rejects.toThrow(
|
|
/downloadDirectory/i,
|
|
);
|
|
expect(source.originalCalls).toBe(0);
|
|
lib.close();
|
|
});
|
|
|
|
it("writes the expected on-disk layout for every file", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.totalFiles).toBe(3);
|
|
expect(result.downloaded).toBe(3);
|
|
expect(result.skipped).toBe(0);
|
|
expect(result.failed).toBe(0);
|
|
expect(result.errors).toEqual([]);
|
|
|
|
// Originals at YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>.
|
|
expect(readFileSync(saved(outDir, "100.jpg")).length).toBe(3000);
|
|
expect(readFileSync(saved(outDir, "101.jpg")).length).toBe(2000);
|
|
expect(readFileSync(saved(outDir, "200.png")).length).toBe(1500);
|
|
|
|
// Per-file metadata sidecar.
|
|
const sidecar = JSON.parse(
|
|
readFileSync(saved(outDir, "100.json"), "utf-8"),
|
|
);
|
|
expect(sidecar.id).toBe(100);
|
|
expect(sidecar.metadata.title).toBe("beach.jpg");
|
|
|
|
// Collection dirs contain symlinks into the date folders.
|
|
const beach = join(outDir, "collections", "Vacation", "beach.jpg");
|
|
expect(lstatSync(beach).isSymbolicLink()).toBe(true);
|
|
expect(readlinkSync(beach)).toContain(DAY);
|
|
expect(readFileSync(beach).length).toBe(3000);
|
|
|
|
// Per-collection metadata JSON.
|
|
const vacation = JSON.parse(
|
|
readFileSync(join(outDir, "collections", "Vacation.json"), "utf-8"),
|
|
);
|
|
expect(vacation.name).toBe("Vacation");
|
|
expect(vacation.files.length).toBe(2);
|
|
expect(vacation.files[0].metadata.title).toBeDefined();
|
|
|
|
// A clean run leaves no failure ledger behind.
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
lib.close();
|
|
});
|
|
|
|
it("keeps server-supplied album and file names inside the backup", async () => {
|
|
const lib = await openLibrary(stubSource(), new HostileClient());
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.failed).toBe(0);
|
|
// The title has no usable extension, so the original is `.bin`.
|
|
expect(existsSync(saved(outDir, "300.bin"))).toBe(true);
|
|
const link = join(
|
|
outDir,
|
|
"collections",
|
|
"__escape",
|
|
"__.._.ssh_authorized_keys",
|
|
);
|
|
expect(lstatSync(link).isSymbolicLink()).toBe(true);
|
|
expect(existsSync(join(outDir, "collections", "__escape.json"))).toBe(
|
|
true,
|
|
);
|
|
// Nothing landed beside or above the backup directory.
|
|
expect(readdirSync(root).sort()).toEqual(["backup", "cache"]);
|
|
lib.close();
|
|
});
|
|
|
|
it("is an idempotent no-op when every original is already present", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const first = await lib.backup({ downloadDirectory: outDir });
|
|
expect(first.downloaded).toBe(3);
|
|
const callsAfterFirst = source.originalCalls;
|
|
|
|
const second = await lib.backup({ downloadDirectory: outDir });
|
|
expect(second.downloaded).toBe(0);
|
|
expect(second.skipped).toBe(3);
|
|
expect(second.failed).toBe(0);
|
|
// A present original is neither fetched nor copied again.
|
|
expect(source.originalCalls).toBe(callsAfterFirst);
|
|
lib.close();
|
|
});
|
|
|
|
it("continues past a download failure and records it in failures.json", async () => {
|
|
const source = stubSource();
|
|
source.failID = 101;
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.totalFiles).toBe(3);
|
|
expect(result.downloaded).toBe(2);
|
|
expect(result.failed).toBe(1);
|
|
expect(result.errors.length).toBe(1);
|
|
expect(result.errors[0]!.fileID).toBe(101);
|
|
expect(result.errors[0]!.title).toBe("sunset.jpg");
|
|
|
|
// The two good files are on disk; the failed one is not.
|
|
expect(existsSync(saved(outDir, "100.jpg"))).toBe(true);
|
|
expect(existsSync(saved(outDir, "200.png"))).toBe(true);
|
|
expect(existsSync(saved(outDir, "101.jpg"))).toBe(false);
|
|
expect(
|
|
existsSync(join(outDir, "collections", "Vacation", "sunset.jpg")),
|
|
).toBe(false);
|
|
|
|
// Durable ledger with classification, attempts, last-tried.
|
|
const ledger = readLedger(outDir);
|
|
const entry = ledger.files["101"]!;
|
|
expect(entry.attempts).toBe(1);
|
|
expect(entry.classification).toBeDefined();
|
|
expect(typeof entry.lastTriedAt).toBe("number");
|
|
lib.close();
|
|
});
|
|
|
|
it("increments the attempt count across runs and clears the ledger once resolved", async () => {
|
|
const source = stubSource();
|
|
source.failID = 101;
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const r1 = await lib.backup({ downloadDirectory: outDir });
|
|
expect(r1.failed).toBe(1);
|
|
expect(readLedger(outDir).files["101"]!.attempts).toBe(1);
|
|
|
|
// Second run: the two good files are present, only 101 is retried.
|
|
const r2 = await lib.backup({ downloadDirectory: outDir });
|
|
expect(r2.failed).toBe(1);
|
|
expect(r2.skipped).toBe(2);
|
|
expect(readLedger(outDir).files["101"]!.attempts).toBe(2);
|
|
|
|
// Resume with a healthy source: 101 downloads, the rest are skipped.
|
|
source.failID = undefined;
|
|
const r3 = await lib.backup({ downloadDirectory: outDir });
|
|
expect(r3.failed).toBe(0);
|
|
expect(r3.skipped).toBe(2);
|
|
expect(existsSync(saved(outDir, "101.jpg"))).toBe(true);
|
|
// A ledger with no remaining failures is removed.
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
lib.close();
|
|
});
|
|
|
|
it("does not abort when a symlink cannot be created (issue #8)", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
// Occupy beach.jpg's symlink path with a directory so symlink creation
|
|
// fails for that one file.
|
|
mkdirSync(join(outDir, "collections", "Vacation", "beach.jpg"), {
|
|
recursive: true,
|
|
});
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
// Every original still downloads despite the symlink failure.
|
|
expect(existsSync(saved(outDir, "100.jpg"))).toBe(true);
|
|
expect(existsSync(saved(outDir, "200.png"))).toBe(true);
|
|
// The other symlinks are still built.
|
|
expect(
|
|
lstatSync(
|
|
join(outDir, "collections", "Vacation", "sunset.jpg"),
|
|
).isSymbolicLink(),
|
|
).toBe(true);
|
|
expect(
|
|
lstatSync(
|
|
join(outDir, "collections", "Work", "diagram.png"),
|
|
).isSymbolicLink(),
|
|
).toBe(true);
|
|
|
|
// The symlink failure is recorded, not thrown.
|
|
expect(result.failed).toBeGreaterThanOrEqual(1);
|
|
const err = result.errors.find((e) => e.fileID === 100);
|
|
expect(err).toBeDefined();
|
|
expect(err!.collection).toBe("Vacation");
|
|
expect(readLedger(outDir).files["100"]).toBeDefined();
|
|
lib.close();
|
|
});
|
|
|
|
it("rebuilds a stale sidecar and a missing symlink on a later run", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
// Corrupt a sidecar and delete a symlink between runs.
|
|
writeFileSync(saved(outDir, "100.json"), "not json");
|
|
rmSync(join(outDir, "collections", "Vacation", "beach.jpg"));
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
expect(result.failed).toBe(0);
|
|
|
|
// The derived views are repaired from the model.
|
|
const sidecar = JSON.parse(
|
|
readFileSync(saved(outDir, "100.json"), "utf-8"),
|
|
);
|
|
expect(sidecar.metadata.title).toBe("beach.jpg");
|
|
expect(
|
|
lstatSync(
|
|
join(outDir, "collections", "Vacation", "beach.jpg"),
|
|
).isSymbolicLink(),
|
|
).toBe(true);
|
|
lib.close();
|
|
});
|
|
|
|
it("backs up only the named albums when onlyAlbumNames is given", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
onlyAlbumNames: ["Work"],
|
|
});
|
|
|
|
expect(result.totalFiles).toBe(1);
|
|
expect(result.downloaded).toBe(1);
|
|
expect(existsSync(saved(outDir, "200.png"))).toBe(true);
|
|
expect(existsSync(saved(outDir, "100.jpg"))).toBe(false);
|
|
expect(existsSync(join(outDir, "collections", "Work.json"))).toBe(true);
|
|
expect(existsSync(join(outDir, "collections", "Vacation.json"))).toBe(
|
|
false,
|
|
);
|
|
lib.close();
|
|
});
|
|
|
|
it("prunes a ledger entry for a file no longer in the library and exits zero", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
// A prior failure for a file that has since left the library (deleted
|
|
// from the account). This run has no way to resolve it, so it must not
|
|
// keep the exit code non-zero forever.
|
|
seedLedger(outDir, 999, "gone.jpg");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
// Everything still present is backed up cleanly, and the stale entry is
|
|
// dropped rather than counted.
|
|
expect(result.downloaded).toBe(3);
|
|
expect(result.failed).toBe(0);
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
lib.close();
|
|
});
|
|
|
|
it("prunes an out-of-scope ledger entry on a scoped run and exits zero", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
// A prior failure for a Vacation file; this run is scoped to Work and
|
|
// never attempts it, so it must not poison the scoped run's exit code.
|
|
seedLedger(outDir, 100, "beach.jpg");
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
onlyAlbumNames: ["Work"],
|
|
});
|
|
|
|
expect(result.totalFiles).toBe(1);
|
|
expect(result.failed).toBe(0);
|
|
expect(existsSync(saved(outDir, "200.png"))).toBe(true);
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
lib.close();
|
|
});
|
|
|
|
it("also stores thumbnails when includeThumbnails is set", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({
|
|
downloadDirectory: outDir,
|
|
includeThumbnails: true,
|
|
});
|
|
|
|
expect(existsSync(join(outDir, "thumbnails", "100.jpg"))).toBe(true);
|
|
expect(existsSync(join(outDir, "thumbnails", "200.jpg"))).toBe(true);
|
|
lib.close();
|
|
});
|
|
|
|
it("counts one attempt when a file fails both its original and thumbnail in a run", async () => {
|
|
const source = stubSource();
|
|
source.failID = 101;
|
|
source.failThumbID = 101;
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
includeThumbnails: true,
|
|
});
|
|
|
|
// Both kinds fail for 101, but the run counts it once.
|
|
const errs = result.errors.filter((e) => e.fileID === 101);
|
|
expect(errs.length).toBe(1);
|
|
expect(readLedger(outDir).files["101"]!.attempts).toBe(1);
|
|
lib.close();
|
|
});
|
|
|
|
it("fetches each original once and writes it only into the backup", async () => {
|
|
// A backup of a 500 GB account must write 500 GB, not a copy in the
|
|
// cache as well: an original fetched for the backup goes straight
|
|
// to its save path, and the cache records it there.
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.downloaded).toBe(3);
|
|
expect(source.originalCalls).toBe(3);
|
|
expect(readdirSync(join(root, "cache", "originals"))).toEqual([]);
|
|
const stored = readdirSync(join(outDir, DAY)).filter(
|
|
(name) => !name.endsWith(".json"),
|
|
);
|
|
expect(stored.sort()).toEqual([
|
|
"2026-03-01.100.jpg",
|
|
"2026-03-01.101.jpg",
|
|
"2026-03-01.200.png",
|
|
]);
|
|
// The cache counts the backup's copy as present: reading the
|
|
// original afterwards fetches nothing and answers with that copy.
|
|
const read = await lib.photos.byID({ fileID: 100 })!.original();
|
|
expect(read.path).toBe(saved(outDir, "100.jpg"));
|
|
expect(source.originalCalls).toBe(3);
|
|
await lib.close();
|
|
});
|
|
|
|
it("saves a file in two albums at its photo's save path and links it from both", async () => {
|
|
// The date edited in Ente has reached Work's copy, synced later, but
|
|
// Vacation's copy still has an earlier edit.
|
|
const older = file(100, 1, "beach.jpg");
|
|
older.pubMagicMetadata = {
|
|
editedTime: new Date(2026, 1, 1, 12).getTime() * 1000,
|
|
};
|
|
const newer = file(100, 2, "beach.jpg");
|
|
newer.updationTime = 2;
|
|
newer.pubMagicMetadata = {
|
|
editedTime: new Date(2026, 3, 15, 12).getTime() * 1000,
|
|
};
|
|
class SharedFileClient extends MockClient {
|
|
override async filesSince(args: {
|
|
collectionID: number;
|
|
}): Promise<FilesPage> {
|
|
const files = [args.collectionID === 1 ? older : newer];
|
|
return { files, deleted: [], cursor: 1 };
|
|
}
|
|
}
|
|
const outDir = join(root, "backup");
|
|
const lib = await Library.open({
|
|
client: new SharedFileClient(),
|
|
cacheDirectory: join(root, "cache"),
|
|
downloadDirectory: outDir,
|
|
contentSource: stubSource(),
|
|
refreshIntervalSeconds: 3600,
|
|
precacheThumbnails: false,
|
|
precacheOriginals: false,
|
|
});
|
|
const photo = lib.photos.byID({ fileID: 100 })!;
|
|
|
|
const result = await lib.backup();
|
|
|
|
expect(result.totalFiles).toBe(1);
|
|
expect(result.downloaded).toBe(1);
|
|
expect(result.failed).toBe(0);
|
|
expect(photo.savePath).toBe(
|
|
join(outDir, "2026", "2026-04", "2026-04-15", "2026-04-15.100.jpg"),
|
|
);
|
|
expect(photo.isLocal).toBe(true);
|
|
expect(existsSync(join(outDir, "2026", "2026-02"))).toBe(false);
|
|
const target = "../../2026/2026-04/2026-04-15/2026-04-15.100.jpg";
|
|
for (const album of ["Vacation", "Work"]) {
|
|
expect(
|
|
readlinkSync(join(outDir, "collections", album, "beach.jpg")),
|
|
).toBe(target);
|
|
}
|
|
await lib.close();
|
|
});
|
|
|
|
it("fsyncs an original copied from the cache before the rename and its directory after", async () => {
|
|
const lib = await openLibrary(stubSource());
|
|
const outDir = join(root, "backup");
|
|
const day = join(outDir, DAY);
|
|
const dest = saved(outDir, "100.jpg");
|
|
// Only an original already in the cache is copied into the backup;
|
|
// one fetched for the backup is written there by the download writer.
|
|
await lib.photos.byID({ fileID: 100 })!.original();
|
|
fsEvents.length = 0;
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
const at = fsEvents.indexOf(`rename:${dest}`);
|
|
expect(at).toBeGreaterThan(0);
|
|
expect(fsEvents[at - 1]).toMatch(
|
|
/^sync:.*\/\.quak-backup-2026-03-01\.100\.jpg-\d+-[0-9a-z]*\.tmp$/,
|
|
);
|
|
expect(fsEvents[at + 1]).toBe(`sync:${day}`);
|
|
lib.close();
|
|
});
|
|
|
|
it("removes temp files left by a killed backup but not those of one still running", async () => {
|
|
const outDir = join(root, "backup");
|
|
const day = join(outDir, DAY);
|
|
mkdirSync(day, { recursive: true });
|
|
// A child that has already exited: its process ID is not running.
|
|
const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid;
|
|
const leftover = `.quak-backup-2026-03-01.100.jpg-${exitedPID}-abc123.tmp`;
|
|
// This test's own process stands in for a backup running at the same
|
|
// time.
|
|
const inProgress = `.quak-backup-2026-03-01.101.jpg-${process.pid}-def456.tmp`;
|
|
writeFileSync(join(day, leftover), "partial");
|
|
writeFileSync(join(day, inProgress), "partial");
|
|
const lib = await openLibrary(stubSource());
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
const names = readdirSync(day);
|
|
expect(names).not.toContain(leftover);
|
|
expect(names).toContain(inProgress);
|
|
lib.close();
|
|
});
|
|
|
|
it("removes temp files left in a date folder no file in the backup is saved in", async () => {
|
|
const outDir = join(root, "backup");
|
|
// As for a file since deleted, or given another date, after a run was
|
|
// killed while writing it.
|
|
const otherDay = join(outDir, "2025", "2025-01", "2025-01-02");
|
|
mkdirSync(otherDay, { recursive: true });
|
|
const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid;
|
|
writeFileSync(join(otherDay, `.quak-${exitedPID}-abc123.tmp`), "x");
|
|
const lib = await openLibrary(stubSource());
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(readdirSync(otherDay)).toEqual([]);
|
|
lib.close();
|
|
});
|
|
|
|
it("removes leftover temp files in thumbnails/ but not those of a backup still running", async () => {
|
|
const outDir = join(root, "backup");
|
|
const thumbnails = join(outDir, "thumbnails");
|
|
mkdirSync(thumbnails, { recursive: true });
|
|
const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid;
|
|
const leftover = `.quak-backup-100.jpg-${exitedPID}-abc123.tmp`;
|
|
const inProgress = `.quak-backup-101.jpg-${process.pid}-def456.tmp`;
|
|
writeFileSync(join(thumbnails, leftover), "partial");
|
|
writeFileSync(join(thumbnails, inProgress), "partial");
|
|
const lib = await openLibrary(stubSource());
|
|
|
|
await lib.backup({
|
|
downloadDirectory: outDir,
|
|
includeThumbnails: true,
|
|
});
|
|
|
|
const names = readdirSync(thumbnails);
|
|
expect(names).not.toContain(leftover);
|
|
expect(names).toContain(inProgress);
|
|
lib.close();
|
|
});
|
|
});
|
|
|
|
// Every refresh fails, as with an expired session or no network.
|
|
class FailingClient extends MockClient {
|
|
override async collectionsSince(): Promise<CollectionsPage> {
|
|
throw new Error("HTTP 401 from server");
|
|
}
|
|
}
|
|
|
|
// Holds its refresh open until `release()` is called, then reports a third
|
|
// album, so a backup can be started while that refresh is still running.
|
|
class HeldClient extends MockClient {
|
|
release!: () => void;
|
|
private held = new Promise<void>((resolve) => {
|
|
this.release = resolve;
|
|
});
|
|
override async collectionsSince(): Promise<CollectionsPage> {
|
|
await this.held;
|
|
return {
|
|
collections: [collection(3, "Later")],
|
|
deleted: [],
|
|
cursor: 2,
|
|
};
|
|
}
|
|
override async filesSince(args: {
|
|
collectionID: number;
|
|
}): Promise<FilesPage> {
|
|
if (args.collectionID !== 3) return super.filesSince(args);
|
|
return { files: [file(300, 3, "late.jpg")], deleted: [], cursor: 2 };
|
|
}
|
|
}
|
|
|
|
// Fill the library cache on disk, so the next open starts its refresh in the
|
|
// background instead of waiting for it.
|
|
const fillCache = async (): Promise<void> => {
|
|
const lib = await openLibrary(stubSource());
|
|
await lib.close();
|
|
};
|
|
|
|
describe("the refresh before a backup", () => {
|
|
it("waits for a refresh already running and backs up what it found", async () => {
|
|
await fillCache();
|
|
const client = new HeldClient();
|
|
const lib = await openLibrary(stubSource(), client);
|
|
const outDir = join(root, "backup");
|
|
|
|
const backup = lib.backup({ downloadDirectory: outDir });
|
|
client.release();
|
|
const result = await backup;
|
|
|
|
expect(result.totalFiles).toBe(4);
|
|
expect(existsSync(saved(outDir, "300.jpg"))).toBe(true);
|
|
await lib.close();
|
|
});
|
|
|
|
it("fails before any download when the refresh fails, leaving failures.json as it was", async () => {
|
|
await fillCache();
|
|
const outDir = join(root, "backup");
|
|
seedLedger(outDir, 100, "beach.jpg");
|
|
const ledgerPath = join(outDir, "failures.json");
|
|
const ledgerBefore = readFileSync(ledgerPath, "utf-8");
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source, new FailingClient());
|
|
|
|
await expect(lib.backup({ downloadDirectory: outDir })).rejects.toThrow(
|
|
"HTTP 401 from server",
|
|
);
|
|
|
|
expect(source.originalCalls).toBe(0);
|
|
expect(readFileSync(ledgerPath, "utf-8")).toBe(ledgerBefore);
|
|
expect(existsSync(join(outDir, DAY))).toBe(false);
|
|
await lib.close();
|
|
});
|
|
|
|
it("fails when the refresh fails on an empty cache, instead of backing up nothing", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source, new FailingClient());
|
|
const outDir = join(root, "backup");
|
|
|
|
await expect(lib.backup({ downloadDirectory: outDir })).rejects.toThrow(
|
|
"HTTP 401 from server",
|
|
);
|
|
|
|
expect(source.originalCalls).toBe(0);
|
|
// The backup made the directory for its lock, and removed the lock.
|
|
expect(readdirSync(outDir)).toEqual([]);
|
|
await lib.close();
|
|
});
|
|
});
|
|
|
|
describe("the backup lock", () => {
|
|
const lockPath = (outDir: string): string => join(outDir, "backup.lock");
|
|
|
|
it("refuses a second backup of the directory while one runs", async () => {
|
|
await fillCache();
|
|
const client = new HeldClient();
|
|
const lib = await openLibrary(stubSource(), client);
|
|
const outDir = join(root, "backup");
|
|
// The first backup holds the lock once it starts its refresh, which
|
|
// `HeldClient` keeps from finishing.
|
|
let refreshing!: () => void;
|
|
const started = new Promise<void>((resolve) => {
|
|
refreshing = resolve;
|
|
});
|
|
const first = lib.backup({
|
|
downloadDirectory: outDir,
|
|
onProgress: (msg) => {
|
|
if (msg === "Refreshing library...") refreshing();
|
|
},
|
|
});
|
|
await started;
|
|
|
|
await expect(lib.backup({ downloadDirectory: outDir })).rejects.toThrow(
|
|
`another backup of ${outDir} is running`,
|
|
);
|
|
|
|
client.release();
|
|
expect((await first).failed).toBe(0);
|
|
await lib.close();
|
|
});
|
|
|
|
it("releases the lock after a backup succeeds", async () => {
|
|
const lib = await openLibrary(stubSource());
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(existsSync(lockPath(outDir))).toBe(false);
|
|
// So the next backup of the directory runs.
|
|
expect((await lib.backup({ downloadDirectory: outDir })).skipped).toBe(
|
|
3,
|
|
);
|
|
await lib.close();
|
|
});
|
|
|
|
it("releases the lock after a backup fails", async () => {
|
|
const lib = await openLibrary(stubSource(), new FailingClient());
|
|
const outDir = join(root, "backup");
|
|
|
|
await expect(lib.backup({ downloadDirectory: outDir })).rejects.toThrow(
|
|
"HTTP 401 from server",
|
|
);
|
|
|
|
expect(existsSync(lockPath(outDir))).toBe(false);
|
|
await lib.close();
|
|
});
|
|
|
|
it("takes over a lock left by a run that was killed", async () => {
|
|
const outDir = join(root, "backup");
|
|
// A killed run's lock, last touched a minute ago.
|
|
mkdirSync(lockPath(outDir), { recursive: true });
|
|
const minuteAgo = new Date(Date.now() - 60_000);
|
|
utimesSync(lockPath(outDir), minuteAgo, minuteAgo);
|
|
const lib = await openLibrary(stubSource());
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.downloaded).toBe(3);
|
|
expect(existsSync(lockPath(outDir))).toBe(false);
|
|
await lib.close();
|
|
});
|
|
});
|
|
|
|
// Also serves ML data: file 100 has `ML_PAYLOAD`, the other two have none.
|
|
// While `mlError` is set, every ML data request fails with it.
|
|
const ML_PAYLOAD = { face: { faces: [] }, clip: { embedding: [0.5, 0.25] } };
|
|
|
|
class MLClient extends MockClient {
|
|
mlError?: string;
|
|
async fetchMLData(args: {
|
|
fileIDs: number[];
|
|
}): Promise<Map<number, MLData>> {
|
|
if (this.mlError) throw new Error(this.mlError);
|
|
const result = new Map<number, MLData>();
|
|
if (args.fileIDs.includes(100)) result.set(100, ML_PAYLOAD);
|
|
return result;
|
|
}
|
|
}
|
|
|
|
// The JSON the backup in `outDir` wrote beside the original of `fileID`.
|
|
const fileJSON = (outDir: string, fileID: number): Record<string, unknown> =>
|
|
JSON.parse(readFileSync(saved(outDir, `${fileID}.json`), "utf-8"));
|
|
|
|
describe("ML data in each file's JSON", () => {
|
|
it("writes a file's ML data, and no ML field for a file that has none", async () => {
|
|
const lib = await openLibrary(stubSource(), new MLClient());
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.failed).toBe(0);
|
|
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
|
|
expect(fileJSON(outDir, 101)).not.toHaveProperty("mlData");
|
|
expect(fileJSON(outDir, 101)).not.toHaveProperty("mlDataError");
|
|
await lib.close();
|
|
});
|
|
|
|
it("gives each file with no cached ML data the reason when the fetch fails, and fails the run", async () => {
|
|
const client = new MLClient();
|
|
const outDir = join(root, "backup");
|
|
const first = await openLibrary(stubSource(), client);
|
|
await first.backup({ downloadDirectory: outDir });
|
|
await first.close();
|
|
|
|
client.mlError = "HTTP 503 from server";
|
|
const lib = await openLibrary(stubSource(), client);
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
// File 100's ML data was cached by the first run and is kept.
|
|
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
|
|
expect(fileJSON(outDir, 100)).not.toHaveProperty("mlDataError");
|
|
for (const fileID of [101, 200]) {
|
|
expect(fileJSON(outDir, fileID).mlDataError).toBe(
|
|
"HTTP 503 from server",
|
|
);
|
|
}
|
|
expect(result.failed).toBe(2);
|
|
expect(result.errors.map((e) => [e.fileID, e.error])).toEqual([
|
|
[101, "ML data: HTTP 503 from server"],
|
|
[200, "ML data: HTTP 503 from server"],
|
|
]);
|
|
expect(Object.keys(readLedger(outDir).files)).toEqual(["101", "200"]);
|
|
await lib.close();
|
|
});
|
|
|
|
it("writes the ML data on the run after a failed fetch", async () => {
|
|
const client = new MLClient();
|
|
client.mlError = "HTTP 503 from server";
|
|
const outDir = join(root, "backup");
|
|
const first = await openLibrary(stubSource(), client);
|
|
expect((await first.backup({ downloadDirectory: outDir })).failed).toBe(
|
|
3,
|
|
);
|
|
expect(fileJSON(outDir, 100)).not.toHaveProperty("mlData");
|
|
await first.close();
|
|
|
|
client.mlError = undefined;
|
|
const lib = await openLibrary(stubSource(), client);
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.failed).toBe(0);
|
|
expect(result.skipped).toBe(3);
|
|
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
|
|
for (const fileID of [100, 101, 200]) {
|
|
expect(fileJSON(outDir, fileID)).not.toHaveProperty("mlDataError");
|
|
}
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
await lib.close();
|
|
});
|
|
});
|
|
|
|
// The fields `RecordsClient` gives the Vacation album: shared by another
|
|
// account, with all three layers of magic metadata.
|
|
const SHARED_ALBUM = {
|
|
ownerID: 7,
|
|
isShared: true,
|
|
updationTime: 1700,
|
|
magicMetadata: { visibility: 0 },
|
|
pubMagicMetadata: { coverID: 100 },
|
|
sharedMagicMetadata: { visibility: 2 },
|
|
};
|
|
|
|
// Serves the Vacation album with `SHARED_ALBUM`'s fields, and each file with
|
|
// the update time 5000 + its ID.
|
|
class RecordsClient extends MockClient {
|
|
override async collectionsSince(): Promise<CollectionsPage> {
|
|
const page = await super.collectionsSince();
|
|
return {
|
|
...page,
|
|
collections: page.collections.map((c) =>
|
|
c.id === 1 ? { ...c, ...SHARED_ALBUM } : c,
|
|
),
|
|
};
|
|
}
|
|
override async filesSince(args: {
|
|
collectionID: number;
|
|
}): Promise<FilesPage> {
|
|
const page = await super.filesSince(args);
|
|
return {
|
|
...page,
|
|
files: page.files.map((f) => ({
|
|
...f,
|
|
updationTime: 5000 + f.id,
|
|
})),
|
|
};
|
|
}
|
|
}
|
|
|
|
// The JSON the backup in `outDir` wrote for the album named `name`.
|
|
const albumJSON = (outDir: string, name: string): Record<string, unknown> =>
|
|
JSON.parse(
|
|
readFileSync(join(outDir, "collections", `${name}.json`), "utf-8"),
|
|
);
|
|
|
|
describe("account and album records", () => {
|
|
it("writes account.json with the account's email and user ID", async () => {
|
|
const lib = await openLibrary(stubSource());
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(
|
|
JSON.parse(readFileSync(join(outDir, "account.json"), "utf-8")),
|
|
).toEqual({ email: "backup@example.com", userID: USER_ID });
|
|
await lib.close();
|
|
});
|
|
|
|
it("writes each album's owner, sharing, update time and magic metadata into its JSON", async () => {
|
|
const lib = await openLibrary(stubSource(), new RecordsClient());
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(albumJSON(outDir, "Vacation")).toEqual({
|
|
id: 1,
|
|
name: "Vacation",
|
|
type: "album",
|
|
...SHARED_ALBUM,
|
|
files: [
|
|
{ id: 100, metadata: file(100, 1, "beach.jpg").metadata },
|
|
{ id: 101, metadata: file(101, 1, "sunset.jpg").metadata },
|
|
],
|
|
});
|
|
// An album with no magic metadata gets no magic metadata fields.
|
|
expect(albumJSON(outDir, "Work")).toEqual({
|
|
id: 2,
|
|
name: "Work",
|
|
type: "album",
|
|
ownerID: USER_ID,
|
|
isShared: false,
|
|
updationTime: 1,
|
|
files: [
|
|
{ id: 200, metadata: file(200, 2, "diagram.png").metadata },
|
|
],
|
|
});
|
|
await lib.close();
|
|
});
|
|
|
|
it("writes each file's update time into its JSON", async () => {
|
|
const lib = await openLibrary(stubSource(), new RecordsClient());
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
for (const fileID of [100, 101, 200]) {
|
|
expect(fileJSON(outDir, fileID).updationTime).toBe(5000 + fileID);
|
|
}
|
|
await lib.close();
|
|
});
|
|
});
|
|
|
|
// What a file's JSON holds as `imageMetadata` for an original of `bytes`.
|
|
const imageMetadataOf = (bytes: Uint8Array): unknown =>
|
|
JSON.parse(JSON.stringify(extractImageMetadata(bytes)));
|
|
|
|
describe("image metadata in each file's JSON", () => {
|
|
// Serves `files` in the Vacation album and nothing in Work.
|
|
class FilesClient extends MockClient {
|
|
constructor(private readonly files: EnteFile[]) {
|
|
super();
|
|
}
|
|
override async filesSince(args: {
|
|
collectionID: number;
|
|
}): Promise<FilesPage> {
|
|
return {
|
|
files: args.collectionID === 1 ? this.files : [],
|
|
deleted: [],
|
|
cursor: 1,
|
|
};
|
|
}
|
|
}
|
|
|
|
// Writes `originals.get(fileID)` as each file's original, looked up at
|
|
// each fetch, so a test can change it between runs.
|
|
const bytesSource = (
|
|
originals: Map<number, Uint8Array>,
|
|
): ContentSource => ({
|
|
original: async ({ file: f, destination }) => {
|
|
const bytes = originals.get(f.id)!;
|
|
writeFileSync(destination, bytes);
|
|
return { bytesWritten: bytes.length };
|
|
},
|
|
thumbnail: async () => {
|
|
throw new Error("no thumbnails in this source");
|
|
},
|
|
});
|
|
|
|
// A JPEG, a HEIC, a video, and a file holding no image metadata. The
|
|
// video's bytes are the JPEG's, so reading it would find EXIF.
|
|
const clip = file(602, 1, "clip.mov");
|
|
const files = [
|
|
file(600, 1, "photo.jpg"),
|
|
file(601, 1, "photo.heic"),
|
|
{ ...clip, metadata: { ...clip.metadata, fileType: "video" as const } },
|
|
file(603, 1, "notes.png"),
|
|
];
|
|
const originals = (): Map<number, Uint8Array> =>
|
|
new Map([
|
|
[600, JPEG_WITH_EXIF],
|
|
[601, HEIC_WITH_EXIF],
|
|
[602, JPEG_WITH_EXIF],
|
|
[603, new TextEncoder().encode("not an image")],
|
|
]);
|
|
const open = (bytes = originals()): Promise<Library> =>
|
|
openLibrary(bytesSource(bytes), new FilesClient(files));
|
|
|
|
// Rewrite the JSON of `fileID` without its image metadata, as an earlier
|
|
// version of quak wrote it.
|
|
const dropImageMetadata = (outDir: string, fileID: number): void => {
|
|
const json = fileJSON(outDir, fileID);
|
|
delete json.imageMetadata;
|
|
writeFileSync(saved(outDir, `${fileID}.json`), JSON.stringify(json));
|
|
};
|
|
|
|
it("writes each new original's image metadata, and none for a video", async () => {
|
|
const lib = await open();
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result).toMatchObject({ downloaded: 4, failed: 0 });
|
|
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
|
|
imageMetadataOf(JPEG_WITH_EXIF),
|
|
);
|
|
expect(fileJSON(outDir, 601).imageMetadata).toEqual(
|
|
imageMetadataOf(HEIC_WITH_EXIF),
|
|
);
|
|
expect(fileJSON(outDir, 602)).not.toHaveProperty("imageMetadata");
|
|
expect(fileJSON(outDir, 602)).not.toHaveProperty("imageMetadataError");
|
|
// Empty, so that the next run does not read it again.
|
|
expect(fileJSON(outDir, 603).imageMetadata).toEqual({});
|
|
await lib.close();
|
|
});
|
|
|
|
it("reads a live photo's image", async () => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(500, 1, "IMG_0500.HEIC"),
|
|
livePhotoZip({ "image.heic": HEIC_WITH_EXIF, "video.mov": VIDEO }),
|
|
livePhotoHash(HEIC_WITH_EXIF, VIDEO),
|
|
);
|
|
const lib = await openLibrary(
|
|
cdnSource(new Map([[500, body]])),
|
|
new FilesClient([live]),
|
|
);
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(fileJSON(outDir, 500).imageMetadata).toEqual(
|
|
imageMetadataOf(HEIC_WITH_EXIF),
|
|
);
|
|
await lib.close();
|
|
});
|
|
|
|
it("keeps the image metadata of an original stored by an earlier run without reading it again", async () => {
|
|
const lib = await open();
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
// A read of the original now would give the HEIC's.
|
|
writeFileSync(saved(outDir, "600.jpg"), HEIC_WITH_EXIF);
|
|
|
|
const second = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(second).toMatchObject({ downloaded: 0, failed: 0 });
|
|
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
|
|
imageMetadataOf(JPEG_WITH_EXIF),
|
|
);
|
|
await lib.close();
|
|
});
|
|
|
|
it("reads an original stored by an earlier version, whose JSON has no image metadata", async () => {
|
|
const lib = await open();
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
dropImageMetadata(outDir, 600);
|
|
|
|
const second = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(second).toMatchObject({ downloaded: 0, failed: 0 });
|
|
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
|
|
imageMetadataOf(JPEG_WITH_EXIF),
|
|
);
|
|
await lib.close();
|
|
});
|
|
|
|
it("reads an original the run stores again, though its JSON has image metadata", async () => {
|
|
const bytes = originals();
|
|
const lib = await open(bytes);
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
rmSync(saved(outDir, "600.jpg"));
|
|
bytes.set(600, HEIC_WITH_EXIF);
|
|
|
|
const second = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(second).toMatchObject({ downloaded: 1, failed: 0 });
|
|
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
|
|
imageMetadataOf(HEIC_WITH_EXIF),
|
|
);
|
|
await lib.close();
|
|
});
|
|
|
|
// Root ignores file permissions, so this fails when run as root. The
|
|
// `test` phase of the `Dockerfile` runs as the `node` user.
|
|
it("gives the reason an original could not be read, failing neither the file nor the run", async () => {
|
|
const lib = await open();
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
dropImageMetadata(outDir, 600);
|
|
const original = saved(outDir, "600.jpg");
|
|
chmodSync(original, 0o000);
|
|
|
|
const second = await lib
|
|
.backup({ downloadDirectory: outDir })
|
|
.finally(() => chmodSync(original, 0o600));
|
|
|
|
expect(second).toMatchObject({ failed: 0, errors: [] });
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
expect(fileJSON(outDir, 600)).not.toHaveProperty("imageMetadata");
|
|
expect(fileJSON(outDir, 600).imageMetadataError).toMatch(/EACCES/);
|
|
await lib.close();
|
|
});
|
|
});
|
|
|
|
// With `verify`, each original already stored is hashed, and one that does not
|
|
// match the content hash its metadata records is downloaded again.
|
|
describe("backup with verify", () => {
|
|
// MockClient's files, each recording the hash of what `stubSource` writes
|
|
// for it, except diagram.png (200), which records none.
|
|
class HashedClient extends MockClient {
|
|
override async filesSince(args: {
|
|
collectionID: number;
|
|
}): Promise<FilesPage> {
|
|
const page = await super.filesSince(args);
|
|
const files = page.files.map((f) =>
|
|
f.id === 200
|
|
? f
|
|
: {
|
|
...f,
|
|
metadata: {
|
|
...f.metadata,
|
|
hash: blake2b(Buffer.alloc(SIZE_BY_ID[f.id]!)),
|
|
},
|
|
},
|
|
);
|
|
return { ...page, files };
|
|
}
|
|
}
|
|
|
|
// A backup of the account in `root/backup`, the library that made it, and
|
|
// the source it fetched from.
|
|
const backedUp = async (): Promise<{
|
|
lib: Library;
|
|
source: StubSource;
|
|
outDir: string;
|
|
}> => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source, new HashedClient());
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
return { lib, source, outDir };
|
|
};
|
|
|
|
// What `stubSource` writes for beach.jpg (100), and other bytes of the
|
|
// same length.
|
|
const good = Buffer.alloc(SIZE_BY_ID[100]!);
|
|
const corrupt = Buffer.alloc(SIZE_BY_ID[100]!, 1);
|
|
|
|
it("leaves an original that matches its hash, and one with no hash, as they are", async () => {
|
|
const { lib, source, outDir } = await backedUp();
|
|
const calls = source.originalCalls;
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
verify: true,
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
downloaded: 0,
|
|
skipped: 3,
|
|
verified: 2,
|
|
mismatched: 0,
|
|
unchecked: 1,
|
|
failed: 0,
|
|
});
|
|
expect(source.originalCalls).toBe(calls);
|
|
await lib.close();
|
|
});
|
|
|
|
it("downloads again an original that does not match its hash", async () => {
|
|
const { lib, outDir } = await backedUp();
|
|
writeFileSync(saved(outDir, "100.jpg"), corrupt);
|
|
const log: string[] = [];
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
verify: true,
|
|
onProgress: (msg) => log.push(msg),
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
downloaded: 1,
|
|
skipped: 2,
|
|
verified: 1,
|
|
mismatched: 1,
|
|
unchecked: 1,
|
|
failed: 0,
|
|
});
|
|
expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(good);
|
|
expect(log.filter((msg) => msg.startsWith("MISMATCH"))).toEqual([
|
|
"MISMATCH original beach.jpg (100): its bytes do not match its content hash",
|
|
]);
|
|
await lib.close();
|
|
});
|
|
|
|
it("records a failed download in failures.json, with the original removed", async () => {
|
|
const { lib, source, outDir } = await backedUp();
|
|
writeFileSync(saved(outDir, "100.jpg"), corrupt);
|
|
source.failID = 100;
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
verify: true,
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
downloaded: 0,
|
|
mismatched: 1,
|
|
failed: 1,
|
|
});
|
|
expect(Object.keys(readLedger(outDir).files)).toEqual(["100"]);
|
|
expect(existsSync(saved(outDir, "100.jpg"))).toBe(false);
|
|
await lib.close();
|
|
});
|
|
|
|
it("records in failures.json an original the cache puts back with the same bad bytes", async () => {
|
|
const lib = await openLibrary(stubSource(), new HashedClient());
|
|
const outDir = join(root, "backup");
|
|
// The cache holds a bad copy, and a backup copies an original the
|
|
// cache holds to its save path.
|
|
const cached = await lib.photos.byID({ fileID: 100 })!.original();
|
|
writeFileSync(cached.path, corrupt);
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
verify: true,
|
|
});
|
|
|
|
expect(result).toMatchObject({
|
|
downloaded: 0,
|
|
mismatched: 1,
|
|
failed: 1,
|
|
});
|
|
expect(result.errors.map((e) => e.error)).toEqual([
|
|
"the original put back does not match its content hash either",
|
|
]);
|
|
expect(Object.keys(readLedger(outDir).files)).toEqual(["100"]);
|
|
expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(corrupt);
|
|
await lib.close();
|
|
});
|
|
|
|
it("hashes nothing without verify", async () => {
|
|
const { lib, outDir } = await backedUp();
|
|
writeFileSync(saved(outDir, "100.jpg"), corrupt);
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result).toMatchObject({
|
|
downloaded: 0,
|
|
skipped: 3,
|
|
verified: 0,
|
|
mismatched: 0,
|
|
unchecked: 0,
|
|
failed: 0,
|
|
});
|
|
expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(corrupt);
|
|
await lib.close();
|
|
});
|
|
|
|
// Root ignores file permissions, so this fails when run as root. The
|
|
// `test` phase of the `Dockerfile` runs as the `node` user.
|
|
it("records an original it cannot read as failed, and leaves it", async () => {
|
|
const { lib, outDir } = await backedUp();
|
|
const original = saved(outDir, "100.jpg");
|
|
chmodSync(original, 0o000);
|
|
|
|
const result = await lib
|
|
.backup({ downloadDirectory: outDir, verify: true })
|
|
.finally(() => chmodSync(original, 0o600));
|
|
|
|
expect(result).toMatchObject({ downloaded: 0, verified: 1, failed: 1 });
|
|
expect(result.errors.map((e) => e.fileID)).toEqual([100]);
|
|
expect(result.errors[0]!.error).toMatch(/EACCES/);
|
|
expect(readFileSync(original)).toEqual(good);
|
|
await lib.close();
|
|
});
|
|
});
|
|
|
|
// Every entry under collections/, one level of directories deep, with each
|
|
// symlink's target.
|
|
const tree = (outDir: string): string[] => {
|
|
const lines: string[] = [];
|
|
const list = (dir: string, prefix: string): void => {
|
|
for (const name of readdirSync(dir).sort()) {
|
|
const path = join(dir, name);
|
|
const st = lstatSync(path);
|
|
if (st.isSymbolicLink()) {
|
|
lines.push(`${prefix}${name} -> ${readlinkSync(path)}`);
|
|
} else if (st.isDirectory() && prefix === "") {
|
|
lines.push(`${name}/`);
|
|
list(path, `${name}/`);
|
|
} else {
|
|
lines.push(`${prefix}${name}`);
|
|
}
|
|
}
|
|
};
|
|
list(join(outDir, "collections"), "");
|
|
return lines;
|
|
};
|
|
|
|
// The album folders under collections/, driven through `runBackup` with a
|
|
// stand-in library whose albums a test changes between runs.
|
|
describe("backup album folders", () => {
|
|
interface Album {
|
|
collection: Collection;
|
|
files: EnteFile[];
|
|
}
|
|
|
|
const libraryOf = (albums: Album[]): BackupLibrary => ({
|
|
refresh: async () => {},
|
|
listCollections: () => albums.map((a) => a.collection),
|
|
listFiles: (id) =>
|
|
albums.find((a) => a.collection.id === id)?.files ?? [],
|
|
undecryptable: () => [],
|
|
original: async (fileID) => {
|
|
const path = join(root, `source-${fileID}`);
|
|
writeFileSync(path, `original ${fileID}`);
|
|
return { path };
|
|
},
|
|
thumbnail: async () => {
|
|
throw new Error("no thumbnails in this stand-in");
|
|
},
|
|
fetchMLData: async () => {},
|
|
mlData: async () => undefined,
|
|
whoami: () => ({ email: "backup@example.com", userID: USER_ID }),
|
|
});
|
|
|
|
const albumID = (outDir: string, jsonName: string): number =>
|
|
JSON.parse(readFileSync(join(outDir, "collections", jsonName), "utf-8"))
|
|
.id;
|
|
|
|
it("gives every file and every album its own name when names repeat", async () => {
|
|
const outDir = join(root, "backup");
|
|
const lib = libraryOf([
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [
|
|
file(1, 10, "IMG_0001.JPG"),
|
|
file(2, 10, "IMG_0001.JPG"),
|
|
file(4, 10, "img_0001.jpg"),
|
|
file(3, 10, "other.jpg"),
|
|
],
|
|
},
|
|
{
|
|
collection: collection(11, "Trip"),
|
|
files: [file(3, 11, "other.jpg")],
|
|
},
|
|
]);
|
|
|
|
const result = await runBackup(lib, { downloadDirectory: outDir });
|
|
|
|
expect(result.failed).toBe(0);
|
|
expect(tree(outDir)).toEqual([
|
|
"Trip (10)/",
|
|
`Trip (10)/IMG_0001 (1).JPG -> ${linkTo("1.JPG")}`,
|
|
`Trip (10)/IMG_0001 (2).JPG -> ${linkTo("2.JPG")}`,
|
|
`Trip (10)/img_0001 (4).jpg -> ${linkTo("4.jpg")}`,
|
|
`Trip (10)/other.jpg -> ${linkTo("3.jpg")}`,
|
|
"Trip (10).json",
|
|
"Trip (11)/",
|
|
`Trip (11)/other.jpg -> ${linkTo("3.jpg")}`,
|
|
"Trip (11).json",
|
|
]);
|
|
expect(albumID(outDir, "Trip (10).json")).toBe(10);
|
|
expect(albumID(outDir, "Trip (11).json")).toBe(11);
|
|
});
|
|
|
|
it("keeps names unique when a name with an ID added is another entry's own name", async () => {
|
|
const outDir = join(root, "backup");
|
|
const lib = libraryOf([
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [
|
|
file(5, 10, "IMG (6).JPG"),
|
|
file(6, 10, "IMG.JPG"),
|
|
file(7, 10, "IMG.JPG"),
|
|
],
|
|
},
|
|
{
|
|
collection: collection(11, "Trip"),
|
|
files: [file(8, 11, "a.jpg")],
|
|
},
|
|
{
|
|
collection: collection(12, "Trip (11)"),
|
|
files: [file(9, 12, "b.jpg")],
|
|
},
|
|
]);
|
|
|
|
const result = await runBackup(lib, { downloadDirectory: outDir });
|
|
|
|
expect(result.failed).toBe(0);
|
|
expect(tree(outDir)).toEqual([
|
|
"Trip (10)/",
|
|
`Trip (10)/IMG (6) (5).JPG -> ${linkTo("5.JPG")}`,
|
|
`Trip (10)/IMG (6).JPG -> ${linkTo("6.JPG")}`,
|
|
`Trip (10)/IMG (7).JPG -> ${linkTo("7.JPG")}`,
|
|
"Trip (10).json",
|
|
"Trip (11)/",
|
|
`Trip (11)/a.jpg -> ${linkTo("8.jpg")}`,
|
|
"Trip (11) (12)/",
|
|
`Trip (11) (12)/b.jpg -> ${linkTo("9.jpg")}`,
|
|
"Trip (11) (12).json",
|
|
"Trip (11).json",
|
|
]);
|
|
expect(albumID(outDir, "Trip (10).json")).toBe(10);
|
|
expect(albumID(outDir, "Trip (11).json")).toBe(11);
|
|
expect(albumID(outDir, "Trip (11) (12).json")).toBe(12);
|
|
});
|
|
|
|
it("changes nothing on a second run over an unchanged account", async () => {
|
|
const outDir = join(root, "backup");
|
|
const lib = libraryOf([
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [
|
|
file(1, 10, "IMG_0001.JPG"),
|
|
file(2, 10, "IMG_0001.JPG"),
|
|
],
|
|
},
|
|
{
|
|
collection: collection(11, "Trip"),
|
|
files: [file(3, 11, "other.jpg")],
|
|
},
|
|
]);
|
|
|
|
await runBackup(lib, { downloadDirectory: outDir });
|
|
const before = tree(outDir);
|
|
const second = await runBackup(lib, { downloadDirectory: outDir });
|
|
|
|
expect(second.downloaded).toBe(0);
|
|
expect(second.failed).toBe(0);
|
|
expect(tree(outDir)).toEqual(before);
|
|
});
|
|
|
|
it("leaves the albums an onlyAlbumNames run skips as they were", async () => {
|
|
const outDir = join(root, "backup");
|
|
// "trip" is skipped by the scoped run but its name clashes with the
|
|
// in-scope "Trip", so "Trip" must keep its ID suffix.
|
|
const lib = libraryOf([
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [file(1, 10, "a.jpg")],
|
|
},
|
|
{
|
|
collection: collection(11, "trip"),
|
|
files: [file(2, 11, "b.jpg")],
|
|
},
|
|
{
|
|
collection: collection(12, "Work"),
|
|
files: [file(3, 12, "c.jpg")],
|
|
},
|
|
]);
|
|
const json = (name: string): string =>
|
|
readFileSync(join(outDir, "collections", name), "utf-8");
|
|
|
|
await runBackup(lib, { downloadDirectory: outDir });
|
|
const before = tree(outDir);
|
|
const skippedJSON = [json("trip (11).json"), json("Work.json")];
|
|
const scoped = await runBackup(lib, {
|
|
downloadDirectory: outDir,
|
|
onlyAlbumNames: ["Trip"],
|
|
});
|
|
|
|
expect(scoped.failed).toBe(0);
|
|
expect(before).toEqual([
|
|
"Trip (10)/",
|
|
`Trip (10)/a.jpg -> ${linkTo("1.jpg")}`,
|
|
"Trip (10).json",
|
|
"Work/",
|
|
`Work/c.jpg -> ${linkTo("3.jpg")}`,
|
|
"Work.json",
|
|
"trip (11)/",
|
|
`trip (11)/b.jpg -> ${linkTo("2.jpg")}`,
|
|
"trip (11).json",
|
|
]);
|
|
expect(tree(outDir)).toEqual(before);
|
|
expect([json("trip (11).json"), json("Work.json")]).toEqual(
|
|
skippedJSON,
|
|
);
|
|
});
|
|
|
|
it("removes links and album folders that are gone, and nothing the user added", async () => {
|
|
const outDir = join(root, "backup");
|
|
const albums: Album[] = [
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [
|
|
file(1, 10, "IMG_0001.JPG"),
|
|
file(2, 10, "IMG_0001.JPG"),
|
|
file(3, 10, "other.jpg"),
|
|
],
|
|
},
|
|
{
|
|
collection: collection(12, "Work"),
|
|
files: [file(5, 12, "a.jpg")],
|
|
},
|
|
{
|
|
collection: collection(13, "Old"),
|
|
files: [file(5, 13, "a.jpg")],
|
|
},
|
|
];
|
|
const lib = libraryOf(albums);
|
|
await runBackup(lib, { downloadDirectory: outDir });
|
|
|
|
// What the user put in the tree: a note and a symlink of their own in
|
|
// an album, a note in an album about to be renamed, and a folder quak
|
|
// did not create.
|
|
const collectionsDir = join(outDir, "collections");
|
|
writeFileSync(join(collectionsDir, "Trip", "notes.txt"), "mine");
|
|
symlinkSync("../elsewhere", join(collectionsDir, "Trip", "mine"));
|
|
writeFileSync(join(collectionsDir, "Work", "keep.txt"), "mine");
|
|
mkdirSync(join(collectionsDir, "Mine"));
|
|
writeFileSync(join(collectionsDir, "Mine", "keep.txt"), "mine");
|
|
|
|
// File 2 leaves Trip, Work is renamed Office, Old is deleted.
|
|
albums[0]!.files.splice(1, 1);
|
|
albums[1]!.collection = collection(12, "Office");
|
|
albums.splice(2, 1);
|
|
const result = await runBackup(lib, { downloadDirectory: outDir });
|
|
|
|
expect(result.failed).toBe(0);
|
|
expect(tree(outDir)).toEqual([
|
|
"Mine/",
|
|
"Mine/keep.txt",
|
|
"Office/",
|
|
`Office/a.jpg -> ${linkTo("5.jpg")}`,
|
|
"Office.json",
|
|
"Trip/",
|
|
`Trip/IMG_0001.JPG -> ${linkTo("1.JPG")}`,
|
|
"Trip/mine -> ../elsewhere",
|
|
"Trip/notes.txt",
|
|
`Trip/other.jpg -> ${linkTo("3.jpg")}`,
|
|
"Trip.json",
|
|
"Work/",
|
|
"Work/keep.txt",
|
|
"Work.json",
|
|
]);
|
|
});
|
|
|
|
// One album backed up, then a folder the user made beside it holding a
|
|
// symlink to an original, with `json` (if given) as its sibling JSON.
|
|
const backupWithUserFolder = async (
|
|
json: string | undefined,
|
|
): Promise<{ outDir: string; failed: number }> => {
|
|
const outDir = join(root, "backup");
|
|
const lib = libraryOf([
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [file(1, 10, "a.jpg")],
|
|
},
|
|
]);
|
|
await runBackup(lib, { downloadDirectory: outDir });
|
|
const collectionsDir = join(outDir, "collections");
|
|
mkdirSync(join(collectionsDir, "Mine"));
|
|
symlinkSync(linkTo("1.jpg"), join(collectionsDir, "Mine", "a.jpg"));
|
|
if (json !== undefined) {
|
|
writeFileSync(join(collectionsDir, "Mine.json"), json);
|
|
}
|
|
const result = await runBackup(lib, { downloadDirectory: outDir });
|
|
return { outDir, failed: result.failed };
|
|
};
|
|
|
|
it("leaves a user folder with no JSON beside it as it was", async () => {
|
|
const { outDir, failed } = await backupWithUserFolder(undefined);
|
|
|
|
expect(failed).toBe(0);
|
|
expect(tree(outDir)).toEqual([
|
|
"Mine/",
|
|
`Mine/a.jpg -> ${linkTo("1.jpg")}`,
|
|
"Trip/",
|
|
`Trip/a.jpg -> ${linkTo("1.jpg")}`,
|
|
"Trip.json",
|
|
]);
|
|
});
|
|
|
|
it("leaves a user folder whose JSON has no album ID as it was", async () => {
|
|
const json = '{"name":"Mine"}';
|
|
const { outDir, failed } = await backupWithUserFolder(json);
|
|
|
|
expect(failed).toBe(0);
|
|
expect(tree(outDir)).toEqual([
|
|
"Mine/",
|
|
`Mine/a.jpg -> ${linkTo("1.jpg")}`,
|
|
"Mine.json",
|
|
"Trip/",
|
|
`Trip/a.jpg -> ${linkTo("1.jpg")}`,
|
|
"Trip.json",
|
|
]);
|
|
expect(
|
|
readFileSync(join(outDir, "collections", "Mine.json"), "utf-8"),
|
|
).toBe(json);
|
|
});
|
|
});
|
|
|
|
// A live photo, which Ente stores as one ZIP, is backed up as its image and
|
|
// its video, which a photo viewer can open, beside a JSON file naming them,
|
|
// and its album folder links both. These tests download a live photo ZIP
|
|
// through the real download layer (test/live-photo.ts).
|
|
describe("backup of live photos", () => {
|
|
// An account of one album, Trip (10), holding `files`.
|
|
class TripClient extends MockClient {
|
|
constructor(private readonly files: EnteFile[]) {
|
|
super();
|
|
}
|
|
override async collectionsSince(): Promise<CollectionsPage> {
|
|
return {
|
|
collections: [collection(10, "Trip")],
|
|
deleted: [],
|
|
cursor: 1,
|
|
};
|
|
}
|
|
override async filesSince(): Promise<FilesPage> {
|
|
return { files: this.files, deleted: [], cursor: 1 };
|
|
}
|
|
}
|
|
|
|
const open = (files: EnteFile[], bodies: Map<number, Uint8Array>) =>
|
|
openLibrary(cdnSource(bodies), new TripClient(files));
|
|
|
|
const stored = [
|
|
"2026-03-01.500.heic",
|
|
"2026-03-01.500.json",
|
|
"2026-03-01.500.livephoto.json",
|
|
"2026-03-01.500.mov",
|
|
];
|
|
const linked = [
|
|
"Trip/",
|
|
`Trip/IMG_0500.heic -> ${linkTo("500.heic")}`,
|
|
`Trip/IMG_0500.mov -> ${linkTo("500.mov")}`,
|
|
"Trip.json",
|
|
];
|
|
|
|
it("stores a live photo as its image and its video and links both", async () => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(500, 10, "IMG_0500.HEIC"),
|
|
);
|
|
const lib = await open([live], new Map([[500, body]]));
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
|
|
expect(readdirSync(join(outDir, DAY)).sort()).toEqual(stored);
|
|
expect(readFileSync(saved(outDir, "500.heic"))).toEqual(
|
|
Buffer.from(IMAGE),
|
|
);
|
|
expect(readFileSync(saved(outDir, "500.mov"))).toEqual(
|
|
Buffer.from(VIDEO),
|
|
);
|
|
expect(tree(outDir)).toEqual(linked);
|
|
|
|
// Both parts are there, so the next run fetches nothing.
|
|
const second = await lib.backup({ downloadDirectory: outDir });
|
|
expect(second).toMatchObject({ downloaded: 0, skipped: 1, failed: 0 });
|
|
await lib.close();
|
|
});
|
|
|
|
it("gives both links of each live photo their own names when titles clash", async () => {
|
|
const a = await asLivePhoto(file(500, 10, "IMG_0001.HEIC"));
|
|
const b = await asLivePhoto(file(501, 10, "IMG_0001.HEIC"));
|
|
const lib = await open(
|
|
[a.file, b.file],
|
|
new Map([
|
|
[500, a.body],
|
|
[501, b.body],
|
|
]),
|
|
);
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(tree(outDir)).toEqual([
|
|
"Trip/",
|
|
`Trip/IMG_0001 (500).heic -> ${linkTo("500.heic")}`,
|
|
`Trip/IMG_0001 (500).mov -> ${linkTo("500.mov")}`,
|
|
`Trip/IMG_0001 (501).heic -> ${linkTo("501.heic")}`,
|
|
`Trip/IMG_0001 (501).mov -> ${linkTo("501.mov")}`,
|
|
"Trip.json",
|
|
]);
|
|
await lib.close();
|
|
});
|
|
|
|
it("stores nothing for a live photo that fails its hash", async () => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(500, 10, "IMG_0500.HEIC"),
|
|
livePhotoZip(),
|
|
"not:the recorded hash",
|
|
);
|
|
const outDir = join(root, "backup");
|
|
const lib = await open([live], new Map([[500, body]]));
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result).toMatchObject({ downloaded: 0, failed: 1 });
|
|
expect(result.errors.map((e) => e.fileID)).toEqual([500]);
|
|
expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]);
|
|
expect(readdirSync(join(outDir, DAY))).toEqual([]);
|
|
expect(tree(outDir)).toEqual(["Trip/", "Trip.json"]);
|
|
await lib.close();
|
|
});
|
|
|
|
it("copies both parts of a live photo the cache already holds", async () => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(500, 10, "IMG_0500.HEIC"),
|
|
);
|
|
const lib = await open([live], new Map([[500, body]]));
|
|
const cached = await lib.photos.byID({ fileID: 500 })!.original();
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
|
|
expect(readdirSync(join(outDir, DAY)).sort()).toEqual(stored);
|
|
expect(readFileSync(saved(outDir, "500.mov"))).toEqual(
|
|
Buffer.from(VIDEO),
|
|
);
|
|
expect(tree(outDir)).toEqual(linked);
|
|
// The cache keeps its own copy.
|
|
expect(existsSync(cached.videoPath!)).toBe(true);
|
|
await lib.close();
|
|
});
|
|
|
|
it.each(["missing", "empty"])(
|
|
"fetches a live photo again when the video its JSON file names is %s",
|
|
async (state) => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(500, 10, "IMG_0500.HEIC"),
|
|
);
|
|
const lib = await open([live], new Map([[500, body]]));
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
const video = saved(outDir, "500.mov");
|
|
if (state === "missing") rmSync(video);
|
|
else writeFileSync(video, "");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result).toMatchObject({
|
|
downloaded: 1,
|
|
skipped: 0,
|
|
failed: 0,
|
|
});
|
|
expect(readFileSync(video)).toEqual(Buffer.from(VIDEO));
|
|
expect(tree(outDir)).toEqual(linked);
|
|
await lib.close();
|
|
},
|
|
);
|
|
|
|
it.each(["missing", "empty"])(
|
|
"fetches a live photo again when the image its JSON file names is %s",
|
|
async (state) => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(500, 10, "IMG_0500.HEIC"),
|
|
);
|
|
const lib = await open([live], new Map([[500, body]]));
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
const image = saved(outDir, "500.heic");
|
|
if (state === "missing") rmSync(image);
|
|
else writeFileSync(image, "");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result).toMatchObject({
|
|
downloaded: 1,
|
|
skipped: 0,
|
|
failed: 0,
|
|
});
|
|
expect(readFileSync(image)).toEqual(Buffer.from(IMAGE));
|
|
expect(tree(outDir)).toEqual(linked);
|
|
await lib.close();
|
|
},
|
|
);
|
|
|
|
it("verifies a live photo's image and video together, and downloads it again when one does not match", async () => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(500, 10, "IMG_0500.HEIC"),
|
|
);
|
|
const lib = await open([live], new Map([[500, body]]));
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
const good = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
verify: true,
|
|
});
|
|
expect(good).toMatchObject({ skipped: 1, verified: 1, mismatched: 0 });
|
|
|
|
const video = saved(outDir, "500.mov");
|
|
writeFileSync(video, "another few seconds of video");
|
|
const bad = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
verify: true,
|
|
});
|
|
|
|
expect(bad).toMatchObject({
|
|
downloaded: 1,
|
|
verified: 0,
|
|
mismatched: 1,
|
|
failed: 0,
|
|
});
|
|
expect(readFileSync(saved(outDir, "500.heic"))).toEqual(
|
|
Buffer.from(IMAGE),
|
|
);
|
|
expect(readFileSync(video)).toEqual(Buffer.from(VIDEO));
|
|
expect(tree(outDir)).toEqual(linked);
|
|
await lib.close();
|
|
});
|
|
|
|
it("removes both files of a live photo that does not match its hash when downloading it again fails", async () => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(500, 10, "IMG_0500.HEIC"),
|
|
);
|
|
const bodies = new Map([[500, body]]);
|
|
const lib = await open([live], bodies);
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
writeFileSync(saved(outDir, "500.mov"), "another few seconds of video");
|
|
bodies.delete(500);
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
verify: true,
|
|
});
|
|
|
|
expect(result).toMatchObject({ mismatched: 1, failed: 1 });
|
|
expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]);
|
|
expect(existsSync(saved(outDir, "500.heic"))).toBe(false);
|
|
expect(existsSync(saved(outDir, "500.mov"))).toBe(false);
|
|
await lib.close();
|
|
});
|
|
|
|
it("serves a live photo the backup stored to a library reading the backup", async () => {
|
|
const { file: live, body } = await asLivePhoto(
|
|
file(500, 10, "IMG_0500.HEIC"),
|
|
);
|
|
const lib = await open([live], new Map([[500, body]]));
|
|
const outDir = join(root, "backup");
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
await lib.close();
|
|
|
|
// Another cache over the same backup, whose server has nothing.
|
|
const reader = await Library.open({
|
|
client: new TripClient([live]),
|
|
cacheDirectory: join(root, "other-cache"),
|
|
downloadDirectory: outDir,
|
|
contentSource: cdnSource(new Map()),
|
|
refreshIntervalSeconds: 3600,
|
|
precacheThumbnails: false,
|
|
precacheOriginals: false,
|
|
});
|
|
const read = await reader.photos.byID({ fileID: 500 })!.original();
|
|
|
|
expect(read).toEqual({
|
|
path: saved(outDir, "500.heic"),
|
|
videoPath: saved(outDir, "500.mov"),
|
|
bytes: IMAGE.length,
|
|
});
|
|
await reader.close();
|
|
});
|
|
});
|