Save path ./photos/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.fileID.ext; download() from the cache first (closes #143)
check / check (push) Successful in 1m25s

Originals are saved at `{downloadDirectory}/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.{fileID}{ext}`. The date is the photo's `takenAt` in local time, and `downloadDirectory` defaults to `./photos`, resolved when the library opens. The old `originals/` layout is gone.

`photo.download()` writes the original to `savePath`. It copies from the cache when the cache holds the original, and fetches otherwise. `lib.backup()` uses the same path and rule, and every album in `collections/` links to it. `isLocal` is true only when the original is at `savePath`.

For a file in several albums, one rule picks the copy everything uses: the most recently synced, with the lowest album ID breaking a tie.

Model: opus-5-5
This commit was merged in pull request #146.
This commit is contained in:
2026-10-01 23:20:58 +02:00
parent 67d554fb46
commit 10e1a9ef39
16 changed files with 975 additions and 499 deletions
+178 -146
View File
@@ -1,16 +1,16 @@
/**
* Tests for the `quak backup` logic, now built on the library API (issue #51).
*
* `lib.backup({ downloadDirectory })` refreshes the library, fetches each
* pending file's original through the content cache/pools, and materialises the
* unchanged on-disk layout:
* `lib.backup({ downloadDirectory })` refreshes the library, puts each pending
* file's original at its save path through the content cache/pools, and
* materialises the on-disk layout:
*
* <downloadDirectory>/
* originals/
* <fileID>.<ext> the decrypted bytes ("present means complete")
* <fileID>.json per-file metadata sidecar (rebuilt each run)
* YYYY/YYYY-MM/YYYY-MM-DD/
* YYYY-MM-DD.<fileID>.<ext> the decrypted bytes ("present means complete")
* YYYY-MM-DD.<fileID>.json per-file metadata sidecar (rebuilt each run)
* collections/
* <name>/<title> symlink into ../originals (rebuilt each run)
* <name>/<title> symlink to the original (rebuilt each run)
* <name>.json per-collection metadata (rebuilt each run)
* failures.json durable ledger of unresolved failures
*
@@ -94,6 +94,18 @@ const USER_ID = 42;
// Decrypted-byte length each stub original writes, keyed by fileID.
const SIZE_BY_ID: Record<number, number> = { 100: 3000, 101: 2000, 200: 1500 };
// Every file is taken at noon local time on 2026-03-01, in microseconds as Ente
// stores times, so the machine's time zone cannot move it to another day. Its
// original is saved in the folder `DAY`, named `2026-03-01.<fileID>.<ext>`.
const TAKEN = new Date(2026, 2, 1, 12).getTime() * 1000;
const DAY = join("2026", "2026-03", "2026-03-01");
// Where the backup in `outDir` saves `name` (`<fileID>.<ext>`), and the target
// of an album folder's symlink to it.
const saved = (outDir: string, name: string): string =>
join(outDir, DAY, `2026-03-01.${name}`);
const linkTo = (name: string): string => `../../${DAY}/2026-03-01.${name}`;
const collection = (id: number, name: string): Collection => ({
id,
ownerID: USER_ID,
@@ -112,7 +124,7 @@ const file = (id: number, collectionID: number, title: string): EnteFile => ({
metadata: {
title,
fileType: "image",
creationTime: 1,
creationTime: TAKEN,
modificationTime: 1,
},
file: { decryptionHeader: "aGVhZGVy" },
@@ -253,7 +265,11 @@ describe("lib.backup", () => {
it("throws before any network when no downloadDirectory is given", async () => {
const source = stubSource();
const lib = await openLibrary(source);
await expect(lib.backup()).rejects.toThrow(/downloadDirectory/i);
// The library always has a download directory, so only an empty one
// given to backup() reaches runBackup as none.
await expect(lib.backup({ downloadDirectory: "" })).rejects.toThrow(
/downloadDirectory/i,
);
expect(source.originalCalls).toBe(0);
lib.close();
});
@@ -271,28 +287,22 @@ describe("lib.backup", () => {
expect(result.failed).toBe(0);
expect(result.errors).toEqual([]);
// Originals under originals/<fileID>.<ext>.
expect(readFileSync(join(outDir, "originals", "100.jpg")).length).toBe(
3000,
);
expect(readFileSync(join(outDir, "originals", "101.jpg")).length).toBe(
2000,
);
expect(readFileSync(join(outDir, "originals", "200.png")).length).toBe(
1500,
);
// Originals at YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>.
expect(readFileSync(saved(outDir, "100.jpg")).length).toBe(3000);
expect(readFileSync(saved(outDir, "101.jpg")).length).toBe(2000);
expect(readFileSync(saved(outDir, "200.png")).length).toBe(1500);
// Per-file metadata sidecar.
const sidecar = JSON.parse(
readFileSync(join(outDir, "originals", "100.json"), "utf-8"),
readFileSync(saved(outDir, "100.json"), "utf-8"),
);
expect(sidecar.id).toBe(100);
expect(sidecar.metadata.title).toBe("beach.jpg");
// Collection dirs contain symlinks into ../originals.
// Collection dirs contain symlinks into the date folders.
const beach = join(outDir, "collections", "Vacation", "beach.jpg");
expect(lstatSync(beach).isSymbolicLink()).toBe(true);
expect(readlinkSync(beach)).toContain("originals");
expect(readlinkSync(beach)).toContain(DAY);
expect(readFileSync(beach).length).toBe(3000);
// Per-collection metadata JSON.
@@ -316,7 +326,7 @@ describe("lib.backup", () => {
expect(result.failed).toBe(0);
// The title has no usable extension, so the original is `.bin`.
expect(existsSync(join(outDir, "originals", "300.bin"))).toBe(true);
expect(existsSync(saved(outDir, "300.bin"))).toBe(true);
const link = join(
outDir,
"collections",
@@ -366,9 +376,9 @@ describe("lib.backup", () => {
expect(result.errors[0]!.title).toBe("sunset.jpg");
// The two good files are on disk; the failed one is not.
expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true);
expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true);
expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(false);
expect(existsSync(saved(outDir, "100.jpg"))).toBe(true);
expect(existsSync(saved(outDir, "200.png"))).toBe(true);
expect(existsSync(saved(outDir, "101.jpg"))).toBe(false);
expect(
existsSync(join(outDir, "collections", "Vacation", "sunset.jpg")),
).toBe(false);
@@ -403,7 +413,7 @@ describe("lib.backup", () => {
const r3 = await lib.backup({ downloadDirectory: outDir });
expect(r3.failed).toBe(0);
expect(r3.skipped).toBe(2);
expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(true);
expect(existsSync(saved(outDir, "101.jpg"))).toBe(true);
// A ledger with no remaining failures is removed.
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
lib.close();
@@ -423,8 +433,8 @@ describe("lib.backup", () => {
const result = await lib.backup({ downloadDirectory: outDir });
// Every original still downloads despite the symlink failure.
expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true);
expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true);
expect(existsSync(saved(outDir, "100.jpg"))).toBe(true);
expect(existsSync(saved(outDir, "200.png"))).toBe(true);
// The other symlinks are still built.
expect(
lstatSync(
@@ -454,7 +464,7 @@ describe("lib.backup", () => {
await lib.backup({ downloadDirectory: outDir });
// Corrupt a sidecar and delete a symlink between runs.
writeFileSync(join(outDir, "originals", "100.json"), "not json");
writeFileSync(saved(outDir, "100.json"), "not json");
rmSync(join(outDir, "collections", "Vacation", "beach.jpg"));
const result = await lib.backup({ downloadDirectory: outDir });
@@ -462,7 +472,7 @@ describe("lib.backup", () => {
// The derived views are repaired from the model.
const sidecar = JSON.parse(
readFileSync(join(outDir, "originals", "100.json"), "utf-8"),
readFileSync(saved(outDir, "100.json"), "utf-8"),
);
expect(sidecar.metadata.title).toBe("beach.jpg");
expect(
@@ -485,8 +495,8 @@ describe("lib.backup", () => {
expect(result.totalFiles).toBe(1);
expect(result.downloaded).toBe(1);
expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true);
expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(false);
expect(existsSync(saved(outDir, "200.png"))).toBe(true);
expect(existsSync(saved(outDir, "100.jpg"))).toBe(false);
expect(existsSync(join(outDir, "collections", "Work.json"))).toBe(true);
expect(existsSync(join(outDir, "collections", "Vacation.json"))).toBe(
false,
@@ -530,7 +540,7 @@ describe("lib.backup", () => {
expect(result.totalFiles).toBe(1);
expect(result.failed).toBe(0);
expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true);
expect(existsSync(saved(outDir, "200.png"))).toBe(true);
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
lib.close();
});
@@ -572,7 +582,7 @@ describe("lib.backup", () => {
it("fetches each original once and writes it only into the backup", async () => {
// A backup of a 500 GB account must write 500 GB, not a copy in the
// cache as well: an original fetched for the backup goes straight
// into its originals/, and the cache records it there.
// to its save path, and the cache records it there.
const source = stubSource();
const lib = await openLibrary(source);
const outDir = join(root, "backup");
@@ -582,23 +592,78 @@ describe("lib.backup", () => {
expect(result.downloaded).toBe(3);
expect(source.originalCalls).toBe(3);
expect(readdirSync(join(root, "cache", "originals"))).toEqual([]);
const stored = readdirSync(join(outDir, "originals")).filter(
const stored = readdirSync(join(outDir, DAY)).filter(
(name) => !name.endsWith(".json"),
);
expect(stored.sort()).toEqual(["100.jpg", "101.jpg", "200.png"]);
expect(stored.sort()).toEqual([
"2026-03-01.100.jpg",
"2026-03-01.101.jpg",
"2026-03-01.200.png",
]);
// The cache counts the backup's copy as present: reading the
// original afterwards fetches nothing and answers with that copy.
const read = await lib.photos.byID({ fileID: 100 })!.original();
expect(read.path).toBe(join(outDir, "originals", "100.jpg"));
expect(read.path).toBe(saved(outDir, "100.jpg"));
expect(source.originalCalls).toBe(3);
await lib.close();
});
it("saves a file in two albums at its photo's save path and links it from both", async () => {
// The date edited in Ente has reached Work's copy, synced later, but
// Vacation's copy still has an earlier edit.
const older = file(100, 1, "beach.jpg");
older.pubMagicMetadata = {
editedTime: new Date(2026, 1, 1, 12).getTime() * 1000,
};
const newer = file(100, 2, "beach.jpg");
newer.updationTime = 2;
newer.pubMagicMetadata = {
editedTime: new Date(2026, 3, 15, 12).getTime() * 1000,
};
class SharedFileClient extends MockClient {
override async filesSince(args: {
collectionID: number;
}): Promise<FilesPage> {
const files = [args.collectionID === 1 ? older : newer];
return { files, deleted: [], cursor: 1 };
}
}
const outDir = join(root, "backup");
const lib = await Library.open({
client: new SharedFileClient(),
cacheDirectory: join(root, "cache"),
downloadDirectory: outDir,
contentSource: stubSource(),
refreshIntervalSeconds: 3600,
precacheThumbnails: false,
precacheOriginals: false,
});
const photo = lib.photos.byID({ fileID: 100 })!;
const result = await lib.backup();
expect(result.totalFiles).toBe(1);
expect(result.downloaded).toBe(1);
expect(result.failed).toBe(0);
expect(photo.savePath).toBe(
join(outDir, "2026", "2026-04", "2026-04-15", "2026-04-15.100.jpg"),
);
expect(photo.isLocal).toBe(true);
expect(existsSync(join(outDir, "2026", "2026-02"))).toBe(false);
const target = "../../2026/2026-04/2026-04-15/2026-04-15.100.jpg";
for (const album of ["Vacation", "Work"]) {
expect(
readlinkSync(join(outDir, "collections", album, "beach.jpg")),
).toBe(target);
}
await lib.close();
});
it("fsyncs an original copied from the cache before the rename and its directory after", async () => {
const lib = await openLibrary(stubSource());
const outDir = join(root, "backup");
const originals = join(outDir, "originals");
const dest = join(originals, "100.jpg");
const day = join(outDir, DAY);
const dest = saved(outDir, "100.jpg");
// Only an original already in the cache is copied into the backup;
// one fetched for the backup is written there by the download writer.
await lib.photos.byID({ fileID: 100 })!.original();
@@ -609,34 +674,50 @@ describe("lib.backup", () => {
const at = fsEvents.indexOf(`rename:${dest}`);
expect(at).toBeGreaterThan(0);
expect(fsEvents[at - 1]).toMatch(
/^sync:.*\/\.quak-backup-100\.jpg-\d+-[0-9a-z]*\.tmp$/,
/^sync:.*\/\.quak-backup-2026-03-01\.100\.jpg-\d+-[0-9a-z]*\.tmp$/,
);
expect(fsEvents[at + 1]).toBe(`sync:${originals}`);
expect(fsEvents[at + 1]).toBe(`sync:${day}`);
lib.close();
});
it("removes temp files left by a killed backup but not those of one still running", async () => {
const outDir = join(root, "backup");
const originals = join(outDir, "originals");
mkdirSync(originals, { recursive: true });
const day = join(outDir, DAY);
mkdirSync(day, { recursive: true });
// A child that has already exited: its process ID is not running.
const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid;
const leftover = `.quak-backup-100.jpg-${exitedPID}-abc123.tmp`;
const leftover = `.quak-backup-2026-03-01.100.jpg-${exitedPID}-abc123.tmp`;
// This test's own process stands in for a backup running at the same
// time.
const inProgress = `.quak-backup-101.jpg-${process.pid}-def456.tmp`;
writeFileSync(join(originals, leftover), "partial");
writeFileSync(join(originals, inProgress), "partial");
const inProgress = `.quak-backup-2026-03-01.101.jpg-${process.pid}-def456.tmp`;
writeFileSync(join(day, leftover), "partial");
writeFileSync(join(day, inProgress), "partial");
const lib = await openLibrary(stubSource());
await lib.backup({ downloadDirectory: outDir });
const names = readdirSync(originals);
const names = readdirSync(day);
expect(names).not.toContain(leftover);
expect(names).toContain(inProgress);
lib.close();
});
it("removes temp files left in a date folder no file in the backup is saved in", async () => {
const outDir = join(root, "backup");
// As for a file since deleted, or given another date, after a run was
// killed while writing it.
const otherDay = join(outDir, "2025", "2025-01", "2025-01-02");
mkdirSync(otherDay, { recursive: true });
const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid;
writeFileSync(join(otherDay, `.quak-${exitedPID}-abc123.tmp`), "x");
const lib = await openLibrary(stubSource());
await lib.backup({ downloadDirectory: outDir });
expect(readdirSync(otherDay)).toEqual([]);
lib.close();
});
it("removes leftover temp files in thumbnails/ but not those of a backup still running", async () => {
const outDir = join(root, "backup");
const thumbnails = join(outDir, "thumbnails");
@@ -709,7 +790,7 @@ describe("the refresh before a backup", () => {
const result = await backup;
expect(result.totalFiles).toBe(4);
expect(existsSync(join(outDir, "originals", "300.jpg"))).toBe(true);
expect(existsSync(saved(outDir, "300.jpg"))).toBe(true);
await lib.close();
});
@@ -728,7 +809,7 @@ describe("the refresh before a backup", () => {
expect(source.originalCalls).toBe(0);
expect(readFileSync(ledgerPath, "utf-8")).toBe(ledgerBefore);
expect(existsSync(join(outDir, "originals"))).toBe(false);
expect(existsSync(join(outDir, DAY))).toBe(false);
await lib.close();
});
@@ -819,13 +900,13 @@ describe("backup album folders", () => {
expect(result.failed).toBe(0);
expect(tree(outDir)).toEqual([
"Trip (10)/",
"Trip (10)/IMG_0001 (1).JPG -> ../../originals/1.JPG",
"Trip (10)/IMG_0001 (2).JPG -> ../../originals/2.JPG",
"Trip (10)/img_0001 (4).jpg -> ../../originals/4.jpg",
"Trip (10)/other.jpg -> ../../originals/3.jpg",
`Trip (10)/IMG_0001 (1).JPG -> ${linkTo("1.JPG")}`,
`Trip (10)/IMG_0001 (2).JPG -> ${linkTo("2.JPG")}`,
`Trip (10)/img_0001 (4).jpg -> ${linkTo("4.jpg")}`,
`Trip (10)/other.jpg -> ${linkTo("3.jpg")}`,
"Trip (10).json",
"Trip (11)/",
"Trip (11)/other.jpg -> ../../originals/3.jpg",
`Trip (11)/other.jpg -> ${linkTo("3.jpg")}`,
"Trip (11).json",
]);
expect(albumID(outDir, "Trip (10).json")).toBe(10);
@@ -858,14 +939,14 @@ describe("backup album folders", () => {
expect(result.failed).toBe(0);
expect(tree(outDir)).toEqual([
"Trip (10)/",
"Trip (10)/IMG (6) (5).JPG -> ../../originals/5.JPG",
"Trip (10)/IMG (6).JPG -> ../../originals/6.JPG",
"Trip (10)/IMG (7).JPG -> ../../originals/7.JPG",
`Trip (10)/IMG (6) (5).JPG -> ${linkTo("5.JPG")}`,
`Trip (10)/IMG (6).JPG -> ${linkTo("6.JPG")}`,
`Trip (10)/IMG (7).JPG -> ${linkTo("7.JPG")}`,
"Trip (10).json",
"Trip (11)/",
"Trip (11)/a.jpg -> ../../originals/8.jpg",
`Trip (11)/a.jpg -> ${linkTo("8.jpg")}`,
"Trip (11) (12)/",
"Trip (11) (12)/b.jpg -> ../../originals/9.jpg",
`Trip (11) (12)/b.jpg -> ${linkTo("9.jpg")}`,
"Trip (11) (12).json",
"Trip (11).json",
]);
@@ -931,13 +1012,13 @@ describe("backup album folders", () => {
expect(scoped.failed).toBe(0);
expect(before).toEqual([
"Trip (10)/",
"Trip (10)/a.jpg -> ../../originals/1.jpg",
`Trip (10)/a.jpg -> ${linkTo("1.jpg")}`,
"Trip (10).json",
"Work/",
"Work/c.jpg -> ../../originals/3.jpg",
`Work/c.jpg -> ${linkTo("3.jpg")}`,
"Work.json",
"trip (11)/",
"trip (11)/b.jpg -> ../../originals/2.jpg",
`trip (11)/b.jpg -> ${linkTo("2.jpg")}`,
"trip (11).json",
]);
expect(tree(outDir)).toEqual(before);
@@ -990,13 +1071,13 @@ describe("backup album folders", () => {
"Mine/",
"Mine/keep.txt",
"Office/",
"Office/a.jpg -> ../../originals/5.jpg",
`Office/a.jpg -> ${linkTo("5.jpg")}`,
"Office.json",
"Trip/",
"Trip/IMG_0001.JPG -> ../../originals/1.JPG",
`Trip/IMG_0001.JPG -> ${linkTo("1.JPG")}`,
"Trip/mine -> ../elsewhere",
"Trip/notes.txt",
"Trip/other.jpg -> ../../originals/3.jpg",
`Trip/other.jpg -> ${linkTo("3.jpg")}`,
"Trip.json",
"Work/",
"Work/keep.txt",
@@ -1005,7 +1086,7 @@ describe("backup album folders", () => {
});
// One album backed up, then a folder the user made beside it holding a
// symlink into originals/, with `json` (if given) as its sibling JSON.
// symlink to an original, with `json` (if given) as its sibling JSON.
const backupWithUserFolder = async (
json: string | undefined,
): Promise<{ outDir: string; failed: number }> => {
@@ -1019,10 +1100,7 @@ describe("backup album folders", () => {
await runBackup(lib, { downloadDirectory: outDir });
const collectionsDir = join(outDir, "collections");
mkdirSync(join(collectionsDir, "Mine"));
symlinkSync(
"../../originals/1.jpg",
join(collectionsDir, "Mine", "a.jpg"),
);
symlinkSync(linkTo("1.jpg"), join(collectionsDir, "Mine", "a.jpg"));
if (json !== undefined) {
writeFileSync(join(collectionsDir, "Mine.json"), json);
}
@@ -1036,9 +1114,9 @@ describe("backup album folders", () => {
expect(failed).toBe(0);
expect(tree(outDir)).toEqual([
"Mine/",
"Mine/a.jpg -> ../../originals/1.jpg",
`Mine/a.jpg -> ${linkTo("1.jpg")}`,
"Trip/",
"Trip/a.jpg -> ../../originals/1.jpg",
`Trip/a.jpg -> ${linkTo("1.jpg")}`,
"Trip.json",
]);
});
@@ -1050,10 +1128,10 @@ describe("backup album folders", () => {
expect(failed).toBe(0);
expect(tree(outDir)).toEqual([
"Mine/",
"Mine/a.jpg -> ../../originals/1.jpg",
`Mine/a.jpg -> ${linkTo("1.jpg")}`,
"Mine.json",
"Trip/",
"Trip/a.jpg -> ../../originals/1.jpg",
`Trip/a.jpg -> ${linkTo("1.jpg")}`,
"Trip.json",
]);
expect(
@@ -1087,23 +1165,16 @@ describe("backup of live photos", () => {
const open = (files: EnteFile[], bodies: Map<number, Uint8Array>) =>
openLibrary(cdnSource(bodies), new TripClient(files));
// What an earlier version stored for live photo 500: the ZIP under the
// image's name, and its link.
const earlierZIP = (outDir: string): void => {
mkdirSync(join(outDir, "originals"), { recursive: true });
mkdirSync(join(outDir, "collections", "Trip"), { recursive: true });
writeFileSync(join(outDir, "originals", "500.HEIC"), livePhotoZip());
symlinkSync(
"../../originals/500.HEIC",
join(outDir, "collections", "Trip", "IMG_0500.HEIC"),
);
};
const stored = ["500.heic", "500.json", "500.livephoto.json", "500.mov"];
const stored = [
"2026-03-01.500.heic",
"2026-03-01.500.json",
"2026-03-01.500.livephoto.json",
"2026-03-01.500.mov",
];
const linked = [
"Trip/",
"Trip/IMG_0500.heic -> ../../originals/500.heic",
"Trip/IMG_0500.mov -> ../../originals/500.mov",
`Trip/IMG_0500.heic -> ${linkTo("500.heic")}`,
`Trip/IMG_0500.mov -> ${linkTo("500.mov")}`,
"Trip.json",
];
@@ -1113,16 +1184,15 @@ describe("backup of live photos", () => {
);
const lib = await open([live], new Map([[500, body]]));
const outDir = join(root, "backup");
const originals = join(outDir, "originals");
const result = await lib.backup({ downloadDirectory: outDir });
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
expect(readdirSync(originals).sort()).toEqual(stored);
expect(readFileSync(join(originals, "500.heic"))).toEqual(
expect(readdirSync(join(outDir, DAY)).sort()).toEqual(stored);
expect(readFileSync(saved(outDir, "500.heic"))).toEqual(
Buffer.from(IMAGE),
);
expect(readFileSync(join(originals, "500.mov"))).toEqual(
expect(readFileSync(saved(outDir, "500.mov"))).toEqual(
Buffer.from(VIDEO),
);
expect(tree(outDir)).toEqual(linked);
@@ -1149,39 +1219,22 @@ describe("backup of live photos", () => {
expect(tree(outDir)).toEqual([
"Trip/",
"Trip/IMG_0001 (500).heic -> ../../originals/500.heic",
"Trip/IMG_0001 (500).mov -> ../../originals/500.mov",
"Trip/IMG_0001 (501).heic -> ../../originals/501.heic",
"Trip/IMG_0001 (501).mov -> ../../originals/501.mov",
`Trip/IMG_0001 (500).heic -> ${linkTo("500.heic")}`,
`Trip/IMG_0001 (500).mov -> ${linkTo("500.mov")}`,
`Trip/IMG_0001 (501).heic -> ${linkTo("501.heic")}`,
`Trip/IMG_0001 (501).mov -> ${linkTo("501.mov")}`,
"Trip.json",
]);
await lib.close();
});
it("replaces the ZIP an earlier version stored, and its link", async () => {
const { file: live, body } = await asLivePhoto(
file(500, 10, "IMG_0500.HEIC"),
);
const outDir = join(root, "backup");
earlierZIP(outDir);
const lib = await open([live], new Map([[500, body]]));
const result = await lib.backup({ downloadDirectory: outDir });
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored);
expect(tree(outDir)).toEqual(linked);
await lib.close();
});
it("stores nothing for a live photo that fails its hash, and keeps what was there", async () => {
it("stores nothing for a live photo that fails its hash", async () => {
const { file: live, body } = await asLivePhoto(
file(500, 10, "IMG_0500.HEIC"),
livePhotoZip(),
"not:the recorded hash",
);
const outDir = join(root, "backup");
earlierZIP(outDir);
const lib = await open([live], new Map([[500, body]]));
const result = await lib.backup({ downloadDirectory: outDir });
@@ -1189,12 +1242,8 @@ describe("backup of live photos", () => {
expect(result).toMatchObject({ downloaded: 0, failed: 1 });
expect(result.errors.map((e) => e.fileID)).toEqual([500]);
expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]);
expect(readdirSync(join(outDir, "originals"))).toEqual(["500.HEIC"]);
expect(tree(outDir)).toEqual([
"Trip/",
"Trip/IMG_0500.HEIC -> ../../originals/500.HEIC",
"Trip.json",
]);
expect(readdirSync(join(outDir, DAY))).toEqual([]);
expect(tree(outDir)).toEqual(["Trip/", "Trip.json"]);
await lib.close();
});
@@ -1209,8 +1258,8 @@ describe("backup of live photos", () => {
const result = await lib.backup({ downloadDirectory: outDir });
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored);
expect(readFileSync(join(outDir, "originals", "500.mov"))).toEqual(
expect(readdirSync(join(outDir, DAY)).sort()).toEqual(stored);
expect(readFileSync(saved(outDir, "500.mov"))).toEqual(
Buffer.from(VIDEO),
);
expect(tree(outDir)).toEqual(linked);
@@ -1219,23 +1268,6 @@ describe("backup of live photos", () => {
await lib.close();
});
it("replaces an earlier ZIP and its link with the image and video the cache holds", async () => {
const { file: live, body } = await asLivePhoto(
file(500, 10, "IMG_0500.HEIC"),
);
const lib = await open([live], new Map([[500, body]]));
await lib.photos.byID({ fileID: 500 })!.original();
const outDir = join(root, "backup");
earlierZIP(outDir);
const result = await lib.backup({ downloadDirectory: outDir });
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored);
expect(tree(outDir)).toEqual(linked);
await lib.close();
});
it.each(["missing", "empty"])(
"fetches a live photo again when the video its JSON file names is %s",
async (state) => {
@@ -1245,7 +1277,7 @@ describe("backup of live photos", () => {
const lib = await open([live], new Map([[500, body]]));
const outDir = join(root, "backup");
await lib.backup({ downloadDirectory: outDir });
const video = join(outDir, "originals", "500.mov");
const video = saved(outDir, "500.mov");
if (state === "missing") rmSync(video);
else writeFileSync(video, "");
@@ -1271,7 +1303,7 @@ describe("backup of live photos", () => {
const lib = await open([live], new Map([[500, body]]));
const outDir = join(root, "backup");
await lib.backup({ downloadDirectory: outDir });
const image = join(outDir, "originals", "500.heic");
const image = saved(outDir, "500.heic");
if (state === "missing") rmSync(image);
else writeFileSync(image, "");
@@ -1310,8 +1342,8 @@ describe("backup of live photos", () => {
const read = await reader.photos.byID({ fileID: 500 })!.original();
expect(read).toEqual({
path: join(outDir, "originals", "500.heic"),
videoPath: join(outDir, "originals", "500.mov"),
path: saved(outDir, "500.heic"),
videoPath: saved(outDir, "500.mov"),
bytes: IMAGE.length,
});
await reader.close();
+1 -1
View File
@@ -757,7 +757,7 @@ describe("backup", () => {
expect(code).toBe(1);
expect(runText).toBe("quak: HTTP 401 from server\n");
expect(stderr.text).toBe("Starting backup...\nRefreshing library...\n");
expect(existsSync(join(dir, "originals"))).toBe(false);
expect(existsSync(dir)).toBe(false);
});
});
+1
View File
@@ -142,6 +142,7 @@ const buildCache = (args: {
pools: new RequestPools(),
source,
cacheDirectory: cacheDir,
downloadDirectory: join(root, "photos"),
getFile: (id) => byID.get(id),
statfs: args.statfs,
cacheOriginalsMaxBytes: args.cacheOriginalsMaxBytes,
+237 -13
View File
@@ -6,16 +6,18 @@
* `Photo` objects that fetch through it, `lib.thumbnails.ensure` drives it, and
* a cached path shows up on the projected record. A library opened without a
* content source leaves those methods throwing rather than silently doing
* nothing. It also covers a `Photo`'s `savePath`, `isLocal`, `content()` and
* `exif()`.
* nothing. It also covers a `Photo`'s `savePath`, `isLocal`, `download()`,
* `content()` and `exif()`.
*/
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import {
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
existsSync,
statSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
@@ -38,6 +40,12 @@ import {
const USER_ID = 7;
// Every file is taken at noon local time on 2026-03-01, in microseconds as Ente
// stores times, so the machine's time zone cannot move it to another day; it
// is saved in the folder `DAY`.
const TAKEN = new Date(2026, 2, 1, 12).getTime() * 1000;
const DAY = join("2026", "2026-03", "2026-03-01");
const collection = (id: number): Collection => ({
id,
ownerID: USER_ID,
@@ -56,7 +64,7 @@ const file = (id: number, collectionID: number): EnteFile => ({
metadata: {
title: `file-${id}.jpg`,
fileType: "image",
creationTime: 1,
creationTime: TAKEN,
modificationTime: 1,
},
file: { decryptionHeader: "aGVhZGVy" },
@@ -224,6 +232,9 @@ describe("Library content wiring", () => {
await expect(
lib.photos.byID({ fileID: 1 })!.thumbnail(),
).rejects.toThrow(/content cache/i);
await expect(
lib.photos.byID({ fileID: 1 })!.download(),
).rejects.toThrow(/content cache/i);
await expect(
lib.thumbnails.ensure({ fileIDs: [1], priority: "visible" }),
).rejects.toThrow(/content cache/i);
@@ -341,7 +352,7 @@ describe("Photo save path, local copy, content and EXIF", () => {
it("names where a backup writes the original, which is local once the backup has written it", async () => {
const lib = await open();
const photo = lib.photos.byID({ fileID: 1 })!;
const savePath = join(root, "backup", "originals", "1.jpg");
const savePath = join(root, "backup", DAY, "2026-03-01.1.jpg");
expect(photo.savePath).toBe(savePath);
expect(photo.isLocal).toBe(false);
@@ -359,23 +370,150 @@ describe("Photo save path, local copy, content and EXIF", () => {
expect(existsSync(join(root, "cache", "originals", "1.jpg"))).toBe(
true,
);
expect(existsSync(photo.savePath!)).toBe(false);
expect(existsSync(photo.savePath)).toBe(false);
expect(photo.isLocal).toBe(false);
await lib.close();
});
it("has no save path and is not local without a download directory", async () => {
it("saves under photos/ in the working directory at open without a download directory", async () => {
const cwd = vi.spyOn(process, "cwd").mockReturnValue(root);
const lib = await open({ downloadDirectory: undefined });
cwd.mockRestore();
const photo = lib.photos.byID({ fileID: 1 })!;
expect(photo.savePath).toBeUndefined();
expect(lib.downloadDirectory).toBe(join(root, "photos"));
expect(photo.savePath).toBe(
join(root, "photos", DAY, "2026-03-01.1.jpg"),
);
expect(photo.isLocal).toBe(false);
await lib.close();
});
it("has no save path and is not local without a content source", async () => {
it("refuses an empty download directory", async () => {
await expect(open({ downloadDirectory: "" })).rejects.toThrow(
/downloadDirectory is empty/,
);
});
it("keeps a photo's save path after a refresh removes its file", async () => {
// The same account, whose album is deleted on the second refresh.
class AlbumDeletedClient extends MockClient {
override async collectionsSince(): Promise<CollectionsPage> {
if (!this.served) return super.collectionsSince();
return { collections: [], deleted: [1], cursor: 2 };
}
}
const lib = await open({ client: new AlbumDeletedClient() });
const photo = lib.photos.byID({ fileID: 1 })!;
await photo.download();
await lib.fresh();
expect(lib.photos.byID({ fileID: 1 })).toBeUndefined();
expect(photo.savePath).toBe(
join(root, "backup", DAY, "2026-03-01.1.jpg"),
);
expect(photo.isLocal).toBe(true);
await lib.close();
});
it("dates the save path and download() by the membership its takenAt comes from", async () => {
// One file in two albums. The date edited in Ente has reached album 2's
// copy, synced later, but album 1 still has an earlier edit.
const older = file(1, 1);
older.pubMagicMetadata = {
editedTime: new Date(2026, 1, 1, 12).getTime() * 1000,
};
const newer = file(1, 2);
newer.updationTime = 2;
newer.pubMagicMetadata = {
editedTime: new Date(2026, 3, 15, 12).getTime() * 1000,
};
const client = {
whoami: () => ({ email: "u@example.com", userID: USER_ID }),
collectionsSince: async (): Promise<CollectionsPage> => ({
collections: [collection(1), collection(2)],
deleted: [],
cursor: 1,
}),
filesSince: async (args: {
collectionID: number;
}): Promise<FilesPage> => ({
files: [args.collectionID === 1 ? older : newer],
deleted: [],
cursor: 1,
}),
};
const lib = await open({ client });
const photo = lib.photos.byID({ fileID: 1 })!;
expect(photo.takenAt).toBe(new Date(2026, 3, 15, 12).getTime());
expect(photo.savePath).toBe(
join(
root,
"backup",
"2026",
"2026-04",
"2026-04-15",
"2026-04-15.1.jpg",
),
);
// download() writes to that same path, so the photo is then local.
const saved = await photo.download();
expect(saved.path).toBe(photo.savePath);
expect(photo.isLocal).toBe(true);
expect(existsSync(join(root, "backup", "2026", "2026-02"))).toBe(false);
await lib.close();
});
it("downloads a photo held across a refresh that edits its date to the save path it names", async () => {
// The same account, whose second refresh brings a date edited in Ente.
const edited = file(1, 1);
edited.updationTime = 2;
edited.pubMagicMetadata = {
editedTime: new Date(2026, 3, 15, 12).getTime() * 1000,
};
class DateEditedClient extends MockClient {
refreshes = 0;
override async collectionsSince(): Promise<CollectionsPage> {
this.refreshes++;
return {
collections: [
{ ...collection(1), updationTime: this.refreshes },
],
deleted: [],
cursor: this.refreshes,
};
}
override async filesSince(): Promise<FilesPage> {
return {
files: [this.refreshes === 1 ? file(1, 1) : edited],
deleted: [],
cursor: this.refreshes,
};
}
}
const lib = await open({ client: new DateEditedClient() });
const photo = lib.photos.byID({ fileID: 1 })!;
await lib.fresh();
expect(lib.photos.byID({ fileID: 1 })!.takenAt).toBe(
new Date(2026, 3, 15, 12).getTime(),
);
const saved = await photo.download();
expect(saved.path).toBe(photo.savePath);
expect(saved.path).toBe(join(root, "backup", DAY, "2026-03-01.1.jpg"));
expect(photo.isLocal).toBe(true);
await lib.close();
});
it("has a save path, and is not local, without a content source", async () => {
const lib = await open({ contentSource: undefined });
const photo = lib.photos.byID({ fileID: 1 })!;
expect(photo.savePath).toBeUndefined();
expect(photo.savePath).toBe(
join(root, "backup", DAY, "2026-03-01.1.jpg"),
);
expect(photo.isLocal).toBe(false);
await lib.close();
});
@@ -387,18 +525,104 @@ describe("Photo save path, local copy, content and EXIF", () => {
contentSource: cdnSource(new Map([[1, body]])),
});
const photo = lib.photos.byID({ fileID: 1 })!;
const originals = join(root, "backup", "originals");
const day = join(root, "backup", DAY);
// Until then the name comes from the title, file-1.jpg; the backup
// stores the image with the extension found inside the live photo.
expect(photo.savePath).toBe(join(originals, "1.jpg"));
expect(photo.savePath).toBe(join(day, "2026-03-01.1.jpg"));
await lib.backup();
expect(photo.savePath).toBe(join(originals, "1.heic"));
expect(photo.savePath).toBe(join(day, "2026-03-01.1.heic"));
expect(photo.isLocal).toBe(true);
expect(await photo.content()).toEqual(Buffer.from(IMAGE));
await lib.close();
});
it("downloads an original the cache holds by copying it, without fetching it again", async () => {
const source = stubSource();
const lib = await open({ contentSource: source });
const photo = lib.photos.byID({ fileID: 1 })!;
await photo.original();
expect(source.originalCalls()).toBe(1);
expect(photo.isLocal).toBe(false);
const saved = await photo.download();
expect(source.originalCalls()).toBe(1);
expect(saved).toEqual({
path: photo.savePath,
bytes: "orig-bytes".length,
});
expect(readFileSync(photo.savePath, "utf-8")).toBe("orig-bytes");
expect(photo.isLocal).toBe(true);
// The cache keeps its own copy.
expect(existsSync(join(root, "cache", "originals", "1.jpg"))).toBe(
true,
);
await lib.close();
});
it("downloads an original the cache does not hold straight to its save path", async () => {
const source = stubSource();
const lib = await open({ contentSource: source });
const photo = lib.photos.byID({ fileID: 1 })!;
const saved = await photo.download();
expect(source.originalCalls()).toBe(1);
expect(saved.path).toBe(join(root, "backup", DAY, "2026-03-01.1.jpg"));
expect(readFileSync(saved.path, "utf-8")).toBe("orig-bytes");
expect(photo.isLocal).toBe(true);
expect(readdirSync(join(root, "cache", "originals"))).toEqual([]);
await lib.close();
});
it("does nothing when download() finds the original already at its save path", async () => {
const source = stubSource();
const lib = await open({ contentSource: source });
const photo = lib.photos.byID({ fileID: 1 })!;
const first = await photo.download();
const written = statSync(first.path).ino;
const second = await photo.download();
expect(second).toEqual(first);
expect(source.originalCalls()).toBe(1);
expect(statSync(second.path).ino).toBe(written);
await lib.close();
});
it("downloads a live photo the cache holds as its image, its video and the JSON file naming them", async () => {
const { file: live, body } = await asLivePhoto(file(1, 1));
const lib = await open({
client: new FilesClient([live]),
contentSource: cdnSource(new Map([[1, body]])),
});
const photo = lib.photos.byID({ fileID: 1 })!;
await photo.original();
const day = join(root, "backup", DAY);
const saved = await photo.download();
expect(saved).toEqual({
path: join(day, "2026-03-01.1.heic"),
videoPath: join(day, "2026-03-01.1.mov"),
bytes: IMAGE.length,
});
expect(readdirSync(day).sort()).toEqual([
"2026-03-01.1.heic",
"2026-03-01.1.livephoto.json",
"2026-03-01.1.mov",
]);
expect(
JSON.parse(
readFileSync(join(day, "2026-03-01.1.livephoto.json"), "utf-8"),
),
).toEqual({ image: "2026-03-01.1.heic", video: "2026-03-01.1.mov" });
expect(photo.savePath).toBe(saved.path);
expect(photo.isLocal).toBe(true);
await lib.close();
});
it("reads the common EXIF fields of a JPEG original", async () => {
const lib = await open({ contentSource: stubSource(JPEG_WITH_EXIF) });
expect(await lib.photos.byID({ fileID: 1 })!.exif()).toStrictEqual({
+50 -6
View File
@@ -7,8 +7,8 @@
* 1. **Fetch once, then serve from disk.** The first `original`/`thumbnail`
* fetches through the request pool and stores the bytes; the next finds the
* file present and returns its path with a single `skipped` event and no
* network. A file already sitting in the backup `downloadDirectory` counts
* as present too.
* network. A file already stored at its save path under the
* `downloadDirectory` counts as present too.
* 2. **Present-means-complete.** Content appears only by the streaming atomic
* writer's rename, so a file that exists is whole. The directory listing
* taken at `open()` is the record of what is cached, and the orphan temp
@@ -41,6 +41,7 @@ import { join } from "node:path";
import {
ContentCache,
savePath,
type ContentSource,
type EnsureEvent,
} from "../../src/library/content.js";
@@ -152,12 +153,48 @@ const buildCache = (
pools: args.pools ?? new RequestPools(),
source,
cacheDirectory: cacheDir,
downloadDirectory: args.downloadDirectory,
downloadDirectory: args.downloadDirectory ?? join(root, "photos"),
getFile: (id) => byID.get(id),
});
return { cache, source };
};
// Microseconds, as Ente stores times, for noon local time on a day, so the
// machine's time zone cannot move the photo to another day.
const noon = (year: number, month: number, day: number): number =>
new Date(year, month - 1, day, 12).getTime() * 1000;
describe("savePath", () => {
it("files an original by year, month and day under the root", () => {
const f = file(12345, "IMG_0001.HEIC");
f.metadata.creationTime = noon(2026, 3, 1);
expect(savePath("/photos", f)).toBe(
"/photos/2026/2026-03/2026-03-01/2026-03-01.12345.HEIC",
);
});
it("dates an original by the date the user set, when there is one", () => {
const f = file(7, "a.jpg");
f.metadata.creationTime = noon(2026, 3, 1);
f.pubMagicMetadata = { editedTime: noon(1999, 12, 31) };
expect(savePath("/photos", f)).toBe(
"/photos/1999/1999-12/1999-12-31/1999-12-31.7.jpg",
);
});
it("takes the extension from the title it was uploaded with, not a new name", () => {
const f = file(8, "upload");
f.metadata.creationTime = noon(2026, 3, 1);
f.pubMagicMetadata = { editedName: "renamed.png" };
expect(savePath("/photos", f)).toBe(
"/photos/2026/2026-03/2026-03-01/2026-03-01.8.bin",
);
});
});
describe("ContentCache.open", () => {
it("creates the cache directories with 0700 permissions", async () => {
const { cache } = buildCache();
@@ -274,11 +311,17 @@ describe("ContentCache.original / thumbnail", () => {
it("serves a file already present in the download directory without fetching", async () => {
const downloadDirectory = join(root, "backup");
mkdirSync(join(downloadDirectory, "originals"), { recursive: true });
const backupPath = join(downloadDirectory, "originals", "1.jpg");
const day = join(downloadDirectory, "2026", "2026-03", "2026-03-01");
mkdirSync(day, { recursive: true });
const backupPath = join(day, "2026-03-01.1.jpg");
writeFileSync(backupPath, "from-backup");
const f = file(1);
f.metadata.creationTime = noon(2026, 3, 1);
const { cache, source } = buildCache({ downloadDirectory });
const { cache, source } = buildCache({
downloadDirectory,
files: [f],
});
await cache.open();
const events: EnsureEvent["status"][] = [];
@@ -649,6 +692,7 @@ describe("ContentCache live photos", () => {
]),
),
cacheDirectory: cacheDir,
downloadDirectory: join(root, "photos"),
getFile: (id) => [a.file, b.file].find((f) => f.id === id),
// Room for one live photo, on a disk with plenty free.
cacheOriginalsMaxBytes: size,
+2
View File
@@ -280,6 +280,7 @@ describe("Precache eviction integration", () => {
pools: new RequestPools(),
source,
cacheDirectory: cacheDir,
downloadDirectory: join(root, "photos"),
getFile: (id) => byID.get(id),
statfs,
cacheOriginalsMaxBytes: 25, // holds two 10-byte originals
@@ -348,6 +349,7 @@ describe("Precache preemption", () => {
pools: new RequestPools({ contentConcurrency: 1 }),
source,
cacheDirectory: join(root, "cache"),
downloadDirectory: join(root, "photos"),
getFile: (id) => byID.get(id),
statfs: async () => ({ bsize: 1, bavail: 1_000_000_000 }),
freeBelowBytes: 0,
+11 -3
View File
@@ -36,6 +36,7 @@ import {
makeAlbumsAPI,
makePhotosAPI,
makeTimelineAPI,
type SavePathLookup,
type TimelineGroup,
} from "../../src/library/read.js";
import { Library } from "../../src/library/index.js";
@@ -101,12 +102,19 @@ const file = (
};
// Build the three API objects over one fixed projection, the way `Library`
// wires them over its live store.
// wires them over its live store. Save paths are covered in
// content-library.test.ts; these tests never ask for one.
const apis = (records: DerivedRecords) => {
const derive = () => records;
const saves: SavePathLookup = {
savePath: () => {
throw new Error("no save paths in these tests");
},
isLocal: () => false,
};
return {
albums: makeAlbumsAPI(derive),
photos: makePhotosAPI(derive),
albums: makeAlbumsAPI(derive, saves),
photos: makePhotosAPI(derive, saves),
timeline: makeTimelineAPI(derive),
};
};