Save path ./photos/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.fileID.ext; download() from the cache first (closes #143)
check / check (push) Successful in 1m25s
check / check (push) Successful in 1m25s
Originals are saved at `{downloadDirectory}/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.{fileID}{ext}`. The date is the photo's `takenAt` in local time, and `downloadDirectory` defaults to `./photos`, resolved when the library opens. The old `originals/` layout is gone.
`photo.download()` writes the original to `savePath`. It copies from the cache when the cache holds the original, and fetches otherwise. `lib.backup()` uses the same path and rule, and every album in `collections/` links to it. `isLocal` is true only when the original is at `savePath`.
For a file in several albums, one rule picks the copy everything uses: the most recently synced, with the lowest album ID breaking a tie.
Model: opus-5-5
This commit was merged in pull request #146.
This commit is contained in:
@@ -142,6 +142,7 @@ const buildCache = (args: {
|
||||
pools: new RequestPools(),
|
||||
source,
|
||||
cacheDirectory: cacheDir,
|
||||
downloadDirectory: join(root, "photos"),
|
||||
getFile: (id) => byID.get(id),
|
||||
statfs: args.statfs,
|
||||
cacheOriginalsMaxBytes: args.cacheOriginalsMaxBytes,
|
||||
|
||||
@@ -6,16 +6,18 @@
|
||||
* `Photo` objects that fetch through it, `lib.thumbnails.ensure` drives it, and
|
||||
* a cached path shows up on the projected record. A library opened without a
|
||||
* content source leaves those methods throwing rather than silently doing
|
||||
* nothing. It also covers a `Photo`'s `savePath`, `isLocal`, `content()` and
|
||||
* `exif()`.
|
||||
* nothing. It also covers a `Photo`'s `savePath`, `isLocal`, `download()`,
|
||||
* `content()` and `exif()`.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||
import {
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
existsSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -38,6 +40,12 @@ import {
|
||||
|
||||
const USER_ID = 7;
|
||||
|
||||
// Every file is taken at noon local time on 2026-03-01, in microseconds as Ente
|
||||
// stores times, so the machine's time zone cannot move it to another day; it
|
||||
// is saved in the folder `DAY`.
|
||||
const TAKEN = new Date(2026, 2, 1, 12).getTime() * 1000;
|
||||
const DAY = join("2026", "2026-03", "2026-03-01");
|
||||
|
||||
const collection = (id: number): Collection => ({
|
||||
id,
|
||||
ownerID: USER_ID,
|
||||
@@ -56,7 +64,7 @@ const file = (id: number, collectionID: number): EnteFile => ({
|
||||
metadata: {
|
||||
title: `file-${id}.jpg`,
|
||||
fileType: "image",
|
||||
creationTime: 1,
|
||||
creationTime: TAKEN,
|
||||
modificationTime: 1,
|
||||
},
|
||||
file: { decryptionHeader: "aGVhZGVy" },
|
||||
@@ -224,6 +232,9 @@ describe("Library content wiring", () => {
|
||||
await expect(
|
||||
lib.photos.byID({ fileID: 1 })!.thumbnail(),
|
||||
).rejects.toThrow(/content cache/i);
|
||||
await expect(
|
||||
lib.photos.byID({ fileID: 1 })!.download(),
|
||||
).rejects.toThrow(/content cache/i);
|
||||
await expect(
|
||||
lib.thumbnails.ensure({ fileIDs: [1], priority: "visible" }),
|
||||
).rejects.toThrow(/content cache/i);
|
||||
@@ -341,7 +352,7 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
||||
it("names where a backup writes the original, which is local once the backup has written it", async () => {
|
||||
const lib = await open();
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
const savePath = join(root, "backup", "originals", "1.jpg");
|
||||
const savePath = join(root, "backup", DAY, "2026-03-01.1.jpg");
|
||||
expect(photo.savePath).toBe(savePath);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
|
||||
@@ -359,23 +370,150 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
||||
expect(existsSync(join(root, "cache", "originals", "1.jpg"))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(existsSync(photo.savePath!)).toBe(false);
|
||||
expect(existsSync(photo.savePath)).toBe(false);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("has no save path and is not local without a download directory", async () => {
|
||||
it("saves under photos/ in the working directory at open without a download directory", async () => {
|
||||
const cwd = vi.spyOn(process, "cwd").mockReturnValue(root);
|
||||
const lib = await open({ downloadDirectory: undefined });
|
||||
cwd.mockRestore();
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
expect(photo.savePath).toBeUndefined();
|
||||
expect(lib.downloadDirectory).toBe(join(root, "photos"));
|
||||
expect(photo.savePath).toBe(
|
||||
join(root, "photos", DAY, "2026-03-01.1.jpg"),
|
||||
);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("has no save path and is not local without a content source", async () => {
|
||||
it("refuses an empty download directory", async () => {
|
||||
await expect(open({ downloadDirectory: "" })).rejects.toThrow(
|
||||
/downloadDirectory is empty/,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps a photo's save path after a refresh removes its file", async () => {
|
||||
// The same account, whose album is deleted on the second refresh.
|
||||
class AlbumDeletedClient extends MockClient {
|
||||
override async collectionsSince(): Promise<CollectionsPage> {
|
||||
if (!this.served) return super.collectionsSince();
|
||||
return { collections: [], deleted: [1], cursor: 2 };
|
||||
}
|
||||
}
|
||||
const lib = await open({ client: new AlbumDeletedClient() });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
await photo.download();
|
||||
|
||||
await lib.fresh();
|
||||
|
||||
expect(lib.photos.byID({ fileID: 1 })).toBeUndefined();
|
||||
expect(photo.savePath).toBe(
|
||||
join(root, "backup", DAY, "2026-03-01.1.jpg"),
|
||||
);
|
||||
expect(photo.isLocal).toBe(true);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("dates the save path and download() by the membership its takenAt comes from", async () => {
|
||||
// One file in two albums. The date edited in Ente has reached album 2's
|
||||
// copy, synced later, but album 1 still has an earlier edit.
|
||||
const older = file(1, 1);
|
||||
older.pubMagicMetadata = {
|
||||
editedTime: new Date(2026, 1, 1, 12).getTime() * 1000,
|
||||
};
|
||||
const newer = file(1, 2);
|
||||
newer.updationTime = 2;
|
||||
newer.pubMagicMetadata = {
|
||||
editedTime: new Date(2026, 3, 15, 12).getTime() * 1000,
|
||||
};
|
||||
const client = {
|
||||
whoami: () => ({ email: "u@example.com", userID: USER_ID }),
|
||||
collectionsSince: async (): Promise<CollectionsPage> => ({
|
||||
collections: [collection(1), collection(2)],
|
||||
deleted: [],
|
||||
cursor: 1,
|
||||
}),
|
||||
filesSince: async (args: {
|
||||
collectionID: number;
|
||||
}): Promise<FilesPage> => ({
|
||||
files: [args.collectionID === 1 ? older : newer],
|
||||
deleted: [],
|
||||
cursor: 1,
|
||||
}),
|
||||
};
|
||||
const lib = await open({ client });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
|
||||
expect(photo.takenAt).toBe(new Date(2026, 3, 15, 12).getTime());
|
||||
expect(photo.savePath).toBe(
|
||||
join(
|
||||
root,
|
||||
"backup",
|
||||
"2026",
|
||||
"2026-04",
|
||||
"2026-04-15",
|
||||
"2026-04-15.1.jpg",
|
||||
),
|
||||
);
|
||||
|
||||
// download() writes to that same path, so the photo is then local.
|
||||
const saved = await photo.download();
|
||||
expect(saved.path).toBe(photo.savePath);
|
||||
expect(photo.isLocal).toBe(true);
|
||||
expect(existsSync(join(root, "backup", "2026", "2026-02"))).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("downloads a photo held across a refresh that edits its date to the save path it names", async () => {
|
||||
// The same account, whose second refresh brings a date edited in Ente.
|
||||
const edited = file(1, 1);
|
||||
edited.updationTime = 2;
|
||||
edited.pubMagicMetadata = {
|
||||
editedTime: new Date(2026, 3, 15, 12).getTime() * 1000,
|
||||
};
|
||||
class DateEditedClient extends MockClient {
|
||||
refreshes = 0;
|
||||
override async collectionsSince(): Promise<CollectionsPage> {
|
||||
this.refreshes++;
|
||||
return {
|
||||
collections: [
|
||||
{ ...collection(1), updationTime: this.refreshes },
|
||||
],
|
||||
deleted: [],
|
||||
cursor: this.refreshes,
|
||||
};
|
||||
}
|
||||
override async filesSince(): Promise<FilesPage> {
|
||||
return {
|
||||
files: [this.refreshes === 1 ? file(1, 1) : edited],
|
||||
deleted: [],
|
||||
cursor: this.refreshes,
|
||||
};
|
||||
}
|
||||
}
|
||||
const lib = await open({ client: new DateEditedClient() });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
|
||||
await lib.fresh();
|
||||
expect(lib.photos.byID({ fileID: 1 })!.takenAt).toBe(
|
||||
new Date(2026, 3, 15, 12).getTime(),
|
||||
);
|
||||
|
||||
const saved = await photo.download();
|
||||
expect(saved.path).toBe(photo.savePath);
|
||||
expect(saved.path).toBe(join(root, "backup", DAY, "2026-03-01.1.jpg"));
|
||||
expect(photo.isLocal).toBe(true);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("has a save path, and is not local, without a content source", async () => {
|
||||
const lib = await open({ contentSource: undefined });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
expect(photo.savePath).toBeUndefined();
|
||||
expect(photo.savePath).toBe(
|
||||
join(root, "backup", DAY, "2026-03-01.1.jpg"),
|
||||
);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
@@ -387,18 +525,104 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
||||
contentSource: cdnSource(new Map([[1, body]])),
|
||||
});
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
const originals = join(root, "backup", "originals");
|
||||
const day = join(root, "backup", DAY);
|
||||
// Until then the name comes from the title, file-1.jpg; the backup
|
||||
// stores the image with the extension found inside the live photo.
|
||||
expect(photo.savePath).toBe(join(originals, "1.jpg"));
|
||||
expect(photo.savePath).toBe(join(day, "2026-03-01.1.jpg"));
|
||||
|
||||
await lib.backup();
|
||||
expect(photo.savePath).toBe(join(originals, "1.heic"));
|
||||
expect(photo.savePath).toBe(join(day, "2026-03-01.1.heic"));
|
||||
expect(photo.isLocal).toBe(true);
|
||||
expect(await photo.content()).toEqual(Buffer.from(IMAGE));
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("downloads an original the cache holds by copying it, without fetching it again", async () => {
|
||||
const source = stubSource();
|
||||
const lib = await open({ contentSource: source });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
await photo.original();
|
||||
expect(source.originalCalls()).toBe(1);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
|
||||
const saved = await photo.download();
|
||||
|
||||
expect(source.originalCalls()).toBe(1);
|
||||
expect(saved).toEqual({
|
||||
path: photo.savePath,
|
||||
bytes: "orig-bytes".length,
|
||||
});
|
||||
expect(readFileSync(photo.savePath, "utf-8")).toBe("orig-bytes");
|
||||
expect(photo.isLocal).toBe(true);
|
||||
// The cache keeps its own copy.
|
||||
expect(existsSync(join(root, "cache", "originals", "1.jpg"))).toBe(
|
||||
true,
|
||||
);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("downloads an original the cache does not hold straight to its save path", async () => {
|
||||
const source = stubSource();
|
||||
const lib = await open({ contentSource: source });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
|
||||
const saved = await photo.download();
|
||||
|
||||
expect(source.originalCalls()).toBe(1);
|
||||
expect(saved.path).toBe(join(root, "backup", DAY, "2026-03-01.1.jpg"));
|
||||
expect(readFileSync(saved.path, "utf-8")).toBe("orig-bytes");
|
||||
expect(photo.isLocal).toBe(true);
|
||||
expect(readdirSync(join(root, "cache", "originals"))).toEqual([]);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("does nothing when download() finds the original already at its save path", async () => {
|
||||
const source = stubSource();
|
||||
const lib = await open({ contentSource: source });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
const first = await photo.download();
|
||||
const written = statSync(first.path).ino;
|
||||
|
||||
const second = await photo.download();
|
||||
|
||||
expect(second).toEqual(first);
|
||||
expect(source.originalCalls()).toBe(1);
|
||||
expect(statSync(second.path).ino).toBe(written);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("downloads a live photo the cache holds as its image, its video and the JSON file naming them", async () => {
|
||||
const { file: live, body } = await asLivePhoto(file(1, 1));
|
||||
const lib = await open({
|
||||
client: new FilesClient([live]),
|
||||
contentSource: cdnSource(new Map([[1, body]])),
|
||||
});
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
await photo.original();
|
||||
const day = join(root, "backup", DAY);
|
||||
|
||||
const saved = await photo.download();
|
||||
|
||||
expect(saved).toEqual({
|
||||
path: join(day, "2026-03-01.1.heic"),
|
||||
videoPath: join(day, "2026-03-01.1.mov"),
|
||||
bytes: IMAGE.length,
|
||||
});
|
||||
expect(readdirSync(day).sort()).toEqual([
|
||||
"2026-03-01.1.heic",
|
||||
"2026-03-01.1.livephoto.json",
|
||||
"2026-03-01.1.mov",
|
||||
]);
|
||||
expect(
|
||||
JSON.parse(
|
||||
readFileSync(join(day, "2026-03-01.1.livephoto.json"), "utf-8"),
|
||||
),
|
||||
).toEqual({ image: "2026-03-01.1.heic", video: "2026-03-01.1.mov" });
|
||||
expect(photo.savePath).toBe(saved.path);
|
||||
expect(photo.isLocal).toBe(true);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("reads the common EXIF fields of a JPEG original", async () => {
|
||||
const lib = await open({ contentSource: stubSource(JPEG_WITH_EXIF) });
|
||||
expect(await lib.photos.byID({ fileID: 1 })!.exif()).toStrictEqual({
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
* 1. **Fetch once, then serve from disk.** The first `original`/`thumbnail`
|
||||
* fetches through the request pool and stores the bytes; the next finds the
|
||||
* file present and returns its path with a single `skipped` event and no
|
||||
* network. A file already sitting in the backup `downloadDirectory` counts
|
||||
* as present too.
|
||||
* network. A file already stored at its save path under the
|
||||
* `downloadDirectory` counts as present too.
|
||||
* 2. **Present-means-complete.** Content appears only by the streaming atomic
|
||||
* writer's rename, so a file that exists is whole. The directory listing
|
||||
* taken at `open()` is the record of what is cached, and the orphan temp
|
||||
@@ -41,6 +41,7 @@ import { join } from "node:path";
|
||||
|
||||
import {
|
||||
ContentCache,
|
||||
savePath,
|
||||
type ContentSource,
|
||||
type EnsureEvent,
|
||||
} from "../../src/library/content.js";
|
||||
@@ -152,12 +153,48 @@ const buildCache = (
|
||||
pools: args.pools ?? new RequestPools(),
|
||||
source,
|
||||
cacheDirectory: cacheDir,
|
||||
downloadDirectory: args.downloadDirectory,
|
||||
downloadDirectory: args.downloadDirectory ?? join(root, "photos"),
|
||||
getFile: (id) => byID.get(id),
|
||||
});
|
||||
return { cache, source };
|
||||
};
|
||||
|
||||
// Microseconds, as Ente stores times, for noon local time on a day, so the
|
||||
// machine's time zone cannot move the photo to another day.
|
||||
const noon = (year: number, month: number, day: number): number =>
|
||||
new Date(year, month - 1, day, 12).getTime() * 1000;
|
||||
|
||||
describe("savePath", () => {
|
||||
it("files an original by year, month and day under the root", () => {
|
||||
const f = file(12345, "IMG_0001.HEIC");
|
||||
f.metadata.creationTime = noon(2026, 3, 1);
|
||||
|
||||
expect(savePath("/photos", f)).toBe(
|
||||
"/photos/2026/2026-03/2026-03-01/2026-03-01.12345.HEIC",
|
||||
);
|
||||
});
|
||||
|
||||
it("dates an original by the date the user set, when there is one", () => {
|
||||
const f = file(7, "a.jpg");
|
||||
f.metadata.creationTime = noon(2026, 3, 1);
|
||||
f.pubMagicMetadata = { editedTime: noon(1999, 12, 31) };
|
||||
|
||||
expect(savePath("/photos", f)).toBe(
|
||||
"/photos/1999/1999-12/1999-12-31/1999-12-31.7.jpg",
|
||||
);
|
||||
});
|
||||
|
||||
it("takes the extension from the title it was uploaded with, not a new name", () => {
|
||||
const f = file(8, "upload");
|
||||
f.metadata.creationTime = noon(2026, 3, 1);
|
||||
f.pubMagicMetadata = { editedName: "renamed.png" };
|
||||
|
||||
expect(savePath("/photos", f)).toBe(
|
||||
"/photos/2026/2026-03/2026-03-01/2026-03-01.8.bin",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ContentCache.open", () => {
|
||||
it("creates the cache directories with 0700 permissions", async () => {
|
||||
const { cache } = buildCache();
|
||||
@@ -274,11 +311,17 @@ describe("ContentCache.original / thumbnail", () => {
|
||||
|
||||
it("serves a file already present in the download directory without fetching", async () => {
|
||||
const downloadDirectory = join(root, "backup");
|
||||
mkdirSync(join(downloadDirectory, "originals"), { recursive: true });
|
||||
const backupPath = join(downloadDirectory, "originals", "1.jpg");
|
||||
const day = join(downloadDirectory, "2026", "2026-03", "2026-03-01");
|
||||
mkdirSync(day, { recursive: true });
|
||||
const backupPath = join(day, "2026-03-01.1.jpg");
|
||||
writeFileSync(backupPath, "from-backup");
|
||||
const f = file(1);
|
||||
f.metadata.creationTime = noon(2026, 3, 1);
|
||||
|
||||
const { cache, source } = buildCache({ downloadDirectory });
|
||||
const { cache, source } = buildCache({
|
||||
downloadDirectory,
|
||||
files: [f],
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
const events: EnsureEvent["status"][] = [];
|
||||
@@ -649,6 +692,7 @@ describe("ContentCache live photos", () => {
|
||||
]),
|
||||
),
|
||||
cacheDirectory: cacheDir,
|
||||
downloadDirectory: join(root, "photos"),
|
||||
getFile: (id) => [a.file, b.file].find((f) => f.id === id),
|
||||
// Room for one live photo, on a disk with plenty free.
|
||||
cacheOriginalsMaxBytes: size,
|
||||
|
||||
@@ -280,6 +280,7 @@ describe("Precache eviction integration", () => {
|
||||
pools: new RequestPools(),
|
||||
source,
|
||||
cacheDirectory: cacheDir,
|
||||
downloadDirectory: join(root, "photos"),
|
||||
getFile: (id) => byID.get(id),
|
||||
statfs,
|
||||
cacheOriginalsMaxBytes: 25, // holds two 10-byte originals
|
||||
@@ -348,6 +349,7 @@ describe("Precache preemption", () => {
|
||||
pools: new RequestPools({ contentConcurrency: 1 }),
|
||||
source,
|
||||
cacheDirectory: join(root, "cache"),
|
||||
downloadDirectory: join(root, "photos"),
|
||||
getFile: (id) => byID.get(id),
|
||||
statfs: async () => ({ bsize: 1, bavail: 1_000_000_000 }),
|
||||
freeBelowBytes: 0,
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
makeAlbumsAPI,
|
||||
makePhotosAPI,
|
||||
makeTimelineAPI,
|
||||
type SavePathLookup,
|
||||
type TimelineGroup,
|
||||
} from "../../src/library/read.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
@@ -101,12 +102,19 @@ const file = (
|
||||
};
|
||||
|
||||
// Build the three API objects over one fixed projection, the way `Library`
|
||||
// wires them over its live store.
|
||||
// wires them over its live store. Save paths are covered in
|
||||
// content-library.test.ts; these tests never ask for one.
|
||||
const apis = (records: DerivedRecords) => {
|
||||
const derive = () => records;
|
||||
const saves: SavePathLookup = {
|
||||
savePath: () => {
|
||||
throw new Error("no save paths in these tests");
|
||||
},
|
||||
isLocal: () => false,
|
||||
};
|
||||
return {
|
||||
albums: makeAlbumsAPI(derive),
|
||||
photos: makePhotosAPI(derive),
|
||||
albums: makeAlbumsAPI(derive, saves),
|
||||
photos: makePhotosAPI(derive, saves),
|
||||
timeline: makeTimelineAPI(derive),
|
||||
};
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user