Save originals at photos/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.fileID.ext; add photo.download() (closes #143)
check / check (push) Successful in 1m25s
check / check (push) Successful in 1m25s
Each original's save path is now YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID><ext> under the library's download directory, dated by takenAt in the machine's time zone. The directory defaults to photos/ in the working directory, resolved once at open. savePath is always a string, with or without a content cache, and isLocal is true only when the original is there. photo.download() puts the original at its save path: copied from the cache when the cache holds it, fetched straight there otherwise. lib.backup() uses the same code for each file, writes each file's JSON beside it, and links collections/ to the save paths. The backup has no originals/ folder. Model: opus-5-5
This commit is contained in:
@@ -142,6 +142,7 @@ const buildCache = (args: {
|
||||
pools: new RequestPools(),
|
||||
source,
|
||||
cacheDirectory: cacheDir,
|
||||
downloadDirectory: join(root, "photos"),
|
||||
getFile: (id) => byID.get(id),
|
||||
statfs: args.statfs,
|
||||
cacheOriginalsMaxBytes: args.cacheOriginalsMaxBytes,
|
||||
|
||||
@@ -6,16 +6,18 @@
|
||||
* `Photo` objects that fetch through it, `lib.thumbnails.ensure` drives it, and
|
||||
* a cached path shows up on the projected record. A library opened without a
|
||||
* content source leaves those methods throwing rather than silently doing
|
||||
* nothing. It also covers a `Photo`'s `savePath`, `isLocal`, `content()` and
|
||||
* `exif()`.
|
||||
* nothing. It also covers a `Photo`'s `savePath`, `isLocal`, `download()`,
|
||||
* `content()` and `exif()`.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||
import {
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
existsSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -29,6 +31,12 @@ import { asLivePhoto, cdnSource, IMAGE, livePhotoZip } from "../live-photo.js";
|
||||
|
||||
const USER_ID = 7;
|
||||
|
||||
// Every file is taken at noon local time on 2026-03-01, in microseconds as Ente
|
||||
// stores times, so the machine's time zone cannot move it to another day; it
|
||||
// is saved in the folder `DAY`.
|
||||
const TAKEN = new Date(2026, 2, 1, 12).getTime() * 1000;
|
||||
const DAY = join("2026", "2026-03", "2026-03-01");
|
||||
|
||||
const collection = (id: number): Collection => ({
|
||||
id,
|
||||
ownerID: USER_ID,
|
||||
@@ -47,7 +55,7 @@ const file = (id: number, collectionID: number): EnteFile => ({
|
||||
metadata: {
|
||||
title: `file-${id}.jpg`,
|
||||
fileType: "image",
|
||||
creationTime: 1,
|
||||
creationTime: TAKEN,
|
||||
modificationTime: 1,
|
||||
},
|
||||
file: { decryptionHeader: "aGVhZGVy" },
|
||||
@@ -215,6 +223,9 @@ describe("Library content wiring", () => {
|
||||
await expect(
|
||||
lib.photos.byID({ fileID: 1 })!.thumbnail(),
|
||||
).rejects.toThrow(/content cache/i);
|
||||
await expect(
|
||||
lib.photos.byID({ fileID: 1 })!.download(),
|
||||
).rejects.toThrow(/content cache/i);
|
||||
await expect(
|
||||
lib.thumbnails.ensure({ fileIDs: [1], priority: "visible" }),
|
||||
).rejects.toThrow(/content cache/i);
|
||||
@@ -332,7 +343,7 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
||||
it("names where a backup writes the original, which is local once the backup has written it", async () => {
|
||||
const lib = await open();
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
const savePath = join(root, "backup", "originals", "1.jpg");
|
||||
const savePath = join(root, "backup", DAY, "2026-03-01.1.jpg");
|
||||
expect(photo.savePath).toBe(savePath);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
|
||||
@@ -350,23 +361,30 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
||||
expect(existsSync(join(root, "cache", "originals", "1.jpg"))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(existsSync(photo.savePath!)).toBe(false);
|
||||
expect(existsSync(photo.savePath)).toBe(false);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("has no save path and is not local without a download directory", async () => {
|
||||
it("saves under photos/ in the working directory at open without a download directory", async () => {
|
||||
const cwd = vi.spyOn(process, "cwd").mockReturnValue(root);
|
||||
const lib = await open({ downloadDirectory: undefined });
|
||||
cwd.mockRestore();
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
expect(photo.savePath).toBeUndefined();
|
||||
expect(lib.downloadDirectory).toBe(join(root, "photos"));
|
||||
expect(photo.savePath).toBe(
|
||||
join(root, "photos", DAY, "2026-03-01.1.jpg"),
|
||||
);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("has no save path and is not local without a content source", async () => {
|
||||
it("has a save path, and is not local, without a content source", async () => {
|
||||
const lib = await open({ contentSource: undefined });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
expect(photo.savePath).toBeUndefined();
|
||||
expect(photo.savePath).toBe(
|
||||
join(root, "backup", DAY, "2026-03-01.1.jpg"),
|
||||
);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
@@ -378,18 +396,104 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
||||
contentSource: cdnSource(new Map([[1, body]])),
|
||||
});
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
const originals = join(root, "backup", "originals");
|
||||
const day = join(root, "backup", DAY);
|
||||
// Until then the name comes from the title, file-1.jpg; the backup
|
||||
// stores the image with the extension found inside the live photo.
|
||||
expect(photo.savePath).toBe(join(originals, "1.jpg"));
|
||||
expect(photo.savePath).toBe(join(day, "2026-03-01.1.jpg"));
|
||||
|
||||
await lib.backup();
|
||||
expect(photo.savePath).toBe(join(originals, "1.heic"));
|
||||
expect(photo.savePath).toBe(join(day, "2026-03-01.1.heic"));
|
||||
expect(photo.isLocal).toBe(true);
|
||||
expect(await photo.content()).toEqual(Buffer.from(IMAGE));
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("downloads an original the cache holds by copying it, without fetching it again", async () => {
|
||||
const source = stubSource();
|
||||
const lib = await open({ contentSource: source });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
await photo.original();
|
||||
expect(source.originalCalls()).toBe(1);
|
||||
expect(photo.isLocal).toBe(false);
|
||||
|
||||
const saved = await photo.download();
|
||||
|
||||
expect(source.originalCalls()).toBe(1);
|
||||
expect(saved).toEqual({
|
||||
path: photo.savePath,
|
||||
bytes: "orig-bytes".length,
|
||||
});
|
||||
expect(readFileSync(photo.savePath, "utf-8")).toBe("orig-bytes");
|
||||
expect(photo.isLocal).toBe(true);
|
||||
// The cache keeps its own copy.
|
||||
expect(existsSync(join(root, "cache", "originals", "1.jpg"))).toBe(
|
||||
true,
|
||||
);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("downloads an original the cache does not hold straight to its save path", async () => {
|
||||
const source = stubSource();
|
||||
const lib = await open({ contentSource: source });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
|
||||
const saved = await photo.download();
|
||||
|
||||
expect(source.originalCalls()).toBe(1);
|
||||
expect(saved.path).toBe(join(root, "backup", DAY, "2026-03-01.1.jpg"));
|
||||
expect(readFileSync(saved.path, "utf-8")).toBe("orig-bytes");
|
||||
expect(photo.isLocal).toBe(true);
|
||||
expect(readdirSync(join(root, "cache", "originals"))).toEqual([]);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("does nothing when download() finds the original already at its save path", async () => {
|
||||
const source = stubSource();
|
||||
const lib = await open({ contentSource: source });
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
const first = await photo.download();
|
||||
const written = statSync(first.path).ino;
|
||||
|
||||
const second = await photo.download();
|
||||
|
||||
expect(second).toEqual(first);
|
||||
expect(source.originalCalls()).toBe(1);
|
||||
expect(statSync(second.path).ino).toBe(written);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("downloads a live photo the cache holds as its image, its video and the JSON file naming them", async () => {
|
||||
const { file: live, body } = await asLivePhoto(file(1, 1));
|
||||
const lib = await open({
|
||||
client: new FilesClient([live]),
|
||||
contentSource: cdnSource(new Map([[1, body]])),
|
||||
});
|
||||
const photo = lib.photos.byID({ fileID: 1 })!;
|
||||
await photo.original();
|
||||
const day = join(root, "backup", DAY);
|
||||
|
||||
const saved = await photo.download();
|
||||
|
||||
expect(saved).toEqual({
|
||||
path: join(day, "2026-03-01.1.heic"),
|
||||
videoPath: join(day, "2026-03-01.1.mov"),
|
||||
bytes: IMAGE.length,
|
||||
});
|
||||
expect(readdirSync(day).sort()).toEqual([
|
||||
"2026-03-01.1.heic",
|
||||
"2026-03-01.1.livephoto.json",
|
||||
"2026-03-01.1.mov",
|
||||
]);
|
||||
expect(
|
||||
JSON.parse(
|
||||
readFileSync(join(day, "2026-03-01.1.livephoto.json"), "utf-8"),
|
||||
),
|
||||
).toEqual({ image: "2026-03-01.1.heic", video: "2026-03-01.1.mov" });
|
||||
expect(photo.savePath).toBe(saved.path);
|
||||
expect(photo.isLocal).toBe(true);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("reads the common EXIF fields of a JPEG original", async () => {
|
||||
const lib = await open({ contentSource: stubSource(JPEG_WITH_EXIF) });
|
||||
expect(await lib.photos.byID({ fileID: 1 })!.exif()).toStrictEqual({
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
* 1. **Fetch once, then serve from disk.** The first `original`/`thumbnail`
|
||||
* fetches through the request pool and stores the bytes; the next finds the
|
||||
* file present and returns its path with a single `skipped` event and no
|
||||
* network. A file already sitting in the backup `downloadDirectory` counts
|
||||
* as present too.
|
||||
* network. A file already stored at its save path under the
|
||||
* `downloadDirectory` counts as present too.
|
||||
* 2. **Present-means-complete.** Content appears only by the streaming atomic
|
||||
* writer's rename, so a file that exists is whole. The directory listing
|
||||
* taken at `open()` is the record of what is cached, and the orphan temp
|
||||
@@ -41,6 +41,7 @@ import { join } from "node:path";
|
||||
|
||||
import {
|
||||
ContentCache,
|
||||
savePath,
|
||||
type ContentSource,
|
||||
type EnsureEvent,
|
||||
} from "../../src/library/content.js";
|
||||
@@ -152,12 +153,48 @@ const buildCache = (
|
||||
pools: args.pools ?? new RequestPools(),
|
||||
source,
|
||||
cacheDirectory: cacheDir,
|
||||
downloadDirectory: args.downloadDirectory,
|
||||
downloadDirectory: args.downloadDirectory ?? join(root, "photos"),
|
||||
getFile: (id) => byID.get(id),
|
||||
});
|
||||
return { cache, source };
|
||||
};
|
||||
|
||||
// Microseconds, as Ente stores times, for noon local time on a day, so the
|
||||
// machine's time zone cannot move the photo to another day.
|
||||
const noon = (year: number, month: number, day: number): number =>
|
||||
new Date(year, month - 1, day, 12).getTime() * 1000;
|
||||
|
||||
describe("savePath", () => {
|
||||
it("files an original by year, month and day under the root", () => {
|
||||
const f = file(12345, "IMG_0001.HEIC");
|
||||
f.metadata.creationTime = noon(2026, 3, 1);
|
||||
|
||||
expect(savePath("/photos", f)).toBe(
|
||||
"/photos/2026/2026-03/2026-03-01/2026-03-01.12345.HEIC",
|
||||
);
|
||||
});
|
||||
|
||||
it("dates an original by the date the user set, when there is one", () => {
|
||||
const f = file(7, "a.jpg");
|
||||
f.metadata.creationTime = noon(2026, 3, 1);
|
||||
f.pubMagicMetadata = { editedTime: noon(1999, 12, 31) };
|
||||
|
||||
expect(savePath("/photos", f)).toBe(
|
||||
"/photos/1999/1999-12/1999-12-31/1999-12-31.7.jpg",
|
||||
);
|
||||
});
|
||||
|
||||
it("takes the extension from the title it was uploaded with, not a new name", () => {
|
||||
const f = file(8, "upload");
|
||||
f.metadata.creationTime = noon(2026, 3, 1);
|
||||
f.pubMagicMetadata = { editedName: "renamed.png" };
|
||||
|
||||
expect(savePath("/photos", f)).toBe(
|
||||
"/photos/2026/2026-03/2026-03-01/2026-03-01.8.bin",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ContentCache.open", () => {
|
||||
it("creates the cache directories with 0700 permissions", async () => {
|
||||
const { cache } = buildCache();
|
||||
@@ -274,11 +311,17 @@ describe("ContentCache.original / thumbnail", () => {
|
||||
|
||||
it("serves a file already present in the download directory without fetching", async () => {
|
||||
const downloadDirectory = join(root, "backup");
|
||||
mkdirSync(join(downloadDirectory, "originals"), { recursive: true });
|
||||
const backupPath = join(downloadDirectory, "originals", "1.jpg");
|
||||
const day = join(downloadDirectory, "2026", "2026-03", "2026-03-01");
|
||||
mkdirSync(day, { recursive: true });
|
||||
const backupPath = join(day, "2026-03-01.1.jpg");
|
||||
writeFileSync(backupPath, "from-backup");
|
||||
const f = file(1);
|
||||
f.metadata.creationTime = noon(2026, 3, 1);
|
||||
|
||||
const { cache, source } = buildCache({ downloadDirectory });
|
||||
const { cache, source } = buildCache({
|
||||
downloadDirectory,
|
||||
files: [f],
|
||||
});
|
||||
await cache.open();
|
||||
|
||||
const events: EnsureEvent["status"][] = [];
|
||||
@@ -649,6 +692,7 @@ describe("ContentCache live photos", () => {
|
||||
]),
|
||||
),
|
||||
cacheDirectory: cacheDir,
|
||||
downloadDirectory: join(root, "photos"),
|
||||
getFile: (id) => [a.file, b.file].find((f) => f.id === id),
|
||||
// Room for one live photo, on a disk with plenty free.
|
||||
cacheOriginalsMaxBytes: size,
|
||||
|
||||
@@ -280,6 +280,7 @@ describe("Precache eviction integration", () => {
|
||||
pools: new RequestPools(),
|
||||
source,
|
||||
cacheDirectory: cacheDir,
|
||||
downloadDirectory: join(root, "photos"),
|
||||
getFile: (id) => byID.get(id),
|
||||
statfs,
|
||||
cacheOriginalsMaxBytes: 25, // holds two 10-byte originals
|
||||
@@ -348,6 +349,7 @@ describe("Precache preemption", () => {
|
||||
pools: new RequestPools({ contentConcurrency: 1 }),
|
||||
source,
|
||||
cacheDirectory: join(root, "cache"),
|
||||
downloadDirectory: join(root, "photos"),
|
||||
getFile: (id) => byID.get(id),
|
||||
statfs: async () => ({ bsize: 1, bavail: 1_000_000_000 }),
|
||||
freeBelowBytes: 0,
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
makeAlbumsAPI,
|
||||
makePhotosAPI,
|
||||
makeTimelineAPI,
|
||||
type SavePathLookup,
|
||||
type TimelineGroup,
|
||||
} from "../../src/library/read.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
@@ -101,12 +102,19 @@ const file = (
|
||||
};
|
||||
|
||||
// Build the three API objects over one fixed projection, the way `Library`
|
||||
// wires them over its live store.
|
||||
// wires them over its live store. Save paths are covered in
|
||||
// content-library.test.ts; these tests never ask for one.
|
||||
const apis = (records: DerivedRecords) => {
|
||||
const derive = () => records;
|
||||
const saves: SavePathLookup = {
|
||||
savePath: () => {
|
||||
throw new Error("no save paths in these tests");
|
||||
},
|
||||
isLocal: () => false,
|
||||
};
|
||||
return {
|
||||
albums: makeAlbumsAPI(derive),
|
||||
photos: makePhotosAPI(derive),
|
||||
albums: makeAlbumsAPI(derive, saves),
|
||||
photos: makePhotosAPI(derive, saves),
|
||||
timeline: makeTimelineAPI(derive),
|
||||
};
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user