Save path ./photos/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.fileID.ext; download() from the cache first (closes #143) #146
+26
-6
@@ -59,6 +59,7 @@ import {
|
||||
savePath,
|
||||
storedAtSavePath,
|
||||
} from "./library/content.js";
|
||||
import { representative } from "./library/records.js";
|
||||
import type { Collection, EnteFile } from "./model/types.js";
|
||||
|
||||
export type ProgressCallback = (message: string) => void;
|
||||
@@ -384,20 +385,34 @@ export const runBackup = async (
|
||||
const now = Date.now();
|
||||
|
||||
// Collections in scope, and the distinct files across them (a file shared
|
||||
// by two albums is one original).
|
||||
// by two albums is one original). Each file is the membership
|
||||
// `representative` picks from all of its albums, in scope or not, so it is
|
||||
// saved at the path `photo.savePath` names.
|
||||
const allCollections = lib.listCollections();
|
||||
const collections = allCollections.filter((c) =>
|
||||
only ? only.has(c.name) : true,
|
||||
);
|
||||
const collectionName = new Map<number, string>();
|
||||
for (const c of collections) collectionName.set(c.id, c.name);
|
||||
for (const c of allCollections) collectionName.set(c.id, c.name);
|
||||
|
||||
const distinct = new Map<number, EnteFile>();
|
||||
const memberships = new Map<number, EnteFile[]>();
|
||||
const filesByCollection = new Map<number, EnteFile[]>();
|
||||
for (const c of collections) {
|
||||
for (const c of allCollections) {
|
||||
const files = lib.listFiles(c.id);
|
||||
filesByCollection.set(c.id, files);
|
||||
for (const f of files) if (!distinct.has(f.id)) distinct.set(f.id, f);
|
||||
for (const f of files) {
|
||||
const arr = memberships.get(f.id);
|
||||
if (arr) arr.push(f);
|
||||
else memberships.set(f.id, [f]);
|
||||
}
|
||||
}
|
||||
const distinct = new Map<number, EnteFile>();
|
||||
for (const c of collections) {
|
||||
for (const f of filesByCollection.get(c.id)!) {
|
||||
if (!distinct.has(f.id)) {
|
||||
distinct.set(f.id, representative(memberships.get(f.id)!));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const errors: BackupError[] = [];
|
||||
@@ -523,9 +538,14 @@ export const runBackup = async (
|
||||
const colDir = join(collectionsDir, colDirName);
|
||||
mkdirSync(colDir, { recursive: true });
|
||||
|
||||
// Every album links the one original, saved from the file's entry in
|
||||
// `distinct`.
|
||||
const files = filesByCollection.get(c.id) ?? [];
|
||||
const links = files.flatMap((f) =>
|
||||
linksFor(f, storedAtSavePath(downloadDirectory, f)),
|
||||
linksFor(
|
||||
f,
|
||||
storedAtSavePath(downloadDirectory, distinct.get(f.id)!),
|
||||
),
|
||||
);
|
||||
const linkNames = uniqueNames(links, true);
|
||||
try {
|
||||
|
||||
@@ -489,9 +489,10 @@ export class Library {
|
||||
return this.store.getFile(collectionID, fileID);
|
||||
}
|
||||
|
||||
// Any membership of a file, addressed by file id alone. A file's own
|
||||
// metadata (title, creationTime) is identical across the collections it
|
||||
// belongs to, so this serves the point commands that hold only a fileID.
|
||||
// The membership of a file its record is read from, addressed by file id
|
||||
// alone. A file's own metadata (title, creationTime) is identical across
|
||||
// the collections it belongs to, so this serves the point commands that
|
||||
// hold only a fileID.
|
||||
getFileByID(fileID: number): EnteFile | undefined {
|
||||
return this.store.getFileByID(fileID);
|
||||
}
|
||||
|
||||
@@ -114,8 +114,9 @@ export const takenAtOf = (file: EnteFile): number =>
|
||||
);
|
||||
|
||||
// The membership a file's record is read from: the most recently synced, lowest
|
||||
// collection id to break ties.
|
||||
const representative = (memberships: EnteFile[]): EnteFile =>
|
||||
// collection id to break ties. Whatever dates a file's save path takes this
|
||||
// membership too, so the path always carries the record's `takenAt`.
|
||||
export const representative = (memberships: EnteFile[]): EnteFile =>
|
||||
memberships.reduce((best, m) =>
|
||||
m.updationTime > best.updationTime ||
|
||||
(m.updationTime === best.updationTime &&
|
||||
|
||||
@@ -16,6 +16,7 @@ import { dirname } from "node:path";
|
||||
|
||||
import { writeAtomic } from "../download/index.js";
|
||||
import type { Collection, EnteFile, Microseconds } from "../model/types.js";
|
||||
import { representative } from "./records.js";
|
||||
|
||||
// Bumped only when the on-disk shape changes incompatibly. A file written
|
||||
// under a different version is discarded on load (see `load`): re-fetching
|
||||
@@ -179,14 +180,16 @@ export class MetadataStore {
|
||||
return this.files.get(fileKey(collectionID, fileID));
|
||||
}
|
||||
|
||||
// Any membership of a file, or undefined. Every membership re-wraps the
|
||||
// same underlying content key, so any one is enough to fetch the bytes;
|
||||
// the content cache resolves a fileID to a file this way.
|
||||
// The membership of a file its record is read from (`representative`), or
|
||||
// undefined. Any membership could fetch the bytes, but the content cache
|
||||
// resolves a fileID to a file this way so that it dates the save path
|
||||
// from the same membership as `photo.savePath`.
|
||||
getFileByID(fileID: number): EnteFile | undefined {
|
||||
const memberships: EnteFile[] = [];
|
||||
for (const file of this.files.values()) {
|
||||
if (file.id === fileID) return file;
|
||||
if (file.id === fileID) memberships.push(file);
|
||||
}
|
||||
return undefined;
|
||||
return memberships.length > 0 ? representative(memberships) : undefined;
|
||||
}
|
||||
|
||||
listFiles(collectionID: number): EnteFile[] {
|
||||
|
||||
@@ -608,6 +608,57 @@ describe("lib.backup", () => {
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("saves a file in two albums at its photo's save path and links it from both", async () => {
|
||||
// The date edited in Ente has reached Work's copy, synced later, but
|
||||
// Vacation's copy still has an earlier edit.
|
||||
const older = file(100, 1, "beach.jpg");
|
||||
older.pubMagicMetadata = {
|
||||
editedTime: new Date(2026, 1, 1, 12).getTime() * 1000,
|
||||
};
|
||||
const newer = file(100, 2, "beach.jpg");
|
||||
newer.updationTime = 2;
|
||||
newer.pubMagicMetadata = {
|
||||
editedTime: new Date(2026, 3, 15, 12).getTime() * 1000,
|
||||
};
|
||||
class SharedFileClient extends MockClient {
|
||||
override async filesSince(args: {
|
||||
collectionID: number;
|
||||
}): Promise<FilesPage> {
|
||||
const files = [args.collectionID === 1 ? older : newer];
|
||||
return { files, deleted: [], cursor: 1 };
|
||||
}
|
||||
}
|
||||
const outDir = join(root, "backup");
|
||||
const lib = await Library.open({
|
||||
client: new SharedFileClient(),
|
||||
cacheDirectory: join(root, "cache"),
|
||||
downloadDirectory: outDir,
|
||||
contentSource: stubSource(),
|
||||
refreshIntervalSeconds: 3600,
|
||||
precacheThumbnails: false,
|
||||
precacheOriginals: false,
|
||||
});
|
||||
const photo = lib.photos.byID({ fileID: 100 })!;
|
||||
|
||||
const result = await lib.backup();
|
||||
|
||||
expect(result.totalFiles).toBe(1);
|
||||
expect(result.downloaded).toBe(1);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(photo.savePath).toBe(
|
||||
join(outDir, "2026", "2026-04", "2026-04-15", "2026-04-15.100.jpg"),
|
||||
);
|
||||
expect(photo.isLocal).toBe(true);
|
||||
expect(existsSync(join(outDir, "2026", "2026-02"))).toBe(false);
|
||||
const target = "../../2026/2026-04/2026-04-15/2026-04-15.100.jpg";
|
||||
for (const album of ["Vacation", "Work"]) {
|
||||
expect(
|
||||
readlinkSync(join(outDir, "collections", album, "beach.jpg")),
|
||||
).toBe(target);
|
||||
}
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("fsyncs an original copied from the cache before the rename and its directory after", async () => {
|
||||
const lib = await openLibrary(stubSource());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
@@ -416,7 +416,7 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("dates the save path by the membership its takenAt comes from", async () => {
|
||||
it("dates the save path and download() by the membership its takenAt comes from", async () => {
|
||||
// One file in two albums. The date edited in Ente has reached album 2's
|
||||
// copy, synced later, but album 1 still has an earlier edit.
|
||||
const older = file(1, 1);
|
||||
@@ -457,6 +457,12 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
||||
"2026-04-15.1.jpg",
|
||||
),
|
||||
);
|
||||
|
||||
// download() writes to that same path, so the photo is then local.
|
||||
const saved = await photo.download();
|
||||
expect(saved.path).toBe(photo.savePath);
|
||||
expect(photo.isLocal).toBe(true);
|
||||
expect(existsSync(join(root, "backup", "2026", "2026-02"))).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user