Save path ./photos/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.fileID.ext; download() from the cache first (closes #143) #146
+26
-6
@@ -59,6 +59,7 @@ import {
|
|||||||
savePath,
|
savePath,
|
||||||
storedAtSavePath,
|
storedAtSavePath,
|
||||||
} from "./library/content.js";
|
} from "./library/content.js";
|
||||||
|
import { representative } from "./library/records.js";
|
||||||
import type { Collection, EnteFile } from "./model/types.js";
|
import type { Collection, EnteFile } from "./model/types.js";
|
||||||
|
|
||||||
export type ProgressCallback = (message: string) => void;
|
export type ProgressCallback = (message: string) => void;
|
||||||
@@ -384,20 +385,34 @@ export const runBackup = async (
|
|||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
|
||||||
// Collections in scope, and the distinct files across them (a file shared
|
// Collections in scope, and the distinct files across them (a file shared
|
||||||
// by two albums is one original).
|
// by two albums is one original). Each file is the membership
|
||||||
|
// `representative` picks from all of its albums, in scope or not, so it is
|
||||||
|
// saved at the path `photo.savePath` names.
|
||||||
const allCollections = lib.listCollections();
|
const allCollections = lib.listCollections();
|
||||||
const collections = allCollections.filter((c) =>
|
const collections = allCollections.filter((c) =>
|
||||||
only ? only.has(c.name) : true,
|
only ? only.has(c.name) : true,
|
||||||
);
|
);
|
||||||
const collectionName = new Map<number, string>();
|
const collectionName = new Map<number, string>();
|
||||||
for (const c of collections) collectionName.set(c.id, c.name);
|
for (const c of allCollections) collectionName.set(c.id, c.name);
|
||||||
|
|
||||||
const distinct = new Map<number, EnteFile>();
|
const memberships = new Map<number, EnteFile[]>();
|
||||||
const filesByCollection = new Map<number, EnteFile[]>();
|
const filesByCollection = new Map<number, EnteFile[]>();
|
||||||
for (const c of collections) {
|
for (const c of allCollections) {
|
||||||
const files = lib.listFiles(c.id);
|
const files = lib.listFiles(c.id);
|
||||||
filesByCollection.set(c.id, files);
|
filesByCollection.set(c.id, files);
|
||||||
for (const f of files) if (!distinct.has(f.id)) distinct.set(f.id, f);
|
for (const f of files) {
|
||||||
|
const arr = memberships.get(f.id);
|
||||||
|
if (arr) arr.push(f);
|
||||||
|
else memberships.set(f.id, [f]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const distinct = new Map<number, EnteFile>();
|
||||||
|
for (const c of collections) {
|
||||||
|
for (const f of filesByCollection.get(c.id)!) {
|
||||||
|
if (!distinct.has(f.id)) {
|
||||||
|
distinct.set(f.id, representative(memberships.get(f.id)!));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const errors: BackupError[] = [];
|
const errors: BackupError[] = [];
|
||||||
@@ -523,9 +538,14 @@ export const runBackup = async (
|
|||||||
const colDir = join(collectionsDir, colDirName);
|
const colDir = join(collectionsDir, colDirName);
|
||||||
mkdirSync(colDir, { recursive: true });
|
mkdirSync(colDir, { recursive: true });
|
||||||
|
|
||||||
|
// Every album links the one original, saved from the file's entry in
|
||||||
|
// `distinct`.
|
||||||
const files = filesByCollection.get(c.id) ?? [];
|
const files = filesByCollection.get(c.id) ?? [];
|
||||||
const links = files.flatMap((f) =>
|
const links = files.flatMap((f) =>
|
||||||
linksFor(f, storedAtSavePath(downloadDirectory, f)),
|
linksFor(
|
||||||
|
f,
|
||||||
|
storedAtSavePath(downloadDirectory, distinct.get(f.id)!),
|
||||||
|
),
|
||||||
);
|
);
|
||||||
const linkNames = uniqueNames(links, true);
|
const linkNames = uniqueNames(links, true);
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -489,9 +489,10 @@ export class Library {
|
|||||||
return this.store.getFile(collectionID, fileID);
|
return this.store.getFile(collectionID, fileID);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Any membership of a file, addressed by file id alone. A file's own
|
// The membership of a file its record is read from, addressed by file id
|
||||||
// metadata (title, creationTime) is identical across the collections it
|
// alone. A file's own metadata (title, creationTime) is identical across
|
||||||
// belongs to, so this serves the point commands that hold only a fileID.
|
// the collections it belongs to, so this serves the point commands that
|
||||||
|
// hold only a fileID.
|
||||||
getFileByID(fileID: number): EnteFile | undefined {
|
getFileByID(fileID: number): EnteFile | undefined {
|
||||||
return this.store.getFileByID(fileID);
|
return this.store.getFileByID(fileID);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -114,8 +114,9 @@ export const takenAtOf = (file: EnteFile): number =>
|
|||||||
);
|
);
|
||||||
|
|
||||||
// The membership a file's record is read from: the most recently synced, lowest
|
// The membership a file's record is read from: the most recently synced, lowest
|
||||||
// collection id to break ties.
|
// collection id to break ties. Whatever dates a file's save path takes this
|
||||||
const representative = (memberships: EnteFile[]): EnteFile =>
|
// membership too, so the path always carries the record's `takenAt`.
|
||||||
|
export const representative = (memberships: EnteFile[]): EnteFile =>
|
||||||
memberships.reduce((best, m) =>
|
memberships.reduce((best, m) =>
|
||||||
m.updationTime > best.updationTime ||
|
m.updationTime > best.updationTime ||
|
||||||
(m.updationTime === best.updationTime &&
|
(m.updationTime === best.updationTime &&
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import { dirname } from "node:path";
|
|||||||
|
|
||||||
import { writeAtomic } from "../download/index.js";
|
import { writeAtomic } from "../download/index.js";
|
||||||
import type { Collection, EnteFile, Microseconds } from "../model/types.js";
|
import type { Collection, EnteFile, Microseconds } from "../model/types.js";
|
||||||
|
import { representative } from "./records.js";
|
||||||
|
|
||||||
// Bumped only when the on-disk shape changes incompatibly. A file written
|
// Bumped only when the on-disk shape changes incompatibly. A file written
|
||||||
// under a different version is discarded on load (see `load`): re-fetching
|
// under a different version is discarded on load (see `load`): re-fetching
|
||||||
@@ -179,14 +180,16 @@ export class MetadataStore {
|
|||||||
return this.files.get(fileKey(collectionID, fileID));
|
return this.files.get(fileKey(collectionID, fileID));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Any membership of a file, or undefined. Every membership re-wraps the
|
// The membership of a file its record is read from (`representative`), or
|
||||||
// same underlying content key, so any one is enough to fetch the bytes;
|
// undefined. Any membership could fetch the bytes, but the content cache
|
||||||
// the content cache resolves a fileID to a file this way.
|
// resolves a fileID to a file this way so that it dates the save path
|
||||||
|
// from the same membership as `photo.savePath`.
|
||||||
getFileByID(fileID: number): EnteFile | undefined {
|
getFileByID(fileID: number): EnteFile | undefined {
|
||||||
|
const memberships: EnteFile[] = [];
|
||||||
for (const file of this.files.values()) {
|
for (const file of this.files.values()) {
|
||||||
if (file.id === fileID) return file;
|
if (file.id === fileID) memberships.push(file);
|
||||||
}
|
}
|
||||||
return undefined;
|
return memberships.length > 0 ? representative(memberships) : undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
listFiles(collectionID: number): EnteFile[] {
|
listFiles(collectionID: number): EnteFile[] {
|
||||||
|
|||||||
@@ -608,6 +608,57 @@ describe("lib.backup", () => {
|
|||||||
await lib.close();
|
await lib.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("saves a file in two albums at its photo's save path and links it from both", async () => {
|
||||||
|
// The date edited in Ente has reached Work's copy, synced later, but
|
||||||
|
// Vacation's copy still has an earlier edit.
|
||||||
|
const older = file(100, 1, "beach.jpg");
|
||||||
|
older.pubMagicMetadata = {
|
||||||
|
editedTime: new Date(2026, 1, 1, 12).getTime() * 1000,
|
||||||
|
};
|
||||||
|
const newer = file(100, 2, "beach.jpg");
|
||||||
|
newer.updationTime = 2;
|
||||||
|
newer.pubMagicMetadata = {
|
||||||
|
editedTime: new Date(2026, 3, 15, 12).getTime() * 1000,
|
||||||
|
};
|
||||||
|
class SharedFileClient extends MockClient {
|
||||||
|
override async filesSince(args: {
|
||||||
|
collectionID: number;
|
||||||
|
}): Promise<FilesPage> {
|
||||||
|
const files = [args.collectionID === 1 ? older : newer];
|
||||||
|
return { files, deleted: [], cursor: 1 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const outDir = join(root, "backup");
|
||||||
|
const lib = await Library.open({
|
||||||
|
client: new SharedFileClient(),
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
downloadDirectory: outDir,
|
||||||
|
contentSource: stubSource(),
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
|
});
|
||||||
|
const photo = lib.photos.byID({ fileID: 100 })!;
|
||||||
|
|
||||||
|
const result = await lib.backup();
|
||||||
|
|
||||||
|
expect(result.totalFiles).toBe(1);
|
||||||
|
expect(result.downloaded).toBe(1);
|
||||||
|
expect(result.failed).toBe(0);
|
||||||
|
expect(photo.savePath).toBe(
|
||||||
|
join(outDir, "2026", "2026-04", "2026-04-15", "2026-04-15.100.jpg"),
|
||||||
|
);
|
||||||
|
expect(photo.isLocal).toBe(true);
|
||||||
|
expect(existsSync(join(outDir, "2026", "2026-02"))).toBe(false);
|
||||||
|
const target = "../../2026/2026-04/2026-04-15/2026-04-15.100.jpg";
|
||||||
|
for (const album of ["Vacation", "Work"]) {
|
||||||
|
expect(
|
||||||
|
readlinkSync(join(outDir, "collections", album, "beach.jpg")),
|
||||||
|
).toBe(target);
|
||||||
|
}
|
||||||
|
await lib.close();
|
||||||
|
});
|
||||||
|
|
||||||
it("fsyncs an original copied from the cache before the rename and its directory after", async () => {
|
it("fsyncs an original copied from the cache before the rename and its directory after", async () => {
|
||||||
const lib = await openLibrary(stubSource());
|
const lib = await openLibrary(stubSource());
|
||||||
const outDir = join(root, "backup");
|
const outDir = join(root, "backup");
|
||||||
|
|||||||
@@ -416,7 +416,7 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
|||||||
await lib.close();
|
await lib.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("dates the save path by the membership its takenAt comes from", async () => {
|
it("dates the save path and download() by the membership its takenAt comes from", async () => {
|
||||||
// One file in two albums. The date edited in Ente has reached album 2's
|
// One file in two albums. The date edited in Ente has reached album 2's
|
||||||
// copy, synced later, but album 1 still has an earlier edit.
|
// copy, synced later, but album 1 still has an earlier edit.
|
||||||
const older = file(1, 1);
|
const older = file(1, 1);
|
||||||
@@ -457,6 +457,12 @@ describe("Photo save path, local copy, content and EXIF", () => {
|
|||||||
"2026-04-15.1.jpg",
|
"2026-04-15.1.jpg",
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// download() writes to that same path, so the photo is then local.
|
||||||
|
const saved = await photo.download();
|
||||||
|
expect(saved.path).toBe(photo.savePath);
|
||||||
|
expect(photo.isLocal).toBe(true);
|
||||||
|
expect(existsSync(join(root, "backup", "2026", "2026-02"))).toBe(false);
|
||||||
await lib.close();
|
await lib.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user