sneak/quak
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

Save path ./photos/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.fileID.ext; download() from the cache first (closes #143)
check / check (push) Successful in 1m25s
Details

Browse Source 
Originals are saved at `{downloadDirectory}/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.{fileID}{ext}`. The date is the photo's `takenAt` in local time, and `downloadDirectory` defaults to `./photos`, resolved when the library opens. The old `originals/` layout is gone.

`photo.download()` writes the original to `savePath`. It copies from the cache when the cache holds the original, and fetches otherwise. `lib.backup()` uses the same path and rule, and every album in `collections/` links to it. `isLocal` is true only when the original is at `savePath`.

For a file in several albums, one rule picks the copy everything uses: the most recently synced, with the lowest album ID breaking a tie.

Model: opus-5-5
This commit was merged in pull request #146.
This commit is contained in:
clawbot
2026-10-01 23:20:58 +02:00
parent 67d554fb46
commit 10e1a9ef39
16 changed files with 975 additions and 499 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/library/content.ts
+149 -69
View File
@@ -23,6 +23,11 @@
// original with no recorded hash is stored unchecked, as the upstream client
// does; thumbnails have none. On top of that this module refuses to record a
// stored file that came out empty.
//
// It also names each original's save path under the download directory
// (`savePath`), where `Photo.download()` and `lib.backup()` put it. A copy in
// the cache does not count as saved there, but is copied there rather than
// fetched again.
import {
closeSync,
@@ -34,8 +39,10 @@ import {
} from "node:fs";
import {
chmod,
copyFile,
mkdir,
readdir,
rename,
rm,
stat,
statfs,
@@ -47,13 +54,15 @@ import type { ApiClient } from "../api/client.js";
import {
downloadFile,
downloadThumbnail,
fsyncPath,
type ProgressCallback,
removeLeftoverTempFiles,
writeAtomic,
} from "../download/index.js";
import { safeExtension } from "../filename.js";
import { safeExtension, withExtension } from "../filename.js";
import type { EnteFile } from "../model/types.js";
import type { Priority, RequestPools } from "./pools.js";
import { takenAtOf } from "./records.js";
const DIR_MODE = 0o700;
const FILE_MODE = 0o600;
@@ -108,11 +117,9 @@ export interface ContentOptions {
export interface PhotoContent {
original(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
thumbnail(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
// Where a backup stores the original, whether or not it is there yet. For
// a live photo not yet stored, it carries the title's extension, and the
// backup may store the image under a different one.
savePath(fileID: number): string | undefined;
isLocal(fileID: number): boolean;
// Put the original at the save path of `file`, the copy the `Photo` holds,
// and return it there.
download(file: EnteFile): Promise<ContentResult>;
}
export interface EnsureResult {
@@ -199,10 +206,10 @@ export interface ContentCacheOptions {
pools: RequestPools;
source: ContentSource;
cacheDirectory: string;
// The backup destination (issue-level `downloadDirectory`). An original
// already stored there by a backup counts as present, so the cache serves
// it rather than fetching a second copy.
downloadDirectory?: string;
// The root of the save paths. An original already stored at its save path
// counts as present, so the cache serves it rather than fetching a second
// copy.
downloadDirectory: string;
// Resolve any membership of a file; every membership shares the underlying
// content key, so any one decrypts the same bytes.
getFile: (fileID: number) => EnteFile | undefined;
@@ -229,11 +236,27 @@ class AbortDrop extends Error {
}
}
// The name of a file's original in originals/: `<fileID><ext>`, the extension
// taken from the title (or `.bin`). A backup names its originals the same way.
// The name of a file's original in the cache's originals/: `<fileID><ext>`, the
// extension taken from the title (or `.bin`).
export const nameInOriginals = (file: EnteFile): string =>
`${file.id}${safeExtension(file.metadata.title)}`;
const pad = (n: number): string => String(n).padStart(2, "0");
// Where the original of `file` is saved under `root`:
// `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID><ext>`, dated by the photo's
// `takenAt` in this machine's time zone, with the extension taken from the
// title (or `.bin`). A live photo is stored as its image and its video beside
// this path, each with the extension found inside the live photo.
export const savePath = (root: string, file: EnteFile): string => {
const taken = new Date(takenAtOf(file));
const year = String(taken.getFullYear());
const month = `${year}-${pad(taken.getMonth() + 1)}`;
const day = `${month}-${pad(taken.getDate())}`;
const ext = safeExtension(file.metadata.title);
return join(root, year, month, day, `${day}.${file.id}${ext}`);
};
// The fileID a cache filename encodes, or undefined when the name is not one
// the cache writes (`<digits><ext>`).
const fileIDFromName = (name: string): number | undefined => {
@@ -279,28 +302,29 @@ const isZip = (path: string): boolean => {
// A live photo's image and video are named with the extensions from inside its
// ZIP, so their names alone do not say which is which. Wherever the cache or a
// backup stores one, a JSON file of this name beside them names both.
const livePhotoJSONName = (fileID: number): string =>
`${fileID}.livephoto.json`;
// save path stores one, a JSON file of this name beside them names both.
// `name` is the original's name without its extension: `<fileID>` in the
// cache, `YYYY-MM-DD.<fileID>` at the save path.
const livePhotoJSONName = (name: string): string => `${name}.livephoto.json`;
// The image and video that the live photo's JSON file in `dir` names, or
// undefined when there is none. Only names of the form the cache writes are
// taken, so the file cannot point outside `dir`.
// undefined when there is none. Only `name` with an extension is taken, so the
// file cannot point outside `dir`.
const readLivePhotoJSON = (
dir: string,
fileID: number,
name: string,
): { path: string; videoPath: string } | undefined => {
const valid = (name: unknown): name is string =>
typeof name === "string" && name === `${fileID}${safeExtension(name)}`;
const valid = (part: unknown): part is string =>
typeof part === "string" && part === `${name}${safeExtension(part)}`;
try {
const { image, video } = JSON.parse(
readFileSync(join(dir, livePhotoJSONName(fileID)), "utf-8"),
readFileSync(join(dir, livePhotoJSONName(name)), "utf-8"),
);
if (valid(image) && valid(video)) {
return { path: join(dir, image), videoPath: join(dir, video) };
}
} catch {
// No such file, or not one the cache wrote.
// No such file, or not one quak wrote.
}
return undefined;
};
@@ -308,11 +332,11 @@ const readLivePhotoJSON = (
// Write the JSON file naming a live photo's image and video, both in `dir`.
export const writeLivePhotoJSON = (
dir: string,
fileID: number,
name: string,
stored: { path: string; videoPath: string },
): Promise<void> =>
writeAtomic(
join(dir, livePhotoJSONName(fileID)),
join(dir, livePhotoJSONName(name)),
new TextEncoder().encode(
JSON.stringify({
image: basename(stored.path),
@@ -321,17 +345,19 @@ export const writeLivePhotoJSON = (
),
);
// The original of `file` as the cache or a backup stored it in `dir`, when all
// of it is there: `<fileID><ext>`, or a live photo's image and video.
// The original of `file` as stored in `dir` under `name` (without its
// extension), when all of it is there: `<name><ext>`, or a live photo's image
// and video.
export const storedOriginal = (
dir: string,
name: string,
file: EnteFile,
): { path: string; videoPath?: string } | undefined => {
if (file.metadata.fileType !== "livePhoto") {
const path = join(dir, nameInOriginals(file));
const path = join(dir, `${name}${safeExtension(file.metadata.title)}`);
return hasContent(path) ? { path } : undefined;
}
const stored = readLivePhotoJSON(dir, file.id);
const stored = readLivePhotoJSON(dir, name);
return stored !== undefined &&
hasContent(stored.path) &&
hasContent(stored.videoPath)
@@ -339,10 +365,76 @@ export const storedOriginal = (
: undefined;
};
// The original of `file` as stored at its save path under `root`, when all of
// it is there.
export const storedAtSavePath = (
root: string,
file: EnteFile,
): { path: string; videoPath?: string } | undefined => {
const path = savePath(root, file);
return storedOriginal(dirname(path), basename(path, extname(path)), file);
};
// Copy bytes into `dest` via a temp file in the same directory plus rename, so
// `dest` appears only once it is whole ("present means complete"). As in the
// download writer, the temp file is fsynced before the rename and the directory
// after it, so a power cut cannot leave a correctly named but short original.
// The temp name carries this process's ID so a later run can tell a leftover
// from a copy still in progress (see `removeLeftoverTempFiles`).
export const copyAtomic = async (src: string, dest: string): Promise<void> => {
if (src === dest) return;
const tmp = join(
dirname(dest),
`.quak-backup-${basename(dest)}-${process.pid}-${Math.random()
.toString(36)
.slice(2)}.tmp`,
);
try {
await copyFile(src, tmp);
await fsyncPath(tmp);
// `rename` replaces the destination's directory entry: an existing
// symlink at `dest` is replaced, not followed, and the new file has
// the temp file's permissions (copied from `src`).
await rename(tmp, dest);
await fsyncPath(dirname(dest));
} finally {
await rm(tmp, { force: true });
}
};
// Put the original of `file` at its save path under `root`, creating its
// folders. `get` is given the save path and returns where the original is: a
// fetch writes it there, and a copy the cache holds is copied there. A live
// photo's image and video go beside the save path, each with its own
// extension: when they came from the cache they are copied. Then the JSON file
// naming them is written, which is what makes the live photo count as stored.
export const placeOriginal = async (
root: string,
file: EnteFile,
get: (dest: string) => Promise<{ path: string; videoPath?: string }>,
): Promise<{ path: string; videoPath?: string }> => {
const dest = savePath(root, file);
await mkdir(dirname(dest), { recursive: true });
const got = await get(dest);
if (got.videoPath === undefined) {
await copyAtomic(got.path, dest);
return { path: dest };
}
const path = withExtension(dest, extname(got.path));
const videoPath = withExtension(dest, extname(got.videoPath));
if (got.path !== path) {
await copyAtomic(got.path, path);
await copyAtomic(got.videoPath, videoPath);
}
const name = basename(dest, extname(dest));
await writeLivePhotoJSON(dirname(dest), name, { path, videoPath });
return { path, videoPath };
};
export class ContentCache implements PhotoContent, ThumbnailsAPI {
private readonly pools: RequestPools;
private readonly source: ContentSource;
private readonly downloadDirectory?: string;
private readonly downloadDirectory: string;
private readonly getFile: (fileID: number) => EnteFile | undefined;
private readonly originalsDir: string;
private readonly thumbnailsDir: string;
@@ -440,32 +532,23 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
return this.get(fileID, "thumbnail", "on-demand", opts?.onProgress);
}
// Where a backup to the download directory stores the file's original,
// whether or not it is there yet: for a live photo already stored, its
// image. For a live photo not yet stored, it carries the title's
// extension, and the backup may store the image under a different one.
// Undefined with no download directory.
savePath(fileID: number): string | undefined {
const file = this.getFile(fileID);
if (this.downloadDirectory === undefined || file === undefined)
return undefined;
const dir = join(this.downloadDirectory, "originals");
return (
storedOriginal(dir, file)?.path ?? join(dir, nameInOriginals(file))
);
// Put the original at the save path of `file` under the download directory
// and return it there. `file` is the copy the `Photo` holds, so the path is
// the one its `savePath` names, even after a refresh changed the date. One
// already stored there is returned as it is; one the cache holds is copied
// from it; any other is fetched straight to the save path, with no copy
// left in the cache.
async download(file: EnteFile): Promise<ContentResult> {
const root = this.downloadDirectory;
const saved =
storedAtSavePath(root, file) ??
(await placeOriginal(root, file, (dest) =>
this.backupOriginal(file.id, dest),
));
return { ...saved, bytes: fileSize(saved.path) ?? 0 };
}
// Whether the whole original is in the download directory, as a backup
// stores it. A copy only in the cache does not count.
isLocal(fileID: number): boolean {
const file = this.getFile(fileID);
if (this.downloadDirectory === undefined || file === undefined)
return false;
const dir = join(this.downloadDirectory, "originals");
return storedOriginal(dir, file) !== undefined;
}
// Get an original for a backup. One not present anywhere is written
// Get an original for a save path. One not present anywhere is written
// straight to `destination` and recorded there, so no second copy lands
// in the cache; one already present is returned where it is.
async backupOriginal(
@@ -635,12 +718,9 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
known.delete(fileID);
}
// An original a backup already stored counts as present.
if (kind === "original" && this.downloadDirectory !== undefined) {
const stored = storedOriginal(
join(this.downloadDirectory, "originals"),
file,
);
// An original already stored at its save path counts as present.
if (kind === "original") {
const stored = storedAtSavePath(this.downloadDirectory, file);
if (stored !== undefined) {
this.originals.set(fileID, stored);
return {
@@ -676,7 +756,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
? this.beginOriginalWrite(fileID)
: null;
try {
const stored = await this.download(
const stored = await this.fetchInto(
file,
dest,
kind,
@@ -691,12 +771,12 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
);
}
}
// A backup records its own live photos.
// `placeOriginal` records a live photo it saves.
if (
stored.videoPath !== undefined &&
opts?.destination === undefined
) {
await writeLivePhotoJSON(dir, fileID, {
await writeLivePhotoJSON(dir, String(fileID), {
path: stored.path,
videoPath: stored.videoPath,
});
@@ -719,7 +799,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
// Fetch into `destination`, returning where the bytes landed: there, or
// for a live photo, its image and video beside it.
private async download(
private async fetchInto(
file: EnteFile,
destination: string,
kind: Kind,
@@ -744,10 +824,10 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
await utimes(path, now, now).catch(() => undefined);
}
// Every stored original that lives under `originalsDir` (a backup-directory
// hit recorded in the map is excluded), with its size and mtime; a live
// Every stored original that lives under `originalsDir` (a save-path hit
// recorded in the map is excluded), with its size and mtime; a live
// photo's size includes its video. Entries whose file has vanished are
// dropped from the map. Backups and thumbnails are never counted.
// dropped from the map. Save paths and thumbnails are never counted.
private async measureOriginals(): Promise<{
entries: {
fileID: number;
@@ -853,7 +933,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
await rm(
join(
this.originalsDir,
livePhotoJSONName(e.fileID),
livePhotoJSONName(String(e.fileID)),
),
{ force: true },
);
@@ -907,8 +987,8 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
// earlier version stored under the image's name, and is removed.
// Any other is left alone: another process may have just stored
// it and not yet written the JSON file.
const livePhoto = names.has(livePhotoJSONName(id))
? readLivePhotoJSON(dir, id)
const livePhoto = names.has(livePhotoJSONName(String(id)))
? readLivePhotoJSON(dir, String(id))
: undefined;
if (livePhoto !== undefined) {
into.set(id, livePhoto);
src/library/index.ts
+41 -28
View File
@@ -27,7 +27,7 @@
// never masked by a subsequent empty refresh.
import { rm } from "node:fs/promises";
import { join } from "node:path";
import { join, resolve } from "node:path";
import envPaths from "env-paths";
import { MetadataStore } from "./store.js";
@@ -49,9 +49,12 @@ import {
type PhotosAPI,
type TimelineAPI,
type FreshReads,
type SavePathLookup,
} from "./read.js";
import {
ContentCache,
savePath,
storedAtSavePath,
type ContentSource,
type ThumbnailsAPI,
type EnsureOptions,
@@ -70,6 +73,7 @@ export {
type PhotoFilter,
type TimelineGroup,
type GroupBy,
type SavePathLookup,
} from "./read.js";
export {
type ContentSource,
@@ -169,8 +173,10 @@ export interface LibraryOptions {
// Where `metadata.json` lives. Defaults to the env-paths cache directory
// plus the user id, so each account has its own cache.
cacheDirectory?: string;
// Persistent backup destination. The refresh loop does not use it; the
// content cache treats an original already stored there as present.
// The root of every photo's save path, where `Photo.download()` and
// `lib.backup()` put originals. Defaults to `photos` in the working
// directory at open. The content cache treats an original already stored
// at its save path as present.
downloadDirectory?: string;
refreshIntervalSeconds?: number;
onProgress?: RefreshProgressCallback;
@@ -234,7 +240,7 @@ export interface LibraryStatus {
export class Library {
readonly cacheDirectory: string;
readonly downloadDirectory?: string;
readonly downloadDirectory: string;
// The in-process read surface (issue #44). Each namespace answers
// synchronously from the live record projection; no read touches the
@@ -296,7 +302,7 @@ export class Library {
store: MetadataStore;
userID: number;
cacheDirectory: string;
downloadDirectory?: string;
downloadDirectory: string;
intervalMs: number;
onProgress?: RefreshProgressCallback;
pools: RequestPools;
@@ -320,8 +326,14 @@ export class Library {
// The read namespaces derive fresh from the store on each call, so they
// always reflect the latest refresh.
const derive = (): DerivedRecords => this.deriveNow();
this.albums = makeAlbumsAPI(derive, this.cache);
this.photos = makePhotosAPI(derive, this.cache);
const root = this.downloadDirectory;
const saves: SavePathLookup = {
savePath: (file) =>
storedAtSavePath(root, file)?.path ?? savePath(root, file),
isLocal: (file) => storedAtSavePath(root, file) !== undefined,
};
this.albums = makeAlbumsAPI(derive, saves, this.cache);
this.photos = makePhotosAPI(derive, saves, this.cache);
this.timeline = makeTimelineAPI(derive);
this.thumbnails = {
ensure: (opts: EnsureOptions): Promise<EnsureResult[]> => {
@@ -350,6 +362,13 @@ export class Library {
const { userID } = opts.client.whoami();
const cacheDirectory =
opts.cacheDirectory ?? defaultCacheDirectory(userID);
if (opts.downloadDirectory === "") {
throw new Error(
"library: downloadDirectory is empty (leave it out to save " +
"under photos/ in the working directory)",
);
}
const downloadDirectory = opts.downloadDirectory ?? resolve("photos");
const metadataPath = join(cacheDirectory, "metadata.json");
let store = await MetadataStore.load(metadataPath);
// A cache directory given explicitly can hold another account's cache.
@@ -404,7 +423,7 @@ export class Library {
pools,
source,
cacheDirectory,
downloadDirectory: opts.downloadDirectory,
downloadDirectory,
getFile: (fileID) => store.getFileByID(fileID),
cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes,
freeBelowBytes: opts.freeBelowBytes,
@@ -426,7 +445,7 @@ export class Library {
store,
userID,
cacheDirectory,
downloadDirectory: opts.downloadDirectory,
downloadDirectory,
intervalMs,
onProgress: opts.onProgress,
pools,
@@ -470,9 +489,10 @@ export class Library {
return this.store.getFile(collectionID, fileID);
}
// Any membership of a file, addressed by file id alone. A file's own
// metadata (title, creationTime) is identical across the collections it
// belongs to, so this serves the point commands that hold only a fileID.
// The membership of a file its record is read from, addressed by file id
// alone. A file's own metadata (title, creationTime) is identical across
// the collections it belongs to, so this serves the point commands that
// hold only a fileID.
getFileByID(fileID: number): EnteFile | undefined {
return this.store.getFileByID(fileID);
}
@@ -543,27 +563,20 @@ export class Library {
};
}
// Back up every in-scope file to `downloadDirectory` in the historical
// on-disk layout, with a durable failure ledger (issue #51). Waits for a
// completed refresh first, as `fresh()` does, joining one already running,
// and rejects before touching any file when it fails. Then fetches pending
// originals (and optional thumbnails) through the content cache and pools,
// and rebuilds the derived symlink/JSON views from the model. Throws before
// any network work when no download directory is available or no content
// cache backs the originals it must fetch.
// Back up every in-scope file to `opts.downloadDirectory`, or else the
// library's, each original at its save path, with a durable failure
// ledger (issue #51). Waits for a completed refresh first, as `fresh()`
// does, joining one already running, and rejects before touching any file
// when it fails. Then puts pending originals at their save paths as
// `Photo.download()` does (and optional thumbnails) through the content
// cache and pools, and rebuilds the derived symlink/JSON views from the
// model. Throws before any network work when no content cache backs the
// originals it must fetch.
backup(opts?: BackupOptions): Promise<BackupResult> {
const downloadDirectory =
opts?.downloadDirectory ?? this.downloadDirectory;
const includeOriginals = opts?.includeOriginals ?? true;
const includeThumbnails = opts?.includeThumbnails ?? false;
if (!downloadDirectory) {
return Promise.reject(
new Error(
"backup requires a downloadDirectory (pass one to " +
"backup() or open the library with one)",
),
);
}
if ((includeOriginals || includeThumbnails) && !this.cache) {
return Promise.reject(
new Error(
src/library/read.ts
+51 -23
View File
@@ -12,18 +12,26 @@
// plain records are the serializable surface, and `record()` returns one.
//
// A `Photo` also fetches its own bytes: `original()`, `thumbnail()`,
// `content()` and `exif()` go through the on-disk content cache (issue #46),
// and are the one place in this module that may touch the network. A library
// opened without a content source leaves that cache absent, and those methods
// then throw. `savePath` and `isLocal` look only at the disk.
// `download()`, `content()` and `exif()` go through the on-disk content cache
// (issue #46), and are the one place in this module that may touch the
// network. A library opened without a content source leaves that cache absent,
// and those methods then throw. `savePath` and `isLocal` look only at the disk
// and need no cache.
import { readFile } from "node:fs/promises";
import { readPhotoExif, type PhotoExif } from "../exif.js";
import type { CollectionType, FileType } from "../model/types.js";
import type { CollectionType, EnteFile, FileType } from "../model/types.js";
import type { ContentOptions, ContentResult, PhotoContent } from "./content.js";
import type { AlbumRecord, PhotoRecord, DerivedRecords } from "./records.js";
// Where a photo's original is saved, and whether all of it is there. The
// library answers both from the disk, with or without a content cache.
export interface SavePathLookup {
savePath(file: EnteFile): string;
isLocal(file: EnteFile): boolean;
}
// Newest first, with fileID as a stable tiebreak so equal-timed files order
// deterministically — the same order the record projection uses.
const byNewest = (a: PhotoRecord, b: PhotoRecord): number =>
@@ -35,10 +43,14 @@ const byNewestAlbum = (a: AlbumRecord, b: AlbumRecord): number =>
b.updationTime - a.updationTime || b.collectionID - a.collectionID;
// A single photo. Field access mirrors `PhotoRecord`; `record()` returns the
// underlying plain record for callers that need the IPC-safe value.
// underlying plain record for callers that need the IPC-safe value. `file` is
// the membership the record is read from, so the save path carries the date of
// `takenAt` and stays known after a refresh removes the file from the library.
export class Photo {
constructor(
private readonly rec: PhotoRecord,
private readonly file: EnteFile,
private readonly saves: SavePathLookup,
private readonly cache?: PhotoContent,
) {}
@@ -89,20 +101,20 @@ export class Photo {
return this.rec.isHidden;
}
// Where `lib.backup()` stores the original in the library's download
// directory, whether or not it is there yet: for a live photo already
// stored, its image. For a live photo not yet stored, it carries the
// title's extension, and the backup may store the image under a different
// one. Undefined when the library has no download directory or no content
// cache.
get savePath(): string | undefined {
return this.cache?.savePath(this.rec.fileID);
// Where `download()` and `lib.backup()` put the original under the
// library's download directory, whether or not it is there yet:
// `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID><ext>`. For a live photo
// already stored, its image. For a live photo not yet stored, it carries
// the title's extension, and the image may be stored under a different
// one.
get savePath(): string {
return this.saves.savePath(this.file);
}
// Whether the whole original is at `savePath`. A copy only in the cache
// does not count.
get isLocal(): boolean {
return this.cache?.isLocal(this.rec.fileID) ?? false;
return this.saves.isLocal(this.file);
}
record(): PhotoRecord {
@@ -111,12 +123,20 @@ export class Photo {
// Fetch and cache the full-resolution original, returning its on-disk path
// and byte length; for a live photo, its image's, and its video's path as
// `videoPath`. Served from the cache (or the backup download directory)
// when already present, otherwise fetched through the content pool.
// `videoPath`. Served from the cache (or the save path) when already
// present, otherwise fetched through the content pool.
async original(opts?: ContentOptions): Promise<ContentResult> {
return this.cacheOrThrow().original(this.rec.fileID, opts);
}
// Put the original at `savePath` and return it there, as `original()`
// does. When it is already there, nothing is written. When the cache holds
// it, it is copied from there; otherwise it is fetched straight to
// `savePath`.
async download(): Promise<ContentResult> {
return this.cacheOrThrow().download(this.file);
}
// As `original`, for the thumbnail, through the thumbnail pool.
async thumbnail(opts?: ContentOptions): Promise<ContentResult> {
return this.cacheOrThrow().thumbnail(this.rec.fileID, opts);
@@ -156,6 +176,7 @@ export class Album {
constructor(
private readonly rec: AlbumRecord,
private readonly records: DerivedRecords,
private readonly saves: SavePathLookup,
private readonly content?: PhotoContent,
) {}
@@ -190,7 +211,10 @@ export class Album {
const out: Photo[] = [];
for (const id of this.rec.fileIDs) {
const p = this.records.photos.get(id);
if (p) out.push(new Photo(p, this.content));
const file = this.records.files.get(id);
if (p && file) {
out.push(new Photo(p, file, this.saves, this.content));
}
}
return out;
}
@@ -253,18 +277,19 @@ export interface FreshReads {
export const makeAlbumsAPI = (
derive: () => DerivedRecords,
saves: SavePathLookup,
content?: PhotoContent,
): AlbumsAPI => ({
list: (): Album[] => {
const records = derive();
return [...records.albums.values()]
.sort(byNewestAlbum)
.map((rec) => new Album(rec, records, content));
.map((rec) => new Album(rec, records, saves, content));
},
byID: ({ collectionID }): Album | undefined => {
const records = derive();
const rec = records.albums.get(collectionID);
return rec ? new Album(rec, records, content) : undefined;
return rec ? new Album(rec, records, saves, content) : undefined;
},
byName: ({ albumName }): Album | undefined => {
const records = derive();
@@ -273,17 +298,20 @@ export const makeAlbumsAPI = (
const match = [...records.albums.values()]
.sort(byNewestAlbum)
.find((rec) => rec.name === albumName);
return match ? new Album(match, records, content) : undefined;
return match ? new Album(match, records, saves, content) : undefined;
},
});
export const makePhotosAPI = (
derive: () => DerivedRecords,
saves: SavePathLookup,
content?: PhotoContent,
): PhotosAPI => ({
byID: ({ fileID }): Photo | undefined => {
const rec = derive().photos.get(fileID);
return rec ? new Photo(rec, content) : undefined;
const records = derive();
const rec = records.photos.get(fileID);
const file = records.files.get(fileID);
return rec && file ? new Photo(rec, file, saves, content) : undefined;
},
records: ({ fileIDs }): PhotoRecord[] => {
const { photos } = derive();
src/library/records.ts
+33 -15
View File
@@ -86,6 +86,10 @@ export interface LibraryChange {
export interface DerivedRecords {
albums: Map<number, AlbumRecord>;
photos: Map<number, PhotoRecord>;
// The membership each photo's record is read from, for its `Photo`'s save
// path. It holds the file's key, so it stays in this process: no snapshot
// or change carries it.
files: Map<number, EnteFile>;
}
const asString = (v: unknown): string | undefined =>
@@ -101,18 +105,19 @@ const microsToMillis = (micros: number): number => Math.floor(micros / 1000);
const byNewestPhoto = (a: PhotoRecord, b: PhotoRecord): number =>
b.takenAt - a.takenAt || b.fileID - a.fileID;
// Build one PhotoRecord from every membership of a file. The memberships share
// the same underlying file, so metadata is read from a single representative
// (the most recently synced, lowest collection id to break ties); `albumIDs`
// gathers them all.
const toPhotoRecord = (
fileID: number,
memberships: EnteFile[],
): PhotoRecord => {
const albumIDs = memberships
.map((m) => m.collectionID)
.sort((a, b) => a - b);
const rep = memberships.reduce((best, m) =>
// A photo's `takenAt` in milliseconds: `pubMagicMetadata.editedTime` when the
// user edited the date, else basic-metadata `creationTime`.
export const takenAtOf = (file: EnteFile): number =>
microsToMillis(
asNumber(file.pubMagicMetadata?.editedTime) ??
file.metadata.creationTime,
);
// The membership a file's record is read from: the most recently synced, lowest
// collection id to break ties. Whatever dates a file's save path takes this
// membership too, so the path always carries the record's `takenAt`.
export const representative = (memberships: EnteFile[]): EnteFile =>
memberships.reduce((best, m) =>
m.updationTime > best.updationTime ||
(m.updationTime === best.updationTime &&
m.collectionID < best.collectionID)
@@ -120,17 +125,28 @@ const toPhotoRecord = (
: best,
);
// Build one PhotoRecord from every membership of a file. The memberships share
// the same underlying file, so metadata is read from a single representative;
// `albumIDs` gathers them all.
const toPhotoRecord = (
fileID: number,
memberships: EnteFile[],
): PhotoRecord => {
const albumIDs = memberships
.map((m) => m.collectionID)
.sort((a, b) => a - b);
const rep = representative(memberships);
const pub = rep.pubMagicMetadata ?? {};
const priv = rep.magicMetadata ?? {};
const takenAtMicros = asNumber(pub.editedTime) ?? rep.metadata.creationTime;
const visibility = asNumber(priv.visibility);
const record: PhotoRecord = {
fileID,
albumIDs,
title: asString(pub.editedName) ?? rep.metadata.title,
takenAt: microsToMillis(takenAtMicros),
takenAt: takenAtOf(rep),
modifiedAt: microsToMillis(rep.metadata.modificationTime),
fileType: rep.metadata.fileType,
isArchived: visibility === VISIBILITY_ARCHIVED,
@@ -198,6 +214,7 @@ export const deriveRecords = (
}
const photos = new Map<number, PhotoRecord>();
const photoFiles = new Map<number, EnteFile>();
const takenAtByFile = new Map<number, number>();
for (const [fileID, memberships] of byFileID) {
const record = toPhotoRecord(fileID, memberships);
@@ -209,6 +226,7 @@ export const deriveRecords = (
record.thumbnailPath = paths.thumbnailPath;
}
photos.set(fileID, record);
photoFiles.set(fileID, representative(memberships));
takenAtByFile.set(fileID, record.takenAt);
}
@@ -217,7 +235,7 @@ export const deriveRecords = (
albums.set(c.id, toAlbumRecord(c, files, takenAtByFile));
}
return { albums, photos };
return { albums, photos, files: photoFiles };
};
// Sorted, GUI-ready arrays: albums newest updated first, photos newest first.
src/library/store.ts
+8 -5
View File
@@ -16,6 +16,7 @@ import { dirname } from "node:path";
import { writeAtomic } from "../download/index.js";
import type { Collection, EnteFile, Microseconds } from "../model/types.js";
import { representative } from "./records.js";
// Bumped only when the on-disk shape changes incompatibly. A file written
// under a different version is discarded on load (see `load`): re-fetching
@@ -179,14 +180,16 @@ export class MetadataStore {
return this.files.get(fileKey(collectionID, fileID));
}
// Any membership of a file, or undefined. Every membership re-wraps the
// same underlying content key, so any one is enough to fetch the bytes;
// the content cache resolves a fileID to a file this way.
// The membership of a file its record is read from (`representative`), or
// undefined. Any membership could fetch the bytes, but the content cache
// resolves a fileID to a file this way so that it dates the save path
// from the same membership as `photo.savePath`.
getFileByID(fileID: number): EnteFile | undefined {
const memberships: EnteFile[] = [];
for (const file of this.files.values()) {
if (file.id === fileID) return file;
if (file.id === fileID) memberships.push(file);
}
return undefined;
return memberships.length > 0 ? representative(memberships) : undefined;
}
listFiles(collectionID: number): EnteFile[] {