Save originals at photos/YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.fileID.ext; add photo.download() (closes #143)

Each original's save path is now
YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID><ext> under the library's
download directory, dated by takenAt in the machine's time zone. The
directory defaults to photos/ in the working directory, resolved once at
open. savePath is always a string, with or without a content cache, and
isLocal is true only when the original is there.

photo.download() puts the original at its save path: copied from the
cache when the cache holds it, fetched straight there otherwise.
lib.backup() uses the same code for each file, writes each file's JSON
beside it, and links collections/ to the save paths. The backup has no
originals/ folder.

Model: opus-5-5

src/library/content.ts

@@ -23,6 +23,11 @@
 // original with no recorded hash is stored unchecked, as the upstream client
 // does; thumbnails have none. On top of that this module refuses to record a
 // stored file that came out empty.
+//
+// It also names each original's save path under the download directory
+// (`savePath`), where `Photo.download()` and `lib.backup()` put it. A copy in
+// the cache does not count as saved there, but is copied there rather than
+// fetched again.
 
 import {
     closeSync,
@@ -34,8 +39,10 @@ import {
 } from "node:fs";
 import {
     chmod,
+    copyFile,
     mkdir,
     readdir,
+    rename,
     rm,
     stat,
     statfs,
@@ -47,13 +54,15 @@ import type { ApiClient } from "../api/client.js";
 import {
     downloadFile,
     downloadThumbnail,
+    fsyncPath,
     type ProgressCallback,
     removeLeftoverTempFiles,
     writeAtomic,
 } from "../download/index.js";
-import { safeExtension } from "../filename.js";
+import { safeExtension, withExtension } from "../filename.js";
 import type { EnteFile } from "../model/types.js";
 import type { Priority, RequestPools } from "./pools.js";
+import { takenAtOf } from "./records.js";
 
 const DIR_MODE = 0o700;
 const FILE_MODE = 0o600;
@@ -108,11 +117,8 @@ export interface ContentOptions {
 export interface PhotoContent {
     original(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
     thumbnail(fileID: number, opts?: ContentOptions): Promise<ContentResult>;
-    // Where a backup stores the original, whether or not it is there yet. For
-    // a live photo not yet stored, it carries the title's extension, and the
-    // backup may store the image under a different one.
-    savePath(fileID: number): string | undefined;
-    isLocal(fileID: number): boolean;
+    // Put the original at its save path and return it there.
+    download(fileID: number): Promise<ContentResult>;
 }
 
 export interface EnsureResult {
@@ -199,10 +205,10 @@ export interface ContentCacheOptions {
     pools: RequestPools;
     source: ContentSource;
     cacheDirectory: string;
-    // The backup destination (issue-level `downloadDirectory`). An original
-    // already stored there by a backup counts as present, so the cache serves
-    // it rather than fetching a second copy.
-    downloadDirectory?: string;
+    // The root of the save paths. An original already stored at its save path
+    // counts as present, so the cache serves it rather than fetching a second
+    // copy.
+    downloadDirectory: string;
     // Resolve any membership of a file; every membership shares the underlying
     // content key, so any one decrypts the same bytes.
     getFile: (fileID: number) => EnteFile | undefined;
@@ -229,11 +235,27 @@ class AbortDrop extends Error {
     }
 }
 
-// The name of a file's original in originals/: `<fileID><ext>`, the extension
-// taken from the title (or `.bin`). A backup names its originals the same way.
+// The name of a file's original in the cache's originals/: `<fileID><ext>`, the
+// extension taken from the title (or `.bin`).
 export const nameInOriginals = (file: EnteFile): string =>
     `${file.id}${safeExtension(file.metadata.title)}`;
 
+const pad = (n: number): string => String(n).padStart(2, "0");
+
+// Where the original of `file` is saved under `root`:
+// `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID><ext>`, dated by the photo's
+// `takenAt` in this machine's time zone, with the extension taken from the
+// title (or `.bin`). A live photo is stored as its image and its video beside
+// this path, each with the extension found inside the live photo.
+export const savePath = (root: string, file: EnteFile): string => {
+    const taken = new Date(takenAtOf(file));
+    const year = String(taken.getFullYear());
+    const month = `${year}-${pad(taken.getMonth() + 1)}`;
+    const day = `${month}-${pad(taken.getDate())}`;
+    const ext = safeExtension(file.metadata.title);
+    return join(root, year, month, day, `${day}.${file.id}${ext}`);
+};
+
 // The fileID a cache filename encodes, or undefined when the name is not one
 // the cache writes (`<digits><ext>`).
 const fileIDFromName = (name: string): number | undefined => {
@@ -279,28 +301,29 @@ const isZip = (path: string): boolean => {
 
 // A live photo's image and video are named with the extensions from inside its
 // ZIP, so their names alone do not say which is which. Wherever the cache or a
-// backup stores one, a JSON file of this name beside them names both.
-const livePhotoJSONName = (fileID: number): string =>
-    `${fileID}.livephoto.json`;
+// save path stores one, a JSON file of this name beside them names both.
+// `name` is the original's name without its extension: `<fileID>` in the
+// cache, `YYYY-MM-DD.<fileID>` at the save path.
+const livePhotoJSONName = (name: string): string => `${name}.livephoto.json`;
 
 // The image and video that the live photo's JSON file in `dir` names, or
-// undefined when there is none. Only names of the form the cache writes are
-// taken, so the file cannot point outside `dir`.
+// undefined when there is none. Only `name` with an extension is taken, so the
+// file cannot point outside `dir`.
 const readLivePhotoJSON = (
     dir: string,
-    fileID: number,
+    name: string,
 ): { path: string; videoPath: string } | undefined => {
-    const valid = (name: unknown): name is string =>
-        typeof name === "string" && name === `${fileID}${safeExtension(name)}`;
+    const valid = (part: unknown): part is string =>
+        typeof part === "string" && part === `${name}${safeExtension(part)}`;
     try {
         const { image, video } = JSON.parse(
-            readFileSync(join(dir, livePhotoJSONName(fileID)), "utf-8"),
+            readFileSync(join(dir, livePhotoJSONName(name)), "utf-8"),
         );
         if (valid(image) && valid(video)) {
             return { path: join(dir, image), videoPath: join(dir, video) };
         }
     } catch {
-        // No such file, or not one the cache wrote.
+        // No such file, or not one quak wrote.
     }
     return undefined;
 };
@@ -308,11 +331,11 @@ const readLivePhotoJSON = (
 // Write the JSON file naming a live photo's image and video, both in `dir`.
 export const writeLivePhotoJSON = (
     dir: string,
-    fileID: number,
+    name: string,
     stored: { path: string; videoPath: string },
 ): Promise<void> =>
     writeAtomic(
-        join(dir, livePhotoJSONName(fileID)),
+        join(dir, livePhotoJSONName(name)),
         new TextEncoder().encode(
             JSON.stringify({
                 image: basename(stored.path),
@@ -321,17 +344,19 @@ export const writeLivePhotoJSON = (
         ),
     );
 
-// The original of `file` as the cache or a backup stored it in `dir`, when all
-// of it is there: `<fileID><ext>`, or a live photo's image and video.
+// The original of `file` as stored in `dir` under `name` (without its
+// extension), when all of it is there: `<name><ext>`, or a live photo's image
+// and video.
 export const storedOriginal = (
     dir: string,
+    name: string,
     file: EnteFile,
 ): { path: string; videoPath?: string } | undefined => {
     if (file.metadata.fileType !== "livePhoto") {
-        const path = join(dir, nameInOriginals(file));
+        const path = join(dir, `${name}${safeExtension(file.metadata.title)}`);
         return hasContent(path) ? { path } : undefined;
     }
-    const stored = readLivePhotoJSON(dir, file.id);
+    const stored = readLivePhotoJSON(dir, name);
     return stored !== undefined &&
         hasContent(stored.path) &&
         hasContent(stored.videoPath)
@@ -339,10 +364,79 @@ export const storedOriginal = (
         : undefined;
 };
 
+// The original of `file` as stored at its save path under `root`, when all of
+// it is there.
+export const storedAtSavePath = (
+    root: string,
+    file: EnteFile,
+): { path: string; videoPath?: string } | undefined => {
+    const path = savePath(root, file);
+    return storedOriginal(dirname(path), basename(path, extname(path)), file);
+};
+
+// Copy bytes into `dest` via a temp file in the same directory plus rename, so
+// `dest` appears only once it is whole ("present means complete"). As in the
+// download writer, the temp file is fsynced before the rename and the directory
+// after it, so a power cut cannot leave a correctly named but short original.
+// The temp name carries this process's ID so a later run can tell a leftover
+// from a copy still in progress (see `removeLeftoverTempFiles`).
+export const copyAtomic = async (src: string, dest: string): Promise<void> => {
+    if (src === dest) return;
+    const tmp = join(
+        dirname(dest),
+        `.quak-backup-${basename(dest)}-${process.pid}-${Math.random()
+            .toString(36)
+            .slice(2)}.tmp`,
+    );
+    try {
+        await copyFile(src, tmp);
+        await fsyncPath(tmp);
+        // `rename` replaces the destination's directory entry: an existing
+        // symlink at `dest` is replaced, not followed, and the new file has
+        // the temp file's permissions (copied from `src`).
+        await rename(tmp, dest);
+        await fsyncPath(dirname(dest));
+    } finally {
+        await rm(tmp, { force: true });
+    }
+};
+
+// Put the original of `file` at its save path under `root`, creating its
+// folders. `get` is given the save path and returns where the original is: a
+// fetch writes it there, and a copy the cache holds is copied there. A live
+// photo's image and video go beside the save path, each with its own
+// extension: when they came from the cache they are copied, after removing
+// whatever was at the save path (an earlier version's ZIP of the two). Then
+// the JSON file naming them is written, which is what makes the live photo
+// count as stored.
+export const placeOriginal = async (
+    root: string,
+    file: EnteFile,
+    get: (dest: string) => Promise<{ path: string; videoPath?: string }>,
+): Promise<{ path: string; videoPath?: string }> => {
+    const dest = savePath(root, file);
+    await mkdir(dirname(dest), { recursive: true });
+    const got = await get(dest);
+    if (got.videoPath === undefined) {
+        await copyAtomic(got.path, dest);
+        return { path: dest };
+    }
+    const path = withExtension(dest, extname(got.path));
+    const videoPath = withExtension(dest, extname(got.videoPath));
+    if (got.path !== path) {
+        await rm(dest, { force: true });
+        await copyAtomic(got.path, path);
+        await copyAtomic(got.videoPath, videoPath);
+    }
+    const name = basename(dest, extname(dest));
+    await writeLivePhotoJSON(dirname(dest), name, { path, videoPath });
+    return { path, videoPath };
+};
+
 export class ContentCache implements PhotoContent, ThumbnailsAPI {
     private readonly pools: RequestPools;
     private readonly source: ContentSource;
-    private readonly downloadDirectory?: string;
+    private readonly downloadDirectory: string;
     private readonly getFile: (fileID: number) => EnteFile | undefined;
     private readonly originalsDir: string;
     private readonly thumbnailsDir: string;
@@ -440,32 +534,23 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
         return this.get(fileID, "thumbnail", "on-demand", opts?.onProgress);
     }
 
-    // Where a backup to the download directory stores the file's original,
-    // whether or not it is there yet: for a live photo already stored, its
-    // image. For a live photo not yet stored, it carries the title's
-    // extension, and the backup may store the image under a different one.
-    // Undefined with no download directory.
-    savePath(fileID: number): string | undefined {
+    // Put the original at its save path under the download directory and
+    // return it there. One already stored there is returned as it is; one the
+    // cache holds is copied from it; any other is fetched straight to the
+    // save path, with no copy left in the cache.
+    async download(fileID: number): Promise<ContentResult> {
         const file = this.getFile(fileID);
-        if (this.downloadDirectory === undefined || file === undefined)
-            return undefined;
-        const dir = join(this.downloadDirectory, "originals");
-        return (
-            storedOriginal(dir, file)?.path ?? join(dir, nameInOriginals(file))
-        );
+        if (!file) throw new Error(`content cache: unknown file ${fileID}`);
+        const root = this.downloadDirectory;
+        const saved =
+            storedAtSavePath(root, file) ??
+            (await placeOriginal(root, file, (dest) =>
+                this.backupOriginal(fileID, dest),
+            ));
+        return { ...saved, bytes: fileSize(saved.path) ?? 0 };
     }
 
-    // Whether the whole original is in the download directory, as a backup
-    // stores it. A copy only in the cache does not count.
-    isLocal(fileID: number): boolean {
-        const file = this.getFile(fileID);
-        if (this.downloadDirectory === undefined || file === undefined)
-            return false;
-        const dir = join(this.downloadDirectory, "originals");
-        return storedOriginal(dir, file) !== undefined;
-    }
-
-    // Get an original for a backup. One not present anywhere is written
+    // Get an original for a save path. One not present anywhere is written
     // straight to `destination` and recorded there, so no second copy lands
     // in the cache; one already present is returned where it is.
     async backupOriginal(
@@ -635,12 +720,9 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
             known.delete(fileID);
         }
 
-        // An original a backup already stored counts as present.
-        if (kind === "original" && this.downloadDirectory !== undefined) {
-            const stored = storedOriginal(
-                join(this.downloadDirectory, "originals"),
-                file,
-            );
+        // An original already stored at its save path counts as present.
+        if (kind === "original") {
+            const stored = storedAtSavePath(this.downloadDirectory, file);
             if (stored !== undefined) {
                 this.originals.set(fileID, stored);
                 return {
@@ -676,7 +758,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
                         ? this.beginOriginalWrite(fileID)
                         : null;
                 try {
-                    const stored = await this.download(
+                    const stored = await this.fetchInto(
                         file,
                         dest,
                         kind,
@@ -691,12 +773,12 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
                             );
                         }
                     }
-                    // A backup records its own live photos.
+                    // `placeOriginal` records a live photo it saves.
                     if (
                         stored.videoPath !== undefined &&
                         opts?.destination === undefined
                     ) {
-                        await writeLivePhotoJSON(dir, fileID, {
+                        await writeLivePhotoJSON(dir, String(fileID), {
                             path: stored.path,
                             videoPath: stored.videoPath,
                         });
@@ -719,7 +801,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
 
     // Fetch into `destination`, returning where the bytes landed: there, or
     // for a live photo, its image and video beside it.
-    private async download(
+    private async fetchInto(
         file: EnteFile,
         destination: string,
         kind: Kind,
@@ -744,10 +826,10 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
         await utimes(path, now, now).catch(() => undefined);
     }
 
-    // Every stored original that lives under `originalsDir` (a backup-directory
-    // hit recorded in the map is excluded), with its size and mtime; a live
+    // Every stored original that lives under `originalsDir` (a save-path hit
+    // recorded in the map is excluded), with its size and mtime; a live
     // photo's size includes its video. Entries whose file has vanished are
-    // dropped from the map. Backups and thumbnails are never counted.
+    // dropped from the map. Save paths and thumbnails are never counted.
     private async measureOriginals(): Promise<{
         entries: {
             fileID: number;
@@ -853,7 +935,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
                         await rm(
                             join(
                                 this.originalsDir,
-                                livePhotoJSONName(e.fileID),
+                                livePhotoJSONName(String(e.fileID)),
                             ),
                             { force: true },
                         );
@@ -907,8 +989,8 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
             // earlier version stored under the image's name, and is removed.
             // Any other is left alone: another process may have just stored
             // it and not yet written the JSON file.
-            const livePhoto = names.has(livePhotoJSONName(id))
-                ? readLivePhotoJSON(dir, id)
+            const livePhoto = names.has(livePhotoJSONName(String(id)))
+                ? readLivePhotoJSON(dir, String(id))
                 : undefined;
             if (livePhoto !== undefined) {
                 into.set(id, livePhoto);