diff --git a/README.md b/README.md index 3903483..3fad579 100644 --- a/README.md +++ b/README.md @@ -470,10 +470,14 @@ cache lives; without it each account gets its own directory under the per-user cache path. `get` and `get-thumb` resolve the file by ID directly, so `--collection` is -accepted for backward compatibility but ignored. `backup-metadata --exif` (alias -`--all`) additionally downloads each file to extract full EXIF/IPTC/XMP -metadata. The listing and backup commands support `--json` for machine-readable -output. +accepted for backward compatibility but ignored. For a live photo, `get` writes +its image and its video, each named after the title with its own extension, as +Ente's clients name them (`IMG_0001.heic` and `IMG_0001.mov`). With +`--out PATH`, the image is written to `PATH` and the video beside it, with +`PATH`'s name and the video's extension; a `PATH` with the video's extension is +refused. `backup-metadata --exif` (alias `--all`) additionally downloads each +file to extract full EXIF/IPTC/XMP metadata. The listing and backup commands +support `--json` for machine-readable output. `backup-metadata` fetches ML data in requests of up to 200 files. When a request still fails after its retries, the error is logged, each of its files is written @@ -499,8 +503,10 @@ the smallest does not. ``` / originals/ - . actual file content (one per unique file) + . actual file content (one per unique file, + two for a live photo: see below) .json all decrypted metadata for that file + .livephoto.json which of a live photo's two files is which collections/ / -> ../../originals/<fileID>.<ext> (symlink) @@ -516,12 +522,22 @@ library's `lib.backup({ includeThumbnails: true })` also writes A collection's directory and JSON are named after the collection, and a symlink after the file's title, both with unsafe characters replaced. When two -collections would get the same name, or two files in one collection the same -title (ignoring case in both), each of them gets its ID added: two albums named +collections would get the same name, or two symlinks in one collection the same +name (ignoring case in both), each of them gets its ID added: two albums named `Trip` become `Trip (10)/` and `Trip (11)/`, and two files titled `IMG_0001.JPG` become `IMG_0001 (12345).JPG` and `IMG_0001 (12346).JPG`. IDs never change, so a name stays the same from run to run until such a clash appears or goes away. +A live photo, which Ente stores as one ZIP of its image and its video, is stored +as those two files, which a photo viewer can open: each is +`originals/<fileID>.<ext>` with the extension it has inside the ZIP (for example +`12345.heic` and `12345.mov`), and `<fileID>.livephoto.json` names the two. The +live photo counts as stored only when both files are present and not empty. Its +album folder links both, each named after the title with that file's extension +(`IMG_0001.heic` and `IMG_0001.mov`). A live photo that an earlier version of +quak stored as the ZIP, under the image's name, is replaced by its two files on +the next run, and the ZIP and its link are removed. + Each run removes the symlinks into `originals/` that no longer belong in their collection's directory, and the directories (and JSON) of collections that were deleted or renamed. Nothing else in `collections/` is touched: a file or a @@ -555,9 +571,9 @@ temporary file's permissions, not those of the file it replaced. errors - [x] Update the API reference section below to match the current implementation - [x] `make docker` green -- [ ] Store live photos in a form a photo viewer can open - (https://git.eeqj.de/sneak/quak/issues/107), once sneak has chosen between - keeping the ZIP and unpacking it +- [x] Store live photos in a form a photo viewer can open + (https://git.eeqj.de/sneak/quak/issues/107): unpacked into the image and + the video Tagging and releases are decided by sneak alone, and happen only when he declares one. @@ -649,7 +665,9 @@ An `Album` exposes its record fields and `album.photos.list()` → `Photo[]` (newest first). A `Photo` exposes its record fields, `photo.record()` → `PhotoRecord`, and two content methods: -- `await photo.original(opts?)` → `{ path, bytes }` — the full-resolution file. +- `await photo.original(opts?)` → `{ path, bytes, videoPath? }` — the + full-resolution file. For a live photo, `path` and `bytes` are its image's and + `videoPath` is its video. - `await photo.thumbnail(opts?)` → `{ path, bytes }`. Both serve from the on-disk content cache when the bytes are present and @@ -669,7 +687,7 @@ The GUI-facing records hold no key material and no binary, so they survive - `PhotoRecord`: `fileID`, `albumIDs`, `title`, `takenAt` (milliseconds), `fileType`, optional `caption` / `width` / `height` / `latitude` / `longitude`, `isArchived`, `isHidden`, and `thumbnailPath` / `originalPath` - once the bytes are cached. + once the bytes are cached (for a live photo, `originalPath` is its image). - `AlbumRecord`: `collectionID`, `name`, `type`, `isShared`, `updationTime`, and `fileIDs` (newest first). - `LibrarySnapshot`: `{ albums, photos, takenAt }`. @@ -720,6 +738,8 @@ Under `cacheDirectory`: <cacheDirectory>/ metadata.json decrypted account state + refresh cursor originals/<fileID>.<ext> cached full-resolution files + originals/<fileID>.livephoto.json + which of a live photo's two files is which thumbnails/<fileID>.jpg cached thumbnails mldata/ <fileID>.json one decrypted ML payload per file @@ -732,14 +752,21 @@ When `metadata.json` belongs to a different account than the client's, originals and thumbnails are kept; they are reached only through the files the current account's records name. +A live photo's original is cached as in the backup: its image and its video, +each `originals/<fileID>.<ext>` with its own extension, and +`originals/<fileID>.livephoto.json` naming them; the two are evicted together. A +live photo that an earlier version cached as its ZIP is not served, and is +replaced by its two files the next time it is read. + A stored file appears only via an atomic temp-then-rename, so its presence means it is complete. Every downloaded original (by `quak get`, the cache, or `backup`) whose metadata records a content hash (`FileMetadata.hash`) is hashed -as it is written: unkeyed BLAKE2b with a 64-byte output, standard base64. For a -live photo, which is stored as a ZIP, the image and the video are hashed -separately and joined as `<imageHash>:<videoHash>`. A mismatch stores nothing -and fails the download with an error naming the file ID. An original with no -recorded hash, from a very old client, is stored unchecked. +as it is written: unkeyed BLAKE2b with a 64-byte output, standard base64. A live +photo arrives as a ZIP and is unpacked as it is written; its image and its video +are hashed separately and joined as `<imageHash>:<videoHash>`, and neither is +stored unless both are complete and match. A mismatch stores nothing and fails +the download with an error naming the file ID. An original with no recorded +hash, from a very old client, is stored unchecked. ### Key types by source file diff --git a/TODO.md b/TODO.md index 35b9e6f..ae1c1e6 100644 --- a/TODO.md +++ b/TODO.md @@ -14,15 +14,25 @@ pre-1.0 # Next Step -Store live photos in a form a photo viewer can open -(https://git.eeqj.de/sneak/quak/issues/107). This waits on sneak's choice -between keeping the ZIP and unpacking it into the image and the video. +None: every issue still open is done on `next` or `next2` and waits for it to +reach `main`. Tagging and releases are decided by sneak alone, and happen only when he declares one. # Completed Steps +- 2026-09-23: Live photos are stored as their image and their video (issue 107). + A live photo, which Ente stores as one ZIP, is unpacked as it downloads into + `<fileID>.<ext>` for the image and for the video, each with its extension from + the ZIP, beside `<fileID>.livephoto.json`, which names the two. Both are + checked against the recorded hash and renamed into place only when both are + complete. The backup and the content cache count the live photo as stored only + with both files, the album folder links both, `quak get` writes both, and the + content result gives the video as `videoPath`. A ZIP an earlier version stored + is replaced on the next backup run, and in the cache when the photo is next + read. + - 2026-09-23: Settled the package metadata (issue 6). quak is not published, so `package.json` is marked `"private": true` and the `files` field is gone. `engines.node` is `>=22`, the major version `script/bootstrap` and the diff --git a/src/backup.ts b/src/backup.ts index 019619c..e8de23f 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -5,7 +5,7 @@ // gets its original bytes onto disk under `downloadDirectory` and rebuilds the // derived views (per-file sidecars, per-collection symlink trees, // per-collection JSON) from the model. The on-disk layout is the historical -// one, unchanged: +// one: // // <downloadDirectory>/ // originals/<fileID>.<ext> the decrypted bytes @@ -14,6 +14,10 @@ // collections/<name>.json per-collection metadata // failures.json durable ledger of unresolved failures // +// A live photo's original is its image and its video, `<fileID>.<ext>` each +// with its own extension, and `originals/<fileID>.livephoto.json` naming them; +// its album folders link both. +// // Crash-safety rests on two properties. Bytes are present-means-complete: an // original appears under `originals/` only via the content layer's atomic // temp-then-rename, so a file that exists is whole and is never re-fetched — an @@ -48,7 +52,12 @@ import { copyFile, rename, rm } from "node:fs/promises"; import { basename, dirname, extname, join, relative } from "node:path"; import { fsyncPath, removeLeftoverTempFiles } from "./download/index.js"; -import { safeExtension, sanitizeFileName } from "./filename.js"; +import { sanitizeFileName, withExtension } from "./filename.js"; +import { + originalName, + storedOriginal, + writeLivePhoto, +} from "./library/content.js"; import type { Collection, EnteFile } from "./model/types.js"; export type ProgressCallback = (message: string) => void; @@ -98,8 +107,13 @@ export interface BackupLibrary { listFiles(collectionID: number): EnteFile[]; // Get an original's bytes onto disk through the content cache/pools, // returning where they landed: `destination` when they were fetched now, - // otherwise wherever they already were (the cache, or a prior backup). - original(fileID: number, destination: string): Promise<{ path: string }>; + // otherwise wherever they already were (the cache, or a prior backup). A + // live photo lands as its image and its video, fetched now beside + // `destination`. + original( + fileID: number, + destination: string, + ): Promise<{ path: string; videoPath?: string }>; thumbnail(fileID: number): Promise<{ path: string }>; } @@ -116,12 +130,6 @@ interface FailureEntry { const LEDGER_VERSION = 1; -// The originals/ filename for a file: `<id><ext>`, the extension taken from the -// title (or `.bin`). Matches the content cache's own naming so a present check -// lines up with what a fetch would write. -const originalName = (file: EnteFile): string => - `${file.id}${safeExtension(file.metadata.title)}`; - // A regular file with content is treated as complete. A zero-byte file is not: // it is the shape an aborted write leaves and must be re-fetched. const isPresent = (path: string): boolean => { @@ -187,6 +195,31 @@ const copyAtomic = async (src: string, dest: string): Promise<void> => { } }; +// Put an original the library returned at `dest` in originals/, where a fresh +// fetch already wrote it. A live photo's image and video go beside `dest`: when +// they came from the cache they are copied, after removing whatever was at +// `dest` (an earlier version's ZIP of the two). Then the JSON file naming them +// is written, which is what makes the live photo count as stored. +const placeOriginal = async ( + file: EnteFile, + dest: string, + got: { path: string; videoPath?: string }, +): Promise<void> => { + if (got.videoPath === undefined) { + await copyAtomic(got.path, dest); + return; + } + const originalsDir = dirname(dest); + const path = join(originalsDir, basename(got.path)); + const videoPath = join(originalsDir, basename(got.videoPath)); + if (got.path !== path) { + await rm(dest, { force: true }); + await copyAtomic(got.path, path); + await copyAtomic(got.videoPath, videoPath); + } + await writeLivePhoto(originalsDir, file.id, { path, videoPath }); +}; + // Ensure `linkPath` is a symlink to `target`, rebuilding a missing, wrong, or // non-symlink entry. Throws on failure (a directory in the way, no permission) // so the caller records it and moves on rather than aborting the run. @@ -203,43 +236,61 @@ const rebuildSymlink = (linkPath: string, target: string): void => { symlinkSync(target, linkPath); }; -// The on-disk names for the entries of one directory, keyed by ID. Each name -// is used as is unless another entry would get the same name, ignoring case -// (two names that differ only in case are one entry on a case-insensitive +// The on-disk names for the entries of one directory, in entry order. Each +// name is used as is unless another entry would get the same name, ignoring +// case (two names that differ only in case are one entry on a case-insensitive // file system); then every entry sharing it gets ` (<id>)`, before the // extension when `beforeExtension` is set. A name with an ID added can match // another entry's own name (`IMG (6).JPG`), so this repeats until no name is // shared. IDs are stable, so the names are too. -const namesByID = ( +const uniqueNames = ( entries: { id: number; name: string }[], beforeExtension: boolean, -): Map<number, string> => { +): string[] => { const withID = (id: number, name: string): string => { const ext = beforeExtension ? extname(name) : ""; const stem = name.slice(0, name.length - ext.length); return `${stem} (${id})${ext}`; }; - const names = new Map<number, string>(); - for (const { id, name } of entries) names.set(id, name); + const names = entries.map((e) => e.name); const suffixed = new Set<number>(); for (;;) { const counts = new Map<string, number>(); - for (const name of names.values()) { + for (const name of names) { const key = name.toLowerCase(); counts.set(key, (counts.get(key) ?? 0) + 1); } let changed = false; - for (const { id, name } of entries) { - if (suffixed.has(id)) continue; + for (const [i, { id, name }] of entries.entries()) { + if (suffixed.has(i)) continue; if (counts.get(name.toLowerCase()) === 1) continue; - names.set(id, withID(id, name)); - suffixed.add(id); + names[i] = withID(id, name); + suffixed.add(i); changed = true; } if (!changed) return names; } }; +// The links a file gets in its album's folder: one named after its title, to +// its original if that is stored. A stored live photo gets two, to its image +// and its video, each named after the title with that file's extension. +const linksFor = ( + file: EnteFile, + stored: { path: string; videoPath?: string } | undefined, +): { id: number; name: string; file: EnteFile; target?: string }[] => { + const name = sanitizeFileName(file.metadata.title, `file-${file.id}`); + if (stored?.videoPath === undefined) { + return [{ id: file.id, name, file, target: stored?.path }]; + } + return [stored.path, stored.videoPath].map((target) => ({ + id: file.id, + name: withExtension(name, extname(target)), + file, + target, + })); +}; + // Remove the symlinks in the album directory `dir` that point into // `originalsDir` and are not named in `keep`. Nothing else in the directory // is touched: anything else there was put there by the user. @@ -419,17 +470,21 @@ export const runBackup = async ( // tree; a present file is left as is. if (includeOriginals) { for (const [fileID, file] of distinct) { - const dest = join(originalsDir, originalName(file)); - if (isPresent(dest)) { + if (storedOriginal(originalsDir, file) !== undefined) { skipped++; continue; } + const dest = join(originalsDir, originalName(file)); try { log(`Fetching original ${file.metadata.title} (${fileID})...`); - // A fetched original is written straight to `dest`; only one - // that was already cached elsewhere is copied. - const { path } = await lib.original(fileID, dest); - await copyAtomic(path, dest); + // A fetched original is written straight to `dest` (a live + // photo beside it); only one that was already cached elsewhere + // is copied. + await placeOriginal( + file, + dest, + await lib.original(fileID, dest), + ); downloaded++; } catch (err) { log( @@ -465,8 +520,7 @@ export const runBackup = async ( // every present original (this repairs stale ones). if (includeOriginals) { for (const [fileID, file] of distinct) { - const orig = join(originalsDir, originalName(file)); - if (isPresent(orig)) { + if (storedOriginal(originalsDir, file) !== undefined) { writeSidecar(join(originalsDir, `${fileID}.json`), file); } } @@ -478,19 +532,18 @@ export const runBackup = async ( // an album it skipped. Stale entries are removed before anything is // rebuilt, so on a case-insensitive file system removing an old name can // never remove the new one. - const albumDirNames = namesByID( + const dirNames = uniqueNames( allCollections.map((c) => ({ id: c.id, name: sanitizeFileName(c.name, `collection-${c.id}`), })), false, ); + const albumDirNames = new Map( + allCollections.map((c, i) => [c.id, dirNames[i]!]), + ); try { - removeStaleAlbumDirs( - collectionsDir, - new Set(albumDirNames.values()), - originalsDir, - ); + removeStaleAlbumDirs(collectionsDir, new Set(dirNames), originalsDir); } catch (err) { log(`FAILED removing old album directories: ${errorMessage(err)}`); } @@ -501,34 +554,33 @@ export const runBackup = async ( mkdirSync(colDir, { recursive: true }); const files = filesByCollection.get(c.id) ?? []; - const linkNames = namesByID( - files.map((f) => ({ - id: f.id, - name: sanitizeFileName(f.metadata.title, `file-${f.id}`), - })), - true, + const links = files.flatMap((f) => + linksFor(f, storedOriginal(originalsDir, f)), ); + const linkNames = uniqueNames(links, true); try { - removeStaleLinks(colDir, new Set(linkNames.values()), originalsDir); + removeStaleLinks(colDir, new Set(linkNames), originalsDir); } catch (err) { log(`FAILED removing old links in ${c.name}: ${errorMessage(err)}`); } - const metaFiles: { id: number; metadata: EnteFile["metadata"] }[] = []; - for (const file of files) { - metaFiles.push({ id: file.id, metadata: file.metadata }); - if (!includeOriginals) continue; - const orig = join(originalsDir, originalName(file)); - if (!isPresent(orig)) continue; - const linkName = linkNames.get(file.id)!; - const linkPath = join(colDir, linkName); + const metaFiles = files.map((f) => ({ + id: f.id, + metadata: f.metadata, + })); + for (const [i, link] of links.entries()) { + if (!includeOriginals || link.target === undefined) continue; + const linkName = linkNames[i]!; try { - rebuildSymlink(linkPath, relative(colDir, orig)); + rebuildSymlink( + join(colDir, linkName), + relative(colDir, link.target), + ); } catch (err) { log( `FAILED symlink ${c.name}/${linkName}: ${errorMessage(err)}`, ); - recordFailure(file, c.name, err); + recordFailure(link.file, c.name, err); } } diff --git a/src/cli-commands.ts b/src/cli-commands.ts index ea4ff18..3ea8f34 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -10,10 +10,11 @@ import { copyFileSync, existsSync, mkdirSync, + statSync, unlinkSync, writeFileSync, } from "node:fs"; -import { join } from "node:path"; +import { extname, join } from "node:path"; import { type Client, type ClientSnapshot, @@ -32,6 +33,7 @@ import { thumbnailName, } from "./cli-output.js"; import { freshCollections, freshFiles, freshFile } from "./cli-read.js"; +import { withExtension } from "./filename.js"; import { runMetadataBackup } from "./metadata-backup.js"; import { listMissingThumbnails, fixMissingThumbnails } from "./thumbnails.js"; @@ -305,8 +307,29 @@ export const getCommand = async ( // Default name is the file's own title, as the pre-library CLI used // (not the editedName-preferring projection title) (issue #52). const outPath = opts.out ?? originalName(file); - copyFileSync(result.path, outPath); - ctx.stderr.write(`${result.bytes} bytes -> ${outPath}\n`); + if (result.videoPath === undefined) { + copyFileSync(result.path, outPath); + ctx.stderr.write(`${result.bytes} bytes -> ${outPath}\n`); + return 0; + } + // A live photo is written as its image and its video, each named after + // the title with its own extension, as Ente's clients name them. With + // --out, the image goes there and the video beside it. + const imageOut = + opts.out ?? withExtension(outPath, extname(result.path)); + const videoOut = withExtension(outPath, extname(result.videoPath)); + if (imageOut.toLowerCase() === videoOut.toLowerCase()) { + ctx.stderr.write( + `File ${fileID} is a live photo, and its video would also be written to ${imageOut}\n`, + ); + return 1; + } + copyFileSync(result.path, imageOut); + copyFileSync(result.videoPath, videoOut); + ctx.stderr.write( + `${result.bytes} bytes -> ${imageOut}\n` + + `${statSync(videoOut).size} bytes -> ${videoOut}\n`, + ); return 0; } finally { await lib.close(); diff --git a/src/download/index.ts b/src/download/index.ts index 06cd18a..eed2114 100644 --- a/src/download/index.ts +++ b/src/download/index.ts @@ -16,14 +16,18 @@ import { streamTagFinal, } from "../crypto/index.js"; import { TruncatedStreamError } from "../errors.js"; -import { sanitizeFileName } from "../filename.js"; +import { safeExtension, sanitizeFileName, withExtension } from "../filename.js"; import { withRetry } from "../retry.js"; import type { ApiClient } from "../api/client.js"; import type { EnteFile } from "../model/types.js"; export interface DownloadResult { + // Where the file was written. A live photo is written as two files, its + // image here and its video at `videoPath` (see `decryptLivePhoto`). path: string; + // The decrypted length; for a live photo, that of the ZIP it arrives as. bytesWritten: number; + videoPath?: string; } // Fired as decrypted plaintext accumulates, with the running total of @@ -45,9 +49,9 @@ const ENC_CHUNK_SIZE = STREAM_CHUNK_SIZE + STREAM_CHUNK_OVERHEAD; // new: a body cut short still decrypts and authenticates up to its last whole // chunk, so the absence of TAG_FINAL is the sole evidence it was cut short, and // this throws rather than let a caller keep a short file. The sink has already -// seen those chunks by then; the caller (`decryptToTemp`) stages them in a temp -// file that is renamed into place only on a clean return, so a throw leaves -// nothing on disk. +// seen those chunks by then; the callers (`decryptToTemp`, `decryptLivePhoto`) +// stage them in temp files that are renamed into place only on a clean return, +// so a throw leaves nothing on disk. const streamDecrypt = async ( stream: ReadableStream<Uint8Array>, header: Uint8Array, @@ -213,6 +217,13 @@ export const removeLeftoverTempFiles = (dir: string): void => { } }; +// A new temp file name in `dir`. The random suffix keeps concurrent downloads +// of the same destination from stepping on each other's temporary file; the +// process ID lets `removeLeftoverTempFiles` tell a leftover from a write in +// progress. +const tempPathIn = (dir: string): string => + join(dir, `.quak-${process.pid}-${randomBytes(16).toString("hex")}.tmp`); + // Stage a write to `destination` atomically and durably, then rename it into // place. `fill` writes the contents into the open temp file handle — either the // whole buffer at once (`writeAtomic`) or chunk by chunk as they decrypt @@ -237,13 +248,7 @@ const stageAtomic = async ( fill: (handle: FileHandle) => Promise<void>, ): Promise<void> => { const dir = dirname(destination); - // The random suffix keeps concurrent downloads of the same destination - // from stepping on each other's temporary file; the process ID lets - // `removeLeftoverTempFiles` tell a leftover from a write in progress. - const tmpPath = join( - dir, - `.quak-${process.pid}-${randomBytes(16).toString("hex")}.tmp`, - ); + const tmpPath = tempPathIn(dir); try { const handle = await open(tmpPath, "w"); try { @@ -278,81 +283,14 @@ export const writeAtomic = async ( ): Promise<void> => stageAtomic(destination, (handle) => handle.writeFile(plaintext)); -// Hashes an original's bytes as they are decrypted, for comparison with the -// hash its uploader recorded. -interface ContentHasher { - update: (plaintext: Uint8Array) => void; - digest: () => string; -} - -const fileHasher = (): ContentHasher => { - const state = chunkHashInit(); - return { - update: (plaintext) => chunkHashUpdate(state, plaintext), - digest: () => chunkHashFinal(state), - }; -}; - -// A live photo is stored as a ZIP of its image and its video, and its recorded -// hash is `<imageHash>:<videoHash>`, each over that part's own bytes. Like the -// upstream client's decoder, this takes the first entries whose names start -// with `image` and `video`. -// -// The ZIP is chosen by its uploader and may expand enormously, so entries are -// hashed as they decompress and never held. fflate's `Unzip` inflates each -// push in one piece, and deflate expands at most about 1000-fold, so the ZIP -// is pushed in 4 KiB slices to keep each decompressed piece near 4 MiB, one -// plaintext chunk. Every entry is started, even one that is not hashed, -// because fflate keeps an unstarted entry's data in memory. -const livePhotoHasher = (fileID: number): ContentHasher => { - const sliceSize = 4096; - const fail = (message: string, cause?: unknown): Error => - new Error(`download: file ${fileID}: ${message}`, { cause }); - const claimed = new Set<string>(); - const hashes = new Map<string, string>(); - const unzip = new Unzip((entry) => { - const part = ["image", "video"].find((p) => entry.name.startsWith(p)); - const target = - part === undefined || claimed.has(part) - ? undefined - : { part, state: chunkHashInit() }; - if (target !== undefined) claimed.add(target.part); - entry.ondata = (err, data, final) => { - if (err) throw err; - if (target === undefined) return; - chunkHashUpdate(target.state, data); - if (final) hashes.set(target.part, chunkHashFinal(target.state)); - }; - entry.start(); - }); - unzip.register(UnzipInflate); - // fflate reports a bad ZIP by throwing, sometimes a TypeError, which the - // retry would take for a network failure; a bad ZIP is never retried. - const push = (data: Uint8Array, final: boolean): void => { - try { - unzip.push(data, final); - } catch (err) { - throw fail("live photo is not a readable ZIP", err); - } - }; - return { - update: (plaintext) => { - for (let i = 0; i < plaintext.length; i += sliceSize) { - push(plaintext.subarray(i, i + sliceSize), false); - } - }, - digest: () => { - push(new Uint8Array(0), true); - const image = hashes.get("image"); - const video = hashes.get("video"); - if (image === undefined || video === undefined) { - throw fail( - "live photo ZIP does not hold both an image and a video", - ); - } - return `${image}:${video}`; - }, - }; +// Refuse an original whose bytes do not hash to what its uploader recorded. +// The error is not retried. +const checkHash = (file: EnteFile, actual: string): void => { + if (actual !== file.metadata.hash) { + throw new Error( + `download: file ${file.id}: content hash ${actual} does not match the hash its uploader recorded, ${file.metadata.hash}`, + ); + } }; // Decrypt `stream` straight to `destination`, one plaintext chunk at a time, @@ -363,9 +301,8 @@ const livePhotoHasher = (fileID: number): ContentHasher => { // Returns the plaintext length written. // // `original` is the file whose original this is (none for a thumbnail, which -// has no recorded hash). When its metadata has a hash, the decrypted bytes -// must match it or nothing is stored. Both a plain file and a live photo's -// parts are hashed as they stream. The mismatch error is not retried. +// has no recorded hash). When its metadata has a hash, the decrypted bytes are +// hashed as they stream and must match it, or nothing is stored. const decryptToTemp = async ( destination: string, stream: ReadableStream<Uint8Array>, @@ -374,44 +311,173 @@ const decryptToTemp = async ( onProgress?: ProgressCallback, original?: EnteFile, ): Promise<number> => { - const expected = original?.metadata.hash; - const hasher = - original === undefined || expected === undefined - ? undefined - : original.metadata.fileType === "livePhoto" - ? livePhotoHasher(original.id) - : fileHasher(); + const hash = + original?.metadata.hash === undefined ? undefined : chunkHashInit(); let bytesWritten = 0; + await stageAtomic(destination, async (handle) => { + bytesWritten = await streamDecrypt( + stream, + header, + key, + async (plaintext) => { + if (hash !== undefined) chunkHashUpdate(hash, plaintext); + await handle.write(plaintext); + }, + onProgress, + ); + if (original !== undefined && hash !== undefined) { + checkHash(original, chunkHashFinal(hash)); + } + }); + return bytesWritten; +}; + +// One of the two parts of a live photo being unpacked: the ZIP entry whose +// name starts with `kind`, written to its own temp file. +interface LivePhotoPart { + kind: "image" | "video"; + tmpPath: string; + handle: FileHandle; + hash: ReturnType<typeof chunkHashInit>; + // Decompressed bytes not yet written. + pending: Uint8Array[]; + // The entry's extension, set once all of the entry has been read. + ext?: string; +} + +const openPart = async ( + kind: "image" | "video", + dir: string, +): Promise<LivePhotoPart> => { + const tmpPath = tempPathIn(dir); + const handle = await open(tmpPath, "w"); + return { kind, tmpPath, handle, hash: chunkHashInit(), pending: [] }; +}; + +// A live photo arrives as a ZIP of its image and its video. Ente's clients +// name the entries `image.<ext>` and `video.<ext>`, and like the upstream +// client's decoder this takes the first entries whose names start with `image` +// and `video`. It is written unpacked: each part is named `destination` with +// the extension replaced by its own entry's, and the two must differ ignoring +// case. When the file records a hash, `<imageHash>:<videoHash>` must match it, +// each over that part's own bytes. Only then is whatever was at `destination` +// removed and the image, then the video, renamed into place; on any failure +// neither is stored. +// +// The ZIP is chosen by its uploader and may expand enormously, so each part is +// written as it decompresses and never held. fflate's `Unzip` inflates each +// push in one piece before `push` returns, and deflate expands at most about +// 1000-fold, so the ZIP is pushed in 4 KiB slices, keeping each decompressed +// piece near 4 MiB, one plaintext chunk, and each piece is written before the +// next slice is pushed. Every entry is started, even one that is not kept, +// because fflate keeps an unstarted entry's data in memory. +const decryptLivePhoto = async ( + destination: string, + stream: ReadableStream<Uint8Array>, + header: Uint8Array, + key: Uint8Array, + onProgress: ProgressCallback | undefined, + file: EnteFile, +): Promise<DownloadResult> => { + const sliceSize = 4096; + const dir = dirname(destination); + const fail = (message: string, cause?: unknown): Error => + new Error(`download: file ${file.id}: ${message}`, { cause }); + const parts: LivePhotoPart[] = []; try { - await stageAtomic(destination, async (handle) => { - bytesWritten = await streamDecrypt( - stream, - header, - key, - async (plaintext) => { - hasher?.update(plaintext); - await handle.write(plaintext); - }, - onProgress, + const image = await openPart("image", dir); + parts.push(image); + const video = await openPart("video", dir); + parts.push(video); + + const claimed = new Set<LivePhotoPart>(); + const unzip = new Unzip((entry) => { + const part = parts.find( + (p) => !claimed.has(p) && entry.name.startsWith(p.kind), ); - if (original === undefined || hasher === undefined) return; - const actual = hasher.digest(); - if (actual !== expected) { - throw new Error( - `download: file ${original.id}: content hash ${actual} does not match the hash its uploader recorded, ${expected}`, - ); - } + if (part !== undefined) claimed.add(part); + entry.ondata = (err, data, final) => { + if (err) throw err; + if (part === undefined) return; + chunkHashUpdate(part.hash, data); + part.pending.push(data); + if (final) part.ext = safeExtension(entry.name); + }; + entry.start(); }); + unzip.register(UnzipInflate); + const push = async ( + data: Uint8Array, + final: boolean, + ): Promise<void> => { + // fflate reports a bad ZIP by throwing, sometimes a TypeError, + // which the retry would take for a network failure; a bad ZIP is + // never retried. + try { + unzip.push(data, final); + } catch (err) { + throw fail("live photo is not a readable ZIP", err); + } + for (const part of parts) { + for (const piece of part.pending) + await part.handle.write(piece); + part.pending = []; + } + }; + + const bytesWritten = await streamDecrypt( + stream, + header, + key, + async (plaintext) => { + for (let i = 0; i < plaintext.length; i += sliceSize) { + await push(plaintext.subarray(i, i + sliceSize), false); + } + }, + onProgress, + ); + await push(new Uint8Array(0), true); + + if (image.ext === undefined || video.ext === undefined) { + throw fail( + "live photo ZIP does not hold both an image and a video", + ); + } + if (file.metadata.hash !== undefined) { + checkHash( + file, + `${chunkHashFinal(image.hash)}:${chunkHashFinal(video.hash)}`, + ); + } + if (image.ext.toLowerCase() === video.ext.toLowerCase()) { + throw fail( + `live photo's image and video have the same extension, ${video.ext}`, + ); + } + const path = withExtension(destination, image.ext); + const videoPath = withExtension(destination, video.ext); + for (const part of parts) { + await part.handle.sync(); + await part.handle.close(); + } + await rm(destination, { force: true }); + await rename(image.tmpPath, path); + try { + await rename(video.tmpPath, videoPath); + } catch (err) { + await rm(path, { force: true }).catch(() => undefined); + throw err; + } + await fsyncPath(dir); + return { path, bytesWritten, videoPath }; } catch (err) { - // Cancel the body so its connection is closed now rather than held - // until the stream is garbage collected. A backup run carries on past - // a failed file, so without this every failure would hold a socket. - // This covers every failure, including a temp file that cannot be - // opened and a header that is rejected before the body is read. - await stream.cancel(err).catch(() => undefined); + // Best-effort cleanup, as in `stageAtomic`. + for (const part of parts) { + await part.handle.close().catch(() => undefined); + await rm(part.tmpPath, { force: true }).catch(() => undefined); + } throw err; } - return bytesWritten; }; // Fetch a stream and decrypt it to `destination`, retrying the whole sequence. @@ -442,19 +508,44 @@ const fetchAndDecrypt = async ( destination: string, onProgress?: ProgressCallback, original?: EnteFile, -): Promise<number> => +): Promise<DownloadResult> => withRetry(async () => { const stream = await openStream(); - return decryptToTemp( - destination, - stream, - header, - key, - onProgress, - original, - ); + try { + if (original?.metadata.fileType === "livePhoto") { + return await decryptLivePhoto( + destination, + stream, + header, + key, + onProgress, + original, + ); + } + const bytesWritten = await decryptToTemp( + destination, + stream, + header, + key, + onProgress, + original, + ); + return { path: destination, bytesWritten }; + } catch (err) { + // Cancel the body so its connection is closed now rather than held + // until the stream is garbage collected. A backup run carries on + // past a failed file, so without this every failure would hold a + // socket. This covers every failure, including a temp file that + // cannot be opened and a header that is rejected before the body + // is read. + await stream.cancel(err).catch(() => undefined); + throw err; + } }, api.getRetryOptions()); +// Write `file`'s original to `outPath`. A live photo is written as its image +// and its video beside `outPath` instead, and whatever was at `outPath` is +// removed (see `decryptLivePhoto`). export const downloadFile = async ( api: ApiClient, file: EnteFile, @@ -466,7 +557,7 @@ export const downloadFile = async ( const resolvedPath = outPath ?? sanitizeFileName(file.metadata.title, `file-${file.id}`); const header = fromBase64(file.file.decryptionHeader); - const bytesWritten = await fetchAndDecrypt( + return fetchAndDecrypt( api, () => api.getFileStream(file.id, { retry: false }), header, @@ -475,7 +566,6 @@ export const downloadFile = async ( onProgress, file, ); - return { path: resolvedPath, bytesWritten }; }; export const downloadThumbnail = async ( @@ -488,7 +578,7 @@ export const downloadThumbnail = async ( outPath ?? `thumb_${sanitizeFileName(file.metadata.title, `file-${file.id}`)}`; const header = fromBase64(file.thumbnail.decryptionHeader); - const bytesWritten = await fetchAndDecrypt( + return fetchAndDecrypt( api, () => api.getThumbnailStream(file.id, { retry: false }), header, @@ -496,5 +586,4 @@ export const downloadThumbnail = async ( resolvedPath, onProgress, ); - return { path: resolvedPath, bytesWritten }; }; diff --git a/src/filename.ts b/src/filename.ts index 963257f..1e6250b 100644 --- a/src/filename.ts +++ b/src/filename.ts @@ -35,3 +35,7 @@ export const safeExtension = (title: string): string => { const ext = extname(title); return /^\.[A-Za-z0-9]+$/.test(ext) ? ext : ".bin"; }; + +// `name` with its extension, if it has one, replaced by `ext` (".mov"). +export const withExtension = (name: string, ext: string): string => + name.slice(0, name.length - extname(name).length) + ext; diff --git a/src/library/content.ts b/src/library/content.ts index fce4c9b..812a3fb 100644 --- a/src/library/content.ts +++ b/src/library/content.ts @@ -1,11 +1,13 @@ // The on-disk content and thumbnail cache keyed by fileID (issue #46). // // Layout under `cacheDirectory`: `originals/<fileID>.<ext>` and -// `thumbnails/<fileID>.<ext>`, flat directories at 0700 with files at 0600. -// Content appears only by the streaming atomic writer's rename (the download -// layer, #40), so a file that exists is whole — "present means complete". The -// directory listing taken at `open()` is the record of what is cached, and the -// orphan temp files a crashed write may have left are reaped there. +// `thumbnails/<fileID>.<ext>`, flat directories at 0700 with files at 0600. A +// live photo's original is two files, its image and its video, with +// `originals/<fileID>.livephoto.json` naming them. Content appears only by the +// streaming atomic writer's rename (the download layer, #40), so a file that +// exists is whole — "present means complete". The directory listing taken at +// `open()` is the record of what is cached, and the orphan temp files a crashed +// write may have left are reaped there. // // A fetch goes through the shared request pools (#45): the content pool for // originals, the thumbnail pool for thumbnails. The pool limits concurrency, @@ -22,7 +24,7 @@ // does; thumbnails have none. On top of that this module refuses to record a // stored file that came out empty. -import { existsSync, statSync } from "node:fs"; +import { existsSync, readFileSync, statSync } from "node:fs"; import { chmod, mkdir, @@ -32,7 +34,7 @@ import { statfs, utimes, } from "node:fs/promises"; -import { dirname, extname, join } from "node:path"; +import { basename, dirname, extname, join } from "node:path"; import type { ApiClient } from "../api/client.js"; import { @@ -40,6 +42,7 @@ import { downloadThumbnail, type ProgressCallback, removeLeftoverTempFiles, + writeAtomic, } from "../download/index.js"; import { safeExtension } from "../filename.js"; import type { EnteFile } from "../model/types.js"; @@ -77,6 +80,8 @@ const poolPriorityOf = (priority: ThumbnailPriority): Priority => export interface ContentResult { path: string; bytes: number; + // A live photo's video. `path` and `bytes` are then its image's. + videoPath?: string; } // Progress for a single `original`/`thumbnail` call. A present file emits one @@ -127,11 +132,14 @@ export interface ThumbnailsAPI { // stand-in so the cache logic runs with no crypto and no network. Pool routing, // dedup, present-checks and integrity live in the cache, not here. export interface ContentSource { + // Writes the original at `destination`. A live photo is written beside it + // as its image and its video instead, and their paths are returned, as + // `downloadFile` does. original(args: { file: EnteFile; destination: string; onProgress?: ProgressCallback; - }): Promise<{ bytesWritten: number }>; + }): Promise<{ bytesWritten: number; path?: string; videoPath?: string }>; thumbnail(args: { file: EnteFile; destination: string; @@ -209,7 +217,9 @@ class AbortDrop extends Error { } } -const originalName = (file: EnteFile): string => +// The originals/ name for a file: `<fileID><ext>`, the extension taken from the +// title (or `.bin`). A backup names its originals the same way. +export const originalName = (file: EnteFile): string => `${file.id}${safeExtension(file.metadata.title)}`; // The fileID a cache filename encodes, or undefined when the name is not one @@ -231,6 +241,73 @@ const fileSize = (path: string): number | undefined => { } }; +// Whether `path` is a regular file with content. A zero-byte file is the shape +// an aborted write leaves, so it does not count. +const hasContent = (path: string | undefined): boolean => + path !== undefined && (fileSize(path) ?? 0) > 0; + +// A live photo's image and video are named with the extensions from inside its +// ZIP, so their names alone do not say which is which. Wherever the cache or a +// backup stores one, this JSON file beside them names both. +const livePhotoFileName = (fileID: number): string => + `${fileID}.livephoto.json`; + +// The image and video that the live photo's JSON file in `dir` names, or +// undefined when there is none. Only names of the form the cache writes are +// taken, so the file cannot point outside `dir`. +const readLivePhoto = ( + dir: string, + fileID: number, +): { path: string; videoPath: string } | undefined => { + const valid = (name: unknown): name is string => + typeof name === "string" && name === `${fileID}${safeExtension(name)}`; + try { + const { image, video } = JSON.parse( + readFileSync(join(dir, livePhotoFileName(fileID)), "utf-8"), + ); + if (valid(image) && valid(video)) { + return { path: join(dir, image), videoPath: join(dir, video) }; + } + } catch { + // No such file, or not one the cache wrote. + } + return undefined; +}; + +// Write the JSON file naming a live photo's image and video, both in `dir`. +export const writeLivePhoto = ( + dir: string, + fileID: number, + stored: { path: string; videoPath: string }, +): Promise<void> => + writeAtomic( + join(dir, livePhotoFileName(fileID)), + new TextEncoder().encode( + JSON.stringify({ + image: basename(stored.path), + video: basename(stored.videoPath), + }), + ), + ); + +// The original of `file` as the cache or a backup stored it in `dir`, when all +// of it is there: `<fileID><ext>`, or a live photo's image and video. +export const storedOriginal = ( + dir: string, + file: EnteFile, +): { path: string; videoPath?: string } | undefined => { + if (file.metadata.fileType !== "livePhoto") { + const path = join(dir, originalName(file)); + return hasContent(path) ? { path } : undefined; + } + const stored = readLivePhoto(dir, file.id); + return stored !== undefined && + hasContent(stored.path) && + hasContent(stored.videoPath) + ? stored + : undefined; +}; + export class ContentCache implements PhotoContent, ThumbnailsAPI { private readonly pools: RequestPools; private readonly source: ContentSource; @@ -238,10 +315,17 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { private readonly getFile: (fileID: number) => EnteFile | undefined; private readonly originalsDir: string; private readonly thumbnailsDir: string; - // fileID -> absolute path of the cached bytes, seeded from the directory - // listing at open() and extended as fetches store new files. - private readonly originals = new Map<number, string>(); - private readonly thumbnails = new Map<number, string>(); + // fileID -> absolute path of the cached bytes, and for a live photo's + // original its video's, seeded from the directory listing at open() and + // extended as fetches store new files. + private readonly originals = new Map< + number, + { path: string; videoPath?: string } + >(); + private readonly thumbnails = new Map< + number, + { path: string; videoPath?: string } + >(); private readonly maxOriginalsBytes: number; private readonly freeBelowBytes: number; private readonly isPinned: (fileID: number) => boolean; @@ -301,9 +385,9 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { pathsFor(fileID: number): CachedPaths { const out: CachedPaths = {}; const original = this.originals.get(fileID); - if (original !== undefined) out.originalPath = original; + if (original !== undefined) out.originalPath = original.path; const thumbnail = this.thumbnails.get(fileID); - if (thumbnail !== undefined) out.thumbnailPath = thumbnail; + if (thumbnail !== undefined) out.thumbnailPath = thumbnail.path; return out; } @@ -335,7 +419,11 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { undefined, { destination }, ); - return { path: result.path, bytes: result.bytes }; + return { + path: result.path, + bytes: result.bytes, + videoPath: result.videoPath, + }; } async ensure(args: EnsureOptions): Promise<EnsureResult[]> { @@ -434,52 +522,72 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { ? { status: "skipped", bytes: result.bytes } : { status: "done", bytes: result.bytes }, ); - return { path: result.path, bytes: result.bytes }; + return { + path: result.path, + bytes: result.bytes, + videoPath: result.videoPath, + }; } // The core: return the cached path if present, else fetch through the pool, // store, and return it. `cached` distinguishes a present hit (no network, // no download event) from a fresh fetch. A fetched original is stored at - // `opts.destination` when given, instead of in `originalsDir`. + // `opts.destination` when given, instead of in `originalsDir`. A live + // photo's original is present only with its video, and is returned with + // it. private async acquire( fileID: number, kind: Kind, priority: Priority, signal: AbortSignal | undefined, opts?: { onByte?: ProgressCallback; destination?: string }, - ): Promise<{ path: string; bytes: number; cached: boolean }> { + ): Promise<{ + path: string; + bytes: number; + videoPath?: string; + cached: boolean; + }> { const file = this.getFile(fileID); if (!file) throw new Error(`content cache: unknown file ${fileID}`); + const isLivePhoto = + kind === "original" && file.metadata.fileType === "livePhoto"; const known = kind === "original" ? this.originals : this.thumbnails; const cached = known.get(fileID); if (cached !== undefined) { - const size = fileSize(cached); - if (size !== undefined && size > 0) { + const size = fileSize(cached.path); + if ( + size !== undefined && + size > 0 && + (!isLivePhoto || hasContent(cached.videoPath)) + ) { // Returning an original's path is a use: bump its mtime so LRU // order reflects it and survives a restart with no ledger. if ( kind === "original" && - dirname(cached) === this.originalsDir + dirname(cached.path) === this.originalsDir ) - await this.touch(cached); - return { path: cached, bytes: size, cached: true }; + await this.touch(cached.path); + return { ...cached, bytes: size, cached: true }; } - // A recorded file that has since gone re-fetches below. + // A recorded file that has since gone, or a live photo an earlier + // version stored as one ZIP, re-fetches below. known.delete(fileID); } // An original a backup already stored counts as present. if (kind === "original" && this.downloadDirectory !== undefined) { - const backupPath = join( - this.downloadDirectory, - "originals", - originalName(file), + const stored = storedOriginal( + join(this.downloadDirectory, "originals"), + file, ); - const size = fileSize(backupPath); - if (size !== undefined && size > 0) { - this.originals.set(fileID, backupPath); - return { path: backupPath, bytes: size, cached: true }; + if (stored !== undefined) { + this.originals.set(fileID, stored); + return { + ...stored, + bytes: fileSize(stored.path) ?? 0, + cached: true, + }; } } @@ -508,21 +616,39 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { ? this.beginOriginalWrite(fileID) : null; try { - await this.download(file, dest, kind, opts?.onByte); - await chmod(dest, FILE_MODE); - const size = (await stat(dest)).size; - if (size === 0) { - throw new Error( - `content cache: ${kind} ${fileID} stored empty`, - ); + const stored = await this.download( + file, + dest, + kind, + opts?.onByte, + ); + for (const path of [stored.path, stored.videoPath]) { + if (path === undefined) continue; + await chmod(path, FILE_MODE); + if ((await stat(path)).size === 0) { + throw new Error( + `content cache: ${kind} ${fileID} stored empty`, + ); + } } - known.set(fileID, dest); + // A backup records its own live photos. + if ( + stored.videoPath !== undefined && + opts?.destination === undefined + ) { + await writeLivePhoto(dir, fileID, { + path: stored.path, + videoPath: stored.videoPath, + }); + } + known.set(fileID, stored); // A fresh original may have crossed the limit; make room by // evicting least-recently-used originals. An over-budget // fetch keeps the file it returns, and no overlapping // sibling is evicted. Thumbnails are never bounded. if (write) await this.enforceOriginalsLimit(write); - return { path: dest, bytes: size, cached: false }; + const size = (await stat(stored.path)).size; + return { ...stored, bytes: size, cached: false }; } finally { if (write) this.inFlightOriginals.delete(write); } @@ -531,18 +657,24 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { ); } + // Fetch into `destination`, returning where the bytes landed: there, or + // for a live photo, its image and video beside it. private async download( file: EnteFile, destination: string, kind: Kind, onProgress: ProgressCallback | undefined, - ): Promise<number> { + ): Promise<{ path: string; videoPath?: string }> { const args = { file, destination, onProgress }; - const result = - kind === "original" - ? await this.source.original(args) - : await this.source.thumbnail(args); - return result.bytesWritten; + if (kind === "thumbnail") { + await this.source.thumbnail(args); + return { path: destination }; + } + const result = await this.source.original(args); + return { + path: result.path ?? destination, + videoPath: result.videoPath, + }; } // Best-effort bump of a file's mtime to now; a failed touch must never fail @@ -553,13 +685,14 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { } // Every stored original that lives under `originalsDir` (a backup-directory - // hit recorded in the map is excluded), with its size and mtime. Entries - // whose file has vanished are dropped from the map. Backups and thumbnails - // are never counted. + // hit recorded in the map is excluded), with its size and mtime; a live + // photo's size includes its video. Entries whose file has vanished are + // dropped from the map. Backups and thumbnails are never counted. private async measureOriginals(): Promise<{ entries: { fileID: number; path: string; + videoPath?: string; size: number; mtimeMs: number; }[]; @@ -568,21 +701,28 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { const entries: { fileID: number; path: string; + videoPath?: string; size: number; mtimeMs: number; }[] = []; let used = 0; - for (const [fileID, path] of this.originals) { + for (const [fileID, { path, videoPath }] of this.originals) { if (dirname(path) !== this.originalsDir) continue; try { const s = await stat(path); + const size = + s.size + + (videoPath === undefined + ? 0 + : (await stat(videoPath)).size); entries.push({ fileID, path, - size: s.size, + videoPath, + size, mtimeMs: s.mtimeMs, }); - used += s.size; + used += size; } catch { this.originals.delete(fileID); } @@ -646,6 +786,18 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { for (const e of evictable) { if (remaining <= limit) break; await rm(e.path, { force: true }); + // A live photo goes whole: its video and the JSON file + // naming the two go with its image. + if (e.videoPath !== undefined) { + await rm(e.videoPath, { force: true }); + await rm( + join( + this.originalsDir, + livePhotoFileName(e.fileID), + ), + { force: true }, + ); + } this.originals.delete(e.fileID); remaining -= e.size; } @@ -670,7 +822,10 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { await chmod(dir, DIR_MODE); } - private async scan(dir: string, into: Map<number, string>): Promise<void> { + private async scan( + dir: string, + into: Map<number, { path: string; videoPath?: string }>, + ): Promise<void> { // Another process sharing this cache may still be writing its temp // files, so only those whose process has exited are removed. removeLeftoverTempFiles(dir); @@ -680,10 +835,17 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { } catch { return; } + const names = new Set(entries); for (const name of entries) { const id = fileIDFromName(name); const path = join(dir, name); - if (id !== undefined && existsSync(path)) into.set(id, path); + if (id === undefined || !existsSync(path)) continue; + // A live photo's image and video are one entry, as the JSON file + // beside them names them. + const livePhoto = names.has(livePhotoFileName(id)) + ? readLivePhoto(dir, id) + : undefined; + into.set(id, livePhoto ?? { path }); } } } diff --git a/src/library/read.ts b/src/library/read.ts index 7443ead..09b95cf 100644 --- a/src/library/read.ts +++ b/src/library/read.ts @@ -80,7 +80,8 @@ export class Photo { } // Fetch and cache the full-resolution original, returning its on-disk path - // and byte length. Served from the cache (or the backup download directory) + // and byte length; for a live photo, its image's, and its video's path as + // `videoPath`. Served from the cache (or the backup download directory) // when already present, otherwise fetched through the content pool. async original(opts?: ContentOptions): Promise<ContentResult> { return this.contentOrThrow().original(this.rec.fileID, opts); diff --git a/src/library/records.ts b/src/library/records.ts index f71aa2f..c06681f 100644 --- a/src/library/records.ts +++ b/src/library/records.ts @@ -42,6 +42,7 @@ export interface PhotoRecord { isArchived: boolean; isHidden: boolean; // Local cache paths, set once a later phase caches the bytes; unset here. + // A live photo's `originalPath` is its image. thumbnailPath?: string; originalPath?: string; } diff --git a/src/metadata-backup.ts b/src/metadata-backup.ts index 6021f87..2bee63e 100644 --- a/src/metadata-backup.ts +++ b/src/metadata-backup.ts @@ -128,7 +128,8 @@ export const extractImageMetadata = ( // Read a file's original bytes through the library's content cache and extract // its embedded image metadata. The bytes come from `photo.original()` — the // same on-disk cache the rest of the library fills — rather than a fresh -// per-call download to a throwaway temp file. +// per-call download to a throwaway temp file. For a live photo, its `path` is +// the image. const extractExif = async ( photo: Photo, ): Promise<Record<string, unknown> | undefined> => { diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 7bcf3bd..7a6b83f 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -53,6 +53,13 @@ import { Library } from "../../src/library/index.js"; import type { ContentSource } from "../../src/library/content.js"; import type { CollectionsPage, FilesPage } from "../../src/client.js"; import type { Collection, EnteFile } from "../../src/model/types.js"; +import { + asLivePhoto, + cdnSource, + IMAGE, + livePhotoZip, + VIDEO, +} from "../live-photo.js"; // `open` and `rename` are wrapped to record, in order, every fsync and rename, // so a test can pin the sequence "fsync the temp file, rename, fsync the @@ -740,6 +747,28 @@ describe("the refresh before a backup", () => { }); }); +// Every entry under collections/, one level of directories deep, with each +// symlink's target. +const tree = (outDir: string): string[] => { + const lines: string[] = []; + const list = (dir: string, prefix: string): void => { + for (const name of readdirSync(dir).sort()) { + const path = join(dir, name); + const st = lstatSync(path); + if (st.isSymbolicLink()) { + lines.push(`${prefix}${name} -> ${readlinkSync(path)}`); + } else if (st.isDirectory() && prefix === "") { + lines.push(`${name}/`); + list(path, `${name}/`); + } else { + lines.push(`${prefix}${name}`); + } + } + }; + list(join(outDir, "collections"), ""); + return lines; +}; + // The album folders under collections/, driven through `runBackup` with a // stand-in library whose albums a test changes between runs. describe("backup album folders", () => { @@ -763,28 +792,6 @@ describe("backup album folders", () => { }, }); - // Every entry under collections/, one level of directories deep, with each - // symlink's target. - const tree = (outDir: string): string[] => { - const lines: string[] = []; - const list = (dir: string, prefix: string): void => { - for (const name of readdirSync(dir).sort()) { - const path = join(dir, name); - const st = lstatSync(path); - if (st.isSymbolicLink()) { - lines.push(`${prefix}${name} -> ${readlinkSync(path)}`); - } else if (st.isDirectory() && prefix === "") { - lines.push(`${name}/`); - list(path, `${name}/`); - } else { - lines.push(`${prefix}${name}`); - } - } - }; - list(join(outDir, "collections"), ""); - return lines; - }; - const albumID = (outDir: string, jsonName: string): number => JSON.parse(readFileSync(join(outDir, "collections", jsonName), "utf-8")) .id; @@ -1054,3 +1061,190 @@ describe("backup album folders", () => { ).toBe(json); }); }); + +// A live photo, which Ente stores as one ZIP, is backed up as its image and +// its video, which a photo viewer can open, beside a JSON file naming them, +// and its album folder links both. These tests download a live photo ZIP +// through the real download layer (test/live-photo.ts). +describe("backup of live photos", () => { + // An account of one album, Trip (10), holding `files`. + class TripClient extends MockClient { + constructor(private readonly files: EnteFile[]) { + super(); + } + override async collectionsSince(): Promise<CollectionsPage> { + return { + collections: [collection(10, "Trip")], + deleted: [], + cursor: 1, + }; + } + override async filesSince(): Promise<FilesPage> { + return { files: this.files, deleted: [], cursor: 1 }; + } + } + + const open = (files: EnteFile[], bodies: Map<number, Uint8Array>) => + openLibrary(cdnSource(bodies), new TripClient(files)); + + // What an earlier version stored for live photo 500: the ZIP under the + // image's name, and its link. + const earlierZIP = (outDir: string): void => { + mkdirSync(join(outDir, "originals"), { recursive: true }); + mkdirSync(join(outDir, "collections", "Trip"), { recursive: true }); + writeFileSync(join(outDir, "originals", "500.HEIC"), livePhotoZip()); + symlinkSync( + "../../originals/500.HEIC", + join(outDir, "collections", "Trip", "IMG_0500.HEIC"), + ); + }; + + const stored = ["500.heic", "500.json", "500.livephoto.json", "500.mov"]; + const linked = [ + "Trip/", + "Trip/IMG_0500.heic -> ../../originals/500.heic", + "Trip/IMG_0500.mov -> ../../originals/500.mov", + "Trip.json", + ]; + + it("stores a live photo as its image and its video and links both", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + const outDir = join(root, "backup"); + const originals = join(outDir, "originals"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ downloaded: 1, failed: 0 }); + expect(readdirSync(originals).sort()).toEqual(stored); + expect(readFileSync(join(originals, "500.heic"))).toEqual( + Buffer.from(IMAGE), + ); + expect(readFileSync(join(originals, "500.mov"))).toEqual( + Buffer.from(VIDEO), + ); + expect(tree(outDir)).toEqual(linked); + + // Both parts are there, so the next run fetches nothing. + const second = await lib.backup({ downloadDirectory: outDir }); + expect(second).toMatchObject({ downloaded: 0, skipped: 1, failed: 0 }); + await lib.close(); + }); + + it("gives both links of each live photo their own names when titles clash", async () => { + const a = await asLivePhoto(file(500, 10, "IMG_0001.HEIC")); + const b = await asLivePhoto(file(501, 10, "IMG_0001.HEIC")); + const lib = await open( + [a.file, b.file], + new Map([ + [500, a.body], + [501, b.body], + ]), + ); + const outDir = join(root, "backup"); + + await lib.backup({ downloadDirectory: outDir }); + + expect(tree(outDir)).toEqual([ + "Trip/", + "Trip/IMG_0001 (500).heic -> ../../originals/500.heic", + "Trip/IMG_0001 (500).mov -> ../../originals/500.mov", + "Trip/IMG_0001 (501).heic -> ../../originals/501.heic", + "Trip/IMG_0001 (501).mov -> ../../originals/501.mov", + "Trip.json", + ]); + await lib.close(); + }); + + it("replaces the ZIP an earlier version stored, and its link", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const outDir = join(root, "backup"); + earlierZIP(outDir); + const lib = await open([live], new Map([[500, body]])); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ downloaded: 1, failed: 0 }); + expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored); + expect(tree(outDir)).toEqual(linked); + await lib.close(); + }); + + it("stores nothing for a live photo that fails its hash, and keeps what was there", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + livePhotoZip(), + "not:the recorded hash", + ); + const outDir = join(root, "backup"); + earlierZIP(outDir); + const lib = await open([live], new Map([[500, body]])); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ downloaded: 0, failed: 1 }); + expect(result.errors.map((e) => e.fileID)).toEqual([500]); + expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]); + expect(readdirSync(join(outDir, "originals"))).toEqual(["500.HEIC"]); + expect(tree(outDir)).toEqual([ + "Trip/", + "Trip/IMG_0500.HEIC -> ../../originals/500.HEIC", + "Trip.json", + ]); + await lib.close(); + }); + + it("copies both parts of a live photo the cache already holds", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + const cached = await lib.photos.byID({ fileID: 500 })!.original(); + const outDir = join(root, "backup"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ downloaded: 1, failed: 0 }); + expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored); + expect(readFileSync(join(outDir, "originals", "500.mov"))).toEqual( + Buffer.from(VIDEO), + ); + expect(tree(outDir)).toEqual(linked); + // The cache keeps its own copy. + expect(existsSync(cached.videoPath!)).toBe(true); + await lib.close(); + }); + + it("serves a live photo the backup stored to a library reading the backup", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + const outDir = join(root, "backup"); + await lib.backup({ downloadDirectory: outDir }); + await lib.close(); + + // Another cache over the same backup, whose server has nothing. + const reader = await Library.open({ + client: new TripClient([live]), + cacheDirectory: join(root, "other-cache"), + downloadDirectory: outDir, + contentSource: cdnSource(new Map()), + refreshIntervalSeconds: 3600, + precacheThumbnails: false, + precacheOriginals: false, + }); + const read = await reader.photos.byID({ fileID: 500 })!.original(); + + expect(read).toEqual({ + path: join(outDir, "originals", "500.heic"), + videoPath: join(outDir, "originals", "500.mov"), + bytes: IMAGE.length, + }); + await reader.close(); + }); +}); diff --git a/test/cli/commands.test.ts b/test/cli/commands.test.ts index 7585335..5b441e5 100644 --- a/test/cli/commands.test.ts +++ b/test/cli/commands.test.ts @@ -12,6 +12,7 @@ import { existsSync, + mkdirSync, mkdtempSync, readdirSync, readFileSync, @@ -22,6 +23,7 @@ import { import { join } from "node:path"; import { tmpdir } from "node:os"; import { PassThrough } from "node:stream"; +import * as jpegJs from "jpeg-js"; import { describe, it, @@ -54,6 +56,14 @@ import type { ContentSource } from "../../src/library/content.js"; import type { Collection, EnteFile } from "../../src/model/types.js"; import { init, toBase64 } from "../../src/crypto/index.js"; import { defaultCacheDirectory } from "../../src/library/index.js"; +import { + asLivePhoto, + cdnSource, + IMAGE, + livePhotoHash, + livePhotoZip, + VIDEO, +} from "../live-photo.js"; const USER_ID = 42; @@ -531,6 +541,120 @@ describe("get and get-thumb", () => { }); }); +// A live photo, which Ente stores as one ZIP, is written as its image and its +// video, which a photo viewer can open. The account here is Vacation holding +// one live photo, 300, downloaded through the real download layer +// (test/live-photo.ts). +describe("a live photo", () => { + const livePhotoClient = async (image = IMAGE): Promise<Client> => { + const { file: live, body } = await asLivePhoto( + file(300, 1, "IMG_0300.HEIC"), + livePhotoZip({ "image.heic": image, "video.mov": VIDEO }), + livePhotoHash(image, VIDEO), + ); + const fake = { + whoami: () => ({ email: "cli@example.com", userID: USER_ID }), + collectionsSince: async () => ({ + collections: [collection(1, "Vacation")], + deleted: [], + cursor: 1, + }), + filesSince: async () => ({ files: [live], deleted: [], cursor: 1 }), + contentSource: () => cdnSource(new Map([[300, body]])), + // The ML data request of `backup-metadata`: no file has any. + getApiClient: () => ({ postJSON: async () => ({ data: [] }) }), + }; + return fake as unknown as Client; + }; + + it("get writes its image and video, named after the title with their own extensions", async () => { + const ctx = context(await livePhotoClient()); + const dir = join(root, "cwd"); + mkdirSync(dir); + const previous = process.cwd(); + process.chdir(dir); + try { + expect(await getCommand(ctx, "300", {})).toBe(0); + } finally { + process.chdir(previous); + } + + expect(readdirSync(dir).sort()).toEqual([ + "IMG_0300.heic", + "IMG_0300.mov", + ]); + expect(readFileSync(join(dir, "IMG_0300.heic"))).toEqual( + Buffer.from(IMAGE), + ); + expect(readFileSync(join(dir, "IMG_0300.mov"))).toEqual( + Buffer.from(VIDEO), + ); + expect(stderr.text).toBe( + `${IMAGE.length} bytes -> IMG_0300.heic\n` + + `${VIDEO.length} bytes -> IMG_0300.mov\n`, + ); + }); + + it("get --out writes the image there and the video beside it", async () => { + const out = join(root, "photo.jpg"); + const video = join(root, "photo.mov"); + + expect( + await getCommand(context(await livePhotoClient()), "300", { out }), + ).toBe(0); + + expect(readFileSync(out)).toEqual(Buffer.from(IMAGE)); + expect(readFileSync(video)).toEqual(Buffer.from(VIDEO)); + expect(stderr.text).toBe( + `${IMAGE.length} bytes -> ${out}\n${VIDEO.length} bytes -> ${video}\n`, + ); + }); + + it("get exits 1 and writes nothing when --out has the video's extension", async () => { + const out = join(root, "photo.MOV"); + + expect( + await getCommand(context(await livePhotoClient()), "300", { out }), + ).toBe(1); + + expect(existsSync(out)).toBe(false); + expect(existsSync(join(root, "photo.mov"))).toBe(false); + expect(stderr.text).toBe( + `File 300 is a live photo, and its video would also be written to ${out}\n`, + ); + }); + + it("backup-metadata --exif reads its image", async () => { + // A 4x4 JPEG, whose size can only come from reading the image; the + // ZIP and the video are not JPEGs. + const jpeg = jpegJs.encode( + { data: new Uint8Array(4 * 4 * 4), width: 4, height: 4 }, + 50, + ).data; + const dir = join(root, "dump"); + + expect( + await backupMetadataCommand( + context(await livePhotoClient(new Uint8Array(jpeg))), + dir, + { exif: true }, + ), + ).toBe(0); + + const record = JSON.parse( + readFileSync( + join(dir, "collections", "1-Vacation", "300.json"), + "utf-8", + ), + ); + expect(record.imageMetadata).toMatchObject({ + format: "jpeg", + width: 4, + height: 4, + }); + }); +}); + describe("backup", () => { it("exits 0 and prints a summary when every file is saved", async () => { const dir = join(root, "backup"); diff --git a/test/download/download.test.ts b/test/download/download.test.ts index f9e144d..f3a06b5 100644 --- a/test/download/download.test.ts +++ b/test/download/download.test.ts @@ -55,6 +55,7 @@ import { readFileSync, rmSync, mkdtempSync, + statSync, writeFileSync, } from "node:fs"; import { dirname, join } from "node:path"; @@ -81,6 +82,7 @@ import { writeAtomic, } from "../../src/download/index.js"; import type { EnteFile, FileMetadata } from "../../src/model/types.js"; +import { IMAGE, livePhotoHash, livePhotoZip, VIDEO } from "../live-photo.js"; // --------------------------------------------------------------------------- // Test helpers @@ -106,6 +108,8 @@ import type { EnteFile, FileMetadata } from "../../src/model/types.js"; const renameHook = vi.hoisted(() => ({ calls: [] as { from: string; to: string; sourceExisted: boolean }[], failWith: null as Error | null, + // When set, only a rename to this path fails. + failTo: null as string | null, })); /** @@ -181,7 +185,10 @@ vi.mock("node:fs/promises", async (importOriginal) => { sourceExisted: sourceExists(from), }); durabilityHook.events.push(`rename:${to}`); - if (renameHook.failWith !== null) { + if ( + renameHook.failWith !== null && + (renameHook.failTo === null || renameHook.failTo === to) + ) { throw renameHook.failWith; } await actual.rename(from, to); @@ -216,6 +223,7 @@ beforeEach(() => { hashHook.lengths.length = 0; renameHook.calls.length = 0; renameHook.failWith = null; + renameHook.failTo = null; durabilityHook.events.length = 0; writeHook.writes.length = 0; }); @@ -1642,38 +1650,35 @@ describe.each(entryPoints)("$name progress", ({ name, download }) => { }); }); -describe("downloadFile content hash", () => { - // Node's own BLAKE2b-512 is the reference, so these tests do not depend - // on the code under test to compute what they expect. - const blake2b = (bytes: Uint8Array): string => - createHash("blake2b512").update(bytes).digest("base64"); +// Node's own BLAKE2b-512 is the reference, so the tests below do not depend on +// the code under test to compute what they expect. +const blake2b = (bytes: Uint8Array): string => + createHash("blake2b512").update(bytes).digest("base64"); - // Serve `plaintext` encrypted as file 999 with the given metadata. Four - // responses are scripted so a retried mismatch would show in `requests`. - const setup = (plaintext: Uint8Array, metadata: Partial<FileMetadata>) => { - const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); - const { header, ciphertext } = encryptFileBody(plaintext, key); - const file = buildMockEnteFile(key, header, header); - file.metadata = { ...file.metadata, ...metadata }; - const body = { kind: "body", bytes: ciphertext } as const; - const { fetch, requests } = scriptedCdnFetch(body, body, body, body); - const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } }); - const dir = mkdtempSync(join(testDir, "hash-")); - const outPath = join(dir, "f.bin"); - return { - run: () => downloadFile(api, file, outPath), - dir, - outPath, - requests, - }; +// Serve `plaintext` encrypted as file 999 with the given metadata, to be +// written to `f.bin` in a fresh directory. Four responses are scripted so a +// retried failure would show in `requests`. +const setup = (plaintext: Uint8Array, metadata: Partial<FileMetadata>) => { + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptFileBody(plaintext, key); + const file = buildMockEnteFile(key, header, header); + file.metadata = { ...file.metadata, ...metadata }; + const body = { kind: "body", bytes: ciphertext } as const; + const { fetch, requests } = scriptedCdnFetch(body, body, body, body); + const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } }); + const dir = mkdtempSync(join(testDir, "hash-")); + const outPath = join(dir, "f.bin"); + return { + run: () => downloadFile(api, file, outPath), + api, + file, + dir, + outPath, + requests, }; +}; - const livePhotoZip = zipSync({ - "image.heic": patternBytes(500, 81), - "video.mov": patternBytes(900, 82), - }); - const livePhotoHash = `${blake2b(patternBytes(500, 81))}:${blake2b(patternBytes(900, 82))}`; - +describe("downloadFile content hash", () => { it("stores a file whose hash matches", async () => { const plaintext = patternBytes(700, 80); const t = setup(plaintext, { hash: blake2b(plaintext) }); @@ -1706,17 +1711,18 @@ describe("downloadFile content hash", () => { }); it("stores a live photo whose image and video hashes match", async () => { - const t = setup(livePhotoZip, { + const t = setup(livePhotoZip(), { fileType: "livePhoto", - hash: livePhotoHash, + hash: livePhotoHash(), }); await t.run(); - expectSameBytes(readFileSync(t.outPath), livePhotoZip); + expect(readFileSync(join(t.dir, "f.heic"))).toEqual(Buffer.from(IMAGE)); + expect(readFileSync(join(t.dir, "f.mov"))).toEqual(Buffer.from(VIDEO)); }); - it("hashes a large live photo entry as it decompresses, never whole", async () => { + it("writes a large live photo entry as it decompresses, never whole", async () => { // 64 MiB of zeros deflates to a few kilobytes, the shape of a ZIP // that would exhaust memory if expanded whole. const image = new Uint8Array(64 * 1024 * 1024); @@ -1724,40 +1730,45 @@ describe("downloadFile content hash", () => { const zip = zipSync({ "image.heic": image, "video.mov": video }); const t = setup(zip, { fileType: "livePhoto", - hash: `${blake2b(image)}:${blake2b(video)}`, + hash: livePhotoHash(image, video), }); await t.run(); - expectSameBytes(readFileSync(t.outPath), zip); + expect(statSync(join(t.dir, "f.heic")).size).toBe(image.length); + expectSameBytes(readFileSync(join(t.dir, "f.mov")), video); const hashed = hashHook.lengths.reduce((a, b) => a + b, 0); expect(hashed).toBe(image.length + video.length); expect(Math.max(...hashHook.lengths)).toBeLessThanOrEqual( 2 * STREAM_CHUNK_SIZE, ); + const written = writeHook.writes.filter((w) => w.path.endsWith(".tmp")); + expect(Math.max(...written.map((w) => w.length))).toBeLessThanOrEqual( + 2 * STREAM_CHUNK_SIZE, + ); }); - it("rejects a live photo whose hash does not match", async () => { + it("rejects a live photo whose hash does not match, keeping what was there", async () => { // The whole ZIP's hash is not the recorded one: each part is hashed. - const t = setup(livePhotoZip, { - fileType: "livePhoto", - hash: blake2b(livePhotoZip), - }); + const zip = livePhotoZip(); + const t = setup(zip, { fileType: "livePhoto", hash: blake2b(zip) }); + writeFileSync(t.outPath, "an earlier download"); await expect(t.run()).rejects.toThrow( /file 999: content hash .* does not match/, ); - expect(readdirSync(t.dir)).toEqual([]); + expect(readdirSync(t.dir)).toEqual(["f.bin"]); + expect(readFileSync(t.outPath, "utf-8")).toBe("an earlier download"); }); it("rejects a live photo that is not a readable ZIP and does not retry", async () => { // Bytes 8-9 of a ZIP entry's local header name its compression // method; 99 is one no reader knows, so the entry cannot be read. - const zip = livePhotoZip.slice(); + const zip = livePhotoZip(); zip[8] = 99; zip[9] = 0; - const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash }); + const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() }); await expect(t.run()).rejects.toThrow( /file 999: live photo is not a readable ZIP/, @@ -1768,8 +1779,8 @@ describe("downloadFile content hash", () => { }); it("rejects a live photo ZIP with no image entry", async () => { - const zip = zipSync({ "video.mov": patternBytes(900, 82) }); - const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash }); + const zip = livePhotoZip({ "video.mov": VIDEO }); + const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() }); await expect(t.run()).rejects.toThrow( /file 999: live photo ZIP does not hold both an image and a video/, @@ -1779,8 +1790,8 @@ describe("downloadFile content hash", () => { }); it("rejects a live photo ZIP with no video entry", async () => { - const zip = zipSync({ "image.heic": patternBytes(500, 81) }); - const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash }); + const zip = livePhotoZip({ "image.heic": IMAGE }); + const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() }); await expect(t.run()).rejects.toThrow( /file 999: live photo ZIP does not hold both an image and a video/, @@ -1789,3 +1800,99 @@ describe("downloadFile content hash", () => { expect(readdirSync(t.dir)).toEqual([]); }); }); + +// --------------------------------------------------------------------------- +// Live photos +// +// A live photo arrives as a ZIP of its image and its video. It is written as +// those two files, which a photo viewer can open, each named after the +// destination with its own extension from the ZIP, the way Ente's clients name +// them when they save one. +// --------------------------------------------------------------------------- + +describe("downloadFile live photos", () => { + const livePhoto = { fileType: "livePhoto", hash: livePhotoHash() } as const; + + it("names the image and the video after the title when no outPath is given", async () => { + const t = setup(livePhotoZip(), { + ...livePhoto, + title: "IMG_1234.HEIC", + }); + + const result = await inDirectory(t.dir, () => + downloadFile(t.api, t.file), + ); + + // `bytesWritten` is the length of the decrypted ZIP. + expect(result).toEqual({ + path: "IMG_1234.heic", + videoPath: "IMG_1234.mov", + bytesWritten: livePhotoZip().length, + }); + expect(readdirSync(t.dir).sort()).toEqual([ + "IMG_1234.heic", + "IMG_1234.mov", + ]); + }); + + it("gives each part its own extension from the ZIP, letters and digits only", async () => { + const zip = livePhotoZip({ "image.JPG": IMAGE, "video.m-4v": VIDEO }); + const t = setup(zip, livePhoto); + + const result = await downloadFile(t.api, t.file, join(t.dir, "f.HEIC")); + + expect(result.path).toBe(join(t.dir, "f.JPG")); + expect(result.videoPath).toBe(join(t.dir, "f.bin")); + expect(readdirSync(t.dir).sort()).toEqual(["f.JPG", "f.bin"]); + }); + + it("replaces what was at the destination, such as an earlier ZIP of the two", async () => { + const t = setup(livePhotoZip(), livePhoto); + writeFileSync(t.outPath, livePhotoZip()); + + await t.run(); + + expect(readdirSync(t.dir).sort()).toEqual(["f.heic", "f.mov"]); + }); + + it("renames the image and then the video into place, each from its own temp file", async () => { + const t = setup(livePhotoZip(), livePhoto); + + await t.run(); + + expect( + renameHook.calls.map((c) => [ + dirname(c.from), + c.to, + c.sourceExisted, + ]), + ).toEqual([ + [t.dir, join(t.dir, "f.heic"), true], + [t.dir, join(t.dir, "f.mov"), true], + ]); + }); + + it("stores neither part when the video cannot be renamed into place", async () => { + const t = setup(livePhotoZip(), livePhoto); + renameHook.failWith = new Error("simulated rename failure"); + renameHook.failTo = join(t.dir, "f.mov"); + + await expect(t.run()).rejects.toThrow("simulated rename failure"); + + expect(readdirSync(t.dir)).toEqual([]); + }); + + it("refuses an image and a video with the same extension, storing nothing", async () => { + // On a file system that ignores case, the two would be one file. + const zip = livePhotoZip({ "image.mov": IMAGE, "video.MOV": VIDEO }); + const t = setup(zip, livePhoto); + writeFileSync(t.outPath, "an earlier download"); + + await expect(t.run()).rejects.toThrow( + /file 999: live photo's image and video have the same extension/, + ); + + expect(readdirSync(t.dir)).toEqual(["f.bin"]); + expect(t.requests()).toBe(1); + }); +}); diff --git a/test/library/content.test.ts b/test/library/content.test.ts index 5db599c..5134bde 100644 --- a/test/library/content.test.ts +++ b/test/library/content.test.ts @@ -31,6 +31,8 @@ import { existsSync, writeFileSync, mkdirSync, + readdirSync, + readFileSync, statSync, } from "node:fs"; import { spawnSync } from "node:child_process"; @@ -44,6 +46,13 @@ import { } from "../../src/library/content.js"; import { RequestPools } from "../../src/library/pools.js"; import type { EnteFile } from "../../src/model/types.js"; +import { + asLivePhoto, + cdnSource, + IMAGE, + livePhotoZip, + VIDEO, +} from "../live-photo.js"; const file = (id: number, title = `file-${id}.jpg`): EnteFile => ({ id, @@ -458,3 +467,125 @@ describe("ContentCache.ensureThumbnails", () => { expect(results[1]?.error).toMatch(/unknown file/i); }); }); + +// A live photo's original is two files, its image and its video, which a +// photo viewer can open, and a JSON file naming them: the two are named with +// the extensions from inside the ZIP, so the names alone do not say which is +// which. These tests download a live photo ZIP through the real download +// layer (test/live-photo.ts). +describe("ContentCache live photos", () => { + const originals = (): string => join(cacheDir, "originals"); + + // A cache over the stand-in server, which holds `bodies` by file ID. + const cacheOf = ( + files: EnteFile[], + bodies: Map<number, Uint8Array>, + ): ContentCache => buildCache({ files, source: cdnSource(bodies) }).cache; + + it("stores a live photo as its image and its video and a JSON file naming them", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + const cache = cacheOf([live], new Map([[5, body]])); + await cache.open(); + + const result = await cache.original(5); + + expect(result).toEqual({ + path: join(originals(), "5.heic"), + videoPath: join(originals(), "5.mov"), + bytes: IMAGE.length, + }); + expect(readFileSync(result.path)).toEqual(Buffer.from(IMAGE)); + expect(readFileSync(result.videoPath!)).toEqual(Buffer.from(VIDEO)); + expect(statSync(result.videoPath!).mode & 0o777).toBe(0o600); + expect( + JSON.parse( + readFileSync(join(originals(), "5.livephoto.json"), "utf-8"), + ), + ).toEqual({ image: "5.heic", video: "5.mov" }); + expect(cache.pathsFor(5)).toEqual({ originalPath: result.path }); + }); + + it("serves a stored live photo from disk after the cache is opened again", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + const first = cacheOf([live], new Map([[5, body]])); + await first.open(); + const stored = await first.original(5); + + // This server has nothing, so a fetch would fail. + const second = cacheOf([live], new Map()); + await second.open(); + const events: string[] = []; + const served = await second.original(5, { + onProgress: (e) => events.push(e.status), + }); + + expect(served).toEqual(stored); + expect(events).toEqual(["skipped"]); + }); + + it("replaces a live photo an earlier version stored as a ZIP under the image's name", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + mkdirSync(originals(), { recursive: true }); + writeFileSync(join(originals(), "5.HEIC"), livePhotoZip()); + const cache = cacheOf([live], new Map([[5, body]])); + await cache.open(); + + const result = await cache.original(5); + + expect(result.videoPath).toBe(join(originals(), "5.mov")); + expect(readdirSync(originals()).sort()).toEqual([ + "5.heic", + "5.livephoto.json", + "5.mov", + ]); + }); + + it("evicts a live photo's image, video and JSON file together", async () => { + const a = await asLivePhoto(file(5, "a.HEIC")); + const b = await asLivePhoto(file(6, "b.HEIC")); + const size = IMAGE.length + VIDEO.length; + const cache = new ContentCache({ + pools: new RequestPools(), + source: cdnSource( + new Map([ + [5, a.body], + [6, b.body], + ]), + ), + cacheDirectory: cacheDir, + getFile: (id) => [a.file, b.file].find((f) => f.id === id), + // Room for one live photo, on a disk with plenty free. + cacheOriginalsMaxBytes: size, + freeBelowBytes: 0, + statfs: async () => ({ bsize: 1, bavail: 1e12 }), + }); + await cache.open(); + + await cache.original(5); + await cache.original(6); + + expect(readdirSync(originals()).sort()).toEqual([ + "6.heic", + "6.livephoto.json", + "6.mov", + ]); + expect(cache.originalsStatus().usedBytes).toBe(size); + }); + + it("stores nothing when a live photo does not match its recorded hash", async () => { + const { file: live, body } = await asLivePhoto( + file(5, "IMG_5.HEIC"), + livePhotoZip(), + "not:the recorded hash", + ); + const cache = cacheOf([live], new Map([[5, body]])); + await cache.open(); + + await expect(cache.original(5)).rejects.toThrow( + /file 5: content hash .* does not match/, + ); + + expect(readdirSync(originals())).toEqual([]); + expect(cache.pathsFor(5)).toEqual({}); + }); +}); diff --git a/test/live-photo.ts b/test/live-photo.ts new file mode 100644 index 0000000..1e3f17e --- /dev/null +++ b/test/live-photo.ts @@ -0,0 +1,77 @@ +/** + * Live photo fixtures for the download, content cache, backup and CLI tests. + * + * Ente stores a live photo as one ZIP holding its image and its video, which + * Ente's clients name `image.<ext>` and `video.<ext>`. The ZIP is built here + * with fflate from small fixed bytes and encrypted the way the server serves a + * file under 4 MiB, as one secretstream chunk. `cdnSource` serves it to the + * real download layer, so a test checks what quak stores for a real one. + */ + +import { createHash } from "node:crypto"; +import { zipSync } from "fflate"; + +import { ApiClient } from "../src/api/client.js"; +import { encryptBlob, init, toBase64 } from "../src/crypto/index.js"; +import { + makeDownloadContentSource, + type ContentSource, +} from "../src/library/content.js"; +import type { EnteFile } from "../src/model/types.js"; + +export const IMAGE = new TextEncoder().encode("the still image"); +export const VIDEO = new TextEncoder().encode("the few seconds of video"); + +// A live photo's ZIP; by default the entries an iPhone's live photo gets. +export const livePhotoZip = ( + entries: Record<string, Uint8Array> = { + "image.heic": IMAGE, + "video.mov": VIDEO, + }, +): Uint8Array => zipSync(entries); + +const blake2b = (bytes: Uint8Array): string => + createHash("blake2b512").update(bytes).digest("base64"); + +// The hash Ente's clients record for a live photo: the unkeyed BLAKE2b-512 of +// the image and of the video, each in standard base64, joined by a colon. +export const livePhotoHash = (image = IMAGE, video = VIDEO): string => + `${blake2b(image)}:${blake2b(video)}`; + +// `file` as a live photo whose original is `zip` and whose recorded hash is +// `hash`, and `body`, what the server serves for it: `zip` encrypted under the +// file's key and header. +export const asLivePhoto = async ( + file: EnteFile, + zip = livePhotoZip(), + hash = livePhotoHash(), +): Promise<{ file: EnteFile; body: Uint8Array }> => { + await init(); + const key = new Uint8Array(32).fill(file.id & 0xff); + const { header, ciphertext } = encryptBlob(zip, key); + return { + file: { + ...file, + key, + metadata: { ...file.metadata, fileType: "livePhoto", hash }, + file: { decryptionHeader: toBase64(header) }, + }, + body: ciphertext, + }; +}; + +// A content source that downloads through the real download layer from a +// stand-in server, which serves `bodies` by file ID and a 404 for any other. +export const cdnSource = (bodies: Map<number, Uint8Array>): ContentSource => + makeDownloadContentSource( + new ApiClient({ + fetch: (async (url: string | URL) => { + const fileID = new URL(String(url)).searchParams.get("fileID"); + const body = bodies.get(Number(fileID)); + return body === undefined + ? new Response("not found", { status: 404 }) + : new Response(body); + }) as typeof globalThis.fetch, + retry: { attempts: 1 }, + }), + );