diff --git a/.dockerignore b/.dockerignore index 4beaf77..e2a5e34 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ # Mirrors .gitignore, with one deliberate exception: .gitignore itself stays # in the build context, because prettier 3 reads it as a default ignore file -# and dropping it would change what `make fmt-check` sees inside the image. +# and dropping it would change what the lint phase's prettier check sees. # VCS .git diff --git a/Dockerfile b/Dockerfile index 0caabf2..144308a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,37 +1,69 @@ -# Lint stage — fast feedback on formatting and lint issues +# Lint phase. The linters are invoked directly rather than through `make +# lint` or `script/lint`, which are themselves a docker build and would +# recurse into a daemon that does not exist in a build step. +# # node 22.22.0 on Alpine 3.23.3 (node:22-alpine), 2026-08-09 FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS lint + WORKDIR /app + COPY script/ script/ COPY package.json yarn.lock ./ RUN script/bootstrap -COPY . . -RUN make fmt-check -RUN make lint -# Check stage — the full suite and the build +COPY . . + +RUN yarn run eslint . +RUN yarn run prettier --check . + +# Test phase, same shape and for the same reason. The suite runs without +# verbose output first and is rerun verbosely only if it fails; the timeout +# catches a hung test. +# # node 22.22.0 on Alpine 3.23.3 (node:22-alpine), 2026-08-09 -FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS check -WORKDIR /app +FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS test -# Force BuildKit to run the lint stage before proceeding. Without this the -# two stages run in parallel and a lint failure can lose the race. -COPY --from=lint /app/yarn.lock /dev/null +WORKDIR /app COPY script/ script/ COPY package.json yarn.lock ./ RUN script/bootstrap + COPY . . -# CHECK_EPOCH is a cache buster: without it Docker serves `make check` from -# cache on an unchanged tree, the suite never executes, and the build still -# exits 0. The guard makes an absent argument a hard failure — an unset ARG -# is the empty string, which is a perfectly stable cache key, so a plain -# `docker build .` would otherwise still get the false green. Fail closed. -ARG CHECK_EPOCH -RUN [ -n "$CHECK_EPOCH" ] || exit 1 -RUN make check +# Unlike the template, the suite runs as the image's non-root `node` user: +# root ignores directory permissions, so the tests of a destination that is +# not writable would otherwise fail. vitest writes into /app. +RUN chown -R node:node /app +USER node + +RUN timeout 90 yarn run vitest run --reporter=dot || \ + { echo "--- Rerunning with verbose for details ---"; \ + timeout 90 yarn run vitest run --reporter=verbose; exit 1; } + +# Build stage, and the last stage: a plain `docker build .` names no target +# and so builds this one. Nothing is wanted from the two phases above; the +# copies are what make BuildKit build them first, so this image cannot be +# produced unless lint and test passed. A stage appended after this one +# would drop all three out of a plain build. +# +# node 22.22.0 on Alpine 3.23.3 (node:22-alpine), 2026-08-09 +FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 + +WORKDIR /app + +COPY --from=lint /app/package.json /dev/null +COPY --from=test /app/package.json /dev/null + +COPY script/ script/ +COPY package.json yarn.lock ./ +RUN script/bootstrap + +COPY . . + +# The version is computed on the host and passed in, because +# .dockerignore excludes .git. +ARG VERSION=dev +LABEL org.opencontainers.image.version="${VERSION}" -ARG CHECK_EPOCH -RUN [ -n "$CHECK_EPOCH" ] || exit 1 RUN make build diff --git a/README.md b/README.md index 8df251c..37832c9 100644 --- a/README.md +++ b/README.md @@ -38,34 +38,47 @@ yarn quak get 67890 --out ./photo.jpg yarn quak backup ./my-backup ``` -For library use: +For library use, the primary surface is the cache-backed `Library`: ```ts -import { Client } from "quak"; +import { Client, Library } from "quak"; +// Log in once; the client satisfies the library's client interface. const client = await Client.login({ email: "you@example.com", password: "your-password", }); -for (const c of await client.listCollections()) { - console.log(c.id, c.name); - const files = await client.listFiles(c.id, c.key); - for (const f of files) { - console.log(` ${f.metadata.title} [${f.metadata.fileType}]`); +// Open a cache-backed library. On an empty cache this awaits one server +// refresh; on an existing cache it returns immediately and refreshes in the +// background every `refreshIntervalSeconds` (default 3). +const lib = await Library.open({ client }); + +// Default reads answer synchronously from the local cache — no network. +for (const album of lib.albums.list()) { + console.log(album.collectionID, album.name); + for (const photo of album.photos.list()) { + console.log(` ${photo.title} [${photo.fileType}]`); } } -// Download a file -const files = await client.listFiles(collectionID, collectionKey); -await client.downloadFile(files[0], "./photo.jpg"); +// Fresh reads await a server round-trip and answer with current state. +const { albums } = await lib.fresh(); +console.log(`${albums.list().length} albums as of now`); -// Serialize session for later (consumer handles persistence) -const snapshot = client.toJSON(); -// ... later: -const restored = Client.fromJSON(snapshot); +// Fetch (and cache) one photo's full-resolution bytes. +const photo = lib.photos.byID({ fileID: 12345 }); +if (photo) { + const { path } = await photo.original(); + console.log(`original at ${path}`); +} + +await lib.close(); ``` +The lower-level `Client` (login, session serialization, and the raw +enumeration/download calls) is exported too and documented under Design below. + ## Entrypoints This repository adheres to the @@ -84,33 +97,52 @@ alpine. We provide: - `script/build` — compile the TypeScript sources into `dist/`, then verify that the entrypoints `package.json` declares (`main`, `types`, `bin`) are among the files the compiler wrote, and make the CLI executable (our own extension) -- `script/test` — run the test suite (vitest, hard-capped at 30s where `timeout` - is available, verbose rerun on failure) -- `script/lint` — run eslint and a prettier check +- `script/test` — run the test suite, by building the `test` phase of the + `Dockerfile` (vitest, 90s timeout, verbose rerun on failure); requires docker +- `script/lint` — run eslint and a prettier check, by building the `lint` phase + of the `Dockerfile`; requires docker (see Linting and testing below) - `script/fmt` — format all files with prettier (writes) -- `script/fmt-check` — check formatting (read-only) -- `script/check` — run all checks: `test`, `lint`, `fmt-check` (our own - extension) -- `script/docker` — build the Docker image, tagged via `script/projectname` -- `script/cibuild` — cd to the repo root and run the image build (what CI runs; - the build runs `make fmt-check` and `make lint` in a first stage, then - `make check` and `make build` in a second) +- `script/fmt-check` — check formatting on the host (read-only); standalone, and + not called by `script/check` or `script/precommit`, because `script/lint` + already checks formatting in the container +- `script/check` — run all checks: `test`, `lint` (our own extension) +- `script/docker` — build the image, tagged via `script/projectname` +- `script/cibuild` — build the image (what CI runs); its last stage depends on + the `lint` and `test` phases, so this one build lints, tests and compiles - `script/precommit` — run by the git pre-commit hook (our own extension); runs - `script/lint` and `script/fmt-check` but deliberately not the tests, so the - TDD red-phase commit can land + `script/lint`, which checks both lint and formatting, but deliberately not the + tests, so the TDD red-phase commit can land - `script/install-precommit` — installs the git pre-commit hook (our own extension); `make hooks` shims to it `make hooks` installs the pre-commit hook that runs `script/precommit`. -Both `script/docker` and `script/cibuild` pass -`--build-arg CHECK_EPOCH="$(date +%s)"`. The Dockerfile refuses to build without -it. This is deliberate: on an unchanged tree Docker would otherwise serve the -`make check` layer from cache, so the suite would never run and the build would -still exit 0. A changing epoch invalidates the check and build layers on every -invocation while leaving the dependency layers below them cached, and the -missing-argument guard means a bare `docker build .` fails loudly instead of -quietly reporting a green it did not earn. +### Linting and testing + +Linting and testing are phases of the `Dockerfile`. The `lint` phase copies the +repo into a digest-pinned node image and runs eslint and `prettier --check .`; +the `test` phase does the same with the suite. `script/lint` and `script/test` +each build one phase with `docker build --no-cache --target `. There is +no host lint or test path: docker is required, and that also works where the +docker daemon is remote and bind mounts are impossible. + +The last stage of the `Dockerfile` compiles the package, and it copies a file +from each phase, so it cannot be built unless lint and the tests pass. That is +why `script/cibuild` is a single `docker build`: it runs lint and the tests once +each and then compiles. + +Every `docker build` in `script/` passes `--no-cache`. On an unchanged tree +Docker would otherwise serve the lint and test steps from cache, nothing would +run, and the build would still exit 0. + +The formatting check is part of the `lint` phase, not a step beside it, so +`script/check` and `script/precommit` do not call `script/fmt-check` as well; +that would run prettier a second time over the same tree for the same verdict. +`script/fmt-check` remains as a standalone entrypoint for asking the formatting +question on the host. Its verdict matches the container's: prettier is pinned to +an exact version, installed from `yarn.lock` under `--frozen-lockfile` in both +places, and reads `.gitignore` as its default ignore file — which is why +`.dockerignore` keeps `.gitignore` in the build context. ## Rationale @@ -145,8 +177,9 @@ All work on quak is test-driven. No exceptions. 3. Subsequent commits add the implementation and any refactors needed to make the tests pass. 4. A feature branch can only be merged into `main` when `make check` is green. - `main` is always green. The Dockerfile runs `make check` and `make build`, so - neither a red branch nor one that does not compile can pass CI. + `main` is always green. CI runs `script/cibuild`, which builds the + `Dockerfile`: its `lint` and `test` phases, then the compile, so neither a + red branch nor one that does not compile can pass CI. 5. Tests are the canonical API documentation for this library. Every test file is commented thoroughly enough that a reader who has never seen quak can learn how to use it from the tests alone. Comments explain why a behavior @@ -160,11 +193,11 @@ All work on quak is test-driven. No exceptions. history must still show tests landing before (or with) the matching implementation. 8. The pre-commit hook installed by `make hooks` runs `script/precommit`, which - runs the lint and format checks but not the full `make check`. This is - deliberate so the TDD red-phase commit (failing tests, no implementation yet) - can land. The full `make check` runs as part of the image build, which is - what CI executes via `script/cibuild`, so a red branch still cannot reach - `main`. + runs `script/lint` — eslint and the prettier check, in the container — but + not the tests, and so not the full `make check`. This is deliberate so the + TDD red-phase commit (failing tests, no implementation yet) can land. The + `test` phase is part of the image build, which is what CI executes via + `script/cibuild`, so a red branch still cannot reach `main`. ## Design @@ -181,17 +214,29 @@ quak/ auth/ login flow (SRP + email OTP + TOTP), key unwrap model/ decrypted Collection, File, Metadata types + decrypt fns download/ streaming file/thumbnail download + decryption + library/ the cache-backed Library: metadata store, read + surface, records, content cache, precache, ML data + and search, request pools backup.ts resilient full-account backup with dedup + metadata-backup.ts + backup-metadata: all decrypted metadata as JSON + mldata-fetch.ts fetch + decrypt per-file ML data + filename.ts safe file names from server metadata errors.ts error types shared across layers retry.ts retry classifier + exponential backoff with jitter thumbnails.ts detect + regenerate missing thumbnails client.ts high-level Client class assembled from the above + cli-commands.ts the CLI's commands as functions returning exit codes + cli-output.ts how the CLI prints a file's title and time + cli-read.ts fresh reads for the CLI's read commands + cli-run.ts run a command, print its error, exit with its code + cli-session.ts read the saved session file back into a Client index.ts public library exports bin/ quak.ts CLI entrypoint (commander.js) test/ unit + integration tests (vitest) Makefile - Dockerfile + Dockerfile lint phase, test phase, compile package.json tsconfig.json ``` @@ -263,11 +308,13 @@ Endpoints used: encrypted token plus key attributes. - `POST /users/ott` and `POST /users/verify-email`: email OTP fallback path. - `POST /users/two-factor/verify`: TOTP second factor. +- `POST /users/logout`: end the calling token's session (`quak logout`). - `GET /collections/v2?sinceTime=`: list collections changed since microsecond timestamp; pass 0 for a full enumeration. - `GET /collections/v2/diff?collectionID=&sinceTime=`: list files in a collection; paginate while `hasMore` is true. - `GET https://files.ente.io/?fileID=`: download encrypted file bytes. +- `POST /files/data/fetch`: fetch encrypted ML data for a batch of files. - `POST /files/upload-url`: mint a presigned upload URL (for thumbnail repair). - `PUT /files/thumbnail`: register an uploaded thumbnail's object key. @@ -307,35 +354,41 @@ and a half seconds of waiting. `sleep` and `random` are injectable through the same option, which is how the test suite exercises the whole policy without waiting. -Two deadlines, applied with `AbortSignal.timeout()` and renewed for each -attempt: +Two deadlines, renewed for each attempt: -| Option | Default | Applies to | -| ------------------- | -------- | ------------------------------------------- | -| `requestTimeoutMs` | `30000` | `getJSON`, `postJSON`, `putJSON`, `putFile` | -| `downloadTimeoutMs` | `600000` | file and thumbnail body transfers | +| Option | Default | Applies to | Kind | +| ------------------- | ------- | ------------------------------------------- | ------------------------------------- | +| `requestTimeoutMs` | `30000` | `getJSON`, `postJSON`, `putJSON`, `putFile` | the whole request | +| `downloadTimeoutMs` | `60000` | file and thumbnail downloads | idle: no bytes received for this long | -They are separate because one number cannot serve both: a value short enough to -keep a hung API call from stalling a backup would cancel a legitimate -multi-gigabyte download. The download deadline covers the body, not just the -headers — `getFileStream` returns as soon as headers arrive, so a deadline that -only guarded the initial request would leave the same hang one layer down. +They are different kinds because a download's length depends on the file and the +link: a whole-transfer deadline short enough to catch a hung connection would +cancel a large video on a slow link that is still making progress. The download +deadline restarts every time bytes arrive, so a slow download runs as long as it +keeps moving, and one that stalls is aborted after 60 seconds of silence. It +covers the wait for the headers and the body — `getFileStream` returns as soon +as headers arrive, so a deadline that only guarded the initial request would +leave the same hang one layer down. There is no limit on the total length of a +download. **Non-idempotent requests are not blindly replayed.** `postJSON` and `putJSON` -reach `/users/srp/create-session`, `/users/two-factor/verify` — which consumes -one of a small number of second-factor attempts — and `/files/thumbnail`. They -are retried only on the three failures that establish no TCP connection to the -server ever existed, so no request byte can have been transmitted: `ENOTFOUND` -and `EAI_AGAIN` (name resolution produced no address) and `ECONNREFUSED` (the -peer refused the connection). A 5xx, a mid-flight reset and a deadline are all -left to the caller, because each of them can happen after the server has already -acted. The routing errnos `EHOSTUNREACH`, `ENETUNREACH` and `ENETDOWN` are -excluded for the same reason, despite looking like connect-time failures: on -Linux an ICMP unreachable arriving mid-flight, or a local interface going down -after the request was written, delivers them on an already-established socket. -They stay retryable for the idempotent calls. `putFile` is exempt: a presigned -PUT stores one whole object at one key in one request, so replaying it has no -partial state to damage. +send every `POST` and `PUT` in the endpoint list above; some of them change +server state, and `/users/two-factor/verify` consumes one of a small number of +second-factor attempts. They are retried only when every errno in the error's +`cause` chain is one of the three that establish no TCP connection to the server +ever existed, so no request byte can have been transmitted: `ENOTFOUND` and +`EAI_AGAIN` (name resolution produced no address) and `ECONNREFUSED` (the peer +refused the connection). A 5xx, a mid-flight reset and a deadline are all left +to the caller, because each of them can happen after the server has already +acted. These two do not follow redirects either: a redirect means the server +already received the request, so it is reported as an error and not retried. The +routing errnos `EHOSTUNREACH`, `ENETUNREACH` and `ENETDOWN` are excluded for the +same reason, despite looking like connect-time failures: on Linux an ICMP +unreachable arriving mid-flight, or a local interface going down after the +request was written, delivers them on an already-established socket. They stay +retryable for the idempotent calls. `putFile` is exempt: a presigned PUT stores +one whole object at one key in one request, so replaying it has no partial state +to damage. A download is retried as a whole — request, stream consumption, and decryption — because a socket reset after the response headers have arrived surfaces in the @@ -367,32 +420,81 @@ decides how to persist sessions. `client.toJSON()` returns a `ClientSnapshot` (a plain serializable object with base64-encoded keys) that the consumer can write to disk, a database, or whatever else fits their use case. `Client.fromJSON(snapshot)` restores a -working client from that snapshot without re-authenticating. +working client from that snapshot without re-authenticating; it checks every +field and each key's length first, and throws an error naming the bad field. +`client.logout()` clears the token and zeroes the key buffers in place; every +later call on that client throws. It does not contact the server, so the token +stays valid there and in any saved snapshot; `await client.logoutOnServer()` +first ends the session on the server (`POST /users/logout`). The CLI stores the snapshot at the platform-appropriate data directory via `env-paths`: `~/Library/Application Support/quak/session.json` on macOS, `$XDG_DATA_HOME/quak/session.json` on Linux. The file is written with mode `0600`. The key material is stored in cleartext in the JSON; treat this file as -you would treat the password itself. +you would treat the password itself. A missing file is reported as "not logged +in"; a file that exists but is corrupt is reported as such, naming the bad +field. Both exit with status 1. + +`quak logout` ends the session on the server, so the token in `session.json` +stops working even in a copy of the file, and then deletes the file. If the +server call fails (or the file is corrupt), the file is still deleted, the +command says the server session could not be ended, and it exits with status 1. +It does not delete the cache: it prints the account's cache directory and says +it still holds decrypted data (file keys in `metadata.json`, cached originals +and thumbnails), for the user to delete if they want it gone. ### CLI surface ``` +quak [--cache-dir ] global: local metadata/content cache location quak login interactive or QUAK_EMAIL/QUAK_PASSWORD quak whoami print logged-in account as JSON -quak logout delete saved session +quak logout end the session, delete it quak collections [--json] list all collections quak files --collection [--json] list files in a collection quak get [--out path] [--collection] download and decrypt a file quak get-thumb [--out] [--collection] download and decrypt a thumbnail quak backup [--json] full incremental backup +quak backup-metadata [--exif] dump all decrypted metadata as JSON quak helper list-missing-thumbnails [--json] find files with missing thumbnails quak helper fix-missing-thumbnails [--file ids] generate + upload missing thumbnails ``` -`get` and `get-thumb` search all collections for the file ID when `--collection` -is not specified. All listing and backup commands support `--json` for -machine-readable output. +Every command runs on the same cache-backed library. The read commands — +`collections`, `files`, `get`, `get-thumb`, `backup-metadata`, +`helper list-missing-thumbnails` and `helper fix-missing-thumbnails` — force a +fresh server round-trip before they answer, so they report current account state +rather than whatever the cache last held. If that round-trip fails, the command +prints the error on one line and exits 1. `--cache-dir` overrides where the +cache lives; without it each account gets its own directory under the per-user +cache path. + +`get` and `get-thumb` resolve the file by ID directly, so `--collection` is +accepted for backward compatibility but ignored. For a live photo, `get` writes +its image and its video, each named after the title with its own extension, as +Ente's clients name them (`IMG_0001.heic` and `IMG_0001.mov`). With +`--out PATH`, the image is written to `PATH` and the video beside it, with +`PATH`'s name and the video's extension; a `PATH` with the video's extension is +refused. `backup-metadata --exif` (alias `--all`) additionally downloads each +file to extract full EXIF/IPTC/XMP metadata. The listing and backup commands +support `--json` for machine-readable output. + +`backup-metadata` fetches ML data in requests of up to 200 files. When a request +still fails after its retries, the error is logged, each of its files is written +with the reason in an `mlDataError` field instead of `mlData`, and the dump goes +on. The exit code is non-zero if any ML data request failed. + +`helper fix-missing-thumbnails` regenerates thumbnails for baseline JPEG images +only, because the bundled decoder (`jpeg-js`) decodes only JPEG. A non-JPEG +image (PNG, HEIC) or a video is reported as `skipped` (unsupported format), kept +distinct from a `failed` repair, and does not affect the exit code; a genuine +failure still exits non-zero. The server accepts a new thumbnail only from the +file's owner and only when it is no larger than the thumbnail size it records +for the file. So a file another account owns, in an album shared with you, is +skipped by both thumbnail helpers without being fetched, and the fixer skips a +file whose recorded thumbnail size is 0 or unknown. Otherwise the fixer lowers +the quality and size of the thumbnail until it fits, and skips the file if even +the smallest does not. ### Backup layout @@ -401,26 +503,80 @@ machine-readable output. ``` / originals/ - . actual file content (one per unique file) + . actual file content (one per unique file, + two for a live photo: see below) .json all decrypted metadata for that file + .livephoto.json which of a live photo's two files is which collections/ / -> ../../originals/<fileID>.<ext> (symlink) <name>.json collection metadata + file list + failures.json files that failed and have not yet succeeded ``` +`failures.json` records each failed file with the kind of failure, how many +times it has been tried and when it was last tried. A file leaves it once it +succeeds, or once it is no longer in the library or in the backup's scope. The +library's `lib.backup({ includeThumbnails: true })` also writes +`thumbnails/<fileID>.jpg` beside `originals/`; `quak backup` does not. + +A collection's directory and JSON are named after the collection, and a symlink +after the file's title, both with unsafe characters replaced. When two +collections would get the same name, or two symlinks in one collection the same +name (ignoring case in both), each of them gets its ID added: two albums named +`Trip` become `Trip (10)/` and `Trip (11)/`, and two files titled `IMG_0001.JPG` +become `IMG_0001 (12345).JPG` and `IMG_0001 (12346).JPG`. IDs never change, so a +name stays the same from run to run until such a clash appears or goes away. + +A live photo, which Ente stores as one ZIP of its image and its video, is stored +as those two files, which a photo viewer can open: each is +`originals/<fileID>.<ext>` with the extension it has inside the ZIP (for example +`12345.heic` and `12345.mov`), and `<fileID>.livephoto.json` names the two. The +live photo counts as stored only when both files are present and not empty. Its +album folder links both, each named after the title with that file's extension +(`IMG_0001.heic` and `IMG_0001.mov`). A live photo that an earlier version of +quak stored as the ZIP, under the image's name, is replaced by its two files on +the next run, and the ZIP and its link are removed. + +Each run removes the symlinks into `originals/` that no longer belong in their +collection's directory, and the directories (and JSON) of collections that were +deleted or renamed. Nothing else in `collections/` is touched: a file or a +symlink you put there stays, and a directory that still holds one after its +symlinks are removed stays too, with its JSON. + Each file is downloaded exactly once regardless of how many collections it -appears in. On subsequent runs, existing originals are skipped. If a download -fails, the error is logged and the backup continues with the next file. The exit -code is non-zero if any files failed. +appears in, and written once: straight into `originals/`, with no copy left in +the cache. An original the cache already held is copied from there instead. On +subsequent runs, existing originals are skipped. If a download fails, the error +is logged and the backup continues with the next file. The exit code is non-zero +if any files failed. `quak backup` opens its library with the thumbnail and +originals precache off, so it fetches only what the backup stores. + +Each original is written to a temporary file in the same directory, synced to +disk, and renamed into place, so an original is either complete or absent, even +after a power cut. A downloaded original's temporary file is named +`.quak-<pid>-<random>.tmp`, one copied from the cache +`.quak-backup-<fileID>.<ext>-<pid>-<random>.tmp`. A run that is killed can leave +one of these temporary files behind; the next backup deletes those whose process +is no longer running. The content cache uses the same scheme, and opening a +library deletes the temporary files in the cache whose process is no longer +running, so a download another process has in progress in the same cache is left +alone. The rename replaces whatever was at the destination rather than writing +through it: a symlink there is replaced, not followed, and the new file has the +temporary file's permissions, not those of the file it replaced. ## TODO - [x] Retry policy: no retry on 4xx, exponential backoff on 5xx and network errors -- [ ] Update the API reference section below to match the current implementation +- [x] Update the API reference section below to match the current implementation - [x] `make docker` green -- [ ] Tag `v1.0.0` +- [x] Store live photos in a form a photo viewer can open + (https://git.eeqj.de/sneak/quak/issues/107): unpacked into the image and + the video + +Tagging and releases are decided by sneak alone, and happen only when he +declares one. Future (desktop client, separate repo): @@ -432,25 +588,208 @@ Future (desktop client, separate repo): ## API reference -The API reference section below is from an earlier draft and does not fully -reflect the current implementation. The authoritative API documentation is in -the test files, particularly `test/client/usage.test.ts` which is a literate -tutorial walking through every operation. Run `yarn test` to verify the examples -are correct. +The library's primary surface is the cache-backed `Library`; the lower-level +`Client` sits underneath it and is covered by the Design sections above. The +test suite is the canonical, executable documentation — `test/library/` and +`test/client/usage.test.ts` walk every operation, and `yarn test` verifies them. -The key types and their actual signatures can be found in: +### Opening a library +`Library.open(options)` loads the on-disk cache, starts the background refresh +loop, and resolves to a `Library`. On an empty cache it awaits the first +refresh, so it opens onto the account's data whenever the server is reachable; +if that refresh fails, it opens with no data and records the error in +`lib.status()`. On an existing cache it returns immediately and refreshes in the +background, so an unreachable server does not block opening. + +`LibraryOptions`: + +| Option | Default | Meaning | +| ------------------------ | --------------------------- | --------------------------------------------------------------------- | +| `client` | required | the account client (a `Client`, or any `LibraryClient`) | +| `cacheDirectory` | `<XDG cache>/quak/<userID>` | where `metadata.json` and the content cache live | +| `downloadDirectory` | none | backup destination; an original already stored there counts as cached | +| `refreshIntervalSeconds` | `3` | background refresh cadence | +| `precacheThumbnails` | `true` | prefetch every thumbnail, newest first | +| `precacheOriginals` | `true` | prefetch the favorites album and the latest-window originals | +| `precacheOriginalsDays` | `7` | length in days of that latest window | +| `cacheOriginalsMaxBytes` | 100 GiB | hard ceiling on the originals cache | +| `freeBelowBytes` | 50 GiB | free space to protect on the volume; the effective limit adapts down | +| `isOriginalPinned` | none | extra predicate for originals that must never be evicted | +| `pools` | fresh `RequestPools` | the bounded request pools (sets concurrency) | +| `onProgress` | none | refresh/ML/precache progress callback (`RefreshEvent`) | +| `contentSource` | the client's own | override the byte source (mainly for tests) | + +Concurrency is set through `pools`: construct +`new RequestPools({ metadataConcurrency, contentConcurrency, thumbnailConcurrency })` +and pass it. The three pools default to 10 / 5 / 25 (see Request pools below). + +`lib.status()` returns a `LibraryStatus` (collection/file counts, last +refresh/ML times and errors, originals usage and effective limit, precache +progress, and `closed`). `lib.close()` stops the background timer; it is +idempotent, and an in-flight refresh is left to finish. The promise it returns +resolves once that refresh (including its cache write), the ML data fetch and +the precache fetches already running have all finished, so the cache directory +can then be removed. + +### Default reads vs. fresh reads + +Default reads — `lib.albums`, `lib.photos`, `lib.timeline` — answer +synchronously from the last refreshed copy held in RAM and never touch the +network. The background timer refreshes that copy every +`refreshIntervalSeconds`, so a default read is immediate but may be up to one +interval stale. + +`await lib.fresh()` forces a refresh, waits for it to complete and persist, and +returns the same `{ albums, photos, timeline }` namespaces — now guaranteed to +reflect a completed server round-trip. Concurrent `fresh()` calls coalesce onto +one refresh, and a refresh that fails rejects the caller (default reads stay +silent and keep serving the last good copy). The CLI's read commands use fresh +reads (issue https://git.eeqj.de/sneak/quak/issues/75). + +### Read surface + +- `lib.albums.list()` → `Album[]`, newest-updated first. + `lib.albums.byID({ collectionID })` and `byName({ albumName })` → + `Album | undefined`. +- `lib.photos.byID({ fileID })` → `Photo | undefined`. + `lib.photos.records({ fileIDs })` → `PhotoRecord[]` in the requested order, + each id once, unknown ids dropped. +- `lib.timeline.groups({ groupBy, filter? })` → `TimelineGroup[]`, grouped by + `"day" | "week" | "month"` (keys `YYYY-MM-DD`, ISO `YYYY-Www`, `YYYY-MM`), + newest group first. A `PhotoFilter` combines `albumID`, `text` + (title/caption/album-name substring), `fileTypes`, `hasLocation`, and + `includeArchived`; hidden photos are always excluded. + +An `Album` exposes its record fields and `album.photos.list()` → `Photo[]` +(newest first). A `Photo` exposes its record fields, `photo.record()` → +`PhotoRecord`, and two content methods: + +- `await photo.original(opts?)` → `{ path, bytes, videoPath? }` — the + full-resolution file. For a live photo, `path` and `bytes` are its image's and + `videoPath` is its video. +- `await photo.thumbnail(opts?)` → `{ path, bytes }`. + +Both serve from the on-disk content cache when the bytes are present and +otherwise fetch through the pools; `opts.onProgress` reports per-file progress. +They throw when the library was opened without a content source. + +Lower-level accessors that return decrypted model objects (which hold key +material) are also available: `listCollections()`, `getCollection(id)`, +`listFiles(collectionID)`, `getFile(collectionID, fileID)`, and +`getFileByID(fileID)`. + +### Records and change notifications + +The GUI-facing records hold no key material and no binary, so they survive +`structuredClone`/JSON across the Electron IPC boundary: + +- `PhotoRecord`: `fileID`, `albumIDs`, `title`, `takenAt` (milliseconds), + `fileType`, optional `caption` / `width` / `height` / `latitude` / + `longitude`, `isArchived`, `isHidden`, and `thumbnailPath` / `originalPath` + once the bytes are cached (for a live photo, `originalPath` is its image). +- `AlbumRecord`: `collectionID`, `name`, `type`, `isShared`, `updationTime`, and + `fileIDs` (newest first). +- `LibrarySnapshot`: `{ albums, photos, takenAt }`. + +`lib.snapshot()` returns a `LibrarySnapshot` (albums newest-updated first, +photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange` +(`albumsChanged`, `photosChanged`, `fileIDsRemoved`, `albumIDsRemoved`, +`refreshedAt`) whenever a refresh alters the projection, and returns +`{ unsubscribe }`; a refresh that changes nothing delivers nothing. + +### Thumbnails, ML search, and backup + +- `lib.thumbnails.ensure({ fileIDs, priority, signal?, onProgress? })` + prefetches thumbnails through the thumbnail pool, deduped by fileID, returning + one `EnsureResult` (`{ fileID, path?, error? }`) per file. `priority` is + `"visible" | "ahead" | "background"`; only `"visible"` preempts background + work. +- `lib.mldata` searches the CLIP index built from Ente's per-file ML data: + `forFile({ fileID })` → `Promise<MLData | undefined>` (the whole stored + payload — face boxes, landmarks, embedding — read from disk on demand); + `similar({ fileID, limit? })` and `searchByEmbedding({ embedding, limit? })` → + `SimilarResult[]` (`{ fileID, score }`, cosine similarity, most similar first, + default limit 20). quak bundles no text encoder, so `searchByEmbedding` takes + a query vector the caller produced elsewhere. +- `await lib.backup(opts?)` → `BackupResult`. It refreshes, fetches every + in-scope original not already in the backup (and, with `includeThumbnails`, + thumbnails) through the content cache, and rebuilds the on-disk backup tree + with a durable failure ledger. A fetched original is written straight into the + backup's `originals/` and not into the cache, which then counts it as present; + one the cache already held is copied from there. `BackupOptions`: + `downloadDirectory` (falls back to the one `open()` was given), + `includeOriginals` (default `true`), `includeThumbnails` (default `false`), + `onlyAlbumNames`, and `onProgress`. See Backup layout above for the tree it + writes. + +### Request pools + +`RequestPools` holds three independent bounded pools — metadata (10), content +(5), thumbnails (25) — because Ente meters these traffic classes differently. +Each pool orders on-demand work ahead of background/precache work and dedups +in-flight fetches by key, and an idle pool never lends its slots to a busy one. + +### On-disk cache layout + +Under `cacheDirectory`: + +``` +<cacheDirectory>/ + metadata.json decrypted account state + refresh cursor + originals/<fileID>.<ext> cached full-resolution files + originals/<fileID>.livephoto.json + which of a live photo's two files is which + thumbnails/<fileID>.jpg cached thumbnails + mldata/ + <fileID>.json one decrypted ML payload per file + clip.f32, clip.json the packed CLIP index and its id list + fetched.json per-file fetch bookkeeping +``` + +When `metadata.json` belongs to a different account than the client's, +`Library.open` deletes it and `mldata/` and starts from an empty cache. Cached +originals and thumbnails are kept; they are reached only through the files the +current account's records name. + +A live photo's original is cached as in the backup: its image and its video, +each `originals/<fileID>.<ext>` with its own extension, and +`originals/<fileID>.livephoto.json` naming them; the two are evicted together. A +live photo that an earlier version cached as its ZIP is not served: the library +removes the ZIP when it opens the cache, and fetches the two files when the +photo is next read or precached. + +A stored file appears only via an atomic temp-then-rename, so its presence means +it is complete. Every downloaded original (by `quak get`, the cache, or +`backup`) whose metadata records a content hash (`FileMetadata.hash`) is hashed +as it is written: unkeyed BLAKE2b with a 64-byte output, standard base64. A live +photo arrives as a ZIP and is unpacked as it is written; its image and its video +are hashed separately and joined as `<imageHash>:<videoHash>`, and neither is +stored unless both are complete and match. A live photo ZIP holding a second +image or video, or whose two files together come to more than 20 times its size +plus 16 MiB, is refused. A mismatch or a refusal stores nothing and fails the +download with an error naming the file ID. An original with no recorded hash, +from a very old client, is stored unchecked. + +### Key types by source file + +- `src/library/index.ts`: `Library`, `LibraryOptions`, `LibraryStatus`, + `LibraryClient`, `RefreshEvent` +- `src/library/read.ts`: `Album`, `Photo`, `AlbumsAPI`, `PhotosAPI`, + `TimelineAPI`, `PhotoFilter`, `TimelineGroup`, `GroupBy` +- `src/library/content.ts`: `ContentResult`, `ContentOptions`, `ThumbnailsAPI`, + `EnsureOptions`, `EnsureResult`, `ContentSource` +- `src/library/records.ts`: `PhotoRecord`, `AlbumRecord`, `LibrarySnapshot`, + `LibraryChange` +- `src/library/mlsearch.ts`: `MLDataAPI`, `SimilarResult` +- `src/library/pools.ts`: `RequestPools`, `RequestPoolsOptions`, `BoundedPool` +- `src/backup.ts`: `BackupOptions`, `BackupResult`, `BackupError` - `src/client.ts`: `Client`, `LoginOptions`, `ClientSnapshot` -- `src/api/client.ts`: `ApiClient`, `ApiClientOptions`, `ApiError`, - `StreamOptions` +- `src/api/client.ts`: `ApiClient`, `ApiClientOptions`, `StreamOptions` - `src/errors.ts`: `ApiError`, `TruncatedStreamError` - `src/retry.ts`: `withRetry`, `isRetryable`, `isSafeToReplay`, `RetryOptions` -- `src/auth/types.ts`: `KeyAttributes`, `SRPAttributes`, - `AuthorizationResponse`, `LoginChallenge` -- `src/model/types.ts`: `Collection`, `EnteFile`, `FileMetadata`, `FileBlob`, - `RawCollection`, `RawEnteFile`, `RawMagicMetadata` -- `src/download/index.ts`: `DownloadResult` -- `src/backup.ts`: `BackupResult`, `BackupError` +- `src/model/types.ts`: `Collection`, `EnteFile`, `FileMetadata`, `FileType`, + `CollectionType`, `RawCollection`, `RawEnteFile` - `src/thumbnails.ts`: `MissingThumbnailInfo`, `ThumbnailFixResult` ## Source attribution @@ -478,16 +817,23 @@ documents: implementation. Tests are the canonical API documentation and must be commented thoroughly. `main` is always green. -- **Required checks before every commit:** `make lint` (eslint + prettier check) - and `make fmt-check` must pass. The pre-commit hook enforces this. - `make check` (which also runs tests) must pass before merging to `main`. +- **Required checks before every commit:** `make lint` must pass — that is + eslint plus the prettier check, and it builds the `lint` phase of the + `Dockerfile`, so it needs docker. The pre-commit hook enforces exactly that. + `make check` (which also runs the tests) must pass before merging to `main`. + `make fmt-check` is available for a host-side formatting check on its own, but + it is not a separate requirement: `make lint` already covers it, and running + both would check formatting twice. Never invoke eslint or prettier directly; + linting runs in the container only. - **Formatting:** prettier with 4-space indents and `proseWrap: always` for markdown. Use `make fmt` to format. Use `yarn` not `npm`. - **Testing:** vitest. Tests go in `test/` mirroring the `src/` structure. - `make test` must complete in under 20 seconds. Use `mkdtempSync` for temporary - directories, never manual timestamp paths. + `make test` must finish in under 60 seconds (the hard cap) and should finish + in under 20. The 90-second `timeout` in the `test` phase of the `Dockerfile` + is a backstop that catches a hung test, not the time limit. Use `mkdtempSync` + for temporary directories, never manual timestamp paths. - **Code style:** `const` for everything, `let` if reassignment is needed, never `var`. Avoid unnecessary comments. No hand-rolled crypto. The diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index bc2f161..2256291 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-07-06 +last_modified: 2026-09-08 --- This document covers repository structure, tooling, and workflow standards. Code @@ -60,17 +60,28 @@ style conventions are in separate documents: prerequisite since nvm requires bash. yarn is then pinned via `corepack prepare yarn@<version> --activate`. Never install "latest" or "lts"; always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root and runs `docker build .`; the Gitea workflow calls it. Four further - scripts are our own extensions to the standard: `script/check` runs - `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is - what the git pre-commit hook runs, and it calls `script/check`; - `script/install-precommit` installs the git pre-commit hook (the `make hooks` - target shims to it); and `script/projectname` (literally that filename) simply - outputs the project's name. Scripts that need the name call - `script/projectname` — e.g. `script/docker` assembles its image tag from it — - so those scripts stay byte-identical across all repos. Repo-type-specific - pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in - `script/precommit`, not in the hook itself. Model scripts are at + repo root, runs `script/bootstrap`, runs `script/check`, and builds the image + with the version; the Gitea workflow calls it. **`script/cibuild` runs + `script/bootstrap` first**, because the workflow checks out the repo and runs + nothing else, while `script/fmt-check` runs the formatter on the host: on a + pristine checkout with nothing installed the run dies there, after the + containerised gates have passed. **The bootstrap alone is not enough**: + `script/bootstrap` installs node and yarn under nvm and leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host + entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore + source nvm for the pinned node version before invoking it, exactly as + `script/bootstrap`'s own install step does. A runner carrying nothing but + docker and git then gets through `script/check`. Four further scripts are our + own extensions to the standard: `script/check` runs `script/test`, + `script/lint` and `script/fmt-check`; `script/precommit` is what the git + pre-commit hook runs, and it calls `script/check`; `script/install-precommit` + installs the git pre-commit hook (the `make hooks` target shims to it); and + `script/projectname` (literally that filename) simply outputs the project's + name. Scripts that need the name call `script/projectname` — e.g. + `script/docker` assembles its image tag from it — so those scripts stay + byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. + `go mod tidy` verification in Go repos) belong in `script/precommit`, not in + the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README must document the provided scripts in an **Entrypoints** section (see the README requirements below). @@ -89,87 +100,140 @@ style conventions are in separate documents: contributor should be able to understand the entire development workflow by reading the Makefile. -- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check` - as a build step so the build fails if the branch is not green. For non-server - repos, the Dockerfile should bring up a development environment and run - `make check`. For server repos, `make check` should run as an early build - stage before the final image is assembled. Dockerfiles install development - prerequisites by running `script/bootstrap` rather than duplicating installs - inline; COPY `script/` and the dependency manifests (`package.json` + - `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap - layer stays cached until dependencies change. +- Every repo should have a `Dockerfile`, and it carries the repo's gates: a + `lint` phase and a `test` phase, with the final stage depending on both so the + image cannot be built unless they pass. For non-server repos the final stage + brings up a development environment; for server repos it is the runtime image. + Dockerfiles install development prerequisites by running `script/bootstrap` + rather than duplicating installs inline; COPY `script/` and the dependency + manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before + running it. -- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go - repos use a multistage build where linting runs in an independent stage based - on the `golangci/golangci-lint` image (pinned by hash). This stage runs - `make fmt-check` and `make lint` before the full build begins. The build stage - then declares an explicit dependency on the lint stage via - `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete - linting before proceeding to compilation and tests. This ensures lint failures - surface in seconds rather than minutes, without blocking on dependency - download or compilation in the build stage. +- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is + no separate lint file. `script/lint` and `script/test` each build one phase + and nothing else: - The standard pattern for a Go repo Dockerfile is: + ```sh + docker build --no-cache --target lint -t "$(script/projectname)-lint" . + docker build --no-cache --target test -t "$(script/projectname)-test" . + ``` + + **A stage that is not the last one in the file is built only when the final + stage's chain depends on it, or when `--target` names it.** That is why the + two gates are always invoked by name here, and why the final stage carries a + `COPY --from=` of a harmless file from each of them: without that edge a + plain `docker build .` builds the last stage alone and exits 0 having linted + and tested nothing. + + **Every `docker build` in `script/` is tagged**, here and in + `script/cibuild` and `script/docker`. An untagged build leaves a dangling + image behind on every invocation, on every developer host and every CI + runner; a tagged one replaces the previous image. + + Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, + `eslint`, `prettier` — never through `make lint` or `script/test`, which are + themselves a `docker build` and would recurse into a daemon that does not + exist in a build step. Formatting is the exception and stays on the host: + `script/fmt` writes the working tree, and `script/fmt-check` is its + read-only twin. + + **No lint verdict may come from a host invocation of the linter.** On a + shared host golangci-lint reads a result cache keyed on file content rather + than location, so a second checkout of the same content is served the first + one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs + exit non-zero with `parallel golangci-lint is running` — a status a caller + cannot tell from real findings. Both have produced wrong verdicts in this + org, in both directions. A container has its own cache, its own `TMPDIR` and + a digest-pinned binary, so neither is reachable. + +- **Any build that runs checks is built with `--no-cache`.** Docker invalidates + a `COPY` layer only when the copied content changes, so on an unchanged tree + the check `RUN` is served from cache, nothing executes, and the build still + exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: + `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the + four, and there is no fifth — `script/check` runs the two gate phases and + `script/fmt-check`, and builds no image of its own. A bare `docker build .` is + not evidence that anything ran: a sub-second build reporting success is a + cache hit, not a result. Never invalidate by pruning — `docker builder prune` + and friends destroy a build cache shared with every other build on the host. + +- **The gate phases are separate stages, and the build stage depends on both.** + The lint phase is based on the `golangci/golangci-lint` image (pinned by + hash), so lint failures surface in seconds rather than after a full compile, + and the test phase is based on the Go image. The canonical Go repo + `Dockerfile`: ```dockerfile - # Lint stage — fast feedback on formatting and lint issues + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make fmt-check - RUN make lint + RUN golangci-lint run --config .golangci.yml ./... - # Build stage + # Test phase # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS test + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN go test -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 90s -race -v ./...; exit 1; } + + # Build stage. Nothing is wanted from either phase above; the copies + # are what make BuildKit build them first, so this stage cannot run + # unless lint and test passed. + # golang:1.x-alpine, YYYY-MM-DD FROM golang@sha256:... AS builder + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null WORKDIR /src - - # Force BuildKit to run the lint stage before proceeding - COPY --from=lint /src/go.sum /dev/null - COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make test ARG VERSION=dev RUN CGO_ENABLED=0 go build -trimpath \ -ldflags="-s -w -X main.Version=${VERSION}" \ -o /app ./cmd/app/ - # Runtime stage + # Runtime stage, and the last one FROM alpine@sha256:... COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` Key points: - - The lint stage uses the `golangci/golangci-lint` image directly (it - includes both Go and the linter), so there is no need to install the - linter separately. - - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates - a stage dependency. BuildKit runs stages in parallel by default; without - this line, the build stage would not wait for lint to finish and a lint - failure might not fail the overall build. + - The lint phase uses the `golangci/golangci-lint` image directly (it has + both Go and the linter), so nothing needs installing. + - `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only + purpose is the ordering edge. BuildKit runs stages in parallel by default, + and a stage nothing depends on is not built at all, so without these two + lines a red gate would not fail the build. + - Keep the runtime stage last, and if you add a stage after it, give it the + same two copies. A plain `docker build .` builds the last stage's chain + and nothing else. - If the project uses `//go:embed` directives that reference build artifacts - (e.g. a web frontend compiled in a separate stage), the lint stage must + (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. - The lint stage should not depend on the actual build output — it exists to - fail fast. - If the project requires CGO or system libraries for linting (e.g. - `vips-dev`), install them in the lint stage with `apk add`. - - The build stage runs `make test` after compilation setup. Tests run in the - build stage, not the lint stage, because they may require compiled - artifacts or heavier dependencies. + `vips-dev`), install them in the lint phase with `apk add`. + - `ARG VERSION=dev` is declared in the stage that compiles and supplied by + `script/docker` and `script/cibuild`; no stage may call `git describe`. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` (which runs `docker build .`) on push. Since the - Dockerfile already runs `make check`, a successful build implies all checks - pass. + runs `script/cibuild` on push, and checks out the repo as its only other step. + That script bootstraps, runs the gate phases, and then builds the image, so a + successful run means every check passed; a bare `docker build .` does not + carry the same guarantee, because its gate phases may come from the cache. The + image build is uncached and so runs the gate phases a second time. That is the + price of the rule above, and it is worth paying: the image that ships is built + from a run of its own gates rather than from a cache entry. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -189,14 +253,21 @@ style conventions are in separate documents: module under test to verify it compiles/parses. There is no excuse for `make test` to be a no-op. -- `make test` must complete in under 20 seconds. Add a 30-second timeout in the - Makefile. +- `make test` must complete in under 60 seconds. That is the hard cap, and a + suite that exceeds it fails. Under 20 seconds is the target. A suite between + 20 and 60 seconds is still green, but the overage must be filed as an + improvement bug against that repo. Add a 90-second timeout to the test + invocation (`go test -timeout 90s`). The backstop deliberately sits above the + hard cap so that it catches a genuinely hung test rather than a merely slow + one. -- **`make test` should use the conditional verbose rerun pattern.** Run tests - without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to - show full output. This keeps CI logs and `docker build` output clean on - success (just package/suite summaries) while providing full diagnostic detail - on failure (every test case, every assertion). The general shell pattern: +- **The test command should use the conditional verbose rerun pattern.** Run + tests without `-v` (verbose) first. If tests fail, automatically rerun with + `-v` to show full output. This keeps CI logs and `docker build` output clean + on success (just package/suite summaries) while providing full diagnostic + detail on failure (every test case, every assertion). The command lives in the + `test` phase of the `Dockerfile`, since `script/test` builds that phase; the + Makefile form below is the same pattern for any repo-local invocation: ```makefile test: @@ -209,11 +280,24 @@ style conventions are in separate documents: ```makefile test: - @go test -timeout 30s -race -cover ./... || \ + @go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 30s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } ``` + `-count=1` is required on both invocations: it defeats Go's test _result_ + cache, so the target cannot report a pass it did not earn, and the rerun + reproduces a failure instead of replaying it. It leaves the build cache + alone, so it costs the runtime of the suite and no recompilation. + + Note that this is a second, independent cache, stacked below the Docker + layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26) + addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes; + it does not guarantee `go test` inside that step does any work, because the + `GOCACHE` baked into earlier image layers survives into the re-executed + step. They are two separate defects requiring two separate fixes, and a fix + for one must not be recorded as covering the other. + Python example: ```makefile @@ -239,10 +323,83 @@ style conventions are in separate documents: must be in `.gitignore`. No exceptions. - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`. - Fetch the standard `.gitignore` from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up - a new repo. + editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), + language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` + from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when + setting up a new repo. These patterns are written to `.gitignore`'s own + semantics, in which an unanchored pattern already matches at every depth; they + are not a `.dockerignore` and must not be transplanted into one unmodified. + +- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns + across unmodified leaves secrets in the build context.** Docker matches with + `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so + `*` does not cross `/` and a pattern without a leading `**/` is anchored at + the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key` + therefore excludes only the copies at the repository root, while `config/.env` + and `certs/server.key` still reach the context and can land in an image layer + — which is more dangerous than a short file with no secret patterns at all, + because it reads as solved and stops anyone looking. Give every + depth-independent pattern the `**/` prefix and leave only genuinely + root-anchored entries unprefixed: `.git`, and the repo's own host-built + binary, written `/myapp` and never `**/myapp`, which would also match + `cmd/myapp/` and delete the package directory from the context. Matching is + case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so + secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`, + and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern + also catches something the build needs, re-include it with a negation + (`!docs/example.env`); deleting the pattern reopens the exposure for every + other file it covers. Fetch the standard `.dockerignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend + it with the repo's own artifacts. + +- **In-repo agent scratch belongs in both files, written to each file's own + semantics.** `.claude/` holds one worktree per in-flight agent — an entire + additional checkout of the repo — so under `COPY . .` the build context + inflates by a multiple of the repo and another session's unreviewed work can + be copied into an image layer. In `.gitignore` the entry is `.claude/`, + unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/` + prefix, because the prefixed form would also delete any nested directory of + that name from the build. Anchoring carries a known gap that the canonical + `.dockerignore` states in its own comment, since consuming repos receive the + file and not the tracker: the directory is created in the agent's working + directory, so a repo running agents in subdirectories still ships + `services/api/.claude/` and must add its own anchored entry there. + +- **Excluding `.git` means `git describe` cannot run inside any build stage, and + it fails quietly there.** In a build stage there is no repository, so + `git describe` writes nothing to stdout, `-X main.Version=` comes out empty, + the binary reports no version at all, and the build still exits 0. Compute the + version on the host and thread it in as a build arg. `script/docker` and + `script/cibuild` do this, byte-identically across repos: + + ```sh + # Own line: a failing command substitution inside an argument does not + # trip `set -e`, so the inline form degrades to an empty constant. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$(script/projectname)" . + ``` + + `--always` makes an untagged repo yield an abbreviated commit hash rather + than failing, and the `[ -n "$version" ]` line is the single place the + fallback is applied — a live check that fires on a build from an export with + no `.git` and on a repository with no commits yet. Do not fold it into the + substitution as `|| echo unknown`, which makes the guard unreachable. The + Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared + there because `ARG` is stage-scoped; passing `VERSION` to a repo whose + Dockerfile declares no such `ARG` is ignored and costs nothing, which is why + the scripts stay byte-identical. One consequence for CI: the standard + checkout action clones shallow and fetches no tags, so a repo that embeds a + tag-derived version must set `fetch-depth: 0` on its checkout step. + +- **Verify `.dockerignore` by enumerating the image, not by reading the + patterns.** Plant files at the root _and_ at least two directories deep, build + a probe image that does `COPY . .`, and list what actually landed + (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context` + size is not a substitute: a nested secret is a few bytes, and BuildKit + transfers only the delta from the previous build. - **No build artifacts in version control.** Code-derived data (compiled bundles, minified output, generated assets) must never be committed to the @@ -258,9 +415,45 @@ style conventions are in separate documents: - Make all changes on a feature branch. You can do whatever you want on a feature branch. -- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only - manually by the user. Fetch from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. +- `.golangci.yml` is standardized. The vendored copy in a consuming repo must + _NEVER_ be modified by an agent: fetch it from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it + byte-identical, so that no repo can quietly loosen its own linting. Linter + configuration changes are made to the canonical copy in the `prompts` repo and + reach consuming repos by re-vendoring; an agent may open a PR against + canonical, which only the user merges. One list is exempt from byte-identity, + because it cannot be written once for every repo: the `deny` list of the + `test-support` depguard rule, where a repo names its own test-support packages + by full import path. A repo adds entries there and changes nothing else, and a + re-vendor carries its entries forward. The canonical golangci-lint version is + v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base + image + (`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`, + which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the + only pin, since no repo installs golangci-lint on the host: bumping the + version means changing it and nothing else. + +- **`script/bootstrap` installs a pinned tool by comparing versions, never by + testing presence.** An `if ! command -v <tool>; then install; fi` guard tests + `PATH` only, so on an already-provisioned machine the pin is inert and a + version bump is a silent no-op — while the Dockerfile, installing into a clean + image, gets the pinned version, so a local `make check` and `make docker` can + disagree about what the tool even is. The canonical form: + - compares the installed version against the pin over the **whole** version + token; a parser that stops at the first `-` reports `2.12.2` for a host + running `2.12.2-rc1` and skips the install; + - treats absent, non-zero, empty or unrecognised `--version` output as a + mismatch, so the failure direction is a redundant install and never a + skipped one; + - after installing, re-resolves the binary the way callers do — `hash -r`, + then through `PATH`, not through the directory the installer wrote to — + and fails naming the resolved path, since an install that a shadowing + binary hides succeeds while changing nothing any caller sees; + - is actually called, and prints the version on both success paths: a + function defined and never invoked has the same exit status and the same + empty output as one that worked. + + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). @@ -379,7 +572,9 @@ style conventions are in separate documents: language-specific config). Everything else goes in a subdirectory. Canonical subdirectory names: - `bin/` — executable scripts and tools - - `cmd/` — Go command entrypoints + - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose + body is a single call into `internal/` or `pkg/`, no project logic in + `cmd/` - `configs/` — configuration templates and examples - `deploy/` — deployment manifests (k8s, compose, terraform) - `docs/` — documentation and markdown (README.md stays in root) diff --git a/TODO.md b/TODO.md index 5671aff..e6c2c8a 100644 --- a/TODO.md +++ b/TODO.md @@ -14,10 +14,294 @@ pre-1.0 # Next Step -Update the README API reference section to match the current implementation. +None: every issue still open is done on `next` and waits for it to reach `main`. + +Tagging and releases are decided by sneak alone, and happen only when he +declares one. # Completed Steps +- 2026-09-28: Tested the live-photo writer's fsyncs (issue 130). A test checks + that the image's and the video's temp files are fsynced before either is + renamed into place, and the directory after both renames, as the `writeAtomic` + test does for one file. + +- 2026-09-23: Live photos are stored as their image and their video (issue 107). + A live photo, which Ente stores as one ZIP, is unpacked as it downloads into + `<fileID>.<ext>` for the image and for the video, each with its extension from + the ZIP, beside `<fileID>.livephoto.json`, which names the two. Both are + checked against the recorded hash and renamed into place only when both are + complete. A ZIP holding a second image or video, or whose two files come to + more than 20 times its size plus 16 MiB, is refused. The backup and the + content cache count the live photo as stored only with both files, the album + folder links both, `quak get` writes both, and the content result gives the + video as `videoPath`. A ZIP an earlier version stored is replaced on the next + backup run, and removed from the cache when the library opens it. + +- 2026-09-23: Settled the package metadata (issue 6). quak is not published, so + `package.json` is marked `"private": true` and the `files` field is gone. + `engines.node` is `>=22`, the major version `script/bootstrap` and the + `Dockerfile` use. An `exports` map makes `.` and `./package.json` the only + importable paths; `runMetadataBackup`, the thumbnail helpers and their types + stay internal to the CLI. + +- 2026-09-23: Brought the README and this file in line with the tree (issue + 111). The layout lists `src/library/` and the other source files, the backup + layout names `failures.json` and the optional `thumbnails/`, "Opening a + library" says what happens when the first refresh fails, the Testing section + gives the 60-second hard cap and 20-second target for `make test` and names + the 90-second `timeout` in the `Dockerfile` as the backstop for a hung test, + and "Tag v1.0.0" is no longer listed as the next step. + +- 2026-09-23: Tested `quak login` and `backup-metadata --exif` (issue 110). + `loginCommand` takes its login function and its prompts from `CliContext`, and + `bin/quak.ts` passes `Client.login` and the terminal prompts. Tests cover a + login from `QUAK_EMAIL` and `QUAK_PASSWORD` with no prompt, the TOTP prompt, a + failed login, and the saved session's modes, and show that `--exif` and + `--all` each turn on EXIF extraction and that it is off without them. + +- 2026-09-23: `quak backup` writes each original once and no longer fills the + cache (issue 106). An original fetched for a backup is written by the download + writer straight into the backup's `originals/`, and the content cache records + it there instead of keeping its own copy; one the cache already held is still + copied. `quak backup` opens its library with the thumbnail and originals + precache off. + +- 2026-09-23: `backup-metadata`, `helper list-missing-thumbnails` and + `helper fix-missing-thumbnails` refresh before they answer (issue 100). Each + awaits `lib.fresh()` before reading, so a file added since the cache was + written is included, and a failed refresh prints one line and exits 1 instead + of answering from a stale or empty cache. The README lists them among the + commands that refresh first. + +- 2026-09-23: `quak backup` waits for the server refresh and fails when it fails + (issue 99). `lib.backup()` joins a refresh already running or starts one, as + `fresh()` does, and rejects before touching any file when it fails, leaving + `failures.json` as it was, so `quak backup` prints the error as one line and + exits 1 instead of backing up the previous run's file list, or nothing, and + exiting 0. + +- 2026-09-23: CLI errors print a message instead of a stack trace (issue 102). + An error a command throws is printed as one `quak: MESSAGE` line on stderr and + the CLI exits 1 once output has drained. The wrapper that does this moved from + `bin/quak.ts` to `src/cli-run.ts`, and `bin/quak.ts` now awaits + `program.parseAsync()`. + +- 2026-09-23: Opening a library no longer deletes another process's download in + progress (issue 105). The download writer's temp files are named + `.quak-<pid>-<random>.tmp`, and `removeLeftoverTempFiles`, moved from the + backup into the download module, deletes a `.quak-*.tmp` file only when the + process ID in its name is no longer running. The content cache calls it at + `open()` for `originals/` and `thumbnails/`, the backup as before. + +- 2026-09-23: Re-vendored the lint and test setup from the template (issue 96). + Linting and testing are the `lint` and `test` phases of the `Dockerfile`; + `script/lint` and `script/test` each build one with `--no-cache`, and the last + stage compiles and depends on both, so `script/cibuild` is one build. + `Dockerfile.lint`, `CHECK_EPOCH`, `LINT_EPOCH` and the tests that checked them + are gone; `REPO_POLICIES.md` is re-copied. + +- 2026-09-23: Stopped `helper fix-missing-thumbnails` retrying files the server + always refuses (issue 109). Both thumbnail helpers skip a file another account + owns without fetching it. The fixer skips a file whose recorded thumbnail size + is 0 or unknown before downloading it, and otherwise tries smaller encodings + (720 px quality 50 down to 160 px quality 20) until the encrypted thumbnail is + no larger than that size, skipping the file if none fits. + +- 2026-09-23: Tested the live-photo hash check's error paths (issue 117). Tests + download a live photo whose ZIP names an unknown compression method, one whose + ZIP has no image entry and one with no video entry, and check that nothing is + stored and the error names the file ID; the unreadable one is not retried. + +- 2026-09-23: `quak logout` ends the session on the server (issue 108). It calls + `POST /users/logout` through the new `Client.logoutOnServer()`, then deletes + `session.json` even when that call fails, says so and exits 1. It prints the + account's cache directory and says it still holds decrypted data. The default + cache path is now `defaultCacheDirectory()` in the library, shared with + `Library.open`. + +- 2026-09-23: Fixed the backup's per-collection folders (issue 103). Two files + in one collection with the same title, and two collections with the same name, + each get their ID added to the name (`IMG_0001 (12345).JPG`, `Trip (10)/`), so + none replaces another's symlink or JSON. Each run removes symlinks into + `originals/` for files no longer in the collection, and the folders of deleted + or renamed collections, leaving anything else in `collections/` alone. The + README backup layout states the naming rule. + +- 2026-09-23: Checked downloaded originals against their recorded content hash + (issue 68). `downloadFile`, which `quak get`, the content cache and backup all + use, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and + stores nothing on a mismatch, failing with an error naming the file ID. A live + photo ZIP is unpacked as it streams with `fflate` and its image and video + hashed separately as `<imageHash>:<videoHash>`. `decryptFile` reads older + clients' `imageHash` and `videoHash` fields for live photos. A file with no + recorded hash is stored unchecked. + +- 2026-09-23: Kept one account's cache from mixing with another's (issue 104). + When `metadata.json` in the cache directory was written for a different, + non-zero user ID than the client's, `Library.open` deletes it and `mldata/` + and starts empty, so the first refresh enumerates from 0. This only happens + with `--cache-dir` or an explicit `cacheDirectory`; the default path already + includes the user ID. A test opens one account's cache as another account. + +- 2026-09-23: `backup-metadata` no longer stops on one failed ML data request + (issue 101). Each request of up to 200 files is tried on its own; a failed one + is logged, its files are written with the reason in `mlDataError`, and the + command exits 1 once the dump is complete. `fetchMLData`, which only this + command used, is gone; the command calls `fetchMLDataBatch` per batch. +- 2026-09-23: Single-sourced the version string (issue 5). `package.json` is the + only place it is written: `src/index.ts` imports it for `VERSION` and + `bin/quak.ts` passes `VERSION` to commander. tsc copies `package.json` to + `dist/package.json`, so the import resolves from the built output too, and + `script/build` runs the built CLI with `--version` to prove it. A test checks + that `VERSION` and `quak --version` both equal the `package.json` version. +- 2026-09-23: Tested that `Library.close()` waits for the originals precache + (issue 93). The test that holds a precache fetch open while `close()` runs now + runs once with only the thumbnail fill and once with only the originals fill, + so dropping either wait from `Precache.close()` fails a test. +- 2026-09-23: Fixed two intermittently failing library tests (issue 90). + `Library.close()` now returns a promise that resolves once an in-flight + refresh (including its cache write), the ML data fetch and running precache + sweeps have finished; the library tests await it, so `afterEach` no longer + removes the cache directory while something is still writing into it. The + precache test waits for both fills to report "done" instead of for its stub + source to be called, which happened before the cache recorded the file. +- 2026-09-23: Pinned three guards reviewers found untested (issue 89). The + download idle deadline's timer is unref'd, so it can never keep the process + alive, and a test checks no timer is left after a download completes or fails. + A test covers the rejection of `#` in a request path. The EXIF scan compares + the `Exif` header only in an APP1 segment of length 8 or more, so it never + reads the next segment's bytes, with a test for a short one. +- 2026-09-23: Made the download deadline an idle deadline (issue 24). + `downloadTimeoutMs` now aborts a file or thumbnail download only after no + bytes have arrived for that long, default 60 seconds, instead of bounding the + whole transfer at 10 minutes, so a slow download that keeps making progress + completes. A download that fails before reading the whole body cancels it, so + a failed file no longer holds its connection. +- 2026-09-23: Every `ApiClient` request URL is now built by one function next to + the class (issue 18), so a self-hosted `apiOrigin` with a base path keeps it + on every request, a path works with or without a leading slash, and query + parameters are percent-encoded. A path containing `?` or `#` is rejected with + an error instead of being silently cut. +- 2026-09-23: Made the CLI testable and tested it (issue 12). The command bodies + moved from `bin/quak.ts` into `src/cli-commands.ts` as functions that take + their options and a context (output streams, session directory, cache + directory, session loader) and return an exit code; `bin/quak.ts` only wires + them to commander and exits with the code once stdout and stderr have drained, + so nothing below it calls `process.exit`. `test/cli/commands.test.ts` drives + them with a fake client: session file modes, logout, the missing and corrupt + session paths, and the output and exit code of `whoami`, `collections`, + `files`, `get`, `get-thumb`, `backup` and `helper list-missing-thumbnails`. +- 2026-09-23: Hardened the backup tree's atomic copy (issue 22). `copyAtomic` + fsyncs its temp file before the rename and the directory after it, through the + download writer's `fsyncPath`; each backup run deletes `.quak-backup-*.tmp` + files whose process is no longer running. The README backup layout names the + temp files and states that the rename replaces a symlink and takes the temp + file's permissions. Added tests for a missing and an unwritable destination + directory for `downloadFile` and `downloadThumbnail`. +- 2026-09-23: Hardened the JPEG EXIF scan behind `backup-metadata --exif` (issue + 11). Every segment length is checked against the remaining bytes and lengths + under 2 stop the scan, so a truncated or corrupt original can neither throw + nor loop. A malformed or unparseable EXIF segment is recorded as + `imageMetadata.exifError`, and a failure to read the original as + `imageMetadataError` in the per-file JSON, instead of the field being left + out. +- 2026-09-22: Hardened the retry classifier (issue 80). A `POST` or `PUT` is + replayed only when every errno in the cause chain is a connect errno, and it + no longer follows redirects. `getRetryOptions()` returns a copy. Tests pin + every errno the classifier names, the cause-chain depth limit, cycle + termination, and a fresh deadline per attempt for every retrying entry point. + The README's endpoint list is the one place that names the requests the replay + rule covers. +- 2026-09-22: Stopped `make test` collecting tests from checkouts nested under + `.claude/` (issue 25). vitest ignores `.gitignore` when finding tests, so a + nested checkout ran the whole suite again; `vitest.config.ts` now adds + `.claude/**` to vitest's default excludes, and + `test/packaging/nested-checkout.test.ts` plants a nested checkout in a temp + directory and fails if vitest would collect it. +- 2026-09-22: Dropped the deprecated `@types/libsodium-wrappers-sumo` stub from + `devDependencies` (issue 27). It shipped no declarations; the types come from + `libsodium-wrappers-sumo` itself. `yarn.lock` regenerated by `yarn remove`. +- 2026-09-22: Hardened the client session lifecycle (issue 10). + `Client.fromJSON` checks every snapshot field and each key's decoded length + and names the bad field; `toJSON` reads the token through + `ApiClient.getAuthToken` and throws when there is none; `logout` zeroes the + key buffers, and `collectionsSince` re-checks for logout after its request so + it never decrypts with zeroed keys. The CLI reports a corrupt session file + separately from a missing one (`src/cli-session.ts`). +- 2026-09-22: Sanitized file names taken from server metadata (issue 9). A new + `src/filename.ts` holds the one sanitizer, used by `quak get`/`get-thumb` + without `--out`, `downloadFile`/`downloadThumbnail` without `outPath`, and the + backup and metadata backup trees; it removes separators, control characters, + leading dots and Windows device names, and falls back to a name built from the + ID for an empty title. Originals-cache extensions are letters and digits only, + else `.bin`. A user-supplied path is used as is. `decryptFile` reads a missing + or non-string title as "" and rejects metadata that is not a JSON object. +- 2026-09-22: Rewrote the README API reference (and the Getting Started / usage + snippets) to match the shipped cache/API library on `next` (issue 53, issue + 13). Documented `Library.open` and its options, the default-read vs `fresh()` + distinction (and that the CLI's read commands are fresh), the record types and + `snapshot()`/`subscribe()`, the `albums`/`photos`/`timeline` read surface, + `Photo` content methods, `thumbnails.ensure`, the `mldata` search surface, + `backup()`, the three request pools (10/5/25), and the on-disk cache layout; + noted the deferred content-hash integrity check (issue 68). Docs-only; no code + changed. +- 2026-09-22: Added resumable, deletion-aware enumeration to `Client` (issue 38, + closes issue 7). `collectionsSince`/`filesSince` take a starting cursor, + decrypt live records, surface tombstoned ids in a separate `deleted` list (a + tombstone has nothing to decrypt, so it is a bare id, not a hollow record), + and return the max `updationTime` seen as the cursor to resume from. + `filesSince` refuses to loop when the diff reports `hasMore` without advancing + the cursor (issue 7). `listCollections`/`listFiles` are now thin wrappers that + enumerate from `sinceTime: 0` and drop deletions, so existing callers are + unaffected. +- 2026-09-22: Carried file size, thumbnail size, and the deletion flag through + `decryptFile` (issue 37, foundation for the cache/API design). Live files now + populate `file.size`/`thumbnail.size` from the server's `info` (left + `undefined` when the server omits it), and `isDeleted` is carried from the + diff row onto `EnteFile`. No caller change: `listFiles` still filters deleted + rows before decrypting. Surfacing a tombstone through decryption belongs to + the enumeration unit (issue 38). +- 2026-08-10: Made `lint-once.test.ts` enforce what its header claims. It walked + `make check` only, so it never read `Dockerfile` — the image CI builds through + `script/cibuild` — and a second `prettier --check .` could be added there with + the suite staying green. The walk now also starts at + `.gitea/workflows/check.yml` and follows its `run:` steps, so the graph under + test is the one CI executes rather than the one someone assumed it executes. + The lockfile assertion was a substring check against the whole of + `script/bootstrap`, which has two install sites and so reported the branch the + containers never take; the two branches are now resolved separately and every + `yarn install` in each is required to be `--frozen-lockfile`. Prettier is + counted per occurrence instead of per line, so two invocations chained with + `&&` no longer read as one, and edges are followed on counted lines instead of + being skipped. Every way for the walk to reach nothing — an unknown target, an + unknown script, a missing file, a node with no commands, an unknown node kind + — is a thrown error rather than a quiet zero. Every assertion in the file was + mutation-tested individually. +- 2026-08-10: Stopped `make check` running `prettier --check .` twice. Since + linting moved into Docker, the duplicate was one container pass and one host + pass of the same check: `script/lint` builds `Dockerfile.lint`, which runs + prettier as a build step, and `script/check` then called `script/fmt-check` as + well. The host call is gone from `script/check` and from `script/precommit`; + the container keeps checking formatting, because a successful + `Dockerfile.lint` build is what CI treats as proof of a clean tree, and it is + also what still fails the pre-commit hook on a badly formatted tree. + `script/fmt-check` survives as a standalone entrypoint, whose verdict cannot + drift from the container's. A test walks the invocation graph from each + entrypoint — through the Makefile shims, the `script/` calls and the + `docker build` — and asserts the prettier count, so the duplication cannot + come back unnoticed. +- 2026-08-10: Moved all linting into Docker. `script/lint` builds a new root + `Dockerfile.lint`, which copies the repo into the digest-pinned node image and + runs eslint and prettier as build steps, so a successful build is a clean + lint; no host lint path remains and `yarn lint` is gone from `package.json`. A + fail-closed `LINT_EPOCH` guard stops Docker serving the linter layers from + cache, which is how a lint build returns success in under a second having + linted nothing. The lint stage inside `Dockerfile` and its `COPY --from=lint` + ordering hack are gone: that image now runs `make test` and `make build` only, + because `script/check` calls `script/lint` and running it in a container would + mean docker inside docker. `script/cibuild` builds the lint image first, then + the test and build image. - 2026-08-09: Made `make docker` green and policy-conformant. Multi-stage Dockerfile: a lint stage runs `make fmt-check` and `make lint`, and the check stage takes a `COPY --from=lint` dependency on it before running `make check` @@ -66,7 +350,6 @@ Update the README API reference section to match the current implementation. # Future Steps -- Tag v1.0.0. - Future desktop client, separate repo: - Electron app skeleton consuming this library. - Local SQLite cache keyed on (collectionID, fileID, updationTime). diff --git a/bin/quak.ts b/bin/quak.ts index 68ab983..4c02183 100644 --- a/bin/quak.ts +++ b/bin/quak.ts @@ -1,146 +1,79 @@ #!/usr/bin/env node -import { input, password as passwordPrompt } from "@inquirer/prompts"; import { stdout, stderr } from "node:process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import { input, password } from "@inquirer/prompts"; import { Command } from "commander"; import envPaths from "env-paths"; -import { Client, type ClientSnapshot } from "../src/client.js"; import { init } from "../src/crypto/index.js"; -import { runBackup } from "../src/backup.js"; -import { runMetadataBackup } from "../src/metadata-backup.js"; import { - listMissingThumbnails, - fixMissingThumbnails, -} from "../src/thumbnails.js"; + type CliContext, + loginCommand, + whoamiCommand, + logoutCommand, + collectionsCommand, + filesCommand, + getCommand, + getThumbCommand, + backupMetadataCommand, + backupCommand, + listMissingThumbnailsCommand, + fixMissingThumbnailsCommand, +} from "../src/cli-commands.js"; +import { run as runCommand } from "../src/cli-run.js"; +import { loadSession } from "../src/cli-session.js"; +import { Client } from "../src/client.js"; +import { VERSION } from "../src/index.js"; const paths = envPaths("quak", { suffix: "" }); -const sessionPath = join(paths.data, "session.json"); - -const loadSession = (): ClientSnapshot | null => { - if (!existsSync(sessionPath)) return null; - try { - return JSON.parse(readFileSync(sessionPath, "utf-8")) as ClientSnapshot; - } catch { - return null; - } -}; - -const saveSession = (snapshot: ClientSnapshot): void => { - mkdirSync(paths.data, { recursive: true, mode: 0o700 }); - writeFileSync(sessionPath, JSON.stringify(snapshot, null, 2), { - mode: 0o600, - }); -}; - -const requireSession = (): Client => { - const snapshot = loadSession(); - if (!snapshot) { - stderr.write( - `Not logged in. Run "quak login" first.\nSession file: ${sessionPath}\n`, - ); - process.exit(1); - } - return Client.fromJSON(snapshot); -}; - -const prompt = async (message: string): Promise<string> => input({ message }); - -const promptSecret = async (message: string): Promise<string> => - passwordPrompt({ message, mask: true }); const program = new Command(); program .name("quak") .description("CLI for the Ente end-to-end encrypted photo service") - .version("0.0.0"); + .version(VERSION) + .option( + "--cache-dir <path>", + "Directory for the local metadata/content cache " + + "(default: the per-user cache directory)", + ); + +const context = (): CliContext => ({ + stdout, + stderr, + sessionDir: paths.data, + cacheDir: program.opts<{ cacheDir?: string }>().cacheDir, + loadSession, + login: (opts) => Client.login(opts), + prompt: (message) => input({ message }), + promptSecret: (message) => password({ message, mask: true }), +}); + +const run = (command: Promise<number>): Promise<void> => + runCommand(command, stdout, stderr, (code) => process.exit(code)); program .command("login") .description("Log in to an Ente account and save the session") - .action(async () => { - await init(); - const email = process.env.QUAK_EMAIL ?? (await prompt("Email")); - const password = - process.env.QUAK_PASSWORD ?? (await promptSecret("Password")); - - stderr.write("Authenticating...\n"); - try { - const client = await Client.login({ - email, - password, - totp: async () => prompt("TOTP code: "), - emailOTP: async () => prompt("Email verification code: "), - }); - - saveSession(client.toJSON()); - const info = client.whoami(); - stderr.write(`Logged in as ${info.email} (user ${info.userID})\n`); - stderr.write(`Session saved to ${sessionPath}\n`); - } catch (err) { - stderr.write( - `Login failed: ${err instanceof Error ? err.message : err}\n`, - ); - process.exit(1); - } - }); + .action(() => run(loginCommand(context()))); program .command("whoami") .description("Print the logged-in account") - .action(() => { - const client = requireSession(); - const info = client.whoami(); - stdout.write(JSON.stringify(info) + "\n"); - }); + .action(() => run(whoamiCommand(context()))); program .command("logout") - .description("Delete the saved session") - .action(async () => { - if (existsSync(sessionPath)) { - const { unlinkSync } = await import("node:fs"); - unlinkSync(sessionPath); - stderr.write("Session deleted.\n"); - } else { - stderr.write("No session found.\n"); - } - }); + .description("End the session on the server and delete the saved session") + .action(() => run(logoutCommand(context()))); program .command("collections") .description("List all collections (albums)") .option("--json", "Output as JSON array") - .action(async (opts: { json?: boolean }) => { - await init(); - const client = requireSession(); - const collections = await client.listCollections(); - - if (opts.json) { - stdout.write( - JSON.stringify( - collections.map((c) => ({ - id: c.id, - name: c.name, - type: c.type, - ownerID: c.ownerID, - isShared: c.isShared, - updationTime: c.updationTime, - })), - null, - 2, - ) + "\n", - ); - } else { - for (const c of collections) { - stdout.write( - `${c.id}\t${c.type}\t${c.name}${c.isShared ? " (shared)" : ""}\n`, - ); - } - } - }); + .action((opts: { json?: boolean }) => + run(collectionsCommand(context(), opts)), + ); program .command("files") @@ -150,93 +83,18 @@ program "Collection ID (from `quak collections`)", ) .option("--json", "Output as JSON array") - .action(async (opts: { collection: string; json?: boolean }) => { - await init(); - const client = requireSession(); - const collectionID = Number(opts.collection); - if (!Number.isFinite(collectionID)) { - stderr.write("Invalid collection ID\n"); - process.exit(1); - } - - const collections = await client.listCollections(); - const col = collections.find((c) => c.id === collectionID); - if (!col) { - stderr.write(`Collection ${collectionID} not found\n`); - process.exit(1); - } - - const files = await client.listFiles(col.id, col.key); - - if (opts.json) { - stdout.write( - JSON.stringify( - files.map((f) => ({ - id: f.id, - title: f.metadata.title, - fileType: f.metadata.fileType, - creationTime: f.metadata.creationTime, - collectionID: f.collectionID, - })), - null, - 2, - ) + "\n", - ); - } else { - for (const f of files) { - stdout.write( - `${f.id}\t${f.metadata.fileType}\t${f.metadata.title}\n`, - ); - } - } - }); + .action((opts: { collection: string; json?: boolean }) => + run(filesCommand(context(), opts)), + ); program .command("get") .description("Download and decrypt a single file") .argument("<fileID>", "File ID (from `quak files`)") .option("--out <path>", "Output file path") - .option( - "--collection <id>", - "Collection ID (required to look up the file key)", - ) - .action( - async ( - fileIDStr: string, - opts: { out?: string; collection?: string }, - ) => { - await init(); - const client = requireSession(); - const fileID = Number(fileIDStr); - if (!Number.isFinite(fileID)) { - stderr.write("Invalid file ID\n"); - process.exit(1); - } - - const collections = await client.listCollections(); - let targetCol; - if (opts.collection) { - targetCol = collections.find( - (c) => c.id === Number(opts.collection), - ); - } - - // Search all collections (or the specified one) for the file - const searchCols = targetCol ? [targetCol] : collections; - for (const col of searchCols) { - const files = await client.listFiles(col.id, col.key); - const file = files.find((f) => f.id === fileID); - if (file) { - const result = await client.downloadFile(file, opts.out); - stderr.write( - `${result.bytesWritten} bytes -> ${result.path}\n`, - ); - return; - } - } - stderr.write(`File ${fileID} not found\n`); - process.exit(1); - }, + .option("--collection <id>", "Accepted for compatibility; ignored") + .action((fileID: string, opts: { out?: string }) => + run(getCommand(context(), fileID, opts)), ); program @@ -244,49 +102,9 @@ program .description("Download and decrypt a thumbnail") .argument("<fileID>", "File ID (from `quak files`)") .option("--out <path>", "Output file path") - .option( - "--collection <id>", - "Collection ID (required to look up the file key)", - ) - .action( - async ( - fileIDStr: string, - opts: { out?: string; collection?: string }, - ) => { - await init(); - const client = requireSession(); - const fileID = Number(fileIDStr); - if (!Number.isFinite(fileID)) { - stderr.write("Invalid file ID\n"); - process.exit(1); - } - - const collections = await client.listCollections(); - let targetCol; - if (opts.collection) { - targetCol = collections.find( - (c) => c.id === Number(opts.collection), - ); - } - - const searchCols = targetCol ? [targetCol] : collections; - for (const col of searchCols) { - const files = await client.listFiles(col.id, col.key); - const file = files.find((f) => f.id === fileID); - if (file) { - const result = await client.downloadThumbnail( - file, - opts.out, - ); - stderr.write( - `${result.bytesWritten} bytes -> ${result.path}\n`, - ); - return; - } - } - stderr.write(`File ${fileID} not found\n`); - process.exit(1); - }, + .option("--collection <id>", "Accepted for compatibility; ignored") + .action((fileID: string, opts: { out?: string }) => + run(getThumbCommand(context(), fileID, opts)), ); program @@ -300,14 +118,9 @@ program "Download each file and extract full EXIF/IPTC/XMP metadata (slow)", ) .option("--all", "Alias for --exif") - .action(async (dir: string, opts: { exif?: boolean; all?: boolean }) => { - await init(); - const client = requireSession(); - await runMetadataBackup(client, dir, { - exif: opts.exif || opts.all, - onProgress: (msg) => stderr.write(msg + "\n"), - }); - }); + .action((dir: string, opts: { exif?: boolean; all?: boolean }) => + run(backupMetadataCommand(context(), dir, opts)), + ); program .command("backup") @@ -316,35 +129,9 @@ program ) .argument("<dir>", "Output directory") .option("--json", "Print result as JSON instead of human-readable summary") - .action(async (dir: string, opts: { json?: boolean }) => { - await init(); - const client = requireSession(); - - stderr.write("Starting backup...\n"); - const result = await runBackup(client, dir, (msg) => { - if (!opts.json) stderr.write(msg + "\n"); - }); - - if (opts.json) { - stdout.write(JSON.stringify(result, null, 2) + "\n"); - } else { - stderr.write("\n--- Backup complete ---\n"); - stderr.write(` Total files: ${result.totalFiles}\n`); - stderr.write(` Downloaded: ${result.downloaded}\n`); - stderr.write(` Skipped: ${result.skipped}\n`); - stderr.write(` Failed: ${result.failed}\n`); - if (result.errors.length > 0) { - stderr.write("\nFailed files:\n"); - for (const e of result.errors) { - stderr.write( - ` [${e.collection}] ${e.title} (id ${e.fileID}): ${e.error}\n`, - ); - } - } - } - - process.exit(result.failed > 0 ? 1 : 0); - }); + .action((dir: string, opts: { json?: boolean }) => + run(backupCommand(context(), dir, opts)), + ); const helper = program .command("helper") @@ -354,30 +141,9 @@ helper .command("list-missing-thumbnails") .description("List files whose thumbnails are missing or empty") .option("--json", "Output as JSON array") - .action(async (opts: { json?: boolean }) => { - await init(); - const client = requireSession(); - const missing = await listMissingThumbnails(client, (msg) => { - if (!opts.json) stderr.write(msg + "\n"); - }); - - if (opts.json) { - stdout.write(JSON.stringify(missing, null, 2) + "\n"); - } else { - if (missing.length === 0) { - stderr.write("No missing thumbnails found.\n"); - } else { - stderr.write( - `\n${missing.length} file(s) with missing thumbnails:\n`, - ); - for (const m of missing) { - stdout.write( - `${m.fileID}\t${m.title}\t${m.collection}\t${m.reason}\n`, - ); - } - } - } - }); + .action((opts: { json?: boolean }) => + run(listMissingThumbnailsCommand(context(), opts)), + ); helper .command("fix-missing-thumbnails") @@ -389,48 +155,9 @@ helper "Specific file IDs to fix (default: fix all missing)", ) .option("--json", "Output as JSON") - .action(async (opts: { file?: string[]; json?: boolean }) => { - await init(); - const client = requireSession(); - - let fileIDs: number[]; - if (opts.file && opts.file.length > 0) { - fileIDs = opts.file.map(Number).filter(Number.isFinite); - } else { - stderr.write("Scanning for missing thumbnails...\n"); - const missing = await listMissingThumbnails(client, (msg) => { - if (!opts.json) stderr.write(msg + "\n"); - }); - fileIDs = missing.map((m) => m.fileID); - if (fileIDs.length === 0) { - stderr.write("No missing thumbnails found.\n"); - return; - } - stderr.write(`Found ${fileIDs.length} file(s) to fix.\n`); - } - - const results = await fixMissingThumbnails(client, fileIDs, (msg) => { - if (!opts.json) stderr.write(msg + "\n"); - }); - - if (opts.json) { - stdout.write(JSON.stringify(results, null, 2) + "\n"); - } else { - const ok = results.filter((r) => r.success).length; - const fail = results.filter((r) => !r.success).length; - stderr.write(`\n--- Done ---\n`); - stderr.write(` Fixed: ${ok}\n`); - stderr.write(` Failed: ${fail}\n`); - if (fail > 0) { - stderr.write("\nFailed files:\n"); - for (const r of results.filter((r) => !r.success)) { - stderr.write(` ${r.fileID}\t${r.title}\t${r.error}\n`); - } - } - } - - process.exit(results.some((r) => !r.success) ? 1 : 0); - }); + .action((opts: { file?: string[]; json?: boolean }) => + run(fixMissingThumbnailsCommand(context(), opts)), + ); await init(); -program.parse(); +await program.parseAsync(); diff --git a/package.json b/package.json index 746ba14..f1a1d6f 100644 --- a/package.json +++ b/package.json @@ -9,28 +9,32 @@ "type": "git", "url": "https://git.eeqj.de/sneak/quak.git" }, + "private": true, "type": "module", + "engines": { + "node": ">=22" + }, "main": "./dist/src/index.js", "types": "./dist/src/index.d.ts", + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./package.json": "./package.json" + }, "bin": { "quak": "./dist/bin/quak.js" }, - "files": [ - "dist/", - "README.md", - "LICENSE" - ], "scripts": { "build": "script/build", "quak": "node ./dist/bin/quak.js", "test": "vitest run", - "lint": "eslint .", "fmt": "prettier --write .", "fmt-check": "prettier --check ." }, "devDependencies": { "@eslint/js": "9.38.0", - "@types/libsodium-wrappers-sumo": "0.8.2", "@types/node": "22.18.13", "eslint": "9.38.0", "prettier": "3.8.1", @@ -44,6 +48,7 @@ "env-paths": "4.0.0", "exif-reader": "2.0.3", "fast-srp-hap": "2.0.4", + "fflate": "0.8.3", "jpeg-js": "0.4.4", "libsodium-wrappers-sumo": "0.8.4" } diff --git a/script/build b/script/build index 05d5180..0a074b7 100755 --- a/script/build +++ b/script/build @@ -45,10 +45,24 @@ for (const bin of bins) { ' } +# src/index.ts imports ../package.json for the version, which tsc copies to +# dist/package.json. Running the built CLI proves that import resolves from +# dist/ and reports the version package.json declares. +verify_version() { + built="$(node dist/bin/quak.js --version)" + declared="$(node -p 'require("./package.json").version')" + if [ "$built" != "$declared" ]; then + echo "build: dist/bin/quak.js reports $built, package.json declares $declared" >&2 + exit 1 + fi + echo "build: dist/bin/quak.js reports version $built" +} + main() { cd "$ROOT" yarn run tsc verify_entrypoints + verify_version } main "$@" diff --git a/script/check b/script/check index 3e1778c..1607369 100755 --- a/script/check +++ b/script/check @@ -1,6 +1,11 @@ #!/bin/sh -# script/check: run all checks (test, lint, fmt-check). Our own -# extension to scripts-to-rule-them-all. Must not modify any files. +# script/check: run all checks (test, lint). Our own extension to +# scripts-to-rule-them-all. Both are Docker phases. Must not modify any +# files. +# +# script/fmt-check is not called here, unlike the template: the lint +# phase already runs `prettier --check .`, so calling it would run +# prettier a second time over the same tree for the same verdict. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -8,7 +13,6 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" main() { "$SCRIPT_DIR/test" "$SCRIPT_DIR/lint" - "$SCRIPT_DIR/fmt-check" } main "$@" diff --git a/script/cibuild b/script/cibuild index 51224da..38f5706 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,17 +1,28 @@ #!/bin/sh -# script/cibuild: run the CI build. The Dockerfile runs script/check and -# script/build, and CHECK_EPOCH differs on every invocation, so those two -# layers cannot be served from Docker's cache: a green build here means -# the checks ran now, not that a previous run was remembered. The layers -# below the epoch (bootstrap, yarn install) are unaffected and stay -# cached. A build that omits the argument fails by design. +# script/cibuild: run the CI build. The image's last stage depends on the +# lint and test phases, so this one build runs eslint, prettier and the +# suite once each and then compiles. Unlike the template it does not run +# script/check first, which would run lint and the tests a second time. +# --no-cache for the same reason as script/docker: the gate phases the +# final stage depends on are RUN steps, and a cached one is a check that +# did not run. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build --build-arg CHECK_EPOCH="$(date +%s)" . + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. VERSION is computed here because .dockerignore + # excludes .git, so `git describe` in a build stage yields an empty + # version without failing. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . } main "$@" diff --git a/script/docker b/script/docker index 0bd2a83..c4688e8 100755 --- a/script/docker +++ b/script/docker @@ -1,9 +1,8 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. # Identical in all repos; the tag comes from script/projectname. -# CHECK_EPOCH is passed for the same reason script/cibuild passes it: the -# Dockerfile refuses to build without it, so that no path to an image can -# quietly serve the check and build layers from cache. +# --no-cache because the gate phases the final stage depends on are RUN +# steps, and a cached one is a check that did not run. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -11,7 +10,15 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build --build-arg CHECK_EPOCH="$(date +%s)" \ + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. VERSION is computed here because .dockerignore + # excludes .git, so `git describe` in a build stage yields an empty + # version without failing. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ -t "$("$SCRIPT_DIR/projectname")" . } diff --git a/script/lint b/script/lint index 3ede02b..2d8b075 100755 --- a/script/lint +++ b/script/lint @@ -1,13 +1,23 @@ #!/bin/sh -# script/lint: run the linter (eslint plus a prettier check). +# script/lint: run the linter. Linting is a phase of the Dockerfile and +# this builds that phase alone; the linter is never installed or run on +# a developer host, where a shared result cache and a host-global lock +# make its answer untrustworthy. +# +# The phase is not the last stage in the file, so it is built only when +# --target names it. --no-cache because a cached lint layer is a lint +# that did not run. The tag makes each build replace the previous image +# instead of leaving a dangling one behind. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - yarn run eslint . - yarn run prettier --check . + docker build --no-cache \ + --target lint \ + -t "$("$SCRIPT_DIR/projectname")-lint" . } main "$@" diff --git a/script/precommit b/script/precommit index 07b9e7e..e87c5c8 100755 --- a/script/precommit +++ b/script/precommit @@ -2,17 +2,17 @@ # script/precommit: run by the git pre-commit hook; fails the commit if # checks fail. Our own extension to scripts-to-rule-them-all. # -# Runs lint and fmt-check but deliberately NOT the tests, so the TDD -# red-phase commit (failing tests, no implementation yet) can land. CI -# runs make check via docker build, which catches any branch that -# ships red. +# Runs lint but deliberately NOT the tests, so the TDD red-phase commit +# (failing tests, no implementation yet) can land. CI runs +# script/cibuild, whose image build includes the test phase, and so +# catches any branch that ships red. The lint phase includes the +# prettier check, so a badly formatted tree still fails the commit. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" main() { "$SCRIPT_DIR/lint" - "$SCRIPT_DIR/fmt-check" } main "$@" diff --git a/script/test b/script/test index d4cec4b..cd239f2 100755 --- a/script/test +++ b/script/test @@ -1,25 +1,19 @@ #!/bin/sh -# script/test: run the test suite. Uses `timeout` (GNU coreutils) when -# available so the run is hard-capped at 30s; on macOS without -# coreutils the cap is skipped. +# script/test: run the test suite. Testing is a phase of the Dockerfile +# and this builds that phase alone, on the same terms as script/lint: +# --target because a phase that is not the last stage is built only when +# named, --no-cache because a cached test layer is a test that did not +# run, and a tag so each build replaces the previous image. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" - -rerun_verbose() { - echo "--- Rerunning with verbose for details ---" - yarn run vitest run --reporter=verbose - exit 1 -} +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - TIMEOUT="$(command -v timeout 2>/dev/null || command -v gtimeout 2>/dev/null || true)" - if [ -n "$TIMEOUT" ]; then - "$TIMEOUT" 30s yarn run vitest run --reporter=dot || rerun_verbose - else - yarn run vitest run --reporter=dot || rerun_verbose - fi + docker build --no-cache \ + --target test \ + -t "$("$SCRIPT_DIR/projectname")-test" . } main "$@" diff --git a/src/api/client.ts b/src/api/client.ts index f3b9996..316f9cc 100644 --- a/src/api/client.ts +++ b/src/api/client.ts @@ -19,14 +19,12 @@ const DEFAULT_FILES_ORIGIN = "https://files.ente.io"; const DEFAULT_THUMBS_ORIGIN = "https://thumbnails.ente.io"; const CLIENT_PACKAGE = "berlin.sneak.quak"; -// Two deadlines rather than one, because a single number cannot serve both -// jobs. Thirty seconds is generous for a JSON call and short enough that a -// hung API connection cannot stall a backup for long. A file body is a -// different shape of problem: the deadline has to cover the whole transfer, -// which for a large video on a slow link is minutes, so a value sane for JSON -// would cancel legitimate downloads. +// Two deadlines of different kinds. `requestTimeoutMs` bounds a whole JSON +// call. `downloadTimeoutMs` is an idle deadline: a file or thumbnail download +// is aborted only when no bytes have arrived for that long, so a large video on +// a slow link that keeps making progress is never cut off. export const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; -export const DEFAULT_DOWNLOAD_TIMEOUT_MS = 600_000; +export const DEFAULT_DOWNLOAD_TIMEOUT_MS = 60_000; export interface ApiClientOptions { apiOrigin?: string; @@ -48,18 +46,44 @@ export interface StreamOptions { retry?: boolean; } -// Enforce a deadline over a response body, not merely over its headers. +// An abort signal that fires once `ms` pass without a call to `restart`. It +// aborts with a `TimeoutError`, the same reason `AbortSignal.timeout()` gives, +// so the retry classifier treats an idle download exactly as it treats any +// other deadline. `stop` must be called when the download ends. The timer is +// unref'd, so even one left running never keeps the process alive. +const idleDeadline = (ms: number) => { + const controller = new AbortController(); + let timer: ReturnType<typeof setTimeout> | undefined; + const stop = (): void => clearTimeout(timer); + const restart = (): void => { + stop(); + timer = setTimeout(() => { + controller.abort( + new DOMException( + `download stalled: no bytes received for ${ms} ms`, + "TimeoutError", + ), + ); + }, ms); + timer.unref(); + }; + restart(); + return { signal: controller.signal, restart, stop }; +}; + +// Enforce the idle deadline over a response body, not merely over its headers. // // `getFileStream` returns as soon as headers arrive; the bytes are pulled // later, in the download layer. Whether the signal passed to `fetch` also // tears down the body afterwards is up to the fetch implementation, so this // wrapper makes it a property of quak instead: every read races the signal, -// and an abort errors the stream with the abort reason — which the retry -// classifier recognises. +// each chunk that arrives restarts the deadline, and an abort errors the +// stream with the abort reason — which the retry classifier recognises. const deadlineStream = ( body: ReadableStream<Uint8Array>, - signal: AbortSignal, + deadline: ReturnType<typeof idleDeadline>, ): ReadableStream<Uint8Array> => { + const { signal } = deadline; const reader = body.getReader(); let rejectOnAbort: (reason: unknown) => void = () => undefined; const aborted = new Promise<never>((_resolve, reject) => { @@ -73,7 +97,10 @@ const deadlineStream = ( const onAbort = (): void => rejectOnAbort(signal.reason); if (signal.aborted) onAbort(); else signal.addEventListener("abort", onAbort, { once: true }); - const release = (): void => signal.removeEventListener("abort", onAbort); + const release = (): void => { + deadline.stop(); + signal.removeEventListener("abort", onAbort); + }; return new ReadableStream<Uint8Array>({ async pull(controller) { @@ -84,6 +111,7 @@ const deadlineStream = ( controller.close(); return; } + deadline.restart(); controller.enqueue(next.value); } catch (err) { release(); @@ -98,6 +126,30 @@ const deadlineStream = ( }); }; +// The one place a request URL is built. `origin` may carry a base path (a +// self-hosted server behind a prefix) and may end in a slash; `path` may or +// may not start with one. Query parameters go only through `query`, which +// percent-encodes them: a `?` or `#` in `path` is an error, because +// `new URL` would otherwise quietly treat what follows as something else. +const buildURL = ( + origin: string, + path: string, + query?: Record<string, string | number | undefined>, +): string => { + if (path.includes("?") || path.includes("#")) { + throw new Error( + `request path must not contain "?" or "#"; pass query parameters separately: ${path}`, + ); + } + const url = new URL( + `${origin.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`, + ); + for (const [k, v] of Object.entries(query ?? {})) { + if (v !== undefined) url.searchParams.set(k, String(v)); + } + return url.href; +}; + export class ApiClient { private readonly apiOrigin: string; private readonly isCustomOrigin: boolean; @@ -139,11 +191,16 @@ export class ApiClient { this.token = undefined; } + getAuthToken(): string | undefined { + return this.token; + } + // The policy this client was configured with, so that a caller wrapping a // whole operation in its own `withRetry` — the download layer — runs under // the same settings rather than under the library defaults. + // A copy, so the caller cannot change this client's settings through it. getRetryOptions(): ResolvedRetryOptions { - return this.retry; + return { ...this.retry }; } private headers(extra?: Record<string, string>): Record<string, string> { @@ -197,22 +254,10 @@ export class ApiClient { path: string, query?: Record<string, string | number | undefined>, ): Promise<T> { - const url = new URL(path, this.apiOrigin + "/"); - // new URL with a base resolves relative paths; ensure we keep the - // origin from apiOrigin even when path starts with / - url.protocol = new URL(this.apiOrigin).protocol; - url.host = new URL(this.apiOrigin).host; - url.pathname = path; - if (query) { - for (const [k, v] of Object.entries(query)) { - if (v !== undefined) { - url.searchParams.set(k, String(v)); - } - } - } + const url = buildURL(this.apiOrigin, path, query); // A GET changes nothing, so it is retried under the full policy. return withRetry(async () => { - const resp = await this._fetch(url.href, { + const resp = await this._fetch(url, { method: "GET", headers: this.headers(), signal: AbortSignal.timeout(this.requestTimeoutMs), @@ -223,16 +268,16 @@ export class ApiClient { } async postJSON<T>(path: string, body: unknown): Promise<T> { - const url = `${this.apiOrigin}${path}`; - // Idempotency: this reaches `/users/srp/create-session`, - // `/users/two-factor/verify` and `/users/ott`, all of which change - // server state — verifying a second factor consumes one of a small - // number of attempts. So a POST is replayed only on a failure that - // establishes no TCP connection to the server ever existed: DNS - // produced no address, or the peer refused the connection. A 5xx, a - // mid-flight reset, a routing errno (which Linux also delivers on an - // established socket) and a timeout are all left to the caller, - // because each of them can occur after the server has already acted. + const url = buildURL(this.apiOrigin, path); + // Not idempotent: a POST is replayed only when `isSafeToReplay` + // says no request byte can have reached the server. The endpoints + // this covers are listed in the README under "Endpoints used". + // + // Redirects are not followed. The origin has already received the + // request when it answers with one, so a connection refused by the + // redirect target would look replay-safe when it is not. The API has + // no legitimate redirect, so one surfaces as an `ApiError` with its + // 3xx status, which is not retried. return withRetry( async () => { const resp = await this._fetch(url, { @@ -241,6 +286,7 @@ export class ApiClient { "Content-Type": "application/json", }), body: JSON.stringify(body), + redirect: "manual", signal: AbortSignal.timeout(this.requestTimeoutMs), }); await this.throwIfError(resp); @@ -255,8 +301,8 @@ export class ApiClient { opts?: StreamOptions, ): Promise<ReadableStream<Uint8Array>> { const url = this.isCustomOrigin - ? `${this.apiOrigin}/files/download/${fileID}` - : `${this.filesOrigin}/?fileID=${fileID}`; + ? buildURL(this.apiOrigin, `/files/download/${fileID}`) + : buildURL(this.filesOrigin, "/", { fileID }); return this.streamRequest(url, opts); } @@ -298,10 +344,8 @@ export class ApiClient { } async putJSON<T>(path: string, body: unknown): Promise<T> { - const url = `${this.apiOrigin}${path}`; - // Same idempotency rule as `postJSON`, for the same reason: this - // reaches `/files/thumbnail`, which registers an uploaded thumbnail - // against a file. + const url = buildURL(this.apiOrigin, path); + // Same replay and redirect rules as `postJSON`, for the same reasons. return withRetry( async () => { const resp = await this._fetch(url, { @@ -310,6 +354,7 @@ export class ApiClient { "Content-Type": "application/json", }), body: JSON.stringify(body), + redirect: "manual", signal: AbortSignal.timeout(this.requestTimeoutMs), }); await this.throwIfError(resp); @@ -335,8 +380,8 @@ export class ApiClient { opts?: StreamOptions, ): Promise<ReadableStream<Uint8Array>> { const url = this.isCustomOrigin - ? `${this.apiOrigin}/files/preview/${fileID}` - : `${this.thumbsOrigin}/?fileID=${fileID}`; + ? buildURL(this.apiOrigin, `/files/preview/${fileID}`) + : buildURL(this.thumbsOrigin, "/", { fileID }); return this.streamRequest(url, opts); } @@ -345,22 +390,27 @@ export class ApiClient { opts?: StreamOptions, ): Promise<ReadableStream<Uint8Array>> { const once = async (): Promise<ReadableStream<Uint8Array>> => { - // A fresh deadline per attempt, so a retry gets the whole budget - // rather than the remainder of the one that just expired. - const signal = AbortSignal.timeout(this.downloadTimeoutMs); - const resp = await this._fetch(url, { - method: "GET", - headers: this.headers(), - signal, - }); - await this.throwIfError(resp); - if (!resp.body) { - // Carries the status, and is not retryable: a response that - // arrived without a body is malformed, and asking again - // produces the same malformed response. - throw new ApiError("response body is null", resp.status); + // A fresh deadline per attempt. It also covers the wait for the + // headers, when no bytes have arrived either. + const deadline = idleDeadline(this.downloadTimeoutMs); + try { + const resp = await this._fetch(url, { + method: "GET", + headers: this.headers(), + signal: deadline.signal, + }); + await this.throwIfError(resp); + if (!resp.body) { + // Carries the status, and is not retryable: a response + // that arrived without a body is malformed, and asking + // again produces the same malformed response. + throw new ApiError("response body is null", resp.status); + } + return deadlineStream(resp.body, deadline); + } catch (err) { + deadline.stop(); + throw err; } - return deadlineStream(resp.body, signal); }; return opts?.retry === false ? once() : withRetry(once, this.retry); } diff --git a/src/backup.ts b/src/backup.ts index b6cece3..9ea9707 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -1,13 +1,81 @@ +// The backup command, rebuilt on the library API (issue #51). +// +// `lib.backup()` waits for a completed refresh of the library (a failed one +// fails the backup before any file is touched), then, for every file in scope, +// gets its original bytes onto disk under `downloadDirectory` and rebuilds the +// derived views (per-file sidecars, per-collection symlink trees, +// per-collection JSON) from the model. The on-disk layout is the historical +// one: +// +// <downloadDirectory>/ +// originals/<fileID>.<ext> the decrypted bytes +// originals/<fileID>.json per-file metadata sidecar +// collections/<name>/<title> symlink into ../../originals +// collections/<name>.json per-collection metadata +// failures.json durable ledger of unresolved failures +// +// A live photo's original is its image and its video, `<fileID>.<ext>` each +// with its own extension, and `originals/<fileID>.livephoto.json` naming them; +// its album folders link both. +// +// Crash-safety rests on two properties. Bytes are present-means-complete: an +// original appears under `originals/` only via the content layer's atomic +// temp-then-rename, so a file that exists is whole and is never re-fetched — an +// interrupted run resumes by listing the directory. The derived views hold no +// unique state, so they are rebuilt every run; that repairs stale sidecars and +// missing or broken symlinks left by an earlier crash. A rebuild also removes +// the symlinks into originals/ that no longer belong to an album, and the +// directories of albums that no longer exist. +// +// Resilience (issue #8): no per-file condition aborts the run. A failed +// download or a failed symlink is caught, recorded in `failures.json` with a +// classification, a running attempt count, and the last-tried time, and the run +// continues. `result.failed` — and thus the CLI's exit code — stays non-zero +// while any failure remains unresolved and clears once every one succeeds. Each +// run reconciles the ledger against the files it attempted, so an entry for a +// file that has since left the library (deleted) or this run's scope is dropped +// rather than counted forever, which would poison a scheduled backup's exit code. + import { - existsSync, + lstatSync, mkdirSync, + readdirSync, + readFileSync, + readlinkSync, + rmdirSync, + rmSync, statSync, symlinkSync, writeFileSync, } from "node:fs"; -import { join, relative, extname } from "node:path"; -import type { Client } from "./client.js"; -import type { EnteFile } from "./model/types.js"; +import { copyFile, rename, rm } from "node:fs/promises"; +import { basename, dirname, extname, join, relative } from "node:path"; + +import { fsyncPath, removeLeftoverTempFiles } from "./download/index.js"; +import { sanitizeFileName, withExtension } from "./filename.js"; +import { + nameInOriginals, + storedOriginal, + writeLivePhotoJSON, +} from "./library/content.js"; +import type { Collection, EnteFile } from "./model/types.js"; + +export type ProgressCallback = (message: string) => void; + +export interface BackupOptions { + // Where the backup tree lives. Required: with none, `backup()` throws + // before any network traffic. A library opened with a `downloadDirectory` + // supplies the default. + downloadDirectory?: string; + // Fetch and store full-resolution originals. Default true. + includeOriginals?: boolean; + // Also fetch and store thumbnails under `thumbnails/<fileID>.jpg`. Default + // false. + includeThumbnails?: boolean; + // Restrict the backup to albums with these names; others are left untouched. + onlyAlbumNames?: string[]; + onProgress?: ProgressCallback; +} export interface BackupError { fileID: number; @@ -17,139 +85,531 @@ export interface BackupError { } export interface BackupResult { + // Distinct files in scope this run. totalFiles: number; + // Originals fetched (or copied from the cache) this run. downloaded: number; + // Originals already present and left untouched. skipped: number; + // Files with an unresolved failure after this run (the ledger size); the + // CLI exits non-zero while this is above zero. A file can be both + // downloaded and failed if its bytes landed but its symlink did not. failed: number; + // This run's per-file errors, in encounter order. errors: BackupError[]; } -export type ProgressCallback = (message: string) => void; +// The slice of the library that backup drives. `Library` implements it; a test +// can drive backup with a stand-in. +export interface BackupLibrary { + refresh(): Promise<void>; + listCollections(): Collection[]; + listFiles(collectionID: number): EnteFile[]; + // Get an original's bytes onto disk through the content cache/pools, + // returning where they landed: `destination` when they were fetched now, + // otherwise wherever they already were (the cache, or a prior backup). A + // live photo lands as its image and its video, fetched now beside + // `destination`. + original( + fileID: number, + destination: string, + ): Promise<{ path: string; videoPath?: string }>; + thumbnail(fileID: number): Promise<{ path: string }>; +} -const sanitizePath = (name: string): string => - name.replace(/[/\\:*?"<>|]/g, "_").replace(/^\.+/, "_"); +type FailureClass = "transient" | "permanent" | "unknown"; -const originalFileName = (file: EnteFile): string => { - const ext = extname(file.metadata.title || "") || ".bin"; - return `${file.id}${ext}`; +interface FailureEntry { + fileID: number; + title: string; + classification: FailureClass; + attempts: number; + lastTriedAt: number; + error: string; +} + +const LEDGER_VERSION = 1; + +// A regular file with content is treated as complete. A zero-byte file is not: +// it is the shape an aborted write leaves and must be re-fetched. +const isPresent = (path: string): boolean => { + try { + const s = statSync(path); + return s.isFile() && s.size > 0; + } catch { + return false; + } +}; + +// Best-effort classification for the ledger. Retryable server/network problems +// are transient; refusals and local filesystem/decrypt errors are permanent; +// anything else is unknown. Both the error code and message are inspected. +const classify = (err: unknown): FailureClass => { + const e = err as NodeJS.ErrnoException; + const text = + `${e?.code ?? ""} ${err instanceof Error ? err.message : String(err)}`.toLowerCase(); + if ( + /timeout|timed out|econnreset|econnrefused|econnaborted|network|socket|eai_again|throttl|temporarily|429|500|502|503|504/.test( + text, + ) + ) { + return "transient"; + } + if ( + /enoent|eacces|eperm|eexist|eisdir|enotempty|erofs|enospc|not found|forbidden|unauthor|decrypt|truncat|401|403|404/.test( + text, + ) + ) { + return "permanent"; + } + return "unknown"; +}; + +const errorMessage = (err: unknown): string => + err instanceof Error ? err.message : String(err); + +// Copy bytes into `dest` via a temp file in the same directory plus rename, so +// `dest` appears only once it is whole ("present means complete"). As in the +// download writer, the temp file is fsynced before the rename and the directory +// after it, so a power cut cannot leave a correctly named but short original. +// The temp name carries this process's ID so a later run can tell a leftover +// from a copy still in progress (see `removeLeftoverTempFiles`). +const copyAtomic = async (src: string, dest: string): Promise<void> => { + if (src === dest) return; + const tmp = join( + dirname(dest), + `.quak-backup-${basename(dest)}-${process.pid}-${Math.random() + .toString(36) + .slice(2)}.tmp`, + ); + try { + await copyFile(src, tmp); + await fsyncPath(tmp); + // `rename` replaces the destination's directory entry: an existing + // symlink at `dest` is replaced, not followed, and the new file has + // the temp file's permissions (copied from `src`). + await rename(tmp, dest); + await fsyncPath(dirname(dest)); + } finally { + await rm(tmp, { force: true }); + } +}; + +// Put an original the library returned at `dest` in originals/, where a fresh +// fetch already wrote it. A live photo's image and video go beside `dest`: when +// they came from the cache they are copied, after removing whatever was at +// `dest` (an earlier version's ZIP of the two). Then the JSON file naming them +// is written, which is what makes the live photo count as stored. +const placeOriginal = async ( + file: EnteFile, + dest: string, + got: { path: string; videoPath?: string }, +): Promise<void> => { + if (got.videoPath === undefined) { + await copyAtomic(got.path, dest); + return; + } + const originalsDir = dirname(dest); + const path = join(originalsDir, basename(got.path)); + const videoPath = join(originalsDir, basename(got.videoPath)); + if (got.path !== path) { + await rm(dest, { force: true }); + await copyAtomic(got.path, path); + await copyAtomic(got.videoPath, videoPath); + } + await writeLivePhotoJSON(originalsDir, file.id, { path, videoPath }); +}; + +// Ensure `linkPath` is a symlink to `target`, rebuilding a missing, wrong, or +// non-symlink entry. Throws on failure (a directory in the way, no permission) +// so the caller records it and moves on rather than aborting the run. +const rebuildSymlink = (linkPath: string, target: string): void => { + try { + const st = lstatSync(linkPath); + if (st.isSymbolicLink() && readlinkSync(linkPath) === target) return; + } catch { + // Nothing there (or unreadable): fall through to create it. + } + // Remove a wrong symlink or stray file. `force` ignores a missing path but + // still refuses a directory (no `recursive`), which surfaces as a failure. + rmSync(linkPath, { force: true }); + symlinkSync(target, linkPath); +}; + +// The on-disk names for the entries of one directory, in entry order. Each +// name is used as is unless another entry would get the same name, ignoring +// case (two names that differ only in case are one entry on a case-insensitive +// file system); then every entry sharing it gets ` (<id>)`, before the +// extension when `beforeExtension` is set. A name with an ID added can match +// another entry's own name (`IMG (6).JPG`), so this repeats until no name is +// shared. IDs are stable, so the names are too. +const uniqueNames = ( + entries: { id: number; name: string }[], + beforeExtension: boolean, +): string[] => { + const withID = (id: number, name: string): string => { + const ext = beforeExtension ? extname(name) : ""; + const stem = name.slice(0, name.length - ext.length); + return `${stem} (${id})${ext}`; + }; + const names = entries.map((e) => e.name); + const suffixed = new Set<number>(); + for (;;) { + const counts = new Map<string, number>(); + for (const name of names) { + const key = name.toLowerCase(); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + let changed = false; + for (const [i, { id, name }] of entries.entries()) { + if (suffixed.has(i)) continue; + if (counts.get(name.toLowerCase()) === 1) continue; + names[i] = withID(id, name); + suffixed.add(i); + changed = true; + } + if (!changed) return names; + } +}; + +// The links a file gets in its album's folder: one named after its title, to +// its original if that is stored. A stored live photo gets two, to its image +// and its video, each named after the title with that file's extension. +const linksFor = ( + file: EnteFile, + stored: { path: string; videoPath?: string } | undefined, +): { id: number; name: string; file: EnteFile; target?: string }[] => { + const name = sanitizeFileName(file.metadata.title, `file-${file.id}`); + if (stored?.videoPath === undefined) { + return [{ id: file.id, name, file, target: stored?.path }]; + } + return [stored.path, stored.videoPath].map((target) => ({ + id: file.id, + name: withExtension(name, extname(target)), + file, + target, + })); +}; + +// Remove the symlinks in the album directory `dir` that point into +// `originalsDir` and are not named in `keep`. Nothing else in the directory +// is touched: anything else there was put there by the user. +const removeStaleLinks = ( + dir: string, + keep: Set<string>, + originalsDir: string, +): void => { + const target = relative(dir, originalsDir); + for (const name of readdirSync(dir)) { + if (keep.has(name)) continue; + const path = join(dir, name); + if ( + lstatSync(path).isSymbolicLink() && + dirname(readlinkSync(path)) === target + ) { + rmSync(path); + } + } +}; + +// Remove the directories under `collectionsDir` that an earlier run wrote for +// an album that is gone or renamed: a directory not named in `current` with a +// `<name>.json` beside it holding an album ID, which is what a run writes. Its +// symlinks into originals/ are removed; if that leaves it empty, it and its +// JSON are deleted, otherwise both stay for what the user put there. +const removeStaleAlbumDirs = ( + collectionsDir: string, + current: Set<string>, + originalsDir: string, +): void => { + for (const entry of readdirSync(collectionsDir, { withFileTypes: true })) { + if (!entry.isDirectory() || current.has(entry.name)) continue; + const jsonPath = join(collectionsDir, `${entry.name}.json`); + try { + const album = JSON.parse(readFileSync(jsonPath, "utf-8")) as { + id?: unknown; + }; + if (typeof album.id !== "number") continue; + } catch { + continue; + } + const dir = join(collectionsDir, entry.name); + removeStaleLinks(dir, new Set(), originalsDir); + if (readdirSync(dir).length > 0) continue; + rmdirSync(dir); + rmSync(jsonPath); + } +}; + +const loadLedger = (path: string): Map<number, FailureEntry> => { + const ledger = new Map<number, FailureEntry>(); + try { + const parsed = JSON.parse(readFileSync(path, "utf-8")) as { + files?: Record<string, FailureEntry>; + }; + for (const entry of Object.values(parsed.files ?? {})) { + if (entry && typeof entry.fileID === "number") { + ledger.set(entry.fileID, entry); + } + } + } catch { + // No ledger yet, or an unreadable one: start clean. + } + return ledger; +}; + +const saveLedger = (path: string, ledger: Map<number, FailureEntry>): void => { + if (ledger.size === 0) { + rmSync(path, { force: true }); + return; + } + const files: Record<string, FailureEntry> = {}; + for (const [fileID, entry] of ledger) files[String(fileID)] = entry; + writeFileSync( + path, + JSON.stringify({ version: LEDGER_VERSION, files }, null, 2), + ); +}; + +const writeSidecar = (path: string, file: EnteFile): void => { + const meta: Record<string, unknown> = { + id: file.id, + collectionID: file.collectionID, + ownerID: file.ownerID, + metadata: file.metadata, + }; + if (file.magicMetadata) meta.magicMetadata = file.magicMetadata; + if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata; + writeFileSync(path, JSON.stringify(meta, null, 2)); }; export const runBackup = async ( - client: Client, - outDir: string, - onProgress?: ProgressCallback, + lib: BackupLibrary, + opts: BackupOptions, ): Promise<BackupResult> => { - const log = onProgress ?? (() => {}); + const downloadDirectory = opts.downloadDirectory; + if (!downloadDirectory) { + throw new Error( + "backup requires a downloadDirectory (pass one to backup() or " + + "open the library with one)", + ); + } + const includeOriginals = opts.includeOriginals ?? true; + const includeThumbnails = opts.includeThumbnails ?? false; + const log = opts.onProgress ?? (() => {}); + const only = opts.onlyAlbumNames ? new Set(opts.onlyAlbumNames) : undefined; - mkdirSync(outDir, { recursive: true }); - const originalsDir = join(outDir, "originals"); + log("Refreshing library..."); + await lib.refresh(); + + const originalsDir = join(downloadDirectory, "originals"); + const collectionsDir = join(downloadDirectory, "collections"); + const thumbnailsDir = join(downloadDirectory, "thumbnails"); mkdirSync(originalsDir, { recursive: true }); - const collectionsDir = join(outDir, "collections"); mkdirSync(collectionsDir, { recursive: true }); + if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true }); + removeLeftoverTempFiles(originalsDir); + removeLeftoverTempFiles(thumbnailsDir); - log("Fetching collections..."); - const collections = await client.listCollections(); - const downloadedIDs = new Set<number>(); + const ledgerPath = join(downloadDirectory, "failures.json"); + const ledger = loadLedger(ledgerPath); + const now = Date.now(); - let totalFiles = 0; + // Collections in scope, and the distinct files across them (a file shared + // by two albums is one original). + const allCollections = lib.listCollections(); + const collections = allCollections.filter((c) => + only ? only.has(c.name) : true, + ); + const collectionName = new Map<number, string>(); + for (const c of collections) collectionName.set(c.id, c.name); + + const distinct = new Map<number, EnteFile>(); + const filesByCollection = new Map<number, EnteFile[]>(); + for (const c of collections) { + const files = lib.listFiles(c.id); + filesByCollection.set(c.id, files); + for (const f of files) if (!distinct.has(f.id)) distinct.set(f.id, f); + } + + const errors: BackupError[] = []; + const failedThisRun = new Set<number>(); let downloaded = 0; let skipped = 0; - let failed = 0; - const errors: BackupError[] = []; - for (const col of collections) { - const colDirName = sanitizePath(col.name || `collection-${col.id}`); + const recordFailure = ( + file: EnteFile, + collection: string, + err: unknown, + ): void => { + // Count at most one attempt per file per run: a file whose original + // and thumbnail both fail this run must not double its attempt count + // or appear twice in errors. + if (failedThisRun.has(file.id)) return; + const error = errorMessage(err); + errors.push({ + fileID: file.id, + title: file.metadata.title, + collection, + error, + }); + const prior = ledger.get(file.id); + ledger.set(file.id, { + fileID: file.id, + title: file.metadata.title, + classification: classify(err), + attempts: (prior?.attempts ?? 0) + 1, + lastTriedAt: now, + error, + }); + failedThisRun.add(file.id); + }; + + // Phase 1: get the bytes. Fetch each pending original (and optional + // thumbnail) through the content cache/pools and place it under the backup + // tree; a present file is left as is. + if (includeOriginals) { + for (const [fileID, file] of distinct) { + if (storedOriginal(originalsDir, file) !== undefined) { + skipped++; + continue; + } + const dest = join(originalsDir, nameInOriginals(file)); + try { + log(`Fetching original ${file.metadata.title} (${fileID})...`); + // A fetched original is written straight to `dest` (a live + // photo beside it); only one that was already cached elsewhere + // is copied. + await placeOriginal( + file, + dest, + await lib.original(fileID, dest), + ); + downloaded++; + } catch (err) { + log( + `FAILED original ${file.metadata.title}: ${errorMessage(err)}`, + ); + recordFailure( + file, + collectionName.get(file.collectionID) ?? "", + err, + ); + } + } + } + + if (includeThumbnails) { + for (const [fileID, file] of distinct) { + const dest = join(thumbnailsDir, `${fileID}.jpg`); + if (isPresent(dest)) continue; + try { + const { path } = await lib.thumbnail(fileID); + await copyAtomic(path, dest); + } catch (err) { + recordFailure( + file, + collectionName.get(file.collectionID) ?? "", + err, + ); + } + } + } + + // Phase 2: rebuild the derived views from the model. Sidecars first, for + // every present original (this repairs stale ones). + if (includeOriginals) { + for (const [fileID, file] of distinct) { + if (storedOriginal(originalsDir, file) !== undefined) { + writeSidecar(join(originalsDir, `${fileID}.json`), file); + } + } + } + + // Then the per-collection symlink trees and JSON. Directory names are + // chosen across every album, not just those in scope, so a scoped run + // names an album the same as a full one and never takes the directory of + // an album it skipped. Stale entries are removed before anything is + // rebuilt, so on a case-insensitive file system removing an old name can + // never remove the new one. + const dirNames = uniqueNames( + allCollections.map((c) => ({ + id: c.id, + name: sanitizeFileName(c.name, `collection-${c.id}`), + })), + false, + ); + const albumDirNames = new Map( + allCollections.map((c, i) => [c.id, dirNames[i]!]), + ); + try { + removeStaleAlbumDirs(collectionsDir, new Set(dirNames), originalsDir); + } catch (err) { + log(`FAILED removing old album directories: ${errorMessage(err)}`); + } + + for (const c of collections) { + const colDirName = albumDirNames.get(c.id)!; const colDir = join(collectionsDir, colDirName); mkdirSync(colDir, { recursive: true }); - log(`[${col.name}] Fetching file list...`); - const files = await client.listFiles(col.id, col.key); - log(`[${col.name}] ${files.length} file(s)`); + const files = filesByCollection.get(c.id) ?? []; + const links = files.flatMap((f) => + linksFor(f, storedOriginal(originalsDir, f)), + ); + const linkNames = uniqueNames(links, true); + try { + removeStaleLinks(colDir, new Set(linkNames), originalsDir); + } catch (err) { + log(`FAILED removing old links in ${c.name}: ${errorMessage(err)}`); + } - const collectionMeta: { - id: number; - name: string; - type: string; - files: { id: number; metadata: EnteFile["metadata"] }[]; - } = { - id: col.id, - name: col.name, - type: col.type, - files: [], - }; - - for (const file of files) { - totalFiles++; - const origName = originalFileName(file); - const origPath = join(originalsDir, origName); - const linkName = sanitizePath( - file.metadata.title || `file-${file.id}`, - ); - const linkPath = join(colDir, linkName); - - if (!downloadedIDs.has(file.id)) { - if (existsSync(origPath) && statSync(origPath).size > 0) { - skipped++; - downloadedIDs.add(file.id); - } else { - try { - log(`[${col.name}] Downloading ${linkName}...`); - await client.downloadFile(file, origPath); - downloaded++; - downloadedIDs.add(file.id); - } catch (err) { - log( - `[${col.name}] FAILED ${linkName}: ${err instanceof Error ? err.message : err}`, - ); - failed++; - errors.push({ - fileID: file.id, - title: file.metadata.title, - collection: col.name, - error: - err instanceof Error - ? err.message - : String(err), - }); - continue; - } - } + const metaFiles = files.map((f) => ({ + id: f.id, + metadata: f.metadata, + })); + for (const [i, link] of links.entries()) { + if (!includeOriginals || link.target === undefined) continue; + const linkName = linkNames[i]!; + try { + rebuildSymlink( + join(colDir, linkName), + relative(colDir, link.target), + ); + } catch (err) { + log( + `FAILED symlink ${c.name}/${linkName}: ${errorMessage(err)}`, + ); + recordFailure(link.file, c.name, err); } - - // Write per-file metadata JSON alongside the original - const metaJsonPath = join(originalsDir, `${file.id}.json`); - if (!existsSync(metaJsonPath)) { - const fileMeta: Record<string, unknown> = { - id: file.id, - collectionID: file.collectionID, - ownerID: file.ownerID, - metadata: file.metadata, - }; - if (file.magicMetadata) { - fileMeta.magicMetadata = file.magicMetadata; - } - if (file.pubMagicMetadata) { - fileMeta.pubMagicMetadata = file.pubMagicMetadata; - } - writeFileSync(metaJsonPath, JSON.stringify(fileMeta, null, 2)); - } - - if (!existsSync(linkPath) && existsSync(origPath)) { - const target = relative(colDir, origPath); - symlinkSync(target, linkPath); - } - - collectionMeta.files.push({ - id: file.id, - metadata: file.metadata, - }); } writeFileSync( join(collectionsDir, `${colDirName}.json`), - JSON.stringify(collectionMeta, null, 2), + JSON.stringify( + { id: c.id, name: c.name, type: c.type, files: metaFiles }, + null, + 2, + ), ); } - return { totalFiles, downloaded, skipped, failed, errors }; + // Reconcile the ledger against what this run actually attempted: an entry + // survives only for a file that failed this run. A file that succeeded had + // its failure resolved; a file gone from the library (deleted) or outside + // this run's scope is not something this run can resolve, so keeping its + // stale entry would keep the exit code non-zero forever — a single + // since-deleted photo would fail every future scheduled backup. + for (const fileID of [...ledger.keys()]) { + if (!failedThisRun.has(fileID)) ledger.delete(fileID); + } + saveLedger(ledgerPath, ledger); + + return { + totalFiles: distinct.size, + downloaded, + skipped, + failed: ledger.size, + errors, + }; }; diff --git a/src/cli-commands.ts b/src/cli-commands.ts new file mode 100644 index 0000000..3ea8f34 --- /dev/null +++ b/src/cli-commands.ts @@ -0,0 +1,559 @@ +// The CLI's commands as plain functions. +// +// Each command takes its options and a `CliContext` and resolves to the exit +// code; a thrown error is left to the caller. Nothing here calls +// `process.exit`: `bin/quak.ts` wires these to the command line, and `run` in +// `cli-run.ts` prints a thrown error as one line and exits once output has +// drained. Output must stay byte-identical (see `cli-output.ts`). + +import { + copyFileSync, + existsSync, + mkdirSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { extname, join } from "node:path"; +import { + type Client, + type ClientSnapshot, + type LoginOptions, +} from "./client.js"; +import { init } from "./crypto/index.js"; +import { + defaultCacheDirectory, + Library, + type LibraryClient, +} from "./library/index.js"; +import { + fileListRow, + fileListLine, + originalName, + thumbnailName, +} from "./cli-output.js"; +import { freshCollections, freshFiles, freshFile } from "./cli-read.js"; +import { withExtension } from "./filename.js"; +import { runMetadataBackup } from "./metadata-backup.js"; +import { listMissingThumbnails, fixMissingThumbnails } from "./thumbnails.js"; + +export interface CliContext { + stdout: { write(text: string): unknown }; + stderr: { write(text: string): unknown }; + // Directory holding `session.json`. + sessionDir: string; + // The `--cache-dir` global, or undefined to let the library pick its + // per-user default keyed by the account id. + cacheDir?: string; + // Reads the session file into a client, or null when there is none. The + // CLI passes `loadSession` from `cli-session.ts`; tests pass a fake client. + loadSession: (path: string) => Client | null; + // Used by `login` only. The CLI passes `Client.login` and terminal + // prompts; tests pass fakes. + login: (opts: LoginOptions) => Promise<Client>; + prompt: (message: string) => Promise<string>; + // Like `prompt`, but the answer is masked as it is typed. + promptSecret: (message: string) => Promise<string>; +} + +const sessionPath = (ctx: CliContext): string => + join(ctx.sessionDir, "session.json"); + +// Write the session readable by its owner only, in a directory only its owner +// can enter. +export const saveSession = ( + sessionDir: string, + snapshot: ClientSnapshot, +): void => { + mkdirSync(sessionDir, { recursive: true, mode: 0o700 }); + writeFileSync( + join(sessionDir, "session.json"), + JSON.stringify(snapshot, null, 2), + { mode: 0o600 }, + ); +}; + +// The saved client, or undefined after telling the user why there is none. +const requireSession = (ctx: CliContext): Client | undefined => { + let client: Client | null; + try { + client = ctx.loadSession(sessionPath(ctx)); + } catch (err) { + ctx.stderr.write( + `${err instanceof Error ? err.message : err}\n` + + `Run "quak logout" and then "quak login" to replace it.\n`, + ); + return undefined; + } + if (!client) { + ctx.stderr.write( + `Not logged in. Run "quak login" first.\nSession file: ${sessionPath(ctx)}\n`, + ); + return undefined; + } + return client; +}; + +// A library client that omits `fetchMLData`, so the point commands below do not +// kick the library's background ML backfill: they read metadata, or fetch one +// file's content, and exit. `backup` and `backup-metadata` handle ML on their +// own terms. The content source is kept so `get`/`get-thumb`/`--exif` can fetch +// originals through the on-disk cache. +const readLibraryClient = (client: Client): LibraryClient => ({ + whoami: () => client.whoami(), + collectionsSince: (args) => client.collectionsSince(args), + filesSince: (args) => client.filesSince(args), + contentSource: () => client.contentSource(), +}); + +// Open a library for a single point command: the aggressive background precache +// (issue #48) is off — a one-shot `collections` or `get` must not start +// downloading the whole account — and the refresh interval is long so no second +// refresh fires mid-command. +const openReadLibrary = (ctx: CliContext, client: Client): Promise<Library> => + Library.open({ + client: readLibraryClient(client), + cacheDirectory: ctx.cacheDir, + refreshIntervalSeconds: 3600, + precacheThumbnails: false, + precacheOriginals: false, + }); + +export const loginCommand = async (ctx: CliContext): Promise<number> => { + await init(); + const email = process.env.QUAK_EMAIL ?? (await ctx.prompt("Email")); + const password = + process.env.QUAK_PASSWORD ?? (await ctx.promptSecret("Password")); + + ctx.stderr.write("Authenticating...\n"); + try { + const client = await ctx.login({ + email, + password, + totp: async () => ctx.prompt("TOTP code: "), + emailOTP: async () => ctx.prompt("Email verification code: "), + }); + + saveSession(ctx.sessionDir, client.toJSON()); + const info = client.whoami(); + ctx.stderr.write(`Logged in as ${info.email} (user ${info.userID})\n`); + ctx.stderr.write(`Session saved to ${sessionPath(ctx)}\n`); + } catch (err) { + ctx.stderr.write( + `Login failed: ${err instanceof Error ? err.message : err}\n`, + ); + return 1; + } + return 0; +}; + +export const whoamiCommand = async (ctx: CliContext): Promise<number> => { + await init(); + const client = requireSession(ctx); + if (!client) return 1; + const info = client.whoami(); + ctx.stdout.write(JSON.stringify(info) + "\n"); + return 0; +}; + +// Ends the session on the server, then deletes the session file even when that +// failed, and exits 1 if it did. The cache is left in place; the user is told +// where it is. +export const logoutCommand = async (ctx: CliContext): Promise<number> => { + const path = sessionPath(ctx); + if (!existsSync(path)) { + ctx.stderr.write("No session found.\n"); + return 0; + } + await init(); + let cacheDir = ctx.cacheDir; + let failure: string | undefined; + try { + const client = ctx.loadSession(path); + if (client) { + cacheDir ??= defaultCacheDirectory(client.whoami().userID); + await client.logoutOnServer(); + client.logout(); + } + } catch (err) { + failure = err instanceof Error ? err.message : String(err); + } + unlinkSync(path); + if (failure === undefined) { + ctx.stderr.write("Session ended on the server.\n"); + } else { + ctx.stderr.write( + `Could not end the session on the server: ${failure}\n`, + ); + } + ctx.stderr.write("Session deleted.\n"); + if (cacheDir !== undefined) { + ctx.stderr.write( + `Cache directory ${cacheDir} still holds decrypted data; delete it to remove that data.\n`, + ); + } + return failure === undefined ? 0 : 1; +}; + +export const collectionsCommand = async ( + ctx: CliContext, + opts: { json?: boolean }, +): Promise<number> => { + await init(); + const client = requireSession(ctx); + if (!client) return 1; + const lib = await openReadLibrary(ctx, client); + try { + // Force a server round-trip and list in enumeration order (issue #36 + // amendment, issue #52): the pre-library CLI printed current state in + // this order, not the albums projection's newest-first order. + const collections = await freshCollections(lib); + + if (opts.json) { + ctx.stdout.write( + JSON.stringify( + collections.map((c) => ({ + id: c.id, + name: c.name, + type: c.type, + ownerID: c.ownerID, + isShared: c.isShared, + updationTime: c.updationTime, + })), + null, + 2, + ) + "\n", + ); + } else { + for (const c of collections) { + ctx.stdout.write( + `${c.id}\t${c.type}\t${c.name}${c.isShared ? " (shared)" : ""}\n`, + ); + } + } + return 0; + } finally { + await lib.close(); + } +}; + +export const filesCommand = async ( + ctx: CliContext, + opts: { collection: string; json?: boolean }, +): Promise<number> => { + await init(); + const client = requireSession(ctx); + if (!client) return 1; + const collectionID = Number(opts.collection); + if (!Number.isFinite(collectionID)) { + ctx.stderr.write("Invalid collection ID\n"); + return 1; + } + + const lib = await openReadLibrary(ctx, client); + try { + // Force a server round-trip and list in enumeration order (issue #36 + // amendment, issue #52). Each file prints from its own decrypted + // metadata (raw title, microsecond creationTime) via cli-output, and in + // the pre-library CLI's enumeration order, not the projection's + // newest-first order. + const files = await freshFiles(lib, collectionID); + if (!files) { + ctx.stderr.write(`Collection ${collectionID} not found\n`); + return 1; + } + + if (opts.json) { + ctx.stdout.write( + JSON.stringify(files.map(fileListRow), null, 2) + "\n", + ); + } else { + for (const file of files) { + ctx.stdout.write(fileListLine(file) + "\n"); + } + } + return 0; + } finally { + await lib.close(); + } +}; + +export const getCommand = async ( + ctx: CliContext, + fileIDStr: string, + opts: { out?: string }, +): Promise<number> => { + await init(); + const client = requireSession(ctx); + if (!client) return 1; + const fileID = Number(fileIDStr); + if (!Number.isFinite(fileID)) { + ctx.stderr.write("Invalid file ID\n"); + return 1; + } + + const lib = await openReadLibrary(ctx, client); + try { + // Force a server round-trip so the file resolves against current state + // (issue #36 amendment, issue #52). + const resolved = await freshFile(lib, fileID); + if (!resolved) { + ctx.stderr.write(`File ${fileID} not found\n`); + return 1; + } + const { photo, file } = resolved; + + const result = await photo.original(); + // Default name is the file's own title, as the pre-library CLI used + // (not the editedName-preferring projection title) (issue #52). + const outPath = opts.out ?? originalName(file); + if (result.videoPath === undefined) { + copyFileSync(result.path, outPath); + ctx.stderr.write(`${result.bytes} bytes -> ${outPath}\n`); + return 0; + } + // A live photo is written as its image and its video, each named after + // the title with its own extension, as Ente's clients name them. With + // --out, the image goes there and the video beside it. + const imageOut = + opts.out ?? withExtension(outPath, extname(result.path)); + const videoOut = withExtension(outPath, extname(result.videoPath)); + if (imageOut.toLowerCase() === videoOut.toLowerCase()) { + ctx.stderr.write( + `File ${fileID} is a live photo, and its video would also be written to ${imageOut}\n`, + ); + return 1; + } + copyFileSync(result.path, imageOut); + copyFileSync(result.videoPath, videoOut); + ctx.stderr.write( + `${result.bytes} bytes -> ${imageOut}\n` + + `${statSync(videoOut).size} bytes -> ${videoOut}\n`, + ); + return 0; + } finally { + await lib.close(); + } +}; + +export const getThumbCommand = async ( + ctx: CliContext, + fileIDStr: string, + opts: { out?: string }, +): Promise<number> => { + await init(); + const client = requireSession(ctx); + if (!client) return 1; + const fileID = Number(fileIDStr); + if (!Number.isFinite(fileID)) { + ctx.stderr.write("Invalid file ID\n"); + return 1; + } + + const lib = await openReadLibrary(ctx, client); + try { + // Force a server round-trip so the file resolves against current state + // (issue #36 amendment, issue #52). + const resolved = await freshFile(lib, fileID); + if (!resolved) { + ctx.stderr.write(`File ${fileID} not found\n`); + return 1; + } + const { photo, file } = resolved; + + const result = await photo.thumbnail(); + // Default name is thumb_<file's own title>, as the pre-library CLI + // used (not the projection title) (issue #52). + const outPath = opts.out ?? thumbnailName(file); + copyFileSync(result.path, outPath); + ctx.stderr.write(`${result.bytes} bytes -> ${outPath}\n`); + return 0; + } finally { + await lib.close(); + } +}; + +export const backupMetadataCommand = async ( + ctx: CliContext, + dir: string, + opts: { exif?: boolean; all?: boolean }, +): Promise<number> => { + await init(); + const client = requireSession(ctx); + if (!client) return 1; + const lib = await openReadLibrary(ctx, client); + try { + // Refresh first so the dump holds current account state, not what the + // cache last held; a failed refresh throws. + await lib.fresh(); + const { failedMLBatches } = await runMetadataBackup(lib, client, dir, { + exif: opts.exif || opts.all, + onProgress: (msg) => ctx.stderr.write(msg + "\n"), + }); + return failedMLBatches > 0 ? 1 : 0; + } finally { + await lib.close(); + } +}; + +export const backupCommand = async ( + ctx: CliContext, + dir: string, + opts: { json?: boolean }, +): Promise<number> => { + await init(); + const client = requireSession(ctx); + if (!client) return 1; + + ctx.stderr.write("Starting backup...\n"); + // The precache is off: the backup fetches what it needs, and must not + // also fill the cache with every thumbnail and the recent originals. + const lib = await Library.open({ + client, + downloadDirectory: dir, + cacheDirectory: ctx.cacheDir, + precacheThumbnails: false, + precacheOriginals: false, + }); + try { + const result = await lib.backup({ + downloadDirectory: dir, + onProgress: (msg) => { + if (!opts.json) ctx.stderr.write(msg + "\n"); + }, + }); + + if (opts.json) { + ctx.stdout.write(JSON.stringify(result, null, 2) + "\n"); + } else { + ctx.stderr.write("\n--- Backup complete ---\n"); + ctx.stderr.write(` Total files: ${result.totalFiles}\n`); + ctx.stderr.write(` Downloaded: ${result.downloaded}\n`); + ctx.stderr.write(` Skipped: ${result.skipped}\n`); + ctx.stderr.write(` Failed: ${result.failed}\n`); + if (result.errors.length > 0) { + ctx.stderr.write("\nFailed files:\n"); + for (const e of result.errors) { + ctx.stderr.write( + ` [${e.collection}] ${e.title} (id ${e.fileID}): ${e.error}\n`, + ); + } + } + } + + return result.failed > 0 ? 1 : 0; + } finally { + await lib.close(); + } +}; + +export const listMissingThumbnailsCommand = async ( + ctx: CliContext, + opts: { json?: boolean }, +): Promise<number> => { + await init(); + const client = requireSession(ctx); + if (!client) return 1; + const lib = await openReadLibrary(ctx, client); + try { + // Refresh first so files added since the cache was written are + // checked; a failed refresh throws. + await lib.fresh(); + const missing = await listMissingThumbnails(lib, client, (msg) => { + if (!opts.json) ctx.stderr.write(msg + "\n"); + }); + + if (opts.json) { + ctx.stdout.write(JSON.stringify(missing, null, 2) + "\n"); + } else { + if (missing.length === 0) { + ctx.stderr.write("No missing thumbnails found.\n"); + } else { + ctx.stderr.write( + `\n${missing.length} file(s) with missing thumbnails:\n`, + ); + for (const m of missing) { + ctx.stdout.write( + `${m.fileID}\t${m.title}\t${m.collection}\t${m.reason}\n`, + ); + } + } + } + return 0; + } finally { + await lib.close(); + } +}; + +export const fixMissingThumbnailsCommand = async ( + ctx: CliContext, + opts: { file?: string[]; json?: boolean }, +): Promise<number> => { + await init(); + const client = requireSession(ctx); + if (!client) return 1; + const lib = await openReadLibrary(ctx, client); + try { + // Refresh first so files added since the cache was written are found; + // a failed refresh throws. + await lib.fresh(); + let fileIDs: number[]; + if (opts.file && opts.file.length > 0) { + fileIDs = opts.file.map(Number).filter(Number.isFinite); + } else { + ctx.stderr.write("Scanning for missing thumbnails...\n"); + const missing = await listMissingThumbnails(lib, client, (msg) => { + if (!opts.json) ctx.stderr.write(msg + "\n"); + }); + fileIDs = missing.map((m) => m.fileID); + if (fileIDs.length === 0) { + ctx.stderr.write("No missing thumbnails found.\n"); + return 0; + } + ctx.stderr.write(`Found ${fileIDs.length} file(s) to fix.\n`); + } + + const results = await fixMissingThumbnails( + lib, + client, + fileIDs, + (msg) => { + if (!opts.json) ctx.stderr.write(msg + "\n"); + }, + ); + + if (opts.json) { + ctx.stdout.write(JSON.stringify(results, null, 2) + "\n"); + } else { + const fixed = results.filter((r) => r.status === "fixed").length; + const skipped = results.filter( + (r) => r.status === "skipped", + ).length; + const failed = results.filter((r) => r.status === "failed").length; + ctx.stderr.write(`\n--- Done ---\n`); + ctx.stderr.write(` Fixed: ${fixed}\n`); + ctx.stderr.write(` Skipped: ${skipped}\n`); + ctx.stderr.write(` Failed: ${failed}\n`); + if (skipped > 0) { + ctx.stderr.write("\nSkipped:\n"); + for (const r of results.filter((r) => r.status === "skipped")) { + ctx.stderr.write( + ` ${r.fileID}\t${r.title}\t${r.reason}\n`, + ); + } + } + if (failed > 0) { + ctx.stderr.write("\nFailed files:\n"); + for (const r of results.filter((r) => r.status === "failed")) { + ctx.stderr.write( + ` ${r.fileID}\t${r.title}\t${r.reason}\n`, + ); + } + } + } + + return results.some((r) => r.status === "failed") ? 1 : 0; + } finally { + await lib.close(); + } +}; diff --git a/src/cli-output.ts b/src/cli-output.ts new file mode 100644 index 0000000..2bf9b21 --- /dev/null +++ b/src/cli-output.ts @@ -0,0 +1,43 @@ +// How the CLI presents a file's identity in `files`, `get`, and `get-thumb`. +// +// These read the file's own decrypted metadata — the raw title and the +// creationTime in microseconds — rather than the `PhotoRecord` projection the +// rest of the library exposes. The projection prefers `editedName`/`editedTime` +// and reports time in milliseconds, which is right for a photo browser but +// would change the CLI's externally-visible output. The pre-library CLI printed +// `metadata.title` and `metadata.creationTime` and named downloads after +// `metadata.title`, and issue #52 requires that output stay byte-identical, so +// the commands shape their output from the raw `EnteFile` through here. + +import { sanitizeFileName } from "./filename.js"; +import type { EnteFile, FileType, Microseconds } from "./model/types.js"; + +// One row of `quak files --json`. +export interface FileListRow { + id: number; + title: string; + fileType: FileType; + creationTime: Microseconds; + collectionID: number; +} + +export const fileListRow = (file: EnteFile): FileListRow => ({ + id: file.id, + title: file.metadata.title, + fileType: file.metadata.fileType, + creationTime: file.metadata.creationTime, + collectionID: file.collectionID, +}); + +// One line of `quak files` in its human, tab-separated form. +export const fileListLine = (file: EnteFile): string => + `${file.id}\t${file.metadata.fileType}\t${file.metadata.title}`; + +// Default output path for `quak get` when `--out` is not given. The title comes +// from the server, so it is sanitized; `--out` is the user's and is used as is. +export const originalName = (file: EnteFile): string => + sanitizeFileName(file.metadata.title, `file-${file.id}`); + +// Default output path for `quak get-thumb` when `--out` is not given. +export const thumbnailName = (file: EnteFile): string => + `thumb_${originalName(file)}`; diff --git a/src/cli-read.ts b/src/cli-read.ts new file mode 100644 index 0000000..6f3c4ad --- /dev/null +++ b/src/cli-read.ts @@ -0,0 +1,62 @@ +// How the CLI's read commands obtain current data. +// +// `collections`, `files --collection`, `get`, and `get-thumb` must answer for +// the account's state at the moment the command runs, not for whatever the +// local cache last happened to hold (owner amendment, issue #36). Each helper +// therefore forces a server round-trip through `Library.fresh()` and only then +// reads — so a collection, file, or metadata change made elsewhere is visible. +// +// `collections` and `files` also list in the library's own enumeration order — +// `listCollections()`/`listFiles()`, the order the pre-library CLI printed — +// rather than the `albums`/`photos` projection's newest-first order, which +// re-sorts the rows. The field values still come from each record's raw +// metadata via `cli-output.ts`. + +import type { Collection, EnteFile } from "./model/types.js"; +import type { Photo, PhotosAPI } from "./library/index.js"; + +// The slice of `Library` these helpers read. `Library` satisfies it +// structurally; a test can drive them with a stand-in that records the +// `fresh()` call and serves records in a known enumeration order. +export interface FreshReadLibrary { + fresh(): Promise<unknown>; + listCollections(): Collection[]; + getCollection(id: number): Collection | undefined; + listFiles(collectionID: number): EnteFile[]; + getFileByID(fileID: number): EnteFile | undefined; + photos: Pick<PhotosAPI, "byID">; +} + +// Every live collection, current as of a forced refresh, in enumeration order. +export const freshCollections = async ( + lib: FreshReadLibrary, +): Promise<Collection[]> => { + await lib.fresh(); + return lib.listCollections(); +}; + +// The files of one collection, current as of a forced refresh, in enumeration +// order. `undefined` (not an empty list) when the collection does not exist, so +// the caller can tell "no such collection" from "an empty collection". +export const freshFiles = async ( + lib: FreshReadLibrary, + collectionID: number, +): Promise<EnteFile[] | undefined> => { + await lib.fresh(); + if (!lib.getCollection(collectionID)) return undefined; + return lib.listFiles(collectionID); +}; + +// One file, current as of a forced refresh, resolved to both its content +// handle (`Photo`, for fetching bytes) and its raw record (`EnteFile`, for the +// default output name and field values). `undefined` when the file is unknown. +export const freshFile = async ( + lib: FreshReadLibrary, + fileID: number, +): Promise<{ photo: Photo; file: EnteFile } | undefined> => { + await lib.fresh(); + const photo = lib.photos.byID({ fileID }); + const file = lib.getFileByID(fileID); + if (!photo || !file) return undefined; + return { photo, file }; +}; diff --git a/src/cli-run.ts b/src/cli-run.ts new file mode 100644 index 0000000..c668b49 --- /dev/null +++ b/src/cli-run.ts @@ -0,0 +1,36 @@ +// Runs one CLI command for `bin/quak.ts` and exits with its code. + +import type { Writable } from "node:stream"; + +// Run a command and exit with its code once stdout/stderr have drained. +// Exiting before the drain can truncate piped output, and the library can keep +// the event loop alive after a command returns, so a plain return could hang. +// An error the command throws is printed as one `quak: MESSAGE` line, without +// the stack trace, and exits 1. +export const run = async ( + command: Promise<number>, + stdout: Writable, + stderr: Writable, + exit: (code: number) => void, +): Promise<void> => { + let code: number; + try { + code = await command; + } catch (err) { + stderr.write( + `quak: ${err instanceof Error ? err.message : String(err)}\n`, + ); + code = 1; + } + const pending = [stdout, stderr].filter((s) => s.writableLength > 0); + if (pending.length === 0) { + exit(code); + return; + } + let remaining = pending.length; + for (const s of pending) { + s.once("drain", () => { + if (--remaining === 0) exit(code); + }); + } +}; diff --git a/src/cli-session.ts b/src/cli-session.ts new file mode 100644 index 0000000..72228d2 --- /dev/null +++ b/src/cli-session.ts @@ -0,0 +1,26 @@ +// How the CLI reads its saved session file back into a `Client`. +// +// A missing file means "not logged in" and returns null. A file that exists but +// cannot be read back into a client (bad JSON, a missing field, a key of the +// wrong length) throws an error saying the session file is corrupt, so the CLI +// can tell the user which of the two it is. Needs `init()` first. + +import { existsSync, readFileSync } from "node:fs"; +import type { ApiClientOptions } from "./api/client.js"; +import { Client } from "./client.js"; + +export const loadSession = ( + path: string, + apiOptions?: ApiClientOptions, +): Client | null => { + if (!existsSync(path)) return null; + try { + return Client.fromJSON( + JSON.parse(readFileSync(path, "utf-8")), + apiOptions, + ); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + throw new Error(`Session file ${path} is corrupt: ${reason}`); + } +}; diff --git a/src/client.ts b/src/client.ts index a4d14ce..62784c1 100644 --- a/src/client.ts +++ b/src/client.ts @@ -7,11 +7,16 @@ import { } from "./auth/login.js"; import { unwrapAuth } from "./auth/unwrap.js"; import { init, fromBase64, toBase64 } from "./crypto/index.js"; +import { fetchMLDataBatch, type MLData } from "./mldata-fetch.js"; import { decryptCollection, decryptFile } from "./model/index.js"; import { downloadFile as dlFile, downloadThumbnail as dlThumb, } from "./download/index.js"; +import { + makeDownloadContentSource, + type ContentSource, +} from "./library/content.js"; import type { Collection, EnteFile, @@ -37,6 +42,22 @@ export interface ClientSnapshot { publicKey: string; } +// The result of a resumable enumeration. Live decrypted records and deleted +// ids are kept apart on purpose: a tombstone carries no key or metadata to +// decrypt, so it is a bare id rather than a hollowed-out record. `cursor` is +// the max `updationTime` seen, to pass back into the next call. +export interface CollectionsPage { + collections: Collection[]; + deleted: number[]; + cursor: number; +} + +export interface FilesPage { + files: EnteFile[]; + deleted: number[]; + cursor: number; +} + export class Client { private readonly api: ApiClient; private readonly email: string; @@ -104,18 +125,57 @@ export class Client { ); } - static fromJSON( - snapshot: ClientSnapshot, - apiOptions?: ApiClientOptions, - ): Client { - const api = new ApiClient({ ...apiOptions, authToken: snapshot.token }); + // Restore a client from a `toJSON()` snapshot. The snapshot usually comes + // straight from `JSON.parse` of a file on disk, so every field is checked + // before use; a bad one throws an error naming it. Needs `init()` first. + static fromJSON(snapshot: unknown, apiOptions?: ApiClientOptions): Client { + const invalid = (field: string, problem: string): Error => + new Error(`Invalid session data: ${field} ${problem}`); + + if (typeof snapshot !== "object" || snapshot === null) { + throw new Error("Invalid session data: not a JSON object"); + } + const s = snapshot as Record<string, unknown>; + for (const field of ["email", "token"]) { + if (typeof s[field] !== "string" || s[field] === "") { + throw invalid(field, "must be a non-empty string"); + } + } + if (!Number.isInteger(s.userID)) { + throw invalid("userID", "must be an integer"); + } + const key = (field: string): Uint8Array => { + const value = s[field]; + if (typeof value !== "string") { + throw invalid(field, "must be a base64 string"); + } + let bytes: Uint8Array; + try { + bytes = fromBase64(value); + } catch { + throw invalid(field, "is not valid base64"); + } + // The master key (secretbox) and the key pair (box) are all 32 bytes. + if (bytes.length !== 32) { + throw invalid( + field, + `must decode to 32 bytes, got ${bytes.length}`, + ); + } + return bytes; + }; + + const api = new ApiClient({ + ...apiOptions, + authToken: s.token as string, + }); return new Client( api, - snapshot.email, - snapshot.userID, - fromBase64(snapshot.masterKey), - fromBase64(snapshot.secretKey), - fromBase64(snapshot.publicKey), + s.email as string, + s.userID as number, + key("masterKey"), + key("secretKey"), + key("publicKey"), ); } @@ -124,6 +184,14 @@ export class Client { return this.api; } + // The content-cache byte source over this client's API: each fetch is the + // download layer's request + streaming decrypt + atomic write. `Library` + // calls this to enable the on-disk content cache. + contentSource(): ContentSource { + this.assertLoggedIn(); + return makeDownloadContentSource(this.api); + } + private assertLoggedIn(): void { if (this.loggedOut) throw new Error("Client has been logged out"); } @@ -135,67 +203,158 @@ export class Client { toJSON(): ClientSnapshot { this.assertLoggedIn(); + const token = this.api.getAuthToken(); + if (!token) { + throw new Error("Cannot serialize client: it has no auth token"); + } return { email: this.email, userID: this.userID, - token: this.api["token"]!, + token, masterKey: toBase64(this.masterKey), secretKey: toBase64(this.secretKey), publicKey: toBase64(this.publicKey), }; } + // Ends this client's session on the server (`POST /users/logout`), so the + // token stops working everywhere, including in any saved copy of it. This + // client is left as it was; call `logout()` to clear it. + async logoutOnServer(): Promise<void> { + this.assertLoggedIn(); + await this.api.postJSON("/users/logout", {}); + } + + // Zeroes the key buffers in place, so any copy of the reference held + // elsewhere is wiped too. Every method checks `assertLoggedIn` before + // touching the keys, so nothing decrypts with the zeroed keys. logout(): void { this.loggedOut = true; this.api.clearAuthToken(); + this.masterKey.fill(0); + this.secretKey.fill(0); + this.publicKey.fill(0); } - async listCollections(): Promise<Collection[]> { + // Enumerate collections changed since `sinceTime`. Live collections are + // decrypted; tombstoned ones (isDeleted) are surfaced as bare ids. The + // returned cursor is the max `updationTime` seen — including tombstones, so + // the next sync resumes past them — falling back to `sinceTime` when the + // response is empty. `/collections/v2` returns the whole changed set in one + // response, so there is no pagination here. + async collectionsSince(args: { + sinceTime: number; + }): Promise<CollectionsPage> { this.assertLoggedIn(); - const { collections } = await this.api.getJSON<{ + const { collections: raws } = await this.api.getJSON<{ collections: RawCollection[]; - }>("/collections/v2", { sinceTime: 0 }); - // The sync API keeps returning deleted collections as tombstones - // (isDeleted: true); their diff endpoint 404s, so drop them. - return collections - .filter((raw) => !raw.isDeleted) - .map((raw) => - decryptCollection( - raw, - { - masterKey: this.masterKey, - publicKey: this.publicKey, - secretKey: this.secretKey, - }, - this.userID, - ), - ); + }>("/collections/v2", { sinceTime: args.sinceTime }); + // logout() may have zeroed the keys while the request was in flight. + this.assertLoggedIn(); + + const collections: Collection[] = []; + const deleted: number[] = []; + let cursor = args.sinceTime; + for (const raw of raws) { + if (raw.isDeleted) { + deleted.push(raw.id); + } else { + collections.push( + decryptCollection( + raw, + { + masterKey: this.masterKey, + publicKey: this.publicKey, + secretKey: this.secretKey, + }, + this.userID, + ), + ); + } + if (raw.updationTime > cursor) cursor = raw.updationTime; + } + return { collections, deleted, cursor }; } - async listFiles( - collectionID: number, - collectionKey: Uint8Array, - ): Promise<EnteFile[]> { + // Enumerate a collection's files changed since `sinceTime`, paginating the + // diff from that cursor. Live rows are decrypted; tombstoned ones are + // surfaced as bare ids. Returns the final cursor to resume from. + async filesSince(args: { + collectionID: number; + collectionKey: Uint8Array; + sinceTime: number; + }): Promise<FilesPage> { this.assertLoggedIn(); - const allFiles: EnteFile[] = []; - let sinceTime = 0; + const { collectionID, collectionKey } = args; + const files: EnteFile[] = []; + const deleted: number[] = []; + let cursor = args.sinceTime; for (;;) { const { diff, hasMore } = await this.api.getJSON<{ diff: RawEnteFile[]; hasMore: boolean; - }>("/collections/v2/diff", { collectionID, sinceTime }); + }>("/collections/v2/diff", { collectionID, sinceTime: cursor }); + let pageMax = cursor; for (const raw of diff) { - if (!raw.isDeleted) { - allFiles.push(decryptFile(raw, collectionKey)); - } - if (raw.updationTime > sinceTime) { - sinceTime = raw.updationTime; + if (raw.isDeleted) { + deleted.push(raw.id); + } else { + files.push(decryptFile(raw, collectionKey)); } + if (raw.updationTime > pageMax) pageMax = raw.updationTime; } - if (!hasMore) break; + + if (!hasMore) { + cursor = pageMax; + break; + } + // The server says there is more, but this page did not advance the + // cursor: following hasMore would refetch the same page forever + // (#7). Stop with a clear error instead of looping. + if (pageMax <= cursor) { + throw new Error( + `/collections/v2/diff for collection ${collectionID} ` + + `returned hasMore with a cursor that did not advance ` + + `(stuck at ${cursor}); refusing to loop`, + ); + } + cursor = pageMax; } - return allFiles; + return { files, deleted, cursor }; + } + + // Whole-account listing: every live collection, deletions hidden. A thin + // wrapper over `collectionsSince` from the beginning of time. + async listCollections(): Promise<Collection[]> { + const { collections } = await this.collectionsSince({ sinceTime: 0 }); + return collections; + } + + // Every live file in a collection, deletions hidden. A thin wrapper over + // `filesSince` from the beginning of time. + async listFiles( + collectionID: number, + collectionKey: Uint8Array, + ): Promise<EnteFile[]> { + const { files } = await this.filesSince({ + collectionID, + collectionKey, + sinceTime: 0, + }); + return files; + } + + // Fetch machine-learning data (face detections + CLIP embeddings) for up + // to a batch of files, each decrypted with its own key. One request; the + // library batches at `MLDATA_BATCH_SIZE` and schedules each batch through + // its metadata request pool. + async fetchMLData(args: { + fileIDs: number[]; + fileKeys: Map<number, Uint8Array>; + }): Promise<Map<number, MLData>> { + this.assertLoggedIn(); + return fetchMLDataBatch(this.api, args.fileIDs, args.fileKeys); } async downloadFile( diff --git a/src/crypto/hash.ts b/src/crypto/hash.ts new file mode 100644 index 0000000..b94d94f --- /dev/null +++ b/src/crypto/hash.ts @@ -0,0 +1,22 @@ +import sodium, { type StateAddress } from "libsodium-wrappers-sumo"; +import { toBase64 } from "./encoding.js"; + +// The content hash an uploading client records in a file's metadata: unkeyed +// BLAKE2b with a 64-byte output over the original's bytes, fed in chunks, as +// standard base64 with padding. Named after the upstream client's functions. +// The output length is read at call time for the same reason as +// `streamTagFinal` in stream.ts: libsodium sets its constants only once ready. + +export const chunkHashInit = (): StateAddress => + sodium.crypto_generichash_init(null, sodium.crypto_generichash_BYTES_MAX); + +export const chunkHashUpdate = (state: StateAddress, chunk: Uint8Array): void => + sodium.crypto_generichash_update(state, chunk); + +export const chunkHashFinal = (state: StateAddress): string => + toBase64( + sodium.crypto_generichash_final( + state, + sodium.crypto_generichash_BYTES_MAX, + ), + ); diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 4e3e53b..d93af09 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -7,6 +7,7 @@ export { } from "./encoding.js"; export { deriveKEK, deriveLoginSubkey } from "./kdf.js"; export { decryptBox, decryptSealed } from "./box.js"; +export { chunkHashFinal, chunkHashInit, chunkHashUpdate } from "./hash.js"; export { decryptBlob, encryptBlob, diff --git a/src/download/index.ts b/src/download/index.ts index bda3ba7..7729890 100644 --- a/src/download/index.ts +++ b/src/download/index.ts @@ -1,7 +1,13 @@ -import { randomUUID } from "node:crypto"; -import { rename, rm, writeFile } from "node:fs/promises"; +import { randomBytes } from "node:crypto"; +import { readdirSync, rmSync } from "node:fs"; +import { open, rename, rm } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; import { dirname, join } from "node:path"; +import { Unzip, UnzipInflate } from "fflate"; import { + chunkHashFinal, + chunkHashInit, + chunkHashUpdate, fromBase64, initStreamPull, pullStreamChunk, @@ -10,82 +16,149 @@ import { streamTagFinal, } from "../crypto/index.js"; import { TruncatedStreamError } from "../errors.js"; +import { safeExtension, sanitizeFileName, withExtension } from "../filename.js"; import { withRetry } from "../retry.js"; import type { ApiClient } from "../api/client.js"; import type { EnteFile } from "../model/types.js"; export interface DownloadResult { + // Where the file was written. A live photo is written as two files, its + // image here and its video at `videoPath` (see `decryptLivePhoto`). path: string; + // The decrypted length; for a live photo, that of the ZIP it arrives as. bytesWritten: number; + videoPath?: string; } +// Fired as decrypted plaintext accumulates, with the running total of +// plaintext bytes recovered so far. Within one download it is non-decreasing +// and its last value equals the final `bytesWritten`. A retry restarts the +// file from byte zero (see `fetchAndDecrypt`), so a fresh attempt begins its +// own count from zero. +export type ProgressCallback = (bytesDone: number) => void; + const ENC_CHUNK_SIZE = STREAM_CHUNK_SIZE + STREAM_CHUNK_OVERHEAD; +// Decrypt a secretstream body, handing each plaintext chunk to `sink` as it is +// produced rather than accumulating the whole file. Peak memory is one +// ciphertext chunk of network buffer plus one plaintext chunk — bounded by +// `STREAM_CHUNK_SIZE` regardless of the file's size — so a multi-gigabyte video +// no longer needs its size again in RAM. Returns the total plaintext length. +// +// The truncation contract is exactly the buffered version's, only the sink is +// new: a body cut short still decrypts and authenticates up to its last whole +// chunk, so the absence of TAG_FINAL is the sole evidence it was cut short, and +// this throws rather than let a caller keep a short file. The sink has already +// seen those chunks by then; the callers (`decryptToTemp`, `decryptLivePhoto`) +// stage them in temp files that are renamed into place only on a clean return, +// so a throw leaves nothing on disk. const streamDecrypt = async ( stream: ReadableStream<Uint8Array>, header: Uint8Array, key: Uint8Array, -): Promise<Uint8Array> => { + sink: (plaintext: Uint8Array) => Promise<void>, + onProgress?: ProgressCallback, +): Promise<number> => { const state = initStreamPull(header, key); const reader = stream.getReader(); - let buffer = new Uint8Array(0); - const plainChunks: Uint8Array[] = []; + // Incoming reads are held as-is and only stitched into a contiguous chunk + // at each `ENC_CHUNK_SIZE` boundary, so every received byte is copied once. + // Concatenating on each read instead — reallocating the whole accumulator + // per read — is O(n^2) in the bytes buffered, and for a 4 MiB chunk that + // memory churn dwarfs the libsodium decryption itself. + const pending: Uint8Array[] = []; + let pendingBytes = 0; let totalPlain = 0; let chunksPulled = 0; let lastTag = -1; - for (;;) { - const { done, value } = await reader.read(); - if (value) { - const merged = new Uint8Array(buffer.length + value.length); - merged.set(buffer); - merged.set(value, buffer.length); - buffer = merged; - } - - while (buffer.length >= ENC_CHUNK_SIZE) { - const encChunk = buffer.slice(0, ENC_CHUNK_SIZE); - buffer = buffer.slice(ENC_CHUNK_SIZE); - const { plaintext, tag } = pullStreamChunk(state, encChunk); - plainChunks.push(plaintext); - totalPlain += plaintext.length; - chunksPulled++; - lastTag = tag; - } - - if (done) { - if (buffer.length > 0) { - // Whatever is left over once every whole chunk has been - // consumed must be the stream's final chunk, and a final - // chunk that actually arrived in full authenticates. If it - // does not, the body stopped part-way through a chunk — the - // ordinary shape of a dropped connection. Poly1305 cannot - // tell a partial chunk from a corrupt one, so this is - // reported as the truncation it almost always is, with the - // authentication failure kept as the error's cause. - let pulled; - try { - pulled = pullStreamChunk(state, buffer); - } catch (err) { - throw new TruncatedStreamError( - `download: stream truncated: response body ended with ${buffer.length} trailing bytes that did not authenticate as a final chunk (transfer stopped mid-chunk, or the data is corrupt)`, - { cause: err }, - ); - } - plainChunks.push(pulled.plaintext); - totalPlain += pulled.plaintext.length; - chunksPulled++; - lastTag = pulled.tag; + // Remove the first `size` bytes from `pending` as one contiguous buffer. + // A read that straddles the boundary is split with `subarray` (a view, no + // copy); its tail stays queued for the next chunk. `size` never exceeds + // `pendingBytes`, so the queue always holds enough. + const takeContiguous = (size: number): Uint8Array => { + const out = new Uint8Array(size); + let offset = 0; + while (offset < size) { + const piece = pending[0]!; + const need = size - offset; + if (piece.length <= need) { + out.set(piece, offset); + offset += piece.length; + pending.shift(); + } else { + out.set(piece.subarray(0, need), offset); + pending[0] = piece.subarray(need); + offset += need; } - break; } + pendingBytes -= size; + return out; + }; + + const consume = async ( + plaintext: Uint8Array, + tag: number, + ): Promise<void> => { + await sink(plaintext); + totalPlain += plaintext.length; + chunksPulled++; + lastTag = tag; + onProgress?.(totalPlain); + }; + + try { + for (;;) { + const { done, value } = await reader.read(); + if (value && value.length > 0) { + pending.push(value); + pendingBytes += value.length; + } + + while (pendingBytes >= ENC_CHUNK_SIZE) { + const encChunk = takeContiguous(ENC_CHUNK_SIZE); + // A whole chunk that fails to authenticate while the stream + // carries on is corruption, not truncation; that error + // propagates unchanged. + const { plaintext, tag } = pullStreamChunk(state, encChunk); + await consume(plaintext, tag); + } + + if (done) { + if (pendingBytes > 0) { + const buffer = takeContiguous(pendingBytes); + // Whatever is left over once every whole chunk has been + // consumed must be the stream's final chunk, and a final + // chunk that actually arrived in full authenticates. If + // it does not, the body stopped part-way through a chunk + // — the ordinary shape of a dropped connection. Poly1305 + // cannot tell a partial chunk from a corrupt one, so this + // is reported as the truncation it almost always is, with + // the authentication failure kept as the error's cause. + // Only the pull is guarded: a sink failure on a chunk + // that did authenticate is a disk error, not a + // truncation. + let pulled; + try { + pulled = pullStreamChunk(state, buffer); + } catch (err) { + throw new TruncatedStreamError( + `download: stream truncated: response body ended with ${buffer.length} trailing bytes that did not authenticate as a final chunk (transfer stopped mid-chunk, or the data is corrupt)`, + { cause: err }, + ); + } + await consume(pulled.plaintext, pulled.tag); + } + break; + } + } + } finally { + reader.releaseLock(); } // Only the last chunk of a secretstream carries TAG_FINAL. Everything a // dropped connection did deliver still decrypts and authenticates, so the // absence of TAG_FINAL is the only evidence that the body was cut short. - // Returning a short plaintext here would put a corrupt file on disk that - // later backup runs would treat as complete. if (chunksPulled === 0) { throw new TruncatedStreamError( "download: stream truncated: response body contained no secretstream chunks", @@ -97,31 +170,102 @@ const streamDecrypt = async ( `download: stream truncated: last chunk tag ${lastTag}, expected TAG_FINAL (${tagFinal})`, ); } - - const result = new Uint8Array(totalPlain); - let offset = 0; - for (const chunk of plainChunks) { - result.set(chunk, offset); - offset += chunk.length; - } - return result; + return totalPlain; }; -// Write `plaintext` to `destination` atomically: stage it in a temporary -// sibling file (same directory, so the rename cannot cross a filesystem -// boundary) and rename it into place. Callers therefore never observe a -// partially written destination, and a pre-existing file at that path is -// replaced only once the new contents are complete on disk. -const writeAtomic = async ( - destination: string, - plaintext: Uint8Array, -): Promise<void> => { - // The random suffix keeps concurrent downloads of the same destination - // from stepping on each other's temporary file. - const tmpPath = join(dirname(destination), `.quak-${randomUUID()}.tmp`); +// Fsync a file or a directory, so its contents (for a directory, its entries) +// are on stable storage. Exported for the backup tree's copy, which needs the +// same durability as the writer below. +export const fsyncPath = async (path: string): Promise<void> => { + const handle = await open(path, "r"); try { - await writeFile(tmpPath, plaintext); + await handle.sync(); + } finally { + await handle.close(); + } +}; + +// A process-ID check: signal 0 delivers nothing and only reports whether the +// process exists. EPERM means it exists but belongs to another user. +const isRunning = (pid: number): boolean => { + try { + process.kill(pid, 0); + return true; + } catch (err) { + return (err as NodeJS.ErrnoException).code === "EPERM"; + } +}; + +// Delete the temp files a killed process left in `dir`: the writer's +// `.quak-<pid>-<random>.tmp` and the backup copy's +// `.quak-backup-<name>-<pid>-<random>.tmp`. Only files whose process is no +// longer running are removed, so another process writing into the same +// directory keeps its own. A reused process ID can only keep a leftover a while +// longer, never remove a live one. +export const removeLeftoverTempFiles = (dir: string): void => { + let names: string[]; + try { + names = readdirSync(dir); + } catch { + return; + } + for (const name of names) { + const match = /^\.quak-(?:.*-)?(\d+)-[0-9a-z]*\.tmp$/.exec(name); + if (match && !isRunning(Number(match[1]))) { + rmSync(join(dir, name), { force: true }); + } + } +}; + +// A new temp file name in `dir`. The random suffix keeps concurrent downloads +// of the same destination from stepping on each other's temporary file; the +// process ID lets `removeLeftoverTempFiles` tell a leftover from a write in +// progress. +const tempPathIn = (dir: string): string => + join(dir, `.quak-${process.pid}-${randomBytes(16).toString("hex")}.tmp`); + +// Stage a write to `destination` atomically and durably, then rename it into +// place. `fill` writes the contents into the open temp file handle — either the +// whole buffer at once (`writeAtomic`) or chunk by chunk as they decrypt +// (`decryptToTemp`). The temp file is a sibling of the destination (same +// directory, so the rename cannot cross a filesystem boundary), so callers +// never observe a partially written destination, and a pre-existing file is +// replaced only once the new contents are complete on disk. +// +// Durability against a power cut needs two fsyncs. Without them the write can +// return while the data or the rename is still only in the kernel's page +// cache, and a crash then resurrects an empty renamed file — exactly the +// corruption a later backup run treats as a complete download. So the temp +// file's contents are fsynced before the rename, and the containing directory +// is fsynced after it, so both the bytes and the new directory entry are on +// stable storage before this returns. +// +// On any failure — including a `fill` that throws because the stream was +// truncated — the temp file is removed, so the destination is untouched and no +// scratch file is left to fill the disk on repeated failures. +const stageAtomic = async ( + destination: string, + fill: (handle: FileHandle) => Promise<void>, +): Promise<void> => { + const dir = dirname(destination); + const tmpPath = tempPathIn(dir); + try { + const handle = await open(tmpPath, "w"); + try { + await fill(handle); + await handle.sync(); + } finally { + await handle.close(); + } + // `rename` replaces the destination's directory entry rather than + // writing through it: an existing symlink at `destination` is + // replaced, not followed, and the new file has the temp file's + // permissions, not those of the file it replaced. await rename(tmpPath, destination); + // Fsync the directory so the rename itself survives a crash: renaming + // over a synced temp file still leaves the new directory entry in the + // page cache until the directory is synced. + await fsyncPath(dir); } catch (err) { // Best-effort cleanup. A failure to remove the temporary file must // never replace the error that actually explains what went wrong. @@ -130,7 +274,239 @@ const writeAtomic = async ( } }; -// Fetch a stream and decrypt it, retrying the whole sequence. +// Write `plaintext` to `destination` atomically and durably. Exported so the +// metadata store can reuse the same durable write for small whole-buffer +// payloads; originals go through `decryptToTemp` instead so they never buffer. +export const writeAtomic = async ( + destination: string, + plaintext: Uint8Array, +): Promise<void> => + stageAtomic(destination, (handle) => handle.writeFile(plaintext)); + +// Refuse an original whose bytes do not hash to what its uploader recorded. +// The error is not retried. +const checkHash = (file: EnteFile, actual: string): void => { + if (actual !== file.metadata.hash) { + throw new Error( + `download: file ${file.id}: content hash ${actual} does not match the hash its uploader recorded, ${file.metadata.hash}`, + ); + } +}; + +// Decrypt `stream` straight to `destination`, one plaintext chunk at a time, +// under the atomic writer's temp-then-rename discipline. Memory stays bounded +// by the chunk size: each decrypted chunk is written to the temp file and +// dropped. The rename happens only after the stream authenticates as terminated +// on TAG_FINAL; a truncated stream throws and leaves the destination untouched. +// Returns the plaintext length written. +// +// `original` is the file whose original this is (none for a thumbnail, which +// has no recorded hash). When its metadata has a hash, the decrypted bytes are +// hashed as they stream and must match it, or nothing is stored. +const decryptToTemp = async ( + destination: string, + stream: ReadableStream<Uint8Array>, + header: Uint8Array, + key: Uint8Array, + onProgress?: ProgressCallback, + original?: EnteFile, +): Promise<number> => { + const hash = + original?.metadata.hash === undefined ? undefined : chunkHashInit(); + let bytesWritten = 0; + await stageAtomic(destination, async (handle) => { + bytesWritten = await streamDecrypt( + stream, + header, + key, + async (plaintext) => { + if (hash !== undefined) chunkHashUpdate(hash, plaintext); + await handle.write(plaintext); + }, + onProgress, + ); + if (original !== undefined && hash !== undefined) { + checkHash(original, chunkHashFinal(hash)); + } + }); + return bytesWritten; +}; + +// One of the two parts of a live photo being unpacked: the ZIP entry whose +// name starts with `kind`, written to its own temp file. +interface LivePhotoPart { + kind: "image" | "video"; + tmpPath: string; + handle: FileHandle; + hash: ReturnType<typeof chunkHashInit>; + // Decompressed bytes not yet written. + pending: Uint8Array[]; + // The entry's extension, set once all of the entry has been read. + ext?: string; +} + +const openPart = async ( + kind: "image" | "video", + dir: string, +): Promise<LivePhotoPart> => { + const tmpPath = tempPathIn(dir); + const handle = await open(tmpPath, "w"); + return { kind, tmpPath, handle, hash: chunkHashInit(), pending: [] }; +}; + +// A live photo arrives as a ZIP of its image and its video. Ente's clients +// name the entries `image.<ext>` and `video.<ext>`. This takes the entry whose +// name starts with `image` as the image and the one whose name starts with +// `video` as the video, and refuses a ZIP holding a second of either. It is +// written unpacked: each part is named `destination` with the extension +// replaced by its own entry's, and the two must differ ignoring case. When the +// file records a hash, `<imageHash>:<videoHash>` must match it, each over that +// part's own bytes. Only then is whatever was at `destination` removed and the +// image, then the video, renamed into place; on any failure neither is stored. +// +// The ZIP is chosen by its uploader and may expand enormously, so each part is +// written as it decompresses and never held, and the ZIP is refused once the +// two together come to more than 20 times its size plus 16 MiB, the limit +// Ente's mobile client and CLI set. The ZIP's size is known only at its end, +// so until then the limit is taken over the part of it decrypted so far. +// fflate's `Unzip` inflates each push in one piece before `push` returns, and +// deflate expands at most about 1000-fold, so the ZIP is pushed in 4 KiB +// slices, keeping each decompressed piece near 4 MiB, one plaintext chunk, and +// each piece is checked and written before the next slice is pushed. Every +// entry is started, even one that is not kept, because fflate keeps an +// unstarted entry's data in memory. +const decryptLivePhoto = async ( + destination: string, + stream: ReadableStream<Uint8Array>, + header: Uint8Array, + key: Uint8Array, + onProgress: ProgressCallback | undefined, + file: EnteFile, +): Promise<DownloadResult> => { + const sliceSize = 4096; + const dir = dirname(destination); + const fail = (message: string, cause?: unknown): Error => + new Error(`download: file ${file.id}: ${message}`, { cause }); + const parts: LivePhotoPart[] = []; + try { + const image = await openPart("image", dir); + parts.push(image); + const video = await openPart("video", dir); + parts.push(video); + + const claimed = new Set<LivePhotoPart>(); + // Set to a part the ZIP holds a second entry for; the ZIP is then + // refused. + let repeated: LivePhotoPart | undefined; + const unzip = new Unzip((entry) => { + const part = parts.find((p) => entry.name.startsWith(p.kind)); + if (part !== undefined) { + if (claimed.has(part)) repeated = part; + claimed.add(part); + } + entry.ondata = (err, data, final) => { + if (err) throw err; + if (part === undefined) return; + chunkHashUpdate(part.hash, data); + part.pending.push(data); + if (final) part.ext = safeExtension(entry.name); + }; + entry.start(); + }); + unzip.register(UnzipInflate); + // The bytes of the ZIP pushed so far, and of the two parts they have + // decompressed to. + let zipBytes = 0; + let expanded = 0; + const push = async ( + data: Uint8Array, + final: boolean, + ): Promise<void> => { + zipBytes += data.length; + // fflate reports a bad ZIP by throwing, sometimes a TypeError, + // which the retry would take for a network failure. A bad ZIP is + // never retried, and nor are the refusals below. + try { + unzip.push(data, final); + } catch (err) { + throw fail("live photo is not a readable ZIP", err); + } + if (repeated !== undefined) { + throw fail( + `live photo ZIP holds more than one ${repeated.kind}`, + ); + } + for (const part of parts) { + for (const piece of part.pending) expanded += piece.length; + } + if (expanded > 20 * zipBytes + 16 * 1024 * 1024) { + throw fail( + "live photo ZIP expands to more than 20 times its size plus 16 MiB", + ); + } + for (const part of parts) { + for (const piece of part.pending) + await part.handle.write(piece); + part.pending = []; + } + }; + + const bytesWritten = await streamDecrypt( + stream, + header, + key, + async (plaintext) => { + for (let i = 0; i < plaintext.length; i += sliceSize) { + await push(plaintext.subarray(i, i + sliceSize), false); + } + }, + onProgress, + ); + await push(new Uint8Array(0), true); + + if (image.ext === undefined || video.ext === undefined) { + throw fail( + "live photo ZIP does not hold both an image and a video", + ); + } + if (file.metadata.hash !== undefined) { + checkHash( + file, + `${chunkHashFinal(image.hash)}:${chunkHashFinal(video.hash)}`, + ); + } + if (image.ext.toLowerCase() === video.ext.toLowerCase()) { + throw fail( + `live photo's image and video have the same extension, ${video.ext}`, + ); + } + const path = withExtension(destination, image.ext); + const videoPath = withExtension(destination, video.ext); + for (const part of parts) { + await part.handle.sync(); + await part.handle.close(); + } + await rm(destination, { force: true }); + await rename(image.tmpPath, path); + try { + await rename(video.tmpPath, videoPath); + } catch (err) { + await rm(path, { force: true }).catch(() => undefined); + throw err; + } + await fsyncPath(dir); + return { path, bytesWritten, videoPath }; + } catch (err) { + // Best-effort cleanup, as in `stageAtomic`. + for (const part of parts) { + await part.handle.close().catch(() => undefined); + await rm(part.tmpPath, { force: true }).catch(() => undefined); + } + throw err; + } +}; + +// Fetch a stream and decrypt it to `destination`, retrying the whole sequence. // // The request is only the first third of a download. `getXStream` returns as // soon as headers arrive, and the bytes are pulled here, so a socket reset @@ -143,53 +519,97 @@ const writeAtomic = async ( // four attempts would mean sixteen requests for one file. The policy comes // from the client so a caller that configured one gets it here too. // -// A retry starts the file over from byte zero: the secretstream pull state is -// not resumable and there is no Range support on these endpoints. +// Because the plaintext is streamed to disk rather than buffered, the atomic +// write is part of the retried unit. A retry starts the file over from byte +// zero — the secretstream pull state is not resumable and there is no Range +// support — staging into a fresh temp file each time: a failed attempt writes +// and then removes its own temp file, and only the attempt that reaches +// TAG_FINAL renames one into place, so a download that needed three tries still +// performs exactly one rename over the destination. const fetchAndDecrypt = async ( api: ApiClient, openStream: () => Promise<ReadableStream<Uint8Array>>, header: Uint8Array, key: Uint8Array, -): Promise<Uint8Array> => + destination: string, + onProgress?: ProgressCallback, + original?: EnteFile, +): Promise<DownloadResult> => withRetry(async () => { const stream = await openStream(); - return streamDecrypt(stream, header, key); + try { + if (original?.metadata.fileType === "livePhoto") { + return await decryptLivePhoto( + destination, + stream, + header, + key, + onProgress, + original, + ); + } + const bytesWritten = await decryptToTemp( + destination, + stream, + header, + key, + onProgress, + original, + ); + return { path: destination, bytesWritten }; + } catch (err) { + // Cancel the body so its connection is closed now rather than held + // until the stream is garbage collected. A backup run carries on + // past a failed file, so without this every failure would hold a + // socket. This covers every failure, including a temp file that + // cannot be opened and a header that is rejected before the body + // is read. + await stream.cancel(err).catch(() => undefined); + throw err; + } }, api.getRetryOptions()); +// Write `file`'s original to `outPath`. A live photo is written as its image +// and its video beside `outPath` instead, and whatever was at `outPath` is +// removed (see `decryptLivePhoto`). export const downloadFile = async ( api: ApiClient, file: EnteFile, outPath?: string, + onProgress?: ProgressCallback, ): Promise<DownloadResult> => { - const resolvedPath = outPath ?? file.metadata.title; + // `outPath` is the caller's and is used as is; the title is the server's + // and is sanitized so it can only name a file in the current directory. + const resolvedPath = + outPath ?? sanitizeFileName(file.metadata.title, `file-${file.id}`); const header = fromBase64(file.file.decryptionHeader); - const plaintext = await fetchAndDecrypt( + return fetchAndDecrypt( api, () => api.getFileStream(file.id, { retry: false }), header, file.key, + resolvedPath, + onProgress, + file, ); - // Outside the retry, deliberately: only the attempt that produced a - // complete, authenticated plaintext gets to stage a temporary file, so a - // download that needed three tries still performs exactly one write and - // one rename. - await writeAtomic(resolvedPath, plaintext); - return { path: resolvedPath, bytesWritten: plaintext.length }; }; export const downloadThumbnail = async ( api: ApiClient, file: EnteFile, outPath?: string, + onProgress?: ProgressCallback, ): Promise<DownloadResult> => { - const resolvedPath = outPath ?? `thumb_${file.metadata.title}`; + const resolvedPath = + outPath ?? + `thumb_${sanitizeFileName(file.metadata.title, `file-${file.id}`)}`; const header = fromBase64(file.thumbnail.decryptionHeader); - const plaintext = await fetchAndDecrypt( + return fetchAndDecrypt( api, () => api.getThumbnailStream(file.id, { retry: false }), header, file.key, + resolvedPath, + onProgress, ); - await writeAtomic(resolvedPath, plaintext); - return { path: resolvedPath, bytesWritten: plaintext.length }; }; diff --git a/src/filename.ts b/src/filename.ts new file mode 100644 index 0000000..1e6250b --- /dev/null +++ b/src/filename.ts @@ -0,0 +1,41 @@ +// File names built from server-supplied metadata. +// +// A file's title and a collection's name are decrypted from data the server +// hands us, and quak does not trust the server. Any name taken from them and +// used on disk goes through here, so it can only ever name one file inside the +// directory the caller chose: never a path, never `..`, never hidden, never a +// Windows device name. +// +// A path the user typed (`--out`, `outPath`) is not passed through here: the +// caller is trusted, the server is not. + +import { extname } from "node:path"; + +// Path separators, characters Windows forbids in file names, and control +// characters (NUL included). +// eslint-disable-next-line no-control-regex +const UNSAFE_CHARACTERS = /[/\\:*?"<>|\x00-\x1f\x7f]/g; + +// Names Windows reserves for devices, with or without an extension. +const RESERVED_DEVICE_NAME = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\.|$)/i; + +// `name` made safe to use as a single file name. Each unsafe character becomes +// `_`, a leading run of dots becomes one `_`, and a device name gets a leading +// `_`. A name with none of these comes back unchanged. An empty name becomes +// `fallback`, which the caller derives from the record's ID. +export const sanitizeFileName = (name: string, fallback: string): string => { + if (name === "") return fallback; + const cleaned = name.replace(UNSAFE_CHARACTERS, "_").replace(/^\.+/, "_"); + return RESERVED_DEVICE_NAME.test(cleaned) ? `_${cleaned}` : cleaned; +}; + +// The extension of `title` (".jpg"), or ".bin" when it has none or it holds +// anything but letters and digits. +export const safeExtension = (title: string): string => { + const ext = extname(title); + return /^\.[A-Za-z0-9]+$/.test(ext) ? ext : ".bin"; +}; + +// `name` with its extension, if it has one, replaced by `ext` (".mov"). +export const withExtension = (name: string, ext: string): string => + name.slice(0, name.length - extname(name).length) + ext; diff --git a/src/index.ts b/src/index.ts index c53ceb9..a0a4186 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,16 @@ -export const VERSION = "0.0.0"; +// package.json is the one place the version is written. tsc copies it to +// dist/package.json, so this path resolves from source and from dist/src/. +import pkg from "../package.json" with { type: "json" }; -export { Client, type LoginOptions, type ClientSnapshot } from "./client.js"; +export const VERSION: string = pkg.version; + +export { + Client, + type LoginOptions, + type ClientSnapshot, + type CollectionsPage, + type FilesPage, +} from "./client.js"; export { ApiClient, ApiError, @@ -27,6 +37,53 @@ export { requestEmailOTP, submitEmailOTP, } from "./auth/login.js"; +export { + Library, + DEFAULT_REFRESH_INTERVAL_SECONDS, + Album, + Photo, + type LibraryClient, + type LibraryOptions, + type LibraryStatus, + type RefreshEvent, + type RefreshProgressCallback, + type AlbumsAPI, + type PhotosAPI, + type TimelineAPI, + type PhotoFilter, + type TimelineGroup, + type GroupBy, + type ContentSource, + type ContentResult, + type ContentEvent, + type ContentOptions, + type PhotoContent, + type ThumbnailsAPI, + type ThumbnailPriority, + type EnsureOptions, + type EnsureResult, + type EnsureEvent, + runBackup, + type BackupOptions, + type BackupResult, + type BackupError, +} from "./library/index.js"; +export { + RequestPools, + BoundedPool, + DEFAULT_METADATA_CONCURRENCY, + DEFAULT_CONTENT_CONCURRENCY, + DEFAULT_THUMBNAIL_CONCURRENCY, + type RequestPoolsOptions, + type Priority, + type RunOptions, +} from "./library/pools.js"; +export type { + AlbumRecord, + PhotoRecord, + LibrarySnapshot, + LibraryChange, +} from "./library/records.js"; export { decryptCollection, decryptFile } from "./model/index.js"; export { downloadFile, downloadThumbnail } from "./download/index.js"; export type { diff --git a/src/library/content.ts b/src/library/content.ts new file mode 100644 index 0000000..2353fef --- /dev/null +++ b/src/library/content.ts @@ -0,0 +1,894 @@ +// The on-disk content and thumbnail cache keyed by fileID (issue #46). +// +// Layout under `cacheDirectory`: `originals/<fileID>.<ext>` and +// `thumbnails/<fileID>.<ext>`, flat directories at 0700 with files at 0600. A +// live photo's original is two files, its image and its video, with +// `originals/<fileID>.livephoto.json` naming them. Content appears only by the +// streaming atomic writer's rename (the download layer, #40), so a file that +// exists is whole — "present means complete". The directory listing taken at +// `open()` is the record of what is cached, and the orphan temp files a crashed +// write may have left are reaped there. +// +// A fetch goes through the shared request pools (#45): the content pool for +// originals, the thumbnail pool for thumbnails. The pool limits concurrency, +// orders on-demand work ahead of background, and dedups by key so a fileID +// requested twice while the first is still in flight downloads once. +// +// Integrity. The reused streaming decrypt is the enforced guarantee: every +// chunk is authenticated and the writer renames the file into place only once +// the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and +// nothing is stored. For an original whose metadata records a content hash +// (`FileMetadata.hash`), the writer also hashes the decrypted bytes and stores +// nothing if they differ, failing the fetch with an error naming the file. An +// original with no recorded hash is stored unchecked, as the upstream client +// does; thumbnails have none. On top of that this module refuses to record a +// stored file that came out empty. + +import { + closeSync, + existsSync, + openSync, + readFileSync, + readSync, + statSync, +} from "node:fs"; +import { + chmod, + mkdir, + readdir, + rm, + stat, + statfs, + utimes, +} from "node:fs/promises"; +import { basename, dirname, extname, join } from "node:path"; + +import type { ApiClient } from "../api/client.js"; +import { + downloadFile, + downloadThumbnail, + type ProgressCallback, + removeLeftoverTempFiles, + writeAtomic, +} from "../download/index.js"; +import { safeExtension } from "../filename.js"; +import type { EnteFile } from "../model/types.js"; +import type { Priority, RequestPools } from "./pools.js"; + +const DIR_MODE = 0o700; +const FILE_MODE = 0o600; +const GIB = 1024 * 1024 * 1024; +// Owner ruling (#36): bound the originals cache at 100 GiB, but back off when +// the volume has under 50 GiB free so the cache never crowds the disk. +export const DEFAULT_ORIGINALS_MAX_BYTES = 100 * GIB; +export const DEFAULT_FREE_BELOW_BYTES = 50 * GIB; +// Ente thumbnails are always JPEG, so the cache stores them with a fixed +// extension rather than deriving one from the (image or video) title. +const THUMBNAIL_EXT = ".jpg"; + +type Kind = "original" | "thumbnail"; + +// An original write in progress, with the IDs of the concurrent original writes +// it overlaps (recorded both ways as writes begin, cleared when the write ends). +interface OriginalWrite { + fileID: number; + overlaps: Set<number>; +} + +// The priority a caller attaches to a thumbnail prefetch. The pool has two +// tiers, so this three-value surface collapses onto them: only a currently +// visible thumbnail preempts (on-demand); "ahead" prefetch and speculative +// "background" work both yield to it. +export type ThumbnailPriority = "visible" | "ahead" | "background"; + +const poolPriorityOf = (priority: ThumbnailPriority): Priority => + priority === "visible" ? "on-demand" : "background"; + +export interface ContentResult { + path: string; + bytes: number; + // A live photo's video. `path` and `bytes` are then its image's. + videoPath?: string; +} + +// Progress for a single `original`/`thumbnail` call. A present file emits one +// `skipped` event and nothing else; a fetched file emits `downloading` as +// plaintext lands and a final `done`. +export type ContentEvent = + | { status: "skipped"; bytes: number } + | { status: "downloading"; bytesDone: number } + | { status: "done"; bytes: number }; + +export interface ContentOptions { + onProgress?: (event: ContentEvent) => void; +} + +// The Photo-facing content surface (the read wrappers call these). The cache +// implements it; a library opened without a content source leaves it absent. +export interface PhotoContent { + original(fileID: number, opts?: ContentOptions): Promise<ContentResult>; + thumbnail(fileID: number, opts?: ContentOptions): Promise<ContentResult>; +} + +export interface EnsureResult { + fileID: number; + path?: string; + error?: string; +} + +export interface EnsureEvent { + fileID: number; + status: "skipped" | "done" | "failed" | "aborted"; + path?: string; + error?: string; +} + +export interface EnsureOptions { + fileIDs: number[]; + priority: ThumbnailPriority; + signal?: AbortSignal; + onProgress?: (event: EnsureEvent) => void; +} + +export interface ThumbnailsAPI { + ensure(args: EnsureOptions): Promise<EnsureResult[]>; +} + +// The byte source the cache fetches through. The real implementation streams +// and decrypts to the destination via the download layer; tests inject a +// stand-in so the cache logic runs with no crypto and no network. Pool routing, +// dedup, present-checks and integrity live in the cache, not here. +export interface ContentSource { + // Writes the original at `destination`. A live photo is written beside it + // as its image and its video instead, and their paths are returned, as + // `downloadFile` does. + original(args: { + file: EnteFile; + destination: string; + onProgress?: ProgressCallback; + }): Promise<{ bytesWritten: number; path?: string; videoPath?: string }>; + thumbnail(args: { + file: EnteFile; + destination: string; + onProgress?: ProgressCallback; + }): Promise<{ bytesWritten: number }>; +} + +// The production source: each fetch is the download layer's request + +// streaming decrypt + atomic write + retry as one unit. +export const makeDownloadContentSource = (api: ApiClient): ContentSource => ({ + original: ({ file, destination, onProgress }) => + downloadFile(api, file, destination, onProgress), + thumbnail: ({ file, destination, onProgress }) => + downloadThumbnail(api, file, destination, onProgress), +}); + +export interface CachedPaths { + originalPath?: string; + thumbnailPath?: string; +} + +// The slice of `fs.statfs` the eviction limit needs: `bavail` is the blocks +// available to an unprivileged writer and `bsize` their size, so +// `bavail * bsize` is the free byte count. Injectable so tests drive the +// adaptive limit without a real volume. +export interface StatFsResult { + bsize: number; + bavail: number; +} +export type StatFsFn = (path: string) => Promise<StatFsResult>; + +const realStatFs: StatFsFn = async (path) => { + const s = await statfs(path); + return { bsize: s.bsize, bavail: s.bavail }; +}; + +// The current usage and effective limit of the originals cache, in bytes. +// `limitBytes` is the adaptive ceiling last computed (see `originalsLimit`). +export interface OriginalsStatus { + usedBytes: number; + limitBytes?: number; +} + +export interface ContentCacheOptions { + pools: RequestPools; + source: ContentSource; + cacheDirectory: string; + // The backup destination (issue-level `downloadDirectory`). An original + // already stored there by a backup counts as present, so the cache serves + // it rather than fetching a second copy. + downloadDirectory?: string; + // Resolve any membership of a file; every membership shares the underlying + // content key, so any one decrypts the same bytes. + getFile: (fileID: number) => EnteFile | undefined; + // Hard ceiling on `cacheDirectory/originals` (default 100 GiB) and the free + // space to protect on the volume (default 50 GiB). The effective limit is + // the lesser of the ceiling and what fits above the protected free space. + cacheOriginalsMaxBytes?: number; + freeBelowBytes?: number; + // Whether an original is pinned (favorites + latest week; the precache unit + // #48 supplies the set). Pinned originals are never evicted; when only + // pinned originals remain the cache runs over-limit until the set shrinks. + isPinned?: (fileID: number) => boolean; + // Free-space probe on the volume holding `cacheDirectory`; defaults to the + // real `fs.statfs`. + statfs?: StatFsFn; +} + +// Thrown inside a pooled task to drop a queued fetch that was aborted before it +// started running. Never escapes `ensureThumbnails`. +class AbortDrop extends Error { + constructor() { + super("aborted"); + this.name = "AbortDrop"; + } +} + +// The name of a file's original in originals/: `<fileID><ext>`, the extension +// taken from the title (or `.bin`). A backup names its originals the same way. +export const nameInOriginals = (file: EnteFile): string => + `${file.id}${safeExtension(file.metadata.title)}`; + +// The fileID a cache filename encodes, or undefined when the name is not one +// the cache writes (`<digits><ext>`). +const fileIDFromName = (name: string): number | undefined => { + const base = name.slice(0, name.length - extname(name).length); + if (!/^\d+$/.test(base)) return undefined; + const id = Number(base); + return Number.isSafeInteger(id) ? id : undefined; +}; + +// Size of a regular file, or undefined if it is absent (or not a regular file). +const fileSize = (path: string): number | undefined => { + try { + const s = statSync(path); + return s.isFile() ? s.size : undefined; + } catch { + return undefined; + } +}; + +// Whether `path` is a regular file with content. A zero-byte file is the shape +// an aborted write leaves, so it does not count. +const hasContent = (path: string | undefined): boolean => + path !== undefined && (fileSize(path) ?? 0) > 0; + +// Whether the file at `path` begins as a ZIP does, with `PK\x03\x04`. False +// when it cannot be read. +const isZip = (path: string): boolean => { + try { + const fd = openSync(path, "r"); + try { + const head = Buffer.alloc(4); + return ( + readSync(fd, head, 0, 4, 0) === 4 && + head.toString("latin1") === "PK\x03\x04" + ); + } finally { + closeSync(fd); + } + } catch { + return false; + } +}; + +// A live photo's image and video are named with the extensions from inside its +// ZIP, so their names alone do not say which is which. Wherever the cache or a +// backup stores one, a JSON file of this name beside them names both. +const livePhotoJSONName = (fileID: number): string => + `${fileID}.livephoto.json`; + +// The image and video that the live photo's JSON file in `dir` names, or +// undefined when there is none. Only names of the form the cache writes are +// taken, so the file cannot point outside `dir`. +const readLivePhotoJSON = ( + dir: string, + fileID: number, +): { path: string; videoPath: string } | undefined => { + const valid = (name: unknown): name is string => + typeof name === "string" && name === `${fileID}${safeExtension(name)}`; + try { + const { image, video } = JSON.parse( + readFileSync(join(dir, livePhotoJSONName(fileID)), "utf-8"), + ); + if (valid(image) && valid(video)) { + return { path: join(dir, image), videoPath: join(dir, video) }; + } + } catch { + // No such file, or not one the cache wrote. + } + return undefined; +}; + +// Write the JSON file naming a live photo's image and video, both in `dir`. +export const writeLivePhotoJSON = ( + dir: string, + fileID: number, + stored: { path: string; videoPath: string }, +): Promise<void> => + writeAtomic( + join(dir, livePhotoJSONName(fileID)), + new TextEncoder().encode( + JSON.stringify({ + image: basename(stored.path), + video: basename(stored.videoPath), + }), + ), + ); + +// The original of `file` as the cache or a backup stored it in `dir`, when all +// of it is there: `<fileID><ext>`, or a live photo's image and video. +export const storedOriginal = ( + dir: string, + file: EnteFile, +): { path: string; videoPath?: string } | undefined => { + if (file.metadata.fileType !== "livePhoto") { + const path = join(dir, nameInOriginals(file)); + return hasContent(path) ? { path } : undefined; + } + const stored = readLivePhotoJSON(dir, file.id); + return stored !== undefined && + hasContent(stored.path) && + hasContent(stored.videoPath) + ? stored + : undefined; +}; + +export class ContentCache implements PhotoContent, ThumbnailsAPI { + private readonly pools: RequestPools; + private readonly source: ContentSource; + private readonly downloadDirectory?: string; + private readonly getFile: (fileID: number) => EnteFile | undefined; + private readonly originalsDir: string; + private readonly thumbnailsDir: string; + // fileID -> absolute path of the cached bytes, and for a live photo's + // original its video's, seeded from the directory listing at open() and + // extended as fetches store new files. + private readonly originals = new Map< + number, + { path: string; videoPath?: string } + >(); + private readonly thumbnails = new Map< + number, + { path: string; videoPath?: string } + >(); + private readonly maxOriginalsBytes: number; + private readonly freeBelowBytes: number; + private readonly isPinned: (fileID: number) => boolean; + private readonly statfs: StatFsFn; + // The last measured usage and effective limit, refreshed at open() and after + // every original write; exposed through `originalsStatus`. + private originalsUsedBytes = 0; + private originalsLimitBytes?: number; + // Serializes limit enforcement so concurrent original writes never race on + // the map or delete each other's just-freed room. + private enforcing: Promise<void> = Promise.resolve(); + // Original writes in progress. Writes for different files run concurrently + // (the content pool), so an eviction pass must never delete a file whose + // fetch has not yet returned. Each entry records the IDs of the concurrent + // original writes it overlaps — noted both ways as writes begin — and a + // write's eviction pass spares them all. Bounded by the pool's concurrency, + // so eviction is never deferred beyond the active working set. + private readonly inFlightOriginals = new Set<OriginalWrite>(); + + constructor(opts: ContentCacheOptions) { + this.pools = opts.pools; + this.source = opts.source; + this.downloadDirectory = opts.downloadDirectory; + this.getFile = opts.getFile; + this.originalsDir = join(opts.cacheDirectory, "originals"); + this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails"); + this.maxOriginalsBytes = + opts.cacheOriginalsMaxBytes ?? DEFAULT_ORIGINALS_MAX_BYTES; + this.freeBelowBytes = opts.freeBelowBytes ?? DEFAULT_FREE_BELOW_BYTES; + this.isPinned = opts.isPinned ?? (() => false); + this.statfs = opts.statfs ?? realStatFs; + } + + // Prepare the cache directories, reap orphan temp files, and take the + // record of what is already cached. Called once before the cache serves. + // `isLivePhoto` says which files are live photos, whose original is only + // the image and video their JSON file names. + async open( + isLivePhoto: (fileID: number) => boolean = () => false, + ): Promise<void> { + await this.ensureDir(this.originalsDir); + await this.ensureDir(this.thumbnailsDir); + await this.scan(this.originalsDir, this.originals, isLivePhoto); + await this.scan(this.thumbnailsDir, this.thumbnails, () => false); + // Publish the current usage and limit without evicting; a restart + // reuses whatever survived on disk. Eviction only ever fires on a write. + await this.refreshOriginalsLimit(); + } + + // The current originals usage and effective limit, both in bytes, as of the + // last write or open. `status().originalsLimitBytes` surfaces this. + originalsStatus(): OriginalsStatus { + return { + usedBytes: this.originalsUsedBytes, + limitBytes: this.originalsLimitBytes, + }; + } + + // The cache paths known for a file, for the record projection to expose as + // `originalPath`/`thumbnailPath`. + pathsFor(fileID: number): CachedPaths { + const out: CachedPaths = {}; + const original = this.originals.get(fileID); + if (original !== undefined) out.originalPath = original.path; + const thumbnail = this.thumbnails.get(fileID); + if (thumbnail !== undefined) out.thumbnailPath = thumbnail.path; + return out; + } + + async original( + fileID: number, + opts?: ContentOptions, + ): Promise<ContentResult> { + return this.get(fileID, "original", "on-demand", opts?.onProgress); + } + + async thumbnail( + fileID: number, + opts?: ContentOptions, + ): Promise<ContentResult> { + return this.get(fileID, "thumbnail", "on-demand", opts?.onProgress); + } + + // Get an original for a backup. One not present anywhere is written + // straight to `destination` and recorded there, so no second copy lands + // in the cache; one already present is returned where it is. + async backupOriginal( + fileID: number, + destination: string, + ): Promise<ContentResult> { + const result = await this.acquire( + fileID, + "original", + "on-demand", + undefined, + { destination }, + ); + return { + path: result.path, + bytes: result.bytes, + videoPath: result.videoPath, + }; + } + + async ensure(args: EnsureOptions): Promise<EnsureResult[]> { + return this.ensureThumbnails(args); + } + + async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> { + return this.ensureMany( + "thumbnail", + poolPriorityOf(args.priority), + args.fileIDs, + args.signal, + args.onProgress, + ); + } + + // Fill originals through the content pool for the precache (#48), always at + // background priority so an on-demand `original()` preempts the fill. A + // present original is a map lookup and no fetch; a per-file failure is + // returned, not thrown, so one bad file never halts a background sweep. + async ensureOriginals(args: { + fileIDs: number[]; + signal?: AbortSignal; + onProgress?: (event: EnsureEvent) => void; + }): Promise<EnsureResult[]> { + return this.ensureMany( + "original", + "background", + args.fileIDs, + args.signal, + args.onProgress, + ); + } + + private async ensureMany( + kind: Kind, + priority: Priority, + fileIDs: number[], + signal: AbortSignal | undefined, + onProgress: ((event: EnsureEvent) => void) | undefined, + ): Promise<EnsureResult[]> { + // Dedup the request list so a repeated fileID is fetched once and + // reported once, in first-requested order. + const seen = new Set<number>(); + const unique: number[] = []; + for (const id of fileIDs) { + if (!seen.has(id)) { + seen.add(id); + unique.push(id); + } + } + return Promise.all( + unique.map((fileID) => + this.ensureOne(fileID, kind, priority, signal, onProgress), + ), + ); + } + + private async ensureOne( + fileID: number, + kind: Kind, + priority: Priority, + signal: AbortSignal | undefined, + onProgress: ((event: EnsureEvent) => void) | undefined, + ): Promise<EnsureResult> { + try { + const result = await this.acquire(fileID, kind, priority, signal); + const status = result.cached ? "skipped" : "done"; + onProgress?.({ fileID, status, path: result.path }); + return { fileID, path: result.path }; + } catch (err) { + if (err instanceof AbortDrop) { + onProgress?.({ fileID, status: "aborted" }); + return { fileID, error: "aborted" }; + } + const error = err instanceof Error ? err.message : String(err); + onProgress?.({ fileID, status: "failed", error }); + return { fileID, error }; + } + } + + private async get( + fileID: number, + kind: Kind, + priority: Priority, + onProgress: ((event: ContentEvent) => void) | undefined, + ): Promise<ContentResult> { + const onByte: ProgressCallback | undefined = onProgress + ? (bytesDone) => onProgress({ status: "downloading", bytesDone }) + : undefined; + const result = await this.acquire(fileID, kind, priority, undefined, { + onByte, + }); + onProgress?.( + result.cached + ? { status: "skipped", bytes: result.bytes } + : { status: "done", bytes: result.bytes }, + ); + return { + path: result.path, + bytes: result.bytes, + videoPath: result.videoPath, + }; + } + + // The core: return the cached path if present, else fetch through the pool, + // store, and return it. `cached` distinguishes a present hit (no network, + // no download event) from a fresh fetch. A fetched original is stored at + // `opts.destination` when given, instead of in `originalsDir`. A live + // photo's original is present only with its video, and is returned with + // it. + private async acquire( + fileID: number, + kind: Kind, + priority: Priority, + signal: AbortSignal | undefined, + opts?: { onByte?: ProgressCallback; destination?: string }, + ): Promise<{ + path: string; + bytes: number; + videoPath?: string; + cached: boolean; + }> { + const file = this.getFile(fileID); + if (!file) throw new Error(`content cache: unknown file ${fileID}`); + const isLivePhoto = + kind === "original" && file.metadata.fileType === "livePhoto"; + + const known = kind === "original" ? this.originals : this.thumbnails; + const cached = known.get(fileID); + if (cached !== undefined) { + const size = fileSize(cached.path); + if ( + size !== undefined && + size > 0 && + (!isLivePhoto || hasContent(cached.videoPath)) + ) { + // Returning an original's path is a use: bump its mtime so LRU + // order reflects it and survives a restart with no ledger. + if ( + kind === "original" && + dirname(cached.path) === this.originalsDir + ) + await this.touch(cached.path); + return { ...cached, bytes: size, cached: true }; + } + // A recorded file that has since gone, or a live photo an earlier + // version stored as one ZIP, re-fetches below. + known.delete(fileID); + } + + // An original a backup already stored counts as present. + if (kind === "original" && this.downloadDirectory !== undefined) { + const stored = storedOriginal( + join(this.downloadDirectory, "originals"), + file, + ); + if (stored !== undefined) { + this.originals.set(fileID, stored); + return { + ...stored, + bytes: fileSize(stored.path) ?? 0, + cached: true, + }; + } + } + + const dir = + kind === "original" ? this.originalsDir : this.thumbnailsDir; + const dest = + kind === "original" + ? (opts?.destination ?? join(dir, nameInOriginals(file))) + : join(dir, `${fileID}${THUMBNAIL_EXT}`); + const pool = + kind === "original" ? this.pools.content : this.pools.thumbnails; + + return pool.run( + async () => { + // Dropping queued work on abort: a task still waiting for a slot + // when the signal fired sees it here and never touches the + // network. A task already past this point is in flight and runs + // to completion. + if (signal?.aborted) throw new AbortDrop(); + + // Register this original among those in flight, linking it with + // every sibling already writing so neither evicts the other's + // file. Non-null iff this is an original. + const write = + kind === "original" + ? this.beginOriginalWrite(fileID) + : null; + try { + const stored = await this.download( + file, + dest, + kind, + opts?.onByte, + ); + for (const path of [stored.path, stored.videoPath]) { + if (path === undefined) continue; + await chmod(path, FILE_MODE); + if ((await stat(path)).size === 0) { + throw new Error( + `content cache: ${kind} ${fileID} stored empty`, + ); + } + } + // A backup records its own live photos. + if ( + stored.videoPath !== undefined && + opts?.destination === undefined + ) { + await writeLivePhotoJSON(dir, fileID, { + path: stored.path, + videoPath: stored.videoPath, + }); + } + known.set(fileID, stored); + // A fresh original may have crossed the limit; make room by + // evicting least-recently-used originals. An over-budget + // fetch keeps the file it returns, and no overlapping + // sibling is evicted. Thumbnails are never bounded. + if (write) await this.enforceOriginalsLimit(write); + const size = (await stat(stored.path)).size; + return { ...stored, bytes: size, cached: false }; + } finally { + if (write) this.inFlightOriginals.delete(write); + } + }, + { priority, key: fileID }, + ); + } + + // Fetch into `destination`, returning where the bytes landed: there, or + // for a live photo, its image and video beside it. + private async download( + file: EnteFile, + destination: string, + kind: Kind, + onProgress: ProgressCallback | undefined, + ): Promise<{ path: string; videoPath?: string }> { + const args = { file, destination, onProgress }; + if (kind === "thumbnail") { + await this.source.thumbnail(args); + return { path: destination }; + } + const result = await this.source.original(args); + return { + path: result.path ?? destination, + videoPath: result.videoPath, + }; + } + + // Best-effort bump of a file's mtime to now; a failed touch must never fail + // the read it accompanies. + private async touch(path: string): Promise<void> { + const now = new Date(); + await utimes(path, now, now).catch(() => undefined); + } + + // Every stored original that lives under `originalsDir` (a backup-directory + // hit recorded in the map is excluded), with its size and mtime; a live + // photo's size includes its video. Entries whose file has vanished are + // dropped from the map. Backups and thumbnails are never counted. + private async measureOriginals(): Promise<{ + entries: { + fileID: number; + path: string; + videoPath?: string; + size: number; + mtimeMs: number; + }[]; + used: number; + }> { + const entries: { + fileID: number; + path: string; + videoPath?: string; + size: number; + mtimeMs: number; + }[] = []; + let used = 0; + for (const [fileID, { path, videoPath }] of this.originals) { + if (dirname(path) !== this.originalsDir) continue; + try { + const s = await stat(path); + const size = + s.size + + (videoPath === undefined + ? 0 + : (await stat(videoPath)).size); + entries.push({ + fileID, + path, + videoPath, + size, + mtimeMs: s.mtimeMs, + }); + used += size; + } catch { + this.originals.delete(fileID); + } + } + return { entries, used }; + } + + // The effective ceiling on originals: the configured max, but no more than + // what fits once the protected free space is set aside. `used + free` is the + // volume space the cache could occupy; subtracting `freeBelowBytes` leaves + // the reserve untouched. Clamped at zero. + private async originalsLimit(used: number): Promise<number> { + const { bsize, bavail } = await this.statfs(this.originalsDir); + const free = bsize * bavail; + const adaptive = used + free - this.freeBelowBytes; + return Math.max(0, Math.min(this.maxOriginalsBytes, adaptive)); + } + + // Recompute and publish usage and limit without evicting (used at open()). + private refreshOriginalsLimit(): Promise<void> { + return this.serializeEnforce(async () => { + const { used } = await this.measureOriginals(); + this.originalsUsedBytes = used; + this.originalsLimitBytes = await this.originalsLimit(used); + }); + } + + // Record a starting original write among those in flight, linking it with + // every sibling already writing so neither can evict the other's file. + private beginOriginalWrite(fileID: number): OriginalWrite { + const write: OriginalWrite = { fileID, overlaps: new Set() }; + for (const other of this.inFlightOriginals) { + write.overlaps.add(other.fileID); + other.overlaps.add(fileID); + } + this.inFlightOriginals.add(write); + return write; + } + + // Evict least-recently-used originals until usage fits the limit. Skipped: + // pinned originals, the file `write` just stored, and every original whose + // write overlaps it (`write.overlaps`). The last two spare any fetch whose + // lifetime overlaps this one, so concurrent over-budget fetches all keep the + // paths they return; when only such originals remain the cache stays + // over-limit until they settle, and a later, non-overlapping write finds + // them eligible again. + private enforceOriginalsLimit(write: OriginalWrite): Promise<void> { + return this.serializeEnforce(async () => { + const { entries, used } = await this.measureOriginals(); + const limit = await this.originalsLimit(used); + let remaining = used; + if (remaining > limit) { + const evictable = entries + .filter( + (e) => + e.fileID !== write.fileID && + !write.overlaps.has(e.fileID) && + !this.isPinned(e.fileID), + ) + .sort((a, b) => a.mtimeMs - b.mtimeMs); + for (const e of evictable) { + if (remaining <= limit) break; + await rm(e.path, { force: true }); + // A live photo goes whole: its video and the JSON file + // naming the two go with its image. + if (e.videoPath !== undefined) { + await rm(e.videoPath, { force: true }); + await rm( + join( + this.originalsDir, + livePhotoJSONName(e.fileID), + ), + { force: true }, + ); + } + this.originals.delete(e.fileID); + remaining -= e.size; + } + } + this.originalsUsedBytes = remaining; + this.originalsLimitBytes = limit; + }); + } + + // Run limit work one at a time; failures are swallowed so a transient + // statfs or unlink error never rejects the read or write that triggered it. + private serializeEnforce(work: () => Promise<void>): Promise<void> { + const next = this.enforcing.then(work).catch(() => undefined); + this.enforcing = next; + return next; + } + + private async ensureDir(dir: string): Promise<void> { + // chmod after mkdir so the mode is tightened even when the directory + // already existed with a looser one; mkdir alone would not. + await mkdir(dir, { recursive: true, mode: DIR_MODE }); + await chmod(dir, DIR_MODE); + } + + private async scan( + dir: string, + into: Map<number, { path: string; videoPath?: string }>, + isLivePhoto: (fileID: number) => boolean, + ): Promise<void> { + // Another process sharing this cache may still be writing its temp + // files, so only those whose process has exited are removed. + removeLeftoverTempFiles(dir); + let entries: string[]; + try { + entries = await readdir(dir); + } catch { + return; + } + const names = new Set(entries); + for (const name of entries) { + const id = fileIDFromName(name); + const path = join(dir, name); + if (id === undefined || !existsSync(path)) continue; + // A live photo's image and video are one entry, as the JSON file + // beside them names them. A live photo's file with no such JSON + // file is not its original. If it is a ZIP, it is the one an + // earlier version stored under the image's name, and is removed. + // Any other is left alone: another process may have just stored + // it and not yet written the JSON file. + const livePhoto = names.has(livePhotoJSONName(id)) + ? readLivePhotoJSON(dir, id) + : undefined; + if (livePhoto !== undefined) { + into.set(id, livePhoto); + } else if (isLivePhoto(id)) { + if (isZip(path)) { + await rm(path, { force: true }).catch(() => undefined); + } + } else { + into.set(id, { path }); + } + } + } +} diff --git a/src/library/index.ts b/src/library/index.ts new file mode 100644 index 0000000..92ddf33 --- /dev/null +++ b/src/library/index.ts @@ -0,0 +1,876 @@ +// The library surface over the local cache. +// +// `Library.open()` loads the on-disk metadata store (issue #41), then starts +// the refresh loop. When the cache loaded empty it awaits the first refresh, +// so the library never opens onto an empty store it could have filled; when an +// existing copy loaded, that first refresh runs in the background and `open()` +// returns as soon as the cached data is ready to serve — a slow or unreachable +// server no longer stalls opening. A background timer then refreshes every +// `refreshIntervalSeconds`. Every default read is answered from RAM — no +// default read touches the network. There is deliberately no `sync()`, no +// `refresh()`, no `serverReachable` flag, and no "before each read" mode +// (design #36). +// +// `fresh()` is the one exception (issue #75, an owner amendment to #36): it +// forces a refresh, awaits it, and only then hands back the read namespaces, so +// a caller that needs server-current data can ask for it. Concurrent `fresh()` +// calls coalesce onto one in-flight refresh, and a refresh that fails rejects +// the caller (the default reads stay silent and serve the last good copy). The +// default methods and the background loop are unchanged. +// +// A refresh stages all of its network work first and only mutates the store +// once every fetch has succeeded. A refresh that fails partway therefore never +// becomes visible to reads: the last good snapshot stays in place, and the +// failure surfaces through `onProgress` and `status()` instead. A commit that +// mutates RAM but then fails to persist keeps `status().lastError` set and the +// store marked unsaved until a later save actually lands, so a stuck disk is +// never masked by a subsequent empty refresh. + +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import envPaths from "env-paths"; + +import { MetadataStore } from "./store.js"; +import { MLDataStore } from "./mldata.js"; +import { RequestPools } from "./pools.js"; +import { + deriveRecords, + snapshotFrom, + diffRecords, + type DerivedRecords, + type LibrarySnapshot, + type LibraryChange, +} from "./records.js"; +import { + makeAlbumsAPI, + makePhotosAPI, + makeTimelineAPI, + type AlbumsAPI, + type PhotosAPI, + type TimelineAPI, + type FreshReads, +} from "./read.js"; +import { + ContentCache, + type ContentSource, + type ThumbnailsAPI, + type EnsureOptions, + type EnsureResult, +} from "./content.js"; +import { makeMLDataAPI, type MLDataAPI } from "./mlsearch.js"; +import { Precache } from "./precache.js"; + +export { + Album, + Photo, + type AlbumsAPI, + type PhotosAPI, + type TimelineAPI, + type FreshReads, + type PhotoFilter, + type TimelineGroup, + type GroupBy, +} from "./read.js"; +export { + type ContentSource, + type ContentResult, + type ContentEvent, + type ContentOptions, + type PhotoContent, + type ThumbnailsAPI, + type ThumbnailPriority, + type EnsureOptions, + type EnsureResult, + type EnsureEvent, +} from "./content.js"; +export { type MLDataAPI, type SimilarResult } from "./mlsearch.js"; +import type { CollectionsPage, FilesPage } from "../client.js"; +import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js"; +import type { Collection, EnteFile } from "../model/types.js"; +import { runBackup, type BackupOptions, type BackupResult } from "../backup.js"; + +export { + runBackup, + type BackupOptions, + type BackupResult, + type BackupError, +} from "../backup.js"; + +export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3; + +// The account's cache directory when `cacheDirectory` is not given: the +// env-paths cache directory plus the user id, so each account has its own. +export const defaultCacheDirectory = (userID: number): string => + join(envPaths("quak", { suffix: "" }).cache, String(userID)); + +// Project a metadata store into by-id records, filling each record's cache +// paths from the content cache when one is given. Shared by the live read +// projection and the precache's initial seeding at open(). +const deriveRecordsFromStore = ( + store: MetadataStore, + cache?: ContentCache, +): DerivedRecords => { + const collections = store.listCollections(); + const files: EnteFile[] = []; + for (const c of collections) files.push(...store.listFiles(c.id)); + return deriveRecords( + collections, + files, + cache ? (fileID) => cache.pathsFor(fileID) : undefined, + ); +}; + +// The slice of `Client` the library depends on. Narrowing to an interface lets +// tests drive a mock with no crypto or network; the real `Client` satisfies it +// structurally. +export interface LibraryClient { + whoami(): { email: string; userID: number }; + collectionsSince(args: { sinceTime: number }): Promise<CollectionsPage>; + filesSince(args: { + collectionID: number; + collectionKey: Uint8Array; + sinceTime: number; + }): Promise<FilesPage>; + // Fetch ML data (face detections + CLIP embeddings) for up to a batch of + // files. Optional: a client without it simply disables ML fetching, leaving + // the metadata refresh untouched. + fetchMLData?(args: { + fileIDs: number[]; + fileKeys: Map<number, Uint8Array>; + }): Promise<Map<number, MLData>>; + // The byte source for the on-disk content cache. Optional so a mock client + // that only serves metadata still satisfies the interface; when absent (and + // no explicit `contentSource` is passed to `open`) the content cache is + // disabled and `Photo.original`/`thumbnail` and `thumbnails.ensure` throw. + contentSource?(): ContentSource; +} + +// A progress event for one unit of background work. A metadata "refresh" or an +// ML "fetchMLData" pass each fire "started" before their network work and then +// exactly one of "done" or "failed"; "failed" carries the error message and +// an ML "done" reports how many payloads it stored. The precache fills +// ("precacheThumbnails"/"precacheOriginals", #48) fire "started"/"done" around +// each sweep that has work, "done" reporting the count newly cached. +export interface RefreshEvent { + operation: + | "refresh" + | "fetchMLData" + | "precacheThumbnails" + | "precacheOriginals"; + status: "started" | "done" | "failed"; + error?: string; + fetched?: number; +} + +export type RefreshProgressCallback = (event: RefreshEvent) => void; + +export interface LibraryOptions { + client: LibraryClient; + // Where `metadata.json` lives. Defaults to the env-paths cache directory + // plus the user id, so each account has its own cache. + cacheDirectory?: string; + // Persistent backup destination. The refresh loop does not use it; the + // content cache treats an original already stored there as present. + downloadDirectory?: string; + refreshIntervalSeconds?: number; + onProgress?: RefreshProgressCallback; + // The bounded request pools (issue #45), shared by the ML-data fetch (the + // metadata pool) and the content cache. Defaults to a fresh set at the + // design's caps. + pools?: RequestPools; + // Overrides the client's own `contentSource()`; mainly for tests that drive + // the cache with a stand-in source. + contentSource?: ContentSource; + // Bound on `cacheDirectory/originals` (default 100 GiB) and the free space + // to protect on its volume (default 50 GiB). The effective limit adapts + // down as the disk fills; `status().originalsLimitBytes` reports it. + cacheOriginalsMaxBytes?: number; + freeBelowBytes?: number; + // An extra pinned predicate OR-ed with the precache's own pinned set + // (favorites + latest week, #48). Pinned originals are never evicted. + isOriginalPinned?: (fileID: number) => boolean; + // The aggressive local precache (#48), all starting inside `open()` with no + // caller input. Thumbnails: every file, newest first, until all are on + // disk. Originals: the favorites album then the latest `precacheOriginalsDays` + // window (the days ending at the newest file). Both default on; the days + // default to 7. + precacheThumbnails?: boolean; + precacheOriginals?: boolean; + precacheOriginalsDays?: number; +} + +export interface LibraryStatus { + userID: number; + collections: number; + files: number; + // Wall-clock ms of the last refresh that succeeded, or undefined if none + // has yet. + lastRefreshAt?: number; + // The message from the most recent refresh, set only while that refresh + // failed; cleared by the next success. + lastError?: string; + // Wall-clock ms of the last ML fetch pass that succeeded, or undefined if + // none has yet (or ML fetching is disabled). + lastMLFetchAt?: number; + // The most recent ML fetch pass's error, set only while it failed. + lastMLError?: string; + // ML payloads stored on disk and CLIP embeddings in the index; undefined + // when ML fetching is disabled. + mlStored?: number; + mlIndexed?: number; + // Bytes stored in the originals cache and the effective size limit as of the + // last write or open; undefined when no content cache is open. + originalsUsedBytes?: number; + originalsLimitBytes?: number; + // Precache progress (#48); undefined when no content cache is open. Totals + // are the files targeted (0 when a fill is disabled); "cached" is how many + // of them are on disk. + thumbnailsCached?: number; + thumbnailsTotal?: number; + originalsCached?: number; + originalsPinned?: number; + closed: boolean; +} + +export class Library { + readonly cacheDirectory: string; + readonly downloadDirectory?: string; + + // The in-process read surface (issue #44). Each namespace answers + // synchronously from the live record projection; no read touches the + // network. + readonly albums: AlbumsAPI; + readonly photos: PhotosAPI; + readonly timeline: TimelineAPI; + // The thumbnail-prefetch surface (issue #46): drives the thumbnail pool + // with priority, dedup, and abort. + readonly thumbnails: ThumbnailsAPI; + // The content-similarity search surface over the CLIP index (issue #50). + // Present whether or not ML fetching is enabled; with no ML store it + // returns empty results. + readonly mldata: MLDataAPI; + + private readonly client: LibraryClient; + private readonly store: MetadataStore; + // The on-disk content cache, or undefined when no content source is + // available (a metadata-only client with no explicit source). + private readonly cache?: ContentCache; + private readonly userID: number; + private readonly intervalMs: number; + private readonly onProgress?: RefreshProgressCallback; + private readonly pools: RequestPools; + // The ML-data cache, present only when the client can fetch ML data. + private readonly mlStore?: MLDataStore; + // The local precache (#48), present only when the content cache is. + private readonly precache?: Precache; + + private timer?: ReturnType<typeof setTimeout>; + // The in-flight refresh cycle, or undefined when none runs. One slot serves + // both paths: the background loop skips when it is set, and a fresh read + // (issue #75) coalesces onto it or starts one. The promise carries the + // cycle's real outcome (it rejects on failure); the background loop ignores + // that, a fresh read propagates it. + private cycle?: Promise<void>; + // Guards the ML fetch pass so a slow backfill never runs twice at once; a + // refresh whose pass is still running kicks nothing new. Holds the running + // pass, so `close()` can wait for it. + private mlFetch?: Promise<void>; + private closed = false; + private lastRefreshAt?: number; + private lastError?: string; + private lastMLFetchAt?: number; + private lastMLError?: string; + // The plain-record projection as of the last refresh, and the GUI change + // subscribers. A refresh that alters the projection notifies each with the + // delta; `lastRecords` is kept current every refresh so a subscriber that + // joins later diffs against the state its own `snapshot()` already returned. + private readonly subscribers = new Set<(change: LibraryChange) => void>(); + private lastRecords: DerivedRecords; + // RAM holds changes disk has not yet accepted (an earlier save failed). + // Cleared only when a save actually succeeds; keeps the store trying to + // persist and the failure visible in `status()` until then. + private unsaved = false; + + private constructor(args: { + client: LibraryClient; + store: MetadataStore; + userID: number; + cacheDirectory: string; + downloadDirectory?: string; + intervalMs: number; + onProgress?: RefreshProgressCallback; + pools: RequestPools; + mldata?: MLDataStore; + cache?: ContentCache; + precache?: Precache; + }) { + this.client = args.client; + this.store = args.store; + this.userID = args.userID; + this.cacheDirectory = args.cacheDirectory; + this.downloadDirectory = args.downloadDirectory; + this.intervalMs = args.intervalMs; + this.onProgress = args.onProgress; + this.pools = args.pools; + this.mlStore = args.mldata; + this.cache = args.cache; + this.precache = args.precache; + this.lastRecords = this.deriveNow(); + + // The read namespaces derive fresh from the store on each call, so they + // always reflect the latest refresh. + const derive = (): DerivedRecords => this.deriveNow(); + this.albums = makeAlbumsAPI(derive, this.cache); + this.photos = makePhotosAPI(derive, this.cache); + this.timeline = makeTimelineAPI(derive); + this.thumbnails = { + ensure: (opts: EnsureOptions): Promise<EnsureResult[]> => { + if (!this.cache) { + return Promise.reject( + new Error( + "thumbnails.ensure requires a library opened with a content cache", + ), + ); + } + return this.cache.ensureThumbnails(opts); + }, + }; + // Reads the ML store live so results grow as ML data is fetched. + this.mldata = makeMLDataAPI(() => this.mlStore); + } + + // Load the cache and start the refresh loop. With an empty cache the first + // refresh is awaited, so `open()` resolves onto populated data whenever the + // server is reachable; that awaited refresh may still fail, and the library + // then opens empty with the failure recorded in `status()`. With an + // existing cache the first refresh runs in the background and `open()` + // returns as soon as the cached data is ready — an unreachable server does + // not block opening. + static async open(opts: LibraryOptions): Promise<Library> { + const { userID } = opts.client.whoami(); + const cacheDirectory = + opts.cacheDirectory ?? defaultCacheDirectory(userID); + const metadataPath = join(cacheDirectory, "metadata.json"); + let store = await MetadataStore.load(metadataPath); + // A cache directory given explicitly can hold another account's cache. + // Its records and cursor are not this account's, so delete it and the + // ML data beside it and start empty. A user ID of 0 means the cache + // was never refreshed and so holds nothing to discard. + if (store.userID !== 0 && store.userID !== userID) { + await rm(metadataPath, { force: true }); + await rm(join(cacheDirectory, "mldata"), { + recursive: true, + force: true, + }); + store = await MetadataStore.load(metadataPath); + } + const intervalMs = + (opts.refreshIntervalSeconds ?? DEFAULT_REFRESH_INTERVAL_SECONDS) * + 1000; + + // One request-pool set serves both the ML-data fetch and the content + // cache, so both honour the same concurrency caps. + const pools = opts.pools ?? new RequestPools(); + + // The ML cache only earns its keep when the client can fetch ML data; + // a client without that capability opens no `mldata/` directory. + const mldata = opts.client.fetchMLData + ? await MLDataStore.open(join(cacheDirectory, "mldata")) + : undefined; + + // Build the content cache from an explicit source or the client's own, + // and take its record of what is already cached (and reap orphan temp + // files) before the first projection, so cached paths are present from + // the start and the first refresh raises no spurious path-change diff. + const source = opts.contentSource ?? opts.client.contentSource?.(); + let cache: ContentCache | undefined; + let precache: Precache | undefined; + if (source) { + // The precache owns the pinned set (favorites + latest week), which + // is the cache's eviction predicate. It is built and seeded from + // the loaded store first so the cache can wire `isPinned` to it, and + // then bound to the cache it fills. A caller-supplied predicate is + // OR-ed in so both survive. + precache = new Precache({ + thumbnails: opts.precacheThumbnails, + originals: opts.precacheOriginals, + originalsDays: opts.precacheOriginalsDays, + onEvent: opts.onProgress, + }); + const records = deriveRecordsFromStore(store); + precache.update(records); + const extraPinned = opts.isOriginalPinned; + cache = new ContentCache({ + pools, + source, + cacheDirectory, + downloadDirectory: opts.downloadDirectory, + getFile: (fileID) => store.getFileByID(fileID), + cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes, + freeBelowBytes: opts.freeBelowBytes, + isPinned: (fileID) => + precache!.isPinned(fileID) || + (extraPinned?.(fileID) ?? false), + }); + // The records say at once which files are live photos, where + // `getFileByID` would search every file for each one cached. + await cache.open( + (fileID) => + records.photos.get(fileID)?.fileType === "livePhoto", + ); + precache.bind(cache); + } + + const lib = new Library({ + client: opts.client, + store, + userID, + cacheDirectory, + downloadDirectory: opts.downloadDirectory, + intervalMs, + onProgress: opts.onProgress, + pools, + mldata, + cache, + precache, + }); + + // Start filling from whatever the loaded store already holds; each + // refresh below re-kicks with the new files (and retries any that + // failed). An empty store starts empty here and fills after its first + // refresh. + precache?.start(); + + if (store.loadedFromDisk) { + // An existing copy already answers reads; refresh in the background + // and start the interval once that first cycle settles. + void lib.runRefresh().then(() => lib.scheduleNext()); + } else { + // Nothing was cached: wait for the first refresh to fill the store + // (or fail) rather than resolve onto an empty library. + await lib.runRefresh(); + lib.scheduleNext(); + } + return lib; + } + + listCollections(): Collection[] { + return this.store.listCollections(); + } + + getCollection(id: number): Collection | undefined { + return this.store.getCollection(id); + } + + listFiles(collectionID: number): EnteFile[] { + return this.store.listFiles(collectionID); + } + + getFile(collectionID: number, fileID: number): EnteFile | undefined { + return this.store.getFile(collectionID, fileID); + } + + // Any membership of a file, addressed by file id alone. A file's own + // metadata (title, creationTime) is identical across the collections it + // belongs to, so this serves the point commands that hold only a fileID. + getFileByID(fileID: number): EnteFile | undefined { + return this.store.getFileByID(fileID); + } + + // A synchronous, RAM-only projection of the whole library into plain + // records (no keys), the surface the GUI reads across IPC. Photos are + // deduplicated to one record per file and ordered newest first. + snapshot(): LibrarySnapshot { + return snapshotFrom(this.deriveNow(), Date.now()); + } + + // Deliver a `LibraryChange` whenever a refresh alters the projection. A + // refresh that changes nothing delivers nothing. The returned handle's + // `unsubscribe` stops delivery. + subscribe(args: { onChange: (change: LibraryChange) => void }): { + unsubscribe: () => void; + } { + const { onChange } = args; + this.subscribers.add(onChange); + return { + unsubscribe: () => { + this.subscribers.delete(onChange); + }, + }; + } + + status(): LibraryStatus { + let files = 0; + const collections = this.store.listCollections(); + for (const c of collections) { + files += this.store.listFiles(c.id).length; + } + const ml = this.mlStore?.stats(); + const originals = this.cache?.originalsStatus(); + const pre = this.precache?.status(); + return { + userID: this.store.userID, + collections: collections.length, + files, + lastRefreshAt: this.lastRefreshAt, + lastError: this.lastError, + lastMLFetchAt: this.lastMLFetchAt, + lastMLError: this.lastMLError, + mlStored: ml?.stored, + mlIndexed: ml?.indexed, + originalsUsedBytes: originals?.usedBytes, + originalsLimitBytes: originals?.limitBytes, + thumbnailsCached: pre?.thumbnailsCached, + thumbnailsTotal: pre?.thumbnailsTotal, + originalsCached: pre?.originalsCached, + originalsPinned: pre?.originalsPinned, + closed: this.closed, + }; + } + + // Fresh reads (issue #75, owner amendment to design #36). Force a refresh, + // wait for it to complete and persist, then hand back the same + // `albums`/`photos`/`timeline` namespaces — now guaranteed to reflect a + // completed server round-trip. Concurrent calls coalesce onto one refresh; + // a refresh that fails rejects here, where the default namespaces would + // instead stay silent and serve the last good copy. + async fresh(): Promise<FreshReads> { + await this.refreshNow(); + return { + albums: this.albums, + photos: this.photos, + timeline: this.timeline, + }; + } + + // Back up every in-scope file to `downloadDirectory` in the historical + // on-disk layout, with a durable failure ledger (issue #51). Waits for a + // completed refresh first, as `fresh()` does, joining one already running, + // and rejects before touching any file when it fails. Then fetches pending + // originals (and optional thumbnails) through the content cache and pools, + // and rebuilds the derived symlink/JSON views from the model. Throws before + // any network work when no download directory is available or no content + // cache backs the originals it must fetch. + backup(opts?: BackupOptions): Promise<BackupResult> { + const downloadDirectory = + opts?.downloadDirectory ?? this.downloadDirectory; + const includeOriginals = opts?.includeOriginals ?? true; + const includeThumbnails = opts?.includeThumbnails ?? false; + if (!downloadDirectory) { + return Promise.reject( + new Error( + "backup requires a downloadDirectory (pass one to " + + "backup() or open the library with one)", + ), + ); + } + if ((includeOriginals || includeThumbnails) && !this.cache) { + return Promise.reject( + new Error( + "backup requires a library opened with a content cache", + ), + ); + } + const cache = this.cache; + return runBackup( + { + refresh: () => this.refreshNow(), + listCollections: () => this.store.listCollections(), + listFiles: (id) => this.store.listFiles(id), + original: (fileID, destination) => + cache!.backupOriginal(fileID, destination), + thumbnail: (fileID) => cache!.thumbnail(fileID), + }, + { ...opts, downloadDirectory }, + ); + } + + // Stop the background timer. Idempotent. An in-flight refresh is left to + // finish; it will not schedule another cycle once closed. The returned + // promise resolves once that refresh (including its cache write), the ML + // fetch pass and the precache fetches already running have all finished, + // so a caller can then remove the cache directory. A refresh failure is + // reported through `status()`, not thrown here. + async close(): Promise<void> { + this.closed = true; + const precacheClosed = this.precache?.close(); + if (this.timer !== undefined) { + clearTimeout(this.timer); + this.timer = undefined; + } + await this.cycle?.catch(() => {}); + await this.mlFetch; + await precacheClosed; + } + + private scheduleNext(): void { + if (this.closed) return; + this.timer = setTimeout(() => { + void this.runRefresh().then(() => this.scheduleNext()); + }, this.intervalMs); + // Do not keep the process alive for the sake of the timer. + this.timer.unref?.(); + } + + // The background loop's refresh: run a cycle unless one is already in flight + // (or the library is closed), and never let a failure escape — the + // background path reports errors through `status()`/`onProgress`, it does + // not throw. Resolves once the cycle it started (or skipped past) settles. + private runRefresh(): Promise<void> { + if (this.closed || this.cycle) return Promise.resolve(); + return this.startCycle().catch(() => {}); + } + + // A fresh read's refresh (issue #75): force a cycle and await it, rejecting + // if it fails. Concurrent fresh reads coalesce onto the one in-flight cycle + // — the background loop's included — so they never fan out into redundant + // server round-trips. + private refreshNow(): Promise<void> { + if (this.closed) { + return Promise.reject(new Error("the library is closed")); + } + return this.cycle ?? this.startCycle(); + } + + // Start one refresh cycle and record it as the in-flight cycle so every + // caller coalesces onto it. The returned promise carries the cycle's real + // outcome; each caller attaches the handling its own path needs, and the + // slot is cleared once the cycle settles. + private startCycle(): Promise<void> { + const cycle = this.refreshCycle(); + this.cycle = cycle; + void cycle.then( + () => { + if (this.cycle === cycle) this.cycle = undefined; + }, + () => { + if (this.cycle === cycle) this.cycle = undefined; + }, + ); + return cycle; + } + + // One refresh cycle: the network fetch and commit, wrapped in the progress + // events and status bookkeeping. Throws when the refresh fails so a fresh + // read can reject; `runRefresh` swallows that throw for the background loop. + private async refreshCycle(): Promise<void> { + this.emit({ operation: "refresh", status: "started" }); + try { + await this.refreshOnce(); + this.lastRefreshAt = Date.now(); + this.lastError = undefined; + this.emit({ operation: "refresh", status: "done" }); + // Backfill ML data for the files this refresh knows about. It runs + // outside the refresh's success/failure so a fetch or disk problem + // there never marks the metadata refresh failed, and it is not + // awaited so it never stalls the refresh interval. + this.mlFetch ??= this.runMLFetch().finally(() => { + this.mlFetch = undefined; + }); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + this.lastError = error; + this.emit({ operation: "refresh", status: "failed", error }); + throw err; + } + } + + // Fetch every change since the stored cursor, then commit. All network + // reads happen before any store mutation, so a fetch that throws leaves the + // store untouched and the previous snapshot intact. + private async refreshOnce(): Promise<void> { + const page = await this.client.collectionsSince({ + sinceTime: this.store.collectionsSinceTime, + }); + + // Stage per-collection file diffs. A collection's files are + // re-enumerated only when its updationTime has advanced past the cached + // copy; an unchanged album's file list cannot have changed. New + // collections enumerate from the beginning of time. + const filePages: { collectionID: number; page: FilesPage }[] = []; + for (const collection of page.collections) { + const known = this.store.getCollection(collection.id); + if (known && collection.updationTime <= known.updationTime) + continue; + const filePage = await this.client.filesSince({ + collectionID: collection.id, + collectionKey: collection.key, + sinceTime: known ? known.updationTime : 0, + }); + filePages.push({ collectionID: collection.id, page: filePage }); + } + + // Network work done; commit to the store and persist only if something + // actually changed. + let changed = false; + + if (this.store.userID !== this.userID) { + this.store.userID = this.userID; + changed = true; + } + + for (const id of page.deleted) { + if (this.store.getCollection(id)) { + this.store.deleteCollection(id); + changed = true; + } + } + for (const collection of page.collections) { + this.store.putCollection(collection); + changed = true; + } + + for (const { collectionID, page: filePage } of filePages) { + for (const id of filePage.deleted) { + if (this.store.getFile(collectionID, id)) { + this.store.deleteFile(collectionID, id); + changed = true; + } + } + for (const f of filePage.files) { + this.store.putFile(f); + changed = true; + } + } + + if (page.cursor !== this.store.collectionsSinceTime) { + this.store.collectionsSinceTime = page.cursor; + changed = true; + } + + if (changed) this.unsaved = true; + + // Reproject and notify subscribers of the delta. This tracks RAM (what + // reads see), so it fires whether or not the save below succeeds; a + // save failure surfaces separately through `status().lastError`. + // `lastRecords` advances every changed refresh so the next diff is + // against current state. + if (changed) { + const next = this.deriveNow(); + if (this.subscribers.size > 0) { + const change = diffRecords(this.lastRecords, next, Date.now()); + if (change) this.notify(change); + } + this.lastRecords = next; + // Recompute the fill orders and pinned set against the new library. + this.precache?.update(next); + } + // Re-kick the fills every cycle: a finished sweep starts afresh to pick + // up new files and retry any that failed, and a running one is left be. + this.precache?.start(); + + // Persist whenever RAM holds changes disk has not accepted — including + // changes an earlier cycle staged whose save failed. `unsaved` clears + // only once a save lands, so a save failure both stays visible through + // `status().lastError` (the throw below records it) and keeps being + // retried, instead of a later empty refresh silently clearing it while + // the on-disk cache is still behind RAM. + if (this.unsaved) { + await this.store.save(); + this.unsaved = false; + } + } + + // One ML fetch pass: fetch, decrypt and store the ML data for every file + // the store knows about that is not cached (or whose `updationTime` has + // advanced), through the metadata pool, and update the CLIP index. Guarded + // so passes never overlap; a failure is reported, not thrown. + private async runMLFetch(): Promise<void> { + const mldata = this.mlStore; + // Bind so the call keeps the client as its receiver when invoked + // through the pool below. + const fetchMLData = this.client.fetchMLData?.bind(this.client); + if (!mldata || !fetchMLData || this.closed) return; + + const files = this.uniqueFiles(); + const needed = mldata.neededFor(files); + if (needed.length === 0) return; + + this.emit({ operation: "fetchMLData", status: "started" }); + try { + const fileKeys = new Map<number, Uint8Array>(); + const updation = new Map<number, number>(); + for (const f of files) { + fileKeys.set(f.id, f.key); + updation.set(f.id, f.updationTime); + } + + let stored = 0; + for (let i = 0; i < needed.length; i += MLDATA_BATCH_SIZE) { + if (this.closed) break; + const batch = needed.slice(i, i + MLDATA_BATCH_SIZE); + const payloads = await this.pools.metadata.run( + () => fetchMLData({ fileIDs: batch, fileKeys }), + { priority: "background" }, + ); + stored += (await mldata.storeFetched(payloads, updation)) + .stored; + } + + this.lastMLFetchAt = Date.now(); + this.lastMLError = undefined; + this.emit({ + operation: "fetchMLData", + status: "done", + fetched: stored, + }); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + this.lastMLError = error; + this.emit({ operation: "fetchMLData", status: "failed", error }); + } + } + + // The distinct files the store holds, one entry per fileID (a file in + // several collections shares its ML data), each carrying the key and the + // newest `updationTime` seen across its memberships. + private uniqueFiles(): { + id: number; + key: Uint8Array; + updationTime: number; + }[] { + const byID = new Map< + number, + { id: number; key: Uint8Array; updationTime: number } + >(); + for (const collection of this.store.listCollections()) { + for (const f of this.store.listFiles(collection.id)) { + const seen = byID.get(f.id); + if (seen === undefined || f.updationTime > seen.updationTime) + byID.set(f.id, { + id: f.id, + key: f.key, + updationTime: f.updationTime, + }); + } + } + return [...byID.values()]; + } + + // Gather every file membership and project the store into by-id records, + // filling each record's cache paths from the content cache when present. + private deriveNow(): DerivedRecords { + return deriveRecordsFromStore(this.store, this.cache); + } + + private notify(change: LibraryChange): void { + for (const onChange of this.subscribers) { + // A misbehaving subscriber must not break the loop or its peers. + try { + onChange(change); + } catch { + // ignore + } + } + } + + private emit(event: RefreshEvent): void { + if (!this.onProgress) return; + // A misbehaving callback must not break the refresh loop. + try { + this.onProgress(event); + } catch { + // ignore + } + } +} diff --git a/src/library/mldata.ts b/src/library/mldata.ts new file mode 100644 index 0000000..06307d8 --- /dev/null +++ b/src/library/mldata.ts @@ -0,0 +1,378 @@ +// The on-disk cache of Ente's per-file machine-learning data and the CLIP +// index derived from it (issue #49). +// +// Under `<cacheDirectory>/mldata/` this keeps: +// +// - `<fileID>.json` — one decrypted, gunzipped payload per file, written by +// rename. Its presence means it is complete: a torn write never leaves a +// half-file, so the set of these files is the source of truth for what is +// cached. The full payload (face boxes, landmarks, embeddings) is read back +// from here on demand and never held in RAM. +// +// - `clip.f32` + `clip.json` — the derived index the content search runs on. +// `clip.json` lists the indexed fileIDs in order plus the embedding length; +// `clip.f32` is those CLIP embeddings packed as one `Float32Array`, so the +// index loads in a single read with no per-vector parse. The index is +// rebuilt from the payloads whenever it is missing or structurally +// disagrees with the files present, and appended to as new payloads arrive. +// +// - `fetched.json` — a small map of fileID to the `updationTime` it was +// fetched at. This is best-effort bookkeeping for refetch decisions (a file +// whose `updationTime` later advances is refetched); the payloads, not this +// file, remain the record of what is cached, so losing it only forgoes +// update-driven refetch until the next fetch rewrites it. +// +// In RAM this holds only the id list and the packed `Float32Array`. + +import { mkdir, readFile, readdir } from "node:fs/promises"; +import { join } from "node:path"; + +import { writeAtomic } from "../download/index.js"; +import type { MLData } from "../mldata-fetch.js"; + +const CLIP_VECTORS = "clip.f32"; +const CLIP_INDEX = "clip.json"; +const FETCHED = "fetched.json"; +// A payload file is named for its fileID alone; the derived files above are +// not, so this pattern picks out payloads and nothing else. +const PAYLOAD_RE = /^(\d+)\.json$/; +const BYTES_PER_FLOAT = 4; + +// The on-disk form of `clip.json`. +interface ClipIndexFile { + fileIDs: number[]; + embeddingLength: number; +} + +// A file the model knows about, for deciding what to fetch. +export interface MLDataFile { + id: number; + updationTime: number; +} + +// The RAM index the search reads: `fileIDs[i]` owns the `embeddingLength` +// floats of `embeddings` starting at `i * embeddingLength`. +export interface MLIndex { + fileIDs: number[]; + embeddingLength: number; + embeddings: Float32Array; +} + +// Pull the CLIP embedding out of a payload, or undefined when it is absent or +// misshapen. Kept strict so a bad payload is skipped rather than corrupting the +// packed index. +const clipEmbedding = (payload: MLData): number[] | undefined => { + const clip = payload.clip; + if (typeof clip !== "object" || clip === null) return undefined; + const embedding = (clip as { embedding?: unknown }).embedding; + if (!Array.isArray(embedding)) return undefined; + if (embedding.some((v) => typeof v !== "number" || !Number.isFinite(v))) + return undefined; + return embedding as number[]; +}; + +export class MLDataStore { + readonly dir: string; + + // fileIDs whose payload JSON is present on disk (present means complete). + private readonly present = new Set<number>(); + // fileID -> updationTime it was fetched at. + private readonly fetched = new Map<number, number>(); + + // The packed index and where each id sits in it. + private ids: number[] = []; + private embeddingLength = 0; + private embeddings = new Float32Array(0); + private readonly pos = new Map<number, number>(); + + private constructor(dir: string) { + this.dir = dir; + } + + // Open (creating the directory) and load the id list and packed index into + // RAM, rebuilding the index from the payloads when it is missing or does + // not match the files present. + static async open(dir: string): Promise<MLDataStore> { + const store = new MLDataStore(dir); + await mkdir(dir, { recursive: true }); + await store.loadPresent(); + await store.loadFetched(); + if (!(await store.tryLoadIndex())) await store.rebuildIndex(); + return store; + } + + // The fileIDs among `files` that must be fetched: every file with no + // payload yet (first run, then new files), plus any whose `updationTime` + // has advanced past the one its cached payload was fetched at. Returned + // sorted and unique. + neededFor(files: MLDataFile[]): number[] { + const latest = new Map<number, number>(); + for (const f of files) { + const seen = latest.get(f.id); + if (seen === undefined || f.updationTime > seen) + latest.set(f.id, f.updationTime); + } + const needed: number[] = []; + for (const [id, updationTime] of latest) { + if (!this.present.has(id)) { + needed.push(id); + continue; + } + const at = this.fetched.get(id); + if (at !== undefined && updationTime > at) needed.push(id); + } + return needed.sort((a, b) => a - b); + } + + // Store a batch of fetched payloads: write one file per id, fold their CLIP + // embeddings into the packed index (in place for a refetch, appended for a + // new file), and persist the derived files. Returns how many payloads were + // stored and how many ids the index now holds. + async storeFetched( + payloads: Map<number, MLData>, + updation: Map<number, number>, + ): Promise<{ stored: number; indexed: number }> { + if (payloads.size === 0) return { stored: 0, indexed: this.ids.length }; + + for (const [id, payload] of payloads) { + await this.writePayload(id, payload); + this.present.add(id); + const at = updation.get(id); + if (at !== undefined) this.fetched.set(id, at); + } + + const updates: { at: number; vector: number[] }[] = []; + const appends: { id: number; vector: number[] }[] = []; + for (const [id, payload] of payloads) { + const vector = clipEmbedding(payload); + if (!vector) continue; + if (this.embeddingLength === 0 && this.ids.length === 0) + this.embeddingLength = vector.length; + // The index is fixed-width; a vector of another length (never seen + // from Ente's CLIP model) is stored but left out of the index. + if (vector.length !== this.embeddingLength) continue; + const at = this.pos.get(id); + if (at !== undefined) updates.push({ at, vector }); + else appends.push({ id, vector }); + } + + for (const { at, vector } of updates) + this.embeddings.set(vector, at * this.embeddingLength); + + if (appends.length > 0) { + const length = this.embeddingLength; + const grown = new Float32Array( + this.embeddings.length + appends.length * length, + ); + grown.set(this.embeddings); + let offset = this.embeddings.length; + for (const { id, vector } of appends) { + grown.set(vector, offset); + this.pos.set(id, this.ids.length); + this.ids.push(id); + offset += length; + } + this.embeddings = grown; + } + + await this.persistIndex(); + await this.persistFetched(); + return { stored: payloads.size, indexed: this.ids.length }; + } + + // The packed index the search runs on. The id list is copied so callers + // cannot disturb the store's own order; the embeddings are the live buffer. + getIndex(): MLIndex { + return { + fileIDs: [...this.ids], + embeddingLength: this.embeddingLength, + embeddings: this.embeddings, + }; + } + + // The full payload for a file, read from disk, or undefined when it is not + // cached or does not parse. + async readPayload(fileID: number): Promise<MLData | undefined> { + if (!this.present.has(fileID)) return undefined; + let raw: string; + try { + raw = await readFile(this.payloadPath(fileID), "utf-8"); + } catch { + return undefined; + } + try { + return JSON.parse(raw) as MLData; + } catch { + return undefined; + } + } + + stats(): { stored: number; indexed: number } { + return { stored: this.present.size, indexed: this.ids.length }; + } + + private payloadPath(id: number): string { + return join(this.dir, `${id}.json`); + } + + private async writePayload(id: number, payload: MLData): Promise<void> { + await writeAtomic( + this.payloadPath(id), + new TextEncoder().encode(JSON.stringify(payload)), + ); + } + + private async loadPresent(): Promise<void> { + let names: string[]; + try { + names = await readdir(this.dir); + } catch { + return; + } + for (const name of names) { + const match = PAYLOAD_RE.exec(name); + if (match) this.present.add(Number(match[1])); + } + } + + private async loadFetched(): Promise<void> { + let raw: string; + try { + raw = await readFile(join(this.dir, FETCHED), "utf-8"); + } catch { + return; + } + try { + const parsed = JSON.parse(raw) as Record<string, unknown>; + for (const [key, value] of Object.entries(parsed)) { + const id = Number(key); + if ( + Number.isInteger(id) && + typeof value === "number" && + this.present.has(id) + ) + this.fetched.set(id, value); + } + } catch { + // Corrupt bookkeeping degrades refetch decisions, never fails open. + } + } + + // Load the packed index if it is present and agrees with the payloads in + // both directions: every id it names must still be present, its vector file + // must be exactly the size the id count and embedding length imply, and no + // embedding-bearing payload on disk may be missing from it. Returns whether + // it loaded. + private async tryLoadIndex(): Promise<boolean> { + let metaRaw: string; + try { + metaRaw = await readFile(join(this.dir, CLIP_INDEX), "utf-8"); + } catch { + return false; + } + let meta: ClipIndexFile; + try { + meta = JSON.parse(metaRaw) as ClipIndexFile; + } catch { + return false; + } + if ( + !Array.isArray(meta.fileIDs) || + typeof meta.embeddingLength !== "number" + ) + return false; + if (meta.fileIDs.some((id) => !this.present.has(id))) return false; + + // The reverse must hold too. A payload carrying an embedding but absent + // from the index means the index is stale — realistically the process + // died after storeFetched renamed the payloads into place but before it + // rewrote clip.json/clip.f32. Loading such an index as "consistent" + // would drop those embeddings for good (neededFor sees the payloads + // present and never refetches), so treat it as a disagreement and + // rebuild. Only present ids the index omits are read; a payload + // legitimately without an embedding stays out and forces no rebuild. + const indexed = new Set(meta.fileIDs); + for (const id of this.present) { + if (indexed.has(id)) continue; + const payload = await this.readPayload(id); + if (payload && clipEmbedding(payload)) return false; + } + + let bytes: Buffer; + try { + bytes = await readFile(join(this.dir, CLIP_VECTORS)); + } catch { + return false; + } + const expected = + meta.fileIDs.length * meta.embeddingLength * BYTES_PER_FLOAT; + if (bytes.byteLength !== expected) return false; + + // One read, no parse: copy into an aligned buffer and view it as + // floats. The copy is needed because a Buffer from the pool can start + // at an offset a Float32Array cannot be laid over. + const aligned = new Uint8Array(bytes.byteLength); + aligned.set(bytes); + this.embeddings = new Float32Array(aligned.buffer); + this.embeddingLength = meta.embeddingLength; + this.ids = [...meta.fileIDs]; + this.pos.clear(); + this.ids.forEach((id, i) => this.pos.set(id, i)); + return true; + } + + // Rebuild the packed index by reading every payload present, then persist + // it. Payloads without a CLIP embedding (or of an unexpected length) are + // simply not indexed. + private async rebuildIndex(): Promise<void> { + this.ids = []; + this.pos.clear(); + this.embeddingLength = 0; + const vectors: number[][] = []; + for (const id of [...this.present].sort((a, b) => a - b)) { + const payload = await this.readPayload(id); + if (!payload) continue; + const vector = clipEmbedding(payload); + if (!vector) continue; + if (this.embeddingLength === 0) + this.embeddingLength = vector.length; + if (vector.length !== this.embeddingLength) continue; + this.pos.set(id, this.ids.length); + this.ids.push(id); + vectors.push(vector); + } + const length = this.embeddingLength; + const packed = new Float32Array(this.ids.length * length); + vectors.forEach((vector, i) => packed.set(vector, i * length)); + this.embeddings = packed; + await this.persistIndex(); + } + + private async persistIndex(): Promise<void> { + const meta: ClipIndexFile = { + fileIDs: this.ids, + embeddingLength: this.embeddingLength, + }; + await writeAtomic( + join(this.dir, CLIP_INDEX), + new TextEncoder().encode(JSON.stringify(meta)), + ); + await writeAtomic( + join(this.dir, CLIP_VECTORS), + new Uint8Array( + this.embeddings.buffer, + this.embeddings.byteOffset, + this.embeddings.byteLength, + ), + ); + } + + private async persistFetched(): Promise<void> { + const record: Record<string, number> = {}; + for (const [id, at] of this.fetched) record[id] = at; + await writeAtomic( + join(this.dir, FETCHED), + new TextEncoder().encode(JSON.stringify(record)), + ); + } +} diff --git a/src/library/mlsearch.ts b/src/library/mlsearch.ts new file mode 100644 index 0000000..d23ef26 --- /dev/null +++ b/src/library/mlsearch.ts @@ -0,0 +1,129 @@ +// The content-similarity search surface over the CLIP index (issue #50). +// +// This is `lib.mldata`. It answers three questions against the ML-data cache +// (#49) without touching the network: +// +// - `forFile` returns the whole stored payload (face boxes, landmarks, +// embedding) for a file, read from disk on demand — the only method here +// that touches the disk, and the only one that is async. +// - `similar` and `searchByEmbedding` rank fileIDs by cosine similarity over +// the packed `Float32Array` index alone. That index (~50k×512) already +// lives in RAM, so each query is a plain loop over it and nothing else. +// +// quak bundles no text encoder (owner-deferred), so `searchByEmbedding` takes +// the query vector the caller has produced elsewhere; `similar` uses the +// query file's own indexed embedding. + +import type { MLData } from "../mldata-fetch.js"; +import type { MLDataStore, MLIndex } from "./mldata.js"; + +// How many nearest files a query returns when the caller names no limit. +const DEFAULT_LIMIT = 20; + +// One ranked result: a fileID and its cosine similarity to the query, in +// [-1, 1]. Callers wanting only the ids read `.fileID`. +export interface SimilarResult { + fileID: number; + score: number; +} + +export interface MLDataAPI { + // The whole stored ML payload for a file, or undefined when it is not + // cached. Reads the payload from disk, so it is async. + forFile(args: { fileID: number }): Promise<MLData | undefined>; + // The files nearest the given file by cosine over their CLIP embeddings, + // most similar first, excluding the file itself. Empty when the file has + // no indexed embedding. + similar(args: { fileID: number; limit?: number }): SimilarResult[]; + // The files nearest a caller-supplied query embedding by cosine, most + // similar first. Empty when the query is the wrong length for the index, + // has zero magnitude, or the index is empty. + searchByEmbedding(args: { + embedding: ArrayLike<number>; + limit?: number; + }): SimilarResult[]; +} + +// Rank the packed index by cosine similarity to `query`, most similar first, +// and return the top `limit`. `skip` (a query file's own id) is left out. Both +// each row's magnitude and the query's are computed here rather than cached: +// the index mutates as ML data is fetched, and one plain pass over ~50k×512 +// floats is fast enough that a norm cache would only add a staleness bug. A +// zero-magnitude vector has no direction, so it is dropped rather than divided +// by zero. +const topByCosine = ( + index: MLIndex, + query: ArrayLike<number>, + limit: number, + skip?: number, +): SimilarResult[] => { + const { fileIDs, embeddingLength, embeddings } = index; + if (embeddingLength === 0 || query.length !== embeddingLength) return []; + + // Every indexed read below is in range: the inner loops run to + // `embeddingLength`, the query is exactly that long (checked above), and + // the packed buffer holds `fileIDs.length * embeddingLength` floats. + // `noUncheckedIndexedAccess` still widens each read to `number | undefined`, + // so they are asserted non-null rather than paying a per-element guard in + // this hot ~50k×512 loop. + let queryNorm = 0; + for (let k = 0; k < embeddingLength; k++) { + const q = query[k]!; + queryNorm += q * q; + } + queryNorm = Math.sqrt(queryNorm); + if (queryNorm === 0) return []; + + const results: SimilarResult[] = []; + for (let i = 0; i < fileIDs.length; i++) { + const id = fileIDs[i]!; + if (id === skip) continue; + const base = i * embeddingLength; + let dot = 0; + let norm = 0; + for (let k = 0; k < embeddingLength; k++) { + const v = embeddings[base + k]!; + dot += query[k]! * v; + norm += v * v; + } + if (norm === 0) continue; + results.push({ + fileID: id, + score: dot / (queryNorm * Math.sqrt(norm)), + }); + } + + // Descending score, ties broken by ascending fileID for a stable order. + results.sort((a, b) => b.score - a.score || a.fileID - b.fileID); + return results.slice(0, Math.max(0, Math.trunc(limit))); +}; + +// Build the search surface over a store the library supplies lazily (the store +// is absent when the client cannot fetch ML data). Reading it per call keeps +// the surface current as the index grows. +export const makeMLDataAPI = ( + store: () => MLDataStore | undefined, +): MLDataAPI => ({ + forFile: ({ fileID }): Promise<MLData | undefined> => { + const s = store(); + return s ? s.readPayload(fileID) : Promise.resolve(undefined); + }, + similar: ({ fileID, limit }): SimilarResult[] => { + const s = store(); + if (!s) return []; + const index = s.getIndex(); + const pos = index.fileIDs.indexOf(fileID); + if (pos < 0) return []; + const base = pos * index.embeddingLength; + const query = index.embeddings.subarray( + base, + base + index.embeddingLength, + ); + return topByCosine(index, query, limit ?? DEFAULT_LIMIT, fileID); + }, + searchByEmbedding: ({ embedding, limit }): SimilarResult[] => { + const s = store(); + if (!s) return []; + return topByCosine(s.getIndex(), embedding, limit ?? DEFAULT_LIMIT); + }, +}); diff --git a/src/library/pools.ts b/src/library/pools.ts new file mode 100644 index 0000000..e1a172d --- /dev/null +++ b/src/library/pools.ts @@ -0,0 +1,181 @@ +// Three bounded request pools for metadata, content, and thumbnails (issue +// #45). +// +// Ente meters differently by traffic class, so quak keeps three independent +// pools instead of one global limit: metadata is cheap and chatty, original +// content is heavy, thumbnails are small but numerous. Each pool is a +// `BoundedPool` — a plain concurrency limiter — and the three run at the +// design's caps (10 / 5 / 25) unless the caller overrides them. +// +// Two behaviours beyond a bare limiter, both per pool: +// +// - Priority: work waiting for a slot is ordered on-demand before +// background/precache, so a slot that frees up serves the request a user is +// waiting on ahead of speculative prefetch. Within one priority the order +// is first-come-first-served. +// +// - In-flight dedup: a task submitted under a `key` that a still-pending task +// already carries is not run a second time; both callers await the one +// result. The key is released the moment that task settles — success or +// failure — so a later request for the same key runs afresh. Callers key by +// the id whose fetch must not be duplicated (a fileID, say). +// +// The pool holds a task's slot for that task's entire lifetime. A task that +// retries internally is doing so inside its slot: the slot is not freed between +// attempts, which is what keeps a retrying request counted against the cap. The +// pool knows nothing of the retry policy; it only holds the slot until the +// task's promise settles. +// +// This module is self-contained infrastructure. Routing a given `Client` call +// to the right pool belongs to the unit that wires the pools into the cache; +// here there is only the machinery. + +export const DEFAULT_METADATA_CONCURRENCY = 10; +export const DEFAULT_CONTENT_CONCURRENCY = 5; +export const DEFAULT_THUMBNAIL_CONCURRENCY = 25; + +// On-demand work is served before background/precache work waiting in the same +// pool. +export type Priority = "on-demand" | "background"; + +export interface RunOptions { + // Defaults to "background": an unmarked request yields to on-demand work. + priority?: Priority; + // When set, a task already pending under this key is shared instead of run + // again. Omit for work that must always execute. + key?: string | number; +} + +// One queued submission awaiting a slot. `start` runs the task and holds the +// slot until it settles. +interface Waiter { + priority: Priority; + // Submission order, used to break ties within a priority (FIFO). + seq: number; + start: () => void; +} + +export class BoundedPool { + readonly concurrency: number; + + private active = 0; + private nextSeq = 0; + private readonly waiting: Waiter[] = []; + // Keyed by a caller-supplied dedup key; holds the shared promise for as + // long as that task is pending, cleared when it settles. + private readonly pending = new Map<string | number, Promise<unknown>>(); + + constructor(concurrency: number) { + if (!Number.isInteger(concurrency) || concurrency < 1) { + throw new RangeError( + `concurrency must be a positive integer, got ${concurrency}`, + ); + } + this.concurrency = concurrency; + } + + // Submit `task` to the pool. It runs once a slot is free, subject to + // priority; the returned promise settles with the task's result. With a + // `key`, a still-pending submission under the same key is returned instead + // of running `task` again. + run<T>(task: () => Promise<T>, opts: RunOptions = {}): Promise<T> { + const { key } = opts; + if (key !== undefined) { + const shared = this.pending.get(key); + if (shared !== undefined) return shared as Promise<T>; + } + + const promise = this.enqueue(task, opts.priority ?? "background"); + + if (key !== undefined) { + this.pending.set(key, promise); + const release = (): void => { + // Only clear our own entry: a fresh submission under the same + // key after this one settled must not be evicted here. + if (this.pending.get(key) === promise) this.pending.delete(key); + }; + promise.then(release, release); + } + return promise; + } + + private enqueue<T>(task: () => Promise<T>, priority: Priority): Promise<T> { + return new Promise<T>((resolve, reject) => { + const start = (): void => { + this.active++; + // Hold the slot until the task fully settles — every internal + // retry included — then admit the next waiter. + void (async () => { + try { + resolve(await task()); + } catch (err) { + reject(err); + } finally { + this.active--; + this.pump(); + } + })(); + }; + this.waiting.push({ priority, seq: this.nextSeq++, start }); + this.pump(); + }); + } + + // Admit waiters until the pool is full or the queue is empty. + private pump(): void { + while (this.active < this.concurrency) { + const next = this.takeNext(); + if (next === undefined) return; + next.start(); + } + } + + // Remove and return the highest-priority waiter: on-demand before + // background, earliest submission first within a priority. + private takeNext(): Waiter | undefined { + let bestIndex = -1; + let best: Waiter | undefined; + for (let i = 0; i < this.waiting.length; i++) { + const w = this.waiting[i]; + if (w === undefined) continue; + if (best === undefined || this.precedes(w, best)) { + best = w; + bestIndex = i; + } + } + if (best === undefined) return undefined; + this.waiting.splice(bestIndex, 1); + return best; + } + + private precedes(a: Waiter, b: Waiter): boolean { + if (a.priority !== b.priority) return a.priority === "on-demand"; + return a.seq < b.seq; + } +} + +export interface RequestPoolsOptions { + metadataConcurrency?: number; + contentConcurrency?: number; + thumbnailConcurrency?: number; +} + +// The three pools the design calls for, each independent: an idle pool never +// lends its slots to a busy one. +export class RequestPools { + readonly metadata: BoundedPool; + readonly content: BoundedPool; + readonly thumbnails: BoundedPool; + + constructor(opts: RequestPoolsOptions = {}) { + this.metadata = new BoundedPool( + opts.metadataConcurrency ?? DEFAULT_METADATA_CONCURRENCY, + ); + this.content = new BoundedPool( + opts.contentConcurrency ?? DEFAULT_CONTENT_CONCURRENCY, + ); + this.thumbnails = new BoundedPool( + opts.thumbnailConcurrency ?? DEFAULT_THUMBNAIL_CONCURRENCY, + ); + } +} diff --git a/src/library/precache.ts b/src/library/precache.ts new file mode 100644 index 0000000..472d627 --- /dev/null +++ b/src/library/precache.ts @@ -0,0 +1,269 @@ +// The aggressive local precache (issue #48), started from `Library.open` with +// no caller input. +// +// Two background fills run concurrently through the shared request pools (#45): +// +// - Thumbnails: every file in the account, newest first, through the +// thumbnail pool until all are on disk. Never evicted. The pool is the same +// one `thumbnails.ensure` uses, so a visible or ahead request always jumps +// ahead of this background fill and a fileID both want is fetched once. +// +// - Originals (the pinned set): through the content pool, the favorites album +// first, then every file whose `takenAt` falls in the latest +// `originalsDays` window — the days ending at the newest file in the +// account. The pinned set is the eviction predicate (#47): a pinned +// original is never evicted, and a file that leaves the set (a favorite +// removed, or the window moving past it on a later refresh) becomes an +// ordinary, evictable original with its bytes left in place. +// +// Both fills yield to on-demand work: every fetch goes to its pool at +// background priority, which the pool serves only after on-demand requests. A +// file already cached costs one map lookup (`pathsFor`) and no fetch. Each +// sweep is driven in bounded chunks so the pool's waiting queue never grows to +// the whole account, keeping on-demand preemption and per-admit cost cheap on a +// large library. Failures are not fatal: an uncached file is retried on the +// next sweep, which `Library` re-kicks after every refresh. + +import type { EnsureResult } from "./content.js"; +import type { DerivedRecords } from "./records.js"; + +const ONE_DAY_MS = 24 * 60 * 60 * 1000; + +export const DEFAULT_PRECACHE_ORIGINALS_DAYS = 7; + +// How many files a sweep submits to a pool before awaiting them. Bounds the +// pool's waiting queue so on-demand work is never stuck behind the whole +// account; the values track each pool's concurrency (#45). +const THUMBNAIL_CHUNK = 25; +const ORIGINAL_CHUNK = 5; + +// The precache metrics `Library.status()` surfaces. +export interface PrecacheStatus { + thumbnailsCached: number; + thumbnailsTotal: number; + originalsCached: number; + originalsPinned: number; +} + +// A background-fill progress event. Each active sweep fires "started" before +// its fetches and "done" (with the count newly on disk) after; a sweep with +// nothing left to fetch is silent. +export interface PrecacheEvent { + operation: "precacheThumbnails" | "precacheOriginals"; + status: "started" | "done"; + fetched?: number; +} + +// The slice of the content cache the precache drives. The real `ContentCache` +// satisfies it; tests inject a fake. +export interface PrecacheCache { + pathsFor(fileID: number): { originalPath?: string; thumbnailPath?: string }; + ensureThumbnails(args: { + fileIDs: number[]; + priority: "background"; + signal?: AbortSignal; + }): Promise<EnsureResult[]>; + ensureOriginals(args: { + fileIDs: number[]; + signal?: AbortSignal; + }): Promise<EnsureResult[]>; +} + +export interface PrecacheOptions { + // Default true; false disables the fill and, for originals, the pinning. + thumbnails?: boolean; + originals?: boolean; + // The latest-week window length in days; default 7. + originalsDays?: number; + onEvent?: (event: PrecacheEvent) => void; +} + +export class Precache { + private readonly doThumbnails: boolean; + private readonly doOriginals: boolean; + private readonly originalsDays: number; + private readonly onEvent?: (event: PrecacheEvent) => void; + + private cache?: PrecacheCache; + // Every file in the account, newest first (the thumbnail fill order). + private thumbOrder: number[] = []; + // The pinned originals in fetch order: favorites first, then the window. + private originalsOrder: number[] = []; + private pinned = new Set<number>(); + + // A sweep runs at most once per fill at a time; a re-kick while one runs is + // a no-op, and the next refresh re-kicks after it finishes. Each holds the + // running sweep, so `close()` can wait for it. + private thumbSweep?: Promise<void>; + private originalsSweep?: Promise<void>; + private readonly aborter = new AbortController(); + private closed = false; + + constructor(opts: PrecacheOptions = {}) { + this.doThumbnails = opts.thumbnails ?? true; + this.doOriginals = opts.originals ?? true; + this.originalsDays = + opts.originalsDays ?? DEFAULT_PRECACHE_ORIGINALS_DAYS; + this.onEvent = opts.onEvent; + } + + // Attach the cache the fills fetch through. `isPinned` works before this is + // called, so the cache can be constructed with `isPinned` wired in and then + // bound here. + bind(cache: PrecacheCache): void { + this.cache = cache; + } + + // Whether an original is pinned (favorites + the latest-week window), the + // eviction predicate (#47). False for every file when originals precaching + // is disabled. + isPinned(fileID: number): boolean { + return this.pinned.has(fileID); + } + + // Recompute the fill orders and the pinned set from the current projection. + // Called at open and after every refresh that changes the library. + update(records: DerivedRecords): void { + const photos = [...records.photos.values()].sort( + (a, b) => b.takenAt - a.takenAt || b.fileID - a.fileID, + ); + + this.thumbOrder = this.doThumbnails ? photos.map((p) => p.fileID) : []; + + const pinned = new Set<number>(); + const order: number[] = []; + if (this.doOriginals) { + // Favorites first, in the album's own newest-first order. + for (const album of records.albums.values()) { + if (album.type !== "favorites") continue; + for (const id of album.fileIDs) { + if (!pinned.has(id)) { + pinned.add(id); + order.push(id); + } + } + } + // Then the latest-week window, ending at the newest file. Photos + // are newest first, so stop once one falls before the window start. + const newest = photos[0]; + if (newest !== undefined) { + const windowStart = + newest.takenAt - this.originalsDays * ONE_DAY_MS; + for (const p of photos) { + if (p.takenAt < windowStart) break; + if (!pinned.has(p.fileID)) { + pinned.add(p.fileID); + order.push(p.fileID); + } + } + } + } + this.pinned = pinned; + this.originalsOrder = order; + } + + status(): PrecacheStatus { + const cache = this.cache; + let thumbnailsCached = 0; + let originalsCached = 0; + if (cache) { + for (const id of this.thumbOrder) + if (cache.pathsFor(id).thumbnailPath !== undefined) + thumbnailsCached++; + for (const id of this.originalsOrder) + if (cache.pathsFor(id).originalPath !== undefined) + originalsCached++; + } + return { + thumbnailsCached, + thumbnailsTotal: this.thumbOrder.length, + originalsCached, + originalsPinned: this.originalsOrder.length, + }; + } + + // Kick both fills. Idempotent: a fill already sweeping is left alone. Safe + // to call after every refresh; a finished fill starts a fresh sweep that + // picks up new files and retries any that failed before. + start(): void { + if (this.closed || !this.cache) return; + if (this.doThumbnails) this.kickThumbnails(); + if (this.doOriginals) this.kickOriginals(); + } + + // Stop the fills. In-flight fetches are left to settle; queued ones drop. + // Resolves once both sweeps have finished, so nothing is still writing. + async close(): Promise<void> { + this.closed = true; + this.aborter.abort(); + await Promise.all([ + this.thumbSweep?.catch(() => {}), + this.originalsSweep?.catch(() => {}), + ]); + } + + private kickThumbnails(): void { + if (this.thumbSweep) return; + this.thumbSweep = this.sweep( + "precacheThumbnails", + () => this.thumbOrder, + (id) => this.cache!.pathsFor(id).thumbnailPath !== undefined, + THUMBNAIL_CHUNK, + (chunk) => + this.cache!.ensureThumbnails({ + fileIDs: chunk, + priority: "background", + signal: this.aborter.signal, + }), + ).finally(() => { + this.thumbSweep = undefined; + }); + } + + private kickOriginals(): void { + if (this.originalsSweep) return; + this.originalsSweep = this.sweep( + "precacheOriginals", + () => this.originalsOrder, + (id) => this.cache!.pathsFor(id).originalPath !== undefined, + ORIGINAL_CHUNK, + (chunk) => + this.cache!.ensureOriginals({ + fileIDs: chunk, + signal: this.aborter.signal, + }), + ).finally(() => { + this.originalsSweep = undefined; + }); + } + + // One fill sweep: skip files already on disk (one lookup each), fetch the + // rest in bounded chunks, and report progress only when there was work. + private async sweep( + operation: PrecacheEvent["operation"], + order: () => number[], + present: (fileID: number) => boolean, + chunkSize: number, + fetch: (chunk: number[]) => Promise<EnsureResult[]>, + ): Promise<void> { + const todo = order().filter((id) => !present(id)); + if (todo.length === 0) return; + this.emit({ operation, status: "started" }); + let fetched = 0; + for (let i = 0; i < todo.length && !this.closed; i += chunkSize) { + const results = await fetch(todo.slice(i, i + chunkSize)); + for (const r of results) if (r.path !== undefined) fetched++; + } + this.emit({ operation, status: "done", fetched }); + } + + private emit(event: PrecacheEvent): void { + if (!this.onEvent) return; + // A misbehaving callback must not break the fill loop. + try { + this.onEvent(event); + } catch { + // ignore + } + } +} diff --git a/src/library/read.ts b/src/library/read.ts new file mode 100644 index 0000000..09b95cf --- /dev/null +++ b/src/library/read.ts @@ -0,0 +1,389 @@ +// The in-process read surface over the local cache (issue #44). +// +// A CLI or an in-process script reads albums, photos, and a grouped timeline +// through `lib.albums`, `lib.photos`, and `lib.timeline`. Every call is +// answered synchronously from the same plain-record projection the GUI reads +// (`deriveRecords`, issue #43); nothing here touches the network. Every method +// takes a single named-argument object. +// +// The `Album` and `Photo` classes are thin, in-process-only wrappers over +// those records: a caller that holds an object reference gets typed field +// access and, for an album, its photos. They are not sent across IPC — the +// plain records are the serializable surface, and `record()` returns one. +// +// A `Photo` also fetches its own bytes: `original()` and `thumbnail()` go +// through the on-disk content cache (issue #46), the one place in this module +// that is not synchronous and RAM-only. A library opened without a content +// source leaves that cache absent, and those two methods then throw. + +import type { CollectionType, FileType } from "../model/types.js"; +import type { ContentOptions, ContentResult, PhotoContent } from "./content.js"; +import type { AlbumRecord, PhotoRecord, DerivedRecords } from "./records.js"; + +// Newest first, with fileID as a stable tiebreak so equal-timed files order +// deterministically — the same order the record projection uses. +const byNewest = (a: PhotoRecord, b: PhotoRecord): number => + b.takenAt - a.takenAt || b.fileID - a.fileID; + +// Albums newest updated first, collection id breaking ties. This is the order +// `albums.list` returns and the order `byName` resolves a name collision in. +const byNewestAlbum = (a: AlbumRecord, b: AlbumRecord): number => + b.updationTime - a.updationTime || b.collectionID - a.collectionID; + +// A single photo. Field access mirrors `PhotoRecord`; `record()` returns the +// underlying plain record for callers that need the IPC-safe value. +export class Photo { + constructor( + private readonly rec: PhotoRecord, + private readonly content?: PhotoContent, + ) {} + + get fileID(): number { + return this.rec.fileID; + } + get albumIDs(): number[] { + return this.rec.albumIDs; + } + get title(): string { + return this.rec.title; + } + get takenAt(): number { + return this.rec.takenAt; + } + get fileType(): FileType { + return this.rec.fileType; + } + get caption(): string | undefined { + return this.rec.caption; + } + get width(): number | undefined { + return this.rec.width; + } + get height(): number | undefined { + return this.rec.height; + } + get latitude(): number | undefined { + return this.rec.latitude; + } + get longitude(): number | undefined { + return this.rec.longitude; + } + get isArchived(): boolean { + return this.rec.isArchived; + } + get isHidden(): boolean { + return this.rec.isHidden; + } + + record(): PhotoRecord { + return this.rec; + } + + // Fetch and cache the full-resolution original, returning its on-disk path + // and byte length; for a live photo, its image's, and its video's path as + // `videoPath`. Served from the cache (or the backup download directory) + // when already present, otherwise fetched through the content pool. + async original(opts?: ContentOptions): Promise<ContentResult> { + return this.contentOrThrow().original(this.rec.fileID, opts); + } + + // As `original`, for the thumbnail, through the thumbnail pool. + async thumbnail(opts?: ContentOptions): Promise<ContentResult> { + return this.contentOrThrow().thumbnail(this.rec.fileID, opts); + } + + private contentOrThrow(): PhotoContent { + if (!this.content) { + throw new Error( + "Photo content requires a library opened with a content cache", + ); + } + return this.content; + } +} + +// A single album. `photos.list()` returns the album's photos as wrappers, +// newest first (the record already stores `fileIDs` in that order). +export class Album { + constructor( + private readonly rec: AlbumRecord, + private readonly records: DerivedRecords, + private readonly content?: PhotoContent, + ) {} + + get collectionID(): number { + return this.rec.collectionID; + } + get name(): string { + return this.rec.name; + } + get type(): CollectionType { + return this.rec.type; + } + get isShared(): boolean { + return this.rec.isShared; + } + get updationTime(): number { + return this.rec.updationTime; + } + get fileIDs(): number[] { + return this.rec.fileIDs; + } + + get photos(): { list: () => Photo[] } { + return { list: (): Photo[] => this.listPhotos() }; + } + + record(): AlbumRecord { + return this.rec; + } + + private listPhotos(): Photo[] { + const out: Photo[] = []; + for (const id of this.rec.fileIDs) { + const p = this.records.photos.get(id); + if (p) out.push(new Photo(p, this.content)); + } + return out; + } +} + +export interface AlbumsAPI { + list(): Album[]; + byName(args: { albumName: string }): Album | undefined; + byID(args: { collectionID: number }): Album | undefined; +} + +export interface PhotosAPI { + byID(args: { fileID: number }): Photo | undefined; + // Plain records for the requested ids, in the order requested, each id at + // most once, unknown ids dropped. + records(args: { fileIDs: number[] }): PhotoRecord[]; +} + +export type GroupBy = "day" | "week" | "month"; + +// A filter over the timeline. All fields are optional and combine with AND. +// Hidden photos are never included, regardless of this filter. +export interface PhotoFilter { + // Keep only photos that belong to this album. + albumID?: number; + // Case-insensitive substring of the title, caption, or any album name the + // photo belongs to. + text?: string; + // Keep only photos of one of these types. + fileTypes?: FileType[]; + // `true` keeps only geotagged photos; `false` keeps only those without a + // location; omitted places no constraint. + hasLocation?: boolean; + // Archived photos are excluded unless this is `true`. Defaults to `false`. + includeArchived?: boolean; +} + +export interface TimelineGroup { + // The period's identity: `YYYY-MM-DD` for day, `YYYY-Www` (ISO 8601 week, + // e.g. `2025-W32`) for week, and `YYYY-MM` for month. + key: string; + // Local-time milliseconds at the start of the period. + startsAt: number; + // The period's files, newest first, each file once. + fileIDs: number[]; +} + +export interface TimelineAPI { + groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[]; +} + +// The surface `Library.fresh()` resolves to (issue #75). It is the same three +// read namespaces as the default `albums`/`photos`/`timeline`, handed back only +// after a forced refresh has brought the local copy current. +export interface FreshReads { + albums: AlbumsAPI; + photos: PhotosAPI; + timeline: TimelineAPI; +} + +export const makeAlbumsAPI = ( + derive: () => DerivedRecords, + content?: PhotoContent, +): AlbumsAPI => ({ + list: (): Album[] => { + const records = derive(); + return [...records.albums.values()] + .sort(byNewestAlbum) + .map((rec) => new Album(rec, records, content)); + }, + byID: ({ collectionID }): Album | undefined => { + const records = derive(); + const rec = records.albums.get(collectionID); + return rec ? new Album(rec, records, content) : undefined; + }, + byName: ({ albumName }): Album | undefined => { + const records = derive(); + // Names are not unique in Ente; resolve a collision deterministically + // to the newest-updated album, matching `list` order. + const match = [...records.albums.values()] + .sort(byNewestAlbum) + .find((rec) => rec.name === albumName); + return match ? new Album(match, records, content) : undefined; + }, +}); + +export const makePhotosAPI = ( + derive: () => DerivedRecords, + content?: PhotoContent, +): PhotosAPI => ({ + byID: ({ fileID }): Photo | undefined => { + const rec = derive().photos.get(fileID); + return rec ? new Photo(rec, content) : undefined; + }, + records: ({ fileIDs }): PhotoRecord[] => { + const { photos } = derive(); + const seen = new Set<number>(); + const out: PhotoRecord[] = []; + for (const id of fileIDs) { + if (seen.has(id)) continue; + const rec = photos.get(id); + if (rec) { + out.push(rec); + seen.add(id); + } + } + return out; + }, +}); + +export const makeTimelineAPI = (derive: () => DerivedRecords): TimelineAPI => ({ + groups: ({ groupBy, filter }): TimelineGroup[] => { + const records = derive(); + return groupPhotos(filterPhotos(records, filter), groupBy); + }, +}); + +// Apply a `PhotoFilter` to the projection. Hidden photos are always dropped; +// archived photos are dropped unless `includeArchived` asks for them. +const filterPhotos = ( + records: DerivedRecords, + filter?: PhotoFilter, +): PhotoRecord[] => { + const f = filter ?? {}; + const includeArchived = f.includeArchived ?? false; + const needle = f.text?.toLowerCase(); + const out: PhotoRecord[] = []; + for (const rec of records.photos.values()) { + if (rec.isHidden) continue; + if (rec.isArchived && !includeArchived) continue; + if (f.albumID !== undefined && !rec.albumIDs.includes(f.albumID)) + continue; + if (f.fileTypes !== undefined && !f.fileTypes.includes(rec.fileType)) + continue; + if (f.hasLocation !== undefined) { + const has = + rec.latitude !== undefined && rec.longitude !== undefined; + if (has !== f.hasLocation) continue; + } + if (needle !== undefined && !matchesText(rec, needle, records)) + continue; + out.push(rec); + } + return out; +}; + +const matchesText = ( + rec: PhotoRecord, + needle: string, + records: DerivedRecords, +): boolean => { + if (rec.title.toLowerCase().includes(needle)) return true; + if (rec.caption !== undefined && rec.caption.toLowerCase().includes(needle)) + return true; + for (const id of rec.albumIDs) { + const album = records.albums.get(id); + if (album && album.name.toLowerCase().includes(needle)) return true; + } + return false; +}; + +// Bucket photos into periods, groups newest first, members newest first. +const groupPhotos = ( + photos: PhotoRecord[], + groupBy: GroupBy, +): TimelineGroup[] => { + const buckets = new Map< + string, + { startsAt: number; recs: PhotoRecord[] } + >(); + for (const rec of photos) { + const { key, startsAt } = periodOf(rec.takenAt, groupBy); + const bucket = buckets.get(key); + if (bucket) bucket.recs.push(rec); + else buckets.set(key, { startsAt, recs: [rec] }); + } + const groups: TimelineGroup[] = []; + for (const [key, bucket] of buckets) { + bucket.recs.sort(byNewest); + groups.push({ + key, + startsAt: bucket.startsAt, + fileIDs: bucket.recs.map((r) => r.fileID), + }); + } + groups.sort((a, b) => b.startsAt - a.startsAt); + return groups; +}; + +const pad = (n: number): string => String(n).padStart(2, "0"); + +const dateKey = (d: Date): string => + `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`; + +const WEEK_MS = 7 * 24 * 60 * 60 * 1000; + +// The ISO 8601 week key `YYYY-Www` for the week starting at the given Monday. +// The week-year is the year of that week's Thursday, so it can differ from the +// calendar year at the January/December boundary (e.g. 2024-12-30 is 2025-W01). +const isoWeekKey = (monday: Date): string => { + const thursday = new Date( + monday.getFullYear(), + monday.getMonth(), + monday.getDate() + 3, + ); + const isoYear = thursday.getFullYear(); + // Thursday of ISO week 1 is the Thursday of the week containing January 4. + const jan4 = new Date(isoYear, 0, 4); + const week1Thursday = new Date( + isoYear, + 0, + 4 + 3 - ((jan4.getDay() + 6) % 7), + ); + const week = + 1 + + Math.round((thursday.getTime() - week1Thursday.getTime()) / WEEK_MS); + return `${isoYear}-W${pad(week)}`; +}; + +// The period a millisecond instant falls in, in local time. Weeks start on +// Monday. `Date` normalizes out-of-range day arguments, so the week's Monday +// is correct across month and year boundaries. +const periodOf = ( + takenAt: number, + groupBy: GroupBy, +): { key: string; startsAt: number } => { + const d = new Date(takenAt); + const year = d.getFullYear(); + const month = d.getMonth(); + const day = d.getDate(); + + if (groupBy === "month") { + const start = new Date(year, month, 1); + return { key: `${year}-${pad(month + 1)}`, startsAt: start.getTime() }; + } + if (groupBy === "week") { + // getDay(): 0=Sunday..6=Saturday; shift so Monday is the week start. + const fromMonday = (d.getDay() + 6) % 7; + const start = new Date(year, month, day - fromMonday); + return { key: isoWeekKey(start), startsAt: start.getTime() }; + } + const start = new Date(year, month, day); + return { key: dateKey(start), startsAt: start.getTime() }; +}; diff --git a/src/library/records.ts b/src/library/records.ts new file mode 100644 index 0000000..c06681f --- /dev/null +++ b/src/library/records.ts @@ -0,0 +1,271 @@ +// Plain records projected from the decrypted store, and the diff between two +// projections. These are the library's GUI-facing surface: they hold no key +// material and no binary, so they survive `structuredClone`/JSON across the +// Electron IPC boundary where methods and file keys cannot go (design #36, +// owner ruling 5). The decrypted `Collection`/`EnteFile` objects stay in RAM in +// the main process; the window only ever sees these records. +// +// Ente holds edited/basic times in microseconds; records expose `takenAt` in +// milliseconds. The magic-metadata field names below are the ones the Ente +// clients write, confirmed against the repo's own fixtures: `w`/`h` in +// test/cli/metadata-backup.test.ts, `visibility` in test/library/store.test.ts. + +import type { + Collection, + CollectionType, + EnteFile, + FileType, +} from "../model/types.js"; + +// Ente private-magic-metadata visibility values. +const VISIBILITY_ARCHIVED = 1; +const VISIBILITY_HIDDEN = 2; + +// A single photo, deduplicated across the collections it belongs to. No key, +// no binary: safe to send to a window. +export interface PhotoRecord { + fileID: number; + // Every collection this file is a member of, ascending. + albumIDs: number[]; + // `pubMagicMetadata.editedName` when the user renamed the file, else the + // basic-metadata title. + title: string; + // Milliseconds. `pubMagicMetadata.editedTime` when the user edited the + // date, else basic-metadata `creationTime`. + takenAt: number; + fileType: FileType; + caption?: string; + width?: number; + height?: number; + latitude?: number; + longitude?: number; + isArchived: boolean; + isHidden: boolean; + // Local cache paths, set once a later phase caches the bytes; unset here. + // A live photo's `originalPath` is its image. + thumbnailPath?: string; + originalPath?: string; +} + +export interface AlbumRecord { + collectionID: number; + name: string; + // `favorites` identifies the account's favorites album. + type: CollectionType; + isShared: boolean; + updationTime: number; + // The album's files, newest first. + fileIDs: number[]; +} + +export interface LibrarySnapshot { + albums: AlbumRecord[]; + photos: PhotoRecord[]; + // Wall-clock milliseconds when the snapshot was taken. + takenAt: number; +} + +export interface LibraryChange { + // Full records for albums/photos added or changed by the refresh. + albumsChanged: AlbumRecord[]; + photosChanged: PhotoRecord[]; + fileIDsRemoved: number[]; + albumIDsRemoved: number[]; + // Wall-clock milliseconds of the refresh that produced this change. + refreshedAt: number; +} + +// The by-id projection of the store at one moment; the source for both +// `snapshotFrom` (sorted arrays for the GUI) and `diffRecords` (change sets). +export interface DerivedRecords { + albums: Map<number, AlbumRecord>; + photos: Map<number, PhotoRecord>; +} + +const asString = (v: unknown): string | undefined => + typeof v === "string" && v.length > 0 ? v : undefined; + +const asNumber = (v: unknown): number | undefined => + typeof v === "number" && Number.isFinite(v) ? v : undefined; + +const microsToMillis = (micros: number): number => Math.floor(micros / 1000); + +// Newest first, with fileID as a stable tiebreak so equal-timed files order +// deterministically. +const byNewestPhoto = (a: PhotoRecord, b: PhotoRecord): number => + b.takenAt - a.takenAt || b.fileID - a.fileID; + +// Build one PhotoRecord from every membership of a file. The memberships share +// the same underlying file, so metadata is read from a single representative +// (the most recently synced, lowest collection id to break ties); `albumIDs` +// gathers them all. +const toPhotoRecord = ( + fileID: number, + memberships: EnteFile[], +): PhotoRecord => { + const albumIDs = memberships + .map((m) => m.collectionID) + .sort((a, b) => a - b); + const rep = memberships.reduce((best, m) => + m.updationTime > best.updationTime || + (m.updationTime === best.updationTime && + m.collectionID < best.collectionID) + ? m + : best, + ); + + const pub = rep.pubMagicMetadata ?? {}; + const priv = rep.magicMetadata ?? {}; + + const takenAtMicros = asNumber(pub.editedTime) ?? rep.metadata.creationTime; + const visibility = asNumber(priv.visibility); + + const record: PhotoRecord = { + fileID, + albumIDs, + title: asString(pub.editedName) ?? rep.metadata.title, + takenAt: microsToMillis(takenAtMicros), + fileType: rep.metadata.fileType, + isArchived: visibility === VISIBILITY_ARCHIVED, + isHidden: visibility === VISIBILITY_HIDDEN, + }; + + const caption = asString(pub.caption); + if (caption !== undefined) record.caption = caption; + const width = asNumber(pub.w); + if (width !== undefined) record.width = width; + const height = asNumber(pub.h); + if (height !== undefined) record.height = height; + if (rep.metadata.latitude !== undefined) + record.latitude = rep.metadata.latitude; + if (rep.metadata.longitude !== undefined) + record.longitude = rep.metadata.longitude; + + return record; +}; + +const toAlbumRecord = ( + collection: Collection, + files: EnteFile[], + takenAtByFile: Map<number, number>, +): AlbumRecord => { + const fileIDs = files + .filter((f) => f.collectionID === collection.id) + .map((f) => f.id) + .sort( + (a, b) => + (takenAtByFile.get(b) ?? 0) - (takenAtByFile.get(a) ?? 0) || + b - a, + ); + return { + collectionID: collection.id, + name: collection.name, + type: collection.type, + isShared: collection.isShared, + updationTime: collection.updationTime, + fileIDs, + }; +}; + +// The cache paths known for a file, so the projection can expose them on the +// record without the read layer reaching into the content cache itself. +export type CachedPathLookup = (fileID: number) => { + originalPath?: string; + thumbnailPath?: string; +}; + +// Project the decrypted collections and file memberships into by-id records. +// `files` is every membership (a file appears once per collection it is in). +// `cachedPaths`, when given, fills each record's cache paths. +export const deriveRecords = ( + collections: Collection[], + files: EnteFile[], + cachedPaths?: CachedPathLookup, +): DerivedRecords => { + const byFileID = new Map<number, EnteFile[]>(); + for (const f of files) { + const arr = byFileID.get(f.id); + if (arr) arr.push(f); + else byFileID.set(f.id, [f]); + } + + const photos = new Map<number, PhotoRecord>(); + const takenAtByFile = new Map<number, number>(); + for (const [fileID, memberships] of byFileID) { + const record = toPhotoRecord(fileID, memberships); + if (cachedPaths) { + const paths = cachedPaths(fileID); + if (paths.originalPath !== undefined) + record.originalPath = paths.originalPath; + if (paths.thumbnailPath !== undefined) + record.thumbnailPath = paths.thumbnailPath; + } + photos.set(fileID, record); + takenAtByFile.set(fileID, record.takenAt); + } + + const albums = new Map<number, AlbumRecord>(); + for (const c of collections) { + albums.set(c.id, toAlbumRecord(c, files, takenAtByFile)); + } + + return { albums, photos }; +}; + +// Sorted, GUI-ready arrays: albums newest updated first, photos newest first. +export const snapshotFrom = ( + records: DerivedRecords, + takenAt: number, +): LibrarySnapshot => ({ + albums: [...records.albums.values()].sort( + (a, b) => + b.updationTime - a.updationTime || b.collectionID - a.collectionID, + ), + photos: [...records.photos.values()].sort(byNewestPhoto), + takenAt, +}); + +// Records compare by value; they are plain and built with a fixed key order, so +// a serialized form is a sound equality key. +const same = (a: unknown, b: unknown): boolean => + JSON.stringify(a) === JSON.stringify(b); + +const diffMap = <T>( + prev: Map<number, T>, + next: Map<number, T>, +): { changed: T[]; removed: number[] } => { + const changed: T[] = []; + for (const [id, record] of next) { + const before = prev.get(id); + if (before === undefined || !same(before, record)) changed.push(record); + } + const removed: number[] = []; + for (const id of prev.keys()) if (!next.has(id)) removed.push(id); + removed.sort((a, b) => a - b); + return { changed, removed }; +}; + +// The change between two projections, or undefined when nothing changed. +export const diffRecords = ( + prev: DerivedRecords, + next: DerivedRecords, + refreshedAt: number, +): LibraryChange | undefined => { + const albums = diffMap(prev.albums, next.albums); + const photos = diffMap(prev.photos, next.photos); + if ( + albums.changed.length === 0 && + albums.removed.length === 0 && + photos.changed.length === 0 && + photos.removed.length === 0 + ) { + return undefined; + } + return { + albumsChanged: albums.changed, + photosChanged: photos.changed, + fileIDsRemoved: photos.removed, + albumIDsRemoved: albums.removed, + refreshedAt, + }; +}; diff --git a/src/library/store.ts b/src/library/store.ts new file mode 100644 index 0000000..56fe0dd --- /dev/null +++ b/src/library/store.ts @@ -0,0 +1,205 @@ +// On-disk JSON metadata store for the local cache. +// +// The store keeps one `metadata.json` file holding the account's server +// state: the user id, a schema version, the cursor for the incremental +// collections listing, and the decrypted collection and file records. The +// whole file is read into RAM on load and rewritten as a whole on save; there +// is no partial update and no lock file. A separate refresh unit populates the +// store from the server — this module only stores what it is given. +// +// The file is a cache, so it is never trusted to exist or to be intact: a +// missing or unreadable file loads as an empty store rather than an error, and +// the refresh unit then repopulates it. + +import { mkdir, chmod, readFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +import { writeAtomic } from "../download/index.js"; +import type { Collection, EnteFile, Microseconds } from "../model/types.js"; + +// Bumped only when the on-disk shape changes incompatibly. A file written +// under a different version is discarded on load (see `load`): re-fetching +// from the server is always safe and cheaper than migrating a cache. +export const METADATA_SCHEMA_VERSION = 1; + +// Directory and file modes match `session.json`: the records hold decrypted +// key material, so on a shared machine only the owner may read them. +const DIR_MODE = 0o700; +const FILE_MODE = 0o600; + +// On-disk shapes. They mirror the in-memory model exactly except for the +// binary `key`, which JSON cannot hold and which is stored as base64. +type StoredCollection = Omit<Collection, "key"> & { key: string }; +type StoredFile = Omit<EnteFile, "key"> & { key: string }; + +interface StoredMetadata { + schemaVersion: number; + userID: number; + collectionsSinceTime: Microseconds; + collections: StoredCollection[]; + files: StoredFile[]; +} + +const encodeKey = (key: Uint8Array): string => + Buffer.from(key).toString("base64"); + +const decodeKey = (encoded: string): Uint8Array => + new Uint8Array(Buffer.from(encoded, "base64")); + +// A file membership is identified by the pair (collectionID, fileID): the same +// underlying file can belong to several collections, each a distinct record +// with its own key. +const fileKey = (collectionID: number, fileID: number): string => + `${collectionID}:${fileID}`; + +export class MetadataStore { + readonly path: string; + readonly schemaVersion = METADATA_SCHEMA_VERSION; + userID = 0; + collectionsSinceTime: Microseconds = 0; + + // True when `load` populated this store from a valid existing file; false + // on a first run or a missing/corrupt/wrong-version file that loaded empty. + // `Library.open` reads it to decide whether the first refresh may run in + // the background (an existing copy already serves reads) or must be awaited. + loadedFromDisk = false; + + private readonly collections = new Map<number, Collection>(); + private readonly files = new Map<string, EnteFile>(); + + private constructor(path: string) { + this.path = path; + } + + // Load the store at `path`. A missing file, an unreadable one, unparseable + // contents, or a mismatched schema version all yield an empty store bound + // to that path — never a thrown error, because the file is only a cache. + static async load(path: string): Promise<MetadataStore> { + const store = new MetadataStore(path); + + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch { + return store; + } + + try { + const parsed = JSON.parse(raw) as StoredMetadata; + if (parsed.schemaVersion !== METADATA_SCHEMA_VERSION) { + return store; + } + store.loadedFromDisk = true; + store.userID = parsed.userID ?? 0; + store.collectionsSinceTime = parsed.collectionsSinceTime ?? 0; + for (const stored of parsed.collections ?? []) { + const collection: Collection = { + ...stored, + key: decodeKey(stored.key), + }; + store.collections.set(collection.id, collection); + } + for (const stored of parsed.files ?? []) { + const file: EnteFile = { + ...stored, + key: decodeKey(stored.key), + }; + store.files.set(fileKey(file.collectionID, file.id), file); + } + } catch { + // Any corruption discards the partial result: a half-read cache is + // worse than an empty one, since the refresh unit will rebuild it. + return new MetadataStore(path); + } + + return store; + } + + // Rewrite the whole file. The directory is created 0700 and the file left + // 0600; the write itself is the download layer's durable atomic writer + // (temp file, fsync, rename, dir fsync), so a reader never sees a partial + // file and a crash cannot leave a truncated one. There is no lock file and + // no `sync()` beyond the writer's own fsyncs. + async save(): Promise<void> { + const model: StoredMetadata = { + schemaVersion: METADATA_SCHEMA_VERSION, + userID: this.userID, + collectionsSinceTime: this.collectionsSinceTime, + collections: [...this.collections.values()].map((c) => ({ + ...c, + key: encodeKey(c.key), + })), + files: [...this.files.values()].map((f) => ({ + ...f, + key: encodeKey(f.key), + })), + }; + + const dir = dirname(this.path); + // chmod after mkdir so the mode is 0700 even when the directory + // already existed with a looser mode; mkdir alone would not tighten + // an existing directory. + await mkdir(dir, { recursive: true, mode: DIR_MODE }); + await chmod(dir, DIR_MODE); + + const payload = new TextEncoder().encode( + JSON.stringify(model, null, 2), + ); + await writeAtomic(this.path, payload); + // The atomic writer's temp file inherits the default mode; tighten the + // renamed file to 0600. The 0700 directory already keeps other users + // out during the brief window before this runs. + await chmod(this.path, FILE_MODE); + } + + getCollection(id: number): Collection | undefined { + return this.collections.get(id); + } + + listCollections(): Collection[] { + return [...this.collections.values()]; + } + + putCollection(collection: Collection): void { + this.collections.set(collection.id, collection); + } + + // Removing a collection also drops its file memberships: a file record is + // only meaningful as part of a collection the cache still knows about. + deleteCollection(id: number): void { + this.collections.delete(id); + for (const [key, file] of this.files) { + if (file.collectionID === id) { + this.files.delete(key); + } + } + } + + getFile(collectionID: number, fileID: number): EnteFile | undefined { + return this.files.get(fileKey(collectionID, fileID)); + } + + // Any membership of a file, or undefined. Every membership re-wraps the + // same underlying content key, so any one is enough to fetch the bytes; + // the content cache resolves a fileID to a file this way. + getFileByID(fileID: number): EnteFile | undefined { + for (const file of this.files.values()) { + if (file.id === fileID) return file; + } + return undefined; + } + + listFiles(collectionID: number): EnteFile[] { + return [...this.files.values()].filter( + (f) => f.collectionID === collectionID, + ); + } + + putFile(file: EnteFile): void { + this.files.set(fileKey(file.collectionID, file.id), file); + } + + deleteFile(collectionID: number, fileID: number): void { + this.files.delete(fileKey(collectionID, fileID)); + } +} diff --git a/src/metadata-backup.ts b/src/metadata-backup.ts index f227ddf..2bee63e 100644 --- a/src/metadata-backup.ts +++ b/src/metadata-backup.ts @@ -1,17 +1,15 @@ -import { gunzipSync } from "node:zlib"; -import { - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - writeFileSync, -} from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; -import { tmpdir } from "node:os"; import * as jpeg from "jpeg-js"; import exifReader from "exif-reader"; import type { Client } from "./client.js"; -import { decryptBlob, fromBase64 } from "./crypto/index.js"; +import type { Library, Photo } from "./library/index.js"; +import { sanitizeFileName } from "./filename.js"; +import { + fetchMLDataBatch, + MLDATA_BATCH_SIZE, + type MLData, +} from "./mldata-fetch.js"; import type { EnteFile } from "./model/types.js"; export type ProgressCallback = (message: string) => void; @@ -21,156 +19,137 @@ export interface MetadataBackupOptions { onProgress?: ProgressCallback; } -const sanitizePath = (name: string): string => - name.replace(/[/\\:*?"<>|]/g, "_").replace(/^\.+/, "_"); - -interface RawRemoteFileData { - fileID: number; - encryptedData: string; - decryptionHeader: string; - updatedAt?: number; -} - -const fetchMLDataForFiles = async ( - client: Client, - fileIDs: number[], - fileKeys: Map<number, Uint8Array>, -): Promise<Map<number, Record<string, unknown>>> => { - const api = client.getApiClient(); - const result = new Map<number, Record<string, unknown>>(); - const batchSize = 200; - - for (let i = 0; i < fileIDs.length; i += batchSize) { - const batch = fileIDs.slice(i, i + batchSize); - const { data } = await api.postJSON<{ data: RawRemoteFileData[] }>( - "/files/data/fetch", - { type: "mldata", fileIDs: batch }, - ); - - for (const entry of data ?? []) { - const key = fileKeys.get(entry.fileID); - if (!key) continue; - try { - const decrypted = decryptBlob( - fromBase64(entry.encryptedData), - fromBase64(entry.decryptionHeader), - key, - ); - const jsonStr = gunzipSync(Buffer.from(decrypted)).toString( - "utf-8", - ); - result.set(entry.fileID, JSON.parse(jsonStr)); - } catch { - // Corrupted ML data for this file; skip it - } - } - } - return result; -}; - -// Extract the raw EXIF APP1 segment from JPEG bytes. Returns the EXIF -// data buffer (starting after the APP1 length field, at the "Exif\0\0" -// header) or undefined if no APP1 marker is found. -const extractExifFromJpeg = (buf: Uint8Array): Buffer | undefined => { - if (buf[0] !== 0xff || buf[1] !== 0xd8) return undefined; +// Find the raw EXIF APP1 segment in JPEG bytes. Returns `exif` (the segment +// data, starting at the "Exif\0\0" header) when there is one, nothing when the +// bytes are not a JPEG or carry no EXIF, and `error` when the segment layout is +// malformed. Each segment length is checked against the bytes that remain and +// each step moves forward by at least 4 bytes, so the scan ends on any input. +export const extractExifFromJpeg = ( + buf: Uint8Array, +): { exif?: Buffer; error?: string } => { + if (buf[0] !== 0xff || buf[1] !== 0xd8) return {}; let offset = 2; - while (offset < buf.length - 1) { - if (buf[offset] !== 0xff) return undefined; + while (offset < buf.length) { + if (offset + 2 > buf.length) + return { error: `truncated segment marker at byte ${offset}` }; + if (buf[offset] !== 0xff) + return { error: `no segment marker at byte ${offset}` }; const marker = buf[offset + 1]!; - if (marker === 0xda) break; // start of scan, no more markers - if (offset + 3 >= buf.length) break; + if (marker === 0xda) return {}; // start of scan, no more markers + if (offset + 4 > buf.length) + return { error: `truncated segment length at byte ${offset}` }; const len = (buf[offset + 2]! << 8) | buf[offset + 3]!; + // The length counts its own two bytes, so anything under 2 is invalid. + if (len < 2) + return { + error: `segment length ${len} at byte ${offset} is too small`, + }; + if (offset + 2 + len > buf.length) + return { + error: `segment length ${len} at byte ${offset} runs past the end of the file`, + }; if (marker === 0xe1) { - // APP1 — check for "Exif\0\0" header + // APP1 — check for "Exif\0\0" header. A length under 8 cannot hold + // the six-byte header, so the segment is not EXIF; below 6 the + // bytes compared would also lie past the segment. if ( + len >= 8 && buf[offset + 4] === 0x45 && buf[offset + 5] === 0x78 && buf[offset + 6] === 0x69 && buf[offset + 7] === 0x66 ) { - return Buffer.from( - buf.buffer, - buf.byteOffset + offset + 4, - len - 2, - ); + return { + exif: Buffer.from( + buf.buffer, + buf.byteOffset + offset + 4, + len - 2, + ), + }; } } offset += 2 + len; } - return undefined; + return { error: "file ends before the image data" }; }; -const extractImageMetadata = ( +// Extract dimensions, EXIF and XMP from a file's bytes. When the EXIF segment +// is malformed or cannot be parsed, the record carries the reason in +// `exifError`. +export const extractImageMetadata = ( fileBytes: Uint8Array, ): Record<string, unknown> | undefined => { - try { - const result: Record<string, unknown> = {}; + const result: Record<string, unknown> = {}; - // Try to get dimensions from JPEG decode + // Try to get dimensions from JPEG decode + try { + const decoded = jpeg.decode(fileBytes, { + useTArray: true, + formatAsRGBA: false, + }); + result.format = "jpeg"; + result.width = decoded.width; + result.height = decoded.height; + } catch { + // Not every original is a JPEG (PNG, HEIC, video), so a failed decode + // is expected and only means no dimensions; a malformed JPEG is still + // reported below through `exifError`. + } + + const { exif, error } = extractExifFromJpeg(fileBytes); + if (error) result.exifError = error; + if (exif) { try { - const decoded = jpeg.decode(fileBytes, { - useTArray: true, - formatAsRGBA: false, - }); - result.format = "jpeg"; - result.width = decoded.width; - result.height = decoded.height; - } catch { - // Not a JPEG or corrupt; still try EXIF extraction + result.exif = exifReader(exif); + } catch (err) { + result.exifRaw = exif.toString("base64"); + result.exifError = err instanceof Error ? err.message : String(err); } - - const exifBuf = extractExifFromJpeg(fileBytes); - if (exifBuf) { - try { - result.exif = exifReader(exifBuf); - } catch { - result.exifRaw = exifBuf.toString("base64"); - } - } - - // Extract XMP (look for "http://ns.adobe.com/xap" in the bytes) - const xmpStart = Buffer.from(fileBytes).indexOf("<?xpacket begin"); - if (xmpStart !== -1) { - const xmpEnd = Buffer.from(fileBytes).indexOf( - "<?xpacket end", - xmpStart, - ); - if (xmpEnd !== -1) { - const end = Buffer.from(fileBytes).indexOf("?>", xmpEnd); - result.xmp = Buffer.from(fileBytes) - .subarray(xmpStart, end !== -1 ? end + 2 : xmpEnd + 50) - .toString("utf-8"); - } - } - - return Object.keys(result).length > 0 ? result : undefined; - } catch { - return undefined; } + + // Extract XMP (look for "http://ns.adobe.com/xap" in the bytes) + const xmpStart = Buffer.from(fileBytes).indexOf("<?xpacket begin"); + if (xmpStart !== -1) { + const xmpEnd = Buffer.from(fileBytes).indexOf( + "<?xpacket end", + xmpStart, + ); + if (xmpEnd !== -1) { + const end = Buffer.from(fileBytes).indexOf("?>", xmpEnd); + result.xmp = Buffer.from(fileBytes) + .subarray(xmpStart, end !== -1 ? end + 2 : xmpEnd + 50) + .toString("utf-8"); + } + } + + return Object.keys(result).length > 0 ? result : undefined; }; +// Read a file's original bytes through the library's content cache and extract +// its embedded image metadata. The bytes come from `photo.original()` — the +// same on-disk cache the rest of the library fills — rather than a fresh +// per-call download to a throwaway temp file. For a live photo, its `path` is +// the image. const extractExif = async ( - client: Client, - file: EnteFile, + photo: Photo, ): Promise<Record<string, unknown> | undefined> => { - const tmpDir = mkdtempSync(join(tmpdir(), "quak-exif-")); - try { - const origPath = join(tmpDir, "original"); - await client.downloadFile(file, origPath); - const fileBytes = new Uint8Array(readFileSync(origPath)); - return extractImageMetadata(fileBytes); - } catch { - return undefined; - } finally { - rmSync(tmpDir, { recursive: true, force: true }); - } + const { path } = await photo.original(); + const fileBytes = new Uint8Array(readFileSync(path)); + return extractImageMetadata(fileBytes); }; +// Dump every decrypted metadata layer the account holds into a directory tree +// of plain JSON: account, per-collection, and per-file records including the +// private and public magic metadata and (by default) the ML data. Collections +// and files are enumerated from the library's cache, which the caller refreshes +// first. Returns how many ML data requests failed; their files are still +// written, with `mlDataError` in place of `mlData`. export const runMetadataBackup = async ( + lib: Library, client: Client, outDir: string, opts?: MetadataBackupOptions, -): Promise<void> => { +): Promise<{ failedMLBatches: number }> => { const log = opts?.onProgress ?? (() => {}); const wantExif = opts?.exif ?? false; @@ -184,14 +163,20 @@ export const runMetadataBackup = async ( ); log("Fetching collections..."); - const collections = await client.listCollections(); - const allFiles: { file: EnteFile; colDirName: string }[] = []; + // Enumerate through the library's read surface. Each album carries its + // photos, but the full decrypted `Collection`/`EnteFile` records (with the + // magic-metadata layers this dump exists to preserve) come from the + // library's by-id accessors. + const allFiles: { file: EnteFile; photo: Photo; colDirName: string }[] = []; const fileKeys = new Map<number, Uint8Array>(); const seenFileIDs = new Set<number>(); - for (const col of collections) { - const dirName = `${col.id}-${sanitizePath(col.name || "unnamed")}`; + for (const album of lib.albums.list()) { + const col = lib.getCollection(album.collectionID); + if (!col) continue; + + const dirName = `${col.id}-${sanitizeFileName(col.name, "unnamed")}`; const colDir = join(outDir, "collections", dirName); mkdirSync(colDir, { recursive: true }); @@ -215,11 +200,13 @@ export const runMetadataBackup = async ( ); log(`[${col.name}] Fetching files...`); - const files = await client.listFiles(col.id, col.key); - log(`[${col.name}] ${files.length} file(s)`); + const photos = album.photos.list(); + log(`[${col.name}] ${photos.length} file(s)`); - for (const file of files) { - allFiles.push({ file, colDirName: dirName }); + for (const photo of photos) { + const file = lib.getFile(col.id, photo.fileID); + if (!file) continue; + allFiles.push({ file, photo, colDirName: dirName }); if (!seenFileIDs.has(file.id)) { fileKeys.set(file.id, file.key); seenFileIDs.add(file.id); @@ -227,16 +214,35 @@ export const runMetadataBackup = async ( } } + // One failed request (retries exhausted) must not end the dump: its files + // get the reason in `mlDataError` and the other batches go on. log("Fetching ML data (face detections, CLIP embeddings)..."); - const mlDataMap = await fetchMLDataForFiles( - client, - [...fileKeys.keys()], - fileKeys, - ); + const mlDataMap = new Map<number, MLData>(); + const mlDataErrors = new Map<number, string>(); + let failedMLBatches = 0; + const fileIDs = [...fileKeys.keys()]; + for (let i = 0; i < fileIDs.length; i += MLDATA_BATCH_SIZE) { + const batch = fileIDs.slice(i, i + MLDATA_BATCH_SIZE); + try { + const result = await fetchMLDataBatch( + client.getApiClient(), + batch, + fileKeys, + ); + for (const [id, payload] of result) mlDataMap.set(id, payload); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + failedMLBatches++; + log( + `ML data request for ${batch.length} file(s) failed: ${reason}`, + ); + for (const id of batch) mlDataErrors.set(id, reason); + } + } log(`Got ML data for ${mlDataMap.size} file(s)`); const writtenFileIDs = new Set<number>(); - for (const { file, colDirName } of allFiles) { + for (const { file, photo, colDirName } of allFiles) { const colDir = join(outDir, "collections", colDirName); const fileMeta: Record<string, unknown> = { @@ -252,11 +258,18 @@ export const runMetadataBackup = async ( const ml = mlDataMap.get(file.id); if (ml) fileMeta.mlData = ml; + const mlError = mlDataErrors.get(file.id); + if (mlError) fileMeta.mlDataError = mlError; if (wantExif && !writtenFileIDs.has(file.id)) { log(`[${file.metadata.title}] Extracting EXIF...`); - const exifData = await extractExif(client, file); - if (exifData) fileMeta.imageMetadata = exifData; + try { + const exifData = await extractExif(photo); + if (exifData) fileMeta.imageMetadata = exifData; + } catch (err) { + fileMeta.imageMetadataError = + err instanceof Error ? err.message : String(err); + } } writtenFileIDs.add(file.id); @@ -267,4 +280,5 @@ export const runMetadataBackup = async ( } log("Metadata backup complete."); + return { failedMLBatches }; }; diff --git a/src/mldata-fetch.ts b/src/mldata-fetch.ts new file mode 100644 index 0000000..e5a254d --- /dev/null +++ b/src/mldata-fetch.ts @@ -0,0 +1,70 @@ +// Fetch and decrypt Ente's per-file machine-learning data ("magic" search +// data: face detections + CLIP embeddings). +// +// The data lives behind `/files/data/fetch` with `type: "mldata"`. Each entry +// comes back encrypted under the file's own key and gzipped; decrypting and +// gunzipping yields the JSON payload +// `{ face: { faces: [...] }, clip: { embedding } }`. Ente caps a request at 200 +// ids, so callers that want many at once split them into batches of +// `MLDATA_BATCH_SIZE` and call `fetchMLDataBatch` once per batch. + +import { gunzipSync } from "node:zlib"; + +import type { ApiClient } from "./api/client.js"; +import { decryptBlob, fromBase64 } from "./crypto/index.js"; + +// The most ids one `/files/data/fetch` request may carry. +export const MLDATA_BATCH_SIZE = 200; + +// The decrypted, gunzipped per-file payload. Its concrete shape is Ente's; the +// store keeps the whole object verbatim and each consumer reads the fields it +// needs, so it stays an open record rather than a fixed interface. +export type MLData = Record<string, unknown>; + +interface RawRemoteFileData { + fileID: number; + encryptedData: string; + decryptionHeader: string; + updatedAt?: number; +} + +// Decrypt one entry with its file key and gunzip the JSON payload. Returns +// undefined when the key is unknown or the entry does not decrypt/parse, so one +// corrupt file never fails a whole batch. +const decodeEntry = ( + entry: RawRemoteFileData, + key: Uint8Array | undefined, +): MLData | undefined => { + if (!key) return undefined; + try { + const decrypted = decryptBlob( + fromBase64(entry.encryptedData), + fromBase64(entry.decryptionHeader), + key, + ); + const json = gunzipSync(Buffer.from(decrypted)).toString("utf-8"); + return JSON.parse(json) as MLData; + } catch { + return undefined; + } +}; + +// Fetch ML data for up to `MLDATA_BATCH_SIZE` ids in a single request. This is +// the unit the request pools schedule; callers with more ids split them into +// batches and submit each batch to the pool. +export const fetchMLDataBatch = async ( + api: ApiClient, + fileIDs: number[], + fileKeys: Map<number, Uint8Array>, +): Promise<Map<number, MLData>> => { + const { data } = await api.postJSON<{ data: RawRemoteFileData[] }>( + "/files/data/fetch", + { type: "mldata", fileIDs }, + ); + const result = new Map<number, MLData>(); + for (const entry of data ?? []) { + const payload = decodeEntry(entry, fileKeys.get(entry.fileID)); + if (payload) result.set(entry.fileID, payload); + } + return result; +}; diff --git a/src/model/decrypt.ts b/src/model/decrypt.ts index fea8f99..9137479 100644 --- a/src/model/decrypt.ts +++ b/src/model/decrypt.ts @@ -34,6 +34,28 @@ const FILE_TYPE_MAP: Record<number, FileType> = { const parseFileType = (n: number): FileType => FILE_TYPE_MAP[n] ?? "unknown"; +// The hash the uploading client recorded for the original's bytes, read the +// way the upstream client's `metadataHash` reads it: `hash` if present, +// otherwise, for a live photo from an older client that wrote the two parts +// separately, `<imageHash>:<videoHash>`. A field that is not a non-empty +// string counts as absent, and a file with no hash at all is normal. +const expectedHash = (json: Record<string, unknown>): string | undefined => { + const text = (v: unknown): string | undefined => + typeof v === "string" && v !== "" ? v : undefined; + const hash = text(json.hash); + if (hash !== undefined) return hash; + const imageHash = text(json.imageHash); + const videoHash = text(json.videoHash); + if ( + json.fileType === 2 && + imageHash !== undefined && + videoHash !== undefined + ) { + return `${imageHash}:${videoHash}`; + } + return undefined; +}; + export const decryptCollection = ( raw: RawCollection, keys: KeyMaterial, @@ -98,15 +120,24 @@ export const decryptFile = ( key, ); const metadataJSON = JSON.parse(new TextDecoder().decode(metadataBytes)); + if ( + typeof metadataJSON !== "object" || + metadataJSON === null || + Array.isArray(metadataJSON) + ) { + throw new Error(`file ${raw.id}: metadata is not a JSON object`); + } const metadata: FileMetadata = { - title: metadataJSON.title ?? "", + // The server controls this JSON: a title that is missing or not a + // string becomes "", never an arbitrary value. + title: typeof metadataJSON.title === "string" ? metadataJSON.title : "", fileType: parseFileType(metadataJSON.fileType ?? -1), creationTime: metadataJSON.creationTime ?? 0, modificationTime: metadataJSON.modificationTime ?? 0, latitude: metadataJSON.latitude, longitude: metadataJSON.longitude, - hash: metadataJSON.hash, + hash: expectedHash(metadataJSON), }; const magicMetadata = decryptMagicMetadata(raw.magicMetadata, key); @@ -120,9 +151,16 @@ export const decryptFile = ( metadata, magicMetadata, pubMagicMetadata, - file: { decryptionHeader: raw.file.decryptionHeader }, - thumbnail: { decryptionHeader: raw.thumbnail.decryptionHeader }, + file: { + decryptionHeader: raw.file.decryptionHeader, + size: raw.info?.fileSize, + }, + thumbnail: { + decryptionHeader: raw.thumbnail.decryptionHeader, + size: raw.info?.thumbSize, + }, updationTime: raw.updationTime, + isDeleted: raw.isDeleted, }; }; diff --git a/src/model/types.ts b/src/model/types.ts index ceb6c79..516407a 100644 --- a/src/model/types.ts +++ b/src/model/types.ts @@ -29,6 +29,9 @@ export interface FileMetadata { modificationTime: Microseconds; latitude?: number; longitude?: number; + // The content hash the uploader recorded (see `expectedHash` in + // decrypt.ts); `downloadFile` refuses an original that does not match it. + // Absent for files from very old clients. hash?: string; } @@ -48,6 +51,9 @@ export interface EnteFile { file: FileBlob; thumbnail: FileBlob; updationTime: Microseconds; + // Set from the diff row's flag. Live files decode with it absent/false; + // deleted rows are filtered out before decryptFile, so it is not set here. + isDeleted?: boolean; } // The key material a logged-in client holds, everything needed to decrypt diff --git a/src/retry.ts b/src/retry.ts index 4d5eb69..1198a79 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -88,18 +88,21 @@ const MAX_CAUSE_DEPTH = 8; // errno on the error it throws — it hangs the underlying socket error off // `cause`, sometimes more than one level down — so a classifier that only read // the top-level error would see a bare `Error` and call every dropped -// connection permanent. -const causeCodes = (err: unknown): string[] => { +// connection permanent. `complete` is false when the walk stopped at the +// depth limit with more of the chain still below it. +const causeCodes = (err: unknown): { codes: string[]; complete: boolean } => { const codes: string[] = []; let current: unknown = err; for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) { - if (current === null || typeof current !== "object") break; + if (current === null || typeof current !== "object") { + return { codes, complete: true }; + } const { code, cause } = current as { code?: unknown; cause?: unknown }; if (typeof code === "string") codes.push(code); - if (cause === current) break; + if (cause === current) return { codes, complete: true }; current = cause; } - return codes; + return { codes, complete: current === null || typeof current !== "object" }; }; const isAbort = (err: unknown): boolean => { @@ -145,15 +148,15 @@ export const isRetryable = (err: unknown): boolean => { // have succeeded; the cost of the imprecision is bounded by the attempt // count. if (err instanceof TypeError) return true; - return causeCodes(err).some((code) => TRANSPORT_CODES.has(code)); + return causeCodes(err).codes.some((code) => TRANSPORT_CODES.has(code)); }; // Could the first attempt already have taken effect on the server? // // `isRetryable` is the wrong question for a request that changes state. -// quak's non-idempotent calls are `/users/srp/create-session`, -// `/users/two-factor/verify` — which consumes one of a small number of 2FA -// attempts — and `/files/thumbnail`. They are replayed only on the failures in +// `postJSON` and `putJSON` use this for every `POST` and `PUT` listed in the +// README under "Endpoints used"; verifying a second factor, for one, consumes +// one of a small number of attempts. They are replayed only on the failures in // `CONNECT_CODES`, which establish that no TCP connection to the server ever // existed: there was no address to connect to, or the peer refused the // connection outright. A request byte cannot have been transmitted, so the @@ -162,9 +165,19 @@ export const isRetryable = (err: unknown): boolean => { // Everything else is ambiguous. A 5xx proves the server did process the // request. A reset or a broken pipe can arrive after it was fully sent and // acted on. A routing errno can be delivered on an established socket. A -// deadline says nothing at all about the server's state. -export const isSafeToReplay = (err: unknown): boolean => - isRetryable(err) && causeCodes(err).some((code) => CONNECT_CODES.has(code)); +// deadline says nothing at all about the server's state. So every errno in the +// cause chain must be a connect errno: one other errno anywhere in the chain +// is doubt, and doubt is not replayed. A chain longer than the walk is doubt +// too: the links below the limit were never read. +export const isSafeToReplay = (err: unknown): boolean => { + const { codes, complete } = causeCodes(err); + return ( + isRetryable(err) && + complete && + codes.length > 0 && + codes.every((code) => CONNECT_CODES.has(code)) + ); +}; export interface WithRetryOptions extends RetryOptions { isRetryable?: (err: unknown) => boolean; diff --git a/src/thumbnails.ts b/src/thumbnails.ts index 254f53d..bb51f1e 100644 --- a/src/thumbnails.ts +++ b/src/thumbnails.ts @@ -2,16 +2,26 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import * as jpeg from "jpeg-js"; import type { Client } from "./client.js"; +import type { Library } from "./library/index.js"; import { ApiError } from "./api/client.js"; import { encryptBlob, toBase64 } from "./crypto/index.js"; -import { downloadFile } from "./download/index.js"; import type { EnteFile } from "./model/types.js"; -import { mkdtempSync, rmSync } from "node:fs"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; -const THUMB_MAX_DIMENSION = 720; -const THUMB_JPEG_QUALITY = 50; +// The server refuses a thumbnail larger than the one it already records for the +// file (`thumbnail.size`, the encrypted size), so these encodings are tried +// from largest to smallest and the first that fits is uploaded. +const THUMB_ENCODINGS = [ + { maxDimension: 720, quality: 50 }, + { maxDimension: 720, quality: 30 }, + { maxDimension: 480, quality: 30 }, + { maxDimension: 320, quality: 20 }, + { maxDimension: 160, quality: 20 }, +]; + +// The server accepts a new thumbnail only from the file's owner, so files other +// people own in albums shared with this account are never checked or repaired. +const NOT_OWNED_REASON = + "owned by another account (only the owner can replace its thumbnail)"; export interface MissingThumbnailInfo { fileID: number; @@ -20,34 +30,61 @@ export interface MissingThumbnailInfo { reason: string; } +// Three outcomes, not two. "fixed": a thumbnail was generated and uploaded. +// "failed": something went wrong (download, encode, upload) and the file still +// has no thumbnail. "skipped": the server would refuse any thumbnail for the +// file or this helper cannot regenerate it — a file another account owns, a +// recorded thumbnail size nothing fits within, a video, or an image that is +// not a baseline JPEG. Skipped is a deliberate, expected outcome, not an error +// (issue #17): the repair path is JPEG-only because `jpeg-js` is, and a PNG or +// HEIC is left for a format-aware tool rather than reported as a failure. +export type ThumbnailFixStatus = "fixed" | "skipped" | "failed"; + export interface ThumbnailFixResult { fileID: number; title: string; collection: string; - success: boolean; - error?: string; + status: ThumbnailFixStatus; + // Why the file was skipped or failed; unset when it was fixed. + reason?: string; } export type ProgressCallback = (message: string) => void; +// Enumerate every file the library knows about, newest album first, each file +// once, and report those whose server-side thumbnail is missing. "Missing" is +// only two answers: an empty body, or a 404. Any other error reaching this +// point has already exhausted its retries — a failing server, a dropped +// connection, a deadline — and says nothing about whether the thumbnail +// exists, so it is logged and the file is left unreported. That distinction is +// what stops `fix-missing-thumbnails` from regenerating and uploading over +// thumbnails that were fine all along while the CDN was briefly returning 500s. +// Files another account owns are logged as skipped and not checked. export const listMissingThumbnails = async ( + lib: Library, client: Client, onProgress?: ProgressCallback, ): Promise<MissingThumbnailInfo[]> => { const log = onProgress ?? (() => {}); + const api = client.getApiClient(); + const { userID } = client.whoami(); const missing: MissingThumbnailInfo[] = []; const seen = new Set<number>(); - const collections = await client.listCollections(); - for (const col of collections) { - log(`[${col.name}] Checking thumbnails...`); - const files = await client.listFiles(col.id, col.key); - for (const file of files) { - if (seen.has(file.id)) continue; - seen.add(file.id); + for (const album of lib.albums.list()) { + log(`[${album.name}] Checking thumbnails...`); + for (const photo of album.photos.list()) { + if (seen.has(photo.fileID)) continue; + seen.add(photo.fileID); + const file = lib.getFile(album.collectionID, photo.fileID); + if (file && file.ownerID !== userID) { + log( + `[${album.name}] Skipping ${photo.title}: ${NOT_OWNED_REASON}`, + ); + continue; + } try { - const api = client.getApiClient(); - const stream = await api.getThumbnailStream(file.id); + const stream = await api.getThumbnailStream(photo.fileID); const reader = stream.getReader(); let totalBytes = 0; for (;;) { @@ -57,35 +94,23 @@ export const listMissingThumbnails = async ( } if (totalBytes === 0) { missing.push({ - fileID: file.id, - title: file.metadata.title, - collection: col.name, + fileID: photo.fileID, + title: photo.title, + collection: album.name, reason: "empty thumbnail (0 bytes)", }); } } catch (err) { - // A 404 is the server stating the thumbnail is not there: - // that, and an empty body, are the only two answers that mean - // "missing". Anything else reaching this point is a failure - // that already exhausted its retries — a failing server, a - // dropped connection, a deadline — and says nothing about - // whether the thumbnail exists. - // - // The distinction is what stops `helper - // fix-missing-thumbnails` from downloading originals, - // regenerating thumbnails and uploading them over thumbnails - // that were fine all along, because the CDN was briefly - // returning 500s while this ran. if (err instanceof ApiError && err.status === 404) { missing.push({ - fileID: file.id, - title: file.metadata.title, - collection: col.name, + fileID: photo.fileID, + title: photo.title, + collection: album.name, reason: "thumbnail not found (HTTP 404)", }); } else { log( - `[${col.name}] Could not check ${file.metadata.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`, + `[${album.name}] Could not check ${photo.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`, ); } } @@ -94,7 +119,7 @@ export const listMissingThumbnails = async ( return missing; }; -// Bilinear resize of RGBA pixel buffer +// Bilinear resize of an RGBA pixel buffer. const resizeRGBA = ( src: Uint8Array, srcW: number, @@ -133,17 +158,13 @@ const resizeRGBA = ( return dst; }; -const generateThumbnail = (fileBytes: Uint8Array): Uint8Array => { - const decoded = jpeg.decode(fileBytes, { - useTArray: true, - formatAsRGBA: true, - }); +const generateThumbnail = ( + decoded: { data: Uint8Array; width: number; height: number }, + maxDimension: number, + quality: number, +): Uint8Array => { const { width: srcW, height: srcH } = decoded; - const scale = Math.min( - THUMB_MAX_DIMENSION / srcW, - THUMB_MAX_DIMENSION / srcH, - 1, - ); + const scale = Math.min(maxDimension / srcW, maxDimension / srcH, 1); const dstW = Math.round(srcW * scale); const dstH = Math.round(srcH * scale); @@ -156,12 +177,59 @@ const generateThumbnail = (fileBytes: Uint8Array): Uint8Array => { const encoded = jpeg.encode( { data: pixels, width: dstW, height: dstH }, - THUMB_JPEG_QUALITY, + quality, ); return new Uint8Array(encoded.data); }; +// A baseline/JFIF JPEG starts with the SOI marker 0xFFD8. `jpeg-js` decodes +// only JPEG, so this signature check is what separates a file the helper can +// regenerate from one it must skip: a PNG, HEIC, or the odd non-image byte +// stream all fail this and are reported as skipped rather than crashing the +// decoder into an opaque failure (issue #17). +const isJpeg = (bytes: Uint8Array): boolean => + bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xd8; + +// The reason a file cannot have a JPEG thumbnail regenerated for it, known from +// its record alone before any bytes are fetched, or undefined when it might. A +// still image still has to be checked against its actual bytes once +// downloaded. +const reasonToSkip = (file: EnteFile, userID: number): string | undefined => { + if (file.ownerID !== userID) { + return NOT_OWNED_REASON; + } + if (file.metadata.fileType !== "image") { + return `unsupported file type: ${file.metadata.fileType} (only JPEG images can be regenerated)`; + } + if (!file.thumbnail.size) { + return `recorded thumbnail size is ${file.thumbnail.size ?? "unknown"} (the server refuses a thumbnail larger than the one it records)`; + } + return undefined; +}; + +// Encrypt the largest encoding of the decoded image whose ciphertext is no +// larger than `maxSize`, or return undefined when even the smallest is larger. +const encryptThumbnailWithin = ( + decoded: { data: Uint8Array; width: number; height: number }, + key: Uint8Array, + maxSize: number, +): { header: Uint8Array; ciphertext: Uint8Array } | undefined => { + for (const { maxDimension, quality } of THUMB_ENCODINGS) { + const thumbJpeg = generateThumbnail(decoded, maxDimension, quality); + const encrypted = encryptBlob(thumbJpeg, key); + if (encrypted.ciphertext.length <= maxSize) return encrypted; + } + return undefined; +}; + +// Regenerate and upload a thumbnail for each requested file. Originals are read +// through the library's content cache (`photo.original()`); the generated +// thumbnail is JPEG-encoded, encrypted under the file's own key, and registered +// with the server — the encrypt-and-upload path is unchanged. Each file is +// resolved to one outcome (fixed / skipped / failed) and a failure on one file +// never stops the others. export const fixMissingThumbnails = async ( + lib: Library, client: Client, fileIDs: number[], onProgress?: ProgressCallback, @@ -169,20 +237,26 @@ export const fixMissingThumbnails = async ( const log = onProgress ?? (() => {}); const results: ThumbnailFixResult[] = []; const api = client.getApiClient(); + const { userID } = client.whoami(); - const collections = await client.listCollections(); + // Resolve each requested fileID to its file record and owning album by + // enumerating the library, each file taken from the first album that holds + // it. The raw `EnteFile` carries the per-file key the thumbnail is + // encrypted under, which the projected records deliberately do not. + const wanted = new Set(fileIDs); const fileMap = new Map< number, { file: EnteFile; collectionName: string } >(); - - for (const col of collections) { - const files = await client.listFiles(col.id, col.key); - for (const file of files) { - if (fileIDs.includes(file.id) && !fileMap.has(file.id)) { - fileMap.set(file.id, { + for (const album of lib.albums.list()) { + for (const photo of album.photos.list()) { + if (!wanted.has(photo.fileID) || fileMap.has(photo.fileID)) + continue; + const file = lib.getFile(album.collectionID, photo.fileID); + if (file) { + fileMap.set(photo.fileID, { file, - collectionName: col.name, + collectionName: album.name, }); } } @@ -195,33 +269,78 @@ export const fixMissingThumbnails = async ( fileID, title: "unknown", collection: "unknown", - success: false, - error: "file not found in any collection", + status: "failed", + reason: "file not found in any collection", }); continue; } const { file, collectionName } = entry; - const tmpDir = mkdtempSync(join(tmpdir(), "quak-thumb-")); + const title = file.metadata.title; + + const skipReason = reasonToSkip(file, userID); + if (skipReason) { + log(`[${collectionName}] Skipping ${title}: ${skipReason}`); + results.push({ + fileID, + title, + collection: collectionName, + status: "skipped", + reason: skipReason, + }); + continue; + } + const maxSize = file.thumbnail.size!; try { - log( - `[${collectionName}] Downloading ${file.metadata.title} for thumbnail generation...`, - ); - const origPath = join(tmpDir, "original"); - await downloadFile(api, file, origPath); + const photo = lib.photos.byID({ fileID }); + if (!photo) { + throw new Error("file not present in the library cache"); + } log( - `[${collectionName}] Generating thumbnail for ${file.metadata.title}...`, + `[${collectionName}] Downloading ${title} for thumbnail generation...`, ); - const fileBytes = readFileSync(origPath); - const thumbJpeg = generateThumbnail(new Uint8Array(fileBytes)); + const { path } = await photo.original(); + const fileBytes = new Uint8Array(readFileSync(path)); + + if (!isJpeg(fileBytes)) { + const reason = + "unsupported image format (only baseline JPEG can be regenerated)"; + log(`[${collectionName}] Skipping ${title}: ${reason}`); + results.push({ + fileID, + title, + collection: collectionName, + status: "skipped", + reason, + }); + continue; + } + + log(`[${collectionName}] Generating thumbnail for ${title}...`); + const decoded = jpeg.decode(fileBytes, { + useTArray: true, + formatAsRGBA: true, + }); + const fitting = encryptThumbnailWithin(decoded, file.key, maxSize); + if (!fitting) { + const reason = `no thumbnail encoding fits the recorded thumbnail size of ${maxSize} bytes`; + log(`[${collectionName}] Skipping ${title}: ${reason}`); + results.push({ + fileID, + title, + collection: collectionName, + status: "skipped", + reason, + }); + continue; + } + const { header, ciphertext } = fitting; log( - `[${collectionName}] Encrypting and uploading thumbnail (${thumbJpeg.length} bytes)...`, + `[${collectionName}] Uploading thumbnail (${ciphertext.length} bytes)...`, ); - const { header, ciphertext } = encryptBlob(thumbJpeg, file.key); - const md5 = createHash("md5").update(ciphertext).digest("base64"); const { objectKey, url } = await api.getUploadURL( ciphertext.length, @@ -230,28 +349,24 @@ export const fixMissingThumbnails = async ( await api.putFile(url, ciphertext); await api.updateThumbnail(file.id, objectKey, toBase64(header)); - log( - `[${collectionName}] Thumbnail uploaded for ${file.metadata.title}`, - ); + log(`[${collectionName}] Thumbnail uploaded for ${title}`); results.push({ fileID, - title: file.metadata.title, + title, collection: collectionName, - success: true, + status: "fixed", }); } catch (err) { log( - `[${collectionName}] FAILED ${file.metadata.title}: ${err instanceof Error ? err.message : err}`, + `[${collectionName}] FAILED ${title}: ${err instanceof Error ? err.message : err}`, ); results.push({ fileID, - title: file.metadata.title, + title, collection: collectionName, - success: false, - error: err instanceof Error ? err.message : String(err), + status: "failed", + reason: err instanceof Error ? err.message : String(err), }); - } finally { - rmSync(tmpDir, { recursive: true, force: true }); } } diff --git a/test/api/client.test.ts b/test/api/client.test.ts index 2faf019..68ab7cf 100644 --- a/test/api/client.test.ts +++ b/test/api/client.test.ts @@ -38,14 +38,18 @@ * the network. The fake records every call for assertion. */ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { ApiClient, ApiError, DEFAULT_DOWNLOAD_TIMEOUT_MS, DEFAULT_REQUEST_TIMEOUT_MS, } from "../../src/api/client.js"; -import type { RetryOptions } from "../../src/retry.js"; +import { + isRetryable, + isSafeToReplay, + type RetryOptions, +} from "../../src/retry.js"; // --------------------------------------------------------------------------- // Test helpers @@ -385,6 +389,103 @@ describe("ApiClient custom origins", () => { }); }); +describe("ApiClient request URLs", () => { + it("accepts a path with or without a leading slash", async () => { + const { fetch, calls } = recordingFetch( + jsonResponse({}), + jsonResponse({}), + jsonResponse({}), + ); + const client = new ApiClient({ fetch }); + await client.getJSON("health"); + await client.postJSON("users/ott", {}); + await client.putJSON("/files/thumbnail", {}); + + expect(calls.map((c) => c.url)).toEqual([ + "https://api.ente.io/health", + "https://api.ente.io/users/ott", + "https://api.ente.io/files/thumbnail", + ]); + }); + + it("accepts an apiOrigin with a trailing slash", async () => { + const { fetch, calls } = recordingFetch(jsonResponse({})); + const client = new ApiClient({ + fetch, + apiOrigin: "https://my-ente.example.com/", + }); + await client.getJSON("/health"); + + expect(calls[0]!.url).toBe("https://my-ente.example.com/health"); + }); + + it("keeps a base path in a self-hosted apiOrigin for every request", async () => { + const body = new Uint8Array([1]); + const { fetch, calls } = recordingFetch( + jsonResponse({}), + jsonResponse({}), + jsonResponse({}), + streamResponse(body), + streamResponse(body), + ); + const client = new ApiClient({ + fetch, + apiOrigin: "https://example.com/ente/", + }); + await client.getJSON("/collections/v2", { sinceTime: 0 }); + await client.postJSON("/users/ott", {}); + await client.putJSON("/files/thumbnail", {}); + await client.getFileStream(99); + await client.getThumbnailStream(77); + + expect(calls.map((c) => c.url)).toEqual([ + "https://example.com/ente/collections/v2?sinceTime=0", + "https://example.com/ente/users/ott", + "https://example.com/ente/files/thumbnail", + "https://example.com/ente/files/download/99", + "https://example.com/ente/files/preview/77", + ]); + }); + + it("percent-encodes query parameters and skips undefined ones", async () => { + const { fetch, calls } = recordingFetch(jsonResponse({})); + const client = new ApiClient({ fetch }); + await client.getJSON("/search", { + q: "a&b=c/d é", + limit: 5, + cursor: undefined, + }); + + const url = new URL(calls[0]!.url); + expect(url.pathname).toBe("/search"); + expect(url.search).toBe("?q=a%26b%3Dc%2Fd+%C3%A9&limit=5"); + expect(url.searchParams.get("q")).toBe("a&b=c/d é"); + }); + + it("rejects a path that carries its own query string", async () => { + const { fetch, calls } = recordingFetch(); + const client = new ApiClient({ fetch }); + + await expect(client.getJSON("/diff?sinceTime=0")).rejects.toThrow( + /must not contain "\?"/, + ); + await expect(client.postJSON("/users/ott?x=1", {})).rejects.toThrow( + /must not contain "\?"/, + ); + expect(calls).toHaveLength(0); + }); + + it("rejects a path that carries a fragment", async () => { + const { fetch, calls } = recordingFetch(); + const client = new ApiClient({ fetch }); + + await expect(client.getJSON("/diff#top")).rejects.toThrow( + /must not contain "\?" or "#"/, + ); + expect(calls).toHaveLength(0); + }); +}); + describe("ApiError", () => { it("throws ApiError on 4xx with status, code, requestID", async () => { const { fetch } = recordingFetch( @@ -639,17 +740,33 @@ describe("ApiClient retries", () => { expect(policy.baseDelayMs).toBe(7); expect(policy.maxDelayMs).toBe(11); }); + + it("does not let a caller change its settings through that policy", async () => { + const { fetch, calls } = scriptedFetch( + textResponse("boom", 500), + textResponse("boom", 500), + textResponse("boom", 500), + ); + const client = new ApiClient({ + fetch, + retry: { ...noWait, attempts: 2 }, + }); + + client.getRetryOptions().attempts = 3; + + expect(client.getRetryOptions().attempts).toBe(2); + await expect(client.getJSON("/x")).rejects.toBeInstanceOf(ApiError); + expect(calls).toHaveLength(2); + }); }); describe("ApiClient timeouts", () => { it("ships bounded default deadlines", () => { - // Asserted here so the README and the code cannot drift. Two numbers - // rather than one, because a deadline that is sane for a JSON call is - // nowhere near enough for a multi-gigabyte body, and a deadline long - // enough for that body would let a hung API call stall a backup for - // ten minutes. + // Asserted here so the README and the code cannot drift. The request + // deadline bounds a whole JSON call; the download deadline is an idle + // one, measured from the last byte that arrived. expect(DEFAULT_REQUEST_TIMEOUT_MS).toBe(30_000); - expect(DEFAULT_DOWNLOAD_TIMEOUT_MS).toBe(600_000); + expect(DEFAULT_DOWNLOAD_TIMEOUT_MS).toBe(60_000); }); it("attaches an abort signal to every request", async () => { @@ -693,6 +810,51 @@ describe("ApiClient timeouts", () => { expect(new Set(signals).size).toBe(3); }, 5000); + it("gives every retrying entry point a fresh deadline per attempt", async () => { + // A refused connection is retried by every entry point, the + // non-idempotent ones included. If the deadline were created once, + // outside the retry, both attempts would carry the same signal. + const entryPoints: [ + string, + () => Response, + (c: ApiClient) => unknown, + ][] = [ + ["getJSON", () => jsonResponse({}), (c) => c.getJSON("/a")], + ["postJSON", () => jsonResponse({}), (c) => c.postJSON("/b", {})], + ["putJSON", () => jsonResponse({}), (c) => c.putJSON("/c", {})], + [ + "putFile", + () => new Response(null, { status: 200 }), + (c) => c.putFile("https://s3.example/x", new Uint8Array([1])), + ], + [ + "getFileStream", + () => streamResponse(new Uint8Array([1])), + (c) => c.getFileStream(1), + ], + [ + "getThumbnailStream", + () => streamResponse(new Uint8Array([1])), + (c) => c.getThumbnailStream(1), + ], + ]; + for (const [name, success, call] of entryPoints) { + const { fetch, calls } = scriptedFetch( + errnoError("ECONNREFUSED", "connect ECONNREFUSED"), + success(), + ); + const client = new ApiClient({ fetch, retry: noWait }); + + await call(client); + + expect(calls, name).toHaveLength(2); + const [first, second] = calls.map((c) => c.init?.signal); + expect(first, name).toBeInstanceOf(AbortSignal); + expect(second, name).toBeInstanceOf(AbortSignal); + expect(second, name).not.toBe(first); + } + }); + it("recovers when a later attempt answers in time", async () => { const { fetch, calls } = scriptedFetch(HANG, jsonResponse({ ok: 1 })); const client = new ApiClient({ @@ -718,25 +880,91 @@ describe("ApiClient timeouts", () => { // that never produces a chunk and never observes the signal, so the // only thing that can unblock the read is quak's own enforcement of // the deadline over the stream it hands out. - const stalling = new Response( - new ReadableStream<Uint8Array>({ - pull: () => new Promise<void>(() => {}), - }), - { status: 200 }, - ); - const { fetch } = scriptedFetch(stalling); - const client = new ApiClient({ - fetch, - downloadTimeoutMs: 20, - retry: { ...noWait, attempts: 1 }, - }); + // + // The clock is faked, so the test runs under the real default + // deadline and waits for nothing. + vi.useFakeTimers(); + try { + const stalling = new Response( + new ReadableStream<Uint8Array>({ + pull: () => new Promise<void>(() => {}), + }), + { status: 200 }, + ); + const { fetch } = scriptedFetch(stalling); + const client = new ApiClient({ + fetch, + retry: { ...noWait, attempts: 1 }, + }); - const stream = await client.getFileStream(42); - const err: unknown = await readAll(stream).catch((e: unknown) => e); + const stream = await client.getFileStream(42); + let settled = false; + const result = readAll(stream).then( + (n) => n, + (e: unknown) => e, + ); + void result.finally(() => { + settled = true; + }); - expect(err).toBeInstanceOf(Error); - expect((err as Error).name).toBe("TimeoutError"); - }, 5000); + await vi.advanceTimersByTimeAsync(DEFAULT_DOWNLOAD_TIMEOUT_MS - 1); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + + const err = await result; + expect(err).toBeInstanceOf(Error); + expect((err as Error).name).toBe("TimeoutError"); + // Classified as every deadline is: retried by the idempotent + // downloads, never replayed for a POST or PUT. + expect(isRetryable(err)).toBe(true); + expect(isSafeToReplay(err)).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + + it("does not abort a slow body that keeps making progress", async () => { + // A deadline over the whole transfer would cut off a large video on a + // slow link however steadily it was arriving. The deadline restarts + // with every chunk, so a body that sends one byte every 600 ms for + // well over the 1000 ms deadline completes. + vi.useFakeTimers(); + try { + let sent = 0; + const trickling = new Response( + new ReadableStream<Uint8Array>({ + async pull(controller) { + await new Promise((resolve) => + setTimeout(resolve, 600), + ); + if (sent === 10) { + controller.close(); + return; + } + controller.enqueue(new Uint8Array([sent++])); + }, + }), + { status: 200 }, + ); + const { fetch } = scriptedFetch(trickling); + const client = new ApiClient({ + fetch, + downloadTimeoutMs: 1000, + retry: { ...noWait, attempts: 1 }, + }); + + const stream = await client.getFileStream(42); + const result = readAll(stream).then( + (n) => n, + (e: unknown) => e, + ); + await vi.advanceTimersByTimeAsync(11 * 600); + + expect(await result).toBe(10); + } finally { + vi.useRealTimers(); + } + }); it("lets a body that arrives in time through untouched", async () => { // The counterpart to the previous test: enforcing the deadline over @@ -762,6 +990,52 @@ describe("ApiClient timeouts", () => { } expect(joined).toEqual(payload); }); + + it("leaves no timer pending after a download completes or fails", async () => { + vi.useFakeTimers(); + try { + const { fetch } = scriptedFetch( + streamResponse(new Uint8Array([1, 2, 3])), + textResponse("gone", 404), + ); + const client = new ApiClient({ + fetch, + retry: { ...noWait, attempts: 1 }, + }); + + expect(await readAll(await client.getFileStream(1))).toBe(3); + expect(vi.getTimerCount()).toBe(0); + + await expect(client.getFileStream(2)).rejects.toBeInstanceOf( + ApiError, + ); + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + + it("never lets the download timer keep the process alive", async () => { + const spy = vi.spyOn(globalThis, "setTimeout"); + try { + const { fetch } = scriptedFetch( + streamResponse(new Uint8Array([1])), + ); + const client = new ApiClient({ + fetch, + downloadTimeoutMs: 12_345, + retry: noWait, + }); + + const stream = await client.getFileStream(1); + const i = spy.mock.calls.findIndex((call) => call[1] === 12_345); + const timer = spy.mock.results[i]!.value as NodeJS.Timeout; + expect(timer.hasRef()).toBe(false); + await stream.cancel(); + } finally { + spy.mockRestore(); + } + }); }); describe("ApiClient error typing", () => { @@ -820,9 +1094,8 @@ describe("ApiClient error typing", () => { describe("ApiClient non-idempotent requests", () => { /** - * `postJSON` and `putJSON` carry quak's only requests that change server - * state: `/users/srp/create-session`, `/users/two-factor/verify` — which - * consumes one of a small number of 2FA attempts — and `/files/thumbnail`. + * `postJSON` and `putJSON` carry quak's requests that can change server + * state; the README lists them under "Endpoints used". * * They are retried only on a failure that establishes no TCP connection to * the server ever existed — DNS produced no address, or the peer refused @@ -922,4 +1195,30 @@ describe("ApiClient non-idempotent requests", () => { await refusedClient.updateThumbnail(1, "key", "header"); expect(refused.calls).toHaveLength(2); }); + + it("does not follow or replay a redirect on POST or PUT", async () => { + // The origin has already received a request it answers with a + // redirect, so following it would let a refused connection to the + // redirect target pass for a request that never went out. + for (const send of [ + (c: ApiClient) => c.postJSON("/users/ott", {}), + (c: ApiClient) => c.putJSON("/files/thumbnail", {}), + ]) { + const { fetch, calls } = scriptedFetch( + new Response(null, { + status: 307, + headers: { location: "https://elsewhere.example/" }, + }), + jsonResponse({}), + ); + const client = new ApiClient({ fetch, retry: noWait }); + + const err: unknown = await send(client).catch((e: unknown) => e); + + expect(calls[0]?.init?.redirect).toBe("manual"); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).status).toBe(307); + expect(calls).toHaveLength(1); + } + }); }); diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 5f2efab..1202ef3 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -1,368 +1,269 @@ /** - * Tests for the `quak backup` command's core logic. + * Tests for the `quak backup` logic, now built on the library API (issue #51). * - * `quak backup <dir>` downloads every file from every collection into - * a local directory tree: + * `lib.backup({ downloadDirectory })` refreshes the library, fetches each + * pending file's original through the content cache/pools, and materialises the + * unchanged on-disk layout: * - * <dir>/ - * <collection-name>/ - * <file-title> - * <file-title> - * <collection-name>/ - * ... - * metadata.json (all decrypted collection + file metadata) + * <downloadDirectory>/ + * originals/ + * <fileID>.<ext> the decrypted bytes ("present means complete") + * <fileID>.json per-file metadata sidecar (rebuilt each run) + * collections/ + * <name>/<title> symlink into ../originals (rebuilt each run) + * <name>.json per-collection metadata (rebuilt each run) + * failures.json durable ledger of unresolved failures * - * The backup command has two properties that distinguish it from a naive - * "download everything" loop: + * The properties that distinguish backup from a naive download loop, and that + * these tests lock down: * - * 1. **Skip existing files.** If `<dir>/<collection>/<title>` already - * exists on disk and its size matches the decrypted content length - * recorded in metadata.json from a prior run, the file is not - * re-downloaded. This makes interrupted backups resumable and - * incremental runs fast. + * 1. Present-means-complete: an original already on disk is not re-fetched, so + * runs are idempotent and interrupted runs resume. + * 2. Per-file resilience: a download failure or a symlink failure is recorded + * and the run continues (issue #8); the derived symlink/JSON views are + * rebuilt from the model every run. + * 3. A durable `failures.json` records each unresolved failure's classification, + * attempt count, and last-tried time; the exit code (result.failed) is + * non-zero while any failure remains and clears once every one is resolved. * - * 2. **Never crash on a single file failure.** If a file download or - * decryption fails, the error is logged and the backup continues - * with the next file. At the end, the exit code is non-zero if any - * files failed, and the summary lists them. The Ente first-party - * CLI crashes entirely when a single file can't be retrieved, - * which defeats the purpose of a backup tool. - * - * These tests exercise the backup logic (in src/backup.ts) using the - * same mock server from the Client usage tests. The CLI binary itself - * is a thin wrapper around this module. + * The cache and download layers are covered elsewhere (content.test.ts, + * download tests); here a mock library client and a stand-in content source + * drive the backup logic with no crypto and no network. */ import { existsSync, lstatSync, + mkdirSync, mkdtempSync, + readdirSync, readFileSync, readlinkSync, rmSync, + symlinkSync, + writeFileSync, } from "node:fs"; +import { spawnSync } from "node:child_process"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import sodium from "libsodium-wrappers-sumo"; -import { SRP, SrpServer } from "fast-srp-hap"; -import { beforeAll, afterAll, describe, expect, it } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; + +import { runBackup, type BackupLibrary } from "../../src/backup.js"; +import { Library } from "../../src/library/index.js"; +import type { ContentSource } from "../../src/library/content.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; import { - init, - toBase64, - deriveKEK, - deriveLoginSubkey, -} from "../../src/crypto/index.js"; -import { Client } from "../../src/client.js"; -import { runBackup } from "../../src/backup.js"; -import type { KeyAttributes } from "../../src/auth/types.js"; + asLivePhoto, + cdnSource, + IMAGE, + livePhotoZip, + VIDEO, +} from "../live-photo.js"; -// --------------------------------------------------------------------------- -// Mock server (condensed from usage.test.ts) -// --------------------------------------------------------------------------- +// `open` and `rename` are wrapped to record, in order, every fsync and rename, +// so a test can pin the sequence "fsync the temp file, rename, fsync the +// directory" that makes a copied original survive a power cut. `vi.hoisted` +// because `vi.mock` factories run before module-level constants exist. +const fsEvents = vi.hoisted(() => [] as string[]); -const TEST_EMAIL = "backup@example.com"; -const TEST_PASSWORD = "backuppass"; -const TEST_OPS = 2; -const TEST_MEM = 64 * 1024 * 1024; +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal<typeof import("node:fs/promises")>(); + return { + ...actual, + open: async ( + ...args: Parameters<typeof actual.open> + ): Promise<Awaited<ReturnType<typeof actual.open>>> => { + const handle = await actual.open(...args); + const realSync = handle.sync.bind(handle); + handle.sync = async (): Promise<void> => { + fsEvents.push(`sync:${String(args[0])}`); + await realSync(); + }; + return handle; + }, + rename: async (from: string, to: string): Promise<void> => { + fsEvents.push(`rename:${to}`); + await actual.rename(from, to); + }, + }; +}); -interface MockState { - verifier: Buffer; - srpAttributes: Record<string, unknown>; - keyAttributes: KeyAttributes; - encryptedToken: string; - collections: Record<string, unknown>[]; - files: Record< - number, - { raw: Record<string, unknown>; plaintext: Uint8Array } - >; +const USER_ID = 42; + +// Decrypted-byte length each stub original writes, keyed by fileID. +const SIZE_BY_ID: Record<number, number> = { 100: 3000, 101: 2000, 200: 1500 }; + +const collection = (id: number, name: string): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + name, + type: "album", + updationTime: 1, + isShared: false, +}); + +const file = (id: number, collectionID: number, title: string): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + metadata: { + title, + fileType: "image", + creationTime: 1, + modificationTime: 1, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 1, +}); + +// A metadata-only client: two albums, three files, served once. No ML. +class MockClient { + private served = false; + whoami(): { email: string; userID: number } { + return { email: "backup@example.com", userID: USER_ID }; + } + async collectionsSince(): Promise<CollectionsPage> { + if (this.served) return { collections: [], deleted: [], cursor: 1 }; + this.served = true; + return { + collections: [collection(1, "Vacation"), collection(2, "Work")], + deleted: [], + cursor: 1, + }; + } + async filesSince(args: { collectionID: number }): Promise<FilesPage> { + const files = + args.collectionID === 1 + ? [file(100, 1, "beach.jpg"), file(101, 1, "sunset.jpg")] + : args.collectionID === 2 + ? [file(200, 2, "diagram.png")] + : []; + return { files, deleted: [], cursor: 1 }; + } } -let mock: MockState; -let testDir: string; - -const buildMock = async (): Promise<MockState> => { - const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES); - const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM); - const loginSubKeyBytes = deriveLoginSubkey(kek); - - const srpUserID = "backup-srp"; - const srpSalt = sodium.randombytes_buf(16); - const verifier = SRP.computeVerifier( - SRP.params["4096"], - Buffer.from(srpSalt), - Buffer.from(srpUserID), - Buffer.from(loginSubKeyBytes), - ); - - const masterKey = sodium.randombytes_buf(32); - const keyNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encryptedKey = sodium.crypto_secretbox_easy(masterKey, keyNonce, kek); - const kp = sodium.crypto_box_keypair(); - const skNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encSK = sodium.crypto_secretbox_easy( - kp.privateKey, - skNonce, - masterKey, - ); - const tokenBytes = sodium.randombytes_buf(32); - const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey); - - const keyAttributes: KeyAttributes = { - kekSalt: toBase64(kekSalt), - encryptedKey: toBase64(encryptedKey), - keyDecryptionNonce: toBase64(keyNonce), - publicKey: toBase64(kp.publicKey), - encryptedSecretKey: toBase64(encSK), - secretKeyDecryptionNonce: toBase64(skNonce), - memLimit: TEST_MEM, - opsLimit: TEST_OPS, - }; - - const makeCollection = (id: number, name: string) => { - const ck = sodium.crypto_secretbox_keygen(); - const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encCK = sodium.crypto_secretbox_easy(ck, ckN, masterKey); - const nameBytes = new TextEncoder().encode(name); - const cnN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encCN = sodium.crypto_secretbox_easy(nameBytes, cnN, ck); +// A server that names an album and a file so as to climb out of the backup +// directory. +class HostileClient extends MockClient { + override async collectionsSince(): Promise<CollectionsPage> { + const page = await super.collectionsSince(); return { - raw: { - id, - owner: { id: 42 }, - encryptedKey: toBase64(encCK), - keyDecryptionNonce: toBase64(ckN), - encryptedName: toBase64(encCN), - nameDecryptionNonce: toBase64(cnN), - type: "album", - updationTime: 1700000000000000, - }, - key: ck, + ...page, + collections: page.collections.length + ? [collection(3, "../escape")] + : [], }; - }; + } + override async filesSince(args: { + collectionID: number; + }): Promise<FilesPage> { + const files = + args.collectionID === 3 + ? [file(300, 3, "../../.ssh/authorized_keys")] + : []; + return { files, deleted: [], cursor: 1 }; + } +} - const makeFile = ( - id: number, - collKey: Uint8Array, - title: string, - plaintext: Uint8Array, - collID: number, - ) => { - const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen(); - const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey); +// A content source that writes byte buffers of the expected length and can be +// told to fail one fileID's original, to exercise per-file resilience. +interface StubSource extends ContentSource { + failID?: number; + failThumbID?: number; + originalCalls: number; +} - const meta = JSON.stringify({ - title, - fileType: 0, - creationTime: 1700000000000000, - modificationTime: 1700000000000000, - }); - const metaPush = - sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); - const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push( - metaPush.state, - new TextEncoder().encode(meta), - null, - sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, - ); - - const filePush = - sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); - const encFile = sodium.crypto_secretstream_xchacha20poly1305_push( - filePush.state, - plaintext, - null, - sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, - ); - - return { - raw: { - id, - collectionID: collID, - ownerID: 42, - encryptedKey: toBase64(encFK), - keyDecryptionNonce: toBase64(fkN), - metadata: { - encryptedData: toBase64(encMeta), - decryptionHeader: toBase64(metaPush.header), - }, - file: { decryptionHeader: toBase64(filePush.header) }, - thumbnail: { - decryptionHeader: toBase64(sodium.randombytes_buf(24)), - }, - updationTime: 1700000000000000, - }, - plaintext, - ciphertext: encFile, - }; - }; - - const col1 = makeCollection(1, "Vacation"); - const col2 = makeCollection(2, "Work"); - const file1 = makeFile( - 100, - col1.key, - "beach.jpg", - sodium.randombytes_buf(3000), - 1, - ); - const file2 = makeFile( - 101, - col1.key, - "sunset.jpg", - sodium.randombytes_buf(2000), - 1, - ); - const file3 = makeFile( - 200, - col2.key, - "diagram.png", - sodium.randombytes_buf(1500), - 2, - ); - - return { - verifier, - srpAttributes: { - srpUserID, - srpSalt: toBase64(srpSalt), - memLimit: TEST_MEM, - opsLimit: TEST_OPS, - kekSalt: toBase64(kekSalt), - isEmailMFAEnabled: false, +const stubSource = (): StubSource => { + const s: StubSource = { + originalCalls: 0, + original: async ({ file: f, destination }) => { + s.originalCalls++; + if (s.failID === f.id) throw new Error("HTTP 500 from server"); + const size = SIZE_BY_ID[f.id] ?? 10; + writeFileSync(destination, Buffer.alloc(size)); + return { bytesWritten: size }; + }, + thumbnail: async ({ file: f, destination }) => { + if (s.failThumbID === f.id) throw new Error("HTTP 500 from server"); + writeFileSync(destination, Buffer.alloc(5)); + return { bytesWritten: 5 }; }, - keyAttributes, - encryptedToken: toBase64(encToken), - collections: [col1.raw, col2.raw], - files: { - 1: { - raw: [file1.raw, file2.raw], - ciphertexts: { 100: file1.ciphertext, 101: file2.ciphertext }, - }, - 2: { raw: [file3.raw], ciphertexts: { 200: file3.ciphertext } }, - 100: { plaintext: file1.plaintext }, - 101: { plaintext: file2.plaintext }, - 200: { plaintext: file3.plaintext }, - } as Record<number, unknown>, }; + return s; }; -const buildMockFetch = (m: MockState, opts?: { failFileID?: number }) => { - let srpServer: SrpServer; +let root: string; - return (async ( - input: RequestInfo | URL, - init?: RequestInit, - ): Promise<Response> => { - const url = - typeof input === "string" - ? input - : input instanceof URL - ? input.href - : input.url; - const parsed = new URL(url); - const path = parsed.pathname; - const json = (body: unknown) => - new Response(JSON.stringify(body), { - status: 200, - headers: { "content-type": "application/json" }, - }); +const openLibrary = ( + source: ContentSource, + client: MockClient = new MockClient(), +): Promise<Library> => + Library.open({ + client, + cacheDirectory: join(root, "cache"), + contentSource: source, + refreshIntervalSeconds: 3600, + // These tests count exact fetches; the background precache (#48) would + // add its own, so it is off here (it is covered in precache.test.ts). + precacheThumbnails: false, + precacheOriginals: false, + }); - if (path === "/users/srp/attributes") - return json({ attributes: m.srpAttributes }); +const readLedger = ( + outDir: string, +): { files: Record<string, Record<string, unknown>> } => + JSON.parse(readFileSync(join(outDir, "failures.json"), "utf-8")); - if (path === "/users/srp/create-session") { - const body = JSON.parse(init?.body as string); - const serverKey = await SRP.genKey(); - srpServer = new SrpServer( - SRP.params["4096"], - m.verifier, - serverKey, - ); - const B = srpServer.computeB(); - srpServer.setA(Buffer.from(body.srpA, "base64")); - return json({ sessionID: "s1", srpB: B.toString("base64") }); - } - - if (path === "/users/srp/verify-session") { - const body = JSON.parse(init?.body as string); - srpServer.checkM1(Buffer.from(body.srpM1, "base64")); - return json({ - srpM2: srpServer.computeM2().toString("base64"), - id: 42, - keyAttributes: m.keyAttributes, - encryptedToken: m.encryptedToken, - }); - } - - if (path === "/collections/v2") - return json({ collections: m.collections }); - - if (path === "/collections/v2/diff") { - const collID = Number(parsed.searchParams.get("collectionID")); - const collData = m.files[collID] as { raw: unknown[] } | undefined; - return json({ diff: collData?.raw ?? [], hasMore: false }); - } - - // File download - if (url.includes("fileID=") || path.startsWith("/files/download/")) { - const fileID = Number( - parsed.searchParams.get("fileID") ?? path.split("/").pop(), - ); - if (opts?.failFileID === fileID) { - return new Response("Internal Server Error", { status: 500 }); - } - const collData = Object.values(m.files).find( - (v: unknown) => - v && - typeof v === "object" && - "ciphertexts" in (v as Record<string, unknown>) && - fileID in - (v as Record<string, Record<number, unknown>>) - .ciphertexts, - ) as { ciphertexts: Record<number, Uint8Array> } | undefined; - if (collData) { - return new Response(collData.ciphertexts[fileID], { - status: 200, - }); - } - return new Response("not found", { status: 404 }); - } - - return new Response("not found", { status: 404 }); - }) as typeof globalThis.fetch; +// Write a durable ledger holding one prior failure, to exercise pruning of +// entries the current run cannot resolve. +const seedLedger = (outDir: string, fileID: number, title: string): void => { + mkdirSync(outDir, { recursive: true }); + writeFileSync( + join(outDir, "failures.json"), + JSON.stringify({ + version: 1, + files: { + [String(fileID)]: { + fileID, + title, + classification: "transient", + attempts: 1, + lastTriedAt: Date.now(), + error: "HTTP 500 from server", + }, + }, + }), + ); }; -// --------------------------------------------------------------------------- -// Setup / teardown -// --------------------------------------------------------------------------- - -beforeAll(async () => { - await init(); - await sodium.ready; - mock = await buildMock(); - testDir = mkdtempSync(join(tmpdir(), "quak-backup-test-")); +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-backup-test-")); }); -afterAll(() => { - if (testDir && existsSync(testDir)) - rmSync(testDir, { recursive: true, force: true }); +afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); }); -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- +describe("lib.backup", () => { + it("throws before any network when no downloadDirectory is given", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + await expect(lib.backup()).rejects.toThrow(/downloadDirectory/i); + expect(source.originalCalls).toBe(0); + lib.close(); + }); -describe("quak backup", () => { - it("downloads all files organized by collection name", async () => { - const outDir = join(testDir, "full-backup"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMockFetch(mock) }, - }); + it("writes the expected on-disk layout for every file", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); - const result = await runBackup(client, outDir); + const result = await lib.backup({ downloadDirectory: outDir }); expect(result.totalFiles).toBe(3); expect(result.downloaded).toBe(3); @@ -370,7 +271,7 @@ describe("quak backup", () => { expect(result.failed).toBe(0); expect(result.errors).toEqual([]); - // Originals are under <outDir>/originals/<fileID>.<ext> + // Originals under originals/<fileID>.<ext>. expect(readFileSync(join(outDir, "originals", "100.jpg")).length).toBe( 3000, ); @@ -381,55 +282,81 @@ describe("quak backup", () => { 1500, ); - // Collection dirs under collections/ contain symlinks to originals - const beachLink = join(outDir, "collections", "Vacation", "beach.jpg"); - expect(lstatSync(beachLink).isSymbolicLink()).toBe(true); - expect(readlinkSync(beachLink)).toContain("originals"); - expect(readFileSync(beachLink).length).toBe(3000); + // Per-file metadata sidecar. + const sidecar = JSON.parse( + readFileSync(join(outDir, "originals", "100.json"), "utf-8"), + ); + expect(sidecar.id).toBe(100); + expect(sidecar.metadata.title).toBe("beach.jpg"); + + // Collection dirs contain symlinks into ../originals. + const beach = join(outDir, "collections", "Vacation", "beach.jpg"); + expect(lstatSync(beach).isSymbolicLink()).toBe(true); + expect(readlinkSync(beach)).toContain("originals"); + expect(readFileSync(beach).length).toBe(3000); + + // Per-collection metadata JSON. + const vacation = JSON.parse( + readFileSync(join(outDir, "collections", "Vacation.json"), "utf-8"), + ); + expect(vacation.name).toBe("Vacation"); + expect(vacation.files.length).toBe(2); + expect(vacation.files[0].metadata.title).toBeDefined(); + + // A clean run leaves no failure ledger behind. + expect(existsSync(join(outDir, "failures.json"))).toBe(false); + lib.close(); }); - it("skips files that already exist on disk with matching size", async () => { - const outDir = join(testDir, "incremental"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMockFetch(mock) }, - }); + it("keeps server-supplied album and file names inside the backup", async () => { + const lib = await openLibrary(stubSource(), new HostileClient()); + const outDir = join(root, "backup"); - // First run: download everything - const first = await runBackup(client, outDir); + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + // The title has no usable extension, so the original is `.bin`. + expect(existsSync(join(outDir, "originals", "300.bin"))).toBe(true); + const link = join( + outDir, + "collections", + "__escape", + "__.._.ssh_authorized_keys", + ); + expect(lstatSync(link).isSymbolicLink()).toBe(true); + expect(existsSync(join(outDir, "collections", "__escape.json"))).toBe( + true, + ); + // Nothing landed beside or above the backup directory. + expect(readdirSync(root).sort()).toEqual(["backup", "cache"]); + lib.close(); + }); + + it("is an idempotent no-op when every original is already present", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + const first = await lib.backup({ downloadDirectory: outDir }); expect(first.downloaded).toBe(3); + const callsAfterFirst = source.originalCalls; - // Second run: everything should be skipped - const second = await runBackup(client, outDir); + const second = await lib.backup({ downloadDirectory: outDir }); expect(second.downloaded).toBe(0); expect(second.skipped).toBe(3); expect(second.failed).toBe(0); + // A present original is neither fetched nor copied again. + expect(source.originalCalls).toBe(callsAfterFirst); + lib.close(); }); - it("continues after a single file download failure", async () => { - // File 101 (sunset.jpg) will return HTTP 500. The other two - // files must still download. The result must report the failure - // without throwing. - // - // A 500 is retryable, so this file now costs several requests before - // it is given up on — that is the point of the retry policy, and - // `runBackup`'s own resilience is unchanged by it: the retry lives - // strictly below this loop, and an exhausted file is still logged, - // counted, and stepped over rather than aborting the run. The - // injected `sleep` is what keeps the suite from actually waiting out - // the backoff. - const outDir = join(testDir, "partial-failure"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { - fetch: buildMockFetch(mock, { failFileID: 101 }), - retry: { sleep: () => Promise.resolve(), random: () => 0 }, - }, - }); + it("continues past a download failure and records it in failures.json", async () => { + const source = stubSource(); + source.failID = 101; + const lib = await openLibrary(source); + const outDir = join(root, "backup"); - const result = await runBackup(client, outDir); + const result = await lib.backup({ downloadDirectory: outDir }); expect(result.totalFiles).toBe(3); expect(result.downloaded).toBe(2); @@ -438,32 +365,955 @@ describe("quak backup", () => { expect(result.errors[0]!.fileID).toBe(101); expect(result.errors[0]!.title).toBe("sunset.jpg"); - // The two successful originals are on disk + // The two good files are on disk; the failed one is not. expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true); expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); - // The failed file has no original and no symlink expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(false); expect( existsSync(join(outDir, "collections", "Vacation", "sunset.jpg")), ).toBe(false); + + // Durable ledger with classification, attempts, last-tried. + const ledger = readLedger(outDir); + const entry = ledger.files["101"]!; + expect(entry.attempts).toBe(1); + expect(entry.classification).toBeDefined(); + expect(typeof entry.lastTriedAt).toBe("number"); + lib.close(); }); - it("writes per-collection JSON metadata", async () => { - const outDir = join(testDir, "metadata-check"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMockFetch(mock) }, + it("increments the attempt count across runs and clears the ledger once resolved", async () => { + const source = stubSource(); + source.failID = 101; + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + const r1 = await lib.backup({ downloadDirectory: outDir }); + expect(r1.failed).toBe(1); + expect(readLedger(outDir).files["101"]!.attempts).toBe(1); + + // Second run: the two good files are present, only 101 is retried. + const r2 = await lib.backup({ downloadDirectory: outDir }); + expect(r2.failed).toBe(1); + expect(r2.skipped).toBe(2); + expect(readLedger(outDir).files["101"]!.attempts).toBe(2); + + // Resume with a healthy source: 101 downloads, the rest are skipped. + source.failID = undefined; + const r3 = await lib.backup({ downloadDirectory: outDir }); + expect(r3.failed).toBe(0); + expect(r3.skipped).toBe(2); + expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(true); + // A ledger with no remaining failures is removed. + expect(existsSync(join(outDir, "failures.json"))).toBe(false); + lib.close(); + }); + + it("does not abort when a symlink cannot be created (issue #8)", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + // Occupy beach.jpg's symlink path with a directory so symlink creation + // fails for that one file. + mkdirSync(join(outDir, "collections", "Vacation", "beach.jpg"), { + recursive: true, }); - await runBackup(client, outDir); + const result = await lib.backup({ downloadDirectory: outDir }); - // Each collection gets a <name>.json next to its image dir - const vacationMeta = join(outDir, "collections", "Vacation.json"); - expect(existsSync(vacationMeta)).toBe(true); - const meta = JSON.parse(readFileSync(vacationMeta, "utf-8")); - expect(meta.name).toBe("Vacation"); - expect(meta.files.length).toBeGreaterThan(0); - expect(meta.files[0].metadata.title).toBeDefined(); + // Every original still downloads despite the symlink failure. + expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true); + expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); + // The other symlinks are still built. + expect( + lstatSync( + join(outDir, "collections", "Vacation", "sunset.jpg"), + ).isSymbolicLink(), + ).toBe(true); + expect( + lstatSync( + join(outDir, "collections", "Work", "diagram.png"), + ).isSymbolicLink(), + ).toBe(true); + + // The symlink failure is recorded, not thrown. + expect(result.failed).toBeGreaterThanOrEqual(1); + const err = result.errors.find((e) => e.fileID === 100); + expect(err).toBeDefined(); + expect(err!.collection).toBe("Vacation"); + expect(readLedger(outDir).files["100"]).toBeDefined(); + lib.close(); + }); + + it("rebuilds a stale sidecar and a missing symlink on a later run", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + await lib.backup({ downloadDirectory: outDir }); + + // Corrupt a sidecar and delete a symlink between runs. + writeFileSync(join(outDir, "originals", "100.json"), "not json"); + rmSync(join(outDir, "collections", "Vacation", "beach.jpg")); + + const result = await lib.backup({ downloadDirectory: outDir }); + expect(result.failed).toBe(0); + + // The derived views are repaired from the model. + const sidecar = JSON.parse( + readFileSync(join(outDir, "originals", "100.json"), "utf-8"), + ); + expect(sidecar.metadata.title).toBe("beach.jpg"); + expect( + lstatSync( + join(outDir, "collections", "Vacation", "beach.jpg"), + ).isSymbolicLink(), + ).toBe(true); + lib.close(); + }); + + it("backs up only the named albums when onlyAlbumNames is given", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + const result = await lib.backup({ + downloadDirectory: outDir, + onlyAlbumNames: ["Work"], + }); + + expect(result.totalFiles).toBe(1); + expect(result.downloaded).toBe(1); + expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); + expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(false); + expect(existsSync(join(outDir, "collections", "Work.json"))).toBe(true); + expect(existsSync(join(outDir, "collections", "Vacation.json"))).toBe( + false, + ); + lib.close(); + }); + + it("prunes a ledger entry for a file no longer in the library and exits zero", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + // A prior failure for a file that has since left the library (deleted + // from the account). This run has no way to resolve it, so it must not + // keep the exit code non-zero forever. + seedLedger(outDir, 999, "gone.jpg"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + // Everything still present is backed up cleanly, and the stale entry is + // dropped rather than counted. + expect(result.downloaded).toBe(3); + expect(result.failed).toBe(0); + expect(existsSync(join(outDir, "failures.json"))).toBe(false); + lib.close(); + }); + + it("prunes an out-of-scope ledger entry on a scoped run and exits zero", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + // A prior failure for a Vacation file; this run is scoped to Work and + // never attempts it, so it must not poison the scoped run's exit code. + seedLedger(outDir, 100, "beach.jpg"); + + const result = await lib.backup({ + downloadDirectory: outDir, + onlyAlbumNames: ["Work"], + }); + + expect(result.totalFiles).toBe(1); + expect(result.failed).toBe(0); + expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); + expect(existsSync(join(outDir, "failures.json"))).toBe(false); + lib.close(); + }); + + it("also stores thumbnails when includeThumbnails is set", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + await lib.backup({ + downloadDirectory: outDir, + includeThumbnails: true, + }); + + expect(existsSync(join(outDir, "thumbnails", "100.jpg"))).toBe(true); + expect(existsSync(join(outDir, "thumbnails", "200.jpg"))).toBe(true); + lib.close(); + }); + + it("counts one attempt when a file fails both its original and thumbnail in a run", async () => { + const source = stubSource(); + source.failID = 101; + source.failThumbID = 101; + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + const result = await lib.backup({ + downloadDirectory: outDir, + includeThumbnails: true, + }); + + // Both kinds fail for 101, but the run counts it once. + const errs = result.errors.filter((e) => e.fileID === 101); + expect(errs.length).toBe(1); + expect(readLedger(outDir).files["101"]!.attempts).toBe(1); + lib.close(); + }); + + it("fetches each original once and writes it only into the backup", async () => { + // A backup of a 500 GB account must write 500 GB, not a copy in the + // cache as well: an original fetched for the backup goes straight + // into its originals/, and the cache records it there. + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result.downloaded).toBe(3); + expect(source.originalCalls).toBe(3); + expect(readdirSync(join(root, "cache", "originals"))).toEqual([]); + const stored = readdirSync(join(outDir, "originals")).filter( + (name) => !name.endsWith(".json"), + ); + expect(stored.sort()).toEqual(["100.jpg", "101.jpg", "200.png"]); + // The cache counts the backup's copy as present: reading the + // original afterwards fetches nothing and answers with that copy. + const read = await lib.photos.byID({ fileID: 100 })!.original(); + expect(read.path).toBe(join(outDir, "originals", "100.jpg")); + expect(source.originalCalls).toBe(3); + await lib.close(); + }); + + it("fsyncs an original copied from the cache before the rename and its directory after", async () => { + const lib = await openLibrary(stubSource()); + const outDir = join(root, "backup"); + const originals = join(outDir, "originals"); + const dest = join(originals, "100.jpg"); + // Only an original already in the cache is copied into the backup; + // one fetched for the backup is written there by the download writer. + await lib.photos.byID({ fileID: 100 })!.original(); + fsEvents.length = 0; + + await lib.backup({ downloadDirectory: outDir }); + + const at = fsEvents.indexOf(`rename:${dest}`); + expect(at).toBeGreaterThan(0); + expect(fsEvents[at - 1]).toMatch( + /^sync:.*\/\.quak-backup-100\.jpg-\d+-[0-9a-z]*\.tmp$/, + ); + expect(fsEvents[at + 1]).toBe(`sync:${originals}`); + lib.close(); + }); + + it("removes temp files left by a killed backup but not those of one still running", async () => { + const outDir = join(root, "backup"); + const originals = join(outDir, "originals"); + mkdirSync(originals, { recursive: true }); + // A child that has already exited: its process ID is not running. + const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid; + const leftover = `.quak-backup-100.jpg-${exitedPID}-abc123.tmp`; + // This test's own process stands in for a backup running at the same + // time. + const inProgress = `.quak-backup-101.jpg-${process.pid}-def456.tmp`; + writeFileSync(join(originals, leftover), "partial"); + writeFileSync(join(originals, inProgress), "partial"); + const lib = await openLibrary(stubSource()); + + await lib.backup({ downloadDirectory: outDir }); + + const names = readdirSync(originals); + expect(names).not.toContain(leftover); + expect(names).toContain(inProgress); + lib.close(); + }); + + it("removes leftover temp files in thumbnails/ but not those of a backup still running", async () => { + const outDir = join(root, "backup"); + const thumbnails = join(outDir, "thumbnails"); + mkdirSync(thumbnails, { recursive: true }); + const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid; + const leftover = `.quak-backup-100.jpg-${exitedPID}-abc123.tmp`; + const inProgress = `.quak-backup-101.jpg-${process.pid}-def456.tmp`; + writeFileSync(join(thumbnails, leftover), "partial"); + writeFileSync(join(thumbnails, inProgress), "partial"); + const lib = await openLibrary(stubSource()); + + await lib.backup({ + downloadDirectory: outDir, + includeThumbnails: true, + }); + + const names = readdirSync(thumbnails); + expect(names).not.toContain(leftover); + expect(names).toContain(inProgress); + lib.close(); + }); +}); + +// Every refresh fails, as with an expired session or no network. +class FailingClient extends MockClient { + override async collectionsSince(): Promise<CollectionsPage> { + throw new Error("HTTP 401 from server"); + } +} + +// Holds its refresh open until `release()` is called, then reports a third +// album, so a backup can be started while that refresh is still running. +class HeldClient extends MockClient { + release!: () => void; + private held = new Promise<void>((resolve) => { + this.release = resolve; + }); + override async collectionsSince(): Promise<CollectionsPage> { + await this.held; + return { + collections: [collection(3, "Later")], + deleted: [], + cursor: 2, + }; + } + override async filesSince(args: { + collectionID: number; + }): Promise<FilesPage> { + if (args.collectionID !== 3) return super.filesSince(args); + return { files: [file(300, 3, "late.jpg")], deleted: [], cursor: 2 }; + } +} + +// Fill the library cache on disk, so the next open starts its refresh in the +// background instead of waiting for it. +const fillCache = async (): Promise<void> => { + const lib = await openLibrary(stubSource()); + await lib.close(); +}; + +describe("the refresh before a backup", () => { + it("waits for a refresh already running and backs up what it found", async () => { + await fillCache(); + const client = new HeldClient(); + const lib = await openLibrary(stubSource(), client); + const outDir = join(root, "backup"); + + const backup = lib.backup({ downloadDirectory: outDir }); + client.release(); + const result = await backup; + + expect(result.totalFiles).toBe(4); + expect(existsSync(join(outDir, "originals", "300.jpg"))).toBe(true); + await lib.close(); + }); + + it("fails before any download when the refresh fails, leaving failures.json as it was", async () => { + await fillCache(); + const outDir = join(root, "backup"); + seedLedger(outDir, 100, "beach.jpg"); + const ledgerPath = join(outDir, "failures.json"); + const ledgerBefore = readFileSync(ledgerPath, "utf-8"); + const source = stubSource(); + const lib = await openLibrary(source, new FailingClient()); + + await expect(lib.backup({ downloadDirectory: outDir })).rejects.toThrow( + "HTTP 401 from server", + ); + + expect(source.originalCalls).toBe(0); + expect(readFileSync(ledgerPath, "utf-8")).toBe(ledgerBefore); + expect(existsSync(join(outDir, "originals"))).toBe(false); + await lib.close(); + }); + + it("fails when the refresh fails on an empty cache, instead of backing up nothing", async () => { + const source = stubSource(); + const lib = await openLibrary(source, new FailingClient()); + const outDir = join(root, "backup"); + + await expect(lib.backup({ downloadDirectory: outDir })).rejects.toThrow( + "HTTP 401 from server", + ); + + expect(source.originalCalls).toBe(0); + expect(existsSync(outDir)).toBe(false); + await lib.close(); + }); +}); + +// Every entry under collections/, one level of directories deep, with each +// symlink's target. +const tree = (outDir: string): string[] => { + const lines: string[] = []; + const list = (dir: string, prefix: string): void => { + for (const name of readdirSync(dir).sort()) { + const path = join(dir, name); + const st = lstatSync(path); + if (st.isSymbolicLink()) { + lines.push(`${prefix}${name} -> ${readlinkSync(path)}`); + } else if (st.isDirectory() && prefix === "") { + lines.push(`${name}/`); + list(path, `${name}/`); + } else { + lines.push(`${prefix}${name}`); + } + } + }; + list(join(outDir, "collections"), ""); + return lines; +}; + +// The album folders under collections/, driven through `runBackup` with a +// stand-in library whose albums a test changes between runs. +describe("backup album folders", () => { + interface Album { + collection: Collection; + files: EnteFile[]; + } + + const libraryOf = (albums: Album[]): BackupLibrary => ({ + refresh: async () => {}, + listCollections: () => albums.map((a) => a.collection), + listFiles: (id) => + albums.find((a) => a.collection.id === id)?.files ?? [], + original: async (fileID) => { + const path = join(root, `source-${fileID}`); + writeFileSync(path, `original ${fileID}`); + return { path }; + }, + thumbnail: async () => { + throw new Error("no thumbnails in this stand-in"); + }, + }); + + const albumID = (outDir: string, jsonName: string): number => + JSON.parse(readFileSync(join(outDir, "collections", jsonName), "utf-8")) + .id; + + it("gives every file and every album its own name when names repeat", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [ + file(1, 10, "IMG_0001.JPG"), + file(2, 10, "IMG_0001.JPG"), + file(4, 10, "img_0001.jpg"), + file(3, 10, "other.jpg"), + ], + }, + { + collection: collection(11, "Trip"), + files: [file(3, 11, "other.jpg")], + }, + ]); + + const result = await runBackup(lib, { downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Trip (10)/", + "Trip (10)/IMG_0001 (1).JPG -> ../../originals/1.JPG", + "Trip (10)/IMG_0001 (2).JPG -> ../../originals/2.JPG", + "Trip (10)/img_0001 (4).jpg -> ../../originals/4.jpg", + "Trip (10)/other.jpg -> ../../originals/3.jpg", + "Trip (10).json", + "Trip (11)/", + "Trip (11)/other.jpg -> ../../originals/3.jpg", + "Trip (11).json", + ]); + expect(albumID(outDir, "Trip (10).json")).toBe(10); + expect(albumID(outDir, "Trip (11).json")).toBe(11); + }); + + it("keeps names unique when a name with an ID added is another entry's own name", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [ + file(5, 10, "IMG (6).JPG"), + file(6, 10, "IMG.JPG"), + file(7, 10, "IMG.JPG"), + ], + }, + { + collection: collection(11, "Trip"), + files: [file(8, 11, "a.jpg")], + }, + { + collection: collection(12, "Trip (11)"), + files: [file(9, 12, "b.jpg")], + }, + ]); + + const result = await runBackup(lib, { downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Trip (10)/", + "Trip (10)/IMG (6) (5).JPG -> ../../originals/5.JPG", + "Trip (10)/IMG (6).JPG -> ../../originals/6.JPG", + "Trip (10)/IMG (7).JPG -> ../../originals/7.JPG", + "Trip (10).json", + "Trip (11)/", + "Trip (11)/a.jpg -> ../../originals/8.jpg", + "Trip (11) (12)/", + "Trip (11) (12)/b.jpg -> ../../originals/9.jpg", + "Trip (11) (12).json", + "Trip (11).json", + ]); + expect(albumID(outDir, "Trip (10).json")).toBe(10); + expect(albumID(outDir, "Trip (11).json")).toBe(11); + expect(albumID(outDir, "Trip (11) (12).json")).toBe(12); + }); + + it("changes nothing on a second run over an unchanged account", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [ + file(1, 10, "IMG_0001.JPG"), + file(2, 10, "IMG_0001.JPG"), + ], + }, + { + collection: collection(11, "Trip"), + files: [file(3, 11, "other.jpg")], + }, + ]); + + await runBackup(lib, { downloadDirectory: outDir }); + const before = tree(outDir); + const second = await runBackup(lib, { downloadDirectory: outDir }); + + expect(second.downloaded).toBe(0); + expect(second.failed).toBe(0); + expect(tree(outDir)).toEqual(before); + }); + + it("leaves the albums an onlyAlbumNames run skips as they were", async () => { + const outDir = join(root, "backup"); + // "trip" is skipped by the scoped run but its name clashes with the + // in-scope "Trip", so "Trip" must keep its ID suffix. + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [file(1, 10, "a.jpg")], + }, + { + collection: collection(11, "trip"), + files: [file(2, 11, "b.jpg")], + }, + { + collection: collection(12, "Work"), + files: [file(3, 12, "c.jpg")], + }, + ]); + const json = (name: string): string => + readFileSync(join(outDir, "collections", name), "utf-8"); + + await runBackup(lib, { downloadDirectory: outDir }); + const before = tree(outDir); + const skippedJSON = [json("trip (11).json"), json("Work.json")]; + const scoped = await runBackup(lib, { + downloadDirectory: outDir, + onlyAlbumNames: ["Trip"], + }); + + expect(scoped.failed).toBe(0); + expect(before).toEqual([ + "Trip (10)/", + "Trip (10)/a.jpg -> ../../originals/1.jpg", + "Trip (10).json", + "Work/", + "Work/c.jpg -> ../../originals/3.jpg", + "Work.json", + "trip (11)/", + "trip (11)/b.jpg -> ../../originals/2.jpg", + "trip (11).json", + ]); + expect(tree(outDir)).toEqual(before); + expect([json("trip (11).json"), json("Work.json")]).toEqual( + skippedJSON, + ); + }); + + it("removes links and album folders that are gone, and nothing the user added", async () => { + const outDir = join(root, "backup"); + const albums: Album[] = [ + { + collection: collection(10, "Trip"), + files: [ + file(1, 10, "IMG_0001.JPG"), + file(2, 10, "IMG_0001.JPG"), + file(3, 10, "other.jpg"), + ], + }, + { + collection: collection(12, "Work"), + files: [file(5, 12, "a.jpg")], + }, + { + collection: collection(13, "Old"), + files: [file(5, 13, "a.jpg")], + }, + ]; + const lib = libraryOf(albums); + await runBackup(lib, { downloadDirectory: outDir }); + + // What the user put in the tree: a note and a symlink of their own in + // an album, a note in an album about to be renamed, and a folder quak + // did not create. + const collectionsDir = join(outDir, "collections"); + writeFileSync(join(collectionsDir, "Trip", "notes.txt"), "mine"); + symlinkSync("../elsewhere", join(collectionsDir, "Trip", "mine")); + writeFileSync(join(collectionsDir, "Work", "keep.txt"), "mine"); + mkdirSync(join(collectionsDir, "Mine")); + writeFileSync(join(collectionsDir, "Mine", "keep.txt"), "mine"); + + // File 2 leaves Trip, Work is renamed Office, Old is deleted. + albums[0]!.files.splice(1, 1); + albums[1]!.collection = collection(12, "Office"); + albums.splice(2, 1); + const result = await runBackup(lib, { downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Mine/", + "Mine/keep.txt", + "Office/", + "Office/a.jpg -> ../../originals/5.jpg", + "Office.json", + "Trip/", + "Trip/IMG_0001.JPG -> ../../originals/1.JPG", + "Trip/mine -> ../elsewhere", + "Trip/notes.txt", + "Trip/other.jpg -> ../../originals/3.jpg", + "Trip.json", + "Work/", + "Work/keep.txt", + "Work.json", + ]); + }); + + // One album backed up, then a folder the user made beside it holding a + // symlink into originals/, with `json` (if given) as its sibling JSON. + const backupWithUserFolder = async ( + json: string | undefined, + ): Promise<{ outDir: string; failed: number }> => { + const outDir = join(root, "backup"); + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [file(1, 10, "a.jpg")], + }, + ]); + await runBackup(lib, { downloadDirectory: outDir }); + const collectionsDir = join(outDir, "collections"); + mkdirSync(join(collectionsDir, "Mine")); + symlinkSync( + "../../originals/1.jpg", + join(collectionsDir, "Mine", "a.jpg"), + ); + if (json !== undefined) { + writeFileSync(join(collectionsDir, "Mine.json"), json); + } + const result = await runBackup(lib, { downloadDirectory: outDir }); + return { outDir, failed: result.failed }; + }; + + it("leaves a user folder with no JSON beside it as it was", async () => { + const { outDir, failed } = await backupWithUserFolder(undefined); + + expect(failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Mine/", + "Mine/a.jpg -> ../../originals/1.jpg", + "Trip/", + "Trip/a.jpg -> ../../originals/1.jpg", + "Trip.json", + ]); + }); + + it("leaves a user folder whose JSON has no album ID as it was", async () => { + const json = '{"name":"Mine"}'; + const { outDir, failed } = await backupWithUserFolder(json); + + expect(failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Mine/", + "Mine/a.jpg -> ../../originals/1.jpg", + "Mine.json", + "Trip/", + "Trip/a.jpg -> ../../originals/1.jpg", + "Trip.json", + ]); + expect( + readFileSync(join(outDir, "collections", "Mine.json"), "utf-8"), + ).toBe(json); + }); +}); + +// A live photo, which Ente stores as one ZIP, is backed up as its image and +// its video, which a photo viewer can open, beside a JSON file naming them, +// and its album folder links both. These tests download a live photo ZIP +// through the real download layer (test/live-photo.ts). +describe("backup of live photos", () => { + // An account of one album, Trip (10), holding `files`. + class TripClient extends MockClient { + constructor(private readonly files: EnteFile[]) { + super(); + } + override async collectionsSince(): Promise<CollectionsPage> { + return { + collections: [collection(10, "Trip")], + deleted: [], + cursor: 1, + }; + } + override async filesSince(): Promise<FilesPage> { + return { files: this.files, deleted: [], cursor: 1 }; + } + } + + const open = (files: EnteFile[], bodies: Map<number, Uint8Array>) => + openLibrary(cdnSource(bodies), new TripClient(files)); + + // What an earlier version stored for live photo 500: the ZIP under the + // image's name, and its link. + const earlierZIP = (outDir: string): void => { + mkdirSync(join(outDir, "originals"), { recursive: true }); + mkdirSync(join(outDir, "collections", "Trip"), { recursive: true }); + writeFileSync(join(outDir, "originals", "500.HEIC"), livePhotoZip()); + symlinkSync( + "../../originals/500.HEIC", + join(outDir, "collections", "Trip", "IMG_0500.HEIC"), + ); + }; + + const stored = ["500.heic", "500.json", "500.livephoto.json", "500.mov"]; + const linked = [ + "Trip/", + "Trip/IMG_0500.heic -> ../../originals/500.heic", + "Trip/IMG_0500.mov -> ../../originals/500.mov", + "Trip.json", + ]; + + it("stores a live photo as its image and its video and links both", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + const outDir = join(root, "backup"); + const originals = join(outDir, "originals"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ downloaded: 1, failed: 0 }); + expect(readdirSync(originals).sort()).toEqual(stored); + expect(readFileSync(join(originals, "500.heic"))).toEqual( + Buffer.from(IMAGE), + ); + expect(readFileSync(join(originals, "500.mov"))).toEqual( + Buffer.from(VIDEO), + ); + expect(tree(outDir)).toEqual(linked); + + // Both parts are there, so the next run fetches nothing. + const second = await lib.backup({ downloadDirectory: outDir }); + expect(second).toMatchObject({ downloaded: 0, skipped: 1, failed: 0 }); + await lib.close(); + }); + + it("gives both links of each live photo their own names when titles clash", async () => { + const a = await asLivePhoto(file(500, 10, "IMG_0001.HEIC")); + const b = await asLivePhoto(file(501, 10, "IMG_0001.HEIC")); + const lib = await open( + [a.file, b.file], + new Map([ + [500, a.body], + [501, b.body], + ]), + ); + const outDir = join(root, "backup"); + + await lib.backup({ downloadDirectory: outDir }); + + expect(tree(outDir)).toEqual([ + "Trip/", + "Trip/IMG_0001 (500).heic -> ../../originals/500.heic", + "Trip/IMG_0001 (500).mov -> ../../originals/500.mov", + "Trip/IMG_0001 (501).heic -> ../../originals/501.heic", + "Trip/IMG_0001 (501).mov -> ../../originals/501.mov", + "Trip.json", + ]); + await lib.close(); + }); + + it("replaces the ZIP an earlier version stored, and its link", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const outDir = join(root, "backup"); + earlierZIP(outDir); + const lib = await open([live], new Map([[500, body]])); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ downloaded: 1, failed: 0 }); + expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored); + expect(tree(outDir)).toEqual(linked); + await lib.close(); + }); + + it("stores nothing for a live photo that fails its hash, and keeps what was there", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + livePhotoZip(), + "not:the recorded hash", + ); + const outDir = join(root, "backup"); + earlierZIP(outDir); + const lib = await open([live], new Map([[500, body]])); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ downloaded: 0, failed: 1 }); + expect(result.errors.map((e) => e.fileID)).toEqual([500]); + expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]); + expect(readdirSync(join(outDir, "originals"))).toEqual(["500.HEIC"]); + expect(tree(outDir)).toEqual([ + "Trip/", + "Trip/IMG_0500.HEIC -> ../../originals/500.HEIC", + "Trip.json", + ]); + await lib.close(); + }); + + it("copies both parts of a live photo the cache already holds", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + const cached = await lib.photos.byID({ fileID: 500 })!.original(); + const outDir = join(root, "backup"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ downloaded: 1, failed: 0 }); + expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored); + expect(readFileSync(join(outDir, "originals", "500.mov"))).toEqual( + Buffer.from(VIDEO), + ); + expect(tree(outDir)).toEqual(linked); + // The cache keeps its own copy. + expect(existsSync(cached.videoPath!)).toBe(true); + await lib.close(); + }); + + it("replaces an earlier ZIP and its link with the image and video the cache holds", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + await lib.photos.byID({ fileID: 500 })!.original(); + const outDir = join(root, "backup"); + earlierZIP(outDir); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ downloaded: 1, failed: 0 }); + expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored); + expect(tree(outDir)).toEqual(linked); + await lib.close(); + }); + + it.each(["missing", "empty"])( + "fetches a live photo again when the video its JSON file names is %s", + async (state) => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + const outDir = join(root, "backup"); + await lib.backup({ downloadDirectory: outDir }); + const video = join(outDir, "originals", "500.mov"); + if (state === "missing") rmSync(video); + else writeFileSync(video, ""); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ + downloaded: 1, + skipped: 0, + failed: 0, + }); + expect(readFileSync(video)).toEqual(Buffer.from(VIDEO)); + expect(tree(outDir)).toEqual(linked); + await lib.close(); + }, + ); + + it.each(["missing", "empty"])( + "fetches a live photo again when the image its JSON file names is %s", + async (state) => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + const outDir = join(root, "backup"); + await lib.backup({ downloadDirectory: outDir }); + const image = join(outDir, "originals", "500.heic"); + if (state === "missing") rmSync(image); + else writeFileSync(image, ""); + + const result = await lib.backup({ downloadDirectory: outDir }); + + expect(result).toMatchObject({ + downloaded: 1, + skipped: 0, + failed: 0, + }); + expect(readFileSync(image)).toEqual(Buffer.from(IMAGE)); + expect(tree(outDir)).toEqual(linked); + await lib.close(); + }, + ); + + it("serves a live photo the backup stored to a library reading the backup", async () => { + const { file: live, body } = await asLivePhoto( + file(500, 10, "IMG_0500.HEIC"), + ); + const lib = await open([live], new Map([[500, body]])); + const outDir = join(root, "backup"); + await lib.backup({ downloadDirectory: outDir }); + await lib.close(); + + // Another cache over the same backup, whose server has nothing. + const reader = await Library.open({ + client: new TripClient([live]), + cacheDirectory: join(root, "other-cache"), + downloadDirectory: outDir, + contentSource: cdnSource(new Map()), + refreshIntervalSeconds: 3600, + precacheThumbnails: false, + precacheOriginals: false, + }); + const read = await reader.photos.byID({ fileID: 500 })!.original(); + + expect(read).toEqual({ + path: join(outDir, "originals", "500.heic"), + videoPath: join(outDir, "originals", "500.mov"), + bytes: IMAGE.length, + }); + await reader.close(); }); }); diff --git a/test/cli/commands.test.ts b/test/cli/commands.test.ts new file mode 100644 index 0000000..5b441e5 --- /dev/null +++ b/test/cli/commands.test.ts @@ -0,0 +1,856 @@ +/** + * Tests for the CLI commands (`src/cli-commands.ts`, issue #12). + * + * Each command is called directly with a context whose output streams collect + * text, whose session directory is a fresh temp directory, and whose session + * loader hands back a fake client. The fake serves two albums and three files + * from memory, writes stand-in bytes for originals and thumbnails, and makes no + * network calls. The helpers the commands call (`cli-read`, `cli-output`, + * backup, thumbnails) have their own tests; these check what each command + * prints and the exit code it returns. + */ + +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { PassThrough } from "node:stream"; +import * as jpegJs from "jpeg-js"; +import { + describe, + it, + expect, + vi, + beforeAll, + beforeEach, + afterEach, +} from "vitest"; + +import { + type CliContext, + saveSession, + loginCommand, + whoamiCommand, + logoutCommand, + collectionsCommand, + filesCommand, + getCommand, + getThumbCommand, + backupCommand, + backupMetadataCommand, + listMissingThumbnailsCommand, + fixMissingThumbnailsCommand, +} from "../../src/cli-commands.js"; +import { run } from "../../src/cli-run.js"; +import { loadSession } from "../../src/cli-session.js"; +import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js"; +import type { ContentSource } from "../../src/library/content.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; +import { init, toBase64 } from "../../src/crypto/index.js"; +import { defaultCacheDirectory } from "../../src/library/index.js"; +import { + asLivePhoto, + cdnSource, + IMAGE, + livePhotoHash, + livePhotoZip, + VIDEO, +} from "../live-photo.js"; + +const USER_ID = 42; + +const collection = ( + id: number, + name: string, + isShared = false, +): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id]), + name, + type: "album", + updationTime: 1, + isShared, +}); + +const file = (id: number, collectionID: number, title: string): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + metadata: { + title, + fileType: "image", + creationTime: 1000, + modificationTime: 1000, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 1, +}); + +const COLLECTIONS = [collection(1, "Vacation"), collection(2, "Work", true)]; + +const FILES: Record<number, EnteFile[]> = { + 1: [file(100, 1, "beach.jpg"), file(101, 1, "sunset.jpg")], + 2: [file(200, 2, "diagram.png")], +}; + +// An original is 7 bytes and a thumbnail 3. `failID` makes that file's +// original fail; `emptyThumbID` makes the server report that file's +// thumbnail as empty. `withNewFile` adds new.jpg (102) to Vacation, advancing +// the collection's updationTime as the server does, and `refreshError` makes +// listing collections fail with that message. +const fakeClient = ( + opts: { + failID?: number; + emptyThumbID?: number; + withNewFile?: boolean; + refreshError?: string; + } = {}, +) => { + const collections = opts.withNewFile + ? [{ ...COLLECTIONS[0], updationTime: 2 }, COLLECTIONS[1]] + : COLLECTIONS; + const files = opts.withNewFile + ? { + ...FILES, + 1: [...FILES[1], { ...file(102, 1, "new.jpg"), updationTime: 2 }], + } + : FILES; + const source: ContentSource = { + original: async ({ file: f, destination }) => { + if (f.id === opts.failID) throw new Error("HTTP 500 from server"); + writeFileSync(destination, Buffer.alloc(7, f.id & 0xff)); + return { bytesWritten: 7 }; + }, + thumbnail: async ({ file: f, destination }) => { + writeFileSync(destination, Buffer.alloc(3, f.id & 0xff)); + return { bytesWritten: 3 }; + }, + }; + const fake = { + whoami: () => ({ email: "cli@example.com", userID: USER_ID }), + collectionsSince: async () => { + if (opts.refreshError) throw new Error(opts.refreshError); + return { collections, deleted: [], cursor: 1 }; + }, + filesSince: async (args: { collectionID: number }) => ({ + files: files[args.collectionID] ?? [], + deleted: [], + cursor: 1, + }), + contentSource: () => source, + getApiClient: () => ({ + // The ML data request of `backup-metadata`: no file has any. + postJSON: async () => ({ data: [] }), + getThumbnailStream: async (fileID: number) => + new ReadableStream<Uint8Array>({ + start(controller) { + if (fileID !== opts.emptyThumbID) { + controller.enqueue(new Uint8Array(3)); + } + controller.close(); + }, + }), + }), + }; + // The commands only call the methods above. + return fake as unknown as Client; +}; + +// Collects everything written to it. +class Output { + text = ""; + write(text: string): void { + this.text += text; + } +} + +let root: string; +let stdout: Output; +let stderr: Output; + +const context = (client: Client | null = fakeClient()): CliContext => ({ + stdout, + stderr, + sessionDir: join(root, "session"), + cacheDir: join(root, "cache"), + loadSession: () => client, + login: async () => { + throw new Error("login not expected"); + }, + prompt: async () => { + throw new Error("prompt not expected"); + }, + promptSecret: async () => { + throw new Error("prompt not expected"); + }, +}); + +beforeAll(async () => { + await init(); +}); + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-cli-test-")); + stdout = new Output(); + stderr = new Output(); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe("session file", () => { + const snapshot: ClientSnapshot = { + email: "cli@example.com", + userID: USER_ID, + token: "token", + masterKey: "a", + secretKey: "b", + publicKey: "c", + }; + + it("is written with mode 0600 in a directory with mode 0700", () => { + const dir = join(root, "new", "session"); + saveSession(dir, snapshot); + expect(statSync(dir).mode & 0o777).toBe(0o700); + const path = join(dir, "session.json"); + expect(statSync(path).mode & 0o777).toBe(0o600); + expect(JSON.parse(readFileSync(path, "utf-8"))).toEqual(snapshot); + }); + + it("a missing session exits 1 with 'Not logged in'", async () => { + const ctx = { ...context(), loadSession }; + expect(await whoamiCommand(ctx)).toBe(1); + expect(stderr.text).toBe( + `Not logged in. Run "quak login" first.\n` + + `Session file: ${join(ctx.sessionDir, "session.json")}\n`, + ); + expect(stdout.text).toBe(""); + }); + + it("a corrupt session exits 1 and says it is corrupt", async () => { + const ctx = { ...context(), loadSession }; + saveSession(ctx.sessionDir, snapshot); + expect(await collectionsCommand(ctx, {})).toBe(1); + expect(stderr.text).toContain("is corrupt"); + expect(stderr.text).toContain( + `Run "quak logout" and then "quak login" to replace it.\n`, + ); + expect(stdout.text).toBe(""); + }); +}); + +// The login function is a fake that hands back a client whose snapshot is +// `snapshot`; each prompt is recorded and answered with "123456". +describe("login", () => { + const snapshot: ClientSnapshot = { + email: "cli@example.com", + userID: USER_ID, + token: "token", + masterKey: "a", + secretKey: "b", + publicKey: "c", + }; + + const loggedIn = { + whoami: () => ({ email: "cli@example.com", userID: USER_ID }), + toJSON: () => snapshot, + } as unknown as Client; + + let prompts: string[]; + + const loginContext = ( + login: (opts: LoginOptions) => Promise<Client>, + ): CliContext => ({ + ...context(), + login, + prompt: async (message) => { + prompts.push(message); + return "123456"; + }, + promptSecret: async (message) => { + prompts.push(message); + return "123456"; + }, + }); + + beforeEach(() => { + prompts = []; + vi.stubEnv("QUAK_EMAIL", "cli@example.com"); + vi.stubEnv("QUAK_PASSWORD", "hunter2"); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("takes the email and password from the environment without a prompt", async () => { + const calls: LoginOptions[] = []; + const ctx = loginContext(async (opts) => { + calls.push(opts); + return loggedIn; + }); + expect(await loginCommand(ctx)).toBe(0); + + expect(prompts).toEqual([]); + expect(calls).toHaveLength(1); + expect(calls[0]!.email).toBe("cli@example.com"); + expect(calls[0]!.password).toBe("hunter2"); + const path = join(ctx.sessionDir, "session.json"); + expect(stderr.text).toBe( + "Authenticating...\n" + + `Logged in as cli@example.com (user ${USER_ID})\n` + + `Session saved to ${path}\n`, + ); + }); + + it("saves the session with mode 0600 in a directory with mode 0700", async () => { + const ctx = loginContext(async () => loggedIn); + expect(await loginCommand(ctx)).toBe(0); + + expect(statSync(ctx.sessionDir).mode & 0o777).toBe(0o700); + const path = join(ctx.sessionDir, "session.json"); + expect(statSync(path).mode & 0o777).toBe(0o600); + expect(JSON.parse(readFileSync(path, "utf-8"))).toEqual(snapshot); + }); + + it("asks for the TOTP code when the account needs one", async () => { + let code: string | undefined; + const ctx = loginContext(async (opts) => { + code = await opts.totp!(); + return loggedIn; + }); + expect(await loginCommand(ctx)).toBe(0); + + expect(prompts).toEqual(["TOTP code: "]); + expect(code).toBe("123456"); + }); + + it("a failed login exits 1, says why and writes no session", async () => { + const ctx = loginContext(async () => { + throw new Error("HTTP 401 from server"); + }); + expect(await loginCommand(ctx)).toBe(1); + + expect(stderr.text).toBe( + "Authenticating...\nLogin failed: HTTP 401 from server\n", + ); + expect(existsSync(join(ctx.sessionDir, "session.json"))).toBe(false); + }); +}); + +// These use a real client read from the session file, over a fake API that +// records each request and answers with `status`. +describe("logout", () => { + const snapshot: ClientSnapshot = { + email: "cli@example.com", + userID: USER_ID, + token: "saved-token", + masterKey: toBase64(new Uint8Array(32)), + secretKey: toBase64(new Uint8Array(32)), + publicKey: toBase64(new Uint8Array(32)), + }; + + const requests: Request[] = []; + + const logoutContext = (status: number): CliContext => ({ + ...context(), + loadSession: (path) => + loadSession(path, { + fetch: async (url, init) => { + requests.push(new Request(url, init)); + return new Response(JSON.stringify({}), { + status, + headers: { "content-type": "application/json" }, + }); + }, + }), + }); + + beforeEach(() => { + requests.length = 0; + }); + + it("ends the session on the server, then deletes the file", async () => { + const ctx = logoutContext(200); + saveSession(ctx.sessionDir, snapshot); + expect(await logoutCommand(ctx)).toBe(0); + + expect(requests).toHaveLength(1); + expect(requests[0]!.method).toBe("POST"); + expect(new URL(requests[0]!.url).pathname).toBe("/users/logout"); + expect(requests[0]!.headers.get("X-Auth-Token")).toBe("saved-token"); + expect(existsSync(join(ctx.sessionDir, "session.json"))).toBe(false); + expect(stderr.text).toBe( + "Session ended on the server.\n" + + "Session deleted.\n" + + `Cache directory ${ctx.cacheDir} still holds decrypted data; delete it to remove that data.\n`, + ); + }); + + it("still deletes the file when the server call fails, and says so", async () => { + const ctx = logoutContext(500); + saveSession(ctx.sessionDir, snapshot); + expect(await logoutCommand(ctx)).toBe(1); + + expect(requests).toHaveLength(1); + expect(existsSync(join(ctx.sessionDir, "session.json"))).toBe(false); + expect(stderr.text).toBe( + "Could not end the session on the server: HTTP 500\n" + + "Session deleted.\n" + + `Cache directory ${ctx.cacheDir} still holds decrypted data; delete it to remove that data.\n`, + ); + }); + + it("names the account's default cache directory without --cache-dir", async () => { + const ctx = { ...logoutContext(200), cacheDir: undefined }; + saveSession(ctx.sessionDir, snapshot); + expect(await logoutCommand(ctx)).toBe(0); + expect(stderr.text).toContain( + `Cache directory ${defaultCacheDirectory(USER_ID)} still holds decrypted data`, + ); + }); + + it("without a session says so, calls nothing and exits 0", async () => { + expect(await logoutCommand(logoutContext(200))).toBe(0); + expect(requests).toHaveLength(0); + expect(stderr.text).toBe("No session found.\n"); + }); +}); + +describe("whoami", () => { + it("prints the account as one line of JSON", async () => { + expect(await whoamiCommand(context())).toBe(0); + expect(stdout.text).toBe( + `{"email":"cli@example.com","userID":${USER_ID}}\n`, + ); + }); +}); + +describe("collections", () => { + it("prints one tab-separated line per album", async () => { + expect(await collectionsCommand(context(), {})).toBe(0); + expect(stdout.text).toBe( + "1\talbum\tVacation\n" + "2\talbum\tWork (shared)\n", + ); + }); + + it("prints a JSON array with --json", async () => { + expect(await collectionsCommand(context(), { json: true })).toBe(0); + expect(JSON.parse(stdout.text)).toEqual([ + { + id: 1, + name: "Vacation", + type: "album", + ownerID: USER_ID, + isShared: false, + updationTime: 1, + }, + { + id: 2, + name: "Work", + type: "album", + ownerID: USER_ID, + isShared: true, + updationTime: 1, + }, + ]); + }); +}); + +describe("files", () => { + it("prints one tab-separated line per file", async () => { + expect(await filesCommand(context(), { collection: "1" })).toBe(0); + expect(stdout.text).toBe( + "100\timage\tbeach.jpg\n" + "101\timage\tsunset.jpg\n", + ); + }); + + it("prints a JSON array with --json", async () => { + const code = await filesCommand(context(), { + collection: "2", + json: true, + }); + expect(code).toBe(0); + expect(JSON.parse(stdout.text)).toEqual([ + { + id: 200, + title: "diagram.png", + fileType: "image", + creationTime: 1000, + collectionID: 2, + }, + ]); + }); + + it("exits 1 for an unknown collection", async () => { + expect(await filesCommand(context(), { collection: "9" })).toBe(1); + expect(stderr.text).toBe("Collection 9 not found\n"); + }); + + it("exits 1 for a collection ID that is not a number", async () => { + expect(await filesCommand(context(), { collection: "abc" })).toBe(1); + expect(stderr.text).toBe("Invalid collection ID\n"); + }); +}); + +describe("get and get-thumb", () => { + it("get finds a file in any album without --collection", async () => { + const out = join(root, "diagram.png"); + expect(await getCommand(context(), "200", { out })).toBe(0); + expect(readFileSync(out)).toEqual(Buffer.alloc(7, 200)); + expect(stderr.text).toBe(`7 bytes -> ${out}\n`); + }); + + it("get-thumb finds a file in any album without --collection", async () => { + const out = join(root, "thumb.jpg"); + expect(await getThumbCommand(context(), "200", { out })).toBe(0); + expect(readFileSync(out)).toEqual(Buffer.alloc(3, 200)); + expect(stderr.text).toBe(`3 bytes -> ${out}\n`); + }); + + it("get exits 1 when no album has the file", async () => { + const out = join(root, "x"); + expect(await getCommand(context(), "999", { out })).toBe(1); + expect(stderr.text).toBe("File 999 not found\n"); + expect(existsSync(out)).toBe(false); + }); + + it("get-thumb exits 1 when no album has the file", async () => { + const out = join(root, "x"); + expect(await getThumbCommand(context(), "999", { out })).toBe(1); + expect(stderr.text).toBe("File 999 not found\n"); + expect(existsSync(out)).toBe(false); + }); + + it("both exit 1 for a file ID that is not a number", async () => { + expect(await getCommand(context(), "abc", {})).toBe(1); + expect(await getThumbCommand(context(), "abc", {})).toBe(1); + expect(stderr.text).toBe("Invalid file ID\nInvalid file ID\n"); + }); +}); + +// A live photo, which Ente stores as one ZIP, is written as its image and its +// video, which a photo viewer can open. The account here is Vacation holding +// one live photo, 300, downloaded through the real download layer +// (test/live-photo.ts). +describe("a live photo", () => { + const livePhotoClient = async (image = IMAGE): Promise<Client> => { + const { file: live, body } = await asLivePhoto( + file(300, 1, "IMG_0300.HEIC"), + livePhotoZip({ "image.heic": image, "video.mov": VIDEO }), + livePhotoHash(image, VIDEO), + ); + const fake = { + whoami: () => ({ email: "cli@example.com", userID: USER_ID }), + collectionsSince: async () => ({ + collections: [collection(1, "Vacation")], + deleted: [], + cursor: 1, + }), + filesSince: async () => ({ files: [live], deleted: [], cursor: 1 }), + contentSource: () => cdnSource(new Map([[300, body]])), + // The ML data request of `backup-metadata`: no file has any. + getApiClient: () => ({ postJSON: async () => ({ data: [] }) }), + }; + return fake as unknown as Client; + }; + + it("get writes its image and video, named after the title with their own extensions", async () => { + const ctx = context(await livePhotoClient()); + const dir = join(root, "cwd"); + mkdirSync(dir); + const previous = process.cwd(); + process.chdir(dir); + try { + expect(await getCommand(ctx, "300", {})).toBe(0); + } finally { + process.chdir(previous); + } + + expect(readdirSync(dir).sort()).toEqual([ + "IMG_0300.heic", + "IMG_0300.mov", + ]); + expect(readFileSync(join(dir, "IMG_0300.heic"))).toEqual( + Buffer.from(IMAGE), + ); + expect(readFileSync(join(dir, "IMG_0300.mov"))).toEqual( + Buffer.from(VIDEO), + ); + expect(stderr.text).toBe( + `${IMAGE.length} bytes -> IMG_0300.heic\n` + + `${VIDEO.length} bytes -> IMG_0300.mov\n`, + ); + }); + + it("get --out writes the image there and the video beside it", async () => { + const out = join(root, "photo.jpg"); + const video = join(root, "photo.mov"); + + expect( + await getCommand(context(await livePhotoClient()), "300", { out }), + ).toBe(0); + + expect(readFileSync(out)).toEqual(Buffer.from(IMAGE)); + expect(readFileSync(video)).toEqual(Buffer.from(VIDEO)); + expect(stderr.text).toBe( + `${IMAGE.length} bytes -> ${out}\n${VIDEO.length} bytes -> ${video}\n`, + ); + }); + + it("get exits 1 and writes nothing when --out has the video's extension", async () => { + const out = join(root, "photo.MOV"); + + expect( + await getCommand(context(await livePhotoClient()), "300", { out }), + ).toBe(1); + + expect(existsSync(out)).toBe(false); + expect(existsSync(join(root, "photo.mov"))).toBe(false); + expect(stderr.text).toBe( + `File 300 is a live photo, and its video would also be written to ${out}\n`, + ); + }); + + it("backup-metadata --exif reads its image", async () => { + // A 4x4 JPEG, whose size can only come from reading the image; the + // ZIP and the video are not JPEGs. + const jpeg = jpegJs.encode( + { data: new Uint8Array(4 * 4 * 4), width: 4, height: 4 }, + 50, + ).data; + const dir = join(root, "dump"); + + expect( + await backupMetadataCommand( + context(await livePhotoClient(new Uint8Array(jpeg))), + dir, + { exif: true }, + ), + ).toBe(0); + + const record = JSON.parse( + readFileSync( + join(dir, "collections", "1-Vacation", "300.json"), + "utf-8", + ), + ); + expect(record.imageMetadata).toMatchObject({ + format: "jpeg", + width: 4, + height: 4, + }); + }); +}); + +describe("backup", () => { + it("exits 0 and prints a summary when every file is saved", async () => { + const dir = join(root, "backup"); + expect(await backupCommand(context(), dir, {})).toBe(0); + expect(stderr.text).toContain( + "\n--- Backup complete ---\n" + + " Total files: 3\n" + + " Downloaded: 3\n" + + " Skipped: 0\n" + + " Failed: 0\n", + ); + expect(stdout.text).toBe(""); + }); + + // The backup opens its library with the precache off: it fetches the + // originals it needs into the backup, and must not also fetch every + // thumbnail in the account, or keep originals, in the per-user cache. + it("leaves nothing in the cache's originals and thumbnails", async () => { + const dir = join(root, "backup"); + expect(await backupCommand(context(), dir, {})).toBe(0); + expect(readdirSync(join(root, "cache", "thumbnails"))).toEqual([]); + expect(readdirSync(join(root, "cache", "originals"))).toEqual([]); + }); + + it("exits 1 and lists the file when one download fails", async () => { + const ctx = context(fakeClient({ failID: 101 })); + expect(await backupCommand(ctx, join(root, "backup"), {})).toBe(1); + expect(stderr.text).toContain(" Failed: 1\n"); + expect(stderr.text).toContain( + "\nFailed files:\n" + + " [Vacation] sunset.jpg (id 101): HTTP 500 from server\n", + ); + }); + + it("prints the result as JSON with --json, still exiting 1 on a failure", async () => { + const ctx = context(fakeClient({ failID: 101 })); + const code = await backupCommand(ctx, join(root, "backup"), { + json: true, + }); + expect(code).toBe(1); + const result = JSON.parse(stdout.text); + expect(result).toMatchObject({ + totalFiles: 3, + downloaded: 2, + skipped: 0, + failed: 1, + }); + expect(result.errors[0].fileID).toBe(101); + expect(stderr.text).toBe("Starting backup...\n"); + }); + + it("exits 1 with the error on one line when the refresh fails", async () => { + const client = { + ...fakeClient(), + collectionsSince: async () => { + throw new Error("HTTP 401 from server"); + }, + } as unknown as Client; + const dir = join(root, "backup"); + // Through `run`, as `bin/quak.ts` does, which prints a thrown error. + const runStderr = new PassThrough(); + let runText = ""; + runStderr.on("data", (chunk: Buffer) => { + runText += chunk.toString(); + }); + const code = await new Promise<number>((resolve) => { + void run( + backupCommand(context(client), dir, {}), + new PassThrough(), + runStderr, + resolve, + ); + }); + expect(code).toBe(1); + expect(runText).toBe("quak: HTTP 401 from server\n"); + expect(stderr.text).toBe("Starting backup...\nRefreshing library...\n"); + expect(existsSync(join(dir, "originals"))).toBe(false); + }); +}); + +describe("helper list-missing-thumbnails", () => { + it("prints one line per file with an empty thumbnail", async () => { + const ctx = context(fakeClient({ emptyThumbID: 200 })); + expect(await listMissingThumbnailsCommand(ctx, {})).toBe(0); + expect(stdout.text).toBe( + "200\tdiagram.png\tWork\tempty thumbnail (0 bytes)\n", + ); + expect(stderr.text).toContain("\n1 file(s) with missing thumbnails:\n"); + }); + + it("says so when nothing is missing", async () => { + expect(await listMissingThumbnailsCommand(context(), {})).toBe(0); + expect(stdout.text).toBe(""); + expect(stderr.text).toContain("No missing thumbnails found.\n"); + }); + + it("prints a JSON array with --json and no progress", async () => { + const ctx = context(fakeClient({ emptyThumbID: 200 })); + expect(await listMissingThumbnailsCommand(ctx, { json: true })).toBe(0); + expect(JSON.parse(stdout.text)).toEqual([ + { + fileID: 200, + title: "diagram.png", + collection: "Work", + reason: "empty thumbnail (0 bytes)", + }, + ]); + expect(stderr.text).toBe(""); + }); +}); + +describe("backup-metadata --exif", () => { + // Runs the command and returns what it printed to stderr. + const backupMetadata = async (opts: { exif?: boolean; all?: boolean }) => { + expect( + await backupMetadataCommand(context(), join(root, "dump"), opts), + ).toBe(0); + return stderr.text; + }; + + it("--exif extracts EXIF", async () => { + expect(await backupMetadata({ exif: true })).toContain( + "[beach.jpg] Extracting EXIF...\n", + ); + }); + + it("--all extracts EXIF", async () => { + expect(await backupMetadata({ all: true })).toContain( + "[beach.jpg] Extracting EXIF...\n", + ); + }); + + it("without either flag extracts no EXIF", async () => { + expect(await backupMetadata({})).not.toContain("Extracting EXIF"); + }); +}); + +// Each test first runs `collections` so the cache holds the account as it was, +// then changes the server under it. +describe("backup-metadata and the thumbnail helpers refresh first", () => { + beforeEach(async () => { + expect(await collectionsCommand(context(), {})).toBe(0); + stdout.text = ""; + stderr.text = ""; + }); + + it("backup-metadata writes a file added since the cache was written", async () => { + const ctx = context(fakeClient({ withNewFile: true })); + const dir = join(root, "dump"); + expect(await backupMetadataCommand(ctx, dir, {})).toBe(0); + expect( + existsSync(join(dir, "collections", "1-Vacation", "102.json")), + ).toBe(true); + }); + + it("list-missing-thumbnails checks a file added since the cache was written", async () => { + const ctx = context( + fakeClient({ withNewFile: true, emptyThumbID: 102 }), + ); + expect(await listMissingThumbnailsCommand(ctx, {})).toBe(0); + expect(stdout.text).toBe( + "102\tnew.jpg\tVacation\tempty thumbnail (0 bytes)\n", + ); + }); + + it("fix-missing-thumbnails finds a file added since the cache was written", async () => { + const ctx = context(fakeClient({ withNewFile: true })); + expect( + await fixMissingThumbnailsCommand(ctx, { + file: ["102"], + json: true, + }), + ).toBe(0); + // Found, then skipped because the server records no thumbnail size + // for it; a file missing from the cache would fail as not found. + expect(JSON.parse(stdout.text)).toMatchObject([ + { fileID: 102, title: "new.jpg", status: "skipped" }, + ]); + }); + + // `run` in `cli-run.ts` prints a thrown error as one line and exits 1. + it("all three throw when the refresh fails", async () => { + const ctx = context( + fakeClient({ refreshError: "HTTP 503 from server" }), + ); + const dir = join(root, "dump"); + await expect(backupMetadataCommand(ctx, dir, {})).rejects.toThrow( + "HTTP 503 from server", + ); + expect(existsSync(dir)).toBe(false); + await expect(listMissingThumbnailsCommand(ctx, {})).rejects.toThrow( + "HTTP 503 from server", + ); + await expect( + fixMissingThumbnailsCommand(ctx, { file: ["100"] }), + ).rejects.toThrow("HTTP 503 from server"); + expect(stdout.text).toBe(""); + }); +}); diff --git a/test/cli/metadata-backup.test.ts b/test/cli/metadata-backup.test.ts index 4a0fb49..6d597e5 100644 --- a/test/cli/metadata-backup.test.ts +++ b/test/cli/metadata-backup.test.ts @@ -2,13 +2,17 @@ * Tests for `quak backup-metadata <dir>`. * * This command dumps all decrypted account metadata into a directory - * tree of plain JSON files, without downloading any file content. It - * is fast (no multi-megabyte downloads) and produces a complete - * plaintext record of every collection name, file title, creation - * date, GPS coordinate, camera model, caption, face label, and any - * other metadata the Ente clients have attached. + * tree of plain JSON files, without downloading any file content (unless + * `--exif` is given). It is fast and produces a complete plaintext record of + * every collection name, file title, creation date, GPS coordinate, camera + * model, caption, face label, and any other metadata the Ente clients have + * attached. * - * Layout: + * As of issue #52 it runs on the library API: `runMetadataBackup(lib, client, + * dir)` enumerates collections and files from the library's cache rather than + * scanning the client directly, and `--exif` reads each original through the + * library's content cache (`photo.original()`). The ML fetch is unchanged. The + * output tree is identical: * * <dir>/ * account.json { email, userID } @@ -34,7 +38,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import sodium from "libsodium-wrappers-sumo"; import { SRP, SrpServer } from "fast-srp-hap"; -import { beforeAll, afterAll, describe, expect, it } from "vitest"; +import { beforeAll, afterAll, describe, expect, it, vi } from "vitest"; import { init, toBase64, @@ -44,9 +48,21 @@ import { } from "../../src/crypto/index.js"; import * as jpegJs from "jpeg-js"; import { Client } from "../../src/client.js"; -import { runMetadataBackup } from "../../src/metadata-backup.js"; +import { Library } from "../../src/library/index.js"; +import { + runMetadataBackup, + type MetadataBackupOptions, +} from "../../src/metadata-backup.js"; +import { backupMetadataCommand } from "../../src/cli-commands.js"; import type { KeyAttributes } from "../../src/auth/types.js"; +// One file per ML data request, so the two files of the mock account are +// fetched in two requests and one of them can fail on its own. +vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({ + ...(await importOriginal<typeof import("../../src/mldata-fetch.js")>()), + MLDATA_BATCH_SIZE: 1, +})); + const TEST_EMAIL = "metabackup@example.com"; const TEST_PASSWORD = "metapass"; const TEST_OPS = 2; @@ -157,14 +173,15 @@ const buildMetaMock = async (): Promise<MetaMockState> => { }, }; - // Collection 2: "Work" with no magic metadata + // Collection 2: "../Work" with no magic metadata. The server chose a name + // that tries to climb out of the backup directory. const ck2 = sodium.crypto_secretbox_keygen(); const { ciphertext: encCK2, nonce: ck2N } = encryptSecretbox( ck2, masterKey, ); const { ciphertext: encCN2, nonce: cn2N } = encryptSecretbox( - new TextEncoder().encode("Work"), + new TextEncoder().encode("../Work"), ck2, ); const rawColl2 = { @@ -338,7 +355,8 @@ const buildMetaMock = async (): Promise<MetaMockState> => { }; }; -const buildMetaFetch = (m: MetaMockState) => { +// `failMLDataFor`: answer 500 to every ML data request that asks for this file. +const buildMetaFetch = (m: MetaMockState, failMLDataFor?: number) => { let srpServer: SrpServer; return (async ( input: RequestInfo | URL, @@ -394,6 +412,8 @@ const buildMetaFetch = (m: MetaMockState) => { } if (path === "/files/data/fetch") { const body = JSON.parse(init?.body as string); + if ((body.fileIDs as number[]).includes(failMLDataFor!)) + return new Response("server error", { status: 500 }); const data = (body.fileIDs as number[]) .filter((id: number) => m.encryptedMLData[id]) .map((id: number) => ({ @@ -431,16 +451,50 @@ afterAll(() => { rmSync(testDir, { recursive: true, force: true }); }); +// Log in against the mock and open a library over its cache. The point commands +// open the library with the background precache off and a long refresh interval; +// the same here keeps the test deterministic (no thumbnail/original prefetch it +// did not ask for, no second refresh mid-test). +const openLib = async (client: Client): Promise<Library> => + Library.open({ + // The library client omits `fetchMLData`, matching how the CLI opens + // point commands: `runMetadataBackup` fetches ML data itself through + // the client, so the library's background backfill would only be a + // redundant second pass over the same endpoint. + client: { + whoami: () => client.whoami(), + collectionsSince: (args) => client.collectionsSince(args), + filesSince: (args) => client.filesSince(args), + contentSource: () => client.contentSource(), + }, + cacheDirectory: mkdtempSync(join(testDir, "cache-")), + refreshIntervalSeconds: 3600, + precacheThumbnails: false, + precacheOriginals: false, + }); + +// Run one metadata backup end to end: fresh client, fresh library, then close. +const runBackup = async ( + outDir: string, + opts?: MetadataBackupOptions, +): Promise<void> => { + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { fetch: buildMetaFetch(mock) }, + }); + const lib = await openLib(client); + try { + await runMetadataBackup(lib, client, outDir, opts); + } finally { + lib.close(); + } +}; + describe("quak backup-metadata", () => { it("writes account.json with email and userID", async () => { const outDir = join(testDir, "full"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMetaFetch(mock) }, - }); - - await runMetadataBackup(client, outDir); + await runBackup(outDir); const account = JSON.parse( readFileSync(join(outDir, "account.json"), "utf-8"), @@ -451,16 +505,11 @@ describe("quak backup-metadata", () => { it("creates per-collection directories with _collection.json", async () => { const outDir = join(testDir, "collections"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMetaFetch(mock) }, - }); - - await runMetadataBackup(client, outDir); + await runBackup(outDir); const collDirs = readdirSync(join(outDir, "collections")); - expect(collDirs.length).toBe(2); + // "../Work" is sanitized into one directory name. + expect(collDirs.sort()).toEqual(["10-Vacation", "20-__Work"]); // Find the Vacation collection dir (prefixed with ID) const vacDir = collDirs.find((d) => d.includes("Vacation"))!; @@ -478,13 +527,7 @@ describe("quak backup-metadata", () => { it("decrypts collection-level pubMagicMetadata", async () => { const outDir = join(testDir, "coll-magic"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMetaFetch(mock) }, - }); - - await runMetadataBackup(client, outDir); + await runBackup(outDir); const collDirs = readdirSync(join(outDir, "collections")); const vacDir = collDirs.find((d) => d.includes("Vacation"))!; @@ -501,13 +544,7 @@ describe("quak backup-metadata", () => { it("writes per-file JSON with all three metadata layers", async () => { const outDir = join(testDir, "file-meta"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMetaFetch(mock) }, - }); - - await runMetadataBackup(client, outDir); + await runBackup(outDir); const collDirs = readdirSync(join(outDir, "collections")); const vacDir = collDirs.find((d) => d.includes("Vacation"))!; @@ -526,13 +563,7 @@ describe("quak backup-metadata", () => { it("handles files with no magic metadata gracefully", async () => { const outDir = join(testDir, "no-magic"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMetaFetch(mock) }, - }); - - await runMetadataBackup(client, outDir); + await runBackup(outDir); const collDirs = readdirSync(join(outDir, "collections")); const workDir = collDirs.find((d) => d.includes("Work"))!; @@ -550,14 +581,8 @@ describe("quak backup-metadata", () => { it("is incremental: second run does not fail", async () => { const outDir = join(testDir, "incremental"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMetaFetch(mock) }, - }); - - await runMetadataBackup(client, outDir); - await runMetadataBackup(client, outDir); + await runBackup(outDir); + await runBackup(outDir); const account = JSON.parse( readFileSync(join(outDir, "account.json"), "utf-8"), @@ -567,13 +592,7 @@ describe("quak backup-metadata", () => { it("fetches and decrypts ML data by default", async () => { const outDir = join(testDir, "ml-data"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMetaFetch(mock) }, - }); - - await runMetadataBackup(client, outDir); + await runBackup(outDir); const collDirs = readdirSync(join(outDir, "collections")); const vacDir = collDirs.find((d) => d.includes("Vacation"))!; @@ -597,13 +616,7 @@ describe("quak backup-metadata", () => { it("extracts EXIF from downloaded files when --exif is set", async () => { const outDir = join(testDir, "exif-data"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMetaFetch(mock) }, - }); - - await runMetadataBackup(client, outDir, { exif: true }); + await runBackup(outDir, { exif: true }); const collDirs = readdirSync(join(outDir, "collections")); const vacDir = collDirs.find((d) => d.includes("Vacation"))!; @@ -619,5 +632,78 @@ describe("quak backup-metadata", () => { expect(fileMeta.imageMetadata.format).toBe("jpeg"); expect(fileMeta.imageMetadata.width).toBe(100); expect(fileMeta.imageMetadata.height).toBe(80); + expect(fileMeta.imageMetadataError).toBeUndefined(); + + // File 200 has no original on the mock server, so extraction fails + // and the reason is recorded instead of the field being left out. + const workDir = collDirs.find((d) => d.includes("Work"))!; + const failedMeta = JSON.parse( + readFileSync( + join(outDir, "collections", workDir, "200.json"), + "utf-8", + ), + ); + expect(failedMeta.imageMetadata).toBeUndefined(); + expect(failedMeta.imageMetadataError).toEqual(expect.any(String)); + }); +}); + +describe("quak backup-metadata when an ML data request fails", () => { + // Run the CLI command against the mock and return its exit code, stderr + // and output directory. + const runCommand = async (failMLDataFor?: number) => { + const client = await Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: { + fetch: buildMetaFetch(mock, failMLDataFor), + retry: { sleep: async () => {} }, + }, + }); + const outDir = mkdtempSync(join(testDir, "ml-fail-")); + let stderr = ""; + const code = await backupMetadataCommand( + { + stdout: { write: () => true }, + stderr: { write: (text: string) => (stderr += text) }, + sessionDir: testDir, + cacheDir: mkdtempSync(join(testDir, "cache-")), + loadSession: () => client, + }, + outDir, + {}, + ); + return { code, stderr, outDir }; + }; + + it("writes every file, marks the failed batch's files, and exits 1", async () => { + const { code, stderr, outDir } = await runCommand(200); + + expect(code).toBe(1); + expect(stderr).toContain("ML data request for 1 file(s) failed"); + + const ok = JSON.parse( + readFileSync( + join(outDir, "collections", "10-Vacation", "100.json"), + "utf-8", + ), + ); + expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]); + expect(ok.mlDataError).toBeUndefined(); + + const failed = JSON.parse( + readFileSync( + join(outDir, "collections", "20-__Work", "200.json"), + "utf-8", + ), + ); + expect(failed.metadata.title).toBe("diagram.png"); + expect(failed.mlData).toBeUndefined(); + expect(failed.mlDataError).toContain("500"); + }); + + it("exits 0 when every ML data request succeeds", async () => { + const { code } = await runCommand(); + expect(code).toBe(0); }); }); diff --git a/test/cli/metadata-exif.test.ts b/test/cli/metadata-exif.test.ts new file mode 100644 index 0000000..4a8cf55 --- /dev/null +++ b/test/cli/metadata-exif.test.ts @@ -0,0 +1,136 @@ +/** + * Tests for the JPEG EXIF scan behind `quak backup-metadata --exif`. + * + * The originals come from users' libraries, so a truncated or corrupt JPEG + * must neither hang the scan nor throw out of it, and a malformed file must be + * told apart from one that simply has no EXIF: the record carries the reason in + * `exifError`. Each input below is a short hand-built byte array. + */ + +import { describe, expect, it } from "vitest"; +import { + extractExifFromJpeg, + extractImageMetadata, +} from "../../src/metadata-backup.js"; + +const SOI = [0xff, 0xd8]; // start of image +const SOS = [0xff, 0xda, 0x00, 0x02]; // start of scan, where the scan stops +const EXIF_HEADER = [0x45, 0x78, 0x69, 0x66, 0x00, 0x00]; // "Exif\0\0" + +// A big-endian TIFF block with one IFD entry: Orientation (0x0112), SHORT, 6. +const TIFF_ORIENTATION_6 = [ + 0x4d, 0x4d, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08, 0x00, 0x01, 0x01, 0x12, + 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, +]; + +// An APP1 segment whose length field matches its data. +const app1 = (data: number[]): number[] => { + const len = data.length + 2; + return [0xff, 0xe1, len >> 8, len & 0xff, ...data]; +}; + +const bytes = (...parts: number[][]): Uint8Array => + new Uint8Array(parts.flat()); + +describe("extractExifFromJpeg", () => { + it("returns the EXIF segment of a valid JPEG", () => { + const data = [...EXIF_HEADER, ...TIFF_ORIENTATION_6]; + const scan = extractExifFromJpeg(bytes(SOI, app1(data), SOS)); + expect(scan.error).toBeUndefined(); + expect([...scan.exif!]).toEqual(data); + }); + + it("returns nothing for a file that is not a JPEG", () => { + const png = bytes([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + expect(extractExifFromJpeg(png)).toEqual({}); + }); + + it("returns nothing for a JPEG without EXIF", () => { + const app0 = [0xff, 0xe0, 0x00, 0x04, 0x00, 0x00]; + expect(extractExifFromJpeg(bytes(SOI, app0, SOS))).toEqual({}); + }); + + it("ignores an APP1 segment too short to hold the Exif header", () => { + // A length under 8 cannot hold the six-byte "Exif\0\0" header, so the + // segment is not EXIF. This one has length 7 and holds only "Exif\0", + // which the old code, lacking the length check, returned as EXIF. + const short = app1(EXIF_HEADER.slice(0, 5)); + expect(extractExifFromJpeg(bytes(SOI, short, SOS))).toEqual({}); + }); + + it("accepts an APP1 segment of length 8 holding just the Exif header", () => { + const scan = extractExifFromJpeg(bytes(SOI, app1(EXIF_HEADER), SOS)); + expect(scan.error).toBeUndefined(); + expect([...scan.exif!]).toEqual(EXIF_HEADER); + }); + + it("reports a JPEG truncated inside a segment header", () => { + const scan = extractExifFromJpeg(bytes(SOI, [0xff, 0xe1, 0x00])); + expect(scan.exif).toBeUndefined(); + expect(scan.error).toMatch(/truncated segment length/); + }); + + it("reports a JPEG that ends before the image data", () => { + const app0 = [0xff, 0xe0, 0x00, 0x04, 0x00, 0x00]; + const scan = extractExifFromJpeg(bytes(SOI, app0)); + expect(scan.error).toMatch(/ends before the image data/); + }); + + it("stops on a zero-length segment instead of looping", () => { + // A length of 0 would otherwise step the scan by 2 bytes at a time + // through the rest of the file, reading garbage as markers. + const zero = [0xff, 0xe0, 0x00, 0x00]; + const scan = extractExifFromJpeg( + bytes(SOI, zero, zero, zero, zero, SOS), + ); + expect(scan.error).toMatch(/segment length 0 at byte 2 is too small/); + }); + + it("stops on a segment length of 1", () => { + const scan = extractExifFromJpeg( + bytes(SOI, [0xff, 0xe0, 0x00, 0x01], SOS), + ); + expect(scan.error).toMatch(/segment length 1 at byte 2 is too small/); + }); + + it("reports a segment length that runs past the end of the file", () => { + // APP1 claims 0x4000 bytes but only the "Exif\0\0" header follows. + const scan = extractExifFromJpeg( + bytes(SOI, [0xff, 0xe1, 0x40, 0x00], EXIF_HEADER), + ); + expect(scan.exif).toBeUndefined(); + expect(scan.error).toMatch(/runs past the end of the file/); + }); +}); + +describe("extractImageMetadata", () => { + it("parses EXIF from a valid JPEG", () => { + const meta = extractImageMetadata( + bytes(SOI, app1([...EXIF_HEADER, ...TIFF_ORIENTATION_6]), SOS), + ); + expect(meta?.exifError).toBeUndefined(); + expect(meta?.exif).toMatchObject({ Image: { Orientation: 6 } }); + }); + + it("returns nothing for a file that is not a JPEG", () => { + const text = new TextEncoder().encode("just some text, not an image"); + expect(extractImageMetadata(text)).toBeUndefined(); + }); + + it("records the reason when the JPEG is malformed", () => { + const meta = extractImageMetadata( + bytes(SOI, [0xff, 0xe1, 0x40, 0x00], EXIF_HEADER), + ); + expect(meta?.exif).toBeUndefined(); + expect(meta?.exifError).toMatch(/runs past the end of the file/); + }); + + it("keeps the raw bytes and the reason when EXIF cannot be parsed", () => { + const data = [...EXIF_HEADER, 0x58, 0x58]; + const meta = extractImageMetadata(bytes(SOI, app1(data), SOS)); + expect(meta?.exif).toBeUndefined(); + expect(meta?.exifRaw).toBe(Buffer.from(data).toString("base64")); + expect(meta?.exifError).toEqual(expect.any(String)); + }); +}); diff --git a/test/cli/output.test.ts b/test/cli/output.test.ts new file mode 100644 index 0000000..ce03e30 --- /dev/null +++ b/test/cli/output.test.ts @@ -0,0 +1,94 @@ +// The CLI presents a file by its own decrypted metadata, not the PhotoRecord +// projection (issue #52). For a renamed file the two disagree: the projection +// prefers `editedName` and reports `editedTime` in milliseconds, while the CLI +// must print the raw `metadata.title` and `metadata.creationTime` (microseconds) +// and name downloads after the raw title, byte-identical to the pre-library CLI. +// +// This locks in that contrast: the shared output helpers emit the raw values, +// and the projection of the same file emits the edited ones — so a regression +// that re-sourced the CLI from the projection would fail here. + +import { describe, it, expect } from "vitest"; + +import { + fileListRow, + fileListLine, + originalName, + thumbnailName, +} from "../../src/cli-output.js"; +import { deriveRecords } from "../../src/library/records.js"; +import type { EnteFile } from "../../src/model/types.js"; + +// Microseconds, as Ente stores times. +const RAW_CREATION = 1700000000000000; +const EDITED_TIME = 1710000000000000; +const RAW_TITLE = "IMG_0001.HEIC"; +const EDITED_NAME = "Sunset.heic"; + +// A file the user has renamed and re-dated: basic metadata holds the original +// title and capture time; public magic metadata holds the edits. +const renamedFile: EnteFile = { + id: 100, + collectionID: 10, + ownerID: 42, + key: new Uint8Array(), + metadata: { + title: RAW_TITLE, + fileType: "image", + creationTime: RAW_CREATION, + modificationTime: RAW_CREATION, + }, + pubMagicMetadata: { editedName: EDITED_NAME, editedTime: EDITED_TIME }, + file: { decryptionHeader: "" }, + thumbnail: { decryptionHeader: "" }, + updationTime: RAW_CREATION, +}; + +describe("CLI file output (issue #52)", () => { + it("emits the raw title and microsecond creationTime for --json", () => { + expect(fileListRow(renamedFile)).toEqual({ + id: 100, + title: RAW_TITLE, + fileType: "image", + creationTime: RAW_CREATION, + collectionID: 10, + }); + }); + + it("emits the raw title in the human column", () => { + expect(fileListLine(renamedFile)).toBe(`100\timage\t${RAW_TITLE}`); + }); + + it("names downloads after the raw title", () => { + expect(originalName(renamedFile)).toBe(RAW_TITLE); + expect(thumbnailName(renamedFile)).toBe(`thumb_${RAW_TITLE}`); + }); + + it("sanitizes the title when naming `quak get` downloads", () => { + // Without `--out`, the server-supplied title names the file, so it must + // not be able to point outside the working directory. + const hostile = { + ...renamedFile, + metadata: { ...renamedFile.metadata, title: "../../.bashrc" }, + }; + expect(originalName(hostile)).toBe("__.._.bashrc"); + expect(thumbnailName(hostile)).toBe("thumb___.._.bashrc"); + + const untitled = { + ...renamedFile, + metadata: { ...renamedFile.metadata, title: "" }, + }; + expect(originalName(untitled)).toBe("file-100"); + expect(thumbnailName(untitled)).toBe("thumb_file-100"); + }); + + it("does not use the editedName/editedTime projection", () => { + const record = deriveRecords([], [renamedFile]).photos.get(100); + // The projection prefers the edits and reports milliseconds; the CLI + // helpers above deliberately do not. + expect(record?.title).toBe(EDITED_NAME); + expect(record?.takenAt).toBe(Math.floor(EDITED_TIME / 1000)); + expect(fileListRow(renamedFile).title).not.toBe(record?.title); + expect(fileListRow(renamedFile).creationTime).not.toBe(record?.takenAt); + }); +}); diff --git a/test/cli/read.test.ts b/test/cli/read.test.ts new file mode 100644 index 0000000..a218b5c --- /dev/null +++ b/test/cli/read.test.ts @@ -0,0 +1,195 @@ +/** + * Tests for the CLI read helpers (`src/cli-read.ts`, owner amendment to + * issue #36, issue #52). + * + * The `collections`, `files`, `get`, and `get-thumb` commands must answer for + * current server state, not the local cache, so each helper forces a + * `Library.fresh()` round-trip before it reads. The stand-in library below + * serves nothing until `fresh()` has been awaited, so a helper that read + * without refreshing would come back empty and fail here. + * + * `collections` and `files` also list in the library's enumeration order + * (`listCollections`/`listFiles`) — the order the pre-library CLI printed — not + * the albums/photos projection's newest-first order. The fixtures are seeded in + * an enumeration order that a newest-first sort would rearrange, so a + * regression to the projection order would fail here too. Field values still + * come from the raw metadata via `cli-output.ts`. + */ + +import { describe, it, expect } from "vitest"; + +import { + freshCollections, + freshFiles, + freshFile, + type FreshReadLibrary, +} from "../../src/cli-read.js"; +import { fileListRow } from "../../src/cli-output.js"; +import type { Photo } from "../../src/library/index.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +const collection = (id: number, updationTime: number): Collection => ({ + id, + ownerID: 42, + key: new Uint8Array(), + name: `album-${id}`, + type: "album", + updationTime, + isShared: false, +}); + +// Microseconds, as Ente stores times. +const file = ( + id: number, + collectionID: number, + creationTime: number, +): EnteFile => ({ + id, + collectionID, + ownerID: 42, + key: new Uint8Array(), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime, + modificationTime: creationTime, + }, + file: { decryptionHeader: "" }, + thumbnail: { decryptionHeader: "" }, + updationTime: creationTime, +}); + +// A library that reveals its records only after `fresh()` has been awaited, and +// serves them in the enumeration order it was given. `photos.byID` returns a +// stand-in `Photo` carrying just the fileID the helper passes through. +class FakeLibrary implements FreshReadLibrary { + freshCalls = 0; + private refreshed = false; + + constructor( + private readonly collections: Collection[], + private readonly files: EnteFile[], + ) {} + + async fresh(): Promise<unknown> { + this.freshCalls++; + this.refreshed = true; + return {}; + } + + listCollections(): Collection[] { + return this.refreshed ? this.collections : []; + } + + getCollection(id: number): Collection | undefined { + return this.listCollections().find((c) => c.id === id); + } + + listFiles(collectionID: number): EnteFile[] { + return this.refreshed + ? this.files.filter((f) => f.collectionID === collectionID) + : []; + } + + getFileByID(fileID: number): EnteFile | undefined { + if (!this.refreshed) return undefined; + return this.files.find((f) => f.id === fileID); + } + + photos = { + byID: ({ fileID }: { fileID: number }): Photo | undefined => { + if (!this.refreshed) return undefined; + if (!this.files.some((f) => f.id === fileID)) return undefined; + return { fileID } as unknown as Photo; + }, + }; +} + +describe("CLI read helpers (issue #36 amendment, issue #52)", () => { + it("freshCollections refreshes first, then lists in enumeration order", async () => { + // Enumeration order 2, 1, 3; a newest-first sort would be 3, 2, 1. + const lib = new FakeLibrary( + [collection(2, 200), collection(1, 300), collection(3, 100)], + [], + ); + const rows = await freshCollections(lib); + + expect(lib.freshCalls).toBe(1); + expect(rows.map((c) => c.id)).toEqual([2, 1, 3]); + // The projection's newest-first order is a different sequence, so this + // is not accidentally that order. + const newestFirst = [...rows] + .sort((a, b) => b.updationTime - a.updationTime) + .map((c) => c.id); + expect(newestFirst).toEqual([1, 2, 3]); + expect(rows.map((c) => c.id)).not.toEqual(newestFirst); + }); + + it("freshFiles refreshes first, lists in enumeration order, keeps raw fields", async () => { + // Enumeration order by id 10, 11, 12; creationTimes ascending, so a + // newest-first sort would reverse them. + const files = [ + file(10, 1, 1_700_000_000_000_000), + file(11, 1, 1_700_000_000_000_001), + file(12, 1, 1_700_000_000_000_002), + ]; + const lib = new FakeLibrary([collection(1, 100)], files); + + const rows = await freshFiles(lib, 1); + expect(lib.freshCalls).toBe(1); + expect(rows?.map((f) => f.id)).toEqual([10, 11, 12]); + + // Field values come from raw metadata: microsecond creationTime and the + // raw title, unchanged. + expect(rows?.map(fileListRow)).toEqual([ + { + id: 10, + title: "file-10.jpg", + fileType: "image", + creationTime: 1_700_000_000_000_000, + collectionID: 1, + }, + { + id: 11, + title: "file-11.jpg", + fileType: "image", + creationTime: 1_700_000_000_000_001, + collectionID: 1, + }, + { + id: 12, + title: "file-12.jpg", + fileType: "image", + creationTime: 1_700_000_000_000_002, + collectionID: 1, + }, + ]); + }); + + it("freshFiles returns undefined for an unknown collection", async () => { + const lib = new FakeLibrary([collection(1, 100)], []); + const rows = await freshFiles(lib, 999); + expect(lib.freshCalls).toBe(1); + expect(rows).toBeUndefined(); + }); + + it("freshFile refreshes first, then resolves the photo and its raw record", async () => { + const f = file(10, 1, 1_700_000_000_000_000); + const lib = new FakeLibrary([collection(1, 100)], [f]); + + const resolved = await freshFile(lib, 10); + expect(lib.freshCalls).toBe(1); + expect(resolved?.photo.fileID).toBe(10); + expect(resolved?.file.metadata.title).toBe("file-10.jpg"); + expect(resolved?.file.metadata.creationTime).toBe( + 1_700_000_000_000_000, + ); + }); + + it("freshFile returns undefined for an unknown file", async () => { + const lib = new FakeLibrary([collection(1, 100)], []); + const resolved = await freshFile(lib, 404); + expect(lib.freshCalls).toBe(1); + expect(resolved).toBeUndefined(); + }); +}); diff --git a/test/cli/run.test.ts b/test/cli/run.test.ts new file mode 100644 index 0000000..29d0a92 --- /dev/null +++ b/test/cli/run.test.ts @@ -0,0 +1,58 @@ +/** + * Tests for `run` in `src/cli-run.ts`, which every CLI command goes through. + */ + +import { PassThrough } from "node:stream"; +import { describe, it, expect } from "vitest"; + +import { run } from "../../src/cli-run.js"; + +// A stream whose written text is kept in `text`; writes finish at once, so +// nothing is left waiting to drain. +const collector = (): { stream: PassThrough; text: () => string } => { + const stream = new PassThrough(); + const chunks: string[] = []; + stream.on("data", (chunk: Buffer) => chunks.push(chunk.toString())); + return { stream, text: () => chunks.join("") }; +}; + +const runToExit = async ( + command: Promise<number>, +): Promise<{ code: number; stdout: string; stderr: string }> => { + const stdout = collector(); + const stderr = collector(); + const code = await new Promise<number>((resolve) => { + void run(command, stdout.stream, stderr.stream, resolve); + }); + return { code, stdout: stdout.text(), stderr: stderr.text() }; +}; + +describe("run", () => { + it("exits with the code the command returns", async () => { + const result = await runToExit(Promise.resolve(3)); + expect(result).toEqual({ code: 3, stdout: "", stderr: "" }); + }); + + it("prints a thrown error as one line without a stack trace and exits 1", async () => { + const failing = async (): Promise<number> => { + throw new Error( + "ENOTDIR: not a directory, mkdir '/dev/null/x/originals'", + ); + }; + const result = await runToExit(failing()); + expect(result.code).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toBe( + "quak: ENOTDIR: not a directory, mkdir '/dev/null/x/originals'\n", + ); + }); + + it("prints a thrown value that is not an Error", async () => { + const result = await runToExit(Promise.reject("offline")); + expect(result).toEqual({ + code: 1, + stdout: "", + stderr: "quak: offline\n", + }); + }); +}); diff --git a/test/client/enumeration.test.ts b/test/client/enumeration.test.ts new file mode 100644 index 0000000..363f27d --- /dev/null +++ b/test/client/enumeration.test.ts @@ -0,0 +1,367 @@ +/** + * Tests for the resumable, deletion-aware enumeration variants on `Client`: + * `collectionsSince` and `filesSince`. + * + * The whole-account methods `listCollections` / `listFiles` always start at + * `sinceTime: 0` and hide deletions. The cache refresh needs the opposite: + * start from a saved cursor, learn what was deleted, and get back a cursor to + * resume from next time. These two methods provide that. + * + * The return shape keeps live records and tombstones apart — `collections` / + * `files` are decrypted live records, `deleted` is a plain list of the ids the + * server tombstoned. A tombstone carries no decryptable key or metadata, so it + * is a bare id rather than a hollowed-out `Collection` / `EnteFile`. + * + * All tests inject a fake `fetch` and drive a real `Client` (built with + * `Client.fromJSON`) so the decryption path runs for real. Live rows are built + * with libsodium exactly as the server would encrypt them; tombstone rows carry + * only the fields the code reads (`id`, `updationTime`, `isDeleted`), because + * they are never decrypted. + */ + +import sodium from "libsodium-wrappers-sumo"; +import { beforeAll, describe, expect, it } from "vitest"; +import { init, toBase64 } from "../../src/crypto/index.js"; +import { Client, type ClientSnapshot } from "../../src/client.js"; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const USER_ID = 42; + +interface Keys { + masterKey: Uint8Array; + publicKey: Uint8Array; + secretKey: Uint8Array; +} + +const buildKeys = (): Keys => { + const kp = sodium.crypto_box_keypair(); + return { + masterKey: sodium.crypto_secretbox_keygen(), + publicKey: kp.publicKey, + secretKey: kp.privateKey, + }; +}; + +const snapshotFor = (keys: Keys): ClientSnapshot => ({ + email: "user@example.com", + userID: USER_ID, + token: "test-token", + masterKey: toBase64(keys.masterKey), + secretKey: toBase64(keys.secretKey), + publicKey: toBase64(keys.publicKey), +}); + +const secretboxEncrypt = ( + plaintext: Uint8Array, + key: Uint8Array, +): { ciphertext: Uint8Array; nonce: Uint8Array } => { + const nonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); + return { + ciphertext: sodium.crypto_secretbox_easy(plaintext, nonce, key), + nonce, + }; +}; + +/** An owned collection row as the server sends it, keyed under the master key. */ +const ownedCollectionRow = ( + masterKey: Uint8Array, + opts: { id: number; name: string; updationTime: number }, +): Record<string, unknown> => { + const collectionKey = sodium.crypto_secretbox_keygen(); + const { ciphertext: encKey, nonce: keyNonce } = secretboxEncrypt( + collectionKey, + masterKey, + ); + const { ciphertext: encName, nonce: nameNonce } = secretboxEncrypt( + new TextEncoder().encode(opts.name), + collectionKey, + ); + return { + id: opts.id, + owner: { id: USER_ID }, + encryptedKey: toBase64(encKey), + keyDecryptionNonce: toBase64(keyNonce), + encryptedName: toBase64(encName), + nameDecryptionNonce: toBase64(nameNonce), + type: "album", + updationTime: opts.updationTime, + }; +}; + +/** A live file row inside a collection, keyed under that collection's key. */ +const fileRow = ( + collectionKey: Uint8Array, + opts: { id: number; title: string; updationTime: number }, +): Record<string, unknown> => { + const fileKey = sodium.crypto_secretbox_keygen(); + const { ciphertext: encFileKey, nonce: fileKeyNonce } = secretboxEncrypt( + fileKey, + collectionKey, + ); + const metadata = { + title: opts.title, + fileType: 0, + creationTime: opts.updationTime, + modificationTime: opts.updationTime, + }; + const push = + sodium.crypto_secretstream_xchacha20poly1305_init_push(fileKey); + const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push( + push.state, + new TextEncoder().encode(JSON.stringify(metadata)), + null, + sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, + ); + return { + id: opts.id, + collectionID: 1, + ownerID: USER_ID, + encryptedKey: toBase64(encFileKey), + keyDecryptionNonce: toBase64(fileKeyNonce), + metadata: { + encryptedData: toBase64(encMeta), + decryptionHeader: toBase64(push.header), + }, + file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, + thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, + updationTime: opts.updationTime, + }; +}; + +/** A tombstone row. Never decrypted, so only these fields are ever read. */ +const tombstoneRow = ( + id: number, + updationTime: number, +): Record<string, unknown> => ({ + id, + updationTime, + isDeleted: true, +}); + +const jsonResponse = (body: unknown): Response => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); + +/** + * A fetch that serves canned responses in order and records the `sinceTime` + * query parameter each request carried, so tests can prove the cursor is + * threaded from one page (and one call) to the next. + */ +const recordingFetch = ( + ...responses: Response[] +): { fetch: typeof globalThis.fetch; sinceTimes: (string | null)[] } => { + const sinceTimes: (string | null)[] = []; + let i = 0; + const fake = async (input: RequestInfo | URL): Promise<Response> => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.href + : input.url; + sinceTimes.push(new URL(url).searchParams.get("sinceTime")); + if (i >= responses.length) { + throw new Error(`recordingFetch: no response for call #${i}`); + } + return responses[i++]!; + }; + return { fetch: fake as typeof globalThis.fetch, sinceTimes }; +}; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("Client.filesSince", () => { + beforeAll(async () => { + await init(); + await sodium.ready; + }); + + it("pages from the given cursor, decrypts live rows, and collects tombstones", async () => { + const keys = buildKeys(); + const collectionKey = sodium.crypto_secretbox_keygen(); + + // Page 1 mixes a live file and a tombstone; the tombstone has the + // higher updationTime, so it — not the live row — sets the cursor the + // second page must be fetched from. + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ + diff: [ + fileRow(collectionKey, { + id: 1001, + title: "first.jpg", + updationTime: 100, + }), + tombstoneRow(1002, 150), + ], + hasMore: true, + }), + jsonResponse({ + diff: [ + fileRow(collectionKey, { + id: 1003, + title: "second.jpg", + updationTime: 200, + }), + ], + hasMore: false, + }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const { files, deleted, cursor } = await client.filesSince({ + collectionID: 1, + collectionKey, + sinceTime: 0, + }); + + expect(files.map((f) => f.id)).toEqual([1001, 1003]); + expect(files.map((f) => f.metadata.title)).toEqual([ + "first.jpg", + "second.jpg", + ]); + expect(deleted).toEqual([1002]); + expect(cursor).toBe(200); + // First request started at the caller's cursor; the second resumed + // from the max updationTime seen on the first page (the tombstone's). + expect(sinceTimes).toEqual(["0", "150"]); + }); + + it("fetches only newer rows when the returned cursor is passed back in", async () => { + const keys = buildKeys(); + const collectionKey = sodium.crypto_secretbox_keygen(); + + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ diff: [], hasMore: false }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const result = await client.filesSince({ + collectionID: 1, + collectionKey, + sinceTime: 200, + }); + + expect(result.files).toEqual([]); + expect(result.deleted).toEqual([]); + // An empty diff advances nothing: the cursor falls back to the input. + expect(result.cursor).toBe(200); + expect(sinceTimes).toEqual(["200"]); + }); + + it("stops and throws when the server claims more but does not advance (#7)", async () => { + const keys = buildKeys(); + const collectionKey = sodium.crypto_secretbox_keygen(); + + // hasMore is true, but the page's max updationTime (50) does not exceed + // the cursor the request was made with (50). Following hasMore here + // would refetch this same page forever. + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }), + jsonResponse({ diff: [tombstoneRow(1, 50)], hasMore: true }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + await expect( + client.filesSince({ + collectionID: 1, + collectionKey, + sinceTime: 50, + }), + ).rejects.toThrow(/not advance|non-advancing/i); + // It gave up after the first page rather than looping. + expect(sinceTimes).toEqual(["50"]); + }); +}); + +describe("Client.collectionsSince", () => { + beforeAll(async () => { + await init(); + await sodium.ready; + }); + + it("decrypts live collections, collects tombstones, and returns a cursor", async () => { + const keys = buildKeys(); + + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ + collections: [ + ownedCollectionRow(keys.masterKey, { + id: 1, + name: "Vacation", + updationTime: 100, + }), + tombstoneRow(3, 150), + ], + }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const { collections, deleted, cursor } = await client.collectionsSince({ + sinceTime: 0, + }); + + expect(collections.map((c) => c.id)).toEqual([1]); + expect(collections[0]!.name).toBe("Vacation"); + expect(deleted).toEqual([3]); + // The tombstone's updationTime advances the cursor too, so the next + // sync starts after it rather than seeing it again. + expect(cursor).toBe(150); + expect(sinceTimes).toEqual(["0"]); + }); + + it("falls back to the input cursor on an empty response", async () => { + const keys = buildKeys(); + + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ collections: [] }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const result = await client.collectionsSince({ sinceTime: 150 }); + + expect(result.collections).toEqual([]); + expect(result.deleted).toEqual([]); + expect(result.cursor).toBe(150); + expect(sinceTimes).toEqual(["150"]); + }); +}); + +describe("Client list wrappers still hide deletions", () => { + beforeAll(async () => { + await init(); + await sodium.ready; + }); + + it("listFiles drops tombstones and returns only live files", async () => { + const keys = buildKeys(); + const collectionKey = sodium.crypto_secretbox_keygen(); + + const { fetch, sinceTimes } = recordingFetch( + jsonResponse({ + diff: [ + fileRow(collectionKey, { + id: 7, + title: "keep.jpg", + updationTime: 100, + }), + tombstoneRow(8, 150), + ], + hasMore: false, + }), + ); + const client = Client.fromJSON(snapshotFor(keys), { fetch }); + + const files = await client.listFiles(1, collectionKey); + + expect(files.map((f) => f.id)).toEqual([7]); + // The wrapper starts a full enumeration from zero. + expect(sinceTimes).toEqual(["0"]); + }); +}); diff --git a/test/client/session.test.ts b/test/client/session.test.ts new file mode 100644 index 0000000..7aa2359 --- /dev/null +++ b/test/client/session.test.ts @@ -0,0 +1,197 @@ +/** + * Tests for the client session lifecycle: `toJSON`, `fromJSON`, `logout`, and + * the CLI's `loadSession`, which reads the saved session file back into a + * client. + */ + +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import sodium from "libsodium-wrappers-sumo"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { init, toBase64 } from "../../src/crypto/index.js"; +import { Client, type ClientSnapshot } from "../../src/client.js"; +import { loadSession } from "../../src/cli-session.js"; + +const validSnapshot = (): ClientSnapshot => { + const kp = sodium.crypto_box_keypair(); + return { + email: "user@example.com", + userID: 42, + token: "test-token", + masterKey: toBase64(sodium.crypto_secretbox_keygen()), + secretKey: toBase64(kp.privateKey), + publicKey: toBase64(kp.publicKey), + }; +}; + +// The client's key buffers are private; the tests read them to prove that +// logout wipes them. +const keyBuffers = (client: Client): Uint8Array[] => [ + client["masterKey"], + client["secretKey"], + client["publicKey"], +]; + +beforeAll(async () => { + await init(); +}); + +describe("Client.toJSON", () => { + it("round-trips through fromJSON unchanged", () => { + const snapshot = validSnapshot(); + expect(Client.fromJSON(snapshot).toJSON()).toEqual(snapshot); + }); + + it("throws instead of emitting a snapshot without a token", () => { + const client = Client.fromJSON(validSnapshot()); + client.getApiClient().clearAuthToken(); + expect(() => client.toJSON()).toThrow(/no auth token/); + }); +}); + +describe("Client.fromJSON", () => { + const shortKey = toBase64(new Uint8Array(16)); + + it.each([ + ["email", undefined], + ["email", 7], + ["email", ""], + ["token", undefined], + ["token", null], + ["token", ""], + ["userID", undefined], + ["userID", "42"], + ["userID", 4.2], + ["masterKey", undefined], + ["masterKey", 7], + ["masterKey", "not base64!"], + ["masterKey", shortKey], + ["secretKey", undefined], + ["secretKey", "not base64!"], + ["secretKey", shortKey], + ["publicKey", undefined], + ["publicKey", "not base64!"], + ["publicKey", shortKey], + ])("rejects %s = %j, naming the field", (field, value) => { + const snapshot: Record<string, unknown> = { ...validSnapshot() }; + snapshot[field] = value; + expect(() => Client.fromJSON(snapshot)).toThrow( + new RegExp(`^Invalid session data: ${field} `), + ); + }); + + it.each([null, "a string", 42])("rejects a non-object %j", (value) => { + expect(() => Client.fromJSON(value)).toThrow( + /^Invalid session data: not a JSON object/, + ); + }); +}); + +describe("Client.logout", () => { + it("zeroes the key buffers and clears the token", () => { + const client = Client.fromJSON(validSnapshot()); + const api = client.getApiClient(); + const keys = keyBuffers(client); + + client.logout(); + + for (const key of keys) { + expect(key.length).toBe(32); + expect(key.every((b) => b === 0)).toBe(true); + } + expect(api.getAuthToken()).toBeUndefined(); + }); + + it("makes every later operation throw", async () => { + const client = Client.fromJSON(validSnapshot()); + client.logout(); + + expect(() => client.whoami()).toThrow(/logged out/); + expect(() => client.toJSON()).toThrow(/logged out/); + expect(() => client.getApiClient()).toThrow(/logged out/); + expect(() => client.contentSource()).toThrow(/logged out/); + await expect(client.listCollections()).rejects.toThrow(/logged out/); + await expect(client.collectionsSince({ sinceTime: 0 })).rejects.toThrow( + /logged out/, + ); + await expect( + client.filesSince({ + collectionID: 1, + collectionKey: new Uint8Array(32), + sinceTime: 0, + }), + ).rejects.toThrow(/logged out/); + await expect( + client.fetchMLData({ fileIDs: [1], fileKeys: new Map() }), + ).rejects.toThrow(/logged out/); + }); + + it("stops a listing in flight from decrypting with the zeroed keys", async () => { + // The server answers only after the client has logged out. If the + // listing went on to decrypt this row with all-zero keys it would fail + // with a decryption error, not the logged-out one. + const row = { + id: 1, + owner: { id: 42 }, + encryptedKey: toBase64(new Uint8Array(48)), + keyDecryptionNonce: toBase64(new Uint8Array(24)), + updationTime: 1, + }; + const client: Client = Client.fromJSON(validSnapshot(), { + fetch: async () => { + client.logout(); + return new Response(JSON.stringify({ collections: [row] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }, + }); + + await expect(client.listCollections()).rejects.toThrow(/logged out/); + }); +}); + +describe("loadSession", () => { + let dir: string; + + beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), "quak-session-test-")); + }); + + afterAll(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("returns null when there is no session file", () => { + expect(loadSession(join(dir, "missing.json"))).toBeNull(); + }); + + it("restores a client from a valid session file", () => { + const path = join(dir, "valid.json"); + writeFileSync(path, JSON.stringify(validSnapshot())); + expect(loadSession(path)!.whoami()).toEqual({ + email: "user@example.com", + userID: 42, + }); + }); + + it("says the file is corrupt when it is not JSON", () => { + const path = join(dir, "truncated.json"); + writeFileSync(path, '{"email": "user@exa'); + expect(() => loadSession(path)).toThrow( + `Session file ${path} is corrupt`, + ); + }); + + it("says the file is corrupt and names the bad field", () => { + const path = join(dir, "bad-key.json"); + writeFileSync( + path, + JSON.stringify({ ...validSnapshot(), secretKey: "AAAA" }), + ); + expect(() => loadSession(path)).toThrow( + new RegExp(`^Session file ${path} is corrupt: .*secretKey`), + ); + }); +}); diff --git a/test/crypto/hash.test.ts b/test/crypto/hash.test.ts new file mode 100644 index 0000000..153ec0a --- /dev/null +++ b/test/crypto/hash.test.ts @@ -0,0 +1,33 @@ +import { beforeAll, describe, expect, it } from "vitest"; +import { + chunkHashFinal, + chunkHashInit, + chunkHashUpdate, + init, +} from "../../src/crypto/index.js"; + +beforeAll(async () => { + await init(); +}); + +describe("content hash", () => { + // RFC 7693 Appendix A: BLAKE2b-512 of "abc". + const abc = Buffer.from( + "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1" + + "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923", + "hex", + ).toString("base64"); + + it("is unkeyed BLAKE2b-512 in standard base64", () => { + const state = chunkHashInit(); + chunkHashUpdate(state, new TextEncoder().encode("abc")); + expect(chunkHashFinal(state)).toBe(abc); + }); + + it("gives the same hash when the input arrives in chunks", () => { + const state = chunkHashInit(); + chunkHashUpdate(state, new TextEncoder().encode("a")); + chunkHashUpdate(state, new TextEncoder().encode("bc")); + expect(chunkHashFinal(state)).toBe(abc); + }); +}); diff --git a/test/crypto/kdf.test.ts b/test/crypto/kdf.test.ts index 670d80e..d252653 100644 --- a/test/crypto/kdf.test.ts +++ b/test/crypto/kdf.test.ts @@ -29,10 +29,10 @@ describe("crypto.deriveKEK (Argon2id)", () => { }); /** - * Cheap parameters used so the test suite stays under the 30-second - * budget. The real production parameters Ente uses are larger - * (memLimit up to 1 GiB, opsLimit 3-16). The algorithm is the same - * regardless of parameters. + * Cheap parameters used so the test suite stays under the 90-second + * `timeout` in the `test` phase of the `Dockerfile`. The real production + * parameters Ente uses are larger (memLimit up to 1 GiB, opsLimit 3-16). + * The algorithm is the same regardless of parameters. */ const TEST_OPS = 2; const TEST_MEM = 64 * 1024 * 1024; // 64 MiB diff --git a/test/download/download.test.ts b/test/download/download.test.ts index 0608381..e08a948 100644 --- a/test/download/download.test.ts +++ b/test/download/download.test.ts @@ -48,17 +48,21 @@ */ import { + chmodSync, existsSync, + mkdirSync, readdirSync, readFileSync, rmSync, mkdtempSync, + statSync, writeFileSync, } from "node:fs"; import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; import { createHash } from "node:crypto"; import sodium from "libsodium-wrappers-sumo"; +import { zipSync } from "fflate"; import { beforeAll, beforeEach, @@ -72,8 +76,13 @@ import { init, toBase64, STREAM_CHUNK_SIZE } from "../../src/crypto/index.js"; import { ApiClient } from "../../src/api/client.js"; import { ApiError, TruncatedStreamError } from "../../src/errors.js"; import type { RetryOptions } from "../../src/retry.js"; -import { downloadFile, downloadThumbnail } from "../../src/download/index.js"; +import { + downloadFile, + downloadThumbnail, + writeAtomic, +} from "../../src/download/index.js"; import type { EnteFile, FileMetadata } from "../../src/model/types.js"; +import { IMAGE, livePhotoHash, livePhotoZip, VIDEO } from "../live-photo.js"; // --------------------------------------------------------------------------- // Test helpers @@ -99,6 +108,36 @@ import type { EnteFile, FileMetadata } from "../../src/model/types.js"; const renameHook = vi.hoisted(() => ({ calls: [] as { from: string; to: string; sourceExisted: boolean }[], failWith: null as Error | null, + // When set, only a rename to this path fails. + failTo: null as string | null, +})); + +/** + * `open` is wrapped so the tests can observe the durability fsyncs the atomic + * writer performs — which are otherwise invisible: an fsync leaves no trace in + * the file's contents. Each `FileHandle.sync()` is recorded, and rename and + * sync events are appended to a single ordered `events` log so a test can pin + * the sequence "fsync the temp file, rename, fsync the directory" that makes a + * write survive a power cut. The flag the handle was opened with distinguishes + * the temp file (`w`) from its containing directory (`r`). + */ +const durabilityHook = vi.hoisted(() => ({ + events: [] as string[], +})); + +/** + * `FileHandle.write` is wrapped so the tests can watch the streaming decrypt + * path put plaintext on disk one chunk at a time. This is the direct evidence + * that memory is bounded by the chunk size and not the file size: a buffered + * downloader would hand the whole file to a single write, whereas the streaming + * one issues one write per secretstream chunk, none larger than + * `STREAM_CHUNK_SIZE`. Each write records the temp path it targeted and its + * length. `writeFile` (which the whole-buffer `writeAtomic` uses) is a distinct + * native call and does not go through this method, so only the streaming path + * is observed here. + */ +const writeHook = vi.hoisted(() => ({ + writes: [] as { path: string; length: number }[], })); vi.mock("node:fs/promises", async (importOriginal) => { @@ -106,13 +145,50 @@ vi.mock("node:fs/promises", async (importOriginal) => { const { existsSync: sourceExists } = await import("node:fs"); return { ...actual, + open: async ( + path: Parameters<typeof actual.open>[0], + flags?: Parameters<typeof actual.open>[1], + ...rest: unknown[] + ): Promise<Awaited<ReturnType<typeof actual.open>>> => { + const handle = await actual.open( + path, + flags as Parameters<typeof actual.open>[1], + ...(rest as []), + ); + const realSync = handle.sync.bind(handle); + handle.sync = async (): Promise<void> => { + durabilityHook.events.push(`sync:${String(flags)}:${path}`); + await realSync(); + }; + const realWrite = handle.write.bind(handle); + handle.write = (async ( + data: unknown, + ...rest2: unknown[] + ): Promise<unknown> => { + if (data instanceof Uint8Array) { + writeHook.writes.push({ + path: String(path), + length: data.length, + }); + } + return (realWrite as (...a: unknown[]) => Promise<unknown>)( + data, + ...rest2, + ); + }) as typeof handle.write; + return handle; + }, rename: async (from: string, to: string): Promise<void> => { renameHook.calls.push({ from, to, sourceExisted: sourceExists(from), }); - if (renameHook.failWith !== null) { + durabilityHook.events.push(`rename:${to}`); + if ( + renameHook.failWith !== null && + (renameHook.failTo === null || renameHook.failTo === to) + ) { throw renameHook.failWith; } await actual.rename(from, to); @@ -120,9 +196,36 @@ vi.mock("node:fs/promises", async (importOriginal) => { }; }); +/** + * `pullStreamChunk` is wrapped to note, as each chunk is decrypted, how many + * writes `writeHook` has seen by then, so a test can tell what reached disk + * before the last chunk was decrypted. + */ +const pullHook = vi.hoisted(() => ({ + writesSeen: [] as number[], +})); + +vi.mock("../../src/crypto/index.js", async (importOriginal) => { + const actual = + await importOriginal<typeof import("../../src/crypto/index.js")>(); + return { + ...actual, + pullStreamChunk: ( + ...args: Parameters<typeof actual.pullStreamChunk> + ): ReturnType<typeof actual.pullStreamChunk> => { + pullHook.writesSeen.push(writeHook.writes.length); + return actual.pullStreamChunk(...args); + }, + }; +}); + beforeEach(() => { + pullHook.writesSeen.length = 0; renameHook.calls.length = 0; renameHook.failWith = null; + renameHook.failTo = null; + durabilityHook.events.length = 0; + writeHook.writes.length = 0; }); let testDir: string; @@ -153,7 +256,8 @@ afterAll(() => { * `sodium.randombytes_buf` goes through the wasm wrapper a byte at a time and * costs roughly 20 seconds for the 4 MiB chunk below — about two hundred * times what it costs to encrypt the same buffer, and on its own enough to - * push `make test` past the 30-second cap in `script/test`. This loop fills + * push `make test` past the 90-second `timeout` in the `test` phase of the + * `Dockerfile`. This loop fills * 4 MiB in a few milliseconds. */ const patternBytes = (length: number, seed: number): Uint8Array => { @@ -187,6 +291,34 @@ const encryptFileBody = ( return { header: push.header, ciphertext }; }; +/** + * Encrypt `plaintext` the way the server does a file of any size: one + * secretstream chunk per `STREAM_CHUNK_SIZE` bytes, the last tagged TAG_FINAL. + */ +const encryptInChunks = ( + plaintext: Uint8Array, + key: Uint8Array, +): { header: Uint8Array; ciphertext: Uint8Array } => { + const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(key); + const chunks: Uint8Array[] = []; + let offset = 0; + do { + const piece = plaintext.subarray(offset, offset + STREAM_CHUNK_SIZE); + offset += STREAM_CHUNK_SIZE; + chunks.push( + sodium.crypto_secretstream_xchacha20poly1305_push( + push.state, + piece, + null, + offset >= plaintext.length + ? sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL + : sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE, + ), + ); + } while (offset < plaintext.length); + return { header: push.header, ciphertext: concat(chunks) }; +}; + /** * Encrypt a body that spans more than one secretstream chunk, the way the * server does for files larger than the 4 MiB plaintext chunk size. @@ -433,6 +565,22 @@ const entryPoints = [ { name: "downloadThumbnail", download: downloadThumbnail }, ]; +// With no `outPath`, the destination is named after `metadata.title`, relative +// to the working directory. Such tests run inside a temporary directory: +// `make check` must not create files in the repo root. +const inDirectory = async <T>( + dir: string, + run: () => Promise<T>, +): Promise<T> => { + const previous = process.cwd(); + process.chdir(dir); + try { + return await run(); + } finally { + process.chdir(previous); + } +}; + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -468,26 +616,59 @@ describe("downloadFile", () => { }); it("uses metadata.title as filename when outPath is omitted", async () => { - // With no `outPath`, the destination is `metadata.title`, used - // verbatim as a path. The title here is therefore given inside the - // test's temporary directory: a bare relative name would resolve - // against the process working directory, i.e. the repo root, and - // `make check` must not create files in the repo — a failure between - // the write and any cleanup would leave one behind. const plaintext = new Uint8Array([1, 2, 3]); const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); const { header, ciphertext } = encryptFileBody(plaintext, key); const thumbPush = sodium.crypto_secretstream_xchacha20poly1305_init_push(key); const file = buildMockEnteFile(key, header, thumbPush.header); - const titlePath = join(testDir, "fallback-name.png"); - file.metadata.title = titlePath; + file.metadata.title = "fallback-name.png"; + const dir = mkdtempSync(join(testDir, "title-")); const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) }); - const result = await downloadFile(api, file); + const result = await inDirectory(dir, () => downloadFile(api, file)); - expect(result.path).toBe(titlePath); - expect(readFileSync(result.path)).toEqual(Buffer.from(plaintext)); + expect(result.path).toBe("fallback-name.png"); + expect(readFileSync(join(dir, "fallback-name.png"))).toEqual( + Buffer.from(plaintext), + ); + }); + + it("keeps a hostile title inside the working directory", async () => { + // The server controls the title. `../escaped.png` must not write to + // the parent directory; it becomes one file name in the current one. + const { api, file } = fixtureFor( + multiChunkKey, + multiChunk.header, + multiChunk.body, + ); + file.metadata.title = "../escaped.png"; + const parent = mkdtempSync(join(testDir, "hostile-")); + const dir = join(parent, "cwd"); + mkdirSync(dir); + + const result = await inDirectory(dir, () => downloadFile(api, file)); + + expect(result.path).toBe("__escaped.png"); + expect(readdirSync(dir)).toEqual(["__escaped.png"]); + expect(readdirSync(parent)).toEqual(["cwd"]); + }); + + it("uses an explicit outPath verbatim, even one with ..", async () => { + // The caller is trusted: its path is not sanitized. + const { api, file } = fixtureFor( + multiChunkKey, + multiChunk.header, + multiChunk.body, + ); + const dir = mkdtempSync(join(testDir, "explicit-")); + mkdirSync(join(dir, "sub")); + const outPath = join(dir, "sub", "..", "explicit.bin"); + + const result = await downloadFile(api, file, outPath); + + expect(result.path).toBe(outPath); + expect(existsSync(join(dir, "explicit.bin"))).toBe(true); }); it("handles a larger single-chunk file (random binary payload)", async () => { @@ -549,6 +730,23 @@ describe("downloadThumbnail", () => { expect(result).toEqual({ path: outPath, bytesWritten: 4 }); expect(readFileSync(outPath)).toEqual(Buffer.from(plaintext)); }); + + it("names the thumbnail thumb_ plus the sanitized title", async () => { + const { api, file } = fixtureFor( + multiChunkKey, + multiChunk.header, + multiChunk.body, + ); + file.metadata.title = "/etc/passwd"; + const dir = mkdtempSync(join(testDir, "thumb-title-")); + + const result = await inDirectory(dir, () => + downloadThumbnail(api, file), + ); + + expect(result.path).toBe("thumb__etc_passwd"); + expect(readdirSync(dir)).toEqual(["thumb__etc_passwd"]); + }); }); // --------------------------------------------------------------------------- @@ -854,6 +1052,47 @@ describe.each(entryPoints)( expect(readFileSync(outPath)).toEqual(Buffer.from(existing)); expect(readdirSync(dir)).toEqual(["rename-fails.bin"]); }); + + it("fails without creating anything when the destination directory does not exist", async () => { + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptFileBody( + patternBytes(64, 33), + key, + ); + const { api, file } = fixtureFor(key, header, ciphertext); + const dir = freshDir(); + const outPath = join(dir, "missing", "never.bin"); + + await expect(download(api, file, outPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + + // The missing directory is not created on the caller's behalf. + expect(readdirSync(dir)).toEqual([]); + }); + + // Root ignores directory permissions, so this fails when run as root. + // The `test` phase of the `Dockerfile` runs as the `node` user. + it("fails without creating anything when the destination directory is not writable", async () => { + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptFileBody( + patternBytes(64, 34), + key, + ); + const { api, file } = fixtureFor(key, header, ciphertext); + const dir = freshDir(); + const outPath = join(dir, "never.bin"); + chmodSync(dir, 0o500); + try { + await expect( + download(api, file, outPath), + ).rejects.toMatchObject({ code: "EACCES" }); + } finally { + chmodSync(dir, 0o700); + } + + expect(readdirSync(dir)).toEqual([]); + }); }, ); @@ -911,10 +1150,13 @@ describe.each(entryPoints)("$name retries", ({ name, download }) => { }); it("stages one temp file for the attempt that succeeded, not one per attempt", async () => { - // The atomic write stays outside the retry loop. A retried download - // must not leave a trail of half-written scratch files, and the - // destination must be touched exactly once — by the attempt that - // produced a complete, authenticated plaintext. + // Each streaming attempt stages into its own temp file, but a retried + // download must not leave a trail of half-written scratch files: a + // failed attempt removes its temp file, and the destination is renamed + // into place exactly once — by the attempt that produced a complete, + // authenticated plaintext. (Here the two failed attempts reset before a + // whole chunk is pulled, so they write nothing; the point stands either + // way — see the retry-restart test below, where they do write.) const { key, header, ciphertext } = smallFixture(42); const { fetch } = scriptedCdnFetch( { kind: "reset", bytes: ciphertext.slice(0, 16) }, @@ -1028,6 +1270,99 @@ describe.each(entryPoints)("$name retries", ({ name, download }) => { }); }); +// --------------------------------------------------------------------------- +// Streaming decrypt to disk +// +// The plaintext is never held whole in memory: each secretstream chunk is +// written to the temp file as it is decrypted, so peak memory is bounded by the +// chunk size rather than the file size. These tests watch the writes directly +// (see `writeHook`) rather than infer memory behaviour from the final file. +// --------------------------------------------------------------------------- + +describe.each(entryPoints)("$name streams to disk", ({ name, download }) => { + const freshDir = (): string => + mkdtempSync(join(testDir, `${name}-stream-`)); + + /** Writes recorded against staged temp files (not the `writeFile` path). */ + const tempWrites = (): { path: string; length: number }[] => + writeHook.writes.filter((w) => w.path.endsWith(".tmp")); + + it("writes one chunk at a time, none larger than STREAM_CHUNK_SIZE", async () => { + // The multi-chunk fixture decrypts to one full 4 MiB chunk plus a small + // final chunk. A streaming writer therefore issues exactly two writes, + // of STREAM_CHUNK_SIZE and then the final chunk's length — never a + // single write carrying the whole 4 MiB + 1 KiB file. That per-chunk + // shape is what "memory bounded by chunk size" means in practice: the + // plaintext is handed to the filesystem and dropped, chunk by chunk. + const { api, file } = fixtureFor( + multiChunkKey, + multiChunk.header, + multiChunk.body, + ); + const outPath = join(freshDir(), "streamed.bin"); + + const result = await download(api, file, outPath); + + const writes = tempWrites(); + expect(writes.map((w) => w.length)).toEqual([ + STREAM_CHUNK_SIZE, + multiChunk.plaintext.length - STREAM_CHUNK_SIZE, + ]); + // No single write ever carried the whole file, and every write fits in + // one chunk's worth of memory. + for (const w of writes) { + expect(w.length).toBeLessThanOrEqual(STREAM_CHUNK_SIZE); + } + expect(result.bytesWritten).toBe(multiChunk.plaintext.length); + expectSameBytes(readFileSync(outPath), multiChunk.plaintext); + }); + + it("restarts from byte zero on a retry, replacing the temp file cleanly", async () => { + // The secretstream pull state is not resumable, so a retry cannot + // continue a half-written file — it must start over. The first attempt + // here delivers a complete leading chunk and then stops before the + // TAG_FINAL chunk: 4 MiB of plaintext lands in a temp file, then the + // download is rejected as truncated and that temp file is discarded. + // The retry streams the whole body into a *fresh* temp file, so the + // destination ends up with exactly the plaintext once — never the + // leading chunk twice, and never a stale temp file left behind. + const truncatedBody = multiChunk.body.slice( + 0, + multiChunk.finalChunkOffset, + ); + const { fetch, requests } = scriptedCdnFetch( + { kind: "body", bytes: truncatedBody }, + { kind: "body", bytes: multiChunk.body }, + ); + const api = new ApiClient({ + fetch, + retry: { ...noWait, attempts: 4 }, + }); + const file = buildMockEnteFile( + multiChunkKey, + multiChunk.header, + multiChunk.header, + ); + const dir = freshDir(); + const outPath = join(dir, "retry-restart.bin"); + + const result = await download(api, file, outPath); + + expect(requests()).toBe(2); + // Both attempts streamed to disk, each into its own temp file: the + // truncated first attempt wrote before it failed, proving the retry did + // not resume a partial file but replaced it. + const distinctTemps = new Set(tempWrites().map((w) => w.path)); + expect(distinctTemps.size).toBe(2); + // The destination holds the complete plaintext exactly once, and no + // temp file survives. + expect(result.bytesWritten).toBe(multiChunk.plaintext.length); + expectSameBytes(readFileSync(outPath), multiChunk.plaintext); + expect(renameHook.calls).toHaveLength(1); + expect(readdirSync(dir)).toEqual(["retry-restart.bin"]); + }); +}); + describe("download retries: corruption is not retried", () => { it("gives up immediately on a chunk that failed to authenticate", async () => { // A whole chunk that failed to authenticate while the stream @@ -1060,4 +1395,603 @@ describe("download retries: corruption is not retried", () => { expect(requests()).toBe(1); }); + + it("cancels the response body when decryption fails", async () => { + // A backup run carries on past a failed file, so a body left open on + // failure would hold its connection until garbage collection, once + // per failed file. This body delivers a corrupt chunk and then stays + // open, so only a cancel from the downloader can close it. + const corrupted = Uint8Array.from(multiChunk.body); + corrupted[10] ^= 0xff; + let cancelled = false; + const fetch = (async () => + new Response( + new ReadableStream<Uint8Array>({ + start(controller) { + controller.enqueue(corrupted); + }, + cancel() { + cancelled = true; + }, + }), + { status: 200 }, + )) as typeof globalThis.fetch; + const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 1 } }); + const file = buildMockEnteFile( + multiChunkKey, + multiChunk.header, + multiChunk.header, + ); + const outPath = join(mkdtempSync(join(testDir, "cancel-")), "c.bin"); + + await expect(downloadFile(api, file, outPath)).rejects.toThrow( + /authentication failed/i, + ); + + expect(cancelled).toBe(true); + }); + + it("cancels the response body when the temp file cannot be opened", async () => { + // The download fails before a byte of the body is read, so the body + // is still open and only a cancel from the downloader can close it. + let cancelled = false; + const fetch = (async () => + new Response( + new ReadableStream<Uint8Array>({ + start(controller) { + controller.enqueue(multiChunk.body); + }, + cancel() { + cancelled = true; + }, + }), + { status: 200 }, + )) as typeof globalThis.fetch; + const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 1 } }); + const file = buildMockEnteFile( + multiChunkKey, + multiChunk.header, + multiChunk.header, + ); + const outPath = join( + mkdtempSync(join(testDir, "cancel-")), + "missing", + "c.bin", + ); + + await expect(downloadFile(api, file, outPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + + expect(cancelled).toBe(true); + }); + + it("cancels the response body when the header is malformed", async () => { + // The header is rejected before the body is read, so the body is + // still open and only a cancel from the downloader can close it. + let cancelled = false; + const fetch = (async () => + new Response( + new ReadableStream<Uint8Array>({ + start(controller) { + controller.enqueue(multiChunk.body); + }, + cancel() { + cancelled = true; + }, + }), + { status: 200 }, + )) as typeof globalThis.fetch; + const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 1 } }); + const shortHeader = multiChunk.header.subarray(0, 5); + const file = buildMockEnteFile(multiChunkKey, shortHeader, shortHeader); + const outPath = join(mkdtempSync(join(testDir, "cancel-")), "c.bin"); + + await expect(downloadFile(api, file, outPath)).rejects.toThrow(); + + expect(cancelled).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Fragmented network reads +// +// A CDN does not hand the body over one secretstream chunk at a time; it +// arrives in whatever pieces the socket produces, many of them far smaller than +// a chunk and most straddling a chunk boundary. `streamDecrypt` reassembles +// those pieces before decrypting, copying each received byte once rather than +// recopying the whole accumulator on every read. This is the path the other +// fixtures never take — their mock fetch delivers each body as a single +// `Response` value, i.e. one read — so it is exercised explicitly here. +// --------------------------------------------------------------------------- + +/** + * A fetch that serves `body` through a `ReadableStream` sliced into many + * fixed-size pieces, imitating a socket that trickles bytes in. `pieceSize` is + * chosen not to divide the chunk framing evenly, so pieces straddle the + * `ENC_CHUNK_SIZE` boundary the downloader splits on — the case a single-value + * body can never produce. `emitted` reports how many pieces were yielded, so a + * test can assert the body really was fragmented and not delivered whole. + */ +const mockFetchForFragmentedBody = ( + body: Uint8Array, + pieceSize: number, +): { fetch: typeof globalThis.fetch; emitted: () => number } => { + let pieces = 0; + const fake = async (): Promise<Response> => + new Response( + new ReadableStream<Uint8Array>({ + start(controller) { + for (let off = 0; off < body.length; off += pieceSize) { + controller.enqueue(body.subarray(off, off + pieceSize)); + pieces++; + } + controller.close(); + }, + }), + { status: 200 }, + ); + return { fetch: fake as typeof globalThis.fetch, emitted: () => pieces }; +}; + +describe("streamDecrypt fragmented reads", () => { + it("decrypts a multi-chunk body delivered in many small pieces", async () => { + // The multi-chunk fixture (one full 4 MiB chunk plus a small final + // chunk) delivered in 1000-byte pieces: several thousand reads, with + // the piece that spans the 4 MiB + 17 byte chunk boundary split across + // two chunks by the reassembler. The plaintext must come out + // byte-identical to the single-read case, and the chunk framing must be + // untouched: exactly two writes, `STREAM_CHUNK_SIZE` then the final + // chunk, the same as when the body arrives whole. If the boundary + // handling were off by a byte under fragmentation, either the pull + // would fail to authenticate or the write sizes would shift. + const { fetch, emitted } = mockFetchForFragmentedBody( + multiChunk.body, + 1000, + ); + const api = new ApiClient({ fetch }); + const file = buildMockEnteFile( + multiChunkKey, + multiChunk.header, + multiChunk.header, + ); + const dir = mkdtempSync(join(testDir, "fragmented-")); + const outPath = join(dir, "fragmented.bin"); + + const result = await downloadFile(api, file, outPath); + + // The body really was trickled in, not handed over whole. + expect(emitted()).toBeGreaterThan(1000); + + const writes = writeHook.writes.filter((w) => w.path.endsWith(".tmp")); + expect(writes.map((w) => w.length)).toEqual([ + STREAM_CHUNK_SIZE, + multiChunk.plaintext.length - STREAM_CHUNK_SIZE, + ]); + expect(result.bytesWritten).toBe(multiChunk.plaintext.length); + expectSameBytes(readFileSync(outPath), multiChunk.plaintext); + }); +}); + +// --------------------------------------------------------------------------- +// Durable atomic writes +// +// `writeAtomic` is exported so the metadata store can reuse the same +// power-cut-safe write. Its durability is the point: the bytes and the new +// directory entry must both be on stable storage before it returns, so a crash +// immediately afterwards cannot resurrect an empty renamed file (#22 area 1). +// --------------------------------------------------------------------------- + +describe("writeAtomic", () => { + it("fsyncs the temp file before the rename and the directory after", async () => { + const dir = mkdtempSync(join(testDir, "atomic-")); + const dest = join(dir, "durable.bin"); + const bytes = patternBytes(2048, 71); + + await writeAtomic(dest, bytes); + + expect(readFileSync(dest)).toEqual(Buffer.from(bytes)); + // The order is the durability contract: fsync the staged temp file so + // its contents are on disk, rename it into place, then fsync the + // directory so that new entry is on disk too. Do the directory fsync + // before the rename, or skip it, and a crash can lose the rename. + expect(durabilityHook.events).toHaveLength(3); + // The temp name carries this process's ID, so a library opening the + // same cache can tell a write in progress from a leftover. + const tempSync = durabilityHook.events[0]!; + expect(tempSync.startsWith(`sync:w:${dir}/`)).toBe(true); + expect(tempSync.slice(`sync:w:${dir}/`.length)).toMatch( + new RegExp(`^\\.quak-${process.pid}-[0-9a-f]{32}\\.tmp$`), + ); + expect(durabilityHook.events[1]).toBe(`rename:${dest}`); + expect(durabilityHook.events[2]).toBe(`sync:r:${dir}`); + }); + + it("leaves no temp file behind when the write cannot be renamed", async () => { + const dir = mkdtempSync(join(testDir, "atomic-fail-")); + const dest = join(dir, "unrenamable.bin"); + renameHook.failWith = new Error("simulated rename failure"); + + await expect(writeAtomic(dest, patternBytes(64, 72))).rejects.toThrow( + "simulated rename failure", + ); + + // The staged temp file was fsynced, then the rename failed; the cleanup + // path must remove it so a repeatedly failing write cannot fill the disk. + expect(existsSync(dest)).toBe(false); + expect(readdirSync(dir)).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Per-chunk progress +// +// Callers streaming a large file want bytes-written as it lands, not only the +// final total. The hook fires as decrypted plaintext accumulates; its values +// are non-decreasing and its last value is exactly `bytesWritten`. +// --------------------------------------------------------------------------- + +describe.each(entryPoints)("$name progress", ({ name, download }) => { + it("reports monotonic progress ending at bytesWritten", async () => { + // The multi-chunk fixture pulls one full 4 MiB chunk and then a small + // final chunk, so the callback fires more than once and monotonicity is + // actually observable rather than trivially true for a single fire. + const { api, file } = fixtureFor( + multiChunkKey, + multiChunk.header, + multiChunk.body, + ); + const outPath = join( + mkdtempSync(join(testDir, `${name}-progress-`)), + "p.bin", + ); + const seen: number[] = []; + + const result = await download(api, file, outPath, (bytesDone) => { + seen.push(bytesDone); + }); + + expect(seen.length).toBeGreaterThan(1); + for (let i = 1; i < seen.length; i++) { + expect(seen[i]!).toBeGreaterThan(seen[i - 1]!); + } + expect(seen[seen.length - 1]).toBe(result.bytesWritten); + expect(result.bytesWritten).toBe(multiChunk.plaintext.length); + }); + + it("downloads normally when no progress callback is given", async () => { + // The callback is optional and its absence must be side-effect-free: + // the download succeeds exactly as it does elsewhere in this file. + const plaintext = patternBytes(300, 73); + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptFileBody(plaintext, key); + const { api, file } = fixtureFor(key, header, ciphertext); + const outPath = join( + mkdtempSync(join(testDir, `${name}-noprog-`)), + "n.bin", + ); + + const result = await download(api, file, outPath); + + expect(result.bytesWritten).toBe(plaintext.length); + expectSameBytes(readFileSync(outPath), plaintext); + }); +}); + +// Node's own BLAKE2b-512 is the reference, so the tests below do not depend on +// the code under test to compute what they expect. +const blake2b = (bytes: Uint8Array): string => + createHash("blake2b512").update(bytes).digest("base64"); + +// Serve `plaintext` encrypted as file 999 with the given metadata, to be +// written to `f.bin` in a fresh directory. Four responses are scripted so a +// retried failure would show in `requests`. +const setup = (plaintext: Uint8Array, metadata: Partial<FileMetadata>) => { + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptInChunks(plaintext, key); + const file = buildMockEnteFile(key, header, header); + file.metadata = { ...file.metadata, ...metadata }; + const body = { kind: "body", bytes: ciphertext } as const; + const { fetch, requests } = scriptedCdnFetch(body, body, body, body); + const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } }); + const dir = mkdtempSync(join(testDir, "hash-")); + const outPath = join(dir, "f.bin"); + return { + run: () => downloadFile(api, file, outPath), + api, + file, + dir, + outPath, + requests, + }; +}; + +describe("downloadFile content hash", () => { + it("stores a file whose hash matches", async () => { + const plaintext = patternBytes(700, 80); + const t = setup(plaintext, { hash: blake2b(plaintext) }); + + await t.run(); + + expectSameBytes(readFileSync(t.outPath), plaintext); + }); + + it("rejects a mismatch, stores nothing, names the file and does not retry", async () => { + const t = setup(patternBytes(700, 80), { + hash: blake2b(patternBytes(700, 79)), + }); + + await expect(t.run()).rejects.toThrow( + /file 999: content hash .* does not match/, + ); + + expect(readdirSync(t.dir)).toEqual([]); + expect(t.requests()).toBe(1); + }); + + it("stores a file with no recorded hash unchecked", async () => { + const plaintext = patternBytes(700, 80); + const t = setup(plaintext, { hash: undefined }); + + await t.run(); + + expectSameBytes(readFileSync(t.outPath), plaintext); + }); + + it("stores a live photo whose image and video hashes match", async () => { + const t = setup(livePhotoZip(), { + fileType: "livePhoto", + hash: livePhotoHash(), + }); + + await t.run(); + + expect(readFileSync(join(t.dir, "f.heic"))).toEqual(Buffer.from(IMAGE)); + expect(readFileSync(join(t.dir, "f.mov"))).toEqual(Buffer.from(VIDEO)); + }); + + it("writes a live photo's image as it decompresses, before the last chunk is decrypted", async () => { + // The image, 12 MiB of zeros, deflates to a few kilobytes at the + // start of the ZIP. The video, stored as it is, carries the ZIP into + // a second chunk. + const image = new Uint8Array(12 * 1024 * 1024); + const video = patternBytes(STREAM_CHUNK_SIZE, 83); + const zip = zipSync({ + "image.heic": image, + "video.mov": [video, { level: 0 }], + }); + const t = setup(zip, { + fileType: "livePhoto", + hash: livePhotoHash(image, video), + }); + + await t.run(); + + // All of the image was written before the second chunk was decrypted, + // and no write held more than two chunks' worth. + expect(pullHook.writesSeen).toHaveLength(2); + const imageTemp = renameHook.calls.find( + (c) => c.to === join(t.dir, "f.heic"), + )!.from; + const imageWrittenBeforeLastChunk = writeHook.writes + .slice(0, pullHook.writesSeen[1]) + .filter((w) => w.path === imageTemp) + .reduce((n, w) => n + w.length, 0); + expect(imageWrittenBeforeLastChunk).toBe(image.length); + expect( + Math.max(...writeHook.writes.map((w) => w.length)), + ).toBeLessThanOrEqual(2 * STREAM_CHUNK_SIZE); + expect(statSync(join(t.dir, "f.heic")).size).toBe(image.length); + expectSameBytes(readFileSync(join(t.dir, "f.mov")), video); + }); + + it("rejects a live photo whose hash does not match, keeping what was there", async () => { + // The whole ZIP's hash is not the recorded one: each part is hashed. + const zip = livePhotoZip(); + const t = setup(zip, { fileType: "livePhoto", hash: blake2b(zip) }); + writeFileSync(t.outPath, "an earlier download"); + + await expect(t.run()).rejects.toThrow( + /file 999: content hash .* does not match/, + ); + + expect(readdirSync(t.dir)).toEqual(["f.bin"]); + expect(readFileSync(t.outPath, "utf-8")).toBe("an earlier download"); + }); + + it("rejects a live photo that is not a readable ZIP and does not retry", async () => { + // Bytes 8-9 of a ZIP entry's local header name its compression + // method; 99 is one no reader knows, so the entry cannot be read. + const zip = livePhotoZip(); + zip[8] = 99; + zip[9] = 0; + const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() }); + + await expect(t.run()).rejects.toThrow( + /file 999: live photo is not a readable ZIP/, + ); + + expect(readdirSync(t.dir)).toEqual([]); + expect(t.requests()).toBe(1); + }); + + it("rejects a live photo ZIP with no image entry", async () => { + const zip = livePhotoZip({ "video.mov": VIDEO }); + const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() }); + + await expect(t.run()).rejects.toThrow( + /file 999: live photo ZIP does not hold both an image and a video/, + ); + + expect(readdirSync(t.dir)).toEqual([]); + }); + + it("rejects a live photo ZIP with no video entry", async () => { + const zip = livePhotoZip({ "image.heic": IMAGE }); + const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() }); + + await expect(t.run()).rejects.toThrow( + /file 999: live photo ZIP does not hold both an image and a video/, + ); + + expect(readdirSync(t.dir)).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Live photos +// +// A live photo arrives as a ZIP of its image and its video. It is written as +// those two files, which a photo viewer can open, each named after the +// destination with its own extension from the ZIP, the way Ente's clients name +// them when they save one. +// --------------------------------------------------------------------------- + +describe("downloadFile live photos", () => { + const livePhoto = { fileType: "livePhoto", hash: livePhotoHash() } as const; + + it("names the image and the video after the title when no outPath is given", async () => { + const t = setup(livePhotoZip(), { + ...livePhoto, + title: "IMG_1234.HEIC", + }); + + const result = await inDirectory(t.dir, () => + downloadFile(t.api, t.file), + ); + + // `bytesWritten` is the length of the decrypted ZIP. + expect(result).toEqual({ + path: "IMG_1234.heic", + videoPath: "IMG_1234.mov", + bytesWritten: livePhotoZip().length, + }); + expect(readdirSync(t.dir).sort()).toEqual([ + "IMG_1234.heic", + "IMG_1234.mov", + ]); + }); + + it("gives each part its own extension from the ZIP, letters and digits only", async () => { + const zip = livePhotoZip({ "image.JPG": IMAGE, "video.m-4v": VIDEO }); + const t = setup(zip, livePhoto); + + const result = await downloadFile(t.api, t.file, join(t.dir, "f.HEIC")); + + expect(result.path).toBe(join(t.dir, "f.JPG")); + expect(result.videoPath).toBe(join(t.dir, "f.bin")); + expect(readdirSync(t.dir).sort()).toEqual(["f.JPG", "f.bin"]); + }); + + it("replaces what was at the destination, such as an earlier ZIP of the two", async () => { + const t = setup(livePhotoZip(), livePhoto); + writeFileSync(t.outPath, livePhotoZip()); + + await t.run(); + + expect(readdirSync(t.dir).sort()).toEqual(["f.heic", "f.mov"]); + }); + + it("renames the image and then the video into place, each from its own temp file", async () => { + const t = setup(livePhotoZip(), livePhoto); + + await t.run(); + + expect( + renameHook.calls.map((c) => [ + dirname(c.from), + c.to, + c.sourceExisted, + ]), + ).toEqual([ + [t.dir, join(t.dir, "f.heic"), true], + [t.dir, join(t.dir, "f.mov"), true], + ]); + }); + + it("fsyncs each part's temp file before the renames and the directory after", async () => { + const t = setup(livePhotoZip(), livePhoto); + + await t.run(); + + // The same durability contract as `writeAtomic`, for two files: both + // temp files are on disk before either is renamed into place, and the + // directory is fsynced once both renames are done. + const [imageTmp, videoTmp] = renameHook.calls.map((c) => c.from); + expect(durabilityHook.events).toEqual([ + `sync:w:${imageTmp}`, + `sync:w:${videoTmp}`, + `rename:${join(t.dir, "f.heic")}`, + `rename:${join(t.dir, "f.mov")}`, + `sync:r:${t.dir}`, + ]); + }); + + it("stores neither part when the video cannot be renamed into place", async () => { + const t = setup(livePhotoZip(), livePhoto); + renameHook.failWith = new Error("simulated rename failure"); + renameHook.failTo = join(t.dir, "f.mov"); + + await expect(t.run()).rejects.toThrow("simulated rename failure"); + + expect(readdirSync(t.dir)).toEqual([]); + }); + + it("refuses an image and a video with the same extension, storing nothing", async () => { + // On a file system that ignores case, the two would be one file. + const zip = livePhotoZip({ "image.mov": IMAGE, "video.MOV": VIDEO }); + const t = setup(zip, livePhoto); + writeFileSync(t.outPath, "an earlier download"); + + await expect(t.run()).rejects.toThrow( + /file 999: live photo's image and video have the same extension/, + ); + + expect(readdirSync(t.dir)).toEqual(["f.bin"]); + expect(t.requests()).toBe(1); + }); + + it.each([ + ["image", "image.jpg"], + ["video", "video.mp4"], + ])( + "refuses a live photo ZIP holding a second %s, storing nothing", + async (kind, second) => { + const zip = livePhotoZip({ + "image.heic": IMAGE, + "video.mov": VIDEO, + [second]: IMAGE, + }); + const t = setup(zip, livePhoto); + writeFileSync(t.outPath, "an earlier download"); + + await expect(t.run()).rejects.toThrow( + `file 999: live photo ZIP holds more than one ${kind}`, + ); + + expect(readdirSync(t.dir)).toEqual(["f.bin"]); + expect(t.requests()).toBe(1); + }, + ); + + it("stops unpacking once the parts pass 20 times the ZIP's size plus 16 MiB, storing nothing", async () => { + // 17 MiB of zeros deflates to a few kilobytes, so its ZIP may expand + // to little more than 16 MiB. + const image = new Uint8Array(17 * 1024 * 1024); + const zip = livePhotoZip({ "image.heic": image, "video.mov": VIDEO }); + const t = setup(zip, { + fileType: "livePhoto", + hash: livePhotoHash(image, VIDEO), + }); + writeFileSync(t.outPath, "an earlier download"); + + await expect(t.run()).rejects.toThrow( + "file 999: live photo ZIP expands to more than 20 times its size plus 16 MiB", + ); + + const written = writeHook.writes.reduce((n, w) => n + w.length, 0); + expect(written).toBeLessThanOrEqual(20 * zip.length + 16 * 1024 * 1024); + expect(readdirSync(t.dir)).toEqual(["f.bin"]); + expect(t.requests()).toBe(1); + }); }); diff --git a/test/filename/filename.test.ts b/test/filename/filename.test.ts new file mode 100644 index 0000000..81e7c56 --- /dev/null +++ b/test/filename/filename.test.ts @@ -0,0 +1,93 @@ +// File names built from server-supplied metadata. +// +// quak does not trust the server. A file's title and a collection's name are +// decrypted from data the server hands us, and a hostile server (or a +// compromised account) can set them to anything. quak uses them to name files +// on disk: `quak get` without `--out`, `downloadFile` without `outPath`, the +// backup's symlink and collection directories, and the extension of every file +// in the originals cache. Each of those goes through `sanitizeFileName` or +// `safeExtension`, so a title can only ever name one file inside the directory +// the caller chose. +// +// A path the user supplies (`--out`, `outPath`) is never sanitized: the caller +// is trusted, the server is not. + +import { describe, expect, it } from "vitest"; + +import { safeExtension, sanitizeFileName } from "../../src/filename.js"; + +const FALLBACK = "file-42"; + +describe("sanitizeFileName", () => { + it("passes a normal title through unchanged", () => { + expect(sanitizeFileName("IMG_0001.HEIC", FALLBACK)).toBe( + "IMG_0001.HEIC", + ); + expect(sanitizeFileName("Holiday 2024 (1).jpg", FALLBACK)).toBe( + "Holiday 2024 (1).jpg", + ); + expect(sanitizeFileName("café.jpg", FALLBACK)).toBe("café.jpg"); + }); + + it("cannot climb out of the directory with ../", () => { + // Without sanitizing, this would overwrite the user's SSH keys. + expect(sanitizeFileName("../../.ssh/authorized_keys", FALLBACK)).toBe( + "__.._.ssh_authorized_keys", + ); + expect(sanitizeFileName("..", FALLBACK)).toBe("_"); + expect(sanitizeFileName("..\\..\\x", FALLBACK)).toBe("__.._x"); + }); + + it("cannot name an absolute path", () => { + expect(sanitizeFileName("/etc/passwd", FALLBACK)).toBe("_etc_passwd"); + expect(sanitizeFileName("C:\\Windows\\x.dll", FALLBACK)).toBe( + "C__Windows_x.dll", + ); + }); + + it("replaces embedded separators, so the name stays one file", () => { + expect(sanitizeFileName("a/b\\c.jpg", FALLBACK)).toBe("a_b_c.jpg"); + }); + + it("replaces NUL and other control characters", () => { + // A NUL truncates the path in C code and makes Node's fs throw. + expect(sanitizeFileName("evil\0.jpg", FALLBACK)).toBe("evil_.jpg"); + expect(sanitizeFileName("line\nbreak\x7f.jpg", FALLBACK)).toBe( + "line_break_.jpg", + ); + }); + + it("does not produce a hidden file", () => { + expect(sanitizeFileName(".bashrc", FALLBACK)).toBe("_bashrc"); + }); + + it("does not produce a Windows device name", () => { + expect(sanitizeFileName("CON", FALLBACK)).toBe("_CON"); + expect(sanitizeFileName("nul.txt", FALLBACK)).toBe("_nul.txt"); + expect(sanitizeFileName("LPT1", FALLBACK)).toBe("_LPT1"); + // Only the exact names are reserved. + expect(sanitizeFileName("console.jpg", FALLBACK)).toBe("console.jpg"); + }); + + it("falls back to the given name for an empty title", () => { + expect(sanitizeFileName("", FALLBACK)).toBe(FALLBACK); + }); +}); + +describe("safeExtension", () => { + it("keeps a normal extension", () => { + expect(safeExtension("IMG_0001.HEIC")).toBe(".HEIC"); + expect(safeExtension("clip.mp4")).toBe(".mp4"); + }); + + it("uses .bin when there is no extension", () => { + expect(safeExtension("")).toBe(".bin"); + expect(safeExtension("README")).toBe(".bin"); + }); + + it("uses .bin when the extension holds anything but letters and digits", () => { + expect(safeExtension("x.j\\..\\pg")).toBe(".bin"); + expect(safeExtension("x.jp g")).toBe(".bin"); + expect(safeExtension("x.jpg\0")).toBe(".bin"); + }); +}); diff --git a/test/library/content-eviction.test.ts b/test/library/content-eviction.test.ts new file mode 100644 index 0000000..d0087b2 --- /dev/null +++ b/test/library/content-eviction.test.ts @@ -0,0 +1,358 @@ +/** + * Tests for the originals cache size limit and LRU eviction (issue #47), + * layered on the on-disk content cache (#46). + * + * Only `cacheDirectory/originals` is bounded and evicted. Before each original + * write the effective limit is + * min(cacheOriginalsMaxBytes, bytesUsedByOriginals + bytesFree - freeBelowBytes) + * with `bytesFree` read from `statfs` on the volume holding `cacheDirectory`. + * The limit therefore falls as the disk fills and rises as space returns. When + * a write would cross the limit, least-recently-used originals are removed until + * it fits; pinned files are skipped, and if only pinned files remain the write + * proceeds over-limit. Last-use is the file `mtime`, bumped whenever a read + * returns an original's path, so ordering survives a restart with no ledger. + * + * `statfs` is injected so the adaptive limit is exercised deterministically: + * `bsize` is 1, so `bavail` is the free byte count the formula sees. The source + * writes a controllable number of bytes per file, and tests set each stored + * file's `mtime` explicitly so LRU order does not depend on wall-clock timing. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, existsSync, statSync, utimesSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + ContentCache, + type ContentSource, + type StatFsFn, +} from "../../src/library/content.js"; +import { RequestPools } from "../../src/library/pools.js"; +import type { EnteFile } from "../../src/model/types.js"; + +const file = (id: number): EnteFile => ({ + id, + collectionID: 1, + ownerID: 1, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: 0, + modificationTime: 0, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 0, +}); + +// A source that writes a controllable number of bytes per file (default 10). +class SizedSource implements ContentSource { + sizeFor = new Map<number, number>(); + + async original(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + const n = this.sizeFor.get(args.file.id) ?? 10; + await writeFile(args.destination, Buffer.alloc(n, 1)); + return { bytesWritten: n }; + } + + async thumbnail(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + await writeFile(args.destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + } +} + +// A source whose original() writes 10 bytes but blocks on a gate before +// returning, so two fetches can be held in flight together. `bothStarted` +// resolves once both originals have entered, and `release()` lets them finish. +class GatedSource implements ContentSource { + private openGate!: () => void; + private readonly gate = new Promise<void>((r) => (this.openGate = r)); + private inFlight = 0; + private reachedTwo!: () => void; + readonly bothStarted = new Promise<void>((r) => (this.reachedTwo = r)); + + async original(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + this.inFlight += 1; + if (this.inFlight === 2) this.reachedTwo(); + await this.gate; + await writeFile(args.destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + } + + async thumbnail(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + await writeFile(args.destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + } + + release(): void { + this.openGate(); + } +} + +let root: string; +let cacheDir: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-eviction-")); + cacheDir = join(root, "cache"); +}); + +afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); +}); + +const originalPath = (id: number): string => + join(cacheDir, "originals", `${id}.jpg`); + +// Pin an original's mtime to a fixed second-resolution instant so LRU order is +// deterministic. Lower `seconds` = older = evicted first. +const setMtime = (id: number, seconds: number): void => { + utimesSync(originalPath(id), seconds, seconds); +}; + +const buildCache = (args: { + source?: ContentSource; + files?: EnteFile[]; + statfs: StatFsFn; + cacheOriginalsMaxBytes?: number; + freeBelowBytes?: number; + isPinned?: (fileID: number) => boolean; +}): { cache: ContentCache; source: SizedSource } => { + const source = (args.source as SizedSource) ?? new SizedSource(); + const byID = new Map<number, EnteFile>(); + for (const f of args.files ?? [file(1), file(2), file(3), file(4), file(5)]) + byID.set(f.id, f); + const cache = new ContentCache({ + pools: new RequestPools(), + source, + cacheDirectory: cacheDir, + getFile: (id) => byID.get(id), + statfs: args.statfs, + cacheOriginalsMaxBytes: args.cacheOriginalsMaxBytes, + freeBelowBytes: args.freeBelowBytes, + isPinned: args.isPinned, + }); + return { cache, source }; +}; + +// Plenty of free space, so the configured max governs the limit unless a test +// dials it down. `bsize` of 1 makes `bavail` the free byte count. +const abundantFree: StatFsFn = async () => ({ + bsize: 1, + bavail: 1_000_000_000, +}); + +describe("originals eviction", () => { + it("removes least-recently-used originals to fit the limit", async () => { + // Each original is 10 bytes; a 25-byte cap holds two. + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 25, + freeBelowBytes: 0, + }); + await cache.open(); + + await cache.original(1); + setMtime(1, 1000); + await cache.original(2); + setMtime(2, 2000); + // Writing the third (total 30 > 25) evicts the oldest, file 1. + await cache.original(3); + + expect(existsSync(originalPath(1))).toBe(false); + expect(existsSync(originalPath(2))).toBe(true); + expect(existsSync(originalPath(3))).toBe(true); + expect(cache.pathsFor(1).originalPath).toBeUndefined(); + expect(cache.originalsStatus().usedBytes).toBe(20); + }); + + it("skips pinned files, evicting the oldest unpinned instead", async () => { + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 25, + freeBelowBytes: 0, + isPinned: (id) => id === 1, + }); + await cache.open(); + + await cache.original(1); + setMtime(1, 1000); // oldest, but pinned + await cache.original(2); + setMtime(2, 2000); + await cache.original(3); + + // File 1 is oldest but pinned, so file 2 is evicted instead. + expect(existsSync(originalPath(1))).toBe(true); + expect(existsSync(originalPath(2))).toBe(false); + expect(existsSync(originalPath(3))).toBe(true); + }); + + it("proceeds over-limit when only pinned files remain", async () => { + // A 5-byte cap cannot hold even one 10-byte original. + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 5, + freeBelowBytes: 0, + isPinned: () => true, + }); + await cache.open(); + + const result = await cache.original(1); + + expect(existsSync(result.path)).toBe(true); + const status = cache.originalsStatus(); + expect(status.usedBytes).toBe(10); + expect(status.usedBytes).toBeGreaterThan(status.limitBytes ?? 0); + }); + + it("keeps a just-written original larger than the limit, evicting it only on a later write", async () => { + // A 5-byte cap cannot hold even one 10-byte original, but a fetch must + // never evict the file it just wrote and is about to return. + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 5, + freeBelowBytes: 0, + }); + await cache.open(); + + const result = await cache.original(1); + + // The just-written original survives its own over-budget write and the + // returned path exists on disk. + expect(existsSync(result.path)).toBe(true); + expect(existsSync(originalPath(1))).toBe(true); + expect(cache.originalsStatus().usedBytes).toBe(10); + setMtime(1, 1000); + + // A later write finds file 1 eligible and evicts it to make room, while + // the newly written file 2 is itself kept over-limit. + await cache.original(2); + expect(existsSync(originalPath(1))).toBe(false); + expect(existsSync(originalPath(2))).toBe(true); + expect(cache.originalsStatus().usedBytes).toBe(10); + }); + + it("adapts the limit down as the disk fills and up as space returns", async () => { + // The configured max is generous; free space drives the limit. Eviction + // only kicks in once free space falls below the protected reserve. + let free = 1_000_000; + const statfs: StatFsFn = async () => ({ bsize: 1, bavail: free }); + const { cache } = buildCache({ + statfs, + cacheOriginalsMaxBytes: 1000, + freeBelowBytes: 100, + }); + await cache.open(); + + // Ample free space: the limit is the configured max and nothing is + // evicted as three 10-byte originals accumulate. + await cache.original(1); + setMtime(1, 1000); + await cache.original(2); + setMtime(2, 2000); + await cache.original(3); + setMtime(3, 3000); + expect(cache.originalsStatus().limitBytes).toBe(1000); + expect(cache.originalsStatus().usedBytes).toBe(30); + + // The disk fills: only 95 bytes free, below the 100-byte reserve. The + // next write (used 40) sees limit = min(1000, 40 + 95 - 100) = 35 and + // evicts the oldest original (file 1) to fit. + free = 95; + await cache.original(4); + expect(cache.originalsStatus().limitBytes).toBe(35); + expect(existsSync(originalPath(1))).toBe(false); + expect(cache.originalsStatus().usedBytes).toBe(30); + setMtime(4, 4000); + + // Space returns: the limit rises back to the configured max and the + // next write is kept without eviction. + free = 1_000_000; + await cache.original(5); + expect(cache.originalsStatus().limitBytes).toBe(1000); + expect(existsSync(originalPath(4))).toBe(true); + expect(existsSync(originalPath(5))).toBe(true); + }); + + it("bumps an original's mtime when a read returns its path", async () => { + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 1000, + freeBelowBytes: 0, + }); + await cache.open(); + + await cache.original(1); + // Age the file well into the past. + setMtime(1, 1000); + expect(statSync(originalPath(1)).mtimeMs).toBeLessThan(2_000_000); + + // A second read is a cache hit that must touch the file. + const before = Date.now(); + await cache.original(1); + expect(statSync(originalPath(1)).mtimeMs).toBeGreaterThanOrEqual( + before - 2000, + ); + }); + + it("keeps both originals when two over-budget fetches race", async () => { + // A 15-byte cap holds one 10-byte original but not two. Two fetches are + // held in flight together; when both stored files cross the limit, + // neither eviction pass may delete the sibling whose path has not yet + // been returned, so both survive over-limit. + const source = new GatedSource(); + const { cache } = buildCache({ + source, + statfs: abundantFree, + cacheOriginalsMaxBytes: 15, + freeBelowBytes: 0, + }); + await cache.open(); + + const p1 = cache.original(1); + const p2 = cache.original(2); + await source.bothStarted; // both downloads are in flight before either stores + source.release(); + const [r1, r2] = await Promise.all([p1, p2]); + + // Both returned paths exist on disk even though together they exceed the + // cap; nothing was evicted out from under a fetch still in progress. + expect(existsSync(r1.path)).toBe(true); + expect(existsSync(r2.path)).toBe(true); + expect(existsSync(originalPath(1))).toBe(true); + expect(existsSync(originalPath(2))).toBe(true); + expect(cache.originalsStatus().usedBytes).toBe(20); + }); + + it("never counts or evicts thumbnails", async () => { + const { cache } = buildCache({ + statfs: abundantFree, + cacheOriginalsMaxBytes: 5, + freeBelowBytes: 0, + }); + await cache.open(); + + await cache.thumbnail(1); + await cache.thumbnail(2); + + expect(cache.pathsFor(1).thumbnailPath).toBeDefined(); + expect(cache.pathsFor(2).thumbnailPath).toBeDefined(); + expect(cache.originalsStatus().usedBytes).toBe(0); + }); +}); diff --git a/test/library/content-library.test.ts b/test/library/content-library.test.ts new file mode 100644 index 0000000..bebc9c4 --- /dev/null +++ b/test/library/content-library.test.ts @@ -0,0 +1,213 @@ +/** + * Integration between `Library` and the content cache (issue #46). + * + * The cache itself is covered in `content.test.ts`; this file locks the wiring: + * `Library.open` builds the cache from a content source, `lib.photos` hands out + * `Photo` objects that fetch through it, `lib.thumbnails.ensure` drives it, and + * a cached path shows up on the projected record. A library opened without a + * content source leaves those methods throwing rather than silently doing + * nothing. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + mkdtempSync, + readdirSync, + rmSync, + existsSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Library, type LibraryOptions } from "../../src/library/index.js"; +import type { ContentSource } from "../../src/library/content.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; +import { asLivePhoto, cdnSource, livePhotoZip } from "../live-photo.js"; + +const USER_ID = 7; + +const collection = (id: number): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + name: `album-${id}`, + type: "album", + updationTime: 1, + isShared: false, +}); + +const file = (id: number, collectionID: number): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: 1, + modificationTime: 1, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 1, +}); + +// A metadata-only client serving one album with one file, once. +class MockClient { + served = false; + whoami(): { email: string; userID: number } { + return { email: "u@example.com", userID: USER_ID }; + } + async collectionsSince(): Promise<CollectionsPage> { + if (this.served) return { collections: [], deleted: [], cursor: 1 }; + this.served = true; + return { collections: [collection(1)], deleted: [], cursor: 1 }; + } + async filesSince(): Promise<FilesPage> { + return { files: [file(1, 1)], deleted: [], cursor: 1 }; + } +} + +// A content source that writes a marker file and counts thumbnail fetches. +const stubSource = (): ContentSource & { thumbCalls: () => number } => { + let thumbCalls = 0; + return { + thumbCalls: () => thumbCalls, + original: async ({ destination }) => { + writeFileSync(destination, "orig-bytes"); + return { bytesWritten: 10 }; + }, + thumbnail: async ({ destination }) => { + thumbCalls++; + writeFileSync(destination, "thumb"); + return { bytesWritten: 5 }; + }, + }; +}; + +let root: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-content-lib-")); +}); + +afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); +}); + +describe("Library content wiring", () => { + it("fetches a thumbnail through a Photo and records its cache path", async () => { + const source = stubSource(); + const lib = await Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + contentSource: source, + refreshIntervalSeconds: 3600, + // On-demand wiring only; the background precache (#48) is covered + // in precache.test.ts and would race the exact-count assertions. + precacheThumbnails: false, + precacheOriginals: false, + }); + + const photo = lib.photos.byID({ fileID: 1 }); + expect(photo).toBeDefined(); + const result = await photo!.thumbnail(); + expect(source.thumbCalls()).toBe(1); + expect(result.path).toBe(join(root, "cache", "thumbnails", "1.jpg")); + expect(existsSync(result.path)).toBe(true); + + // The cached path is now on the projected record. + expect(lib.photos.byID({ fileID: 1 })!.record().thumbnailPath).toBe( + result.path, + ); + await lib.close(); + }); + + it("drives thumbnails.ensure through the cache", async () => { + const source = stubSource(); + const lib = await Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + contentSource: source, + refreshIntervalSeconds: 3600, + // On-demand wiring only; the background precache (#48) is covered + // in precache.test.ts and would race the exact-count assertions. + precacheThumbnails: false, + precacheOriginals: false, + }); + + const results = await lib.thumbnails.ensure({ + fileIDs: [1], + priority: "visible", + }); + expect(results).toEqual([ + { fileID: 1, path: join(root, "cache", "thumbnails", "1.jpg") }, + ]); + await lib.close(); + }); + + it("removes a live photo's ZIP an earlier version cached when it opens, and precaches its image and video", async () => { + const { file: live, body } = await asLivePhoto(file(1, 1)); + class LiveClient extends MockClient { + override async filesSince(): Promise<FilesPage> { + return { files: [live], deleted: [], cursor: 1 }; + } + } + const originals = join(root, "cache", "originals"); + const open = (opts: Partial<LibraryOptions>): Promise<Library> => + Library.open({ + client: new LiveClient(), + cacheDirectory: join(root, "cache"), + contentSource: cdnSource(new Map([[1, body]])), + refreshIntervalSeconds: 3600, + precacheThumbnails: false, + precacheOriginals: false, + ...opts, + }); + // A first run records the library, so the next one knows that file 1 + // is a live photo when it opens the cache. + await (await open({})).close(); + writeFileSync(join(originals, "1.jpg"), livePhotoZip()); + + let precached!: () => void; + const done = new Promise<void>((r) => (precached = r)); + const lib = await open({ + precacheOriginals: true, + onProgress: (e) => { + if (e.operation === "precacheOriginals" && e.status === "done") + precached(); + }, + }); + expect(existsSync(join(originals, "1.jpg"))).toBe(false); + await done; + + expect(readdirSync(originals).sort()).toEqual([ + "1.heic", + "1.livephoto.json", + "1.mov", + ]); + expect(lib.photos.byID({ fileID: 1 })!.record().originalPath).toBe( + join(originals, "1.heic"), + ); + await lib.close(); + }); + + it("throws from content methods when opened without a content source", async () => { + const lib = await Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + refreshIntervalSeconds: 3600, + }); + + await expect( + lib.photos.byID({ fileID: 1 })!.thumbnail(), + ).rejects.toThrow(/content cache/i); + await expect( + lib.thumbnails.ensure({ fileIDs: [1], priority: "visible" }), + ).rejects.toThrow(/content cache/i); + await lib.close(); + }); +}); diff --git a/test/library/content.test.ts b/test/library/content.test.ts new file mode 100644 index 0000000..e0f0e28 --- /dev/null +++ b/test/library/content.test.ts @@ -0,0 +1,687 @@ +/** + * Tests for the on-disk content and thumbnail cache (issue #46). + * + * The cache keys stored bytes by `fileID` under `cacheDirectory`: + * `originals/<fileID>.<ext>` and `thumbnails/<fileID>.<ext>`. Its contract: + * + * 1. **Fetch once, then serve from disk.** The first `original`/`thumbnail` + * fetches through the request pool and stores the bytes; the next finds the + * file present and returns its path with a single `skipped` event and no + * network. A file already sitting in the backup `downloadDirectory` counts + * as present too. + * 2. **Present-means-complete.** Content appears only by the streaming atomic + * writer's rename, so a file that exists is whole. The directory listing + * taken at `open()` is the record of what is cached, and the orphan temp + * files a crashed write may have left are reaped there. + * 3. **`thumbnails.ensure` drives the thumbnail pool with priority, dedup, and + * abort.** A `fileID` asked for twice downloads once; a visible request is + * served ahead of a background one; and an `AbortSignal` drops work still + * queued while letting an in-flight fetch finish. + * + * The `ContentSource` is a stand-in: it writes deterministic bytes to the + * destination and returns the count, so the cache logic is exercised with no + * crypto and no network. Ordering tests gate the stand-in on explicit deferreds + * and assert the persisted result, never a bare call or a timer. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + mkdtempSync, + rmSync, + existsSync, + writeFileSync, + mkdirSync, + readdirSync, + readFileSync, + statSync, +} from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + ContentCache, + type ContentSource, + type EnsureEvent, +} from "../../src/library/content.js"; +import { RequestPools } from "../../src/library/pools.js"; +import type { EnteFile } from "../../src/model/types.js"; +import { + asLivePhoto, + cdnSource, + IMAGE, + livePhotoZip, + VIDEO, +} from "../live-photo.js"; + +const file = (id: number, title = `file-${id}.jpg`): EnteFile => ({ + id, + collectionID: 1, + ownerID: 1, + key: new Uint8Array([id & 0xff]), + metadata: { + title, + fileType: "image", + creationTime: 0, + modificationTime: 0, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 0, +}); + +// A deferred with externally callable resolve, used to gate the stand-in source +// so ordering is controlled by the test rather than by timing. +const deferred = (): { promise: Promise<void>; resolve: () => void } => { + let resolve!: () => void; + const promise = new Promise<void>((r) => { + resolve = r; + }); + return { promise, resolve }; +}; + +// A ContentSource that writes `${kind}:${fileID}` bytes to the destination and +// records every call. `gate` optionally blocks a call until released, and +// `completed` records the order in which fetches finished — the observable used +// by the priority and abort tests instead of a timer. +class StubSource implements ContentSource { + originalCalls: number[] = []; + thumbnailCalls: number[] = []; + completed: number[] = []; + emptyFor = new Set<number>(); + gates = new Map<number, Promise<void>>(); + + private async run( + kind: "original" | "thumbnail", + file: EnteFile, + destination: string, + ): Promise<{ bytesWritten: number }> { + const gate = this.gates.get(file.id); + if (gate) await gate; + const bytes = this.emptyFor.has(file.id) + ? new Uint8Array(0) + : new TextEncoder().encode(`${kind}:${file.id}`); + writeFileSync(destination, bytes); + this.completed.push(file.id); + return { bytesWritten: bytes.length }; + } + + async original(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + this.originalCalls.push(args.file.id); + return this.run("original", args.file, args.destination); + } + + async thumbnail(args: { + file: EnteFile; + destination: string; + }): Promise<{ bytesWritten: number }> { + this.thumbnailCalls.push(args.file.id); + return this.run("thumbnail", args.file, args.destination); + } +} + +let root: string; +let cacheDir: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-content-")); + cacheDir = join(root, "cache"); +}); + +afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); +}); + +const buildCache = ( + args: { + source?: ContentSource; + files?: EnteFile[]; + pools?: RequestPools; + downloadDirectory?: string; + } = {}, +): { cache: ContentCache; source: StubSource } => { + const source = (args.source as StubSource) ?? new StubSource(); + const byID = new Map<number, EnteFile>(); + for (const f of args.files ?? [file(1), file(2), file(3)]) + byID.set(f.id, f); + const cache = new ContentCache({ + pools: args.pools ?? new RequestPools(), + source, + cacheDirectory: cacheDir, + downloadDirectory: args.downloadDirectory, + getFile: (id) => byID.get(id), + }); + return { cache, source }; +}; + +describe("ContentCache.open", () => { + it("creates the cache directories with 0700 permissions", async () => { + const { cache } = buildCache(); + await cache.open(); + + const originals = join(cacheDir, "originals"); + const thumbnails = join(cacheDir, "thumbnails"); + expect(existsSync(originals)).toBe(true); + expect(existsSync(thumbnails)).toBe(true); + expect(statSync(originals).mode & 0o777).toBe(0o700); + expect(statSync(thumbnails).mode & 0o777).toBe(0o700); + }); + + it("removes temp files of an exited process, keeping those of a running one and complete content", async () => { + const originals = join(cacheDir, "originals"); + const thumbnails = join(cacheDir, "thumbnails"); + mkdirSync(originals, { recursive: true }); + mkdirSync(thumbnails, { recursive: true }); + // A child that has already exited: its process ID is not running. + const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid; + const orphan = join(originals, `.quak-${exitedPID}-abc123.tmp`); + const orphanThumb = join(thumbnails, `.quak-${exitedPID}-abc456.tmp`); + // This test's own process stands in for another process still + // downloading into the same cache. + const inProgress = join(originals, `.quak-${process.pid}-def123.tmp`); + const inProgressThumb = join( + thumbnails, + `.quak-${process.pid}-def456.tmp`, + ); + const complete = join(originals, "1.jpg"); + const thumb = join(thumbnails, "2.jpg"); + writeFileSync(orphan, "half-written"); + writeFileSync(orphanThumb, "half-written"); + writeFileSync(inProgress, "half-written"); + writeFileSync(inProgressThumb, "half-written"); + writeFileSync(complete, "whole"); + writeFileSync(thumb, "whole-thumb"); + + const { cache } = buildCache(); + await cache.open(); + + expect(existsSync(orphan)).toBe(false); + expect(existsSync(orphanThumb)).toBe(false); + expect(existsSync(inProgress)).toBe(true); + expect(existsSync(inProgressThumb)).toBe(true); + expect(existsSync(complete)).toBe(true); + expect(existsSync(thumb)).toBe(true); + expect(cache.pathsFor(1)).toEqual({ originalPath: complete }); + }); + + it("records already-cached files so their paths appear in pathsFor", async () => { + const originals = join(cacheDir, "originals"); + const thumbnails = join(cacheDir, "thumbnails"); + mkdirSync(originals, { recursive: true }); + mkdirSync(thumbnails, { recursive: true }); + writeFileSync(join(originals, "1.jpg"), "orig"); + writeFileSync(join(thumbnails, "1.jpg"), "thumb"); + + const { cache } = buildCache(); + await cache.open(); + + expect(cache.pathsFor(1)).toEqual({ + originalPath: join(originals, "1.jpg"), + thumbnailPath: join(thumbnails, "1.jpg"), + }); + expect(cache.pathsFor(2)).toEqual({}); + }); +}); + +describe("ContentCache.original / thumbnail", () => { + it("fetches once, then serves the cached file with a single skipped event", async () => { + const { cache, source } = buildCache(); + await cache.open(); + + const events: string[] = []; + const first = await cache.original(1, { + onProgress: (e) => events.push(e.status), + }); + expect(source.originalCalls).toEqual([1]); + expect(first.path).toBe(join(cacheDir, "originals", "1.jpg")); + expect(first.bytes).toBe("original:1".length); + expect(existsSync(first.path)).toBe(true); + expect(statSync(first.path).mode & 0o777).toBe(0o600); + expect(cache.pathsFor(1).originalPath).toBe(first.path); + + const skips: string[] = []; + const second = await cache.original(1, { + onProgress: (e) => skips.push(e.status), + }); + // No second download, and exactly one skipped event. + expect(source.originalCalls).toEqual([1]); + expect(second.path).toBe(first.path); + expect(skips).toEqual(["skipped"]); + }); + + it("takes only a letters-and-digits extension from the title", async () => { + // The title comes from the server; an extension such as `.\..\x` + // must not reach the cache file name, so it becomes `.bin`. + const { cache } = buildCache({ + files: [file(1, "a.jpg"), file(2, "b.\\..\\x"), file(3, "")], + }); + await cache.open(); + + expect((await cache.original(1)).path).toBe( + join(cacheDir, "originals", "1.jpg"), + ); + expect((await cache.original(2)).path).toBe( + join(cacheDir, "originals", "2.bin"), + ); + expect((await cache.original(3)).path).toBe( + join(cacheDir, "originals", "3.bin"), + ); + }); + + it("serves a file already present in the download directory without fetching", async () => { + const downloadDirectory = join(root, "backup"); + mkdirSync(join(downloadDirectory, "originals"), { recursive: true }); + const backupPath = join(downloadDirectory, "originals", "1.jpg"); + writeFileSync(backupPath, "from-backup"); + + const { cache, source } = buildCache({ downloadDirectory }); + await cache.open(); + + const events: EnsureEvent["status"][] = []; + const result = await cache.original(1, { + onProgress: (e) => events.push(e.status), + }); + + expect(source.originalCalls).toEqual([]); + expect(result.path).toBe(backupPath); + expect(result.bytes).toBe("from-backup".length); + expect(events).toEqual(["skipped"]); + }); + + it("fetches and caches a thumbnail", async () => { + const { cache, source } = buildCache(); + await cache.open(); + + const result = await cache.thumbnail(2); + expect(source.thumbnailCalls).toEqual([2]); + expect(result.path).toBe(join(cacheDir, "thumbnails", "2.jpg")); + expect(existsSync(result.path)).toBe(true); + expect(cache.pathsFor(2).thumbnailPath).toBe(result.path); + }); + + it("shares one download between concurrent callers for the same file", async () => { + const { cache, source } = buildCache(); + await cache.open(); + const gate = deferred(); + source.gates.set(1, gate.promise); + + const a = cache.original(1); + const b = cache.original(1); + gate.resolve(); + const [ra, rb] = await Promise.all([a, b]); + + expect(source.originalCalls).toEqual([1]); + expect(ra.path).toBe(rb.path); + }); + + it("does not record a path when the fetched file is empty", async () => { + const { cache, source } = buildCache(); + source.emptyFor.add(1); + await cache.open(); + + await expect(cache.original(1)).rejects.toThrow(/empty/i); + expect(cache.pathsFor(1).originalPath).toBeUndefined(); + }); + + it("rejects an unknown file", async () => { + const { cache } = buildCache({ files: [] }); + await cache.open(); + await expect(cache.original(999)).rejects.toThrow(/unknown file/i); + }); +}); + +describe("ContentCache.ensureThumbnails", () => { + it("downloads once for a file listed twice and reports every id", async () => { + const { cache, source } = buildCache(); + await cache.open(); + + const results = await cache.ensureThumbnails({ + fileIDs: [1, 1, 2], + priority: "visible", + }); + + expect(source.thumbnailCalls.sort()).toEqual([1, 2]); + expect(results).toEqual([ + { fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") }, + { fileID: 2, path: join(cacheDir, "thumbnails", "2.jpg") }, + ]); + }); + + it("skips present files and reports a skipped event", async () => { + const thumbnails = join(cacheDir, "thumbnails"); + mkdirSync(thumbnails, { recursive: true }); + writeFileSync(join(thumbnails, "1.jpg"), "present"); + + const { cache, source } = buildCache(); + await cache.open(); + + const events: EnsureEvent[] = []; + const results = await cache.ensureThumbnails({ + fileIDs: [1, 2], + priority: "ahead", + onProgress: (e) => events.push(e), + }); + + expect(source.thumbnailCalls).toEqual([2]); + expect(results).toEqual([ + { fileID: 1, path: join(thumbnails, "1.jpg") }, + { fileID: 2, path: join(thumbnails, "2.jpg") }, + ]); + expect(events).toContainEqual({ + fileID: 1, + status: "skipped", + path: join(thumbnails, "1.jpg"), + }); + }); + + it("serves a visible request ahead of an already-queued background one", async () => { + // One thumbnail slot, so exactly one fetch runs at a time and the rest + // wait in the pool. A background fetch takes the slot; a background and + // a visible fetch queue behind it. When the slot frees, the pool must + // pick the visible (on-demand) request ahead of the background one that + // was submitted first. The completion order is the observable. + const pools = new RequestPools({ thumbnailConcurrency: 1 }); + const { cache, source } = buildCache({ pools }); + await cache.open(); + + const gateA = deferred(); + const gateB = deferred(); + const gateC = deferred(); + source.gates.set(1, gateA.promise); + source.gates.set(2, gateB.promise); + source.gates.set(3, gateC.promise); + + const bgFirst = cache.ensureThumbnails({ + fileIDs: [1], + priority: "background", + }); + // Let fetch 1 take the only slot before the others queue. + await Promise.resolve(); + const bgSecond = cache.ensureThumbnails({ + fileIDs: [2], + priority: "background", + }); + const visible = cache.ensureThumbnails({ + fileIDs: [3], + priority: "visible", + }); + + gateA.resolve(); + gateC.resolve(); + gateB.resolve(); + await Promise.all([bgFirst, bgSecond, visible]); + + // 1 ran first (it held the slot). Of the two that were queued, the + // visible id 3 was served before the background id 2. + expect(source.completed).toEqual([1, 3, 2]); + }); + + it("drops queued work on abort but keeps an in-flight fetch", async () => { + const pools = new RequestPools({ thumbnailConcurrency: 1 }); + const { cache, source } = buildCache({ pools }); + await cache.open(); + + const gate = deferred(); + source.gates.set(1, gate.promise); + const controller = new AbortController(); + + const pending = cache.ensureThumbnails({ + fileIDs: [1, 2], + priority: "ahead", + signal: controller.signal, + }); + // Fetch 1 is in flight (holds the slot); 2 is queued. + await Promise.resolve(); + controller.abort(); + gate.resolve(); + + const results = await pending; + + // The in-flight fetch finished and is kept; the queued one was dropped + // before it ran. + expect(source.thumbnailCalls).toEqual([1]); + expect(results).toEqual([ + { fileID: 1, path: join(cacheDir, "thumbnails", "1.jpg") }, + { fileID: 2, error: "aborted" }, + ]); + }); + + it("captures a per-file failure without failing the batch", async () => { + const { cache } = buildCache({ files: [file(1)] }); + await cache.open(); + + const results = await cache.ensureThumbnails({ + fileIDs: [1, 2], + priority: "background", + }); + + expect(results[0]).toEqual({ + fileID: 1, + path: join(cacheDir, "thumbnails", "1.jpg"), + }); + expect(results[1]?.fileID).toBe(2); + expect(results[1]?.error).toMatch(/unknown file/i); + }); +}); + +// A live photo's original is two files, its image and its video, which a +// photo viewer can open, and a JSON file naming them: the two are named with +// the extensions from inside the ZIP, so the names alone do not say which is +// which. These tests download a live photo ZIP through the real download +// layer (test/live-photo.ts). +describe("ContentCache live photos", () => { + const originals = (): string => join(cacheDir, "originals"); + + // A cache over the stand-in server, which holds `bodies` by file ID. + const cacheOf = ( + files: EnteFile[], + bodies: Map<number, Uint8Array>, + ): ContentCache => buildCache({ files, source: cdnSource(bodies) }).cache; + + it("stores a live photo as its image and its video and a JSON file naming them", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + const cache = cacheOf([live], new Map([[5, body]])); + await cache.open(); + + const result = await cache.original(5); + + expect(result).toEqual({ + path: join(originals(), "5.heic"), + videoPath: join(originals(), "5.mov"), + bytes: IMAGE.length, + }); + expect(readFileSync(result.path)).toEqual(Buffer.from(IMAGE)); + expect(readFileSync(result.videoPath!)).toEqual(Buffer.from(VIDEO)); + expect(statSync(result.videoPath!).mode & 0o777).toBe(0o600); + expect( + JSON.parse( + readFileSync(join(originals(), "5.livephoto.json"), "utf-8"), + ), + ).toEqual({ image: "5.heic", video: "5.mov" }); + expect(cache.pathsFor(5)).toEqual({ originalPath: result.path }); + }); + + it("serves a stored live photo from disk after the cache is opened again", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + const first = cacheOf([live], new Map([[5, body]])); + await first.open(); + const stored = await first.original(5); + + // This server has nothing, so a fetch would fail. + const second = cacheOf([live], new Map()); + await second.open(); + const events: string[] = []; + const served = await second.original(5, { + onProgress: (e) => events.push(e.status), + }); + + expect(served).toEqual(stored); + expect(events).toEqual(["skipped"]); + }); + + it("replaces a live photo an earlier version stored as a ZIP under the image's name", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + mkdirSync(originals(), { recursive: true }); + writeFileSync(join(originals(), "5.HEIC"), livePhotoZip()); + const cache = cacheOf([live], new Map([[5, body]])); + // Opened without being told that file 5 is a live photo, the cache + // records the ZIP, and does not serve it. + await cache.open(); + + const result = await cache.original(5); + + expect(result.videoPath).toBe(join(originals(), "5.mov")); + expect(readdirSync(originals()).sort()).toEqual([ + "5.heic", + "5.livephoto.json", + "5.mov", + ]); + }); + + it("removes a live photo's ZIP an earlier version stored when it opens, so the precache fetches the image and video", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + mkdirSync(originals(), { recursive: true }); + writeFileSync(join(originals(), "5.HEIC"), livePhotoZip()); + const cache = cacheOf([live], new Map([[5, body]])); + + await cache.open((fileID) => fileID === 5); + + expect(readdirSync(originals())).toEqual([]); + expect(cache.pathsFor(5)).toEqual({}); + const [fetched] = await cache.ensureOriginals({ fileIDs: [5] }); + expect(fetched).toEqual({ + fileID: 5, + path: join(originals(), "5.heic"), + }); + expect(cache.pathsFor(5)).toEqual({ originalPath: fetched!.path }); + }); + + it("leaves the image and video another process has just stored when it opens before their JSON file is written", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + const server = cdnSource(new Map([[5, body]])); + let second: ContentCache | undefined; + // Once the image and video are in place, and before the first cache + // writes the JSON file naming them, a second cache opens. + const source: ContentSource = { + original: async (args) => { + const stored = await server.original(args); + second = cacheOf([live], new Map()); + await second.open((fileID) => fileID === 5); + return stored; + }, + thumbnail: (args) => server.thumbnail(args), + }; + const first = buildCache({ files: [live], source }).cache; + await first.open((fileID) => fileID === 5); + + const result = await first.original(5); + + expect(readdirSync(originals()).sort()).toEqual([ + "5.heic", + "5.livephoto.json", + "5.mov", + ]); + expect(readFileSync(result.path)).toEqual(Buffer.from(IMAGE)); + expect(readFileSync(result.videoPath!)).toEqual(Buffer.from(VIDEO)); + expect(second!.pathsFor(5)).toEqual({}); + }); + + it.each(["missing", "empty"])( + "fetches a live photo again when the video its JSON file names is %s", + async (state) => { + const { file: live, body } = await asLivePhoto( + file(5, "IMG_5.HEIC"), + ); + const cache = cacheOf([live], new Map([[5, body]])); + await cache.open(); + const stored = await cache.original(5); + if (state === "missing") rmSync(stored.videoPath!); + else writeFileSync(stored.videoPath!, ""); + + const events: string[] = []; + const fetched = await cache.original(5, { + onProgress: (e) => events.push(e.status), + }); + + expect(events.at(-1)).toBe("done"); + expect(fetched).toEqual(stored); + expect(readFileSync(fetched.videoPath!)).toEqual( + Buffer.from(VIDEO), + ); + }, + ); + + it("does not take a file outside originals/ that a live photo's JSON file names", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + mkdirSync(originals(), { recursive: true }); + writeFileSync(join(cacheDir, "5.heic"), "not this photo"); + writeFileSync(join(originals(), "5.mov"), VIDEO); + writeFileSync( + join(originals(), "5.livephoto.json"), + JSON.stringify({ image: "../5.heic", video: "5.mov" }), + ); + const cache = cacheOf([live], new Map([[5, body]])); + await cache.open((fileID) => fileID === 5); + + const result = await cache.original(5); + + expect(result.path).toBe(join(originals(), "5.heic")); + expect(readFileSync(result.path)).toEqual(Buffer.from(IMAGE)); + expect(readFileSync(join(cacheDir, "5.heic"), "utf-8")).toBe( + "not this photo", + ); + }); + + it("evicts a live photo's image, video and JSON file together", async () => { + const a = await asLivePhoto(file(5, "a.HEIC")); + const b = await asLivePhoto(file(6, "b.HEIC")); + const size = IMAGE.length + VIDEO.length; + const cache = new ContentCache({ + pools: new RequestPools(), + source: cdnSource( + new Map([ + [5, a.body], + [6, b.body], + ]), + ), + cacheDirectory: cacheDir, + getFile: (id) => [a.file, b.file].find((f) => f.id === id), + // Room for one live photo, on a disk with plenty free. + cacheOriginalsMaxBytes: size, + freeBelowBytes: 0, + statfs: async () => ({ bsize: 1, bavail: 1e12 }), + }); + await cache.open(); + + await cache.original(5); + await cache.original(6); + + expect(readdirSync(originals()).sort()).toEqual([ + "6.heic", + "6.livephoto.json", + "6.mov", + ]); + expect(cache.originalsStatus().usedBytes).toBe(size); + }); + + it("stores nothing when a live photo does not match its recorded hash", async () => { + const { file: live, body } = await asLivePhoto( + file(5, "IMG_5.HEIC"), + livePhotoZip(), + "not:the recorded hash", + ); + const cache = cacheOf([live], new Map([[5, body]])); + await cache.open(); + + await expect(cache.original(5)).rejects.toThrow( + /file 5: content hash .* does not match/, + ); + + expect(readdirSync(originals())).toEqual([]); + expect(cache.pathsFor(5)).toEqual({}); + }); +}); diff --git a/test/library/fresh.test.ts b/test/library/fresh.test.ts new file mode 100644 index 0000000..10b8ee4 --- /dev/null +++ b/test/library/fresh.test.ts @@ -0,0 +1,272 @@ +/** + * Tests for fresh reads (issue #75, an owner amendment to design #36). + * + * The default read namespaces answer from RAM and never touch the network; a + * background loop keeps the local copy current. `fresh()` adds an awaited path: + * it forces a refresh, waits for it to complete and persist, and only then + * hands back the `albums`/`photos`/`timeline` namespaces, guaranteeing the + * local copy reflects a completed server round-trip. The contracts here: + * + * 1. A fresh read observes a server change that a same-instant default read + * would miss (the default path has not refreshed yet). + * 2. Concurrent fresh reads coalesce onto one in-flight refresh — three + * concurrent `fresh()` calls make exactly one collections round-trip, not + * three. + * 3. A refresh that fails rejects the fresh read (currency was unavailable), + * while the default reads stay silent and keep serving the last good copy. + * + * The client is the same metadata-only mock the background-refresh tests use: + * no crypto, no network, scripted pages, and a record of each call. A long + * refresh interval keeps the background timer out of the way so each test's + * refreshes are exactly the ones it triggers. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Library } from "../../src/library/index.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +const USER_ID = 42; + +// Long enough that no background tick fires during a test; each test's +// refreshes are only the ones its own `fresh()` calls force. +const SLOW_INTERVAL = 3600; + +const collection = (id: number, updationTime: number): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + name: `album-${id}`, + type: "album", + updationTime, + isShared: false, +}); + +const file = ( + id: number, + collectionID: number, + updationTime: number, +): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: updationTime, + modificationTime: updationTime, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime, +}); + +class MockClient { + userID = USER_ID; + failCollections = false; + collectionsQueue: CollectionsPage[] = []; + filesByCollection = new Map<number, FilesPage[]>(); + + collectionsSinceTimes: number[] = []; + filesCalls: { collectionID: number; sinceTime: number }[] = []; + + whoami(): { email: string; userID: number } { + return { email: "user@example.com", userID: this.userID }; + } + + async collectionsSince(args: { + sinceTime: number; + }): Promise<CollectionsPage> { + this.collectionsSinceTimes.push(args.sinceTime); + if (this.failCollections) throw new Error("network down"); + return ( + this.collectionsQueue.shift() ?? { + collections: [], + deleted: [], + cursor: args.sinceTime, + } + ); + } + + async filesSince(args: { + collectionID: number; + collectionKey: Uint8Array; + sinceTime: number; + }): Promise<FilesPage> { + this.filesCalls.push({ + collectionID: args.collectionID, + sinceTime: args.sinceTime, + }); + const queue = this.filesByCollection.get(args.collectionID); + return ( + queue?.shift() ?? { + files: [], + deleted: [], + cursor: args.sinceTime, + } + ); + } + + filesFor(collectionID: number, ...pages: FilesPage[]): void { + this.filesByCollection.set(collectionID, pages); + } +} + +// A client seeded with one collection and one file, opened with an empty cache +// so the initial refresh is awaited and post-`open()` state is deterministic. +const openSeeded = async ( + cacheDirectory: string, +): Promise<{ client: MockClient; lib: Library }> => { + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: SLOW_INTERVAL, + }); + return { client, lib }; +}; + +describe("Library.fresh", () => { + let dir: string; + let cacheDirectory: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "quak-fresh-")); + cacheDirectory = join(dir, "cache"); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("observes a server change a same-instant default read would miss", async () => { + const { client, lib } = await openSeeded(cacheDirectory); + try { + // A new file appears on the server after open, advancing its + // collection so the next refresh re-enumerates it. + client.collectionsQueue.push({ + collections: [collection(1, 200)], + deleted: [], + cursor: 200, + }); + client.filesFor(1, { + files: [file(1002, 1, 190)], + deleted: [], + cursor: 190, + }); + + // A default read at this instant has not refreshed: it misses 1002. + expect(lib.photos.byID({ fileID: 1002 })).toBeUndefined(); + + // A fresh read forces the round-trip and sees it. + const reads = await lib.fresh(); + expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002); + // And the change is now live for the default namespaces too. + expect(lib.photos.byID({ fileID: 1002 })?.fileID).toBe(1002); + } finally { + await lib.close(); + } + }); + + it("coalesces concurrent fresh reads onto one in-flight refresh", async () => { + const { client, lib } = await openSeeded(cacheDirectory); + try { + client.collectionsQueue.push({ + collections: [collection(1, 200)], + deleted: [], + cursor: 200, + }); + client.filesFor(1, { + files: [file(1002, 1, 190)], + deleted: [], + cursor: 190, + }); + + const collectionsBefore = client.collectionsSinceTimes.length; + const filesBefore = client.filesCalls.length; + + // Gate the next collections fetch so all three fresh reads are in + // flight together before any of them completes. + let release: () => void = () => {}; + const gate = new Promise<void>((r) => { + release = r; + }); + const inner = client.collectionsSince.bind(client); + client.collectionsSince = async (args: { sinceTime: number }) => { + await gate; + return inner(args); + }; + + const all = Promise.all([lib.fresh(), lib.fresh(), lib.fresh()]); + release(); + const [a, b, c] = await all; + + // Exactly one collections round-trip and one file round-trip served + // all three fresh reads. + expect(client.collectionsSinceTimes.length).toBe( + collectionsBefore + 1, + ); + expect(client.filesCalls.length).toBe(filesBefore + 1); + + // All three observed the change. + for (const reads of [a, b, c]) { + expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002); + } + } finally { + await lib.close(); + } + }); + + it("rejects the fresh read when the refresh fails, leaving defaults intact", async () => { + const { client, lib } = await openSeeded(cacheDirectory); + try { + client.failCollections = true; + + // Two concurrent fresh reads both reject, and share one failed + // round-trip rather than each making its own. + const collectionsBefore = client.collectionsSinceTimes.length; + const first = lib.fresh(); + const second = lib.fresh(); + await expect(first).rejects.toThrow(/network down/); + await expect(second).rejects.toThrow(/network down/); + expect(client.collectionsSinceTimes.length).toBe( + collectionsBefore + 1, + ); + + // The default reads never rejected: they still serve the last good + // copy, and the failure surfaced through status(). + expect(lib.photos.byID({ fileID: 1001 })?.fileID).toBe(1001); + expect(lib.status().lastError).toMatch(/network down/); + + // Recovery: once the server answers, a fresh read resolves current. + client.failCollections = false; + client.collectionsQueue.push({ + collections: [collection(2, 300)], + deleted: [], + cursor: 300, + }); + const reads = await lib.fresh(); + expect(reads.albums.byID({ collectionID: 2 })?.collectionID).toBe( + 2, + ); + expect(lib.status().lastError).toBeUndefined(); + } finally { + await lib.close(); + } + }); +}); diff --git a/test/library/library.test.ts b/test/library/library.test.ts new file mode 100644 index 0000000..2669871 --- /dev/null +++ b/test/library/library.test.ts @@ -0,0 +1,741 @@ +/** + * Tests for `Library.open()` and its transparent background refresh loop. + * + * The library keeps the account's server state in a `MetadataStore` (issue + * #41) and pulls changes with the resumable, tombstone-aware enumerators on + * `Client` (issue #38: `collectionsSince` / `filesSince`). `open()` loads the + * cache, does one refresh, then refreshes again every `refreshIntervalSeconds` + * on a background timer. The design (#36) forbids an exposed `sync()`, a + * `serverReachable` flag, a `lib.refresh()` method, and a "before each read" + * mode. The contracts exercised here: + * + * 1. Reads are answered from RAM. A read never calls the client. + * 2. `open()` does an initial refresh, then the interval keeps refreshing; + * each refresh resumes from the stored cursor and applies diffs + tombstones. + * 3. The cache is rewritten only when a refresh actually changes something. + * 4. A failed refresh is invisible to reads: the last good data stays, the + * failure surfaces via `onProgress` ("failed") and `status()`, and a later + * success clears the error. `open()` itself resolves even when the first + * refresh fails (offline start from cache). + * 5. `close()` stops the timer and is idempotent, and its promise resolves + * only once an in-flight refresh has written the cache file. + * 6. `cacheDirectory` defaults to the env-paths cache dir plus the user id. + * 7. `open()` branches on the cache: an empty cache awaits the first refresh + * (it has nothing to serve yet); an existing cache serves its copy at once + * and refreshes in the background, so a slow or dead server never stalls + * opening. + * 8. A save failure that leaves RAM ahead of disk keeps `status().lastError` + * set and keeps retrying the write; a later empty refresh does not clear it. + * + * The client is a mock: no crypto, no network. It serves scripted pages and + * records the `sinceTime` each call carried so cursor threading is provable. + * + * On an empty cache `open()` awaits the initial refresh (including its cache + * write), so state right after `open()` is deterministic; the tests that + * inspect post-`open()` state seed no cache and rely on that. Tests for an + * existing-cache open seed a store first and prove `open()` returns without + * waiting for the network. The interval tests then use real timers with a + * short interval and `vi.waitFor`: a fake clock cannot settle the real + * fsync-and-rename cache write, and empty diffs never write, so the eventual + * state is stable to poll for. + * + * A refresh changes RAM before it writes the cache file, so a polled state can + * be visible while that write is still running. Every test therefore awaits + * `close()`, which waits for the in-flight refresh, before `afterEach` removes + * the directory. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import envPaths from "env-paths"; + +import { Library, type RefreshEvent } from "../../src/library/index.js"; +import { MetadataStore } from "../../src/library/store.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +const USER_ID = 42; + +// Short enough that a couple of ticks pass within a test, long enough not to +// spin; interval tests poll for the eventual state rather than counting ticks. +const FAST_INTERVAL = 0.02; + +const collection = ( + id: number, + updationTime: number, + name = `album-${id}`, +): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + name, + type: "album", + updationTime, + isShared: false, +}); + +const file = ( + id: number, + collectionID: number, + updationTime: number, +): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: updationTime, + modificationTime: updationTime, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime, +}); + +/** + * A mock `Client`. `collectionsSince` shifts one page off `collectionsQueue` + * per call (an empty diff that advances nothing when the queue runs dry); + * `filesSince` shifts from a per-collection queue. `failCollections` makes the + * next and all further collection fetches throw, to simulate an offline server. + */ +class MockClient { + userID = USER_ID; + failCollections = false; + collectionsQueue: CollectionsPage[] = []; + filesByCollection = new Map<number, FilesPage[]>(); + + collectionsSinceTimes: number[] = []; + filesCalls: { collectionID: number; sinceTime: number }[] = []; + + whoami(): { email: string; userID: number } { + return { email: "user@example.com", userID: this.userID }; + } + + async collectionsSince(args: { + sinceTime: number; + }): Promise<CollectionsPage> { + this.collectionsSinceTimes.push(args.sinceTime); + if (this.failCollections) throw new Error("network down"); + return ( + this.collectionsQueue.shift() ?? { + collections: [], + deleted: [], + cursor: args.sinceTime, + } + ); + } + + async filesSince(args: { + collectionID: number; + collectionKey: Uint8Array; + sinceTime: number; + }): Promise<FilesPage> { + this.filesCalls.push({ + collectionID: args.collectionID, + sinceTime: args.sinceTime, + }); + const queue = this.filesByCollection.get(args.collectionID); + return ( + queue?.shift() ?? { + files: [], + deleted: [], + cursor: args.sinceTime, + } + ); + } + + filesFor(collectionID: number, ...pages: FilesPage[]): void { + this.filesByCollection.set(collectionID, pages); + } +} + +describe("Library.open and background refresh", () => { + let dir: string; + let cacheDirectory: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "quak-library-")); + cacheDirectory = join(dir, "cache"); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("does an initial refresh and answers reads from the cache", async () => { + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90), file(1002, 1, 95)], + deleted: [], + cursor: 95, + }); + + const lib = await Library.open({ client, cacheDirectory }); + try { + expect(lib.listCollections().map((c) => c.id)).toEqual([1]); + expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001, 1002]); + expect(lib.getFile(1, 1001)?.metadata.title).toBe("file-1001.jpg"); + + const status = lib.status(); + expect(status.userID).toBe(USER_ID); + expect(status.collections).toBe(1); + expect(status.files).toBe(2); + expect(status.lastRefreshAt).toBeGreaterThan(0); + expect(status.lastError).toBeUndefined(); + + // The initial refresh persisted the cache to disk. + const reloaded = await MetadataStore.load( + join(cacheDirectory, "metadata.json"), + ); + expect(reloaded.getFile(1, 1001)?.id).toBe(1001); + expect(reloaded.collectionsSinceTime).toBe(100); + } finally { + await lib.close(); + } + }); + + it("reads never call the client", async () => { + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + + const lib = await Library.open({ client, cacheDirectory }); + try { + const collectionCalls = client.collectionsSinceTimes.length; + const fileCalls = client.filesCalls.length; + + lib.listCollections(); + lib.getCollection(1); + lib.listFiles(1); + lib.getFile(1, 1001); + lib.status(); + + expect(client.collectionsSinceTimes.length).toBe(collectionCalls); + expect(client.filesCalls.length).toBe(fileCalls); + } finally { + await lib.close(); + } + }); + + it("resumes each refresh from the stored cursor", async () => { + // Seed a cache with a cursor and a collection, as a prior run left it. + const path = join(cacheDirectory, "metadata.json"); + const seed = await MetadataStore.load(path); + seed.userID = USER_ID; + seed.collectionsSinceTime = 500; + seed.putCollection(collection(1, 400)); + seed.putFile(file(1001, 1, 400)); + await seed.save(); + + const client = new MockClient(); + // The collection's updationTime advances (400 -> 600), so its files are + // re-enumerated from the collection's stored updationTime (400). + client.collectionsQueue.push({ + collections: [collection(1, 600)], + deleted: [], + cursor: 600, + }); + client.filesFor(1, { + files: [file(1002, 1, 550)], + deleted: [], + cursor: 550, + }); + + // Opening from an existing cache serves the seeded copy at once and + // refreshes in the background, so the refresh's effects are polled for. + const lib = await Library.open({ client, cacheDirectory }); + try { + await vi.waitFor( + () => { + // Collections resumed from the stored cursor, and files were + // re-enumerated from the stored collection updationTime. + expect(client.collectionsSinceTimes[0]).toBe(500); + expect(client.filesCalls).toEqual([ + { collectionID: 1, sinceTime: 400 }, + ]); + expect(lib.listFiles(1).map((f) => f.id)).toEqual([ + 1001, 1002, + ]); + }, + { timeout: 2000, interval: 5 }, + ); + } finally { + await lib.close(); + } + }); + + it("does not re-enumerate a collection whose updationTime did not advance", async () => { + const path = join(cacheDirectory, "metadata.json"); + const seed = await MetadataStore.load(path); + seed.userID = USER_ID; + seed.collectionsSinceTime = 100; + seed.putCollection(collection(1, 400)); + await seed.save(); + + const client = new MockClient(); + // The collection comes back in the diff (its metadata changed) but at + // the same updationTime, so its files must not be re-fetched. + client.collectionsQueue.push({ + collections: [collection(1, 400, "renamed")], + deleted: [], + cursor: 400, + }); + + // Existing cache: the rename lands via the background refresh. + const lib = await Library.open({ client, cacheDirectory }); + try { + await vi.waitFor( + () => expect(lib.getCollection(1)?.name).toBe("renamed"), + { timeout: 2000, interval: 5 }, + ); + // The collection's updationTime did not advance, so its files were + // never re-fetched. + expect(client.filesCalls).toEqual([]); + } finally { + await lib.close(); + } + }); + + it("applies diffs and tombstones on the interval", async () => { + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100), collection(2, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + client.filesFor(2, { + files: [file(2001, 2, 90)], + deleted: [], + cursor: 90, + }); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + }); + try { + expect(lib.listCollections().map((c) => c.id)).toEqual([1, 2]); + expect(lib.listFiles(2).map((f) => f.id)).toEqual([2001]); + + // Next refresh: collection 2 is tombstoned; collection 1 gains a + // file and loses its old one. + client.filesFor(1, { + files: [file(1002, 1, 190)], + deleted: [1001], + cursor: 190, + }); + client.collectionsQueue.push({ + collections: [collection(1, 200)], + deleted: [2], + cursor: 200, + }); + + await vi.waitFor( + () => { + expect(lib.listCollections().map((c) => c.id)).toEqual([1]); + expect(lib.listFiles(1).map((f) => f.id)).toEqual([1002]); + // Collection 2's files went with it. + expect(lib.listFiles(2)).toEqual([]); + }, + { timeout: 2000, interval: 5 }, + ); + } finally { + await lib.close(); + } + }); + + it("rewrites the cache only when a refresh changes something", async () => { + const saveSpy = vi.spyOn(MetadataStore.prototype, "save"); + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + }); + try { + // The initial refresh changed everything, so it saved once. + expect(saveSpy).toHaveBeenCalledTimes(1); + + // Several empty-diff ticks pass; none of them may rewrite the file. + await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 4)); + expect(saveSpy).toHaveBeenCalledTimes(1); + + // A real change triggers exactly one more rewrite; later empty ticks + // still do not, so the count settles at two. + client.collectionsQueue.push({ + collections: [collection(2, 200)], + deleted: [], + cursor: 200, + }); + await vi.waitFor(() => expect(saveSpy).toHaveBeenCalledTimes(2), { + timeout: 2000, + interval: 5, + }); + await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 4)); + expect(saveSpy).toHaveBeenCalledTimes(2); + } finally { + await lib.close(); + saveSpy.mockRestore(); + } + }); + + it("keeps a failed refresh invisible to reads and recovers later", async () => { + const events: RefreshEvent[] = []; + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + onProgress: (e) => events.push(e), + }); + try { + expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]); + + // The server goes away; refreshes now fail. + client.failCollections = true; + await vi.waitFor( + () => expect(lib.status().lastError).toMatch(/network down/), + { timeout: 2000, interval: 5 }, + ); + + // Reads still see the last good data; the failure was reported. + expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]); + expect( + events.some( + (e) => e.operation === "refresh" && e.status === "failed", + ), + ).toBe(true); + + // Recovery: a later refresh succeeds and clears the error. + client.failCollections = false; + client.collectionsQueue.push({ + collections: [collection(2, 300)], + deleted: [], + cursor: 300, + }); + await vi.waitFor( + () => { + expect(lib.status().lastError).toBeUndefined(); + expect(lib.listCollections().map((c) => c.id)).toEqual([ + 1, 2, + ]); + }, + { timeout: 2000, interval: 5 }, + ); + } finally { + await lib.close(); + } + }); + + it("resolves open() even when the first refresh fails", async () => { + const client = new MockClient(); + client.failCollections = true; + const events: RefreshEvent[] = []; + + const lib = await Library.open({ + client, + cacheDirectory, + onProgress: (e) => events.push(e), + }); + try { + // Nothing was cached and the server is unreachable: reads are empty, + // but the library opened and the failure is on record. + expect(lib.listCollections()).toEqual([]); + expect(lib.status().lastError).toMatch(/network down/); + expect(lib.status().lastRefreshAt).toBeUndefined(); + expect( + events.some( + (e) => e.operation === "refresh" && e.status === "failed", + ), + ).toBe(true); + } finally { + await lib.close(); + } + }); + + it("opens from an existing cache without waiting for the first refresh", async () => { + // Seed a cache as a prior run left it. + const path = join(cacheDirectory, "metadata.json"); + const seed = await MetadataStore.load(path); + seed.userID = USER_ID; + seed.collectionsSinceTime = 500; + seed.putCollection(collection(1, 400)); + seed.putFile(file(1001, 1, 400)); + await seed.save(); + + // The server does not answer this run's first refresh until the test + // is done with it. + let answerFirstFetch: (page: CollectionsPage) => void = () => {}; + const client = new MockClient(); + client.collectionsSince = () => + new Promise<CollectionsPage>((resolve) => { + answerFirstFetch = resolve; + }); + + // open() must resolve from the cache without blocking on the network, + // and reads must serve the seeded copy. + const lib = await Library.open({ client, cacheDirectory }); + try { + expect(lib.listCollections().map((c) => c.id)).toEqual([1]); + expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]); + // The first refresh is still outstanding: nothing has completed or + // failed yet. + expect(lib.status().lastRefreshAt).toBeUndefined(); + expect(lib.status().lastError).toBeUndefined(); + } finally { + // close() waits for the outstanding refresh, so let it finish. + answerFirstFetch({ collections: [], deleted: [], cursor: 500 }); + await lib.close(); + } + }); + + it("awaits the first refresh on a first run with an empty cache", async () => { + // No cache on disk: open() must not resolve until the first fetch does, + // so it never hands back an empty library it could have filled. + let releaseFirstFetch: (page: CollectionsPage) => void = () => {}; + const gate = new Promise<CollectionsPage>((resolve) => { + releaseFirstFetch = resolve; + }); + const client = new MockClient(); + client.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + client.collectionsSince = async (args: { sinceTime: number }) => { + client.collectionsSinceTimes.push(args.sinceTime); + return gate; + }; + + let opened = false; + const openPromise = Library.open({ client, cacheDirectory }).then( + (l) => { + opened = true; + return l; + }, + ); + + // While the first fetch is outstanding, open() has not resolved. + await new Promise((r) => setTimeout(r, 20)); + expect(opened).toBe(false); + + // Completing the fetch lets open() resolve with the data in place. + releaseFirstFetch({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + const lib = await openPromise; + try { + expect(opened).toBe(true); + expect(lib.listCollections().map((c) => c.id)).toEqual([1]); + expect(lib.listFiles(1).map((f) => f.id)).toEqual([1001]); + expect(lib.status().lastRefreshAt).toBeGreaterThan(0); + } finally { + await lib.close(); + } + }); + + it("keeps a save failure visible until a save actually succeeds", async () => { + const saveSpy = vi + .spyOn(MetadataStore.prototype, "save") + .mockRejectedValue(new Error("disk full")); + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + }); + try { + // The initial refresh mutated RAM but its save failed, so the error + // is on record and no refresh has counted as successful. + expect(lib.status().lastError).toMatch(/disk full/); + expect(lib.status().lastRefreshAt).toBeUndefined(); + + // Empty-diff ticks pass. Each still retries the unsaved write and + // still fails, so the error never silently clears and the refresh + // clock never advances — RAM must not run ahead of disk unnoticed. + const savesBefore = saveSpy.mock.calls.length; + await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 4)); + expect(saveSpy.mock.calls.length).toBeGreaterThan(savesBefore); + expect(lib.status().lastError).toMatch(/disk full/); + expect(lib.status().lastRefreshAt).toBeUndefined(); + + // Once the disk recovers, the next tick persists the pending change + // and only then clears the error and advances the clock. + saveSpy.mockRestore(); + await vi.waitFor( + () => { + expect(lib.status().lastError).toBeUndefined(); + expect(lib.status().lastRefreshAt).toBeGreaterThan(0); + }, + { timeout: 2000, interval: 5 }, + ); + const reloaded = await MetadataStore.load( + join(cacheDirectory, "metadata.json"), + ); + expect(reloaded.getFile(1, 1001)?.id).toBe(1001); + } finally { + await lib.close(); + saveSpy.mockRestore(); + } + }); + + it("close() stops the timer and is idempotent", async () => { + const client = new MockClient(); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + }); + const callsAfterOpen = client.collectionsSinceTimes.length; + + await lib.close(); + await lib.close(); // second close must not throw + expect(lib.status().closed).toBe(true); + + // No further refreshes fire once closed. + await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 5)); + expect(client.collectionsSinceTimes.length).toBe(callsAfterOpen); + }); + + it("close() resolves only after an in-flight refresh has written the cache", async () => { + const path = join(cacheDirectory, "metadata.json"); + const seed = await MetadataStore.load(path); + seed.userID = USER_ID; + seed.collectionsSinceTime = 500; + seed.putCollection(collection(1, 400)); + seed.putFile(file(1001, 1, 400)); + await seed.save(); + + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 600)], + deleted: [], + cursor: 600, + }); + client.filesFor(1, { + files: [file(1002, 1, 600)], + deleted: [], + cursor: 600, + }); + + // Hold the refresh's cache write until the test releases it. + const realSave = MetadataStore.prototype.save; + let releaseSave: () => void = () => {}; + const saveHeld = new Promise<void>((resolve) => { + releaseSave = resolve; + }); + const saveSpy = vi + .spyOn(MetadataStore.prototype, "save") + .mockImplementation(async function (this: MetadataStore) { + await saveHeld; + return realSave.call(this); + }); + + const lib = await Library.open({ client, cacheDirectory }); + try { + await vi.waitFor(() => expect(saveSpy).toHaveBeenCalled(), { + timeout: 2000, + interval: 5, + }); + + let closed = false; + const closing = lib.close().then(() => { + closed = true; + }); + await new Promise((r) => setTimeout(r, 50)); + expect(closed).toBe(false); + + releaseSave(); + await closing; + const reloaded = await MetadataStore.load(path); + expect(reloaded.getFile(1, 1002)?.id).toBe(1002); + } finally { + releaseSave(); + await lib.close(); + saveSpy.mockRestore(); + } + }); + + it("defaults cacheDirectory to the env-paths cache dir plus user id", async () => { + const xdg = join(dir, "xdg-cache"); + const prev = process.env.XDG_CACHE_HOME; + process.env.XDG_CACHE_HOME = xdg; + try { + const client = new MockClient(); + const lib = await Library.open({ client }); + try { + const expected = join( + envPaths("quak", { suffix: "" }).cache, + String(USER_ID), + ); + expect(lib.cacheDirectory).toBe(expected); + expect(lib.cacheDirectory.startsWith(xdg)).toBe(true); + expect(lib.cacheDirectory.endsWith(String(USER_ID))).toBe(true); + } finally { + await lib.close(); + } + } finally { + if (prev === undefined) delete process.env.XDG_CACHE_HOME; + else process.env.XDG_CACHE_HOME = prev; + } + }); +}); diff --git a/test/library/mldata.test.ts b/test/library/mldata.test.ts new file mode 100644 index 0000000..3de1cdc --- /dev/null +++ b/test/library/mldata.test.ts @@ -0,0 +1,562 @@ +/** + * Tests for the ML-data cache and its derived CLIP index (issue #49). + * + * Two layers are exercised: + * + * 1. `MLDataStore` on its own: storing one payload file per fileID (present + * means complete), building a `clip.f32` + `clip.json` index that reloads + * in a single read, rebuilding that index from the payloads when it is + * missing or disagrees with the files present, appending as new payloads + * arrive, overwriting a refetched file in place, and deciding what to + * (re)fetch as `updationTime` advances. + * + * 2. `Library` wiring: after each refresh the library fetches ML data through + * the metadata pool for every known file not yet cached, is incremental on + * later refreshes, and refetches a file whose `updationTime` advanced. + * Opening a cache directory written by another account starts empty, + * its ML data included (issue #104). + * + * Embedding values are chosen to be exactly representable as float32 so the + * round-trip through `clip.f32` compares equal. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { MLDataStore } from "../../src/library/mldata.js"; +import { Library } from "../../src/library/index.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { MLData } from "../../src/mldata-fetch.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +// A payload shaped like Ente's: a CLIP embedding plus face data that only the +// on-disk payload carries (never the RAM index). +const payload = (embedding: number[]): MLData => ({ + face: { + faces: [{ faceID: "f", detection: { box: { x: 0.5 } } }], + }, + clip: { embedding }, +}); + +describe("MLDataStore", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "quak-mldata-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("stores one payload file per fileID and builds a one-read index", async () => { + const store = await MLDataStore.open(dir); + const res = await store.storeFetched( + new Map([ + [100, payload([0.5, 0.25, 0.75])], + [200, payload([1, -2, 0.5])], + ]), + new Map([ + [100, 10], + [200, 20], + ]), + ); + expect(res).toEqual({ stored: 2, indexed: 2 }); + + // One payload file per fileID, and the derived index files. + expect(existsSync(join(dir, "100.json"))).toBe(true); + expect(existsSync(join(dir, "200.json"))).toBe(true); + expect(existsSync(join(dir, "clip.f32"))).toBe(true); + expect(existsSync(join(dir, "clip.json"))).toBe(true); + + // Reopening loads the index from disk in one read. + const reopened = await MLDataStore.open(dir); + const index = reopened.getIndex(); + expect(index.fileIDs).toEqual([100, 200]); + expect(index.embeddingLength).toBe(3); + expect([...index.embeddings]).toEqual([0.5, 0.25, 0.75, 1, -2, 0.5]); + + // The full payload (face boxes) is read back from disk on demand. + const full = await reopened.readPayload(100); + expect(full?.face).toBeDefined(); + expect(await reopened.readPayload(999)).toBeUndefined(); + }); + + it("rebuilds the index from payloads when it is missing", async () => { + const store = await MLDataStore.open(dir); + await store.storeFetched( + new Map([[100, payload([0.5, 0.25, 0.75])]]), + new Map([[100, 10]]), + ); + + // The derived index is lost but the payloads survive. + rmSync(join(dir, "clip.f32")); + rmSync(join(dir, "clip.json")); + + const reopened = await MLDataStore.open(dir); + const index = reopened.getIndex(); + expect(index.fileIDs).toEqual([100]); + expect([...index.embeddings]).toEqual([0.5, 0.25, 0.75]); + expect(existsSync(join(dir, "clip.f32"))).toBe(true); + }); + + it("rebuilds the index when it disagrees with the files present", async () => { + const store = await MLDataStore.open(dir); + await store.storeFetched( + new Map([ + [100, payload([0.5, 0.25, 0.75])], + [200, payload([1, -2, 0.5])], + ]), + new Map([ + [100, 10], + [200, 20], + ]), + ); + + // A payload disappears out from under the index (leaving it referencing + // a file no longer present); the index must be rebuilt from what is + // actually on disk. + rmSync(join(dir, "200.json")); + + const reopened = await MLDataStore.open(dir); + expect(reopened.getIndex().fileIDs).toEqual([100]); + }); + + it("rebuilds the index when a payload on disk is missing from it", async () => { + const store = await MLDataStore.open(dir); + await store.storeFetched( + new Map([[100, payload([0.5, 0.25, 0.75])]]), + new Map([[100, 10]]), + ); + + // A crash between storeFetched renaming a payload into place and + // rewriting the index leaves the payload complete on disk but absent + // from clip.json. Write a second payload directly to reproduce that + // torn state without touching the index. + writeFileSync( + join(dir, "200.json"), + JSON.stringify(payload([1, -2, 0.5])), + ); + + // Reopening self-heals with no manual delete: the index is rebuilt from + // the payloads to include the orphaned embedding. + const reopened = await MLDataStore.open(dir); + const index = reopened.getIndex(); + expect(index.fileIDs).toEqual([100, 200]); + expect([...index.embeddings]).toEqual([0.5, 0.25, 0.75, 1, -2, 0.5]); + }); + + it("appends new payloads and overwrites a refetched file in place", async () => { + const store = await MLDataStore.open(dir); + await store.storeFetched( + new Map([[100, payload([0.5, 0.25, 0.75])]]), + new Map([[100, 10]]), + ); + // A later batch adds a new file: appended after the first. + await store.storeFetched( + new Map([[200, payload([1, -2, 0.5])]]), + new Map([[200, 20]]), + ); + // Refetching 100 (its embedding changed) updates it in place, not a + // duplicate row. + await store.storeFetched( + new Map([[100, payload([9, 9, 9])]]), + new Map([[100, 30]]), + ); + + const index = store.getIndex(); + expect(index.fileIDs).toEqual([100, 200]); + expect([...index.embeddings]).toEqual([9, 9, 9, 1, -2, 0.5]); + }); + + it("keeps a payload without a CLIP embedding out of the index", async () => { + const store = await MLDataStore.open(dir); + const res = await store.storeFetched( + new Map<number, MLData>([[100, { face: { faces: [] } }]]), + new Map([[100, 10]]), + ); + expect(res.stored).toBe(1); + expect(res.indexed).toBe(0); + // The payload is still cached (present means complete). + expect(existsSync(join(dir, "100.json"))).toBe(true); + expect(store.getIndex().fileIDs).toEqual([]); + }); + + it("fetches only what is missing or has a newer updationTime", async () => { + const store = await MLDataStore.open(dir); + await store.storeFetched( + new Map([[100, payload([0.5, 0.25, 0.75])]]), + new Map([[100, 10]]), + ); + + // 100 is cached and current; 200 has never been fetched. + expect( + store.neededFor([ + { id: 100, updationTime: 10 }, + { id: 200, updationTime: 5 }, + ]), + ).toEqual([200]); + + // 100's updationTime advanced past what it was fetched at: refetch. + expect(store.neededFor([{ id: 100, updationTime: 15 }])).toEqual([100]); + + // Nothing advanced: nothing to fetch. + expect(store.neededFor([{ id: 100, updationTime: 10 }])).toEqual([]); + }); + + it("survives a corrupt index without losing the payloads", async () => { + const store = await MLDataStore.open(dir); + await store.storeFetched( + new Map([[100, payload([0.5, 0.25, 0.75])]]), + new Map([[100, 10]]), + ); + writeFileSync(join(dir, "clip.json"), "not json"); + + const reopened = await MLDataStore.open(dir); + expect(reopened.getIndex().fileIDs).toEqual([100]); + }); +}); + +// --- Library wiring --------------------------------------------------------- + +const USER_ID = 42; +const FAST_INTERVAL = 0.02; + +const collection = (id: number, updationTime: number): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + name: `album-${id}`, + type: "album", + updationTime, + isShared: false, +}); + +const file = ( + id: number, + collectionID: number, + updationTime: number, +): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: updationTime, + modificationTime: updationTime, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime, +}); + +// A mock client that serves scripted collection/file pages and per-file ML +// payloads, recording every ML fetch request so incremental behaviour is +// provable. +class MLMockClient { + userID = USER_ID; + collectionsQueue: CollectionsPage[] = []; + filesByCollection = new Map<number, FilesPage[]>(); + mlByFile = new Map<number, MLData>(); + mlFetchCalls: number[][] = []; + + whoami(): { email: string; userID: number } { + return { email: "user@example.com", userID: this.userID }; + } + + async collectionsSince(args: { + sinceTime: number; + }): Promise<CollectionsPage> { + return ( + this.collectionsQueue.shift() ?? { + collections: [], + deleted: [], + cursor: args.sinceTime, + } + ); + } + + async filesSince(args: { + collectionID: number; + collectionKey: Uint8Array; + sinceTime: number; + }): Promise<FilesPage> { + const queue = this.filesByCollection.get(args.collectionID); + return ( + queue?.shift() ?? { + files: [], + deleted: [], + cursor: args.sinceTime, + } + ); + } + + async fetchMLData(args: { + fileIDs: number[]; + fileKeys: Map<number, Uint8Array>; + }): Promise<Map<number, MLData>> { + this.mlFetchCalls.push([...args.fileIDs]); + const result = new Map<number, MLData>(); + for (const id of args.fileIDs) { + const p = this.mlByFile.get(id); + if (p) result.set(id, p); + } + return result; + } + + filesFor(collectionID: number, ...pages: FilesPage[]): void { + this.filesByCollection.set(collectionID, pages); + } +} + +describe("Library ML-data fetch on refresh", () => { + let dir: string; + let cacheDirectory: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "quak-lib-mldata-")); + cacheDirectory = join(dir, "cache"); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("fetches, stores and indexes ML data for known files, then is incremental", async () => { + const client = new MLMockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90), file(1002, 1, 95)], + deleted: [], + cursor: 95, + }); + client.mlByFile.set(1001, payload([0.5, 0.25, 0.75])); + client.mlByFile.set(1002, payload([1, -2, 0.5])); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + }); + try { + // Wait on `lastMLFetchAt`, set only once the pass has persisted the + // index and payloads — not on the in-RAM counts, which advance + // before `storeFetched` writes to disk, so the reopen below reads + // the committed index rather than racing the write. + await vi.waitFor( + () => { + expect(lib.status().lastMLFetchAt).toBeGreaterThan(0); + expect(lib.status().mlIndexed).toBe(2); + expect(lib.status().mlStored).toBe(2); + }, + { timeout: 2000, interval: 5 }, + ); + + // Both files were fetched, in one batch. + expect(client.mlFetchCalls.flat().sort((a, b) => a - b)).toEqual([ + 1001, 1002, + ]); + const callsAfterFirst = client.mlFetchCalls.length; + + // The index is on disk and reloads to the same shape. + const reopened = await MLDataStore.open( + join(cacheDirectory, "mldata"), + ); + expect(reopened.getIndex().fileIDs).toEqual([1001, 1002]); + + // Later refreshes with nothing new must not refetch. + await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 5)); + expect(client.mlFetchCalls.length).toBe(callsAfterFirst); + } finally { + await lib.close(); + } + }); + + it("refetches a file whose updationTime advanced", async () => { + const client = new MLMockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + client.mlByFile.set(1001, payload([0.5, 0.25, 0.75])); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + }); + try { + // Wait on `lastMLFetchAt`, set only after the first pass has + // persisted, not on `mlIndexed`, which is bumped in RAM before the + // write lands. + await vi.waitFor( + () => expect(lib.status().lastMLFetchAt).toBeGreaterThan(0), + { timeout: 2000, interval: 5 }, + ); + const callsBefore = client.mlFetchCalls.length; + + // The file changes on the server (updationTime advances) with a new + // embedding; the next refresh must refetch it. + client.mlByFile.set(1001, payload([9, 9, 9])); + client.collectionsQueue.push({ + collections: [collection(1, 200)], + deleted: [], + cursor: 200, + }); + client.filesFor(1, { + files: [file(1001, 1, 190)], + deleted: [], + cursor: 190, + }); + + // Poll the persisted index itself, not the fetch-call log: a call + // is recorded the instant the mock is entered, but `storeFetched` + // rewrites `clip.f32` only after it resolves, so an earlier reopen + // would read the pre-refetch vector. Reopening reads only committed + // (atomically renamed) files, so this sees the new embedding once — + // and only once — the store has written it. + await vi.waitFor( + async () => { + const reopened = await MLDataStore.open( + join(cacheDirectory, "mldata"), + ); + expect([...reopened.getIndex().embeddings]).toEqual([ + 9, 9, 9, + ]); + }, + { timeout: 2000, interval: 20 }, + ); + + // The refetch really went back to the server for 1001. + expect(client.mlFetchCalls.length).toBeGreaterThan(callsBefore); + expect(client.mlFetchCalls.flat()).toContain(1001); + } finally { + await lib.close(); + } + }); + + it("close() resolves only after a running ML data fetch has stored its payloads", async () => { + const client = new MLMockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + client.mlByFile.set(1001, payload([0.5, 0.25, 0.75])); + + // Hold the ML data fetch open until the test releases it. + let release!: () => void; + const held = new Promise<void>((r) => (release = r)); + let fetchStarted!: () => void; + const started = new Promise<void>((r) => (fetchStarted = r)); + const realFetch = client.fetchMLData.bind(client); + client.fetchMLData = async (args) => { + fetchStarted(); + await held; + return realFetch(args); + }; + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: 3600, + }); + try { + await started; + + let closed = false; + const closing = lib.close().then(() => { + closed = true; + }); + await new Promise((r) => setTimeout(r, 20)); + expect(closed).toBe(false); + + release(); + await closing; + expect( + existsSync(join(cacheDirectory, "mldata", "1001.json")), + ).toBe(true); + } finally { + release(); + await lib.close(); + } + }); + + it("starts empty when the cache directory holds another account's cache", async () => { + // Account A fills the cache directory: metadata and ML data. + const clientA = new MLMockClient(); + clientA.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + clientA.filesFor(1, { + files: [file(1001, 1, 90)], + deleted: [], + cursor: 90, + }); + clientA.mlByFile.set(1001, payload([0.5, 0.25, 0.75])); + const libA = await Library.open({ + client: clientA, + cacheDirectory, + refreshIntervalSeconds: 3600, + }); + try { + await vi.waitFor( + () => expect(libA.status().lastMLFetchAt).toBeGreaterThan(0), + { timeout: 2000, interval: 5 }, + ); + } finally { + await libA.close(); + } + + // Account B opens the same directory. + const clientB = new MLMockClient(); + clientB.userID = USER_ID + 1; + const sinceTimes: number[] = []; + const realCollectionsSince = clientB.collectionsSince.bind(clientB); + clientB.collectionsSince = async (args) => { + sinceTimes.push(args.sinceTime); + return realCollectionsSince(args); + }; + const libB = await Library.open({ + client: clientB, + cacheDirectory, + refreshIntervalSeconds: 3600, + }); + try { + expect(sinceTimes[0]).toBe(0); + expect(libB.status().userID).toBe(USER_ID + 1); + expect(libB.listCollections()).toEqual([]); + expect(libB.getFile(1, 1001)).toBeUndefined(); + expect(await libB.mldata.forFile({ fileID: 1001 })).toBeUndefined(); + expect( + libB.mldata.searchByEmbedding({ embedding: [0.5, 0.25, 0.75] }), + ).toEqual([]); + expect( + existsSync(join(cacheDirectory, "mldata", "1001.json")), + ).toBe(false); + } finally { + await libB.close(); + } + }); +}); diff --git a/test/library/mlsearch.test.ts b/test/library/mlsearch.test.ts new file mode 100644 index 0000000..877fccc --- /dev/null +++ b/test/library/mlsearch.test.ts @@ -0,0 +1,108 @@ +/** + * Tests for the content-similarity search surface over the CLIP index + * (issue #50). + * + * The surface is `lib.mldata`: `forFile` reads the full stored payload from + * disk, while `similar` and `searchByEmbedding` rank fileIDs by cosine + * similarity over the in-RAM `Float32Array` index alone (no disk, no network). + * The fixture uses axis-aligned vectors so the correct cosine ranking is + * obvious by inspection; cosine ignores magnitude, so `[2, 0, 0]` ranks above + * `[0.8, 0.6, 0]` for a `[1, 0, 0]` query. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { MLDataStore } from "../../src/library/mldata.js"; +import { makeMLDataAPI, type MLDataAPI } from "../../src/library/mlsearch.js"; +import type { MLData } from "../../src/mldata-fetch.js"; + +// A payload shaped like Ente's: a CLIP embedding plus face data that only the +// on-disk payload carries (never the RAM index). +const payload = (embedding: number[]): MLData => ({ + face: { faces: [{ faceID: "f", detection: { box: { x: 0.5 } } }] }, + clip: { embedding }, +}); + +// A small fixture index. Directions are chosen so every cosine ranking below +// is unambiguous. +const fixture = (): Map<number, MLData> => + new Map([ + [10, payload([1, 0, 0])], + [20, payload([0.8, 0.6, 0])], + [30, payload([0, 1, 0])], + [40, payload([-1, 0, 0])], + [50, payload([2, 0, 0])], + ]); + +describe("lib.mldata content-similarity search", () => { + let dir: string; + let store: MLDataStore; + let api: MLDataAPI; + + beforeEach(async () => { + dir = mkdtempSync(join(tmpdir(), "quak-mlsearch-")); + store = await MLDataStore.open(dir); + const updation = new Map([...fixture().keys()].map((id) => [id, 1])); + await store.storeFetched(fixture(), updation); + api = makeMLDataAPI(() => store); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("forFile returns the whole stored payload, or undefined when uncached", async () => { + const full = await api.forFile({ fileID: 20 }); + expect(full).toBeDefined(); + // Face data lives only in the payload, never in the RAM index. + expect(full?.face).toBeDefined(); + expect(full?.clip).toEqual({ embedding: [0.8, 0.6, 0] }); + expect(await api.forFile({ fileID: 999 })).toBeUndefined(); + }); + + it("similar ranks other files by cosine and excludes the query itself", () => { + // Query is file 10 = [1, 0, 0]. By cosine: 50 (1.0) > 20 (0.8) > + // 30 (0) > 40 (-1); 10 itself is left out. + const ranked = api.similar({ fileID: 10 }); + expect(ranked.map((r) => r.fileID)).toEqual([50, 20, 30, 40]); + // Cosine ignores magnitude: [2,0,0] is a perfect match for [1,0,0]. + expect(ranked[0]).toMatchObject({ fileID: 50 }); + expect(ranked[0].score).toBeCloseTo(1, 5); + }); + + it("similar honours limit and returns [] for an unindexed file", () => { + expect( + api.similar({ fileID: 10, limit: 2 }).map((r) => r.fileID), + ).toEqual([50, 20]); + expect(api.similar({ fileID: 999 })).toEqual([]); + }); + + it("searchByEmbedding ranks the index by cosine to the query vector", () => { + // Query [0, 1, 0]: 30 (1.0) > 20 (0.6) > {10, 40, 50} all 0, broken by + // ascending fileID. + const ranked = api.searchByEmbedding({ embedding: [0, 1, 0] }); + expect(ranked.map((r) => r.fileID)).toEqual([30, 20, 10, 40, 50]); + expect(ranked[0].score).toBeCloseTo(1, 5); + + expect( + api + .searchByEmbedding({ embedding: [0, 1, 0], limit: 2 }) + .map((r) => r.fileID), + ).toEqual([30, 20]); + }); + + it("searchByEmbedding returns [] for a wrong-length or zero query", () => { + expect(api.searchByEmbedding({ embedding: [1, 0] })).toEqual([]); + expect(api.searchByEmbedding({ embedding: [0, 0, 0] })).toEqual([]); + }); + + it("degrades to empty results when no ML store is present", async () => { + const none = makeMLDataAPI(() => undefined); + expect(await none.forFile({ fileID: 10 })).toBeUndefined(); + expect(none.similar({ fileID: 10 })).toEqual([]); + expect(none.searchByEmbedding({ embedding: [1, 0, 0] })).toEqual([]); + }); +}); diff --git a/test/library/pools.test.ts b/test/library/pools.test.ts new file mode 100644 index 0000000..9fed142 --- /dev/null +++ b/test/library/pools.test.ts @@ -0,0 +1,345 @@ +/** + * Tests for `src/library/pools.ts` — the three bounded request pools (issue + * #45). + * + * A `BoundedPool` runs submitted tasks with a fixed concurrency cap. Within a + * pool, on-demand work runs before background work, and a task submitted with a + * key that a still-pending task already carries is not run twice — both callers + * share the one result. `RequestPools` bundles the three the design calls for + * (metadata 10, content 5, thumbnails 25); the pools are independent, so an + * idle pool never lends its slots to a busy one. + * + * ## How the tasks are controlled + * + * Every task here is a gate: it reports when it *starts* and then blocks until + * the test *releases* it, so the test decides exactly how many run at once and + * in what order they finish. A shared tracker counts how many tasks are running + * at any instant and records the peak, which is what the concurrency assertions + * read. No assertion is about wall-clock time. + * + * `drain()` returns a promise that settles on a macrotask, which flushes the + * microtask queue the pool schedules its starts on; the tests await it to let + * the pool react to a submission or a release before they inspect it. + */ + +import { describe, it, expect } from "vitest"; + +import { + BoundedPool, + RequestPools, + DEFAULT_METADATA_CONCURRENCY, + DEFAULT_CONTENT_CONCURRENCY, + DEFAULT_THUMBNAIL_CONCURRENCY, +} from "../../src/library/pools.js"; + +// Settle on a macrotask so every microtask the pool queued has run. +const drain = (): Promise<void> => + new Promise((resolve) => setTimeout(resolve, 0)); + +// A controllable task. `task` blocks until `release()` (resolve) or `fail()` +// (reject) is called; `startOrder` records the sequence in which tasks began. +interface Gate<T> { + task: () => Promise<T>; + release: (value: T) => void; + fail: (err: unknown) => void; + started: () => boolean; + runs: () => number; +} + +// Tracks how many gated tasks are running concurrently across a whole test. +class Tracker { + active = 0; + peak = 0; + readonly starts: string[] = []; + + gate<T>(label = ""): Gate<T> { + let settleResolve!: (value: T) => void; + let settleReject!: (err: unknown) => void; + const settled = new Promise<T>((resolve, reject) => { + settleResolve = resolve; + settleReject = reject; + }); + let started = false; + let runs = 0; + const task = async (): Promise<T> => { + started = true; + runs++; + this.active++; + this.peak = Math.max(this.peak, this.active); + this.starts.push(label); + try { + return await settled; + } finally { + this.active--; + } + }; + return { + task, + release: (value: T) => settleResolve(value), + fail: (err: unknown) => settleReject(err), + started: () => started, + runs: () => runs, + }; + } +} + +describe("BoundedPool concurrency cap", () => { + it("never runs more than `concurrency` tasks at once", async () => { + const pool = new BoundedPool(3); + const t = new Tracker(); + const gates = Array.from({ length: 5 }, () => t.gate<void>()); + + const done = gates.map((g) => pool.run(g.task)); + await drain(); + + // Three started, two queued behind the cap. + expect(t.active).toBe(3); + expect(gates.slice(0, 3).every((g) => g.started())).toBe(true); + expect(gates.slice(3).some((g) => g.started())).toBe(false); + + // Finishing one admits exactly one more; the cap holds. + gates[0]!.release(); + await drain(); + expect(t.active).toBe(3); + expect(gates[3]!.started()).toBe(true); + expect(gates[4]!.started()).toBe(false); + + for (const g of gates.slice(1)) g.release(); + await Promise.all(done); + expect(t.peak).toBe(3); + }); + + it("rejects a non-positive or non-integer concurrency", () => { + expect(() => new BoundedPool(0)).toThrow(RangeError); + expect(() => new BoundedPool(-1)).toThrow(RangeError); + expect(() => new BoundedPool(2.5)).toThrow(RangeError); + }); +}); + +describe("BoundedPool priority ordering", () => { + it("runs on-demand work before background, FIFO within a priority", async () => { + const pool = new BoundedPool(1); + const t = new Tracker(); + const a = t.gate<void>("a"); + const b = t.gate<void>("b"); + const c = t.gate<void>("c"); + const d = t.gate<void>("d"); + + // `a` takes the only slot; the rest queue. + void pool.run(a.task, { priority: "background" }); + await drain(); + void pool.run(b.task, { priority: "background" }); + void pool.run(c.task, { priority: "on-demand" }); + void pool.run(d.task, { priority: "background" }); + await drain(); + expect(t.starts).toEqual(["a"]); + + // The on-demand `c` jumps ahead of the earlier-queued background `b`. + a.release(); + await drain(); + expect(t.starts).toEqual(["a", "c"]); + + // Then background work drains in submission order: `b` before `d`. + c.release(); + await drain(); + expect(t.starts).toEqual(["a", "c", "b"]); + b.release(); + await drain(); + expect(t.starts).toEqual(["a", "c", "b", "d"]); + d.release(); + }); + + it("defaults to background priority", async () => { + const pool = new BoundedPool(1); + const t = new Tracker(); + const a = t.gate<void>("a"); + const plain = t.gate<void>("plain"); + const urgent = t.gate<void>("urgent"); + + void pool.run(a.task); + await drain(); + void pool.run(plain.task); // no options -> background + void pool.run(urgent.task, { priority: "on-demand" }); + await drain(); + + a.release(); + await drain(); + expect(t.starts).toEqual(["a", "urgent"]); + urgent.release(); + plain.release(); + }); +}); + +describe("BoundedPool in-flight dedup", () => { + it("fetches a key once and hands both callers the same result", async () => { + const pool = new BoundedPool(5); + const t = new Tracker(); + const g = t.gate<number>(); + + const first = pool.run(g.task, { key: 7 }); + const second = pool.run(g.task, { key: 7 }); + await drain(); + + expect(g.runs()).toBe(1); + expect(first).toBe(second); + + g.release(99); + expect(await first).toBe(99); + expect(await second).toBe(99); + }); + + it("dedups only while in flight; a settled key runs again", async () => { + const pool = new BoundedPool(5); + const t = new Tracker(); + const g1 = t.gate<number>(); + + const first = pool.run(g1.task, { key: 7 }); + await drain(); + g1.release(1); + expect(await first).toBe(1); + + // The key is free again once its task settled. + const g2 = t.gate<number>(); + const third = pool.run(g2.task, { key: 7 }); + await drain(); + expect(g2.started()).toBe(true); + g2.release(2); + expect(await third).toBe(2); + }); + + it("propagates a rejection to every deduped caller", async () => { + const pool = new BoundedPool(5); + const t = new Tracker(); + const g = t.gate<number>(); + + const first = pool.run(g.task, { key: 7 }); + const second = pool.run(g.task, { key: 7 }); + await drain(); + + const boom = new Error("boom"); + g.fail(boom); + await expect(first).rejects.toBe(boom); + await expect(second).rejects.toBe(boom); + + // A failed key is also freed, so it may be retried by a fresh submit. + const g2 = t.gate<number>(); + const retry = pool.run(g2.task, { key: 7 }); + await drain(); + expect(g2.started()).toBe(true); + g2.release(5); + expect(await retry).toBe(5); + }); +}); + +describe("BoundedPool slot lifetime", () => { + it("holds one slot for a task's whole lifetime, retries included", async () => { + const pool = new BoundedPool(1); + const t = new Tracker(); + + // A task that internally makes two attempts before succeeding — the + // shape of a retrying request. It must occupy exactly one slot for the + // whole of that, so no other task may start until it finally settles. + const attempt1 = t.gate<void>("attempt1"); + const attempt2 = t.gate<void>("attempt2"); + const retrying = async (): Promise<void> => { + try { + await attempt1.task(); + } catch { + await attempt2.task(); + } + }; + const other = t.gate<void>("other"); + + const running = pool.run(retrying); + await drain(); + void pool.run(other.task); + await drain(); + + // First attempt is in flight and holds the only slot. + expect(t.starts).toEqual(["attempt1"]); + expect(other.started()).toBe(false); + + // The retry is still the same task in the same slot; `other` waits. + attempt1.fail(new Error("transient")); + await drain(); + expect(t.starts).toEqual(["attempt1", "attempt2"]); + expect(other.started()).toBe(false); + + // Only when the whole task settles does the slot free. + attempt2.release(); + await running; + await drain(); + expect(other.started()).toBe(true); + other.release(); + }); +}); + +describe("RequestPools", () => { + it("exposes three pools at the design's default caps", () => { + expect(DEFAULT_METADATA_CONCURRENCY).toBe(10); + expect(DEFAULT_CONTENT_CONCURRENCY).toBe(5); + expect(DEFAULT_THUMBNAIL_CONCURRENCY).toBe(25); + + const pools = new RequestPools(); + expect(pools.metadata.concurrency).toBe(10); + expect(pools.content.concurrency).toBe(5); + expect(pools.thumbnails.concurrency).toBe(25); + }); + + it("takes overridden caps", () => { + const pools = new RequestPools({ + metadataConcurrency: 1, + contentConcurrency: 2, + thumbnailConcurrency: 3, + }); + expect(pools.metadata.concurrency).toBe(1); + expect(pools.content.concurrency).toBe(2); + expect(pools.thumbnails.concurrency).toBe(3); + }); + + it("keeps pools independent: an idle pool lends no slots", async () => { + const pools = new RequestPools({ contentConcurrency: 1 }); + const t = new Tracker(); + const c1 = t.gate<void>(); + const c2 = t.gate<void>(); + const c3 = t.gate<void>(); + + // The content pool is capped at 1. The thumbnail pool sits idle with 25 + // free slots — none of which may be borrowed to run a second content + // task. + void pools.content.run(c1.task); + void pools.content.run(c2.task); + void pools.content.run(c3.task); + await drain(); + expect(t.active).toBe(1); + + c1.release(); + await drain(); + expect(t.active).toBe(1); + c2.release(); + await drain(); + expect(t.active).toBe(1); + c3.release(); + await drain(); + expect(t.peak).toBe(1); + }); + + it("runs different pools concurrently", async () => { + const pools = new RequestPools({ + metadataConcurrency: 1, + contentConcurrency: 1, + }); + const t = new Tracker(); + const m = t.gate<void>(); + const c = t.gate<void>(); + + void pools.metadata.run(m.task); + void pools.content.run(c.task); + await drain(); + + // One slot each, in two independent pools: both run at once. + expect(t.active).toBe(2); + m.release(); + c.release(); + }); +}); diff --git a/test/library/precache.test.ts b/test/library/precache.test.ts new file mode 100644 index 0000000..977867f --- /dev/null +++ b/test/library/precache.test.ts @@ -0,0 +1,498 @@ +/** + * The aggressive local precache (issue #48), driven from `Library.open`. + * + * Two background fills start with no caller input: every thumbnail in the + * account newest first, and the originals of the pinned set (the favorites + * album then the latest `precacheOriginalsDays` window ending at the newest + * file). Both run through the shared pools at background priority, so on-demand + * work always preempts them; both report through `status()`. The pinned set is + * the eviction predicate (#47), so a pinned original is never evicted and a + * file that leaves the set becomes an ordinary, evictable original. + * + * The unit tests drive `Precache` against a fake cache that records what it was + * asked to fetch (order and kind) with no pool or network; the integration + * tests drive the real wiring through `Library.open` with a stub content + * source, and the eviction test drives the real `ContentCache`. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, existsSync, utimesSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Precache, type PrecacheCache } from "../../src/library/precache.js"; +import { + deriveRecords, + type DerivedRecords, +} from "../../src/library/records.js"; +import { + ContentCache, + type ContentSource, + type EnsureResult, + type StatFsFn, +} from "../../src/library/content.js"; +import { RequestPools } from "../../src/library/pools.js"; +import { Library } from "../../src/library/index.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; + +const DAY_MICROS = 24 * 60 * 60 * 1000 * 1000; + +const collection = ( + id: number, + type: Collection["type"] = "album", +): Collection => ({ + id, + ownerID: 1, + key: new Uint8Array([id & 0xff]), + name: `album-${id}`, + type, + updationTime: 1, + isShared: false, +}); + +// A file whose creationTime (microseconds) places it `daysAgo` days before a +// fixed reference instant, so the latest-week window is deterministic. +const REFERENCE_MICROS = 1_000 * DAY_MICROS; +const file = (id: number, collectionID: number, daysAgo: number): EnteFile => ({ + id, + collectionID, + ownerID: 1, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: REFERENCE_MICROS - daysAgo * DAY_MICROS, + modificationTime: 0, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 1, +}); + +const records = ( + collections: Collection[], + files: EnteFile[], +): DerivedRecords => deriveRecords(collections, files); + +// A fake cache: records every fetch (kind + order) and reports the files it has +// stored via `pathsFor`. `presentThumbs`/`presentOriginals` seed already-cached +// files so the precache skips them with a single lookup. +class FakeCache implements PrecacheCache { + readonly thumbFetched: number[] = []; + readonly originalFetched: number[] = []; + readonly presentThumbs = new Set<number>(); + readonly presentOriginals = new Set<number>(); + + pathsFor(fileID: number): { + originalPath?: string; + thumbnailPath?: string; + } { + const out: { originalPath?: string; thumbnailPath?: string } = {}; + if (this.presentThumbs.has(fileID)) + out.thumbnailPath = `/thumbs/${fileID}`; + if (this.presentOriginals.has(fileID)) + out.originalPath = `/originals/${fileID}`; + return out; + } + + async ensureThumbnails(args: { + fileIDs: number[]; + priority: "background"; + }): Promise<EnsureResult[]> { + return args.fileIDs.map((fileID) => { + this.thumbFetched.push(fileID); + this.presentThumbs.add(fileID); + return { fileID, path: `/thumbs/${fileID}` }; + }); + } + + async ensureOriginals(args: { + fileIDs: number[]; + }): Promise<EnsureResult[]> { + return args.fileIDs.map((fileID) => { + this.originalFetched.push(fileID); + this.presentOriginals.add(fileID); + return { fileID, path: `/originals/${fileID}` }; + }); + } +} + +// Resolve once a predicate holds, polling the microtask queue; fails fast +// rather than hanging the suite. +const until = async (predicate: () => boolean): Promise<void> => { + for (let i = 0; i < 1000; i++) { + if (predicate()) return; + await new Promise((r) => setTimeout(r, 1)); + } + throw new Error("condition not met in time"); +}; + +describe("Precache unit", () => { + it("precaches every thumbnail newest first, skipping present ones", async () => { + const cols = [collection(1)]; + const files = [ + file(1, 1, 0), + file(2, 1, 1), + file(3, 1, 2), + file(4, 1, 3), + ]; + const cache = new FakeCache(); + cache.presentThumbs.add(3); // already on disk: skipped + const pre = new Precache({ originals: false }); + pre.bind(cache); + pre.update(records(cols, files)); + pre.start(); + + await until(() => cache.thumbFetched.length === 3); + // Newest first (file 1 is newest), file 3 skipped by a lookup. + expect(cache.thumbFetched).toEqual([1, 2, 4]); + expect(cache.originalFetched).toEqual([]); + }); + + it("pins favorites then the latest-week window and precaches their originals in that order", async () => { + const cols = [collection(1), collection(2, "favorites")]; + // File 10 is an old favorite (30 days old); files 1..3 are within the + // 7-day window; file 4 is outside it. + const files = [ + file(1, 1, 0), + file(2, 1, 2), + file(3, 1, 6), + file(4, 1, 20), + file(10, 2, 30), // favorite, old + ]; + const cache = new FakeCache(); + const pre = new Precache({ originalsDays: 7 }); + pre.bind(cache); + pre.update(records(cols, files)); + pre.start(); + + await until(() => cache.originalFetched.length === 4); + // Favorite (10) first, then the window newest-first (1, 2, 3). File 4 + // is outside the window and never pinned. + expect(cache.originalFetched).toEqual([10, 1, 2, 3]); + expect(pre.isPinned(10)).toBe(true); + expect(pre.isPinned(1)).toBe(true); + expect(pre.isPinned(4)).toBe(false); + }); + + it("drops a file from the pinned set when the window moves past it", () => { + const cols = [collection(1)]; + const cache = new FakeCache(); + const pre = new Precache({ originalsDays: 7 }); + pre.bind(cache); + + pre.update(records(cols, [file(1, 1, 0), file(2, 1, 3)])); + expect(pre.isPinned(2)).toBe(true); + + // A newer file arrives; the window's newest end moves forward so the + // 3-day-old file 2 (now 13 days behind the newest) falls out. + pre.update( + records(cols, [file(3, 1, -10), file(1, 1, 0), file(2, 1, 3)]), + ); + expect(pre.isPinned(3)).toBe(true); + expect(pre.isPinned(2)).toBe(false); + }); + + it("reports progress through status()", async () => { + const cols = [collection(1), collection(2, "favorites")]; + const files = [file(1, 1, 0), file(2, 1, 1), file(10, 2, 0)]; + const cache = new FakeCache(); + const pre = new Precache({ originalsDays: 7 }); + pre.bind(cache); + pre.update(records(cols, files)); + + const before = pre.status(); + expect(before.thumbnailsTotal).toBe(3); + expect(before.thumbnailsCached).toBe(0); + expect(before.originalsPinned).toBe(3); // files 1, 2, 10 all in window + expect(before.originalsCached).toBe(0); + + pre.start(); + await until( + () => + cache.thumbFetched.length === 3 && + cache.originalFetched.length === 3, + ); + const after = pre.status(); + expect(after.thumbnailsCached).toBe(3); + expect(after.originalsCached).toBe(3); + }); + + it("honours the disable flags", async () => { + const cols = [collection(1)]; + const files = [file(1, 1, 0)]; + const cache = new FakeCache(); + const pre = new Precache({ thumbnails: false, originals: false }); + pre.bind(cache); + pre.update(records(cols, files)); + pre.start(); + + await new Promise((r) => setTimeout(r, 20)); + expect(cache.thumbFetched).toEqual([]); + expect(cache.originalFetched).toEqual([]); + expect(pre.isPinned(1)).toBe(false); + expect(pre.status().thumbnailsTotal).toBe(0); + }); +}); + +// ---- Integration through the real ContentCache and Library ---- + +const enteFile = (id: number, collectionID: number): EnteFile => + file(id, collectionID, 0); + +describe("Precache eviction integration", () => { + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-precache-evict-")); + }); + afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); + }); + + it("never evicts a pinned original the precache put in place", async () => { + const cacheDir = join(root, "cache"); + // File 1 is the favorites album's only file (pinned regardless of age); + // files 2 and 3 sit outside the latest-week window, so only file 1 is + // pinned. The by-id map serves the bytes for each fetch. + const cols = [collection(1), collection(2, "favorites")]; + const files = [file(1, 2, 30), file(2, 1, 40), file(3, 1, 50)]; + const byID = new Map<number, EnteFile>(files.map((f) => [f.id, f])); + const source: ContentSource = { + original: async ({ destination }) => { + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + thumbnail: async ({ destination }) => { + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + }; + const statfs: StatFsFn = async () => ({ + bsize: 1, + bavail: 1_000_000_000, + }); + const pre = new Precache({ originalsDays: 7 }); + const cache = new ContentCache({ + pools: new RequestPools(), + source, + cacheDirectory: cacheDir, + getFile: (id) => byID.get(id), + statfs, + cacheOriginalsMaxBytes: 25, // holds two 10-byte originals + freeBelowBytes: 0, + isPinned: (id) => pre.isPinned(id), + }); + pre.bind(cache); + pre.update(records(cols, files)); + expect(pre.isPinned(1)).toBe(true); + expect(pre.isPinned(2)).toBe(false); + await cache.open(); + + // Fill three originals; the 25-byte cap forces an eviction on the + // third, and the pinned file 1 must survive it even though it is the + // least-recently-used. + await cache.original(1); + utimesSync(join(cacheDir, "originals", "1.jpg"), 1000, 1000); // oldest + await cache.original(2); + utimesSync(join(cacheDir, "originals", "2.jpg"), 2000, 2000); + await cache.original(3); + + expect(existsSync(join(cacheDir, "originals", "1.jpg"))).toBe(true); + expect(existsSync(join(cacheDir, "originals", "2.jpg"))).toBe(false); + expect(existsSync(join(cacheDir, "originals", "3.jpg"))).toBe(true); + }); +}); + +describe("Precache preemption", () => { + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-precache-preempt-")); + }); + afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); + }); + + it("lets an on-demand original preempt the background originals fill", async () => { + const byID = new Map<number, EnteFile>([ + [1, enteFile(1, 1)], + [2, enteFile(2, 1)], + [3, enteFile(3, 1)], + ]); + const finished: number[] = []; + let openGate!: () => void; + const gate = new Promise<void>((r) => (openGate = r)); + let sawFirst!: () => void; + const firstStarted = new Promise<void>((r) => (sawFirst = r)); + let started = 0; + const source: ContentSource = { + original: async ({ file: f, destination }) => { + if (++started === 1) sawFirst(); + await gate; + await writeFile(destination, Buffer.alloc(10, 1)); + finished.push(f.id); + return { bytesWritten: 10 }; + }, + thumbnail: async ({ destination }) => { + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + }; + // One content slot, so file 1 holds it while 2 and 3 wait. + const cache = new ContentCache({ + pools: new RequestPools({ contentConcurrency: 1 }), + source, + cacheDirectory: join(root, "cache"), + getFile: (id) => byID.get(id), + statfs: async () => ({ bsize: 1, bavail: 1_000_000_000 }), + freeBelowBytes: 0, + }); + await cache.open(); + + const pA = cache.ensureOriginals({ fileIDs: [1] }); // background + await firstStarted; // file 1 now holds the only slot + const pB = cache.original(2); // on-demand, queued behind file 1 + const pC = cache.ensureOriginals({ fileIDs: [3] }); // background, queued + await new Promise((r) => setTimeout(r, 5)); // let both enqueue + openGate(); + await Promise.all([pA, pB, pC]); + + // On-demand file 2 was served before the background file 3. + expect(finished).toEqual([1, 2, 3]); + }); +}); + +describe("Precache through Library.open", () => { + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-precache-lib-")); + }); + afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); + }); + + class MockClient { + served = false; + whoami(): { email: string; userID: number } { + return { email: "u@example.com", userID: 7 }; + } + async collectionsSince(): Promise<CollectionsPage> { + if (this.served) return { collections: [], deleted: [], cursor: 1 }; + this.served = true; + return { + collections: [collection(1), collection(2, "favorites")], + deleted: [], + cursor: 1, + }; + } + async filesSince(args: { collectionID: number }): Promise<FilesPage> { + const files = + args.collectionID === 1 + ? [enteFile(1, 1), enteFile(2, 1)] + : [enteFile(3, 2)]; + return { files, deleted: [], cursor: 1 }; + } + } + + it("starts both precaches from open() and reports them in status()", async () => { + const source: ContentSource = { + original: async ({ destination }) => { + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + thumbnail: async ({ destination }) => { + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }, + }; + // Each fill reports "done" once the cache has recorded its files. The + // source returning is not enough: the cache records a file only after + // it has checked it on disk. + const finished = new Set<string>(); + let bothFinished!: () => void; + const precached = new Promise<void>((r) => (bothFinished = r)); + const lib = await Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + contentSource: source, + refreshIntervalSeconds: 3600, + onProgress: (e) => { + if ( + e.status === "done" && + (e.operation === "precacheThumbnails" || + e.operation === "precacheOriginals") + ) { + finished.add(e.operation); + if (finished.size === 2) bothFinished(); + } + }, + }); + + // Every file's thumbnail is precached; the favorite (file 3) and the + // week's files (1, 2) all have their originals precached. + await precached; + const status = lib.status(); + expect(status.thumbnailsTotal).toBe(3); + expect(status.thumbnailsCached).toBe(3); + expect(status.originalsPinned).toBe(3); + expect(status.originalsCached).toBe(3); + await lib.close(); + }); + + it.each(["thumbnail", "original"] as const)( + "close() resolves only after a running %s precache fetch has written its file", + async (kind) => { + // Only the fill under test runs, and each of its fetches waits + // until the test releases it. + let release!: () => void; + const held = new Promise<void>((r) => (release = r)); + let fetchStarted!: (destination: string) => void; + const started = new Promise<string>((r) => (fetchStarted = r)); + const fetch = async ({ destination }: { destination: string }) => { + fetchStarted(destination); + await held; + await writeFile(destination, Buffer.alloc(10, 1)); + return { bytesWritten: 10 }; + }; + const unused = async () => { + throw new Error("this fill is turned off"); + }; + const source: ContentSource = + kind === "thumbnail" + ? { original: unused, thumbnail: fetch } + : { original: fetch, thumbnail: unused }; + const lib = await Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + contentSource: source, + refreshIntervalSeconds: 3600, + precacheThumbnails: kind === "thumbnail", + precacheOriginals: kind === "original", + }); + try { + const destination = await started; + + let closed = false; + const closing = lib.close().then(() => { + closed = true; + }); + await new Promise((r) => setTimeout(r, 20)); + expect(closed).toBe(false); + + release(); + await closing; + expect(existsSync(destination)).toBe(true); + } finally { + release(); + await lib.close(); + } + }, + ); +}); diff --git a/test/library/read.test.ts b/test/library/read.test.ts new file mode 100644 index 0000000..2fe394a --- /dev/null +++ b/test/library/read.test.ts @@ -0,0 +1,537 @@ +/** + * Tests for the in-process read surface (issue #44). + * + * Phase 1 (#43) projected the decrypted store into plain `AlbumRecord` / + * `PhotoRecord` values. This phase adds the read API a CLI or in-process script + * uses, all served from RAM with no network: + * + * - `lib.albums` — `list` / `byName` / `byID`, returning thin `Album` wrappers. + * - `lib.photos` — `byID` (a `Photo` wrapper) and `records` (plain records). + * - `lib.timeline.groups` — photos bucketed by local day / week / month. + * + * Every call takes a single named-argument object; there are no positional + * arguments. The wrapper classes are for in-process callers only (they hold + * object identity, not JSON); the plain records remain the IPC-safe surface. + * Content-fetch methods (`Photo.original` / `thumbnail`) are a later unit and + * deliberately absent here — this surface is read-only. + * + * The detailed cases drive the API factories directly over a hand-built + * projection (`deriveRecords`), which keeps them free of disk and timers. A + * final section opens a real `Library` to prove the namespaces are wired to the + * live store and that a read never touches the client. + */ + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + deriveRecords, + type DerivedRecords, +} from "../../src/library/records.js"; +import { + Album, + Photo, + makeAlbumsAPI, + makePhotosAPI, + makeTimelineAPI, + type TimelineGroup, +} from "../../src/library/read.js"; +import { Library } from "../../src/library/index.js"; +import { MetadataStore } from "../../src/library/store.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +const OWNER = 42; + +// Ente stores times in microseconds; records expose milliseconds. These +// helpers keep the fixtures readable: `ms(...)` picks an epoch-millisecond +// instant, `micros(...)` is what the fixture stores so the derived record's +// `takenAt` comes back as the same millisecond value. +const ms = (epochMillis: number): number => epochMillis; +const micros = (epochMillis: number): number => epochMillis * 1000; + +const collection = ( + id: number, + opts: Partial<Collection> = {}, +): Collection => ({ + id, + ownerID: OWNER, + key: new Uint8Array([id & 0xff, 1, 2, 3]), + name: `album-${id}`, + type: "album", + updationTime: micros(1_700_000_000_000), + isShared: false, + ...opts, +}); + +const file = ( + id: number, + collectionID: number, + opts: Partial<EnteFile> & { + creationTime?: number; + title?: string; + fileType?: EnteFile["metadata"]["fileType"]; + latitude?: number; + longitude?: number; + } = {}, +): EnteFile => { + const { creationTime, title, fileType, latitude, longitude, ...rest } = + opts; + const metadata: EnteFile["metadata"] = { + title: title ?? `file-${id}.jpg`, + fileType: fileType ?? "image", + creationTime: creationTime ?? micros(1_700_000_000_000), + modificationTime: micros(1_700_000_000_000), + }; + if (latitude !== undefined) metadata.latitude = latitude; + if (longitude !== undefined) metadata.longitude = longitude; + return { + id, + collectionID, + ownerID: OWNER, + key: new Uint8Array([id & 0xff, 9, 8, 7]), + metadata, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: micros(1_700_000_000_000), + ...rest, + }; +}; + +// Build the three API objects over one fixed projection, the way `Library` +// wires them over its live store. +const apis = (records: DerivedRecords) => { + const derive = () => records; + return { + albums: makeAlbumsAPI(derive), + photos: makePhotosAPI(derive), + timeline: makeTimelineAPI(derive), + }; +}; + +// Every file id present across all timeline groups, in group-then-member order. +const allFileIDs = (groups: TimelineGroup[]): number[] => + groups.flatMap((g) => g.fileIDs); + +describe("lib.albums", () => { + it("lists albums as Album wrappers, newest updated first", () => { + const records = deriveRecords( + [ + collection(1, { updationTime: micros(1_700_000_000_000) }), + collection(2, { updationTime: micros(1_705_000_000_000) }), + ], + [file(10, 1), file(20, 2)], + ); + const albums = apis(records).albums.list(); + expect(albums.every((a) => a instanceof Album)).toBe(true); + // Collection 2 updated later, so it sorts ahead of collection 1. + expect(albums.map((a) => a.collectionID)).toEqual([2, 1]); + }); + + it("exposes album fields and its photos newest first", () => { + const records = deriveRecords( + [ + collection(7, { + name: "Trip", + type: "favorites", + isShared: true, + }), + ], + [ + file(1, 7, { creationTime: micros(1_600_000_000_000) }), + file(2, 7, { creationTime: micros(1_800_000_000_000) }), + file(3, 7, { creationTime: micros(1_700_000_000_000) }), + ], + ); + const album = apis(records).albums.byID({ collectionID: 7 })!; + expect(album.name).toBe("Trip"); + expect(album.type).toBe("favorites"); + expect(album.isShared).toBe(true); + expect(album.fileIDs).toEqual([2, 3, 1]); + + const photos = album.photos.list(); + expect(photos.every((p) => p instanceof Photo)).toBe(true); + expect(photos.map((p) => p.fileID)).toEqual([2, 3, 1]); + + // record() hands back the plain, JSON-safe projection. + expect("key" in album.record()).toBe(false); + }); + + it("finds an album by exact name and returns undefined when absent", () => { + const records = deriveRecords( + [ + collection(1, { name: "Berlin" }), + collection(2, { name: "Paris" }), + ], + [file(10, 1), file(20, 2)], + ); + const { albums } = apis(records); + expect(albums.byName({ albumName: "Paris" })?.collectionID).toBe(2); + expect(albums.byName({ albumName: "paris" })).toBeUndefined(); + expect(albums.byName({ albumName: "Nowhere" })).toBeUndefined(); + }); + + it("returns undefined for an unknown collection id", () => { + const records = deriveRecords([collection(1)], [file(10, 1)]); + expect( + apis(records).albums.byID({ collectionID: 999 }), + ).toBeUndefined(); + }); +}); + +describe("lib.photos", () => { + it("byID returns a Photo wrapper carrying the mapped fields", () => { + const records = deriveRecords( + [collection(1)], + [ + file(1001, 1, { + title: "IMG.jpg", + fileType: "video", + creationTime: micros(1_699_000_000_000), + latitude: 52.52, + longitude: 13.405, + pubMagicMetadata: { caption: "at the lake" }, + }), + ], + ); + const photo = apis(records).photos.byID({ fileID: 1001 })!; + expect(photo).toBeInstanceOf(Photo); + expect(photo.title).toBe("IMG.jpg"); + expect(photo.fileType).toBe("video"); + expect(photo.takenAt).toBe(ms(1_699_000_000_000)); + expect(photo.caption).toBe("at the lake"); + expect(photo.latitude).toBeCloseTo(52.52); + expect(photo.isArchived).toBe(false); + expect(photo.isHidden).toBe(false); + // The wrapper hands back the plain record, IPC-safe. + expect("key" in photo.record()).toBe(false); + }); + + it("byID returns undefined for an unknown file id", () => { + const records = deriveRecords([collection(1)], [file(1, 1)]); + expect(apis(records).photos.byID({ fileID: 999 })).toBeUndefined(); + }); + + it("records() returns plain records in requested order, deduped, skipping unknowns", () => { + const records = deriveRecords( + [collection(1)], + [file(1, 1), file(2, 1), file(3, 1)], + ); + const out = apis(records).photos.records({ + fileIDs: [3, 1, 3, 999, 2], + }); + // Requested order preserved; the repeated 3 appears once; 999 is dropped. + expect(out.map((r) => r.fileID)).toEqual([3, 1, 2]); + // Plain records, not wrappers, and JSON round-trips whole. + expect(out[0]).not.toBeInstanceOf(Photo); + expect(JSON.parse(JSON.stringify(out[0]))).toEqual(out[0]); + }); + + it("emits one record for a file even when it belongs to several albums", () => { + // File 1001 is a member of collections 1 and 2. + const records = deriveRecords( + [collection(1), collection(2)], + [file(1001, 1), file(1001, 2)], + ); + const out = apis(records).photos.records({ fileIDs: [1001, 1001] }); + expect(out).toHaveLength(1); + expect(out[0]!.albumIDs).toEqual([1, 2]); + }); +}); + +describe("lib.timeline grouping", () => { + // Group keys and `startsAt` are computed in local time. Pinning the zone to + // UTC makes the expected values exact and lets the fixtures use `Date.UTC`. + const savedTZ = process.env.TZ; + beforeAll(() => { + process.env.TZ = "UTC"; + }); + afterAll(() => { + if (savedTZ === undefined) delete process.env.TZ; + else process.env.TZ = savedTZ; + }); + + it("buckets by local day, newest group and newest member first", () => { + const records = deriveRecords( + [collection(1)], + [ + file(1, 1, { creationTime: micros(Date.UTC(2024, 0, 15, 9)) }), + file(2, 1, { creationTime: micros(Date.UTC(2024, 0, 15, 18)) }), + file(3, 1, { creationTime: micros(Date.UTC(2024, 0, 16, 12)) }), + file(4, 1, { creationTime: micros(Date.UTC(2024, 1, 1, 12)) }), + ], + ); + const groups = apis(records).timeline.groups({ groupBy: "day" }); + expect(groups.map((g) => g.key)).toEqual([ + "2024-02-01", + "2024-01-16", + "2024-01-15", + ]); + // Group start is local midnight of the day. + expect(groups[2]!.startsAt).toBe(Date.UTC(2024, 0, 15)); + // Within the 2024-01-15 group, the later photo (id 2) is first. + expect(groups[2]!.fileIDs).toEqual([2, 1]); + }); + + it("buckets by week with weeks starting on Monday", () => { + // 2024-01-15 is a Monday; the week runs through Sunday 2024-01-21. + const records = deriveRecords( + [collection(1)], + [ + file(1, 1, { creationTime: micros(Date.UTC(2024, 0, 15, 12)) }), // Mon + file(2, 1, { creationTime: micros(Date.UTC(2024, 0, 17, 12)) }), // Wed + file(3, 1, { creationTime: micros(Date.UTC(2024, 0, 21, 12)) }), // Sun + file(4, 1, { creationTime: micros(Date.UTC(2024, 0, 22, 12)) }), // next Mon + ], + ); + const groups = apis(records).timeline.groups({ groupBy: "week" }); + // ISO week keys: 2024-01-15 is in 2024-W03, the next Monday in 2024-W04. + expect(groups.map((g) => g.key)).toEqual(["2024-W04", "2024-W03"]); + const first = groups.find((g) => g.key === "2024-W03")!; + expect(first.startsAt).toBe(Date.UTC(2024, 0, 15)); + // The Sunday belongs to the Monday-started week, not the next one. + expect(first.fileIDs.sort((a, b) => a - b)).toEqual([1, 2, 3]); + }); + + it("assigns a Sunday to the preceding Monday's week across a month boundary", () => { + // 2024-01-14 is a Sunday; its week started Monday 2024-01-08. + const records = deriveRecords( + [collection(1)], + [file(1, 1, { creationTime: micros(Date.UTC(2024, 0, 14, 12)) })], + ); + const groups = apis(records).timeline.groups({ groupBy: "week" }); + expect(groups.map((g) => g.key)).toEqual(["2024-W02"]); + expect(groups[0]!.startsAt).toBe(Date.UTC(2024, 0, 8)); + }); + + it("buckets by month", () => { + const records = deriveRecords( + [collection(1)], + [ + file(1, 1, { creationTime: micros(Date.UTC(2024, 0, 3, 12)) }), + file(2, 1, { creationTime: micros(Date.UTC(2024, 0, 28, 12)) }), + file(3, 1, { creationTime: micros(Date.UTC(2024, 1, 9, 12)) }), + ], + ); + const groups = apis(records).timeline.groups({ groupBy: "month" }); + expect(groups.map((g) => g.key)).toEqual(["2024-02", "2024-01"]); + expect(groups[1]!.startsAt).toBe(Date.UTC(2024, 0, 1)); + expect(groups[1]!.fileIDs).toEqual([2, 1]); + }); + + it("lists each file once even when it belongs to several albums", () => { + // File 1001 is in collections 1 and 2 but must appear once in a group. + const records = deriveRecords( + [collection(1), collection(2)], + [ + file(1001, 1, { + creationTime: micros(Date.UTC(2024, 0, 15, 12)), + }), + file(1001, 2, { + creationTime: micros(Date.UTC(2024, 0, 15, 12)), + }), + ], + ); + const groups = apis(records).timeline.groups({ groupBy: "day" }); + expect(allFileIDs(groups)).toEqual([1001]); + }); +}); + +describe("lib.timeline uses local time, not UTC", () => { + const savedTZ = process.env.TZ; + afterAll(() => { + if (savedTZ === undefined) delete process.env.TZ; + else process.env.TZ = savedTZ; + }); + + it("buckets by the viewer's local day", () => { + // Kolkata is UTC+5:30 with no DST. An instant at 2024-01-14T20:00Z is + // 2024-01-15 01:30 local, so it belongs to the local day 2024-01-15. + process.env.TZ = "Asia/Kolkata"; + const records = deriveRecords( + [collection(1)], + [file(1, 1, { creationTime: micros(Date.UTC(2024, 0, 14, 20)) })], + ); + const groups = apis(records).timeline.groups({ groupBy: "day" }); + expect(groups[0]!.key).toBe("2024-01-15"); + // Local midnight of 2024-01-15, which is 2024-01-14T18:30Z. + expect(groups[0]!.startsAt).toBe(new Date(2024, 0, 15).getTime()); + expect(groups[0]!.startsAt).toBe(Date.UTC(2024, 0, 14, 18, 30)); + }); +}); + +describe("PhotoFilter", () => { + // A fixture spanning albums, file types, geotags, captions, and the two + // visibility states, all on the same local day so grouping is incidental. + const day = (h: number): number => micros(Date.UTC(2024, 2, 4, h)); + const records = (): DerivedRecords => + deriveRecords( + [ + collection(1, { name: "Holidays" }), + collection(2, { name: "Work" }), + ], + [ + file(1, 1, { + title: "Beach sunset", + creationTime: day(1), + latitude: 1, + longitude: 2, + }), + file(2, 1, { + title: "clip.mov", + fileType: "video", + creationTime: day(2), + }), + file(3, 2, { + title: "invoice scan", + creationTime: day(3), + pubMagicMetadata: { caption: "SUNSET colours" }, + }), + file(4, 2, { + title: "archived note", + creationTime: day(4), + magicMetadata: { visibility: 1 }, + }), + file(5, 2, { + title: "secret", + creationTime: day(5), + magicMetadata: { visibility: 2 }, + }), + ], + ); + + const idsWith = ( + filter: Parameters< + ReturnType<typeof apis>["timeline"]["groups"] + >[0]["filter"], + ): number[] => + allFileIDs( + apis(records()).timeline.groups({ groupBy: "day", filter }), + ).sort((a, b) => a - b); + + it("never includes hidden photos and excludes archived by default", () => { + // No filter: hidden (5) always gone, archived (4) gone unless asked for. + expect(idsWith(undefined)).toEqual([1, 2, 3]); + }); + + it("includes archived photos when includeArchived is set, hidden still never", () => { + expect(idsWith({ includeArchived: true })).toEqual([1, 2, 3, 4]); + }); + + it("filters by album membership", () => { + expect(idsWith({ albumID: 1 })).toEqual([1, 2]); + expect(idsWith({ albumID: 2 })).toEqual([3]); + }); + + it("filters by file type", () => { + expect(idsWith({ fileTypes: ["video"] })).toEqual([2]); + expect(idsWith({ fileTypes: ["image", "video"] })).toEqual([1, 2, 3]); + }); + + it("filters by presence or absence of location", () => { + expect(idsWith({ hasLocation: true })).toEqual([1]); + expect(idsWith({ hasLocation: false })).toEqual([2, 3]); + }); + + it("matches text case-insensitively against title, caption, and album name", () => { + // Title match (case-insensitive): "Beach sunset". + expect(idsWith({ text: "SUNSET" })).toEqual([1, 3]); + // Caption-only match: file 3's caption is "SUNSET colours". + expect(idsWith({ text: "colours" })).toEqual([3]); + // Album-name match: everything in "Holidays". + expect(idsWith({ text: "holiday" })).toEqual([1, 2]); + }); + + it("combines filters", () => { + // Images in album 1 with a location: only file 1. + expect( + idsWith({ albumID: 1, fileTypes: ["image"], hasLocation: true }), + ).toEqual([1]); + }); +}); + +/** + * A minimal mock `Client`, enough for `Library.open` to run its refresh loop. + * The queues are empty, so the background refresh over a seeded cache changes + * nothing; the counters prove that a read never calls the client. + */ +class MockClient { + userID = OWNER; + collectionsCalls = 0; + filesCalls = 0; + + whoami(): { email: string; userID: number } { + return { email: "user@example.com", userID: this.userID }; + } + async collectionsSince(args: { + sinceTime: number; + }): Promise<CollectionsPage> { + this.collectionsCalls++; + return { collections: [], deleted: [], cursor: args.sinceTime }; + } + async filesSince(args: { + collectionID: number; + collectionKey: Uint8Array; + sinceTime: number; + }): Promise<FilesPage> { + this.filesCalls++; + return { files: [], deleted: [], cursor: args.sinceTime }; + } +} + +describe("Library exposes the read surface over its live store", () => { + let dir: string; + + beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), "quak-read-")); + }); + afterAll(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("serves albums, photos, and timeline from RAM without calling the client", async () => { + const cacheDirectory = join(dir, "cache"); + const path = join(cacheDirectory, "metadata.json"); + + // Seed a cache as a prior run left it, so open() serves it at once. + const seed = await MetadataStore.load(path); + seed.userID = OWNER; + seed.collectionsSinceTime = 100; + seed.putCollection(collection(1, { name: "Seeded" })); + seed.putFile( + file(1001, 1, { creationTime: micros(Date.UTC(2024, 5, 1, 12)) }), + ); + await seed.save(); + + const client = new MockClient(); + // A long interval keeps the background timer from firing during the test. + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: 3600, + }); + try { + const collectionsBefore = client.collectionsCalls; + const filesBefore = client.filesCalls; + + expect(lib.albums.list().map((a) => a.name)).toEqual(["Seeded"]); + expect( + lib.albums.byName({ albumName: "Seeded" })?.collectionID, + ).toBe(1); + expect(lib.photos.byID({ fileID: 1001 })?.fileID).toBe(1001); + expect( + lib.photos.records({ fileIDs: [1001] }).map((r) => r.fileID), + ).toEqual([1001]); + const groups = lib.timeline.groups({ groupBy: "month" }); + expect(allFileIDs(groups)).toEqual([1001]); + + // Reads are answered from RAM: no read called the client. + expect(client.collectionsCalls).toBe(collectionsBefore); + expect(client.filesCalls).toBe(filesBefore); + } finally { + await lib.close(); + } + }); +}); diff --git a/test/library/records.test.ts b/test/library/records.test.ts new file mode 100644 index 0000000..cf4acd3 --- /dev/null +++ b/test/library/records.test.ts @@ -0,0 +1,279 @@ +/** + * Tests for the plain-record mapping in `src/library/records.ts` (issue #43). + * + * The library keeps decrypted `Collection`/`EnteFile` objects in RAM, but those + * carry binary keys and cannot cross the Electron IPC boundary. `deriveRecords` + * projects them into plain `AlbumRecord`/`PhotoRecord` values — no keys, no + * `Uint8Array`, JSON-safe — that the GUI process consumes. This file pins: + * + * 1. Field mapping from `metadata` and the two magic-metadata layers, using the + * real Ente field names confirmed against the fixtures in + * `test/cli/metadata-backup.test.ts` (`w`/`h`) and `test/library/store.test.ts` + * (`visibility`): title/takenAt precedence, caption, width/height, geo, + * visibility → isArchived/isHidden, fileType. + * 2. `takenAt` is milliseconds; Ente stores creationTime/editedTime in + * microseconds, so the record divides by 1000. + * 3. Deduplication: one `PhotoRecord` per fileID even when the file belongs to + * several collections, with every membership's collection id in `albumIDs`. + * 4. Ordering: photos and album `fileIDs` are newest first. + * 5. No key material survives the projection. + * 6. `diffRecords` reports exactly what changed between two derivations, and + * returns undefined when nothing changed. + */ + +import { describe, it, expect } from "vitest"; +import { + deriveRecords, + snapshotFrom, + diffRecords, +} from "../../src/library/records.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +const OWNER = 42; + +// Microsecond epoch values, as Ente stores times. 1e15 ≈ 2001 in microseconds. +const T = (micros: number): number => micros; + +const collection = ( + id: number, + opts: Partial<Collection> = {}, +): Collection => ({ + id, + ownerID: OWNER, + key: new Uint8Array([id & 0xff, 1, 2, 3]), + name: `album-${id}`, + type: "album", + updationTime: T(1_700_000_000_000_000), + isShared: false, + ...opts, +}); + +const file = ( + id: number, + collectionID: number, + opts: Partial<EnteFile> & { + creationTime?: number; + title?: string; + } = {}, +): EnteFile => { + const { creationTime, title, ...rest } = opts; + return { + id, + collectionID, + ownerID: OWNER, + key: new Uint8Array([id & 0xff, 9, 8, 7]), + metadata: { + title: title ?? `file-${id}.jpg`, + fileType: "image", + creationTime: creationTime ?? T(1_700_000_000_000_000), + modificationTime: T(1_700_000_000_000_000), + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: T(1_700_000_000_000_000), + ...rest, + }; +}; + +describe("deriveRecords: photo mapping", () => { + it("projects a file into a plain PhotoRecord with no key material", () => { + const f = file(1001, 1, { + metadata: { + title: "IMG_1.jpg", + fileType: "image", + creationTime: T(1_699_000_000_000_000), + modificationTime: T(1_699_000_000_000_000), + latitude: 52.52, + longitude: 13.405, + }, + }); + const { photos } = deriveRecords([collection(1)], [f]); + const rec = photos.get(1001)!; + + expect(rec.fileID).toBe(1001); + expect(rec.albumIDs).toEqual([1]); + expect(rec.title).toBe("IMG_1.jpg"); + expect(rec.fileType).toBe("image"); + expect(rec.latitude).toBeCloseTo(52.52); + expect(rec.longitude).toBeCloseTo(13.405); + expect(rec.isArchived).toBe(false); + expect(rec.isHidden).toBe(false); + + // Safe to send over IPC: no key, no Uint8Array, JSON round-trips whole. + expect("key" in rec).toBe(false); + expect(JSON.parse(JSON.stringify(rec))).toEqual(rec); + }); + + it("takenAt is creationTime converted from microseconds to milliseconds", () => { + const f = file(1001, 1, { creationTime: T(1_699_000_000_000_000) }); + const { photos } = deriveRecords([collection(1)], [f]); + expect(photos.get(1001)!.takenAt).toBe(1_699_000_000_000); + }); + + it("prefers pubMagicMetadata.editedName and editedTime over metadata", () => { + const f = file(1001, 1, { + title: "original.jpg", + creationTime: T(1_699_000_000_000_000), + pubMagicMetadata: { + editedName: "Sunset over the bay", + editedTime: T(1_650_000_000_000_000), + }, + }); + const { photos } = deriveRecords([collection(1)], [f]); + const rec = photos.get(1001)!; + expect(rec.title).toBe("Sunset over the bay"); + expect(rec.takenAt).toBe(1_650_000_000_000); + }); + + it("falls back to metadata when the edited fields are empty or absent", () => { + const f = file(1001, 1, { + title: "original.jpg", + creationTime: T(1_699_000_000_000_000), + pubMagicMetadata: { editedName: "" }, + }); + const { photos } = deriveRecords([collection(1)], [f]); + const rec = photos.get(1001)!; + expect(rec.title).toBe("original.jpg"); + expect(rec.takenAt).toBe(1_699_000_000_000); + }); + + it("maps caption and width/height from the public magic metadata", () => { + const f = file(1001, 1, { + pubMagicMetadata: { + caption: "at the beach", + w: 3000, + h: 2000, + }, + }); + const rec = deriveRecords([collection(1)], [f]).photos.get(1001)!; + expect(rec.caption).toBe("at the beach"); + expect(rec.width).toBe(3000); + expect(rec.height).toBe(2000); + }); + + it("omits optional fields that are absent from the metadata", () => { + const rec = deriveRecords([collection(1)], [file(1001, 1)]).photos.get( + 1001, + )!; + expect("caption" in rec).toBe(false); + expect("width" in rec).toBe(false); + expect("height" in rec).toBe(false); + expect("latitude" in rec).toBe(false); + }); + + it("reads archived and hidden from private magicMetadata.visibility", () => { + const archived = file(1, 1, { magicMetadata: { visibility: 1 } }); + const hidden = file(2, 1, { magicMetadata: { visibility: 2 } }); + const visible = file(3, 1, { magicMetadata: { visibility: 0 } }); + const { photos } = deriveRecords( + [collection(1)], + [archived, hidden, visible], + ); + expect(photos.get(1)).toMatchObject({ + isArchived: true, + isHidden: false, + }); + expect(photos.get(2)).toMatchObject({ + isArchived: false, + isHidden: true, + }); + expect(photos.get(3)).toMatchObject({ + isArchived: false, + isHidden: false, + }); + }); +}); + +describe("deriveRecords: dedup and ordering", () => { + it("emits one PhotoRecord per fileID across memberships, all albums listed", () => { + // File 1001 belongs to collections 1 and 2; 2002 only to 2. + const files = [ + file(1001, 1, { creationTime: T(1_700_000_000_000_000) }), + file(1001, 2, { creationTime: T(1_700_000_000_000_000) }), + file(2002, 2, { creationTime: T(1_710_000_000_000_000) }), + ]; + const { photos } = deriveRecords([collection(1), collection(2)], files); + expect([...photos.keys()].sort((a, b) => a - b)).toEqual([1001, 2002]); + expect(photos.get(1001)!.albumIDs).toEqual([1, 2]); + expect(photos.get(2002)!.albumIDs).toEqual([2]); + }); + + it("orders snapshot photos newest first by takenAt", () => { + const files = [ + file(1, 1, { creationTime: T(1_600_000_000_000_000) }), + file(2, 1, { creationTime: T(1_800_000_000_000_000) }), + file(3, 1, { creationTime: T(1_700_000_000_000_000) }), + ]; + const snap = snapshotFrom(deriveRecords([collection(1)], files), 123); + expect(snap.photos.map((p) => p.fileID)).toEqual([2, 3, 1]); + expect(snap.takenAt).toBe(123); + }); + + it("orders album fileIDs newest first", () => { + const files = [ + file(1, 7, { creationTime: T(1_600_000_000_000_000) }), + file(2, 7, { creationTime: T(1_800_000_000_000_000) }), + file(3, 7, { creationTime: T(1_700_000_000_000_000) }), + ]; + const { albums } = deriveRecords([collection(7)], files); + expect(albums.get(7)!.fileIDs).toEqual([2, 3, 1]); + }); +}); + +describe("deriveRecords: album mapping", () => { + it("carries collection identity, sharing, and the favorites type", () => { + const fav = collection(9, { + name: "Favorites", + type: "favorites", + isShared: true, + updationTime: T(1_705_000_000_000_000), + }); + const rec = deriveRecords([fav], [file(1, 9)]).albums.get(9)!; + expect(rec).toMatchObject({ + collectionID: 9, + name: "Favorites", + type: "favorites", + isShared: true, + updationTime: T(1_705_000_000_000_000), + }); + expect("key" in rec).toBe(false); + expect(JSON.parse(JSON.stringify(rec))).toEqual(rec); + }); +}); + +describe("diffRecords", () => { + const at = 999; + + it("returns undefined when nothing changed", () => { + const a = deriveRecords([collection(1)], [file(1, 1)]); + const b = deriveRecords([collection(1)], [file(1, 1)]); + expect(diffRecords(a, b, at)).toBeUndefined(); + }); + + it("reports added and changed albums and photos and removals", () => { + const before = deriveRecords( + [collection(1), collection(2)], + [file(1, 1), file(2, 2)], + ); + // Collection 2 is gone (album + its only file removed). Collection 1 is + // renamed (changed album), gains file 3, and file 1 is retitled. + const after = deriveRecords( + [collection(1, { name: "renamed" })], + [ + file(1, 1, { + pubMagicMetadata: { editedName: "new title" }, + }), + file(3, 1), + ], + ); + const change = diffRecords(before, after, at)!; + expect(change.refreshedAt).toBe(at); + expect(change.albumIDsRemoved).toEqual([2]); + expect(change.fileIDsRemoved).toEqual([2]); + expect(change.albumsChanged.map((a) => a.collectionID)).toEqual([1]); + expect(change.albumsChanged[0]!.name).toBe("renamed"); + expect( + change.photosChanged.map((p) => p.fileID).sort((x, y) => x - y), + ).toEqual([1, 3]); + }); +}); diff --git a/test/library/snapshot.test.ts b/test/library/snapshot.test.ts new file mode 100644 index 0000000..c0f306c --- /dev/null +++ b/test/library/snapshot.test.ts @@ -0,0 +1,303 @@ +/** + * Tests for `Library.snapshot()` and `Library.subscribe()` (issue #43). + * + * These are the surface the GUI consumes across Electron IPC. `snapshot()` is + * synchronous — it reads the in-RAM store and projects it into plain records + * (no keys) — and `subscribe({ onChange })` delivers a `LibraryChange` whenever + * a background refresh actually changes the derived records. The contracts: + * + * 1. `snapshot()` deduplicates a file across memberships into one record with + * every album id, orders photos newest first, and carries no key material. + * 2. `subscribe` fires on a refresh that changes something, with the exact + * changed and removed sets for both albums and photos. + * 3. A refresh that changes nothing (an empty diff) fires no change. + * 4. `unsubscribe()` stops further delivery. + * + * The client is the same scripted mock used by the refresh-loop tests: no + * crypto, no network. Interval tests use a short real interval and `vi.waitFor`. + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Library } from "../../src/library/index.js"; +import type { LibraryChange } from "../../src/library/records.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +const USER_ID = 42; +const FAST_INTERVAL = 0.02; + +const collection = ( + id: number, + updationTime: number, + name = `album-${id}`, +): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + name, + type: "album", + updationTime, + isShared: false, +}); + +const file = ( + id: number, + collectionID: number, + creationTime: number, +): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime, + modificationTime: creationTime, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: creationTime, +}); + +class MockClient { + userID = USER_ID; + collectionsQueue: CollectionsPage[] = []; + filesByCollection = new Map<number, FilesPage[]>(); + + whoami(): { email: string; userID: number } { + return { email: "user@example.com", userID: this.userID }; + } + + async collectionsSince(args: { + sinceTime: number; + }): Promise<CollectionsPage> { + return ( + this.collectionsQueue.shift() ?? { + collections: [], + deleted: [], + cursor: args.sinceTime, + } + ); + } + + async filesSince(args: { + collectionID: number; + collectionKey: Uint8Array; + sinceTime: number; + }): Promise<FilesPage> { + const queue = this.filesByCollection.get(args.collectionID); + return ( + queue?.shift() ?? { + files: [], + deleted: [], + cursor: args.sinceTime, + } + ); + } + + filesFor(collectionID: number, ...pages: FilesPage[]): void { + this.filesByCollection.set(collectionID, pages); + } +} + +describe("Library.snapshot and Library.subscribe", () => { + let dir: string; + let cacheDirectory: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "quak-snapshot-")); + cacheDirectory = join(dir, "cache"); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("snapshot() dedupes across memberships, orders newest first, holds no keys", async () => { + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100), collection(2, 100)], + deleted: [], + cursor: 100, + }); + // File 1001 is in both collections; 2002 only in collection 2 and newer. + client.filesFor(1, { + files: [file(1001, 1, 1_600_000_000_000_000)], + deleted: [], + cursor: 1_600_000_000_000_000, + }); + client.filesFor(2, { + files: [ + file(1001, 2, 1_600_000_000_000_000), + file(2002, 2, 1_800_000_000_000_000), + ], + deleted: [], + cursor: 1_800_000_000_000_000, + }); + + const lib = await Library.open({ client, cacheDirectory }); + try { + const snap = lib.snapshot(); + + // One record per fileID, newest first, both albums on the shared file. + expect(snap.photos.map((p) => p.fileID)).toEqual([2002, 1001]); + const shared = snap.photos.find((p) => p.fileID === 1001)!; + expect(shared.albumIDs).toEqual([1, 2]); + expect(shared.takenAt).toBe(1_600_000_000_000); + + expect(snap.albums.map((a) => a.collectionID).sort()).toEqual([ + 1, 2, + ]); + + // Nothing carries key material; the whole snapshot is JSON-safe. + expect(JSON.parse(JSON.stringify(snap))).toEqual(snap); + for (const p of snap.photos) expect("key" in p).toBe(false); + for (const a of snap.albums) expect("key" in a).toBe(false); + } finally { + await lib.close(); + } + }); + + it("subscribe fires on a refresh change with the correct changed/removed sets", async () => { + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100), collection(2, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 1_600_000_000_000_000)], + deleted: [], + cursor: 1_600_000_000_000_000, + }); + client.filesFor(2, { + files: [file(2002, 2, 1_600_000_000_000_000)], + deleted: [], + cursor: 1_600_000_000_000_000, + }); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + }); + const changes: LibraryChange[] = []; + const { unsubscribe } = lib.subscribe({ + onChange: (c) => changes.push(c), + }); + try { + // Next refresh: collection 2 (and its file) tombstoned; collection 1 + // gains file 1003. + client.filesFor(1, { + files: [file(1003, 1, 1_700_000_000_000_000)], + deleted: [], + cursor: 1_700_000_000_000_000, + }); + client.collectionsQueue.push({ + collections: [collection(1, 200)], + deleted: [2], + cursor: 200, + }); + + await vi.waitFor(() => expect(changes.length).toBeGreaterThan(0), { + timeout: 2000, + interval: 5, + }); + + const change = changes[0]!; + expect(change.albumIDsRemoved).toEqual([2]); + expect(change.fileIDsRemoved).toEqual([2002]); + expect(change.photosChanged.map((p) => p.fileID)).toEqual([1003]); + expect(change.albumsChanged.map((a) => a.collectionID)).toEqual([ + 1, + ]); + expect(change.refreshedAt).toBeGreaterThan(0); + } finally { + unsubscribe(); + await lib.close(); + } + }); + + it("a refresh that changes nothing fires no change", async () => { + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 1_600_000_000_000_000)], + deleted: [], + cursor: 1_600_000_000_000_000, + }); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + }); + const changes: LibraryChange[] = []; + const { unsubscribe } = lib.subscribe({ + onChange: (c) => changes.push(c), + }); + try { + // Let several empty-diff ticks pass; none may deliver a change. + await new Promise((r) => setTimeout(r, FAST_INTERVAL * 1000 * 6)); + expect(changes).toEqual([]); + } finally { + unsubscribe(); + await lib.close(); + } + }); + + it("unsubscribe stops further delivery", async () => { + const client = new MockClient(); + client.collectionsQueue.push({ + collections: [collection(1, 100)], + deleted: [], + cursor: 100, + }); + client.filesFor(1, { + files: [file(1001, 1, 1_600_000_000_000_000)], + deleted: [], + cursor: 1_600_000_000_000_000, + }); + + const lib = await Library.open({ + client, + cacheDirectory, + refreshIntervalSeconds: FAST_INTERVAL, + }); + const changes: LibraryChange[] = []; + const { unsubscribe } = lib.subscribe({ + onChange: (c) => changes.push(c), + }); + unsubscribe(); + try { + client.collectionsQueue.push({ + collections: [collection(3, 300)], + deleted: [], + cursor: 300, + }); + client.filesFor(3, { + files: [file(3003, 3, 1_700_000_000_000_000)], + deleted: [], + cursor: 1_700_000_000_000_000, + }); + // The change lands in the store, but the cancelled subscriber sees + // nothing. + await vi.waitFor( + () => expect(lib.snapshot().albums.length).toBe(2), + { timeout: 2000, interval: 5 }, + ); + expect(changes).toEqual([]); + } finally { + await lib.close(); + } + }); +}); diff --git a/test/library/store.test.ts b/test/library/store.test.ts new file mode 100644 index 0000000..21212b3 --- /dev/null +++ b/test/library/store.test.ts @@ -0,0 +1,246 @@ +/** + * Tests for the on-disk JSON metadata store (`MetadataStore`). + * + * The store is the local cache the library keeps of the account's server + * state: one `metadata.json` file holding the user id, a schema version, the + * cursor for the incremental collections listing, and the decrypted + * collection and file records. The whole file is read into RAM on load and + * rewritten as a whole on save. A separate refresh unit (issue #42) is what + * populates it; this unit only stores. + * + * Four contracts are load-bearing and each is exercised below: + * + * 1. **Round-trip fidelity.** Everything put into the store — including the + * binary decryption keys, which JSON cannot hold directly and which the + * store base64-encodes — comes back byte-for-byte after a save and a fresh + * load. A cache that quietly dropped or mangled a field would hand the + * caller wrong keys or stale metadata. + * + * 2. **A missing or corrupt file loads as an empty store, never an error.** + * The file is only a cache: if it is absent (first run) or unreadable + * (interrupted write on an older build, disk corruption, hand-editing), + * the right answer is to start empty and let the refresh unit repopulate, + * not to crash the whole library. + * + * 3. **Writes are atomic and durable.** The store reuses the same + * fsync-before-rename atomic writer the download layer uses, so a reader + * never sees a half-written file and a crash cannot leave a truncated one. + * The observable consequence tested here is that a save leaves exactly the + * destination file behind — no temporary sibling — and that overwriting an + * existing store preserves a complete, re-loadable file. + * + * 4. **Permissions match `session.json`.** The directory is `0700` and the + * file is `0600`, because the records contain decrypted key material and + * must not be readable by other users on a shared machine. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + mkdtempSync, + rmSync, + readdirSync, + statSync, + writeFileSync, + mkdirSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + MetadataStore, + METADATA_SCHEMA_VERSION, +} from "../../src/library/store.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; + +// A representative decrypted collection, including a binary key and all three +// magic-metadata layers, so the round-trip test proves every field survives. +const sampleCollection = (): Collection => ({ + id: 12345, + ownerID: 42, + key: new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]), + name: "Holiday 2026", + type: "album", + updationTime: 1_700_000_000_000_000, + isShared: true, + magicMetadata: { visibility: 0 }, + pubMagicMetadata: { subType: 0, coverID: 999 }, + sharedMagicMetadata: { note: "shared with a friend" }, +}); + +// A representative decrypted file membership: metadata, both blob headers, a +// binary key, a content hash, and file/thumbnail sizes. +const sampleFile = (): EnteFile => ({ + id: 67890, + collectionID: 12345, + ownerID: 42, + key: new Uint8Array([9, 8, 7, 6, 5, 4, 3, 2, 1]), + metadata: { + title: "IMG_0001.jpg", + fileType: "image", + creationTime: 1_699_000_000_000_000, + modificationTime: 1_699_000_500_000_000, + latitude: 52.52, + longitude: 13.405, + hash: "sha256:deadbeef", + }, + magicMetadata: { editedName: "sunset" }, + pubMagicMetadata: { editedTime: 1_699_000_600_000_000 }, + file: { decryptionHeader: "ZmlsZUhlYWRlcg==", size: 4_194_304 }, + thumbnail: { decryptionHeader: "dGh1bWJIZWFkZXI=", size: 8192 }, + updationTime: 1_700_000_100_000_000, +}); + +describe("MetadataStore", () => { + let dir: string; + let path: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "quak-store-")); + // Deliberately nest the store one level below the temp dir so save() + // has to create its own directory and set its mode. + path = join(dir, "cache", "metadata.json"); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("round-trips the whole model, keys and all", async () => { + const store = await MetadataStore.load(path); + store.userID = 42; + store.collectionsSinceTime = 1_700_000_000_000_000; + store.putCollection(sampleCollection()); + store.putFile(sampleFile()); + await store.save(); + + const reloaded = await MetadataStore.load(path); + expect(reloaded.userID).toBe(42); + expect(reloaded.collectionsSinceTime).toBe(1_700_000_000_000_000); + + // The binary key must come back as the exact bytes, not a base64 + // string or a plain object of numbered keys. + const collection = reloaded.getCollection(12345); + expect(collection).toEqual(sampleCollection()); + expect(collection?.key).toBeInstanceOf(Uint8Array); + + const file = reloaded.getFile(12345, 67890); + expect(file).toEqual(sampleFile()); + expect(file?.key).toBeInstanceOf(Uint8Array); + + expect(reloaded.listCollections()).toHaveLength(1); + expect(reloaded.listFiles(12345)).toHaveLength(1); + }); + + it("writes the declared schema version", async () => { + const store = await MetadataStore.load(path); + await store.save(); + const reloaded = await MetadataStore.load(path); + expect(reloaded.schemaVersion).toBe(METADATA_SCHEMA_VERSION); + }); + + it("loads an empty store when the file is missing", async () => { + const store = await MetadataStore.load(path); + expect(store.userID).toBe(0); + expect(store.listCollections()).toEqual([]); + expect(store.getCollection(1)).toBeUndefined(); + }); + + it("loads an empty store when the file is corrupt", async () => { + mkdirSync(join(dir, "cache"), { recursive: true }); + writeFileSync(path, "{ this is not valid json ]["); + const store = await MetadataStore.load(path); + expect(store.listCollections()).toEqual([]); + expect(store.listFiles(12345)).toEqual([]); + }); + + it("loads an empty store when the schema version does not match", async () => { + // A cache written by a future build with an incompatible schema is + // discarded rather than misread; the refresh unit repopulates it. + mkdirSync(join(dir, "cache"), { recursive: true }); + writeFileSync( + path, + JSON.stringify({ + schemaVersion: METADATA_SCHEMA_VERSION + 1, + userID: 42, + collectionsSinceTime: 0, + collections: [], + files: [], + }), + ); + const store = await MetadataStore.load(path); + expect(store.userID).toBe(0); + expect(store.listCollections()).toEqual([]); + }); + + it("creates the directory 0700 and the file 0600", async () => { + const store = await MetadataStore.load(path); + store.putCollection(sampleCollection()); + await store.save(); + + // Directory 0700, file 0600: on a shared machine the decrypted keys + // in this file must be readable only by their owner. Mask to the + // permission bits; the file-type bits are not part of the assertion. + expect(statSync(join(dir, "cache")).mode & 0o777).toBe(0o700); + expect(statSync(path).mode & 0o777).toBe(0o600); + }); + + it("leaves exactly the destination behind, with no temp sibling", async () => { + const store = await MetadataStore.load(path); + store.putCollection(sampleCollection()); + await store.save(); + // The atomic writer stages a temporary file and renames it into + // place; on success nothing temporary is left in the directory. + expect(readdirSync(join(dir, "cache"))).toEqual(["metadata.json"]); + }); + + it("overwrites an existing store atomically and stays re-loadable", async () => { + const first = await MetadataStore.load(path); + first.userID = 1; + first.putCollection(sampleCollection()); + await first.save(); + + const second = await MetadataStore.load(path); + second.userID = 2; + second.deleteCollection(12345); + await second.save(); + + const reloaded = await MetadataStore.load(path); + expect(reloaded.userID).toBe(2); + expect(reloaded.getCollection(12345)).toBeUndefined(); + expect(readdirSync(join(dir, "cache"))).toEqual(["metadata.json"]); + }); + + it("deletes a collection together with its file memberships", async () => { + const store = await MetadataStore.load(path); + store.putCollection(sampleCollection()); + store.putFile(sampleFile()); + store.deleteCollection(12345); + expect(store.getCollection(12345)).toBeUndefined(); + expect(store.getFile(12345, 67890)).toBeUndefined(); + expect(store.listFiles(12345)).toEqual([]); + }); + + it("scopes file records to their collection membership", async () => { + // The same underlying file can be a member of two collections, each a + // separate record with its own key. Storing one must not touch the + // other, and lookups are per membership. + const store = await MetadataStore.load(path); + const inA = sampleFile(); + const inB: EnteFile = { + ...sampleFile(), + collectionID: 55555, + key: new Uint8Array([100, 101, 102]), + }; + store.putFile(inA); + store.putFile(inB); + + expect(store.getFile(12345, 67890)?.key).toEqual(inA.key); + expect(store.getFile(55555, 67890)?.key).toEqual(inB.key); + expect(store.listFiles(12345)).toHaveLength(1); + expect(store.listFiles(55555)).toHaveLength(1); + + store.deleteFile(12345, 67890); + expect(store.getFile(12345, 67890)).toBeUndefined(); + expect(store.getFile(55555, 67890)?.key).toEqual(inB.key); + }); +}); diff --git a/test/live-photo.ts b/test/live-photo.ts new file mode 100644 index 0000000..1e3f17e --- /dev/null +++ b/test/live-photo.ts @@ -0,0 +1,77 @@ +/** + * Live photo fixtures for the download, content cache, backup and CLI tests. + * + * Ente stores a live photo as one ZIP holding its image and its video, which + * Ente's clients name `image.<ext>` and `video.<ext>`. The ZIP is built here + * with fflate from small fixed bytes and encrypted the way the server serves a + * file under 4 MiB, as one secretstream chunk. `cdnSource` serves it to the + * real download layer, so a test checks what quak stores for a real one. + */ + +import { createHash } from "node:crypto"; +import { zipSync } from "fflate"; + +import { ApiClient } from "../src/api/client.js"; +import { encryptBlob, init, toBase64 } from "../src/crypto/index.js"; +import { + makeDownloadContentSource, + type ContentSource, +} from "../src/library/content.js"; +import type { EnteFile } from "../src/model/types.js"; + +export const IMAGE = new TextEncoder().encode("the still image"); +export const VIDEO = new TextEncoder().encode("the few seconds of video"); + +// A live photo's ZIP; by default the entries an iPhone's live photo gets. +export const livePhotoZip = ( + entries: Record<string, Uint8Array> = { + "image.heic": IMAGE, + "video.mov": VIDEO, + }, +): Uint8Array => zipSync(entries); + +const blake2b = (bytes: Uint8Array): string => + createHash("blake2b512").update(bytes).digest("base64"); + +// The hash Ente's clients record for a live photo: the unkeyed BLAKE2b-512 of +// the image and of the video, each in standard base64, joined by a colon. +export const livePhotoHash = (image = IMAGE, video = VIDEO): string => + `${blake2b(image)}:${blake2b(video)}`; + +// `file` as a live photo whose original is `zip` and whose recorded hash is +// `hash`, and `body`, what the server serves for it: `zip` encrypted under the +// file's key and header. +export const asLivePhoto = async ( + file: EnteFile, + zip = livePhotoZip(), + hash = livePhotoHash(), +): Promise<{ file: EnteFile; body: Uint8Array }> => { + await init(); + const key = new Uint8Array(32).fill(file.id & 0xff); + const { header, ciphertext } = encryptBlob(zip, key); + return { + file: { + ...file, + key, + metadata: { ...file.metadata, fileType: "livePhoto", hash }, + file: { decryptionHeader: toBase64(header) }, + }, + body: ciphertext, + }; +}; + +// A content source that downloads through the real download layer from a +// stand-in server, which serves `bodies` by file ID and a 404 for any other. +export const cdnSource = (bodies: Map<number, Uint8Array>): ContentSource => + makeDownloadContentSource( + new ApiClient({ + fetch: (async (url: string | URL) => { + const fileID = new URL(String(url)).searchParams.get("fileID"); + const body = bodies.get(Number(fileID)); + return body === undefined + ? new Response("not found", { status: 404 }) + : new Response(body); + }) as typeof globalThis.fetch, + retry: { attempts: 1 }, + }), + ); diff --git a/test/model/decrypt.test.ts b/test/model/decrypt.test.ts index 13af5c2..f0db636 100644 --- a/test/model/decrypt.test.ts +++ b/test/model/decrypt.test.ts @@ -145,7 +145,15 @@ const buildSharedRawCollection = ( const buildRawFile = ( collectionKey: Uint8Array, - opts?: { title?: string; fileType?: number; creationTime?: number }, + opts?: { + // Any JSON value; `undefined` leaves the title out of the metadata. + title?: unknown; + fileType?: number; + creationTime?: number; + info?: { fileSize?: number; thumbSize?: number }; + // Replaces the whole metadata JSON value. + metadata?: unknown; + }, ): RawEnteFile => { const fileKey = sodium.crypto_secretbox_keygen(); const { ciphertext: encFileKey, nonce: fileKeyNonce } = secretboxEncrypt( @@ -153,8 +161,8 @@ const buildRawFile = ( collectionKey, ); - const metadata = { - title: opts?.title ?? "IMG_0001.jpg", + const defaultMetadata = { + title: opts && "title" in opts ? opts.title : "IMG_0001.jpg", fileType: opts?.fileType ?? 0, creationTime: opts?.creationTime ?? 1700000000000000, modificationTime: 1700000000000000, @@ -162,6 +170,8 @@ const buildRawFile = ( longitude: 2.3522, hash: "abcdef1234567890", }; + const metadata = + opts && "metadata" in opts ? opts.metadata : defaultMetadata; // File metadata is encrypted as a single-chunk secretstream blob // (not secretbox). The decryptionHeader is the secretstream init header. const metadataBytes = new TextEncoder().encode(JSON.stringify(metadata)); @@ -186,6 +196,7 @@ const buildRawFile = ( }, file: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)) }, + info: opts?.info, updationTime: 1700000000000000, }; }; @@ -315,6 +326,71 @@ describe("model.decryptFile", () => { expect(file.metadata.longitude).toBeCloseTo(2.3522); }); + it("reads a missing or non-string title as an empty string", () => { + // The server controls the metadata JSON. A title that is not a + // string must not reach code that builds file names from it. + const masterKey = sodium.crypto_secretbox_keygen(); + const { collectionKey } = buildRawCollection(masterKey); + for (const title of [undefined, null, 42, ["a"], { x: "../y" }]) { + const file = decryptFile( + buildRawFile(collectionKey, { title }), + collectionKey, + ); + expect(file.metadata.title).toBe(""); + } + }); + + it("rejects metadata that is not a JSON object", () => { + const masterKey = sodium.crypto_secretbox_keygen(); + const { collectionKey } = buildRawCollection(masterKey); + for (const metadata of [null, "IMG_0001.jpg", 7, []]) { + const raw = buildRawFile(collectionKey, { metadata }); + expect(() => decryptFile(raw, collectionKey)).toThrow( + "file 200: metadata is not a JSON object", + ); + } + }); + + it("reads the recorded content hash, joining an older live photo's two parts", () => { + const masterKey = sodium.crypto_secretbox_keygen(); + const { collectionKey } = buildRawCollection(masterKey); + const hashOf = (metadata: Record<string, unknown>) => + decryptFile( + buildRawFile(collectionKey, { + metadata: { title: "x", ...metadata }, + }), + collectionKey, + ).metadata.hash; + + expect(hashOf({ fileType: 0, hash: "H" })).toBe("H"); + expect( + hashOf({ fileType: 2, hash: "H", imageHash: "I", videoHash: "V" }), + ).toBe("H"); + expect(hashOf({ fileType: 2, imageHash: "I", videoHash: "V" })).toBe( + "I:V", + ); + expect(hashOf({ fileType: 2, imageHash: "I" })).toBeUndefined(); + expect( + hashOf({ fileType: 0, imageHash: "I", videoHash: "V" }), + ).toBeUndefined(); + expect(hashOf({ fileType: 0 })).toBeUndefined(); + expect(hashOf({ fileType: 0, hash: 42 })).toBeUndefined(); + expect( + hashOf({ fileType: 2, imageHash: "I", videoHash: 7 }), + ).toBeUndefined(); + // An empty string counts as absent, not as a hash to match. + expect(hashOf({ fileType: 0, hash: "" })).toBeUndefined(); + expect( + hashOf({ fileType: 2, hash: "", imageHash: "I", videoHash: "V" }), + ).toBe("I:V"); + expect( + hashOf({ fileType: 2, imageHash: "", videoHash: "V" }), + ).toBeUndefined(); + expect( + hashOf({ fileType: 2, imageHash: "I", videoHash: "" }), + ).toBeUndefined(); + }); + it("maps fileType numbers to FileType strings", () => { // Ente uses: 0=image, 1=video, 2=livePhoto const masterKey = sodium.crypto_secretbox_keygen(); @@ -357,4 +433,47 @@ describe("model.decryptFile", () => { expect(() => decryptFile(raw, wrongKey)).toThrow(); }); + + it("carries the file and thumbnail byte sizes from info", () => { + // The server reports the encrypted-blob sizes in `info`; the cache + // needs them without a HEAD request, so decryptFile must copy them + // onto the file and thumbnail blobs. + const masterKey = sodium.crypto_secretbox_keygen(); + const { collectionKey } = buildRawCollection(masterKey); + const raw = buildRawFile(collectionKey, { + info: { fileSize: 4096, thumbSize: 512 }, + }); + + const file = decryptFile(raw, collectionKey); + + expect(file.file.size).toBe(4096); + expect(file.thumbnail.size).toBe(512); + }); + + it("leaves the sizes undefined when the server omits info", () => { + // Older files predate the info field; the sizes must stay undefined + // rather than become 0, so callers can tell "unknown" from "empty". + const masterKey = sodium.crypto_secretbox_keygen(); + const { collectionKey } = buildRawCollection(masterKey); + const raw = buildRawFile(collectionKey); + expect(raw.info).toBeUndefined(); + + const file = decryptFile(raw, collectionKey); + + expect(file.file.size).toBeUndefined(); + expect(file.thumbnail.size).toBeUndefined(); + }); + + it("carries the deletion flag from the diff row", () => { + // The diff marks a deleted row with isDeleted; decryptFile copies it + // onto the file so a caller can tell a deleted row from a live one. + const masterKey = sodium.crypto_secretbox_keygen(); + const { collectionKey } = buildRawCollection(masterKey); + const raw = buildRawFile(collectionKey); + raw.isDeleted = true; + + const file = decryptFile(raw, collectionKey); + + expect(file.isDeleted).toBe(true); + }); }); diff --git a/test/packaging/build-context.test.ts b/test/packaging/build-context.test.ts index 9ffa5a1..d6c24e4 100644 --- a/test/packaging/build-context.test.ts +++ b/test/packaging/build-context.test.ts @@ -2,17 +2,17 @@ // failures are silent. // // Excluding too little: a worktree left under `.claude/` is copied into the -// image, vitest globs its `test/` tree as well as the real one, and the -// containerised `make check` runs the whole suite twice over while reporting -// success. A compiled `bin/quak` is ~100 MB of context nobody needs. +// image, vitest globs its `test/` tree as well as the real one, and the test +// phase runs the whole suite twice over while reporting success. A compiled +// `bin/quak` is ~100 MB of context nobody needs. // // Excluding too much: Prettier 3 reads `.gitignore` as a default ignore file, -// so dropping it from the context silently changes which files -// `make fmt-check` looks at inside the image compared to the host. +// so dropping it from the context silently changes which files the lint +// phase's prettier check looks at compared to `make fmt-check` on the host. // // Neither shows up as a build failure, so they are asserted here. import { describe, expect, it } from "vitest"; -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { join } from "node:path"; @@ -46,4 +46,14 @@ describe(".dockerignore", () => { it("leaves .gitignore in the build context for prettier", () => { expect(dockerignore).not.toContain(".gitignore"); }); + + // BuildKit lets a `Dockerfile.dockerignore` shadow the root one; such a + // file would silently give the build a different, unreviewed context — + // and eslint's flat config does not ignore dot-directories, so a stray + // `.claude/` worktree would be linted. + it("is not shadowed by a Dockerfile.dockerignore", () => { + expect(existsSync(join(repoRoot, "Dockerfile.dockerignore"))).toBe( + false, + ); + }); }); diff --git a/test/packaging/entrypoints.test.ts b/test/packaging/entrypoints.test.ts index 3a09f64..a9b5426 100644 --- a/test/packaging/entrypoints.test.ts +++ b/test/packaging/entrypoints.test.ts @@ -1,6 +1,6 @@ // The package manifest promises three files that only exist after a build: // `main`, `types`, and the `quak` binary. Nothing in the test suite used to -// look at them, and `make check` runs test, lint and fmt-check but never the +// look at them, and `make check` runs the test and lint phases but never the // build, so `tsconfig.json` and `package.json` were free to drift apart. They // did: `rootDir` was `./src` while `include` also pulled in `bin/**/*`, which // is TS6059, and no build had succeeded for as long as that was true. diff --git a/test/packaging/nested-checkout.test.ts b/test/packaging/nested-checkout.test.ts new file mode 100644 index 0000000..af26af6 --- /dev/null +++ b/test/packaging/nested-checkout.test.ts @@ -0,0 +1,50 @@ +// A checkout nested under `.claude/` must not add its tests to this suite. +// The test plants one in a temporary directory next to a real test file and +// asks vitest, with this repo's config, which test files it would run. +import { afterEach, describe, expect, it } from "vitest"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = fileURLToPath(new URL("../../", import.meta.url)); + +let root = ""; + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +const writeTest = (path: string): void => { + mkdirSync(join(root, path, ".."), { recursive: true }); + writeFileSync( + join(root, path), + 'import { it } from "vitest";\nit("runs", () => {});\n', + ); +}; + +describe("vitest.config.ts", () => { + it("does not collect tests from a checkout nested under .claude/", () => { + root = mkdtempSync(join(tmpdir(), "quak-nested-checkout-")); + writeTest("test/real.test.ts"); + writeTest(".claude/worktrees/other/test/real.test.ts"); + + const output = execFileSync( + process.execPath, + [ + join(repoRoot, "node_modules/vitest/vitest.mjs"), + "list", + "--filesOnly", + "--config", + join(repoRoot, "vitest.config.ts"), + "--root", + root, + ], + { cwd: root, encoding: "utf-8" }, + ); + + const files = output.split("\n").filter((line) => line !== ""); + expect(files).toEqual(["test/real.test.ts"]); + }); +}); diff --git a/test/retry/retry.test.ts b/test/retry/retry.test.ts index 7baf8c0..394d96e 100644 --- a/test/retry/retry.test.ts +++ b/test/retry/retry.test.ts @@ -149,8 +149,11 @@ describe("isRetryable: transport failures", () => { }); it("retries an errno carried on the error itself", () => { + // Every errno the classifier names, so none can be reclassified + // unnoticed. for (const code of [ "ECONNRESET", + "ECONNABORTED", "ETIMEDOUT", "EPIPE", "ENOTFOUND", @@ -158,6 +161,8 @@ describe("isRetryable: transport failures", () => { "ECONNREFUSED", "EHOSTUNREACH", "ENETUNREACH", + "ENETRESET", + "ENETDOWN", ]) { expect(isRetryable(errnoError(code))).toBe(true); } @@ -215,6 +220,27 @@ describe("isRetryable: transport failures", () => { looped.cause = looped; expect(isRetryable(looped)).toBe(false); }); + + it("terminates on a cause chain that loops through two errors", () => { + const first: Error & { cause?: unknown } = new Error("first"); + const second = new Error("second", { cause: first }); + first.cause = second; + expect(isRetryable(first)).toBe(false); + }); + + it("reads the error and at most seven causes below it", () => { + // The walk is bounded at eight links. An errno at the eighth link is + // found; one at the ninth is not. + const buried = (causes: number): Error => { + let err = errnoError("ECONNRESET"); + for (let i = 0; i < causes; i++) { + err = new Error(`wrapper ${i}`, { cause: err }); + } + return err; + }; + expect(isRetryable(buried(7))).toBe(true); + expect(isRetryable(buried(8))).toBe(false); + }); }); describe("isRetryable: stream truncation versus corruption", () => { @@ -279,10 +305,9 @@ describe("isSafeToReplay", () => { * that is not the whole question: the other half is "could the first * attempt already have taken effect on the server?". * - * quak's non-idempotent calls are `/users/srp/create-session`, - * `/users/two-factor/verify` (which consumes one of a limited number of - * 2FA attempts) and `/files/thumbnail`. A blind replay of any of them can - * do real damage, so they retry only on the failures that establish no TCP + * The calls this guards are the `POST` and `PUT` requests listed in the + * README under "Endpoints used". A blind replay of some of them can do + * real damage, so they retry only on the failures that establish no TCP * connection to the server ever existed — DNS produced no address, or the * peer refused the connection — and therefore that no request byte can * have been transmitted. @@ -333,6 +358,52 @@ describe("isSafeToReplay", () => { ).toBe(false); expect(isSafeToReplay(new TypeError("fetch failed"))).toBe(false); }); + + it("does not replay any other errno the classifier names", () => { + for (const code of [ + "ECONNRESET", + "ECONNABORTED", + "ETIMEDOUT", + "EPIPE", + "EHOSTUNREACH", + "ENETUNREACH", + "ENETRESET", + "ENETDOWN", + ]) { + expect(isSafeToReplay(errnoError(code))).toBe(false); + } + }); + + it("does not replay a chain that also shows the request may have gone out", () => { + // A connect errno somewhere in the chain is not enough: any other + // errno beside it is doubt, and doubt is not replayed. + const reset = Object.assign( + new Error("read ECONNRESET", { cause: errnoError("ECONNREFUSED") }), + { code: "ECONNRESET" }, + ); + const mixed = new TypeError("fetch failed", { cause: reset }); + expect(isRetryable(mixed)).toBe(true); + expect(isSafeToReplay(mixed)).toBe(false); + }); + + it("does not replay a chain longer than the walk reads", () => { + // Eight connect errnos, then a reset at the ninth link, below the + // limit. The walk never sees the reset, so it cannot rule it out. + const refusedChain = (below: Error | undefined): Error => { + let err = below; + for (let i = 0; i < 8; i++) { + err = Object.assign(new Error(`refused ${i}`, { cause: err }), { + code: "ECONNREFUSED", + }); + } + return err as Error; + }; + expect(isSafeToReplay(refusedChain(errnoError("ECONNRESET")))).toBe( + false, + ); + // The same eight links with nothing below them are replayable. + expect(isSafeToReplay(refusedChain(undefined))).toBe(true); + }); }); // --------------------------------------------------------------------------- diff --git a/test/smoke.test.ts b/test/smoke.test.ts index 77fe023..a33c747 100644 --- a/test/smoke.test.ts +++ b/test/smoke.test.ts @@ -1,9 +1,43 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { readFileSync } from "node:fs"; import { VERSION } from "../src/index.js"; -describe("quak", () => { - it("exports a version string", () => { - expect(typeof VERSION).toBe("string"); - expect(VERSION.length).toBeGreaterThan(0); +const packageVersion = ( + JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf-8"), + ) as { version: string } +).version; + +class ExitCalled extends Error {} + +describe("version", () => { + const argv = process.argv; + + afterEach(() => { + process.argv = argv; + vi.restoreAllMocks(); + }); + + it("exports the version from package.json", () => { + expect(VERSION).toBe(packageVersion); + }); + + // Runs bin/quak.ts with --version. commander prints the version and then + // calls process.exit, which is stubbed to throw so the test survives. + it("reports the version from package.json in quak --version", async () => { + const printed: string[] = []; + vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { + printed.push(String(chunk)); + return true; + }); + vi.spyOn(process, "exit").mockImplementation(() => { + throw new ExitCalled(); + }); + process.argv = ["node", "quak", "--version"]; + + await expect(import("../bin/quak.js")).rejects.toBeInstanceOf( + ExitCalled, + ); + expect(printed.join("").trim()).toBe(packageVersion); }); }); diff --git a/test/thumbnails/thumbnails.test.ts b/test/thumbnails/thumbnails.test.ts index 2743d17..580a312 100644 --- a/test/thumbnails/thumbnails.test.ts +++ b/test/thumbnails/thumbnails.test.ts @@ -6,17 +6,30 @@ * working thumbnails, others return 404 or empty bodies. The tests * verify that the detection and repair logic handles each case correctly. * + * As of issue #52 both helpers take an open `Library` for enumeration and the + * `Client` for the API operations that stay unchanged (the thumbnail existence + * check, and the encrypt-and-upload path). `fixMissingThumbnails` reads each + * original through the library's content cache (`photo.original()`). + * * `fixMissingThumbnails` is the most complex function in quak: it * downloads the original file, generates a JPEG thumbnail with jpeg-js, * encrypts it with secretstream push, gets a presigned upload URL, * uploads to S3, and registers the new thumbnail with the API. The * test verifies each step actually happened and the uploaded data is * a valid encrypted blob that decrypts to a JPEG. + * + * It regenerates thumbnails for baseline JPEGs only, because `jpeg-js` decodes + * only JPEG. A non-JPEG image (PNG, HEIC) or a video is reported as "skipped + * (unsupported)" rather than crashing the decoder into an opaque failure + * (issue #17); the mixed test below locks that distinction down. */ +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; import sodium from "libsodium-wrappers-sumo"; import * as jpegJs from "jpeg-js"; -import { beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { init, toBase64, @@ -27,6 +40,7 @@ import { } from "../../src/crypto/index.js"; import { SRP, SrpServer } from "fast-srp-hap"; import { Client } from "../../src/client.js"; +import { Library } from "../../src/library/index.js"; import { listMissingThumbnails, fixMissingThumbnails, @@ -42,6 +56,7 @@ const TEST_EMAIL = "thumb@example.com"; const TEST_PASSWORD = "thumbpass"; const TEST_OPS = 2; const TEST_MEM = 64 * 1024 * 1024; +const TEST_TIME = 1700000000000000; interface ThumbMockState { verifier: Buffer; @@ -63,8 +78,17 @@ interface ThumbMockState { } let mock: ThumbMockState; +let tmpRoot: string; -const buildThumbMock = async (): Promise<ThumbMockState> => { +// PNG signature bytes — enough for `fixMissingThumbnails` to recognise a +// non-JPEG image and skip it. It need not be a decodable PNG. +const PNG_BYTES = new Uint8Array([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, +]); + +const buildThumbMock = async (opts?: { + extraFormats?: boolean; +}): Promise<ThumbMockState> => { const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES); const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM); const loginSubKeyBytes = deriveLoginSubkey(kek); @@ -102,7 +126,6 @@ const buildThumbMock = async (): Promise<ThumbMockState> => { opsLimit: TEST_OPS, }; - // One collection with 3 files: ok thumbnail, empty thumbnail, 404 thumbnail const collKey = sodium.crypto_secretbox_keygen(); const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); const encCK = sodium.crypto_secretbox_easy(collKey, ckN, masterKey); @@ -118,10 +141,11 @@ const buildThumbMock = async (): Promise<ThumbMockState> => { encryptedName: toBase64(encCN), nameDecryptionNonce: toBase64(cnN), type: "album", - updationTime: 1700000000000000, + updationTime: TEST_TIME, }; - // Generate a real tiny JPEG via jpeg-js + // Generate a real tiny JPEG via jpeg-js, used as the encrypted body of the + // JPEG files so a repair actually decodes and re-encodes real pixels. const w = 100; const h = 80; const pixels = new Uint8Array(w * h * 4); @@ -131,26 +155,32 @@ const buildThumbMock = async (): Promise<ThumbMockState> => { pixels[i + 2] = 0; // B pixels[i + 3] = 255; // A } - const tinyJpeg = jpegJs.encode( - { data: pixels, width: w, height: h }, - 80, - ).data; + const tinyJpeg = new Uint8Array( + jpegJs.encode({ data: pixels, width: w, height: h }, 80).data, + ); const fileKeys: Record<number, Uint8Array> = {}; const fileCiphertexts: Record<number, Uint8Array> = {}; - const rawFiles: Record<string, unknown>[] = []; - for (const fileID of [100, 101, 102]) { + // Build one raw file record: encrypt its metadata and its body under a + // fresh per-file key, and record the key and ciphertext for the mock to + // serve and for the test to verify against. + const makeRawFile = ( + fileID: number, + fileType: number, + title: string, + body: Uint8Array, + ): Record<string, unknown> => { const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen(); fileKeys[fileID] = fk; const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey); const meta = JSON.stringify({ - title: `file-${fileID}.jpg`, - fileType: 0, - creationTime: 1700000000000000, - modificationTime: 1700000000000000, + title, + fileType, + creationTime: TEST_TIME, + modificationTime: TEST_TIME, }); const metaPush = sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); @@ -161,18 +191,17 @@ const buildThumbMock = async (): Promise<ThumbMockState> => { sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, ); - // Encrypt the tiny JPEG as the file body const filePush = sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); const encFile = sodium.crypto_secretstream_xchacha20poly1305_push( filePush.state, - new Uint8Array(tinyJpeg), + body, null, sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, ); fileCiphertexts[fileID] = encFile; - rawFiles.push({ + return { id: fileID, collectionID: 1, ownerID: 42, @@ -186,8 +215,33 @@ const buildThumbMock = async (): Promise<ThumbMockState> => { thumbnail: { decryptionHeader: toBase64(sodium.randombytes_buf(24)), }, - updationTime: 1700000000000000, - }); + // The encrypted size of the thumbnail the server records; large + // enough here that the default encoding fits. + info: { thumbSize: 1_000_000 }, + updationTime: TEST_TIME, + }; + }; + + // Three JPEG files: ok thumbnail, empty thumbnail, 404 thumbnail. + const rawFiles: Record<string, unknown>[] = []; + for (const fileID of [100, 101, 102]) { + rawFiles.push(makeRawFile(fileID, 0, `file-${fileID}.jpg`, tinyJpeg)); + } + const thumbnailBehavior: Record<number, "ok" | "empty" | "404" | "500"> = { + 100: "ok", + 101: "empty", + 102: "404", + }; + + // For the issue #17 mixed test: a non-JPEG image and a video, both with a + // missing (404) thumbnail so they surface in the missing list too. + if (opts?.extraFormats) { + rawFiles.push(makeRawFile(103, 0, "file-103.png", PNG_BYTES)); + rawFiles.push( + makeRawFile(104, 1, "file-104.mp4", new Uint8Array([0, 0, 0, 1])), + ); + thumbnailBehavior[103] = "404"; + thumbnailBehavior[104] = "404"; } return { @@ -206,11 +260,7 @@ const buildThumbMock = async (): Promise<ThumbMockState> => { filesByCollection: { 1: rawFiles }, fileCiphertexts, fileKeys, - thumbnailBehavior: { - 100: "ok", - 101: "empty", - 102: "404", - }, + thumbnailBehavior, uploadedThumbnails: [], }; }; @@ -381,6 +431,57 @@ const countingFetch = ( return { fetch: fake as typeof globalThis.fetch, matched: () => matched }; }; +// Open a library over a mock-backed client. As the CLI does for point commands, +// the background precache is off and the refresh interval is long, and the +// library client omits `fetchMLData` so no background ML fetch runs. The real +// `Client` is still used for the API operations the helpers perform directly. +const openLib = (client: Client): Promise<Library> => + Library.open({ + client: { + whoami: () => client.whoami(), + collectionsSince: (args) => client.collectionsSince(args), + filesSince: (args) => client.filesSince(args), + contentSource: () => client.contentSource(), + }, + cacheDirectory: mkdtempSync(join(tmpRoot, "cache-")), + refreshIntervalSeconds: 3600, + precacheThumbnails: false, + precacheOriginals: false, + }); + +/** The mock's raw record for one file, for a test to change before login. */ +const rawFile = (m: ThumbMockState, fileID: number): Record<string, unknown> => + m.filesByCollection[1]!.find((f) => f.id === fileID)!; + +/** Replace the original the mock serves for one file. */ +const replaceOriginal = ( + m: ThumbMockState, + fileID: number, + body: Uint8Array, +): void => { + const push = sodium.crypto_secretstream_xchacha20poly1305_init_push( + m.fileKeys[fileID]!, + ); + m.fileCiphertexts[fileID] = + sodium.crypto_secretstream_xchacha20poly1305_push( + push.state, + body, + null, + sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, + ); + rawFile(m, fileID).file = { decryptionHeader: toBase64(push.header) }; +}; + +const isOriginalDownload = (url: string): boolean => + url.includes("files.ente.io") || url.includes("/files/download/"); + +const login = (fetch: typeof globalThis.fetch, retry?: RetryOptions) => + Client.login({ + email: TEST_EMAIL, + password: TEST_PASSWORD, + apiOptions: retry ? { fetch, retry } : { fetch }, + }); + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -389,17 +490,21 @@ beforeAll(async () => { await init(); await sodium.ready; mock = await buildThumbMock(); + tmpRoot = mkdtempSync(join(tmpdir(), "quak-thumb-test-")); +}); + +afterAll(() => { + if (tmpRoot && existsSync(tmpRoot)) + rmSync(tmpRoot, { recursive: true, force: true }); }); describe("listMissingThumbnails", () => { it("identifies files with empty and 404 thumbnails, ignores working ones", async () => { - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildThumbFetch(mock) }, - }); + const client = await login(buildThumbFetch(mock)); + const lib = await openLib(client); - const missing = await listMissingThumbnails(client); + const missing = await listMissingThumbnails(lib, client); + lib.close(); // File 100 has a working thumbnail → not reported // File 101 has an empty thumbnail → reported @@ -436,13 +541,11 @@ describe("listMissingThumbnails", () => { buildThumbFetch(failingMock), (url) => url.includes("thumbnails.ente.io") && url.includes("102"), ); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: counted.fetch, retry: { ...noWait } }, - }); + const client = await login(counted.fetch, { ...noWait }); + const lib = await openLib(client); - const missing = await listMissingThumbnails(client); + const missing = await listMissingThumbnails(lib, client); + lib.close(); // Only the genuinely empty thumbnail is reported. expect(missing.map((m) => m.fileID)).toEqual([101]); @@ -475,13 +578,11 @@ describe("listMissingThumbnails", () => { return inner(input, init); }) as typeof globalThis.fetch; - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch, retry: { ...noWait } }, - }); + const client = await login(fetch, { ...noWait }); + const lib = await openLib(client); - const missing = await listMissingThumbnails(client); + const missing = await listMissingThumbnails(lib, client); + lib.close(); expect(missing.map((m) => m.fileID)).toEqual([101]); expect(thumbRequests).toBe(4); @@ -500,32 +601,55 @@ describe("listMissingThumbnails", () => { mockWithDupes.filesByCollection[2] = mockWithDupes.filesByCollection[1]!; - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildThumbFetch(mockWithDupes) }, - }); + const client = await login(buildThumbFetch(mockWithDupes)); + const lib = await openLib(client); - const missing = await listMissingThumbnails(client); + const missing = await listMissingThumbnails(lib, client); + lib.close(); // Should still be 2, not 4 (each file checked only once) expect(missing.length).toBe(2); }); + + it("skips a file another account owns without fetching its thumbnail", async () => { + const otherMock = await buildThumbMock(); + rawFile(otherMock, 102).ownerID = 7; + const logs: string[] = []; + const counted = countingFetch( + buildThumbFetch(otherMock), + (url) => url.includes("thumbnails.ente.io") && url.includes("102"), + ); + const client = await login(counted.fetch); + const lib = await openLib(client); + + const missing = await listMissingThumbnails(lib, client, (msg) => + logs.push(msg), + ); + lib.close(); + + expect(missing.map((m) => m.fileID)).toEqual([101]); + expect(counted.matched()).toBe(0); + expect( + logs.some( + (l) => + l.includes("Skipping file-102.jpg") && + l.includes("another account"), + ), + ).toBe(true); + }); }); describe("fixMissingThumbnails", () => { it("downloads original, generates thumbnail, encrypts, uploads, and registers", async () => { const fixMock = await buildThumbMock(); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildThumbFetch(fixMock) }, - }); + const client = await login(buildThumbFetch(fixMock)); + const lib = await openLib(client); - const results = await fixMissingThumbnails(client, [101]); + const results = await fixMissingThumbnails(lib, client, [101]); + lib.close(); expect(results.length).toBe(1); - expect(results[0]!.success).toBe(true); + expect(results[0]!.status).toBe("fixed"); expect(results[0]!.fileID).toBe(101); expect(results[0]!.title).toBe("file-101.jpg"); expect(results[0]!.collection).toBe("Photos"); @@ -555,48 +679,163 @@ describe("fixMissingThumbnails", () => { it("reports failure for nonexistent file IDs without crashing", async () => { const fixMock = await buildThumbMock(); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildThumbFetch(fixMock) }, - }); + const client = await login(buildThumbFetch(fixMock)); + const lib = await openLib(client); - const results = await fixMissingThumbnails(client, [999]); + const results = await fixMissingThumbnails(lib, client, [999]); + lib.close(); expect(results.length).toBe(1); - expect(results[0]!.success).toBe(false); + expect(results[0]!.status).toBe("failed"); expect(results[0]!.fileID).toBe(999); - expect(results[0]!.error).toContain("not found"); + expect(results[0]!.reason).toContain("not found"); }); it("continues after one file fails and reports mixed results", async () => { const fixMock = await buildThumbMock(); - // Make file 102 fail by removing its ciphertext so download fails + // Make file 102 fail by removing its ciphertext so the download 404s. delete fixMock.fileCiphertexts[102]; - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildThumbFetch(fixMock) }, - }); + const client = await login(buildThumbFetch(fixMock)); + const lib = await openLib(client); - const results = await fixMissingThumbnails(client, [101, 102]); + const results = await fixMissingThumbnails(lib, client, [101, 102]); + lib.close(); expect(results.length).toBe(2); const success = results.find((r) => r.fileID === 101)!; const failure = results.find((r) => r.fileID === 102)!; - expect(success.success).toBe(true); - expect(failure.success).toBe(false); + expect(success.status).toBe("fixed"); + expect(failure.status).toBe("failed"); + }); + + it("skips a non-JPEG image and a video as unsupported, not failed (issue #17)", async () => { + // A PNG and a video both throw inside the JPEG decoder. The helper must + // recognise them up front and report "skipped", distinct from a genuine + // "failed", and must not upload anything for them. The JPEG in the same + // batch is still repaired. + const fixMock = await buildThumbMock({ extraFormats: true }); + const client = await login(buildThumbFetch(fixMock)); + const lib = await openLib(client); + + const results = await fixMissingThumbnails( + lib, + client, + [101, 103, 104], + ); + lib.close(); + + const jpeg = results.find((r) => r.fileID === 101)!; + const png = results.find((r) => r.fileID === 103)!; + const video = results.find((r) => r.fileID === 104)!; + + expect(jpeg.status).toBe("fixed"); + + // The PNG is a still image but not a JPEG: skipped only after its bytes + // are inspected. + expect(png.status).toBe("skipped"); + expect(png.reason).toContain("JPEG"); + + // The video is skipped from its type alone, before any download. + expect(video.status).toBe("skipped"); + expect(video.reason).toContain("video"); + + // Only the JPEG was uploaded; the two skipped files touched no upload. + expect(fixMock.uploadedThumbnails.length).toBe(1); + expect(fixMock.uploadedThumbnails[0]!.fileID).toBe(101); + }); + + it("skips a file another account owns without downloading it", async () => { + // The server accepts a thumbnail only from the file's owner. + const fixMock = await buildThumbMock(); + rawFile(fixMock, 101).ownerID = 7; + const counted = countingFetch( + buildThumbFetch(fixMock), + isOriginalDownload, + ); + const client = await login(counted.fetch); + const lib = await openLib(client); + + const results = await fixMissingThumbnails(lib, client, [101]); + lib.close(); + + expect(results[0]!.status).toBe("skipped"); + expect(results[0]!.reason).toContain("another account"); + expect(counted.matched()).toBe(0); + expect(fixMock.uploadedThumbnails.length).toBe(0); + }); + + it("skips a file whose recorded thumbnail size is 0 without downloading it", async () => { + // The server refuses a thumbnail larger than the one it records, and + // no thumbnail is 0 bytes. + const fixMock = await buildThumbMock(); + rawFile(fixMock, 101).info = { thumbSize: 0 }; + const counted = countingFetch( + buildThumbFetch(fixMock), + isOriginalDownload, + ); + const client = await login(counted.fetch); + const lib = await openLib(client); + + const results = await fixMissingThumbnails(lib, client, [101]); + lib.close(); + + expect(results[0]!.status).toBe("skipped"); + expect(results[0]!.reason).toContain("recorded thumbnail size is 0"); + expect(counted.matched()).toBe(0); + expect(fixMock.uploadedThumbnails.length).toBe(0); + }); + + it("re-encodes smaller until the thumbnail fits the recorded size", async () => { + // A noisy 400x300 JPEG, which the default encoding (quality 50, not + // resized because it is under 720 px) cannot compress below the size + // recorded here: one byte less than that encoding's ciphertext. + const fixMock = await buildThumbMock(); + const w = 400; + const h = 300; + const noisy = new Uint8Array( + jpegJs.encode( + { + data: sodium.randombytes_buf(w * h * 4), + width: w, + height: h, + }, + 90, + ).data, + ); + replaceOriginal(fixMock, 101, noisy); + const decoded = jpegJs.decode(noisy, { + useTArray: true, + formatAsRGBA: true, + }); + const defaultSize = + jpegJs.encode(decoded, 50).data.length + + sodium.crypto_secretstream_xchacha20poly1305_ABYTES; + const recordedSize = defaultSize - 1; + rawFile(fixMock, 101).info = { thumbSize: recordedSize }; + + const client = await login(buildThumbFetch(fixMock)); + const lib = await openLib(client); + + const results = await fixMissingThumbnails(lib, client, [101]); + lib.close(); + + expect(results[0]!.status).toBe("fixed"); + const upload = fixMock.uploadedThumbnails[0]!; + expect(upload.ciphertext.length).toBeLessThanOrEqual(recordedSize); + const decrypted = decryptBlob( + upload.ciphertext, + fromBase64(upload.decryptionHeader), + fixMock.fileKeys[101]!, + ); + expect(decrypted[0]).toBe(0xff); + expect(decrypted[1]).toBe(0xd8); }); }); describe("Client.getApiClient", () => { it("returns the ApiClient when logged in", async () => { - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildThumbFetch(mock) }, - }); + const client = await login(buildThumbFetch(mock)); const api = client.getApiClient(); expect(api).toBeDefined(); @@ -604,11 +843,7 @@ describe("Client.getApiClient", () => { }); it("throws after logout", async () => { - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildThumbFetch(mock) }, - }); + const client = await login(buildThumbFetch(mock)); client.logout(); expect(() => client.getApiClient()).toThrow(/logged out/); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..e39bf41 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,10 @@ +import { configDefaults, defineConfig } from "vitest/config"; + +// vitest does not read .gitignore when looking for tests. A checkout nested +// under .claude/ has its own test/ tree, and without this exclude the suite +// runs once per nested checkout and still reports success. +export default defineConfig({ + test: { + exclude: [...configDefaults.exclude, ".claude/**"], + }, +}); diff --git a/yarn.lock b/yarn.lock index ab845b6..7759c2d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -528,13 +528,6 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== -"@types/libsodium-wrappers-sumo@0.8.2": - version "0.8.2" - resolved "https://registry.yarnpkg.com/@types/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.8.2.tgz#488e8747fbb982fe901020b5afeaddfa63da6830" - integrity sha512-uFOBpg/r21hExVlh2ty8YpDfSR+Yy3Jn8XS4+SSjitbhTxdYq+pBz/49XRxyUFe8SzqujHf/Wu0/O4d+FUtNfQ== - dependencies: - libsodium-wrappers-sumo "*" - "@types/node@22.18.13": version "22.18.13" resolved "https://registry.yarnpkg.com/@types/node/-/node-22.18.13.tgz#a037c4f474b860be660e05dbe92a9ef945472e28" @@ -1076,6 +1069,11 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" +fflate@0.8.3: + version "0.8.3" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.3.tgz#bc27d8eb30343d4d512abb03480202ce65d825fc" + integrity sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA== + file-entry-cache@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f" @@ -1249,7 +1247,7 @@ libsodium-sumo@^0.8.0: resolved "https://registry.yarnpkg.com/libsodium-sumo/-/libsodium-sumo-0.8.4.tgz#6d4687781fa0ad398af14a7df872d5c27cf8cd31" integrity sha512-TMtHShQfVVsaxDygyapvUC3o7YsPgXa/hRWeIgzyFz6w5k/1hirGptCxp1U7XwW3rCskaTTYKgV10v86UiGgNw== -libsodium-wrappers-sumo@*, libsodium-wrappers-sumo@0.8.4: +libsodium-wrappers-sumo@0.8.4: version "0.8.4" resolved "https://registry.yarnpkg.com/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.8.4.tgz#6656a3e7e0551ecce08ddee4bfb501a092eac6fa" integrity sha512-ql7hcgulKZ3ekfa2DGAogcCKsWU0diA/0nArz1CFzh93WQdb46/Kj18ka/Hifq6uA3Ush34Pc6vU/6HXeRwUkg==