sneak/quak
1
0
Fork 0
Code Issues 8 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Compare commits

base: sneak/quak:issue-107-live-photos
Branches Tags
sneak/quak:main sneak/quak:issue-107-live-photos sneak/quak:next2 sneak/quak:next
..
compare: sneak/quak:573de7e1045efe5c101f188a805f4ed67fb3915a
Branches Tags
sneak/quak:issue-107-live-photos sneak/quak:next2 sneak/quak:next sneak/quak:main
1 Commits
issue-107-live-photos .. 573de7e104
Author SHA1 Message Date
sneak 573de7e104 quak logout ends the session on the server (closes #108)
check / check (push) Successful in 47s
Details 
logout now calls POST /users/logout with the saved token through the new
Client.logoutOnServer(), then deletes session.json even when that call
fails; in that case it says the server session could not be ended and
exits 1. It prints the account's cache directory and says it still holds
decrypted data. The default cache path moves into
defaultCacheDirectory(), shared with Library.open, so both name the same
directory.

Model: opus-5-5
 2026-09-23 04:05:54 +00:00
44 changed files with 1309 additions and 3536 deletions
Expand all files Collapse all files
.dockerignore
+1 -1
View File
@@ -1,6 +1,6 @@
# Mirrors .gitignore, with one deliberate exception: .gitignore itself stays
# in the build context, because prettier 3 reads it as a default ignore file
# and dropping it would change what the lint phase's prettier check sees.
# and dropping it would change what `make fmt-check` sees inside the image.
# VCS
.git
Dockerfile
+17 -58
View File
@@ -1,69 +1,28 @@
# Lint phase. The linters are invoked directly rather than through `make
# lint` or `script/lint`, which are themselves a docker build and would
# recurse into a daemon that does not exist in a build step.
# Test and build image: the suite, then the compile.
#
# Linting deliberately does not happen here. `script/lint` is a build of
# Dockerfile.lint, and `script/check` calls `script/lint`, so running
# `make check` in this image would mean running `docker build` inside a
# container. Lint runs exactly once, in Dockerfile.lint; script/cibuild
# builds that first and this second.
# node 22.22.0 on Alpine 3.23.3 (node:22-alpine), 2026-08-09
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS lint
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS check
WORKDIR /app
COPY script/ script/
COPY package.json yarn.lock ./
RUN script/bootstrap
COPY . .
RUN yarn run eslint .
RUN yarn run prettier --check .
# Test phase, same shape and for the same reason. The suite runs without
# verbose output first and is rerun verbosely only if it fails; the timeout
# catches a hung test.
#
# node 22.22.0 on Alpine 3.23.3 (node:22-alpine), 2026-08-09
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS test
WORKDIR /app
COPY script/ script/
COPY package.json yarn.lock ./
RUN script/bootstrap
COPY . .
# Unlike the template, the suite runs as the image's non-root `node` user:
# root ignores directory permissions, so the tests of a destination that is
# not writable would otherwise fail. vitest writes into /app.
RUN chown -R node:node /app
USER node
RUN timeout 90 yarn run vitest run --reporter=dot || \
{ echo "--- Rerunning with verbose for details ---"; \
timeout 90 yarn run vitest run --reporter=verbose; exit 1; }
# Build stage, and the last stage: a plain `docker build .` names no target
# and so builds this one. Nothing is wanted from the two phases above; the
# copies are what make BuildKit build them first, so this image cannot be
# produced unless lint and test passed. A stage appended after this one
# would drop all three out of a plain build.
#
# node 22.22.0 on Alpine 3.23.3 (node:22-alpine), 2026-08-09
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34
WORKDIR /app
COPY --from=lint /app/package.json /dev/null
COPY --from=test /app/package.json /dev/null
COPY script/ script/
COPY package.json yarn.lock ./
RUN script/bootstrap
COPY . .
# The version is computed on the host and passed in, because
# .dockerignore excludes .git.
ARG VERSION=dev
LABEL org.opencontainers.image.version="${VERSION}"
# CHECK_EPOCH is a cache buster: without it Docker serves the test layer from
# cache on an unchanged tree, the suite never executes, and the build still
# exits 0. The guard makes an absent argument a hard failure — an unset ARG
# is the empty string, which is a perfectly stable cache key, so a plain
# `docker build .` would otherwise still get the false green. Fail closed.
ARG CHECK_EPOCH
RUN [ -n "$CHECK_EPOCH" ] || exit 1
RUN make test
ARG CHECK_EPOCH
RUN [ -n "$CHECK_EPOCH" ] || exit 1
RUN make build
Dockerfile.lint
+35
View File
@@ -0,0 +1,35 @@
# Lint image: every lint run happens here, and nowhere else. The repo is
# COPYed into a digest-pinned image and the linters run as build steps, so a
# successful build IS a clean lint. `script/lint` does nothing but build this
# file, which also works where the docker daemon is remote and bind mounts are
# impossible. Nothing that runs inside a container may call `script/lint`:
# that is why Dockerfile no longer runs `make check`.
# node 22.22.0 on Alpine 3.23.3 (node:22-alpine), 2026-08-09
FROM node@sha256:e4bf2a82ad0a4037d28035ae71529873c069b13eb0455466ae0bc13363826e34 AS lint
WORKDIR /app
# Manifests before sources, so the dependency install layer stays cached
# until package.json or yarn.lock changes. script/bootstrap ends in
# `yarn install --frozen-lockfile`; the lint steps below are deliberately
# not cached.
COPY script/ script/
COPY package.json yarn.lock ./
RUN script/bootstrap
COPY . .
# LINT_EPOCH is a cache buster, with the same fail-closed contract as
# CHECK_EPOCH in Dockerfile. No lint cache is wanted: on an unchanged tree
# Docker serves the linter layers in well under a second, having linted
# nothing, and the build still exits 0. The guard makes an absent argument a
# hard failure — an unset ARG is the empty string, which is a perfectly
# stable cache key, so a plain `docker build -f Dockerfile.lint .` would
# otherwise get exactly that false green. Every layer below this one is a
# child of the guard, so a fresh epoch forces all of them to execute.
ARG LINT_EPOCH
RUN [ -n "$LINT_EPOCH" ] || exit 1
# The linters are invoked directly rather than through `make lint`, because
# `make lint` is the build of this file.
RUN yarn run eslint .
RUN yarn run prettier --check .
README.md
+103 -170
View File
@@ -97,18 +97,20 @@ alpine. We provide:
- `script/build` — compile the TypeScript sources into `dist/`, then verify that
the entrypoints `package.json` declares (`main`, `types`, `bin`) are among the
files the compiler wrote, and make the CLI executable (our own extension)
- `script/test` — run the test suite, by building the `test` phase of the
`Dockerfile` (vitest, 90s timeout, verbose rerun on failure); requires docker
- `script/lint` — run eslint and a prettier check, by building the `lint` phase
of the `Dockerfile`; requires docker (see Linting and testing below)
- `script/test` — run the test suite (vitest, hard-capped at 30s where `timeout`
is available, verbose rerun on failure)
- `script/lint` — run eslint and a prettier check, by building
`Dockerfile.lint`; requires docker (see Linting below)
- `script/fmt` — format all files with prettier (writes)
- `script/fmt-check` — check formatting on the host (read-only); standalone, and
not called by `script/check` or `script/precommit`, because `script/lint`
already checks formatting in the container
already checks formatting in the container (see Linting below)
- `script/check` — run all checks: `test`, `lint` (our own extension)
- `script/docker` — build the image, tagged via `script/projectname`
- `script/cibuild` — build the image (what CI runs); its last stage depends on
the `lint` and `test` phases, so this one build lints, tests and compiles
- `script/docker` — build the test and build image, tagged via
`script/projectname`
- `script/cibuild` — cd to the repo root and build both images (what CI runs):
`script/lint` first, then the `Dockerfile` image, which runs `make test` and
`make build`
- `script/precommit` — run by the git pre-commit hook (our own extension); runs
`script/lint`, which checks both lint and formatting, but deliberately not the
tests, so the TDD red-phase commit can land
@@ -117,32 +119,50 @@ alpine. We provide:
`make hooks` installs the pre-commit hook that runs `script/precommit`.
### Linting and testing
### Linting
Linting and testing are phases of the `Dockerfile`. The `lint` phase copies the
repo into a digest-pinned node image and runs eslint and `prettier --check .`;
the `test` phase does the same with the suite. `script/lint` and `script/test`
each build one phase with `docker build --no-cache --target <phase>`. There is
no host lint or test path: docker is required, and that also works where the
docker daemon is remote and bind mounts are impossible.
Linting runs in a container, one way, everywhere. `script/lint` builds
`Dockerfile.lint`, which copies the repo into a digest-pinned node image and
runs eslint and prettier as build steps, so a successful build is a clean lint.
There is no host lint path: docker is required to lint, and that also works
where the docker daemon is remote and bind mounts are impossible.
The last stage of the `Dockerfile` compiles the package, and it copies a file
from each phase, so it cannot be built unless lint and the tests pass. That is
why `script/cibuild` is a single `docker build`: it runs lint and the tests once
each and then compiles.
The formatting check is part of that, not a step beside it. `script/check` and
`script/precommit` therefore call `script/lint` and stop; neither calls
`script/fmt-check` as well, which would run prettier a second time over the same
tree for the same verdict — and the weaker of the two, since the host's prettier
is whatever the working tree has installed. So `make check` and the pre-commit
hook both still fail on a badly formatted tree, and prettier runs exactly once
in each. `test/packaging/lint-once.test.ts` asserts that count by walking the
invocation graph, so a second pass cannot creep back in unnoticed.
Every `docker build` in `script/` passes `--no-cache`. On an unchanged tree
Docker would otherwise serve the lint and test steps from cache, nothing would
run, and the build would still exit 0.
The formatting check is part of the `lint` phase, not a step beside it, so
`script/check` and `script/precommit` do not call `script/fmt-check` as well;
that would run prettier a second time over the same tree for the same verdict.
`script/fmt-check` remains as a standalone entrypoint for asking the formatting
question on the host. Its verdict matches the container's: prettier is pinned to
an exact version, installed from `yarn.lock` under `--frozen-lockfile` in both
places, and reads `.gitignore` as its default ignore file — which is why
`.dockerignore` keeps `.gitignore` in the build context.
question on its own, without docker and without the rest of lint. Its verdict
cannot drift from the container's: prettier is pinned to an exact version,
installed from `yarn.lock` under `--frozen-lockfile` in both places, and reads
`.gitignore` as its default ignore file — which is why `.dockerignore`
deliberately keeps `.gitignore` in the build context.
Lint happens in exactly one place, which constrains the rest of the build.
`script/check` calls `script/lint`, so `make check` cannot run inside a
container without asking for docker inside docker. The image built from
`Dockerfile` therefore runs `make test` and `make build` and does not lint;
`script/cibuild` builds `Dockerfile.lint` first and that image second, so CI
gets both verdicts.
### Build epochs
`script/lint` passes `--build-arg LINT_EPOCH="$(date +%s)"`, and `script/docker`
and `script/cibuild` pass `--build-arg CHECK_EPOCH="$(date +%s)"`. Both
Dockerfiles refuse to build without their argument. This is deliberate: on an
unchanged tree Docker would otherwise serve the linter and test layers from
cache, so nothing would run and the build would still exit 0 — a lint build over
an untouched tree returns success in well under a second, having linted nothing.
A changing epoch invalidates every layer below the guard on every invocation
while leaving the dependency layers above them cached, and the missing-argument
guard means a bare `docker build .` fails loudly instead of quietly reporting a
green it did not earn: an unset build argument is the empty string, which is a
perfectly stable cache key.
## Rationale
@@ -177,9 +197,10 @@ All work on quak is test-driven. No exceptions.
3. Subsequent commits add the implementation and any refactors needed to make
the tests pass.
4. A feature branch can only be merged into `main` when `make check` is green.
`main` is always green. CI runs `script/cibuild`, which builds the
`Dockerfile`: its `lint` and `test` phases, then the compile, so neither a
red branch nor one that does not compile can pass CI.
`main` is always green. CI runs `script/cibuild`, which lints via
`Dockerfile.lint` and then runs `make test` and `make build` in the
`Dockerfile` image, so neither a red branch nor one that does not compile can
pass CI.
5. Tests are the canonical API documentation for this library. Every test file
is commented thoroughly enough that a reader who has never seen quak can
learn how to use it from the tests alone. Comments explain why a behavior
@@ -196,7 +217,7 @@ All work on quak is test-driven. No exceptions.
runs `script/lint` — eslint and the prettier check, in the container — but
not the tests, and so not the full `make check`. This is deliberate so the
TDD red-phase commit (failing tests, no implementation yet) can land. The
`test` phase is part of the image build, which is what CI executes via
suite runs as part of the image build, which is what CI executes via
`script/cibuild`, so a red branch still cannot reach `main`.
## Design
@@ -214,29 +235,18 @@ quak/
auth/ login flow (SRP + email OTP + TOTP), key unwrap
model/ decrypted Collection, File, Metadata types + decrypt fns
download/ streaming file/thumbnail download + decryption
library/ the cache-backed Library: metadata store, read
surface, records, content cache, precache, ML data
and search, request pools
backup.ts resilient full-account backup with dedup
metadata-backup.ts
backup-metadata: all decrypted metadata as JSON
mldata-fetch.ts fetch + decrypt per-file ML data
filename.ts safe file names from server metadata
errors.ts error types shared across layers
retry.ts retry classifier + exponential backoff with jitter
thumbnails.ts detect + regenerate missing thumbnails
client.ts high-level Client class assembled from the above
cli-commands.ts the CLI's commands as functions returning exit codes
cli-output.ts how the CLI prints a file's title and time
cli-read.ts fresh reads for the CLI's read commands
cli-run.ts run a command, print its error, exit with its code
cli-session.ts read the saved session file back into a Client
index.ts public library exports
bin/
quak.ts CLI entrypoint (commander.js)
test/ unit + integration tests (vitest)
Makefile
Dockerfile lint phase, test phase, compile
Dockerfile test suite and compile
Dockerfile.lint eslint and prettier, as build steps
package.json
tsconfig.json
```
@@ -461,23 +471,16 @@ quak helper fix-missing-thumbnails [--file ids] generate + upload missing thumbn
```
Every command runs on the same cache-backed library. The read commands —
`collections`, `files`, `get`, `get-thumb`, `backup-metadata`,
`helper list-missing-thumbnails` and `helper fix-missing-thumbnails` — force a
fresh server round-trip before they answer, so they report current account state
rather than whatever the cache last held. If that round-trip fails, the command
prints the error on one line and exits 1. `--cache-dir` overrides where the
cache lives; without it each account gets its own directory under the per-user
cache path.
`collections`, `files`, `get`, and `get-thumb` — force a fresh server round-trip
before they answer, so they report current account state rather than whatever
the cache last held. `--cache-dir` overrides where the cache lives; without it
each account gets its own directory under the per-user cache path.
`get` and `get-thumb` resolve the file by ID directly, so `--collection` is
accepted for backward compatibility but ignored. For a live photo, `get` writes
its image and its video, each named after the title with its own extension, as
Ente's clients name them (`IMG_0001.heic` and `IMG_0001.mov`). With
`--out PATH`, the image is written to `PATH` and the video beside it, with
`PATH`'s name and the video's extension; a `PATH` with the video's extension is
refused. `backup-metadata --exif` (alias `--all`) additionally downloads each
file to extract full EXIF/IPTC/XMP metadata. The listing and backup commands
support `--json` for machine-readable output.
accepted for backward compatibility but ignored. `backup-metadata --exif` (alias
`--all`) additionally downloads each file to extract full EXIF/IPTC/XMP
metadata. The listing and backup commands support `--json` for machine-readable
output.
`backup-metadata` fetches ML data in requests of up to 200 files. When a request
still fails after its retries, the error is logged, each of its files is written
@@ -488,13 +491,7 @@ on. The exit code is non-zero if any ML data request failed.
only, because the bundled decoder (`jpeg-js`) decodes only JPEG. A non-JPEG
image (PNG, HEIC) or a video is reported as `skipped` (unsupported format), kept
distinct from a `failed` repair, and does not affect the exit code; a genuine
failure still exits non-zero. The server accepts a new thumbnail only from the
file's owner and only when it is no larger than the thumbnail size it records
for the file. So a file another account owns, in an album shared with you, is
skipped by both thumbnail helpers without being fetched, and the fixer skips a
file whose recorded thumbnail size is 0 or unknown. Otherwise the fixer lowers
the quality and size of the thumbnail until it fits, and skips the file if even
the smallest does not.
failure still exits non-zero.
### Backup layout
@@ -503,65 +500,26 @@ the smallest does not.
```
<dir>/
originals/
<fileID>.<ext> actual file content (one per unique file,
two for a live photo: see below)
<fileID>.<ext> actual file content (one per unique file)
<fileID>.json all decrypted metadata for that file
<fileID>.livephoto.json which of a live photo's two files is which
collections/
<name>/
<title> -> ../../originals/<fileID>.<ext> (symlink)
<name>.json collection metadata + file list
failures.json files that failed and have not yet succeeded
```
`failures.json` records each failed file with the kind of failure, how many
times it has been tried and when it was last tried. A file leaves it once it
succeeds, or once it is no longer in the library or in the backup's scope. The
library's `lib.backup({ includeThumbnails: true })` also writes
`thumbnails/<fileID>.jpg` beside `originals/`; `quak backup` does not.
A collection's directory and JSON are named after the collection, and a symlink
after the file's title, both with unsafe characters replaced. When two
collections would get the same name, or two symlinks in one collection the same
name (ignoring case in both), each of them gets its ID added: two albums named
`Trip` become `Trip (10)/` and `Trip (11)/`, and two files titled `IMG_0001.JPG`
become `IMG_0001 (12345).JPG` and `IMG_0001 (12346).JPG`. IDs never change, so a
name stays the same from run to run until such a clash appears or goes away.
A live photo, which Ente stores as one ZIP of its image and its video, is stored
as those two files, which a photo viewer can open: each is
`originals/<fileID>.<ext>` with the extension it has inside the ZIP (for example
`12345.heic` and `12345.mov`), and `<fileID>.livephoto.json` names the two. The
live photo counts as stored only when both files are present and not empty. Its
album folder links both, each named after the title with that file's extension
(`IMG_0001.heic` and `IMG_0001.mov`). A live photo that an earlier version of
quak stored as the ZIP, under the image's name, is replaced by its two files on
the next run, and the ZIP and its link are removed.
Each run removes the symlinks into `originals/` that no longer belong in their
collection's directory, and the directories (and JSON) of collections that were
deleted or renamed. Nothing else in `collections/` is touched: a file or a
symlink you put there stays, and a directory that still holds one after its
symlinks are removed stays too, with its JSON.
Each file is downloaded exactly once regardless of how many collections it
appears in, and written once: straight into `originals/`, with no copy left in
the cache. An original the cache already held is copied from there instead. On
subsequent runs, existing originals are skipped. If a download fails, the error
is logged and the backup continues with the next file. The exit code is non-zero
if any files failed. `quak backup` opens its library with the thumbnail and
originals precache off, so it fetches only what the backup stores.
appears in. On subsequent runs, existing originals are skipped. If a download
fails, the error is logged and the backup continues with the next file. The exit
code is non-zero if any files failed.
Each original is written to a temporary file in the same directory, synced to
disk, and renamed into place, so an original is either complete or absent, even
after a power cut. A downloaded original's temporary file is named
`.quak-<pid>-<random>.tmp`, one copied from the cache
`.quak-backup-<fileID>.<ext>-<pid>-<random>.tmp`. A run that is killed can leave
one of these temporary files behind; the next backup deletes those whose process
is no longer running. The content cache uses the same scheme, and opening a
library deletes the temporary files in the cache whose process is no longer
running, so a download another process has in progress in the same cache is left
alone. The rename replaces whatever was at the destination rather than writing
Each original is copied to a temporary file named
`.quak-backup-<fileID>.<ext>-<pid>-<random>.tmp` in the same directory, synced
to disk, and renamed into place, so an original is either complete or absent,
even after a power cut. A run that is killed can leave one of these temporary
files behind; the next backup deletes those whose process is no longer running.
Downloads and the content cache use the same scheme with `.quak-<random>.tmp`
names. The rename replaces whatever was at the destination rather than writing
through it: a symlink there is replaced, not followed, and the new file has the
temporary file's permissions, not those of the file it replaced.
@@ -571,12 +529,7 @@ temporary file's permissions, not those of the file it replaced.
errors
- [x] Update the API reference section below to match the current implementation
- [x] `make docker` green
- [x] Store live photos in a form a photo viewer can open
(https://git.eeqj.de/sneak/quak/issues/107): unpacked into the image and
the video
Tagging and releases are decided by sneak alone, and happen only when he
declares one.
- [ ] Tag `v1.0.0`
Future (desktop client, separate repo):
@@ -596,11 +549,10 @@ test suite is the canonical, executable documentation — `test/library/` and
### Opening a library
`Library.open(options)` loads the on-disk cache, starts the background refresh
loop, and resolves to a `Library`. On an empty cache it awaits the first
refresh, so it opens onto the account's data whenever the server is reachable;
if that refresh fails, it opens with no data and records the error in
`lib.status()`. On an existing cache it returns immediately and refreshes in the
background, so an unreachable server does not block opening.
loop, and resolves to a `Library`. On an empty cache it awaits the first refresh
so it never opens onto empty data; on an existing cache it returns immediately
and refreshes in the background, so an unreachable server does not block
opening.
`LibraryOptions`:
@@ -665,9 +617,7 @@ An `Album` exposes its record fields and `album.photos.list()` → `Photo[]`
(newest first). A `Photo` exposes its record fields, `photo.record()` →
`PhotoRecord`, and two content methods:
- `await photo.original(opts?)` → `{ path, bytes, videoPath? }` — the
full-resolution file. For a live photo, `path` and `bytes` are its image's and
`videoPath` is its video.
- `await photo.original(opts?)` → `{ path, bytes }` — the full-resolution file.
- `await photo.thumbnail(opts?)` → `{ path, bytes }`.
Both serve from the on-disk content cache when the bytes are present and
@@ -687,7 +637,7 @@ The GUI-facing records hold no key material and no binary, so they survive
- `PhotoRecord`: `fileID`, `albumIDs`, `title`, `takenAt` (milliseconds),
`fileType`, optional `caption` / `width` / `height` / `latitude` /
`longitude`, `isArchived`, `isHidden`, and `thumbnailPath` / `originalPath`
once the bytes are cached (for a live photo, `originalPath` is its image).
once the bytes are cached.
- `AlbumRecord`: `collectionID`, `name`, `type`, `isShared`, `updationTime`, and
`fileIDs` (newest first).
- `LibrarySnapshot`: `{ albums, photos, takenAt }`.
@@ -713,15 +663,12 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange`
default limit 20). quak bundles no text encoder, so `searchByEmbedding` takes
a query vector the caller produced elsewhere.
- `await lib.backup(opts?)` → `BackupResult`. It refreshes, fetches every
in-scope original not already in the backup (and, with `includeThumbnails`,
thumbnails) through the content cache, and rebuilds the on-disk backup tree
with a durable failure ledger. A fetched original is written straight into the
backup's `originals/` and not into the cache, which then counts it as present;
one the cache already held is copied from there. `BackupOptions`:
`downloadDirectory` (falls back to the one `open()` was given),
`includeOriginals` (default `true`), `includeThumbnails` (default `false`),
`onlyAlbumNames`, and `onProgress`. See Backup layout above for the tree it
writes.
in-scope original (and, with `includeThumbnails`, thumbnails) through the
content cache, and rebuilds the on-disk backup tree with a durable failure
ledger. `BackupOptions`: `downloadDirectory` (falls back to the one `open()`
was given), `includeOriginals` (default `true`), `includeThumbnails` (default
`false`), `onlyAlbumNames`, and `onProgress`. See Backup layout above for the
tree it writes.
### Request pools
@@ -738,8 +685,6 @@ Under `cacheDirectory`:
<cacheDirectory>/
metadata.json decrypted account state + refresh cursor
originals/<fileID>.<ext> cached full-resolution files
originals/<fileID>.livephoto.json
which of a live photo's two files is which
thumbnails/<fileID>.jpg cached thumbnails
mldata/
<fileID>.json one decrypted ML payload per file
@@ -752,21 +697,11 @@ When `metadata.json` belongs to a different account than the client's,
originals and thumbnails are kept; they are reached only through the files the
current account's records name.
A live photo's original is cached as in the backup: its image and its video,
each `originals/<fileID>.<ext>` with its own extension, and
`originals/<fileID>.livephoto.json` naming them; the two are evicted together. A
live photo that an earlier version cached as its ZIP is not served, and is
replaced by its two files the next time it is read.
A stored file appears only via an atomic temp-then-rename, so its presence means
it is complete. Every downloaded original (by `quak get`, the cache, or
`backup`) whose metadata records a content hash (`FileMetadata.hash`) is hashed
as it is written: unkeyed BLAKE2b with a 64-byte output, standard base64. A live
photo arrives as a ZIP and is unpacked as it is written; its image and its video
are hashed separately and joined as `<imageHash>:<videoHash>`, and neither is
stored unless both are complete and match. A mismatch stores nothing and fails
the download with an error naming the file ID. An original with no recorded
hash, from a very old client, is stored unchecked.
it is complete. The design also calls for a content-hash comparison against
`FileMetadata.hash` on each fetched original; that check is deferred (issue
https://git.eeqj.de/sneak/quak/issues/68) because the exact hash construction
cannot yet be confirmed against the repo's fixtures.
### Key types by source file
@@ -815,22 +750,20 @@ documents:
commented thoroughly. `main` is always green.
- **Required checks before every commit:** `make lint` must pass — that is
eslint plus the prettier check, and it builds the `lint` phase of the
`Dockerfile`, so it needs docker. The pre-commit hook enforces exactly that.
`make check` (which also runs the tests) must pass before merging to `main`.
`make fmt-check` is available for a host-side formatting check on its own, but
it is not a separate requirement: `make lint` already covers it, and running
both would check formatting twice. Never invoke eslint or prettier directly;
linting runs in the container only.
eslint plus the prettier check, and it builds `Dockerfile.lint`, so it needs
docker. The pre-commit hook enforces exactly that. `make check` (which also
runs the tests) must pass before merging to `main`. `make fmt-check` is
available for a host-side formatting check on its own, but it is not a
separate requirement: `make lint` already covers it, and running both would
check formatting twice. Never invoke eslint or prettier directly; linting runs
in the container only.
- **Formatting:** prettier with 4-space indents and `proseWrap: always` for
markdown. Use `make fmt` to format. Use `yarn` not `npm`.
- **Testing:** vitest. Tests go in `test/` mirroring the `src/` structure.
`make test` must finish in under 60 seconds (the hard cap) and should finish
in under 20. The 90-second `timeout` in the `test` phase of the `Dockerfile`
is a backstop that catches a hung test, not the time limit. Use `mkdtempSync`
for temporary directories, never manual timestamp paths.
`make test` must complete in under 20 seconds. Use `mkdtempSync` for temporary
directories, never manual timestamp paths.
- **Code style:** `const` for everything, `let` if reassignment is needed, never
`var`. Avoid unnecessary comments. No hand-rolled crypto. The
REPO_POLICIES.md
+75 -270
View File
@@ -1,6 +1,6 @@
---
title: Repository Policies
last_modified: 2026-09-08
last_modified: 2026-07-06
---
This document covers repository structure, tooling, and workflow standards. Code
@@ -60,28 +60,17 @@ style conventions are in separate documents:
prerequisite since nvm requires bash. yarn is then pinned via
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
always exact versions. `script/cibuild` runs the CI build: it changes to the
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
with the version; the Gitea workflow calls it. **`script/cibuild` runs
`script/bootstrap` first**, because the workflow checks out the repo and runs
nothing else, while `script/fmt-check` runs the formatter on the host: on a
pristine checkout with nothing installed the run dies there, after the
containerised gates have passed. **The bootstrap alone is not enough**:
`script/bootstrap` installs node and yarn under nvm and leaves neither on the
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
source nvm for the pinned node version before invoking it, exactly as
`script/bootstrap`'s own install step does. A runner carrying nothing but
docker and git then gets through `script/check`. Four further scripts are our
own extensions to the standard: `script/check` runs `script/test`,
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
installs the git pre-commit hook (the `make hooks` target shims to it); and
`script/projectname` (literally that filename) simply outputs the project's
name. Scripts that need the name call `script/projectname` — e.g.
`script/docker` assembles its image tag from it — so those scripts stay
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
the hook itself. Model scripts are at
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
scripts are our own extensions to the standard: `script/check` runs
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
what the git pre-commit hook runs, and it calls `script/check`;
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
target shims to it); and `script/projectname` (literally that filename) simply
outputs the project's name. Scripts that need the name call
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
so those scripts stay byte-identical across all repos. Repo-type-specific
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
`script/precommit`, not in the hook itself. Model scripts are at
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
must document the provided scripts in an **Entrypoints** section (see the
README requirements below).
@@ -100,140 +89,87 @@ style conventions are in separate documents:
contributor should be able to understand the entire development workflow by
reading the Makefile.
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a
`lint` phase and a `test` phase, with the final stage depending on both so the
image cannot be built unless they pass. For non-server repos the final stage
brings up a development environment; for server repos it is the runtime image.
Dockerfiles install development prerequisites by running `script/bootstrap`
rather than duplicating installs inline; COPY `script/` and the dependency
manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before
running it.
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
as a build step so the build fails if the branch is not green. For non-server
repos, the Dockerfile should bring up a development environment and run
`make check`. For server repos, `make check` should run as an early build
stage before the final image is assembled. Dockerfiles install development
prerequisites by running `script/bootstrap` rather than duplicating installs
inline; COPY `script/` and the dependency manifests (`package.json` +
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
layer stays cached until dependencies change.
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
no separate lint file. `script/lint` and `script/test` each build one phase
and nothing else:
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
repos use a multistage build where linting runs in an independent stage based
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
`make fmt-check` and `make lint` before the full build begins. The build stage
then declares an explicit dependency on the lint stage via
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
linting before proceeding to compilation and tests. This ensures lint failures
surface in seconds rather than minutes, without blocking on dependency
download or compilation in the build stage.
```sh
docker build --no-cache --target lint -t "$(script/projectname)-lint" .
docker build --no-cache --target test -t "$(script/projectname)-test" .
```
**A stage that is not the last one in the file is built only when the final
stage's chain depends on it, or when `--target` names it.** That is why the
two gates are always invoked by name here, and why the final stage carries a
`COPY --from=` of a harmless file from each of them: without that edge a
plain `docker build .` builds the last stage alone and exits 0 having linted
and tested nothing.
**Every `docker build` in `script/` is tagged**, here and in
`script/cibuild` and `script/docker`. An untagged build leaves a dangling
image behind on every invocation, on every developer host and every CI
runner; a tagged one replaces the previous image.
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
themselves a `docker build` and would recurse into a daemon that does not
exist in a build step. Formatting is the exception and stays on the host:
`script/fmt` writes the working tree, and `script/fmt-check` is its
read-only twin.
**No lint verdict may come from a host invocation of the linter.** On a
shared host golangci-lint reads a result cache keyed on file content rather
than location, so a second checkout of the same content is served the first
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
exit non-zero with `parallel golangci-lint is running` — a status a caller
cannot tell from real findings. Both have produced wrong verdicts in this
org, in both directions. A container has its own cache, its own `TMPDIR` and
a digest-pinned binary, so neither is reachable.
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
a `COPY` layer only when the copied content changes, so on an unchanged tree
the check `RUN` is served from cache, nothing executes, and the build still
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
four, and there is no fifth — `script/check` runs the two gate phases and
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
not evidence that anything ran: a sub-second build reporting success is a
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
and friends destroy a build cache shared with every other build on the host.
- **The gate phases are separate stages, and the build stage depends on both.**
The lint phase is based on the `golangci/golangci-lint` image (pinned by
hash), so lint failures surface in seconds rather than after a full compile,
and the test phase is based on the Go image. The canonical Go repo
`Dockerfile`:
The standard pattern for a Go repo Dockerfile is:
```dockerfile
# Lint phase
# Lint stage — fast feedback on formatting and lint issues
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
FROM golangci/golangci-lint@sha256:... AS lint
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN golangci-lint run --config .golangci.yml ./...
RUN make fmt-check
RUN make lint
# Test phase
# golang:1.x-alpine, YYYY-MM-DD
FROM golang@sha256:... AS test
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go test -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so this stage cannot run
# unless lint and test passed.
# Build stage
# golang:1.x-alpine, YYYY-MM-DD
FROM golang@sha256:... AS builder
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
WORKDIR /src
# Force BuildKit to run the lint stage before proceeding
COPY --from=lint /src/go.sum /dev/null
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN make test
ARG VERSION=dev
RUN CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X main.Version=${VERSION}" \
-o /app ./cmd/app/
# Runtime stage, and the last one
# Runtime stage
FROM alpine@sha256:...
COPY --from=builder /app /usr/local/bin/app
ENTRYPOINT ["app"]
```
Key points:
- The lint phase uses the `golangci/golangci-lint` image directly (it has
both Go and the linter), so nothing needs installing.
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only
purpose is the ordering edge. BuildKit runs stages in parallel by default,
and a stage nothing depends on is not built at all, so without these two
lines a red gate would not fail the build.
- Keep the runtime stage last, and if you add a stage after it, give it the
same two copies. A plain `docker build .` builds the last stage's chain
and nothing else.
- The lint stage uses the `golangci/golangci-lint` image directly (it
includes both Go and the linter), so there is no need to install the
linter separately.
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
a stage dependency. BuildKit runs stages in parallel by default; without
this line, the build stage would not wait for lint to finish and a lint
failure might not fail the overall build.
- If the project uses `//go:embed` directives that reference build artifacts
(e.g. a web frontend compiled in a separate stage), the lint phase must
(e.g. a web frontend compiled in a separate stage), the lint stage must
create placeholder files so the embed directives resolve. Example:
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
The lint stage should not depend on the actual build output — it exists to
fail fast.
- If the project requires CGO or system libraries for linting (e.g.
`vips-dev`), install them in the lint phase with `apk add`.
- `ARG VERSION=dev` is declared in the stage that compiles and supplied by
`script/docker` and `script/cibuild`; no stage may call `git describe`.
`vips-dev`), install them in the lint stage with `apk add`.
- The build stage runs `make test` after compilation setup. Tests run in the
build stage, not the lint stage, because they may require compiled
artifacts or heavier dependencies.
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
runs `script/cibuild` on push, and checks out the repo as its only other step.
That script bootstraps, runs the gate phases, and then builds the image, so a
successful run means every check passed; a bare `docker build .` does not
carry the same guarantee, because its gate phases may come from the cache. The
image build is uncached and so runs the gate phases a second time. That is the
price of the rule above, and it is worth paying: the image that ships is built
from a run of its own gates rather than from a cache entry.
runs `script/cibuild` (which runs `docker build .`) on push. Since the
Dockerfile already runs `make check`, a successful build implies all checks
pass.
- Use platform-standard formatters: `black` for Python, `prettier` for
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
@@ -253,21 +189,14 @@ style conventions are in separate documents:
module under test to verify it compiles/parses. There is no excuse for
`make test` to be a no-op.
- `make test` must complete in under 60 seconds. That is the hard cap, and a
suite that exceeds it fails. Under 20 seconds is the target. A suite between
20 and 60 seconds is still green, but the overage must be filed as an
improvement bug against that repo. Add a 90-second timeout to the test
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
hard cap so that it catches a genuinely hung test rather than a merely slow
one.
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
Makefile.
- **The test command should use the conditional verbose rerun pattern.** Run
tests without `-v` (verbose) first. If tests fail, automatically rerun with
`-v` to show full output. This keeps CI logs and `docker build` output clean
on success (just package/suite summaries) while providing full diagnostic
detail on failure (every test case, every assertion). The command lives in the
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
Makefile form below is the same pattern for any repo-local invocation:
- **`make test` should use the conditional verbose rerun pattern.** Run tests
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
show full output. This keeps CI logs and `docker build` output clean on
success (just package/suite summaries) while providing full diagnostic detail
on failure (every test case, every assertion). The general shell pattern:
```makefile
test:
@@ -280,24 +209,11 @@ style conventions are in separate documents:
```makefile
test:
@go test -count=1 -timeout 90s -race -cover ./... || \
@go test -timeout 30s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
go test -timeout 30s -race -v ./...; exit 1; }
```
`-count=1` is required on both invocations: it defeats Go's test _result_
cache, so the target cannot report a pass it did not earn, and the rerun
reproduces a failure instead of replaying it. It leaves the build cache
alone, so it costs the runtime of the suite and no recompilation.
Note that this is a second, independent cache, stacked below the Docker
layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26)
addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes;
it does not guarantee `go test` inside that step does any work, because the
`GOCACHE` baked into earlier image layers survives into the re-executed
step. They are two separate defects requiring two separate fixes, and a fix
for one must not be recorded as covering the other.
Python example:
```makefile
@@ -323,83 +239,10 @@ style conventions are in separate documents:
must be in `.gitignore`. No exceptions.
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
language build artifacts, and `node_modules/`. Fetch the standard `.gitignore`
from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when
setting up a new repo. These patterns are written to `.gitignore`'s own
semantics, in which an unanchored pattern already matches at every depth; they
are not a `.dockerignore` and must not be transplanted into one unmodified.
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
across unmodified leaves secrets in the build context.** Docker matches with
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
therefore excludes only the copies at the repository root, while `config/.env`
and `certs/server.key` still reach the context and can land in an image layer
— which is more dangerous than a short file with no secret patterns at all,
because it reads as solved and stops anyone looking. Give every
depth-independent pattern the `**/` prefix and leave only genuinely
root-anchored entries unprefixed: `.git`, and the repo's own host-built
binary, written `/myapp` and never `**/myapp`, which would also match
`cmd/myapp/` and delete the package directory from the context. Matching is
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
also catches something the build needs, re-include it with a negation
(`!docs/example.env`); deleting the pattern reopens the exposure for every
other file it covers. Fetch the standard `.dockerignore` from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
it with the repo's own artifacts.
- **In-repo agent scratch belongs in both files, written to each file's own
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
additional checkout of the repo — so under `COPY . .` the build context
inflates by a multiple of the repo and another session's unreviewed work can
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
prefix, because the prefixed form would also delete any nested directory of
that name from the build. Anchoring carries a known gap that the canonical
`.dockerignore` states in its own comment, since consuming repos receive the
file and not the tracker: the directory is created in the agent's working
directory, so a repo running agents in subdirectories still ships
`services/api/.claude/` and must add its own anchored entry there.
- **Excluding `.git` means `git describe` cannot run inside any build stage, and
it fails quietly there.** In a build stage there is no repository, so
`git describe` writes nothing to stdout, `-X main.Version=` comes out empty,
the binary reports no version at all, and the build still exits 0. Compute the
version on the host and thread it in as a build arg. `script/docker` and
`script/cibuild` do this, byte-identically across repos:
```sh
# Own line: a failing command substitution inside an argument does not
# trip `set -e`, so the inline form degrades to an empty constant.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$(script/projectname)" .
```
`--always` makes an untagged repo yield an abbreviated commit hash rather
than failing, and the `[ -n "$version" ]` line is the single place the
fallback is applied — a live check that fires on a build from an export with
no `.git` and on a repository with no commits yet. Do not fold it into the
substitution as `|| echo unknown`, which makes the guard unreachable. The
Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
the scripts stay byte-identical. One consequence for CI: the standard
checkout action clones shallow and fetches no tags, so a repo that embeds a
tag-derived version must set `fetch-depth: 0` on its checkout step.
- **Verify `.dockerignore` by enumerating the image, not by reading the
patterns.** Plant files at the root _and_ at least two directories deep, build
a probe image that does `COPY . .`, and list what actually landed
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
size is not a substitute: a nested secret is a few bytes, and BuildKit
transfers only the delta from the previous build.
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
Fetch the standard `.gitignore` from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
a new repo.
- **No build artifacts in version control.** Code-derived data (compiled
bundles, minified output, generated assets) must never be committed to the
@@ -415,45 +258,9 @@ style conventions are in separate documents:
- Make all changes on a feature branch. You can do whatever you want on a
feature branch.
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must
_NEVER_ be modified by an agent: fetch it from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it
byte-identical, so that no repo can quietly loosen its own linting. Linter
configuration changes are made to the canonical copy in the `prompts` repo and
reach consuming repos by re-vendoring; an agent may open a PR against
canonical, which only the user merges. One list is exempt from byte-identity,
because it cannot be written once for every repo: the `deny` list of the
`test-support` depguard rule, where a repo names its own test-support packages
by full import path. A repo adds entries there and changes nothing else, and a
re-vendor carries its entries forward. The canonical golangci-lint version is
v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base
image
(`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`,
which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the
only pin, since no repo installs golangci-lint on the host: bumping the
version means changing it and nothing else.
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
`PATH` only, so on an already-provisioned machine the pin is inert and a
version bump is a silent no-op — while the Dockerfile, installing into a clean
image, gets the pinned version, so a local `make check` and `make docker` can
disagree about what the tool even is. The canonical form:
- compares the installed version against the pin over the **whole** version
token; a parser that stops at the first `-` reports `2.12.2` for a host
running `2.12.2-rc1` and skips the install;
- treats absent, non-zero, empty or unrecognised `--version` output as a
mismatch, so the failure direction is a redundant install and never a
skipped one;
- after installing, re-resolves the binary the way callers do — `hash -r`,
then through `PATH`, not through the directory the installer wrote to —
and fails naming the resolved path, since an install that a shadowing
binary hides succeeds while changing nothing any caller sees;
- is actually called, and prints the version on both success paths: a
function defined and never invoked has the same exit status and the same
empty output as one that worked.
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
manually by the user. Fetch from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
- When pinning images or packages by hash, add a comment above the reference
with the version and date (YYYY-MM-DD).
@@ -572,9 +379,7 @@ style conventions are in separate documents:
language-specific config). Everything else goes in a subdirectory. Canonical
subdirectory names:
- `bin/` — executable scripts and tools
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
body is a single call into `internal/` or `pkg/`, no project logic in
`cmd/`
- `cmd/` — Go command entrypoints
- `configs/` — configuration templates and examples
- `deploy/` — deployment manifests (k8s, compose, terraform)
- `docs/` — documentation and markdown (README.md stays in root)
TODO.md
+1 -108
View File
@@ -14,100 +14,10 @@ pre-1.0
# Next Step
None: every issue still open is done on `next` or `next2` and waits for it to
reach `main`.
Tagging and releases are decided by sneak alone, and happen only when he
declares one.
Tag v1.0.0.
# Completed Steps
- 2026-09-23: Live photos are stored as their image and their video (issue 107).
A live photo, which Ente stores as one ZIP, is unpacked as it downloads into
`<fileID>.<ext>` for the image and for the video, each with its extension from
the ZIP, beside `<fileID>.livephoto.json`, which names the two. Both are
checked against the recorded hash and renamed into place only when both are
complete. The backup and the content cache count the live photo as stored only
with both files, the album folder links both, `quak get` writes both, and the
content result gives the video as `videoPath`. A ZIP an earlier version stored
is replaced on the next backup run, and in the cache when the photo is next
read.
- 2026-09-23: Settled the package metadata (issue 6). quak is not published, so
`package.json` is marked `"private": true` and the `files` field is gone.
`engines.node` is `>=22`, the major version `script/bootstrap` and the
`Dockerfile` use. An `exports` map makes `.` and `./package.json` the only
importable paths; `runMetadataBackup`, the thumbnail helpers and their types
stay internal to the CLI.
- 2026-09-23: Brought the README and this file in line with the tree (issue
111). The layout lists `src/library/` and the other source files, the backup
layout names `failures.json` and the optional `thumbnails/`, "Opening a
library" says what happens when the first refresh fails, the Testing section
gives the 60-second hard cap and 20-second target for `make test` and names
the 90-second `timeout` in the `Dockerfile` as the backstop for a hung test,
and "Tag v1.0.0" is no longer listed as the next step.
- 2026-09-23: Tested `quak login` and `backup-metadata --exif` (issue 110).
`loginCommand` takes its login function and its prompts from `CliContext`, and
`bin/quak.ts` passes `Client.login` and the terminal prompts. Tests cover a
login from `QUAK_EMAIL` and `QUAK_PASSWORD` with no prompt, the TOTP prompt, a
failed login, and the saved session's modes, and show that `--exif` and
`--all` each turn on EXIF extraction and that it is off without them.
- 2026-09-23: `quak backup` writes each original once and no longer fills the
cache (issue 106). An original fetched for a backup is written by the download
writer straight into the backup's `originals/`, and the content cache records
it there instead of keeping its own copy; one the cache already held is still
copied. `quak backup` opens its library with the thumbnail and originals
precache off.
- 2026-09-23: `backup-metadata`, `helper list-missing-thumbnails` and
`helper fix-missing-thumbnails` refresh before they answer (issue 100). Each
awaits `lib.fresh()` before reading, so a file added since the cache was
written is included, and a failed refresh prints one line and exits 1 instead
of answering from a stale or empty cache. The README lists them among the
commands that refresh first.
- 2026-09-23: `quak backup` waits for the server refresh and fails when it fails
(issue 99). `lib.backup()` joins a refresh already running or starts one, as
`fresh()` does, and rejects before touching any file when it fails, leaving
`failures.json` as it was, so `quak backup` prints the error as one line and
exits 1 instead of backing up the previous run's file list, or nothing, and
exiting 0.
- 2026-09-23: CLI errors print a message instead of a stack trace (issue 102).
An error a command throws is printed as one `quak: MESSAGE` line on stderr and
the CLI exits 1 once output has drained. The wrapper that does this moved from
`bin/quak.ts` to `src/cli-run.ts`, and `bin/quak.ts` now awaits
`program.parseAsync()`.
- 2026-09-23: Opening a library no longer deletes another process's download in
progress (issue 105). The download writer's temp files are named
`.quak-<pid>-<random>.tmp`, and `removeLeftoverTempFiles`, moved from the
backup into the download module, deletes a `.quak-*.tmp` file only when the
process ID in its name is no longer running. The content cache calls it at
`open()` for `originals/` and `thumbnails/`, the backup as before.
- 2026-09-23: Re-vendored the lint and test setup from the template (issue 96).
Linting and testing are the `lint` and `test` phases of the `Dockerfile`;
`script/lint` and `script/test` each build one with `--no-cache`, and the last
stage compiles and depends on both, so `script/cibuild` is one build.
`Dockerfile.lint`, `CHECK_EPOCH`, `LINT_EPOCH` and the tests that checked them
are gone; `REPO_POLICIES.md` is re-copied.
- 2026-09-23: Stopped `helper fix-missing-thumbnails` retrying files the server
always refuses (issue 109). Both thumbnail helpers skip a file another account
owns without fetching it. The fixer skips a file whose recorded thumbnail size
is 0 or unknown before downloading it, and otherwise tries smaller encodings
(720 px quality 50 down to 160 px quality 20) until the encrypted thumbnail is
no larger than that size, skipping the file if none fits.
- 2026-09-23: Tested the live-photo hash check's error paths (issue 117). Tests
download a live photo whose ZIP names an unknown compression method, one whose
ZIP has no image entry and one with no video entry, and check that nothing is
stored and the error names the file ID; the unreadable one is not retried.
- 2026-09-23: `quak logout` ends the session on the server (issue 108). It calls
`POST /users/logout` through the new `Client.logoutOnServer()`, then deletes
`session.json` even when that call fails, says so and exits 1. It prints the
@@ -115,23 +25,6 @@ declares one.
cache path is now `defaultCacheDirectory()` in the library, shared with
`Library.open`.
- 2026-09-23: Fixed the backup's per-collection folders (issue 103). Two files
in one collection with the same title, and two collections with the same name,
each get their ID added to the name (`IMG_0001 (12345).JPG`, `Trip (10)/`), so
none replaces another's symlink or JSON. Each run removes symlinks into
`originals/` for files no longer in the collection, and the folders of deleted
or renamed collections, leaving anything else in `collections/` alone. The
README backup layout states the naming rule.
- 2026-09-23: Checked downloaded originals against their recorded content hash
(issue 68). `downloadFile`, which `quak get`, the content cache and backup all
use, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and
stores nothing on a mismatch, failing with an error naming the file ID. A live
photo ZIP is unpacked as it streams with `fflate` and its image and video
hashed separately as `<imageHash>:<videoHash>`. `decryptFile` reads older
clients' `imageHash` and `videoHash` fields for live photos. A file with no
recorded hash is stored unchecked.
- 2026-09-23: Kept one account's cache from mixing with another's (issue 104).
When `metadata.json` in the cache directory was written for a different,
non-zero user ID than the client's, `Library.open` deletes it and `mldata/`
bin/quak.ts
+18 -9
View File
@@ -1,7 +1,6 @@
#!/usr/bin/env node
import { stdout, stderr } from "node:process";
import { input, password } from "@inquirer/prompts";
import { Command } from "commander";
import envPaths from "env-paths";
import { init } from "../src/crypto/index.js";
@@ -19,9 +18,7 @@ import {
listMissingThumbnailsCommand,
fixMissingThumbnailsCommand,
} from "../src/cli-commands.js";
import { run as runCommand } from "../src/cli-run.js";
import { loadSession } from "../src/cli-session.js";
import { Client } from "../src/client.js";
import { VERSION } from "../src/index.js";
const paths = envPaths("quak", { suffix: "" });
@@ -44,13 +41,25 @@ const context = (): CliContext => ({
sessionDir: paths.data,
cacheDir: program.opts<{ cacheDir?: string }>().cacheDir,
loadSession,
login: (opts) => Client.login(opts),
prompt: (message) => input({ message }),
promptSecret: (message) => password({ message, mask: true }),
});
const run = (command: Promise<number>): Promise<void> =>
runCommand(command, stdout, stderr, (code) => process.exit(code));
// Run a command and exit with its code once stdout/stderr have drained.
// Exiting before the drain can truncate piped output, and the library can keep
// the event loop alive after a command returns, so a plain return could hang.
const run = async (command: Promise<number>): Promise<void> => {
process.exitCode = await command;
const pending = [stdout, stderr].filter((s) => s.writableLength > 0);
if (pending.length === 0) {
process.exit();
return;
}
let remaining = pending.length;
for (const s of pending) {
s.once("drain", () => {
if (--remaining === 0) process.exit();
});
}
};
program
.command("login")
@@ -160,4 +169,4 @@ helper
);
await init();
await program.parseAsync();
program.parse();
package.json
+5 -12
View File
@@ -9,23 +9,17 @@
"type": "git",
"url": "https://git.eeqj.de/sneak/quak.git"
},
"private": true,
"type": "module",
"engines": {
"node": ">=22"
},
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
"exports": {
".": {
"types": "./dist/src/index.d.ts",
"import": "./dist/src/index.js"
},
"./package.json": "./package.json"
},
"bin": {
"quak": "./dist/bin/quak.js"
},
"files": [
"dist/",
"README.md",
"LICENSE"
],
"scripts": {
"build": "script/build",
"quak": "node ./dist/bin/quak.js",
@@ -48,7 +42,6 @@
"env-paths": "4.0.0",
"exif-reader": "2.0.3",
"fast-srp-hap": "2.0.4",
"fflate": "0.8.3",
"jpeg-js": "0.4.4",
"libsodium-wrappers-sumo": "0.8.4"
}
script/check
+13 -5
View File
@@ -1,11 +1,19 @@
#!/bin/sh
# script/check: run all checks (test, lint). Our own extension to
# scripts-to-rule-them-all. Both are Docker phases. Must not modify any
# files.
# scripts-to-rule-them-all. Must not modify any files.
#
# script/fmt-check is not called here, unlike the template: the lint
# phase already runs `prettier --check .`, so calling it would run
# prettier a second time over the same tree for the same verdict.
# The formatting check is part of lint, not a step of its own:
# script/lint builds Dockerfile.lint, which runs eslint AND
# `prettier --check .` as build steps. Calling script/fmt-check here as
# well would run prettier a second time over the same tree for the same
# verdict — the weaker of the two, since the host toolchain is whatever
# the working tree happens to have installed while the container's is
# digest-pinned. script/fmt-check remains a standalone entrypoint for
# asking the formatting question by itself.
#
# script/lint builds Dockerfile.lint, so this script requires docker and
# must never be run from inside a container: that is why the Dockerfile
# image runs script/test and script/build rather than this.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
script/cibuild
+13 -17
View File
@@ -1,11 +1,15 @@
#!/bin/sh
# script/cibuild: run the CI build. The image's last stage depends on the
# lint and test phases, so this one build runs eslint, prettier and the
# suite once each and then compiles. Unlike the template it does not run
# script/check first, which would run lint and the tests a second time.
# --no-cache for the same reason as script/docker: the gate phases the
# final stage depends on are RUN steps, and a cached one is a check that
# did not run.
# script/cibuild: run the CI build, which is both images in a defined order.
#
# First script/lint, which builds Dockerfile.lint and is the one and only
# place linting happens — it goes first so a lint failure is reported before
# the slower suite runs. Then the Dockerfile image, which runs script/test
# and script/build. CHECK_EPOCH and LINT_EPOCH differ on every invocation, so
# neither the linters nor the suite can be served from Docker's cache: a
# green build here means the checks ran now, not that a previous run was
# remembered. The layers below the epochs (bootstrap, yarn install) are
# unaffected and stay cached. A build that omits the arguments fails by
# design.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -13,16 +17,8 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
# Own line: a failing command substitution inside an argument does
# not trip `set -e`, so the inline form degrades silently to an
# empty constant. VERSION is computed here because .dockerignore
# excludes .git, so `git describe` in a build stage yields an empty
# version without failing.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$("$SCRIPT_DIR/projectname")" .
"$SCRIPT_DIR/lint"
docker build --build-arg CHECK_EPOCH="$(date +%s)" .
}
main "$@"
script/docker
+5 -11
View File
@@ -1,8 +1,10 @@
#!/bin/sh
# script/docker: build the Docker image tagged with the project name.
# Identical in all repos; the tag comes from script/projectname.
# --no-cache because the gate phases the final stage depends on are RUN
# steps, and a cached one is a check that did not run.
# CHECK_EPOCH is passed for the same reason script/cibuild passes it: the
# Dockerfile refuses to build without it, so that no path to an image can
# quietly serve the test and build layers from cache. This builds the test
# and build image only; linting is a separate image, built by script/lint.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -10,15 +12,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
# Own line: a failing command substitution inside an argument does
# not trip `set -e`, so the inline form degrades silently to an
# empty constant. VERSION is computed here because .dockerignore
# excludes .git, so `git describe` in a build stage yields an empty
# version without failing.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
docker build --no-cache \
--build-arg VERSION="$version" \
docker build --build-arg CHECK_EPOCH="$(date +%s)" \
-t "$("$SCRIPT_DIR/projectname")" .
}
script/lint
+14 -13
View File
@@ -1,23 +1,24 @@
#!/bin/sh
# script/lint: run the linter. Linting is a phase of the Dockerfile and
# this builds that phase alone; the linter is never installed or run on
# a developer host, where a shared result cache and a host-global lock
# make its answer untrustworthy.
# script/lint: run the linters. eslint and prettier are never run against
# the working tree from here: linting runs via docker only, one way,
# everywhere — script/lint builds Dockerfile.lint, which COPYs the repo into
# the pinned node image and runs the linters as build steps. That works even
# when the docker daemon is remote and bind mounts are impossible.
#
# The phase is not the last stage in the file, so it is built only when
# --target names it. --no-cache because a cached lint layer is a lint
# that did not run. The tag makes each build replace the previous image
# instead of leaving a dangling one behind.
# LINT_EPOCH is passed on every invocation because no lint cache is wanted:
# on an unchanged tree Docker would otherwise serve the linter layers, having
# linted nothing, and still exit 0. Dockerfile.lint refuses to build without
# the argument, so no path to a lint result can quietly come from cache.
#
# Nothing that runs inside a container may call this script; see the header
# of Dockerfile.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
docker build --no-cache \
--target lint \
-t "$("$SCRIPT_DIR/projectname")-lint" .
docker build --build-arg LINT_EPOCH="$(date +%s)" -f Dockerfile.lint .
}
main "$@"
script/precommit
+11 -3
View File
@@ -4,9 +4,17 @@
#
# Runs lint but deliberately NOT the tests, so the TDD red-phase commit
# (failing tests, no implementation yet) can land. CI runs
# script/cibuild, whose image build includes the test phase, and so
# catches any branch that ships red. The lint phase includes the
# prettier check, so a badly formatted tree still fails the commit.
# script/cibuild, which builds both images and so catches any branch
# that ships red.
#
# The formatting check is still enforced here, because script/lint is a
# build of Dockerfile.lint and that runs `prettier --check .` as a build
# step: a badly formatted tree fails this hook, and therefore the
# commit. Calling script/fmt-check as well would only run prettier a
# second time over the same tree for the same verdict.
#
# script/lint is a docker build (Dockerfile.lint); docker is required to
# commit, which is the point of linting one way, everywhere.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
script/test
+16 -10
View File
@@ -1,19 +1,25 @@
#!/bin/sh
# script/test: run the test suite. Testing is a phase of the Dockerfile
# and this builds that phase alone, on the same terms as script/lint:
# --target because a phase that is not the last stage is built only when
# named, --no-cache because a cached test layer is a test that did not
# run, and a tag so each build replaces the previous image.
# script/test: run the test suite. Uses `timeout` (GNU coreutils) when
# available so the run is hard-capped at 30s; on macOS without
# coreutils the cap is skipped.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
rerun_verbose() {
echo "--- Rerunning with verbose for details ---"
yarn run vitest run --reporter=verbose
exit 1
}
main() {
cd "$ROOT"
docker build --no-cache \
--target test \
-t "$("$SCRIPT_DIR/projectname")-test" .
TIMEOUT="$(command -v timeout 2>/dev/null || command -v gtimeout 2>/dev/null || true)"
if [ -n "$TIMEOUT" ]; then
"$TIMEOUT" 30s yarn run vitest run --reporter=dot || rerun_verbose
else
yarn run vitest run --reporter=dot || rerun_verbose
fi
}
main "$@"
src/backup.ts
+64 -214
View File
@@ -1,11 +1,9 @@
// The backup command, rebuilt on the library API (issue #51).
//
// `lib.backup()` waits for a completed refresh of the library (a failed one
// fails the backup before any file is touched), then, for every file in scope,
// gets its original bytes onto disk under `downloadDirectory` and rebuilds the
// derived views (per-file sidecars, per-collection symlink trees,
// per-collection JSON) from the model. The on-disk layout is the historical
// one:
// `lib.backup()` refreshes the library, then, for every file in scope, gets its
// original bytes onto disk under `downloadDirectory` and rebuilds the derived
// views (per-file sidecars, per-collection symlink trees, per-collection JSON)
// from the model. The on-disk layout is the historical one, unchanged:
//
// <downloadDirectory>/
// originals/<fileID>.<ext> the decrypted bytes
@@ -14,18 +12,12 @@
// collections/<name>.json per-collection metadata
// failures.json durable ledger of unresolved failures
//
// A live photo's original is its image and its video, `<fileID>.<ext>` each
// with its own extension, and `originals/<fileID>.livephoto.json` naming them;
// its album folders link both.
//
// Crash-safety rests on two properties. Bytes are present-means-complete: an
// original appears under `originals/` only via the content layer's atomic
// temp-then-rename, so a file that exists is whole and is never re-fetched — an
// interrupted run resumes by listing the directory. The derived views hold no
// unique state, so they are rebuilt every run; that repairs stale sidecars and
// missing or broken symlinks left by an earlier crash. A rebuild also removes
// the symlinks into originals/ that no longer belong to an album, and the
// directories of albums that no longer exist.
// missing or broken symlinks left by an earlier crash.
//
// Resilience (issue #8): no per-file condition aborts the run. A failed
// download or a failed symlink is caught, recorded in `failures.json` with a
@@ -42,22 +34,16 @@ import {
readdirSync,
readFileSync,
readlinkSync,
rmdirSync,
rmSync,
statSync,
symlinkSync,
writeFileSync,
} from "node:fs";
import { copyFile, rename, rm } from "node:fs/promises";
import { basename, dirname, extname, join, relative } from "node:path";
import { basename, dirname, join, relative } from "node:path";
import { fsyncPath, removeLeftoverTempFiles } from "./download/index.js";
import { sanitizeFileName, withExtension } from "./filename.js";
import {
originalName,
storedOriginal,
writeLivePhoto,
} from "./library/content.js";
import { fsyncPath } from "./download/index.js";
import { safeExtension, sanitizeFileName } from "./filename.js";
import type { Collection, EnteFile } from "./model/types.js";
export type ProgressCallback = (message: string) => void;
@@ -106,14 +92,8 @@ export interface BackupLibrary {
listCollections(): Collection[];
listFiles(collectionID: number): EnteFile[];
// Get an original's bytes onto disk through the content cache/pools,
// returning where they landed: `destination` when they were fetched now,
// otherwise wherever they already were (the cache, or a prior backup). A
// live photo lands as its image and its video, fetched now beside
// `destination`.
original(
fileID: number,
destination: string,
): Promise<{ path: string; videoPath?: string }>;
// returning where they landed (the cache, or a prior backup).
original(fileID: number): Promise<{ path: string }>;
thumbnail(fileID: number): Promise<{ path: string }>;
}
@@ -130,6 +110,12 @@ interface FailureEntry {
const LEDGER_VERSION = 1;
// The originals/ filename for a file: `<id><ext>`, the extension taken from the
// title (or `.bin`). Matches the content cache's own naming so a present check
// lines up with what a fetch would write.
const originalName = (file: EnteFile): string =>
`${file.id}${safeExtension(file.metadata.title)}`;
// A regular file with content is treated as complete. A zero-byte file is not:
// it is the shape an aborted write leaves and must be re-fetched.
const isPresent = (path: string): boolean => {
@@ -195,29 +181,34 @@ const copyAtomic = async (src: string, dest: string): Promise<void> => {
}
};
// Put an original the library returned at `dest` in originals/, where a fresh
// fetch already wrote it. A live photo's image and video go beside `dest`: when
// they came from the cache they are copied, after removing whatever was at
// `dest` (an earlier version's ZIP of the two). Then the JSON file naming them
// is written, which is what makes the live photo count as stored.
const placeOriginal = async (
file: EnteFile,
dest: string,
got: { path: string; videoPath?: string },
): Promise<void> => {
if (got.videoPath === undefined) {
await copyAtomic(got.path, dest);
// A process-ID check: signal 0 delivers nothing and only reports whether the
// process exists. EPERM means it exists but belongs to another user.
const isRunning = (pid: number): boolean => {
try {
process.kill(pid, 0);
return true;
} catch (err) {
return (err as NodeJS.ErrnoException).code === "EPERM";
}
};
// Delete the temp files `copyAtomic` leaves behind when a backup is killed
// before its rename. Only files whose process is no longer running are
// removed, so a backup running at the same time keeps its own. A reused
// process ID can only keep a leftover a while longer, never remove a live one.
const removeLeftoverTempFiles = (dir: string): void => {
let names: string[];
try {
names = readdirSync(dir);
} catch {
return;
}
const originalsDir = dirname(dest);
const path = join(originalsDir, basename(got.path));
const videoPath = join(originalsDir, basename(got.videoPath));
if (got.path !== path) {
await rm(dest, { force: true });
await copyAtomic(got.path, path);
await copyAtomic(got.videoPath, videoPath);
for (const name of names) {
const match = /^\.quak-backup-.*-(\d+)-[0-9a-z]*\.tmp$/.exec(name);
if (match && !isRunning(Number(match[1]))) {
rmSync(join(dir, name), { force: true });
}
}
await writeLivePhoto(originalsDir, file.id, { path, videoPath });
};
// Ensure `linkPath` is a symlink to `target`, rebuilding a missing, wrong, or
@@ -236,111 +227,6 @@ const rebuildSymlink = (linkPath: string, target: string): void => {
symlinkSync(target, linkPath);
};
// The on-disk names for the entries of one directory, in entry order. Each
// name is used as is unless another entry would get the same name, ignoring
// case (two names that differ only in case are one entry on a case-insensitive
// file system); then every entry sharing it gets ` (<id>)`, before the
// extension when `beforeExtension` is set. A name with an ID added can match
// another entry's own name (`IMG (6).JPG`), so this repeats until no name is
// shared. IDs are stable, so the names are too.
const uniqueNames = (
entries: { id: number; name: string }[],
beforeExtension: boolean,
): string[] => {
const withID = (id: number, name: string): string => {
const ext = beforeExtension ? extname(name) : "";
const stem = name.slice(0, name.length - ext.length);
return `${stem} (${id})${ext}`;
};
const names = entries.map((e) => e.name);
const suffixed = new Set<number>();
for (;;) {
const counts = new Map<string, number>();
for (const name of names) {
const key = name.toLowerCase();
counts.set(key, (counts.get(key) ?? 0) + 1);
}
let changed = false;
for (const [i, { id, name }] of entries.entries()) {
if (suffixed.has(i)) continue;
if (counts.get(name.toLowerCase()) === 1) continue;
names[i] = withID(id, name);
suffixed.add(i);
changed = true;
}
if (!changed) return names;
}
};
// The links a file gets in its album's folder: one named after its title, to
// its original if that is stored. A stored live photo gets two, to its image
// and its video, each named after the title with that file's extension.
const linksFor = (
file: EnteFile,
stored: { path: string; videoPath?: string } | undefined,
): { id: number; name: string; file: EnteFile; target?: string }[] => {
const name = sanitizeFileName(file.metadata.title, `file-${file.id}`);
if (stored?.videoPath === undefined) {
return [{ id: file.id, name, file, target: stored?.path }];
}
return [stored.path, stored.videoPath].map((target) => ({
id: file.id,
name: withExtension(name, extname(target)),
file,
target,
}));
};
// Remove the symlinks in the album directory `dir` that point into
// `originalsDir` and are not named in `keep`. Nothing else in the directory
// is touched: anything else there was put there by the user.
const removeStaleLinks = (
dir: string,
keep: Set<string>,
originalsDir: string,
): void => {
const target = relative(dir, originalsDir);
for (const name of readdirSync(dir)) {
if (keep.has(name)) continue;
const path = join(dir, name);
if (
lstatSync(path).isSymbolicLink() &&
dirname(readlinkSync(path)) === target
) {
rmSync(path);
}
}
};
// Remove the directories under `collectionsDir` that an earlier run wrote for
// an album that is gone or renamed: a directory not named in `current` with a
// `<name>.json` beside it holding an album ID, which is what a run writes. Its
// symlinks into originals/ are removed; if that leaves it empty, it and its
// JSON are deleted, otherwise both stay for what the user put there.
const removeStaleAlbumDirs = (
collectionsDir: string,
current: Set<string>,
originalsDir: string,
): void => {
for (const entry of readdirSync(collectionsDir, { withFileTypes: true })) {
if (!entry.isDirectory() || current.has(entry.name)) continue;
const jsonPath = join(collectionsDir, `${entry.name}.json`);
try {
const album = JSON.parse(readFileSync(jsonPath, "utf-8")) as {
id?: unknown;
};
if (typeof album.id !== "number") continue;
} catch {
continue;
}
const dir = join(collectionsDir, entry.name);
removeStaleLinks(dir, new Set(), originalsDir);
if (readdirSync(dir).length > 0) continue;
rmdirSync(dir);
rmSync(jsonPath);
}
};
const loadLedger = (path: string): Map<number, FailureEntry> => {
const ledger = new Map<number, FailureEntry>();
try {
@@ -417,10 +303,9 @@ export const runBackup = async (
// Collections in scope, and the distinct files across them (a file shared
// by two albums is one original).
const allCollections = lib.listCollections();
const collections = allCollections.filter((c) =>
only ? only.has(c.name) : true,
);
const collections = lib
.listCollections()
.filter((c) => (only ? only.has(c.name) : true));
const collectionName = new Map<number, string>();
for (const c of collections) collectionName.set(c.id, c.name);
@@ -470,21 +355,15 @@ export const runBackup = async (
// tree; a present file is left as is.
if (includeOriginals) {
for (const [fileID, file] of distinct) {
if (storedOriginal(originalsDir, file) !== undefined) {
const dest = join(originalsDir, originalName(file));
if (isPresent(dest)) {
skipped++;
continue;
}
const dest = join(originalsDir, originalName(file));
try {
log(`Fetching original ${file.metadata.title} (${fileID})...`);
// A fetched original is written straight to `dest` (a live
// photo beside it); only one that was already cached elsewhere
// is copied.
await placeOriginal(
file,
dest,
await lib.original(fileID, dest),
);
const { path } = await lib.original(fileID);
await copyAtomic(path, dest);
downloaded++;
} catch (err) {
log(
@@ -520,67 +399,38 @@ export const runBackup = async (
// every present original (this repairs stale ones).
if (includeOriginals) {
for (const [fileID, file] of distinct) {
if (storedOriginal(originalsDir, file) !== undefined) {
const orig = join(originalsDir, originalName(file));
if (isPresent(orig)) {
writeSidecar(join(originalsDir, `${fileID}.json`), file);
}
}
}
// Then the per-collection symlink trees and JSON. Directory names are
// chosen across every album, not just those in scope, so a scoped run
// names an album the same as a full one and never takes the directory of
// an album it skipped. Stale entries are removed before anything is
// rebuilt, so on a case-insensitive file system removing an old name can
// never remove the new one.
const dirNames = uniqueNames(
allCollections.map((c) => ({
id: c.id,
name: sanitizeFileName(c.name, `collection-${c.id}`),
})),
false,
);
const albumDirNames = new Map(
allCollections.map((c, i) => [c.id, dirNames[i]!]),
);
try {
removeStaleAlbumDirs(collectionsDir, new Set(dirNames), originalsDir);
} catch (err) {
log(`FAILED removing old album directories: ${errorMessage(err)}`);
}
// Then the per-collection symlink trees and JSON.
for (const c of collections) {
const colDirName = albumDirNames.get(c.id)!;
const colDirName = sanitizeFileName(c.name, `collection-${c.id}`);
const colDir = join(collectionsDir, colDirName);
mkdirSync(colDir, { recursive: true });
const files = filesByCollection.get(c.id) ?? [];
const links = files.flatMap((f) =>
linksFor(f, storedOriginal(originalsDir, f)),
const metaFiles: { id: number; metadata: EnteFile["metadata"] }[] = [];
for (const file of files) {
metaFiles.push({ id: file.id, metadata: file.metadata });
if (!includeOriginals) continue;
const orig = join(originalsDir, originalName(file));
if (!isPresent(orig)) continue;
const linkName = sanitizeFileName(
file.metadata.title,
`file-${file.id}`,
);
const linkNames = uniqueNames(links, true);
const linkPath = join(colDir, linkName);
try {
removeStaleLinks(colDir, new Set(linkNames), originalsDir);
} catch (err) {
log(`FAILED removing old links in ${c.name}: ${errorMessage(err)}`);
}
const metaFiles = files.map((f) => ({
id: f.id,
metadata: f.metadata,
}));
for (const [i, link] of links.entries()) {
if (!includeOriginals || link.target === undefined) continue;
const linkName = linkNames[i]!;
try {
rebuildSymlink(
join(colDir, linkName),
relative(colDir, link.target),
);
rebuildSymlink(linkPath, relative(colDir, orig));
} catch (err) {
log(
`FAILED symlink ${c.name}/${linkName}: ${errorMessage(err)}`,
);
recordFailure(link.file, c.name, err);
recordFailure(file, c.name, err);
}
}
src/cli-commands.ts
+17 -57
View File
@@ -2,24 +2,20 @@
//
// Each command takes its options and a `CliContext` and resolves to the exit
// code; a thrown error is left to the caller. Nothing here calls
// `process.exit`: `bin/quak.ts` wires these to the command line, and `run` in
// `cli-run.ts` prints a thrown error as one line and exits once output has
// drained. Output must stay byte-identical (see `cli-output.ts`).
// `process.exit`: `bin/quak.ts` wires these to the command line and exits with
// the returned code once output has drained. Output must stay byte-identical
// (see `cli-output.ts`).
import { input, password as passwordPrompt } from "@inquirer/prompts";
import {
copyFileSync,
existsSync,
mkdirSync,
statSync,
unlinkSync,
writeFileSync,
} from "node:fs";
import { extname, join } from "node:path";
import {
type Client,
type ClientSnapshot,
type LoginOptions,
} from "./client.js";
import { join } from "node:path";
import { Client, type ClientSnapshot } from "./client.js";
import { init } from "./crypto/index.js";
import {
defaultCacheDirectory,
@@ -33,7 +29,6 @@ import {
thumbnailName,
} from "./cli-output.js";
import { freshCollections, freshFiles, freshFile } from "./cli-read.js";
import { withExtension } from "./filename.js";
import { runMetadataBackup } from "./metadata-backup.js";
import { listMissingThumbnails, fixMissingThumbnails } from "./thumbnails.js";
@@ -48,12 +43,6 @@ export interface CliContext {
// Reads the session file into a client, or null when there is none. The
// CLI passes `loadSession` from `cli-session.ts`; tests pass a fake client.
loadSession: (path: string) => Client | null;
// Used by `login` only. The CLI passes `Client.login` and terminal
// prompts; tests pass fakes.
login: (opts: LoginOptions) => Promise<Client>;
prompt: (message: string) => Promise<string>;
// Like `prompt`, but the answer is masked as it is typed.
promptSecret: (message: string) => Promise<string>;
}
const sessionPath = (ctx: CliContext): string =>
@@ -119,19 +108,24 @@ const openReadLibrary = (ctx: CliContext, client: Client): Promise<Library> =>
precacheOriginals: false,
});
const prompt = async (message: string): Promise<string> => input({ message });
const promptSecret = async (message: string): Promise<string> =>
passwordPrompt({ message, mask: true });
export const loginCommand = async (ctx: CliContext): Promise<number> => {
await init();
const email = process.env.QUAK_EMAIL ?? (await ctx.prompt("Email"));
const email = process.env.QUAK_EMAIL ?? (await prompt("Email"));
const password =
process.env.QUAK_PASSWORD ?? (await ctx.promptSecret("Password"));
process.env.QUAK_PASSWORD ?? (await promptSecret("Password"));
ctx.stderr.write("Authenticating...\n");
try {
const client = await ctx.login({
const client = await Client.login({
email,
password,
totp: async () => ctx.prompt("TOTP code: "),
emailOTP: async () => ctx.prompt("Email verification code: "),
totp: async () => prompt("TOTP code: "),
emailOTP: async () => prompt("Email verification code: "),
});
saveSession(ctx.sessionDir, client.toJSON());
@@ -307,30 +301,9 @@ export const getCommand = async (
// Default name is the file's own title, as the pre-library CLI used
// (not the editedName-preferring projection title) (issue #52).
const outPath = opts.out ?? originalName(file);
if (result.videoPath === undefined) {
copyFileSync(result.path, outPath);
ctx.stderr.write(`${result.bytes} bytes -> ${outPath}\n`);
return 0;
}
// A live photo is written as its image and its video, each named after
// the title with its own extension, as Ente's clients name them. With
// --out, the image goes there and the video beside it.
const imageOut =
opts.out ?? withExtension(outPath, extname(result.path));
const videoOut = withExtension(outPath, extname(result.videoPath));
if (imageOut.toLowerCase() === videoOut.toLowerCase()) {
ctx.stderr.write(
`File ${fileID} is a live photo, and its video would also be written to ${imageOut}\n`,
);
return 1;
}
copyFileSync(result.path, imageOut);
copyFileSync(result.videoPath, videoOut);
ctx.stderr.write(
`${result.bytes} bytes -> ${imageOut}\n` +
`${statSync(videoOut).size} bytes -> ${videoOut}\n`,
);
return 0;
} finally {
await lib.close();
}
@@ -383,9 +356,6 @@ export const backupMetadataCommand = async (
if (!client) return 1;
const lib = await openReadLibrary(ctx, client);
try {
// Refresh first so the dump holds current account state, not what the
// cache last held; a failed refresh throws.
await lib.fresh();
const { failedMLBatches } = await runMetadataBackup(lib, client, dir, {
exif: opts.exif || opts.all,
onProgress: (msg) => ctx.stderr.write(msg + "\n"),
@@ -406,14 +376,10 @@ export const backupCommand = async (
if (!client) return 1;
ctx.stderr.write("Starting backup...\n");
// The precache is off: the backup fetches what it needs, and must not
// also fill the cache with every thumbnail and the recent originals.
const lib = await Library.open({
client,
downloadDirectory: dir,
cacheDirectory: ctx.cacheDir,
precacheThumbnails: false,
precacheOriginals: false,
});
try {
const result = await lib.backup({
@@ -456,9 +422,6 @@ export const listMissingThumbnailsCommand = async (
if (!client) return 1;
const lib = await openReadLibrary(ctx, client);
try {
// Refresh first so files added since the cache was written are
// checked; a failed refresh throws.
await lib.fresh();
const missing = await listMissingThumbnails(lib, client, (msg) => {
if (!opts.json) ctx.stderr.write(msg + "\n");
});
@@ -494,9 +457,6 @@ export const fixMissingThumbnailsCommand = async (
if (!client) return 1;
const lib = await openReadLibrary(ctx, client);
try {
// Refresh first so files added since the cache was written are found;
// a failed refresh throws.
await lib.fresh();
let fileIDs: number[];
if (opts.file && opts.file.length > 0) {
fileIDs = opts.file.map(Number).filter(Number.isFinite);
@@ -535,7 +495,7 @@ export const fixMissingThumbnailsCommand = async (
ctx.stderr.write(` Skipped: ${skipped}\n`);
ctx.stderr.write(` Failed: ${failed}\n`);
if (skipped > 0) {
ctx.stderr.write("\nSkipped:\n");
ctx.stderr.write("\nSkipped (unsupported format):\n");
for (const r of results.filter((r) => r.status === "skipped")) {
ctx.stderr.write(
` ${r.fileID}\t${r.title}\t${r.reason}\n`,
src/cli-run.ts
-36
View File
@@ -1,36 +0,0 @@
// Runs one CLI command for `bin/quak.ts` and exits with its code.
import type { Writable } from "node:stream";
// Run a command and exit with its code once stdout/stderr have drained.
// Exiting before the drain can truncate piped output, and the library can keep
// the event loop alive after a command returns, so a plain return could hang.
// An error the command throws is printed as one `quak: MESSAGE` line, without
// the stack trace, and exits 1.
export const run = async (
command: Promise<number>,
stdout: Writable,
stderr: Writable,
exit: (code: number) => void,
): Promise<void> => {
let code: number;
try {
code = await command;
} catch (err) {
stderr.write(
`quak: ${err instanceof Error ? err.message : String(err)}\n`,
);
code = 1;
}
const pending = [stdout, stderr].filter((s) => s.writableLength > 0);
if (pending.length === 0) {
exit(code);
return;
}
let remaining = pending.length;
for (const s of pending) {
s.once("drain", () => {
if (--remaining === 0) exit(code);
});
}
};
src/crypto/hash.ts
-22
View File
@@ -1,22 +0,0 @@
import sodium, { type StateAddress } from "libsodium-wrappers-sumo";
import { toBase64 } from "./encoding.js";
// The content hash an uploading client records in a file's metadata: unkeyed
// BLAKE2b with a 64-byte output over the original's bytes, fed in chunks, as
// standard base64 with padding. Named after the upstream client's functions.
// The output length is read at call time for the same reason as
// `streamTagFinal` in stream.ts: libsodium sets its constants only once ready.
export const chunkHashInit = (): StateAddress =>
sodium.crypto_generichash_init(null, sodium.crypto_generichash_BYTES_MAX);
export const chunkHashUpdate = (state: StateAddress, chunk: Uint8Array): void =>
sodium.crypto_generichash_update(state, chunk);
export const chunkHashFinal = (state: StateAddress): string =>
toBase64(
sodium.crypto_generichash_final(
state,
sodium.crypto_generichash_BYTES_MAX,
),
);
src/crypto/index.ts
-1
View File
@@ -7,7 +7,6 @@ export {
} from "./encoding.js";
export { deriveKEK, deriveLoginSubkey } from "./kdf.js";
export { decryptBox, decryptSealed } from "./box.js";
export { chunkHashFinal, chunkHashInit, chunkHashUpdate } from "./hash.js";
export {
decryptBlob,
encryptBlob,
src/download/index.ts
+24 -261
View File
@@ -1,13 +1,8 @@
import { randomBytes } from "node:crypto";
import { readdirSync, rmSync } from "node:fs";
import { randomUUID } from "node:crypto";
import { open, rename, rm } from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
import { dirname, join } from "node:path";
import { Unzip, UnzipInflate } from "fflate";
import {
chunkHashFinal,
chunkHashInit,
chunkHashUpdate,
fromBase64,
initStreamPull,
pullStreamChunk,
@@ -16,18 +11,14 @@ import {
streamTagFinal,
} from "../crypto/index.js";
import { TruncatedStreamError } from "../errors.js";
import { safeExtension, sanitizeFileName, withExtension } from "../filename.js";
import { sanitizeFileName } from "../filename.js";
import { withRetry } from "../retry.js";
import type { ApiClient } from "../api/client.js";
import type { EnteFile } from "../model/types.js";
export interface DownloadResult {
// Where the file was written. A live photo is written as two files, its
// image here and its video at `videoPath` (see `decryptLivePhoto`).
path: string;
// The decrypted length; for a live photo, that of the ZIP it arrives as.
bytesWritten: number;
videoPath?: string;
}
// Fired as decrypted plaintext accumulates, with the running total of
@@ -49,9 +40,9 @@ const ENC_CHUNK_SIZE = STREAM_CHUNK_SIZE + STREAM_CHUNK_OVERHEAD;
// new: a body cut short still decrypts and authenticates up to its last whole
// chunk, so the absence of TAG_FINAL is the sole evidence it was cut short, and
// this throws rather than let a caller keep a short file. The sink has already
// seen those chunks by then; the callers (`decryptToTemp`, `decryptLivePhoto`)
// stage them in temp files that are renamed into place only on a clean return,
// so a throw leaves nothing on disk.
// seen those chunks by then; the caller (`decryptToTemp`) stages them in a temp
// file that is renamed into place only on a clean return, so a throw leaves
// nothing on disk.
const streamDecrypt = async (
stream: ReadableStream<Uint8Array>,
header: Uint8Array,
@@ -185,45 +176,6 @@ export const fsyncPath = async (path: string): Promise<void> => {
}
};
// A process-ID check: signal 0 delivers nothing and only reports whether the
// process exists. EPERM means it exists but belongs to another user.
const isRunning = (pid: number): boolean => {
try {
process.kill(pid, 0);
return true;
} catch (err) {
return (err as NodeJS.ErrnoException).code === "EPERM";
}
};
// Delete the temp files a killed process left in `dir`: the writer's
// `.quak-<pid>-<random>.tmp` and the backup copy's
// `.quak-backup-<name>-<pid>-<random>.tmp`. Only files whose process is no
// longer running are removed, so another process writing into the same
// directory keeps its own. A reused process ID can only keep a leftover a while
// longer, never remove a live one.
export const removeLeftoverTempFiles = (dir: string): void => {
let names: string[];
try {
names = readdirSync(dir);
} catch {
return;
}
for (const name of names) {
const match = /^\.quak-(?:.*-)?(\d+)-[0-9a-z]*\.tmp$/.exec(name);
if (match && !isRunning(Number(match[1]))) {
rmSync(join(dir, name), { force: true });
}
}
};
// A new temp file name in `dir`. The random suffix keeps concurrent downloads
// of the same destination from stepping on each other's temporary file; the
// process ID lets `removeLeftoverTempFiles` tell a leftover from a write in
// progress.
const tempPathIn = (dir: string): string =>
join(dir, `.quak-${process.pid}-${randomBytes(16).toString("hex")}.tmp`);
// Stage a write to `destination` atomically and durably, then rename it into
// place. `fill` writes the contents into the open temp file handle — either the
// whole buffer at once (`writeAtomic`) or chunk by chunk as they decrypt
@@ -248,7 +200,9 @@ const stageAtomic = async (
fill: (handle: FileHandle) => Promise<void>,
): Promise<void> => {
const dir = dirname(destination);
const tmpPath = tempPathIn(dir);
// The random suffix keeps concurrent downloads of the same destination
// from stepping on each other's temporary file.
const tmpPath = join(dir, `.quak-${randomUUID()}.tmp`);
try {
const handle = await open(tmpPath, "w");
try {
@@ -283,203 +237,44 @@ export const writeAtomic = async (
): Promise<void> =>
stageAtomic(destination, (handle) => handle.writeFile(plaintext));
// Refuse an original whose bytes do not hash to what its uploader recorded.
// The error is not retried.
const checkHash = (file: EnteFile, actual: string): void => {
if (actual !== file.metadata.hash) {
throw new Error(
`download: file ${file.id}: content hash ${actual} does not match the hash its uploader recorded, ${file.metadata.hash}`,
);
}
};
// Decrypt `stream` straight to `destination`, one plaintext chunk at a time,
// under the atomic writer's temp-then-rename discipline. Memory stays bounded
// by the chunk size: each decrypted chunk is written to the temp file and
// dropped. The rename happens only after the stream authenticates as terminated
// on TAG_FINAL; a truncated stream throws and leaves the destination untouched.
// Returns the plaintext length written.
//
// `original` is the file whose original this is (none for a thumbnail, which
// has no recorded hash). When its metadata has a hash, the decrypted bytes are
// hashed as they stream and must match it, or nothing is stored.
const decryptToTemp = async (
destination: string,
stream: ReadableStream<Uint8Array>,
header: Uint8Array,
key: Uint8Array,
onProgress?: ProgressCallback,
original?: EnteFile,
): Promise<number> => {
const hash =
original?.metadata.hash === undefined ? undefined : chunkHashInit();
let bytesWritten = 0;
try {
await stageAtomic(destination, async (handle) => {
bytesWritten = await streamDecrypt(
stream,
header,
key,
async (plaintext) => {
if (hash !== undefined) chunkHashUpdate(hash, plaintext);
await handle.write(plaintext);
},
onProgress,
);
if (original !== undefined && hash !== undefined) {
checkHash(original, chunkHashFinal(hash));
}
});
} catch (err) {
// Cancel the body so its connection is closed now rather than held
// until the stream is garbage collected. A backup run carries on past
// a failed file, so without this every failure would hold a socket.
// This covers every failure, including a temp file that cannot be
// opened and a header that is rejected before the body is read.
await stream.cancel(err).catch(() => undefined);
throw err;
}
return bytesWritten;
};
// One of the two parts of a live photo being unpacked: the ZIP entry whose
// name starts with `kind`, written to its own temp file.
interface LivePhotoPart {
kind: "image" | "video";
tmpPath: string;
handle: FileHandle;
hash: ReturnType<typeof chunkHashInit>;
// Decompressed bytes not yet written.
pending: Uint8Array[];
// The entry's extension, set once all of the entry has been read.
ext?: string;
}
const openPart = async (
kind: "image" | "video",
dir: string,
): Promise<LivePhotoPart> => {
const tmpPath = tempPathIn(dir);
const handle = await open(tmpPath, "w");
return { kind, tmpPath, handle, hash: chunkHashInit(), pending: [] };
};
// A live photo arrives as a ZIP of its image and its video. Ente's clients
// name the entries `image.<ext>` and `video.<ext>`, and like the upstream
// client's decoder this takes the first entries whose names start with `image`
// and `video`. It is written unpacked: each part is named `destination` with
// the extension replaced by its own entry's, and the two must differ ignoring
// case. When the file records a hash, `<imageHash>:<videoHash>` must match it,
// each over that part's own bytes. Only then is whatever was at `destination`
// removed and the image, then the video, renamed into place; on any failure
// neither is stored.
//
// The ZIP is chosen by its uploader and may expand enormously, so each part is
// written as it decompresses and never held. fflate's `Unzip` inflates each
// push in one piece before `push` returns, and deflate expands at most about
// 1000-fold, so the ZIP is pushed in 4 KiB slices, keeping each decompressed
// piece near 4 MiB, one plaintext chunk, and each piece is written before the
// next slice is pushed. Every entry is started, even one that is not kept,
// because fflate keeps an unstarted entry's data in memory.
const decryptLivePhoto = async (
destination: string,
stream: ReadableStream<Uint8Array>,
header: Uint8Array,
key: Uint8Array,
onProgress: ProgressCallback | undefined,
file: EnteFile,
): Promise<DownloadResult> => {
const sliceSize = 4096;
const dir = dirname(destination);
const fail = (message: string, cause?: unknown): Error =>
new Error(`download: file ${file.id}: ${message}`, { cause });
const parts: LivePhotoPart[] = [];
try {
const image = await openPart("image", dir);
parts.push(image);
const video = await openPart("video", dir);
parts.push(video);
const claimed = new Set<LivePhotoPart>();
const unzip = new Unzip((entry) => {
const part = parts.find(
(p) => !claimed.has(p) && entry.name.startsWith(p.kind),
);
if (part !== undefined) claimed.add(part);
entry.ondata = (err, data, final) => {
if (err) throw err;
if (part === undefined) return;
chunkHashUpdate(part.hash, data);
part.pending.push(data);
if (final) part.ext = safeExtension(entry.name);
};
entry.start();
});
unzip.register(UnzipInflate);
const push = async (
data: Uint8Array,
final: boolean,
): Promise<void> => {
// fflate reports a bad ZIP by throwing, sometimes a TypeError,
// which the retry would take for a network failure; a bad ZIP is
// never retried.
try {
unzip.push(data, final);
} catch (err) {
throw fail("live photo is not a readable ZIP", err);
}
for (const part of parts) {
for (const piece of part.pending)
await part.handle.write(piece);
part.pending = [];
}
};
const bytesWritten = await streamDecrypt(
stream,
header,
key,
async (plaintext) => {
for (let i = 0; i < plaintext.length; i += sliceSize) {
await push(plaintext.subarray(i, i + sliceSize), false);
}
},
onProgress,
);
await push(new Uint8Array(0), true);
if (image.ext === undefined || video.ext === undefined) {
throw fail(
"live photo ZIP does not hold both an image and a video",
);
}
if (file.metadata.hash !== undefined) {
checkHash(
file,
`${chunkHashFinal(image.hash)}:${chunkHashFinal(video.hash)}`,
);
}
if (image.ext.toLowerCase() === video.ext.toLowerCase()) {
throw fail(
`live photo's image and video have the same extension, ${video.ext}`,
);
}
const path = withExtension(destination, image.ext);
const videoPath = withExtension(destination, video.ext);
for (const part of parts) {
await part.handle.sync();
await part.handle.close();
}
await rm(destination, { force: true });
await rename(image.tmpPath, path);
try {
await rename(video.tmpPath, videoPath);
} catch (err) {
await rm(path, { force: true }).catch(() => undefined);
throw err;
}
await fsyncPath(dir);
return { path, bytesWritten, videoPath };
} catch (err) {
// Best-effort cleanup, as in `stageAtomic`.
for (const part of parts) {
await part.handle.close().catch(() => undefined);
await rm(part.tmpPath, { force: true }).catch(() => undefined);
}
throw err;
}
};
// Fetch a stream and decrypt it to `destination`, retrying the whole sequence.
//
// The request is only the first third of a download. `getXStream` returns as
@@ -507,45 +302,12 @@ const fetchAndDecrypt = async (
key: Uint8Array,
destination: string,
onProgress?: ProgressCallback,
original?: EnteFile,
): Promise<DownloadResult> =>
): Promise<number> =>
withRetry(async () => {
const stream = await openStream();
try {
if (original?.metadata.fileType === "livePhoto") {
return await decryptLivePhoto(
destination,
stream,
header,
key,
onProgress,
original,
);
}
const bytesWritten = await decryptToTemp(
destination,
stream,
header,
key,
onProgress,
original,
);
return { path: destination, bytesWritten };
} catch (err) {
// Cancel the body so its connection is closed now rather than held
// until the stream is garbage collected. A backup run carries on
// past a failed file, so without this every failure would hold a
// socket. This covers every failure, including a temp file that
// cannot be opened and a header that is rejected before the body
// is read.
await stream.cancel(err).catch(() => undefined);
throw err;
}
return decryptToTemp(destination, stream, header, key, onProgress);
}, api.getRetryOptions());
// Write `file`'s original to `outPath`. A live photo is written as its image
// and its video beside `outPath` instead, and whatever was at `outPath` is
// removed (see `decryptLivePhoto`).
export const downloadFile = async (
api: ApiClient,
file: EnteFile,
@@ -557,15 +319,15 @@ export const downloadFile = async (
const resolvedPath =
outPath ?? sanitizeFileName(file.metadata.title, `file-${file.id}`);
const header = fromBase64(file.file.decryptionHeader);
return fetchAndDecrypt(
const bytesWritten = await fetchAndDecrypt(
api,
() => api.getFileStream(file.id, { retry: false }),
header,
file.key,
resolvedPath,
onProgress,
file,
);
return { path: resolvedPath, bytesWritten };
};
export const downloadThumbnail = async (
@@ -578,7 +340,7 @@ export const downloadThumbnail = async (
outPath ??
`thumb_${sanitizeFileName(file.metadata.title, `file-${file.id}`)}`;
const header = fromBase64(file.thumbnail.decryptionHeader);
return fetchAndDecrypt(
const bytesWritten = await fetchAndDecrypt(
api,
() => api.getThumbnailStream(file.id, { retry: false }),
header,
@@ -586,4 +348,5 @@ export const downloadThumbnail = async (
resolvedPath,
onProgress,
);
return { path: resolvedPath, bytesWritten };
};
src/filename.ts
-4
View File
@@ -35,7 +35,3 @@ export const safeExtension = (title: string): string => {
const ext = extname(title);
return /^\.[A-Za-z0-9]+$/.test(ext) ? ext : ".bin";
};
// `name` with its extension, if it has one, replaced by `ext` (".mov").
export const withExtension = (name: string, ext: string): string =>
name.slice(0, name.length - extname(name).length) + ext;
src/library/content.ts
+68 -244
View File
@@ -1,13 +1,11 @@
// The on-disk content and thumbnail cache keyed by fileID (issue #46).
//
// Layout under `cacheDirectory`: `originals/<fileID>.<ext>` and
// `thumbnails/<fileID>.<ext>`, flat directories at 0700 with files at 0600. A
// live photo's original is two files, its image and its video, with
// `originals/<fileID>.livephoto.json` naming them. Content appears only by the
// streaming atomic writer's rename (the download layer, #40), so a file that
// exists is whole — "present means complete". The directory listing taken at
// `open()` is the record of what is cached, and the orphan temp files a crashed
// write may have left are reaped there.
// `thumbnails/<fileID>.<ext>`, flat directories at 0700 with files at 0600.
// Content appears only by the streaming atomic writer's rename (the download
// layer, #40), so a file that exists is whole — "present means complete". The
// directory listing taken at `open()` is the record of what is cached, and the
// orphan temp files a crashed write may have left are reaped there.
//
// A fetch goes through the shared request pools (#45): the content pool for
// originals, the thumbnail pool for thumbnails. The pool limits concurrency,
@@ -17,14 +15,14 @@
// Integrity. The reused streaming decrypt is the enforced guarantee: every
// chunk is authenticated and the writer renames the file into place only once
// the stream ends on TAG_FINAL, so a truncated or corrupt fetch throws and
// nothing is stored. For an original whose metadata records a content hash
// (`FileMetadata.hash`), the writer also hashes the decrypted bytes and stores
// nothing if they differ, failing the fetch with an error naming the file. An
// original with no recorded hash is stored unchecked, as the upstream client
// does; thumbnails have none. On top of that this module refuses to record a
// stored file that came out empty.
// nothing is stored. On top of that this module refuses to record a stored file
// that came out empty. The design also asks for a content-hash comparison
// against `FileMetadata.hash` (with a `fileSize` fallback); that is deferred —
// see the PR — because the exact hash construction cannot be confirmed against
// the repo's fixtures and `FileBlob.size` is the encrypted object size, not the
// decrypted length this layer has.
import { existsSync, readFileSync, statSync } from "node:fs";
import { existsSync, statSync } from "node:fs";
import {
chmod,
mkdir,
@@ -34,15 +32,13 @@ import {
statfs,
utimes,
} from "node:fs/promises";
import { basename, dirname, extname, join } from "node:path";
import { dirname, extname, join } from "node:path";
import type { ApiClient } from "../api/client.js";
import {
downloadFile,
downloadThumbnail,
type ProgressCallback,
removeLeftoverTempFiles,
writeAtomic,
} from "../download/index.js";
import { safeExtension } from "../filename.js";
import type { EnteFile } from "../model/types.js";
@@ -50,6 +46,8 @@ import type { Priority, RequestPools } from "./pools.js";
const DIR_MODE = 0o700;
const FILE_MODE = 0o600;
const TEMP_PREFIX = ".quak-";
const TEMP_SUFFIX = ".tmp";
const GIB = 1024 * 1024 * 1024;
// Owner ruling (#36): bound the originals cache at 100 GiB, but back off when
// the volume has under 50 GiB free so the cache never crowds the disk.
@@ -80,8 +78,6 @@ const poolPriorityOf = (priority: ThumbnailPriority): Priority =>
export interface ContentResult {
path: string;
bytes: number;
// A live photo's video. `path` and `bytes` are then its image's.
videoPath?: string;
}
// Progress for a single `original`/`thumbnail` call. A present file emits one
@@ -132,14 +128,11 @@ export interface ThumbnailsAPI {
// stand-in so the cache logic runs with no crypto and no network. Pool routing,
// dedup, present-checks and integrity live in the cache, not here.
export interface ContentSource {
// Writes the original at `destination`. A live photo is written beside it
// as its image and its video instead, and their paths are returned, as
// `downloadFile` does.
original(args: {
file: EnteFile;
destination: string;
onProgress?: ProgressCallback;
}): Promise<{ bytesWritten: number; path?: string; videoPath?: string }>;
}): Promise<{ bytesWritten: number }>;
thumbnail(args: {
file: EnteFile;
destination: string;
@@ -217,9 +210,7 @@ class AbortDrop extends Error {
}
}
// The originals/ name for a file: `<fileID><ext>`, the extension taken from the
// title (or `.bin`). A backup names its originals the same way.
export const originalName = (file: EnteFile): string =>
const originalName = (file: EnteFile): string =>
`${file.id}${safeExtension(file.metadata.title)}`;
// The fileID a cache filename encodes, or undefined when the name is not one
@@ -241,73 +232,6 @@ const fileSize = (path: string): number | undefined => {
}
};
// Whether `path` is a regular file with content. A zero-byte file is the shape
// an aborted write leaves, so it does not count.
const hasContent = (path: string | undefined): boolean =>
path !== undefined && (fileSize(path) ?? 0) > 0;
// A live photo's image and video are named with the extensions from inside its
// ZIP, so their names alone do not say which is which. Wherever the cache or a
// backup stores one, this JSON file beside them names both.
const livePhotoFileName = (fileID: number): string =>
`${fileID}.livephoto.json`;
// The image and video that the live photo's JSON file in `dir` names, or
// undefined when there is none. Only names of the form the cache writes are
// taken, so the file cannot point outside `dir`.
const readLivePhoto = (
dir: string,
fileID: number,
): { path: string; videoPath: string } | undefined => {
const valid = (name: unknown): name is string =>
typeof name === "string" && name === `${fileID}${safeExtension(name)}`;
try {
const { image, video } = JSON.parse(
readFileSync(join(dir, livePhotoFileName(fileID)), "utf-8"),
);
if (valid(image) && valid(video)) {
return { path: join(dir, image), videoPath: join(dir, video) };
}
} catch {
// No such file, or not one the cache wrote.
}
return undefined;
};
// Write the JSON file naming a live photo's image and video, both in `dir`.
export const writeLivePhoto = (
dir: string,
fileID: number,
stored: { path: string; videoPath: string },
): Promise<void> =>
writeAtomic(
join(dir, livePhotoFileName(fileID)),
new TextEncoder().encode(
JSON.stringify({
image: basename(stored.path),
video: basename(stored.videoPath),
}),
),
);
// The original of `file` as the cache or a backup stored it in `dir`, when all
// of it is there: `<fileID><ext>`, or a live photo's image and video.
export const storedOriginal = (
dir: string,
file: EnteFile,
): { path: string; videoPath?: string } | undefined => {
if (file.metadata.fileType !== "livePhoto") {
const path = join(dir, originalName(file));
return hasContent(path) ? { path } : undefined;
}
const stored = readLivePhoto(dir, file.id);
return stored !== undefined &&
hasContent(stored.path) &&
hasContent(stored.videoPath)
? stored
: undefined;
};
export class ContentCache implements PhotoContent, ThumbnailsAPI {
private readonly pools: RequestPools;
private readonly source: ContentSource;
@@ -315,17 +239,10 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
private readonly getFile: (fileID: number) => EnteFile | undefined;
private readonly originalsDir: string;
private readonly thumbnailsDir: string;
// fileID -> absolute path of the cached bytes, and for a live photo's
// original its video's, seeded from the directory listing at open() and
// extended as fetches store new files.
private readonly originals = new Map<
number,
{ path: string; videoPath?: string }
>();
private readonly thumbnails = new Map<
number,
{ path: string; videoPath?: string }
>();
// fileID -> absolute path of the cached bytes, seeded from the directory
// listing at open() and extended as fetches store new files.
private readonly originals = new Map<number, string>();
private readonly thumbnails = new Map<number, string>();
private readonly maxOriginalsBytes: number;
private readonly freeBelowBytes: number;
private readonly isPinned: (fileID: number) => boolean;
@@ -385,9 +302,9 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
pathsFor(fileID: number): CachedPaths {
const out: CachedPaths = {};
const original = this.originals.get(fileID);
if (original !== undefined) out.originalPath = original.path;
if (original !== undefined) out.originalPath = original;
const thumbnail = this.thumbnails.get(fileID);
if (thumbnail !== undefined) out.thumbnailPath = thumbnail.path;
if (thumbnail !== undefined) out.thumbnailPath = thumbnail;
return out;
}
@@ -405,27 +322,6 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
return this.get(fileID, "thumbnail", "on-demand", opts?.onProgress);
}
// Get an original for a backup. One not present anywhere is written
// straight to `destination` and recorded there, so no second copy lands
// in the cache; one already present is returned where it is.
async backupOriginal(
fileID: number,
destination: string,
): Promise<ContentResult> {
const result = await this.acquire(
fileID,
"original",
"on-demand",
undefined,
{ destination },
);
return {
path: result.path,
bytes: result.bytes,
videoPath: result.videoPath,
};
}
async ensure(args: EnsureOptions): Promise<EnsureResult[]> {
return this.ensureThumbnails(args);
}
@@ -522,72 +418,51 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
? { status: "skipped", bytes: result.bytes }
: { status: "done", bytes: result.bytes },
);
return {
path: result.path,
bytes: result.bytes,
videoPath: result.videoPath,
};
return { path: result.path, bytes: result.bytes };
}
// The core: return the cached path if present, else fetch through the pool,
// store, and return it. `cached` distinguishes a present hit (no network,
// no download event) from a fresh fetch. A fetched original is stored at
// `opts.destination` when given, instead of in `originalsDir`. A live
// photo's original is present only with its video, and is returned with
// it.
// no download event) from a fresh fetch.
private async acquire(
fileID: number,
kind: Kind,
priority: Priority,
signal: AbortSignal | undefined,
opts?: { onByte?: ProgressCallback; destination?: string },
): Promise<{
path: string;
bytes: number;
videoPath?: string;
cached: boolean;
}> {
opts?: { onByte?: ProgressCallback },
): Promise<{ path: string; bytes: number; cached: boolean }> {
const file = this.getFile(fileID);
if (!file) throw new Error(`content cache: unknown file ${fileID}`);
const isLivePhoto =
kind === "original" && file.metadata.fileType === "livePhoto";
const known = kind === "original" ? this.originals : this.thumbnails;
const cached = known.get(fileID);
if (cached !== undefined) {
const size = fileSize(cached.path);
if (
size !== undefined &&
size > 0 &&
(!isLivePhoto || hasContent(cached.videoPath))
) {
const size = fileSize(cached);
if (size !== undefined && size > 0) {
// Returning an original's path is a use: bump its mtime so LRU
// order reflects it and survives a restart with no ledger.
if (
kind === "original" &&
dirname(cached.path) === this.originalsDir
dirname(cached) === this.originalsDir
)
await this.touch(cached.path);
return { ...cached, bytes: size, cached: true };
await this.touch(cached);
return { path: cached, bytes: size, cached: true };
}
// A recorded file that has since gone, or a live photo an earlier
// version stored as one ZIP, re-fetches below.
// A recorded file that has since gone re-fetches below.
known.delete(fileID);
}
// An original a backup already stored counts as present.
if (kind === "original" && this.downloadDirectory !== undefined) {
const stored = storedOriginal(
join(this.downloadDirectory, "originals"),
file,
const backupPath = join(
this.downloadDirectory,
"originals",
originalName(file),
);
if (stored !== undefined) {
this.originals.set(fileID, stored);
return {
...stored,
bytes: fileSize(stored.path) ?? 0,
cached: true,
};
const size = fileSize(backupPath);
if (size !== undefined && size > 0) {
this.originals.set(fileID, backupPath);
return { path: backupPath, bytes: size, cached: true };
}
}
@@ -595,7 +470,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
kind === "original" ? this.originalsDir : this.thumbnailsDir;
const dest =
kind === "original"
? (opts?.destination ?? join(dir, originalName(file)))
? join(dir, originalName(file))
: join(dir, `${fileID}${THUMBNAIL_EXT}`);
const pool =
kind === "original" ? this.pools.content : this.pools.thumbnails;
@@ -616,39 +491,21 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
? this.beginOriginalWrite(fileID)
: null;
try {
const stored = await this.download(
file,
dest,
kind,
opts?.onByte,
);
for (const path of [stored.path, stored.videoPath]) {
if (path === undefined) continue;
await chmod(path, FILE_MODE);
if ((await stat(path)).size === 0) {
await this.download(file, dest, kind, opts?.onByte);
await chmod(dest, FILE_MODE);
const size = (await stat(dest)).size;
if (size === 0) {
throw new Error(
`content cache: ${kind} ${fileID} stored empty`,
);
}
}
// A backup records its own live photos.
if (
stored.videoPath !== undefined &&
opts?.destination === undefined
) {
await writeLivePhoto(dir, fileID, {
path: stored.path,
videoPath: stored.videoPath,
});
}
known.set(fileID, stored);
known.set(fileID, dest);
// A fresh original may have crossed the limit; make room by
// evicting least-recently-used originals. An over-budget
// fetch keeps the file it returns, and no overlapping
// sibling is evicted. Thumbnails are never bounded.
if (write) await this.enforceOriginalsLimit(write);
const size = (await stat(stored.path)).size;
return { ...stored, bytes: size, cached: false };
return { path: dest, bytes: size, cached: false };
} finally {
if (write) this.inFlightOriginals.delete(write);
}
@@ -657,24 +514,18 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
);
}
// Fetch into `destination`, returning where the bytes landed: there, or
// for a live photo, its image and video beside it.
private async download(
file: EnteFile,
destination: string,
kind: Kind,
onProgress: ProgressCallback | undefined,
): Promise<{ path: string; videoPath?: string }> {
): Promise<number> {
const args = { file, destination, onProgress };
if (kind === "thumbnail") {
await this.source.thumbnail(args);
return { path: destination };
}
const result = await this.source.original(args);
return {
path: result.path ?? destination,
videoPath: result.videoPath,
};
const result =
kind === "original"
? await this.source.original(args)
: await this.source.thumbnail(args);
return result.bytesWritten;
}
// Best-effort bump of a file's mtime to now; a failed touch must never fail
@@ -685,14 +536,13 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
}
// Every stored original that lives under `originalsDir` (a backup-directory
// hit recorded in the map is excluded), with its size and mtime; a live
// photo's size includes its video. Entries whose file has vanished are
// dropped from the map. Backups and thumbnails are never counted.
// hit recorded in the map is excluded), with its size and mtime. Entries
// whose file has vanished are dropped from the map. Backups and thumbnails
// are never counted.
private async measureOriginals(): Promise<{
entries: {
fileID: number;
path: string;
videoPath?: string;
size: number;
mtimeMs: number;
}[];
@@ -701,28 +551,21 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
const entries: {
fileID: number;
path: string;
videoPath?: string;
size: number;
mtimeMs: number;
}[] = [];
let used = 0;
for (const [fileID, { path, videoPath }] of this.originals) {
for (const [fileID, path] of this.originals) {
if (dirname(path) !== this.originalsDir) continue;
try {
const s = await stat(path);
const size =
s.size +
(videoPath === undefined
? 0
: (await stat(videoPath)).size);
entries.push({
fileID,
path,
videoPath,
size,
size: s.size,
mtimeMs: s.mtimeMs,
});
used += size;
used += s.size;
} catch {
this.originals.delete(fileID);
}
@@ -786,18 +629,6 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
for (const e of evictable) {
if (remaining <= limit) break;
await rm(e.path, { force: true });
// A live photo goes whole: its video and the JSON file
// naming the two go with its image.
if (e.videoPath !== undefined) {
await rm(e.videoPath, { force: true });
await rm(
join(
this.originalsDir,
livePhotoFileName(e.fileID),
),
{ force: true },
);
}
this.originals.delete(e.fileID);
remaining -= e.size;
}
@@ -822,30 +653,23 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
await chmod(dir, DIR_MODE);
}
private async scan(
dir: string,
into: Map<number, { path: string; videoPath?: string }>,
): Promise<void> {
// Another process sharing this cache may still be writing its temp
// files, so only those whose process has exited are removed.
removeLeftoverTempFiles(dir);
private async scan(dir: string, into: Map<number, string>): Promise<void> {
let entries: string[];
try {
entries = await readdir(dir);
} catch {
return;
}
const names = new Set(entries);
for (const name of entries) {
if (name.startsWith(TEMP_PREFIX) && name.endsWith(TEMP_SUFFIX)) {
await rm(join(dir, name), { force: true }).catch(
() => undefined,
);
continue;
}
const id = fileIDFromName(name);
const path = join(dir, name);
if (id === undefined || !existsSync(path)) continue;
// A live photo's image and video are one entry, as the JSON file
// beside them names them.
const livePhoto = names.has(livePhotoFileName(id))
? readLivePhoto(dir, id)
: undefined;
into.set(id, livePhoto ?? { path });
if (id !== undefined && existsSync(path)) into.set(id, path);
}
}
}
src/library/index.ts
+7 -10
View File
@@ -538,13 +538,11 @@ export class Library {
}
// Back up every in-scope file to `downloadDirectory` in the historical
// on-disk layout, with a durable failure ledger (issue #51). Waits for a
// completed refresh first, as `fresh()` does, joining one already running,
// and rejects before touching any file when it fails. Then fetches pending
// originals (and optional thumbnails) through the content cache and pools,
// and rebuilds the derived symlink/JSON views from the model. Throws before
// any network work when no download directory is available or no content
// cache backs the originals it must fetch.
// on-disk layout, with a durable failure ledger (issue #51). Refreshes
// first, fetches pending originals (and optional thumbnails) through the
// content cache and pools, then rebuilds the derived symlink/JSON views
// from the model. Throws before any network work when no download directory
// is available or no content cache backs the originals it must fetch.
backup(opts?: BackupOptions): Promise<BackupResult> {
const downloadDirectory =
opts?.downloadDirectory ?? this.downloadDirectory;
@@ -568,11 +566,10 @@ export class Library {
const cache = this.cache;
return runBackup(
{
refresh: () => this.refreshNow(),
refresh: () => this.runRefresh(),
listCollections: () => this.store.listCollections(),
listFiles: (id) => this.store.listFiles(id),
original: (fileID, destination) =>
cache!.backupOriginal(fileID, destination),
original: (fileID) => cache!.original(fileID),
thumbnail: (fileID) => cache!.thumbnail(fileID),
},
{ ...opts, downloadDirectory },
src/library/read.ts
+1 -2
View File
@@ -80,8 +80,7 @@ export class Photo {
}
// Fetch and cache the full-resolution original, returning its on-disk path
// and byte length; for a live photo, its image's, and its video's path as
// `videoPath`. Served from the cache (or the backup download directory)
// and byte length. Served from the cache (or the backup download directory)
// when already present, otherwise fetched through the content pool.
async original(opts?: ContentOptions): Promise<ContentResult> {
return this.contentOrThrow().original(this.rec.fileID, opts);
src/library/records.ts
-1
View File
@@ -42,7 +42,6 @@ export interface PhotoRecord {
isArchived: boolean;
isHidden: boolean;
// Local cache paths, set once a later phase caches the bytes; unset here.
// A live photo's `originalPath` is its image.
thumbnailPath?: string;
originalPath?: string;
}
src/metadata-backup.ts
+3 -4
View File
@@ -128,8 +128,7 @@ export const extractImageMetadata = (
// Read a file's original bytes through the library's content cache and extract
// its embedded image metadata. The bytes come from `photo.original()` — the
// same on-disk cache the rest of the library fills — rather than a fresh
// per-call download to a throwaway temp file. For a live photo, its `path` is
// the image.
// per-call download to a throwaway temp file.
const extractExif = async (
photo: Photo,
): Promise<Record<string, unknown> | undefined> => {
@@ -141,8 +140,8 @@ const extractExif = async (
// Dump every decrypted metadata layer the account holds into a directory tree
// of plain JSON: account, per-collection, and per-file records including the
// private and public magic metadata and (by default) the ML data. Collections
// and files are enumerated from the library's cache, which the caller refreshes
// first. Returns how many ML data requests failed; their files are still
// and files are enumerated from the library's cache rather than a fresh server
// scan. Returns how many ML data requests failed; their files are still
// written, with `mlDataError` in place of `mlData`.
export const runMetadataBackup = async (
lib: Library,
src/model/decrypt.ts
+1 -23
View File
@@ -34,28 +34,6 @@ const FILE_TYPE_MAP: Record<number, FileType> = {
const parseFileType = (n: number): FileType => FILE_TYPE_MAP[n] ?? "unknown";
// The hash the uploading client recorded for the original's bytes, read the
// way the upstream client's `metadataHash` reads it: `hash` if present,
// otherwise, for a live photo from an older client that wrote the two parts
// separately, `<imageHash>:<videoHash>`. A field that is not a non-empty
// string counts as absent, and a file with no hash at all is normal.
const expectedHash = (json: Record<string, unknown>): string | undefined => {
const text = (v: unknown): string | undefined =>
typeof v === "string" && v !== "" ? v : undefined;
const hash = text(json.hash);
if (hash !== undefined) return hash;
const imageHash = text(json.imageHash);
const videoHash = text(json.videoHash);
if (
json.fileType === 2 &&
imageHash !== undefined &&
videoHash !== undefined
) {
return `${imageHash}:${videoHash}`;
}
return undefined;
};
export const decryptCollection = (
raw: RawCollection,
keys: KeyMaterial,
@@ -137,7 +115,7 @@ export const decryptFile = (
modificationTime: metadataJSON.modificationTime ?? 0,
latitude: metadataJSON.latitude,
longitude: metadataJSON.longitude,
hash: expectedHash(metadataJSON),
hash: metadataJSON.hash,
};
const magicMetadata = decryptMagicMetadata(raw.magicMetadata, key);
src/model/types.ts
-3
View File
@@ -29,9 +29,6 @@ export interface FileMetadata {
modificationTime: Microseconds;
latitude?: number;
longitude?: number;
// The content hash the uploader recorded (see `expectedHash` in
// decrypt.ts); `downloadFile` refuses an original that does not match it.
// Absent for files from very old clients.
hash?: string;
}
src/thumbnails.ts
+30 -88
View File
@@ -7,21 +7,8 @@ import { ApiError } from "./api/client.js";
import { encryptBlob, toBase64 } from "./crypto/index.js";
import type { EnteFile } from "./model/types.js";
// The server refuses a thumbnail larger than the one it already records for the
// file (`thumbnail.size`, the encrypted size), so these encodings are tried
// from largest to smallest and the first that fits is uploaded.
const THUMB_ENCODINGS = [
{ maxDimension: 720, quality: 50 },
{ maxDimension: 720, quality: 30 },
{ maxDimension: 480, quality: 30 },
{ maxDimension: 320, quality: 20 },
{ maxDimension: 160, quality: 20 },
];
// The server accepts a new thumbnail only from the file's owner, so files other
// people own in albums shared with this account are never checked or repaired.
const NOT_OWNED_REASON =
"owned by another account (only the owner can replace its thumbnail)";
const THUMB_MAX_DIMENSION = 720;
const THUMB_JPEG_QUALITY = 50;
export interface MissingThumbnailInfo {
fileID: number;
@@ -32,12 +19,11 @@ export interface MissingThumbnailInfo {
// Three outcomes, not two. "fixed": a thumbnail was generated and uploaded.
// "failed": something went wrong (download, encode, upload) and the file still
// has no thumbnail. "skipped": the server would refuse any thumbnail for the
// file or this helper cannot regenerate it — a file another account owns, a
// recorded thumbnail size nothing fits within, a video, or an image that is
// not a baseline JPEG. Skipped is a deliberate, expected outcome, not an error
// (issue #17): the repair path is JPEG-only because `jpeg-js` is, and a PNG or
// HEIC is left for a format-aware tool rather than reported as a failure.
// has no thumbnail. "skipped": the file is a format this helper cannot
// regenerate — a video, or an image that is not a baseline JPEG. Skipped is a
// deliberate, expected outcome, not an error (issue #17): the repair path is
// JPEG-only because `jpeg-js` is, and a PNG or HEIC is left for a format-aware
// tool rather than reported as a failure.
export type ThumbnailFixStatus = "fixed" | "skipped" | "failed";
export interface ThumbnailFixResult {
@@ -59,7 +45,6 @@ export type ProgressCallback = (message: string) => void;
// exists, so it is logged and the file is left unreported. That distinction is
// what stops `fix-missing-thumbnails` from regenerating and uploading over
// thumbnails that were fine all along while the CDN was briefly returning 500s.
// Files another account owns are logged as skipped and not checked.
export const listMissingThumbnails = async (
lib: Library,
client: Client,
@@ -67,7 +52,6 @@ export const listMissingThumbnails = async (
): Promise<MissingThumbnailInfo[]> => {
const log = onProgress ?? (() => {});
const api = client.getApiClient();
const { userID } = client.whoami();
const missing: MissingThumbnailInfo[] = [];
const seen = new Set<number>();
@@ -76,13 +60,6 @@ export const listMissingThumbnails = async (
for (const photo of album.photos.list()) {
if (seen.has(photo.fileID)) continue;
seen.add(photo.fileID);
const file = lib.getFile(album.collectionID, photo.fileID);
if (file && file.ownerID !== userID) {
log(
`[${album.name}] Skipping ${photo.title}: ${NOT_OWNED_REASON}`,
);
continue;
}
try {
const stream = await api.getThumbnailStream(photo.fileID);
const reader = stream.getReader();
@@ -158,13 +135,17 @@ const resizeRGBA = (
return dst;
};
const generateThumbnail = (
decoded: { data: Uint8Array; width: number; height: number },
maxDimension: number,
quality: number,
): Uint8Array => {
const generateThumbnail = (fileBytes: Uint8Array): Uint8Array => {
const decoded = jpeg.decode(fileBytes, {
useTArray: true,
formatAsRGBA: true,
});
const { width: srcW, height: srcH } = decoded;
const scale = Math.min(maxDimension / srcW, maxDimension / srcH, 1);
const scale = Math.min(
THUMB_MAX_DIMENSION / srcW,
THUMB_MAX_DIMENSION / srcH,
1,
);
const dstW = Math.round(srcW * scale);
const dstH = Math.round(srcH * scale);
@@ -177,7 +158,7 @@ const generateThumbnail = (
const encoded = jpeg.encode(
{ data: pixels, width: dstW, height: dstH },
quality,
THUMB_JPEG_QUALITY,
);
return new Uint8Array(encoded.data);
};
@@ -190,35 +171,14 @@ const generateThumbnail = (
const isJpeg = (bytes: Uint8Array): boolean =>
bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xd8;
// The reason a file cannot have a JPEG thumbnail regenerated for it, known from
// its record alone before any bytes are fetched, or undefined when it might. A
// still image still has to be checked against its actual bytes once
// downloaded.
const reasonToSkip = (file: EnteFile, userID: number): string | undefined => {
if (file.ownerID !== userID) {
return NOT_OWNED_REASON;
}
// The reason a file cannot have a JPEG thumbnail regenerated for it from its
// metadata alone, before any bytes are fetched, or undefined when it might. A
// non-image (video, live photo) is unsupported outright; a still image still
// has to be checked against its actual bytes once downloaded.
const unsupportedByType = (file: EnteFile): string | undefined => {
if (file.metadata.fileType !== "image") {
return `unsupported file type: ${file.metadata.fileType} (only JPEG images can be regenerated)`;
}
if (!file.thumbnail.size) {
return `recorded thumbnail size is ${file.thumbnail.size ?? "unknown"} (the server refuses a thumbnail larger than the one it records)`;
}
return undefined;
};
// Encrypt the largest encoding of the decoded image whose ciphertext is no
// larger than `maxSize`, or return undefined when even the smallest is larger.
const encryptThumbnailWithin = (
decoded: { data: Uint8Array; width: number; height: number },
key: Uint8Array,
maxSize: number,
): { header: Uint8Array; ciphertext: Uint8Array } | undefined => {
for (const { maxDimension, quality } of THUMB_ENCODINGS) {
const thumbJpeg = generateThumbnail(decoded, maxDimension, quality);
const encrypted = encryptBlob(thumbJpeg, key);
if (encrypted.ciphertext.length <= maxSize) return encrypted;
}
return undefined;
};
@@ -237,7 +197,6 @@ export const fixMissingThumbnails = async (
const log = onProgress ?? (() => {});
const results: ThumbnailFixResult[] = [];
const api = client.getApiClient();
const { userID } = client.whoami();
// Resolve each requested fileID to its file record and owning album by
// enumerating the library, each file taken from the first album that holds
@@ -278,19 +237,18 @@ export const fixMissingThumbnails = async (
const { file, collectionName } = entry;
const title = file.metadata.title;
const skipReason = reasonToSkip(file, userID);
if (skipReason) {
log(`[${collectionName}] Skipping ${title}: ${skipReason}`);
const typeReason = unsupportedByType(file);
if (typeReason) {
log(`[${collectionName}] Skipping ${title}: ${typeReason}`);
results.push({
fileID,
title,
collection: collectionName,
status: "skipped",
reason: skipReason,
reason: typeReason,
});
continue;
}
const maxSize = file.thumbnail.size!;
try {
const photo = lib.photos.byID({ fileID });
@@ -319,28 +277,12 @@ export const fixMissingThumbnails = async (
}
log(`[${collectionName}] Generating thumbnail for ${title}...`);
const decoded = jpeg.decode(fileBytes, {
useTArray: true,
formatAsRGBA: true,
});
const fitting = encryptThumbnailWithin(decoded, file.key, maxSize);
if (!fitting) {
const reason = `no thumbnail encoding fits the recorded thumbnail size of ${maxSize} bytes`;
log(`[${collectionName}] Skipping ${title}: ${reason}`);
results.push({
fileID,
title,
collection: collectionName,
status: "skipped",
reason,
});
continue;
}
const { header, ciphertext } = fitting;
const thumbJpeg = generateThumbnail(fileBytes);
log(
`[${collectionName}] Uploading thumbnail (${ciphertext.length} bytes)...`,
`[${collectionName}] Encrypting and uploading thumbnail (${thumbJpeg.length} bytes)...`,
);
const { header, ciphertext } = encryptBlob(thumbJpeg, file.key);
const md5 = createHash("md5").update(ciphertext).digest("base64");
const { objectKey, url } = await api.getUploadURL(
ciphertext.length,
test/cli/backup.test.ts
+1 -627
View File
@@ -40,7 +40,6 @@ import {
readFileSync,
readlinkSync,
rmSync,
symlinkSync,
writeFileSync,
} from "node:fs";
import { spawnSync } from "node:child_process";
@@ -48,18 +47,10 @@ import { join } from "node:path";
import { tmpdir } from "node:os";
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { runBackup, type BackupLibrary } from "../../src/backup.js";
import { Library } from "../../src/library/index.js";
import type { ContentSource } from "../../src/library/content.js";
import type { CollectionsPage, FilesPage } from "../../src/client.js";
import type { Collection, EnteFile } from "../../src/model/types.js";
import {
asLivePhoto,
cdnSource,
IMAGE,
livePhotoZip,
VIDEO,
} from "../live-photo.js";
// `open` and `rename` are wrapped to record, in order, every fsync and rename,
// so a test can pin the sequence "fsync the temp file, rename, fsync the
@@ -569,39 +560,11 @@ describe("lib.backup", () => {
lib.close();
});
it("fetches each original once and writes it only into the backup", async () => {
// A backup of a 500 GB account must write 500 GB, not a copy in the
// cache as well: an original fetched for the backup goes straight
// into its originals/, and the cache records it there.
const source = stubSource();
const lib = await openLibrary(source);
const outDir = join(root, "backup");
const result = await lib.backup({ downloadDirectory: outDir });
expect(result.downloaded).toBe(3);
expect(source.originalCalls).toBe(3);
expect(readdirSync(join(root, "cache", "originals"))).toEqual([]);
const stored = readdirSync(join(outDir, "originals")).filter(
(name) => !name.endsWith(".json"),
);
expect(stored.sort()).toEqual(["100.jpg", "101.jpg", "200.png"]);
// The cache counts the backup's copy as present: reading the
// original afterwards fetches nothing and answers with that copy.
const read = await lib.photos.byID({ fileID: 100 })!.original();
expect(read.path).toBe(join(outDir, "originals", "100.jpg"));
expect(source.originalCalls).toBe(3);
await lib.close();
});
it("fsyncs an original copied from the cache before the rename and its directory after", async () => {
it("fsyncs a copied original before the rename and its directory after", async () => {
const lib = await openLibrary(stubSource());
const outDir = join(root, "backup");
const originals = join(outDir, "originals");
const dest = join(originals, "100.jpg");
// Only an original already in the cache is copied into the backup;
// one fetched for the backup is written there by the download writer.
await lib.photos.byID({ fileID: 100 })!.original();
fsEvents.length = 0;
await lib.backup({ downloadDirectory: outDir });
@@ -659,592 +622,3 @@ describe("lib.backup", () => {
lib.close();
});
});
// Every refresh fails, as with an expired session or no network.
class FailingClient extends MockClient {
override async collectionsSince(): Promise<CollectionsPage> {
throw new Error("HTTP 401 from server");
}
}
// Holds its refresh open until `release()` is called, then reports a third
// album, so a backup can be started while that refresh is still running.
class HeldClient extends MockClient {
release!: () => void;
private held = new Promise<void>((resolve) => {
this.release = resolve;
});
override async collectionsSince(): Promise<CollectionsPage> {
await this.held;
return {
collections: [collection(3, "Later")],
deleted: [],
cursor: 2,
};
}
override async filesSince(args: {
collectionID: number;
}): Promise<FilesPage> {
if (args.collectionID !== 3) return super.filesSince(args);
return { files: [file(300, 3, "late.jpg")], deleted: [], cursor: 2 };
}
}
// Fill the library cache on disk, so the next open starts its refresh in the
// background instead of waiting for it.
const fillCache = async (): Promise<void> => {
const lib = await openLibrary(stubSource());
await lib.close();
};
describe("the refresh before a backup", () => {
it("waits for a refresh already running and backs up what it found", async () => {
await fillCache();
const client = new HeldClient();
const lib = await openLibrary(stubSource(), client);
const outDir = join(root, "backup");
const backup = lib.backup({ downloadDirectory: outDir });
client.release();
const result = await backup;
expect(result.totalFiles).toBe(4);
expect(existsSync(join(outDir, "originals", "300.jpg"))).toBe(true);
await lib.close();
});
it("fails before any download when the refresh fails, leaving failures.json as it was", async () => {
await fillCache();
const outDir = join(root, "backup");
seedLedger(outDir, 100, "beach.jpg");
const ledgerPath = join(outDir, "failures.json");
const ledgerBefore = readFileSync(ledgerPath, "utf-8");
const source = stubSource();
const lib = await openLibrary(source, new FailingClient());
await expect(lib.backup({ downloadDirectory: outDir })).rejects.toThrow(
"HTTP 401 from server",
);
expect(source.originalCalls).toBe(0);
expect(readFileSync(ledgerPath, "utf-8")).toBe(ledgerBefore);
expect(existsSync(join(outDir, "originals"))).toBe(false);
await lib.close();
});
it("fails when the refresh fails on an empty cache, instead of backing up nothing", async () => {
const source = stubSource();
const lib = await openLibrary(source, new FailingClient());
const outDir = join(root, "backup");
await expect(lib.backup({ downloadDirectory: outDir })).rejects.toThrow(
"HTTP 401 from server",
);
expect(source.originalCalls).toBe(0);
expect(existsSync(outDir)).toBe(false);
await lib.close();
});
});
// Every entry under collections/, one level of directories deep, with each
// symlink's target.
const tree = (outDir: string): string[] => {
const lines: string[] = [];
const list = (dir: string, prefix: string): void => {
for (const name of readdirSync(dir).sort()) {
const path = join(dir, name);
const st = lstatSync(path);
if (st.isSymbolicLink()) {
lines.push(`${prefix}${name} -> ${readlinkSync(path)}`);
} else if (st.isDirectory() && prefix === "") {
lines.push(`${name}/`);
list(path, `${name}/`);
} else {
lines.push(`${prefix}${name}`);
}
}
};
list(join(outDir, "collections"), "");
return lines;
};
// The album folders under collections/, driven through `runBackup` with a
// stand-in library whose albums a test changes between runs.
describe("backup album folders", () => {
interface Album {
collection: Collection;
files: EnteFile[];
}
const libraryOf = (albums: Album[]): BackupLibrary => ({
refresh: async () => {},
listCollections: () => albums.map((a) => a.collection),
listFiles: (id) =>
albums.find((a) => a.collection.id === id)?.files ?? [],
original: async (fileID) => {
const path = join(root, `source-${fileID}`);
writeFileSync(path, `original ${fileID}`);
return { path };
},
thumbnail: async () => {
throw new Error("no thumbnails in this stand-in");
},
});
const albumID = (outDir: string, jsonName: string): number =>
JSON.parse(readFileSync(join(outDir, "collections", jsonName), "utf-8"))
.id;
it("gives every file and every album its own name when names repeat", async () => {
const outDir = join(root, "backup");
const lib = libraryOf([
{
collection: collection(10, "Trip"),
files: [
file(1, 10, "IMG_0001.JPG"),
file(2, 10, "IMG_0001.JPG"),
file(4, 10, "img_0001.jpg"),
file(3, 10, "other.jpg"),
],
},
{
collection: collection(11, "Trip"),
files: [file(3, 11, "other.jpg")],
},
]);
const result = await runBackup(lib, { downloadDirectory: outDir });
expect(result.failed).toBe(0);
expect(tree(outDir)).toEqual([
"Trip (10)/",
"Trip (10)/IMG_0001 (1).JPG -> ../../originals/1.JPG",
"Trip (10)/IMG_0001 (2).JPG -> ../../originals/2.JPG",
"Trip (10)/img_0001 (4).jpg -> ../../originals/4.jpg",
"Trip (10)/other.jpg -> ../../originals/3.jpg",
"Trip (10).json",
"Trip (11)/",
"Trip (11)/other.jpg -> ../../originals/3.jpg",
"Trip (11).json",
]);
expect(albumID(outDir, "Trip (10).json")).toBe(10);
expect(albumID(outDir, "Trip (11).json")).toBe(11);
});
it("keeps names unique when a name with an ID added is another entry's own name", async () => {
const outDir = join(root, "backup");
const lib = libraryOf([
{
collection: collection(10, "Trip"),
files: [
file(5, 10, "IMG (6).JPG"),
file(6, 10, "IMG.JPG"),
file(7, 10, "IMG.JPG"),
],
},
{
collection: collection(11, "Trip"),
files: [file(8, 11, "a.jpg")],
},
{
collection: collection(12, "Trip (11)"),
files: [file(9, 12, "b.jpg")],
},
]);
const result = await runBackup(lib, { downloadDirectory: outDir });
expect(result.failed).toBe(0);
expect(tree(outDir)).toEqual([
"Trip (10)/",
"Trip (10)/IMG (6) (5).JPG -> ../../originals/5.JPG",
"Trip (10)/IMG (6).JPG -> ../../originals/6.JPG",
"Trip (10)/IMG (7).JPG -> ../../originals/7.JPG",
"Trip (10).json",
"Trip (11)/",
"Trip (11)/a.jpg -> ../../originals/8.jpg",
"Trip (11) (12)/",
"Trip (11) (12)/b.jpg -> ../../originals/9.jpg",
"Trip (11) (12).json",
"Trip (11).json",
]);
expect(albumID(outDir, "Trip (10).json")).toBe(10);
expect(albumID(outDir, "Trip (11).json")).toBe(11);
expect(albumID(outDir, "Trip (11) (12).json")).toBe(12);
});
it("changes nothing on a second run over an unchanged account", async () => {
const outDir = join(root, "backup");
const lib = libraryOf([
{
collection: collection(10, "Trip"),
files: [
file(1, 10, "IMG_0001.JPG"),
file(2, 10, "IMG_0001.JPG"),
],
},
{
collection: collection(11, "Trip"),
files: [file(3, 11, "other.jpg")],
},
]);
await runBackup(lib, { downloadDirectory: outDir });
const before = tree(outDir);
const second = await runBackup(lib, { downloadDirectory: outDir });
expect(second.downloaded).toBe(0);
expect(second.failed).toBe(0);
expect(tree(outDir)).toEqual(before);
});
it("leaves the albums an onlyAlbumNames run skips as they were", async () => {
const outDir = join(root, "backup");
// "trip" is skipped by the scoped run but its name clashes with the
// in-scope "Trip", so "Trip" must keep its ID suffix.
const lib = libraryOf([
{
collection: collection(10, "Trip"),
files: [file(1, 10, "a.jpg")],
},
{
collection: collection(11, "trip"),
files: [file(2, 11, "b.jpg")],
},
{
collection: collection(12, "Work"),
files: [file(3, 12, "c.jpg")],
},
]);
const json = (name: string): string =>
readFileSync(join(outDir, "collections", name), "utf-8");
await runBackup(lib, { downloadDirectory: outDir });
const before = tree(outDir);
const skippedJSON = [json("trip (11).json"), json("Work.json")];
const scoped = await runBackup(lib, {
downloadDirectory: outDir,
onlyAlbumNames: ["Trip"],
});
expect(scoped.failed).toBe(0);
expect(before).toEqual([
"Trip (10)/",
"Trip (10)/a.jpg -> ../../originals/1.jpg",
"Trip (10).json",
"Work/",
"Work/c.jpg -> ../../originals/3.jpg",
"Work.json",
"trip (11)/",
"trip (11)/b.jpg -> ../../originals/2.jpg",
"trip (11).json",
]);
expect(tree(outDir)).toEqual(before);
expect([json("trip (11).json"), json("Work.json")]).toEqual(
skippedJSON,
);
});
it("removes links and album folders that are gone, and nothing the user added", async () => {
const outDir = join(root, "backup");
const albums: Album[] = [
{
collection: collection(10, "Trip"),
files: [
file(1, 10, "IMG_0001.JPG"),
file(2, 10, "IMG_0001.JPG"),
file(3, 10, "other.jpg"),
],
},
{
collection: collection(12, "Work"),
files: [file(5, 12, "a.jpg")],
},
{
collection: collection(13, "Old"),
files: [file(5, 13, "a.jpg")],
},
];
const lib = libraryOf(albums);
await runBackup(lib, { downloadDirectory: outDir });
// What the user put in the tree: a note and a symlink of their own in
// an album, a note in an album about to be renamed, and a folder quak
// did not create.
const collectionsDir = join(outDir, "collections");
writeFileSync(join(collectionsDir, "Trip", "notes.txt"), "mine");
symlinkSync("../elsewhere", join(collectionsDir, "Trip", "mine"));
writeFileSync(join(collectionsDir, "Work", "keep.txt"), "mine");
mkdirSync(join(collectionsDir, "Mine"));
writeFileSync(join(collectionsDir, "Mine", "keep.txt"), "mine");
// File 2 leaves Trip, Work is renamed Office, Old is deleted.
albums[0]!.files.splice(1, 1);
albums[1]!.collection = collection(12, "Office");
albums.splice(2, 1);
const result = await runBackup(lib, { downloadDirectory: outDir });
expect(result.failed).toBe(0);
expect(tree(outDir)).toEqual([
"Mine/",
"Mine/keep.txt",
"Office/",
"Office/a.jpg -> ../../originals/5.jpg",
"Office.json",
"Trip/",
"Trip/IMG_0001.JPG -> ../../originals/1.JPG",
"Trip/mine -> ../elsewhere",
"Trip/notes.txt",
"Trip/other.jpg -> ../../originals/3.jpg",
"Trip.json",
"Work/",
"Work/keep.txt",
"Work.json",
]);
});
// One album backed up, then a folder the user made beside it holding a
// symlink into originals/, with `json` (if given) as its sibling JSON.
const backupWithUserFolder = async (
json: string | undefined,
): Promise<{ outDir: string; failed: number }> => {
const outDir = join(root, "backup");
const lib = libraryOf([
{
collection: collection(10, "Trip"),
files: [file(1, 10, "a.jpg")],
},
]);
await runBackup(lib, { downloadDirectory: outDir });
const collectionsDir = join(outDir, "collections");
mkdirSync(join(collectionsDir, "Mine"));
symlinkSync(
"../../originals/1.jpg",
join(collectionsDir, "Mine", "a.jpg"),
);
if (json !== undefined) {
writeFileSync(join(collectionsDir, "Mine.json"), json);
}
const result = await runBackup(lib, { downloadDirectory: outDir });
return { outDir, failed: result.failed };
};
it("leaves a user folder with no JSON beside it as it was", async () => {
const { outDir, failed } = await backupWithUserFolder(undefined);
expect(failed).toBe(0);
expect(tree(outDir)).toEqual([
"Mine/",
"Mine/a.jpg -> ../../originals/1.jpg",
"Trip/",
"Trip/a.jpg -> ../../originals/1.jpg",
"Trip.json",
]);
});
it("leaves a user folder whose JSON has no album ID as it was", async () => {
const json = '{"name":"Mine"}';
const { outDir, failed } = await backupWithUserFolder(json);
expect(failed).toBe(0);
expect(tree(outDir)).toEqual([
"Mine/",
"Mine/a.jpg -> ../../originals/1.jpg",
"Mine.json",
"Trip/",
"Trip/a.jpg -> ../../originals/1.jpg",
"Trip.json",
]);
expect(
readFileSync(join(outDir, "collections", "Mine.json"), "utf-8"),
).toBe(json);
});
});
// A live photo, which Ente stores as one ZIP, is backed up as its image and
// its video, which a photo viewer can open, beside a JSON file naming them,
// and its album folder links both. These tests download a live photo ZIP
// through the real download layer (test/live-photo.ts).
describe("backup of live photos", () => {
// An account of one album, Trip (10), holding `files`.
class TripClient extends MockClient {
constructor(private readonly files: EnteFile[]) {
super();
}
override async collectionsSince(): Promise<CollectionsPage> {
return {
collections: [collection(10, "Trip")],
deleted: [],
cursor: 1,
};
}
override async filesSince(): Promise<FilesPage> {
return { files: this.files, deleted: [], cursor: 1 };
}
}
const open = (files: EnteFile[], bodies: Map<number, Uint8Array>) =>
openLibrary(cdnSource(bodies), new TripClient(files));
// What an earlier version stored for live photo 500: the ZIP under the
// image's name, and its link.
const earlierZIP = (outDir: string): void => {
mkdirSync(join(outDir, "originals"), { recursive: true });
mkdirSync(join(outDir, "collections", "Trip"), { recursive: true });
writeFileSync(join(outDir, "originals", "500.HEIC"), livePhotoZip());
symlinkSync(
"../../originals/500.HEIC",
join(outDir, "collections", "Trip", "IMG_0500.HEIC"),
);
};
const stored = ["500.heic", "500.json", "500.livephoto.json", "500.mov"];
const linked = [
"Trip/",
"Trip/IMG_0500.heic -> ../../originals/500.heic",
"Trip/IMG_0500.mov -> ../../originals/500.mov",
"Trip.json",
];
it("stores a live photo as its image and its video and links both", async () => {
const { file: live, body } = await asLivePhoto(
file(500, 10, "IMG_0500.HEIC"),
);
const lib = await open([live], new Map([[500, body]]));
const outDir = join(root, "backup");
const originals = join(outDir, "originals");
const result = await lib.backup({ downloadDirectory: outDir });
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
expect(readdirSync(originals).sort()).toEqual(stored);
expect(readFileSync(join(originals, "500.heic"))).toEqual(
Buffer.from(IMAGE),
);
expect(readFileSync(join(originals, "500.mov"))).toEqual(
Buffer.from(VIDEO),
);
expect(tree(outDir)).toEqual(linked);
// Both parts are there, so the next run fetches nothing.
const second = await lib.backup({ downloadDirectory: outDir });
expect(second).toMatchObject({ downloaded: 0, skipped: 1, failed: 0 });
await lib.close();
});
it("gives both links of each live photo their own names when titles clash", async () => {
const a = await asLivePhoto(file(500, 10, "IMG_0001.HEIC"));
const b = await asLivePhoto(file(501, 10, "IMG_0001.HEIC"));
const lib = await open(
[a.file, b.file],
new Map([
[500, a.body],
[501, b.body],
]),
);
const outDir = join(root, "backup");
await lib.backup({ downloadDirectory: outDir });
expect(tree(outDir)).toEqual([
"Trip/",
"Trip/IMG_0001 (500).heic -> ../../originals/500.heic",
"Trip/IMG_0001 (500).mov -> ../../originals/500.mov",
"Trip/IMG_0001 (501).heic -> ../../originals/501.heic",
"Trip/IMG_0001 (501).mov -> ../../originals/501.mov",
"Trip.json",
]);
await lib.close();
});
it("replaces the ZIP an earlier version stored, and its link", async () => {
const { file: live, body } = await asLivePhoto(
file(500, 10, "IMG_0500.HEIC"),
);
const outDir = join(root, "backup");
earlierZIP(outDir);
const lib = await open([live], new Map([[500, body]]));
const result = await lib.backup({ downloadDirectory: outDir });
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored);
expect(tree(outDir)).toEqual(linked);
await lib.close();
});
it("stores nothing for a live photo that fails its hash, and keeps what was there", async () => {
const { file: live, body } = await asLivePhoto(
file(500, 10, "IMG_0500.HEIC"),
livePhotoZip(),
"not:the recorded hash",
);
const outDir = join(root, "backup");
earlierZIP(outDir);
const lib = await open([live], new Map([[500, body]]));
const result = await lib.backup({ downloadDirectory: outDir });
expect(result).toMatchObject({ downloaded: 0, failed: 1 });
expect(result.errors.map((e) => e.fileID)).toEqual([500]);
expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]);
expect(readdirSync(join(outDir, "originals"))).toEqual(["500.HEIC"]);
expect(tree(outDir)).toEqual([
"Trip/",
"Trip/IMG_0500.HEIC -> ../../originals/500.HEIC",
"Trip.json",
]);
await lib.close();
});
it("copies both parts of a live photo the cache already holds", async () => {
const { file: live, body } = await asLivePhoto(
file(500, 10, "IMG_0500.HEIC"),
);
const lib = await open([live], new Map([[500, body]]));
const cached = await lib.photos.byID({ fileID: 500 })!.original();
const outDir = join(root, "backup");
const result = await lib.backup({ downloadDirectory: outDir });
expect(result).toMatchObject({ downloaded: 1, failed: 0 });
expect(readdirSync(join(outDir, "originals")).sort()).toEqual(stored);
expect(readFileSync(join(outDir, "originals", "500.mov"))).toEqual(
Buffer.from(VIDEO),
);
expect(tree(outDir)).toEqual(linked);
// The cache keeps its own copy.
expect(existsSync(cached.videoPath!)).toBe(true);
await lib.close();
});
it("serves a live photo the backup stored to a library reading the backup", async () => {
const { file: live, body } = await asLivePhoto(
file(500, 10, "IMG_0500.HEIC"),
);
const lib = await open([live], new Map([[500, body]]));
const outDir = join(root, "backup");
await lib.backup({ downloadDirectory: outDir });
await lib.close();
// Another cache over the same backup, whose server has nothing.
const reader = await Library.open({
client: new TripClient([live]),
cacheDirectory: join(root, "other-cache"),
downloadDirectory: outDir,
contentSource: cdnSource(new Map()),
refreshIntervalSeconds: 3600,
precacheThumbnails: false,
precacheOriginals: false,
});
const read = await reader.photos.byID({ fileID: 500 })!.original();
expect(read).toEqual({
path: join(outDir, "originals", "500.heic"),
videoPath: join(outDir, "originals", "500.mov"),
bytes: IMAGE.length,
});
await reader.close();
});
});
test/cli/commands.test.ts
+10 -402
View File
@@ -12,9 +12,7 @@
import {
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
statSync,
@@ -22,22 +20,11 @@ import {
} from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { PassThrough } from "node:stream";
import * as jpegJs from "jpeg-js";
import {
describe,
it,
expect,
vi,
beforeAll,
beforeEach,
afterEach,
} from "vitest";
import { describe, it, expect, beforeAll, beforeEach, afterEach } from "vitest";
import {
type CliContext,
saveSession,
loginCommand,
whoamiCommand,
logoutCommand,
collectionsCommand,
@@ -45,25 +32,14 @@ import {
getCommand,
getThumbCommand,
backupCommand,
backupMetadataCommand,
listMissingThumbnailsCommand,
fixMissingThumbnailsCommand,
} from "../../src/cli-commands.js";
import { run } from "../../src/cli-run.js";
import { loadSession } from "../../src/cli-session.js";
import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js";
import type { Client, ClientSnapshot } from "../../src/client.js";
import type { ContentSource } from "../../src/library/content.js";
import type { Collection, EnteFile } from "../../src/model/types.js";
import { init, toBase64 } from "../../src/crypto/index.js";
import { defaultCacheDirectory } from "../../src/library/index.js";
import {
asLivePhoto,
cdnSource,
IMAGE,
livePhotoHash,
livePhotoZip,
VIDEO,
} from "../live-photo.js";
const USER_ID = 42;
@@ -106,26 +82,8 @@ const FILES: Record<number, EnteFile[]> = {
// An original is 7 bytes and a thumbnail 3. `failID` makes that file's
// original fail; `emptyThumbID` makes the server report that file's
// thumbnail as empty. `withNewFile` adds new.jpg (102) to Vacation, advancing
// the collection's updationTime as the server does, and `refreshError` makes
// listing collections fail with that message.
const fakeClient = (
opts: {
failID?: number;
emptyThumbID?: number;
withNewFile?: boolean;
refreshError?: string;
} = {},
) => {
const collections = opts.withNewFile
? [{ ...COLLECTIONS[0], updationTime: 2 }, COLLECTIONS[1]]
: COLLECTIONS;
const files = opts.withNewFile
? {
...FILES,
1: [...FILES[1], { ...file(102, 1, "new.jpg"), updationTime: 2 }],
}
: FILES;
// thumbnail as empty.
const fakeClient = (opts: { failID?: number; emptyThumbID?: number } = {}) => {
const source: ContentSource = {
original: async ({ file: f, destination }) => {
if (f.id === opts.failID) throw new Error("HTTP 500 from server");
@@ -139,19 +97,18 @@ const fakeClient = (
};
const fake = {
whoami: () => ({ email: "cli@example.com", userID: USER_ID }),
collectionsSince: async () => {
if (opts.refreshError) throw new Error(opts.refreshError);
return { collections, deleted: [], cursor: 1 };
},
collectionsSince: async () => ({
collections: COLLECTIONS,
deleted: [],
cursor: 1,
}),
filesSince: async (args: { collectionID: number }) => ({
files: files[args.collectionID] ?? [],
files: FILES[args.collectionID] ?? [],
deleted: [],
cursor: 1,
}),
contentSource: () => source,
getApiClient: () => ({
// The ML data request of `backup-metadata`: no file has any.
postJSON: async () => ({ data: [] }),
getThumbnailStream: async (fileID: number) =>
new ReadableStream<Uint8Array>({
start(controller) {
@@ -185,15 +142,6 @@ const context = (client: Client | null = fakeClient()): CliContext => ({
sessionDir: join(root, "session"),
cacheDir: join(root, "cache"),
loadSession: () => client,
login: async () => {
throw new Error("login not expected");
},
prompt: async () => {
throw new Error("prompt not expected");
},
promptSecret: async () => {
throw new Error("prompt not expected");
},
});
beforeAll(async () => {
@@ -251,105 +199,6 @@ describe("session file", () => {
});
});
// The login function is a fake that hands back a client whose snapshot is
// `snapshot`; each prompt is recorded and answered with "123456".
describe("login", () => {
const snapshot: ClientSnapshot = {
email: "cli@example.com",
userID: USER_ID,
token: "token",
masterKey: "a",
secretKey: "b",
publicKey: "c",
};
const loggedIn = {
whoami: () => ({ email: "cli@example.com", userID: USER_ID }),
toJSON: () => snapshot,
} as unknown as Client;
let prompts: string[];
const loginContext = (
login: (opts: LoginOptions) => Promise<Client>,
): CliContext => ({
...context(),
login,
prompt: async (message) => {
prompts.push(message);
return "123456";
},
promptSecret: async (message) => {
prompts.push(message);
return "123456";
},
});
beforeEach(() => {
prompts = [];
vi.stubEnv("QUAK_EMAIL", "cli@example.com");
vi.stubEnv("QUAK_PASSWORD", "hunter2");
});
afterEach(() => {
vi.unstubAllEnvs();
});
it("takes the email and password from the environment without a prompt", async () => {
const calls: LoginOptions[] = [];
const ctx = loginContext(async (opts) => {
calls.push(opts);
return loggedIn;
});
expect(await loginCommand(ctx)).toBe(0);
expect(prompts).toEqual([]);
expect(calls).toHaveLength(1);
expect(calls[0]!.email).toBe("cli@example.com");
expect(calls[0]!.password).toBe("hunter2");
const path = join(ctx.sessionDir, "session.json");
expect(stderr.text).toBe(
"Authenticating...\n" +
`Logged in as cli@example.com (user ${USER_ID})\n` +
`Session saved to ${path}\n`,
);
});
it("saves the session with mode 0600 in a directory with mode 0700", async () => {
const ctx = loginContext(async () => loggedIn);
expect(await loginCommand(ctx)).toBe(0);
expect(statSync(ctx.sessionDir).mode & 0o777).toBe(0o700);
const path = join(ctx.sessionDir, "session.json");
expect(statSync(path).mode & 0o777).toBe(0o600);
expect(JSON.parse(readFileSync(path, "utf-8"))).toEqual(snapshot);
});
it("asks for the TOTP code when the account needs one", async () => {
let code: string | undefined;
const ctx = loginContext(async (opts) => {
code = await opts.totp!();
return loggedIn;
});
expect(await loginCommand(ctx)).toBe(0);
expect(prompts).toEqual(["TOTP code: "]);
expect(code).toBe("123456");
});
it("a failed login exits 1, says why and writes no session", async () => {
const ctx = loginContext(async () => {
throw new Error("HTTP 401 from server");
});
expect(await loginCommand(ctx)).toBe(1);
expect(stderr.text).toBe(
"Authenticating...\nLogin failed: HTTP 401 from server\n",
);
expect(existsSync(join(ctx.sessionDir, "session.json"))).toBe(false);
});
});
// These use a real client read from the session file, over a fake API that
// records each request and answers with `status`.
describe("logout", () => {
@@ -541,120 +390,6 @@ describe("get and get-thumb", () => {
});
});
// A live photo, which Ente stores as one ZIP, is written as its image and its
// video, which a photo viewer can open. The account here is Vacation holding
// one live photo, 300, downloaded through the real download layer
// (test/live-photo.ts).
describe("a live photo", () => {
const livePhotoClient = async (image = IMAGE): Promise<Client> => {
const { file: live, body } = await asLivePhoto(
file(300, 1, "IMG_0300.HEIC"),
livePhotoZip({ "image.heic": image, "video.mov": VIDEO }),
livePhotoHash(image, VIDEO),
);
const fake = {
whoami: () => ({ email: "cli@example.com", userID: USER_ID }),
collectionsSince: async () => ({
collections: [collection(1, "Vacation")],
deleted: [],
cursor: 1,
}),
filesSince: async () => ({ files: [live], deleted: [], cursor: 1 }),
contentSource: () => cdnSource(new Map([[300, body]])),
// The ML data request of `backup-metadata`: no file has any.
getApiClient: () => ({ postJSON: async () => ({ data: [] }) }),
};
return fake as unknown as Client;
};
it("get writes its image and video, named after the title with their own extensions", async () => {
const ctx = context(await livePhotoClient());
const dir = join(root, "cwd");
mkdirSync(dir);
const previous = process.cwd();
process.chdir(dir);
try {
expect(await getCommand(ctx, "300", {})).toBe(0);
} finally {
process.chdir(previous);
}
expect(readdirSync(dir).sort()).toEqual([
"IMG_0300.heic",
"IMG_0300.mov",
]);
expect(readFileSync(join(dir, "IMG_0300.heic"))).toEqual(
Buffer.from(IMAGE),
);
expect(readFileSync(join(dir, "IMG_0300.mov"))).toEqual(
Buffer.from(VIDEO),
);
expect(stderr.text).toBe(
`${IMAGE.length} bytes -> IMG_0300.heic\n` +
`${VIDEO.length} bytes -> IMG_0300.mov\n`,
);
});
it("get --out writes the image there and the video beside it", async () => {
const out = join(root, "photo.jpg");
const video = join(root, "photo.mov");
expect(
await getCommand(context(await livePhotoClient()), "300", { out }),
).toBe(0);
expect(readFileSync(out)).toEqual(Buffer.from(IMAGE));
expect(readFileSync(video)).toEqual(Buffer.from(VIDEO));
expect(stderr.text).toBe(
`${IMAGE.length} bytes -> ${out}\n${VIDEO.length} bytes -> ${video}\n`,
);
});
it("get exits 1 and writes nothing when --out has the video's extension", async () => {
const out = join(root, "photo.MOV");
expect(
await getCommand(context(await livePhotoClient()), "300", { out }),
).toBe(1);
expect(existsSync(out)).toBe(false);
expect(existsSync(join(root, "photo.mov"))).toBe(false);
expect(stderr.text).toBe(
`File 300 is a live photo, and its video would also be written to ${out}\n`,
);
});
it("backup-metadata --exif reads its image", async () => {
// A 4x4 JPEG, whose size can only come from reading the image; the
// ZIP and the video are not JPEGs.
const jpeg = jpegJs.encode(
{ data: new Uint8Array(4 * 4 * 4), width: 4, height: 4 },
50,
).data;
const dir = join(root, "dump");
expect(
await backupMetadataCommand(
context(await livePhotoClient(new Uint8Array(jpeg))),
dir,
{ exif: true },
),
).toBe(0);
const record = JSON.parse(
readFileSync(
join(dir, "collections", "1-Vacation", "300.json"),
"utf-8",
),
);
expect(record.imageMetadata).toMatchObject({
format: "jpeg",
width: 4,
height: 4,
});
});
});
describe("backup", () => {
it("exits 0 and prints a summary when every file is saved", async () => {
const dir = join(root, "backup");
@@ -669,16 +404,6 @@ describe("backup", () => {
expect(stdout.text).toBe("");
});
// The backup opens its library with the precache off: it fetches the
// originals it needs into the backup, and must not also fetch every
// thumbnail in the account, or keep originals, in the per-user cache.
it("leaves nothing in the cache's originals and thumbnails", async () => {
const dir = join(root, "backup");
expect(await backupCommand(context(), dir, {})).toBe(0);
expect(readdirSync(join(root, "cache", "thumbnails"))).toEqual([]);
expect(readdirSync(join(root, "cache", "originals"))).toEqual([]);
});
it("exits 1 and lists the file when one download fails", async () => {
const ctx = context(fakeClient({ failID: 101 }));
expect(await backupCommand(ctx, join(root, "backup"), {})).toBe(1);
@@ -705,34 +430,6 @@ describe("backup", () => {
expect(result.errors[0].fileID).toBe(101);
expect(stderr.text).toBe("Starting backup...\n");
});
it("exits 1 with the error on one line when the refresh fails", async () => {
const client = {
...fakeClient(),
collectionsSince: async () => {
throw new Error("HTTP 401 from server");
},
} as unknown as Client;
const dir = join(root, "backup");
// Through `run`, as `bin/quak.ts` does, which prints a thrown error.
const runStderr = new PassThrough();
let runText = "";
runStderr.on("data", (chunk: Buffer) => {
runText += chunk.toString();
});
const code = await new Promise<number>((resolve) => {
void run(
backupCommand(context(client), dir, {}),
new PassThrough(),
runStderr,
resolve,
);
});
expect(code).toBe(1);
expect(runText).toBe("quak: HTTP 401 from server\n");
expect(stderr.text).toBe("Starting backup...\nRefreshing library...\n");
expect(existsSync(join(dir, "originals"))).toBe(false);
});
});
describe("helper list-missing-thumbnails", () => {
@@ -765,92 +462,3 @@ describe("helper list-missing-thumbnails", () => {
expect(stderr.text).toBe("");
});
});
describe("backup-metadata --exif", () => {
// Runs the command and returns what it printed to stderr.
const backupMetadata = async (opts: { exif?: boolean; all?: boolean }) => {
expect(
await backupMetadataCommand(context(), join(root, "dump"), opts),
).toBe(0);
return stderr.text;
};
it("--exif extracts EXIF", async () => {
expect(await backupMetadata({ exif: true })).toContain(
"[beach.jpg] Extracting EXIF...\n",
);
});
it("--all extracts EXIF", async () => {
expect(await backupMetadata({ all: true })).toContain(
"[beach.jpg] Extracting EXIF...\n",
);
});
it("without either flag extracts no EXIF", async () => {
expect(await backupMetadata({})).not.toContain("Extracting EXIF");
});
});
// Each test first runs `collections` so the cache holds the account as it was,
// then changes the server under it.
describe("backup-metadata and the thumbnail helpers refresh first", () => {
beforeEach(async () => {
expect(await collectionsCommand(context(), {})).toBe(0);
stdout.text = "";
stderr.text = "";
});
it("backup-metadata writes a file added since the cache was written", async () => {
const ctx = context(fakeClient({ withNewFile: true }));
const dir = join(root, "dump");
expect(await backupMetadataCommand(ctx, dir, {})).toBe(0);
expect(
existsSync(join(dir, "collections", "1-Vacation", "102.json")),
).toBe(true);
});
it("list-missing-thumbnails checks a file added since the cache was written", async () => {
const ctx = context(
fakeClient({ withNewFile: true, emptyThumbID: 102 }),
);
expect(await listMissingThumbnailsCommand(ctx, {})).toBe(0);
expect(stdout.text).toBe(
"102\tnew.jpg\tVacation\tempty thumbnail (0 bytes)\n",
);
});
it("fix-missing-thumbnails finds a file added since the cache was written", async () => {
const ctx = context(fakeClient({ withNewFile: true }));
expect(
await fixMissingThumbnailsCommand(ctx, {
file: ["102"],
json: true,
}),
).toBe(0);
// Found, then skipped because the server records no thumbnail size
// for it; a file missing from the cache would fail as not found.
expect(JSON.parse(stdout.text)).toMatchObject([
{ fileID: 102, title: "new.jpg", status: "skipped" },
]);
});
// `run` in `cli-run.ts` prints a thrown error as one line and exits 1.
it("all three throw when the refresh fails", async () => {
const ctx = context(
fakeClient({ refreshError: "HTTP 503 from server" }),
);
const dir = join(root, "dump");
await expect(backupMetadataCommand(ctx, dir, {})).rejects.toThrow(
"HTTP 503 from server",
);
expect(existsSync(dir)).toBe(false);
await expect(listMissingThumbnailsCommand(ctx, {})).rejects.toThrow(
"HTTP 503 from server",
);
await expect(
fixMissingThumbnailsCommand(ctx, { file: ["100"] }),
).rejects.toThrow("HTTP 503 from server");
expect(stdout.text).toBe("");
});
});
test/cli/run.test.ts
-58
View File
@@ -1,58 +0,0 @@
/**
* Tests for `run` in `src/cli-run.ts`, which every CLI command goes through.
*/
import { PassThrough } from "node:stream";
import { describe, it, expect } from "vitest";
import { run } from "../../src/cli-run.js";
// A stream whose written text is kept in `text`; writes finish at once, so
// nothing is left waiting to drain.
const collector = (): { stream: PassThrough; text: () => string } => {
const stream = new PassThrough();
const chunks: string[] = [];
stream.on("data", (chunk: Buffer) => chunks.push(chunk.toString()));
return { stream, text: () => chunks.join("") };
};
const runToExit = async (
command: Promise<number>,
): Promise<{ code: number; stdout: string; stderr: string }> => {
const stdout = collector();
const stderr = collector();
const code = await new Promise<number>((resolve) => {
void run(command, stdout.stream, stderr.stream, resolve);
});
return { code, stdout: stdout.text(), stderr: stderr.text() };
};
describe("run", () => {
it("exits with the code the command returns", async () => {
const result = await runToExit(Promise.resolve(3));
expect(result).toEqual({ code: 3, stdout: "", stderr: "" });
});
it("prints a thrown error as one line without a stack trace and exits 1", async () => {
const failing = async (): Promise<number> => {
throw new Error(
"ENOTDIR: not a directory, mkdir '/dev/null/x/originals'",
);
};
const result = await runToExit(failing());
expect(result.code).toBe(1);
expect(result.stdout).toBe("");
expect(result.stderr).toBe(
"quak: ENOTDIR: not a directory, mkdir '/dev/null/x/originals'\n",
);
});
it("prints a thrown value that is not an Error", async () => {
const result = await runToExit(Promise.reject("offline"));
expect(result).toEqual({
code: 1,
stdout: "",
stderr: "quak: offline\n",
});
});
});
test/crypto/hash.test.ts
-33
View File
@@ -1,33 +0,0 @@
import { beforeAll, describe, expect, it } from "vitest";
import {
chunkHashFinal,
chunkHashInit,
chunkHashUpdate,
init,
} from "../../src/crypto/index.js";
beforeAll(async () => {
await init();
});
describe("content hash", () => {
// RFC 7693 Appendix A: BLAKE2b-512 of "abc".
const abc = Buffer.from(
"ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1" +
"7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923",
"hex",
).toString("base64");
it("is unkeyed BLAKE2b-512 in standard base64", () => {
const state = chunkHashInit();
chunkHashUpdate(state, new TextEncoder().encode("abc"));
expect(chunkHashFinal(state)).toBe(abc);
});
it("gives the same hash when the input arrives in chunks", () => {
const state = chunkHashInit();
chunkHashUpdate(state, new TextEncoder().encode("a"));
chunkHashUpdate(state, new TextEncoder().encode("bc"));
expect(chunkHashFinal(state)).toBe(abc);
});
});
test/crypto/kdf.test.ts
+4 -4
View File
@@ -29,10 +29,10 @@ describe("crypto.deriveKEK (Argon2id)", () => {
});
/**
* Cheap parameters used so the test suite stays under the 90-second
* `timeout` in the `test` phase of the `Dockerfile`. The real production
* parameters Ente uses are larger (memLimit up to 1 GiB, opsLimit 3-16).
* The algorithm is the same regardless of parameters.
* Cheap parameters used so the test suite stays under the 30-second
* budget. The real production parameters Ente uses are larger
* (memLimit up to 1 GiB, opsLimit 3-16). The algorithm is the same
* regardless of parameters.
*/
const TEST_OPS = 2;
const TEST_MEM = 64 * 1024 * 1024; // 64 MiB
test/download/download.test.ts
+12 -295
View File
@@ -55,14 +55,12 @@ import {
readFileSync,
rmSync,
mkdtempSync,
statSync,
writeFileSync,
} from "node:fs";
import { dirname, join } from "node:path";
import { tmpdir } from "node:os";
import { createHash } from "node:crypto";
import sodium from "libsodium-wrappers-sumo";
import { zipSync } from "fflate";
import {
beforeAll,
beforeEach,
@@ -82,7 +80,6 @@ import {
writeAtomic,
} from "../../src/download/index.js";
import type { EnteFile, FileMetadata } from "../../src/model/types.js";
import { IMAGE, livePhotoHash, livePhotoZip, VIDEO } from "../live-photo.js";
// ---------------------------------------------------------------------------
// Test helpers
@@ -108,8 +105,6 @@ import { IMAGE, livePhotoHash, livePhotoZip, VIDEO } from "../live-photo.js";
const renameHook = vi.hoisted(() => ({
calls: [] as { from: string; to: string; sourceExisted: boolean }[],
failWith: null as Error | null,
// When set, only a rename to this path fails.
failTo: null as string | null,
}));
/**
@@ -185,10 +180,7 @@ vi.mock("node:fs/promises", async (importOriginal) => {
sourceExisted: sourceExists(from),
});
durabilityHook.events.push(`rename:${to}`);
if (
renameHook.failWith !== null &&
(renameHook.failTo === null || renameHook.failTo === to)
) {
if (renameHook.failWith !== null) {
throw renameHook.failWith;
}
await actual.rename(from, to);
@@ -196,34 +188,9 @@ vi.mock("node:fs/promises", async (importOriginal) => {
};
});
/**
* `chunkHashUpdate` is wrapped to record the length of every piece hashed, so
* a test can show that a live photo entry reaches the hash in pieces far
* smaller than the entry, rather than decompressed whole first.
*/
const hashHook = vi.hoisted(() => ({
lengths: [] as number[],
}));
vi.mock("../../src/crypto/index.js", async (importOriginal) => {
const actual =
await importOriginal<typeof import("../../src/crypto/index.js")>();
return {
...actual,
chunkHashUpdate: (
...args: Parameters<typeof actual.chunkHashUpdate>
): void => {
hashHook.lengths.push(args[1].length);
actual.chunkHashUpdate(...args);
},
};
});
beforeEach(() => {
hashHook.lengths.length = 0;
renameHook.calls.length = 0;
renameHook.failWith = null;
renameHook.failTo = null;
durabilityHook.events.length = 0;
writeHook.writes.length = 0;
});
@@ -256,8 +223,7 @@ afterAll(() => {
* `sodium.randombytes_buf` goes through the wasm wrapper a byte at a time and
* costs roughly 20 seconds for the 4 MiB chunk below — about two hundred
* times what it costs to encrypt the same buffer, and on its own enough to
* push `make test` past the 90-second `timeout` in the `test` phase of the
* `Dockerfile`. This loop fills
* push `make test` past the 30-second cap in `script/test`. This loop fills
* 4 MiB in a few milliseconds.
*/
const patternBytes = (length: number, seed: number): Uint8Array => {
@@ -1043,10 +1009,13 @@ describe.each(entryPoints)(
expect(readdirSync(dir)).toEqual([]);
});
// Root ignores directory permissions, so this fails when run as root.
// The `test` phase of the `Dockerfile` runs as the `node` user.
it("fails without creating anything when the destination directory is not writable", async () => {
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
// Root ignores directory permissions, so this cannot fail as root
// (the Docker test image runs as root).
it.skipIf(process.getuid?.() === 0)(
"fails without creating anything when the destination directory is not writable",
async () => {
const key =
sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(
patternBytes(64, 34),
key,
@@ -1064,7 +1033,8 @@ describe.each(entryPoints)(
}
expect(readdirSync(dir)).toEqual([]);
});
},
);
},
);
@@ -1568,13 +1538,7 @@ describe("writeAtomic", () => {
// directory so that new entry is on disk too. Do the directory fsync
// before the rename, or skip it, and a crash can lose the rename.
expect(durabilityHook.events).toHaveLength(3);
// The temp name carries this process's ID, so a library opening the
// same cache can tell a write in progress from a leftover.
const tempSync = durabilityHook.events[0]!;
expect(tempSync.startsWith(`sync:w:${dir}/`)).toBe(true);
expect(tempSync.slice(`sync:w:${dir}/`.length)).toMatch(
new RegExp(`^\\.quak-${process.pid}-[0-9a-f]{32}\\.tmp$`),
);
expect(durabilityHook.events[0]).toMatch(/^sync:w:.*\.tmp$/);
expect(durabilityHook.events[1]).toBe(`rename:${dest}`);
expect(durabilityHook.events[2]).toBe(`sync:r:${dir}`);
});
@@ -1649,250 +1613,3 @@ describe.each(entryPoints)("$name progress", ({ name, download }) => {
expectSameBytes(readFileSync(outPath), plaintext);
});
});
// Node's own BLAKE2b-512 is the reference, so the tests below do not depend on
// the code under test to compute what they expect.
const blake2b = (bytes: Uint8Array): string =>
createHash("blake2b512").update(bytes).digest("base64");
// Serve `plaintext` encrypted as file 999 with the given metadata, to be
// written to `f.bin` in a fresh directory. Four responses are scripted so a
// retried failure would show in `requests`.
const setup = (plaintext: Uint8Array, metadata: Partial<FileMetadata>) => {
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(plaintext, key);
const file = buildMockEnteFile(key, header, header);
file.metadata = { ...file.metadata, ...metadata };
const body = { kind: "body", bytes: ciphertext } as const;
const { fetch, requests } = scriptedCdnFetch(body, body, body, body);
const api = new ApiClient({ fetch, retry: { ...noWait, attempts: 4 } });
const dir = mkdtempSync(join(testDir, "hash-"));
const outPath = join(dir, "f.bin");
return {
run: () => downloadFile(api, file, outPath),
api,
file,
dir,
outPath,
requests,
};
};
describe("downloadFile content hash", () => {
it("stores a file whose hash matches", async () => {
const plaintext = patternBytes(700, 80);
const t = setup(plaintext, { hash: blake2b(plaintext) });
await t.run();
expectSameBytes(readFileSync(t.outPath), plaintext);
});
it("rejects a mismatch, stores nothing, names the file and does not retry", async () => {
const t = setup(patternBytes(700, 80), {
hash: blake2b(patternBytes(700, 79)),
});
await expect(t.run()).rejects.toThrow(
/file 999: content hash .* does not match/,
);
expect(readdirSync(t.dir)).toEqual([]);
expect(t.requests()).toBe(1);
});
it("stores a file with no recorded hash unchecked", async () => {
const plaintext = patternBytes(700, 80);
const t = setup(plaintext, { hash: undefined });
await t.run();
expectSameBytes(readFileSync(t.outPath), plaintext);
});
it("stores a live photo whose image and video hashes match", async () => {
const t = setup(livePhotoZip(), {
fileType: "livePhoto",
hash: livePhotoHash(),
});
await t.run();
expect(readFileSync(join(t.dir, "f.heic"))).toEqual(Buffer.from(IMAGE));
expect(readFileSync(join(t.dir, "f.mov"))).toEqual(Buffer.from(VIDEO));
});
it("writes a large live photo entry as it decompresses, never whole", async () => {
// 64 MiB of zeros deflates to a few kilobytes, the shape of a ZIP
// that would exhaust memory if expanded whole.
const image = new Uint8Array(64 * 1024 * 1024);
const video = patternBytes(900, 83);
const zip = zipSync({ "image.heic": image, "video.mov": video });
const t = setup(zip, {
fileType: "livePhoto",
hash: livePhotoHash(image, video),
});
await t.run();
expect(statSync(join(t.dir, "f.heic")).size).toBe(image.length);
expectSameBytes(readFileSync(join(t.dir, "f.mov")), video);
const hashed = hashHook.lengths.reduce((a, b) => a + b, 0);
expect(hashed).toBe(image.length + video.length);
expect(Math.max(...hashHook.lengths)).toBeLessThanOrEqual(
2 * STREAM_CHUNK_SIZE,
);
const written = writeHook.writes.filter((w) => w.path.endsWith(".tmp"));
expect(Math.max(...written.map((w) => w.length))).toBeLessThanOrEqual(
2 * STREAM_CHUNK_SIZE,
);
});
it("rejects a live photo whose hash does not match, keeping what was there", async () => {
// The whole ZIP's hash is not the recorded one: each part is hashed.
const zip = livePhotoZip();
const t = setup(zip, { fileType: "livePhoto", hash: blake2b(zip) });
writeFileSync(t.outPath, "an earlier download");
await expect(t.run()).rejects.toThrow(
/file 999: content hash .* does not match/,
);
expect(readdirSync(t.dir)).toEqual(["f.bin"]);
expect(readFileSync(t.outPath, "utf-8")).toBe("an earlier download");
});
it("rejects a live photo that is not a readable ZIP and does not retry", async () => {
// Bytes 8-9 of a ZIP entry's local header name its compression
// method; 99 is one no reader knows, so the entry cannot be read.
const zip = livePhotoZip();
zip[8] = 99;
zip[9] = 0;
const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() });
await expect(t.run()).rejects.toThrow(
/file 999: live photo is not a readable ZIP/,
);
expect(readdirSync(t.dir)).toEqual([]);
expect(t.requests()).toBe(1);
});
it("rejects a live photo ZIP with no image entry", async () => {
const zip = livePhotoZip({ "video.mov": VIDEO });
const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() });
await expect(t.run()).rejects.toThrow(
/file 999: live photo ZIP does not hold both an image and a video/,
);
expect(readdirSync(t.dir)).toEqual([]);
});
it("rejects a live photo ZIP with no video entry", async () => {
const zip = livePhotoZip({ "image.heic": IMAGE });
const t = setup(zip, { fileType: "livePhoto", hash: livePhotoHash() });
await expect(t.run()).rejects.toThrow(
/file 999: live photo ZIP does not hold both an image and a video/,
);
expect(readdirSync(t.dir)).toEqual([]);
});
});
// ---------------------------------------------------------------------------
// Live photos
//
// A live photo arrives as a ZIP of its image and its video. It is written as
// those two files, which a photo viewer can open, each named after the
// destination with its own extension from the ZIP, the way Ente's clients name
// them when they save one.
// ---------------------------------------------------------------------------
describe("downloadFile live photos", () => {
const livePhoto = { fileType: "livePhoto", hash: livePhotoHash() } as const;
it("names the image and the video after the title when no outPath is given", async () => {
const t = setup(livePhotoZip(), {
...livePhoto,
title: "IMG_1234.HEIC",
});
const result = await inDirectory(t.dir, () =>
downloadFile(t.api, t.file),
);
// `bytesWritten` is the length of the decrypted ZIP.
expect(result).toEqual({
path: "IMG_1234.heic",
videoPath: "IMG_1234.mov",
bytesWritten: livePhotoZip().length,
});
expect(readdirSync(t.dir).sort()).toEqual([
"IMG_1234.heic",
"IMG_1234.mov",
]);
});
it("gives each part its own extension from the ZIP, letters and digits only", async () => {
const zip = livePhotoZip({ "image.JPG": IMAGE, "video.m-4v": VIDEO });
const t = setup(zip, livePhoto);
const result = await downloadFile(t.api, t.file, join(t.dir, "f.HEIC"));
expect(result.path).toBe(join(t.dir, "f.JPG"));
expect(result.videoPath).toBe(join(t.dir, "f.bin"));
expect(readdirSync(t.dir).sort()).toEqual(["f.JPG", "f.bin"]);
});
it("replaces what was at the destination, such as an earlier ZIP of the two", async () => {
const t = setup(livePhotoZip(), livePhoto);
writeFileSync(t.outPath, livePhotoZip());
await t.run();
expect(readdirSync(t.dir).sort()).toEqual(["f.heic", "f.mov"]);
});
it("renames the image and then the video into place, each from its own temp file", async () => {
const t = setup(livePhotoZip(), livePhoto);
await t.run();
expect(
renameHook.calls.map((c) => [
dirname(c.from),
c.to,
c.sourceExisted,
]),
).toEqual([
[t.dir, join(t.dir, "f.heic"), true],
[t.dir, join(t.dir, "f.mov"), true],
]);
});
it("stores neither part when the video cannot be renamed into place", async () => {
const t = setup(livePhotoZip(), livePhoto);
renameHook.failWith = new Error("simulated rename failure");
renameHook.failTo = join(t.dir, "f.mov");
await expect(t.run()).rejects.toThrow("simulated rename failure");
expect(readdirSync(t.dir)).toEqual([]);
});
it("refuses an image and a video with the same extension, storing nothing", async () => {
// On a file system that ignores case, the two would be one file.
const zip = livePhotoZip({ "image.mov": IMAGE, "video.MOV": VIDEO });
const t = setup(zip, livePhoto);
writeFileSync(t.outPath, "an earlier download");
await expect(t.run()).rejects.toThrow(
/file 999: live photo's image and video have the same extension/,
);
expect(readdirSync(t.dir)).toEqual(["f.bin"]);
expect(t.requests()).toBe(1);
});
});
test/library/content.test.ts
+2 -151
View File
@@ -31,11 +31,8 @@ import {
existsSync,
writeFileSync,
mkdirSync,
readdirSync,
readFileSync,
statSync,
} from "node:fs";
import { spawnSync } from "node:child_process";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -46,13 +43,6 @@ import {
} from "../../src/library/content.js";
import { RequestPools } from "../../src/library/pools.js";
import type { EnteFile } from "../../src/model/types.js";
import {
asLivePhoto,
cdnSource,
IMAGE,
livePhotoZip,
VIDEO,
} from "../live-photo.js";
const file = (id: number, title = `file-${id}.jpg`): EnteFile => ({
id,
@@ -171,28 +161,15 @@ describe("ContentCache.open", () => {
expect(statSync(thumbnails).mode & 0o777).toBe(0o700);
});
it("removes temp files of an exited process, keeping those of a running one and complete content", async () => {
it("reaps orphan temp files but keeps complete content", async () => {
const originals = join(cacheDir, "originals");
const thumbnails = join(cacheDir, "thumbnails");
mkdirSync(originals, { recursive: true });
mkdirSync(thumbnails, { recursive: true });
// A child that has already exited: its process ID is not running.
const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid;
const orphan = join(originals, `.quak-${exitedPID}-abc123.tmp`);
const orphanThumb = join(thumbnails, `.quak-${exitedPID}-abc456.tmp`);
// This test's own process stands in for another process still
// downloading into the same cache.
const inProgress = join(originals, `.quak-${process.pid}-def123.tmp`);
const inProgressThumb = join(
thumbnails,
`.quak-${process.pid}-def456.tmp`,
);
const orphan = join(originals, ".quak-abc123.tmp");
const complete = join(originals, "1.jpg");
const thumb = join(thumbnails, "2.jpg");
writeFileSync(orphan, "half-written");
writeFileSync(orphanThumb, "half-written");
writeFileSync(inProgress, "half-written");
writeFileSync(inProgressThumb, "half-written");
writeFileSync(complete, "whole");
writeFileSync(thumb, "whole-thumb");
@@ -200,12 +177,8 @@ describe("ContentCache.open", () => {
await cache.open();
expect(existsSync(orphan)).toBe(false);
expect(existsSync(orphanThumb)).toBe(false);
expect(existsSync(inProgress)).toBe(true);
expect(existsSync(inProgressThumb)).toBe(true);
expect(existsSync(complete)).toBe(true);
expect(existsSync(thumb)).toBe(true);
expect(cache.pathsFor(1)).toEqual({ originalPath: complete });
});
it("records already-cached files so their paths appear in pathsFor", async () => {
@@ -467,125 +440,3 @@ describe("ContentCache.ensureThumbnails", () => {
expect(results[1]?.error).toMatch(/unknown file/i);
});
});
// A live photo's original is two files, its image and its video, which a
// photo viewer can open, and a JSON file naming them: the two are named with
// the extensions from inside the ZIP, so the names alone do not say which is
// which. These tests download a live photo ZIP through the real download
// layer (test/live-photo.ts).
describe("ContentCache live photos", () => {
const originals = (): string => join(cacheDir, "originals");
// A cache over the stand-in server, which holds `bodies` by file ID.
const cacheOf = (
files: EnteFile[],
bodies: Map<number, Uint8Array>,
): ContentCache => buildCache({ files, source: cdnSource(bodies) }).cache;
it("stores a live photo as its image and its video and a JSON file naming them", async () => {
const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC"));
const cache = cacheOf([live], new Map([[5, body]]));
await cache.open();
const result = await cache.original(5);
expect(result).toEqual({
path: join(originals(), "5.heic"),
videoPath: join(originals(), "5.mov"),
bytes: IMAGE.length,
});
expect(readFileSync(result.path)).toEqual(Buffer.from(IMAGE));
expect(readFileSync(result.videoPath!)).toEqual(Buffer.from(VIDEO));
expect(statSync(result.videoPath!).mode & 0o777).toBe(0o600);
expect(
JSON.parse(
readFileSync(join(originals(), "5.livephoto.json"), "utf-8"),
),
).toEqual({ image: "5.heic", video: "5.mov" });
expect(cache.pathsFor(5)).toEqual({ originalPath: result.path });
});
it("serves a stored live photo from disk after the cache is opened again", async () => {
const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC"));
const first = cacheOf([live], new Map([[5, body]]));
await first.open();
const stored = await first.original(5);
// This server has nothing, so a fetch would fail.
const second = cacheOf([live], new Map());
await second.open();
const events: string[] = [];
const served = await second.original(5, {
onProgress: (e) => events.push(e.status),
});
expect(served).toEqual(stored);
expect(events).toEqual(["skipped"]);
});
it("replaces a live photo an earlier version stored as a ZIP under the image's name", async () => {
const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC"));
mkdirSync(originals(), { recursive: true });
writeFileSync(join(originals(), "5.HEIC"), livePhotoZip());
const cache = cacheOf([live], new Map([[5, body]]));
await cache.open();
const result = await cache.original(5);
expect(result.videoPath).toBe(join(originals(), "5.mov"));
expect(readdirSync(originals()).sort()).toEqual([
"5.heic",
"5.livephoto.json",
"5.mov",
]);
});
it("evicts a live photo's image, video and JSON file together", async () => {
const a = await asLivePhoto(file(5, "a.HEIC"));
const b = await asLivePhoto(file(6, "b.HEIC"));
const size = IMAGE.length + VIDEO.length;
const cache = new ContentCache({
pools: new RequestPools(),
source: cdnSource(
new Map([
[5, a.body],
[6, b.body],
]),
),
cacheDirectory: cacheDir,
getFile: (id) => [a.file, b.file].find((f) => f.id === id),
// Room for one live photo, on a disk with plenty free.
cacheOriginalsMaxBytes: size,
freeBelowBytes: 0,
statfs: async () => ({ bsize: 1, bavail: 1e12 }),
});
await cache.open();
await cache.original(5);
await cache.original(6);
expect(readdirSync(originals()).sort()).toEqual([
"6.heic",
"6.livephoto.json",
"6.mov",
]);
expect(cache.originalsStatus().usedBytes).toBe(size);
});
it("stores nothing when a live photo does not match its recorded hash", async () => {
const { file: live, body } = await asLivePhoto(
file(5, "IMG_5.HEIC"),
livePhotoZip(),
"not:the recorded hash",
);
const cache = cacheOf([live], new Map([[5, body]]));
await cache.open();
await expect(cache.original(5)).rejects.toThrow(
/file 5: content hash .* does not match/,
);
expect(readdirSync(originals())).toEqual([]);
expect(cache.pathsFor(5)).toEqual({});
});
});
test/live-photo.ts
-77
View File
@@ -1,77 +0,0 @@
/**
* Live photo fixtures for the download, content cache, backup and CLI tests.
*
* Ente stores a live photo as one ZIP holding its image and its video, which
* Ente's clients name `image.<ext>` and `video.<ext>`. The ZIP is built here
* with fflate from small fixed bytes and encrypted the way the server serves a
* file under 4 MiB, as one secretstream chunk. `cdnSource` serves it to the
* real download layer, so a test checks what quak stores for a real one.
*/
import { createHash } from "node:crypto";
import { zipSync } from "fflate";
import { ApiClient } from "../src/api/client.js";
import { encryptBlob, init, toBase64 } from "../src/crypto/index.js";
import {
makeDownloadContentSource,
type ContentSource,
} from "../src/library/content.js";
import type { EnteFile } from "../src/model/types.js";
export const IMAGE = new TextEncoder().encode("the still image");
export const VIDEO = new TextEncoder().encode("the few seconds of video");
// A live photo's ZIP; by default the entries an iPhone's live photo gets.
export const livePhotoZip = (
entries: Record<string, Uint8Array> = {
"image.heic": IMAGE,
"video.mov": VIDEO,
},
): Uint8Array => zipSync(entries);
const blake2b = (bytes: Uint8Array): string =>
createHash("blake2b512").update(bytes).digest("base64");
// The hash Ente's clients record for a live photo: the unkeyed BLAKE2b-512 of
// the image and of the video, each in standard base64, joined by a colon.
export const livePhotoHash = (image = IMAGE, video = VIDEO): string =>
`${blake2b(image)}:${blake2b(video)}`;
// `file` as a live photo whose original is `zip` and whose recorded hash is
// `hash`, and `body`, what the server serves for it: `zip` encrypted under the
// file's key and header.
export const asLivePhoto = async (
file: EnteFile,
zip = livePhotoZip(),
hash = livePhotoHash(),
): Promise<{ file: EnteFile; body: Uint8Array }> => {
await init();
const key = new Uint8Array(32).fill(file.id & 0xff);
const { header, ciphertext } = encryptBlob(zip, key);
return {
file: {
...file,
key,
metadata: { ...file.metadata, fileType: "livePhoto", hash },
file: { decryptionHeader: toBase64(header) },
},
body: ciphertext,
};
};
// A content source that downloads through the real download layer from a
// stand-in server, which serves `bodies` by file ID and a 404 for any other.
export const cdnSource = (bodies: Map<number, Uint8Array>): ContentSource =>
makeDownloadContentSource(
new ApiClient({
fetch: (async (url: string | URL) => {
const fileID = new URL(String(url)).searchParams.get("fileID");
const body = bodies.get(Number(fileID));
return body === undefined
? new Response("not found", { status: 404 })
: new Response(body);
}) as typeof globalThis.fetch,
retry: { attempts: 1 },
}),
);
test/model/decrypt.test.ts
-40
View File
@@ -351,46 +351,6 @@ describe("model.decryptFile", () => {
}
});
it("reads the recorded content hash, joining an older live photo's two parts", () => {
const masterKey = sodium.crypto_secretbox_keygen();
const { collectionKey } = buildRawCollection(masterKey);
const hashOf = (metadata: Record<string, unknown>) =>
decryptFile(
buildRawFile(collectionKey, {
metadata: { title: "x", ...metadata },
}),
collectionKey,
).metadata.hash;
expect(hashOf({ fileType: 0, hash: "H" })).toBe("H");
expect(
hashOf({ fileType: 2, hash: "H", imageHash: "I", videoHash: "V" }),
).toBe("H");
expect(hashOf({ fileType: 2, imageHash: "I", videoHash: "V" })).toBe(
"I:V",
);
expect(hashOf({ fileType: 2, imageHash: "I" })).toBeUndefined();
expect(
hashOf({ fileType: 0, imageHash: "I", videoHash: "V" }),
).toBeUndefined();
expect(hashOf({ fileType: 0 })).toBeUndefined();
expect(hashOf({ fileType: 0, hash: 42 })).toBeUndefined();
expect(
hashOf({ fileType: 2, imageHash: "I", videoHash: 7 }),
).toBeUndefined();
// An empty string counts as absent, not as a hash to match.
expect(hashOf({ fileType: 0, hash: "" })).toBeUndefined();
expect(
hashOf({ fileType: 2, hash: "", imageHash: "I", videoHash: "V" }),
).toBe("I:V");
expect(
hashOf({ fileType: 2, imageHash: "", videoHash: "V" }),
).toBeUndefined();
expect(
hashOf({ fileType: 2, imageHash: "I", videoHash: "" }),
).toBeUndefined();
});
it("maps fileType numbers to FileType strings", () => {
// Ente uses: 0=image, 1=video, 2=livePhoto
const masterKey = sodium.crypto_secretbox_keygen();
test/packaging/build-context.test.ts
+17 -12
View File
@@ -2,13 +2,13 @@
// failures are silent.
//
// Excluding too little: a worktree left under `.claude/` is copied into the
// image, vitest globs its `test/` tree as well as the real one, and the test
// phase runs the whole suite twice over while reporting success. A compiled
// `bin/quak` is ~100 MB of context nobody needs.
// image, vitest globs its `test/` tree as well as the real one, and the
// containerised `make check` runs the whole suite twice over while reporting
// success. A compiled `bin/quak` is ~100 MB of context nobody needs.
//
// Excluding too much: Prettier 3 reads `.gitignore` as a default ignore file,
// so dropping it from the context silently changes which files the lint
// phase's prettier check looks at compared to `make fmt-check` on the host.
// so dropping it from the context silently changes which files
// `make fmt-check` looks at inside the image compared to the host.
//
// Neither shows up as a build failure, so they are asserted here.
import { describe, expect, it } from "vitest";
@@ -47,13 +47,18 @@ describe(".dockerignore", () => {
expect(dockerignore).not.toContain(".gitignore");
});
// BuildKit lets a `Dockerfile.dockerignore` shadow the root one; such a
// file would silently give the build a different, unreviewed context —
// and eslint's flat config does not ignore dot-directories, so a stray
// `.claude/` worktree would be linted.
it("is not shadowed by a Dockerfile.dockerignore", () => {
expect(existsSync(join(repoRoot, "Dockerfile.dockerignore"))).toBe(
// Both images are built from this same context, and the lint image runs
// eslint and prettier across it. BuildKit lets a `<dockerfile>.dockerignore`
// shadow the root one for a single build; such a file would silently give
// the lint build a different, unreviewed context — and eslint's flat config
// does not ignore dot-directories, so a stray `.claude/` worktree would be
// linted.
it.each(["Dockerfile", "Dockerfile.lint"])(
"is not shadowed by a per-Dockerfile ignore file for %s",
(name) => {
expect(existsSync(join(repoRoot, `${name}.dockerignore`))).toBe(
false,
);
});
},
);
});
test/packaging/entrypoints.test.ts
+4 -2
View File
@@ -1,7 +1,9 @@
// The package manifest promises three files that only exist after a build:
// `main`, `types`, and the `quak` binary. Nothing in the test suite used to
// look at them, and `make check` runs the test and lint phases but never the
// build, so `tsconfig.json` and `package.json` were free to drift apart. They
// look at them, and `make check` runs the suite and the lint container but
// never the build, so `tsconfig.json` and `package.json` were free to drift
// apart. (The formatting check is part of the lint container, not a step of
// its own; `test/packaging/lint-once.test.ts` is what holds that shape.) They
// did: `rootDir` was `./src` while `include` also pulled in `bin/**/*`, which
// is TS6059, and no build had succeeded for as long as that was true.
//
test/packaging/lint-docker.test.ts
+184
View File
@@ -0,0 +1,184 @@
// Linting runs in Docker, one way, everywhere: `script/lint` builds
// `Dockerfile.lint`, which COPYs the repo into a digest-pinned image and runs
// eslint and prettier as build steps, so a successful build IS a clean lint.
//
// Three things can quietly undo that, and none of them shows up as a build
// failure, which is why they are asserted here:
//
// 1. Recursion. `script/check` calls `script/lint`, and `script/lint` is now a
// `docker build`. Anything that runs `make check` inside a container is
// therefore asking for Docker inside Docker, and CI breaks. The image built
// from `Dockerfile` runs the suite and the compile only; lint happens once,
// in `Dockerfile.lint`.
// 2. Cache. A lint build over an unchanged tree returns success in well under a
// second having linted nothing. The `LINT_EPOCH` guard is what forces the
// linter layers to execute, and it has to fail closed: an unset build
// argument is the empty string, which is a perfectly stable cache key, so an
// invocation that omits it must be rejected rather than served a cached
// green.
// 3. A host lint path surviving alongside the container one, which would let a
// lint result come from an unpinned local toolchain.
import { describe, expect, it } from "vitest";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { join } from "node:path";
const repoRoot = fileURLToPath(new URL("../../", import.meta.url));
const read = (name: string): string =>
readFileSync(join(repoRoot, name), "utf-8");
// The executable lines of a shell script or Dockerfile: comments carry the
// reasoning and frequently name the very commands these tests forbid, so they
// would otherwise trigger every assertion below.
const instructions = (name: string): string[] =>
read(name)
.split("\n")
.map((line) => line.trim())
.filter((line) => line !== "" && !line.startsWith("#"));
const lintScript = instructions("script/lint");
const dockerfileLint = instructions("Dockerfile.lint");
const dockerfile = instructions("Dockerfile");
const cibuild = instructions("script/cibuild");
const has = (lines: string[], pattern: RegExp): boolean =>
lines.some((line) => pattern.test(line));
describe("script/lint", () => {
it("lints by building Dockerfile.lint", () => {
expect(has(lintScript, /docker build .*-f Dockerfile\.lint/)).toBe(
true,
);
});
// The whole point of the ruling: no invocation of a linter against the
// working tree survives, so a lint verdict can only come from the pinned
// image.
it("runs no linter on the host", () => {
expect(has(lintScript, /eslint|prettier/)).toBe(false);
});
// Without a fresh epoch the build is served from cache in under a second,
// having linted nothing, and still exits 0.
it("passes a fresh LINT_EPOCH on every run", () => {
expect(
has(lintScript, /--build-arg LINT_EPOCH="\$\(date \+%s\)"/),
).toBe(true);
});
});
describe("Dockerfile.lint", () => {
// Tag references are server-mutable, so they are remote code execution.
it("pins its base image by digest", () => {
expect(has(dockerfileLint, /^FROM \S+@sha256:[0-9a-f]{64}/)).toBe(true);
});
it("runs eslint as a build step", () => {
expect(has(dockerfileLint, /^RUN .*eslint \./)).toBe(true);
});
it("runs prettier as a build step", () => {
expect(has(dockerfileLint, /^RUN .*prettier --check \./)).toBe(true);
});
// An unset ARG is the empty string, and an empty string is a perfectly
// stable cache key. Rejecting it is what stops a bare
// `docker build -f Dockerfile.lint .` from reporting a green it did not
// earn.
it("refuses to build without LINT_EPOCH", () => {
expect(has(dockerfileLint, /^ARG LINT_EPOCH$/)).toBe(true);
expect(
has(dockerfileLint, /^RUN \[ -n "\$LINT_EPOCH" \] \|\| exit 1$/),
).toBe(true);
});
// The guard only forces execution of the layers below it, so both linters
// have to sit after it. Layer order is the mechanism, not a style choice.
it("puts both linters below the epoch guard", () => {
const guard = dockerfileLint.findIndex((line) =>
/^RUN \[ -n "\$LINT_EPOCH" \]/.test(line),
);
const linters = dockerfileLint
.map((line, index) => ({ line, index }))
.filter(({ line }) => /^RUN .*(eslint|prettier)/.test(line));
expect(linters.length).toBeGreaterThan(0);
for (const { line, index } of linters) {
expect(
index,
`${line} must run below the LINT_EPOCH guard`,
).toBeGreaterThan(guard);
}
});
// Dependency installation is the slow layer and has nothing to do with the
// sources, so it caches separately: manifests first, sources afterwards.
it("copies the manifests before the sources", () => {
const manifests = dockerfileLint.findIndex((line) =>
/^COPY package\.json yarn\.lock/.test(line),
);
const sources = dockerfileLint.findIndex((line) =>
/^COPY \. \.$/.test(line),
);
expect(manifests).toBeGreaterThanOrEqual(0);
expect(sources).toBeGreaterThan(manifests);
});
// script/lint is a docker build; a lint step that shelled out to it would
// recurse.
it("does not call script/lint or make lint", () => {
expect(has(dockerfileLint, /make lint|script\/lint/)).toBe(false);
});
});
describe("Dockerfile", () => {
// `make check` runs script/lint, which is a docker build, so an image that
// ran it would need a Docker daemon inside the container.
it("does not run make check, make lint or script/lint", () => {
expect(
has(dockerfile, /make check|make lint|script\/(check|lint)/),
).toBe(false);
});
// The replaced lint stage took a `COPY --from=lint` dependency to order
// itself before the check stage. Dockerfile.lint is that stage now, and
// two definitions of how to lint is one too many.
it("has no lint stage", () => {
expect(has(dockerfile, /AS lint\b|--from=lint\b/)).toBe(false);
});
it("still runs the suite and the build under the epoch guard", () => {
expect(has(dockerfile, /^RUN make test$/)).toBe(true);
expect(has(dockerfile, /^RUN make build$/)).toBe(true);
expect(
has(dockerfile, /^RUN \[ -n "\$CHECK_EPOCH" \] \|\| exit 1$/),
).toBe(true);
});
});
describe("script/cibuild", () => {
// CI has to get both verdicts. Lint goes first so the fast failure is
// reported before the suite runs.
it("builds the lint image before the test and build image", () => {
const lint = cibuild.findIndex((line) => /\/lint"/.test(line));
const check = cibuild.findIndex((line) =>
/docker build .*CHECK_EPOCH/.test(line),
);
expect(lint).toBeGreaterThanOrEqual(0);
expect(check).toBeGreaterThan(lint);
});
});
describe("package.json", () => {
// `yarn lint` was a second, unpinned way to get a lint verdict, from
// whatever eslint the working tree happened to have installed.
it("exposes no host lint script", () => {
const pkg = JSON.parse(read("package.json")) as {
scripts: Record<string, string>;
};
expect(pkg.scripts.lint).toBeUndefined();
});
});
test/packaging/lint-once.test.ts
+503
View File
@@ -0,0 +1,503 @@
// `make check` used to run `prettier --check .` twice: once inside the lint
// container (`script/lint` builds `Dockerfile.lint`, which runs eslint and
// prettier as build steps) and once again on the host, because `script/check`
// also called `script/fmt-check`. Two passes, one verdict, and the host one is
// the weaker of the two — its prettier is whatever the working tree happens to
// have installed, while the container's is digest-pinned and installed under
// `--frozen-lockfile`.
//
// The fix was to delete the host call from `script/check` and `script/precommit`.
// Nothing about that fix is self-enforcing: anyone can wire `script/fmt-check`
// back in, or add a prettier step to a Dockerfile, and every build stays green
// while quietly doing the work twice again. So the count is asserted here
// rather than promised in a comment.
//
// The assertion is a static walk of the invocation graph, not a string match
// against one file. Starting from an entrypoint, it follows every edge the repo
// actually uses to reach another command — `run:` steps in the CI workflow,
// `"$SCRIPT_DIR/<name>"` and `script/<name>` into other scripts, `make <target>`
// through the Makefile shims, `yarn run <name>` through the `package.json`
// scripts, and `docker build -f <file>` into that Dockerfile's `RUN` steps — and
// counts the prettier invocations it finds. A prettier call added anywhere in
// that graph is therefore caught, wherever it is added.
//
// Two entrypoints are walked, because they cover different graphs: `make check`
// is what a developer runs, and `.gitea/workflows/check.yml` is what CI runs.
// The CI walk starts at the workflow file rather than at a hand-picked script,
// so "the path CI executes" is read out of the repo instead of assumed; it
// reaches `script/cibuild`, and through it the `Dockerfile` image that `make
// check` never touches. Walking only `make check` is how a duplicate prettier
// pass in `Dockerfile` stayed invisible.
//
// Undercounting is the failure mode that would make this test worthless. Three
// things guard against it: the walk is asserted to have reached the nodes that
// matter, an unresolvable or empty node is a thrown error rather than a quiet
// zero, and prettier is counted per occurrence rather than per line, so two
// invocations chained with `&&` cannot read as one.
import { describe, expect, it } from "vitest";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { join } from "node:path";
const repoRoot = fileURLToPath(new URL("../../", import.meta.url));
const read = (name: string): string =>
readFileSync(join(repoRoot, name), "utf-8");
// A backslash at end of line continues the command; the resolver has to see the
// whole invocation, since the interesting flags (`-f Dockerfile.lint`) can sit
// on the continuation.
const joinContinuations = (text: string): string[] => {
const joined: string[] = [];
for (const raw of text.split("\n")) {
const line = raw.trim();
const previous = joined[joined.length - 1];
if (previous !== undefined && previous.endsWith("\\")) {
joined[joined.length - 1] =
`${previous.slice(0, -1).trim()} ${line}`;
} else {
joined.push(line);
}
}
return joined;
};
// Comments are stripped everywhere. The headers of these scripts explain the
// duplication this test exists to prevent, and therefore name `prettier` and
// `script/fmt-check` repeatedly; counting them would make the test assert the
// prose instead of the behaviour.
const executable = (text: string): string[] =>
joinContinuations(text).filter(
(line) => line !== "" && !line.startsWith("#"),
);
// Every occurrence, not "does this line mention prettier": a line that reads
// `yarn run prettier --check . && yarn run prettier --check src` is two passes
// over the same tree, which is exactly the bug this file exists to catch, and
// counting it as one would hide it. `.prettierrc` and `.prettierignore` are not
// invocations and do not match, because `\b` requires a non-word character
// after the name.
const countPrettier = (line: string): number =>
(line.match(/\bprettier\b/g) ?? []).length;
// Makefile targets are thin shims (`check:` / tab / `@script/check`), so a
// `make <target>` edge has to resolve through them to keep "per `make check`"
// meaning what it says. Recipe lines are the tab-indented ones.
const makeRecipes = (): Map<string, string[]> => {
const recipes = new Map<string, string[]>();
let current: string | null = null;
for (const raw of read("Makefile").split("\n")) {
if (raw.startsWith("\t")) {
if (current !== null) {
recipes.get(current)?.push(raw.trim().replace(/^[@-]+/, ""));
}
continue;
}
const target = /^([a-z][a-z-]*)\s*:(?!=)/.exec(raw);
current = target === null ? null : target[1];
if (current !== null && !recipes.has(current)) {
recipes.set(current, []);
}
}
return recipes;
};
const recipes = makeRecipes();
const packageScripts = (): Record<string, string> => {
const pkg = JSON.parse(read("package.json")) as {
scripts?: Record<string, string>;
};
return pkg.scripts ?? {};
};
const scripts = packageScripts();
// Node keys: `script/<name>`, `docker:<Dockerfile>`, `make:<target>`,
// `yarn:<package.json script>`, `workflow:<CI workflow file>`.
const resolve = (node: string): string[] => {
if (node.startsWith("script/")) return executable(read(node));
if (node.startsWith("docker:")) {
return executable(read(node.slice("docker:".length)))
.filter((line) => line.startsWith("RUN "))
.map((line) => line.slice("RUN ".length));
}
// The `run:` steps of a workflow, in file order. `uses:` steps are actions,
// not commands, and have no edges into this repo's graph. A `run: |` block
// would resolve to the bare `|`, which reaches nothing and therefore fails
// the count rather than passing quietly.
if (node.startsWith("workflow:")) {
return executable(read(node.slice("workflow:".length)))
.filter((line) => /^-?\s*run:\s*\S/.test(line))
.map((line) => line.replace(/^-?\s*run:\s*/, ""));
}
if (node.startsWith("make:")) {
const target = node.slice("make:".length);
const recipe = recipes.get(target);
// A renamed or deleted target must be a loud failure: silently walking
// an empty recipe would report zero prettier invocations, which reads
// like the tidiest possible result.
if (recipe === undefined) {
throw new Error(`no such Makefile target: ${target}`);
}
return recipe;
}
if (node.startsWith("yarn:")) {
const name = node.slice("yarn:".length);
const script = scripts[name];
if (script === undefined) {
throw new Error(`no such package.json script: ${name}`);
}
return [script];
}
throw new Error(`unresolvable node: ${node}`);
};
// Same reasoning as the missing-target error, applied to every node kind: a
// node that resolves to no commands contributes zero prettier invocations and
// zero edges, which is indistinguishable from a clean result. Fail instead.
const commandsOf = (node: string): string[] => {
const commands = resolve(node);
if (commands.length === 0) {
throw new Error(`node resolved to no commands: ${node}`);
}
return commands;
};
const edgesOf = (line: string): string[] => {
const edges: string[] = [];
// `"$SCRIPT_DIR/lint"`, `"$ROOT/script/lint"` and a bare `script/lint` are
// all the same edge.
for (const match of line.matchAll(
/(?:\$SCRIPT_DIR|\$\{SCRIPT_DIR\}|script)\/([a-z][a-z-]*)/g,
)) {
edges.push(`script/${match[1]}`);
}
// Only real targets: `pkg_install gnumake make make make` in
// script/bootstrap is a package name, not an invocation of this Makefile.
for (const match of line.matchAll(/\bmake\s+([a-z][a-z-]*)/g)) {
if (recipes.has(match[1] ?? "")) edges.push(`make:${match[1]}`);
}
// Same rule for yarn: `yarn run prettier` is the linter itself (counted,
// not followed), `yarn run fmt-check` would be a package.json script that
// runs it indirectly.
for (const match of line.matchAll(/\byarn(?:\s+run)?\s+([a-z][a-z-]*)/g)) {
if ((match[1] ?? "") in scripts) edges.push(`yarn:${match[1]}`);
}
// The container lint pass lives behind a `docker build`; without following
// it the count would miss the one invocation that is supposed to survive.
if (/\bdocker\s+build\b/.test(line)) {
const file = /\s-f\s+(\S+)/.exec(line);
edges.push(`docker:${file === null ? "Dockerfile" : file[1]}`);
}
return edges;
};
interface Walk {
prettier: number;
reached: Set<string>;
}
// Repeated invocations must count repeatedly — running the same script twice is
// exactly the bug — so nodes are not deduplicated. The path stack is only there
// to turn a cycle into a loud failure instead of a hang.
//
// Counting and edge-following both happen for every line: a line that invokes
// prettier can also invoke something else, and skipping the edges of counted
// lines silently truncated the graph.
const walk = (node: string, path: string[] = [], into?: Walk): Walk => {
const result = into ?? { prettier: 0, reached: new Set<string>() };
if (path.includes(node)) {
throw new Error(`invocation cycle: ${[...path, node].join(" -> ")}`);
}
result.reached.add(node);
for (const line of commandsOf(node)) {
result.prettier += countPrettier(line);
for (const edge of edgesOf(line)) {
walk(edge, [...path, node], result);
}
}
return result;
};
describe("prettier runs exactly once per make check", () => {
const check = walk("make:check");
// The headline assertion, and the one the issue is about.
it("invokes prettier once for the whole of make check", () => {
expect(check.prettier).toBe(1);
});
// Guards against the count being 1 (or 0) because the walk never got
// anywhere. `make check` has to reach the suite, the lint script, and the
// Dockerfile whose build IS the lint verdict.
it.each(["script/check", "script/test", "script/lint", "Dockerfile.lint"])(
"reaches %s while counting",
(node) => {
const key = node.startsWith("script/") ? node : `docker:${node}`;
expect([...check.reached]).toContain(key);
},
);
// The one that survives is the container's, not the host's: that is the
// authoritative verdict, since a successful Dockerfile.lint build is what
// CI treats as proof of a clean tree.
it("keeps the surviving invocation inside the lint container", () => {
expect(walk("docker:Dockerfile.lint").prettier).toBe(1);
});
it("does not reach the host formatting check from make check", () => {
expect([...check.reached]).not.toContain("script/fmt-check");
});
});
describe("prettier runs exactly once per CI build", () => {
// Rooted at the workflow file, so this is the graph CI executes rather than
// the graph someone believed CI executes. `make check` cannot stand in for
// it: CI runs script/cibuild, which builds Dockerfile as well as
// Dockerfile.lint, and nothing under `make check` ever reads Dockerfile.
const ci = walk("workflow:.gitea/workflows/check.yml");
it("invokes prettier once for the whole CI build", () => {
expect(ci.prettier).toBe(1);
});
// script/cibuild is here because the workflow is asserted to run it;
// Dockerfile is here because it is the half of the CI graph that the
// `make check` walk cannot see.
it.each([
"script/cibuild",
"script/lint",
"docker:Dockerfile.lint",
"docker:Dockerfile",
])("reaches %s while counting", (node) => {
expect([...ci.reached]).toContain(node);
});
// The test and build image must not lint: linting is Dockerfile.lint's job,
// and a prettier step added here would be a second pass over the same tree
// for the same verdict — on the one path where it matters most.
it("keeps prettier out of the test and build image", () => {
expect(walk("docker:Dockerfile").prettier).toBe(0);
});
});
describe("the standalone entrypoints still do what their names say", () => {
// REPO_POLICIES.md requires both `make lint` and `make fmt-check` to exist
// and mean something. Dropping fmt-check from script/check must not turn it
// into a target nobody can use, and must not leave `make check` passing
// because both halves became no-ops.
it("still checks formatting under make fmt-check", () => {
expect(walk("make:fmt-check").prettier).toBe(1);
});
it("still checks formatting under make lint", () => {
expect(walk("make:lint").prettier).toBe(1);
});
});
describe("script/precommit", () => {
// Same duplication as script/check, same fix. The hook still catches a
// badly formatted tree before the commit lands, because script/lint is the
// container prettier run — that is the whole reason the host call could go.
it("checks formatting exactly once", () => {
expect(walk("script/precommit").prettier).toBe(1);
});
it("gets that check from the lint container", () => {
expect([...walk("script/precommit").reached]).toContain(
"docker:Dockerfile.lint",
);
});
});
// script/bootstrap installs the dependencies, and it has two install sites: one
// for the case where yarn has to be reached through nvm, and one for the case
// where yarn is already on PATH. A substring check against the whole file
// cannot tell them apart, so it reports the first and says nothing about the
// second — which is the one the containers take, because the pinned node image
// ships yarn. Both are resolved separately here.
const installBranches = (): { withoutYarn: string[]; withYarn: string[] } => {
const lines = executable(read("script/bootstrap"));
const open = lines.findIndex((line) =>
/^install_js_deps\s*\(\)/.test(line),
);
if (open === -1) {
throw new Error("script/bootstrap: no install_js_deps function");
}
const close = lines.indexOf("}", open);
const body = lines.slice(open + 1, close === -1 ? undefined : close);
const guard = body.findIndex((line) =>
/^if\b.*\bmissing yarn\b/.test(line),
);
const otherwise = body.indexOf("else", guard);
const end = body.indexOf("fi", otherwise);
if (guard === -1 || otherwise === -1 || end === -1) {
throw new Error(
"script/bootstrap: install_js_deps is not the expected " +
"if missing yarn / else / fi shape",
);
}
return {
withoutYarn: body.slice(guard + 1, otherwise),
withYarn: body.slice(otherwise + 1, end),
};
};
// Every `yarn install` in the given lines, with its flags, so an unpinned
// install cannot hide next to a pinned one.
const yarnInstalls = (lines: string[]): string[] =>
lines.flatMap((line) =>
[...line.matchAll(/\byarn install\b[^"'&|;]*/g)].map((match) =>
match[0].trim(),
),
);
describe("host and container prettier cannot disagree", () => {
// With the host pass gone from `make check`, `make fmt-check` is the only
// host-side formatting check left, and the container is the gate. The two
// must keep producing the same verdict on the same tree, or a developer
// running `make fmt-check` gets a green that CI then rejects.
//
// Three things make them agree, and all three are load-bearing:
it("pins the same prettier for both", () => {
const pkg = JSON.parse(read("package.json")) as {
devDependencies: Record<string, string>;
};
// An exact version, not a range: `^3.8.1` would let the container and
// the host resolve different builds with different formatting.
expect(pkg.devDependencies.prettier).toMatch(/^\d+\.\d+\.\d+$/);
});
it("installs from the lockfile on the branch the container takes", () => {
// Both images are FROM a node image, which ships yarn, so `missing
// yarn` is false and this is the branch that runs in the container.
const installs = yarnInstalls(installBranches().withYarn);
expect(installs).not.toHaveLength(0);
for (const install of installs) {
expect(install).toContain("--frozen-lockfile");
}
});
it("installs from the lockfile on the nvm branch too", () => {
// Not the container's branch, but it is the one a developer without
// yarn on PATH gets, and their prettier has to match the container's.
const installs = yarnInstalls(installBranches().withoutYarn);
expect(installs).not.toHaveLength(0);
for (const install of installs) {
expect(install).toContain("--frozen-lockfile");
}
});
it("runs script/bootstrap inside the lint container", () => {
// Without this the lockfile assertions above would be about a script
// the container never executes.
expect([...walk("docker:Dockerfile.lint").reached]).toContain(
"script/bootstrap",
);
});
it("keeps .gitignore in the build context", () => {
// Prettier 3 reads .gitignore as a default ignore file, so excluding it
// from the context would change which files the container checks.
const dockerignore = read(".dockerignore")
.split("\n")
.map((line) => line.trim());
expect(dockerignore).not.toContain(".gitignore");
});
});
describe("the walk cannot pass vacuously", () => {
// An earlier draft of this file computed a Makefile target as
// `node.slice("make:")` — a string where a number belongs, which coerces to
// NaN and made every target resolve to nothing. The count went to zero and
// an assertion of "not twice" would have been satisfied by a walk that had
// read nothing at all. Every way of reaching nothing is therefore an
// error here, and the ways are tested rather than assumed.
it("reports zero for a subgraph that does not run prettier", () => {
expect(walk("make:clean").prettier).toBe(0);
});
it("refuses a Makefile target that does not exist", () => {
expect(() => walk("make:no-such-target")).toThrow(
/no such Makefile target/,
);
});
it("refuses a package.json script that does not exist", () => {
expect(() => walk("yarn:no-such-script")).toThrow(
/no such package.json script/,
);
});
it("refuses a script that does not exist", () => {
expect(() => walk("script/no-such-script")).toThrow(/ENOENT/);
});
it("refuses a node that resolves to no commands", () => {
// .dockerignore has no RUN steps, standing in for a Dockerfile whose
// steps a restructure moved somewhere the resolver cannot see.
expect(() => walk("docker:.dockerignore")).toThrow(
/resolved to no commands/,
);
});
it("refuses a node kind it does not understand", () => {
expect(() => walk("nonsense")).toThrow(/unresolvable node/);
});
it("refuses to walk in circles", () => {
expect(() => walk("make:check", ["script/check"])).toThrow(
/invocation cycle/,
);
});
});
describe("the resolver reads what the shell would run", () => {
// Counting per line is how `yarn run prettier --check . && yarn run
// prettier --check src` read as a single invocation.
it("counts every prettier invocation on a line", () => {
expect(
countPrettier(
"yarn run prettier --check . && yarn run prettier --check src",
),
).toBe(2);
});
it("does not count the config files as invocations", () => {
expect(countPrettier("COPY .prettierrc .prettierignore ./")).toBe(0);
});
// The counting `continue` also dropped every edge that shared a line with a
// prettier call, so a whole subtree could be hidden behind one `&&`.
it("still follows the edges of a line that invokes prettier", () => {
expect(
edgesOf('yarn run prettier --check . && "$SCRIPT_DIR/lint"'),
).toContain("script/lint");
});
it("resolves every spelling of a script call to one node", () => {
expect(
edgesOf('"$SCRIPT_DIR/lint" "${SCRIPT_DIR}/test" script/fmt'),
).toEqual(["script/lint", "script/test", "script/fmt"]);
});
it("follows a bare docker build to Dockerfile and -f to its file", () => {
expect(edgesOf("docker build .")).toContain("docker:Dockerfile");
expect(edgesOf("docker build -f Dockerfile.lint .")).toContain(
"docker:Dockerfile.lint",
);
});
it("reads the run steps of the CI workflow and not its uses steps", () => {
expect(commandsOf("workflow:.gitea/workflows/check.yml")).toEqual([
"script/cibuild",
]);
});
});
test/thumbnails/thumbnails.test.ts
-143
View File
@@ -215,9 +215,6 @@ const buildThumbMock = async (opts?: {
thumbnail: {
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
},
// The encrypted size of the thumbnail the server records; large
// enough here that the default encoding fits.
info: { thumbSize: 1_000_000 },
updationTime: TEST_TIME,
};
};
@@ -449,32 +446,6 @@ const openLib = (client: Client): Promise<Library> =>
precacheOriginals: false,
});
/** The mock's raw record for one file, for a test to change before login. */
const rawFile = (m: ThumbMockState, fileID: number): Record<string, unknown> =>
m.filesByCollection[1]!.find((f) => f.id === fileID)!;
/** Replace the original the mock serves for one file. */
const replaceOriginal = (
m: ThumbMockState,
fileID: number,
body: Uint8Array,
): void => {
const push = sodium.crypto_secretstream_xchacha20poly1305_init_push(
m.fileKeys[fileID]!,
);
m.fileCiphertexts[fileID] =
sodium.crypto_secretstream_xchacha20poly1305_push(
push.state,
body,
null,
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
);
rawFile(m, fileID).file = { decryptionHeader: toBase64(push.header) };
};
const isOriginalDownload = (url: string): boolean =>
url.includes("files.ente.io") || url.includes("/files/download/");
const login = (fetch: typeof globalThis.fetch, retry?: RetryOptions) =>
Client.login({
email: TEST_EMAIL,
@@ -610,33 +581,6 @@ describe("listMissingThumbnails", () => {
// Should still be 2, not 4 (each file checked only once)
expect(missing.length).toBe(2);
});
it("skips a file another account owns without fetching its thumbnail", async () => {
const otherMock = await buildThumbMock();
rawFile(otherMock, 102).ownerID = 7;
const logs: string[] = [];
const counted = countingFetch(
buildThumbFetch(otherMock),
(url) => url.includes("thumbnails.ente.io") && url.includes("102"),
);
const client = await login(counted.fetch);
const lib = await openLib(client);
const missing = await listMissingThumbnails(lib, client, (msg) =>
logs.push(msg),
);
lib.close();
expect(missing.map((m) => m.fileID)).toEqual([101]);
expect(counted.matched()).toBe(0);
expect(
logs.some(
(l) =>
l.includes("Skipping file-102.jpg") &&
l.includes("another account"),
),
).toBe(true);
});
});
describe("fixMissingThumbnails", () => {
@@ -744,93 +688,6 @@ describe("fixMissingThumbnails", () => {
expect(fixMock.uploadedThumbnails.length).toBe(1);
expect(fixMock.uploadedThumbnails[0]!.fileID).toBe(101);
});
it("skips a file another account owns without downloading it", async () => {
// The server accepts a thumbnail only from the file's owner.
const fixMock = await buildThumbMock();
rawFile(fixMock, 101).ownerID = 7;
const counted = countingFetch(
buildThumbFetch(fixMock),
isOriginalDownload,
);
const client = await login(counted.fetch);
const lib = await openLib(client);
const results = await fixMissingThumbnails(lib, client, [101]);
lib.close();
expect(results[0]!.status).toBe("skipped");
expect(results[0]!.reason).toContain("another account");
expect(counted.matched()).toBe(0);
expect(fixMock.uploadedThumbnails.length).toBe(0);
});
it("skips a file whose recorded thumbnail size is 0 without downloading it", async () => {
// The server refuses a thumbnail larger than the one it records, and
// no thumbnail is 0 bytes.
const fixMock = await buildThumbMock();
rawFile(fixMock, 101).info = { thumbSize: 0 };
const counted = countingFetch(
buildThumbFetch(fixMock),
isOriginalDownload,
);
const client = await login(counted.fetch);
const lib = await openLib(client);
const results = await fixMissingThumbnails(lib, client, [101]);
lib.close();
expect(results[0]!.status).toBe("skipped");
expect(results[0]!.reason).toContain("recorded thumbnail size is 0");
expect(counted.matched()).toBe(0);
expect(fixMock.uploadedThumbnails.length).toBe(0);
});
it("re-encodes smaller until the thumbnail fits the recorded size", async () => {
// A noisy 400x300 JPEG, which the default encoding (quality 50, not
// resized because it is under 720 px) cannot compress below the size
// recorded here: one byte less than that encoding's ciphertext.
const fixMock = await buildThumbMock();
const w = 400;
const h = 300;
const noisy = new Uint8Array(
jpegJs.encode(
{
data: sodium.randombytes_buf(w * h * 4),
width: w,
height: h,
},
90,
).data,
);
replaceOriginal(fixMock, 101, noisy);
const decoded = jpegJs.decode(noisy, {
useTArray: true,
formatAsRGBA: true,
});
const defaultSize =
jpegJs.encode(decoded, 50).data.length +
sodium.crypto_secretstream_xchacha20poly1305_ABYTES;
const recordedSize = defaultSize - 1;
rawFile(fixMock, 101).info = { thumbSize: recordedSize };
const client = await login(buildThumbFetch(fixMock));
const lib = await openLib(client);
const results = await fixMissingThumbnails(lib, client, [101]);
lib.close();
expect(results[0]!.status).toBe("fixed");
const upload = fixMock.uploadedThumbnails[0]!;
expect(upload.ciphertext.length).toBeLessThanOrEqual(recordedSize);
const decrypted = decryptBlob(
upload.ciphertext,
fromBase64(upload.decryptionHeader),
fixMock.fileKeys[101]!,
);
expect(decrypted[0]).toBe(0xff);
expect(decrypted[1]).toBe(0xd8);
});
});
describe("Client.getApiClient", () => {
yarn.lock
-5
View File
@@ -1069,11 +1069,6 @@ fastq@^1.6.0:
dependencies:
reusify "^1.0.4"
fflate@0.8.3:
version "0.8.3"
resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.3.tgz#bc27d8eb30343d4d512abb03480202ce65d825fc"
integrity sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==
file-entry-cache@^8.0.0:
version "8.0.0"
resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f"