Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
1283048ce2 |
+42
-75
@@ -1,86 +1,53 @@
|
||||
# .dockerignore does NOT use .gitignore semantics. Docker matches with
|
||||
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
|
||||
# `/` and an unprefixed pattern is anchored at the context root. Every
|
||||
# depth-independent pattern therefore needs `**/`, or `config/.env` and
|
||||
# `certs/server.key` still ship while this file reads as solved. Only
|
||||
# genuinely root-anchored entries go unprefixed. Never transplant these
|
||||
# into .gitignore, where `**/` is wrong.
|
||||
# Mirrors .gitignore, with one deliberate exception: .gitignore itself stays
|
||||
# in the build context, because prettier 3 reads it as a default ignore file
|
||||
# and dropping it would change what the lint phase's prettier check sees.
|
||||
#
|
||||
# Matching is case-sensitive, so secrets use character ranges rather
|
||||
# than an ALL-CAPS twin, which would still miss `Server.Key`.
|
||||
#
|
||||
# Extend with this repo's own host-built artifacts, written anchored:
|
||||
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
|
||||
# deletes the package directory from the context.
|
||||
# .git is deliberately NOT excluded: the build derives the version it stamps
|
||||
# from it (script/version). It is sent without its config, which holds the
|
||||
# clone's remote URL and any credential in it, and which the build stage, the
|
||||
# final image, would otherwise carry. git describe does not need it.
|
||||
.git/config
|
||||
|
||||
# .git is sent without its config. Without a VERSION build argument the
|
||||
# stage that compiles runs `git describe --tags --always` on .git, which
|
||||
# does not need .git/config; that file can hold a credential, such as a
|
||||
# password in a remote URL or the token the CI checkout step stores there.
|
||||
# Each submodule keeps a config with the same exposure in its git directory
|
||||
# under .git/modules/, nested again for a submodule's own submodules, or in
|
||||
# its own .git directory when it keeps one.
|
||||
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
|
||||
# `deploy/config`, `config/lib`) loses its whole git directory, because
|
||||
# `**/.git/modules/**/config` also matches that segment's directory
|
||||
# under .git/modules/. Go's version stamping then fails the build;
|
||||
# nothing leaks. Name such a submodule without that segment:
|
||||
# `git submodule add --name`.
|
||||
**/.git/config
|
||||
**/.git/modules/**/config
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Agent scratch: one full checkout of the repo per in-flight agent.
|
||||
# Anchored because it occurs once where agents run at the repo root.
|
||||
# KNOWN GAP: a repo running agents in subdirectories still ships
|
||||
# `services/api/.claude/` and must add its own anchored entry.
|
||||
.claude
|
||||
# Editors
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
*.bak
|
||||
.idea/
|
||||
.vscode/
|
||||
*.sublime-*
|
||||
|
||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
||||
# convention. Re-include a committed template with a negation if the
|
||||
# build needs one: `!docs/example.env`.
|
||||
**/*.[eE][nN][vV]
|
||||
**/.[eE][nN][vV].*
|
||||
**/.[eE][nN][vV][rR][cC]
|
||||
# Node
|
||||
node_modules
|
||||
|
||||
# Private keys and the bundles carrying them. Public certificates
|
||||
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
|
||||
**/*.[pP][eE][mM]
|
||||
**/*.[kK][eE][yY]
|
||||
**/*.[pP]12
|
||||
**/*.[pP][fF][xX]
|
||||
**/[iI][dD]_[rR][sS][aA]
|
||||
**/[iI][dD]_[dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
||||
**/[iI][dD]_[eE][dD]25519
|
||||
**/[iI][dD]_[eE][dD]25519_[sS][kK]
|
||||
# TypeScript / build artifacts
|
||||
dist
|
||||
build
|
||||
*.tsbuildinfo
|
||||
coverage
|
||||
.nyc_output/
|
||||
|
||||
# Dependencies: restored inside the image, never copied in.
|
||||
**/node_modules
|
||||
# Vitest
|
||||
.vitest-cache/
|
||||
|
||||
# OS metadata.
|
||||
**/.DS_Store
|
||||
**/Thumbs.db
|
||||
|
||||
# Editor state: never a build input, and it churns COPY.
|
||||
**/*.swp
|
||||
**/*.swo
|
||||
**/*~
|
||||
**/*.bak
|
||||
**/.idea
|
||||
**/.vscode
|
||||
**/*.sublime-*
|
||||
|
||||
# TypeScript / build artifacts: the image compiles its own.
|
||||
/dist
|
||||
/build
|
||||
/*.tsbuildinfo
|
||||
/coverage
|
||||
/.nyc_output
|
||||
/.vitest-cache
|
||||
# Environment / secrets
|
||||
.env
|
||||
.env.*
|
||||
*.pem
|
||||
*.key
|
||||
|
||||
# Compiled binary (built by make build-bin); around 100 MB
|
||||
/bin/quak
|
||||
bin/quak
|
||||
|
||||
# quak runtime data (in case anyone runs the CLI from inside the repo)
|
||||
/.quak
|
||||
.quak/
|
||||
|
||||
# Local per-developer tool state, including agent worktrees. Correctness,
|
||||
# not context size: a worktree copied in here has its own test/ tree, which
|
||||
# vitest globs alongside the real one, so the containerised suite runs N+1
|
||||
# times over and still reports success.
|
||||
.claude/
|
||||
|
||||
+10
-32
@@ -11,41 +11,9 @@ Thumbs.db
|
||||
.vscode/
|
||||
*.sublime-*
|
||||
|
||||
# Agent scratch (worktrees of this repo, created and destroyed by
|
||||
# in-flight tooling). Unanchored: .gitignore patterns already match at
|
||||
# every depth, so no prefix is wanted here. This is not a .dockerignore
|
||||
# entry and must not be given a `**/` prefix on the way into one.
|
||||
.claude/
|
||||
|
||||
# Node
|
||||
node_modules/
|
||||
|
||||
# Secrets. Unanchored like every entry above, so each matches at every
|
||||
# depth. Matching is case-sensitive on Linux, so names use character
|
||||
# ranges rather than a lowercase form that misses `Server.Key`.
|
||||
|
||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
||||
# convention. Only the templates `example.env` and `sample.env` are
|
||||
# re-included below. A repository that commits any other template adds
|
||||
# its own negation after these lines, for example `!.env.example`.
|
||||
*.[eE][nN][vV]
|
||||
.[eE][nN][vV].*
|
||||
.[eE][nN][vV][rR][cC]
|
||||
!example.env
|
||||
!sample.env
|
||||
|
||||
# Private keys and the bundles carrying them.
|
||||
*.[pP][eE][mM]
|
||||
*.[kK][eE][yY]
|
||||
*.[pP]12
|
||||
*.[pP][fF][xX]
|
||||
[iI][dD]_[rR][sS][aA]
|
||||
[iI][dD]_[dD][sS][aA]
|
||||
[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
||||
[iI][dD]_[eE][dD]25519
|
||||
[iI][dD]_[eE][dD]25519_[sS][kK]
|
||||
|
||||
# TypeScript / build artifacts
|
||||
dist/
|
||||
build/
|
||||
@@ -56,8 +24,18 @@ coverage/
|
||||
# Vitest
|
||||
.vitest-cache/
|
||||
|
||||
# Environment / secrets
|
||||
.env
|
||||
.env.*
|
||||
*.pem
|
||||
*.key
|
||||
|
||||
# Compiled binary (built by make build-bin)
|
||||
bin/quak
|
||||
|
||||
# quak runtime data (in case anyone runs the CLI from inside the repo)
|
||||
.quak/
|
||||
|
||||
# Local per-developer tool settings and scratch state, including the
|
||||
# worktrees agents check out under this directory
|
||||
.claude/
|
||||
|
||||
@@ -1,2 +1,5 @@
|
||||
node_modules/
|
||||
yarn.lock
|
||||
dist/
|
||||
build/
|
||||
coverage/
|
||||
|
||||
@@ -58,8 +58,6 @@ COPY --from=test /app/package.json /dev/null
|
||||
COPY script/ script/
|
||||
COPY package.json yarn.lock ./
|
||||
RUN script/bootstrap
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /app
|
||||
|
||||
COPY . .
|
||||
|
||||
|
||||
@@ -137,15 +137,16 @@ alpine. We provide:
|
||||
- `script/lint` — run eslint and a prettier check, by building the `lint` phase
|
||||
of the `Dockerfile`; requires docker (see Linting and testing below)
|
||||
- `script/fmt` — format all files with prettier (writes)
|
||||
- `script/fmt-check` — check formatting on the host (read-only)
|
||||
- `script/check` — run all checks: `test`, `lint`, `fmt-check` (our own
|
||||
extension)
|
||||
- `script/fmt-check` — check formatting on the host (read-only); standalone, and
|
||||
not called by `script/check` or `script/precommit`, because `script/lint`
|
||||
already checks formatting in the container
|
||||
- `script/check` — run all checks: `test`, `lint` (our own extension)
|
||||
- `script/docker` — build the image, tagged via `script/projectname`
|
||||
- `script/cibuild` — what CI runs: `script/bootstrap`, then `script/check`, then
|
||||
the image build
|
||||
- `script/cibuild` — build the image (what CI runs); its last stage depends on
|
||||
the `lint` and `test` phases, so this one build lints, tests and compiles
|
||||
- `script/precommit` — run by the git pre-commit hook (our own extension); runs
|
||||
`script/lint` and `script/fmt-check` but deliberately not the tests, so the
|
||||
TDD red-phase commit can land
|
||||
`script/lint`, which checks both lint and formatting, but deliberately not the
|
||||
tests, so the TDD red-phase commit can land
|
||||
- `script/install-precommit` — installs the git pre-commit hook (our own
|
||||
extension); `make hooks` shims to it
|
||||
|
||||
@@ -161,19 +162,22 @@ script runs the tools on the host: docker is required, and that also works where
|
||||
the docker daemon is remote and bind mounts are impossible.
|
||||
|
||||
The last stage of the `Dockerfile` compiles the package, and it copies a file
|
||||
from each phase, so it cannot be built unless lint and the tests pass. The image
|
||||
build in `script/cibuild` therefore runs lint and the tests a second time, after
|
||||
`script/check` has run them.
|
||||
from each phase, so it cannot be built unless lint and the tests pass. That is
|
||||
why `script/cibuild` is a single `docker build`: it runs lint and the tests once
|
||||
each and then compiles.
|
||||
|
||||
Every `docker build` in `script/` passes `--no-cache`. On an unchanged tree
|
||||
Docker would otherwise serve the lint and test steps from cache, nothing would
|
||||
run, and the build would still exit 0.
|
||||
|
||||
`script/fmt-check` runs prettier on the host. Its verdict matches the `lint`
|
||||
phase's: prettier is pinned to an exact version, installed from `yarn.lock`
|
||||
under `--frozen-lockfile` in both places, and reads `.gitignore` as its default
|
||||
ignore file — which is why `.dockerignore` keeps `.gitignore` in the build
|
||||
context.
|
||||
The formatting check is part of the `lint` phase, not a step beside it, so
|
||||
`script/check` and `script/precommit` do not call `script/fmt-check` as well;
|
||||
that would run prettier a second time over the same tree for the same verdict.
|
||||
`script/fmt-check` remains as a standalone entrypoint for asking the formatting
|
||||
question on the host. Its verdict matches the container's: prettier is pinned to
|
||||
an exact version, installed from `yarn.lock` under `--frozen-lockfile` in both
|
||||
places, and reads `.gitignore` as its default ignore file — which is why
|
||||
`.dockerignore` keeps `.gitignore` in the build context.
|
||||
|
||||
### Version
|
||||
|
||||
@@ -255,10 +259,11 @@ All work on quak is test-driven. No exceptions.
|
||||
history must still show tests landing before (or with) the matching
|
||||
implementation.
|
||||
8. The pre-commit hook installed by `make hooks` runs `script/precommit`, which
|
||||
runs `script/lint` and `script/fmt-check` but not the tests, and so not the
|
||||
full `make check`. This is deliberate so the TDD red-phase commit (failing
|
||||
tests, no implementation yet) can land. CI executes `script/cibuild`, which
|
||||
runs the tests, so a red branch still cannot reach `next`.
|
||||
runs `script/lint` — eslint and the prettier check, in the container — but
|
||||
not the tests, and so not the full `make check`. This is deliberate so the
|
||||
TDD red-phase commit (failing tests, no implementation yet) can land. The
|
||||
`test` phase is part of the image build, which is what CI executes via
|
||||
`script/cibuild`, so a red branch still cannot reach `next`.
|
||||
|
||||
## Design
|
||||
|
||||
@@ -415,24 +420,18 @@ Backoff is exponential with full jitter: the delay before retry _n_ is
|
||||
`random() * min(maxDelayMs, baseDelayMs * 2 ** (n - 1))`. The exponential term
|
||||
is the ceiling and the wait is drawn below it, so a client that lost many
|
||||
parallel downloads to one CDN blip does not send them all again at the same
|
||||
instant. The numbers, configurable through `ApiClientOptions.retry`:
|
||||
instant. Defaults, configurable through `ApiClientOptions.retry`:
|
||||
|
||||
| Option | Default | `quak backup` | Meaning |
|
||||
| ------------- | ------- | ------------- | ----------------------------------- |
|
||||
| `attempts` | `4` | `10` | total calls, not retries |
|
||||
| `baseDelayMs` | `500` | `1000` | ceiling for the first retry's delay |
|
||||
| `maxDelayMs` | `10000` | `60000` | upper bound on that ceiling |
|
||||
| Option | Default | Meaning |
|
||||
| ------------- | ------- | ----------------------------------- |
|
||||
| `attempts` | `4` | total calls, not retries |
|
||||
| `baseDelayMs` | `500` | ceiling for the first retry's delay |
|
||||
| `maxDelayMs` | `10000` | upper bound on that ceiling |
|
||||
|
||||
With the defaults a file that is going to fail gives up after at most three and
|
||||
a half seconds of waiting. `quak backup` usually runs from cron with nobody
|
||||
watching, so every request it makes uses the `quak backup` column instead,
|
||||
exported as `UNATTENDED_RETRY_OPTIONS`: a request that keeps failing gives up
|
||||
after at most 243 seconds of waiting, and usually after about half that, since
|
||||
each wait is drawn at random below its ceiling. Every other command uses the
|
||||
defaults. A library user gets the same budget by passing
|
||||
`UNATTENDED_RETRY_OPTIONS` as `ApiClientOptions.retry`. `sleep` and `random` are
|
||||
injectable through the same option, which is how the test suite exercises the
|
||||
whole policy without waiting.
|
||||
With those defaults a file that is going to fail gives up after at most three
|
||||
and a half seconds of waiting. `sleep` and `random` are injectable through the
|
||||
same option, which is how the test suite exercises the whole policy without
|
||||
waiting.
|
||||
|
||||
Two deadlines, renewed for each attempt:
|
||||
|
||||
@@ -549,7 +548,7 @@ quak collections [--json] list all collections
|
||||
quak files --collection <id> [--json] list files in a collection
|
||||
quak get <fileID> [--out path] [--collection] download and decrypt a file
|
||||
quak get-thumb <fileID> [--out] [--collection] download and decrypt a thumbnail
|
||||
quak backup <dir> [--json] [--verify] full incremental backup
|
||||
quak backup <dir> [--json] full incremental backup
|
||||
quak backup-metadata <dir> [--exif] dump the metadata quak keeps as JSON
|
||||
quak helper list-missing-thumbnails [--json] find files with missing thumbnails
|
||||
quak helper fix-missing-thumbnails [--file ids] [--json] generate + upload missing thumbnails
|
||||
@@ -582,9 +581,6 @@ reads no tag from is recorded, base64, as `exifRaw`, with the reason in
|
||||
`exifError`. `collections`, `files`, `backup`, `helper list-missing-thumbnails`
|
||||
and `helper fix-missing-thumbnails` take `--json` for machine-readable output.
|
||||
|
||||
`backup --verify` also hashes the originals already in the backup and downloads
|
||||
again any that do not match the content hash Ente records (see "Backup layout").
|
||||
|
||||
`backup-metadata` fetches ML data in requests of up to 200 files. When a request
|
||||
fails, the error is logged, each of its files is written with the reason in an
|
||||
`mlDataError` field instead of `mlData`, and the dump goes on. The exit code is
|
||||
@@ -613,9 +609,8 @@ the smallest does not.
|
||||
per unique file, two for a live photo: see
|
||||
below)
|
||||
YYYY-MM-DD.<fileID>.json the file's basic metadata fields quak
|
||||
keeps, its update time, its private and
|
||||
public magic metadata, its ML data, and
|
||||
its original's EXIF, XMP and dimensions
|
||||
keeps, and its private and public magic
|
||||
metadata
|
||||
YYYY-MM-DD.<fileID>.livephoto.json
|
||||
which of a live photo's two files is which
|
||||
collections/
|
||||
@@ -623,7 +618,6 @@ the smallest does not.
|
||||
<title> -> ../../YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>
|
||||
(symlink)
|
||||
<name>.json collection metadata + file list
|
||||
account.json the account's email and user ID
|
||||
failures.json files that failed and have not yet succeeded
|
||||
```
|
||||
|
||||
@@ -635,33 +629,6 @@ name as uploaded, case kept, or `.bin` when it has none or it holds anything but
|
||||
letters and digits. When the date or the time zone changes, the next run saves
|
||||
the original at its new path and leaves the old copy where it is.
|
||||
|
||||
`account.json` holds the account's `email` and `userID`, as `backup-metadata`
|
||||
writes it. A collection's JSON holds its `id`, `name`, `type`, `ownerID`,
|
||||
`isShared` and `updationTime`, its `magicMetadata`, `pubMagicMetadata` and
|
||||
`sharedMagicMetadata` when it has them, as `backup-metadata`'s
|
||||
`_collection.json` does, and `files`, each file's `id` and `metadata`. A file's
|
||||
JSON holds its `id`, `collectionID`, `ownerID`, `metadata` and `updationTime`,
|
||||
and its `magicMetadata` and `pubMagicMetadata` when it has them. Update times
|
||||
are in microseconds, as Ente records them.
|
||||
|
||||
A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as
|
||||
`mlData`, the same payload `backup-metadata` writes; a file Ente has no ML data
|
||||
for has no `mlData`. The backup waits for the library's ML data fetch to finish
|
||||
before it writes the JSON files. If that fetch fails, each file whose ML data is
|
||||
not in the cache gets the reason in `mlDataError` instead and counts as failed,
|
||||
and the next run fetches it again. The JSON files are rewritten on every run, so
|
||||
ML data that arrived since the last run appears.
|
||||
|
||||
A file's JSON holds, as `imageMetadata`, what `backup-metadata --exif` records
|
||||
from its original, or from a live photo's image: `format`, `width` and `height`
|
||||
for a JPEG, `exif` (or `exifRaw` and `exifError`), and `xmp`. An original with
|
||||
none of these gets `{}`. A video gets no `imageMetadata`: reading a whole video
|
||||
to look for tags is not worth it. When the original cannot be read, the reason
|
||||
is in `imageMetadataError` instead; neither the file nor the run fails. The
|
||||
original is read when the run stores it, or when the JSON beside it has neither
|
||||
field, as one an earlier version wrote. Otherwise the field is taken from that
|
||||
JSON when it is rewritten, so a run does not read every stored original again.
|
||||
|
||||
`failures.json` records each failed file with the kind of failure, how many
|
||||
times it has been tried and when it was last tried. A file leaves it once it
|
||||
succeeds, or once it is no longer in the library or in the backup's scope. The
|
||||
@@ -698,22 +665,8 @@ subsequent runs, existing originals are skipped. If a download fails, the error
|
||||
is logged and the backup continues with the next file. The exit code is non-zero
|
||||
if any files failed. `quak backup` opens its library with the thumbnail and
|
||||
originals precache off, so the only file content it fetches is the originals the
|
||||
backup stores.
|
||||
|
||||
With `--verify`, or `lib.backup({ verify: true })`, a run also hashes each
|
||||
original already at its save path the way a download is checked (see "On-disk
|
||||
cache layout" below): its bytes, read in chunks, or a live photo's image and
|
||||
video, joined as `<imageHash>:<videoHash>`. An original that matches the content
|
||||
hash its metadata records is left as it is. One that does not is logged on one
|
||||
line naming the file, deleted (a live photo's image and video both), and
|
||||
downloaded again in the same run like a missing one; if that download fails, the
|
||||
file goes into `failures.json`. A file whose metadata records no hash is left as
|
||||
it is and counted as unchecked. A stored original that cannot be read is left as
|
||||
it is and counts as failed. The summary and `--json` add the counts `verified`,
|
||||
`mismatched` and `unchecked`, all of originals that were already stored; one
|
||||
first downloaded in this run is in none of them. A mismatch that was downloaded
|
||||
again does not make the exit code non-zero. Without `--verify` nothing is
|
||||
hashed, the summary is unchanged, and the three counts are 0 in `--json`.
|
||||
backup stores. The library's ML data fetch still runs and fills the cache's
|
||||
`mldata/`.
|
||||
|
||||
Each original is written to a temporary file in the same directory, synced to
|
||||
disk, and renamed into place, so an original is either complete or absent, even
|
||||
@@ -933,15 +886,13 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange`
|
||||
- `await lib.backup(opts?)` → `BackupResult`. It waits for a refresh as
|
||||
`fresh()` does, puts every in-scope original not already at its save path
|
||||
there as `photo.download()` does (and, with `includeThumbnails`, fetches
|
||||
thumbnails) through the content cache, waits for an ML data fetch, and
|
||||
rebuilds the on-disk backup tree, each file's JSON with its ML data and its
|
||||
original's EXIF, XMP and dimensions, with a durable failure ledger. A fetched
|
||||
original is written straight to its save path and not into the cache, which
|
||||
then counts it as present; one the cache already held is copied from there.
|
||||
`BackupOptions`: `downloadDirectory` (falls back to the library's),
|
||||
`includeOriginals` (default `true`), `includeThumbnails` (default `false`),
|
||||
`onlyAlbumNames`, and `onProgress`. See Backup layout above for the tree it
|
||||
writes.
|
||||
thumbnails) through the content cache, and rebuilds the on-disk backup tree
|
||||
with a durable failure ledger. A fetched original is written straight to its
|
||||
save path and not into the cache, which then counts it as present; one the
|
||||
cache already held is copied from there. `BackupOptions`: `downloadDirectory`
|
||||
(falls back to the library's), `includeOriginals` (default `true`),
|
||||
`includeThumbnails` (default `false`), `onlyAlbumNames`, and `onProgress`. See
|
||||
Backup layout above for the tree it writes.
|
||||
|
||||
### Request pools
|
||||
|
||||
@@ -1036,12 +987,14 @@ documents:
|
||||
before the implementation. Tests are the canonical API documentation and must
|
||||
be commented thoroughly. `main` and `next` are always green.
|
||||
|
||||
- **Required checks before every commit:** `make lint` and `make fmt-check` must
|
||||
pass. `make lint` is eslint plus the prettier check, and it builds the `lint`
|
||||
phase of the `Dockerfile`, so it needs docker. The pre-commit hook enforces
|
||||
exactly that. `make check` (which also runs the tests) must pass before
|
||||
merging into `next`. Never invoke eslint or prettier directly; linting runs in
|
||||
the container only.
|
||||
- **Required checks before every commit:** `make lint` must pass — that is
|
||||
eslint plus the prettier check, and it builds the `lint` phase of the
|
||||
`Dockerfile`, so it needs docker. The pre-commit hook enforces exactly that.
|
||||
`make check` (which also runs the tests) must pass before merging into `next`.
|
||||
`make fmt-check` is available for a host-side formatting check on its own, but
|
||||
it is not a separate requirement: `make lint` already covers it, and running
|
||||
both would check formatting twice. Never invoke eslint or prettier directly;
|
||||
linting runs in the container only.
|
||||
|
||||
- **Formatting:** prettier with 4-space indents and `proseWrap: always` for
|
||||
markdown. Use `make fmt` to format. Use `yarn` not `npm`.
|
||||
|
||||
+44
-120
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: Repository Policies
|
||||
last_modified: 2026-10-04
|
||||
last_modified: 2026-09-08
|
||||
---
|
||||
|
||||
This document covers repository structure, tooling, and workflow standards. Code
|
||||
@@ -104,14 +104,10 @@ style conventions are in separate documents:
|
||||
`lint` phase and a `test` phase, with the final stage depending on both so the
|
||||
image cannot be built unless they pass. For non-server repos the final stage
|
||||
brings up a development environment; for server repos it is the runtime image.
|
||||
The gate phases and the build stage start from their pinned base images and
|
||||
install what those images lack either inline, as the canonical Go `Dockerfile`
|
||||
below does for `git`, or by running `script/bootstrap`, as the `prompts`
|
||||
repo's own `Dockerfile` does for its yarn packages. The development
|
||||
environment stage installs development prerequisites by running
|
||||
`script/bootstrap` rather than duplicating its installs inline. A stage that
|
||||
runs `script/bootstrap` COPYs `script/` and the dependency manifests
|
||||
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
|
||||
Dockerfiles install development prerequisites by running `script/bootstrap`
|
||||
rather than duplicating installs inline; COPY `script/` and the dependency
|
||||
manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before
|
||||
running it.
|
||||
|
||||
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
|
||||
no separate lint file. `script/lint` and `script/test` each build one phase
|
||||
@@ -160,14 +156,11 @@ style conventions are in separate documents:
|
||||
not evidence that anything ran: a sub-second build reporting success is a
|
||||
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
|
||||
and friends destroy a build cache shared with every other build on the host.
|
||||
When a check is added or changed, prove it works by planting a defect it must
|
||||
catch and watching the run fail on it, then revert the defect. A green run
|
||||
alone shows neither that the check ran nor that it covers what it should.
|
||||
|
||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
||||
hash), so lint failures surface in seconds rather than after a full compile,
|
||||
and the test phase is based on the Debian Go image. The canonical Go repo
|
||||
and the test phase is based on the Go image. The canonical Go repo
|
||||
`Dockerfile`:
|
||||
|
||||
```dockerfile
|
||||
@@ -180,9 +173,8 @@ style conventions are in separate documents:
|
||||
COPY . .
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||
# image ships and the alpine one does not.
|
||||
# golang:1.x, YYYY-MM-DD
|
||||
# Test phase
|
||||
# golang:1.x-alpine, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS test
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
@@ -199,29 +191,15 @@ style conventions are in separate documents:
|
||||
FROM golang@sha256:... AS builder
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
RUN apk add --no-cache git
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /src
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
|
||||
# The VERSION build arg when one is given, otherwise
|
||||
# `git describe --tags --always` on the .git in the build context. With
|
||||
# .git present, a version that is still empty, dev or unknown fails the
|
||||
# build: git is missing or could not read the checkout.
|
||||
ARG VERSION
|
||||
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||
if [ -e .git ]; then \
|
||||
case "$VERSION" in ""|dev|unknown) \
|
||||
echo "version is '$VERSION' although .git is present" >&2; \
|
||||
exit 1 ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /app ./cmd/app/
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /app ./cmd/app/
|
||||
|
||||
# Runtime stage, and the last one
|
||||
FROM alpine@sha256:...
|
||||
@@ -243,41 +221,10 @@ style conventions are in separate documents:
|
||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
||||
create placeholder files so the embed directives resolve. Example:
|
||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||
- If the project requires CGO or system libraries for linting, install them
|
||||
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
||||
has no `apk`, so install with `apt-get` under the Debian package name
|
||||
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
||||
lists in the same `RUN`, so the layer does not keep them:
|
||||
|
||||
```dockerfile
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
```
|
||||
|
||||
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
||||
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
|
||||
repository's own, each submodule's under `.git/modules/`, and that of a
|
||||
submodule keeping its own `.git` directory. `git describe` does not need
|
||||
them, and each can hold a credential: a password in a remote URL, or the
|
||||
token the CI checkout step stores there. A submodule whose name has a
|
||||
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
|
||||
git directory to `**/.git/modules/**/config`, and Go's version stamping
|
||||
then fails the build: give it a name without that segment
|
||||
(`git submodule add --name`). The stage that compiles has `git` (the
|
||||
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
|
||||
takes the version from the `VERSION` build argument when one is given,
|
||||
otherwise from `git describe --tags --always`. That gives the tag on a
|
||||
tagged commit; on a later commit, the tag, the number of commits since it
|
||||
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
|
||||
tag is reachable. The stage that compiles also marks its working directory
|
||||
safe for git (`git config --system --add safe.directory /src`): a context
|
||||
sent as a tar stream keeps the sender's file owners, and git refuses a
|
||||
checkout owned by another user, so the version would come out empty.
|
||||
`ARG VERSION` has no default, and the build fails if the context carries
|
||||
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
|
||||
`docker build .` with no build arguments must succeed; a Dockerfile that
|
||||
refuses an empty build argument drops that refusal and keeps the argument.
|
||||
- If the project requires CGO or system libraries for linting (e.g.
|
||||
`vips-dev`), install them in the lint phase with `apk add`.
|
||||
- `ARG VERSION=dev` is declared in the stage that compiles and supplied by
|
||||
`script/docker` and `script/cibuild`; no stage may call `git describe`.
|
||||
|
||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||
runs `script/cibuild` on push, and checks out the repo as its only other step.
|
||||
@@ -286,12 +233,7 @@ style conventions are in separate documents:
|
||||
carry the same guarantee, because its gate phases may come from the cache. The
|
||||
image build is uncached and so runs the gate phases a second time. That is the
|
||||
price of the rule above, and it is worth paying: the image that ships is built
|
||||
from a run of its own gates rather than from a cache entry. A separate
|
||||
workflow limited to `main` by a `branches` list under `on: push` cannot be
|
||||
checked by review: to try a change to it, add the feature branch to that list
|
||||
and push, then remove the branch from the list again before merging. Keep any
|
||||
job in it that publishes behind `if: github.ref_name == 'main'`, so the run
|
||||
from the feature branch publishes nothing.
|
||||
from a run of its own gates rather than from a cache entry.
|
||||
|
||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||
@@ -344,19 +286,17 @@ style conventions are in separate documents:
|
||||
```
|
||||
|
||||
`-count=1` is required on both invocations: it defeats Go's test _result_
|
||||
cache, so neither run can report a stored pass in place of running the
|
||||
tests. It leaves the build cache alone, so it costs the runtime of the suite
|
||||
and no recompilation.
|
||||
cache, so the target cannot report a pass it did not earn, and the rerun
|
||||
reproduces a failure instead of replaying it. It leaves the build cache
|
||||
alone, so it costs the runtime of the suite and no recompilation.
|
||||
|
||||
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
||||
passing result in its cache directory (`GOCACHE`), and when the same tests
|
||||
run again on unchanged code it prints that result, marked `(cached)`,
|
||||
without running them. That matters on a developer's machine, where this
|
||||
target runs and the directory lasts from one run to the next. The `test`
|
||||
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
|
||||
result for this repo's tests and nothing before its `go test` step runs a
|
||||
test, so there is nothing to replay. `--no-cache` (above) is what makes that
|
||||
step run on an unchanged tree.
|
||||
Note that this is a second, independent cache, stacked below the Docker
|
||||
layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26)
|
||||
addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes;
|
||||
it does not guarantee `go test` inside that step does any work, because the
|
||||
`GOCACHE` baked into earlier image layers survives into the re-executed
|
||||
step. They are two separate defects requiring two separate fixes, and a fix
|
||||
for one must not be recorded as covering the other.
|
||||
|
||||
Python example:
|
||||
|
||||
@@ -400,7 +340,7 @@ style conventions are in separate documents:
|
||||
— which is more dangerous than a short file with no secret patterns at all,
|
||||
because it reads as solved and stops anyone looking. Give every
|
||||
depth-independent pattern the `**/` prefix and leave only genuinely
|
||||
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
|
||||
root-anchored entries unprefixed: `.git`, and the repo's own host-built
|
||||
binary, written `/myapp` and never `**/myapp`, which would also match
|
||||
`cmd/myapp/` and delete the package directory from the context. Matching is
|
||||
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
|
||||
@@ -425,13 +365,12 @@ style conventions are in separate documents:
|
||||
directory, so a repo running agents in subdirectories still ships
|
||||
`services/api/.claude/` and must add its own anchored entry there.
|
||||
|
||||
- **A plain `docker build .` of a clone stamps the version that
|
||||
`git describe --tags --always` gives**, derived from the `.git` in the build
|
||||
context as the canonical `Dockerfile` above shows. Without its failure check,
|
||||
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
|
||||
and the build would still exit 0. `script/docker` and `script/cibuild` pass
|
||||
the version they compute on the host; it takes precedence. They do this
|
||||
byte-identically across repos:
|
||||
- **Excluding `.git` means `git describe` cannot run inside any build stage, and
|
||||
it fails quietly there.** In a build stage there is no repository, so
|
||||
`git describe` writes nothing to stdout, `-X main.Version=` comes out empty,
|
||||
the binary reports no version at all, and the build still exits 0. Compute the
|
||||
version on the host and thread it in as a build arg. `script/docker` and
|
||||
`script/cibuild` do this, byte-identically across repos:
|
||||
|
||||
```sh
|
||||
# Own line: a failing command substitution inside an argument does not
|
||||
@@ -448,7 +387,7 @@ style conventions are in separate documents:
|
||||
fallback is applied — a live check that fires on a build from an export with
|
||||
no `.git` and on a repository with no commits yet. Do not fold it into the
|
||||
substitution as `|| echo unknown`, which makes the guard unreachable. The
|
||||
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
|
||||
Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared
|
||||
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
|
||||
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
|
||||
the scripts stay byte-identical. One consequence for CI: the standard
|
||||
@@ -487,18 +426,12 @@ style conventions are in separate documents:
|
||||
`test-support` depguard rule, where a repo names its own test-support packages
|
||||
by full import path. A repo adds entries there and changes nothing else, and a
|
||||
re-vendor carries its entries forward. The canonical golangci-lint version is
|
||||
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
|
||||
v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base
|
||||
image
|
||||
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
|
||||
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
|
||||
directive must not name a newer Go minor version than the one golangci-lint
|
||||
was built with, or golangci-lint refuses to lint it: this release lints
|
||||
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
|
||||
installs golangci-lint on the host. A repo sets the lint phase digest to the
|
||||
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
|
||||
the two prompted the change: the canonical copy can name linters that an older
|
||||
golangci-lint rejects, and a newer golangci-lint can add linters that
|
||||
`default: all` switches on until the canonical copy disables them.
|
||||
(`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`,
|
||||
which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the
|
||||
only pin, since no repo installs golangci-lint on the host: bumping the
|
||||
version means changing it and nothing else.
|
||||
|
||||
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
|
||||
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
|
||||
@@ -522,11 +455,6 @@ style conventions are in separate documents:
|
||||
|
||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
||||
|
||||
A Go tool a repo needs on the host is installed with `go install` pinned to
|
||||
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
|
||||
a `go.mod` tool dependency or through a `tools.go` file, either of which
|
||||
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
|
||||
|
||||
- When pinning images or packages by hash, add a comment above the reference
|
||||
with the version and date (YYYY-MM-DD).
|
||||
|
||||
@@ -639,10 +567,10 @@ style conventions are in separate documents:
|
||||
settings.
|
||||
|
||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
||||
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`,
|
||||
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`,
|
||||
and language-specific config). Everything else goes in a subdirectory.
|
||||
Canonical subdirectory names:
|
||||
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
||||
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
||||
language-specific config). Everything else goes in a subdirectory. Canonical
|
||||
subdirectory names:
|
||||
- `bin/` — executable scripts and tools
|
||||
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
|
||||
body is a single call into `internal/` or `pkg/`, no project logic in
|
||||
@@ -673,7 +601,3 @@ style conventions are in separate documents:
|
||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
||||
- Python: `pyproject.toml`
|
||||
|
||||
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
|
||||
is never committed under a file or directory named after one agent tool, such
|
||||
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
|
||||
|
||||
@@ -25,62 +25,14 @@ declares one.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-06: `quak backup --verify` and `lib.backup({ verify: true })` hash
|
||||
each original already at its save path as the download check does, streamed, a
|
||||
live photo as `<imageHash>:<videoHash>` (issue 168). One that does not match
|
||||
the content hash its metadata records is logged, removed (both files of a live
|
||||
photo) and downloaded again in the same run; a failed download goes into
|
||||
`failures.json`. One with no recorded hash is left alone. The result, `--json`
|
||||
and the summary gain `verified`, `mismatched` and `unchecked`. Without
|
||||
`--verify` nothing is hashed.
|
||||
|
||||
- 2026-10-06: `quak backup` writes each original's EXIF, XMP and dimensions into
|
||||
the file's JSON as `imageMetadata`, what `backup-metadata --exif` records
|
||||
(issue 167): for a live photo from its image, for a video nothing, and `{}`
|
||||
for an original with none of them. A failed read puts the reason in
|
||||
`imageMetadataError` and fails neither the file nor the run. An original is
|
||||
read when the run stores it or when its JSON has neither field; otherwise the
|
||||
field is taken from that JSON. The hand-built JPEGs moved to
|
||||
`test/exif-jpeg.ts`, beside the HEIC.
|
||||
|
||||
- 2026-10-06: When the server answers HTTP 401 and that ends a command that
|
||||
loads the saved session, because the server no longer accepts its token, the
|
||||
command prints one line,
|
||||
- 2026-10-05: A command whose saved session the server no longer accepts
|
||||
(HTTP 401) prints one line,
|
||||
`quak: the saved session is no longer valid; run "quak login"`, and exits 3
|
||||
(issue 164). `quak backup` meets that 401 on the refresh it starts with,
|
||||
before it touches any file. For the same commands, a missing or corrupt
|
||||
session file keeps its message and also exits 3, so a cron job can tell that
|
||||
the user must log in again. `quak logout` is unchanged. `run` in
|
||||
`src/cli-run.ts` recognises the 401, which reaches it unchanged.
|
||||
|
||||
- 2026-10-06: `quak backup` retries a failed request for longer than the other
|
||||
commands do (issue 165). `src/retry.ts` exports `UNATTENDED_RETRY_OPTIONS`
|
||||
beside the unchanged default: 10 attempts, a 1 s base delay and a 60 s cap, so
|
||||
a request that keeps failing waits at most 243 s before it gives up.
|
||||
`bin/quak.ts` loads the backup's session with them, so its refresh, ML data
|
||||
and downloads all use them. What is retried and the backoff formula are
|
||||
unchanged.
|
||||
|
||||
- 2026-10-06: The files this repository copies from `sneak/prompts` are copied
|
||||
again from its commit `dd4027b` (issue 171). `.gitignore` and `.dockerignore`
|
||||
keep out more secret files, and this repository's build artifacts follow the
|
||||
copied content. `script/check` runs `script/fmt-check` again, `script/cibuild`
|
||||
runs `script/bootstrap` and `script/check` before the image build,
|
||||
`script/fmt` and `script/fmt-check` find the pinned yarn under nvm, and the
|
||||
image's last stage marks `/app` safe for git.
|
||||
|
||||
- 2026-10-06: `quak backup` writes the account and album records
|
||||
`backup-metadata` writes (issue 166): `account.json` with the account's
|
||||
`email` and `userID`, and in each album's JSON its `ownerID`, `isShared`,
|
||||
`updationTime` and, when present, its three layers of magic metadata. Each
|
||||
file's JSON gains its `updationTime`.
|
||||
|
||||
- 2026-10-05: `quak backup` writes each file's ML data (its faces and CLIP
|
||||
embedding) into the file's JSON as `mlData`, the payload
|
||||
`lib.mldata.forFile()` returns (issue 163). `lib.backup()` waits for an ML
|
||||
data fetch before it writes the JSON files. When that fetch fails, each file
|
||||
whose ML data is not cached gets the reason in `mlDataError` and counts as
|
||||
failed, and the next run fetches it again.
|
||||
(issue 164). A missing or corrupt session file keeps its message and now also
|
||||
exits 3, so a cron job can tell that the user must log in again. `run` in
|
||||
`src/cli-run.ts` recognises the 401, which reaches it unchanged from the
|
||||
refresh, so `quak backup` stops there before it touches any file. Every other
|
||||
error still exits 1.
|
||||
|
||||
- 2026-10-03: The download-albums example test no longer fails on a disk with
|
||||
under 50 GiB free (issue 160). It opens its libraries with
|
||||
|
||||
+2
-19
@@ -23,7 +23,6 @@ import { run as runCommand } from "../src/cli-run.js";
|
||||
import { loadSession } from "../src/cli-session.js";
|
||||
import { Client } from "../src/client.js";
|
||||
import { VERSION } from "../src/index.js";
|
||||
import { UNATTENDED_RETRY_OPTIONS } from "../src/retry.js";
|
||||
|
||||
const paths = envPaths("quak", { suffix: "" });
|
||||
|
||||
@@ -130,24 +129,8 @@ program
|
||||
)
|
||||
.argument("<dir>", "Output directory")
|
||||
.option("--json", "Print result as JSON instead of human-readable summary")
|
||||
.option(
|
||||
"--verify",
|
||||
"Re-hash stored originals and download again any that do not match",
|
||||
)
|
||||
// A backup usually runs from cron with nobody watching, so every request
|
||||
// it makes retries for longer than the other commands' requests do.
|
||||
.action((dir: string, opts: { json?: boolean; verify?: boolean }) =>
|
||||
run(
|
||||
backupCommand(
|
||||
{
|
||||
...context(),
|
||||
loadSession: (path) =>
|
||||
loadSession(path, { retry: UNATTENDED_RETRY_OPTIONS }),
|
||||
},
|
||||
dir,
|
||||
opts,
|
||||
),
|
||||
),
|
||||
.action((dir: string, opts: { json?: boolean }) =>
|
||||
run(backupCommand(context(), dir, opts)),
|
||||
);
|
||||
|
||||
const helper = program
|
||||
|
||||
+7
-5
@@ -1,8 +1,11 @@
|
||||
#!/bin/sh
|
||||
# script/check: run all checks (test, lint, fmt-check). Our own
|
||||
# extension to scripts-to-rule-them-all. test and lint are Docker
|
||||
# phases; fmt-check is native, because a formatter writes the working
|
||||
# tree. Must not modify any files.
|
||||
# script/check: run all checks (test, lint). Our own extension to
|
||||
# scripts-to-rule-them-all. Both are Docker phases. Must not modify any
|
||||
# files.
|
||||
#
|
||||
# script/fmt-check is not called here, unlike the template: the lint
|
||||
# phase already runs `prettier --check .`, so calling it would run
|
||||
# prettier a second time over the same tree for the same verdict.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
@@ -10,7 +13,6 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
main() {
|
||||
"$SCRIPT_DIR/test"
|
||||
"$SCRIPT_DIR/lint"
|
||||
"$SCRIPT_DIR/fmt-check"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+7
-7
@@ -1,7 +1,8 @@
|
||||
#!/bin/sh
|
||||
# script/cibuild: run the CI build. It bootstraps first: a CI runner
|
||||
# checks out and runs this and nothing else, and script/fmt-check runs
|
||||
# the formatter on the host, which a pristine checkout cannot do.
|
||||
# script/cibuild: run the CI build. The image's last stage depends on the
|
||||
# lint and test phases, so this one build runs eslint, prettier and the
|
||||
# suite once each and then compiles. Unlike the template it does not run
|
||||
# script/check first, which would run lint and the tests a second time.
|
||||
# --no-cache for the same reason as script/docker: the gate phases the
|
||||
# final stage depends on are RUN steps, and a cached one is a check that
|
||||
# did not run.
|
||||
@@ -12,12 +13,11 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
"$SCRIPT_DIR/bootstrap"
|
||||
"$SCRIPT_DIR/check"
|
||||
# Own line: a failing command substitution inside an argument does
|
||||
# not trip `set -e`, so the inline form degrades silently to an
|
||||
# empty constant. The VERSION build argument takes precedence over
|
||||
# the version a build stage derives from the .git in the context.
|
||||
# empty constant. The version resolved here goes in as the VERSION
|
||||
# build arg, which takes precedence over what the build would derive
|
||||
# from the .git in its context.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
docker build --no-cache \
|
||||
|
||||
+3
-2
@@ -12,8 +12,9 @@ main() {
|
||||
cd "$ROOT"
|
||||
# Own line: a failing command substitution inside an argument does
|
||||
# not trip `set -e`, so the inline form degrades silently to an
|
||||
# empty constant. The VERSION build argument takes precedence over
|
||||
# the version a build stage derives from the .git in the context.
|
||||
# empty constant. The version resolved here goes in as the VERSION
|
||||
# build arg, which takes precedence over what the build would derive
|
||||
# from the .git in its context.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
docker build --no-cache \
|
||||
|
||||
+1
-20
@@ -4,28 +4,9 @@ set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Must match the pin in script/bootstrap.
|
||||
NODE_VERSION="22.17.0"
|
||||
|
||||
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||
# on the PATH of the shell that called it, so resolve the pinned
|
||||
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||
# bash script, hence the subshell.
|
||||
run_yarn() {
|
||||
if command -v yarn >/dev/null 2>&1; then
|
||||
exec yarn "$@"
|
||||
fi
|
||||
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
echo "fmt: no yarn; run script/bootstrap first" >&2
|
||||
exit 1
|
||||
fi
|
||||
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
|
||||
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
run_yarn run prettier --write .
|
||||
yarn run prettier --write .
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+1
-20
@@ -4,28 +4,9 @@ set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Must match the pin in script/bootstrap.
|
||||
NODE_VERSION="22.17.0"
|
||||
|
||||
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||
# on the PATH of the shell that called it, so resolve the pinned
|
||||
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||
# bash script, hence the subshell.
|
||||
run_yarn() {
|
||||
if command -v yarn >/dev/null 2>&1; then
|
||||
exec yarn "$@"
|
||||
fi
|
||||
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
echo "fmt-check: no yarn; run script/bootstrap first" >&2
|
||||
exit 1
|
||||
fi
|
||||
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
|
||||
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
run_yarn run prettier --check .
|
||||
yarn run prettier --check .
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+5
-5
@@ -2,17 +2,17 @@
|
||||
# script/precommit: run by the git pre-commit hook; fails the commit if
|
||||
# checks fail. Our own extension to scripts-to-rule-them-all.
|
||||
#
|
||||
# Runs lint and fmt-check but deliberately NOT the tests, so the TDD
|
||||
# red-phase commit (failing tests, no implementation yet) can land. CI
|
||||
# runs script/cibuild, which runs the tests, and so catches any branch
|
||||
# that ships red.
|
||||
# Runs lint but deliberately NOT the tests, so the TDD red-phase commit
|
||||
# (failing tests, no implementation yet) can land. CI runs
|
||||
# script/cibuild, whose image build includes the test phase, and so
|
||||
# catches any branch that ships red. The lint phase includes the
|
||||
# prettier check, so a badly formatted tree still fails the commit.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
|
||||
main() {
|
||||
"$SCRIPT_DIR/lint"
|
||||
"$SCRIPT_DIR/fmt-check"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+26
-242
@@ -3,20 +3,16 @@
|
||||
// `lib.backup()` waits for a completed refresh of the library (a failed one
|
||||
// fails the backup before any file is touched), then, for every file in scope,
|
||||
// puts its original at its save path under `downloadDirectory`, as
|
||||
// `Photo.download()` does, waits for an ML data fetch, and rebuilds the derived
|
||||
// views (per-file sidecars, per-collection symlink trees, per-collection JSON)
|
||||
// from the model. The on-disk layout:
|
||||
// `Photo.download()` does, and rebuilds the derived views (per-file sidecars,
|
||||
// per-collection symlink trees, per-collection JSON) from the model. The
|
||||
// on-disk layout:
|
||||
//
|
||||
// <downloadDirectory>/
|
||||
// YYYY/YYYY-MM/YYYY-MM-DD/
|
||||
// YYYY-MM-DD.<fileID>.<ext> the decrypted bytes (the save path)
|
||||
// YYYY-MM-DD.<fileID>.json per-file metadata sidecar, with
|
||||
// the file's ML data and its
|
||||
// original's EXIF, XMP and
|
||||
// dimensions
|
||||
// YYYY-MM-DD.<fileID>.json per-file metadata sidecar
|
||||
// collections/<name>/<title> symlink to the original
|
||||
// collections/<name>.json per-collection metadata
|
||||
// account.json the account's email and user ID
|
||||
// failures.json durable ledger of unresolved failures
|
||||
//
|
||||
// A live photo's original is its image and its video, each with its own
|
||||
@@ -30,28 +26,18 @@
|
||||
// no unique state, so they are rebuilt every run; that repairs stale sidecars
|
||||
// and missing or broken symlinks left by an earlier crash. A rebuild also
|
||||
// removes the symlinks to originals that no longer belong to an album, and the
|
||||
// directories of albums that no longer exist. The one thing a sidecar takes
|
||||
// from the sidecar it replaces is its original's EXIF, XMP and dimensions (or
|
||||
// why they could not be read), so that a run does not read every stored
|
||||
// original again; a sidecar without them gets them read from the original.
|
||||
//
|
||||
// With `verify`, each original already at its save path is hashed as the
|
||||
// download check hashes it, and one that does not match the content hash its
|
||||
// metadata records is removed and fetched again in the same run.
|
||||
// directories of albums that no longer exist.
|
||||
//
|
||||
// Resilience (issue #8): no per-file condition aborts the run. A failed
|
||||
// download, a failed symlink, or ML data missing because the ML data fetch
|
||||
// failed is caught, recorded in `failures.json` with a classification, a
|
||||
// running attempt count, and the last-tried time, and the run continues.
|
||||
// `result.failed` — and thus the CLI's exit code — stays non-zero while any
|
||||
// failure remains unresolved and clears once every one succeeds. Each run
|
||||
// reconciles the ledger against the files it attempted, so an entry for a file
|
||||
// that has since left the library (deleted) or this run's scope is dropped
|
||||
// rather than counted forever, which would poison a scheduled backup's exit
|
||||
// code.
|
||||
// download or a failed symlink is caught, recorded in `failures.json` with a
|
||||
// classification, a running attempt count, and the last-tried time, and the run
|
||||
// continues. `result.failed` — and thus the CLI's exit code — stays non-zero
|
||||
// while any failure remains unresolved and clears once every one succeeds. Each
|
||||
// run reconciles the ledger against the files it attempted, so an entry for a
|
||||
// file that has since left the library (deleted) or this run's scope is dropped
|
||||
// rather than counted forever, which would poison a scheduled backup's exit code.
|
||||
|
||||
import {
|
||||
createReadStream,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
@@ -63,15 +49,8 @@ import {
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, extname, join, relative, resolve } from "node:path";
|
||||
|
||||
import {
|
||||
chunkHashFinal,
|
||||
chunkHashInit,
|
||||
chunkHashUpdate,
|
||||
init,
|
||||
} from "./crypto/index.js";
|
||||
import { removeLeftoverTempFiles } from "./download/index.js";
|
||||
import { sanitizeFileName, withExtension } from "./filename.js";
|
||||
import {
|
||||
@@ -81,9 +60,7 @@ import {
|
||||
storedAtSavePath,
|
||||
} from "./library/content.js";
|
||||
import { representative } from "./library/records.js";
|
||||
import { extractImageMetadata } from "./metadata-backup.js";
|
||||
import type { MLData } from "./mldata-fetch.js";
|
||||
import type { Collection, EnteFile, FileMetadata } from "./model/types.js";
|
||||
import type { Collection, EnteFile } from "./model/types.js";
|
||||
|
||||
export type ProgressCallback = (message: string) => void;
|
||||
|
||||
@@ -99,9 +76,6 @@ export interface BackupOptions {
|
||||
includeThumbnails?: boolean;
|
||||
// Restrict the backup to albums with these names; others are left untouched.
|
||||
onlyAlbumNames?: string[];
|
||||
// Hash each original already at its save path, and fetch again any whose
|
||||
// bytes do not match the content hash its metadata records. Default false.
|
||||
verify?: boolean;
|
||||
onProgress?: ProgressCallback;
|
||||
}
|
||||
|
||||
@@ -119,13 +93,6 @@ export interface BackupResult {
|
||||
downloaded: number;
|
||||
// Originals already at their save path and left untouched.
|
||||
skipped: number;
|
||||
// With `verify`, the originals already at their save path whose hash
|
||||
// matched, those whose hash did not (each removed and fetched again), and
|
||||
// those whose metadata records no hash (left as they are). All three are
|
||||
// zero without `verify`.
|
||||
verified: number;
|
||||
mismatched: number;
|
||||
unchecked: number;
|
||||
// Files with an unresolved failure after this run (the ledger size); the
|
||||
// CLI exits non-zero while this is above zero. A file can be both
|
||||
// downloaded and failed if its bytes landed but its symlink did not.
|
||||
@@ -137,8 +104,6 @@ export interface BackupResult {
|
||||
// The slice of the library that backup drives. `Library` implements it; a test
|
||||
// can drive backup with a stand-in.
|
||||
export interface BackupLibrary {
|
||||
// The account the library belongs to.
|
||||
whoami(): { email: string; userID: number };
|
||||
refresh(): Promise<void>;
|
||||
listCollections(): Collection[];
|
||||
listFiles(collectionID: number): EnteFile[];
|
||||
@@ -152,13 +117,6 @@ export interface BackupLibrary {
|
||||
destination: string,
|
||||
): Promise<{ path: string; videoPath?: string }>;
|
||||
thumbnail(fileID: number): Promise<{ path: string }>;
|
||||
// Wait for an ML data fetch to complete, joining one already running or
|
||||
// starting one. Rejects with the reason when it fails; resolves at once
|
||||
// when the library cannot fetch ML data.
|
||||
fetchMLData(): Promise<void>;
|
||||
// A file's cached ML data, as `lib.mldata.forFile()` returns it, or
|
||||
// undefined when none is cached.
|
||||
mlData(fileID: number): Promise<MLData | undefined>;
|
||||
}
|
||||
|
||||
type FailureClass = "transient" | "permanent" | "unknown";
|
||||
@@ -382,115 +340,18 @@ const saveLedger = (path: string, ledger: Map<number, FailureEntry>): void => {
|
||||
);
|
||||
};
|
||||
|
||||
// The content hash of the original stored at `stored`, computed as the download
|
||||
// check computes it: over each file's bytes, read in chunks, and for a live
|
||||
// photo `<imageHash>:<videoHash>`.
|
||||
const storedHash = async (stored: {
|
||||
path: string;
|
||||
videoPath?: string;
|
||||
}): Promise<string> => {
|
||||
await init();
|
||||
const hashFile = async (path: string): Promise<string> => {
|
||||
const state = chunkHashInit();
|
||||
for await (const chunk of createReadStream(path)) {
|
||||
chunkHashUpdate(state, chunk as Buffer);
|
||||
}
|
||||
return chunkHashFinal(state);
|
||||
};
|
||||
const hash = await hashFile(stored.path);
|
||||
if (stored.videoPath === undefined) return hash;
|
||||
return `${hash}:${await hashFile(stored.videoPath)}`;
|
||||
};
|
||||
|
||||
// A file's EXIF, XMP and dimensions as its JSON holds them: what
|
||||
// `extractImageMetadata` found in its original, or why the original could not
|
||||
// be read.
|
||||
interface ImageMetadata {
|
||||
imageMetadata?: Record<string, unknown>;
|
||||
imageMetadataError?: string;
|
||||
}
|
||||
|
||||
// The image metadata for the file whose original is at `originalPath` (for a
|
||||
// live photo, its image) and whose JSON is at `jsonPath`. A video gets none,
|
||||
// as `photo.exif()` reads none. An original stored before this run is not read
|
||||
// again when its JSON already holds image metadata: that is kept. A failed
|
||||
// read gives the reason, and fails neither the file nor the run. An original
|
||||
// with no EXIF, XMP or JPEG dimensions gets `{}`, so it is not read again.
|
||||
const imageMetadataFor = async (
|
||||
file: EnteFile,
|
||||
originalPath: string,
|
||||
jsonPath: string,
|
||||
storedThisRun: boolean,
|
||||
): Promise<ImageMetadata> => {
|
||||
if (file.metadata.fileType === "video") return {};
|
||||
if (!storedThisRun) {
|
||||
try {
|
||||
const { imageMetadata, imageMetadataError } = JSON.parse(
|
||||
readFileSync(jsonPath, "utf-8"),
|
||||
) as ImageMetadata;
|
||||
if (imageMetadata !== undefined || imageMetadataError !== undefined)
|
||||
return { imageMetadata, imageMetadataError };
|
||||
} catch {
|
||||
// No JSON yet, or one that cannot be parsed: read the original.
|
||||
}
|
||||
}
|
||||
try {
|
||||
const bytes = await readFile(originalPath);
|
||||
return { imageMetadata: extractImageMetadata(bytes) ?? {} };
|
||||
} catch (err) {
|
||||
return { imageMetadataError: errorMessage(err) };
|
||||
}
|
||||
};
|
||||
|
||||
// The file's JSON: its basic fields, its magic metadata, its ML data or the
|
||||
// reason the ML data is missing, and its image metadata.
|
||||
const writeSidecar = (
|
||||
path: string,
|
||||
file: EnteFile,
|
||||
ml: { mlData?: MLData; mlDataError?: string },
|
||||
image: ImageMetadata,
|
||||
): void => {
|
||||
const writeSidecar = (path: string, file: EnteFile): void => {
|
||||
const meta: Record<string, unknown> = {
|
||||
id: file.id,
|
||||
collectionID: file.collectionID,
|
||||
ownerID: file.ownerID,
|
||||
metadata: file.metadata,
|
||||
updationTime: file.updationTime,
|
||||
};
|
||||
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
||||
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
||||
if (ml.mlData) meta.mlData = ml.mlData;
|
||||
if (ml.mlDataError) meta.mlDataError = ml.mlDataError;
|
||||
if (image.imageMetadata) meta.imageMetadata = image.imageMetadata;
|
||||
if (image.imageMetadataError) {
|
||||
meta.imageMetadataError = image.imageMetadataError;
|
||||
}
|
||||
writeFileSync(path, JSON.stringify(meta, null, 2));
|
||||
};
|
||||
|
||||
// The album's JSON: its basic fields, its magic metadata, and its files.
|
||||
const writeAlbumJSON = (
|
||||
path: string,
|
||||
c: Collection,
|
||||
files: { id: number; metadata: FileMetadata }[],
|
||||
): void => {
|
||||
const album: Record<string, unknown> = {
|
||||
id: c.id,
|
||||
name: c.name,
|
||||
type: c.type,
|
||||
ownerID: c.ownerID,
|
||||
isShared: c.isShared,
|
||||
updationTime: c.updationTime,
|
||||
};
|
||||
if (c.magicMetadata) album.magicMetadata = c.magicMetadata;
|
||||
if (c.pubMagicMetadata) album.pubMagicMetadata = c.pubMagicMetadata;
|
||||
if (c.sharedMagicMetadata) {
|
||||
album.sharedMagicMetadata = c.sharedMagicMetadata;
|
||||
}
|
||||
album.files = files;
|
||||
writeFileSync(path, JSON.stringify(album, null, 2));
|
||||
};
|
||||
|
||||
export const runBackup = async (
|
||||
lib: BackupLibrary,
|
||||
opts: BackupOptions,
|
||||
@@ -506,7 +367,6 @@ export const runBackup = async (
|
||||
const includeThumbnails = opts.includeThumbnails ?? false;
|
||||
const log = opts.onProgress ?? (() => {});
|
||||
const only = opts.onlyAlbumNames ? new Set(opts.onlyAlbumNames) : undefined;
|
||||
const verify = opts.verify ?? false;
|
||||
|
||||
log("Refreshing library...");
|
||||
await lib.refresh();
|
||||
@@ -514,11 +374,6 @@ export const runBackup = async (
|
||||
const collectionsDir = join(downloadDirectory, "collections");
|
||||
const thumbnailsDir = join(downloadDirectory, "thumbnails");
|
||||
mkdirSync(collectionsDir, { recursive: true });
|
||||
const { email, userID } = lib.whoami();
|
||||
writeFileSync(
|
||||
join(downloadDirectory, "account.json"),
|
||||
JSON.stringify({ email, userID }, null, 2),
|
||||
);
|
||||
if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true });
|
||||
removeLeftoverTempFiles(thumbnailsDir);
|
||||
for (const dir of dateFolders(downloadDirectory)) {
|
||||
@@ -562,12 +417,8 @@ export const runBackup = async (
|
||||
|
||||
const errors: BackupError[] = [];
|
||||
const failedThisRun = new Set<number>();
|
||||
const storedThisRun = new Set<number>();
|
||||
let downloaded = 0;
|
||||
let skipped = 0;
|
||||
let verified = 0;
|
||||
let mismatched = 0;
|
||||
let unchecked = 0;
|
||||
|
||||
const recordFailure = (
|
||||
file: EnteFile,
|
||||
@@ -599,45 +450,10 @@ export const runBackup = async (
|
||||
|
||||
// Phase 1: get the bytes. Put each pending original at its save path
|
||||
// through the content cache/pools, as `Photo.download()` does, and fetch
|
||||
// the optional thumbnails; a present file is left as is. With `verify`, a
|
||||
// present original is hashed first, and one that does not match the hash
|
||||
// its metadata records is removed and fetched like a missing one. One
|
||||
// that cannot be read is recorded as failed and left where it is.
|
||||
// the optional thumbnails; a present file is left as is.
|
||||
if (includeOriginals) {
|
||||
for (const [fileID, file] of distinct) {
|
||||
let stored = storedAtSavePath(downloadDirectory, file);
|
||||
if (stored !== undefined && verify) {
|
||||
try {
|
||||
if (file.metadata.hash === undefined) {
|
||||
unchecked++;
|
||||
} else if (
|
||||
(await storedHash(stored)) === file.metadata.hash
|
||||
) {
|
||||
verified++;
|
||||
} else {
|
||||
log(
|
||||
`MISMATCH original ${file.metadata.title} (${fileID}): its bytes do not match its content hash`,
|
||||
);
|
||||
mismatched++;
|
||||
rmSync(stored.path);
|
||||
if (stored.videoPath !== undefined) {
|
||||
rmSync(stored.videoPath);
|
||||
}
|
||||
stored = undefined;
|
||||
}
|
||||
} catch (err) {
|
||||
log(
|
||||
`FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
|
||||
);
|
||||
recordFailure(
|
||||
file,
|
||||
collectionName.get(file.collectionID) ?? "",
|
||||
err,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (stored !== undefined) {
|
||||
if (storedAtSavePath(downloadDirectory, file) !== undefined) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
@@ -649,7 +465,6 @@ export const runBackup = async (
|
||||
await placeOriginal(downloadDirectory, file, (dest) =>
|
||||
lib.original(fileID, dest),
|
||||
);
|
||||
storedThisRun.add(fileID);
|
||||
downloaded++;
|
||||
} catch (err) {
|
||||
log(
|
||||
@@ -682,43 +497,12 @@ export const runBackup = async (
|
||||
}
|
||||
|
||||
// Phase 2: rebuild the derived views from the model. Sidecars first, for
|
||||
// every present original (this repairs stale ones), each with the file's
|
||||
// ML data once an ML data fetch has completed, and its image metadata.
|
||||
// When the fetch fails, a file with no cached ML data gets the reason
|
||||
// instead and is recorded as failed. The next run fetches its ML data
|
||||
// again because none is cached.
|
||||
// every present original (this repairs stale ones).
|
||||
if (includeOriginals) {
|
||||
let mlDataError: string | undefined;
|
||||
try {
|
||||
log("Fetching ML data...");
|
||||
await lib.fetchMLData();
|
||||
} catch (err) {
|
||||
mlDataError = errorMessage(err);
|
||||
log(`FAILED ML data: ${mlDataError}`);
|
||||
}
|
||||
for (const file of distinct.values()) {
|
||||
const stored = storedAtSavePath(downloadDirectory, file);
|
||||
if (stored === undefined) continue;
|
||||
const path = withExtension(
|
||||
savePath(downloadDirectory, file),
|
||||
".json",
|
||||
);
|
||||
const image = await imageMetadataFor(
|
||||
file,
|
||||
stored.path,
|
||||
path,
|
||||
storedThisRun.has(file.id),
|
||||
);
|
||||
const mlData = await lib.mlData(file.id);
|
||||
if (mlData === undefined && mlDataError !== undefined) {
|
||||
writeSidecar(path, file, { mlDataError }, image);
|
||||
recordFailure(
|
||||
file,
|
||||
collectionName.get(file.collectionID) ?? "",
|
||||
new Error(`ML data: ${mlDataError}`),
|
||||
);
|
||||
} else {
|
||||
writeSidecar(path, file, { mlData }, image);
|
||||
if (storedAtSavePath(downloadDirectory, file) !== undefined) {
|
||||
const path = savePath(downloadDirectory, file);
|
||||
writeSidecar(withExtension(path, ".json"), file);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -790,10 +574,13 @@ export const runBackup = async (
|
||||
}
|
||||
}
|
||||
|
||||
writeAlbumJSON(
|
||||
writeFileSync(
|
||||
join(collectionsDir, `${colDirName}.json`),
|
||||
c,
|
||||
metaFiles,
|
||||
JSON.stringify(
|
||||
{ id: c.id, name: c.name, type: c.type, files: metaFiles },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -812,9 +599,6 @@ export const runBackup = async (
|
||||
totalFiles: distinct.size,
|
||||
downloaded,
|
||||
skipped,
|
||||
verified,
|
||||
mismatched,
|
||||
unchecked,
|
||||
failed: ledger.size,
|
||||
errors,
|
||||
};
|
||||
|
||||
+1
-7
@@ -401,7 +401,7 @@ export const backupMetadataCommand = async (
|
||||
export const backupCommand = async (
|
||||
ctx: CliContext,
|
||||
dir: string,
|
||||
opts: { json?: boolean; verify?: boolean },
|
||||
opts: { json?: boolean },
|
||||
): Promise<number> => {
|
||||
await init();
|
||||
const client = requireSession(ctx);
|
||||
@@ -420,7 +420,6 @@ export const backupCommand = async (
|
||||
try {
|
||||
const result = await lib.backup({
|
||||
downloadDirectory: dir,
|
||||
verify: opts.verify,
|
||||
onProgress: (msg) => {
|
||||
if (!opts.json) ctx.stderr.write(msg + "\n");
|
||||
},
|
||||
@@ -433,11 +432,6 @@ export const backupCommand = async (
|
||||
ctx.stderr.write(` Total files: ${result.totalFiles}\n`);
|
||||
ctx.stderr.write(` Downloaded: ${result.downloaded}\n`);
|
||||
ctx.stderr.write(` Skipped: ${result.skipped}\n`);
|
||||
if (opts.verify) {
|
||||
ctx.stderr.write(` Verified: ${result.verified}\n`);
|
||||
ctx.stderr.write(` Mismatched: ${result.mismatched}\n`);
|
||||
ctx.stderr.write(` Unchecked: ${result.unchecked}\n`);
|
||||
}
|
||||
ctx.stderr.write(` Failed: ${result.failed}\n`);
|
||||
if (result.errors.length > 0) {
|
||||
ctx.stderr.write("\nFailed files:\n");
|
||||
|
||||
@@ -27,7 +27,6 @@ export {
|
||||
isRetryable,
|
||||
isSafeToReplay,
|
||||
resolveRetryOptions,
|
||||
UNATTENDED_RETRY_OPTIONS,
|
||||
withRetry,
|
||||
type ResolvedRetryOptions,
|
||||
type RetryOptions,
|
||||
|
||||
+10
-25
@@ -279,8 +279,7 @@ export class Library {
|
||||
private cycle?: Promise<void>;
|
||||
// Guards the ML fetch pass so a slow backfill never runs twice at once; a
|
||||
// refresh whose pass is still running kicks nothing new. Holds the running
|
||||
// pass, so `close()` and `backup()` can wait for it. It rejects when the
|
||||
// pass fails.
|
||||
// pass, so `close()` can wait for it.
|
||||
private mlFetch?: Promise<void>;
|
||||
private closed = false;
|
||||
private lastRefreshAt?: number;
|
||||
@@ -570,10 +569,8 @@ export class Library {
|
||||
// does, joining one already running, and rejects before touching any file
|
||||
// when it fails. Then puts pending originals at their save paths as
|
||||
// `Photo.download()` does (and optional thumbnails) through the content
|
||||
// cache and pools, waits for an ML data fetch, and rebuilds the derived
|
||||
// symlink/JSON views from the model, each file's JSON with its ML data,
|
||||
// beside an `account.json` with the account's email and user ID.
|
||||
// Throws before any network work when no content cache backs the
|
||||
// cache and pools, and rebuilds the derived symlink/JSON views from the
|
||||
// model. Throws before any network work when no content cache backs the
|
||||
// originals it must fetch.
|
||||
backup(opts?: BackupOptions): Promise<BackupResult> {
|
||||
const downloadDirectory =
|
||||
@@ -590,15 +587,12 @@ export class Library {
|
||||
const cache = this.cache;
|
||||
return runBackup(
|
||||
{
|
||||
whoami: () => this.client.whoami(),
|
||||
refresh: () => this.refreshNow(),
|
||||
listCollections: () => this.store.listCollections(),
|
||||
listFiles: (id) => this.store.listFiles(id),
|
||||
original: (fileID, destination) =>
|
||||
cache!.backupOriginal(fileID, destination),
|
||||
thumbnail: (fileID) => cache!.thumbnail(fileID),
|
||||
fetchMLData: () => this.fetchMLDataNow(),
|
||||
mlData: (fileID) => this.mldata.forFile({ fileID }),
|
||||
},
|
||||
{ ...opts, downloadDirectory },
|
||||
);
|
||||
@@ -618,7 +612,7 @@ export class Library {
|
||||
this.timer = undefined;
|
||||
}
|
||||
await this.cycle?.catch(() => {});
|
||||
await this.mlFetch?.catch(() => {});
|
||||
await this.mlFetch;
|
||||
await precacheClosed;
|
||||
}
|
||||
|
||||
@@ -682,9 +676,10 @@ export class Library {
|
||||
// Backfill ML data for the files this refresh knows about. It runs
|
||||
// outside the refresh's success/failure so a fetch or disk problem
|
||||
// there never marks the metadata refresh failed, and it is not
|
||||
// awaited so it never stalls the refresh interval. Its failure is
|
||||
// reported through `status()` and `onProgress`.
|
||||
void this.fetchMLDataNow().catch(() => {});
|
||||
// awaited so it never stalls the refresh interval.
|
||||
this.mlFetch ??= this.runMLFetch().finally(() => {
|
||||
this.mlFetch = undefined;
|
||||
});
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
this.lastError = error;
|
||||
@@ -789,19 +784,10 @@ export class Library {
|
||||
}
|
||||
}
|
||||
|
||||
// Join the running ML fetch pass, or start one when none runs. Resolves at
|
||||
// once when the client cannot fetch ML data; rejects when the pass fails.
|
||||
private fetchMLDataNow(): Promise<void> {
|
||||
this.mlFetch ??= this.runMLFetch().finally(() => {
|
||||
this.mlFetch = undefined;
|
||||
});
|
||||
return this.mlFetch;
|
||||
}
|
||||
|
||||
// One ML fetch pass: fetch, decrypt and store the ML data for every file
|
||||
// the store knows about that is not cached (or whose `updationTime` has
|
||||
// advanced), through the metadata pool, and update the CLIP index. A
|
||||
// failure is reported through `status()` and `onProgress`, then thrown.
|
||||
// advanced), through the metadata pool, and update the CLIP index. Guarded
|
||||
// so passes never overlap; a failure is reported, not thrown.
|
||||
private async runMLFetch(): Promise<void> {
|
||||
const mldata = this.mlStore;
|
||||
// Bind so the call keeps the client as its receiver when invoked
|
||||
@@ -845,7 +831,6 @@ export class Library {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
this.lastMLError = error;
|
||||
this.emit({ operation: "fetchMLData", status: "failed", error });
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -37,16 +37,6 @@ export const DEFAULT_RETRY_OPTIONS: ResolvedRetryOptions = {
|
||||
random: Math.random,
|
||||
};
|
||||
|
||||
// For a run nobody is watching, such as `quak backup` from cron. A request that
|
||||
// keeps failing waits at most 243 s in all before it gives up, usually about
|
||||
// half that, since each wait is drawn at random below its ceiling.
|
||||
export const UNATTENDED_RETRY_OPTIONS: ResolvedRetryOptions = {
|
||||
...DEFAULT_RETRY_OPTIONS,
|
||||
attempts: 10,
|
||||
baseDelayMs: 1_000,
|
||||
maxDelayMs: 60_000,
|
||||
};
|
||||
|
||||
export const resolveRetryOptions = (
|
||||
opts?: RetryOptions,
|
||||
): ResolvedRetryOptions => ({
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
* collections/
|
||||
* <name>/<title> symlink to the original (rebuilt each run)
|
||||
* <name>.json per-collection metadata (rebuilt each run)
|
||||
* account.json the account's email and user ID
|
||||
* failures.json durable ledger of unresolved failures
|
||||
*
|
||||
* The properties that distinguish backup from a naive download loop, and that
|
||||
@@ -33,7 +32,6 @@
|
||||
*/
|
||||
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
@@ -54,17 +52,11 @@ import { runBackup, type BackupLibrary } from "../../src/backup.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
import type { ContentSource } from "../../src/library/content.js";
|
||||
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
||||
import { extractImageMetadata } from "../../src/metadata-backup.js";
|
||||
import type { MLData } from "../../src/mldata-fetch.js";
|
||||
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
||||
import { JPEG_WITH_EXIF } from "../exif-jpeg.js";
|
||||
import {
|
||||
asLivePhoto,
|
||||
blake2b,
|
||||
cdnSource,
|
||||
IMAGE,
|
||||
livePhotoHash,
|
||||
livePhotoZip,
|
||||
VIDEO,
|
||||
} from "../live-photo.js";
|
||||
@@ -836,515 +828,6 @@ describe("the refresh before a backup", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// Also serves ML data: file 100 has `ML_PAYLOAD`, the other two have none.
|
||||
// While `mlError` is set, every ML data request fails with it.
|
||||
const ML_PAYLOAD = { face: { faces: [] }, clip: { embedding: [0.5, 0.25] } };
|
||||
|
||||
class MLClient extends MockClient {
|
||||
mlError?: string;
|
||||
async fetchMLData(args: {
|
||||
fileIDs: number[];
|
||||
}): Promise<Map<number, MLData>> {
|
||||
if (this.mlError) throw new Error(this.mlError);
|
||||
const result = new Map<number, MLData>();
|
||||
if (args.fileIDs.includes(100)) result.set(100, ML_PAYLOAD);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// The JSON the backup in `outDir` wrote beside the original of `fileID`.
|
||||
const fileJSON = (outDir: string, fileID: number): Record<string, unknown> =>
|
||||
JSON.parse(readFileSync(saved(outDir, `${fileID}.json`), "utf-8"));
|
||||
|
||||
describe("ML data in each file's JSON", () => {
|
||||
it("writes a file's ML data, and no ML field for a file that has none", async () => {
|
||||
const lib = await openLibrary(stubSource(), new MLClient());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(result.failed).toBe(0);
|
||||
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
|
||||
expect(fileJSON(outDir, 101)).not.toHaveProperty("mlData");
|
||||
expect(fileJSON(outDir, 101)).not.toHaveProperty("mlDataError");
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("gives each file with no cached ML data the reason when the fetch fails, and fails the run", async () => {
|
||||
const client = new MLClient();
|
||||
const outDir = join(root, "backup");
|
||||
const first = await openLibrary(stubSource(), client);
|
||||
await first.backup({ downloadDirectory: outDir });
|
||||
await first.close();
|
||||
|
||||
client.mlError = "HTTP 503 from server";
|
||||
const lib = await openLibrary(stubSource(), client);
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
// File 100's ML data was cached by the first run and is kept.
|
||||
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
|
||||
expect(fileJSON(outDir, 100)).not.toHaveProperty("mlDataError");
|
||||
for (const fileID of [101, 200]) {
|
||||
expect(fileJSON(outDir, fileID).mlDataError).toBe(
|
||||
"HTTP 503 from server",
|
||||
);
|
||||
}
|
||||
expect(result.failed).toBe(2);
|
||||
expect(result.errors.map((e) => [e.fileID, e.error])).toEqual([
|
||||
[101, "ML data: HTTP 503 from server"],
|
||||
[200, "ML data: HTTP 503 from server"],
|
||||
]);
|
||||
expect(Object.keys(readLedger(outDir).files)).toEqual(["101", "200"]);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("writes the ML data on the run after a failed fetch", async () => {
|
||||
const client = new MLClient();
|
||||
client.mlError = "HTTP 503 from server";
|
||||
const outDir = join(root, "backup");
|
||||
const first = await openLibrary(stubSource(), client);
|
||||
expect((await first.backup({ downloadDirectory: outDir })).failed).toBe(
|
||||
3,
|
||||
);
|
||||
expect(fileJSON(outDir, 100)).not.toHaveProperty("mlData");
|
||||
await first.close();
|
||||
|
||||
client.mlError = undefined;
|
||||
const lib = await openLibrary(stubSource(), client);
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.skipped).toBe(3);
|
||||
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
|
||||
for (const fileID of [100, 101, 200]) {
|
||||
expect(fileJSON(outDir, fileID)).not.toHaveProperty("mlDataError");
|
||||
}
|
||||
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
});
|
||||
|
||||
// The fields `RecordsClient` gives the Vacation album: shared by another
|
||||
// account, with all three layers of magic metadata.
|
||||
const SHARED_ALBUM = {
|
||||
ownerID: 7,
|
||||
isShared: true,
|
||||
updationTime: 1700,
|
||||
magicMetadata: { visibility: 0 },
|
||||
pubMagicMetadata: { coverID: 100 },
|
||||
sharedMagicMetadata: { visibility: 2 },
|
||||
};
|
||||
|
||||
// Serves the Vacation album with `SHARED_ALBUM`'s fields, and each file with
|
||||
// the update time 5000 + its ID.
|
||||
class RecordsClient extends MockClient {
|
||||
override async collectionsSince(): Promise<CollectionsPage> {
|
||||
const page = await super.collectionsSince();
|
||||
return {
|
||||
...page,
|
||||
collections: page.collections.map((c) =>
|
||||
c.id === 1 ? { ...c, ...SHARED_ALBUM } : c,
|
||||
),
|
||||
};
|
||||
}
|
||||
override async filesSince(args: {
|
||||
collectionID: number;
|
||||
}): Promise<FilesPage> {
|
||||
const page = await super.filesSince(args);
|
||||
return {
|
||||
...page,
|
||||
files: page.files.map((f) => ({
|
||||
...f,
|
||||
updationTime: 5000 + f.id,
|
||||
})),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// The JSON the backup in `outDir` wrote for the album named `name`.
|
||||
const albumJSON = (outDir: string, name: string): Record<string, unknown> =>
|
||||
JSON.parse(
|
||||
readFileSync(join(outDir, "collections", `${name}.json`), "utf-8"),
|
||||
);
|
||||
|
||||
describe("account and album records", () => {
|
||||
it("writes account.json with the account's email and user ID", async () => {
|
||||
const lib = await openLibrary(stubSource());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(
|
||||
JSON.parse(readFileSync(join(outDir, "account.json"), "utf-8")),
|
||||
).toEqual({ email: "backup@example.com", userID: USER_ID });
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("writes each album's owner, sharing, update time and magic metadata into its JSON", async () => {
|
||||
const lib = await openLibrary(stubSource(), new RecordsClient());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(albumJSON(outDir, "Vacation")).toEqual({
|
||||
id: 1,
|
||||
name: "Vacation",
|
||||
type: "album",
|
||||
...SHARED_ALBUM,
|
||||
files: [
|
||||
{ id: 100, metadata: file(100, 1, "beach.jpg").metadata },
|
||||
{ id: 101, metadata: file(101, 1, "sunset.jpg").metadata },
|
||||
],
|
||||
});
|
||||
// An album with no magic metadata gets no magic metadata fields.
|
||||
expect(albumJSON(outDir, "Work")).toEqual({
|
||||
id: 2,
|
||||
name: "Work",
|
||||
type: "album",
|
||||
ownerID: USER_ID,
|
||||
isShared: false,
|
||||
updationTime: 1,
|
||||
files: [
|
||||
{ id: 200, metadata: file(200, 2, "diagram.png").metadata },
|
||||
],
|
||||
});
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("writes each file's update time into its JSON", async () => {
|
||||
const lib = await openLibrary(stubSource(), new RecordsClient());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
for (const fileID of [100, 101, 200]) {
|
||||
expect(fileJSON(outDir, fileID).updationTime).toBe(5000 + fileID);
|
||||
}
|
||||
await lib.close();
|
||||
});
|
||||
});
|
||||
|
||||
// What a file's JSON holds as `imageMetadata` for an original of `bytes`.
|
||||
const imageMetadataOf = (bytes: Uint8Array): unknown =>
|
||||
JSON.parse(JSON.stringify(extractImageMetadata(bytes)));
|
||||
|
||||
describe("image metadata in each file's JSON", () => {
|
||||
// Serves `files` in the Vacation album and nothing in Work.
|
||||
class FilesClient extends MockClient {
|
||||
constructor(private readonly files: EnteFile[]) {
|
||||
super();
|
||||
}
|
||||
override async filesSince(args: {
|
||||
collectionID: number;
|
||||
}): Promise<FilesPage> {
|
||||
return {
|
||||
files: args.collectionID === 1 ? this.files : [],
|
||||
deleted: [],
|
||||
cursor: 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Writes `originals.get(fileID)` as each file's original, looked up at
|
||||
// each fetch, so a test can change it between runs.
|
||||
const bytesSource = (
|
||||
originals: Map<number, Uint8Array>,
|
||||
): ContentSource => ({
|
||||
original: async ({ file: f, destination }) => {
|
||||
const bytes = originals.get(f.id)!;
|
||||
writeFileSync(destination, bytes);
|
||||
return { bytesWritten: bytes.length };
|
||||
},
|
||||
thumbnail: async () => {
|
||||
throw new Error("no thumbnails in this source");
|
||||
},
|
||||
});
|
||||
|
||||
// A JPEG, a HEIC, a video, and a file holding no image metadata. The
|
||||
// video's bytes are the JPEG's, so reading it would find EXIF.
|
||||
const clip = file(602, 1, "clip.mov");
|
||||
const files = [
|
||||
file(600, 1, "photo.jpg"),
|
||||
file(601, 1, "photo.heic"),
|
||||
{ ...clip, metadata: { ...clip.metadata, fileType: "video" as const } },
|
||||
file(603, 1, "notes.png"),
|
||||
];
|
||||
const originals = (): Map<number, Uint8Array> =>
|
||||
new Map([
|
||||
[600, JPEG_WITH_EXIF],
|
||||
[601, HEIC_WITH_EXIF],
|
||||
[602, JPEG_WITH_EXIF],
|
||||
[603, new TextEncoder().encode("not an image")],
|
||||
]);
|
||||
const open = (bytes = originals()): Promise<Library> =>
|
||||
openLibrary(bytesSource(bytes), new FilesClient(files));
|
||||
|
||||
// Rewrite the JSON of `fileID` without its image metadata, as an earlier
|
||||
// version of quak wrote it.
|
||||
const dropImageMetadata = (outDir: string, fileID: number): void => {
|
||||
const json = fileJSON(outDir, fileID);
|
||||
delete json.imageMetadata;
|
||||
writeFileSync(saved(outDir, `${fileID}.json`), JSON.stringify(json));
|
||||
};
|
||||
|
||||
it("writes each new original's image metadata, and none for a video", async () => {
|
||||
const lib = await open();
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(result).toMatchObject({ downloaded: 4, failed: 0 });
|
||||
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
|
||||
imageMetadataOf(JPEG_WITH_EXIF),
|
||||
);
|
||||
expect(fileJSON(outDir, 601).imageMetadata).toEqual(
|
||||
imageMetadataOf(HEIC_WITH_EXIF),
|
||||
);
|
||||
expect(fileJSON(outDir, 602)).not.toHaveProperty("imageMetadata");
|
||||
expect(fileJSON(outDir, 602)).not.toHaveProperty("imageMetadataError");
|
||||
// Empty, so that the next run does not read it again.
|
||||
expect(fileJSON(outDir, 603).imageMetadata).toEqual({});
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("reads a live photo's image", async () => {
|
||||
const { file: live, body } = await asLivePhoto(
|
||||
file(500, 1, "IMG_0500.HEIC"),
|
||||
livePhotoZip({ "image.heic": HEIC_WITH_EXIF, "video.mov": VIDEO }),
|
||||
livePhotoHash(HEIC_WITH_EXIF, VIDEO),
|
||||
);
|
||||
const lib = await openLibrary(
|
||||
cdnSource(new Map([[500, body]])),
|
||||
new FilesClient([live]),
|
||||
);
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(fileJSON(outDir, 500).imageMetadata).toEqual(
|
||||
imageMetadataOf(HEIC_WITH_EXIF),
|
||||
);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("keeps the image metadata of an original stored by an earlier run without reading it again", async () => {
|
||||
const lib = await open();
|
||||
const outDir = join(root, "backup");
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
// A read of the original now would give the HEIC's.
|
||||
writeFileSync(saved(outDir, "600.jpg"), HEIC_WITH_EXIF);
|
||||
|
||||
const second = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(second).toMatchObject({ downloaded: 0, failed: 0 });
|
||||
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
|
||||
imageMetadataOf(JPEG_WITH_EXIF),
|
||||
);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("reads an original stored by an earlier version, whose JSON has no image metadata", async () => {
|
||||
const lib = await open();
|
||||
const outDir = join(root, "backup");
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
dropImageMetadata(outDir, 600);
|
||||
|
||||
const second = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(second).toMatchObject({ downloaded: 0, failed: 0 });
|
||||
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
|
||||
imageMetadataOf(JPEG_WITH_EXIF),
|
||||
);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("reads an original the run stores again, though its JSON has image metadata", async () => {
|
||||
const bytes = originals();
|
||||
const lib = await open(bytes);
|
||||
const outDir = join(root, "backup");
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
rmSync(saved(outDir, "600.jpg"));
|
||||
bytes.set(600, HEIC_WITH_EXIF);
|
||||
|
||||
const second = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(second).toMatchObject({ downloaded: 1, failed: 0 });
|
||||
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
|
||||
imageMetadataOf(HEIC_WITH_EXIF),
|
||||
);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
// Root ignores file permissions, so this fails when run as root. The
|
||||
// `test` phase of the `Dockerfile` runs as the `node` user.
|
||||
it("gives the reason an original could not be read, failing neither the file nor the run", async () => {
|
||||
const lib = await open();
|
||||
const outDir = join(root, "backup");
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
dropImageMetadata(outDir, 600);
|
||||
const original = saved(outDir, "600.jpg");
|
||||
chmodSync(original, 0o000);
|
||||
|
||||
const second = await lib
|
||||
.backup({ downloadDirectory: outDir })
|
||||
.finally(() => chmodSync(original, 0o600));
|
||||
|
||||
expect(second).toMatchObject({ failed: 0, errors: [] });
|
||||
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
||||
expect(fileJSON(outDir, 600)).not.toHaveProperty("imageMetadata");
|
||||
expect(fileJSON(outDir, 600).imageMetadataError).toMatch(/EACCES/);
|
||||
await lib.close();
|
||||
});
|
||||
});
|
||||
|
||||
// With `verify`, each original already stored is hashed, and one that does not
|
||||
// match the content hash its metadata records is downloaded again.
|
||||
describe("backup with verify", () => {
|
||||
// MockClient's files, each recording the hash of what `stubSource` writes
|
||||
// for it, except diagram.png (200), which records none.
|
||||
class HashedClient extends MockClient {
|
||||
override async filesSince(args: {
|
||||
collectionID: number;
|
||||
}): Promise<FilesPage> {
|
||||
const page = await super.filesSince(args);
|
||||
const files = page.files.map((f) =>
|
||||
f.id === 200
|
||||
? f
|
||||
: {
|
||||
...f,
|
||||
metadata: {
|
||||
...f.metadata,
|
||||
hash: blake2b(Buffer.alloc(SIZE_BY_ID[f.id]!)),
|
||||
},
|
||||
},
|
||||
);
|
||||
return { ...page, files };
|
||||
}
|
||||
}
|
||||
|
||||
// A backup of the account in `root/backup`, the library that made it, and
|
||||
// the source it fetched from.
|
||||
const backedUp = async (): Promise<{
|
||||
lib: Library;
|
||||
source: StubSource;
|
||||
outDir: string;
|
||||
}> => {
|
||||
const source = stubSource();
|
||||
const lib = await openLibrary(source, new HashedClient());
|
||||
const outDir = join(root, "backup");
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
return { lib, source, outDir };
|
||||
};
|
||||
|
||||
// What `stubSource` writes for beach.jpg (100), and other bytes of the
|
||||
// same length.
|
||||
const good = Buffer.alloc(SIZE_BY_ID[100]!);
|
||||
const corrupt = Buffer.alloc(SIZE_BY_ID[100]!, 1);
|
||||
|
||||
it("leaves an original that matches its hash, and one with no hash, as they are", async () => {
|
||||
const { lib, source, outDir } = await backedUp();
|
||||
const calls = source.originalCalls;
|
||||
|
||||
const result = await lib.backup({
|
||||
downloadDirectory: outDir,
|
||||
verify: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
downloaded: 0,
|
||||
skipped: 3,
|
||||
verified: 2,
|
||||
mismatched: 0,
|
||||
unchecked: 1,
|
||||
failed: 0,
|
||||
});
|
||||
expect(source.originalCalls).toBe(calls);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("downloads again an original that does not match its hash", async () => {
|
||||
const { lib, outDir } = await backedUp();
|
||||
writeFileSync(saved(outDir, "100.jpg"), corrupt);
|
||||
const log: string[] = [];
|
||||
|
||||
const result = await lib.backup({
|
||||
downloadDirectory: outDir,
|
||||
verify: true,
|
||||
onProgress: (msg) => log.push(msg),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
downloaded: 1,
|
||||
skipped: 2,
|
||||
verified: 1,
|
||||
mismatched: 1,
|
||||
unchecked: 1,
|
||||
failed: 0,
|
||||
});
|
||||
expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(good);
|
||||
expect(log.filter((msg) => msg.startsWith("MISMATCH"))).toEqual([
|
||||
"MISMATCH original beach.jpg (100): its bytes do not match its content hash",
|
||||
]);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("records a failed download in failures.json, with the original removed", async () => {
|
||||
const { lib, source, outDir } = await backedUp();
|
||||
writeFileSync(saved(outDir, "100.jpg"), corrupt);
|
||||
source.failID = 100;
|
||||
|
||||
const result = await lib.backup({
|
||||
downloadDirectory: outDir,
|
||||
verify: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
downloaded: 0,
|
||||
mismatched: 1,
|
||||
failed: 1,
|
||||
});
|
||||
expect(Object.keys(readLedger(outDir).files)).toEqual(["100"]);
|
||||
expect(existsSync(saved(outDir, "100.jpg"))).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("hashes nothing without verify", async () => {
|
||||
const { lib, outDir } = await backedUp();
|
||||
writeFileSync(saved(outDir, "100.jpg"), corrupt);
|
||||
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
downloaded: 0,
|
||||
skipped: 3,
|
||||
verified: 0,
|
||||
mismatched: 0,
|
||||
unchecked: 0,
|
||||
failed: 0,
|
||||
});
|
||||
expect(readFileSync(saved(outDir, "100.jpg"))).toEqual(corrupt);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
// Root ignores file permissions, so this fails when run as root. The
|
||||
// `test` phase of the `Dockerfile` runs as the `node` user.
|
||||
it("records an original it cannot read as failed, and leaves it", async () => {
|
||||
const { lib, outDir } = await backedUp();
|
||||
const original = saved(outDir, "100.jpg");
|
||||
chmodSync(original, 0o000);
|
||||
|
||||
const result = await lib
|
||||
.backup({ downloadDirectory: outDir, verify: true })
|
||||
.finally(() => chmodSync(original, 0o600));
|
||||
|
||||
expect(result).toMatchObject({ downloaded: 0, verified: 1, failed: 1 });
|
||||
expect(result.errors.map((e) => e.fileID)).toEqual([100]);
|
||||
expect(result.errors[0]!.error).toMatch(/EACCES/);
|
||||
expect(readFileSync(original)).toEqual(good);
|
||||
await lib.close();
|
||||
});
|
||||
});
|
||||
|
||||
// Every entry under collections/, one level of directories deep, with each
|
||||
// symlink's target.
|
||||
const tree = (outDir: string): string[] => {
|
||||
@@ -1388,9 +871,6 @@ describe("backup album folders", () => {
|
||||
thumbnail: async () => {
|
||||
throw new Error("no thumbnails in this stand-in");
|
||||
},
|
||||
fetchMLData: async () => {},
|
||||
mlData: async () => undefined,
|
||||
whoami: () => ({ email: "backup@example.com", userID: USER_ID }),
|
||||
});
|
||||
|
||||
const albumID = (outDir: string, jsonName: string): number =>
|
||||
@@ -1840,64 +1320,6 @@ describe("backup of live photos", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("verifies a live photo's image and video together, and downloads it again when one does not match", async () => {
|
||||
const { file: live, body } = await asLivePhoto(
|
||||
file(500, 10, "IMG_0500.HEIC"),
|
||||
);
|
||||
const lib = await open([live], new Map([[500, body]]));
|
||||
const outDir = join(root, "backup");
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
const good = await lib.backup({
|
||||
downloadDirectory: outDir,
|
||||
verify: true,
|
||||
});
|
||||
expect(good).toMatchObject({ skipped: 1, verified: 1, mismatched: 0 });
|
||||
|
||||
const video = saved(outDir, "500.mov");
|
||||
writeFileSync(video, "another few seconds of video");
|
||||
const bad = await lib.backup({
|
||||
downloadDirectory: outDir,
|
||||
verify: true,
|
||||
});
|
||||
|
||||
expect(bad).toMatchObject({
|
||||
downloaded: 1,
|
||||
verified: 0,
|
||||
mismatched: 1,
|
||||
failed: 0,
|
||||
});
|
||||
expect(readFileSync(saved(outDir, "500.heic"))).toEqual(
|
||||
Buffer.from(IMAGE),
|
||||
);
|
||||
expect(readFileSync(video)).toEqual(Buffer.from(VIDEO));
|
||||
expect(tree(outDir)).toEqual(linked);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("removes both files of a live photo that does not match its hash when downloading it again fails", async () => {
|
||||
const { file: live, body } = await asLivePhoto(
|
||||
file(500, 10, "IMG_0500.HEIC"),
|
||||
);
|
||||
const bodies = new Map([[500, body]]);
|
||||
const lib = await open([live], bodies);
|
||||
const outDir = join(root, "backup");
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
writeFileSync(saved(outDir, "500.mov"), "another few seconds of video");
|
||||
bodies.delete(500);
|
||||
|
||||
const result = await lib.backup({
|
||||
downloadDirectory: outDir,
|
||||
verify: true,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ mismatched: 1, failed: 1 });
|
||||
expect(Object.keys(readLedger(outDir).files)).toEqual(["500"]);
|
||||
expect(existsSync(saved(outDir, "500.heic"))).toBe(false);
|
||||
expect(existsSync(saved(outDir, "500.mov"))).toBe(false);
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("serves a live photo the backup stored to a library reading the backup", async () => {
|
||||
const { file: live, body } = await asLivePhoto(
|
||||
file(500, 10, "IMG_0500.HEIC"),
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
/**
|
||||
* Tests for the retry options `bin/quak.ts` loads the saved session with:
|
||||
* `quak backup` gets the unattended ones, every other command the default.
|
||||
*
|
||||
* Each test runs `bin/quak.ts`, as the smoke test does, with its session loader
|
||||
* replaced by one that records the options it is given and reports no session,
|
||||
* so the command stops before it makes any request.
|
||||
*/
|
||||
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ApiClientOptions } from "../../src/api/client.js";
|
||||
import { UNATTENDED_RETRY_OPTIONS } from "../../src/retry.js";
|
||||
|
||||
const loaded = vi.hoisted(() => [] as (ApiClientOptions | undefined)[]);
|
||||
|
||||
vi.mock("../../src/cli-session.js", () => ({
|
||||
loadSession: (_path: string, apiOptions?: ApiClientOptions) => {
|
||||
loaded.push(apiOptions);
|
||||
return null;
|
||||
},
|
||||
}));
|
||||
|
||||
const argv = process.argv;
|
||||
const dir = mkdtempSync(join(tmpdir(), "quak-bin-test-"));
|
||||
|
||||
// Run `quak <args>` to completion. The "Not logged in" message and the exit
|
||||
// are swallowed.
|
||||
const quak = async (...args: string[]): Promise<void> => {
|
||||
vi.spyOn(process.stderr, "write").mockImplementation(() => true);
|
||||
vi.spyOn(process, "exit").mockImplementation(() => undefined as never);
|
||||
process.argv = ["node", "quak", ...args];
|
||||
vi.resetModules();
|
||||
await import("../../bin/quak.js");
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
process.argv = argv;
|
||||
loaded.length = 0;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("bin/quak.ts session loading", () => {
|
||||
it("loads the session for backup with the unattended retry options", async () => {
|
||||
await quak("backup", dir);
|
||||
// `vi.resetModules()` gave `bin/quak.ts` its own copy of
|
||||
// `src/retry.ts`, whose `sleep` is a different function, so the
|
||||
// options are compared by their numbers.
|
||||
const { attempts, baseDelayMs, maxDelayMs } = UNATTENDED_RETRY_OPTIONS;
|
||||
expect(loaded).toEqual([
|
||||
{
|
||||
retry: expect.objectContaining({
|
||||
attempts,
|
||||
baseDelayMs,
|
||||
maxDelayMs,
|
||||
}),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("loads the session for another command with the default options", async () => {
|
||||
await quak("collections");
|
||||
expect(loaded).toEqual([undefined]);
|
||||
});
|
||||
});
|
||||
@@ -52,14 +52,13 @@ import {
|
||||
import { run } from "../../src/cli-run.js";
|
||||
import { loadSession } from "../../src/cli-session.js";
|
||||
import type { Client, ClientSnapshot, LoginOptions } from "../../src/client.js";
|
||||
import { savePath, type ContentSource } from "../../src/library/content.js";
|
||||
import type { ContentSource } from "../../src/library/content.js";
|
||||
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||
import { init, toBase64 } from "../../src/crypto/index.js";
|
||||
import { defaultCacheDirectory } from "../../src/library/index.js";
|
||||
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
||||
import {
|
||||
asLivePhoto,
|
||||
blake2b,
|
||||
cdnSource,
|
||||
IMAGE,
|
||||
livePhotoHash,
|
||||
@@ -743,77 +742,6 @@ describe("backup", () => {
|
||||
expect(stderr.text).toBe("Starting backup...\n");
|
||||
});
|
||||
|
||||
it("--verify downloads again an original that does not match its hash, prints the counts, and exits 0", async () => {
|
||||
// Each file records the hash of the original the fake writes for it.
|
||||
const client = {
|
||||
...fakeClient(),
|
||||
filesSince: async (args: { collectionID: number }) => ({
|
||||
files: (FILES[args.collectionID] ?? []).map((f) => ({
|
||||
...f,
|
||||
metadata: {
|
||||
...f.metadata,
|
||||
hash: blake2b(Buffer.alloc(7, f.id & 0xff)),
|
||||
},
|
||||
})),
|
||||
deleted: [],
|
||||
cursor: 1,
|
||||
}),
|
||||
} as unknown as Client;
|
||||
const ctx = context(client);
|
||||
const dir = join(root, "backup");
|
||||
expect(await backupCommand(ctx, dir, {})).toBe(0);
|
||||
writeFileSync(savePath(dir, FILES[1]![0]!), "corrupt");
|
||||
|
||||
expect(await backupCommand(ctx, dir, { verify: true })).toBe(0);
|
||||
|
||||
expect(stderr.text).toContain(
|
||||
"MISMATCH original beach.jpg (100): its bytes do not match its content hash\n",
|
||||
);
|
||||
expect(stderr.text).toContain(
|
||||
" Downloaded: 1\n" +
|
||||
" Skipped: 2\n" +
|
||||
" Verified: 2\n" +
|
||||
" Mismatched: 1\n" +
|
||||
" Unchecked: 0\n" +
|
||||
" Failed: 0\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("--verify --json adds the verified, mismatched and unchecked counts", async () => {
|
||||
const dir = join(root, "backup");
|
||||
expect(await backupCommand(context(), dir, {})).toBe(0);
|
||||
|
||||
const code = await backupCommand(context(), dir, {
|
||||
verify: true,
|
||||
json: true,
|
||||
});
|
||||
|
||||
expect(code).toBe(0);
|
||||
expect(JSON.parse(stdout.text)).toMatchObject({
|
||||
skipped: 3,
|
||||
verified: 0,
|
||||
mismatched: 0,
|
||||
unchecked: 3,
|
||||
failed: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("exits 1 and lists each file when the ML data fetch fails", async () => {
|
||||
const client = {
|
||||
...fakeClient(),
|
||||
fetchMLData: async () => {
|
||||
throw new Error("HTTP 503 from server");
|
||||
},
|
||||
} as unknown as Client;
|
||||
expect(
|
||||
await backupCommand(context(client), join(root, "backup"), {}),
|
||||
).toBe(1);
|
||||
expect(stderr.text).toContain(" Failed: 3\n");
|
||||
expect(stderr.text).toContain(
|
||||
" [Vacation] beach.jpg (id 100): ML data: HTTP 503 from server\n",
|
||||
);
|
||||
});
|
||||
|
||||
// Runs `backup` through `run`, as `bin/quak.ts` does, which prints a thrown
|
||||
// error; returns the exit code and what `run` printed.
|
||||
const backupThroughRun = async (
|
||||
|
||||
@@ -12,10 +12,6 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { init, toBase64 } from "../../src/crypto/index.js";
|
||||
import { Client, type ClientSnapshot } from "../../src/client.js";
|
||||
import { loadSession } from "../../src/cli-session.js";
|
||||
import {
|
||||
DEFAULT_RETRY_OPTIONS,
|
||||
UNATTENDED_RETRY_OPTIONS,
|
||||
} from "../../src/retry.js";
|
||||
|
||||
const validSnapshot = (): ClientSnapshot => {
|
||||
const kp = sodium.crypto_box_keypair();
|
||||
@@ -180,20 +176,6 @@ describe("loadSession", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("gives the restored client the retry options it is passed", () => {
|
||||
const path = join(dir, "retry.json");
|
||||
writeFileSync(path, JSON.stringify(validSnapshot()));
|
||||
const retryOptions = (client: Client | null) =>
|
||||
client!.getApiClient().getRetryOptions();
|
||||
|
||||
expect(
|
||||
retryOptions(
|
||||
loadSession(path, { retry: UNATTENDED_RETRY_OPTIONS }),
|
||||
),
|
||||
).toEqual(UNATTENDED_RETRY_OPTIONS);
|
||||
expect(retryOptions(loadSession(path))).toEqual(DEFAULT_RETRY_OPTIONS);
|
||||
});
|
||||
|
||||
it("says the file is corrupt when it is not JSON", () => {
|
||||
const path = join(dir, "truncated.json");
|
||||
writeFileSync(path, '{"email": "user@exa');
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* `exif.heic`, beside this file: a real 64x64 HEIC whose EXIF holds the same
|
||||
* values as the hand-built JPEG in `exif-jpeg.ts`, for the tests of `exif()`,
|
||||
* `backup-metadata --exif` and the image metadata `quak backup` records.
|
||||
* values as the hand-built JPEG in `library/content-library.test.ts`, for the
|
||||
* tests of `exif()` and `backup-metadata --exif`.
|
||||
*
|
||||
* It was made once, in a throwaway node:22-alpine container (Alpine 3.23.3),
|
||||
* with libheif 1.23.0 and exiftool 13.55:
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
/**
|
||||
* Hand-built JPEGs for the tests of `exif()` and of the image metadata
|
||||
* `quak backup` records: one whose EXIF holds the same values as `exif.heic`
|
||||
* (see `exif-heic.ts`), and one whose EXIF cannot be parsed.
|
||||
*/
|
||||
|
||||
// Big-endian bytes for the hand-built JPEG below.
|
||||
const u16 = (n: number): number[] => [n >> 8, n & 0xff];
|
||||
const u32 = (n: number): number[] => [...u16(n >>> 16), ...u16(n & 0xffff)];
|
||||
const ascii = (s: string): number[] => [...new TextEncoder().encode(s), 0];
|
||||
const rational = (num: number, den: number): number[] => [
|
||||
...u32(num),
|
||||
...u32(den),
|
||||
];
|
||||
// One IFD entry: tag, type (1 BYTE, 2 ASCII, 3 SHORT, 4 LONG, 5 RATIONAL),
|
||||
// count, then the value when it fits in 4 bytes, else its offset.
|
||||
const entry = (
|
||||
tag: number,
|
||||
type: number,
|
||||
count: number,
|
||||
value: number[],
|
||||
): number[] => [...u16(tag), ...u16(type), ...u32(count), ...value];
|
||||
|
||||
// The TIFF block of a JPEG's EXIF segment, holding every field `Photo`'s typed
|
||||
// methods return: the camera in the first IFD, the exposure in the Exif IFD,
|
||||
// and a GPS position of 40°26'46" N, 79°58'56" W, 12.5 m below sea level.
|
||||
// Offsets count from the start of this block.
|
||||
const TIFF = [
|
||||
...[0x4d, 0x4d, 0x00, 0x2a], // big-endian TIFF
|
||||
...u32(8), // the first IFD's offset
|
||||
// The first IFD, at 8: five entries, then no next IFD.
|
||||
...u16(5),
|
||||
...entry(0x010f, 2, 6, u32(74)), // Make
|
||||
...entry(0x0110, 2, 7, u32(80)), // Model
|
||||
...entry(0x0112, 3, 1, [...u16(6), 0, 0]), // Orientation
|
||||
...entry(0x8769, 4, 1, u32(88)), // the Exif IFD's offset
|
||||
...entry(0x8825, 4, 1, u32(246)), // the GPS IFD's offset
|
||||
...u32(0),
|
||||
...ascii("Canon"), // at 74
|
||||
...ascii("EOS R5"), // at 80
|
||||
0, // a pad byte
|
||||
// The Exif IFD, at 88: seven entries, then no next IFD.
|
||||
...u16(7),
|
||||
...entry(0x829a, 5, 1, u32(178)), // ExposureTime
|
||||
...entry(0x829d, 5, 1, u32(186)), // FNumber
|
||||
...entry(0x8827, 3, 1, [...u16(400), 0, 0]), // ISOSpeedRatings
|
||||
...entry(0x9003, 2, 20, u32(194)), // DateTimeOriginal
|
||||
...entry(0x9011, 2, 7, u32(214)), // OffsetTimeOriginal
|
||||
...entry(0x920a, 5, 1, u32(222)), // FocalLength
|
||||
...entry(0xa434, 2, 16, u32(230)), // LensModel
|
||||
...u32(0),
|
||||
...rational(1, 250), // at 178
|
||||
...rational(28, 10), // at 186
|
||||
...ascii("2021:07:15 14:30:00"), // at 194
|
||||
...ascii("+02:00"), // at 214
|
||||
0, // a pad byte
|
||||
...rational(50, 1), // at 222
|
||||
...ascii("RF50mm F1.8 STM"), // at 230
|
||||
// The GPS IFD, at 246: six entries, then no next IFD.
|
||||
...u16(6),
|
||||
...entry(0x0001, 2, 2, [...ascii("N"), 0, 0]), // GPSLatitudeRef
|
||||
...entry(0x0002, 5, 3, u32(324)), // GPSLatitude
|
||||
...entry(0x0003, 2, 2, [...ascii("W"), 0, 0]), // GPSLongitudeRef
|
||||
...entry(0x0004, 5, 3, u32(348)), // GPSLongitude
|
||||
...entry(0x0005, 1, 1, [1, 0, 0, 0]), // GPSAltitudeRef: below sea level
|
||||
...entry(0x0006, 5, 1, u32(372)), // GPSAltitude
|
||||
...u32(0),
|
||||
...[...rational(40, 1), ...rational(26, 1), ...rational(46, 1)], // at 324
|
||||
...[...rational(79, 1), ...rational(58, 1), ...rational(56, 1)], // at 348
|
||||
...rational(25, 2), // at 372
|
||||
];
|
||||
|
||||
export const JPEG_WITH_EXIF = new Uint8Array([
|
||||
...[0xff, 0xd8], // start of image
|
||||
...[0xff, 0xe1, ...u16(2 + 6 + TIFF.length)], // APP1 and its length
|
||||
...[...ascii("Exif"), 0], // "Exif\0\0"
|
||||
...TIFF,
|
||||
...[0xff, 0xda, 0x00, 0x02], // start of scan
|
||||
]);
|
||||
|
||||
// A JPEG whose EXIF segment is laid out correctly but holds "XX" where the TIFF
|
||||
// byte order belongs, so exifreader cannot parse it.
|
||||
export const JPEG_WITH_BAD_EXIF = new Uint8Array([
|
||||
...[0xff, 0xd8], // start of image
|
||||
...[0xff, 0xe1, ...u16(2 + 6 + 2)], // APP1 and its length
|
||||
...[...ascii("Exif"), 0], // "Exif\0\0"
|
||||
...[0x58, 0x58], // "XX"
|
||||
...[0xff, 0xda, 0x00, 0x02], // start of scan
|
||||
]);
|
||||
@@ -29,7 +29,6 @@ import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
||||
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||
import { readPhotoExif, type PhotoExif } from "../../src/exif.js";
|
||||
import { HEIC_WITH_EXIF } from "../exif-heic.js";
|
||||
import { JPEG_WITH_BAD_EXIF, JPEG_WITH_EXIF } from "../exif-jpeg.js";
|
||||
import {
|
||||
asLivePhoto,
|
||||
cdnSource,
|
||||
@@ -257,6 +256,90 @@ describe("Library content wiring", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// Big-endian bytes for the hand-built JPEG below.
|
||||
const u16 = (n: number): number[] => [n >> 8, n & 0xff];
|
||||
const u32 = (n: number): number[] => [...u16(n >>> 16), ...u16(n & 0xffff)];
|
||||
const ascii = (s: string): number[] => [...new TextEncoder().encode(s), 0];
|
||||
const rational = (num: number, den: number): number[] => [
|
||||
...u32(num),
|
||||
...u32(den),
|
||||
];
|
||||
// One IFD entry: tag, type (1 BYTE, 2 ASCII, 3 SHORT, 4 LONG, 5 RATIONAL),
|
||||
// count, then the value when it fits in 4 bytes, else its offset.
|
||||
const entry = (
|
||||
tag: number,
|
||||
type: number,
|
||||
count: number,
|
||||
value: number[],
|
||||
): number[] => [...u16(tag), ...u16(type), ...u32(count), ...value];
|
||||
|
||||
// The TIFF block of a JPEG's EXIF segment, holding every field `Photo`'s typed
|
||||
// methods return: the camera in the first IFD, the exposure in the Exif IFD,
|
||||
// and a GPS position of 40°26'46" N, 79°58'56" W, 12.5 m below sea level.
|
||||
// Offsets count from the start of this block.
|
||||
const TIFF = [
|
||||
...[0x4d, 0x4d, 0x00, 0x2a], // big-endian TIFF
|
||||
...u32(8), // the first IFD's offset
|
||||
// The first IFD, at 8: five entries, then no next IFD.
|
||||
...u16(5),
|
||||
...entry(0x010f, 2, 6, u32(74)), // Make
|
||||
...entry(0x0110, 2, 7, u32(80)), // Model
|
||||
...entry(0x0112, 3, 1, [...u16(6), 0, 0]), // Orientation
|
||||
...entry(0x8769, 4, 1, u32(88)), // the Exif IFD's offset
|
||||
...entry(0x8825, 4, 1, u32(246)), // the GPS IFD's offset
|
||||
...u32(0),
|
||||
...ascii("Canon"), // at 74
|
||||
...ascii("EOS R5"), // at 80
|
||||
0, // a pad byte
|
||||
// The Exif IFD, at 88: seven entries, then no next IFD.
|
||||
...u16(7),
|
||||
...entry(0x829a, 5, 1, u32(178)), // ExposureTime
|
||||
...entry(0x829d, 5, 1, u32(186)), // FNumber
|
||||
...entry(0x8827, 3, 1, [...u16(400), 0, 0]), // ISOSpeedRatings
|
||||
...entry(0x9003, 2, 20, u32(194)), // DateTimeOriginal
|
||||
...entry(0x9011, 2, 7, u32(214)), // OffsetTimeOriginal
|
||||
...entry(0x920a, 5, 1, u32(222)), // FocalLength
|
||||
...entry(0xa434, 2, 16, u32(230)), // LensModel
|
||||
...u32(0),
|
||||
...rational(1, 250), // at 178
|
||||
...rational(28, 10), // at 186
|
||||
...ascii("2021:07:15 14:30:00"), // at 194
|
||||
...ascii("+02:00"), // at 214
|
||||
0, // a pad byte
|
||||
...rational(50, 1), // at 222
|
||||
...ascii("RF50mm F1.8 STM"), // at 230
|
||||
// The GPS IFD, at 246: six entries, then no next IFD.
|
||||
...u16(6),
|
||||
...entry(0x0001, 2, 2, [...ascii("N"), 0, 0]), // GPSLatitudeRef
|
||||
...entry(0x0002, 5, 3, u32(324)), // GPSLatitude
|
||||
...entry(0x0003, 2, 2, [...ascii("W"), 0, 0]), // GPSLongitudeRef
|
||||
...entry(0x0004, 5, 3, u32(348)), // GPSLongitude
|
||||
...entry(0x0005, 1, 1, [1, 0, 0, 0]), // GPSAltitudeRef: below sea level
|
||||
...entry(0x0006, 5, 1, u32(372)), // GPSAltitude
|
||||
...u32(0),
|
||||
...[...rational(40, 1), ...rational(26, 1), ...rational(46, 1)], // at 324
|
||||
...[...rational(79, 1), ...rational(58, 1), ...rational(56, 1)], // at 348
|
||||
...rational(25, 2), // at 372
|
||||
];
|
||||
|
||||
const JPEG_WITH_EXIF = new Uint8Array([
|
||||
...[0xff, 0xd8], // start of image
|
||||
...[0xff, 0xe1, ...u16(2 + 6 + TIFF.length)], // APP1 and its length
|
||||
...[...ascii("Exif"), 0], // "Exif\0\0"
|
||||
...TIFF,
|
||||
...[0xff, 0xda, 0x00, 0x02], // start of scan
|
||||
]);
|
||||
|
||||
// A JPEG whose EXIF segment is laid out correctly but holds "XX" where the TIFF
|
||||
// byte order belongs, so exifreader cannot parse it.
|
||||
const JPEG_WITH_BAD_EXIF = new Uint8Array([
|
||||
...[0xff, 0xd8], // start of image
|
||||
...[0xff, 0xe1, ...u16(2 + 6 + 2)], // APP1 and its length
|
||||
...[...ascii("Exif"), 0], // "Exif\0\0"
|
||||
...[0x58, 0x58], // "XX"
|
||||
...[0xff, 0xda, 0x00, 0x02], // start of scan
|
||||
]);
|
||||
|
||||
describe("Photo save path, local copy, content and EXIF", () => {
|
||||
// The same account, with `files` in its album instead.
|
||||
class FilesClient extends MockClient {
|
||||
|
||||
+1
-2
@@ -30,8 +30,7 @@ export const livePhotoZip = (
|
||||
},
|
||||
): Uint8Array => zipSync(entries);
|
||||
|
||||
// The content hash Ente's clients record for an original's bytes.
|
||||
export const blake2b = (bytes: Uint8Array): string =>
|
||||
const blake2b = (bytes: Uint8Array): string =>
|
||||
createHash("blake2b512").update(bytes).digest("base64");
|
||||
|
||||
// The hash Ente's clients record for a live photo: the unkeyed BLAKE2b-512 of
|
||||
|
||||
@@ -29,19 +29,18 @@ const patterns = (name: string): string[] =>
|
||||
const dockerignore = patterns(".dockerignore");
|
||||
|
||||
describe(".dockerignore", () => {
|
||||
// Everything here is either generated, enormous, or secret. `.claude` is
|
||||
// the correctness one: see the header comment and issue #25. The leading
|
||||
// `/` anchors an entry at the root of the context.
|
||||
// Everything here is either generated, enormous, or secret. `.claude/` is
|
||||
// the correctness one: see the header comment and issue #25.
|
||||
it.each([
|
||||
".claude",
|
||||
"/.quak",
|
||||
"/bin/quak",
|
||||
"**/node_modules",
|
||||
"/coverage",
|
||||
"/dist",
|
||||
"/.vitest-cache",
|
||||
"/.nyc_output",
|
||||
"/*.tsbuildinfo",
|
||||
".claude/",
|
||||
".quak/",
|
||||
"bin/quak",
|
||||
"node_modules",
|
||||
"coverage",
|
||||
"dist",
|
||||
".vitest-cache/",
|
||||
".nyc_output/",
|
||||
"*.tsbuildinfo",
|
||||
])("keeps %s out of the build context", (pattern) => {
|
||||
expect(dockerignore).toContain(pattern);
|
||||
});
|
||||
@@ -58,7 +57,7 @@ describe(".dockerignore", () => {
|
||||
// The build stage is the final image, so a .git/config sent in would
|
||||
// ship the clone's remote URL and any credential in it.
|
||||
it("sends .git without its config", () => {
|
||||
expect(dockerignore).toContain("**/.git/config");
|
||||
expect(dockerignore).toContain(".git/config");
|
||||
});
|
||||
|
||||
// BuildKit lets a `Dockerfile.dockerignore` shadow the root one; such a
|
||||
|
||||
@@ -43,7 +43,6 @@ import {
|
||||
isRetryable,
|
||||
isSafeToReplay,
|
||||
resolveRetryOptions,
|
||||
UNATTENDED_RETRY_OPTIONS,
|
||||
withRetry,
|
||||
} from "../../src/retry.js";
|
||||
import { ApiError, TruncatedStreamError } from "../../src/errors.js";
|
||||
@@ -640,33 +639,3 @@ describe("retry defaults", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("unattended retry options", () => {
|
||||
it("allow ten attempts, a 1 s base delay and a 60 s cap", () => {
|
||||
// The numbers the README documents for `quak backup`.
|
||||
expect(UNATTENDED_RETRY_OPTIONS.attempts).toBe(10);
|
||||
expect(UNATTENDED_RETRY_OPTIONS.baseDelayMs).toBe(1_000);
|
||||
expect(UNATTENDED_RETRY_OPTIONS.maxDelayMs).toBe(60_000);
|
||||
});
|
||||
|
||||
it("give up on a request that keeps failing after at most 243 s of waiting", async () => {
|
||||
// `random: () => 1` makes every wait its ceiling, the worst case.
|
||||
const { sleep, delays } = recordingSleep();
|
||||
let calls = 0;
|
||||
await expect(
|
||||
withRetry(
|
||||
() => {
|
||||
calls++;
|
||||
return Promise.reject(new ApiError("HTTP 503", 503));
|
||||
},
|
||||
{ ...UNATTENDED_RETRY_OPTIONS, sleep, random: () => 1 },
|
||||
),
|
||||
).rejects.toThrow("HTTP 503");
|
||||
|
||||
expect(calls).toBe(10);
|
||||
expect(delays).toEqual([
|
||||
1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 60_000,
|
||||
]);
|
||||
expect(delays.reduce((sum, ms) => sum + ms, 0)).toBe(243_000);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user