Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d9852af13 |
No files matched your search
@@ -26,8 +26,10 @@ check:
|
|||||||
build:
|
build:
|
||||||
@script/build
|
@script/build
|
||||||
|
|
||||||
build-bin:
|
# Bundles the built dist/, so the binary reports the version script/build
|
||||||
@script/build-bin
|
# stamped.
|
||||||
|
build-bin: build
|
||||||
|
nix-shell -p bun --run "bun build dist/bin/quak.js --compile --outfile bin/quak"
|
||||||
|
|
||||||
install: build-bin
|
install: build-bin
|
||||||
mkdir -p ~/bin
|
mkdir -p ~/bin
|
||||||
|
|||||||
@@ -31,8 +31,7 @@ again on the next run. Every download is decrypted on its way to disk.
|
|||||||
|
|
||||||
## Getting Started
|
## Getting Started
|
||||||
|
|
||||||
The commands below need `git`, `yarn` and Node 22.13 or later. Node 23 needs
|
The commands below need `git`, Node 22 or later and `yarn`.
|
||||||
23.5 or later.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://git.eeqj.de/sneak/quak.git
|
git clone https://git.eeqj.de/sneak/quak.git
|
||||||
@@ -70,29 +69,12 @@ The library keeps a local copy of the account's album and file lists, the cache.
|
|||||||
It brings the cache up to date with the server in rounds called refreshes.
|
It brings the cache up to date with the server in rounds called refreshes.
|
||||||
|
|
||||||
quak is not published to a package registry: `package.json` marks it `private`,
|
quak is not published to a package registry: `package.json` marks it `private`,
|
||||||
and `dist/` is not committed. Clone and build it as above, then link the clone
|
and `dist/` is not committed. Clone and build it as above, then add the clone to
|
||||||
into your own program:
|
your own program's `package.json` by its path, for example
|
||||||
|
`yarn add file:../quak`. The package's `main`, `types` and `exports` fields
|
||||||
```bash
|
point into `dist/`, so the clone must be built first. The package is an ES
|
||||||
# In the quak clone:
|
module only: `package.json` sets `"type": "module"`, and its `exports` field
|
||||||
yarn link
|
offers only `import`, so a CommonJS program cannot `require` it.
|
||||||
|
|
||||||
# In your program's directory:
|
|
||||||
yarn link quak
|
|
||||||
```
|
|
||||||
|
|
||||||
`yarn link` also installs the `quak` command in yarn's global bin directory, and
|
|
||||||
`yarn unlink` in the clone removes the link and the command.
|
|
||||||
|
|
||||||
A linked quak uses the dependencies that `yarn install` put in the clone, at the
|
|
||||||
versions in quak's `yarn.lock`. Avoid `yarn add file:../quak`: it resolves
|
|
||||||
quak's dependencies again, and their newer releases can need a newer Node.
|
|
||||||
`yarn link quak` does not add quak to your `package.json`, so run it again in
|
|
||||||
each new checkout of your program.
|
|
||||||
|
|
||||||
The package's `main`, `types` and `exports` fields point into `dist/`, so the
|
|
||||||
clone must be built first. The package is an ES module only, so a CommonJS
|
|
||||||
program cannot `require` it.
|
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { Client, Library } from "quak";
|
import { Client, Library } from "quak";
|
||||||
@@ -108,9 +90,8 @@ const client = await Client.login({
|
|||||||
const lib = await Library.open({
|
const lib = await Library.open({
|
||||||
client,
|
client,
|
||||||
// Without these two, open() also starts downloading every thumbnail in
|
// Without these two, open() also starts downloading every thumbnail in
|
||||||
// the account, and the originals of the favorites album and the latest
|
// the account and the latest week's originals into the cache. It fetches
|
||||||
// week, into the cache. It fetches machine-learning data after each
|
// machine-learning data after each refresh either way.
|
||||||
// refresh either way.
|
|
||||||
precacheThumbnails: false,
|
precacheThumbnails: false,
|
||||||
precacheOriginals: false,
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
@@ -141,12 +122,10 @@ await lib.close();
|
|||||||
for. It refreshes every 3 seconds until `close()`. It also downloads, in the
|
for. It refreshes every 3 seconds until `close()`. It also downloads, in the
|
||||||
background, every thumbnail in the account and the originals of the favorites
|
background, every thumbnail in the account and the originals of the favorites
|
||||||
album and of the latest week (see "Opening a library"). The example turns those
|
album and of the latest week (see "Opening a library"). The example turns those
|
||||||
two downloads off. After each refresh that succeeds, the library also fetches
|
two downloads off. After each refresh the library also fetches each file's
|
||||||
each file's machine-learning data into the cache. That is the faces and the
|
machine-learning data into the cache. That is the faces and the other data
|
||||||
other data Ente's apps compute for a photo (see "The JSON beside each
|
Ente's apps compute for a photo (see "The JSON beside each original"). No option
|
||||||
original"). No option in `LibraryOptions` turns this off. A `client` without a
|
turns this off.
|
||||||
`fetchMLData` method fetches none, which is how quak's own read commands open
|
|
||||||
their library.
|
|
||||||
|
|
||||||
The `Client` under the library handles login, saved sessions and the raw
|
The `Client` under the library handles login, saved sessions and the raw
|
||||||
requests. The "Library reference" below covers both.
|
requests. The "Library reference" below covers both.
|
||||||
@@ -154,16 +133,10 @@ requests. The "Library reference" below covers both.
|
|||||||
## Running the backup from cron
|
## Running the backup from cron
|
||||||
|
|
||||||
`quak backup` never prompts, so cron can run it. `make install` builds quak as
|
`quak backup` never prompts, so cron can run it. `make install` builds quak as
|
||||||
one binary and copies it to `~/bin/quak`. It uses bun, a JavaScript runtime, to
|
one binary and copies it to `~/bin/quak`. Besides what Getting Started needs,
|
||||||
bundle the built `dist/` into one executable. Besides what Getting Started
|
`make install` needs [nix](https://nixos.org/download). It uses bun, a
|
||||||
needs, `make install` needs `make`. On Linux it downloads bun from GitHub with
|
JavaScript runtime, to bundle the built `dist/` into one executable. It runs bun
|
||||||
`curl` and unpacks it with `unzip`. It checks the download against a SHA-256
|
through `nix-shell -p bun`.
|
||||||
hash in the repo. On macOS it runs bun through `nix-shell -p bun`, so it needs
|
|
||||||
[nix](https://nixos.org/download).
|
|
||||||
|
|
||||||
On Linux systems that use the musl C library, such as Alpine, `make install`
|
|
||||||
downloads bun's musl build. On Alpine, both that bun and `~/bin/quak` need the
|
|
||||||
`libstdc++` and `libgcc` packages.
|
|
||||||
|
|
||||||
Log in once with `~/bin/quak`, as the user the cron job will run as:
|
Log in once with `~/bin/quak`, as the user the cron job will run as:
|
||||||
|
|
||||||
@@ -183,10 +156,9 @@ output to `~/quak-backup.log`. Sunday's run adds `--verify`:
|
|||||||
|
|
||||||
A run with `--verify` does all a plain run does. It also hashes each original
|
A run with `--verify` does all a plain run does. It also hashes each original
|
||||||
already in the backup and compares the result with the content hash. The content
|
already in the backup and compares the result with the content hash. The content
|
||||||
hash is the hash of the file's bytes that Ente recorded at upload. The run
|
hash is the hash of the file's bytes that Ente recorded at upload. It replaces
|
||||||
fetches a new copy of each original that does not match, and replaces the
|
any original that does not match. Sunday's run is the `--verify` run, rather
|
||||||
original once the copy matches. Sunday's run is the `--verify` run, rather than
|
than a second job that night, because two backups of one directory cannot run at
|
||||||
a second job that night, because two backups of one directory cannot run at
|
|
||||||
once. The second one exits 2 without doing anything.
|
once. The second one exits 2 without doing anything.
|
||||||
|
|
||||||
Cron runs the job with a short `PATH`, usually `/usr/bin:/bin`, so the crontab
|
Cron runs the job with a short `PATH`, usually `/usr/bin:/bin`, so the crontab
|
||||||
@@ -255,12 +227,9 @@ set, it asks on the terminal. The password shows as `*` characters while it is
|
|||||||
typed.
|
typed.
|
||||||
|
|
||||||
When the account has a second factor, quak asks for the code. It asks
|
When the account has a second factor, quak asks for the code. It asks
|
||||||
`TOTP code: ` for a time-based code from an authenticator app. An account whose
|
`TOTP code: ` for a time-based code from an authenticator app, and
|
||||||
only second factor is a passkey cannot log in with quak. When the account has
|
`Email verification code: ` for a code sent by email. An account whose only
|
||||||
email verification on, quak asks `Email verification code: ` for the code sent
|
second factor is a passkey cannot log in with quak.
|
||||||
by email. The password is then used only to unlock the keys. After the email
|
|
||||||
code quak expects the login to complete. When the server asks for a second
|
|
||||||
factor at that point, login fails.
|
|
||||||
|
|
||||||
On success it logs `Logged in as <email> (user <id>)` and
|
On success it logs `Logged in as <email> (user <id>)` and
|
||||||
`Session saved to <path>`. On failure it logs `Login failed: <reason>`, and the
|
`Session saved to <path>`. On failure it logs `Login failed: <reason>`, and the
|
||||||
@@ -270,9 +239,7 @@ to stdout.
|
|||||||
#### `quak whoami`
|
#### `quak whoami`
|
||||||
|
|
||||||
Prints the saved session's account on stdout as one line of JSON:
|
Prints the saved session's account on stdout as one line of JSON:
|
||||||
`{"email":"you@example.com","userID":12345}`. It reads the session file only and
|
`{"email":"you@example.com","userID":12345}`.
|
||||||
sends no request, so it succeeds even when the server no longer accepts the
|
|
||||||
session.
|
|
||||||
|
|
||||||
#### `quak logout`
|
#### `quak logout`
|
||||||
|
|
||||||
@@ -327,7 +294,7 @@ title, with unsafe characters replaced (see "File names from the server"). With
|
|||||||
original is kept in the cache as well.
|
original is kept in the cache as well.
|
||||||
|
|
||||||
A file ID that no collection holds gives `File <id> not found`, and the exit
|
A file ID that no collection holds gives `File <id> not found`, and the exit
|
||||||
status is 1.
|
status is 1. The option `--collection <id>` is accepted and ignored.
|
||||||
|
|
||||||
A live photo is a still image and a short video that Ente stores as one file.
|
A live photo is a still image and a short video that Ente stores as one file.
|
||||||
`get` writes both, each named after the title with its own extension, as Ente's
|
`get` writes both, each named after the title with its own extension, as Ente's
|
||||||
@@ -339,16 +306,16 @@ refused, because both files would land on it.
|
|||||||
#### `quak get-thumb <fileID> [--out <path>]`
|
#### `quak get-thumb <fileID> [--out <path>]`
|
||||||
|
|
||||||
Downloads and decrypts one file's thumbnail, the small JPEG Ente shows in a
|
Downloads and decrypts one file's thumbnail, the small JPEG Ente shows in a
|
||||||
grid. It writes one file, also for a live photo. Without `--out`, the name is
|
grid. It works as `get` does. Without `--out`, the name is `thumb_` followed by
|
||||||
`thumb_` followed by the title. In all else it works as `get` does.
|
the title.
|
||||||
|
|
||||||
#### `quak backup <dir> [--json] [--verify]`
|
#### `quak backup <dir> [--json] [--verify]`
|
||||||
|
|
||||||
Backs up every file in the account to `<dir>`. The layout is described under
|
Backs up every file in the account to `<dir>`. The layout is described under
|
||||||
"The backup directory". The command downloads only the originals that are not
|
"The backup directory". It downloads only the originals that are not yet in the
|
||||||
yet in the backup, and it goes on past a file that fails. A file is downloaded
|
backup, and it goes on past a file that fails. A file is downloaded once,
|
||||||
once, however many collections hold it. An original the cache already holds is
|
however many collections hold it. An original the cache already holds is copied
|
||||||
copied from there instead.
|
from there instead.
|
||||||
|
|
||||||
Progress goes to stderr: one line per original fetched, and a `FAILED` line for
|
Progress goes to stderr: one line per original fetched, and a `FAILED` line for
|
||||||
each original or album that failed. The run ends with a summary of counts and,
|
each original or album that failed. The run ends with a summary of counts and,
|
||||||
@@ -387,8 +354,7 @@ Writes the metadata quak keeps about the account, its collections and its files
|
|||||||
as JSON files under `<dir>`. The layout is described under "The metadata dump
|
as JSON files under `<dir>`. The layout is described under "The metadata dump
|
||||||
directory". Photo content is not written. With `--exif`, or its alias `--all`,
|
directory". Photo content is not written. With `--exif`, or its alias `--all`,
|
||||||
the command also fetches each file's original and records the image's own
|
the command also fetches each file's original and records the image's own
|
||||||
metadata from it (see "The metadata dump directory"). Each original it fetches
|
metadata from it (see "The metadata dump directory").
|
||||||
stays in the cache.
|
|
||||||
|
|
||||||
Ente's apps compute machine-learning data for each photo, such as the faces in
|
Ente's apps compute machine-learning data for each photo, such as the faces in
|
||||||
it (see "The JSON beside each original"). The command fetches it in requests of
|
it (see "The JSON beside each original"). The command fetches it in requests of
|
||||||
@@ -403,13 +369,12 @@ word, and does not change the exit status.
|
|||||||
|
|
||||||
#### `quak helper list-missing-thumbnails [--json]`
|
#### `quak helper list-missing-thumbnails [--json]`
|
||||||
|
|
||||||
Finds the files whose thumbnail is missing on the server. It downloads the
|
Finds the files whose thumbnail is missing on the server. It downloads every
|
||||||
thumbnail of every file you own, one collection at a time, newest updated first,
|
thumbnail in the account, one collection at a time, newest collection first, so
|
||||||
so on a large account it takes a while. A thumbnail counts as missing when the
|
on a large account it takes a while. A thumbnail counts as missing when the
|
||||||
server answers with an empty body or with HTTP 404. Any other failure is logged
|
server answers with an empty body or with HTTP 404. Any other failure is logged
|
||||||
as `[<collection>] Could not check <title>: <reason> (not reported as missing)`,
|
as `[<collection>] Could not check <title>: <reason> (not reported as missing)`,
|
||||||
and the file is left out of the list. The command exits 0 even when some files
|
and the file is left out of the list.
|
||||||
could not be checked.
|
|
||||||
|
|
||||||
The server lets only a file's owner replace its thumbnail. A file another
|
The server lets only a file's owner replace its thumbnail. A file another
|
||||||
account owns, in a collection shared with you, is skipped and not checked.
|
account owns, in a collection shared with you, is skipped and not checked.
|
||||||
@@ -452,13 +417,12 @@ Each file ends in one of three states:
|
|||||||
`--file` is held by no collection.
|
`--file` is held by no collection.
|
||||||
|
|
||||||
Without `--json`, the command ends with counts of fixed, skipped and failed
|
Without `--json`, the command ends with counts of fixed, skipped and failed
|
||||||
files. A `Skipped:` list and a `Failed files:` list follow when they have
|
files, a `Skipped:` list and a `Failed files:` list. With `--json`, one JSON
|
||||||
entries. With `--json`, one JSON array on stdout replaces them; each object
|
array on stdout replaces them; each object holds `fileID`, `title`,
|
||||||
holds `fileID`, `title`, `collection`, `status` and, for a skipped or failed
|
`collection`, `status` and, for a skipped or failed file, `reason`. When any
|
||||||
file, `reason`. When any file failed, the exit status is 1. A skipped file does
|
file failed, the exit status is 1. A skipped file does not change it. When
|
||||||
not change it. When nothing needs fixing, the command logs
|
nothing needs fixing, the command logs `No missing thumbnails found.` and
|
||||||
`No missing thumbnails found.` and exits 0. With `--json` it then writes nothing
|
exits 0. With `--json` it then writes nothing on stdout.
|
||||||
on stdout.
|
|
||||||
|
|
||||||
quak refuses the whole command, before any request, when one `--file` ID is
|
quak refuses the whole command, before any request, when one `--file` ID is
|
||||||
invalid (see "IDs").
|
invalid (see "IDs").
|
||||||
@@ -489,9 +453,9 @@ collection fetched, such as `Fetched the files of 3 of 120 collections`.
|
|||||||
|
|
||||||
When a refresh or a search fails, the command logs the error on one line and
|
When a refresh or a search fails, the command logs the error on one line and
|
||||||
exits 1, or 3 when the server no longer accepts the saved session. A collection
|
exits 1, or 3 when the server no longer accepts the saved session. A collection
|
||||||
or file that quak cannot decrypt does not stop a refresh or a search: the
|
or file that quak cannot decrypt does not stop a refresh: the command answers
|
||||||
command leaves it out and answers from what did decrypt (see "Default reads and
|
from what did decrypt (see "Default reads and fresh reads"). `backup` counts it
|
||||||
fresh reads"). `backup` counts it as failed (see "What quak cannot decrypt").
|
as failed (see "What quak cannot decrypt").
|
||||||
|
|
||||||
### IDs
|
### IDs
|
||||||
|
|
||||||
@@ -503,11 +467,20 @@ before any request is sent. The command logs `Invalid file ID: <arg>` or
|
|||||||
|
|
||||||
### What quak prints
|
### What quak prints
|
||||||
|
|
||||||
stdout holds a command's output, such as the listing or the JSON, and the
|
stdout holds only a command's output: the listing, the JSON, or nothing. The
|
||||||
prompts of `login`. Every other line goes to stderr as a log line at one of six
|
prompts of `login` go to stdout too. Every other line goes to stderr as a log
|
||||||
levels. From the quietest, the levels are fatal, error, warning, info, debug and
|
line at one of six levels. From the quietest, the levels are fatal, error,
|
||||||
trace. A run prints the lines at its own level and at every quieter one. The
|
warning, info, debug and trace. A run prints the lines at its own level and at
|
||||||
level is info unless `-v`, `-vv` or `-q` says otherwise.
|
every quieter one. The level is info unless `-v`, `-vv` or `-q` says otherwise.
|
||||||
|
|
||||||
|
The one exception is a command line quak cannot parse, such as a missing
|
||||||
|
argument, an unknown option or `-q` with `-v`. quak refuses it with one
|
||||||
|
`error: …` line that is not a log line, and exits 1. `-q` does not hide that
|
||||||
|
line. `quak` with no command prints the usage on stderr and exits 1.
|
||||||
|
|
||||||
|
`yarn quak` prints yarn's own lines on stdout as well. To pipe a command's
|
||||||
|
output, run `yarn --silent quak files --json` or
|
||||||
|
`node dist/bin/quak.js files --json` instead.
|
||||||
|
|
||||||
- Fatal: why a command exits non-zero. A command that runs and exits non-zero
|
- Fatal: why a command exits non-zero. A command that runs and exits non-zero
|
||||||
logs at least one fatal line. A command that exits 1 because files or albums
|
logs at least one fatal line. A command that exits 1 because files or albums
|
||||||
@@ -533,22 +506,12 @@ level is info unless `-v`, `-vv` or `-q` says otherwise.
|
|||||||
one that exits non-zero prints only its fatal lines. The prompts of `login` are
|
one that exits non-zero prints only its fatal lines. The prompts of `login` are
|
||||||
not log lines, and `-q` leaves them alone.
|
not log lines, and `-q` leaves them alone.
|
||||||
|
|
||||||
Two things on stderr are not log lines. A command line that quak cannot parse,
|
|
||||||
such as a missing argument, an unknown option or `-q` with `-v`, is refused with
|
|
||||||
an `error: …` message, and quak exits 1. `-q` does not hide the message. `quak`
|
|
||||||
with no command prints the usage on stderr and exits 1.
|
|
||||||
|
|
||||||
`yarn quak` prints yarn's own lines on stdout as well. To pipe a command's
|
|
||||||
output, run `yarn --silent quak files --json` or
|
|
||||||
`node dist/bin/quak.js files --json` instead.
|
|
||||||
|
|
||||||
With `--json`, a command's JSON on stdout takes the place of its summary on
|
With `--json`, a command's JSON on stdout takes the place of its summary on
|
||||||
stderr. For `backup`, that is the `--- Backup complete ---` line, the counts and
|
stderr. For `backup`, that is the lines from `--- Backup complete ---` to the
|
||||||
the `Failed files:` list. For `list-missing-thumbnails`, it is
|
end of the `Failed files:` list. For `list-missing-thumbnails`, it is
|
||||||
`No missing thumbnails found.` or `<n> file(s) with missing thumbnails:`. For
|
`No missing thumbnails found.` or `<n> file(s) with missing thumbnails:`. For
|
||||||
`fix-missing-thumbnails`, it is the `--- Done ---` line, the counts, the
|
`fix-missing-thumbnails`, it is the lines from `--- Done ---` to the end of the
|
||||||
`Skipped:` list and the `Failed files:` list. Every other line is logged as
|
`Failed files:` list. Every other line is logged as without `--json`.
|
||||||
without `--json`.
|
|
||||||
|
|
||||||
### Exit codes
|
### Exit codes
|
||||||
|
|
||||||
@@ -696,9 +659,6 @@ but letters and digits, or when it is longer than 16 characters.
|
|||||||
When the date or the time zone changes, the next run stores the original at its
|
When the date or the time zone changes, the next run stores the original at its
|
||||||
new path and leaves the old copy where it is.
|
new path and leaves the old copy where it is.
|
||||||
|
|
||||||
A file deleted in Ente keeps its original at its save path, and the JSON beside
|
|
||||||
it. The next run removes its symlinks from the collection directories.
|
|
||||||
|
|
||||||
The library gives the same path as `photo.savePath` and writes it with
|
The library gives the same path as `photo.savePath` and writes it with
|
||||||
`photo.download()` (see "A photo's content").
|
`photo.download()` (see "A photo's content").
|
||||||
|
|
||||||
@@ -717,13 +677,13 @@ The JSON file beside an original holds:
|
|||||||
|
|
||||||
Update times are in microseconds, as Ente records them.
|
Update times are in microseconds, as Ente records them.
|
||||||
|
|
||||||
The JSON also holds the machine-learning data Ente computed for the photo, as
|
The file also holds the machine-learning data Ente computed for the file, as
|
||||||
`mlData`. Ente's apps detect faces in each photo and compute a CLIP embedding
|
`mlData`. Ente's apps detect faces in each photo and compute a CLIP embedding
|
||||||
for it, a list of 512 numbers that describes what the photo shows. `mlData`
|
for it, a list of 512 numbers that describes what the photo shows. `mlData`
|
||||||
holds everything but the `clip` block, which holds that embedding. In its place
|
holds everything but that embedding. In its place the file holds `keywords`,
|
||||||
the JSON holds `keywords`, derived from the embedding as described under
|
derived from the embedding as described under "Keywords". A file with no
|
||||||
"Keywords". A file with no machine-learning data has neither field. A file whose
|
machine-learning data has neither field. A file whose data holds no usable
|
||||||
data holds no usable embedding has no `keywords`.
|
embedding has no `keywords`.
|
||||||
|
|
||||||
The backup waits for the library's fetch of machine-learning data to finish
|
The backup waits for the library's fetch of machine-learning data to finish
|
||||||
before it writes the JSON files. When a file's data cannot be fetched,
|
before it writes the JSON files. When a file's data cannot be fetched,
|
||||||
@@ -776,11 +736,11 @@ derives a file's keywords the way the app sorts it. The categories are:
|
|||||||
|
|
||||||
Each category has a query, and a text embedding of that query from Ente's CLIP
|
Each category has a query, and a text embedding of that query from Ente's CLIP
|
||||||
text model. A file's score for a category is the cosine similarity of the file's
|
text model. A file's score for a category is the cosine similarity of the file's
|
||||||
embedding and the category's. That is a number from -1 to 1; the more alike the
|
embedding and the category's: a number from -1 to 1, higher the more alike the
|
||||||
two embeddings are, the higher it is. The file falls in the category when its
|
two embeddings are. The file falls in the category when its score reaches the
|
||||||
score reaches the category's minimum score. The keywords are the titles of the
|
category's minimum score. The keywords are the titles of the categories the file
|
||||||
categories the file falls in, highest score first. A video never falls in a
|
falls in, highest score first. A video never falls in a category marked "no
|
||||||
category marked "no videos", as in the app.
|
videos", as in the app.
|
||||||
|
|
||||||
#### Collection directories
|
#### Collection directories
|
||||||
|
|
||||||
@@ -803,21 +763,17 @@ ignoring case, each gets its ID added: two albums named `Trip` become
|
|||||||
`Trip (10)/` and `Trip (11)/`. Two files in one collection whose titles match,
|
`Trip (10)/` and `Trip (11)/`. Two files in one collection whose titles match,
|
||||||
ignoring case, get their IDs added before the extension: `IMG_0001 (12345).JPG`
|
ignoring case, get their IDs added before the extension: `IMG_0001 (12345).JPG`
|
||||||
and `IMG_0001 (12346).JPG`. A collection named like another collection's JSON
|
and `IMG_0001 (12346).JPG`. A collection named like another collection's JSON
|
||||||
file, such as `Trip.json` beside `Trip`, also gets its ID added. A file name
|
file, such as `Trip.json` beside `Trip`, also gets its ID added. A name too long
|
||||||
longer than 255 bytes, the limit on most file systems, gets its ID added too. So
|
for a file name, which most file systems limit to 255 bytes, gets its ID added
|
||||||
does a collection name longer than 250 bytes, which leaves room for `.json`.
|
too. It is then cut short before the ID, just far enough to fit. IDs never
|
||||||
Either name is then cut short before the ID, just far enough to fit. IDs never
|
|
||||||
change, so a name stays the same from run to run until such a clash appears or
|
change, so a name stays the same from run to run until such a clash appears or
|
||||||
goes away.
|
goes away.
|
||||||
|
|
||||||
Each run removes every symlink in a collection's directory that points into the
|
Each run removes the symlinks that no longer belong in a collection's directory.
|
||||||
date folders and is not one of its current links, yours included. A file or
|
It removes the directory and JSON of a collection that was deleted or renamed,
|
||||||
symlink of yours at the name of one of its links is replaced by the link. A
|
once the directory holds nothing else. Nothing else under `collections/` is
|
||||||
directory of yours at that name stays, and the photo whose link it blocks goes
|
touched: a file or symlink you put there stays, and a directory that still holds
|
||||||
into `failures.json`. Anything else you put under `collections/` stays. A run
|
one stays too, with its JSON.
|
||||||
removes the directory and JSON of a collection that was deleted or renamed, once
|
|
||||||
the directory holds nothing else. One that still holds something of yours stays,
|
|
||||||
with its JSON.
|
|
||||||
|
|
||||||
A collection whose directory or JSON cannot be written does not stop the run.
|
A collection whose directory or JSON cannot be written does not stop the run.
|
||||||
For example, a file you put there may already have the directory's name. The run
|
For example, a file you put there may already have the directory's name. The run
|
||||||
@@ -862,9 +818,8 @@ Each entry holds:
|
|||||||
|
|
||||||
- `fileID` and `title`;
|
- `fileID` and `title`;
|
||||||
- `classification`: the kind of failure;
|
- `classification`: the kind of failure;
|
||||||
- `attempts`: how many runs in a row have failed the file;
|
- `attempts`: how many runs have tried the file;
|
||||||
- `lastTriedAt`: when the run that last tried it finished its refresh, as a Unix
|
- `lastTriedAt`: when it was last tried, as a Unix time in milliseconds;
|
||||||
time in milliseconds;
|
|
||||||
- `error`: the error, as quak printed it.
|
- `error`: the error, as quak printed it.
|
||||||
|
|
||||||
The kinds are:
|
The kinds are:
|
||||||
@@ -882,13 +837,13 @@ file outside them. `failures.json` is deleted when the list is empty.
|
|||||||
`backup.lock` keeps two backups of the same directory from running at once, such
|
`backup.lock` keeps two backups of the same directory from running at once, such
|
||||||
as a cron run that starts while the previous one is still going. A backup
|
as a cron run that starts while the previous one is still going. A backup
|
||||||
creates `<dir>` if it is missing and takes the lock before its refresh, so a
|
creates `<dir>` if it is missing and takes the lock before its refresh, so a
|
||||||
refused run sends no request. A run that still holds the lock removes it when it
|
refused run sends no request. It removes the lock when it ends, whether it
|
||||||
ends, whether it succeeded or failed.
|
succeeded or failed.
|
||||||
|
|
||||||
The lock is a directory that
|
The lock is a directory that
|
||||||
[proper-lockfile](https://github.com/moxystudio/node-proper-lockfile) creates.
|
[proper-lockfile](https://github.com/moxystudio/node-proper-lockfile) creates.
|
||||||
While the backup runs, it updates the modification time of `backup.lock` every 5
|
While the backup runs, the backup updates the directory's modification time
|
||||||
seconds. A second backup of `<dir>` fails at once: it prints
|
every 5 seconds. A second backup of the directory fails at once: it prints
|
||||||
`quak: another backup of <dir> is running` and exits 2. A run that is killed, or
|
`quak: another backup of <dir> is running` and exits 2. A run that is killed, or
|
||||||
cut off by a crash, can leave the lock behind. Once the lock has gone 10 seconds
|
cut off by a crash, can leave the lock behind. Once the lock has gone 10 seconds
|
||||||
without an update, the next run takes it over, so nobody has to remove it by
|
without an update, the next run takes it over, so nobody has to remove it by
|
||||||
@@ -931,13 +886,13 @@ The summary and the `--json` output add the counts `verified`, `mismatched` and
|
|||||||
`unchecked`. Each counts only originals that were already stored. A mismatch
|
`unchecked`. Each counts only originals that were already stored. A mismatch
|
||||||
that was replaced with matching bytes does not make the exit code non-zero.
|
that was replaced with matching bytes does not make the exit code non-zero.
|
||||||
|
|
||||||
A run without `--verify` drops two kinds of failure from `failures.json`:
|
Two kinds of failure stay in `failures.json` only until the next run:
|
||||||
|
|
||||||
- a mismatched original whose new copy failed;
|
- a mismatched original whose new copy failed;
|
||||||
- a stored original that cannot be read.
|
- a stored original that cannot be read.
|
||||||
|
|
||||||
That run finds the original in place, counts it as skipped and drops the entry,
|
A run without `--verify` finds the original in place, counts it as skipped, and
|
||||||
so it can exit 0. Only a `--verify` run hashes the original again.
|
drops the entry, so that run can exit 0. Only a `--verify` run hashes it again.
|
||||||
|
|
||||||
Without `--verify`, no stored original is hashed, the summary is unchanged, and
|
Without `--verify`, no stored original is hashed, the summary is unchanged, and
|
||||||
the three counts are 0 in the JSON.
|
the three counts are 0 in the JSON.
|
||||||
@@ -963,11 +918,7 @@ scheme. Opening a library deletes such leftovers in the cache's `originals/` and
|
|||||||
The rename replaces whatever was at the destination rather than writing through
|
The rename replaces whatever was at the destination rather than writing through
|
||||||
it. A symlink there is replaced, not followed, and the new file has the
|
it. A symlink there is replaced, not followed, and the new file has the
|
||||||
temporary file's permissions, not those of the file it replaced. Each original
|
temporary file's permissions, not those of the file it replaced. Each original
|
||||||
is written with mode `0600`, readable by its owner only. Everything else the
|
is written with mode `0600`, readable by its owner only.
|
||||||
backup writes, its folders and JSON files included, gets the mode your umask
|
|
||||||
allows, which usually lets every user on the machine read it. The JSON beside an
|
|
||||||
original holds the file's location and faces. The `backup-metadata` tree is
|
|
||||||
written the same way.
|
|
||||||
|
|
||||||
### The metadata dump directory
|
### The metadata dump directory
|
||||||
|
|
||||||
@@ -992,9 +943,9 @@ of the same name apart.
|
|||||||
`files`. Each `<fileID>.json` holds the fields a backup's JSON beside an
|
`files`. Each `<fileID>.json` holds the fields a backup's JSON beside an
|
||||||
original holds, with these differences:
|
original holds, with these differences:
|
||||||
|
|
||||||
- `imageMetadata` is written only with `--exif`. The original of every file,
|
- `imageMetadata` is written only with `--exif`. The original of every file is
|
||||||
videos included, is then downloaded and read for its EXIF, XMP and JPEG
|
then downloaded, videos included, and read as a backup reads it (see "The JSON
|
||||||
dimensions.
|
beside each original").
|
||||||
- A file whose original holds no image metadata gets no `imageMetadata` field.
|
- A file whose original holds no image metadata gets no `imageMetadata` field.
|
||||||
- An original that cannot be downloaded or read gives `imageMetadataError`
|
- An original that cannot be downloaded or read gives `imageMetadataError`
|
||||||
instead.
|
instead.
|
||||||
@@ -1075,24 +1026,23 @@ onto what the cache holds, empty or not. A refresh then starts only when
|
|||||||
- `contentSource`: a replacement for the client's own byte source. Mainly for
|
- `contentSource`: a replacement for the client's own byte source. Mainly for
|
||||||
tests.
|
tests.
|
||||||
|
|
||||||
In the content and thumbnail pools, your own requests take the next free slot
|
In each pool, your own requests take the next free slot ahead of the two
|
||||||
ahead of the two background downloads. A file already in the cache is not
|
background downloads. A file already in the cache is not fetched again. When the
|
||||||
fetched again. When the originals cache grows past its limit, the least recently
|
originals cache grows past its limit, the least recently used originals are
|
||||||
used originals are evicted, but never a pinned one, and never one whose fetch is
|
evicted, but never a pinned one, and never one whose fetch is still in flight.
|
||||||
still in flight.
|
|
||||||
|
|
||||||
`onProgress` receives a `RefreshEvent`. Its `operation` names the work:
|
`onProgress` receives a `RefreshEvent`. Its `operation` names the work:
|
||||||
`"refresh"`, `"fetchMLData"` for a machine-learning fetch, and
|
`"refresh"`, `"fetchMLData"` for a machine-learning fetch, and
|
||||||
`"precacheThumbnails"` or `"precacheOriginals"` for the two background
|
`"precacheThumbnails"` or `"precacheOriginals"` for the two background
|
||||||
downloads. Its `status` is `"started"` as the work starts and `"done"` as it
|
downloads. Its `status` is `"started"` as the work starts and `"done"` as it
|
||||||
ends. A refresh or a machine-learning fetch that fails ends with `"failed"`
|
ends. A refresh or a machine-learning fetch that fails ends with `"failed"` and
|
||||||
instead, with the message in `error`. In between, a refresh sends `"progress"`
|
the error message instead. In between, a refresh sends `"progress"` after each
|
||||||
after each collection whose files it fetched: `fetched` is the number of
|
collection whose files it fetched: `fetched` is the number of collections
|
||||||
collections fetched so far, and `total` the number it will fetch. A refresh
|
fetched so far, and `total` the number it will fetch. A refresh fetches the
|
||||||
fetches the files of each collection that is new or changed since the cache's
|
files of each collection that is new or changed since the cache's copy, so on an
|
||||||
copy, so on an empty cache it fetches those of every collection. The `"done"`
|
empty cache it fetches those of every collection. The `"done"` event of a
|
||||||
event of a machine-learning fetch, and of each background download, also carries
|
machine-learning fetch, and of each background download, also carries `fetched`:
|
||||||
`fetched`: how many items it newly stored.
|
how many items it newly stored.
|
||||||
|
|
||||||
`lib.status()` returns a `LibraryStatus`. It holds the collection and file
|
`lib.status()` returns a `LibraryStatus`. It holds the collection and file
|
||||||
counts. `lastRefreshAt` is the time of the last refresh that succeeded, and
|
counts. `lastRefreshAt` is the time of the last refresh that succeeded, and
|
||||||
@@ -1107,10 +1057,10 @@ cached, so the next fetch asks for it again. The reasons are kept in memory
|
|||||||
only.
|
only.
|
||||||
|
|
||||||
`lib.close()` stops the background timer. It can be called more than once. A
|
`lib.close()` stops the background timer. It can be called more than once. A
|
||||||
refresh in flight is left to finish, with its write to the cache. `close()`
|
refresh in flight is left to finish, with its write to the cache. The promise
|
||||||
returns a promise. It resolves once that refresh, the machine-learning fetch and
|
`close()` returns resolves once that refresh, the machine-learning fetch and the
|
||||||
the background downloads already running have all finished. The cache directory
|
background downloads already running have all finished. The cache directory can
|
||||||
can then be removed. After `close()`, `fresh()`, `freshCollections()`,
|
then be removed. After `close()`, `fresh()`, `freshCollections()`,
|
||||||
`freshFiles()` and `lib.backup()` reject with `the library is closed`.
|
`freshFiles()` and `lib.backup()` reject with `the library is closed`.
|
||||||
|
|
||||||
### Default reads and fresh reads
|
### Default reads and fresh reads
|
||||||
@@ -1215,11 +1165,11 @@ progress of that one file.
|
|||||||
keyed by tag name, each as exifreader decodes it with its `id`, `value`,
|
keyed by tag name, each as exifreader decodes it with its `id`, `value`,
|
||||||
`description` and `computed` value. For example, `Make` is
|
`description` and `computed` value. For example, `Make` is
|
||||||
`{ id: 271, value: ["Example"], description: "Example", computed: "Example" }`.
|
`{ id: 271, value: ["Example"], description: "Example", computed: "Example" }`.
|
||||||
A tag that exifreader has no name for is keyed `undefined-<tag number>`. The
|
A tag exifreader has no name for is keyed `undefined-<tag number>`. The tags
|
||||||
tags of the thumbnail embedded in the file are under `Thumbnail`, so they
|
of the thumbnail embedded in the file are under `Thumbnail`, so they cannot
|
||||||
cannot hide the main image's tags of the same name. The thumbnail image itself
|
hide the main image's tags of the same name. The thumbnail image itself is
|
||||||
is left out. Any file exifreader cannot read gives `{}`, and a video gives
|
left out. Any file exifreader cannot read gives `{}`, and a video gives `{}`
|
||||||
`{}` without being downloaded.
|
without being downloaded.
|
||||||
- `await photo.make(opts?)` returns one common EXIF field, picked from the tags
|
- `await photo.make(opts?)` returns one common EXIF field, picked from the tags
|
||||||
`exif()` returns and typed as in `PhotoExif`, or `undefined` when the file
|
`exif()` returns and typed as in `PhotoExif`, or `undefined` when the file
|
||||||
lacks it. These methods do the same for their fields:
|
lacks it. These methods do the same for their fields:
|
||||||
@@ -1307,9 +1257,7 @@ symlinks. Once the lock is lost, it rejects with
|
|||||||
`BackupOptions`:
|
`BackupOptions`:
|
||||||
|
|
||||||
- `downloadDirectory`: where the tree lives. Default: the library's.
|
- `downloadDirectory`: where the tree lives. Default: the library's.
|
||||||
- `includeOriginals`: fetch and store originals. Default `true`. With `false`,
|
- `includeOriginals`: fetch and store originals. Default `true`.
|
||||||
the backup also fetches no machine-learning data, and writes no JSON beside
|
|
||||||
the originals and no symlinks.
|
|
||||||
- `includeThumbnails`: also store thumbnails as `thumbnails/<fileID>.jpg` beside
|
- `includeThumbnails`: also store thumbnails as `thumbnails/<fileID>.jpg` beside
|
||||||
`collections/`. Default `false`. `quak backup` does not set it.
|
`collections/`. Default `false`. `quak backup` does not set it.
|
||||||
- `onlyAlbumNames`: back up only the albums with these names.
|
- `onlyAlbumNames`: back up only the albums with these names.
|
||||||
@@ -1330,18 +1278,15 @@ releases it once the backup ends.
|
|||||||
many requests run at once, because Ente meters these three kinds of traffic
|
many requests run at once, because Ente meters these three kinds of traffic
|
||||||
differently:
|
differently:
|
||||||
|
|
||||||
- metadata, with 10 slots: only the machine-learning data requests;
|
- metadata, with 10 slots;
|
||||||
- content, with 5: originals;
|
- content, with 5;
|
||||||
- thumbnails, with 25: thumbnails.
|
- thumbnails, with 25.
|
||||||
|
|
||||||
Only the requests named above go through the pools. A refresh,
|
Construct one with
|
||||||
`freshCollections()` and `freshFiles()` send theirs outside them. Construct one
|
|
||||||
with
|
|
||||||
`new RequestPools({ metadataConcurrency, contentConcurrency, thumbnailConcurrency })`
|
`new RequestPools({ metadataConcurrency, contentConcurrency, thumbnailConcurrency })`
|
||||||
and pass it as `pools`. Each pool serves on-demand work ahead of background
|
and pass it as `pools`. Each pool serves on-demand work ahead of background
|
||||||
work. Two tasks given to `BoundedPool.run()` with the same `key` run once while
|
work. A task submitted under a key that is still pending is not run a second
|
||||||
the first is pending, and both get its result. An idle pool never lends its
|
time. An idle pool never lends its slots to a busy one.
|
||||||
slots to a busy one.
|
|
||||||
|
|
||||||
`lib.backup()` works on as many originals at once as the content pool allows, 5
|
`lib.backup()` works on as many originals at once as the content pool allows, 5
|
||||||
by default. `quak backup` uses the default. The limit covers everything it does
|
by default. `quak backup` uses the default. The limit covers everything it does
|
||||||
@@ -1411,9 +1356,9 @@ They differ because a download's length depends on the file and the link. A
|
|||||||
whole-transfer deadline short enough to catch a hung connection would cancel a
|
whole-transfer deadline short enough to catch a hung connection would cancel a
|
||||||
large video on a slow link that is still making progress. The download deadline
|
large video on a slow link that is still making progress. The download deadline
|
||||||
restarts each time bytes arrive, so a slow download runs as long as it keeps
|
restarts each time bytes arrive, so a slow download runs as long as it keeps
|
||||||
moving, and one that stalls is aborted after 60 seconds of silence. The download
|
moving, and one that stalls is aborted after 60 seconds of silence. It covers
|
||||||
deadline covers the wait for the headers as well as the body. There is no limit
|
the wait for the headers as well as the body. There is no limit on the total
|
||||||
on the total length of a download.
|
length of a download.
|
||||||
|
|
||||||
A request that changes state is not blindly replayed. Every `POST` and `PUT` in
|
A request that changes state is not blindly replayed. Every `POST` and `PUT` in
|
||||||
the endpoint list under "HTTP API" goes through this rule. Verifying a second
|
the endpoint list under "HTTP API" goes through this rule. Verifying a second
|
||||||
@@ -1435,13 +1380,13 @@ that rule although they look like connect-time failures. On Linux they are also
|
|||||||
delivered on an established socket, after the request may have been written.
|
delivered on an established socket, after the request may have been written.
|
||||||
They stay retryable for requests that change nothing. Bun's `ConnectionRefused`
|
They stay retryable for requests that change nothing. Bun's `ConnectionRefused`
|
||||||
and `FailedToOpenSocket` are treated the same way: a request that changes
|
and `FailedToOpenSocket` are treated the same way: a request that changes
|
||||||
nothing is retried on them, and a `POST` or `PUT` is not replayed on them. Bun
|
nothing is retried on them, and a `POST` or `PUT` is not replayed on them. When
|
||||||
reuses a connection it kept open from an earlier request. When that connection
|
a connection bun kept open from an earlier request closes before any answer, bun
|
||||||
closes before any answer, bun sends the request again on a new one. If that one
|
sends the request again on a new connection. If that one fails, bun gives one of
|
||||||
fails, bun gives one of these codes although the server may already have the
|
these codes although the server may already have the request. The upload of a
|
||||||
request. The upload of a thumbnail to its presigned URL is exempt from the rule.
|
thumbnail to its presigned URL is exempt from the rule. It stores one whole
|
||||||
It stores one whole object at one key in one request, so replaying it has no
|
object at one key in one request, so replaying it has no partial state to
|
||||||
partial state to damage.
|
damage.
|
||||||
|
|
||||||
A download is retried as a whole: the request, the reading of the body and the
|
A download is retried as a whole: the request, the reading of the body and the
|
||||||
decryption. A socket reset after the response headers have arrived surfaces in
|
decryption. A socket reset after the response headers have arrived surfaces in
|
||||||
@@ -1466,9 +1411,8 @@ return, quak never keeps a truncated file.
|
|||||||
`Client.login({ email, password, totp?, emailOTP?, apiOptions? })` logs in and
|
`Client.login({ email, password, totp?, emailOTP?, apiOptions? })` logs in and
|
||||||
resolves to a `Client`. `totp` and `emailOTP` are callbacks that return the code
|
resolves to a `Client`. `totp` and `emailOTP` are callbacks that return the code
|
||||||
when the account asks for one. Without the callback the account needs, login
|
when the account asks for one. Without the callback the account needs, login
|
||||||
throws. A passkey challenge always throws. After an email code, login expects
|
throws. A passkey challenge always throws. `apiOptions` configures the HTTP
|
||||||
the key attributes and the token. When the server asks for a second factor
|
client (see below).
|
||||||
instead, login throws. `apiOptions` configures the HTTP client (see below).
|
|
||||||
|
|
||||||
The `Client` holds the session token, the master key, the secret key and the
|
The `Client` holds the session token, the master key, the secret key and the
|
||||||
public key in memory. There is no on-disk session store in the library; the
|
public key in memory. There is no on-disk session store in the library; the
|
||||||
@@ -1478,16 +1422,9 @@ write wherever it likes. `Client.fromJSON(snapshot, apiOptions?)` restores a
|
|||||||
working client from that snapshot without logging in again. It checks every
|
working client from that snapshot without logging in again. It checks every
|
||||||
field and each key's length first, and throws an error naming the bad field.
|
field and each key's length first, and throws an error naming the bad field.
|
||||||
|
|
||||||
`Client.fromJSON` works only once libsodium, the cryptography library quak uses,
|
|
||||||
is loaded. `Client.login` loads it, and `Client.fromJSON` does not. A program
|
|
||||||
that restores a session in a new process first runs `await sodium.ready`, with
|
|
||||||
`sodium` imported from `libsodium-wrappers-sumo`, which quak depends on. The
|
|
||||||
exported `beginLogin`, `unwrapAuth`, `decryptCollection`, `decryptFile`,
|
|
||||||
`downloadFile` and `downloadThumbnail` need the same.
|
|
||||||
|
|
||||||
`client.logout()` clears the token and zeroes the key buffers in place. After
|
`client.logout()` clears the token and zeroes the key buffers in place. After
|
||||||
it, every other method on that client throws. `logout()` does not contact the
|
it, every other method on that client throws. It does not contact the server, so
|
||||||
server, so the token stays valid there and in any saved snapshot. Call
|
the token stays valid there and in any saved snapshot. Call
|
||||||
`await client.logoutOnServer()` before `logout()` to end the session on the
|
`await client.logoutOnServer()` before `logout()` to end the session on the
|
||||||
server.
|
server.
|
||||||
|
|
||||||
@@ -1507,10 +1444,8 @@ return only the decrypted records that are not deleted, from `sinceTime` `0`.
|
|||||||
`client.fetchMLData({ fileIDs, fileKeys })` fetches and decrypts the
|
`client.fetchMLData({ fileIDs, fileKeys })` fetches and decrypts the
|
||||||
machine-learning data of up to 200 files. `client.downloadFile(file, outPath?)`
|
machine-learning data of up to 200 files. `client.downloadFile(file, outPath?)`
|
||||||
and `client.downloadThumbnail(file, outPath?)` download and decrypt one file to
|
and `client.downloadThumbnail(file, outPath?)` download and decrypt one file to
|
||||||
`outPath`. For a live photo, `downloadFile` writes the image and the video, each
|
`outPath`. Without `outPath`, they write to the working directory under the
|
||||||
named `outPath` with its own extension. `quak get --out` instead puts the image
|
names `quak get` and `quak get-thumb` would use.
|
||||||
at the path itself. Without `outPath`, they write to the working directory under
|
|
||||||
the names `quak get` and `quak get-thumb` would use.
|
|
||||||
|
|
||||||
`ApiClientOptions`:
|
`ApiClientOptions`:
|
||||||
|
|
||||||
@@ -1705,8 +1640,8 @@ Login derives the keys in this order:
|
|||||||
the code (`POST /users/two-factor/verify`); the server then returns the key
|
the code (`POST /users/two-factor/verify`); the server then returns the key
|
||||||
attributes and the token. quak cannot answer a passkey request. When the
|
attributes and the token. quak cannot answer a passkey request. When the
|
||||||
account has email verification on (`isEmailMFAEnabled`), an email code
|
account has email verification on (`isEmailMFAEnabled`), an email code
|
||||||
replaces SRP. quak then expects the key attributes and the token in the
|
replaces SRP, and the server returns the key attributes and the token after
|
||||||
answer, and fails the login when the server asks for a second factor instead.
|
it.
|
||||||
5. `crypto_secretbox_open_easy` over the encrypted master key with the KEK
|
5. `crypto_secretbox_open_easy` over the encrypted master key with the KEK
|
||||||
yields the 32-byte master key.
|
yields the 32-byte master key.
|
||||||
6. `crypto_secretbox_open_easy` over the encrypted secret key with the master
|
6. `crypto_secretbox_open_easy` over the encrypted secret key with the master
|
||||||
@@ -1785,37 +1720,36 @@ This repository adheres to the
|
|||||||
standard. Normalized scripts in `script/` are the entrypoints for the
|
standard. Normalized scripts in `script/` are the entrypoints for the
|
||||||
development workflow, and most Makefile targets are thin shims that call them.
|
development workflow, and most Makefile targets are thin shims that call them.
|
||||||
The scripts are POSIX `sh`, not bash, so they run in minimal containers such as
|
The scripts are POSIX `sh`, not bash, so they run in minimal containers such as
|
||||||
Alpine. Of these, only `bootstrap`, `setup`, `test` and `cibuild` come from the
|
Alpine. The scripts marked "ours" are this repository's own additions to the
|
||||||
standard. The others are additions to it.
|
standard.
|
||||||
|
|
||||||
- `script/bootstrap`: install all dependencies. It installs `make`, `git`, Node
|
- `script/bootstrap`: install all dependencies. It installs `make`, `git`, Node
|
||||||
and `yarn` when they are missing, then runs `yarn install --frozen-lockfile`.
|
and `yarn` when they are missing, then runs `yarn install --frozen-lockfile`.
|
||||||
- `script/setup`: set up the repo for development after a fresh clone. It runs
|
- `script/setup`: set up the repo for development after a fresh clone. It runs
|
||||||
`script/bootstrap`, then `script/install-precommit`.
|
`script/bootstrap`, then `script/install-precommit`.
|
||||||
- `script/projectname`: print the project name. `script/docker` uses it for the
|
- `script/projectname` (ours): print the project name. `script/docker` uses it
|
||||||
image tag.
|
for the image tag.
|
||||||
- `script/build`: compile the TypeScript sources into `dist/` and stamp the
|
- `script/build` (ours): compile the TypeScript sources into `dist/` and stamp
|
||||||
version into `dist/package.json`. Then check that the entrypoints
|
the version into `dist/package.json`. Then check that the entrypoints
|
||||||
`package.json` declares (`main`, `types`, `bin`) were written, and make the
|
`package.json` declares (`main`, `types`, `bin`) were written, and make the
|
||||||
command line executable.
|
command line executable.
|
||||||
- `script/version`: print the version `script/build` stamps (see "Version").
|
- `script/version` (ours): print the version `script/build` stamps (see
|
||||||
- `script/build-bin`: run `script/build`, then bundle the built `dist/` into one
|
"Version").
|
||||||
binary, `bin/quak`, with bun (see "Running the backup from cron").
|
|
||||||
- `script/test`: run the test suite, by building the `test` phase of the
|
- `script/test`: run the test suite, by building the `test` phase of the
|
||||||
`Dockerfile`. Requires Docker.
|
`Dockerfile`. Requires Docker.
|
||||||
- `script/lint`: run eslint and a prettier check, by building the `lint` phase
|
- `script/lint`: run eslint and a prettier check, by building the `lint` phase
|
||||||
of the `Dockerfile`. Requires Docker.
|
of the `Dockerfile`. Requires Docker.
|
||||||
- `script/fmt`: format all files with prettier. It rewrites files in place.
|
- `script/fmt`: format all files with prettier. It rewrites files in place.
|
||||||
- `script/fmt-check`: check formatting on the host. It is read-only.
|
- `script/fmt-check`: check formatting on the host. It is read-only.
|
||||||
- `script/check`: run all checks: `test`, `lint`, `fmt-check`.
|
- `script/check` (ours): run all checks: `test`, `lint`, `fmt-check`.
|
||||||
- `script/docker`: build the image, tagged with `script/projectname`.
|
- `script/docker`: build the image, tagged with `script/projectname`.
|
||||||
- `script/cibuild`: what CI runs: `script/bootstrap`, then `script/check`, then
|
- `script/cibuild`: what CI runs: `script/bootstrap`, then `script/check`, then
|
||||||
the image build.
|
the image build.
|
||||||
- `script/precommit`: run by the git pre-commit hook. It runs `script/lint` and
|
- `script/precommit` (ours): run by the git pre-commit hook. It runs
|
||||||
`script/fmt-check`, and not the tests, so that a commit of failing tests can
|
`script/lint` and `script/fmt-check`, and not the tests, so that a commit of
|
||||||
land (see "Development workflow").
|
failing tests can land (see "Development workflow").
|
||||||
- `script/install-precommit`: install the git pre-commit hook. `make hooks`
|
- `script/install-precommit` (ours): install the git pre-commit hook.
|
||||||
calls it.
|
`make hooks` calls it.
|
||||||
|
|
||||||
The Makefile targets are:
|
The Makefile targets are:
|
||||||
|
|
||||||
@@ -1829,7 +1763,8 @@ The Makefile targets are:
|
|||||||
- `make build`: runs `script/build`.
|
- `make build`: runs `script/build`.
|
||||||
- `make docker`: runs `script/docker`.
|
- `make docker`: runs `script/docker`.
|
||||||
- `make hooks`: runs `script/install-precommit`.
|
- `make hooks`: runs `script/install-precommit`.
|
||||||
- `make build-bin`: runs `script/build-bin`.
|
- `make build-bin`: bundles the built `dist/` into one binary, `bin/quak`. It
|
||||||
|
runs bun through `nix-shell -p bun`.
|
||||||
- `make install`: runs `make build-bin` and copies the binary to `~/bin/quak`.
|
- `make install`: runs `make build-bin` and copies the binary to `~/bin/quak`.
|
||||||
- `make dev`: runs the TypeScript compiler in watch mode.
|
- `make dev`: runs the TypeScript compiler in watch mode.
|
||||||
- `make clean`: removes `dist/`, `coverage/`, `.vitest-cache/` and the
|
- `make clean`: removes `dist/`, `coverage/`, `.vitest-cache/` and the
|
||||||
@@ -1881,8 +1816,8 @@ empty, `dev` or `unknown`, because such a build could not be traced to its
|
|||||||
commit.
|
commit.
|
||||||
|
|
||||||
`.dockerignore` therefore does not leave out `.git`, so any `docker build .` of
|
`.dockerignore` therefore does not leave out `.git`, so any `docker build .` of
|
||||||
a clone stamps the commit it was built from. A clone of depth 1 stamps a tag
|
a clone stamps the commit it was built from. A shallow clone stamps a tag only
|
||||||
only when its commit carries one, and otherwise the short commit.
|
when the cloned commit itself carries one, and otherwise the short commit.
|
||||||
`.dockerignore` does leave out `.git/config`, which holds the clone's remote URL
|
`.dockerignore` does leave out `.git/config`, which holds the clone's remote URL
|
||||||
and any credential in it; `git describe` does not need that file.
|
and any credential in it; `git describe` does not need that file.
|
||||||
`script/docker` and `script/cibuild` work out the version on the host, with
|
`script/docker` and `script/cibuild` work out the version on the host, with
|
||||||
@@ -1937,15 +1872,8 @@ These apply to people and to coding agents alike.
|
|||||||
- container images by digest;
|
- container images by digest;
|
||||||
- the nvm archive `script/bootstrap` downloads, and the Python packages
|
- the nvm archive `script/bootstrap` downloads, and the Python packages
|
||||||
`bin/clip-keyword-vectors` installs, by SHA-256 hash;
|
`bin/clip-keyword-vectors` installs, by SHA-256 hash;
|
||||||
- bun's glibc and musl builds, which `script/build-bin` downloads on Linux,
|
|
||||||
by SHA-256 hash;
|
|
||||||
- Node and yarn, when `script/bootstrap` installs them, by version.
|
- Node and yarn, when `script/bootstrap` installs them, by version.
|
||||||
- Not pinned:
|
- bun is not pinned: `make build-bin` uses whatever `nix-shell -p bun` provides.
|
||||||
- bun on macOS, which `script/build-bin` takes from `nix-shell -p bun`;
|
|
||||||
- `make` and `git`, which `script/bootstrap` installs from the system
|
|
||||||
package manager when they are missing, as every `Dockerfile` stage does;
|
|
||||||
- `bash` and `curl`, which `script/bootstrap` installs the same way when it
|
|
||||||
has to install Node.
|
|
||||||
- Follow the workflow above: branches off `next`, tests first and committed
|
- Follow the workflow above: branches off `next`, tests first and committed
|
||||||
failing, and `next` merged into `main` by sneak alone.
|
failing, and `next` merged into `main` by sneak alone.
|
||||||
- Before every commit, `make lint` and `make fmt-check` must pass. `make lint`
|
- Before every commit, `make lint` and `make fmt-check` must pass. `make lint`
|
||||||
@@ -1966,7 +1894,8 @@ These apply to people and to coding agents alike.
|
|||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
`TODO.md` in the repo root holds the next step and the steps planned. Open now:
|
`TODO.md` in the repo root is the work log: the next step, the steps done and
|
||||||
|
the steps planned. Open now:
|
||||||
|
|
||||||
- [ ] Check quak against a live Ente account. This needs an account only sneak
|
- [ ] Check quak against a live Ente account. This needs an account only sneak
|
||||||
has.
|
has.
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
- branch from `next`
|
- branch from `next`
|
||||||
- do the work in Next Step
|
- do the work in Next Step
|
||||||
- delete Next Step from `TODO.md`
|
- move Next Step to the top of Completed Steps
|
||||||
- move the top item of Future Steps into Next Step
|
- move the top item of Future Steps into Next Step
|
||||||
- commit (`TODO.md` changes in the same commit as the work)
|
- commit (`TODO.md` changes in the same commit as the work)
|
||||||
- push
|
- push
|
||||||
@@ -24,6 +24,733 @@ only sneak has, so it waits on him.
|
|||||||
Tagging and releases are decided by sneak alone, and happen only when he
|
Tagging and releases are decided by sneak alone, and happen only when he
|
||||||
declares one.
|
declares one.
|
||||||
|
|
||||||
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-26: On an empty cache, `quak helper fix-missing-thumbnails --file`
|
||||||
|
stops fetching collections' files once it finds every file it names
|
||||||
|
(https://git.eeqj.de/sneak/quak/issues/223). It used to wait for a refresh,
|
||||||
|
which on an empty cache fetched the files of every collection. Now it reads
|
||||||
|
the collection list, then one collection's files at a time, the most recently
|
||||||
|
updated first. It stores none of the collections or file records it reads. It
|
||||||
|
fetches each original through `lib.originalOf()`, which keeps it in the cache.
|
||||||
|
Once the cache holds the account's collections, it still waits for a refresh.
|
||||||
|
|
||||||
|
- 2026-10-26: On an empty cache, `quak get` and `quak get-thumb` stop fetching
|
||||||
|
collections' files once they find the file
|
||||||
|
(https://git.eeqj.de/sneak/quak/issues/222). They used to wait for a refresh,
|
||||||
|
which on an empty cache fetched the files of every collection. Now they read
|
||||||
|
the collection list, then one collection's files at a time. They store none of
|
||||||
|
the collections or file records they read, and fetch the original or thumbnail
|
||||||
|
through the new `lib.originalOf()` and `lib.thumbnailOf()`. Once the cache
|
||||||
|
holds the account's collections, they still wait for a refresh.
|
||||||
|
|
||||||
|
- 2026-10-26: `quak files` with a collection ID fetches that collection's files
|
||||||
|
alone (https://git.eeqj.de/sneak/quak/issues/217). It used to wait for a
|
||||||
|
refresh, which on an empty cache fetched the files of every collection. It now
|
||||||
|
reads the collection list through `lib.freshCollections()`, then the one
|
||||||
|
collection's files through the new `lib.freshFiles()`. It stores nothing, and
|
||||||
|
the files print in the server's order.
|
||||||
|
|
||||||
|
- 2026-10-22: The JSON `quak backup` and `quak backup-metadata` write for a file
|
||||||
|
holds `keywords` in place of the CLIP embedding in its ML data
|
||||||
|
(https://git.eeqj.de/sneak/quak/issues/218). The keywords are the titles of
|
||||||
|
the Discover categories of Ente's photos app that the file falls in, by the
|
||||||
|
app's rule: the cosine similarity of the file's CLIP embedding and the text
|
||||||
|
embedding of a category's query is at least the category's minimum score, and
|
||||||
|
videos stay out of the categories that leave them out. `src/clip-keywords.ts`
|
||||||
|
applies the rule. `src/clip-keyword-vectors.ts` holds the 16 categories with
|
||||||
|
their text embeddings. `bin/clip-keyword-vectors` writes it with Ente's CLIP
|
||||||
|
text model, and writes the CLIP embedding of a public-domain photo of a sunset
|
||||||
|
that the tests use.
|
||||||
|
|
||||||
|
- 2026-10-18: `-v`/`--verbose` raises the CLI's log level once for each repeat,
|
||||||
|
and `-q`/`--quiet` prints only fatal errors
|
||||||
|
(https://git.eeqj.de/sneak/quak/issues/212). The levels are fatal, error,
|
||||||
|
warning, info, debug and trace; the default is info, everything quak printed
|
||||||
|
on stderr before, `-v` gives debug and `-vv` trace. `src/cli-log.ts` holds the
|
||||||
|
level and the log the commands write stderr lines through. Debug adds each
|
||||||
|
retry, which `withRetry` reports through the new `onRetry`, and the start and
|
||||||
|
end of each refresh and ML data fetch; trace adds each request, which
|
||||||
|
`ApiClient` reports through the new `onRequest`. `--json` no longer keeps the
|
||||||
|
progress lines off stderr, so `backup` and `--json` runs log a refresh's
|
||||||
|
progress too. Every line that reports a failed file or album is at error,
|
||||||
|
which the progress callbacks of `backup`, `backup-metadata` and the thumbnail
|
||||||
|
helpers now say, and a command that exits 1 because files or albums failed
|
||||||
|
ends with one fatal line saying how many and where to look.
|
||||||
|
|
||||||
|
- 2026-10-18: `quak files` takes the collection ID as an optional argument
|
||||||
|
instead of the `--collection` option, which is gone
|
||||||
|
(https://git.eeqj.de/sneak/quak/issues/211). With no argument it lists every
|
||||||
|
file in the account once, collection by collection; a file in several
|
||||||
|
collections is listed with the first of them, which is its `collectionID` in
|
||||||
|
`--json`. The fresh read in `src/cli-read.ts` does this.
|
||||||
|
|
||||||
|
- 2026-10-18: Names quak builds from an album name or a title fit in a file name
|
||||||
|
(https://git.eeqj.de/sneak/quak/issues/205). `shortenKeepingExtension` in
|
||||||
|
`src/filename.ts` cuts a name short to a number of bytes, keeping its
|
||||||
|
extension when it fits and always its first character, and can put text right
|
||||||
|
before the extension; the backup's `withID` uses it for an ID.
|
||||||
|
`defaultFileName` there gives the name a file is saved under when the caller
|
||||||
|
gives no path: a prefix such as `thumb_` and the title with unsafe characters
|
||||||
|
replaced, cut short with it to 255 bytes, and, for a live photo's image or
|
||||||
|
video, with that part's own extension before the cut. `quak get`,
|
||||||
|
`quak get-thumb`, `downloadFile` and `downloadThumbnail` use it without
|
||||||
|
`--out` or `outPath`. `backup-metadata` cuts an album's name short so that its
|
||||||
|
folder, `<id>-<name>`, fits in 255 bytes. `safeExtension` gives `.bin` for an
|
||||||
|
extension of more than 16 letters and digits, so a save path's name always
|
||||||
|
fits.
|
||||||
|
|
||||||
|
- 2026-10-18: `quak collections` no longer waits for the files of every
|
||||||
|
collection (issue 210). On an empty cache it opened its library with a refresh
|
||||||
|
that fetched and decrypted the files of each collection in turn, printing
|
||||||
|
nothing and storing nothing until all were done, so on a large account it
|
||||||
|
seemed to hang. It now reads the collection list alone through the new
|
||||||
|
`lib.freshCollections()`, one request that stores nothing. The read commands
|
||||||
|
open their library with the new `refreshOnOpen: false`, so each makes only the
|
||||||
|
refresh it asks for. A refresh sends a `"progress"` event after each
|
||||||
|
collection whose files it fetched, and the commands that wait on one print it
|
||||||
|
on stderr, except with `--json`.
|
||||||
|
|
||||||
|
- 2026-10-16: ML data the server sends that does not decrypt, gunzip or parse is
|
||||||
|
no longer taken for no ML data (issue 198). `fetchMLDataBatch` returns why for
|
||||||
|
each such file beside the payloads; ML data that parses to something other
|
||||||
|
than a JSON object, such as `null`, is one of them. The library keeps the
|
||||||
|
reasons from its last ML data fetch that succeeded in memory, not in the
|
||||||
|
cache, and gives them in `status().mlDataErrors` and in the `fetchMLData`
|
||||||
|
event with status `"done"`; a fetch still running or one that fails does not
|
||||||
|
change them. `quak backup` writes the reason as `mlDataError` in the file's
|
||||||
|
JSON when no ML data is cached for it, and records the file in `failures.json`
|
||||||
|
with an error starting `ML data: `; `backup-metadata` writes `mlDataError`
|
||||||
|
too. Both exit 1. Tests serve ML data encrypted under a key that is not the
|
||||||
|
file's.
|
||||||
|
|
||||||
|
- 2026-10-16: With `--verify`, an original whose hash does not match is no
|
||||||
|
longer removed before its replacement is downloaded (issue 199).
|
||||||
|
`replaceOriginal` in `src/library/content.ts` downloads the replacement to a
|
||||||
|
temporary file beside it, a live photo's image and video each to its own,
|
||||||
|
hashes it and renames it over the original only when it matches. When the
|
||||||
|
download fails or does not match either, the original keeps its bytes and
|
||||||
|
links, and the file goes into `failures.json`. The replacement is always
|
||||||
|
downloaded, through the new `fetchOriginal` of the content cache, never copied
|
||||||
|
from the cache.
|
||||||
|
|
||||||
|
- 2026-10-16: The `~/bin/quak` that `make install` builds retries a refused
|
||||||
|
connection and a name that does not resolve, and the backup records either as
|
||||||
|
transient (issue 206). Bun's `fetch` reports them with codes of its own,
|
||||||
|
`ConnectionRefused` and `FailedToOpenSocket`, which `src/retry.ts` now counts
|
||||||
|
as transport failures. A `POST` or `PUT` is not replayed on them, because bun
|
||||||
|
also gives them when it resends a request on a new connection after a
|
||||||
|
connection it kept open closed, and the server may already have the request.
|
||||||
|
Bun already gave `ECONNRESET` for a reset and a `TimeoutError` for a deadline.
|
||||||
|
`classify` in `src/backup.ts` now records any `TimeoutError` as transient, so
|
||||||
|
a stalled download, whose error says only that it stalled, is no longer
|
||||||
|
`unknown`. The shapes were found by running the binary against local
|
||||||
|
listeners.
|
||||||
|
|
||||||
|
- 2026-10-15: An album whose folder or JSON `quak backup` cannot write no longer
|
||||||
|
ends the run before the other albums and `failures.json` are written (issue
|
||||||
|
197). The run prints `FAILED album <name>: <reason>`, lists the album's ID in
|
||||||
|
`failedAlbums` in the result and in `--json`, goes on, and exits 1. An album
|
||||||
|
whose folder would have the name of another album's JSON, as `x.json` beside
|
||||||
|
`x` would, gets its ID added. A folder or link name over 255 bytes is cut
|
||||||
|
short between two characters, by `shortenToBytes` in `src/filename.ts`, and
|
||||||
|
gets the ID.
|
||||||
|
|
||||||
|
- 2026-10-15: A network failure is printed and recorded with its reason, not as
|
||||||
|
just `fetch failed` (issue 196). `describeError` in `src/errors.ts` gives an
|
||||||
|
error's message followed by the message of each error on its `cause` chain,
|
||||||
|
such as `fetch failed: connect ECONNREFUSED 1.2.3.4:443`, and each address's
|
||||||
|
reason when a host has several and none could be reached. `run` prints that,
|
||||||
|
and the backup logs it and records it in `failures.json`, where `classify` now
|
||||||
|
calls the failure transient when the chain holds an errno that
|
||||||
|
`hasTransportCode` in `src/retry.ts` recognises, for a failed ML data fetch
|
||||||
|
too. The walk along the chain moved from `src/retry.ts` to `src/errors.ts`,
|
||||||
|
which both use. Tests refuse every connection of a backup, then only one
|
||||||
|
file's download, and fail the ML data fetch.
|
||||||
|
|
||||||
|
- 2026-10-15: The README says that `make install` needs nix (issue 200), in
|
||||||
|
Getting Started and in "Running the backup from cron": `make build-bin` runs
|
||||||
|
bun through `nix-shell -p bun`. The `Makefile` is unchanged.
|
||||||
|
|
||||||
|
- 2026-10-14: `lib.backup()`, and so `quak backup`, works on up to the content
|
||||||
|
pool's limit of originals at once, 5 by default, instead of one at a time
|
||||||
|
(issue 182). That many workers take files from one shared list, each doing all
|
||||||
|
the work for one original, its `--verify` hashing included, so no more
|
||||||
|
originals than that are open at once. `BackupLibrary` says the limit as
|
||||||
|
`contentConcurrency`. A test holds each download at the fake Ente server to
|
||||||
|
count those in flight. Once a worker has thrown, as one does when `onProgress`
|
||||||
|
throws, or the lock is lost, no worker takes another file; the backup fails
|
||||||
|
once the originals under way are in place, and a lost lock stops it before it
|
||||||
|
fetches ML data.
|
||||||
|
|
||||||
|
- 2026-10-14: Next Step said the cache design, issue 36, waits on sneak's
|
||||||
|
review, though that issue closed on 2026-10-03 (issue 194). It now names the
|
||||||
|
issues open on the tracker.
|
||||||
|
|
||||||
|
- 2026-10-14: A backup whose lock is lost no longer dies with a stack trace
|
||||||
|
(issue 184). `lockBackupDirectory` gives proper-lockfile an `onCompromised`
|
||||||
|
that records why the lock was lost instead of throwing from its timer, and
|
||||||
|
returns the lock, which `quak backup` passes to `lib.backup()` as `lock`. The
|
||||||
|
backup then stops before it starts another file with
|
||||||
|
`lost the lock on DIR: REASON`, which `quak backup` prints as one line before
|
||||||
|
it exits 1. Releasing a lost lock leaves `backup.lock` alone. Tests in
|
||||||
|
`test/cli/backup-end-to-end.test.ts` remove the lock, or put another backup's
|
||||||
|
in its place, while a download is held open.
|
||||||
|
|
||||||
|
- 2026-10-14: An album or file quak cannot decrypt, or whose metadata is not
|
||||||
|
JSON, no longer fails the refresh, so one bad record no longer stops
|
||||||
|
`quak backup` from storing anything (issue 181). The refresh leaves it out, an
|
||||||
|
album with its files, and the library keeps it with the reason in
|
||||||
|
`metadata.json` until the server sends a copy that decrypts or deletes it.
|
||||||
|
`quak backup` stores every other file, records each such file in
|
||||||
|
`failures.json` with an error starting `could not be decrypted:`, names each
|
||||||
|
such album's ID in a printed line, or with `--json` in `undecryptableAlbums`,
|
||||||
|
and exits 1 while any remain. A file whose copy in another album decrypts is
|
||||||
|
not recorded: its original is stored from that copy, and the album where it
|
||||||
|
does not decrypt neither links nor lists it. The read commands answer from
|
||||||
|
what decrypted.
|
||||||
|
|
||||||
|
- 2026-10-14: `quak backup` no longer asks the server for changes every 3
|
||||||
|
seconds while it runs (issue 183). It opens its library with
|
||||||
|
`refreshIntervalSeconds: Infinity`, which the library now takes as no
|
||||||
|
background timer, so the backup refreshes only as it starts and no timed
|
||||||
|
refresh starts another ML data pass during the run. A test holds a backup's
|
||||||
|
download open while two hours pass and counts its `/collections/v2` requests.
|
||||||
|
|
||||||
|
- 2026-10-14: `test/cli/backup-end-to-end.test.ts` runs `quak backup` as cron
|
||||||
|
does (issue 186): `backupCommand` and `run`, the real `loadSession` reading a
|
||||||
|
session saved after logging in, and every request answered by a fake of the
|
||||||
|
Ente HTTP API serving data encrypted with libsodium. It covers the backup tree
|
||||||
|
of an account with two albums of its own and one shared with it, a second run
|
||||||
|
that downloads nothing, a download that succeeds after two 503s, a 404 that
|
||||||
|
lands in `failures.json` with exit 1, a refresh answered 401 with exit 3, and
|
||||||
|
`--verify` putting back an original changed on disk. The fake server and the
|
||||||
|
account it logs in moved from the backup-metadata test into
|
||||||
|
`test/ente-server.ts`, unchanged.
|
||||||
|
|
||||||
|
- 2026-10-14: `files --collection`, `get`, `get-thumb` and
|
||||||
|
`helper fix-missing-thumbnails --file` take an ID only when it is written in
|
||||||
|
decimal digits and is a safe integer (issue 185). Before, `Number()` read
|
||||||
|
`0x10` as 16, `1e3` as 1000 and an empty string as 0, and `--file` dropped an
|
||||||
|
ID it could not read without a word. Anything else now prints
|
||||||
|
`Invalid file ID: <arg>` or `Invalid collection ID: <arg>` and exits 1 before
|
||||||
|
any request; one bad `--file` ID fails the whole command. `parseID` in
|
||||||
|
`src/cli-commands.ts` does the check for all four.
|
||||||
|
|
||||||
|
- 2026-10-06: The README says how to run `quak backup` from cron (issue 170):
|
||||||
|
log in once as the job's user, a crontab with a backup every night and
|
||||||
|
`--verify` on Sundays appending to a log file, and the `HOME` and
|
||||||
|
`XDG_DATA_HOME` the job needs to find the saved session. A table gives each
|
||||||
|
exit code: 0, the backup is complete; 1, files are in `failures.json` or
|
||||||
|
another error stopped the run; 2, another backup of the directory is running;
|
||||||
|
3, there is no usable session. The introduction lists everything the backup
|
||||||
|
keeps for each file.
|
||||||
|
|
||||||
|
- 2026-10-06: `quak backup --verify` and `lib.backup({ verify: true })` hash
|
||||||
|
each original already at its save path as the download check does, streamed, a
|
||||||
|
live photo as `<imageHash>:<videoHash>` (issue 168). One that does not match
|
||||||
|
the content hash its metadata records is logged, removed (both files of a live
|
||||||
|
photo) and put back in the same run, downloaded or copied from the cache, and
|
||||||
|
what is put back is hashed too. One that still does not match, or whose
|
||||||
|
download fails, goes into `failures.json`. One with no recorded hash is left
|
||||||
|
alone. The result, `--json` and the summary gain `verified`, `mismatched` and
|
||||||
|
`unchecked`. Without `--verify` nothing is hashed.
|
||||||
|
|
||||||
|
- 2026-10-06: Two backups of the same directory never run at once (issue 169).
|
||||||
|
`lib.backup()` takes a lock, `backup.lock` in its download directory, made
|
||||||
|
with `proper-lockfile`, before its refresh, and removes it when it ends,
|
||||||
|
whether it succeeds or fails. A second backup of the directory, from another
|
||||||
|
process or the same one, fails at once with an error naming the directory.
|
||||||
|
`quak backup` takes the lock before it opens its library, so a refused run
|
||||||
|
sends no request; it prints the error as one line and exits 2. A lock that has
|
||||||
|
gone 10 seconds untouched, left by a run that could not remove it, is taken
|
||||||
|
over by the next run.
|
||||||
|
|
||||||
|
- 2026-10-06: `quak backup` writes each original's EXIF, XMP and dimensions into
|
||||||
|
the file's JSON as `imageMetadata`, what `backup-metadata --exif` records
|
||||||
|
(issue 167): for a live photo from its image, for a video nothing, and `{}`
|
||||||
|
for an original with none of them. A failed read puts the reason in
|
||||||
|
`imageMetadataError` and fails neither the file nor the run. An original is
|
||||||
|
read when the run stores it or when its JSON has neither field; otherwise the
|
||||||
|
field is taken from that JSON. The hand-built JPEGs moved to
|
||||||
|
`test/exif-jpeg.ts`, beside the HEIC.
|
||||||
|
|
||||||
|
- 2026-10-06: When the server answers HTTP 401 and that ends a command that
|
||||||
|
loads the saved session, because the server no longer accepts its token, the
|
||||||
|
command prints one line,
|
||||||
|
`quak: the saved session is no longer valid; run "quak login"`, and exits 3
|
||||||
|
(issue 164). `quak backup` meets that 401 on the refresh it starts with,
|
||||||
|
before it touches any file. For the same commands, a missing or corrupt
|
||||||
|
session file keeps its message and also exits 3, so a cron job can tell that
|
||||||
|
the user must log in again. `quak logout` is unchanged. `run` in
|
||||||
|
`src/cli-run.ts` recognises the 401, which reaches it unchanged.
|
||||||
|
|
||||||
|
- 2026-10-06: `quak backup` retries a failed request for longer than the other
|
||||||
|
commands do (issue 165). `src/retry.ts` exports `UNATTENDED_RETRY_OPTIONS`
|
||||||
|
beside the unchanged default: 10 attempts, a 1 s base delay and a 60 s cap, so
|
||||||
|
a request that keeps failing waits at most 243 s before it gives up.
|
||||||
|
`bin/quak.ts` loads the backup's session with them, so its refresh, ML data
|
||||||
|
and downloads all use them. What is retried and the backoff formula are
|
||||||
|
unchanged.
|
||||||
|
|
||||||
|
- 2026-10-06: The files this repository copies from `sneak/prompts` are copied
|
||||||
|
again from its commit `dd4027b` (issue 171). `.gitignore` and `.dockerignore`
|
||||||
|
keep out more secret files, and this repository's build artifacts follow the
|
||||||
|
copied content. `script/check` runs `script/fmt-check` again, `script/cibuild`
|
||||||
|
runs `script/bootstrap` and `script/check` before the image build,
|
||||||
|
`script/fmt` and `script/fmt-check` find the pinned yarn under nvm, and the
|
||||||
|
image's last stage marks `/app` safe for git.
|
||||||
|
|
||||||
|
- 2026-10-06: `quak backup` writes the account and album records
|
||||||
|
`backup-metadata` writes (issue 166): `account.json` with the account's
|
||||||
|
`email` and `userID`, and in each album's JSON its `ownerID`, `isShared`,
|
||||||
|
`updationTime` and, when present, its three layers of magic metadata. Each
|
||||||
|
file's JSON gains its `updationTime`.
|
||||||
|
|
||||||
|
- 2026-10-05: `quak backup` writes each file's ML data (its faces and CLIP
|
||||||
|
embedding) into the file's JSON as `mlData`, the payload
|
||||||
|
`lib.mldata.forFile()` returns (issue 163). `lib.backup()` waits for an ML
|
||||||
|
data fetch before it writes the JSON files. When that fetch fails, each file
|
||||||
|
whose ML data is not cached gets the reason in `mlDataError` and counts as
|
||||||
|
failed, and the next run fetches it again.
|
||||||
|
|
||||||
|
- 2026-10-03: The download-albums example test no longer fails on a disk with
|
||||||
|
under 50 GiB free (issue 160). It opens its libraries with
|
||||||
|
`freeBelowBytes: 0`, so the free space of the disk it runs on cannot shrink
|
||||||
|
the cache, evict the original it cached and add a fetch to the count. No other
|
||||||
|
test's result depends on that free space. The 50 GiB default is unchanged.
|
||||||
|
|
||||||
|
- 2026-10-02: `docker build .` stamps the commit's tag or short commit, not
|
||||||
|
`dev` (issue 154). `script/build` writes the version `script/version` prints
|
||||||
|
into `dist/package.json`: the `VERSION` environment variable or build arg when
|
||||||
|
one is given, otherwise `git describe --tags --always`, otherwise the version
|
||||||
|
`package.json` declares. `.dockerignore` sends `.git`, and a checkout with
|
||||||
|
`.git` whose version comes out empty, `dev` or `unknown` fails the build.
|
||||||
|
`make build-bin` bundles the built `dist/`, so the single binary reports the
|
||||||
|
same version.
|
||||||
|
|
||||||
|
- 2026-10-02: `photo.exif()` returns every EXIF tag in the file as `ExifTags`,
|
||||||
|
keyed by tag name, each as exifreader decodes it, not only the thirteen common
|
||||||
|
fields (issue 156). The embedded thumbnail's tags are under `Thumbnail`,
|
||||||
|
without the thumbnail image. The thirteen typed methods stay, each picking its
|
||||||
|
field from the tags `exif()` returns, typed as in `PhotoExif`. The example
|
||||||
|
script's JSON files now carry every tag.
|
||||||
|
|
||||||
|
- 2026-10-01: The content cache no longer looks for a live photo that an earlier
|
||||||
|
version cached as one ZIP (issue 151). When the cache opens, a live photo's
|
||||||
|
file that no JSON file names is now always left alone.
|
||||||
|
|
||||||
|
- 2026-10-01: `examples/download-albums.ts` logs in, opens the library, and for
|
||||||
|
every album downloads each photo to its save path, writes the photo's record
|
||||||
|
and EXIF fields to a JSON file beside it, and writes the album's photos to
|
||||||
|
`albums/<collectionID>.json` (issue 144). The build compiles it to
|
||||||
|
`dist/examples/`; the README's "Examples" section says how to run it.
|
||||||
|
|
||||||
|
- 2026-10-01: A `Photo` has one async method for each field of `exif()`, named
|
||||||
|
and typed as in `PhotoExif`: `make()`, `model()`, `lensModel()`,
|
||||||
|
`dateTimeOriginal()`, `offsetTimeOriginal()`, `exposureTime()`, `fNumber()`,
|
||||||
|
`iso()`, `focalLength()`, `orientation()`, `gpsLatitude()`, `gpsLongitude()`
|
||||||
|
and `gpsAltitude()` (issue 148). Each calls `exif()` and returns its one
|
||||||
|
field, or undefined when the file lacks it. `Photo` implements a type with one
|
||||||
|
method per `PhotoExif` field, so the build's type check fails when a field has
|
||||||
|
no method. A test checks, on the JPEG and the HEIC, that each method gives the
|
||||||
|
same value as `exif()`.
|
||||||
|
|
||||||
|
- 2026-10-01: Each original's save path is
|
||||||
|
`YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>` under the library's
|
||||||
|
download directory, which defaults to `photos` in the working directory (issue
|
||||||
|
143). The date is the photo's `takenAt` in the machine's time zone.
|
||||||
|
`photo.savePath` is always a string, with or without a content cache, and
|
||||||
|
`isLocal` is true only when the original is at its save path.
|
||||||
|
`photo.download()` puts the original there, copied from the cache when the
|
||||||
|
cache holds it and fetched otherwise. `lib.backup()` does the same for each
|
||||||
|
file, writes each file's JSON beside its original, and links `collections/` to
|
||||||
|
the save paths.
|
||||||
|
|
||||||
|
- 2026-10-01: `photo.exif()` and `backup-metadata --exif` read EXIF from
|
||||||
|
HEIC/HEIF originals, a live photo's HEIC image included, as well as JPEG and
|
||||||
|
the other image formats `exifreader` reads (issue 145). `exifreader` replaces
|
||||||
|
`exif-reader` and the JPEG segment scan; `PhotoExif` is unchanged. The `exif`
|
||||||
|
field of `backup-metadata --exif` is now exifreader's tag output, and
|
||||||
|
`exifRaw` holds the whole EXIF block it could not read. The tests use a real
|
||||||
|
HEIC, `test/exif.heic`.
|
||||||
|
|
||||||
|
- 2026-10-01: A `Photo` has `savePath`, `isLocal`, `content()`, `exif()`,
|
||||||
|
`modifiedAt`, `hash` and `year` (issue 141). `savePath` is where
|
||||||
|
`lib.backup()` writes the original under the library's download directory; for
|
||||||
|
a live photo not yet stored, it carries the title's extension, and the backup
|
||||||
|
may store the image under a different one. `isLocal` says whether the whole
|
||||||
|
original is there. Both look only at the disk. `content()` returns the
|
||||||
|
original's bytes and `exif()` the common EXIF fields of a JPEG; both may
|
||||||
|
download the original, and `exif()` downloads no video. `PhotoRecord` gains
|
||||||
|
`modifiedAt` and `hash`, and the JPEG EXIF scan moved to `src/exif.ts`.
|
||||||
|
|
||||||
|
- 2026-09-29: The "Workflow" list at the top of this file now says to branch
|
||||||
|
from `next` and open a pull request that targets `next`, that the repository
|
||||||
|
manager squash-merges a reviewed pull request into `next`, and that only sneak
|
||||||
|
merges `next` into `main`, as the README does (issue 137).
|
||||||
|
|
||||||
|
- 2026-09-29: The README's "Development workflow" and "For LLMs" sections now
|
||||||
|
say work branches from `next`, every pull request targets `next`, the
|
||||||
|
repository manager squash-merges reviewed pull requests into `next`, and only
|
||||||
|
sneak merges `next` into `main` (issue 135).
|
||||||
|
|
||||||
|
- 2026-09-29: Brought the README and this file in line with `next` after the
|
||||||
|
milestone merge (issue 132). The Next Step says no implementation work is open
|
||||||
|
and the cache design (issue 36) waits on sneak's review. README corrections:
|
||||||
|
the Getting Started comments say when the library refreshes; most, not all,
|
||||||
|
Makefile targets call a script; `script/lint` and `script/test` have no host
|
||||||
|
path, though `yarn test` does; the SRP handshake uses `fast-srp-hap`, outside
|
||||||
|
`crypto/`, and a thumbnail upload's MD5 uses `node:crypto`; the SRP password
|
||||||
|
is the first 16 bytes of a 32-byte subkey; the key attributes and token come
|
||||||
|
after SRP, after the TOTP code SRP may ask for, or after the email OTP that
|
||||||
|
replaces SRP when the account has email MFA on, and quak cannot answer a
|
||||||
|
passkey; the auth token is sent as URL-safe base64 with padding; every
|
||||||
|
`TypeError` is retried; each download attempt writes its own temporary files,
|
||||||
|
two for a live photo, and only the attempt that completes renames them into
|
||||||
|
place; `runMetadataBackup` records a failed download in the file's JSON; a
|
||||||
|
second `client.logout()` does not throw; `quak logout` with no session exits
|
||||||
|
0, and names the cache directory only when it knows it; `login`, `whoami` and
|
||||||
|
`logout` open no library; `--exif` records XMP and, for a JPEG, EXIF, and no
|
||||||
|
IPTC; which commands take `--json`; a failed ML data request may not have been
|
||||||
|
retried; the thumbnail fixer is not limited to baseline JPEG; `quak backup`
|
||||||
|
still fetches ML data; a backup's JSON holds the basic metadata fields quak
|
||||||
|
keeps and the private and public magic metadata, not every decrypted field,
|
||||||
|
and the README no longer lists what the magic metadata holds; the default
|
||||||
|
cache directory is the per-user one, not an XDG path on macOS; pinned
|
||||||
|
originals can exceed `cacheOriginalsMaxBytes`; which tests cover which
|
||||||
|
operations; a default read is only as current as the last refresh whose
|
||||||
|
requests all succeeded; `fresh()` and `lib.backup()` join a refresh already
|
||||||
|
running; a `Photo` has no `thumbnailPath` or `originalPath`; and `408` and
|
||||||
|
`429` are retried.
|
||||||
|
|
||||||
|
- 2026-09-28: Tested the live-photo writer's fsyncs (issue 130). A test checks
|
||||||
|
that the image's and the video's temp files are fsynced before either is
|
||||||
|
renamed into place, and the directory after both renames, as the `writeAtomic`
|
||||||
|
test does for one file.
|
||||||
|
|
||||||
|
- 2026-09-23: Live photos are stored as their image and their video (issue 107).
|
||||||
|
A live photo, which Ente stores as one ZIP, is unpacked as it downloads into
|
||||||
|
`<fileID>.<ext>` for the image and for the video, each with its extension from
|
||||||
|
the ZIP, beside `<fileID>.livephoto.json`, which names the two. Both are
|
||||||
|
checked against the recorded hash and renamed into place only when both are
|
||||||
|
complete. A ZIP holding a second image or video, or whose two files come to
|
||||||
|
more than 20 times its size plus 16 MiB, is refused. The backup and the
|
||||||
|
content cache count the live photo as stored only with both files, the album
|
||||||
|
folder links both, `quak get` writes both, and the content result gives the
|
||||||
|
video as `videoPath`. A ZIP an earlier version stored is replaced on the next
|
||||||
|
backup run, and removed from the cache when the library opens it.
|
||||||
|
|
||||||
|
- 2026-09-23: Settled the package metadata (issue 6). quak is not published, so
|
||||||
|
`package.json` is marked `"private": true` and the `files` field is gone.
|
||||||
|
`engines.node` is `>=22`, the major version `script/bootstrap` and the
|
||||||
|
`Dockerfile` use. An `exports` map makes `.` and `./package.json` the only
|
||||||
|
importable paths; `runMetadataBackup`, the thumbnail helpers and their types
|
||||||
|
stay internal to the CLI.
|
||||||
|
|
||||||
|
- 2026-09-23: Brought the README and this file in line with the tree (issue
|
||||||
|
111). The layout lists `src/library/` and the other source files, the backup
|
||||||
|
layout names `failures.json` and the optional `thumbnails/`, "Opening a
|
||||||
|
library" says what happens when the first refresh fails, the Testing section
|
||||||
|
gives the 60-second hard cap and 20-second target for `make test` and names
|
||||||
|
the 90-second `timeout` in the `Dockerfile` as the backstop for a hung test,
|
||||||
|
and "Tag v1.0.0" is no longer listed as the next step.
|
||||||
|
|
||||||
|
- 2026-09-23: Tested `quak login` and `backup-metadata --exif` (issue 110).
|
||||||
|
`loginCommand` takes its login function and its prompts from `CliContext`, and
|
||||||
|
`bin/quak.ts` passes `Client.login` and the terminal prompts. Tests cover a
|
||||||
|
login from `QUAK_EMAIL` and `QUAK_PASSWORD` with no prompt, the TOTP prompt, a
|
||||||
|
failed login, and the saved session's modes, and show that `--exif` and
|
||||||
|
`--all` each turn on EXIF extraction and that it is off without them.
|
||||||
|
|
||||||
|
- 2026-09-23: `quak backup` writes each original once and no longer fills the
|
||||||
|
cache (issue 106). An original fetched for a backup is written by the download
|
||||||
|
writer straight into the backup's `originals/`, and the content cache records
|
||||||
|
it there instead of keeping its own copy; one the cache already held is still
|
||||||
|
copied. `quak backup` opens its library with the thumbnail and originals
|
||||||
|
precache off.
|
||||||
|
|
||||||
|
- 2026-09-23: `backup-metadata`, `helper list-missing-thumbnails` and
|
||||||
|
`helper fix-missing-thumbnails` refresh before they answer (issue 100). Each
|
||||||
|
awaits `lib.fresh()` before reading, so a file added since the cache was
|
||||||
|
written is included, and a failed refresh prints one line and exits 1 instead
|
||||||
|
of answering from a stale or empty cache. The README lists them among the
|
||||||
|
commands that refresh first.
|
||||||
|
|
||||||
|
- 2026-09-23: `quak backup` waits for the server refresh and fails when it fails
|
||||||
|
(issue 99). `lib.backup()` joins a refresh already running or starts one, as
|
||||||
|
`fresh()` does, and rejects before touching any file when it fails, leaving
|
||||||
|
`failures.json` as it was, so `quak backup` prints the error as one line and
|
||||||
|
exits 1 instead of backing up the previous run's file list, or nothing, and
|
||||||
|
exiting 0.
|
||||||
|
|
||||||
|
- 2026-09-23: CLI errors print a message instead of a stack trace (issue 102).
|
||||||
|
An error a command throws is printed as one `quak: MESSAGE` line on stderr and
|
||||||
|
the CLI exits 1 once output has drained. The wrapper that does this moved from
|
||||||
|
`bin/quak.ts` to `src/cli-run.ts`, and `bin/quak.ts` now awaits
|
||||||
|
`program.parseAsync()`.
|
||||||
|
|
||||||
|
- 2026-09-23: Opening a library no longer deletes another process's download in
|
||||||
|
progress (issue 105). The download writer's temp files are named
|
||||||
|
`.quak-<pid>-<random>.tmp`, and `removeLeftoverTempFiles`, moved from the
|
||||||
|
backup into the download module, deletes a `.quak-*.tmp` file only when the
|
||||||
|
process ID in its name is no longer running. The content cache calls it at
|
||||||
|
`open()` for `originals/` and `thumbnails/`, the backup as before.
|
||||||
|
|
||||||
|
- 2026-09-23: Re-vendored the lint and test setup from the template (issue 96).
|
||||||
|
Linting and testing are the `lint` and `test` phases of the `Dockerfile`;
|
||||||
|
`script/lint` and `script/test` each build one with `--no-cache`, and the last
|
||||||
|
stage compiles and depends on both, so `script/cibuild` is one build.
|
||||||
|
`Dockerfile.lint`, `CHECK_EPOCH`, `LINT_EPOCH` and the tests that checked them
|
||||||
|
are gone; `REPO_POLICIES.md` is re-copied.
|
||||||
|
|
||||||
|
- 2026-09-23: Stopped `helper fix-missing-thumbnails` retrying files the server
|
||||||
|
always refuses (issue 109). Both thumbnail helpers skip a file another account
|
||||||
|
owns without fetching it. The fixer skips a file whose recorded thumbnail size
|
||||||
|
is 0 or unknown before downloading it, and otherwise tries smaller encodings
|
||||||
|
(720 px quality 50 down to 160 px quality 20) until the encrypted thumbnail is
|
||||||
|
no larger than that size, skipping the file if none fits.
|
||||||
|
|
||||||
|
- 2026-09-23: Tested the live-photo hash check's error paths (issue 117). Tests
|
||||||
|
download a live photo whose ZIP names an unknown compression method, one whose
|
||||||
|
ZIP has no image entry and one with no video entry, and check that nothing is
|
||||||
|
stored and the error names the file ID; the unreadable one is not retried.
|
||||||
|
|
||||||
|
- 2026-09-23: `quak logout` ends the session on the server (issue 108). It calls
|
||||||
|
`POST /users/logout` through the new `Client.logoutOnServer()`, then deletes
|
||||||
|
`session.json` even when that call fails, says so and exits 1. It prints the
|
||||||
|
account's cache directory and says it still holds decrypted data. The default
|
||||||
|
cache path is now `defaultCacheDirectory()` in the library, shared with
|
||||||
|
`Library.open`.
|
||||||
|
|
||||||
|
- 2026-09-23: Fixed the backup's per-collection folders (issue 103). Two files
|
||||||
|
in one collection with the same title, and two collections with the same name,
|
||||||
|
each get their ID added to the name (`IMG_0001 (12345).JPG`, `Trip (10)/`), so
|
||||||
|
none replaces another's symlink or JSON. Each run removes symlinks into
|
||||||
|
`originals/` for files no longer in the collection, and the folders of deleted
|
||||||
|
or renamed collections, leaving anything else in `collections/` alone. The
|
||||||
|
README backup layout states the naming rule.
|
||||||
|
|
||||||
|
- 2026-09-23: Checked downloaded originals against their recorded content hash
|
||||||
|
(issue 68). `downloadFile`, which `quak get`, the content cache and backup all
|
||||||
|
use, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and
|
||||||
|
stores nothing on a mismatch, failing with an error naming the file ID. A live
|
||||||
|
photo ZIP is unpacked as it streams with `fflate` and its image and video
|
||||||
|
hashed separately as `<imageHash>:<videoHash>`. `decryptFile` reads older
|
||||||
|
clients' `imageHash` and `videoHash` fields for live photos. A file with no
|
||||||
|
recorded hash is stored unchecked.
|
||||||
|
|
||||||
|
- 2026-09-23: Kept one account's cache from mixing with another's (issue 104).
|
||||||
|
When `metadata.json` in the cache directory was written for a different,
|
||||||
|
non-zero user ID than the client's, `Library.open` deletes it and `mldata/`
|
||||||
|
and starts empty, so the first refresh enumerates from 0. This only happens
|
||||||
|
with `--cache-dir` or an explicit `cacheDirectory`; the default path already
|
||||||
|
includes the user ID. A test opens one account's cache as another account.
|
||||||
|
|
||||||
|
- 2026-09-23: `backup-metadata` no longer stops on one failed ML data request
|
||||||
|
(issue 101). Each request of up to 200 files is tried on its own; a failed one
|
||||||
|
is logged, its files are written with the reason in `mlDataError`, and the
|
||||||
|
command exits 1 once the dump is complete. `fetchMLData`, which only this
|
||||||
|
command used, is gone; the command calls `fetchMLDataBatch` per batch.
|
||||||
|
- 2026-09-23: Single-sourced the version string (issue 5). `package.json` is the
|
||||||
|
only place it is written: `src/index.ts` imports it for `VERSION` and
|
||||||
|
`bin/quak.ts` passes `VERSION` to commander. tsc copies `package.json` to
|
||||||
|
`dist/package.json`, so the import resolves from the built output too, and
|
||||||
|
`script/build` runs the built CLI with `--version` to prove it. A test checks
|
||||||
|
that `VERSION` and `quak --version` both equal the `package.json` version.
|
||||||
|
- 2026-09-23: Tested that `Library.close()` waits for the originals precache
|
||||||
|
(issue 93). The test that holds a precache fetch open while `close()` runs now
|
||||||
|
runs once with only the thumbnail fill and once with only the originals fill,
|
||||||
|
so dropping either wait from `Precache.close()` fails a test.
|
||||||
|
- 2026-09-23: Fixed two intermittently failing library tests (issue 90).
|
||||||
|
`Library.close()` now returns a promise that resolves once an in-flight
|
||||||
|
refresh (including its cache write), the ML data fetch and running precache
|
||||||
|
sweeps have finished; the library tests await it, so `afterEach` no longer
|
||||||
|
removes the cache directory while something is still writing into it. The
|
||||||
|
precache test waits for both fills to report "done" instead of for its stub
|
||||||
|
source to be called, which happened before the cache recorded the file.
|
||||||
|
- 2026-09-23: Pinned three guards reviewers found untested (issue 89). The
|
||||||
|
download idle deadline's timer is unref'd, so it can never keep the process
|
||||||
|
alive, and a test checks no timer is left after a download completes or fails.
|
||||||
|
A test covers the rejection of `#` in a request path. The EXIF scan compares
|
||||||
|
the `Exif` header only in an APP1 segment of length 8 or more, so it never
|
||||||
|
reads the next segment's bytes, with a test for a short one.
|
||||||
|
- 2026-09-23: Made the download deadline an idle deadline (issue 24).
|
||||||
|
`downloadTimeoutMs` now aborts a file or thumbnail download only after no
|
||||||
|
bytes have arrived for that long, default 60 seconds, instead of bounding the
|
||||||
|
whole transfer at 10 minutes, so a slow download that keeps making progress
|
||||||
|
completes. A download that fails before reading the whole body cancels it, so
|
||||||
|
a failed file no longer holds its connection.
|
||||||
|
- 2026-09-23: Every `ApiClient` request URL is now built by one function next to
|
||||||
|
the class (issue 18), so a self-hosted `apiOrigin` with a base path keeps it
|
||||||
|
on every request, a path works with or without a leading slash, and query
|
||||||
|
parameters are percent-encoded. A path containing `?` or `#` is rejected with
|
||||||
|
an error instead of being silently cut.
|
||||||
|
- 2026-09-23: Made the CLI testable and tested it (issue 12). The command bodies
|
||||||
|
moved from `bin/quak.ts` into `src/cli-commands.ts` as functions that take
|
||||||
|
their options and a context (output streams, session directory, cache
|
||||||
|
directory, session loader) and return an exit code; `bin/quak.ts` only wires
|
||||||
|
them to commander and exits with the code once stdout and stderr have drained,
|
||||||
|
so nothing below it calls `process.exit`. `test/cli/commands.test.ts` drives
|
||||||
|
them with a fake client: session file modes, logout, the missing and corrupt
|
||||||
|
session paths, and the output and exit code of `whoami`, `collections`,
|
||||||
|
`files`, `get`, `get-thumb`, `backup` and `helper list-missing-thumbnails`.
|
||||||
|
- 2026-09-23: Hardened the backup tree's atomic copy (issue 22). `copyAtomic`
|
||||||
|
fsyncs its temp file before the rename and the directory after it, through the
|
||||||
|
download writer's `fsyncPath`; each backup run deletes `.quak-backup-*.tmp`
|
||||||
|
files whose process is no longer running. The README backup layout names the
|
||||||
|
temp files and states that the rename replaces a symlink and takes the temp
|
||||||
|
file's permissions. Added tests for a missing and an unwritable destination
|
||||||
|
directory for `downloadFile` and `downloadThumbnail`.
|
||||||
|
- 2026-09-23: Hardened the JPEG EXIF scan behind `backup-metadata --exif` (issue
|
||||||
|
11). Every segment length is checked against the remaining bytes and lengths
|
||||||
|
under 2 stop the scan, so a truncated or corrupt original can neither throw
|
||||||
|
nor loop. A malformed or unparseable EXIF segment is recorded as
|
||||||
|
`imageMetadata.exifError`, and a failure to read the original as
|
||||||
|
`imageMetadataError` in the per-file JSON, instead of the field being left
|
||||||
|
out.
|
||||||
|
- 2026-09-22: Hardened the retry classifier (issue 80). A `POST` or `PUT` is
|
||||||
|
replayed only when every errno in the cause chain is a connect errno, and it
|
||||||
|
no longer follows redirects. `getRetryOptions()` returns a copy. Tests pin
|
||||||
|
every errno the classifier names, the cause-chain depth limit, cycle
|
||||||
|
termination, and a fresh deadline per attempt for every retrying entry point.
|
||||||
|
The README's endpoint list is the one place that names the requests the replay
|
||||||
|
rule covers.
|
||||||
|
- 2026-09-22: Stopped `make test` collecting tests from checkouts nested under
|
||||||
|
`.claude/` (issue 25). vitest ignores `.gitignore` when finding tests, so a
|
||||||
|
nested checkout ran the whole suite again; `vitest.config.ts` now adds
|
||||||
|
`.claude/**` to vitest's default excludes, and
|
||||||
|
`test/packaging/nested-checkout.test.ts` plants a nested checkout in a temp
|
||||||
|
directory and fails if vitest would collect it.
|
||||||
|
- 2026-09-22: Dropped the deprecated `@types/libsodium-wrappers-sumo` stub from
|
||||||
|
`devDependencies` (issue 27). It shipped no declarations; the types come from
|
||||||
|
`libsodium-wrappers-sumo` itself. `yarn.lock` regenerated by `yarn remove`.
|
||||||
|
- 2026-09-22: Hardened the client session lifecycle (issue 10).
|
||||||
|
`Client.fromJSON` checks every snapshot field and each key's decoded length
|
||||||
|
and names the bad field; `toJSON` reads the token through
|
||||||
|
`ApiClient.getAuthToken` and throws when there is none; `logout` zeroes the
|
||||||
|
key buffers, and `collectionsSince` re-checks for logout after its request so
|
||||||
|
it never decrypts with zeroed keys. The CLI reports a corrupt session file
|
||||||
|
separately from a missing one (`src/cli-session.ts`).
|
||||||
|
- 2026-09-22: Sanitized file names taken from server metadata (issue 9). A new
|
||||||
|
`src/filename.ts` holds the one sanitizer, used by `quak get`/`get-thumb`
|
||||||
|
without `--out`, `downloadFile`/`downloadThumbnail` without `outPath`, and the
|
||||||
|
backup and metadata backup trees; it removes separators, control characters,
|
||||||
|
leading dots and Windows device names, and falls back to a name built from the
|
||||||
|
ID for an empty title. Originals-cache extensions are letters and digits only,
|
||||||
|
else `.bin`. A user-supplied path is used as is. `decryptFile` reads a missing
|
||||||
|
or non-string title as "" and rejects metadata that is not a JSON object.
|
||||||
|
- 2026-09-22: Rewrote the README API reference (and the Getting Started / usage
|
||||||
|
snippets) to match the shipped cache/API library on `next` (issue 53, issue
|
||||||
|
13). Documented `Library.open` and its options, the default-read vs `fresh()`
|
||||||
|
distinction (and that the CLI's read commands are fresh), the record types and
|
||||||
|
`snapshot()`/`subscribe()`, the `albums`/`photos`/`timeline` read surface,
|
||||||
|
`Photo` content methods, `thumbnails.ensure`, the `mldata` search surface,
|
||||||
|
`backup()`, the three request pools (10/5/25), and the on-disk cache layout;
|
||||||
|
noted the deferred content-hash integrity check (issue 68). Docs-only; no code
|
||||||
|
changed.
|
||||||
|
- 2026-09-22: Added resumable, deletion-aware enumeration to `Client` (issue 38,
|
||||||
|
closes issue 7). `collectionsSince`/`filesSince` take a starting cursor,
|
||||||
|
decrypt live records, surface tombstoned ids in a separate `deleted` list (a
|
||||||
|
tombstone has nothing to decrypt, so it is a bare id, not a hollow record),
|
||||||
|
and return the max `updationTime` seen as the cursor to resume from.
|
||||||
|
`filesSince` refuses to loop when the diff reports `hasMore` without advancing
|
||||||
|
the cursor (issue 7). `listCollections`/`listFiles` are now thin wrappers that
|
||||||
|
enumerate from `sinceTime: 0` and drop deletions, so existing callers are
|
||||||
|
unaffected.
|
||||||
|
- 2026-09-22: Carried file size, thumbnail size, and the deletion flag through
|
||||||
|
`decryptFile` (issue 37, foundation for the cache/API design). Live files now
|
||||||
|
populate `file.size`/`thumbnail.size` from the server's `info` (left
|
||||||
|
`undefined` when the server omits it), and `isDeleted` is carried from the
|
||||||
|
diff row onto `EnteFile`. No caller change: `listFiles` still filters deleted
|
||||||
|
rows before decrypting. Surfacing a tombstone through decryption belongs to
|
||||||
|
the enumeration unit (issue 38).
|
||||||
|
- 2026-08-10: Made `lint-once.test.ts` enforce what its header claims. It walked
|
||||||
|
`make check` only, so it never read `Dockerfile` — the image CI builds through
|
||||||
|
`script/cibuild` — and a second `prettier --check .` could be added there with
|
||||||
|
the suite staying green. The walk now also starts at
|
||||||
|
`.gitea/workflows/check.yml` and follows its `run:` steps, so the graph under
|
||||||
|
test is the one CI executes rather than the one someone assumed it executes.
|
||||||
|
The lockfile assertion was a substring check against the whole of
|
||||||
|
`script/bootstrap`, which has two install sites and so reported the branch the
|
||||||
|
containers never take; the two branches are now resolved separately and every
|
||||||
|
`yarn install` in each is required to be `--frozen-lockfile`. Prettier is
|
||||||
|
counted per occurrence instead of per line, so two invocations chained with
|
||||||
|
`&&` no longer read as one, and edges are followed on counted lines instead of
|
||||||
|
being skipped. Every way for the walk to reach nothing — an unknown target, an
|
||||||
|
unknown script, a missing file, a node with no commands, an unknown node kind
|
||||||
|
— is a thrown error rather than a quiet zero. Every assertion in the file was
|
||||||
|
mutation-tested individually.
|
||||||
|
- 2026-08-10: Stopped `make check` running `prettier --check .` twice. Since
|
||||||
|
linting moved into Docker, the duplicate was one container pass and one host
|
||||||
|
pass of the same check: `script/lint` builds `Dockerfile.lint`, which runs
|
||||||
|
prettier as a build step, and `script/check` then called `script/fmt-check` as
|
||||||
|
well. The host call is gone from `script/check` and from `script/precommit`;
|
||||||
|
the container keeps checking formatting, because a successful
|
||||||
|
`Dockerfile.lint` build is what CI treats as proof of a clean tree, and it is
|
||||||
|
also what still fails the pre-commit hook on a badly formatted tree.
|
||||||
|
`script/fmt-check` survives as a standalone entrypoint, whose verdict cannot
|
||||||
|
drift from the container's. A test walks the invocation graph from each
|
||||||
|
entrypoint — through the Makefile shims, the `script/` calls and the
|
||||||
|
`docker build` — and asserts the prettier count, so the duplication cannot
|
||||||
|
come back unnoticed.
|
||||||
|
- 2026-08-10: Moved all linting into Docker. `script/lint` builds a new root
|
||||||
|
`Dockerfile.lint`, which copies the repo into the digest-pinned node image and
|
||||||
|
runs eslint and prettier as build steps, so a successful build is a clean
|
||||||
|
lint; no host lint path remains and `yarn lint` is gone from `package.json`. A
|
||||||
|
fail-closed `LINT_EPOCH` guard stops Docker serving the linter layers from
|
||||||
|
cache, which is how a lint build returns success in under a second having
|
||||||
|
linted nothing. The lint stage inside `Dockerfile` and its `COPY --from=lint`
|
||||||
|
ordering hack are gone: that image now runs `make test` and `make build` only,
|
||||||
|
because `script/check` calls `script/lint` and running it in a container would
|
||||||
|
mean docker inside docker. `script/cibuild` builds the lint image first, then
|
||||||
|
the test and build image.
|
||||||
|
- 2026-08-09: Made `make docker` green and policy-conformant. Multi-stage
|
||||||
|
Dockerfile: a lint stage runs `make fmt-check` and `make lint`, and the check
|
||||||
|
stage takes a `COPY --from=lint` dependency on it before running `make check`
|
||||||
|
and `make build`. `CHECK_EPOCH` and a fail-closed guard stop Docker serving
|
||||||
|
those two layers from cache, which is what let a build report success without
|
||||||
|
running the suite. `script/projectname` says `quak`, so the image is tagged
|
||||||
|
`quak`; `script/bootstrap` updates apt lists before installing, so a Debian
|
||||||
|
base works; `.dockerignore` no longer ships the compiled binary, the caches or
|
||||||
|
agent worktrees into the build context, and keeps `.gitignore` in it for
|
||||||
|
prettier.
|
||||||
|
- 2026-08-09: Fixed the TypeScript build. `rootDir` is the repo root, so `bin/`
|
||||||
|
compiles alongside `src/` instead of failing with TS6059; output is
|
||||||
|
`dist/src/` and `dist/bin/`, which is where `main`, `types` and `bin.quak` now
|
||||||
|
point. `script/build` verifies the declared entrypoints exist after the
|
||||||
|
compiler runs and makes the CLI executable, the Dockerfile runs `make build`
|
||||||
|
as well as `make check`, and a `quak` script makes the README's
|
||||||
|
`yarn quak <command>` examples work.
|
||||||
|
- 2026-08-09: Retry policy: no retry on 4xx (except `408` and `429`),
|
||||||
|
exponential backoff with full jitter on 5xx, transport failures and truncated
|
||||||
|
transfers, under per-attempt deadlines that cover the response body as well as
|
||||||
|
the request. Downloads retry request, stream consumption and decryption as one
|
||||||
|
unit; `postJSON` and `putJSON` are replayed only when the connection was never
|
||||||
|
established.
|
||||||
|
- 2026-08-09: Downloads verify the secretstream terminated on `TAG_FINAL` and
|
||||||
|
write output atomically: a truncated body is rejected instead of landing on
|
||||||
|
disk as a short file, and plaintext is staged in a sibling temp file and
|
||||||
|
renamed into place, so a failed download leaves the destination untouched.
|
||||||
|
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
|
||||||
|
shims, README Entrypoints section
|
||||||
|
- 2026-06-10: Decrypted collections shared by other users (sealed-box keys);
|
||||||
|
listCollections drops deleted-collection tombstones.
|
||||||
|
- 2026-06-10: Login hardening: dual-2FA empty-string fields handled, TOTP
|
||||||
|
preferred when a passkey is also enrolled, interactive input via
|
||||||
|
@inquirer/prompts.
|
||||||
|
- 2026-06-10: Replaced sharp with pure JS (jpeg-js + exif-reader); added
|
||||||
|
single-binary bun build and make install.
|
||||||
|
- 2026-06-09: Added backup-metadata command (ML data always included, --exif
|
||||||
|
opt-in); rewrote README to match the implementation; added thumbnail helper
|
||||||
|
tests.
|
||||||
|
- 2026-05-13: Full CLI surface: login, backup with dedup symlink layout,
|
||||||
|
collections, files, get, get-thumb, thumbnail repair helpers.
|
||||||
|
- 2026-05-13: Client OO API with literate usage tests; file download and
|
||||||
|
decryption; all three metadata layers decrypted and persisted; renamed quack
|
||||||
|
to quak.
|
||||||
|
- 2026-05-11: SRP login flow (email OTP + TOTP) and ApiClient.
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Future desktop client, separate repo:
|
- Future desktop client, separate repo:
|
||||||
|
|||||||
@@ -123,6 +123,7 @@ program
|
|||||||
.description("Download and decrypt a single file")
|
.description("Download and decrypt a single file")
|
||||||
.argument("<fileID>", "File ID (from `quak files`)")
|
.argument("<fileID>", "File ID (from `quak files`)")
|
||||||
.option("--out <path>", "Output file path")
|
.option("--out <path>", "Output file path")
|
||||||
|
.option("--collection <id>", "Accepted for compatibility; ignored")
|
||||||
.action((fileID: string, opts: { out?: string }) =>
|
.action((fileID: string, opts: { out?: string }) =>
|
||||||
run(getCommand(context(), fileID, opts)),
|
run(getCommand(context(), fileID, opts)),
|
||||||
);
|
);
|
||||||
@@ -132,6 +133,7 @@ program
|
|||||||
.description("Download and decrypt a thumbnail")
|
.description("Download and decrypt a thumbnail")
|
||||||
.argument("<fileID>", "File ID (from `quak files`)")
|
.argument("<fileID>", "File ID (from `quak files`)")
|
||||||
.option("--out <path>", "Output file path")
|
.option("--out <path>", "Output file path")
|
||||||
|
.option("--collection <id>", "Accepted for compatibility; ignored")
|
||||||
.action((fileID: string, opts: { out?: string }) =>
|
.action((fileID: string, opts: { out?: string }) =>
|
||||||
run(getThumbCommand(context(), fileID, opts)),
|
run(getThumbCommand(context(), fileID, opts)),
|
||||||
);
|
);
|
||||||
|
|||||||
+1
-1
@@ -12,7 +12,7 @@
|
|||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": "^22.13.0 || >=23.5.0"
|
"node": ">=22"
|
||||||
},
|
},
|
||||||
"main": "./dist/src/index.js",
|
"main": "./dist/src/index.js",
|
||||||
"types": "./dist/src/index.d.ts",
|
"types": "./dist/src/index.d.ts",
|
||||||
|
|||||||
@@ -1,73 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/build-bin: run script/build, then bundle the built dist/ into one
|
|
||||||
# executable, bin/quak, which reports the version script/build stamped. Our own
|
|
||||||
# extension to scripts-to-rule-them-all.
|
|
||||||
#
|
|
||||||
# bun compiles the binary from a copy of its own executable. On Linux, nix's
|
|
||||||
# bun loads through a C library in the nix store, and so does a binary compiled
|
|
||||||
# from it. nix-collect-garbage can delete that library, and the binary then no
|
|
||||||
# longer starts. On Linux this script therefore compiles with bun's own release
|
|
||||||
# build, which loads through the system's C library. It downloads that build
|
|
||||||
# and checks it against the hash below before running it.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
# bun 1.3.13, 2026-11-11, and the SHA-256 of each release archive below. The
|
|
||||||
# x86-64 archives are the baseline builds, which also run on CPUs without AVX2.
|
|
||||||
# The musl archives are for Linux systems that use the musl C library, such as
|
|
||||||
# Alpine.
|
|
||||||
BUN_VERSION="1.3.13"
|
|
||||||
BUN_LINUX_X64_BASELINE_SHA256="9d8a24292a7068090205daac0a5a223f5f69736f5287e37bf88d3b4031edc750"
|
|
||||||
BUN_LINUX_X64_MUSL_BASELINE_SHA256="88ca7c7ad235b498f549eea2f770f434e9f0f5e9ba95168a2d3a1f235184c394"
|
|
||||||
BUN_LINUX_AARCH64_SHA256="70bae41b3908b0a120e1e58c5c8af30e74afae3b8d11b0d3fdd8e787ddfb4b22"
|
|
||||||
BUN_LINUX_AARCH64_MUSL_SHA256="5385e978107ce4934298d8d6afe9bfbb898683f6cc23e6753a0da60bc60c5b81"
|
|
||||||
|
|
||||||
# compile_with_downloaded_bun <archive-name> <sha256>
|
|
||||||
compile_with_downloaded_bun() {
|
|
||||||
tmp="$(mktemp -d)"
|
|
||||||
trap 'rm -rf "$tmp"' EXIT
|
|
||||||
curl -fsSL -o "$tmp/$1.zip" \
|
|
||||||
"https://github.com/oven-sh/bun/releases/download/bun-v$BUN_VERSION/$1.zip"
|
|
||||||
echo "$2 $tmp/$1.zip" | sha256sum -c -
|
|
||||||
unzip -q "$tmp/$1.zip" -d "$tmp"
|
|
||||||
"$tmp/$1/bun" build dist/bin/quak.js --compile --outfile bin/quak
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
"$ROOT/script/build"
|
|
||||||
platform="$(uname -s)-$(uname -m)"
|
|
||||||
# musl's ldd begins its version output with "musl libc"; glibc's does not.
|
|
||||||
if ldd --version 2>&1 | grep -q '^musl libc'; then
|
|
||||||
platform="$platform-musl"
|
|
||||||
fi
|
|
||||||
case "$platform" in
|
|
||||||
Linux-x86_64)
|
|
||||||
compile_with_downloaded_bun bun-linux-x64-baseline \
|
|
||||||
"$BUN_LINUX_X64_BASELINE_SHA256"
|
|
||||||
;;
|
|
||||||
Linux-x86_64-musl)
|
|
||||||
compile_with_downloaded_bun bun-linux-x64-musl-baseline \
|
|
||||||
"$BUN_LINUX_X64_MUSL_BASELINE_SHA256"
|
|
||||||
;;
|
|
||||||
Linux-aarch64)
|
|
||||||
compile_with_downloaded_bun bun-linux-aarch64 \
|
|
||||||
"$BUN_LINUX_AARCH64_SHA256"
|
|
||||||
;;
|
|
||||||
Linux-aarch64-musl)
|
|
||||||
compile_with_downloaded_bun bun-linux-aarch64-musl \
|
|
||||||
"$BUN_LINUX_AARCH64_MUSL_SHA256"
|
|
||||||
;;
|
|
||||||
Linux-*)
|
|
||||||
echo "build-bin: bun has no release build for Linux on $(uname -m)" >&2
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
nix-shell -p bun --run \
|
|
||||||
"bun build dist/bin/quak.js --compile --outfile bin/quak"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
+1
-8
@@ -354,13 +354,7 @@ export class ApiClient {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// `contentMD5` is the value given to `getUploadURL`. The server signs it
|
async putFile(presignedURL: string, data: Uint8Array): Promise<void> {
|
||||||
// into the URL, and storage refuses a PUT without that `Content-MD5`.
|
|
||||||
async putFile(
|
|
||||||
presignedURL: string,
|
|
||||||
data: Uint8Array,
|
|
||||||
contentMD5: string,
|
|
||||||
): Promise<void> {
|
|
||||||
// Idempotent despite being a write: a presigned PUT stores one whole
|
// Idempotent despite being a write: a presigned PUT stores one whole
|
||||||
// object at one key in one request, so replaying it either overwrites
|
// object at one key in one request, so replaying it either overwrites
|
||||||
// the same bytes or lands them for the first time. There is no partial
|
// the same bytes or lands them for the first time. There is no partial
|
||||||
@@ -376,7 +370,6 @@ export class ApiClient {
|
|||||||
headers: {
|
headers: {
|
||||||
"Content-Type": "application/octet-stream",
|
"Content-Type": "application/octet-stream",
|
||||||
"Content-Length": String(data.length),
|
"Content-Length": String(data.length),
|
||||||
"Content-MD5": contentMD5,
|
|
||||||
},
|
},
|
||||||
body: data,
|
body: data,
|
||||||
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
||||||
|
|||||||
+7
-10
@@ -1,18 +1,15 @@
|
|||||||
// The CLI's log level, and the one function the commands write stderr lines
|
// The CLI's log level, and the one function the commands write stderr lines
|
||||||
// through.
|
// through.
|
||||||
//
|
//
|
||||||
// Every line quak logs goes to stderr, at one of six levels, from the
|
// Every line quak prints on stderr is a log line at one of six levels, from
|
||||||
// quietest: fatal, error, warning, info, debug, trace. A run prints the lines
|
// the quietest: fatal, error, warning, info, debug, trace. A run prints the
|
||||||
// at its own level and at every quieter one. Its level is info unless `-v` or
|
// lines at its own level and at every quieter one. Its level is info unless
|
||||||
// `-q` asks for another (see "What quak prints" in the README). A command that
|
// `-v` or `-q` asks for another (see "What quak prints" in the README). Every
|
||||||
// runs and exits non-zero logs a fatal line saying why, so even `-q`, which
|
// command that exits non-zero logs a fatal line saying why, so even `-q`, which
|
||||||
// prints only fatal lines, never leaves such an exit unexplained. An error
|
// prints only fatal lines, never leaves a non-zero exit unexplained. An error
|
||||||
// line reports something that failed, such as a file or an album, and a
|
// line reports something that failed, such as a file or an album, and a
|
||||||
// warning something that did not fail but needs the user's attention. stdout
|
// warning something that did not fail but needs the user's attention. stdout
|
||||||
// carries a command's output and the prompts of `login`, and is not a log.
|
// carries only a command's output and is not a log.
|
||||||
// Two things on stderr are not log lines: the `error:` message for a
|
|
||||||
// command line that quak cannot parse, and the usage printed when no command
|
|
||||||
// is given.
|
|
||||||
//
|
//
|
||||||
// The library itself writes no lines: its layers report through callbacks, and
|
// The library itself writes no lines: its layers report through callbacks, and
|
||||||
// the CLI logs what they report. `logRequests` gives the API client options
|
// the CLI logs what they report. `logRequests` gives the API client options
|
||||||
|
|||||||
+1
-1
@@ -325,7 +325,7 @@ export const fixMissingThumbnails = async (
|
|||||||
ciphertext.length,
|
ciphertext.length,
|
||||||
md5,
|
md5,
|
||||||
);
|
);
|
||||||
await api.putFile(url, ciphertext, md5);
|
await api.putFile(url, ciphertext);
|
||||||
await api.updateThumbnail(file.id, objectKey, toBase64(header));
|
await api.updateThumbnail(file.id, objectKey, toBase64(header));
|
||||||
|
|
||||||
log(`[${collectionName}] Thumbnail uploaded for ${title}`);
|
log(`[${collectionName}] Thumbnail uploaded for ${title}`);
|
||||||
|
|||||||
+4
-10
@@ -780,7 +780,7 @@ describe("ApiClient timeouts", () => {
|
|||||||
|
|
||||||
await client.getJSON("/a");
|
await client.getJSON("/a");
|
||||||
await client.postJSON("/b", {});
|
await client.postJSON("/b", {});
|
||||||
await client.putFile("https://s3.example/x", new Uint8Array([1]), "m");
|
await client.putFile("https://s3.example/x", new Uint8Array([1]));
|
||||||
await client.putJSON("/c", {});
|
await client.putJSON("/c", {});
|
||||||
|
|
||||||
for (const call of calls) {
|
for (const call of calls) {
|
||||||
@@ -825,8 +825,7 @@ describe("ApiClient timeouts", () => {
|
|||||||
[
|
[
|
||||||
"putFile",
|
"putFile",
|
||||||
() => new Response(null, { status: 200 }),
|
() => new Response(null, { status: 200 }),
|
||||||
(c) =>
|
(c) => c.putFile("https://s3.example/x", new Uint8Array([1])),
|
||||||
c.putFile("https://s3.example/x", new Uint8Array([1]), "m"),
|
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
"getFileStream",
|
"getFileStream",
|
||||||
@@ -1050,7 +1049,7 @@ describe("ApiClient error typing", () => {
|
|||||||
const client = new ApiClient({ fetch, retry: noWait });
|
const client = new ApiClient({ fetch, retry: noWait });
|
||||||
|
|
||||||
const err: unknown = await client
|
const err: unknown = await client
|
||||||
.putFile("https://s3.example/obj", new Uint8Array([1, 2]), "m")
|
.putFile("https://s3.example/obj", new Uint8Array([1, 2]))
|
||||||
.catch((e: unknown) => e);
|
.catch((e: unknown) => e);
|
||||||
|
|
||||||
expect(err).toBeInstanceOf(ApiError);
|
expect(err).toBeInstanceOf(ApiError);
|
||||||
@@ -1068,11 +1067,7 @@ describe("ApiClient error typing", () => {
|
|||||||
);
|
);
|
||||||
const client = new ApiClient({ fetch, retry: noWait });
|
const client = new ApiClient({ fetch, retry: noWait });
|
||||||
|
|
||||||
await client.putFile(
|
await client.putFile("https://s3.example/obj", new Uint8Array([1, 2]));
|
||||||
"https://s3.example/obj",
|
|
||||||
new Uint8Array([1, 2]),
|
|
||||||
"m",
|
|
||||||
);
|
|
||||||
expect(calls).toHaveLength(2);
|
expect(calls).toHaveLength(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -1282,7 +1277,6 @@ describe("ApiClient onRequest", () => {
|
|||||||
await client.putFile(
|
await client.putFile(
|
||||||
"https://s3.example/obj?X-Amz-Signature=abc",
|
"https://s3.example/obj?X-Amz-Signature=abc",
|
||||||
new Uint8Array([1, 2]),
|
new Uint8Array([1, 2]),
|
||||||
"m",
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(requests).toEqual([
|
expect(requests).toEqual([
|
||||||
|
|||||||
+2
-13
@@ -80,20 +80,13 @@ describe("ApiClient.putFile", () => {
|
|||||||
const client = new ApiClient({ fetch, authToken: "secret-tok" });
|
const client = new ApiClient({ fetch, authToken: "secret-tok" });
|
||||||
const data = new Uint8Array([1, 2, 3, 4, 5]);
|
const data = new Uint8Array([1, 2, 3, 4, 5]);
|
||||||
|
|
||||||
await client.putFile(
|
await client.putFile("https://s3.example.com/presigned?sig=abc", data);
|
||||||
"https://s3.example.com/presigned?sig=abc",
|
|
||||||
data,
|
|
||||||
"md5base64==",
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(calls[0]!.url).toBe("https://s3.example.com/presigned?sig=abc");
|
expect(calls[0]!.url).toBe("https://s3.example.com/presigned?sig=abc");
|
||||||
expect(calls[0]!.init?.method).toBe("PUT");
|
expect(calls[0]!.init?.method).toBe("PUT");
|
||||||
const headers = new Headers(calls[0]!.init?.headers as HeadersInit);
|
const headers = new Headers(calls[0]!.init?.headers as HeadersInit);
|
||||||
expect(headers.get("Content-Type")).toBe("application/octet-stream");
|
expect(headers.get("Content-Type")).toBe("application/octet-stream");
|
||||||
expect(headers.get("Content-Length")).toBe("5");
|
expect(headers.get("Content-Length")).toBe("5");
|
||||||
// The server signs the MD5 into the URL; storage refuses the PUT
|
|
||||||
// without it.
|
|
||||||
expect(headers.get("Content-MD5")).toBe("md5base64==");
|
|
||||||
// Must NOT leak auth headers to S3
|
// Must NOT leak auth headers to S3
|
||||||
expect(headers.has("X-Auth-Token")).toBe(false);
|
expect(headers.has("X-Auth-Token")).toBe(false);
|
||||||
expect(headers.has("X-Client-Package")).toBe(false);
|
expect(headers.has("X-Client-Package")).toBe(false);
|
||||||
@@ -106,11 +99,7 @@ describe("ApiClient.putFile", () => {
|
|||||||
const client = new ApiClient({ fetch });
|
const client = new ApiClient({ fetch });
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
client.putFile(
|
client.putFile("https://s3.example.com/bad", new Uint8Array(10)),
|
||||||
"https://s3.example.com/bad",
|
|
||||||
new Uint8Array(10),
|
|
||||||
"md5base64==",
|
|
||||||
),
|
|
||||||
).rejects.toThrow(/403/);
|
).rejects.toThrow(/403/);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
+2
-24
@@ -1,18 +1,12 @@
|
|||||||
/**
|
/**
|
||||||
* Tests for the retry options `bin/quak.ts` loads the saved session with
|
* Tests for the retry options `bin/quak.ts` loads the saved session with
|
||||||
* (`quak backup` gets the unattended ones, every other command the default),
|
* (`quak backup` gets the unattended ones, every other command the default),
|
||||||
* each with the callbacks that log its requests and retries, for `quak files`
|
* each with the callbacks that log its requests and retries, and for
|
||||||
* taking its collection ID as an optional argument, and for `quak get` and
|
* `quak files` taking its collection ID as an optional argument.
|
||||||
* `quak get-thumb` refusing `--collection`.
|
|
||||||
*
|
*
|
||||||
* Each test runs `bin/quak.ts`, as the smoke test does, with its session loader
|
* Each test runs `bin/quak.ts`, as the smoke test does, with its session loader
|
||||||
* replaced by one that records the options it is given and reports no session,
|
* replaced by one that records the options it is given and reports no session,
|
||||||
* so the command stops before it makes any request.
|
* so the command stops before it makes any request.
|
||||||
*
|
|
||||||
* The first load of the sources `bin/quak.ts` imports compiles them, and the
|
|
||||||
* first `init()` starts libsodium. On a loaded host that takes longer than a
|
|
||||||
* test's timeout, so both happen here, before the tests run. Each test's
|
|
||||||
* `vi.resetModules()` then only runs the compiled sources again.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { mkdtempSync, rmSync } from "node:fs";
|
import { mkdtempSync, rmSync } from "node:fs";
|
||||||
@@ -20,13 +14,8 @@ import { tmpdir } from "node:os";
|
|||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
|
import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { ApiClientOptions } from "../../src/api/client.js";
|
import type { ApiClientOptions } from "../../src/api/client.js";
|
||||||
import "../../src/cli-commands.js";
|
|
||||||
import { init } from "../../src/crypto/index.js";
|
|
||||||
import "../../src/index.js";
|
|
||||||
import { UNATTENDED_RETRY_OPTIONS } from "../../src/retry.js";
|
import { UNATTENDED_RETRY_OPTIONS } from "../../src/retry.js";
|
||||||
|
|
||||||
await init();
|
|
||||||
|
|
||||||
const loaded = vi.hoisted(() => [] as (ApiClientOptions | undefined)[]);
|
const loaded = vi.hoisted(() => [] as (ApiClientOptions | undefined)[]);
|
||||||
|
|
||||||
vi.mock("../../src/cli-session.js", () => ({
|
vi.mock("../../src/cli-session.js", () => ({
|
||||||
@@ -106,14 +95,3 @@ describe("bin/quak.ts files", () => {
|
|||||||
expect(await quak("files", "7")).toMatch(/^Not logged in/);
|
expect(await quak("files", "7")).toMatch(/^Not logged in/);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("bin/quak.ts get and get-thumb", () => {
|
|
||||||
it.each(["get", "get-thumb"])(
|
|
||||||
"%s refuses --collection as an unknown option",
|
|
||||||
async (command) => {
|
|
||||||
expect(await quak(command, "1", "--collection", "2")).toMatch(
|
|
||||||
/^error: unknown option '--collection'/,
|
|
||||||
);
|
|
||||||
},
|
|
||||||
);
|
|
||||||
});
|
|
||||||
@@ -569,14 +569,14 @@ describe("files", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("get and get-thumb", () => {
|
describe("get and get-thumb", () => {
|
||||||
it("get finds a file in any album", async () => {
|
it("get finds a file in any album without --collection", async () => {
|
||||||
const out = join(root, "diagram.png");
|
const out = join(root, "diagram.png");
|
||||||
expect(await getCommand(context(), "200", { out })).toBe(0);
|
expect(await getCommand(context(), "200", { out })).toBe(0);
|
||||||
expect(readFileSync(out)).toEqual(Buffer.alloc(7, 200));
|
expect(readFileSync(out)).toEqual(Buffer.alloc(7, 200));
|
||||||
expect(stderr.text).toBe(`7 bytes -> ${out}\n`);
|
expect(stderr.text).toBe(`7 bytes -> ${out}\n`);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("get-thumb finds a file in any album", async () => {
|
it("get-thumb finds a file in any album without --collection", async () => {
|
||||||
const out = join(root, "thumb.jpg");
|
const out = join(root, "thumb.jpg");
|
||||||
expect(await getThumbCommand(context(), "200", { out })).toBe(0);
|
expect(await getThumbCommand(context(), "200", { out })).toBe(0);
|
||||||
expect(readFileSync(out)).toEqual(Buffer.alloc(3, 200));
|
expect(readFileSync(out)).toEqual(Buffer.alloc(3, 200));
|
||||||
|
|||||||
@@ -328,7 +328,7 @@ describe("what the library reports", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Each operation is named in plain words.
|
// Each operation is named in plain words, as the README names it.
|
||||||
it("logs a refresh's progress at info, and its start and end at debug", () => {
|
it("logs a refresh's progress at info, and its start and end at debug", () => {
|
||||||
const { log, lines } = collect();
|
const { log, lines } = collect();
|
||||||
const onProgress = logRefresh(log);
|
const onProgress = logRefresh(log);
|
||||||
|
|||||||
+2
-5
@@ -19,11 +19,8 @@ import { SRP, SrpServer } from "fast-srp-hap";
|
|||||||
import { toBase64, deriveKEK, deriveLoginSubkey } from "../src/crypto/index.js";
|
import { toBase64, deriveKEK, deriveLoginSubkey } from "../src/crypto/index.js";
|
||||||
import type { KeyAttributes } from "../src/auth/types.js";
|
import type { KeyAttributes } from "../src/auth/types.js";
|
||||||
|
|
||||||
// The account's password is stretched at libsodium's lowest Argon2id cost.
|
const TEST_OPS = 2;
|
||||||
// Every login stretches it again, and at a realistic cost that takes seconds on
|
const TEST_MEM = 64 * 1024 * 1024;
|
||||||
// a loaded host.
|
|
||||||
const TEST_OPS = 1;
|
|
||||||
const TEST_MEM = 8192;
|
|
||||||
|
|
||||||
export interface MetaMockState {
|
export interface MetaMockState {
|
||||||
verifier: Buffer;
|
verifier: Buffer;
|
||||||
|
|||||||
@@ -1,180 +0,0 @@
|
|||||||
// `script/build-bin` compiles `bin/quak`, the single binary that the README's
|
|
||||||
// cron setup runs. bun compiles it from a copy of its own executable, so the
|
|
||||||
// binary loads through the same C library as the bun that built it. nix's bun
|
|
||||||
// loads through a C library in the nix store, which garbage collection can
|
|
||||||
// delete. On Linux the script therefore downloads bun's own release build,
|
|
||||||
// which loads through the system's C library.
|
|
||||||
//
|
|
||||||
// What the script promises, and why each test matters:
|
|
||||||
//
|
|
||||||
// - On Linux, each machine and C library that bun publishes a build for gets
|
|
||||||
// its own archive: x86-64 or arm64, with glibc or with musl, as on Alpine. A
|
|
||||||
// bun built for another machine or C library does not start, and
|
|
||||||
// `make install` fails.
|
|
||||||
// - The archive's SHA-256 must match the one in the script before anything in
|
|
||||||
// the archive runs. Without that check, a tampered download would build the
|
|
||||||
// binary that cron runs.
|
|
||||||
// - Linux on any other machine stops with an error. Building there with nix's
|
|
||||||
// bun would bring back the dependency on the nix store.
|
|
||||||
// - Other systems, such as macOS, compile with nix's bun as before, and
|
|
||||||
// download nothing.
|
|
||||||
//
|
|
||||||
// Each test copies the script into a fresh directory, which the script then
|
|
||||||
// treats as the checkout, and runs it there. Stubs first on PATH stand in for
|
|
||||||
// `uname`, `ldd`, `curl`, `unzip` and `nix-shell`, so no test downloads
|
|
||||||
// anything or runs a real bun.
|
|
||||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
||||||
import { spawnSync } from "node:child_process";
|
|
||||||
import {
|
|
||||||
chmodSync,
|
|
||||||
copyFileSync,
|
|
||||||
existsSync,
|
|
||||||
mkdirSync,
|
|
||||||
mkdtempSync,
|
|
||||||
readFileSync,
|
|
||||||
rmSync,
|
|
||||||
writeFileSync,
|
|
||||||
} from "node:fs";
|
|
||||||
import { tmpdir } from "node:os";
|
|
||||||
import { join } from "node:path";
|
|
||||||
import { fileURLToPath } from "node:url";
|
|
||||||
|
|
||||||
const repoRoot = fileURLToPath(new URL("../../", import.meta.url));
|
|
||||||
|
|
||||||
// The stubs, keyed by the command each replaces. `uname` and `ldd` describe the
|
|
||||||
// system that runBuildBin passes in the environment. `curl`, `unzip` and
|
|
||||||
// `nix-shell` write the arguments they were given, one per line, to a file
|
|
||||||
// named after the command in $STUB_LOG_DIR, which also shows that they ran.
|
|
||||||
const stubs: Record<string, string> = {
|
|
||||||
uname: `
|
|
||||||
case "$1" in
|
|
||||||
-s) echo "$STUB_UNAME_S" ;;
|
|
||||||
-m) echo "$STUB_UNAME_M" ;;
|
|
||||||
esac`,
|
|
||||||
// The script looks only at the start of the line: "musl libc" or not.
|
|
||||||
ldd: `
|
|
||||||
if [ "$STUB_LIBC" = musl ]; then
|
|
||||||
echo "musl libc (x86_64)"
|
|
||||||
else
|
|
||||||
echo "ldd (GNU libc) 2.36"
|
|
||||||
fi`,
|
|
||||||
// Writes a file that is not the archive to the path after -o.
|
|
||||||
curl: `
|
|
||||||
printf '%s\\n' "$@" > "$STUB_LOG_DIR/curl"
|
|
||||||
while [ $# -gt 1 ]; do
|
|
||||||
if [ "$1" = -o ]; then echo "not the archive" > "$2"; fi
|
|
||||||
shift
|
|
||||||
done`,
|
|
||||||
// Unpacks a stub bun where the real archive puts bun: in a directory named
|
|
||||||
// after the archive, inside the directory after -d. That bun writes
|
|
||||||
// bin/quak, as a real compile would.
|
|
||||||
unzip: `
|
|
||||||
printf '%s\\n' "$@" > "$STUB_LOG_DIR/unzip"
|
|
||||||
for arg in "$@"; do
|
|
||||||
case "$arg" in *.zip) archive="$arg" ;; esac
|
|
||||||
if [ "$previous" = -d ]; then directory="$arg"; fi
|
|
||||||
previous="$arg"
|
|
||||||
done
|
|
||||||
bun="$directory/$(basename "$archive" .zip)/bun"
|
|
||||||
mkdir -p "$(dirname "$bun")"
|
|
||||||
echo '#!/bin/sh' > "$bun"
|
|
||||||
echo 'mkdir -p bin && echo binary > bin/quak' >> "$bun"
|
|
||||||
chmod +x "$bun"`,
|
|
||||||
"nix-shell": `
|
|
||||||
printf '%s\\n' "$@" > "$STUB_LOG_DIR/nix-shell"`,
|
|
||||||
};
|
|
||||||
|
|
||||||
let checkout = "";
|
|
||||||
|
|
||||||
const writeScript = (path: string, body: string): void => {
|
|
||||||
writeFileSync(path, `#!/bin/sh\n${body}\n`);
|
|
||||||
chmodSync(path, 0o755);
|
|
||||||
};
|
|
||||||
|
|
||||||
// A checkout holding the script, a stub script/build that does nothing, the
|
|
||||||
// stubs, and the directory they log to.
|
|
||||||
beforeEach(() => {
|
|
||||||
checkout = mkdtempSync(join(tmpdir(), "quak-build-bin-"));
|
|
||||||
mkdirSync(join(checkout, "script"));
|
|
||||||
copyFileSync(
|
|
||||||
join(repoRoot, "script/build-bin"),
|
|
||||||
join(checkout, "script/build-bin"),
|
|
||||||
);
|
|
||||||
chmodSync(join(checkout, "script/build-bin"), 0o755);
|
|
||||||
writeScript(join(checkout, "script/build"), "exit 0");
|
|
||||||
mkdirSync(join(checkout, "stubs"));
|
|
||||||
for (const [command, body] of Object.entries(stubs)) {
|
|
||||||
writeScript(join(checkout, "stubs", command), body);
|
|
||||||
}
|
|
||||||
mkdirSync(join(checkout, "log"));
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
rmSync(checkout, { recursive: true, force: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
// Runs the script with nothing in its environment but the stubs' settings and a
|
|
||||||
// PATH that finds the stubs first.
|
|
||||||
const runBuildBin = (system: string, machine: string, libc = "glibc") =>
|
|
||||||
spawnSync(join(checkout, "script/build-bin"), {
|
|
||||||
cwd: checkout,
|
|
||||||
encoding: "utf-8",
|
|
||||||
env: {
|
|
||||||
PATH: `${join(checkout, "stubs")}:${process.env.PATH}`,
|
|
||||||
STUB_UNAME_S: system,
|
|
||||||
STUB_UNAME_M: machine,
|
|
||||||
STUB_LIBC: libc,
|
|
||||||
STUB_LOG_DIR: join(checkout, "log"),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
const ran = (command: string): boolean =>
|
|
||||||
existsSync(join(checkout, "log", command));
|
|
||||||
|
|
||||||
const argumentsOf = (command: string): string[] =>
|
|
||||||
readFileSync(join(checkout, "log", command), "utf-8")
|
|
||||||
.trimEnd()
|
|
||||||
.split("\n");
|
|
||||||
|
|
||||||
describe("script/build-bin", () => {
|
|
||||||
// The stub curl writes a file that is not the archive, so the hash check
|
|
||||||
// must stop the script before unzip runs. A script without the check would
|
|
||||||
// run unzip, and the stub bun it unpacks would write bin/quak.
|
|
||||||
it.each([
|
|
||||||
["x86_64", "glibc", "bun-linux-x64-baseline.zip"],
|
|
||||||
["x86_64", "musl", "bun-linux-x64-musl-baseline.zip"],
|
|
||||||
["aarch64", "glibc", "bun-linux-aarch64.zip"],
|
|
||||||
["aarch64", "musl", "bun-linux-aarch64-musl.zip"],
|
|
||||||
])(
|
|
||||||
"on Linux %s with %s, downloads %s and stops when its hash does not match",
|
|
||||||
(machine, libc, archive) => {
|
|
||||||
const result = runBuildBin("Linux", machine, libc);
|
|
||||||
expect(argumentsOf("curl")).toContain(
|
|
||||||
"https://github.com/oven-sh/bun/releases/download/bun-v1.3.13/" +
|
|
||||||
archive,
|
|
||||||
);
|
|
||||||
expect(result.status).not.toBe(0);
|
|
||||||
expect(ran("unzip")).toBe(false);
|
|
||||||
expect(existsSync(join(checkout, "bin/quak"))).toBe(false);
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
it("stops on a Linux machine that bun publishes no build for", () => {
|
|
||||||
const result = runBuildBin("Linux", "riscv64");
|
|
||||||
expect(result.status).not.toBe(0);
|
|
||||||
expect(ran("curl")).toBe(false);
|
|
||||||
expect(ran("nix-shell")).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("compiles with nix's bun on macOS", () => {
|
|
||||||
const result = runBuildBin("Darwin", "arm64");
|
|
||||||
expect(result.status).toBe(0);
|
|
||||||
expect(ran("curl")).toBe(false);
|
|
||||||
expect(argumentsOf("nix-shell")).toEqual([
|
|
||||||
"-p",
|
|
||||||
"bun",
|
|
||||||
"--run",
|
|
||||||
"bun build dist/bin/quak.js --compile --outfile bin/quak",
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -57,11 +57,8 @@ import type { RetryOptions } from "../../src/retry.js";
|
|||||||
|
|
||||||
const TEST_EMAIL = "thumb@example.com";
|
const TEST_EMAIL = "thumb@example.com";
|
||||||
const TEST_PASSWORD = "thumbpass";
|
const TEST_PASSWORD = "thumbpass";
|
||||||
// The password is stretched at libsodium's lowest Argon2id cost. Each test
|
const TEST_OPS = 2;
|
||||||
// builds an account and logs in, and at a realistic cost that takes seconds on
|
const TEST_MEM = 64 * 1024 * 1024;
|
||||||
// a loaded host.
|
|
||||||
const TEST_OPS = 1;
|
|
||||||
const TEST_MEM = 8192;
|
|
||||||
const TEST_TIME = 1700000000000000;
|
const TEST_TIME = 1700000000000000;
|
||||||
|
|
||||||
interface ThumbMockState {
|
interface ThumbMockState {
|
||||||
@@ -273,7 +270,6 @@ const buildThumbMock = async (opts?: {
|
|||||||
|
|
||||||
const buildThumbFetch = (m: ThumbMockState) => {
|
const buildThumbFetch = (m: ThumbMockState) => {
|
||||||
let srpServer: SrpServer;
|
let srpServer: SrpServer;
|
||||||
let signedMD5: string | undefined;
|
|
||||||
|
|
||||||
return (async (
|
return (async (
|
||||||
input: RequestInfo | URL,
|
input: RequestInfo | URL,
|
||||||
@@ -366,9 +362,8 @@ const buildThumbFetch = (m: ThumbMockState) => {
|
|||||||
return new Response("not found", { status: 404 });
|
return new Response("not found", { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upload URL minting. The real server signs the MD5 into the URL.
|
// Upload URL minting
|
||||||
if (path === "/files/upload-url") {
|
if (path === "/files/upload-url") {
|
||||||
signedMD5 = JSON.parse(init?.body as string).contentMD5;
|
|
||||||
return json({
|
return json({
|
||||||
objectKey: `42/thumb-${Date.now()}`,
|
objectKey: `42/thumb-${Date.now()}`,
|
||||||
url: "https://s3.mock.test/presigned-put",
|
url: "https://s3.mock.test/presigned-put",
|
||||||
@@ -377,11 +372,6 @@ const buildThumbFetch = (m: ThumbMockState) => {
|
|||||||
|
|
||||||
// Presigned PUT (S3 upload)
|
// Presigned PUT (S3 upload)
|
||||||
if (url.startsWith("https://s3.mock.test/")) {
|
if (url.startsWith("https://s3.mock.test/")) {
|
||||||
// Storage refuses a PUT that lacks the signed `Content-MD5`.
|
|
||||||
const headers = new Headers(init?.headers);
|
|
||||||
if (headers.get("Content-MD5") !== signedMD5) {
|
|
||||||
return new Response("AccessDenied", { status: 400 });
|
|
||||||
}
|
|
||||||
const body = init?.body;
|
const body = init?.body;
|
||||||
// Store the uploaded bytes for later inspection
|
// Store the uploaded bytes for later inspection
|
||||||
if (body instanceof Uint8Array) {
|
if (body instanceof Uint8Array) {
|
||||||
|
|||||||
@@ -3,15 +3,8 @@ import { configDefaults, defineConfig } from "vitest/config";
|
|||||||
// vitest does not read .gitignore when looking for tests. A checkout nested
|
// vitest does not read .gitignore when looking for tests. A checkout nested
|
||||||
// under .claude/ has its own test/ tree, and without this exclude the suite
|
// under .claude/ has its own test/ tree, and without this exclude the suite
|
||||||
// runs once per nested checkout and still reports success.
|
// runs once per nested checkout and still reports success.
|
||||||
//
|
|
||||||
// vitest starts one worker per CPU core by default, and a shared host has
|
|
||||||
// dozens. Four keep the suite well inside the Dockerfile's time limit. vitest
|
|
||||||
// 2 rejects a maxWorkers below its default minWorkers, so minWorkers is set
|
|
||||||
// too.
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
test: {
|
test: {
|
||||||
exclude: [...configDefaults.exclude, ".claude/**"],
|
exclude: [...configDefaults.exclude, ".claude/**"],
|
||||||
minWorkers: 1,
|
|
||||||
maxWorkers: 4,
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
Reference in new issue
Block a user