Compare commits
6
Commits
35bd45a575
..
next
| Author | SHA1 | Date | |
|---|---|---|---|
 clawbot
|
657266a60e | ||
 sneak
|
ae0f72971d | ||
|
clawbot
|
68d22b9927 | ||
|
clawbot
|
92e0c6c791 | ||
|
clawbot
|
66e3ac77e8 | ||
|
clawbot
|
c5a588177c |
@@ -10,12 +10,13 @@ account into a deduplicated local directory tree, skipping files that already
|
||||
exist on disk and continuing past individual download failures instead of
|
||||
crashing. For each file it persists the basic metadata fields quak keeps (title,
|
||||
file type, creation and modification time, latitude, longitude, content hash),
|
||||
its update time, the private and public magic metadata in full, Ente's ML data
|
||||
for it when there is any, and, for an image (for a live photo, its image), its
|
||||
original's EXIF and XMP, with its dimensions for a JPEG only; for a video it
|
||||
keeps none of these three. It runs unattended from cron (see "Running the backup
|
||||
from cron"). A helper subcommand can detect and regenerate missing thumbnails,
|
||||
encrypting and uploading them back to the server.
|
||||
its update time, and the private and public magic metadata in full. It keeps any
|
||||
ML data Ente has for the file, except the CLIP embedding, and writes the file's
|
||||
keywords in place of the embedding. For an image (for a live photo, its image),
|
||||
it keeps its original's EXIF and XMP, with its dimensions for a JPEG only. For a
|
||||
video it keeps none of these three. It runs unattended from cron (see "Running
|
||||
the backup from cron"). A helper subcommand can detect and regenerate missing
|
||||
thumbnails, encrypting and uploading them back to the server.
|
||||
|
||||
## Getting Started
|
||||
|
||||
@@ -137,12 +138,12 @@ matters.
|
||||
|
||||
### Exit codes
|
||||
|
||||
| Code | Meaning |
|
||||
| ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `0` | The backup is complete: every original is at its save path, and no file failed. |
|
||||
| `1` | The run finished with files in `failures.json` or albums it could not decrypt or write (see "Backup layout"), or it stopped on another error, printed as one line `quak: <message>`. The next run tries again. |
|
||||
| `2` | Another backup of the same directory is running. This run sent no request and changed nothing. |
|
||||
| `3` | There is no usable session: none is saved, the saved one is corrupt, or the server no longer accepts it. Run `quak login` as the user the cron job runs as. |
|
||||
| Code | Meaning |
|
||||
| ---- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `0` | The backup is complete: every original is at its save path, and no file failed. |
|
||||
| `1` | The run finished with files in `failures.json` or albums it could not decrypt or write (see "Backup layout"), and its last line says how many, or it stopped on another error, printed as one line `quak: <message>`. The next run tries again. |
|
||||
| `2` | Another backup of the same directory is running. This run sent no request and changed nothing. |
|
||||
| `3` | There is no usable session: none is saved, the saved one is corrupt, or the server no longer accepts it. Run `quak login` as the user the cron job runs as. |
|
||||
|
||||
`<message>` is the error's message, followed by the message of each error that
|
||||
caused it. Under Node, as `yarn quak` runs quak, `fetch` reports every failure
|
||||
@@ -371,6 +372,11 @@ quak/
|
||||
backup-metadata: the metadata quak keeps, as JSON
|
||||
exif.ts EXIF read from an image's bytes with exifreader
|
||||
mldata-fetch.ts fetch + decrypt per-file ML data
|
||||
clip-keywords.ts
|
||||
a file's keywords from its CLIP embedding
|
||||
clip-keyword-vectors.ts
|
||||
Ente's Discover categories with their text
|
||||
embeddings, written by bin/clip-keyword-vectors
|
||||
filename.ts safe file names from server metadata
|
||||
errors.ts error types shared across layers, and how an error
|
||||
is described
|
||||
@@ -378,6 +384,8 @@ quak/
|
||||
thumbnails.ts detect + regenerate missing thumbnails
|
||||
client.ts high-level Client class assembled from the above
|
||||
cli-commands.ts the CLI's commands as functions returning exit codes
|
||||
cli-log.ts the CLI's log level, and the one function the
|
||||
commands write stderr lines through
|
||||
cli-output.ts how the CLI prints a file's title and time
|
||||
cli-read.ts fresh reads for the CLI's read commands
|
||||
cli-run.ts run a command, print its error, exit with its code
|
||||
@@ -385,6 +393,9 @@ quak/
|
||||
index.ts public library exports
|
||||
bin/
|
||||
quak.ts CLI entrypoint (commander.js)
|
||||
clip-keyword-vectors, clip-keyword-vectors.py
|
||||
write src/clip-keyword-vectors.ts and a test
|
||||
fixture (see Source attribution)
|
||||
examples/
|
||||
download-albums.ts
|
||||
download every album's photos and metadata
|
||||
@@ -456,6 +467,12 @@ A custom API endpoint is configurable for self-hosted servers via the
|
||||
constructor option `apiOrigin`. When set, file downloads route through
|
||||
`<apiOrigin>/files/download/<id>` instead of the dedicated CDN host.
|
||||
|
||||
The constructor option `onRequest` is called once for each request sent, a
|
||||
retried request once for each attempt, with its method, its URL and the status
|
||||
of the answer or the error the request failed with. The URL of a presigned
|
||||
upload is given without its query, which holds the upload's signature. The CLI
|
||||
logs each request with `-vv` (see "CLI surface").
|
||||
|
||||
Required request headers on every authenticated call:
|
||||
|
||||
- `X-Auth-Token`: the decrypted auth token from login.
|
||||
@@ -532,7 +549,9 @@ each wait is drawn at random below its ceiling. Every other command uses the
|
||||
defaults. A library user gets the same budget by passing
|
||||
`UNATTENDED_RETRY_OPTIONS` as `ApiClientOptions.retry`. `sleep` and `random` are
|
||||
injectable through the same option, which is how the test suite exercises the
|
||||
whole policy without waiting.
|
||||
whole policy without waiting. `onRetry`, given there too, is called before each
|
||||
wait with the error the attempt failed with, the attempt's number and the wait;
|
||||
the CLI logs it with `-v` (see "CLI surface").
|
||||
|
||||
Two deadlines, renewed for each attempt:
|
||||
|
||||
@@ -646,6 +665,7 @@ thumbnails), for the user to delete if they want it gone.
|
||||
|
||||
```
|
||||
quak [--cache-dir <path>] <command> global: local metadata/content cache location
|
||||
quak [-v | -vv | -q] <command> global: log more on stderr, or only fatal errors
|
||||
quak login interactive or QUAK_EMAIL/QUAK_PASSWORD
|
||||
quak whoami print logged-in account as JSON
|
||||
quak logout end the session, delete it
|
||||
@@ -669,16 +689,66 @@ request, and neither fetches any collection's files nor changes the cache. The
|
||||
others wait for a refresh. With an empty cache, as on a first run, that refresh
|
||||
fetches the files of every collection, one collection after another, and stores
|
||||
nothing until it has them all, which on a large account takes a while. Meanwhile
|
||||
the command prints a line on stderr for each collection whose files it has
|
||||
fetched, such as `Fetched the files of 3 of 120 collections`; with `--json`,
|
||||
`files` and the two helpers print no such line. stdout holds only the command's
|
||||
output. If that round-trip fails, the command prints the error on one line and
|
||||
exits 1, or 3 when the server no longer accepts the saved session (see "Session
|
||||
handling"). An album or file quak cannot decrypt does not fail it; the command
|
||||
answers from what did decrypt (see "Default reads vs. fresh reads").
|
||||
the command logs a line on stderr for each collection whose files it has
|
||||
fetched, such as `Fetched the files of 3 of 120 collections`, with `--json` too;
|
||||
`backup` does the same while the refresh it starts with runs. stdout holds only
|
||||
the command's output. If that round-trip fails, the command prints the error on
|
||||
one line and exits 1, or 3 when the server no longer accepts the saved session
|
||||
(see "Session handling"). An album or file quak cannot decrypt does not fail it;
|
||||
the command answers from what did decrypt (see "Default reads vs. fresh reads").
|
||||
`--cache-dir` overrides where the cache lives; without it each account gets its
|
||||
own directory under the per-user cache path.
|
||||
|
||||
`-v` (`--verbose`) and `-q` (`--quiet`) set how much quak logs on stderr. Like
|
||||
`--cache-dir`, they go before or after any command's name. Every line quak
|
||||
prints on stderr is a log line at one of six levels, from the quietest: fatal,
|
||||
error, warning, info, debug and trace. A run prints the lines at its own level
|
||||
and at every quieter one. stdout holds only the command's output, at every
|
||||
level.
|
||||
|
||||
- Every command that exits non-zero logs a fatal line saying why, such as
|
||||
`quak: <message>`, `Not logged in`, an invalid ID or a collection that does
|
||||
not exist. A command that exits 1 because files or albums failed logs it last,
|
||||
saying how many failed and where to look: `backup` points to `failures.json`
|
||||
and its `FAILED` lines, `backup-metadata` to the `mlDataError` in the JSON of
|
||||
each such file, and `helper fix-missing-thumbnails` to its `Failed files:`
|
||||
list, or to its JSON with `--json`.
|
||||
- An error line reports something that failed: the `FAILED` lines that `backup`
|
||||
and `helper fix-missing-thumbnails` log as they work,
|
||||
`ML data of file <id> could not be read` and
|
||||
`ML data request for <n> file(s) failed` from `backup-metadata`,
|
||||
`Could not check <title>` from both thumbnail helpers, and the `Failed files:`
|
||||
list at the end of `backup` and of `helper fix-missing-thumbnails`.
|
||||
- A warning reports something that did not fail but needs your attention: after
|
||||
`logout`, that the cache directory still holds decrypted data.
|
||||
- Every other line quak logs by default is info.
|
||||
|
||||
With `--json`, a command's result on stdout takes the place of its summary on
|
||||
stderr: for `backup`, the lines from `--- Backup complete ---` to the end of its
|
||||
`Failed files:` list; for `helper list-missing-thumbnails`,
|
||||
`No missing thumbnails found.` or `<n> file(s) with missing thumbnails:`; and
|
||||
for `helper fix-missing-thumbnails`, the lines from `--- Done ---` to the end of
|
||||
its `Failed files:` list. Every other line is logged as it is without `--json`.
|
||||
|
||||
- With neither option the level is info: the lines quak prints as it works, such
|
||||
as a refresh's progress, the files a backup fetches, its summary and the files
|
||||
that failed.
|
||||
- `-v` raises the level to debug, which adds each retry of a request, with why
|
||||
the attempt failed and how long quak waits before the next, and the start and
|
||||
end of each refresh and each ML data fetch, with the error of one that failed.
|
||||
- `-vv` raises it to trace, which adds each request quak sends: its method, its
|
||||
URL and the status of the answer, or the error the request failed with. The
|
||||
URL of an upload of a thumbnail is logged without its query, which holds the
|
||||
upload's signature. More repeats stay at trace, and `--verbose --verbose` is
|
||||
`-vv`.
|
||||
- `-q` lowers it to fatal: a run that exits 0 prints nothing on stderr, and one
|
||||
that exits non-zero prints only its fatal lines. The prompts of `login` are
|
||||
not log lines, and `-q` leaves them as they are.
|
||||
|
||||
`-q` with `-v` is refused with one line,
|
||||
`error: option '-q, --quiet' cannot be used with option '-v, --verbose'`, and
|
||||
exit status 1.
|
||||
|
||||
`files <collectionID>` lists the files in that collection. `files` with no
|
||||
argument lists every file in the account, collection by collection, each file
|
||||
once: a file in several collections is listed with the first of them, which is
|
||||
@@ -691,33 +761,49 @@ before any request with one line, `Invalid file ID: <arg>` or
|
||||
`Invalid collection ID: <arg>`, and exit status 1. One bad `--file` ID refuses
|
||||
the whole command, so nothing is fixed.
|
||||
|
||||
Without `--out`, `get` writes the file to the working directory under its title,
|
||||
and `get-thumb` writes the thumbnail there under `thumb_` followed by the title,
|
||||
each with unsafe characters replaced. A name over 255 bytes, which most file
|
||||
systems refuse, is cut short between two characters before its extension, so the
|
||||
extension is kept whenever it fits. An extension too long to leave room for the
|
||||
name's first character is cut short too, so the name never starts with a dot.
|
||||
|
||||
`get` and `get-thumb` resolve the file by ID directly, so `--collection` is
|
||||
accepted for backward compatibility but ignored. For a live photo, `get` writes
|
||||
its image and its video, each named after the title with its own extension, as
|
||||
Ente's clients name them (`IMG_0001.heic` and `IMG_0001.mov`). With
|
||||
`--out PATH`, the image is written to `PATH` and the video beside it, with
|
||||
`PATH`'s name and the video's extension; a `PATH` with the video's extension is
|
||||
refused. `backup-metadata --exif` (alias `--all`) additionally fetches each
|
||||
file's original through the cache and records, from it or a live photo's image,
|
||||
its XMP metadata, its EXIF metadata and, for a JPEG, its dimensions. EXIF is
|
||||
read with [exifreader](https://github.com/mattiasw/ExifReader) from any image
|
||||
format it reads, JPEG, HEIC/HEIF, AVIF, PNG and WebP among them. The record's
|
||||
`exif` field is exifreader's EXIF tag output: each tag by name, with its
|
||||
`value`, `description` and `computed` value. An EXIF block exifreader finds but
|
||||
reads no tag from is recorded, base64, as `exifRaw`, with the reason in
|
||||
`exifError`. `collections`, `files`, `backup`, `helper list-missing-thumbnails`
|
||||
and `helper fix-missing-thumbnails` take `--json` for machine-readable output.
|
||||
Ente's clients name them (`IMG_0001.heic` and `IMG_0001.mov`), and each name is
|
||||
cut short as above once its own extension is in it. With `--out PATH`, the image
|
||||
is written to `PATH` and the video beside it, with `PATH`'s name and the video's
|
||||
extension; a `PATH` with the video's extension is refused.
|
||||
`backup-metadata --exif` (alias `--all`) additionally fetches each file's
|
||||
original through the cache and records, from it or a live photo's image, its XMP
|
||||
metadata, its EXIF metadata and, for a JPEG, its dimensions. EXIF is read with
|
||||
[exifreader](https://github.com/mattiasw/ExifReader) from any image format it
|
||||
reads, JPEG, HEIC/HEIF, AVIF, PNG and WebP among them. The record's `exif` field
|
||||
is exifreader's EXIF tag output: each tag by name, with its `value`,
|
||||
`description` and `computed` value. An EXIF block exifreader finds but reads no
|
||||
tag from is recorded, base64, as `exifRaw`, with the reason in `exifError`.
|
||||
`collections`, `files`, `backup`, `helper list-missing-thumbnails` and
|
||||
`helper fix-missing-thumbnails` take `--json` for machine-readable output.
|
||||
|
||||
`backup --verify` also hashes the originals already in the backup and replaces
|
||||
any that do not match the content hash Ente records; one it cannot replace stays
|
||||
as it was and goes into `failures.json` (see "Backup layout").
|
||||
|
||||
`backup-metadata <dir>` writes `account.json` and, for each collection, a
|
||||
directory `collections/<id>-<name>/` holding `_collection.json` and a
|
||||
`<fileID>.json` for each of its files. Each `<fileID>.json` holds the file's ML
|
||||
data and keywords as `quak backup` writes them (see "Backup layout"). The
|
||||
collection's name has unsafe characters replaced and is cut short between two
|
||||
characters so that the directory's name fits in 255 bytes; the ID keeps the
|
||||
directories apart.
|
||||
|
||||
`backup-metadata` fetches ML data in requests of up to 200 files. When a request
|
||||
fails, the error is logged, each of its files is written with the reason in an
|
||||
`mlDataError` field instead of `mlData`, and the dump goes on. A file whose ML
|
||||
data does not decrypt, gunzip or parse is logged and written the same way, with
|
||||
why in `mlDataError`. The exit code is non-zero if any file was written with
|
||||
`mlDataError`.
|
||||
`mlDataError`, and the last line then says how many were.
|
||||
|
||||
`helper fix-missing-thumbnails` regenerates thumbnails for JPEG images only,
|
||||
because the bundled decoder (`jpeg-js`) decodes only JPEG. A non-JPEG image
|
||||
@@ -743,11 +829,11 @@ the smallest does not.
|
||||
below)
|
||||
YYYY-MM-DD.<fileID>.json the file's basic metadata fields quak
|
||||
keeps, its update time, its private and
|
||||
public magic metadata, its ML data, and,
|
||||
for an image (for a live photo, its
|
||||
image), its original's EXIF and XMP, with
|
||||
dimensions for a JPEG only; none of these
|
||||
three for a video
|
||||
public magic metadata, its ML data and
|
||||
keywords, and, for an image (for a live
|
||||
photo, its image), its original's EXIF
|
||||
and XMP, with dimensions for a JPEG
|
||||
only; none of these three for a video
|
||||
YYYY-MM-DD.<fileID>.livephoto.json
|
||||
which of a live photo's two files is which
|
||||
collections/
|
||||
@@ -764,9 +850,10 @@ Each original is saved at its save path, the same path `photo.savePath` gives
|
||||
and `photo.download()` writes (see Read surface below). The date is the photo's
|
||||
`takenAt` (the date set in Ente if it was edited, else its creation time) in the
|
||||
time zone of the machine running quak. The extension is the one in the file's
|
||||
name as uploaded, case kept, or `.bin` when it has none or it holds anything but
|
||||
letters and digits. When the date or the time zone changes, the next run saves
|
||||
the original at its new path and leaves the old copy where it is.
|
||||
name as uploaded, case kept, or `.bin` when it has none, it holds anything but
|
||||
letters and digits, or it is more than 16 of them long. When the date or the
|
||||
time zone changes, the next run saves the original at its new path and leaves
|
||||
the old copy where it is.
|
||||
|
||||
`account.json` holds the account's `email` and `userID`, as `backup-metadata`
|
||||
writes it. A collection's JSON holds its `id`, `name`, `type`, `ownerID`,
|
||||
@@ -777,20 +864,51 @@ JSON holds its `id`, `collectionID`, `ownerID`, `metadata` and `updationTime`,
|
||||
and its `magicMetadata` and `pubMagicMetadata` when it has them. Update times
|
||||
are in microseconds, as Ente records them.
|
||||
|
||||
A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as
|
||||
`mlData`, the same payload `backup-metadata` writes; a file Ente has no ML data
|
||||
for has no `mlData`. The backup waits for the library's ML data fetch to finish
|
||||
before it writes the JSON files. If that fetch fails, each file whose ML data is
|
||||
not in the cache gets the reason in `mlDataError` instead and counts as failed,
|
||||
and the next run fetches it again. The same goes for a file with no ML data in
|
||||
the cache whose ML data the server sent but quak could not decrypt, gunzip or
|
||||
parse. Its `mlDataError` says why, and `failures.json` records it with an error
|
||||
starting `ML data: `. quak keeps that reason in memory only and caches nothing
|
||||
for the file, so the next run asks for its ML data again. An ML data fetch that
|
||||
fails, such as one the library's refresh starts while the backup writes the JSON
|
||||
files, leaves the reasons as the last fetch that succeeded left them. The JSON
|
||||
files are rewritten on every run, so ML data that arrived since the last run
|
||||
appears.
|
||||
A file's JSON holds Ente's ML data for it as `mlData`, such as the faces Ente
|
||||
found, without its CLIP embedding. It holds `keywords` in place of the
|
||||
embedding.
|
||||
|
||||
The Discover section of Ente's photos app has 16 categories:
|
||||
|
||||
- Identity (no videos)
|
||||
- Screenshots (no videos)
|
||||
- QR Codes (no videos)
|
||||
- Receipts
|
||||
- Notes
|
||||
- Memes
|
||||
- Visiting Cards (no videos)
|
||||
- Babies
|
||||
- Pets
|
||||
- Selfies
|
||||
- Wallpapers (no videos)
|
||||
- Food
|
||||
- Celebrations
|
||||
- Sunset
|
||||
- Hills
|
||||
- Greenery
|
||||
|
||||
A file's keywords are the titles of the categories it falls in. A file's score
|
||||
for a category is the cosine similarity of its CLIP embedding and the text
|
||||
embedding of the category's query. As in Ente's app, the file falls in the
|
||||
category when that score is at least the category's minimum score. A video never
|
||||
falls in a category marked "no videos". The keywords are listed highest score
|
||||
first.
|
||||
|
||||
A file with no ML data has neither `mlData` nor `keywords`. A file whose ML data
|
||||
holds no usable CLIP embedding has no `keywords`. `backup-metadata` writes both
|
||||
fields the same way.
|
||||
|
||||
The backup waits for the library's ML data fetch to finish before it writes the
|
||||
JSON files. If that fetch fails, each file whose ML data is not in the cache
|
||||
gets the reason in `mlDataError` instead and counts as failed, and the next run
|
||||
fetches it again. The same goes for a file with no ML data in the cache whose ML
|
||||
data the server sent but quak could not decrypt, gunzip or parse. Its
|
||||
`mlDataError` says why, and `failures.json` records it with an error starting
|
||||
`ML data: `. quak keeps that reason in memory only and caches nothing for the
|
||||
file, so the next run asks for its ML data again. An ML data fetch that fails,
|
||||
such as one the library's refresh starts while the backup writes the JSON files,
|
||||
leaves the reasons as the last fetch that succeeded left them. The JSON files
|
||||
are rewritten on every run, so ML data that arrived since the last run appears.
|
||||
|
||||
A file's JSON holds, as `imageMetadata`, what `backup-metadata --exif` records
|
||||
from its original, or from a live photo's image: `format`, `width` and `height`
|
||||
@@ -813,8 +931,8 @@ reads vs. fresh reads") goes into `failures.json` with an error that starts
|
||||
`could not be decrypted:` and gives the reason. If a copy of it in another album
|
||||
decrypted, it does not: its original is stored from that copy, and the album
|
||||
where it does not decrypt neither links nor lists it. An album the library could
|
||||
not decrypt has no name quak can read and no files it can list. A run without
|
||||
`--json` prints one line for it,
|
||||
not decrypt has no name quak can read and no files it can list. `quak backup`
|
||||
prints one line for it on stderr,
|
||||
`FAILED album <id>: could not be decrypted: <reason>`, and `--json` and the
|
||||
`BackupResult` list its ID in `undecryptableAlbums`. Such a file in
|
||||
`failures.json`, or such an album, keeps the exit code at 1, run after run,
|
||||
@@ -859,15 +977,16 @@ gets its ID added as well: beside an album `x`, whose JSON is `x.json`, an album
|
||||
named `x.json` gets the directory `x.json (12)/`. A name too long for a file
|
||||
name, which most file systems limit to 255 bytes, gets its ID added too. A name
|
||||
with its ID added is cut short between two characters in the part before the ID,
|
||||
just far enough that it fits. When a symlink's extension alone leaves no room
|
||||
for the ID, the extension is cut short and the part before the ID keeps only the
|
||||
bytes that cut leaves over. A collection's directory name also leaves room for
|
||||
the `.json` its JSON adds. IDs never change, so a name stays the same from run
|
||||
to run until such a clash appears or goes away.
|
||||
just far enough that it fits. When a symlink's extension leaves no room for the
|
||||
ID and the first character of the name, the extension is cut short, and the part
|
||||
before the ID keeps that character and any bytes the cut leaves over. A
|
||||
collection's directory name also leaves room for the `.json` its JSON adds. IDs
|
||||
never change, so a name stays the same from run to run until such a clash
|
||||
appears or goes away.
|
||||
|
||||
A collection whose directory or JSON cannot be written, such as one whose
|
||||
directory name is taken by a file you put in `collections/`, does not stop the
|
||||
run. A run without `--json` prints `FAILED album <name>: <reason>` for it,
|
||||
run. `quak backup` prints `FAILED album <name>: <reason>` for it on stderr,
|
||||
`--json` and the `BackupResult` list its ID in `failedAlbums`, and the exit code
|
||||
is 1. The other collections and `failures.json` are still written.
|
||||
|
||||
@@ -1193,18 +1312,18 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange`
|
||||
not already at its save path there as `photo.download()` does (and, with
|
||||
`includeThumbnails`, fetches thumbnails) through the content cache, waits for
|
||||
an ML data fetch, and rebuilds the on-disk backup tree with a durable failure
|
||||
ledger. Each file's JSON holds its ML data and, for an image (for a live
|
||||
photo, its image), its original's EXIF and XMP, with its dimensions for a JPEG
|
||||
only; a video's JSON holds none of these three. A fetched original is written
|
||||
straight to its save path and not into the cache, which then counts it as
|
||||
present; one the cache already held is copied from there. `BackupOptions`:
|
||||
`downloadDirectory` (falls back to the library's), `includeOriginals` (default
|
||||
`true`), `includeThumbnails` (default `false`), `onlyAlbumNames`, `verify`
|
||||
(default `false`), `onProgress`, and `lock`. See Backup layout above for the
|
||||
tree it writes, and for the albums `BackupResult` lists in
|
||||
`undecryptableAlbums` and `failedAlbums`. Whether the backup took the lock or
|
||||
its caller did, once the lock is lost the backup rejects with
|
||||
`lost the lock on <dir>: <reason>` before it starts another file.
|
||||
ledger. Each file's JSON holds its ML data and keywords. An image's JSON also
|
||||
holds the EXIF and XMP of its original, and its dimensions if it is a JPEG. A
|
||||
live photo's JSON holds these for its image. A video's JSON holds no EXIF, XMP
|
||||
or dimensions. A fetched original is written straight to its save path and not
|
||||
into the cache, which then counts it as present; one the cache already held is
|
||||
copied from there. `BackupOptions`: `downloadDirectory` (falls back to the
|
||||
library's), `includeOriginals` (default `true`), `includeThumbnails` (default
|
||||
`false`), `onlyAlbumNames`, `verify` (default `false`), `onProgress`, and
|
||||
`lock`. See Backup layout above for the tree it writes, and for the albums
|
||||
`BackupResult` lists in `undecryptableAlbums` and `failedAlbums`. Whether the
|
||||
backup took the lock or its caller did, once the lock is lost the backup
|
||||
rejects with `lost the lock on <dir>: <reason>` before it starts another file.
|
||||
|
||||
### Request pools
|
||||
|
||||
@@ -1287,6 +1406,14 @@ is rewritten in TypeScript in this repository. Protocol fidelity is verified
|
||||
against the upstream implementations in `web/packages/base/`,
|
||||
`mobile/apps/photos/lib/`, and `cli/`.
|
||||
|
||||
The keywords come from Ente too. `src/clip-keyword-vectors.ts` holds the
|
||||
Discover categories of Ente's photos app, from
|
||||
`mobile/apps/photos/assets/discover.json`, each with its query's text embedding
|
||||
from Ente's CLIP text model. `bin/clip-keyword-vectors` writes that file and the
|
||||
CLIP embedding of a public-domain photo that the tests use. It runs Ente's
|
||||
models in Docker, with every download pinned by sha256. It is run by hand only,
|
||||
followed by `make fmt`.
|
||||
|
||||
## For LLMs
|
||||
|
||||
If you are an LLM agent working on this repository, read and follow these
|
||||
|
||||
@@ -17,20 +17,63 @@ pre-1.0
|
||||
|
||||
# Next Step
|
||||
|
||||
One bug is open: https://git.eeqj.de/sneak/quak/issues/205. The check of quak
|
||||
against a live Ente account, https://git.eeqj.de/sneak/quak/issues/187, needs an
|
||||
account only sneak has, so it waits on him.
|
||||
No bug is open, and no change sneak asked for is open. The check of quak against
|
||||
a live Ente account, https://git.eeqj.de/sneak/quak/issues/187, needs an account
|
||||
only sneak has, so it waits on him.
|
||||
|
||||
Tagging and releases are decided by sneak alone, and happen only when he
|
||||
declares one.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-22: The JSON `quak backup` and `quak backup-metadata` write for a file
|
||||
holds `keywords` in place of the CLIP embedding in its ML data
|
||||
(https://git.eeqj.de/sneak/quak/issues/218). The keywords are the titles of
|
||||
the Discover categories of Ente's photos app that the file falls in, by the
|
||||
app's rule: the cosine similarity of the file's CLIP embedding and the text
|
||||
embedding of a category's query is at least the category's minimum score, and
|
||||
videos stay out of the categories that leave them out. `src/clip-keywords.ts`
|
||||
applies the rule. `src/clip-keyword-vectors.ts` holds the 16 categories with
|
||||
their text embeddings. `bin/clip-keyword-vectors` writes it with Ente's CLIP
|
||||
text model, and writes the CLIP embedding of a public-domain photo of a sunset
|
||||
that the tests use.
|
||||
|
||||
- 2026-10-18: `-v`/`--verbose` raises the CLI's log level once for each repeat,
|
||||
and `-q`/`--quiet` prints only fatal errors
|
||||
(https://git.eeqj.de/sneak/quak/issues/212). The levels are fatal, error,
|
||||
warning, info, debug and trace; the default is info, everything quak printed
|
||||
on stderr before, `-v` gives debug and `-vv` trace. `src/cli-log.ts` holds the
|
||||
level and the log the commands write stderr lines through. Debug adds each
|
||||
retry, which `withRetry` reports through the new `onRetry`, and the start and
|
||||
end of each refresh and ML data fetch; trace adds each request, which
|
||||
`ApiClient` reports through the new `onRequest`. `--json` no longer keeps the
|
||||
progress lines off stderr, so `backup` and `--json` runs log a refresh's
|
||||
progress too. Every line that reports a failed file or album is at error,
|
||||
which the progress callbacks of `backup`, `backup-metadata` and the thumbnail
|
||||
helpers now say, and a command that exits 1 because files or albums failed
|
||||
ends with one fatal line saying how many and where to look.
|
||||
|
||||
- 2026-10-18: `quak files` takes the collection ID as an optional argument
|
||||
instead of the `--collection` option, which is gone (issue 211). With no
|
||||
argument it lists every file in the account once, collection by collection; a
|
||||
file in several collections is listed with the first of them, which is its
|
||||
`collectionID` in `--json`. The fresh read in `src/cli-read.ts` does this.
|
||||
instead of the `--collection` option, which is gone
|
||||
(https://git.eeqj.de/sneak/quak/issues/211). With no argument it lists every
|
||||
file in the account once, collection by collection; a file in several
|
||||
collections is listed with the first of them, which is its `collectionID` in
|
||||
`--json`. The fresh read in `src/cli-read.ts` does this.
|
||||
|
||||
- 2026-10-18: Names quak builds from an album name or a title fit in a file name
|
||||
(https://git.eeqj.de/sneak/quak/issues/205). `shortenKeepingExtension` in
|
||||
`src/filename.ts` cuts a name short to a number of bytes, keeping its
|
||||
extension when it fits and always its first character, and can put text right
|
||||
before the extension; the backup's `withID` uses it for an ID.
|
||||
`defaultFileName` there gives the name a file is saved under when the caller
|
||||
gives no path: a prefix such as `thumb_` and the title with unsafe characters
|
||||
replaced, cut short with it to 255 bytes, and, for a live photo's image or
|
||||
video, with that part's own extension before the cut. `quak get`,
|
||||
`quak get-thumb`, `downloadFile` and `downloadThumbnail` use it without
|
||||
`--out` or `outPath`. `backup-metadata` cuts an album's name short so that its
|
||||
folder, `<id>-<name>`, fits in 255 bytes. `safeExtension` gives `.bin` for an
|
||||
extension of more than 16 letters and digits, so a save path's name always
|
||||
fits.
|
||||
|
||||
- 2026-10-18: `quak collections` no longer waits for the files of every
|
||||
collection (issue 210). On an empty cache it opened its library with a refresh
|
||||
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/bin/sh
|
||||
# Writes src/clip-keyword-vectors.ts and test/clip-keywords-sunset.json by
|
||||
# running bin/clip-keyword-vectors.py in a Python image pinned by digest, with
|
||||
# its packages pinned by hash. Run it by hand from a checkout, then `make fmt`;
|
||||
# no gate runs it. Besides the Python image, it downloads about 265 MB, most of
|
||||
# it Ente's two CLIP models.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# python:3.13-slim (Python 3.13.17 on Debian 13.7), 2026-10-21. The wheels
|
||||
# below are the ones for this image on linux/amd64.
|
||||
IMAGE=python@sha256:57b29a7c050c4e679e4bea572e14b3965cf4b5839de2da49d55e49ffc4024f39
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
docker run --rm -i --platform linux/amd64 \
|
||||
--user "$(id -u):$(id -g)" -e HOME=/tmp \
|
||||
-v "$ROOT:/repo" -w /repo "$IMAGE" sh -c '
|
||||
python -m venv /tmp/venv &&
|
||||
/tmp/venv/bin/pip install --quiet --disable-pip-version-check \
|
||||
--no-cache-dir --no-deps --require-hashes -r /dev/stdin &&
|
||||
/tmp/venv/bin/python bin/clip-keyword-vectors.py' <<'EOF'
|
||||
numpy==2.5.3 --hash=sha256:a5fa86b80fd24bcd1aff83ad23be44ea323de3f787be8f8b15d4a65621e25321
|
||||
onnxruntime==1.30.0 --hash=sha256:86f940afc801ea9681a4da8af84fbe95e1d9ea7d80903952cc1bfad54faad38f
|
||||
pillow==12.3.0 --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3
|
||||
tokenizers==0.23.2 --hash=sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305
|
||||
EOF
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Write src/clip-keyword-vectors.ts and test/clip-keywords-sunset.json.
|
||||
|
||||
bin/clip-keyword-vectors runs this in a pinned Python image. It downloads
|
||||
Ente's Discover list, Ente's CLIP text and image models, the CLIP tokenizer
|
||||
and a public-domain photo of a sunset, checks each against its sha256, and
|
||||
writes:
|
||||
|
||||
- src/clip-keyword-vectors.ts: each Discover category of Ente's photos app,
|
||||
with the text embedding Ente's CLIP text model gives its query;
|
||||
- test/clip-keywords-sunset.json: the CLIP embedding Ente's image model gives
|
||||
the photo, prepared as Ente's apps prepare a photo.
|
||||
|
||||
Both embeddings are scaled to length 1, as Ente's apps scale them.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime
|
||||
from PIL import Image, ImageOps
|
||||
from tokenizers import Tokenizer
|
||||
|
||||
ENTE_COMMIT = "f5ad70a860ed25d0c8057b0efb4bd6eaefe4b0c5"
|
||||
DISCOVER_PATH = "mobile/apps/photos/assets/discover.json"
|
||||
DISCOVER_URL = (
|
||||
f"https://raw.githubusercontent.com/ente-io/ente/{ENTE_COMMIT}/{DISCOVER_PATH}"
|
||||
)
|
||||
DISCOVER_SHA256 = "cf70ac7116b592f7921ca4cf67cdbb421a8e6b287fbe44b2be678b8d198e5c24"
|
||||
|
||||
TEXT_MODEL = "mobileclip_s2_text_opset18_quant.onnx"
|
||||
TEXT_MODEL_SHA256 = "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322"
|
||||
IMAGE_MODEL = "mobileclip_s2_image_gelu_opset20.onnx"
|
||||
IMAGE_MODEL_SHA256 = "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e"
|
||||
|
||||
# The CLIP tokenizer Hugging Face publishes for OpenAI's CLIP. Its vocabulary
|
||||
# and merges are those of bpe_simple_vocab_16e6.txt, the vocabulary Ente's
|
||||
# apps tokenize with.
|
||||
TOKENIZER_URL = (
|
||||
"https://huggingface.co/openai/clip-vit-base-patch32/resolve/"
|
||||
"3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/tokenizer.json"
|
||||
)
|
||||
TOKENIZER_SHA256 = "b556ac8c99757ffb677208af34bc8c6721572114111a6e0aaf5fa69ff0b8d842"
|
||||
|
||||
# "Sunset off the Massachusetts coast", a NOAA photo in the public domain:
|
||||
# https://commons.wikimedia.org/wiki/File:Sunset_off_the_Massachusetts_coast_-_NOAA.jpg
|
||||
PHOTO_URL = (
|
||||
"https://upload.wikimedia.org/wikipedia/commons/d/db/"
|
||||
"Sunset_off_the_Massachusetts_coast_-_NOAA.jpg"
|
||||
)
|
||||
PHOTO_SHA256 = "be62f3289c9bf7b75af68cd6978021d54e8a531d5d4cc493f6e9fbf5e1ab74d3"
|
||||
|
||||
# Ente's text model takes 77 token ids, padded with 0; its image model takes
|
||||
# a 256x256 RGB image.
|
||||
TOKEN_COUNT = 77
|
||||
IMAGE_SIZE = 256
|
||||
|
||||
|
||||
def download(url, sha256):
|
||||
# Wikimedia refuses requests without a User-Agent.
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "quak"})
|
||||
with urllib.request.urlopen(request) as response:
|
||||
data = response.read()
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
if digest != sha256:
|
||||
raise SystemExit(f"{url} has sha256 {digest}, not {sha256}")
|
||||
return data
|
||||
|
||||
|
||||
def ente_model(name, sha256):
|
||||
return onnxruntime.InferenceSession(
|
||||
download(f"https://models.ente.com/{name}", sha256),
|
||||
providers=["CPUExecutionProvider"],
|
||||
)
|
||||
|
||||
|
||||
def run(session, tensor):
|
||||
(output,) = session.run(None, {session.get_inputs()[0].name: tensor})
|
||||
embedding = output[0]
|
||||
return embedding / np.linalg.norm(embedding)
|
||||
|
||||
|
||||
def text_embedding(session, tokenizer, query):
|
||||
ids = tokenizer.encode(query).ids
|
||||
ids += [0] * (TOKEN_COUNT - len(ids))
|
||||
return run(session, np.array([ids], dtype=np.int32))
|
||||
|
||||
|
||||
def image_embedding(session, photo):
|
||||
image = ImageOps.exif_transpose(Image.open(io.BytesIO(photo))).convert("RGB")
|
||||
# Scale to cover IMAGE_SIZE x IMAGE_SIZE and crop the centre.
|
||||
square = ImageOps.fit(image, (IMAGE_SIZE, IMAGE_SIZE), Image.Resampling.BILINEAR)
|
||||
pixels = np.asarray(square, dtype=np.float32) / 255
|
||||
return run(session, pixels.transpose(2, 0, 1)[np.newaxis])
|
||||
|
||||
|
||||
def numbers(embedding):
|
||||
return ", ".join(str(value) for value in embedding)
|
||||
|
||||
|
||||
def write_categories(prompts, embeddings):
|
||||
lines = [
|
||||
"// Written by bin/clip-keyword-vectors; do not edit by hand.",
|
||||
"//",
|
||||
"// The Discover categories of Ente's photos app, in their order, from",
|
||||
f"// {DISCOVER_PATH} in https://github.com/ente-io/ente",
|
||||
f"// at commit {ENTE_COMMIT}. Each has the text",
|
||||
f"// embedding Ente's CLIP text model ({TEXT_MODEL})",
|
||||
"// gives its query, scaled to length 1. `showVideo` is false for a category",
|
||||
"// that leaves out videos.",
|
||||
"",
|
||||
"export const DISCOVER_CATEGORIES = [",
|
||||
]
|
||||
for prompt, embedding in zip(prompts, embeddings):
|
||||
lines += [
|
||||
" {",
|
||||
f" title: {json.dumps(prompt['title'])},",
|
||||
f" query: {json.dumps(prompt['query'])},",
|
||||
f" minScore: {json.dumps(prompt['minScore'])},",
|
||||
f" showVideo: {json.dumps(prompt.get('showVideo', True))},",
|
||||
f" textEmbedding: [{numbers(embedding)}],",
|
||||
" },",
|
||||
]
|
||||
lines.append("];")
|
||||
with open("src/clip-keyword-vectors.ts", "w", encoding="utf-8") as file:
|
||||
file.write("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
def write_fixture(embedding):
|
||||
with open("test/clip-keywords-sunset.json", "w", encoding="utf-8") as file:
|
||||
file.write(
|
||||
"{\n"
|
||||
f' "photo": {json.dumps(PHOTO_URL)},\n'
|
||||
f' "sha256": {json.dumps(PHOTO_SHA256)},\n'
|
||||
f' "embedding": [{numbers(embedding)}]\n'
|
||||
"}\n"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
prompts = json.loads(download(DISCOVER_URL, DISCOVER_SHA256))["prompts"]
|
||||
tokenizer = Tokenizer.from_str(
|
||||
download(TOKENIZER_URL, TOKENIZER_SHA256).decode("utf-8")
|
||||
)
|
||||
text_model = ente_model(TEXT_MODEL, TEXT_MODEL_SHA256)
|
||||
embeddings = [
|
||||
text_embedding(text_model, tokenizer, prompt["query"]) for prompt in prompts
|
||||
]
|
||||
write_categories(prompts, embeddings)
|
||||
|
||||
image_model = ente_model(IMAGE_MODEL, IMAGE_MODEL_SHA256)
|
||||
write_fixture(image_embedding(image_model, download(PHOTO_URL, PHOTO_SHA256)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+40
-8
@@ -2,8 +2,9 @@
|
||||
|
||||
import { stdout, stderr } from "node:process";
|
||||
import { input, password } from "@inquirer/prompts";
|
||||
import { Command } from "commander";
|
||||
import { Command, Option } from "commander";
|
||||
import envPaths from "env-paths";
|
||||
import { logLevel, logRequests, stderrLog, type Log } from "../src/cli-log.js";
|
||||
import { init } from "../src/crypto/index.js";
|
||||
import {
|
||||
type CliContext,
|
||||
@@ -29,6 +30,9 @@ const paths = envPaths("quak", { suffix: "" });
|
||||
|
||||
const program = new Command();
|
||||
|
||||
// Program options, so they are taken before or after any command's name.
|
||||
// `-v` counts its repeats, the way commander's documentation counts a repeated
|
||||
// option.
|
||||
program
|
||||
.name("quak")
|
||||
.description("CLI for the Ente end-to-end encrypted photo service")
|
||||
@@ -37,21 +41,46 @@ program
|
||||
"--cache-dir <path>",
|
||||
"Directory for the local metadata/content cache " +
|
||||
"(default: the per-user cache directory)",
|
||||
);
|
||||
)
|
||||
.option(
|
||||
"-v, --verbose",
|
||||
"Log more on stderr: debug lines with -v, trace lines too with -vv",
|
||||
(_value: string, previous: number) => previous + 1,
|
||||
0,
|
||||
)
|
||||
.addOption(
|
||||
new Option(
|
||||
"-q, --quiet",
|
||||
"Print nothing on stderr but the error a failing command exits with",
|
||||
).conflicts("verbose"),
|
||||
)
|
||||
.configureHelp({ showGlobalOptions: true });
|
||||
|
||||
interface ProgramOptions {
|
||||
cacheDir?: string;
|
||||
verbose: number;
|
||||
quiet?: boolean;
|
||||
}
|
||||
|
||||
// The log for this run, at the level `-v` and `-q` ask for.
|
||||
const log = (): Log => {
|
||||
const { verbose, quiet } = program.opts<ProgramOptions>();
|
||||
return stderrLog(stderr, logLevel(verbose, quiet ?? false));
|
||||
};
|
||||
|
||||
const context = (): CliContext => ({
|
||||
stdout,
|
||||
stderr,
|
||||
log: log(),
|
||||
sessionDir: paths.data,
|
||||
cacheDir: program.opts<{ cacheDir?: string }>().cacheDir,
|
||||
loadSession,
|
||||
login: (opts) => Client.login(opts),
|
||||
cacheDir: program.opts<ProgramOptions>().cacheDir,
|
||||
loadSession: (path) => loadSession(path, logRequests(log())),
|
||||
login: (opts) => Client.login({ ...opts, apiOptions: logRequests(log()) }),
|
||||
prompt: (message) => input({ message }),
|
||||
promptSecret: (message) => password({ message, mask: true }),
|
||||
});
|
||||
|
||||
const run = (command: Promise<number>): Promise<void> =>
|
||||
runCommand(command, stdout, stderr, (code) => process.exit(code));
|
||||
runCommand(command, stdout, stderr, log(), (code) => process.exit(code));
|
||||
|
||||
program
|
||||
.command("login")
|
||||
@@ -143,7 +172,10 @@ program
|
||||
{
|
||||
...context(),
|
||||
loadSession: (path) =>
|
||||
loadSession(path, { retry: UNATTENDED_RETRY_OPTIONS }),
|
||||
loadSession(
|
||||
path,
|
||||
logRequests(log(), UNATTENDED_RETRY_OPTIONS),
|
||||
),
|
||||
},
|
||||
dir,
|
||||
opts,
|
||||
|
||||
+57
-12
@@ -36,6 +36,15 @@ export interface ApiClientOptions {
|
||||
retry?: RetryOptions;
|
||||
requestTimeoutMs?: number;
|
||||
downloadTimeoutMs?: number;
|
||||
// Called once for each request sent, with the status of its response or
|
||||
// the error it failed with, for a caller that logs requests. A request
|
||||
// that is retried is reported once for each attempt.
|
||||
onRequest?: (request: {
|
||||
method: string;
|
||||
url: string;
|
||||
status?: number;
|
||||
error?: unknown;
|
||||
}) => void;
|
||||
}
|
||||
|
||||
export interface StreamOptions {
|
||||
@@ -159,6 +168,7 @@ export class ApiClient {
|
||||
private readonly retry: ResolvedRetryOptions;
|
||||
private readonly requestTimeoutMs: number;
|
||||
private readonly downloadTimeoutMs: number;
|
||||
private readonly onRequest: NonNullable<ApiClientOptions["onRequest"]>;
|
||||
private token: string | undefined;
|
||||
|
||||
constructor(opts?: ApiClientOptions) {
|
||||
@@ -180,6 +190,7 @@ export class ApiClient {
|
||||
opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
||||
this.downloadTimeoutMs =
|
||||
opts?.downloadTimeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS;
|
||||
this.onRequest = opts?.onRequest ?? (() => {});
|
||||
this.token = opts?.authToken;
|
||||
}
|
||||
|
||||
@@ -214,6 +225,33 @@ export class ApiClient {
|
||||
return h;
|
||||
}
|
||||
|
||||
// Send one request and report it to `onRequest`, with the status of its
|
||||
// response or the error it failed with. `reportedURL` is the URL given to
|
||||
// `onRequest`, when that must not be the one sent.
|
||||
private async send(
|
||||
url: string,
|
||||
init: RequestInit & { method: string },
|
||||
reportedURL = url,
|
||||
): Promise<Response> {
|
||||
let resp: Response;
|
||||
try {
|
||||
resp = await this._fetch(url, init);
|
||||
} catch (err) {
|
||||
this.onRequest({
|
||||
method: init.method,
|
||||
url: reportedURL,
|
||||
error: err,
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
this.onRequest({
|
||||
method: init.method,
|
||||
url: reportedURL,
|
||||
status: resp.status,
|
||||
});
|
||||
return resp;
|
||||
}
|
||||
|
||||
private async throwIfError(resp: Response): Promise<void> {
|
||||
if (resp.ok) return;
|
||||
const requestID = resp.headers.get("x-request-id") ?? undefined;
|
||||
@@ -257,7 +295,7 @@ export class ApiClient {
|
||||
const url = buildURL(this.apiOrigin, path, query);
|
||||
// A GET changes nothing, so it is retried under the full policy.
|
||||
return withRetry(async () => {
|
||||
const resp = await this._fetch(url, {
|
||||
const resp = await this.send(url, {
|
||||
method: "GET",
|
||||
headers: this.headers(),
|
||||
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
||||
@@ -280,7 +318,7 @@ export class ApiClient {
|
||||
// 3xx status, which is not retried.
|
||||
return withRetry(
|
||||
async () => {
|
||||
const resp = await this._fetch(url, {
|
||||
const resp = await this.send(url, {
|
||||
method: "POST",
|
||||
headers: this.headers({
|
||||
"Content-Type": "application/json",
|
||||
@@ -321,16 +359,23 @@ export class ApiClient {
|
||||
// object at one key in one request, so replaying it either overwrites
|
||||
// the same bytes or lands them for the first time. There is no partial
|
||||
// state to protect, hence the full policy rather than the POST rule.
|
||||
// It is reported without its query, which holds the upload's
|
||||
// signature: a log keeps what it is given.
|
||||
const { origin, pathname } = new URL(presignedURL);
|
||||
await withRetry(async () => {
|
||||
const resp = await this._fetch(presignedURL, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Length": String(data.length),
|
||||
const resp = await this.send(
|
||||
presignedURL,
|
||||
{
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Length": String(data.length),
|
||||
},
|
||||
body: data,
|
||||
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
||||
},
|
||||
body: data,
|
||||
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
||||
});
|
||||
origin + pathname,
|
||||
);
|
||||
if (!resp.ok) {
|
||||
// An ApiError, not a bare Error: without the status on the
|
||||
// error the upload path cannot be classified at all, and a
|
||||
@@ -348,7 +393,7 @@ export class ApiClient {
|
||||
// Same replay and redirect rules as `postJSON`, for the same reasons.
|
||||
return withRetry(
|
||||
async () => {
|
||||
const resp = await this._fetch(url, {
|
||||
const resp = await this.send(url, {
|
||||
method: "PUT",
|
||||
headers: this.headers({
|
||||
"Content-Type": "application/json",
|
||||
@@ -394,7 +439,7 @@ export class ApiClient {
|
||||
// headers, when no bytes have arrived either.
|
||||
const deadline = idleDeadline(this.downloadTimeoutMs);
|
||||
try {
|
||||
const resp = await this._fetch(url, {
|
||||
const resp = await this.send(url, {
|
||||
method: "GET",
|
||||
headers: this.headers(),
|
||||
signal: deadline.signal,
|
||||
|
||||
+37
-19
@@ -14,9 +14,9 @@
|
||||
// YYYY/YYYY-MM/YYYY-MM-DD/
|
||||
// YYYY-MM-DD.<fileID>.<ext> the decrypted bytes (the save path)
|
||||
// YYYY-MM-DD.<fileID>.json per-file metadata sidecar, with
|
||||
// the file's ML data and its
|
||||
// original's EXIF, XMP and
|
||||
// dimensions
|
||||
// the file's ML data, its
|
||||
// keywords, and its original's
|
||||
// EXIF, XMP and dimensions
|
||||
// collections/<name>/<title> symlink to the original
|
||||
// collections/<name>.json per-collection metadata
|
||||
// account.json the account's email and user ID
|
||||
@@ -77,11 +77,13 @@ import { readFile } from "node:fs/promises";
|
||||
import { dirname, extname, join, relative, resolve } from "node:path";
|
||||
import lockfile from "proper-lockfile";
|
||||
|
||||
import { clipKeywords } from "./clip-keywords.js";
|
||||
import { removeLeftoverTempFiles } from "./download/index.js";
|
||||
import { describeError } from "./errors.js";
|
||||
import {
|
||||
MAX_NAME_BYTES,
|
||||
sanitizeFileName,
|
||||
shortenKeepingExtension,
|
||||
shortenToBytes,
|
||||
withExtension,
|
||||
} from "./filename.js";
|
||||
@@ -100,7 +102,10 @@ import type { MLData } from "./mldata-fetch.js";
|
||||
import type { Collection, EnteFile, FileMetadata } from "./model/types.js";
|
||||
import { hasTransportCode } from "./retry.js";
|
||||
|
||||
export type ProgressCallback = (message: string) => void;
|
||||
// Reports one line of the backup's progress. `level` is "error" when the line
|
||||
// reports something that failed, such as a file or an album, and is left out
|
||||
// for every other line.
|
||||
export type ProgressCallback = (message: string, level?: "error") => void;
|
||||
|
||||
// The lock in a backup directory, as `lockBackupDirectory` returns it.
|
||||
export interface BackupLock {
|
||||
@@ -289,9 +294,10 @@ const rebuildSymlink = (linkPath: string, target: string): void => {
|
||||
// case (two names that differ only in case are one entry on a case-insensitive
|
||||
// file system), or it is too long for a file name; then it gets ` (<id>)`,
|
||||
// right before the extension when `beforeExtension` is set. To fit, the part
|
||||
// before the ID is cut short as far as it must be. When the extension alone
|
||||
// leaves no room for the ID, the extension is cut short and the part before
|
||||
// the ID keeps only the bytes that cut leaves over. With `withJSON`, each
|
||||
// before the ID is cut short as far as it must be. When the extension leaves
|
||||
// no room for the ID and the first character of the name, the extension is
|
||||
// cut short, and the part before the ID keeps that character and any bytes
|
||||
// the cut leaves over (see `shortenKeepingExtension`). With `withJSON`, each
|
||||
// entry also has the JSON file `<name>.json` beside it, as an album does: a
|
||||
// name then leaves room for `.json`, and one that is another entry's JSON name
|
||||
// gets its ID added too.
|
||||
@@ -305,11 +311,9 @@ const uniqueNames = (
|
||||
const maxBytes = MAX_NAME_BYTES - (opts.withJSON ? ".json".length : 0);
|
||||
const withID = (id: number, name: string): string => {
|
||||
const tag = ` (${id})`;
|
||||
const fullExt = opts.beforeExtension ? extname(name) : "";
|
||||
const stem = name.slice(0, name.length - fullExt.length);
|
||||
const ext = shortenToBytes(fullExt, maxBytes - Buffer.byteLength(tag));
|
||||
const room = maxBytes - Buffer.byteLength(tag + ext);
|
||||
return shortenToBytes(stem, room) + tag + ext;
|
||||
return opts.beforeExtension
|
||||
? shortenKeepingExtension(name, maxBytes, tag)
|
||||
: shortenToBytes(name, maxBytes - Buffer.byteLength(tag)) + tag;
|
||||
};
|
||||
const names = entries.map((e) => e.name);
|
||||
const suffixed = new Set<number>();
|
||||
@@ -494,8 +498,9 @@ const imageMetadataFor = async (
|
||||
}
|
||||
};
|
||||
|
||||
// The file's JSON: its basic fields, its magic metadata, its ML data or the
|
||||
// reason the ML data is missing, and its image metadata.
|
||||
// The file's JSON: its basic fields, its magic metadata, its ML data without
|
||||
// the CLIP embedding and the keywords that embedding gives, or the reason the
|
||||
// ML data is missing, and its image metadata.
|
||||
const writeSidecar = (
|
||||
path: string,
|
||||
file: EnteFile,
|
||||
@@ -511,7 +516,13 @@ const writeSidecar = (
|
||||
};
|
||||
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
||||
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
||||
if (ml.mlData) meta.mlData = ml.mlData;
|
||||
if (ml.mlData) {
|
||||
const { clip: _clip, ...mlDataWithoutClip } = ml.mlData;
|
||||
meta.mlData = mlDataWithoutClip;
|
||||
const isVideo = file.metadata.fileType === "video";
|
||||
const keywords = clipKeywords(ml.mlData, isVideo);
|
||||
if (keywords) meta.keywords = keywords;
|
||||
}
|
||||
if (ml.mlDataError) meta.mlDataError = ml.mlDataError;
|
||||
if (image.imageMetadata) meta.imageMetadata = image.imageMetadata;
|
||||
if (image.imageMetadataError) {
|
||||
@@ -654,7 +665,7 @@ const runLockedBackup = async (
|
||||
for (const record of lib.undecryptable()) {
|
||||
const error = `could not be decrypted: ${record.error}`;
|
||||
if (record.fileID === undefined) {
|
||||
log(`FAILED album ${record.collectionID}: ${error}`);
|
||||
log(`FAILED album ${record.collectionID}: ${error}`, "error");
|
||||
undecryptableAlbums.push(record.collectionID);
|
||||
} else if (scope.has(record.collectionID)) {
|
||||
const copies = memberships.get(record.fileID);
|
||||
@@ -699,6 +710,7 @@ const runLockedBackup = async (
|
||||
} catch (err) {
|
||||
log(
|
||||
`FAILED verifying original ${file.metadata.title}: ${describeError(err)}`,
|
||||
"error",
|
||||
);
|
||||
recordFailure(
|
||||
file.id,
|
||||
@@ -732,6 +744,7 @@ const runLockedBackup = async (
|
||||
} catch (err) {
|
||||
log(
|
||||
`FAILED original ${file.metadata.title}: ${describeError(err)}`,
|
||||
"error",
|
||||
);
|
||||
recordFailure(
|
||||
file.id,
|
||||
@@ -807,7 +820,7 @@ const runLockedBackup = async (
|
||||
await lib.fetchMLData();
|
||||
} catch (err) {
|
||||
mlDataFailure = err;
|
||||
log(`FAILED ML data: ${describeError(err)}`);
|
||||
log(`FAILED ML data: ${describeError(err)}`, "error");
|
||||
}
|
||||
for (const file of distinct.values()) {
|
||||
if (lock.lost) throw lock.lost;
|
||||
@@ -868,7 +881,10 @@ const runLockedBackup = async (
|
||||
downloadDirectory,
|
||||
);
|
||||
} catch (err) {
|
||||
log(`FAILED removing old album directories: ${describeError(err)}`);
|
||||
log(
|
||||
`FAILED removing old album directories: ${describeError(err)}`,
|
||||
"error",
|
||||
);
|
||||
}
|
||||
|
||||
// An album whose folder or JSON cannot be written, such as one whose
|
||||
@@ -897,6 +913,7 @@ const runLockedBackup = async (
|
||||
} catch (err) {
|
||||
log(
|
||||
`FAILED removing old links in ${c.name}: ${describeError(err)}`,
|
||||
"error",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -915,6 +932,7 @@ const runLockedBackup = async (
|
||||
} catch (err) {
|
||||
log(
|
||||
`FAILED symlink ${c.name}/${linkName}: ${describeError(err)}`,
|
||||
"error",
|
||||
);
|
||||
recordFailure(
|
||||
link.file.id,
|
||||
@@ -931,7 +949,7 @@ const runLockedBackup = async (
|
||||
metaFiles,
|
||||
);
|
||||
} catch (err) {
|
||||
log(`FAILED album ${c.name}: ${describeError(err)}`);
|
||||
log(`FAILED album ${c.name}: ${describeError(err)}`, "error");
|
||||
failedAlbums.push(c.id);
|
||||
}
|
||||
}
|
||||
|
||||
+113
-105
@@ -4,7 +4,10 @@
|
||||
// code; a thrown error is left to the caller. Nothing here calls
|
||||
// `process.exit`: `bin/quak.ts` wires these to the command line, and `run` in
|
||||
// `cli-run.ts` prints a thrown error as one line and exits once output has
|
||||
// drained. Output must stay byte-identical (see `cli-output.ts`).
|
||||
// drained. Output must stay byte-identical (see `cli-output.ts`). Each line on
|
||||
// stderr goes through `ctx.log` at its level (see `cli-log.ts`). Every command
|
||||
// that exits non-zero logs a fatal line saying why; one that exits 1 because
|
||||
// files or albums failed logs it last, saying how many and where to look.
|
||||
|
||||
import {
|
||||
copyFileSync,
|
||||
@@ -33,6 +36,7 @@ import {
|
||||
originalName,
|
||||
thumbnailName,
|
||||
} from "./cli-output.js";
|
||||
import { type Log, logRefresh } from "./cli-log.js";
|
||||
import { freshCollections, freshFiles, freshFile } from "./cli-read.js";
|
||||
import { withExtension } from "./filename.js";
|
||||
import { runMetadataBackup } from "./metadata-backup.js";
|
||||
@@ -40,7 +44,8 @@ import { listMissingThumbnails, fixMissingThumbnails } from "./thumbnails.js";
|
||||
|
||||
export interface CliContext {
|
||||
stdout: { write(text: string): unknown };
|
||||
stderr: { write(text: string): unknown };
|
||||
// Writes a line on stderr at a log level, if the run's level prints it.
|
||||
log: Log;
|
||||
// Directory holding `session.json`.
|
||||
sessionDir: string;
|
||||
// The `--cache-dir` global, or undefined to let the library pick its
|
||||
@@ -82,16 +87,16 @@ const requireSession = (ctx: CliContext): Client | undefined => {
|
||||
try {
|
||||
client = ctx.loadSession(sessionPath(ctx));
|
||||
} catch (err) {
|
||||
ctx.stderr.write(
|
||||
`${err instanceof Error ? err.message : err}\n` +
|
||||
`Run "quak logout" and then "quak login" to replace it.\n`,
|
||||
ctx.log("fatal", err instanceof Error ? err.message : String(err));
|
||||
ctx.log(
|
||||
"fatal",
|
||||
`Run "quak logout" and then "quak login" to replace it.`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
if (!client) {
|
||||
ctx.stderr.write(
|
||||
`Not logged in. Run "quak login" first.\nSession file: ${sessionPath(ctx)}\n`,
|
||||
);
|
||||
ctx.log("fatal", `Not logged in. Run "quak login" first.`);
|
||||
ctx.log("fatal", `Session file: ${sessionPath(ctx)}`);
|
||||
return undefined;
|
||||
}
|
||||
return client;
|
||||
@@ -107,7 +112,7 @@ const parseID = (
|
||||
): number | undefined => {
|
||||
const id = Number(text);
|
||||
if (/^[0-9]+$/.test(text) && Number.isSafeInteger(id)) return id;
|
||||
ctx.stderr.write(`Invalid ${kind} ID: ${text}\n`);
|
||||
ctx.log("fatal", `Invalid ${kind} ID: ${text}`);
|
||||
return undefined;
|
||||
};
|
||||
|
||||
@@ -128,13 +133,8 @@ const readLibraryClient = (client: Client): LibraryClient => ({
|
||||
// downloading the whole account — and no refresh starts as it opens or on a
|
||||
// timer: each command asks for the one it needs, and `collections` needs none
|
||||
// (issue #210). While a refresh runs, each collection whose files it has
|
||||
// fetched is counted on stderr, unless `json` is set, so a large account shows
|
||||
// progress rather than nothing.
|
||||
const openReadLibrary = (
|
||||
ctx: CliContext,
|
||||
client: Client,
|
||||
opts: { json?: boolean } = {},
|
||||
): Promise<Library> =>
|
||||
// fetched is logged, so a large account shows progress rather than nothing.
|
||||
const openReadLibrary = (ctx: CliContext, client: Client): Promise<Library> =>
|
||||
Library.open({
|
||||
client: readLibraryClient(client),
|
||||
cacheDirectory: ctx.cacheDir,
|
||||
@@ -142,12 +142,7 @@ const openReadLibrary = (
|
||||
refreshIntervalSeconds: Infinity,
|
||||
precacheThumbnails: false,
|
||||
precacheOriginals: false,
|
||||
onProgress: (event) => {
|
||||
if (opts.json || event.status !== "progress") return;
|
||||
ctx.stderr.write(
|
||||
`Fetched the files of ${event.fetched} of ${event.total} collections\n`,
|
||||
);
|
||||
},
|
||||
onProgress: logRefresh(ctx.log),
|
||||
});
|
||||
|
||||
export const loginCommand = async (ctx: CliContext): Promise<number> => {
|
||||
@@ -156,7 +151,7 @@ export const loginCommand = async (ctx: CliContext): Promise<number> => {
|
||||
const password =
|
||||
process.env.QUAK_PASSWORD ?? (await ctx.promptSecret("Password"));
|
||||
|
||||
ctx.stderr.write("Authenticating...\n");
|
||||
ctx.log("info", "Authenticating...");
|
||||
try {
|
||||
const client = await ctx.login({
|
||||
email,
|
||||
@@ -167,11 +162,12 @@ export const loginCommand = async (ctx: CliContext): Promise<number> => {
|
||||
|
||||
saveSession(ctx.sessionDir, client.toJSON());
|
||||
const info = client.whoami();
|
||||
ctx.stderr.write(`Logged in as ${info.email} (user ${info.userID})\n`);
|
||||
ctx.stderr.write(`Session saved to ${sessionPath(ctx)}\n`);
|
||||
ctx.log("info", `Logged in as ${info.email} (user ${info.userID})`);
|
||||
ctx.log("info", `Session saved to ${sessionPath(ctx)}`);
|
||||
} catch (err) {
|
||||
ctx.stderr.write(
|
||||
`Login failed: ${err instanceof Error ? err.message : err}\n`,
|
||||
ctx.log(
|
||||
"fatal",
|
||||
`Login failed: ${err instanceof Error ? err.message : err}`,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
@@ -193,7 +189,7 @@ export const whoamiCommand = async (ctx: CliContext): Promise<number> => {
|
||||
export const logoutCommand = async (ctx: CliContext): Promise<number> => {
|
||||
const path = sessionPath(ctx);
|
||||
if (!existsSync(path)) {
|
||||
ctx.stderr.write("No session found.\n");
|
||||
ctx.log("info", "No session found.");
|
||||
return 0;
|
||||
}
|
||||
await init();
|
||||
@@ -211,16 +207,15 @@ export const logoutCommand = async (ctx: CliContext): Promise<number> => {
|
||||
}
|
||||
unlinkSync(path);
|
||||
if (failure === undefined) {
|
||||
ctx.stderr.write("Session ended on the server.\n");
|
||||
ctx.log("info", "Session ended on the server.");
|
||||
} else {
|
||||
ctx.stderr.write(
|
||||
`Could not end the session on the server: ${failure}\n`,
|
||||
);
|
||||
ctx.log("fatal", `Could not end the session on the server: ${failure}`);
|
||||
}
|
||||
ctx.stderr.write("Session deleted.\n");
|
||||
ctx.log("info", "Session deleted.");
|
||||
if (cacheDir !== undefined) {
|
||||
ctx.stderr.write(
|
||||
`Cache directory ${cacheDir} still holds decrypted data; delete it to remove that data.\n`,
|
||||
ctx.log(
|
||||
"warning",
|
||||
`Cache directory ${cacheDir} still holds decrypted data; delete it to remove that data.`,
|
||||
);
|
||||
}
|
||||
return failure === undefined ? 0 : 1;
|
||||
@@ -285,7 +280,7 @@ export const filesCommand = async (
|
||||
: parseID(ctx, "collection", collectionIDStr);
|
||||
if (collectionIDStr !== undefined && collectionID === undefined) return 1;
|
||||
|
||||
const lib = await openReadLibrary(ctx, client, opts);
|
||||
const lib = await openReadLibrary(ctx, client);
|
||||
try {
|
||||
// Force a server round-trip and list in enumeration order (issue #36
|
||||
// amendment, issue #52). Each file prints from its own decrypted
|
||||
@@ -294,7 +289,7 @@ export const filesCommand = async (
|
||||
// newest-first order.
|
||||
const files = await freshFiles(lib, collectionID);
|
||||
if (!files) {
|
||||
ctx.stderr.write(`Collection ${collectionID} not found\n`);
|
||||
ctx.log("fatal", `Collection ${collectionID} not found`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -330,7 +325,7 @@ export const getCommand = async (
|
||||
// (issue #36 amendment, issue #52).
|
||||
const resolved = await freshFile(lib, fileID);
|
||||
if (!resolved) {
|
||||
ctx.stderr.write(`File ${fileID} not found\n`);
|
||||
ctx.log("fatal", `File ${fileID} not found`);
|
||||
return 1;
|
||||
}
|
||||
const { photo, file } = resolved;
|
||||
@@ -338,30 +333,33 @@ export const getCommand = async (
|
||||
const result = await photo.original();
|
||||
// Default name is the file's own title, as the pre-library CLI used
|
||||
// (not the editedName-preferring projection title) (issue #52).
|
||||
const outPath = opts.out ?? originalName(file);
|
||||
if (result.videoPath === undefined) {
|
||||
const outPath = opts.out ?? originalName(file);
|
||||
copyFileSync(result.path, outPath);
|
||||
ctx.stderr.write(`${result.bytes} bytes -> ${outPath}\n`);
|
||||
ctx.log("info", `${result.bytes} bytes -> ${outPath}`);
|
||||
return 0;
|
||||
}
|
||||
// A live photo is written as its image and its video, each named after
|
||||
// the title with its own extension, as Ente's clients name them. With
|
||||
// --out, the image goes there and the video beside it.
|
||||
const imageOut =
|
||||
opts.out ?? withExtension(outPath, extname(result.path));
|
||||
const videoOut = withExtension(outPath, extname(result.videoPath));
|
||||
const imageExt = extname(result.path);
|
||||
const videoExt = extname(result.videoPath);
|
||||
const imageOut = opts.out ?? originalName(file, imageExt);
|
||||
const videoOut =
|
||||
opts.out === undefined
|
||||
? originalName(file, videoExt)
|
||||
: withExtension(opts.out, videoExt);
|
||||
if (imageOut.toLowerCase() === videoOut.toLowerCase()) {
|
||||
ctx.stderr.write(
|
||||
`File ${fileID} is a live photo, and its video would also be written to ${imageOut}\n`,
|
||||
ctx.log(
|
||||
"fatal",
|
||||
`File ${fileID} is a live photo, and its video would also be written to ${imageOut}`,
|
||||
);
|
||||
return 1;
|
||||
}
|
||||
copyFileSync(result.path, imageOut);
|
||||
copyFileSync(result.videoPath, videoOut);
|
||||
ctx.stderr.write(
|
||||
`${result.bytes} bytes -> ${imageOut}\n` +
|
||||
`${statSync(videoOut).size} bytes -> ${videoOut}\n`,
|
||||
);
|
||||
ctx.log("info", `${result.bytes} bytes -> ${imageOut}`);
|
||||
ctx.log("info", `${statSync(videoOut).size} bytes -> ${videoOut}`);
|
||||
return 0;
|
||||
} finally {
|
||||
await lib.close();
|
||||
@@ -385,7 +383,7 @@ export const getThumbCommand = async (
|
||||
// (issue #36 amendment, issue #52).
|
||||
const resolved = await freshFile(lib, fileID);
|
||||
if (!resolved) {
|
||||
ctx.stderr.write(`File ${fileID} not found\n`);
|
||||
ctx.log("fatal", `File ${fileID} not found`);
|
||||
return 1;
|
||||
}
|
||||
const { photo, file } = resolved;
|
||||
@@ -395,7 +393,7 @@ export const getThumbCommand = async (
|
||||
// used (not the projection title) (issue #52).
|
||||
const outPath = opts.out ?? thumbnailName(file);
|
||||
copyFileSync(result.path, outPath);
|
||||
ctx.stderr.write(`${result.bytes} bytes -> ${outPath}\n`);
|
||||
ctx.log("info", `${result.bytes} bytes -> ${outPath}`);
|
||||
return 0;
|
||||
} finally {
|
||||
await lib.close();
|
||||
@@ -417,9 +415,14 @@ export const backupMetadataCommand = async (
|
||||
await lib.fresh();
|
||||
const result = await runMetadataBackup(lib, client, dir, {
|
||||
exif: opts.exif || opts.all,
|
||||
onProgress: (msg) => ctx.stderr.write(msg + "\n"),
|
||||
onProgress: (msg, level) => ctx.log(level ?? "info", msg),
|
||||
});
|
||||
return result.filesWithMLDataError > 0 ? 1 : 0;
|
||||
if (result.filesWithMLDataError === 0) return 0;
|
||||
ctx.log(
|
||||
"fatal",
|
||||
`quak: the ML data of ${result.filesWithMLDataError} file(s) could not be read; see mlDataError in their JSON in ${dir}`,
|
||||
);
|
||||
return 1;
|
||||
} finally {
|
||||
await lib.close();
|
||||
}
|
||||
@@ -434,7 +437,7 @@ export const backupCommand = async (
|
||||
const client = requireSession(ctx);
|
||||
if (!client) return 3;
|
||||
|
||||
ctx.stderr.write("Starting backup...\n");
|
||||
ctx.log("info", "Starting backup...");
|
||||
// The lock is taken before the library opens and starts its refresh, so a
|
||||
// run refused while another backup of `dir` runs sends no request.
|
||||
let lock: BackupLock;
|
||||
@@ -442,7 +445,7 @@ export const backupCommand = async (
|
||||
lock = await lockBackupDirectory(dir);
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ELOCKED") throw err;
|
||||
ctx.stderr.write(`quak: ${(err as Error).message}\n`);
|
||||
ctx.log("fatal", `quak: ${(err as Error).message}`);
|
||||
return 2;
|
||||
}
|
||||
try {
|
||||
@@ -457,45 +460,48 @@ export const backupCommand = async (
|
||||
refreshIntervalSeconds: Infinity,
|
||||
precacheThumbnails: false,
|
||||
precacheOriginals: false,
|
||||
onProgress: logRefresh(ctx.log),
|
||||
});
|
||||
try {
|
||||
const result = await lib.backup({
|
||||
downloadDirectory: dir,
|
||||
lock,
|
||||
verify: opts.verify,
|
||||
onProgress: (msg) => {
|
||||
if (!opts.json) ctx.stderr.write(msg + "\n");
|
||||
},
|
||||
onProgress: (msg, level) => ctx.log(level ?? "info", msg),
|
||||
});
|
||||
|
||||
if (opts.json) {
|
||||
ctx.stdout.write(JSON.stringify(result, null, 2) + "\n");
|
||||
} else {
|
||||
ctx.stderr.write("\n--- Backup complete ---\n");
|
||||
ctx.stderr.write(` Total files: ${result.totalFiles}\n`);
|
||||
ctx.stderr.write(` Downloaded: ${result.downloaded}\n`);
|
||||
ctx.stderr.write(` Skipped: ${result.skipped}\n`);
|
||||
ctx.log("info", "\n--- Backup complete ---");
|
||||
ctx.log("info", ` Total files: ${result.totalFiles}`);
|
||||
ctx.log("info", ` Downloaded: ${result.downloaded}`);
|
||||
ctx.log("info", ` Skipped: ${result.skipped}`);
|
||||
if (opts.verify) {
|
||||
ctx.stderr.write(` Verified: ${result.verified}\n`);
|
||||
ctx.stderr.write(` Mismatched: ${result.mismatched}\n`);
|
||||
ctx.stderr.write(` Unchecked: ${result.unchecked}\n`);
|
||||
ctx.log("info", ` Verified: ${result.verified}`);
|
||||
ctx.log("info", ` Mismatched: ${result.mismatched}`);
|
||||
ctx.log("info", ` Unchecked: ${result.unchecked}`);
|
||||
}
|
||||
ctx.stderr.write(` Failed: ${result.failed}\n`);
|
||||
ctx.log("info", ` Failed: ${result.failed}`);
|
||||
if (result.errors.length > 0) {
|
||||
ctx.stderr.write("\nFailed files:\n");
|
||||
ctx.log("error", "\nFailed files:");
|
||||
for (const e of result.errors) {
|
||||
ctx.stderr.write(
|
||||
` [${e.collection}] ${e.title} (id ${e.fileID}): ${e.error}\n`,
|
||||
ctx.log(
|
||||
"error",
|
||||
` [${e.collection}] ${e.title} (id ${e.fileID}): ${e.error}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result.failed > 0 ||
|
||||
result.undecryptableAlbums.length > 0 ||
|
||||
result.failedAlbums.length > 0
|
||||
? 1
|
||||
: 0;
|
||||
const albums =
|
||||
result.undecryptableAlbums.length + result.failedAlbums.length;
|
||||
if (result.failed === 0 && albums === 0) return 0;
|
||||
ctx.log(
|
||||
"fatal",
|
||||
`quak: ${result.failed} file(s) and ${albums} album(s) failed; see ${join(dir, "failures.json")} and the FAILED lines`,
|
||||
);
|
||||
return 1;
|
||||
} finally {
|
||||
await lib.close();
|
||||
}
|
||||
@@ -511,23 +517,24 @@ export const listMissingThumbnailsCommand = async (
|
||||
await init();
|
||||
const client = requireSession(ctx);
|
||||
if (!client) return 3;
|
||||
const lib = await openReadLibrary(ctx, client, opts);
|
||||
const lib = await openReadLibrary(ctx, client);
|
||||
try {
|
||||
// Refresh first so files added since the cache was written are
|
||||
// checked; a failed refresh throws.
|
||||
await lib.fresh();
|
||||
const missing = await listMissingThumbnails(lib, client, (msg) => {
|
||||
if (!opts.json) ctx.stderr.write(msg + "\n");
|
||||
});
|
||||
const missing = await listMissingThumbnails(lib, client, (msg, level) =>
|
||||
ctx.log(level ?? "info", msg),
|
||||
);
|
||||
|
||||
if (opts.json) {
|
||||
ctx.stdout.write(JSON.stringify(missing, null, 2) + "\n");
|
||||
} else {
|
||||
if (missing.length === 0) {
|
||||
ctx.stderr.write("No missing thumbnails found.\n");
|
||||
ctx.log("info", "No missing thumbnails found.");
|
||||
} else {
|
||||
ctx.stderr.write(
|
||||
`\n${missing.length} file(s) with missing thumbnails:\n`,
|
||||
ctx.log(
|
||||
"info",
|
||||
`\n${missing.length} file(s) with missing thumbnails:`,
|
||||
);
|
||||
for (const m of missing) {
|
||||
ctx.stdout.write(
|
||||
@@ -556,33 +563,34 @@ export const fixMissingThumbnailsCommand = async (
|
||||
if (fileID === undefined) return 1;
|
||||
fileIDs.push(fileID);
|
||||
}
|
||||
const lib = await openReadLibrary(ctx, client, opts);
|
||||
const lib = await openReadLibrary(ctx, client);
|
||||
try {
|
||||
// Refresh first so files added since the cache was written are found;
|
||||
// a failed refresh throws.
|
||||
await lib.fresh();
|
||||
if (fileIDs.length === 0) {
|
||||
ctx.stderr.write("Scanning for missing thumbnails...\n");
|
||||
const missing = await listMissingThumbnails(lib, client, (msg) => {
|
||||
if (!opts.json) ctx.stderr.write(msg + "\n");
|
||||
});
|
||||
ctx.log("info", "Scanning for missing thumbnails...");
|
||||
const missing = await listMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
(msg, level) => ctx.log(level ?? "info", msg),
|
||||
);
|
||||
fileIDs = missing.map((m) => m.fileID);
|
||||
if (fileIDs.length === 0) {
|
||||
ctx.stderr.write("No missing thumbnails found.\n");
|
||||
ctx.log("info", "No missing thumbnails found.");
|
||||
return 0;
|
||||
}
|
||||
ctx.stderr.write(`Found ${fileIDs.length} file(s) to fix.\n`);
|
||||
ctx.log("info", `Found ${fileIDs.length} file(s) to fix.`);
|
||||
}
|
||||
|
||||
const results = await fixMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
fileIDs,
|
||||
(msg) => {
|
||||
if (!opts.json) ctx.stderr.write(msg + "\n");
|
||||
},
|
||||
(msg, level) => ctx.log(level ?? "info", msg),
|
||||
);
|
||||
|
||||
const failed = results.filter((r) => r.status === "failed").length;
|
||||
if (opts.json) {
|
||||
ctx.stdout.write(JSON.stringify(results, null, 2) + "\n");
|
||||
} else {
|
||||
@@ -590,30 +598,30 @@ export const fixMissingThumbnailsCommand = async (
|
||||
const skipped = results.filter(
|
||||
(r) => r.status === "skipped",
|
||||
).length;
|
||||
const failed = results.filter((r) => r.status === "failed").length;
|
||||
ctx.stderr.write(`\n--- Done ---\n`);
|
||||
ctx.stderr.write(` Fixed: ${fixed}\n`);
|
||||
ctx.stderr.write(` Skipped: ${skipped}\n`);
|
||||
ctx.stderr.write(` Failed: ${failed}\n`);
|
||||
ctx.log("info", `\n--- Done ---`);
|
||||
ctx.log("info", ` Fixed: ${fixed}`);
|
||||
ctx.log("info", ` Skipped: ${skipped}`);
|
||||
ctx.log("info", ` Failed: ${failed}`);
|
||||
if (skipped > 0) {
|
||||
ctx.stderr.write("\nSkipped:\n");
|
||||
ctx.log("info", "\nSkipped:");
|
||||
for (const r of results.filter((r) => r.status === "skipped")) {
|
||||
ctx.stderr.write(
|
||||
` ${r.fileID}\t${r.title}\t${r.reason}\n`,
|
||||
);
|
||||
ctx.log("info", ` ${r.fileID}\t${r.title}\t${r.reason}`);
|
||||
}
|
||||
}
|
||||
if (failed > 0) {
|
||||
ctx.stderr.write("\nFailed files:\n");
|
||||
ctx.log("error", "\nFailed files:");
|
||||
for (const r of results.filter((r) => r.status === "failed")) {
|
||||
ctx.stderr.write(
|
||||
` ${r.fileID}\t${r.title}\t${r.reason}\n`,
|
||||
);
|
||||
ctx.log("error", ` ${r.fileID}\t${r.title}\t${r.reason}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results.some((r) => r.status === "failed") ? 1 : 0;
|
||||
if (failed === 0) return 0;
|
||||
ctx.log(
|
||||
"fatal",
|
||||
`quak: ${failed} file(s) failed; see the "Failed files:" list, or the JSON with --json`,
|
||||
);
|
||||
return 1;
|
||||
} finally {
|
||||
await lib.close();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
// The CLI's log level, and the one function the commands write stderr lines
|
||||
// through.
|
||||
//
|
||||
// Every line quak prints on stderr is a log line at one of six levels, from
|
||||
// the quietest: fatal, error, warning, info, debug, trace. A run prints the
|
||||
// lines at its own level and at every quieter one. Its level is info unless
|
||||
// `-v` or `-q` asks for another (see "CLI surface" in the README). Every
|
||||
// command that exits non-zero logs a fatal line saying why, so even `-q`, which
|
||||
// prints only fatal lines, never leaves a non-zero exit unexplained. An error
|
||||
// line reports something that failed, such as a file or an album, and a
|
||||
// warning something that did not fail but needs the user's attention. stdout
|
||||
// carries only a command's output and is not a log.
|
||||
//
|
||||
// The library itself writes no lines: its layers report through callbacks, and
|
||||
// the CLI logs what they report. `logRequests` gives the API client options
|
||||
// that log each request and each retry, and `logRefresh` the library's
|
||||
// `onProgress` that logs what a refresh and an ML data fetch report.
|
||||
|
||||
import type { ApiClientOptions } from "./api/client.js";
|
||||
import { describeError } from "./errors.js";
|
||||
import type { RefreshEvent } from "./library/index.js";
|
||||
import type { RetryOptions } from "./retry.js";
|
||||
|
||||
const LEVELS = ["fatal", "error", "warning", "info", "debug", "trace"] as const;
|
||||
|
||||
export type LogLevel = (typeof LEVELS)[number];
|
||||
|
||||
// Writes `line` on stderr when the run's level prints lines at `level`.
|
||||
export type Log = (level: LogLevel, line: string) => void;
|
||||
|
||||
// The level asked for by `-v` given `verbose` times, or by `-q`: info without
|
||||
// either, debug for `-v`, trace for `-vv` and more, fatal for `-q`.
|
||||
// `bin/quak.ts` refuses `-q` and `-v` together.
|
||||
export const logLevel = (verbose: number, quiet: boolean): LogLevel => {
|
||||
if (quiet) return "fatal";
|
||||
if (verbose === 0) return "info";
|
||||
return verbose === 1 ? "debug" : "trace";
|
||||
};
|
||||
|
||||
// The log that writes each line at `level` or quieter to `stderr`, ending it
|
||||
// with a newline.
|
||||
export const stderrLog =
|
||||
(stderr: { write(text: string): unknown }, level: LogLevel): Log =>
|
||||
(lineLevel, line) => {
|
||||
if (LEVELS.indexOf(lineLevel) <= LEVELS.indexOf(level)) {
|
||||
stderr.write(line + "\n");
|
||||
}
|
||||
};
|
||||
|
||||
// API client options that log each request at trace, with the status of its
|
||||
// response or the error it failed with, and each retry at debug, with why and
|
||||
// how long it waits. `retry` is the retry policy, the default when not given.
|
||||
export const logRequests = (
|
||||
log: Log,
|
||||
retry?: RetryOptions,
|
||||
): ApiClientOptions => ({
|
||||
retry: {
|
||||
...retry,
|
||||
onRetry: (err, attempt, delayMs) =>
|
||||
log(
|
||||
"debug",
|
||||
`Attempt ${attempt} failed, retrying in ${Math.round(delayMs)} ms: ${describeError(err)}`,
|
||||
),
|
||||
},
|
||||
onRequest: ({ method, url, status, error }) =>
|
||||
log(
|
||||
"trace",
|
||||
`${method} ${url}: ${error === undefined ? `HTTP ${status}` : describeError(error)}`,
|
||||
),
|
||||
});
|
||||
|
||||
// How a log line names each operation the library reports the start and end
|
||||
// of.
|
||||
const OPERATION_NAMES: Record<RefreshEvent["operation"], string> = {
|
||||
refresh: "Refresh",
|
||||
fetchMLData: "ML data fetch",
|
||||
precacheThumbnails: "Thumbnail precache",
|
||||
precacheOriginals: "Original precache",
|
||||
};
|
||||
|
||||
// A library `onProgress` that logs, at info, each collection whose files a
|
||||
// refresh has fetched, and at debug the start and end of each refresh and ML
|
||||
// data fetch, with the error of one that failed.
|
||||
export const logRefresh =
|
||||
(log: Log) =>
|
||||
(event: RefreshEvent): void => {
|
||||
if (event.status === "progress") {
|
||||
log(
|
||||
"info",
|
||||
`Fetched the files of ${event.fetched} of ${event.total} collections`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const error = event.error === undefined ? "" : `: ${event.error}`;
|
||||
log(
|
||||
"debug",
|
||||
`${OPERATION_NAMES[event.operation]} ${event.status}${error}`,
|
||||
);
|
||||
};
|
||||
+9
-7
@@ -9,7 +9,7 @@
|
||||
// `metadata.title`, and issue #52 requires that output stay byte-identical, so
|
||||
// the commands shape their output from the raw `EnteFile` through here.
|
||||
|
||||
import { sanitizeFileName } from "./filename.js";
|
||||
import { defaultFileName } from "./filename.js";
|
||||
import type { EnteFile, FileType, Microseconds } from "./model/types.js";
|
||||
|
||||
// One row of `quak files --json`.
|
||||
@@ -33,11 +33,13 @@ export const fileListRow = (file: EnteFile): FileListRow => ({
|
||||
export const fileListLine = (file: EnteFile): string =>
|
||||
`${file.id}\t${file.metadata.fileType}\t${file.metadata.title}`;
|
||||
|
||||
// Default output path for `quak get` when `--out` is not given. The title comes
|
||||
// from the server, so it is sanitized; `--out` is the user's and is used as is.
|
||||
export const originalName = (file: EnteFile): string =>
|
||||
sanitizeFileName(file.metadata.title, `file-${file.id}`);
|
||||
// Default output path for `quak get` when `--out` is not given, made from the
|
||||
// title (see `defaultFileName`), with `ext` in place of its extension for a
|
||||
// live photo's image or video; `--out` is the user's and is used as is.
|
||||
export const originalName = (file: EnteFile, ext?: string): string =>
|
||||
defaultFileName(file.metadata.title, file.id, { ext });
|
||||
|
||||
// Default output path for `quak get-thumb` when `--out` is not given.
|
||||
// Default output path for `quak get-thumb` when `--out` is not given: `thumb_`
|
||||
// and the title.
|
||||
export const thumbnailName = (file: EnteFile): string =>
|
||||
`thumb_${originalName(file)}`;
|
||||
defaultFileName(file.metadata.title, file.id, { prefix: "thumb_" });
|
||||
|
||||
+12
-9
@@ -2,21 +2,23 @@
|
||||
|
||||
import type { Writable } from "node:stream";
|
||||
import { ApiError } from "./api/client.js";
|
||||
import type { Log } from "./cli-log.js";
|
||||
import { describeError } from "./errors.js";
|
||||
|
||||
// Run a command and exit with its code once stdout/stderr have drained.
|
||||
// Exiting before the drain can truncate piped output, and the library can keep
|
||||
// the event loop alive after a command returns, so a plain return could hang.
|
||||
// An error the command throws is printed as one `quak: MESSAGE` line, without
|
||||
// the stack trace, and exits 1. MESSAGE is the error as `describeError` gives
|
||||
// it, so it carries the reason Node's `fetch` puts on `cause`. A 401 from the
|
||||
// server means it no longer accepts the saved session: that prints one line
|
||||
// saying to log in again and exits 3, as a missing or corrupt session file
|
||||
// does.
|
||||
// An error the command throws is logged as one fatal `quak: MESSAGE` line,
|
||||
// without the stack trace, and exits 1. MESSAGE is the error as
|
||||
// `describeError` gives it, so it carries the reason Node's `fetch` puts on
|
||||
// `cause`. A 401 from the server means it no longer accepts the saved session:
|
||||
// that logs one fatal line saying to log in again and exits 3, as a missing or
|
||||
// corrupt session file does.
|
||||
export const run = async (
|
||||
command: Promise<number>,
|
||||
stdout: Writable,
|
||||
stderr: Writable,
|
||||
log: Log,
|
||||
exit: (code: number) => void,
|
||||
): Promise<void> => {
|
||||
let code: number;
|
||||
@@ -24,12 +26,13 @@ export const run = async (
|
||||
code = await command;
|
||||
} catch (err) {
|
||||
if (err instanceof ApiError && err.status === 401) {
|
||||
stderr.write(
|
||||
`quak: the saved session is no longer valid; run "quak login"\n`,
|
||||
log(
|
||||
"fatal",
|
||||
`quak: the saved session is no longer valid; run "quak login"`,
|
||||
);
|
||||
code = 3;
|
||||
} else {
|
||||
stderr.write(`quak: ${describeError(err)}\n`);
|
||||
log("fatal", `quak: ${describeError(err)}`);
|
||||
code = 1;
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,57 @@
|
||||
// Keywords for a file from its CLIP embedding.
|
||||
//
|
||||
// Ente stores no keywords. A file's ML data holds a CLIP embedding, the 512
|
||||
// numbers Ente's image model gave the photo, and Ente's photos app sorts files
|
||||
// into the categories of its Discover section with it: a file falls in a
|
||||
// category when the cosine similarity of its CLIP embedding and the text
|
||||
// embedding of the category's query is at least the category's `minScore`,
|
||||
// unless the file is a video and the category leaves out videos. A file's
|
||||
// keywords are the titles of the categories it falls in. The categories and
|
||||
// their text embeddings are in `clip-keyword-vectors.ts`.
|
||||
|
||||
import { DISCOVER_CATEGORIES } from "./clip-keyword-vectors.js";
|
||||
import { clipEmbedding } from "./library/mldata.js";
|
||||
import type { MLData } from "./mldata-fetch.js";
|
||||
|
||||
// How many numbers a CLIP embedding from Ente's image model has.
|
||||
const EMBEDDING_LENGTH = 512;
|
||||
|
||||
const norm = (vector: number[]): number => Math.hypot(...vector);
|
||||
|
||||
const cosineSimilarity = (a: number[], b: number[]): number => {
|
||||
let dot = 0;
|
||||
a.forEach((value, i) => {
|
||||
dot += value * (b[i] ?? 0);
|
||||
});
|
||||
return dot / (norm(a) * norm(b));
|
||||
};
|
||||
|
||||
// The titles of the Discover categories a file falls in, highest score first,
|
||||
// or undefined when its ML data has no CLIP embedding of 512 finite numbers
|
||||
// that are not all zero. Tests pass their own `categories`.
|
||||
export const clipKeywords = (
|
||||
mlData: MLData,
|
||||
isVideo: boolean,
|
||||
categories = DISCOVER_CATEGORIES,
|
||||
): string[] | undefined => {
|
||||
const embedding = clipEmbedding(mlData);
|
||||
if (
|
||||
!embedding ||
|
||||
embedding.length !== EMBEDDING_LENGTH ||
|
||||
norm(embedding) === 0
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const matches: { title: string; score: number }[] = [];
|
||||
for (const category of categories) {
|
||||
if (isVideo && !category.showVideo) continue;
|
||||
const score = cosineSimilarity(embedding, category.textEmbedding);
|
||||
if (score >= category.minScore) {
|
||||
matches.push({ title: category.title, score });
|
||||
}
|
||||
}
|
||||
return matches
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.map((match) => match.title);
|
||||
};
|
||||
+28
-11
@@ -16,7 +16,7 @@ import {
|
||||
streamTagFinal,
|
||||
} from "../crypto/index.js";
|
||||
import { TruncatedStreamError } from "../errors.js";
|
||||
import { safeExtension, sanitizeFileName, withExtension } from "../filename.js";
|
||||
import { defaultFileName, safeExtension, withExtension } from "../filename.js";
|
||||
import { withRetry } from "../retry.js";
|
||||
import type { ApiClient } from "../api/client.js";
|
||||
import type { EnteFile } from "../model/types.js";
|
||||
@@ -365,10 +365,14 @@ const openPart = async (
|
||||
// name starts with `image` as the image and the one whose name starts with
|
||||
// `video` as the video, and refuses a ZIP holding a second of either. It is
|
||||
// written unpacked: each part is named `destination` with the extension
|
||||
// replaced by its own entry's, and the two must differ ignoring case. When the
|
||||
// file records a hash, `<imageHash>:<videoHash>` must match it, each over that
|
||||
// part's own bytes. Only then are the image, then the video, renamed into
|
||||
// place; on any failure neither is stored.
|
||||
// replaced by its own entry's, and the two must differ ignoring case. When
|
||||
// `defaultName` is set, `destination` is the name `defaultFileName` made from
|
||||
// the title rather than a path the caller gave, and each part is named as
|
||||
// `defaultFileName` names it with its own extension, so that the name is cut
|
||||
// short to fit with that extension. When the file records a hash,
|
||||
// `<imageHash>:<videoHash>` must match it, each over that part's own bytes.
|
||||
// Only then are the image, then the video, renamed into place; on any failure
|
||||
// neither is stored.
|
||||
//
|
||||
// The ZIP is chosen by its uploader and may expand enormously, so each part is
|
||||
// written as it decompresses and never held, and the ZIP is refused once the
|
||||
@@ -388,6 +392,7 @@ const decryptLivePhoto = async (
|
||||
key: Uint8Array,
|
||||
onProgress: ProgressCallback | undefined,
|
||||
file: EnteFile,
|
||||
defaultName: boolean,
|
||||
): Promise<DownloadResult> => {
|
||||
const sliceSize = 4096;
|
||||
const dir = dirname(destination);
|
||||
@@ -486,8 +491,12 @@ const decryptLivePhoto = async (
|
||||
`live photo's image and video have the same extension, ${video.ext}`,
|
||||
);
|
||||
}
|
||||
const path = withExtension(destination, image.ext);
|
||||
const videoPath = withExtension(destination, video.ext);
|
||||
const partPath = (ext: string): string =>
|
||||
defaultName
|
||||
? defaultFileName(file.metadata.title, file.id, { ext })
|
||||
: withExtension(destination, ext);
|
||||
const path = partPath(image.ext);
|
||||
const videoPath = partPath(video.ext);
|
||||
for (const part of parts) {
|
||||
await part.handle.sync();
|
||||
await part.handle.close();
|
||||
@@ -531,6 +540,9 @@ const decryptLivePhoto = async (
|
||||
// and then removes its own temp file, and only the attempt that reaches
|
||||
// TAG_FINAL renames one into place, so a download that needed three tries still
|
||||
// performs exactly one rename over the destination.
|
||||
//
|
||||
// `defaultName` says `destination` is the name `defaultFileName` made, not a
|
||||
// path the caller gave (see `decryptLivePhoto`).
|
||||
const fetchAndDecrypt = async (
|
||||
api: ApiClient,
|
||||
openStream: () => Promise<ReadableStream<Uint8Array>>,
|
||||
@@ -539,6 +551,7 @@ const fetchAndDecrypt = async (
|
||||
destination: string,
|
||||
onProgress?: ProgressCallback,
|
||||
original?: EnteFile,
|
||||
defaultName = false,
|
||||
): Promise<DownloadResult> =>
|
||||
withRetry(async () => {
|
||||
const stream = await openStream();
|
||||
@@ -551,6 +564,7 @@ const fetchAndDecrypt = async (
|
||||
key,
|
||||
onProgress,
|
||||
original,
|
||||
defaultName,
|
||||
);
|
||||
}
|
||||
const bytesWritten = await decryptToTemp(
|
||||
@@ -582,10 +596,11 @@ export const downloadFile = async (
|
||||
outPath?: string,
|
||||
onProgress?: ProgressCallback,
|
||||
): Promise<DownloadResult> => {
|
||||
// `outPath` is the caller's and is used as is; the title is the server's
|
||||
// and is sanitized so it can only name a file in the current directory.
|
||||
// `outPath` is the caller's and is used as is. Without it, the file is
|
||||
// written in the current directory under the name `defaultFileName` makes
|
||||
// from the title, which is the server's.
|
||||
const resolvedPath =
|
||||
outPath ?? sanitizeFileName(file.metadata.title, `file-${file.id}`);
|
||||
outPath ?? defaultFileName(file.metadata.title, file.id);
|
||||
const header = fromBase64(file.file.decryptionHeader);
|
||||
return fetchAndDecrypt(
|
||||
api,
|
||||
@@ -595,6 +610,7 @@ export const downloadFile = async (
|
||||
resolvedPath,
|
||||
onProgress,
|
||||
file,
|
||||
outPath === undefined,
|
||||
);
|
||||
};
|
||||
|
||||
@@ -604,9 +620,10 @@ export const downloadThumbnail = async (
|
||||
outPath?: string,
|
||||
onProgress?: ProgressCallback,
|
||||
): Promise<DownloadResult> => {
|
||||
// As in `downloadFile`, with `thumb_` before the title.
|
||||
const resolvedPath =
|
||||
outPath ??
|
||||
`thumb_${sanitizeFileName(file.metadata.title, `file-${file.id}`)}`;
|
||||
defaultFileName(file.metadata.title, file.id, { prefix: "thumb_" });
|
||||
const header = fromBase64(file.thumbnail.decryptionHeader);
|
||||
return fetchAndDecrypt(
|
||||
api,
|
||||
|
||||
+56
-4
@@ -5,7 +5,9 @@
|
||||
// used on disk goes through here, so it can only ever name one file inside the
|
||||
// directory the caller chose: never a path, never `..`, never hidden, never a
|
||||
// Windows device name. Such a name can still be too long for a file name, and
|
||||
// `shortenToBytes` cuts one short.
|
||||
// `shortenToBytes` or `shortenKeepingExtension` cuts one short.
|
||||
// `defaultFileName` does both for the name a file is saved under when the
|
||||
// caller gives no path.
|
||||
//
|
||||
// A path the user typed (`--out`, `outPath`) is not passed through here: the
|
||||
// caller is trusted, the server is not.
|
||||
@@ -30,11 +32,11 @@ export const sanitizeFileName = (name: string, fallback: string): string => {
|
||||
return RESERVED_DEVICE_NAME.test(cleaned) ? `_${cleaned}` : cleaned;
|
||||
};
|
||||
|
||||
// The extension of `title` (".jpg"), or ".bin" when it has none or it holds
|
||||
// anything but letters and digits.
|
||||
// The extension of `title` (".jpg"), or ".bin" when it has none, it holds
|
||||
// anything but letters and digits, or it is more than 16 of them long.
|
||||
export const safeExtension = (title: string): string => {
|
||||
const ext = extname(title);
|
||||
return /^\.[A-Za-z0-9]+$/.test(ext) ? ext : ".bin";
|
||||
return /^\.[A-Za-z0-9]{1,16}$/.test(ext) ? ext : ".bin";
|
||||
};
|
||||
|
||||
// `name` with its extension, if it has one, replaced by `ext` (".mov").
|
||||
@@ -57,3 +59,53 @@ export const shortenToBytes = (name: string, maxBytes: number): string => {
|
||||
}
|
||||
return name.slice(0, end);
|
||||
};
|
||||
|
||||
// `name` cut short to fit in `maxBytes` bytes with `insert` put right before
|
||||
// its extension (`IMG_0001 (12).jpg`), each cut made as `shortenToBytes`
|
||||
// makes it. The part before the extension is cut first, so the extension is
|
||||
// kept whenever it fits, but that part always keeps its first character:
|
||||
// when the extension leaves no room for `insert` and that character, the
|
||||
// extension is cut short too, and the part before it keeps the bytes that cut
|
||||
// leaves over. So a name that does not start with a dot never comes back
|
||||
// starting with one, which would hide it. `insert` is never cut. With no
|
||||
// `insert`, a name that fits comes back unchanged.
|
||||
export const shortenKeepingExtension = (
|
||||
name: string,
|
||||
maxBytes: number,
|
||||
insert = "",
|
||||
): string => {
|
||||
const fullExt = extname(name);
|
||||
const stem = name.slice(0, name.length - fullExt.length);
|
||||
const first = Array.from(stem)[0] ?? "";
|
||||
const ext = shortenToBytes(
|
||||
fullExt,
|
||||
maxBytes - Buffer.byteLength(insert + first),
|
||||
);
|
||||
const room = maxBytes - Buffer.byteLength(insert + ext);
|
||||
return shortenToBytes(stem, room) + insert + ext;
|
||||
};
|
||||
|
||||
// The name a file is saved under in the working directory when the caller
|
||||
// gives no path (`quak get` without `--out`, `downloadFile` without
|
||||
// `outPath`): `prefix` (`thumb_` for a thumbnail), then the file's title with
|
||||
// unsafe characters replaced, or `file-<id>` when the title is empty, cut
|
||||
// short so that the whole fits in a file name, keeping the title's extension
|
||||
// whenever it fits. `ext` (".mov"), when given, takes the place of the
|
||||
// title's extension before the name is cut short; a live photo's image and
|
||||
// video are each named with their own. The cut can leave a device name
|
||||
// (`CONSOLE.<long extension>` becomes `CON.<long extension>`); that name gets
|
||||
// a leading `_`, as `sanitizeFileName` gives one, and is cut a byte shorter to
|
||||
// leave room for it.
|
||||
export const defaultFileName = (
|
||||
title: string,
|
||||
fileID: number,
|
||||
opts: { prefix?: string; ext?: string } = {},
|
||||
): string => {
|
||||
const { prefix = "", ext } = opts;
|
||||
const safe = sanitizeFileName(title, `file-${fileID}`);
|
||||
const name = ext === undefined ? safe : withExtension(safe, ext);
|
||||
const room = MAX_NAME_BYTES - Buffer.byteLength(prefix);
|
||||
const cut = prefix + shortenKeepingExtension(name, room);
|
||||
if (!RESERVED_DEVICE_NAME.test(cut)) return cut;
|
||||
return `_${prefix}${shortenKeepingExtension(name, room - 1)}`;
|
||||
};
|
||||
|
||||
@@ -61,7 +61,7 @@ export interface MLIndex {
|
||||
// Pull the CLIP embedding out of a payload, or undefined when it is absent or
|
||||
// misshapen. Kept strict so a bad payload is skipped rather than corrupting the
|
||||
// packed index.
|
||||
const clipEmbedding = (payload: MLData): number[] | undefined => {
|
||||
export const clipEmbedding = (payload: MLData): number[] | undefined => {
|
||||
const clip = payload.clip;
|
||||
if (typeof clip !== "object" || clip === null) return undefined;
|
||||
const embedding = (clip as { embedding?: unknown }).embedding;
|
||||
|
||||
+35
-6
@@ -1,11 +1,16 @@
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import * as jpeg from "jpeg-js";
|
||||
import { clipKeywords } from "./clip-keywords.js";
|
||||
import type { Client } from "./client.js";
|
||||
import { describeError } from "./errors.js";
|
||||
import { readExifTags } from "./exif.js";
|
||||
import type { Library, Photo } from "./library/index.js";
|
||||
import { sanitizeFileName } from "./filename.js";
|
||||
import {
|
||||
MAX_NAME_BYTES,
|
||||
sanitizeFileName,
|
||||
shortenToBytes,
|
||||
} from "./filename.js";
|
||||
import {
|
||||
fetchMLDataBatch,
|
||||
MLDATA_BATCH_SIZE,
|
||||
@@ -13,7 +18,10 @@ import {
|
||||
} from "./mldata-fetch.js";
|
||||
import type { EnteFile } from "./model/types.js";
|
||||
|
||||
export type ProgressCallback = (message: string) => void;
|
||||
// Reports one line of the dump's progress. `level` is "error" when the line
|
||||
// reports ML data that could not be read, and is left out for every other
|
||||
// line.
|
||||
export type ProgressCallback = (message: string, level?: "error") => void;
|
||||
|
||||
export interface MetadataBackupOptions {
|
||||
exif?: boolean;
|
||||
@@ -90,7 +98,8 @@ const extractExif = async (
|
||||
|
||||
// Dump every decrypted metadata layer the account holds into a directory tree
|
||||
// of plain JSON: account, per-collection, and per-file records including the
|
||||
// private and public magic metadata and (by default) the ML data. Collections
|
||||
// private and public magic metadata and the ML data, with the keywords of its
|
||||
// CLIP embedding in place of the embedding (see clip-keywords.ts). Collections
|
||||
// and files are enumerated from the library's cache, which the caller refreshes
|
||||
// first. Returns how many files have no ML data because their ML data request
|
||||
// failed or their ML data did not decrypt, gunzip or parse; they are still
|
||||
@@ -127,7 +136,15 @@ export const runMetadataBackup = async (
|
||||
const col = lib.getCollection(album.collectionID);
|
||||
if (!col) continue;
|
||||
|
||||
const dirName = `${col.id}-${sanitizeFileName(col.name, "unnamed")}`;
|
||||
// The album's ID keeps the folders apart, so the name after it can be
|
||||
// cut short to fit in a file name.
|
||||
const prefix = `${col.id}-`;
|
||||
const dirName =
|
||||
prefix +
|
||||
shortenToBytes(
|
||||
sanitizeFileName(col.name, "unnamed"),
|
||||
MAX_NAME_BYTES - prefix.length,
|
||||
);
|
||||
const colDir = join(outDir, "collections", dirName);
|
||||
mkdirSync(colDir, { recursive: true });
|
||||
|
||||
@@ -183,13 +200,17 @@ export const runMetadataBackup = async (
|
||||
for (const [id, payload] of payloads) mlDataMap.set(id, payload);
|
||||
for (const [id, err] of errors) {
|
||||
const reason = describeError(err);
|
||||
log(`ML data of file ${id} could not be read: ${reason}`);
|
||||
log(
|
||||
`ML data of file ${id} could not be read: ${reason}`,
|
||||
"error",
|
||||
);
|
||||
mlDataErrors.set(id, reason);
|
||||
}
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
log(
|
||||
`ML data request for ${batch.length} file(s) failed: ${reason}`,
|
||||
"error",
|
||||
);
|
||||
for (const id of batch) mlDataErrors.set(id, reason);
|
||||
}
|
||||
@@ -211,8 +232,16 @@ export const runMetadataBackup = async (
|
||||
if (file.pubMagicMetadata)
|
||||
fileMeta.pubMagicMetadata = file.pubMagicMetadata;
|
||||
|
||||
// The ML data without the CLIP embedding, and the keywords that
|
||||
// embedding gives.
|
||||
const ml = mlDataMap.get(file.id);
|
||||
if (ml) fileMeta.mlData = ml;
|
||||
if (ml) {
|
||||
const { clip: _clip, ...mlDataWithoutClip } = ml;
|
||||
fileMeta.mlData = mlDataWithoutClip;
|
||||
const isVideo = file.metadata.fileType === "video";
|
||||
const keywords = clipKeywords(ml, isVideo);
|
||||
if (keywords) fileMeta.keywords = keywords;
|
||||
}
|
||||
const mlError = mlDataErrors.get(file.id);
|
||||
if (mlError) fileMeta.mlDataError = mlError;
|
||||
|
||||
|
||||
+8
-1
@@ -24,6 +24,9 @@ export interface RetryOptions {
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
// Injected so the jitter is reproducible under test.
|
||||
random?: () => number;
|
||||
// Called before each wait with the error that failed attempt number
|
||||
// `attempt` and how long the wait is, for a caller that logs retries.
|
||||
onRetry?: (err: unknown, attempt: number, delayMs: number) => void;
|
||||
}
|
||||
|
||||
export type ResolvedRetryOptions = Required<RetryOptions>;
|
||||
@@ -35,6 +38,7 @@ export const DEFAULT_RETRY_OPTIONS: ResolvedRetryOptions = {
|
||||
sleep: (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
random: Math.random,
|
||||
onRetry: () => {},
|
||||
};
|
||||
|
||||
// For a run nobody is watching, such as `quak backup` from cron. A request that
|
||||
@@ -55,6 +59,7 @@ export const resolveRetryOptions = (
|
||||
maxDelayMs: opts?.maxDelayMs ?? DEFAULT_RETRY_OPTIONS.maxDelayMs,
|
||||
sleep: opts?.sleep ?? DEFAULT_RETRY_OPTIONS.sleep,
|
||||
random: opts?.random ?? DEFAULT_RETRY_OPTIONS.random,
|
||||
onRetry: opts?.onRetry ?? DEFAULT_RETRY_OPTIONS.onRetry,
|
||||
});
|
||||
|
||||
// Transport failures: the request did not complete, for reasons below HTTP.
|
||||
@@ -234,7 +239,9 @@ export const withRetry = async <T>(
|
||||
// and its status intact.
|
||||
throw err;
|
||||
}
|
||||
await policy.sleep(backoffMs(attempt, policy));
|
||||
const delayMs = backoffMs(attempt, policy);
|
||||
policy.onRetry(err, attempt, delayMs);
|
||||
await policy.sleep(delayMs);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
+6
-1
@@ -49,7 +49,10 @@ export interface ThumbnailFixResult {
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export type ProgressCallback = (message: string) => void;
|
||||
// Reports one line of progress. `level` is "error" when the line reports a
|
||||
// file that could not be checked or fixed, and is left out for every other
|
||||
// line.
|
||||
export type ProgressCallback = (message: string, level?: "error") => void;
|
||||
|
||||
// Enumerate every file the library knows about, newest album first, each file
|
||||
// once, and report those whose server-side thumbnail is missing. "Missing" is
|
||||
@@ -111,6 +114,7 @@ export const listMissingThumbnails = async (
|
||||
} else {
|
||||
log(
|
||||
`[${album.name}] Could not check ${photo.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`,
|
||||
"error",
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -359,6 +363,7 @@ export const fixMissingThumbnails = async (
|
||||
} catch (err) {
|
||||
log(
|
||||
`[${collectionName}] FAILED ${title}: ${err instanceof Error ? err.message : err}`,
|
||||
"error",
|
||||
);
|
||||
results.push({
|
||||
fileID,
|
||||
|
||||
@@ -1222,3 +1222,65 @@ describe("ApiClient non-idempotent requests", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// `onRequest` hears of each request the client sends, as its response arrives
|
||||
// or as it fails. The CLI logs these with `-vv`. A retried request is reported
|
||||
// once for each attempt.
|
||||
describe("ApiClient onRequest", () => {
|
||||
it("reports each request with its status, or with the error it failed with", async () => {
|
||||
const refused = new TypeError("fetch failed");
|
||||
const { fetch } = scriptedFetch(
|
||||
refused,
|
||||
textResponse("unavailable", 503),
|
||||
jsonResponse({ collections: [] }),
|
||||
jsonResponse({}),
|
||||
streamResponse(new Uint8Array([1, 2, 3])),
|
||||
);
|
||||
const requests: unknown[] = [];
|
||||
const client = new ApiClient({
|
||||
fetch,
|
||||
retry: noWait,
|
||||
onRequest: (request) => requests.push(request),
|
||||
});
|
||||
|
||||
await client.getJSON("/collections/v2", { sinceTime: 0 });
|
||||
await client.postJSON("/users/logout", {});
|
||||
await readAll(await client.getFileStream(7));
|
||||
|
||||
const collections = "https://api.ente.io/collections/v2?sinceTime=0";
|
||||
expect(requests).toEqual([
|
||||
{ method: "GET", url: collections, error: refused },
|
||||
{ method: "GET", url: collections, status: 503 },
|
||||
{ method: "GET", url: collections, status: 200 },
|
||||
{
|
||||
method: "POST",
|
||||
url: "https://api.ente.io/users/logout",
|
||||
status: 200,
|
||||
},
|
||||
{
|
||||
method: "GET",
|
||||
url: "https://files.ente.io/?fileID=7",
|
||||
status: 200,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports a presigned upload without its query, which holds the upload's signature", async () => {
|
||||
const { fetch } = scriptedFetch(new Response(null, { status: 200 }));
|
||||
const requests: unknown[] = [];
|
||||
const client = new ApiClient({
|
||||
fetch,
|
||||
retry: noWait,
|
||||
onRequest: (request) => requests.push(request),
|
||||
});
|
||||
|
||||
await client.putFile(
|
||||
"https://s3.example/obj?X-Amz-Signature=abc",
|
||||
new Uint8Array([1, 2]),
|
||||
);
|
||||
|
||||
expect(requests).toEqual([
|
||||
{ method: "PUT", url: "https://s3.example/obj", status: 200 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -54,6 +54,7 @@ import {
|
||||
saveSession,
|
||||
type CliContext,
|
||||
} from "../../src/cli-commands.js";
|
||||
import { stderrLog } from "../../src/cli-log.js";
|
||||
import { run } from "../../src/cli-run.js";
|
||||
import { loadSession } from "../../src/cli-session.js";
|
||||
import { Client } from "../../src/client.js";
|
||||
@@ -85,10 +86,17 @@ const DAY = "2024/2024-03/2024-03-01";
|
||||
const TOAST = new TextEncoder().encode("a few seconds of the wedding toast");
|
||||
|
||||
// Ente's ML data for beach.jpg: a face it found, and the photo's CLIP
|
||||
// embedding.
|
||||
// embedding, here that of a photo of a sunset (see clip-keywords.test.ts).
|
||||
const ML_DATA = {
|
||||
face: { faces: [{ faceID: "100_1", score: 0.98 }] },
|
||||
clip: { embedding: [0.5, 0.6, 0.7] },
|
||||
clip: {
|
||||
embedding: JSON.parse(
|
||||
readFileSync(
|
||||
new URL("../clip-keywords-sunset.json", import.meta.url),
|
||||
"utf-8",
|
||||
),
|
||||
).embedding,
|
||||
},
|
||||
};
|
||||
|
||||
// Each file's ID, its owner, its original as uploaded, and the metadata its
|
||||
@@ -303,9 +311,10 @@ const backup = async (
|
||||
const prompted = async (): Promise<never> => {
|
||||
throw new Error("quak backup prompted");
|
||||
};
|
||||
const log = stderrLog(stderr, "info");
|
||||
const ctx: CliContext = {
|
||||
stdout,
|
||||
stderr,
|
||||
log,
|
||||
sessionDir,
|
||||
cacheDir: `${dir}-cache`,
|
||||
loadSession: (path) =>
|
||||
@@ -319,7 +328,7 @@ const backup = async (
|
||||
promptSecret: prompted,
|
||||
};
|
||||
const code = await new Promise<number>((resolve) => {
|
||||
void run(backupCommand(ctx, dir, opts), stdout, stderr, resolve);
|
||||
void run(backupCommand(ctx, dir, opts), stdout, stderr, log, resolve);
|
||||
});
|
||||
return { code, stderr: printed, stdout: output };
|
||||
};
|
||||
@@ -484,7 +493,10 @@ describe("the first backup of the account", () => {
|
||||
},
|
||||
updationTime: TAKEN,
|
||||
});
|
||||
expect(beach.mlData).toEqual(ML_DATA);
|
||||
// The JSON holds the ML data without its CLIP embedding, and the
|
||||
// keywords the embedding gives in its place.
|
||||
expect(beach.mlData).toEqual({ face: ML_DATA.face });
|
||||
expect(beach.keywords).toEqual(["Sunset"]);
|
||||
expect(beach.imageMetadata.exif.Make.description).toBe("Canon");
|
||||
expect(beach.imageMetadata.exif.Model.description).toBe("EOS R5");
|
||||
|
||||
@@ -495,6 +507,7 @@ describe("the first backup of the account", () => {
|
||||
metadata: { title: "toast.mp4", fileType: "video" },
|
||||
});
|
||||
expect(toast).not.toHaveProperty("mlData");
|
||||
expect(toast).not.toHaveProperty("keywords");
|
||||
expect(toast).not.toHaveProperty("imageMetadata");
|
||||
expect(toast).not.toHaveProperty("imageMetadataError");
|
||||
|
||||
@@ -1104,6 +1117,9 @@ describe("a backup whose lock is lost", () => {
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toBe(
|
||||
"Starting backup...\n" +
|
||||
"Fetched the files of 1 of 3 collections\n" +
|
||||
"Fetched the files of 2 of 3 collections\n" +
|
||||
"Fetched the files of 3 of 3 collections\n" +
|
||||
"Refreshing library...\n" +
|
||||
"Fetching original beach.jpg (100)...\n" +
|
||||
"Fetching original IMG_0002.HEIC (300)...\n" +
|
||||
|
||||
@@ -42,6 +42,7 @@ import {
|
||||
saveSession,
|
||||
type CliContext,
|
||||
} from "../../src/cli-commands.js";
|
||||
import { stderrLog } from "../../src/cli-log.js";
|
||||
import { loadSession } from "../../src/cli-session.js";
|
||||
import { Client, type ClientSnapshot } from "../../src/client.js";
|
||||
import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js";
|
||||
@@ -304,7 +305,7 @@ const context = (ente: FakeEnte): CliContext => {
|
||||
saveSession(join(root, "session"), ente.snapshot());
|
||||
return {
|
||||
stdout,
|
||||
stderr,
|
||||
log: stderrLog(stderr, "info"),
|
||||
sessionDir: join(root, "session"),
|
||||
cacheDir: join(root, "cache"),
|
||||
loadSession: (path) =>
|
||||
@@ -370,6 +371,24 @@ describe("quak backup with records it cannot decrypt", () => {
|
||||
expect(stderr.text).toContain(" Failed: 2\n");
|
||||
});
|
||||
|
||||
// The album is an error line. The last line is fatal and counts the files
|
||||
// and the album, so `-q`, which prints fatal lines only, still says why
|
||||
// the backup exits 1.
|
||||
it("logs the album at error, and ends with one fatal line counting the files and the album", async () => {
|
||||
const ctx = {
|
||||
...context(brokenAccount()),
|
||||
log: stderrLog(stderr, "error"),
|
||||
};
|
||||
expect(await backupCommand(ctx, backupDir(), {})).toBe(1);
|
||||
|
||||
expect(stderr.text).toMatch(
|
||||
/^FAILED album 2: could not be decrypted: /,
|
||||
);
|
||||
expect(stderr.text.split("\n").at(-2)).toBe(
|
||||
`quak: 2 file(s) and 1 album(s) failed; see ${join(backupDir(), "failures.json")} and the FAILED lines`,
|
||||
);
|
||||
});
|
||||
|
||||
it("names each album it cannot decrypt in the --json result", async () => {
|
||||
expect(await backup(brokenAccount(), { json: true })).toBe(1);
|
||||
|
||||
|
||||
+111
-18
@@ -52,6 +52,7 @@ import { tmpdir } from "node:os";
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||
|
||||
import { runBackup, type BackupLibrary } from "../../src/backup.js";
|
||||
import { DISCOVER_CATEGORIES } from "../../src/clip-keyword-vectors.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
import type { ContentSource } from "../../src/library/content.js";
|
||||
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
||||
@@ -946,9 +947,19 @@ describe("the backup lock", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// Also serves ML data: file 100 has `ML_PAYLOAD`, the other two have none.
|
||||
// While `mlError` is set, every ML data request fails with it.
|
||||
const ML_PAYLOAD = { face: { faces: [] }, clip: { embedding: [0.5, 0.25] } };
|
||||
// Also serves ML data: file 100 has `ML_PAYLOAD`, whose CLIP embedding is that
|
||||
// of a photo of a sunset (see clip-keywords.test.ts), and the other two have
|
||||
// none. While `mlError` is set, every ML data request fails with it.
|
||||
const SUNSET_EMBEDDING: number[] = JSON.parse(
|
||||
readFileSync(
|
||||
new URL("../clip-keywords-sunset.json", import.meta.url),
|
||||
"utf-8",
|
||||
),
|
||||
).embedding;
|
||||
const ML_PAYLOAD = {
|
||||
face: { faces: [] },
|
||||
clip: { embedding: SUNSET_EMBEDDING },
|
||||
};
|
||||
|
||||
class MLClient extends MockClient {
|
||||
mlError?: string;
|
||||
@@ -968,16 +979,99 @@ const fileJSON = (outDir: string, fileID: number): Record<string, unknown> =>
|
||||
JSON.parse(readFileSync(saved(outDir, `${fileID}.json`), "utf-8"));
|
||||
|
||||
describe("ML data in each file's JSON", () => {
|
||||
it("writes a file's ML data, and no ML field for a file that has none", async () => {
|
||||
// The JSON holds the ML data without its CLIP embedding, and in its place
|
||||
// the keywords the embedding gives.
|
||||
it("writes a file's ML data and keywords, and no ML field for a file that has none", async () => {
|
||||
const lib = await openLibrary(stubSource(), new MLClient());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(result.failed).toBe(0);
|
||||
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
|
||||
expect(fileJSON(outDir, 100).mlData).toEqual({ face: { faces: [] } });
|
||||
expect(fileJSON(outDir, 100).keywords).toEqual(["Sunset"]);
|
||||
expect(fileJSON(outDir, 101)).not.toHaveProperty("mlData");
|
||||
expect(fileJSON(outDir, 101)).not.toHaveProperty("mlDataError");
|
||||
expect(fileJSON(outDir, 101)).not.toHaveProperty("keywords");
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
// A file's ML data can hold faces and no CLIP embedding. Its JSON then has
|
||||
// the ML data and no keywords.
|
||||
it("writes no keywords for ML data without a CLIP embedding", async () => {
|
||||
const faces = { face: { faces: [{ faceID: "101_1", score: 0.9 }] } };
|
||||
|
||||
// Gives file 101 ML data holding only faces.
|
||||
class FacesClient extends MockClient {
|
||||
async fetchMLData(args: { fileIDs: number[] }): Promise<{
|
||||
payloads: Map<number, MLData>;
|
||||
errors: Map<number, unknown>;
|
||||
}> {
|
||||
const payloads = new Map<number, MLData>();
|
||||
if (args.fileIDs.includes(101)) payloads.set(101, faces);
|
||||
return { payloads, errors: new Map() };
|
||||
}
|
||||
}
|
||||
const lib = await openLibrary(stubSource(), new FacesClient());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(result.failed).toBe(0);
|
||||
expect(fileJSON(outDir, 101).mlData).toEqual(faces);
|
||||
expect(fileJSON(outDir, 101)).not.toHaveProperty("keywords");
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
// Ente's photos app keeps videos out of some Discover categories, Identity
|
||||
// among them. Here a photo and a video share one CLIP embedding, Identity's
|
||||
// own text embedding, so the photo gets the keyword Identity and the video
|
||||
// does not.
|
||||
it("keeps a video out of a category that leaves out videos", async () => {
|
||||
const identity = DISCOVER_CATEGORIES.find(
|
||||
(c) => c.title === "Identity",
|
||||
)!;
|
||||
expect(identity.showVideo).toBe(false);
|
||||
const toast = file(102, 1, "toast.mp4");
|
||||
const video = {
|
||||
...toast,
|
||||
metadata: { ...toast.metadata, fileType: "video" as const },
|
||||
};
|
||||
|
||||
// Serves beach.jpg and the video in the Vacation album, and gives
|
||||
// each ML data holding only that embedding.
|
||||
class VideoClient extends MockClient {
|
||||
override async filesSince(args: {
|
||||
collectionID: number;
|
||||
}): Promise<FilesPage> {
|
||||
const files =
|
||||
args.collectionID === 1
|
||||
? [file(100, 1, "beach.jpg"), video]
|
||||
: [];
|
||||
return { files, deleted: [], cursor: 1 };
|
||||
}
|
||||
async fetchMLData(args: { fileIDs: number[] }): Promise<{
|
||||
payloads: Map<number, MLData>;
|
||||
errors: Map<number, unknown>;
|
||||
}> {
|
||||
const payloads = new Map<number, MLData>();
|
||||
for (const id of args.fileIDs) {
|
||||
payloads.set(id, {
|
||||
clip: { embedding: identity.textEmbedding },
|
||||
});
|
||||
}
|
||||
return { payloads, errors: new Map() };
|
||||
}
|
||||
}
|
||||
const lib = await openLibrary(stubSource(), new VideoClient());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(result.failed).toBe(0);
|
||||
expect(fileJSON(outDir, 100).keywords).toContain("Identity");
|
||||
expect(fileJSON(outDir, 102)).toHaveProperty("keywords");
|
||||
expect(fileJSON(outDir, 102).keywords).not.toContain("Identity");
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
@@ -993,7 +1087,7 @@ describe("ML data in each file's JSON", () => {
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
// File 100's ML data was cached by the first run and is kept.
|
||||
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
|
||||
expect(fileJSON(outDir, 100).mlData).toEqual({ face: { faces: [] } });
|
||||
expect(fileJSON(outDir, 100)).not.toHaveProperty("mlDataError");
|
||||
for (const fileID of [101, 200]) {
|
||||
expect(fileJSON(outDir, fileID).mlDataError).toBe(
|
||||
@@ -1026,7 +1120,7 @@ describe("ML data in each file's JSON", () => {
|
||||
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.skipped).toBe(3);
|
||||
expect(fileJSON(outDir, 100).mlData).toEqual(ML_PAYLOAD);
|
||||
expect(fileJSON(outDir, 100).mlData).toEqual({ face: { faces: [] } });
|
||||
for (const fileID of [100, 101, 200]) {
|
||||
expect(fileJSON(outDir, fileID)).not.toHaveProperty("mlDataError");
|
||||
}
|
||||
@@ -1795,12 +1889,11 @@ describe("backup album folders", () => {
|
||||
expect(readLedger(outDir).files["5"]).toMatchObject({ attempts: 1 });
|
||||
});
|
||||
|
||||
// The extensions of these titles leave no room for the ID, so each is cut
|
||||
// short and the name before the ID keeps only the bytes that cut leaves
|
||||
// over. File 1's extension is 303 bytes and is cut to 251, which leaves
|
||||
// none. File 2's is 271 bytes and is cut to 250, since "€" takes three
|
||||
// bytes, which leaves one, for the "a". Neither extension is only letters
|
||||
// and digits, so both originals are saved with `.bin`, a name that fits.
|
||||
// The extensions of these titles leave no room for the ID and the "a"
|
||||
// before them, so each is cut short to 250 bytes, which leaves one, for
|
||||
// the "a". File 1's extension is 303 bytes and file 2's 271. Neither
|
||||
// extension is only letters and digits, so both originals are saved with
|
||||
// `.bin`, a name that fits.
|
||||
it("cuts the extension of a title short when it leaves no room for the ID", async () => {
|
||||
const outDir = join(root, "backup");
|
||||
const lib = libraryOf([
|
||||
@@ -1815,7 +1908,7 @@ describe("backup album folders", () => {
|
||||
|
||||
const result = await runBackup(lib, { downloadDirectory: outDir });
|
||||
|
||||
const link = ` (1).${"L".repeat(250)}`;
|
||||
const link = `a (1).${"L".repeat(249)}`;
|
||||
const link2 = `a (2).${"€".repeat(83)}`;
|
||||
expect(Buffer.byteLength(link)).toBe(255);
|
||||
expect(Buffer.byteLength(link2)).toBe(255);
|
||||
@@ -1830,9 +1923,9 @@ describe("backup album folders", () => {
|
||||
|
||||
// Files 1 and 3 share a title, so each gets its ID before the
|
||||
// extension. File 2's title is too long, and its extension leaves no
|
||||
// room for the ID. Were the ID put at the end of file 2's title cut
|
||||
// short, its link would have file 1's name, and the link made second
|
||||
// would replace the first.
|
||||
// room for the ID and the "x" before it. Were the ID put at the end of
|
||||
// file 2's title cut short, its link would have file 1's name, and the
|
||||
// link made second would replace the first.
|
||||
it("keeps names unique when a title's extension leaves no room for the ID", async () => {
|
||||
const outDir = join(root, "backup");
|
||||
const shared = `x.${"Q".repeat(245)} (2)`;
|
||||
@@ -1852,8 +1945,8 @@ describe("backup album folders", () => {
|
||||
expect(result.failed).toBe(0);
|
||||
expect(tree(outDir)).toEqual([
|
||||
"Trip/",
|
||||
`Trip/ (2).${"Q".repeat(250)} -> ${linkTo("2.bin")}`,
|
||||
`Trip/x (1).${"Q".repeat(245)} (2) -> ${linkTo("1.bin")}`,
|
||||
`Trip/x (2).${"Q".repeat(249)} -> ${linkTo("2.bin")}`,
|
||||
`Trip/x (3).${"Q".repeat(245)} (2) -> ${linkTo("3.bin")}`,
|
||||
"Trip.json",
|
||||
]);
|
||||
|
||||
+11
-3
@@ -1,7 +1,8 @@
|
||||
/**
|
||||
* Tests for the retry options `bin/quak.ts` loads the saved session with
|
||||
* (`quak backup` gets the unattended ones, every other command the default),
|
||||
* and for `quak files` taking its collection ID as an optional argument.
|
||||
* each with the callbacks that log its requests and retries, and for
|
||||
* `quak files` taking its collection ID as an optional argument.
|
||||
*
|
||||
* Each test runs `bin/quak.ts`, as the smoke test does, with its session loader
|
||||
* replaced by one that records the options it is given and reports no session,
|
||||
@@ -65,14 +66,21 @@ describe("bin/quak.ts session loading", () => {
|
||||
attempts,
|
||||
baseDelayMs,
|
||||
maxDelayMs,
|
||||
onRetry: expect.any(Function),
|
||||
}),
|
||||
onRequest: expect.any(Function),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("loads the session for another command with the default options", async () => {
|
||||
it("loads the session for another command with the default retry options", async () => {
|
||||
await quak("collections");
|
||||
expect(loaded).toEqual([undefined]);
|
||||
expect(loaded).toEqual([
|
||||
{
|
||||
retry: { onRetry: expect.any(Function) },
|
||||
onRequest: expect.any(Function),
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+246
-15
@@ -21,7 +21,7 @@ import {
|
||||
utimesSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { basename, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { PassThrough } from "node:stream";
|
||||
import * as jpegJs from "jpeg-js";
|
||||
@@ -51,6 +51,7 @@ import {
|
||||
listMissingThumbnailsCommand,
|
||||
fixMissingThumbnailsCommand,
|
||||
} from "../../src/cli-commands.js";
|
||||
import { stderrLog } from "../../src/cli-log.js";
|
||||
import { run } from "../../src/cli-run.js";
|
||||
import { loadSession } from "../../src/cli-session.js";
|
||||
import {
|
||||
@@ -193,6 +194,9 @@ const FETCHED_BOTH_ALBUMS =
|
||||
"Fetched the files of 1 of 2 collections\n" +
|
||||
"Fetched the files of 2 of 2 collections\n";
|
||||
|
||||
// The same for an account with one album.
|
||||
const FETCHED_ONE_ALBUM = "Fetched the files of 1 of 1 collections\n";
|
||||
|
||||
// Collects everything written to it.
|
||||
class Output {
|
||||
text = "";
|
||||
@@ -205,9 +209,11 @@ let root: string;
|
||||
let stdout: Output;
|
||||
let stderr: Output;
|
||||
|
||||
// The commands log at the default level, info, as `quak` without `-v` or `-q`
|
||||
// does; the levels have their own tests, in `log.test.ts`.
|
||||
const context = (client: Client | null = fakeClient()): CliContext => ({
|
||||
stdout,
|
||||
stderr,
|
||||
log: stderrLog(stderr, "info"),
|
||||
sessionDir: join(root, "session"),
|
||||
cacheDir: join(root, "cache"),
|
||||
loadSession: () => client,
|
||||
@@ -563,7 +569,7 @@ describe("files", () => {
|
||||
[101, 1],
|
||||
[200, 2],
|
||||
]);
|
||||
expect(stderr.text).toBe("");
|
||||
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -597,17 +603,109 @@ describe("get and get-thumb", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// A file system refuses a file name over 255 bytes. The account here is one
|
||||
// album, whose name is 300 bytes, holding one file, 700, with the title given.
|
||||
describe("names from an album name or a title over 255 bytes", () => {
|
||||
const longNames = (title: string): Client =>
|
||||
({
|
||||
...fakeClient(),
|
||||
collectionsSince: async () => ({
|
||||
collections: [collection(7, "A".repeat(300))],
|
||||
deleted: [],
|
||||
cursor: 1,
|
||||
}),
|
||||
filesSince: async () => ({
|
||||
files: [file(700, 7, title)],
|
||||
deleted: [],
|
||||
cursor: 1,
|
||||
}),
|
||||
}) as unknown as Client;
|
||||
const longTitle = `${"t".repeat(296)}.jpg`;
|
||||
|
||||
// Run `command` in a new directory, where `get` and `get-thumb` without
|
||||
// `--out` write, and return its exit code and the directory.
|
||||
const inNewDirectory = async (
|
||||
command: () => Promise<number>,
|
||||
): Promise<{ code: number; dir: string }> => {
|
||||
const dir = mkdtempSync(join(root, "cwd-"));
|
||||
const previous = process.cwd();
|
||||
process.chdir(dir);
|
||||
try {
|
||||
return { code: await command(), dir };
|
||||
} finally {
|
||||
process.chdir(previous);
|
||||
}
|
||||
};
|
||||
|
||||
it("get cuts the title short to 255 bytes, keeping its extension, and exits 0", async () => {
|
||||
const { code, dir } = await inNewDirectory(() =>
|
||||
getCommand(context(longNames(longTitle)), "700", {}),
|
||||
);
|
||||
|
||||
const name = `${"t".repeat(251)}.jpg`;
|
||||
expect(Buffer.byteLength(name)).toBe(255);
|
||||
expect(code).toBe(0);
|
||||
expect(readdirSync(dir)).toEqual([name]);
|
||||
expect(stderr.text).toBe(FETCHED_ONE_ALBUM + `7 bytes -> ${name}\n`);
|
||||
});
|
||||
|
||||
it("get-thumb cuts the title short so that thumb_ and it fit in 255 bytes, and exits 0", async () => {
|
||||
const { code, dir } = await inNewDirectory(() =>
|
||||
getThumbCommand(context(longNames(longTitle)), "700", {}),
|
||||
);
|
||||
|
||||
const name = `thumb_${"t".repeat(245)}.jpg`;
|
||||
expect(Buffer.byteLength(name)).toBe(255);
|
||||
expect(code).toBe(0);
|
||||
expect(readdirSync(dir)).toEqual([name]);
|
||||
expect(stderr.text).toBe(FETCHED_ONE_ALBUM + `3 bytes -> ${name}\n`);
|
||||
});
|
||||
|
||||
it("backup-metadata cuts the album name short so that its folder's name fits in 255 bytes, and exits 0", async () => {
|
||||
const dir = join(root, "dump");
|
||||
|
||||
const code = await backupMetadataCommand(
|
||||
context(longNames("beach.jpg")),
|
||||
dir,
|
||||
{},
|
||||
);
|
||||
|
||||
const folder = `7-${"A".repeat(253)}`;
|
||||
expect(Buffer.byteLength(folder)).toBe(255);
|
||||
expect(code).toBe(0);
|
||||
expect(readdirSync(join(dir, "collections"))).toEqual([folder]);
|
||||
expect(readdirSync(join(dir, "collections", folder)).sort()).toEqual([
|
||||
"700.json",
|
||||
"_collection.json",
|
||||
]);
|
||||
});
|
||||
|
||||
// The title's extension is 300 letters, too long for the save path's
|
||||
// name, so the original is saved with `.bin`.
|
||||
it("backup saves a title with a 300-letter extension as .bin, and exits 0", async () => {
|
||||
const dir = join(root, "backup");
|
||||
const title = `b.${"e".repeat(300)}`;
|
||||
|
||||
const code = await backupCommand(context(longNames(title)), dir, {});
|
||||
|
||||
const path = savePath(dir, file(700, 7, title));
|
||||
expect(code).toBe(0);
|
||||
expect(basename(path)).toMatch(/^\d{4}-\d{2}-\d{2}\.700\.bin$/);
|
||||
expect(readFileSync(path)).toEqual(Buffer.alloc(7, 700 & 0xff));
|
||||
});
|
||||
});
|
||||
|
||||
// A live photo, which Ente stores as one ZIP, is written as its image and its
|
||||
// video, which a photo viewer can open. The account here is Vacation holding
|
||||
// one live photo, 300, downloaded through the real download layer
|
||||
// (test/live-photo.ts).
|
||||
describe("a live photo", () => {
|
||||
// What `get` prints on stderr while its refresh fetches Vacation's files.
|
||||
const FETCHED_ONE_ALBUM = "Fetched the files of 1 of 1 collections\n";
|
||||
|
||||
const livePhotoClient = async (image = IMAGE): Promise<Client> => {
|
||||
const livePhotoClient = async (
|
||||
image = IMAGE,
|
||||
title = "IMG_0300.HEIC",
|
||||
): Promise<Client> => {
|
||||
const { file: live, body } = await asLivePhoto(
|
||||
file(300, 1, "IMG_0300.HEIC"),
|
||||
file(300, 1, title),
|
||||
livePhotoZip({ "image.heic": image, "video.mov": VIDEO }),
|
||||
livePhotoHash(image, VIDEO),
|
||||
);
|
||||
@@ -655,6 +753,44 @@ describe("a live photo", () => {
|
||||
);
|
||||
});
|
||||
|
||||
// Run `get` without `--out` in a new directory for a live photo titled
|
||||
// `title`, and return its exit code and the names it wrote.
|
||||
const getInNewDirectory = async (
|
||||
title: string,
|
||||
): Promise<{ code: number; names: string[] }> => {
|
||||
const ctx = context(await livePhotoClient(IMAGE, title));
|
||||
const dir = mkdtempSync(join(root, "cwd-"));
|
||||
const previous = process.cwd();
|
||||
process.chdir(dir);
|
||||
try {
|
||||
const code = await getCommand(ctx, "300", {});
|
||||
return { code, names: readdirSync(dir).sort() };
|
||||
} finally {
|
||||
process.chdir(previous);
|
||||
}
|
||||
};
|
||||
const imageName = `${"t".repeat(250)}.heic`;
|
||||
const videoName = `${"t".repeat(251)}.mov`;
|
||||
|
||||
// The title fits, but ".heic" is one byte longer than ".jpg", so the
|
||||
// image's name is cut short once its extension is in it.
|
||||
it("get cuts its image's name short for a 255-byte title ending in .jpg, and exits 0", async () => {
|
||||
const title = `${"t".repeat(251)}.jpg`;
|
||||
|
||||
const result = await getInNewDirectory(title);
|
||||
|
||||
expect(Buffer.byteLength(title)).toBe(255);
|
||||
expect(Buffer.byteLength(imageName)).toBe(255);
|
||||
expect(Buffer.byteLength(videoName)).toBe(255);
|
||||
expect(result).toEqual({ code: 0, names: [imageName, videoName] });
|
||||
});
|
||||
|
||||
it("get cuts the names of its image and video short for a 300-byte title with no extension, and exits 0", async () => {
|
||||
const result = await getInNewDirectory("t".repeat(300));
|
||||
|
||||
expect(result).toEqual({ code: 0, names: [imageName, videoName] });
|
||||
});
|
||||
|
||||
it("get --out writes the image there and the video beside it", async () => {
|
||||
const out = join(root, "photo.jpg");
|
||||
const video = join(root, "photo.mov");
|
||||
@@ -777,6 +913,40 @@ describe("backup", () => {
|
||||
);
|
||||
});
|
||||
|
||||
// A failed file is an error line both where the backup fails it and in
|
||||
// the list at the end. The last line is fatal, so `-q`, which prints
|
||||
// fatal lines only, still says why the backup exits 1.
|
||||
it("logs a failed file at error, and ends with one fatal line counting the failures", async () => {
|
||||
const dir = join(root, "backup");
|
||||
const last = `quak: 1 file(s) and 0 album(s) failed; see ${join(dir, "failures.json")} and the FAILED lines\n`;
|
||||
const ctx = context(fakeClient({ failID: 101 }));
|
||||
expect(
|
||||
await backupCommand(
|
||||
{ ...ctx, log: stderrLog(stderr, "error") },
|
||||
dir,
|
||||
{},
|
||||
),
|
||||
).toBe(1);
|
||||
expect(stderr.text).toBe(
|
||||
"FAILED original sunset.jpg: HTTP 500 from server\n" +
|
||||
"\nFailed files:\n" +
|
||||
" [Vacation] sunset.jpg (id 101): HTTP 500 from server\n" +
|
||||
last,
|
||||
);
|
||||
|
||||
stderr.text = "";
|
||||
expect(
|
||||
await backupCommand(
|
||||
{ ...ctx, log: stderrLog(stderr, "fatal") },
|
||||
dir,
|
||||
{ json: true },
|
||||
),
|
||||
).toBe(1);
|
||||
expect(stderr.text).toBe(last);
|
||||
});
|
||||
|
||||
// `--json` shapes stdout only: the log lines stay on stderr, and the JSON
|
||||
// takes the place of the summary.
|
||||
it("prints the result as JSON with --json, still exiting 1 on a failure", async () => {
|
||||
const ctx = context(fakeClient({ failID: 101 }));
|
||||
const code = await backupCommand(ctx, join(root, "backup"), {
|
||||
@@ -791,7 +961,31 @@ describe("backup", () => {
|
||||
failed: 1,
|
||||
});
|
||||
expect(result.errors[0].fileID).toBe(101);
|
||||
expect(stderr.text).toBe("Starting backup...\n");
|
||||
expect(stderr.text).toContain(
|
||||
"Fetching original sunset.jpg (101)...\n",
|
||||
);
|
||||
expect(stderr.text).not.toContain("Backup complete");
|
||||
});
|
||||
|
||||
// On an empty cache the library's first refresh fetches the files of
|
||||
// every album before the backup can start, which on a large account takes
|
||||
// a while; each album it has fetched the files of is counted on stderr,
|
||||
// with `--json` too.
|
||||
it("counts on stderr each album whose files its first refresh fetched, with --json too", async () => {
|
||||
const start =
|
||||
"Starting backup...\n" +
|
||||
FETCHED_BOTH_ALBUMS +
|
||||
"Refreshing library...\n";
|
||||
for (const json of [false, true]) {
|
||||
stderr.text = "";
|
||||
const dir = mkdtempSync(join(root, "backup-"));
|
||||
const ctx = {
|
||||
...context(),
|
||||
cacheDir: mkdtempSync(join(root, "cache-")),
|
||||
};
|
||||
expect(await backupCommand(ctx, dir, { json })).toBe(0);
|
||||
expect(stderr.text.slice(0, start.length)).toBe(start);
|
||||
}
|
||||
});
|
||||
|
||||
it("--verify downloads again an original that does not match its hash, prints the counts, and exits 0", async () => {
|
||||
@@ -881,6 +1075,7 @@ describe("backup", () => {
|
||||
backupCommand(ctx, dir, {}),
|
||||
new PassThrough(),
|
||||
runStderr,
|
||||
stderrLog(runStderr, "info"),
|
||||
resolve,
|
||||
);
|
||||
});
|
||||
@@ -1121,7 +1316,9 @@ describe("helper list-missing-thumbnails", () => {
|
||||
expect(stderr.text).toContain("No missing thumbnails found.\n");
|
||||
});
|
||||
|
||||
it("prints a JSON array with --json and no progress", async () => {
|
||||
// `--json` shapes stdout only: the progress stays on stderr, and the JSON
|
||||
// takes the place of the count and the list.
|
||||
it("prints a JSON array with --json, and its progress on stderr", async () => {
|
||||
const ctx = context(fakeClient({ emptyThumbID: 200 }));
|
||||
expect(await listMissingThumbnailsCommand(ctx, { json: true })).toBe(0);
|
||||
expect(JSON.parse(stdout.text)).toEqual([
|
||||
@@ -1132,7 +1329,37 @@ describe("helper list-missing-thumbnails", () => {
|
||||
reason: "empty thumbnail (0 bytes)",
|
||||
},
|
||||
]);
|
||||
expect(stderr.text).toBe("");
|
||||
expect(stderr.text).toContain(FETCHED_BOTH_ALBUMS);
|
||||
expect(stderr.text).toContain("[Work] Checking thumbnails...\n");
|
||||
expect(stderr.text).not.toContain("with missing thumbnails");
|
||||
});
|
||||
});
|
||||
|
||||
describe("helper fix-missing-thumbnails", () => {
|
||||
// File 999 is in no album, so fixing it fails. The list of failed files
|
||||
// is at error, and the last line is fatal, so `-q`, which prints fatal
|
||||
// lines only, still says why the command exits 1.
|
||||
it("lists a failed file at error, and ends with one fatal line counting the failures", async () => {
|
||||
const last = `quak: 1 file(s) failed; see the "Failed files:" list, or the JSON with --json\n`;
|
||||
const ctx = { ...context(), log: stderrLog(stderr, "error") };
|
||||
expect(await fixMissingThumbnailsCommand(ctx, { file: ["999"] })).toBe(
|
||||
1,
|
||||
);
|
||||
expect(stderr.text).toBe(
|
||||
"\nFailed files:\n" +
|
||||
" 999\tunknown\tfile not found in any collection\n" +
|
||||
last,
|
||||
);
|
||||
|
||||
stderr.text = "";
|
||||
const quiet = { ...context(), log: stderrLog(stderr, "fatal") };
|
||||
expect(
|
||||
await fixMissingThumbnailsCommand(quiet, {
|
||||
file: ["999"],
|
||||
json: true,
|
||||
}),
|
||||
).toBe(1);
|
||||
expect(stderr.text).toBe(last);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1491,9 +1718,9 @@ describe("an account with many albums", () => {
|
||||
expect(stderr.text).toBe(progress);
|
||||
});
|
||||
|
||||
// With `--json` the refresh's progress is not printed, as `backup` and
|
||||
// the thumbnail helpers print none of theirs.
|
||||
it("files --json prints no progress", async () => {
|
||||
// `--json` shapes stdout only, so the refresh's progress is still counted
|
||||
// on stderr.
|
||||
it("files --json counts on stderr each album whose files its refresh fetched", async () => {
|
||||
expect(await filesCommand(accountContext(), "7", { json: true })).toBe(
|
||||
0,
|
||||
);
|
||||
@@ -1507,6 +1734,10 @@ describe("an account with many albums", () => {
|
||||
collectionID: 7,
|
||||
},
|
||||
]);
|
||||
expect(stderr.text).toBe("");
|
||||
let progress = "";
|
||||
for (let n = 1; n <= ALBUMS; n++) {
|
||||
progress += `Fetched the files of ${n} of ${ALBUMS} collections\n`;
|
||||
}
|
||||
expect(stderr.text).toBe(progress);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,357 @@
|
||||
/**
|
||||
* Tests for the CLI's log levels, set with `-v`/`--verbose` and `-q`/`--quiet`
|
||||
* (`src/cli-log.ts`, issue #212).
|
||||
*
|
||||
* Every line quak prints on stderr is a log line at one of six levels, from
|
||||
* the quietest: fatal, error, warning, info, debug, trace. A run prints the
|
||||
* lines at its own level and at every quieter one. The level is info unless
|
||||
* asked otherwise: `-v` raises it to debug, `-vv` to trace, and more repeats
|
||||
* stay at trace. `-q` lowers it to fatal, so a run prints only the line a
|
||||
* command prints as it fails with a non-zero exit code. stdout holds the
|
||||
* command's output at every level.
|
||||
*
|
||||
* Most tests run `bin/quak.ts`, as the smoke test does, against an account of
|
||||
* one album holding one photo: the saved session is in a temporary directory,
|
||||
* and `fetch` is a fake of the Ente server that serves the album and the photo
|
||||
* encrypted as Ente's clients encrypt them. Each run has a cache directory of
|
||||
* its own, so its refresh fetches the album's files, as on a first run.
|
||||
*/
|
||||
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import sodium from "libsodium-wrappers-sumo";
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "vitest";
|
||||
|
||||
import { ApiClient } from "../../src/api/client.js";
|
||||
import { saveSession } from "../../src/cli-commands.js";
|
||||
import { logRefresh, logRequests, type Log } from "../../src/cli-log.js";
|
||||
import { init, toBase64 } from "../../src/crypto/index.js";
|
||||
import { encryptSecretbox, encryptStreamBlob } from "../ente-server.js";
|
||||
|
||||
// `bin/quak.ts` reads the saved session from the data directory env-paths
|
||||
// gives it, which here is a temporary directory made before the first run.
|
||||
const paths = vi.hoisted(() => ({ data: "" }));
|
||||
vi.mock("env-paths", () => ({ default: () => paths }));
|
||||
|
||||
const USER_ID = 42;
|
||||
const root = mkdtempSync(join(tmpdir(), "quak-log-test-"));
|
||||
const argv = process.argv;
|
||||
|
||||
class ExitCalled extends Error {}
|
||||
|
||||
// Run `quak <args>` to its exit, with a cache of its own, and return its exit
|
||||
// code and what it wrote on stdout and on stderr, which are kept off the
|
||||
// terminal. `process.exit` is stubbed to throw, so nothing runs after it, as
|
||||
// nothing does after a real exit.
|
||||
const quak = async (
|
||||
...args: string[]
|
||||
): Promise<{ code: number; stdout: string; stderr: string }> => {
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let code = NaN;
|
||||
vi.spyOn(process.stdout, "write").mockImplementation((text) => {
|
||||
stdout += String(text);
|
||||
return true;
|
||||
});
|
||||
vi.spyOn(process.stderr, "write").mockImplementation((text) => {
|
||||
stderr += String(text);
|
||||
return true;
|
||||
});
|
||||
vi.spyOn(process, "exit").mockImplementation((exitCode) => {
|
||||
code = Number(exitCode ?? 0);
|
||||
throw new ExitCalled();
|
||||
});
|
||||
const cache = mkdtempSync(join(root, "cache-"));
|
||||
process.argv = ["node", "quak", "--cache-dir", cache, ...args];
|
||||
vi.resetModules();
|
||||
await expect(import("../../bin/quak.js")).rejects.toBeInstanceOf(
|
||||
ExitCalled,
|
||||
);
|
||||
return { code, stdout, stderr };
|
||||
};
|
||||
|
||||
// Album 1, "Trips", of user 42, holding photo 100, "beach.jpg". The album's
|
||||
// key is encrypted with the account's master key, the photo's key with the
|
||||
// album's, and the photo's metadata with its own key. Only the session and
|
||||
// these two records are needed: the tests never log in.
|
||||
beforeAll(async () => {
|
||||
await init();
|
||||
await sodium.ready;
|
||||
const masterKey = sodium.crypto_secretbox_keygen();
|
||||
const keyPair = sodium.crypto_box_keypair();
|
||||
paths.data = join(root, "session");
|
||||
saveSession(paths.data, {
|
||||
email: "log@example.com",
|
||||
userID: USER_ID,
|
||||
token: "token",
|
||||
masterKey: toBase64(masterKey),
|
||||
secretKey: toBase64(keyPair.privateKey),
|
||||
publicKey: toBase64(keyPair.publicKey),
|
||||
});
|
||||
|
||||
const albumKey = sodium.crypto_secretbox_keygen();
|
||||
const albumKeyBox = encryptSecretbox(albumKey, masterKey);
|
||||
const name = encryptSecretbox(new TextEncoder().encode("Trips"), albumKey);
|
||||
const album = {
|
||||
id: 1,
|
||||
owner: { id: USER_ID },
|
||||
encryptedKey: toBase64(albumKeyBox.ciphertext),
|
||||
keyDecryptionNonce: toBase64(albumKeyBox.nonce),
|
||||
encryptedName: toBase64(name.ciphertext),
|
||||
nameDecryptionNonce: toBase64(name.nonce),
|
||||
type: "album",
|
||||
updationTime: 1,
|
||||
};
|
||||
const photoKey = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
const photoKeyBox = encryptSecretbox(photoKey, albumKey);
|
||||
const metadata = encryptStreamBlob(
|
||||
new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
title: "beach.jpg",
|
||||
fileType: 0,
|
||||
creationTime: 1000,
|
||||
modificationTime: 1000,
|
||||
}),
|
||||
),
|
||||
photoKey,
|
||||
);
|
||||
const photo = {
|
||||
id: 100,
|
||||
collectionID: 1,
|
||||
ownerID: USER_ID,
|
||||
encryptedKey: toBase64(photoKeyBox.ciphertext),
|
||||
keyDecryptionNonce: toBase64(photoKeyBox.nonce),
|
||||
metadata: {
|
||||
encryptedData: toBase64(metadata.ciphertext),
|
||||
decryptionHeader: toBase64(metadata.header),
|
||||
},
|
||||
// These tests fetch no original and no thumbnail.
|
||||
file: { decryptionHeader: "" },
|
||||
thumbnail: { decryptionHeader: "" },
|
||||
updationTime: 1,
|
||||
};
|
||||
|
||||
const json = (body: unknown): Response =>
|
||||
new Response(JSON.stringify(body), {
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
vi.stubGlobal("fetch", async (input: string) => {
|
||||
const { pathname } = new URL(input);
|
||||
if (pathname === "/collections/v2") {
|
||||
return json({ collections: [album] });
|
||||
}
|
||||
if (pathname === "/collections/v2/diff") {
|
||||
return json({ diff: [photo], hasMore: false });
|
||||
}
|
||||
return new Response("not found", { status: 404 });
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.argv = argv;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
vi.unstubAllGlobals();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// What `quak files` writes on stdout at every level.
|
||||
const FILES = "100\timage\tbeach.jpg\n";
|
||||
|
||||
// What it writes on stderr at each level. Info is the refresh's progress, one
|
||||
// line for the one album whose files it fetched. Debug adds the start and end
|
||||
// of the refresh, and trace each request with the status of its answer.
|
||||
const INFO = "Fetched the files of 1 of 1 collections\n";
|
||||
const DEBUG = "Refresh started\n" + INFO + "Refresh done\n";
|
||||
const TRACE =
|
||||
"Refresh started\n" +
|
||||
"GET https://api.ente.io/collections/v2?sinceTime=0: HTTP 200\n" +
|
||||
"GET https://api.ente.io/collections/v2/diff?collectionID=1&sinceTime=0: HTTP 200\n" +
|
||||
INFO +
|
||||
"Refresh done\n";
|
||||
|
||||
describe("the log level", () => {
|
||||
it("is info by default", async () => {
|
||||
expect(await quak("files")).toEqual({
|
||||
code: 0,
|
||||
stdout: FILES,
|
||||
stderr: INFO,
|
||||
});
|
||||
});
|
||||
|
||||
it("is debug with -v", async () => {
|
||||
expect(await quak("-v", "files")).toEqual({
|
||||
code: 0,
|
||||
stdout: FILES,
|
||||
stderr: DEBUG,
|
||||
});
|
||||
});
|
||||
|
||||
it("is trace with -vv", async () => {
|
||||
expect(await quak("-vv", "files")).toEqual({
|
||||
code: 0,
|
||||
stdout: FILES,
|
||||
stderr: TRACE,
|
||||
});
|
||||
});
|
||||
|
||||
it("stays at trace with -vvv", async () => {
|
||||
expect((await quak("-vvv", "files")).stderr).toBe(TRACE);
|
||||
});
|
||||
|
||||
it("counts --verbose --verbose as -vv", async () => {
|
||||
expect((await quak("--verbose", "--verbose", "files")).stderr).toBe(
|
||||
TRACE,
|
||||
);
|
||||
});
|
||||
|
||||
// Both are options of the program, taken before or after the command's
|
||||
// name, as `--cache-dir` is.
|
||||
it("is set by an option after the command's name too", async () => {
|
||||
expect((await quak("files", "-v")).stderr).toBe(DEBUG);
|
||||
expect((await quak("files", "--json", "-vv")).stderr).toBe(TRACE);
|
||||
expect((await quak("files", "-q")).stderr).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("-q", () => {
|
||||
it("prints nothing on stderr when the command works", async () => {
|
||||
expect(await quak("-q", "files")).toEqual({
|
||||
code: 0,
|
||||
stdout: FILES,
|
||||
stderr: "",
|
||||
});
|
||||
});
|
||||
|
||||
// The refresh's progress is left out; the line saying why the command
|
||||
// failed is the fatal one, and it is printed.
|
||||
it("prints only the line a failing command exits with", async () => {
|
||||
expect(await quak("-q", "files", "9")).toEqual({
|
||||
code: 1,
|
||||
stdout: "",
|
||||
stderr: "Collection 9 not found\n",
|
||||
});
|
||||
});
|
||||
|
||||
it("is refused with -v, in one line, before the command runs", async () => {
|
||||
for (const args of [
|
||||
["-q", "-v", "files"],
|
||||
["files", "--verbose", "--quiet"],
|
||||
]) {
|
||||
const { code, stdout, stderr } = await quak(...args);
|
||||
expect(code).toBe(1);
|
||||
expect(stdout).toBe("");
|
||||
expect(stderr).toBe(
|
||||
"error: option '-q, --quiet' cannot be used with option '-v, --verbose'\n",
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// `--help` lists both options, and each command's help lists them too, since
|
||||
// each command takes them.
|
||||
describe("--help", () => {
|
||||
it("lists -v and -q for the program and for each command", async () => {
|
||||
for (const args of [["--help"], ["files", "--help"]]) {
|
||||
const { stdout } = await quak(...args);
|
||||
expect(stdout).toContain("-v, --verbose");
|
||||
expect(stdout).toContain("-q, --quiet");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// The library writes no lines itself. Its layers report through callbacks,
|
||||
// and the CLI logs what they report.
|
||||
describe("what the library reports", () => {
|
||||
const collect = (): { log: Log; lines: string[] } => {
|
||||
const lines: string[] = [];
|
||||
return { log: (level, line) => lines.push(`${level}: ${line}`), lines };
|
||||
};
|
||||
|
||||
// A request answered 503 and retried: each attempt is a request, logged
|
||||
// at trace with its status, and the retry is logged at debug with how
|
||||
// long it waits and why.
|
||||
it("logs each request at trace and each retry at debug", async () => {
|
||||
const { log, lines } = collect();
|
||||
const answers = [
|
||||
new Response("unavailable", { status: 503 }),
|
||||
new Response("{}", {
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
];
|
||||
const client = new ApiClient({
|
||||
...logRequests(log, {
|
||||
sleep: async () => {},
|
||||
random: () => 0.5,
|
||||
}),
|
||||
fetch: async () => answers.shift()!,
|
||||
});
|
||||
|
||||
await client.getJSON("/collections/v2", { sinceTime: 0 });
|
||||
|
||||
const url = "https://api.ente.io/collections/v2?sinceTime=0";
|
||||
expect(lines).toEqual([
|
||||
`trace: GET ${url}: HTTP 503`,
|
||||
"debug: Attempt 1 failed, retrying in 250 ms: HTTP 503",
|
||||
`trace: GET ${url}: HTTP 200`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("logs a request that failed with its error", async () => {
|
||||
const { log, lines } = collect();
|
||||
const client = new ApiClient({
|
||||
...logRequests(log, { attempts: 1 }),
|
||||
fetch: async () => {
|
||||
throw new TypeError("fetch failed", {
|
||||
cause: new Error("connect ECONNREFUSED 1.2.3.4:443"),
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
await expect(client.getJSON("/collections/v2")).rejects.toThrow();
|
||||
|
||||
expect(lines).toEqual([
|
||||
"trace: GET https://api.ente.io/collections/v2: fetch failed: connect ECONNREFUSED 1.2.3.4:443",
|
||||
]);
|
||||
});
|
||||
|
||||
// Each operation is named in plain words, as the README names it.
|
||||
it("logs a refresh's progress at info, and its start and end at debug", () => {
|
||||
const { log, lines } = collect();
|
||||
const onProgress = logRefresh(log);
|
||||
|
||||
onProgress({ operation: "refresh", status: "started" });
|
||||
onProgress({
|
||||
operation: "refresh",
|
||||
status: "progress",
|
||||
fetched: 1,
|
||||
total: 2,
|
||||
});
|
||||
onProgress({
|
||||
operation: "refresh",
|
||||
status: "failed",
|
||||
error: "HTTP 503",
|
||||
});
|
||||
onProgress({ operation: "fetchMLData", status: "started" });
|
||||
onProgress({ operation: "fetchMLData", status: "done", fetched: 3 });
|
||||
|
||||
expect(lines).toEqual([
|
||||
"debug: Refresh started",
|
||||
"info: Fetched the files of 1 of 2 collections",
|
||||
"debug: Refresh failed: HTTP 503",
|
||||
"debug: ML data fetch started",
|
||||
"debug: ML data fetch done",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -41,12 +41,14 @@ import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
|
||||
import { init, toBase64, encryptBlob } from "../../src/crypto/index.js";
|
||||
import * as jpegJs from "jpeg-js";
|
||||
import { Client } from "../../src/client.js";
|
||||
import { DISCOVER_CATEGORIES } from "../../src/clip-keyword-vectors.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
import {
|
||||
runMetadataBackup,
|
||||
type MetadataBackupOptions,
|
||||
} from "../../src/metadata-backup.js";
|
||||
import { backupMetadataCommand } from "../../src/cli-commands.js";
|
||||
import { stderrLog, type LogLevel } from "../../src/cli-log.js";
|
||||
import {
|
||||
buildAccount,
|
||||
buildMetaFetch,
|
||||
@@ -55,8 +57,8 @@ import {
|
||||
type MetaMockState,
|
||||
} from "../ente-server.js";
|
||||
|
||||
// One file per ML data request, so the two files of the mock account are
|
||||
// fetched in two requests and one of them can fail on its own.
|
||||
// One file per ML data request, so each file of the mock account is fetched in
|
||||
// a request of its own and one of them can fail on its own.
|
||||
vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../../src/mldata-fetch.js")>()),
|
||||
MLDATA_BATCH_SIZE: 1,
|
||||
@@ -65,6 +67,21 @@ vi.mock("../../src/mldata-fetch.js", async (importOriginal) => ({
|
||||
const TEST_EMAIL = "metabackup@example.com";
|
||||
const TEST_PASSWORD = "metapass";
|
||||
|
||||
// The CLIP embedding of a photo of a sunset (see clip-keywords.test.ts), which
|
||||
// the ML data of beach.jpg holds.
|
||||
const SUNSET_EMBEDDING: number[] = JSON.parse(
|
||||
readFileSync(
|
||||
new URL("../clip-keywords-sunset.json", import.meta.url),
|
||||
"utf-8",
|
||||
),
|
||||
).embedding;
|
||||
|
||||
// Identity, a Discover category of Ente's photos app that leaves out videos.
|
||||
const IDENTITY = DISCOVER_CATEGORIES.find((c) => c.title === "Identity")!;
|
||||
|
||||
// ML data holding faces and no CLIP embedding, which file 302 has.
|
||||
const FACES_ONLY = { face: { faces: [{ faceID: "face-def", score: 0.9 }] } };
|
||||
|
||||
let mock: MetaMockState;
|
||||
let testDir: string;
|
||||
|
||||
@@ -203,6 +220,65 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
|
||||
updationTime: 1710000000000000,
|
||||
};
|
||||
|
||||
// A file in coll 10 with the ML data given. `fileType` is Ente's number: 0
|
||||
// an image, 1 a video.
|
||||
const vacationFile = (
|
||||
id: number,
|
||||
title: string,
|
||||
fileType: number,
|
||||
mlData: object,
|
||||
) => {
|
||||
const fileKey = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
const { ciphertext: encKey, nonce } = encryptSecretbox(fileKey, ck1);
|
||||
const meta = encryptStreamBlob(
|
||||
new TextEncoder().encode(
|
||||
JSON.stringify({
|
||||
title,
|
||||
fileType,
|
||||
creationTime: 1700000000000000,
|
||||
modificationTime: 1700000000000000,
|
||||
}),
|
||||
),
|
||||
fileKey,
|
||||
);
|
||||
const ml = encryptBlob(
|
||||
new Uint8Array(gzipSync(Buffer.from(JSON.stringify(mlData)))),
|
||||
fileKey,
|
||||
);
|
||||
return {
|
||||
raw: {
|
||||
id,
|
||||
collectionID: 10,
|
||||
ownerID: 42,
|
||||
encryptedKey: toBase64(encKey),
|
||||
keyDecryptionNonce: toBase64(nonce),
|
||||
metadata: {
|
||||
encryptedData: toBase64(meta.ciphertext),
|
||||
decryptionHeader: toBase64(meta.header),
|
||||
},
|
||||
file: {
|
||||
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
||||
},
|
||||
thumbnail: {
|
||||
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
||||
},
|
||||
updationTime: 1700000000000000,
|
||||
},
|
||||
mlData: {
|
||||
encryptedData: toBase64(ml.ciphertext),
|
||||
decryptionHeader: toBase64(ml.header),
|
||||
},
|
||||
};
|
||||
};
|
||||
// Files 300 and 301 in coll 10: a photo and a video whose ML data holds
|
||||
// only a CLIP embedding, Identity's own text embedding.
|
||||
const identityML = { clip: { embedding: IDENTITY.textEmbedding } };
|
||||
const identityPhoto = vacationFile(300, "passport.jpg", 0, identityML);
|
||||
const identityVideo = vacationFile(301, "passport.mp4", 1, identityML);
|
||||
// File 302 in coll 10: a photo whose ML data holds faces and no CLIP
|
||||
// embedding.
|
||||
const facesPhoto = vacationFile(302, "party.jpg", 0, FACES_ONLY);
|
||||
|
||||
// Encrypt ML data for file 100 (gzipped JSON, encrypted with file key)
|
||||
const mlPayload = JSON.stringify({
|
||||
face: {
|
||||
@@ -229,7 +305,7 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
|
||||
clip: {
|
||||
version: 1,
|
||||
client: "test",
|
||||
embedding: [0.5, 0.6, 0.7],
|
||||
embedding: SUNSET_EMBEDDING,
|
||||
},
|
||||
});
|
||||
const gzipped = gzipSync(Buffer.from(mlPayload));
|
||||
@@ -269,12 +345,23 @@ const buildMetaMock = async (): Promise<MetaMockState> => {
|
||||
keyAttributes: account.keyAttributes,
|
||||
encryptedToken: account.encryptedToken,
|
||||
collections: [rawColl1, rawColl2],
|
||||
filesByCollection: { 10: [rawFile1], 20: [rawFile2] },
|
||||
filesByCollection: {
|
||||
10: [
|
||||
rawFile1,
|
||||
identityPhoto.raw,
|
||||
identityVideo.raw,
|
||||
facesPhoto.raw,
|
||||
],
|
||||
20: [rawFile2],
|
||||
},
|
||||
encryptedMLData: {
|
||||
100: {
|
||||
encryptedData: toBase64(mlCiphertext),
|
||||
decryptionHeader: toBase64(mlHeader),
|
||||
},
|
||||
300: identityPhoto.mlData,
|
||||
301: identityVideo.mlData,
|
||||
302: facesPhoto.mlData,
|
||||
},
|
||||
fileCiphertexts: { 100: encFileBody1 },
|
||||
};
|
||||
@@ -431,7 +518,9 @@ describe("quak backup-metadata", () => {
|
||||
expect(account.email).toBe(TEST_EMAIL);
|
||||
});
|
||||
|
||||
it("fetches and decrypts ML data by default", async () => {
|
||||
// The CLIP embedding is left out of the ML data, and the keywords it gives
|
||||
// are written in its place.
|
||||
it("fetches and decrypts ML data by default, with keywords for the CLIP embedding", async () => {
|
||||
const outDir = join(testDir, "ml-data");
|
||||
await runBackup(outDir);
|
||||
|
||||
@@ -451,8 +540,41 @@ describe("quak backup-metadata", () => {
|
||||
expect(fileMeta.mlData.face.faces[0].faceID).toBe("face-abc");
|
||||
expect(fileMeta.mlData.face.faces[0].score).toBeCloseTo(0.98);
|
||||
expect(fileMeta.mlData.face.faces[0].detection.box.x).toBeCloseTo(0.1);
|
||||
expect(fileMeta.mlData.clip).toBeDefined();
|
||||
expect(fileMeta.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
|
||||
expect(fileMeta.mlData).not.toHaveProperty("clip");
|
||||
expect(fileMeta.keywords).toEqual(["Sunset"]);
|
||||
});
|
||||
|
||||
// Ente's photos app keeps videos out of some Discover categories, Identity
|
||||
// among them. Files 300 and 301, a photo and a video, share one CLIP
|
||||
// embedding, Identity's own text embedding, so the photo gets the keyword
|
||||
// Identity and the video does not.
|
||||
it("keeps a video out of a category that leaves out videos", async () => {
|
||||
expect(IDENTITY.showVideo).toBe(false);
|
||||
const outDir = join(testDir, "video-keywords");
|
||||
await runBackup(outDir);
|
||||
|
||||
const vacation = join(outDir, "collections", "10-Vacation");
|
||||
const fileJSON = (fileID: number) =>
|
||||
JSON.parse(readFileSync(join(vacation, `${fileID}.json`), "utf-8"));
|
||||
expect(fileJSON(300).keywords).toContain("Identity");
|
||||
expect(fileJSON(301)).toHaveProperty("keywords");
|
||||
expect(fileJSON(301).keywords).not.toContain("Identity");
|
||||
});
|
||||
|
||||
// A file's ML data can hold faces and no CLIP embedding. File 302's JSON
|
||||
// then has the ML data and no keywords.
|
||||
it("writes no keywords for ML data without a CLIP embedding", async () => {
|
||||
const outDir = join(testDir, "no-clip");
|
||||
await runBackup(outDir);
|
||||
|
||||
const fileMeta = JSON.parse(
|
||||
readFileSync(
|
||||
join(outDir, "collections", "10-Vacation", "302.json"),
|
||||
"utf-8",
|
||||
),
|
||||
);
|
||||
expect(fileMeta.mlData).toEqual(FACES_ONLY);
|
||||
expect(fileMeta).not.toHaveProperty("keywords");
|
||||
});
|
||||
|
||||
it("extracts EXIF from downloaded files when --exif is set", async () => {
|
||||
@@ -490,9 +612,12 @@ describe("quak backup-metadata", () => {
|
||||
});
|
||||
|
||||
describe("quak backup-metadata when a file's ML data cannot be read", () => {
|
||||
// Run the CLI command against `fetch`, the mock unless given, and return
|
||||
// its exit code, stderr and output directory.
|
||||
const runCommand = async (fetch = buildMetaFetch(mock)) => {
|
||||
// Run the CLI command against `fetch`, the mock unless given, logging at
|
||||
// `level`, and return its exit code, stderr and output directory.
|
||||
const runCommand = async (
|
||||
fetch = buildMetaFetch(mock),
|
||||
level: LogLevel = "info",
|
||||
) => {
|
||||
const client = await Client.login({
|
||||
email: TEST_EMAIL,
|
||||
password: TEST_PASSWORD,
|
||||
@@ -506,7 +631,10 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => {
|
||||
const code = await backupMetadataCommand(
|
||||
{
|
||||
stdout: { write: () => true },
|
||||
stderr: { write: (text: string) => (stderr += text) },
|
||||
log: stderrLog(
|
||||
{ write: (text: string) => (stderr += text) },
|
||||
level,
|
||||
),
|
||||
sessionDir: testDir,
|
||||
cacheDir: mkdtempSync(join(testDir, "cache-")),
|
||||
loadSession: () => client,
|
||||
@@ -517,13 +645,16 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => {
|
||||
return { code, stderr, outDir };
|
||||
};
|
||||
|
||||
// The failed request is logged at error, so it is the first line printed
|
||||
// at that level.
|
||||
it("writes every file, marks the failed batch's files, and exits 1", async () => {
|
||||
const { code, stderr, outDir } = await runCommand(
|
||||
buildMetaFetch(mock, 200),
|
||||
"error",
|
||||
);
|
||||
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("ML data request for 1 file(s) failed");
|
||||
expect(stderr).toMatch(/^ML data request for 1 file\(s\) failed: /);
|
||||
|
||||
const ok = JSON.parse(
|
||||
readFileSync(
|
||||
@@ -531,7 +662,7 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => {
|
||||
"utf-8",
|
||||
),
|
||||
);
|
||||
expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
|
||||
expect(ok.keywords).toEqual(["Sunset"]);
|
||||
expect(ok.mlDataError).toBeUndefined();
|
||||
|
||||
const failed = JSON.parse(
|
||||
@@ -546,7 +677,9 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => {
|
||||
});
|
||||
|
||||
// The server sends ML data for diagram.png encrypted under a key that is
|
||||
// not the file's, so quak cannot decrypt it.
|
||||
// not the file's, so quak cannot decrypt it. That is logged at error, and
|
||||
// the last line is fatal, so `-q`, which prints fatal lines only, still
|
||||
// says why the command exits 1.
|
||||
it("writes the reason for ML data that does not decrypt, and exits 1", async () => {
|
||||
const otherKey = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
const { header, ciphertext } = encryptBlob(
|
||||
@@ -564,12 +697,14 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => {
|
||||
},
|
||||
},
|
||||
}),
|
||||
"error",
|
||||
);
|
||||
|
||||
expect(code).toBe(1);
|
||||
const reason = "secretstream chunk authentication failed";
|
||||
expect(stderr).toContain(
|
||||
`ML data of file 200 could not be read: ${reason}\n`,
|
||||
expect(stderr).toBe(
|
||||
`ML data of file 200 could not be read: ${reason}\n` +
|
||||
`quak: the ML data of 1 file(s) could not be read; see mlDataError in their JSON in ${outDir}\n`,
|
||||
);
|
||||
const failed = JSON.parse(
|
||||
readFileSync(
|
||||
@@ -585,7 +720,7 @@ describe("quak backup-metadata when a file's ML data cannot be read", () => {
|
||||
"utf-8",
|
||||
),
|
||||
);
|
||||
expect(ok.mlData.clip.embedding).toEqual([0.5, 0.6, 0.7]);
|
||||
expect(ok.keywords).toEqual(["Sunset"]);
|
||||
});
|
||||
|
||||
it("exits 0 when every ML data request succeeds", async () => {
|
||||
|
||||
@@ -6,6 +6,7 @@ import { PassThrough } from "node:stream";
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
import { ApiError } from "../../src/api/client.js";
|
||||
import { stderrLog } from "../../src/cli-log.js";
|
||||
import { run } from "../../src/cli-run.js";
|
||||
|
||||
// A stream whose written text is kept in `text`; writes finish at once, so
|
||||
@@ -23,7 +24,13 @@ const runToExit = async (
|
||||
const stdout = collector();
|
||||
const stderr = collector();
|
||||
const code = await new Promise<number>((resolve) => {
|
||||
void run(command, stdout.stream, stderr.stream, resolve);
|
||||
void run(
|
||||
command,
|
||||
stdout.stream,
|
||||
stderr.stream,
|
||||
stderrLog(stderr.stream, "info"),
|
||||
resolve,
|
||||
);
|
||||
});
|
||||
return { code, stdout: stdout.text(), stderr: stderr.text() };
|
||||
};
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
{
|
||||
"photo": "https://upload.wikimedia.org/wikipedia/commons/d/db/Sunset_off_the_Massachusetts_coast_-_NOAA.jpg",
|
||||
"sha256": "be62f3289c9bf7b75af68cd6978021d54e8a531d5d4cc493f6e9fbf5e1ab74d3",
|
||||
"embedding": [
|
||||
-0.0021123015, -0.04437615, -0.062354308, 0.006249992, 0.022141185,
|
||||
0.011159019, 0.01227392, -0.00030532264, 0.03186437, 0.09196922,
|
||||
0.039111234, -0.037084445, -0.032962628, -0.019820206, -0.013141825,
|
||||
0.0047183544, 0.017137334, 0.046581306, 0.0047622235, 0.04982521,
|
||||
0.023128154, 0.032080866, 0.037709303, 0.019241406, 0.0032100384,
|
||||
-0.014275843, 0.049337693, 0.025953151, 0.014263995, -0.0062328726,
|
||||
-0.040504098, -0.015083303, -0.008123875, -0.006233639, -0.05352683,
|
||||
-0.0048083956, 0.0763486, -0.013155067, 0.025114346, 0.022016674,
|
||||
0.0001588023, -0.010341572, -0.0037518279, -0.019181008, 0.04549337,
|
||||
-0.0010725886, 0.004636866, 0.017177915, -0.0117317885, 0.0026097358,
|
||||
0.030738998, 0.013158572, -0.08856403, -0.0121391015, 0.011477229,
|
||||
-0.0033104012, 0.035524756, -0.018607879, -0.0071739843, -0.031906743,
|
||||
0.0056713386, -0.0044195866, 0.00292471, 0.00061686034, -0.043075223,
|
||||
0.09040427, 0.001522537, -0.011445652, 0.009609886, -0.0053201183,
|
||||
-0.0057452377, 0.00972853, -0.0024397348, -0.021255083, 0.015893811,
|
||||
-0.006020548, -0.09428023, 0.014277583, -0.017975299, -0.011878315,
|
||||
0.011825521, 0.00014203975, -0.0020437292, -0.05305343, -0.024840152,
|
||||
-0.011336872, -0.002796541, -0.00036171873, 0.004530499, 0.044486605,
|
||||
0.021602718, -0.01292857, 0.024530916, 0.019559948, -0.039928272,
|
||||
-0.0681174, 0.049101, 0.02461532, 0.0365531, -0.04070358, -0.011011939,
|
||||
-0.0027188526, -0.020370876, 0.012904856, 0.043157287, -0.051658224,
|
||||
-0.3726012, 0.019549074, -0.0499252, -0.04800559, -0.011493859,
|
||||
0.02893256, 0.060341954, 0.0013929688, -0.051283866, -0.024286803,
|
||||
0.044124287, -0.005995806, -0.062817656, 0.007369614, -0.021470634,
|
||||
-0.028147545, 0.00411659, 0.04234773, 0.03787647, 0.008579682,
|
||||
-0.038568925, -0.027498025, -0.028416378, 0.0022012168, 0.010906191,
|
||||
-0.013745787, 0.018267838, 0.013531097, -0.061559845, 0.004682517,
|
||||
-0.005993303, 0.0028113858, -0.0028286583, 0.0026029332, -0.06693403,
|
||||
-0.0072381413, -0.02419584, 0.0037009937, -0.0011296495, -0.030310456,
|
||||
0.0005999609, 0.009945354, 0.024915686, 0.038672045, 0.023009432,
|
||||
-0.05093051, -0.09101409, -0.06686167, -0.06211481, -0.022600474,
|
||||
0.0024902367, -0.0067171026, -0.016769778, -0.0032803628, 0.008623083,
|
||||
-0.0016110836, 0.014925073, 0.033811048, 0.012829449, -0.052575592,
|
||||
0.03728281, -0.0051201633, 0.021238064, -0.031086605, -0.0043402985,
|
||||
0.25054494, -0.028396713, -0.010725523, -0.023445267, 0.021751683,
|
||||
0.003130971, 0.012270548, -0.014836475, 0.014466885, 0.0021478906,
|
||||
-0.088153616, 0.015356315, 0.039639276, -0.040283255, 0.014923924,
|
||||
-0.029303031, -0.004475201, 0.047299225, -0.06726505, -0.012491676,
|
||||
-0.002080582, 0.03506341, 0.04146718, 0.0074114986, -0.009582633,
|
||||
0.006623749, -0.05454898, -0.0034387247, -0.014780821, 0.03698328,
|
||||
-0.034441486, 0.22032379, -0.021792699, 0.008001907, 0.0341019,
|
||||
0.0025784625, -0.0071689896, -0.014151225, -0.050547987, -0.030372301,
|
||||
-0.03288388, -0.042987447, -0.0033656165, 0.021502906, -0.0030374376,
|
||||
-0.024413, -0.03506603, 0.030891743, 0.0069994032, 0.004887366,
|
||||
0.030707076, 0.0065348223, 0.0008240555, 0.016809713, -0.024151647,
|
||||
-0.0040458064, 0.044567548, 0.03908212, -0.061782435, 0.0026002831,
|
||||
0.02003242, -0.0115451, -0.045135964, -0.0663542, 0.005240975,
|
||||
0.0010777109, -0.009276002, -0.011919081, 0.027654119, -0.013634551,
|
||||
0.0129154315, -0.013195512, 0.040571272, -0.099290505, -0.04656693,
|
||||
-0.20979854, 0.04181004, -0.06769166, -0.04121946, 0.018260444,
|
||||
0.005617506, -0.15694222, 0.032472793, 0.014189598, 0.008345103,
|
||||
-0.041118417, 0.0074527003, -0.011494104, 0.004906474, -0.0027100458,
|
||||
0.0009994594, -0.041061398, -0.045274835, 0.015187793, 0.034275036,
|
||||
-0.02813289, 0.02098019, -0.015043198, 0.03716667, 0.013542642,
|
||||
-0.008104835, 0.014965486, -0.027978558, 0.021163663, -0.006942737,
|
||||
-0.034969814, -0.038961593, 0.016802894, -0.012318175, -0.047821593,
|
||||
-0.01933216, 0.0064432374, 0.0322358, -0.020437784, 0.02659447,
|
||||
-0.03022022, -0.019443465, -0.094178624, -0.017009107, 0.03860671,
|
||||
-0.00023695534, -0.033089854, -0.014677462, 0.012356399, 0.018040499,
|
||||
-0.19360113, 0.011710817, -0.019871691, 0.020124774, 0.024948105,
|
||||
-0.024573864, -0.028349072, -0.041731454, 0.032646254, 0.015098165,
|
||||
0.0018447012, -0.01129152, -0.02481687, 0.036048, 0.20797811,
|
||||
0.04317757, -0.017239755, 0.033431605, -0.011384422, 0.0099174455,
|
||||
-0.19739223, -0.008901062, -0.056426458, 0.014591864, 0.0044474266,
|
||||
0.008209326, -0.012241819, 0.042553645, -0.021552453, 0.008611734,
|
||||
0.048944127, -0.013872049, 0.019179989, 0.008183264, -0.053223096,
|
||||
-0.027327595, -0.0033542586, -0.012353858, -0.010941209, 0.010212546,
|
||||
0.029714134, -0.011385593, -0.0013529479, 0.014505385, 0.01148663,
|
||||
0.012613636, -0.0074430504, -0.014008101, -0.028146038, -0.042778917,
|
||||
0.02143165, 0.016812252, -0.06286215, 0.02085532, -0.035128277,
|
||||
0.0241282, 0.01397908, -0.034992814, 0.013153766, 0.0068236147,
|
||||
0.013594184, -0.06859393, 0.051849406, 0.029675119, -0.0037699796,
|
||||
-0.00013216575, 0.02916647, 0.031859048, -0.015006051, 0.0001439966,
|
||||
0.0026584554, -0.0046948595, 0.039754897, 0.034873154, 0.0012870439,
|
||||
-0.011516082, -0.074089095, 0.019241294, 0.00076195505, 0.0027334464,
|
||||
-0.036206458, -0.0011109262, -0.048838835, -0.00204685, 0.01179991,
|
||||
0.0068136845, 0.009852495, 0.017308988, 0.03159545, 0.058803353,
|
||||
0.038116634, 0.008836143, -0.010460637, -0.010994099, -0.0012926942,
|
||||
-0.048370842, 0.037696566, -0.08093239, -0.0041242186, -0.005866373,
|
||||
0.02048607, -0.12549639, 0.011705739, -0.0021425558, 0.018078592,
|
||||
-0.038228087, 0.004245971, -0.11560507, 0.01381203, -0.02734893,
|
||||
0.029595908, 0.101082794, -0.042842254, -0.08397301, -0.1546237,
|
||||
-0.037624564, 0.0069606556, -0.008020479, 0.0043268185, 0.016504597,
|
||||
0.044554908, -0.05240209, 0.020519335, -0.050994243, -0.019861093,
|
||||
0.022066997, -0.0188911, 0.012919696, -0.022872692, -0.012961374,
|
||||
-0.020071112, 0.0074553066, 0.06402014, 0.04065211, 0.014139608,
|
||||
0.06221546, -0.02574805, 0.010114545, 0.020619567, 0.032264773,
|
||||
-0.009148828, 0.022849955, -0.016593235, -0.032456633, -0.006737303,
|
||||
-0.0037493357, -0.065912865, -0.0024181998, -0.0037048983, -0.019304488,
|
||||
0.009085037, 0.057529982, -0.0378338, 0.01792042, 0.01885012,
|
||||
-0.0043600523, 0.017382368, -0.027792772, 0.006749118, -0.0053297766,
|
||||
-0.011628587, 0.015736576, 0.024801968, 0.011543913, -0.01159873,
|
||||
-0.000759976, 0.042183932, 0.05764386, -0.06422416, 0.010577374,
|
||||
-0.02261249, 0.02282793, -0.05141075, -0.11287078, -0.020670144,
|
||||
-0.020953743, 0.030885734, -0.012597732, -0.121472664, 0.028447151,
|
||||
-0.0415664, -0.010061708, 0.06441338, 0.0271207, -0.0016693748,
|
||||
-0.0045694243, 0.020342957, 0.030528478, 0.007129517, -0.0029600926,
|
||||
-0.007061397, 0.06036486, -0.004736018, -0.0090786135, -0.07601116,
|
||||
-0.039556272, -0.051269546, -0.019473685, 0.029858151, 0.028464556,
|
||||
0.012424476, 0.013742254, 0.008742927, 0.006065751, -0.009892663,
|
||||
0.07699999, -0.003443197, 0.0027863425, 0.029197995, 0.0211013,
|
||||
0.01098379, 0.016684106, -0.028917195, 0.008122037, 0.0265241,
|
||||
-0.04461223
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// `clipKeywords`: the keywords quak writes for a file in place of its CLIP
|
||||
// embedding.
|
||||
//
|
||||
// Ente's ML data for a file holds a CLIP embedding, 512 numbers its image
|
||||
// model gave the photo. Ente's photos app puts a file in a category of its
|
||||
// Discover section when the cosine similarity of that embedding and the text
|
||||
// embedding of the category's query is at least the category's minimum score,
|
||||
// and keeps videos out of some categories. `clipKeywords` applies that rule
|
||||
// and returns the titles of the categories a file falls in, highest score
|
||||
// first; `quak backup` and `quak backup-metadata` write them as `keywords`.
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { DISCOVER_CATEGORIES } from "../src/clip-keyword-vectors.js";
|
||||
import { clipKeywords } from "../src/clip-keywords.js";
|
||||
|
||||
// 512 numbers: the ones given, then zeros.
|
||||
const vector = (...start: number[]): number[] => [
|
||||
...start,
|
||||
...new Array<number>(512 - start.length).fill(0),
|
||||
];
|
||||
|
||||
const withEmbedding = (embedding: unknown) => ({
|
||||
face: { faces: [] },
|
||||
clip: { embedding },
|
||||
});
|
||||
|
||||
// Two made-up categories, each with a text embedding along one axis. The
|
||||
// embedding [3, 4] has a cosine similarity of 3/5 with "Across" and 4/5 with
|
||||
// "Up". Videos are kept out of "Across".
|
||||
const ACROSS = {
|
||||
title: "Across",
|
||||
query: "across",
|
||||
minScore: 0.6,
|
||||
showVideo: false,
|
||||
textEmbedding: vector(1),
|
||||
};
|
||||
const UP = {
|
||||
title: "Up",
|
||||
query: "up",
|
||||
minScore: 0.5,
|
||||
showVideo: true,
|
||||
textEmbedding: vector(0, 1),
|
||||
};
|
||||
|
||||
describe("clipKeywords", () => {
|
||||
it("gives each category whose minimum score the similarity reaches, highest score first", () => {
|
||||
expect(
|
||||
clipKeywords(withEmbedding(vector(3, 4)), false, [ACROSS, UP]),
|
||||
).toEqual(["Up", "Across"]);
|
||||
});
|
||||
|
||||
it("leaves out a category when the similarity is just below its minimum score", () => {
|
||||
const stricter = { ...ACROSS, minScore: 0.6 + 1e-9 };
|
||||
|
||||
expect(
|
||||
clipKeywords(withEmbedding(vector(3, 4)), false, [stricter, UP]),
|
||||
).toEqual(["Up"]);
|
||||
});
|
||||
|
||||
it("keeps a video out of a category that leaves out videos", () => {
|
||||
expect(
|
||||
clipKeywords(withEmbedding(vector(3, 4)), true, [ACROSS, UP]),
|
||||
).toEqual(["Up"]);
|
||||
});
|
||||
|
||||
it("gives an empty list for an embedding that falls in no category", () => {
|
||||
expect(
|
||||
clipKeywords(withEmbedding(vector(0, 0, 1)), false, [ACROSS, UP]),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["no CLIP embedding", { face: { faces: [] } }],
|
||||
["a CLIP part that is not an object", { clip: null }],
|
||||
["a CLIP part without an embedding", { clip: {} }],
|
||||
["an embedding of 511 numbers", withEmbedding(vector(3, 4).slice(1))],
|
||||
["an embedding with NaN in it", withEmbedding(vector(3, NaN))],
|
||||
["an embedding with Infinity in it", withEmbedding(vector(Infinity))],
|
||||
["an embedding of strings", withEmbedding(vector(3, 4).map(String))],
|
||||
["an embedding of zeros", withEmbedding(vector())],
|
||||
])("gives no keywords for ML data with %s", (_, mlData) => {
|
||||
expect(clipKeywords(mlData, false, [ACROSS, UP])).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// The CLIP embedding Ente's image model gives a public-domain NOAA photo of a
|
||||
// sunset, made by bin/clip-keyword-vectors from the photo at `photo`.
|
||||
const SUNSET: { photo: string; sha256: string; embedding: number[] } =
|
||||
JSON.parse(
|
||||
readFileSync(
|
||||
new URL("clip-keywords-sunset.json", import.meta.url),
|
||||
"utf-8",
|
||||
),
|
||||
);
|
||||
|
||||
describe("Ente's Discover categories", () => {
|
||||
it("are the 16 of Ente's photos app, each with a text embedding of norm 1", () => {
|
||||
expect(DISCOVER_CATEGORIES.map((c) => c.title)).toEqual([
|
||||
"Identity",
|
||||
"Screenshots",
|
||||
"QR Codes",
|
||||
"Receipts",
|
||||
"Notes",
|
||||
"Memes",
|
||||
"Visiting Cards",
|
||||
"Babies",
|
||||
"Pets",
|
||||
"Selfies",
|
||||
"Wallpapers",
|
||||
"Food",
|
||||
"Celebrations",
|
||||
"Sunset",
|
||||
"Hills",
|
||||
"Greenery",
|
||||
]);
|
||||
for (const category of DISCOVER_CATEGORIES) {
|
||||
expect(category.textEmbedding).toHaveLength(512);
|
||||
expect(Math.hypot(...category.textEmbedding)).toBeCloseTo(1, 5);
|
||||
}
|
||||
});
|
||||
|
||||
it("give a photo of a sunset the keyword Sunset and no other", () => {
|
||||
expect(
|
||||
clipKeywords({ clip: { embedding: SUNSET.embedding } }, false),
|
||||
).toEqual(["Sunset"]);
|
||||
});
|
||||
});
|
||||
@@ -658,6 +658,26 @@ describe("downloadFile", () => {
|
||||
expect(readdirSync(parent)).toEqual(["cwd"]);
|
||||
});
|
||||
|
||||
// A file system refuses a file name over 255 bytes.
|
||||
it("cuts a title too long for a file name short, keeping its extension", async () => {
|
||||
const plaintext = new Uint8Array([1, 2, 3]);
|
||||
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
||||
const thumbPush =
|
||||
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
||||
const file = buildMockEnteFile(key, header, thumbPush.header);
|
||||
file.metadata.title = `${"t".repeat(296)}.jpg`;
|
||||
const dir = mkdtempSync(join(testDir, "long-title-"));
|
||||
|
||||
const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) });
|
||||
const result = await inDirectory(dir, () => downloadFile(api, file));
|
||||
|
||||
const name = `${"t".repeat(251)}.jpg`;
|
||||
expect(Buffer.byteLength(name)).toBe(255);
|
||||
expect(result.path).toBe(name);
|
||||
expect(readdirSync(dir)).toEqual([name]);
|
||||
});
|
||||
|
||||
it("uses an explicit outPath verbatim, even one with ..", async () => {
|
||||
// The caller is trusted: its path is not sanitized.
|
||||
const { api, file } = fixtureFor(
|
||||
@@ -751,6 +771,28 @@ describe("downloadThumbnail", () => {
|
||||
expect(result.path).toBe("thumb__etc_passwd");
|
||||
expect(readdirSync(dir)).toEqual(["thumb__etc_passwd"]);
|
||||
});
|
||||
|
||||
it("leaves room for thumb_ when it cuts a title too long for a file name short", async () => {
|
||||
const plaintext = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]);
|
||||
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||
const filePush =
|
||||
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
||||
const { header: thumbHeader, ciphertext: thumbCipher } =
|
||||
encryptFileBody(plaintext, key);
|
||||
const file = buildMockEnteFile(key, filePush.header, thumbHeader);
|
||||
file.metadata.title = `${"t".repeat(296)}.jpg`;
|
||||
const dir = mkdtempSync(join(testDir, "long-thumb-title-"));
|
||||
|
||||
const api = new ApiClient({ fetch: mockFetchForBody(thumbCipher) });
|
||||
const result = await inDirectory(dir, () =>
|
||||
downloadThumbnail(api, file),
|
||||
);
|
||||
|
||||
const name = `thumb_${"t".repeat(245)}.jpg`;
|
||||
expect(Buffer.byteLength(name)).toBe(255);
|
||||
expect(result.path).toBe(name);
|
||||
expect(readdirSync(dir)).toEqual([name]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -1881,6 +1923,53 @@ describe("downloadFile live photos", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
// A file system refuses a file name over 255 bytes. Each name is cut
|
||||
// short once its own extension is in it.
|
||||
const imageName = `${"t".repeat(250)}.heic`;
|
||||
const videoName = `${"t".repeat(251)}.mov`;
|
||||
|
||||
// The title fits, but ".heic" is one byte longer than ".jpg".
|
||||
it("cuts the image's name short for a 255-byte title ending in .jpg when no outPath is given", async () => {
|
||||
const title = `${"t".repeat(251)}.jpg`;
|
||||
const t = setup(livePhotoZip(), { ...livePhoto, title });
|
||||
|
||||
const result = await inDirectory(t.dir, () =>
|
||||
downloadFile(t.api, t.file),
|
||||
);
|
||||
|
||||
expect(Buffer.byteLength(title)).toBe(255);
|
||||
expect(Buffer.byteLength(imageName)).toBe(255);
|
||||
expect(Buffer.byteLength(videoName)).toBe(255);
|
||||
expect(result.path).toBe(imageName);
|
||||
expect(result.videoPath).toBe(videoName);
|
||||
expect(readdirSync(t.dir).sort()).toEqual([imageName, videoName]);
|
||||
});
|
||||
|
||||
it("cuts the names of the image and the video short for a 300-byte title with no extension when no outPath is given", async () => {
|
||||
const t = setup(livePhotoZip(), {
|
||||
...livePhoto,
|
||||
title: "t".repeat(300),
|
||||
});
|
||||
|
||||
const result = await inDirectory(t.dir, () =>
|
||||
downloadFile(t.api, t.file),
|
||||
);
|
||||
|
||||
expect(result.path).toBe(imageName);
|
||||
expect(result.videoPath).toBe(videoName);
|
||||
expect(readdirSync(t.dir).sort()).toEqual([imageName, videoName]);
|
||||
});
|
||||
|
||||
// The caller's path is used as is; only the extension is each part's own.
|
||||
it("keeps an outPath too long for a file name as given", async () => {
|
||||
const t = setup(livePhotoZip(), livePhoto);
|
||||
const outPath = join(t.dir, `${"t".repeat(251)}.jpg`);
|
||||
|
||||
await expect(downloadFile(t.api, t.file, outPath)).rejects.toThrow(
|
||||
/ENAMETOOLONG/,
|
||||
);
|
||||
});
|
||||
|
||||
it("gives each part its own extension from the ZIP, letters and digits only", async () => {
|
||||
const zip = livePhotoZip({ "image.JPG": IMAGE, "video.m-4v": VIDEO });
|
||||
const t = setup(zip, livePhoto);
|
||||
|
||||
@@ -15,8 +15,10 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
defaultFileName,
|
||||
safeExtension,
|
||||
sanitizeFileName,
|
||||
shortenKeepingExtension,
|
||||
shortenToBytes,
|
||||
} from "../../src/filename.js";
|
||||
|
||||
@@ -94,6 +96,12 @@ describe("safeExtension", () => {
|
||||
expect(safeExtension("x.jp g")).toBe(".bin");
|
||||
expect(safeExtension("x.jpg\0")).toBe(".bin");
|
||||
});
|
||||
|
||||
it("keeps an extension of 16 letters and digits, and uses .bin for a longer one", () => {
|
||||
expect(safeExtension(`x.${"e".repeat(16)}`)).toBe(`.${"e".repeat(16)}`);
|
||||
expect(safeExtension(`x.${"e".repeat(17)}`)).toBe(".bin");
|
||||
expect(safeExtension(`b.${"e".repeat(300)}`)).toBe(".bin");
|
||||
});
|
||||
});
|
||||
|
||||
describe("shortenToBytes", () => {
|
||||
@@ -118,3 +126,103 @@ describe("shortenToBytes", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shortenKeepingExtension", () => {
|
||||
it("returns a name that fits unchanged", () => {
|
||||
expect(shortenKeepingExtension("beach.jpg", 9)).toBe("beach.jpg");
|
||||
expect(shortenKeepingExtension("beach", 5)).toBe("beach");
|
||||
});
|
||||
|
||||
it("cuts the part before the extension short and keeps the extension", () => {
|
||||
expect(shortenKeepingExtension(`${"t".repeat(296)}.jpg`, 255)).toBe(
|
||||
`${"t".repeat(251)}.jpg`,
|
||||
);
|
||||
expect(shortenKeepingExtension("é".repeat(150), 255)).toBe(
|
||||
"é".repeat(127),
|
||||
);
|
||||
});
|
||||
|
||||
// A name cut down to its extension would start with a dot, and so be
|
||||
// hidden.
|
||||
it("keeps the first character before the extension, cutting the extension short to leave room for it", () => {
|
||||
expect(shortenKeepingExtension(`b.${"e".repeat(300)}`, 255)).toBe(
|
||||
`b.${"e".repeat(253)}`,
|
||||
);
|
||||
expect(shortenKeepingExtension(`ab.${"e".repeat(300)}`, 255)).toBe(
|
||||
`a.${"e".repeat(253)}`,
|
||||
);
|
||||
expect(shortenKeepingExtension(`é.${"e".repeat(300)}`, 255)).toBe(
|
||||
`é.${"e".repeat(252)}`,
|
||||
);
|
||||
// "€" takes three bytes, so the extension is cut to 253 and the
|
||||
// part before it keeps the two bytes left over, for the "a".
|
||||
expect(shortenKeepingExtension(`a.${"€".repeat(90)}`, 255)).toBe(
|
||||
`a.${"€".repeat(84)}`,
|
||||
);
|
||||
});
|
||||
|
||||
it("puts `insert` right before the extension and leaves room for it", () => {
|
||||
expect(shortenKeepingExtension("beach.jpg", 255, " (7)")).toBe(
|
||||
"beach (7).jpg",
|
||||
);
|
||||
expect(
|
||||
shortenKeepingExtension(`${"t".repeat(296)}.jpg`, 255, " (7)"),
|
||||
).toBe(`${"t".repeat(247)} (7).jpg`);
|
||||
expect(
|
||||
shortenKeepingExtension(`b.${"e".repeat(300)}`, 255, " (7)"),
|
||||
).toBe(`b (7).${"e".repeat(249)}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("defaultFileName", () => {
|
||||
it("is the title when it is safe and fits", () => {
|
||||
expect(defaultFileName("IMG_0001.HEIC", 42)).toBe("IMG_0001.HEIC");
|
||||
});
|
||||
|
||||
it("replaces unsafe characters, and is file-<id> for an empty title", () => {
|
||||
expect(defaultFileName("../x.jpg", 42)).toBe("__x.jpg");
|
||||
expect(defaultFileName("", 42)).toBe("file-42");
|
||||
});
|
||||
|
||||
it("cuts a title over 255 bytes short, keeping its extension", () => {
|
||||
expect(defaultFileName(`${"t".repeat(296)}.jpg`, 42)).toBe(
|
||||
`${"t".repeat(251)}.jpg`,
|
||||
);
|
||||
expect(defaultFileName(`b.${"e".repeat(300)}`, 42)).toBe(
|
||||
`b.${"e".repeat(253)}`,
|
||||
);
|
||||
});
|
||||
|
||||
it("puts `prefix` before the title and leaves room for it", () => {
|
||||
expect(defaultFileName("beach.jpg", 42, { prefix: "thumb_" })).toBe(
|
||||
"thumb_beach.jpg",
|
||||
);
|
||||
expect(
|
||||
defaultFileName(`${"t".repeat(296)}.jpg`, 42, { prefix: "thumb_" }),
|
||||
).toBe(`thumb_${"t".repeat(245)}.jpg`);
|
||||
});
|
||||
|
||||
// A live photo's image and video are each named with their own
|
||||
// extension, and the name is cut short once that extension is in it.
|
||||
it("puts `ext` in place of the title's extension before cutting the name short", () => {
|
||||
expect(defaultFileName("IMG_0001.HEIC", 42, { ext: ".mov" })).toBe(
|
||||
"IMG_0001.mov",
|
||||
);
|
||||
expect(defaultFileName("", 42, { ext: ".mov" })).toBe("file-42.mov");
|
||||
expect(
|
||||
defaultFileName(`${"t".repeat(251)}.jpg`, 42, { ext: ".heic" }),
|
||||
).toBe(`${"t".repeat(250)}.heic`);
|
||||
expect(defaultFileName("t".repeat(300), 42, { ext: ".heic" })).toBe(
|
||||
`${"t".repeat(250)}.heic`,
|
||||
);
|
||||
});
|
||||
|
||||
// Cutting `CONSOLE` or `NULL` short before a long extension leaves `CON`
|
||||
// or `NUL`, so the name gets the `_` `sanitizeFileName` gives a device
|
||||
// name, and the part before the extension gives up a byte for it.
|
||||
it("never returns a Windows device name, even once cut short", () => {
|
||||
const ext = `.${"e".repeat(251)}`;
|
||||
expect(defaultFileName(`CONSOLE${ext}`, 42)).toBe(`_CO${ext}`);
|
||||
expect(defaultFileName(`NULL${ext}`, 42)).toBe(`_NU${ext}`);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -555,6 +555,36 @@ describe("withRetry", () => {
|
||||
expect(delays).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("tells `onRetry` of each attempt it retries, before the wait", async () => {
|
||||
// A caller that logs retries, as the CLI does with `-v`, learns why
|
||||
// each attempt failed and how long the next one is put off. The last
|
||||
// attempt is not retried, so it is not reported.
|
||||
const { sleep, delays } = recordingSleep();
|
||||
const retries: [string, number, number][] = [];
|
||||
let calls = 0;
|
||||
const failure = withRetry(
|
||||
() => {
|
||||
calls++;
|
||||
return Promise.reject(new ApiError(`attempt ${calls}`, 503));
|
||||
},
|
||||
{
|
||||
attempts: 3,
|
||||
sleep,
|
||||
random: () => 0.5,
|
||||
onRetry: (err, attempt, delayMs) => {
|
||||
retries.push([(err as Error).message, attempt, delayMs]);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
await expect(failure).rejects.toThrow("attempt 3");
|
||||
expect(retries).toEqual([
|
||||
["attempt 1", 1, 250],
|
||||
["attempt 2", 2, 500],
|
||||
]);
|
||||
expect(delays).toEqual([250, 500]);
|
||||
});
|
||||
|
||||
it("does not retry at all when attempts is 1", async () => {
|
||||
const { sleep, delays } = recordingSleep();
|
||||
let calls = 0;
|
||||
@@ -708,6 +738,7 @@ describe("retry defaults", () => {
|
||||
expect(resolved.maxDelayMs).toBe(DEFAULT_RETRY_OPTIONS.maxDelayMs);
|
||||
expect(typeof resolved.sleep).toBe("function");
|
||||
expect(typeof resolved.random).toBe("function");
|
||||
expect(typeof resolved.onRetry).toBe("function");
|
||||
});
|
||||
|
||||
it("resolves to the defaults when given nothing", () => {
|
||||
|
||||
@@ -544,11 +544,22 @@ describe("listMissingThumbnails", () => {
|
||||
const client = await login(counted.fetch, { ...noWait });
|
||||
const lib = await openLib(client);
|
||||
|
||||
const missing = await listMissingThumbnails(lib, client);
|
||||
const errors: string[] = [];
|
||||
const missing = await listMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
(msg, level) => {
|
||||
if (level === "error") errors.push(msg);
|
||||
},
|
||||
);
|
||||
lib.close();
|
||||
|
||||
// Only the genuinely empty thumbnail is reported.
|
||||
// Only the genuinely empty thumbnail is reported. The file that
|
||||
// could not be checked is reported as an error.
|
||||
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
||||
expect(errors).toEqual([
|
||||
expect.stringMatching(/^\[.*\] Could not check file-102\.jpg: /),
|
||||
]);
|
||||
// And the 500 was retried rather than accepted as an answer: four
|
||||
// attempts is the library default.
|
||||
expect(counted.matched()).toBe(4);
|
||||
@@ -699,7 +710,15 @@ describe("fixMissingThumbnails", () => {
|
||||
const client = await login(buildThumbFetch(fixMock));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(lib, client, [101, 102]);
|
||||
const errors: string[] = [];
|
||||
const results = await fixMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
[101, 102],
|
||||
(msg, level) => {
|
||||
if (level === "error") errors.push(msg);
|
||||
},
|
||||
);
|
||||
lib.close();
|
||||
|
||||
expect(results.length).toBe(2);
|
||||
@@ -707,6 +726,10 @@ describe("fixMissingThumbnails", () => {
|
||||
const failure = results.find((r) => r.fileID === 102)!;
|
||||
expect(success.status).toBe("fixed");
|
||||
expect(failure.status).toBe("failed");
|
||||
// The failed file, and only it, is reported as an error.
|
||||
expect(errors).toEqual([
|
||||
expect.stringMatching(/^\[.*\] FAILED file-102\.jpg: /),
|
||||
]);
|
||||
});
|
||||
|
||||
it("skips a non-JPEG image and a video as unsupported, not failed (issue #17)", async () => {
|
||||
|
||||
Reference in New Issue
Block a user