Compare commits
7
Commits
5f7b786dda
..
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe952d3e62 | ||
|
|
d23d3f8f47 | ||
|
|
aeccb489b5 | ||
|
|
2e00139d3c | ||
|
|
e8575780e4 | ||
|
|
17d1d74615 | ||
|
|
c05d63a2f0 |
@@ -38,34 +38,47 @@ yarn quak get 67890 --out ./photo.jpg
|
|||||||
yarn quak backup ./my-backup
|
yarn quak backup ./my-backup
|
||||||
```
|
```
|
||||||
|
|
||||||
For library use:
|
For library use, the primary surface is the cache-backed `Library`:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { Client } from "quak";
|
import { Client, Library } from "quak";
|
||||||
|
|
||||||
|
// Log in once; the client satisfies the library's client interface.
|
||||||
const client = await Client.login({
|
const client = await Client.login({
|
||||||
email: "you@example.com",
|
email: "you@example.com",
|
||||||
password: "your-password",
|
password: "your-password",
|
||||||
});
|
});
|
||||||
|
|
||||||
for (const c of await client.listCollections()) {
|
// Open a cache-backed library. On an empty cache this awaits one server
|
||||||
console.log(c.id, c.name);
|
// refresh; on an existing cache it returns immediately and refreshes in the
|
||||||
const files = await client.listFiles(c.id, c.key);
|
// background every `refreshIntervalSeconds` (default 3).
|
||||||
for (const f of files) {
|
const lib = await Library.open({ client });
|
||||||
console.log(` ${f.metadata.title} [${f.metadata.fileType}]`);
|
|
||||||
|
// Default reads answer synchronously from the local cache — no network.
|
||||||
|
for (const album of lib.albums.list()) {
|
||||||
|
console.log(album.collectionID, album.name);
|
||||||
|
for (const photo of album.photos.list()) {
|
||||||
|
console.log(` ${photo.title} [${photo.fileType}]`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download a file
|
// Fresh reads await a server round-trip and answer with current state.
|
||||||
const files = await client.listFiles(collectionID, collectionKey);
|
const { albums } = await lib.fresh();
|
||||||
await client.downloadFile(files[0], "./photo.jpg");
|
console.log(`${albums.list().length} albums as of now`);
|
||||||
|
|
||||||
// Serialize session for later (consumer handles persistence)
|
// Fetch (and cache) one photo's full-resolution bytes.
|
||||||
const snapshot = client.toJSON();
|
const photo = lib.photos.byID({ fileID: 12345 });
|
||||||
// ... later:
|
if (photo) {
|
||||||
const restored = Client.fromJSON(snapshot);
|
const { path } = await photo.original();
|
||||||
|
console.log(`original at ${path}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
lib.close();
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The lower-level `Client` (login, session serialization, and the raw
|
||||||
|
enumeration/download calls) is exported too and documented under Design below.
|
||||||
|
|
||||||
## Entrypoints
|
## Entrypoints
|
||||||
|
|
||||||
This repository adheres to the
|
This repository adheres to the
|
||||||
@@ -420,6 +433,7 @@ you would treat the password itself.
|
|||||||
### CLI surface
|
### CLI surface
|
||||||
|
|
||||||
```
|
```
|
||||||
|
quak [--cache-dir <path>] <command> global: local metadata/content cache location
|
||||||
quak login interactive or QUAK_EMAIL/QUAK_PASSWORD
|
quak login interactive or QUAK_EMAIL/QUAK_PASSWORD
|
||||||
quak whoami print logged-in account as JSON
|
quak whoami print logged-in account as JSON
|
||||||
quak logout delete saved session
|
quak logout delete saved session
|
||||||
@@ -428,13 +442,28 @@ quak files --collection <id> [--json] list files in a collection
|
|||||||
quak get <fileID> [--out path] [--collection] download and decrypt a file
|
quak get <fileID> [--out path] [--collection] download and decrypt a file
|
||||||
quak get-thumb <fileID> [--out] [--collection] download and decrypt a thumbnail
|
quak get-thumb <fileID> [--out] [--collection] download and decrypt a thumbnail
|
||||||
quak backup <dir> [--json] full incremental backup
|
quak backup <dir> [--json] full incremental backup
|
||||||
|
quak backup-metadata <dir> [--exif] dump all decrypted metadata as JSON
|
||||||
quak helper list-missing-thumbnails [--json] find files with missing thumbnails
|
quak helper list-missing-thumbnails [--json] find files with missing thumbnails
|
||||||
quak helper fix-missing-thumbnails [--file ids] generate + upload missing thumbnails
|
quak helper fix-missing-thumbnails [--file ids] generate + upload missing thumbnails
|
||||||
```
|
```
|
||||||
|
|
||||||
`get` and `get-thumb` search all collections for the file ID when `--collection`
|
Every command runs on the same cache-backed library. The read commands —
|
||||||
is not specified. All listing and backup commands support `--json` for
|
`collections`, `files`, `get`, and `get-thumb` — force a fresh server round-trip
|
||||||
machine-readable output.
|
before they answer, so they report current account state rather than whatever
|
||||||
|
the cache last held. `--cache-dir` overrides where the cache lives; without it
|
||||||
|
each account gets its own directory under the per-user cache path.
|
||||||
|
|
||||||
|
`get` and `get-thumb` resolve the file by ID directly, so `--collection` is
|
||||||
|
accepted for backward compatibility but ignored. `backup-metadata --exif` (alias
|
||||||
|
`--all`) additionally downloads each file to extract full EXIF/IPTC/XMP
|
||||||
|
metadata. The listing and backup commands support `--json` for machine-readable
|
||||||
|
output.
|
||||||
|
|
||||||
|
`helper fix-missing-thumbnails` regenerates thumbnails for baseline JPEG images
|
||||||
|
only, because the bundled decoder (`jpeg-js`) decodes only JPEG. A non-JPEG
|
||||||
|
image (PNG, HEIC) or a video is reported as `skipped` (unsupported format), kept
|
||||||
|
distinct from a `failed` repair, and does not affect the exit code; a genuine
|
||||||
|
failure still exits non-zero.
|
||||||
|
|
||||||
### Backup layout
|
### Backup layout
|
||||||
|
|
||||||
@@ -460,7 +489,7 @@ code is non-zero if any files failed.
|
|||||||
|
|
||||||
- [x] Retry policy: no retry on 4xx, exponential backoff on 5xx and network
|
- [x] Retry policy: no retry on 4xx, exponential backoff on 5xx and network
|
||||||
errors
|
errors
|
||||||
- [ ] Update the API reference section below to match the current implementation
|
- [x] Update the API reference section below to match the current implementation
|
||||||
- [x] `make docker` green
|
- [x] `make docker` green
|
||||||
- [ ] Tag `v1.0.0`
|
- [ ] Tag `v1.0.0`
|
||||||
|
|
||||||
@@ -474,25 +503,179 @@ Future (desktop client, separate repo):
|
|||||||
|
|
||||||
## API reference
|
## API reference
|
||||||
|
|
||||||
The API reference section below is from an earlier draft and does not fully
|
The library's primary surface is the cache-backed `Library`; the lower-level
|
||||||
reflect the current implementation. The authoritative API documentation is in
|
`Client` sits underneath it and is covered by the Design sections above. The
|
||||||
the test files, particularly `test/client/usage.test.ts` which is a literate
|
test suite is the canonical, executable documentation — `test/library/` and
|
||||||
tutorial walking through every operation. Run `yarn test` to verify the examples
|
`test/client/usage.test.ts` walk every operation, and `yarn test` verifies them.
|
||||||
are correct.
|
|
||||||
|
|
||||||
The key types and their actual signatures can be found in:
|
### Opening a library
|
||||||
|
|
||||||
|
`Library.open(options)` loads the on-disk cache, starts the background refresh
|
||||||
|
loop, and resolves to a `Library`. On an empty cache it awaits the first refresh
|
||||||
|
so it never opens onto empty data; on an existing cache it returns immediately
|
||||||
|
and refreshes in the background, so an unreachable server does not block
|
||||||
|
opening.
|
||||||
|
|
||||||
|
`LibraryOptions`:
|
||||||
|
|
||||||
|
| Option | Default | Meaning |
|
||||||
|
| ------------------------ | --------------------------- | --------------------------------------------------------------------- |
|
||||||
|
| `client` | required | the account client (a `Client`, or any `LibraryClient`) |
|
||||||
|
| `cacheDirectory` | `<XDG cache>/quak/<userID>` | where `metadata.json` and the content cache live |
|
||||||
|
| `downloadDirectory` | none | backup destination; an original already stored there counts as cached |
|
||||||
|
| `refreshIntervalSeconds` | `3` | background refresh cadence |
|
||||||
|
| `precacheThumbnails` | `true` | prefetch every thumbnail, newest first |
|
||||||
|
| `precacheOriginals` | `true` | prefetch the favorites album and the latest-window originals |
|
||||||
|
| `precacheOriginalsDays` | `7` | length in days of that latest window |
|
||||||
|
| `cacheOriginalsMaxBytes` | 100 GiB | hard ceiling on the originals cache |
|
||||||
|
| `freeBelowBytes` | 50 GiB | free space to protect on the volume; the effective limit adapts down |
|
||||||
|
| `isOriginalPinned` | none | extra predicate for originals that must never be evicted |
|
||||||
|
| `pools` | fresh `RequestPools` | the bounded request pools (sets concurrency) |
|
||||||
|
| `onProgress` | none | refresh/ML/precache progress callback (`RefreshEvent`) |
|
||||||
|
| `contentSource` | the client's own | override the byte source (mainly for tests) |
|
||||||
|
|
||||||
|
Concurrency is set through `pools`: construct
|
||||||
|
`new RequestPools({ metadataConcurrency, contentConcurrency, thumbnailConcurrency })`
|
||||||
|
and pass it. The three pools default to 10 / 5 / 25 (see Request pools below).
|
||||||
|
|
||||||
|
`lib.status()` returns a `LibraryStatus` (collection/file counts, last
|
||||||
|
refresh/ML times and errors, originals usage and effective limit, precache
|
||||||
|
progress, and `closed`). `lib.close()` stops the background timer; it is
|
||||||
|
idempotent, and an in-flight refresh is left to finish.
|
||||||
|
|
||||||
|
### Default reads vs. fresh reads
|
||||||
|
|
||||||
|
Default reads — `lib.albums`, `lib.photos`, `lib.timeline` — answer
|
||||||
|
synchronously from the last refreshed copy held in RAM and never touch the
|
||||||
|
network. The background timer refreshes that copy every
|
||||||
|
`refreshIntervalSeconds`, so a default read is immediate but may be up to one
|
||||||
|
interval stale.
|
||||||
|
|
||||||
|
`await lib.fresh()` forces a refresh, waits for it to complete and persist, and
|
||||||
|
returns the same `{ albums, photos, timeline }` namespaces — now guaranteed to
|
||||||
|
reflect a completed server round-trip. Concurrent `fresh()` calls coalesce onto
|
||||||
|
one refresh, and a refresh that fails rejects the caller (default reads stay
|
||||||
|
silent and keep serving the last good copy). The CLI's read commands use fresh
|
||||||
|
reads (issue https://git.eeqj.de/sneak/quak/issues/75).
|
||||||
|
|
||||||
|
### Read surface
|
||||||
|
|
||||||
|
- `lib.albums.list()` → `Album[]`, newest-updated first.
|
||||||
|
`lib.albums.byID({ collectionID })` and `byName({ albumName })` →
|
||||||
|
`Album | undefined`.
|
||||||
|
- `lib.photos.byID({ fileID })` → `Photo | undefined`.
|
||||||
|
`lib.photos.records({ fileIDs })` → `PhotoRecord[]` in the requested order,
|
||||||
|
each id once, unknown ids dropped.
|
||||||
|
- `lib.timeline.groups({ groupBy, filter? })` → `TimelineGroup[]`, grouped by
|
||||||
|
`"day" | "week" | "month"` (keys `YYYY-MM-DD`, ISO `YYYY-Www`, `YYYY-MM`),
|
||||||
|
newest group first. A `PhotoFilter` combines `albumID`, `text`
|
||||||
|
(title/caption/album-name substring), `fileTypes`, `hasLocation`, and
|
||||||
|
`includeArchived`; hidden photos are always excluded.
|
||||||
|
|
||||||
|
An `Album` exposes its record fields and `album.photos.list()` → `Photo[]`
|
||||||
|
(newest first). A `Photo` exposes its record fields, `photo.record()` →
|
||||||
|
`PhotoRecord`, and two content methods:
|
||||||
|
|
||||||
|
- `await photo.original(opts?)` → `{ path, bytes }` — the full-resolution file.
|
||||||
|
- `await photo.thumbnail(opts?)` → `{ path, bytes }`.
|
||||||
|
|
||||||
|
Both serve from the on-disk content cache when the bytes are present and
|
||||||
|
otherwise fetch through the pools; `opts.onProgress` reports per-file progress.
|
||||||
|
They throw when the library was opened without a content source.
|
||||||
|
|
||||||
|
Lower-level accessors that return decrypted model objects (which hold key
|
||||||
|
material) are also available: `listCollections()`, `getCollection(id)`,
|
||||||
|
`listFiles(collectionID)`, `getFile(collectionID, fileID)`, and
|
||||||
|
`getFileByID(fileID)`.
|
||||||
|
|
||||||
|
### Records and change notifications
|
||||||
|
|
||||||
|
The GUI-facing records hold no key material and no binary, so they survive
|
||||||
|
`structuredClone`/JSON across the Electron IPC boundary:
|
||||||
|
|
||||||
|
- `PhotoRecord`: `fileID`, `albumIDs`, `title`, `takenAt` (milliseconds),
|
||||||
|
`fileType`, optional `caption` / `width` / `height` / `latitude` /
|
||||||
|
`longitude`, `isArchived`, `isHidden`, and `thumbnailPath` / `originalPath`
|
||||||
|
once the bytes are cached.
|
||||||
|
- `AlbumRecord`: `collectionID`, `name`, `type`, `isShared`, `updationTime`, and
|
||||||
|
`fileIDs` (newest first).
|
||||||
|
- `LibrarySnapshot`: `{ albums, photos, takenAt }`.
|
||||||
|
|
||||||
|
`lib.snapshot()` returns a `LibrarySnapshot` (albums newest-updated first,
|
||||||
|
photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange`
|
||||||
|
(`albumsChanged`, `photosChanged`, `fileIDsRemoved`, `albumIDsRemoved`,
|
||||||
|
`refreshedAt`) whenever a refresh alters the projection, and returns
|
||||||
|
`{ unsubscribe }`; a refresh that changes nothing delivers nothing.
|
||||||
|
|
||||||
|
### Thumbnails, ML search, and backup
|
||||||
|
|
||||||
|
- `lib.thumbnails.ensure({ fileIDs, priority, signal?, onProgress? })`
|
||||||
|
prefetches thumbnails through the thumbnail pool, deduped by fileID, returning
|
||||||
|
one `EnsureResult` (`{ fileID, path?, error? }`) per file. `priority` is
|
||||||
|
`"visible" | "ahead" | "background"`; only `"visible"` preempts background
|
||||||
|
work.
|
||||||
|
- `lib.mldata` searches the CLIP index built from Ente's per-file ML data:
|
||||||
|
`forFile({ fileID })` → `Promise<MLData | undefined>` (the whole stored
|
||||||
|
payload — face boxes, landmarks, embedding — read from disk on demand);
|
||||||
|
`similar({ fileID, limit? })` and `searchByEmbedding({ embedding, limit? })` →
|
||||||
|
`SimilarResult[]` (`{ fileID, score }`, cosine similarity, most similar first,
|
||||||
|
default limit 20). quak bundles no text encoder, so `searchByEmbedding` takes
|
||||||
|
a query vector the caller produced elsewhere.
|
||||||
|
- `await lib.backup(opts?)` → `BackupResult`. It refreshes, fetches every
|
||||||
|
in-scope original (and, with `includeThumbnails`, thumbnails) through the
|
||||||
|
content cache, and rebuilds the on-disk backup tree with a durable failure
|
||||||
|
ledger. `BackupOptions`: `downloadDirectory` (falls back to the one `open()`
|
||||||
|
was given), `includeOriginals` (default `true`), `includeThumbnails` (default
|
||||||
|
`false`), `onlyAlbumNames`, and `onProgress`. See Backup layout above for the
|
||||||
|
tree it writes.
|
||||||
|
|
||||||
|
### Request pools
|
||||||
|
|
||||||
|
`RequestPools` holds three independent bounded pools — metadata (10), content
|
||||||
|
(5), thumbnails (25) — because Ente meters these traffic classes differently.
|
||||||
|
Each pool orders on-demand work ahead of background/precache work and dedups
|
||||||
|
in-flight fetches by key, and an idle pool never lends its slots to a busy one.
|
||||||
|
|
||||||
|
### On-disk cache layout
|
||||||
|
|
||||||
|
Under `cacheDirectory`:
|
||||||
|
|
||||||
|
```
|
||||||
|
<cacheDirectory>/
|
||||||
|
metadata.json decrypted account state + refresh cursor
|
||||||
|
originals/<fileID>.<ext> cached full-resolution files
|
||||||
|
thumbnails/<fileID>.jpg cached thumbnails
|
||||||
|
mldata/
|
||||||
|
<fileID>.json one decrypted ML payload per file
|
||||||
|
clip.f32, clip.json the packed CLIP index and its id list
|
||||||
|
fetched.json per-file fetch bookkeeping
|
||||||
|
```
|
||||||
|
|
||||||
|
A stored file appears only via an atomic temp-then-rename, so its presence means
|
||||||
|
it is complete. The design also calls for a content-hash comparison against
|
||||||
|
`FileMetadata.hash` on each fetched original; that check is deferred (issue
|
||||||
|
https://git.eeqj.de/sneak/quak/issues/68) because the exact hash construction
|
||||||
|
cannot yet be confirmed against the repo's fixtures.
|
||||||
|
|
||||||
|
### Key types by source file
|
||||||
|
|
||||||
|
- `src/library/index.ts`: `Library`, `LibraryOptions`, `LibraryStatus`,
|
||||||
|
`LibraryClient`, `RefreshEvent`
|
||||||
|
- `src/library/read.ts`: `Album`, `Photo`, `AlbumsAPI`, `PhotosAPI`,
|
||||||
|
`TimelineAPI`, `PhotoFilter`, `TimelineGroup`, `GroupBy`
|
||||||
|
- `src/library/content.ts`: `ContentResult`, `ContentOptions`, `ThumbnailsAPI`,
|
||||||
|
`EnsureOptions`, `EnsureResult`, `ContentSource`
|
||||||
|
- `src/library/records.ts`: `PhotoRecord`, `AlbumRecord`, `LibrarySnapshot`,
|
||||||
|
`LibraryChange`
|
||||||
|
- `src/library/mlsearch.ts`: `MLDataAPI`, `SimilarResult`
|
||||||
|
- `src/library/pools.ts`: `RequestPools`, `RequestPoolsOptions`, `BoundedPool`
|
||||||
|
- `src/backup.ts`: `BackupOptions`, `BackupResult`, `BackupError`
|
||||||
- `src/client.ts`: `Client`, `LoginOptions`, `ClientSnapshot`
|
- `src/client.ts`: `Client`, `LoginOptions`, `ClientSnapshot`
|
||||||
- `src/api/client.ts`: `ApiClient`, `ApiClientOptions`, `ApiError`,
|
- `src/api/client.ts`: `ApiClient`, `ApiClientOptions`, `StreamOptions`
|
||||||
`StreamOptions`
|
|
||||||
- `src/errors.ts`: `ApiError`, `TruncatedStreamError`
|
- `src/errors.ts`: `ApiError`, `TruncatedStreamError`
|
||||||
- `src/retry.ts`: `withRetry`, `isRetryable`, `isSafeToReplay`, `RetryOptions`
|
- `src/retry.ts`: `withRetry`, `isRetryable`, `isSafeToReplay`, `RetryOptions`
|
||||||
- `src/auth/types.ts`: `KeyAttributes`, `SRPAttributes`,
|
- `src/model/types.ts`: `Collection`, `EnteFile`, `FileMetadata`, `FileType`,
|
||||||
`AuthorizationResponse`, `LoginChallenge`
|
`CollectionType`, `RawCollection`, `RawEnteFile`
|
||||||
- `src/model/types.ts`: `Collection`, `EnteFile`, `FileMetadata`, `FileBlob`,
|
|
||||||
`RawCollection`, `RawEnteFile`, `RawMagicMetadata`
|
|
||||||
- `src/download/index.ts`: `DownloadResult`
|
|
||||||
- `src/backup.ts`: `BackupResult`, `BackupError`
|
|
||||||
- `src/thumbnails.ts`: `MissingThumbnailInfo`, `ThumbnailFixResult`
|
- `src/thumbnails.ts`: `MissingThumbnailInfo`, `ThumbnailFixResult`
|
||||||
|
|
||||||
## Source attribution
|
## Source attribution
|
||||||
|
|||||||
@@ -14,10 +14,19 @@ pre-1.0
|
|||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
Update the README API reference section to match the current implementation.
|
Tag v1.0.0.
|
||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-09-22: Rewrote the README API reference (and the Getting Started / usage
|
||||||
|
snippets) to match the shipped cache/API library on `next` (issue 53, issue
|
||||||
|
13). Documented `Library.open` and its options, the default-read vs `fresh()`
|
||||||
|
distinction (and that the CLI's read commands are fresh), the record types and
|
||||||
|
`snapshot()`/`subscribe()`, the `albums`/`photos`/`timeline` read surface,
|
||||||
|
`Photo` content methods, `thumbnails.ensure`, the `mldata` search surface,
|
||||||
|
`backup()`, the three request pools (10/5/25), and the on-disk cache layout;
|
||||||
|
noted the deferred content-hash integrity check (issue 68). Docs-only; no code
|
||||||
|
changed.
|
||||||
- 2026-09-22: Added resumable, deletion-aware enumeration to `Client` (issue 38,
|
- 2026-09-22: Added resumable, deletion-aware enumeration to `Client` (issue 38,
|
||||||
closes issue 7). `collectionsSince`/`filesSince` take a starting cursor,
|
closes issue 7). `collectionsSince`/`filesSince` take a starting cursor,
|
||||||
decrypt live records, surface tombstoned ids in a separate `deleted` list (a
|
decrypt live records, surface tombstoned ids in a separate `deleted` list (a
|
||||||
@@ -122,7 +131,6 @@ Update the README API reference section to match the current implementation.
|
|||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Tag v1.0.0.
|
|
||||||
- Future desktop client, separate repo:
|
- Future desktop client, separate repo:
|
||||||
- Electron app skeleton consuming this library.
|
- Electron app skeleton consuming this library.
|
||||||
- Local SQLite cache keyed on (collectionID, fileID, updationTime).
|
- Local SQLite cache keyed on (collectionID, fileID, updationTime).
|
||||||
|
|||||||
+187
-126
@@ -2,13 +2,26 @@
|
|||||||
|
|
||||||
import { input, password as passwordPrompt } from "@inquirer/prompts";
|
import { input, password as passwordPrompt } from "@inquirer/prompts";
|
||||||
import { stdout, stderr } from "node:process";
|
import { stdout, stderr } from "node:process";
|
||||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
import {
|
||||||
|
copyFileSync,
|
||||||
|
existsSync,
|
||||||
|
mkdirSync,
|
||||||
|
readFileSync,
|
||||||
|
writeFileSync,
|
||||||
|
} from "node:fs";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { Command } from "commander";
|
import { Command } from "commander";
|
||||||
import envPaths from "env-paths";
|
import envPaths from "env-paths";
|
||||||
import { Client, type ClientSnapshot } from "../src/client.js";
|
import { Client, type ClientSnapshot } from "../src/client.js";
|
||||||
import { init } from "../src/crypto/index.js";
|
import { init } from "../src/crypto/index.js";
|
||||||
import { Library } from "../src/library/index.js";
|
import { Library, type LibraryClient } from "../src/library/index.js";
|
||||||
|
import {
|
||||||
|
fileListRow,
|
||||||
|
fileListLine,
|
||||||
|
originalName,
|
||||||
|
thumbnailName,
|
||||||
|
} from "../src/cli-output.js";
|
||||||
|
import { freshCollections, freshFiles, freshFile } from "../src/cli-read.js";
|
||||||
import { runMetadataBackup } from "../src/metadata-backup.js";
|
import { runMetadataBackup } from "../src/metadata-backup.js";
|
||||||
import {
|
import {
|
||||||
listMissingThumbnails,
|
listMissingThumbnails,
|
||||||
@@ -55,7 +68,61 @@ const program = new Command();
|
|||||||
program
|
program
|
||||||
.name("quak")
|
.name("quak")
|
||||||
.description("CLI for the Ente end-to-end encrypted photo service")
|
.description("CLI for the Ente end-to-end encrypted photo service")
|
||||||
.version("0.0.0");
|
.version("0.0.0")
|
||||||
|
.option(
|
||||||
|
"--cache-dir <path>",
|
||||||
|
"Directory for the local metadata/content cache " +
|
||||||
|
"(default: the per-user cache directory)",
|
||||||
|
);
|
||||||
|
|
||||||
|
// The `--cache-dir` global, or undefined to let the library pick its per-user
|
||||||
|
// default keyed by the account id.
|
||||||
|
const cacheDirOption = (): string | undefined =>
|
||||||
|
program.opts<{ cacheDir?: string }>().cacheDir;
|
||||||
|
|
||||||
|
// A library client that omits `fetchMLData`, so the point commands below do not
|
||||||
|
// kick the library's background ML backfill: they read metadata, or fetch one
|
||||||
|
// file's content, and exit. `backup` and `backup-metadata` handle ML on their
|
||||||
|
// own terms. The content source is kept so `get`/`get-thumb`/`--exif` can fetch
|
||||||
|
// originals through the on-disk cache.
|
||||||
|
const readLibraryClient = (client: Client): LibraryClient => ({
|
||||||
|
whoami: () => client.whoami(),
|
||||||
|
collectionsSince: (args) => client.collectionsSince(args),
|
||||||
|
filesSince: (args) => client.filesSince(args),
|
||||||
|
contentSource: () => client.contentSource(),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Open a library for a single point command: the aggressive background precache
|
||||||
|
// (issue #48) is off — a one-shot `collections` or `get` must not start
|
||||||
|
// downloading the whole account — and the refresh interval is long so no second
|
||||||
|
// refresh fires mid-command.
|
||||||
|
const openReadLibrary = (client: Client): Promise<Library> =>
|
||||||
|
Library.open({
|
||||||
|
client: readLibraryClient(client),
|
||||||
|
cacheDirectory: cacheDirOption(),
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Close the library and exit once stdout/stderr have drained. `process.exit`
|
||||||
|
// alone can truncate buffered piped output, and the library keeps the event
|
||||||
|
// loop alive with a background refresh, so a plain return could hang; this does
|
||||||
|
// neither.
|
||||||
|
const finish = (lib: Library | undefined, code: number): void => {
|
||||||
|
lib?.close();
|
||||||
|
const pending = [stdout, stderr].filter((s) => s.writableLength > 0);
|
||||||
|
if (pending.length === 0) {
|
||||||
|
process.exit(code);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let remaining = pending.length;
|
||||||
|
for (const s of pending) {
|
||||||
|
s.once("drain", () => {
|
||||||
|
if (--remaining === 0) process.exit(code);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
program
|
program
|
||||||
.command("login")
|
.command("login")
|
||||||
@@ -116,7 +183,11 @@ program
|
|||||||
.action(async (opts: { json?: boolean }) => {
|
.action(async (opts: { json?: boolean }) => {
|
||||||
await init();
|
await init();
|
||||||
const client = requireSession();
|
const client = requireSession();
|
||||||
const collections = await client.listCollections();
|
const lib = await openReadLibrary(client);
|
||||||
|
// Force a server round-trip and list in enumeration order (issue #36
|
||||||
|
// amendment, issue #52): the pre-library CLI printed current state in
|
||||||
|
// this order, not the albums projection's newest-first order.
|
||||||
|
const collections = await freshCollections(lib);
|
||||||
|
|
||||||
if (opts.json) {
|
if (opts.json) {
|
||||||
stdout.write(
|
stdout.write(
|
||||||
@@ -140,6 +211,7 @@ program
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
finish(lib, 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
program
|
program
|
||||||
@@ -159,36 +231,29 @@ program
|
|||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
const collections = await client.listCollections();
|
const lib = await openReadLibrary(client);
|
||||||
const col = collections.find((c) => c.id === collectionID);
|
// Force a server round-trip and list in enumeration order (issue #36
|
||||||
if (!col) {
|
// amendment, issue #52). Each file prints from its own decrypted
|
||||||
|
// metadata (raw title, microsecond creationTime) via cli-output, and in
|
||||||
|
// the pre-library CLI's enumeration order, not the projection's
|
||||||
|
// newest-first order.
|
||||||
|
const files = await freshFiles(lib, collectionID);
|
||||||
|
if (!files) {
|
||||||
stderr.write(`Collection ${collectionID} not found\n`);
|
stderr.write(`Collection ${collectionID} not found\n`);
|
||||||
process.exit(1);
|
finish(lib, 1);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const files = await client.listFiles(col.id, col.key);
|
|
||||||
|
|
||||||
if (opts.json) {
|
if (opts.json) {
|
||||||
stdout.write(
|
stdout.write(
|
||||||
JSON.stringify(
|
JSON.stringify(files.map(fileListRow), null, 2) + "\n",
|
||||||
files.map((f) => ({
|
|
||||||
id: f.id,
|
|
||||||
title: f.metadata.title,
|
|
||||||
fileType: f.metadata.fileType,
|
|
||||||
creationTime: f.metadata.creationTime,
|
|
||||||
collectionID: f.collectionID,
|
|
||||||
})),
|
|
||||||
null,
|
|
||||||
2,
|
|
||||||
) + "\n",
|
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
for (const f of files) {
|
for (const file of files) {
|
||||||
stdout.write(
|
stdout.write(fileListLine(file) + "\n");
|
||||||
`${f.id}\t${f.metadata.fileType}\t${f.metadata.title}\n`,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
finish(lib, 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
program
|
program
|
||||||
@@ -196,98 +261,70 @@ program
|
|||||||
.description("Download and decrypt a single file")
|
.description("Download and decrypt a single file")
|
||||||
.argument("<fileID>", "File ID (from `quak files`)")
|
.argument("<fileID>", "File ID (from `quak files`)")
|
||||||
.option("--out <path>", "Output file path")
|
.option("--out <path>", "Output file path")
|
||||||
.option(
|
.option("--collection <id>", "Accepted for compatibility; ignored")
|
||||||
"--collection <id>",
|
.action(async (fileIDStr: string, opts: { out?: string }) => {
|
||||||
"Collection ID (required to look up the file key)",
|
await init();
|
||||||
)
|
const client = requireSession();
|
||||||
.action(
|
const fileID = Number(fileIDStr);
|
||||||
async (
|
if (!Number.isFinite(fileID)) {
|
||||||
fileIDStr: string,
|
stderr.write("Invalid file ID\n");
|
||||||
opts: { out?: string; collection?: string },
|
|
||||||
) => {
|
|
||||||
await init();
|
|
||||||
const client = requireSession();
|
|
||||||
const fileID = Number(fileIDStr);
|
|
||||||
if (!Number.isFinite(fileID)) {
|
|
||||||
stderr.write("Invalid file ID\n");
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
const collections = await client.listCollections();
|
|
||||||
let targetCol;
|
|
||||||
if (opts.collection) {
|
|
||||||
targetCol = collections.find(
|
|
||||||
(c) => c.id === Number(opts.collection),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Search all collections (or the specified one) for the file
|
|
||||||
const searchCols = targetCol ? [targetCol] : collections;
|
|
||||||
for (const col of searchCols) {
|
|
||||||
const files = await client.listFiles(col.id, col.key);
|
|
||||||
const file = files.find((f) => f.id === fileID);
|
|
||||||
if (file) {
|
|
||||||
const result = await client.downloadFile(file, opts.out);
|
|
||||||
stderr.write(
|
|
||||||
`${result.bytesWritten} bytes -> ${result.path}\n`,
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stderr.write(`File ${fileID} not found\n`);
|
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
},
|
}
|
||||||
);
|
|
||||||
|
const lib = await openReadLibrary(client);
|
||||||
|
// Force a server round-trip so the file resolves against current state
|
||||||
|
// (issue #36 amendment, issue #52).
|
||||||
|
const resolved = await freshFile(lib, fileID);
|
||||||
|
if (!resolved) {
|
||||||
|
stderr.write(`File ${fileID} not found\n`);
|
||||||
|
finish(lib, 1);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const { photo, file } = resolved;
|
||||||
|
|
||||||
|
const result = await photo.original();
|
||||||
|
// Default name is the file's own title, as the pre-library CLI used
|
||||||
|
// (not the editedName-preferring projection title) (issue #52).
|
||||||
|
const outPath = opts.out ?? originalName(file);
|
||||||
|
copyFileSync(result.path, outPath);
|
||||||
|
stderr.write(`${result.bytes} bytes -> ${outPath}\n`);
|
||||||
|
finish(lib, 0);
|
||||||
|
});
|
||||||
|
|
||||||
program
|
program
|
||||||
.command("get-thumb")
|
.command("get-thumb")
|
||||||
.description("Download and decrypt a thumbnail")
|
.description("Download and decrypt a thumbnail")
|
||||||
.argument("<fileID>", "File ID (from `quak files`)")
|
.argument("<fileID>", "File ID (from `quak files`)")
|
||||||
.option("--out <path>", "Output file path")
|
.option("--out <path>", "Output file path")
|
||||||
.option(
|
.option("--collection <id>", "Accepted for compatibility; ignored")
|
||||||
"--collection <id>",
|
.action(async (fileIDStr: string, opts: { out?: string }) => {
|
||||||
"Collection ID (required to look up the file key)",
|
await init();
|
||||||
)
|
const client = requireSession();
|
||||||
.action(
|
const fileID = Number(fileIDStr);
|
||||||
async (
|
if (!Number.isFinite(fileID)) {
|
||||||
fileIDStr: string,
|
stderr.write("Invalid file ID\n");
|
||||||
opts: { out?: string; collection?: string },
|
|
||||||
) => {
|
|
||||||
await init();
|
|
||||||
const client = requireSession();
|
|
||||||
const fileID = Number(fileIDStr);
|
|
||||||
if (!Number.isFinite(fileID)) {
|
|
||||||
stderr.write("Invalid file ID\n");
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
const collections = await client.listCollections();
|
|
||||||
let targetCol;
|
|
||||||
if (opts.collection) {
|
|
||||||
targetCol = collections.find(
|
|
||||||
(c) => c.id === Number(opts.collection),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const searchCols = targetCol ? [targetCol] : collections;
|
|
||||||
for (const col of searchCols) {
|
|
||||||
const files = await client.listFiles(col.id, col.key);
|
|
||||||
const file = files.find((f) => f.id === fileID);
|
|
||||||
if (file) {
|
|
||||||
const result = await client.downloadThumbnail(
|
|
||||||
file,
|
|
||||||
opts.out,
|
|
||||||
);
|
|
||||||
stderr.write(
|
|
||||||
`${result.bytesWritten} bytes -> ${result.path}\n`,
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stderr.write(`File ${fileID} not found\n`);
|
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
},
|
}
|
||||||
);
|
|
||||||
|
const lib = await openReadLibrary(client);
|
||||||
|
// Force a server round-trip so the file resolves against current state
|
||||||
|
// (issue #36 amendment, issue #52).
|
||||||
|
const resolved = await freshFile(lib, fileID);
|
||||||
|
if (!resolved) {
|
||||||
|
stderr.write(`File ${fileID} not found\n`);
|
||||||
|
finish(lib, 1);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const { photo, file } = resolved;
|
||||||
|
|
||||||
|
const result = await photo.thumbnail();
|
||||||
|
// Default name is thumb_<file's own title>, as the pre-library CLI
|
||||||
|
// used (not the projection title) (issue #52).
|
||||||
|
const outPath = opts.out ?? thumbnailName(file);
|
||||||
|
copyFileSync(result.path, outPath);
|
||||||
|
stderr.write(`${result.bytes} bytes -> ${outPath}\n`);
|
||||||
|
finish(lib, 0);
|
||||||
|
});
|
||||||
|
|
||||||
program
|
program
|
||||||
.command("backup-metadata")
|
.command("backup-metadata")
|
||||||
@@ -303,10 +340,12 @@ program
|
|||||||
.action(async (dir: string, opts: { exif?: boolean; all?: boolean }) => {
|
.action(async (dir: string, opts: { exif?: boolean; all?: boolean }) => {
|
||||||
await init();
|
await init();
|
||||||
const client = requireSession();
|
const client = requireSession();
|
||||||
await runMetadataBackup(client, dir, {
|
const lib = await openReadLibrary(client);
|
||||||
|
await runMetadataBackup(lib, client, dir, {
|
||||||
exif: opts.exif || opts.all,
|
exif: opts.exif || opts.all,
|
||||||
onProgress: (msg) => stderr.write(msg + "\n"),
|
onProgress: (msg) => stderr.write(msg + "\n"),
|
||||||
});
|
});
|
||||||
|
finish(lib, 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
program
|
program
|
||||||
@@ -321,14 +360,17 @@ program
|
|||||||
const client = requireSession();
|
const client = requireSession();
|
||||||
|
|
||||||
stderr.write("Starting backup...\n");
|
stderr.write("Starting backup...\n");
|
||||||
const lib = await Library.open({ client, downloadDirectory: dir });
|
const lib = await Library.open({
|
||||||
|
client,
|
||||||
|
downloadDirectory: dir,
|
||||||
|
cacheDirectory: cacheDirOption(),
|
||||||
|
});
|
||||||
const result = await lib.backup({
|
const result = await lib.backup({
|
||||||
downloadDirectory: dir,
|
downloadDirectory: dir,
|
||||||
onProgress: (msg) => {
|
onProgress: (msg) => {
|
||||||
if (!opts.json) stderr.write(msg + "\n");
|
if (!opts.json) stderr.write(msg + "\n");
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
lib.close();
|
|
||||||
|
|
||||||
if (opts.json) {
|
if (opts.json) {
|
||||||
stdout.write(JSON.stringify(result, null, 2) + "\n");
|
stdout.write(JSON.stringify(result, null, 2) + "\n");
|
||||||
@@ -348,7 +390,7 @@ program
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
process.exit(result.failed > 0 ? 1 : 0);
|
finish(lib, result.failed > 0 ? 1 : 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
const helper = program
|
const helper = program
|
||||||
@@ -362,7 +404,8 @@ helper
|
|||||||
.action(async (opts: { json?: boolean }) => {
|
.action(async (opts: { json?: boolean }) => {
|
||||||
await init();
|
await init();
|
||||||
const client = requireSession();
|
const client = requireSession();
|
||||||
const missing = await listMissingThumbnails(client, (msg) => {
|
const lib = await openReadLibrary(client);
|
||||||
|
const missing = await listMissingThumbnails(lib, client, (msg) => {
|
||||||
if (!opts.json) stderr.write(msg + "\n");
|
if (!opts.json) stderr.write(msg + "\n");
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -382,6 +425,7 @@ helper
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
finish(lib, 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
helper
|
helper
|
||||||
@@ -397,44 +441,61 @@ helper
|
|||||||
.action(async (opts: { file?: string[]; json?: boolean }) => {
|
.action(async (opts: { file?: string[]; json?: boolean }) => {
|
||||||
await init();
|
await init();
|
||||||
const client = requireSession();
|
const client = requireSession();
|
||||||
|
const lib = await openReadLibrary(client);
|
||||||
|
|
||||||
let fileIDs: number[];
|
let fileIDs: number[];
|
||||||
if (opts.file && opts.file.length > 0) {
|
if (opts.file && opts.file.length > 0) {
|
||||||
fileIDs = opts.file.map(Number).filter(Number.isFinite);
|
fileIDs = opts.file.map(Number).filter(Number.isFinite);
|
||||||
} else {
|
} else {
|
||||||
stderr.write("Scanning for missing thumbnails...\n");
|
stderr.write("Scanning for missing thumbnails...\n");
|
||||||
const missing = await listMissingThumbnails(client, (msg) => {
|
const missing = await listMissingThumbnails(lib, client, (msg) => {
|
||||||
if (!opts.json) stderr.write(msg + "\n");
|
if (!opts.json) stderr.write(msg + "\n");
|
||||||
});
|
});
|
||||||
fileIDs = missing.map((m) => m.fileID);
|
fileIDs = missing.map((m) => m.fileID);
|
||||||
if (fileIDs.length === 0) {
|
if (fileIDs.length === 0) {
|
||||||
stderr.write("No missing thumbnails found.\n");
|
stderr.write("No missing thumbnails found.\n");
|
||||||
|
finish(lib, 0);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
stderr.write(`Found ${fileIDs.length} file(s) to fix.\n`);
|
stderr.write(`Found ${fileIDs.length} file(s) to fix.\n`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(client, fileIDs, (msg) => {
|
const results = await fixMissingThumbnails(
|
||||||
if (!opts.json) stderr.write(msg + "\n");
|
lib,
|
||||||
});
|
client,
|
||||||
|
fileIDs,
|
||||||
|
(msg) => {
|
||||||
|
if (!opts.json) stderr.write(msg + "\n");
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
if (opts.json) {
|
if (opts.json) {
|
||||||
stdout.write(JSON.stringify(results, null, 2) + "\n");
|
stdout.write(JSON.stringify(results, null, 2) + "\n");
|
||||||
} else {
|
} else {
|
||||||
const ok = results.filter((r) => r.success).length;
|
const fixed = results.filter((r) => r.status === "fixed").length;
|
||||||
const fail = results.filter((r) => !r.success).length;
|
const skipped = results.filter(
|
||||||
|
(r) => r.status === "skipped",
|
||||||
|
).length;
|
||||||
|
const failed = results.filter((r) => r.status === "failed").length;
|
||||||
stderr.write(`\n--- Done ---\n`);
|
stderr.write(`\n--- Done ---\n`);
|
||||||
stderr.write(` Fixed: ${ok}\n`);
|
stderr.write(` Fixed: ${fixed}\n`);
|
||||||
stderr.write(` Failed: ${fail}\n`);
|
stderr.write(` Skipped: ${skipped}\n`);
|
||||||
if (fail > 0) {
|
stderr.write(` Failed: ${failed}\n`);
|
||||||
|
if (skipped > 0) {
|
||||||
|
stderr.write("\nSkipped (unsupported format):\n");
|
||||||
|
for (const r of results.filter((r) => r.status === "skipped")) {
|
||||||
|
stderr.write(` ${r.fileID}\t${r.title}\t${r.reason}\n`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (failed > 0) {
|
||||||
stderr.write("\nFailed files:\n");
|
stderr.write("\nFailed files:\n");
|
||||||
for (const r of results.filter((r) => !r.success)) {
|
for (const r of results.filter((r) => r.status === "failed")) {
|
||||||
stderr.write(` ${r.fileID}\t${r.title}\t${r.error}\n`);
|
stderr.write(` ${r.fileID}\t${r.title}\t${r.reason}\n`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
process.exit(results.some((r) => !r.success) ? 1 : 0);
|
finish(lib, results.some((r) => r.status === "failed") ? 1 : 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
await init();
|
await init();
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
// How the CLI presents a file's identity in `files`, `get`, and `get-thumb`.
|
||||||
|
//
|
||||||
|
// These read the file's own decrypted metadata — the raw title and the
|
||||||
|
// creationTime in microseconds — rather than the `PhotoRecord` projection the
|
||||||
|
// rest of the library exposes. The projection prefers `editedName`/`editedTime`
|
||||||
|
// and reports time in milliseconds, which is right for a photo browser but
|
||||||
|
// would change the CLI's externally-visible output. The pre-library CLI printed
|
||||||
|
// `metadata.title` and `metadata.creationTime` and named downloads after
|
||||||
|
// `metadata.title`, and issue #52 requires that output stay byte-identical, so
|
||||||
|
// the commands shape their output from the raw `EnteFile` through here.
|
||||||
|
|
||||||
|
import type { EnteFile, FileType, Microseconds } from "./model/types.js";
|
||||||
|
|
||||||
|
// One row of `quak files --json`.
|
||||||
|
export interface FileListRow {
|
||||||
|
id: number;
|
||||||
|
title: string;
|
||||||
|
fileType: FileType;
|
||||||
|
creationTime: Microseconds;
|
||||||
|
collectionID: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const fileListRow = (file: EnteFile): FileListRow => ({
|
||||||
|
id: file.id,
|
||||||
|
title: file.metadata.title,
|
||||||
|
fileType: file.metadata.fileType,
|
||||||
|
creationTime: file.metadata.creationTime,
|
||||||
|
collectionID: file.collectionID,
|
||||||
|
});
|
||||||
|
|
||||||
|
// One line of `quak files` in its human, tab-separated form.
|
||||||
|
export const fileListLine = (file: EnteFile): string =>
|
||||||
|
`${file.id}\t${file.metadata.fileType}\t${file.metadata.title}`;
|
||||||
|
|
||||||
|
// Default output path for `quak get` when `--out` is not given.
|
||||||
|
export const originalName = (file: EnteFile): string => file.metadata.title;
|
||||||
|
|
||||||
|
// Default output path for `quak get-thumb` when `--out` is not given.
|
||||||
|
export const thumbnailName = (file: EnteFile): string =>
|
||||||
|
`thumb_${file.metadata.title}`;
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
// How the CLI's read commands obtain current data.
|
||||||
|
//
|
||||||
|
// `collections`, `files --collection`, `get`, and `get-thumb` must answer for
|
||||||
|
// the account's state at the moment the command runs, not for whatever the
|
||||||
|
// local cache last happened to hold (owner amendment, issue #36). Each helper
|
||||||
|
// therefore forces a server round-trip through `Library.fresh()` and only then
|
||||||
|
// reads — so a collection, file, or metadata change made elsewhere is visible.
|
||||||
|
//
|
||||||
|
// `collections` and `files` also list in the library's own enumeration order —
|
||||||
|
// `listCollections()`/`listFiles()`, the order the pre-library CLI printed —
|
||||||
|
// rather than the `albums`/`photos` projection's newest-first order, which
|
||||||
|
// re-sorts the rows. The field values still come from each record's raw
|
||||||
|
// metadata via `cli-output.ts`.
|
||||||
|
|
||||||
|
import type { Collection, EnteFile } from "./model/types.js";
|
||||||
|
import type { Photo, PhotosAPI } from "./library/index.js";
|
||||||
|
|
||||||
|
// The slice of `Library` these helpers read. `Library` satisfies it
|
||||||
|
// structurally; a test can drive them with a stand-in that records the
|
||||||
|
// `fresh()` call and serves records in a known enumeration order.
|
||||||
|
export interface FreshReadLibrary {
|
||||||
|
fresh(): Promise<unknown>;
|
||||||
|
listCollections(): Collection[];
|
||||||
|
getCollection(id: number): Collection | undefined;
|
||||||
|
listFiles(collectionID: number): EnteFile[];
|
||||||
|
getFileByID(fileID: number): EnteFile | undefined;
|
||||||
|
photos: Pick<PhotosAPI, "byID">;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every live collection, current as of a forced refresh, in enumeration order.
|
||||||
|
export const freshCollections = async (
|
||||||
|
lib: FreshReadLibrary,
|
||||||
|
): Promise<Collection[]> => {
|
||||||
|
await lib.fresh();
|
||||||
|
return lib.listCollections();
|
||||||
|
};
|
||||||
|
|
||||||
|
// The files of one collection, current as of a forced refresh, in enumeration
|
||||||
|
// order. `undefined` (not an empty list) when the collection does not exist, so
|
||||||
|
// the caller can tell "no such collection" from "an empty collection".
|
||||||
|
export const freshFiles = async (
|
||||||
|
lib: FreshReadLibrary,
|
||||||
|
collectionID: number,
|
||||||
|
): Promise<EnteFile[] | undefined> => {
|
||||||
|
await lib.fresh();
|
||||||
|
if (!lib.getCollection(collectionID)) return undefined;
|
||||||
|
return lib.listFiles(collectionID);
|
||||||
|
};
|
||||||
|
|
||||||
|
// One file, current as of a forced refresh, resolved to both its content
|
||||||
|
// handle (`Photo`, for fetching bytes) and its raw record (`EnteFile`, for the
|
||||||
|
// default output name and field values). `undefined` when the file is unknown.
|
||||||
|
export const freshFile = async (
|
||||||
|
lib: FreshReadLibrary,
|
||||||
|
fileID: number,
|
||||||
|
): Promise<{ photo: Photo; file: EnteFile } | undefined> => {
|
||||||
|
await lib.fresh();
|
||||||
|
const photo = lib.photos.byID({ fileID });
|
||||||
|
const file = lib.getFileByID(fileID);
|
||||||
|
if (!photo || !file) return undefined;
|
||||||
|
return { photo, file };
|
||||||
|
};
|
||||||
+281
-21
@@ -23,8 +23,16 @@
|
|||||||
// decrypted length this layer has.
|
// decrypted length this layer has.
|
||||||
|
|
||||||
import { existsSync, statSync } from "node:fs";
|
import { existsSync, statSync } from "node:fs";
|
||||||
import { chmod, mkdir, readdir, rm, stat } from "node:fs/promises";
|
import {
|
||||||
import { extname, join } from "node:path";
|
chmod,
|
||||||
|
mkdir,
|
||||||
|
readdir,
|
||||||
|
rm,
|
||||||
|
stat,
|
||||||
|
statfs,
|
||||||
|
utimes,
|
||||||
|
} from "node:fs/promises";
|
||||||
|
import { dirname, extname, join } from "node:path";
|
||||||
|
|
||||||
import type { ApiClient } from "../api/client.js";
|
import type { ApiClient } from "../api/client.js";
|
||||||
import {
|
import {
|
||||||
@@ -39,12 +47,24 @@ const DIR_MODE = 0o700;
|
|||||||
const FILE_MODE = 0o600;
|
const FILE_MODE = 0o600;
|
||||||
const TEMP_PREFIX = ".quak-";
|
const TEMP_PREFIX = ".quak-";
|
||||||
const TEMP_SUFFIX = ".tmp";
|
const TEMP_SUFFIX = ".tmp";
|
||||||
|
const GIB = 1024 * 1024 * 1024;
|
||||||
|
// Owner ruling (#36): bound the originals cache at 100 GiB, but back off when
|
||||||
|
// the volume has under 50 GiB free so the cache never crowds the disk.
|
||||||
|
export const DEFAULT_ORIGINALS_MAX_BYTES = 100 * GIB;
|
||||||
|
export const DEFAULT_FREE_BELOW_BYTES = 50 * GIB;
|
||||||
// Ente thumbnails are always JPEG, so the cache stores them with a fixed
|
// Ente thumbnails are always JPEG, so the cache stores them with a fixed
|
||||||
// extension rather than deriving one from the (image or video) title.
|
// extension rather than deriving one from the (image or video) title.
|
||||||
const THUMBNAIL_EXT = ".jpg";
|
const THUMBNAIL_EXT = ".jpg";
|
||||||
|
|
||||||
type Kind = "original" | "thumbnail";
|
type Kind = "original" | "thumbnail";
|
||||||
|
|
||||||
|
// An original write in progress, with the IDs of the concurrent original writes
|
||||||
|
// it overlaps (recorded both ways as writes begin, cleared when the write ends).
|
||||||
|
interface OriginalWrite {
|
||||||
|
fileID: number;
|
||||||
|
overlaps: Set<number>;
|
||||||
|
}
|
||||||
|
|
||||||
// The priority a caller attaches to a thumbnail prefetch. The pool has two
|
// The priority a caller attaches to a thumbnail prefetch. The pool has two
|
||||||
// tiers, so this three-value surface collapses onto them: only a currently
|
// tiers, so this three-value surface collapses onto them: only a currently
|
||||||
// visible thumbnail preempts (on-demand); "ahead" prefetch and speculative
|
// visible thumbnail preempts (on-demand); "ahead" prefetch and speculative
|
||||||
@@ -133,6 +153,28 @@ export interface CachedPaths {
|
|||||||
thumbnailPath?: string;
|
thumbnailPath?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The slice of `fs.statfs` the eviction limit needs: `bavail` is the blocks
|
||||||
|
// available to an unprivileged writer and `bsize` their size, so
|
||||||
|
// `bavail * bsize` is the free byte count. Injectable so tests drive the
|
||||||
|
// adaptive limit without a real volume.
|
||||||
|
export interface StatFsResult {
|
||||||
|
bsize: number;
|
||||||
|
bavail: number;
|
||||||
|
}
|
||||||
|
export type StatFsFn = (path: string) => Promise<StatFsResult>;
|
||||||
|
|
||||||
|
const realStatFs: StatFsFn = async (path) => {
|
||||||
|
const s = await statfs(path);
|
||||||
|
return { bsize: s.bsize, bavail: s.bavail };
|
||||||
|
};
|
||||||
|
|
||||||
|
// The current usage and effective limit of the originals cache, in bytes.
|
||||||
|
// `limitBytes` is the adaptive ceiling last computed (see `originalsLimit`).
|
||||||
|
export interface OriginalsStatus {
|
||||||
|
usedBytes: number;
|
||||||
|
limitBytes?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface ContentCacheOptions {
|
export interface ContentCacheOptions {
|
||||||
pools: RequestPools;
|
pools: RequestPools;
|
||||||
source: ContentSource;
|
source: ContentSource;
|
||||||
@@ -144,6 +186,18 @@ export interface ContentCacheOptions {
|
|||||||
// Resolve any membership of a file; every membership shares the underlying
|
// Resolve any membership of a file; every membership shares the underlying
|
||||||
// content key, so any one decrypts the same bytes.
|
// content key, so any one decrypts the same bytes.
|
||||||
getFile: (fileID: number) => EnteFile | undefined;
|
getFile: (fileID: number) => EnteFile | undefined;
|
||||||
|
// Hard ceiling on `cacheDirectory/originals` (default 100 GiB) and the free
|
||||||
|
// space to protect on the volume (default 50 GiB). The effective limit is
|
||||||
|
// the lesser of the ceiling and what fits above the protected free space.
|
||||||
|
cacheOriginalsMaxBytes?: number;
|
||||||
|
freeBelowBytes?: number;
|
||||||
|
// Whether an original is pinned (favorites + latest week; the precache unit
|
||||||
|
// #48 supplies the set). Pinned originals are never evicted; when only
|
||||||
|
// pinned originals remain the cache runs over-limit until the set shrinks.
|
||||||
|
isPinned?: (fileID: number) => boolean;
|
||||||
|
// Free-space probe on the volume holding `cacheDirectory`; defaults to the
|
||||||
|
// real `fs.statfs`.
|
||||||
|
statfs?: StatFsFn;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Thrown inside a pooled task to drop a queued fetch that was aborted before it
|
// Thrown inside a pooled task to drop a queued fetch that was aborted before it
|
||||||
@@ -190,6 +244,24 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
// listing at open() and extended as fetches store new files.
|
// listing at open() and extended as fetches store new files.
|
||||||
private readonly originals = new Map<number, string>();
|
private readonly originals = new Map<number, string>();
|
||||||
private readonly thumbnails = new Map<number, string>();
|
private readonly thumbnails = new Map<number, string>();
|
||||||
|
private readonly maxOriginalsBytes: number;
|
||||||
|
private readonly freeBelowBytes: number;
|
||||||
|
private readonly isPinned: (fileID: number) => boolean;
|
||||||
|
private readonly statfs: StatFsFn;
|
||||||
|
// The last measured usage and effective limit, refreshed at open() and after
|
||||||
|
// every original write; exposed through `originalsStatus`.
|
||||||
|
private originalsUsedBytes = 0;
|
||||||
|
private originalsLimitBytes?: number;
|
||||||
|
// Serializes limit enforcement so concurrent original writes never race on
|
||||||
|
// the map or delete each other's just-freed room.
|
||||||
|
private enforcing: Promise<void> = Promise.resolve();
|
||||||
|
// Original writes in progress. Writes for different files run concurrently
|
||||||
|
// (the content pool), so an eviction pass must never delete a file whose
|
||||||
|
// fetch has not yet returned. Each entry records the IDs of the concurrent
|
||||||
|
// original writes it overlaps — noted both ways as writes begin — and a
|
||||||
|
// write's eviction pass spares them all. Bounded by the pool's concurrency,
|
||||||
|
// so eviction is never deferred beyond the active working set.
|
||||||
|
private readonly inFlightOriginals = new Set<OriginalWrite>();
|
||||||
|
|
||||||
constructor(opts: ContentCacheOptions) {
|
constructor(opts: ContentCacheOptions) {
|
||||||
this.pools = opts.pools;
|
this.pools = opts.pools;
|
||||||
@@ -198,6 +270,11 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
this.getFile = opts.getFile;
|
this.getFile = opts.getFile;
|
||||||
this.originalsDir = join(opts.cacheDirectory, "originals");
|
this.originalsDir = join(opts.cacheDirectory, "originals");
|
||||||
this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails");
|
this.thumbnailsDir = join(opts.cacheDirectory, "thumbnails");
|
||||||
|
this.maxOriginalsBytes =
|
||||||
|
opts.cacheOriginalsMaxBytes ?? DEFAULT_ORIGINALS_MAX_BYTES;
|
||||||
|
this.freeBelowBytes = opts.freeBelowBytes ?? DEFAULT_FREE_BELOW_BYTES;
|
||||||
|
this.isPinned = opts.isPinned ?? (() => false);
|
||||||
|
this.statfs = opts.statfs ?? realStatFs;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare the cache directories, reap orphan temp files, and take the
|
// Prepare the cache directories, reap orphan temp files, and take the
|
||||||
@@ -207,6 +284,18 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
await this.ensureDir(this.thumbnailsDir);
|
await this.ensureDir(this.thumbnailsDir);
|
||||||
await this.scan(this.originalsDir, this.originals);
|
await this.scan(this.originalsDir, this.originals);
|
||||||
await this.scan(this.thumbnailsDir, this.thumbnails);
|
await this.scan(this.thumbnailsDir, this.thumbnails);
|
||||||
|
// Publish the current usage and limit without evicting; a restart
|
||||||
|
// reuses whatever survived on disk. Eviction only ever fires on a write.
|
||||||
|
await this.refreshOriginalsLimit();
|
||||||
|
}
|
||||||
|
|
||||||
|
// The current originals usage and effective limit, both in bytes, as of the
|
||||||
|
// last write or open. `status().originalsLimitBytes` surfaces this.
|
||||||
|
originalsStatus(): OriginalsStatus {
|
||||||
|
return {
|
||||||
|
usedBytes: this.originalsUsedBytes,
|
||||||
|
limitBytes: this.originalsLimitBytes,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// The cache paths known for a file, for the record projection to expose as
|
// The cache paths known for a file, for the record projection to expose as
|
||||||
@@ -239,12 +328,45 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> {
|
async ensureThumbnails(args: EnsureOptions): Promise<EnsureResult[]> {
|
||||||
const priority = poolPriorityOf(args.priority);
|
return this.ensureMany(
|
||||||
|
"thumbnail",
|
||||||
|
poolPriorityOf(args.priority),
|
||||||
|
args.fileIDs,
|
||||||
|
args.signal,
|
||||||
|
args.onProgress,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fill originals through the content pool for the precache (#48), always at
|
||||||
|
// background priority so an on-demand `original()` preempts the fill. A
|
||||||
|
// present original is a map lookup and no fetch; a per-file failure is
|
||||||
|
// returned, not thrown, so one bad file never halts a background sweep.
|
||||||
|
async ensureOriginals(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
signal?: AbortSignal;
|
||||||
|
onProgress?: (event: EnsureEvent) => void;
|
||||||
|
}): Promise<EnsureResult[]> {
|
||||||
|
return this.ensureMany(
|
||||||
|
"original",
|
||||||
|
"background",
|
||||||
|
args.fileIDs,
|
||||||
|
args.signal,
|
||||||
|
args.onProgress,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async ensureMany(
|
||||||
|
kind: Kind,
|
||||||
|
priority: Priority,
|
||||||
|
fileIDs: number[],
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
onProgress: ((event: EnsureEvent) => void) | undefined,
|
||||||
|
): Promise<EnsureResult[]> {
|
||||||
// Dedup the request list so a repeated fileID is fetched once and
|
// Dedup the request list so a repeated fileID is fetched once and
|
||||||
// reported once, in first-requested order.
|
// reported once, in first-requested order.
|
||||||
const seen = new Set<number>();
|
const seen = new Set<number>();
|
||||||
const unique: number[] = [];
|
const unique: number[] = [];
|
||||||
for (const id of args.fileIDs) {
|
for (const id of fileIDs) {
|
||||||
if (!seen.has(id)) {
|
if (!seen.has(id)) {
|
||||||
seen.add(id);
|
seen.add(id);
|
||||||
unique.push(id);
|
unique.push(id);
|
||||||
@@ -252,24 +374,20 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
}
|
}
|
||||||
return Promise.all(
|
return Promise.all(
|
||||||
unique.map((fileID) =>
|
unique.map((fileID) =>
|
||||||
this.ensureOne(fileID, priority, args.signal, args.onProgress),
|
this.ensureOne(fileID, kind, priority, signal, onProgress),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async ensureOne(
|
private async ensureOne(
|
||||||
fileID: number,
|
fileID: number,
|
||||||
|
kind: Kind,
|
||||||
priority: Priority,
|
priority: Priority,
|
||||||
signal: AbortSignal | undefined,
|
signal: AbortSignal | undefined,
|
||||||
onProgress: ((event: EnsureEvent) => void) | undefined,
|
onProgress: ((event: EnsureEvent) => void) | undefined,
|
||||||
): Promise<EnsureResult> {
|
): Promise<EnsureResult> {
|
||||||
try {
|
try {
|
||||||
const result = await this.acquire(
|
const result = await this.acquire(fileID, kind, priority, signal);
|
||||||
fileID,
|
|
||||||
"thumbnail",
|
|
||||||
priority,
|
|
||||||
signal,
|
|
||||||
);
|
|
||||||
const status = result.cached ? "skipped" : "done";
|
const status = result.cached ? "skipped" : "done";
|
||||||
onProgress?.({ fileID, status, path: result.path });
|
onProgress?.({ fileID, status, path: result.path });
|
||||||
return { fileID, path: result.path };
|
return { fileID, path: result.path };
|
||||||
@@ -321,8 +439,16 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
const cached = known.get(fileID);
|
const cached = known.get(fileID);
|
||||||
if (cached !== undefined) {
|
if (cached !== undefined) {
|
||||||
const size = fileSize(cached);
|
const size = fileSize(cached);
|
||||||
if (size !== undefined && size > 0)
|
if (size !== undefined && size > 0) {
|
||||||
|
// Returning an original's path is a use: bump its mtime so LRU
|
||||||
|
// order reflects it and survives a restart with no ledger.
|
||||||
|
if (
|
||||||
|
kind === "original" &&
|
||||||
|
dirname(cached) === this.originalsDir
|
||||||
|
)
|
||||||
|
await this.touch(cached);
|
||||||
return { path: cached, bytes: size, cached: true };
|
return { path: cached, bytes: size, cached: true };
|
||||||
|
}
|
||||||
// A recorded file that has since gone re-fetches below.
|
// A recorded file that has since gone re-fetches below.
|
||||||
known.delete(fileID);
|
known.delete(fileID);
|
||||||
}
|
}
|
||||||
@@ -358,16 +484,32 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
// to completion.
|
// to completion.
|
||||||
if (signal?.aborted) throw new AbortDrop();
|
if (signal?.aborted) throw new AbortDrop();
|
||||||
|
|
||||||
await this.download(file, dest, kind, opts?.onByte);
|
// Register this original among those in flight, linking it with
|
||||||
await chmod(dest, FILE_MODE);
|
// every sibling already writing so neither evicts the other's
|
||||||
const size = (await stat(dest)).size;
|
// file. Non-null iff this is an original.
|
||||||
if (size === 0) {
|
const write =
|
||||||
throw new Error(
|
kind === "original"
|
||||||
`content cache: ${kind} ${fileID} stored empty`,
|
? this.beginOriginalWrite(fileID)
|
||||||
);
|
: null;
|
||||||
|
try {
|
||||||
|
await this.download(file, dest, kind, opts?.onByte);
|
||||||
|
await chmod(dest, FILE_MODE);
|
||||||
|
const size = (await stat(dest)).size;
|
||||||
|
if (size === 0) {
|
||||||
|
throw new Error(
|
||||||
|
`content cache: ${kind} ${fileID} stored empty`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
known.set(fileID, dest);
|
||||||
|
// A fresh original may have crossed the limit; make room by
|
||||||
|
// evicting least-recently-used originals. An over-budget
|
||||||
|
// fetch keeps the file it returns, and no overlapping
|
||||||
|
// sibling is evicted. Thumbnails are never bounded.
|
||||||
|
if (write) await this.enforceOriginalsLimit(write);
|
||||||
|
return { path: dest, bytes: size, cached: false };
|
||||||
|
} finally {
|
||||||
|
if (write) this.inFlightOriginals.delete(write);
|
||||||
}
|
}
|
||||||
known.set(fileID, dest);
|
|
||||||
return { path: dest, bytes: size, cached: false };
|
|
||||||
},
|
},
|
||||||
{ priority, key: fileID },
|
{ priority, key: fileID },
|
||||||
);
|
);
|
||||||
@@ -387,6 +529,124 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
|||||||
return result.bytesWritten;
|
return result.bytesWritten;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Best-effort bump of a file's mtime to now; a failed touch must never fail
|
||||||
|
// the read it accompanies.
|
||||||
|
private async touch(path: string): Promise<void> {
|
||||||
|
const now = new Date();
|
||||||
|
await utimes(path, now, now).catch(() => undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every stored original that lives under `originalsDir` (a backup-directory
|
||||||
|
// hit recorded in the map is excluded), with its size and mtime. Entries
|
||||||
|
// whose file has vanished are dropped from the map. Backups and thumbnails
|
||||||
|
// are never counted.
|
||||||
|
private async measureOriginals(): Promise<{
|
||||||
|
entries: {
|
||||||
|
fileID: number;
|
||||||
|
path: string;
|
||||||
|
size: number;
|
||||||
|
mtimeMs: number;
|
||||||
|
}[];
|
||||||
|
used: number;
|
||||||
|
}> {
|
||||||
|
const entries: {
|
||||||
|
fileID: number;
|
||||||
|
path: string;
|
||||||
|
size: number;
|
||||||
|
mtimeMs: number;
|
||||||
|
}[] = [];
|
||||||
|
let used = 0;
|
||||||
|
for (const [fileID, path] of this.originals) {
|
||||||
|
if (dirname(path) !== this.originalsDir) continue;
|
||||||
|
try {
|
||||||
|
const s = await stat(path);
|
||||||
|
entries.push({
|
||||||
|
fileID,
|
||||||
|
path,
|
||||||
|
size: s.size,
|
||||||
|
mtimeMs: s.mtimeMs,
|
||||||
|
});
|
||||||
|
used += s.size;
|
||||||
|
} catch {
|
||||||
|
this.originals.delete(fileID);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { entries, used };
|
||||||
|
}
|
||||||
|
|
||||||
|
// The effective ceiling on originals: the configured max, but no more than
|
||||||
|
// what fits once the protected free space is set aside. `used + free` is the
|
||||||
|
// volume space the cache could occupy; subtracting `freeBelowBytes` leaves
|
||||||
|
// the reserve untouched. Clamped at zero.
|
||||||
|
private async originalsLimit(used: number): Promise<number> {
|
||||||
|
const { bsize, bavail } = await this.statfs(this.originalsDir);
|
||||||
|
const free = bsize * bavail;
|
||||||
|
const adaptive = used + free - this.freeBelowBytes;
|
||||||
|
return Math.max(0, Math.min(this.maxOriginalsBytes, adaptive));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recompute and publish usage and limit without evicting (used at open()).
|
||||||
|
private refreshOriginalsLimit(): Promise<void> {
|
||||||
|
return this.serializeEnforce(async () => {
|
||||||
|
const { used } = await this.measureOriginals();
|
||||||
|
this.originalsUsedBytes = used;
|
||||||
|
this.originalsLimitBytes = await this.originalsLimit(used);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record a starting original write among those in flight, linking it with
|
||||||
|
// every sibling already writing so neither can evict the other's file.
|
||||||
|
private beginOriginalWrite(fileID: number): OriginalWrite {
|
||||||
|
const write: OriginalWrite = { fileID, overlaps: new Set() };
|
||||||
|
for (const other of this.inFlightOriginals) {
|
||||||
|
write.overlaps.add(other.fileID);
|
||||||
|
other.overlaps.add(fileID);
|
||||||
|
}
|
||||||
|
this.inFlightOriginals.add(write);
|
||||||
|
return write;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Evict least-recently-used originals until usage fits the limit. Skipped:
|
||||||
|
// pinned originals, the file `write` just stored, and every original whose
|
||||||
|
// write overlaps it (`write.overlaps`). The last two spare any fetch whose
|
||||||
|
// lifetime overlaps this one, so concurrent over-budget fetches all keep the
|
||||||
|
// paths they return; when only such originals remain the cache stays
|
||||||
|
// over-limit until they settle, and a later, non-overlapping write finds
|
||||||
|
// them eligible again.
|
||||||
|
private enforceOriginalsLimit(write: OriginalWrite): Promise<void> {
|
||||||
|
return this.serializeEnforce(async () => {
|
||||||
|
const { entries, used } = await this.measureOriginals();
|
||||||
|
const limit = await this.originalsLimit(used);
|
||||||
|
let remaining = used;
|
||||||
|
if (remaining > limit) {
|
||||||
|
const evictable = entries
|
||||||
|
.filter(
|
||||||
|
(e) =>
|
||||||
|
e.fileID !== write.fileID &&
|
||||||
|
!write.overlaps.has(e.fileID) &&
|
||||||
|
!this.isPinned(e.fileID),
|
||||||
|
)
|
||||||
|
.sort((a, b) => a.mtimeMs - b.mtimeMs);
|
||||||
|
for (const e of evictable) {
|
||||||
|
if (remaining <= limit) break;
|
||||||
|
await rm(e.path, { force: true });
|
||||||
|
this.originals.delete(e.fileID);
|
||||||
|
remaining -= e.size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.originalsUsedBytes = remaining;
|
||||||
|
this.originalsLimitBytes = limit;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run limit work one at a time; failures are swallowed so a transient
|
||||||
|
// statfs or unlink error never rejects the read or write that triggered it.
|
||||||
|
private serializeEnforce(work: () => Promise<void>): Promise<void> {
|
||||||
|
const next = this.enforcing.then(work).catch(() => undefined);
|
||||||
|
this.enforcing = next;
|
||||||
|
return next;
|
||||||
|
}
|
||||||
|
|
||||||
private async ensureDir(dir: string): Promise<void> {
|
private async ensureDir(dir: string): Promise<void> {
|
||||||
// chmod after mkdir so the mode is tightened even when the directory
|
// chmod after mkdir so the mode is tightened even when the directory
|
||||||
// already existed with a looser one; mkdir alone would not.
|
// already existed with a looser one; mkdir alone would not.
|
||||||
|
|||||||
+195
-27
@@ -6,10 +6,17 @@
|
|||||||
// existing copy loaded, that first refresh runs in the background and `open()`
|
// existing copy loaded, that first refresh runs in the background and `open()`
|
||||||
// returns as soon as the cached data is ready to serve — a slow or unreachable
|
// returns as soon as the cached data is ready to serve — a slow or unreachable
|
||||||
// server no longer stalls opening. A background timer then refreshes every
|
// server no longer stalls opening. A background timer then refreshes every
|
||||||
// `refreshIntervalSeconds`. Every read is answered from RAM — no read touches
|
// `refreshIntervalSeconds`. Every default read is answered from RAM — no
|
||||||
// the network. There is deliberately no `sync()`, no `refresh()`, no
|
// default read touches the network. There is deliberately no `sync()`, no
|
||||||
// `serverReachable` flag, and no "before each read" mode (design #36): the
|
// `refresh()`, no `serverReachable` flag, and no "before each read" mode
|
||||||
// only ways state changes are the refreshes above.
|
// (design #36).
|
||||||
|
//
|
||||||
|
// `fresh()` is the one exception (issue #75, an owner amendment to #36): it
|
||||||
|
// forces a refresh, awaits it, and only then hands back the read namespaces, so
|
||||||
|
// a caller that needs server-current data can ask for it. Concurrent `fresh()`
|
||||||
|
// calls coalesce onto one in-flight refresh, and a refresh that fails rejects
|
||||||
|
// the caller (the default reads stay silent and serve the last good copy). The
|
||||||
|
// default methods and the background loop are unchanged.
|
||||||
//
|
//
|
||||||
// A refresh stages all of its network work first and only mutates the store
|
// A refresh stages all of its network work first and only mutates the store
|
||||||
// once every fetch has succeeded. A refresh that fails partway therefore never
|
// once every fetch has succeeded. A refresh that fails partway therefore never
|
||||||
@@ -40,6 +47,7 @@ import {
|
|||||||
type AlbumsAPI,
|
type AlbumsAPI,
|
||||||
type PhotosAPI,
|
type PhotosAPI,
|
||||||
type TimelineAPI,
|
type TimelineAPI,
|
||||||
|
type FreshReads,
|
||||||
} from "./read.js";
|
} from "./read.js";
|
||||||
import {
|
import {
|
||||||
ContentCache,
|
ContentCache,
|
||||||
@@ -48,6 +56,8 @@ import {
|
|||||||
type EnsureOptions,
|
type EnsureOptions,
|
||||||
type EnsureResult,
|
type EnsureResult,
|
||||||
} from "./content.js";
|
} from "./content.js";
|
||||||
|
import { makeMLDataAPI, type MLDataAPI } from "./mlsearch.js";
|
||||||
|
import { Precache } from "./precache.js";
|
||||||
|
|
||||||
export {
|
export {
|
||||||
Album,
|
Album,
|
||||||
@@ -55,6 +65,7 @@ export {
|
|||||||
type AlbumsAPI,
|
type AlbumsAPI,
|
||||||
type PhotosAPI,
|
type PhotosAPI,
|
||||||
type TimelineAPI,
|
type TimelineAPI,
|
||||||
|
type FreshReads,
|
||||||
type PhotoFilter,
|
type PhotoFilter,
|
||||||
type TimelineGroup,
|
type TimelineGroup,
|
||||||
type GroupBy,
|
type GroupBy,
|
||||||
@@ -71,6 +82,7 @@ export {
|
|||||||
type EnsureResult,
|
type EnsureResult,
|
||||||
type EnsureEvent,
|
type EnsureEvent,
|
||||||
} from "./content.js";
|
} from "./content.js";
|
||||||
|
export { type MLDataAPI, type SimilarResult } from "./mlsearch.js";
|
||||||
import type { CollectionsPage, FilesPage } from "../client.js";
|
import type { CollectionsPage, FilesPage } from "../client.js";
|
||||||
import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js";
|
import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js";
|
||||||
import type { Collection, EnteFile } from "../model/types.js";
|
import type { Collection, EnteFile } from "../model/types.js";
|
||||||
@@ -85,6 +97,23 @@ export {
|
|||||||
|
|
||||||
export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3;
|
export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3;
|
||||||
|
|
||||||
|
// Project a metadata store into by-id records, filling each record's cache
|
||||||
|
// paths from the content cache when one is given. Shared by the live read
|
||||||
|
// projection and the precache's initial seeding at open().
|
||||||
|
const deriveRecordsFromStore = (
|
||||||
|
store: MetadataStore,
|
||||||
|
cache?: ContentCache,
|
||||||
|
): DerivedRecords => {
|
||||||
|
const collections = store.listCollections();
|
||||||
|
const files: EnteFile[] = [];
|
||||||
|
for (const c of collections) files.push(...store.listFiles(c.id));
|
||||||
|
return deriveRecords(
|
||||||
|
collections,
|
||||||
|
files,
|
||||||
|
cache ? (fileID) => cache.pathsFor(fileID) : undefined,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
// The slice of `Client` the library depends on. Narrowing to an interface lets
|
// The slice of `Client` the library depends on. Narrowing to an interface lets
|
||||||
// tests drive a mock with no crypto or network; the real `Client` satisfies it
|
// tests drive a mock with no crypto or network; the real `Client` satisfies it
|
||||||
// structurally.
|
// structurally.
|
||||||
@@ -113,9 +142,15 @@ export interface LibraryClient {
|
|||||||
// A progress event for one unit of background work. A metadata "refresh" or an
|
// A progress event for one unit of background work. A metadata "refresh" or an
|
||||||
// ML "fetchMLData" pass each fire "started" before their network work and then
|
// ML "fetchMLData" pass each fire "started" before their network work and then
|
||||||
// exactly one of "done" or "failed"; "failed" carries the error message and
|
// exactly one of "done" or "failed"; "failed" carries the error message and
|
||||||
// an ML "done" reports how many payloads it stored.
|
// an ML "done" reports how many payloads it stored. The precache fills
|
||||||
|
// ("precacheThumbnails"/"precacheOriginals", #48) fire "started"/"done" around
|
||||||
|
// each sweep that has work, "done" reporting the count newly cached.
|
||||||
export interface RefreshEvent {
|
export interface RefreshEvent {
|
||||||
operation: "refresh" | "fetchMLData";
|
operation:
|
||||||
|
| "refresh"
|
||||||
|
| "fetchMLData"
|
||||||
|
| "precacheThumbnails"
|
||||||
|
| "precacheOriginals";
|
||||||
status: "started" | "done" | "failed";
|
status: "started" | "done" | "failed";
|
||||||
error?: string;
|
error?: string;
|
||||||
fetched?: number;
|
fetched?: number;
|
||||||
@@ -140,6 +175,22 @@ export interface LibraryOptions {
|
|||||||
// Overrides the client's own `contentSource()`; mainly for tests that drive
|
// Overrides the client's own `contentSource()`; mainly for tests that drive
|
||||||
// the cache with a stand-in source.
|
// the cache with a stand-in source.
|
||||||
contentSource?: ContentSource;
|
contentSource?: ContentSource;
|
||||||
|
// Bound on `cacheDirectory/originals` (default 100 GiB) and the free space
|
||||||
|
// to protect on its volume (default 50 GiB). The effective limit adapts
|
||||||
|
// down as the disk fills; `status().originalsLimitBytes` reports it.
|
||||||
|
cacheOriginalsMaxBytes?: number;
|
||||||
|
freeBelowBytes?: number;
|
||||||
|
// An extra pinned predicate OR-ed with the precache's own pinned set
|
||||||
|
// (favorites + latest week, #48). Pinned originals are never evicted.
|
||||||
|
isOriginalPinned?: (fileID: number) => boolean;
|
||||||
|
// The aggressive local precache (#48), all starting inside `open()` with no
|
||||||
|
// caller input. Thumbnails: every file, newest first, until all are on
|
||||||
|
// disk. Originals: the favorites album then the latest `precacheOriginalsDays`
|
||||||
|
// window (the days ending at the newest file). Both default on; the days
|
||||||
|
// default to 7.
|
||||||
|
precacheThumbnails?: boolean;
|
||||||
|
precacheOriginals?: boolean;
|
||||||
|
precacheOriginalsDays?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface LibraryStatus {
|
export interface LibraryStatus {
|
||||||
@@ -161,6 +212,17 @@ export interface LibraryStatus {
|
|||||||
// when ML fetching is disabled.
|
// when ML fetching is disabled.
|
||||||
mlStored?: number;
|
mlStored?: number;
|
||||||
mlIndexed?: number;
|
mlIndexed?: number;
|
||||||
|
// Bytes stored in the originals cache and the effective size limit as of the
|
||||||
|
// last write or open; undefined when no content cache is open.
|
||||||
|
originalsUsedBytes?: number;
|
||||||
|
originalsLimitBytes?: number;
|
||||||
|
// Precache progress (#48); undefined when no content cache is open. Totals
|
||||||
|
// are the files targeted (0 when a fill is disabled); "cached" is how many
|
||||||
|
// of them are on disk.
|
||||||
|
thumbnailsCached?: number;
|
||||||
|
thumbnailsTotal?: number;
|
||||||
|
originalsCached?: number;
|
||||||
|
originalsPinned?: number;
|
||||||
closed: boolean;
|
closed: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -177,6 +239,10 @@ export class Library {
|
|||||||
// The thumbnail-prefetch surface (issue #46): drives the thumbnail pool
|
// The thumbnail-prefetch surface (issue #46): drives the thumbnail pool
|
||||||
// with priority, dedup, and abort.
|
// with priority, dedup, and abort.
|
||||||
readonly thumbnails: ThumbnailsAPI;
|
readonly thumbnails: ThumbnailsAPI;
|
||||||
|
// The content-similarity search surface over the CLIP index (issue #50).
|
||||||
|
// Present whether or not ML fetching is enabled; with no ML store it
|
||||||
|
// returns empty results.
|
||||||
|
readonly mldata: MLDataAPI;
|
||||||
|
|
||||||
private readonly client: LibraryClient;
|
private readonly client: LibraryClient;
|
||||||
private readonly store: MetadataStore;
|
private readonly store: MetadataStore;
|
||||||
@@ -188,10 +254,17 @@ export class Library {
|
|||||||
private readonly onProgress?: RefreshProgressCallback;
|
private readonly onProgress?: RefreshProgressCallback;
|
||||||
private readonly pools: RequestPools;
|
private readonly pools: RequestPools;
|
||||||
// The ML-data cache, present only when the client can fetch ML data.
|
// The ML-data cache, present only when the client can fetch ML data.
|
||||||
private readonly mldata?: MLDataStore;
|
private readonly mlStore?: MLDataStore;
|
||||||
|
// The local precache (#48), present only when the content cache is.
|
||||||
|
private readonly precache?: Precache;
|
||||||
|
|
||||||
private timer?: ReturnType<typeof setTimeout>;
|
private timer?: ReturnType<typeof setTimeout>;
|
||||||
private refreshing = false;
|
// The in-flight refresh cycle, or undefined when none runs. One slot serves
|
||||||
|
// both paths: the background loop skips when it is set, and a fresh read
|
||||||
|
// (issue #75) coalesces onto it or starts one. The promise carries the
|
||||||
|
// cycle's real outcome (it rejects on failure); the background loop ignores
|
||||||
|
// that, a fresh read propagates it.
|
||||||
|
private cycle?: Promise<void>;
|
||||||
// Guards the ML fetch pass so a slow backfill never runs twice at once; a
|
// Guards the ML fetch pass so a slow backfill never runs twice at once; a
|
||||||
// refresh whose pass is still running kicks nothing new.
|
// refresh whose pass is still running kicks nothing new.
|
||||||
private mlFetching = false;
|
private mlFetching = false;
|
||||||
@@ -222,6 +295,7 @@ export class Library {
|
|||||||
pools: RequestPools;
|
pools: RequestPools;
|
||||||
mldata?: MLDataStore;
|
mldata?: MLDataStore;
|
||||||
cache?: ContentCache;
|
cache?: ContentCache;
|
||||||
|
precache?: Precache;
|
||||||
}) {
|
}) {
|
||||||
this.client = args.client;
|
this.client = args.client;
|
||||||
this.store = args.store;
|
this.store = args.store;
|
||||||
@@ -231,8 +305,9 @@ export class Library {
|
|||||||
this.intervalMs = args.intervalMs;
|
this.intervalMs = args.intervalMs;
|
||||||
this.onProgress = args.onProgress;
|
this.onProgress = args.onProgress;
|
||||||
this.pools = args.pools;
|
this.pools = args.pools;
|
||||||
this.mldata = args.mldata;
|
this.mlStore = args.mldata;
|
||||||
this.cache = args.cache;
|
this.cache = args.cache;
|
||||||
|
this.precache = args.precache;
|
||||||
this.lastRecords = this.deriveNow();
|
this.lastRecords = this.deriveNow();
|
||||||
|
|
||||||
// The read namespaces derive fresh from the store on each call, so they
|
// The read namespaces derive fresh from the store on each call, so they
|
||||||
@@ -253,6 +328,8 @@ export class Library {
|
|||||||
return this.cache.ensureThumbnails(opts);
|
return this.cache.ensureThumbnails(opts);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
// Reads the ML store live so results grow as ML data is fetched.
|
||||||
|
this.mldata = makeMLDataAPI(() => this.mlStore);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load the cache and start the refresh loop. With an empty cache the first
|
// Load the cache and start the refresh loop. With an empty cache the first
|
||||||
@@ -290,15 +367,35 @@ export class Library {
|
|||||||
// the start and the first refresh raises no spurious path-change diff.
|
// the start and the first refresh raises no spurious path-change diff.
|
||||||
const source = opts.contentSource ?? opts.client.contentSource?.();
|
const source = opts.contentSource ?? opts.client.contentSource?.();
|
||||||
let cache: ContentCache | undefined;
|
let cache: ContentCache | undefined;
|
||||||
|
let precache: Precache | undefined;
|
||||||
if (source) {
|
if (source) {
|
||||||
|
// The precache owns the pinned set (favorites + latest week), which
|
||||||
|
// is the cache's eviction predicate. It is built and seeded from
|
||||||
|
// the loaded store first so the cache can wire `isPinned` to it, and
|
||||||
|
// then bound to the cache it fills. A caller-supplied predicate is
|
||||||
|
// OR-ed in so both survive.
|
||||||
|
precache = new Precache({
|
||||||
|
thumbnails: opts.precacheThumbnails,
|
||||||
|
originals: opts.precacheOriginals,
|
||||||
|
originalsDays: opts.precacheOriginalsDays,
|
||||||
|
onEvent: opts.onProgress,
|
||||||
|
});
|
||||||
|
precache.update(deriveRecordsFromStore(store));
|
||||||
|
const extraPinned = opts.isOriginalPinned;
|
||||||
cache = new ContentCache({
|
cache = new ContentCache({
|
||||||
pools,
|
pools,
|
||||||
source,
|
source,
|
||||||
cacheDirectory,
|
cacheDirectory,
|
||||||
downloadDirectory: opts.downloadDirectory,
|
downloadDirectory: opts.downloadDirectory,
|
||||||
getFile: (fileID) => store.getFileByID(fileID),
|
getFile: (fileID) => store.getFileByID(fileID),
|
||||||
|
cacheOriginalsMaxBytes: opts.cacheOriginalsMaxBytes,
|
||||||
|
freeBelowBytes: opts.freeBelowBytes,
|
||||||
|
isPinned: (fileID) =>
|
||||||
|
precache!.isPinned(fileID) ||
|
||||||
|
(extraPinned?.(fileID) ?? false),
|
||||||
});
|
});
|
||||||
await cache.open();
|
await cache.open();
|
||||||
|
precache.bind(cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
const lib = new Library({
|
const lib = new Library({
|
||||||
@@ -312,8 +409,15 @@ export class Library {
|
|||||||
pools,
|
pools,
|
||||||
mldata,
|
mldata,
|
||||||
cache,
|
cache,
|
||||||
|
precache,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Start filling from whatever the loaded store already holds; each
|
||||||
|
// refresh below re-kicks with the new files (and retries any that
|
||||||
|
// failed). An empty store starts empty here and fills after its first
|
||||||
|
// refresh.
|
||||||
|
precache?.start();
|
||||||
|
|
||||||
if (store.loadedFromDisk) {
|
if (store.loadedFromDisk) {
|
||||||
// An existing copy already answers reads; refresh in the background
|
// An existing copy already answers reads; refresh in the background
|
||||||
// and start the interval once that first cycle settles.
|
// and start the interval once that first cycle settles.
|
||||||
@@ -343,6 +447,13 @@ export class Library {
|
|||||||
return this.store.getFile(collectionID, fileID);
|
return this.store.getFile(collectionID, fileID);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Any membership of a file, addressed by file id alone. A file's own
|
||||||
|
// metadata (title, creationTime) is identical across the collections it
|
||||||
|
// belongs to, so this serves the point commands that hold only a fileID.
|
||||||
|
getFileByID(fileID: number): EnteFile | undefined {
|
||||||
|
return this.store.getFileByID(fileID);
|
||||||
|
}
|
||||||
|
|
||||||
// A synchronous, RAM-only projection of the whole library into plain
|
// A synchronous, RAM-only projection of the whole library into plain
|
||||||
// records (no keys), the surface the GUI reads across IPC. Photos are
|
// records (no keys), the surface the GUI reads across IPC. Photos are
|
||||||
// deduplicated to one record per file and ordered newest first.
|
// deduplicated to one record per file and ordered newest first.
|
||||||
@@ -371,7 +482,9 @@ export class Library {
|
|||||||
for (const c of collections) {
|
for (const c of collections) {
|
||||||
files += this.store.listFiles(c.id).length;
|
files += this.store.listFiles(c.id).length;
|
||||||
}
|
}
|
||||||
const ml = this.mldata?.stats();
|
const ml = this.mlStore?.stats();
|
||||||
|
const originals = this.cache?.originalsStatus();
|
||||||
|
const pre = this.precache?.status();
|
||||||
return {
|
return {
|
||||||
userID: this.store.userID,
|
userID: this.store.userID,
|
||||||
collections: collections.length,
|
collections: collections.length,
|
||||||
@@ -382,10 +495,31 @@ export class Library {
|
|||||||
lastMLError: this.lastMLError,
|
lastMLError: this.lastMLError,
|
||||||
mlStored: ml?.stored,
|
mlStored: ml?.stored,
|
||||||
mlIndexed: ml?.indexed,
|
mlIndexed: ml?.indexed,
|
||||||
|
originalsUsedBytes: originals?.usedBytes,
|
||||||
|
originalsLimitBytes: originals?.limitBytes,
|
||||||
|
thumbnailsCached: pre?.thumbnailsCached,
|
||||||
|
thumbnailsTotal: pre?.thumbnailsTotal,
|
||||||
|
originalsCached: pre?.originalsCached,
|
||||||
|
originalsPinned: pre?.originalsPinned,
|
||||||
closed: this.closed,
|
closed: this.closed,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fresh reads (issue #75, owner amendment to design #36). Force a refresh,
|
||||||
|
// wait for it to complete and persist, then hand back the same
|
||||||
|
// `albums`/`photos`/`timeline` namespaces — now guaranteed to reflect a
|
||||||
|
// completed server round-trip. Concurrent calls coalesce onto one refresh;
|
||||||
|
// a refresh that fails rejects here, where the default namespaces would
|
||||||
|
// instead stay silent and serve the last good copy.
|
||||||
|
async fresh(): Promise<FreshReads> {
|
||||||
|
await this.refreshNow();
|
||||||
|
return {
|
||||||
|
albums: this.albums,
|
||||||
|
photos: this.photos,
|
||||||
|
timeline: this.timeline,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
// Back up every in-scope file to `downloadDirectory` in the historical
|
// Back up every in-scope file to `downloadDirectory` in the historical
|
||||||
// on-disk layout, with a durable failure ledger (issue #51). Refreshes
|
// on-disk layout, with a durable failure ledger (issue #51). Refreshes
|
||||||
// first, fetches pending originals (and optional thumbnails) through the
|
// first, fetches pending originals (and optional thumbnails) through the
|
||||||
@@ -429,6 +563,7 @@ export class Library {
|
|||||||
// finish; it will not schedule another cycle once closed.
|
// finish; it will not schedule another cycle once closed.
|
||||||
close(): void {
|
close(): void {
|
||||||
this.closed = true;
|
this.closed = true;
|
||||||
|
this.precache?.close();
|
||||||
if (this.timer !== undefined) {
|
if (this.timer !== undefined) {
|
||||||
clearTimeout(this.timer);
|
clearTimeout(this.timer);
|
||||||
this.timer = undefined;
|
this.timer = undefined;
|
||||||
@@ -444,11 +579,48 @@ export class Library {
|
|||||||
this.timer.unref?.();
|
this.timer.unref?.();
|
||||||
}
|
}
|
||||||
|
|
||||||
// One refresh cycle, guarded so a failure never escapes and overlapping
|
// The background loop's refresh: run a cycle unless one is already in flight
|
||||||
// cycles never run. Errors are reported, not thrown.
|
// (or the library is closed), and never let a failure escape — the
|
||||||
private async runRefresh(): Promise<void> {
|
// background path reports errors through `status()`/`onProgress`, it does
|
||||||
if (this.closed || this.refreshing) return;
|
// not throw. Resolves once the cycle it started (or skipped past) settles.
|
||||||
this.refreshing = true;
|
private runRefresh(): Promise<void> {
|
||||||
|
if (this.closed || this.cycle) return Promise.resolve();
|
||||||
|
return this.startCycle().catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
// A fresh read's refresh (issue #75): force a cycle and await it, rejecting
|
||||||
|
// if it fails. Concurrent fresh reads coalesce onto the one in-flight cycle
|
||||||
|
// — the background loop's included — so they never fan out into redundant
|
||||||
|
// server round-trips.
|
||||||
|
private refreshNow(): Promise<void> {
|
||||||
|
if (this.closed) {
|
||||||
|
return Promise.reject(new Error("the library is closed"));
|
||||||
|
}
|
||||||
|
return this.cycle ?? this.startCycle();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start one refresh cycle and record it as the in-flight cycle so every
|
||||||
|
// caller coalesces onto it. The returned promise carries the cycle's real
|
||||||
|
// outcome; each caller attaches the handling its own path needs, and the
|
||||||
|
// slot is cleared once the cycle settles.
|
||||||
|
private startCycle(): Promise<void> {
|
||||||
|
const cycle = this.refreshCycle();
|
||||||
|
this.cycle = cycle;
|
||||||
|
void cycle.then(
|
||||||
|
() => {
|
||||||
|
if (this.cycle === cycle) this.cycle = undefined;
|
||||||
|
},
|
||||||
|
() => {
|
||||||
|
if (this.cycle === cycle) this.cycle = undefined;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return cycle;
|
||||||
|
}
|
||||||
|
|
||||||
|
// One refresh cycle: the network fetch and commit, wrapped in the progress
|
||||||
|
// events and status bookkeeping. Throws when the refresh fails so a fresh
|
||||||
|
// read can reject; `runRefresh` swallows that throw for the background loop.
|
||||||
|
private async refreshCycle(): Promise<void> {
|
||||||
this.emit({ operation: "refresh", status: "started" });
|
this.emit({ operation: "refresh", status: "started" });
|
||||||
try {
|
try {
|
||||||
await this.refreshOnce();
|
await this.refreshOnce();
|
||||||
@@ -464,8 +636,7 @@ export class Library {
|
|||||||
const error = err instanceof Error ? err.message : String(err);
|
const error = err instanceof Error ? err.message : String(err);
|
||||||
this.lastError = error;
|
this.lastError = error;
|
||||||
this.emit({ operation: "refresh", status: "failed", error });
|
this.emit({ operation: "refresh", status: "failed", error });
|
||||||
} finally {
|
throw err;
|
||||||
this.refreshing = false;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -546,7 +717,12 @@ export class Library {
|
|||||||
if (change) this.notify(change);
|
if (change) this.notify(change);
|
||||||
}
|
}
|
||||||
this.lastRecords = next;
|
this.lastRecords = next;
|
||||||
|
// Recompute the fill orders and pinned set against the new library.
|
||||||
|
this.precache?.update(next);
|
||||||
}
|
}
|
||||||
|
// Re-kick the fills every cycle: a finished sweep starts afresh to pick
|
||||||
|
// up new files and retry any that failed, and a running one is left be.
|
||||||
|
this.precache?.start();
|
||||||
|
|
||||||
// Persist whenever RAM holds changes disk has not accepted — including
|
// Persist whenever RAM holds changes disk has not accepted — including
|
||||||
// changes an earlier cycle staged whose save failed. `unsaved` clears
|
// changes an earlier cycle staged whose save failed. `unsaved` clears
|
||||||
@@ -565,7 +741,7 @@ export class Library {
|
|||||||
// advanced), through the metadata pool, and update the CLIP index. Guarded
|
// advanced), through the metadata pool, and update the CLIP index. Guarded
|
||||||
// so passes never overlap; a failure is reported, not thrown.
|
// so passes never overlap; a failure is reported, not thrown.
|
||||||
private async runMLFetch(): Promise<void> {
|
private async runMLFetch(): Promise<void> {
|
||||||
const mldata = this.mldata;
|
const mldata = this.mlStore;
|
||||||
// Bind so the call keeps the client as its receiver when invoked
|
// Bind so the call keeps the client as its receiver when invoked
|
||||||
// through the pool below.
|
// through the pool below.
|
||||||
const fetchMLData = this.client.fetchMLData?.bind(this.client);
|
const fetchMLData = this.client.fetchMLData?.bind(this.client);
|
||||||
@@ -642,15 +818,7 @@ export class Library {
|
|||||||
// Gather every file membership and project the store into by-id records,
|
// Gather every file membership and project the store into by-id records,
|
||||||
// filling each record's cache paths from the content cache when present.
|
// filling each record's cache paths from the content cache when present.
|
||||||
private deriveNow(): DerivedRecords {
|
private deriveNow(): DerivedRecords {
|
||||||
const collections = this.store.listCollections();
|
return deriveRecordsFromStore(this.store, this.cache);
|
||||||
const files: EnteFile[] = [];
|
|
||||||
for (const c of collections) files.push(...this.store.listFiles(c.id));
|
|
||||||
const cache = this.cache;
|
|
||||||
return deriveRecords(
|
|
||||||
collections,
|
|
||||||
files,
|
|
||||||
cache ? (fileID) => cache.pathsFor(fileID) : undefined,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private notify(change: LibraryChange): void {
|
private notify(change: LibraryChange): void {
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
// The content-similarity search surface over the CLIP index (issue #50).
|
||||||
|
//
|
||||||
|
// This is `lib.mldata`. It answers three questions against the ML-data cache
|
||||||
|
// (#49) without touching the network:
|
||||||
|
//
|
||||||
|
// - `forFile` returns the whole stored payload (face boxes, landmarks,
|
||||||
|
// embedding) for a file, read from disk on demand — the only method here
|
||||||
|
// that touches the disk, and the only one that is async.
|
||||||
|
// - `similar` and `searchByEmbedding` rank fileIDs by cosine similarity over
|
||||||
|
// the packed `Float32Array` index alone. That index (~50k×512) already
|
||||||
|
// lives in RAM, so each query is a plain loop over it and nothing else.
|
||||||
|
//
|
||||||
|
// quak bundles no text encoder (owner-deferred), so `searchByEmbedding` takes
|
||||||
|
// the query vector the caller has produced elsewhere; `similar` uses the
|
||||||
|
// query file's own indexed embedding.
|
||||||
|
|
||||||
|
import type { MLData } from "../mldata-fetch.js";
|
||||||
|
import type { MLDataStore, MLIndex } from "./mldata.js";
|
||||||
|
|
||||||
|
// How many nearest files a query returns when the caller names no limit.
|
||||||
|
const DEFAULT_LIMIT = 20;
|
||||||
|
|
||||||
|
// One ranked result: a fileID and its cosine similarity to the query, in
|
||||||
|
// [-1, 1]. Callers wanting only the ids read `.fileID`.
|
||||||
|
export interface SimilarResult {
|
||||||
|
fileID: number;
|
||||||
|
score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MLDataAPI {
|
||||||
|
// The whole stored ML payload for a file, or undefined when it is not
|
||||||
|
// cached. Reads the payload from disk, so it is async.
|
||||||
|
forFile(args: { fileID: number }): Promise<MLData | undefined>;
|
||||||
|
// The files nearest the given file by cosine over their CLIP embeddings,
|
||||||
|
// most similar first, excluding the file itself. Empty when the file has
|
||||||
|
// no indexed embedding.
|
||||||
|
similar(args: { fileID: number; limit?: number }): SimilarResult[];
|
||||||
|
// The files nearest a caller-supplied query embedding by cosine, most
|
||||||
|
// similar first. Empty when the query is the wrong length for the index,
|
||||||
|
// has zero magnitude, or the index is empty.
|
||||||
|
searchByEmbedding(args: {
|
||||||
|
embedding: ArrayLike<number>;
|
||||||
|
limit?: number;
|
||||||
|
}): SimilarResult[];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rank the packed index by cosine similarity to `query`, most similar first,
|
||||||
|
// and return the top `limit`. `skip` (a query file's own id) is left out. Both
|
||||||
|
// each row's magnitude and the query's are computed here rather than cached:
|
||||||
|
// the index mutates as ML data is fetched, and one plain pass over ~50k×512
|
||||||
|
// floats is fast enough that a norm cache would only add a staleness bug. A
|
||||||
|
// zero-magnitude vector has no direction, so it is dropped rather than divided
|
||||||
|
// by zero.
|
||||||
|
const topByCosine = (
|
||||||
|
index: MLIndex,
|
||||||
|
query: ArrayLike<number>,
|
||||||
|
limit: number,
|
||||||
|
skip?: number,
|
||||||
|
): SimilarResult[] => {
|
||||||
|
const { fileIDs, embeddingLength, embeddings } = index;
|
||||||
|
if (embeddingLength === 0 || query.length !== embeddingLength) return [];
|
||||||
|
|
||||||
|
// Every indexed read below is in range: the inner loops run to
|
||||||
|
// `embeddingLength`, the query is exactly that long (checked above), and
|
||||||
|
// the packed buffer holds `fileIDs.length * embeddingLength` floats.
|
||||||
|
// `noUncheckedIndexedAccess` still widens each read to `number | undefined`,
|
||||||
|
// so they are asserted non-null rather than paying a per-element guard in
|
||||||
|
// this hot ~50k×512 loop.
|
||||||
|
let queryNorm = 0;
|
||||||
|
for (let k = 0; k < embeddingLength; k++) {
|
||||||
|
const q = query[k]!;
|
||||||
|
queryNorm += q * q;
|
||||||
|
}
|
||||||
|
queryNorm = Math.sqrt(queryNorm);
|
||||||
|
if (queryNorm === 0) return [];
|
||||||
|
|
||||||
|
const results: SimilarResult[] = [];
|
||||||
|
for (let i = 0; i < fileIDs.length; i++) {
|
||||||
|
const id = fileIDs[i]!;
|
||||||
|
if (id === skip) continue;
|
||||||
|
const base = i * embeddingLength;
|
||||||
|
let dot = 0;
|
||||||
|
let norm = 0;
|
||||||
|
for (let k = 0; k < embeddingLength; k++) {
|
||||||
|
const v = embeddings[base + k]!;
|
||||||
|
dot += query[k]! * v;
|
||||||
|
norm += v * v;
|
||||||
|
}
|
||||||
|
if (norm === 0) continue;
|
||||||
|
results.push({
|
||||||
|
fileID: id,
|
||||||
|
score: dot / (queryNorm * Math.sqrt(norm)),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Descending score, ties broken by ascending fileID for a stable order.
|
||||||
|
results.sort((a, b) => b.score - a.score || a.fileID - b.fileID);
|
||||||
|
return results.slice(0, Math.max(0, Math.trunc(limit)));
|
||||||
|
};
|
||||||
|
|
||||||
|
// Build the search surface over a store the library supplies lazily (the store
|
||||||
|
// is absent when the client cannot fetch ML data). Reading it per call keeps
|
||||||
|
// the surface current as the index grows.
|
||||||
|
export const makeMLDataAPI = (
|
||||||
|
store: () => MLDataStore | undefined,
|
||||||
|
): MLDataAPI => ({
|
||||||
|
forFile: ({ fileID }): Promise<MLData | undefined> => {
|
||||||
|
const s = store();
|
||||||
|
return s ? s.readPayload(fileID) : Promise.resolve(undefined);
|
||||||
|
},
|
||||||
|
similar: ({ fileID, limit }): SimilarResult[] => {
|
||||||
|
const s = store();
|
||||||
|
if (!s) return [];
|
||||||
|
const index = s.getIndex();
|
||||||
|
const pos = index.fileIDs.indexOf(fileID);
|
||||||
|
if (pos < 0) return [];
|
||||||
|
const base = pos * index.embeddingLength;
|
||||||
|
const query = index.embeddings.subarray(
|
||||||
|
base,
|
||||||
|
base + index.embeddingLength,
|
||||||
|
);
|
||||||
|
return topByCosine(index, query, limit ?? DEFAULT_LIMIT, fileID);
|
||||||
|
},
|
||||||
|
searchByEmbedding: ({ embedding, limit }): SimilarResult[] => {
|
||||||
|
const s = store();
|
||||||
|
if (!s) return [];
|
||||||
|
return topByCosine(s.getIndex(), embedding, limit ?? DEFAULT_LIMIT);
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -0,0 +1,265 @@
|
|||||||
|
// The aggressive local precache (issue #48), started from `Library.open` with
|
||||||
|
// no caller input.
|
||||||
|
//
|
||||||
|
// Two background fills run concurrently through the shared request pools (#45):
|
||||||
|
//
|
||||||
|
// - Thumbnails: every file in the account, newest first, through the
|
||||||
|
// thumbnail pool until all are on disk. Never evicted. The pool is the same
|
||||||
|
// one `thumbnails.ensure` uses, so a visible or ahead request always jumps
|
||||||
|
// ahead of this background fill and a fileID both want is fetched once.
|
||||||
|
//
|
||||||
|
// - Originals (the pinned set): through the content pool, the favorites album
|
||||||
|
// first, then every file whose `takenAt` falls in the latest
|
||||||
|
// `originalsDays` window — the days ending at the newest file in the
|
||||||
|
// account. The pinned set is the eviction predicate (#47): a pinned
|
||||||
|
// original is never evicted, and a file that leaves the set (a favorite
|
||||||
|
// removed, or the window moving past it on a later refresh) becomes an
|
||||||
|
// ordinary, evictable original with its bytes left in place.
|
||||||
|
//
|
||||||
|
// Both fills yield to on-demand work: every fetch goes to its pool at
|
||||||
|
// background priority, which the pool serves only after on-demand requests. A
|
||||||
|
// file already cached costs one map lookup (`pathsFor`) and no fetch. Each
|
||||||
|
// sweep is driven in bounded chunks so the pool's waiting queue never grows to
|
||||||
|
// the whole account, keeping on-demand preemption and per-admit cost cheap on a
|
||||||
|
// large library. Failures are not fatal: an uncached file is retried on the
|
||||||
|
// next sweep, which `Library` re-kicks after every refresh.
|
||||||
|
|
||||||
|
import type { EnsureResult } from "./content.js";
|
||||||
|
import type { DerivedRecords } from "./records.js";
|
||||||
|
|
||||||
|
const ONE_DAY_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
export const DEFAULT_PRECACHE_ORIGINALS_DAYS = 7;
|
||||||
|
|
||||||
|
// How many files a sweep submits to a pool before awaiting them. Bounds the
|
||||||
|
// pool's waiting queue so on-demand work is never stuck behind the whole
|
||||||
|
// account; the values track each pool's concurrency (#45).
|
||||||
|
const THUMBNAIL_CHUNK = 25;
|
||||||
|
const ORIGINAL_CHUNK = 5;
|
||||||
|
|
||||||
|
// The precache metrics `Library.status()` surfaces.
|
||||||
|
export interface PrecacheStatus {
|
||||||
|
thumbnailsCached: number;
|
||||||
|
thumbnailsTotal: number;
|
||||||
|
originalsCached: number;
|
||||||
|
originalsPinned: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A background-fill progress event. Each active sweep fires "started" before
|
||||||
|
// its fetches and "done" (with the count newly on disk) after; a sweep with
|
||||||
|
// nothing left to fetch is silent.
|
||||||
|
export interface PrecacheEvent {
|
||||||
|
operation: "precacheThumbnails" | "precacheOriginals";
|
||||||
|
status: "started" | "done";
|
||||||
|
fetched?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The slice of the content cache the precache drives. The real `ContentCache`
|
||||||
|
// satisfies it; tests inject a fake.
|
||||||
|
export interface PrecacheCache {
|
||||||
|
pathsFor(fileID: number): { originalPath?: string; thumbnailPath?: string };
|
||||||
|
ensureThumbnails(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
priority: "background";
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<EnsureResult[]>;
|
||||||
|
ensureOriginals(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<EnsureResult[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PrecacheOptions {
|
||||||
|
// Default true; false disables the fill and, for originals, the pinning.
|
||||||
|
thumbnails?: boolean;
|
||||||
|
originals?: boolean;
|
||||||
|
// The latest-week window length in days; default 7.
|
||||||
|
originalsDays?: number;
|
||||||
|
onEvent?: (event: PrecacheEvent) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class Precache {
|
||||||
|
private readonly doThumbnails: boolean;
|
||||||
|
private readonly doOriginals: boolean;
|
||||||
|
private readonly originalsDays: number;
|
||||||
|
private readonly onEvent?: (event: PrecacheEvent) => void;
|
||||||
|
|
||||||
|
private cache?: PrecacheCache;
|
||||||
|
// Every file in the account, newest first (the thumbnail fill order).
|
||||||
|
private thumbOrder: number[] = [];
|
||||||
|
// The pinned originals in fetch order: favorites first, then the window.
|
||||||
|
private originalsOrder: number[] = [];
|
||||||
|
private pinned = new Set<number>();
|
||||||
|
|
||||||
|
// A sweep runs at most once per fill at a time; a re-kick while one runs is
|
||||||
|
// a no-op, and the next refresh re-kicks after it finishes.
|
||||||
|
private thumbRunning = false;
|
||||||
|
private originalsRunning = false;
|
||||||
|
private readonly aborter = new AbortController();
|
||||||
|
private closed = false;
|
||||||
|
|
||||||
|
constructor(opts: PrecacheOptions = {}) {
|
||||||
|
this.doThumbnails = opts.thumbnails ?? true;
|
||||||
|
this.doOriginals = opts.originals ?? true;
|
||||||
|
this.originalsDays =
|
||||||
|
opts.originalsDays ?? DEFAULT_PRECACHE_ORIGINALS_DAYS;
|
||||||
|
this.onEvent = opts.onEvent;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attach the cache the fills fetch through. `isPinned` works before this is
|
||||||
|
// called, so the cache can be constructed with `isPinned` wired in and then
|
||||||
|
// bound here.
|
||||||
|
bind(cache: PrecacheCache): void {
|
||||||
|
this.cache = cache;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whether an original is pinned (favorites + the latest-week window), the
|
||||||
|
// eviction predicate (#47). False for every file when originals precaching
|
||||||
|
// is disabled.
|
||||||
|
isPinned(fileID: number): boolean {
|
||||||
|
return this.pinned.has(fileID);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recompute the fill orders and the pinned set from the current projection.
|
||||||
|
// Called at open and after every refresh that changes the library.
|
||||||
|
update(records: DerivedRecords): void {
|
||||||
|
const photos = [...records.photos.values()].sort(
|
||||||
|
(a, b) => b.takenAt - a.takenAt || b.fileID - a.fileID,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.thumbOrder = this.doThumbnails ? photos.map((p) => p.fileID) : [];
|
||||||
|
|
||||||
|
const pinned = new Set<number>();
|
||||||
|
const order: number[] = [];
|
||||||
|
if (this.doOriginals) {
|
||||||
|
// Favorites first, in the album's own newest-first order.
|
||||||
|
for (const album of records.albums.values()) {
|
||||||
|
if (album.type !== "favorites") continue;
|
||||||
|
for (const id of album.fileIDs) {
|
||||||
|
if (!pinned.has(id)) {
|
||||||
|
pinned.add(id);
|
||||||
|
order.push(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Then the latest-week window, ending at the newest file. Photos
|
||||||
|
// are newest first, so stop once one falls before the window start.
|
||||||
|
const newest = photos[0];
|
||||||
|
if (newest !== undefined) {
|
||||||
|
const windowStart =
|
||||||
|
newest.takenAt - this.originalsDays * ONE_DAY_MS;
|
||||||
|
for (const p of photos) {
|
||||||
|
if (p.takenAt < windowStart) break;
|
||||||
|
if (!pinned.has(p.fileID)) {
|
||||||
|
pinned.add(p.fileID);
|
||||||
|
order.push(p.fileID);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.pinned = pinned;
|
||||||
|
this.originalsOrder = order;
|
||||||
|
}
|
||||||
|
|
||||||
|
status(): PrecacheStatus {
|
||||||
|
const cache = this.cache;
|
||||||
|
let thumbnailsCached = 0;
|
||||||
|
let originalsCached = 0;
|
||||||
|
if (cache) {
|
||||||
|
for (const id of this.thumbOrder)
|
||||||
|
if (cache.pathsFor(id).thumbnailPath !== undefined)
|
||||||
|
thumbnailsCached++;
|
||||||
|
for (const id of this.originalsOrder)
|
||||||
|
if (cache.pathsFor(id).originalPath !== undefined)
|
||||||
|
originalsCached++;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
thumbnailsCached,
|
||||||
|
thumbnailsTotal: this.thumbOrder.length,
|
||||||
|
originalsCached,
|
||||||
|
originalsPinned: this.originalsOrder.length,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kick both fills. Idempotent: a fill already sweeping is left alone. Safe
|
||||||
|
// to call after every refresh; a finished fill starts a fresh sweep that
|
||||||
|
// picks up new files and retries any that failed before.
|
||||||
|
start(): void {
|
||||||
|
if (this.closed || !this.cache) return;
|
||||||
|
if (this.doThumbnails) this.kickThumbnails();
|
||||||
|
if (this.doOriginals) this.kickOriginals();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop the fills. In-flight fetches are left to settle; queued ones drop.
|
||||||
|
close(): void {
|
||||||
|
this.closed = true;
|
||||||
|
this.aborter.abort();
|
||||||
|
}
|
||||||
|
|
||||||
|
private kickThumbnails(): void {
|
||||||
|
if (this.thumbRunning) return;
|
||||||
|
this.thumbRunning = true;
|
||||||
|
void this.sweep(
|
||||||
|
"precacheThumbnails",
|
||||||
|
() => this.thumbOrder,
|
||||||
|
(id) => this.cache!.pathsFor(id).thumbnailPath !== undefined,
|
||||||
|
THUMBNAIL_CHUNK,
|
||||||
|
(chunk) =>
|
||||||
|
this.cache!.ensureThumbnails({
|
||||||
|
fileIDs: chunk,
|
||||||
|
priority: "background",
|
||||||
|
signal: this.aborter.signal,
|
||||||
|
}),
|
||||||
|
).finally(() => {
|
||||||
|
this.thumbRunning = false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private kickOriginals(): void {
|
||||||
|
if (this.originalsRunning) return;
|
||||||
|
this.originalsRunning = true;
|
||||||
|
void this.sweep(
|
||||||
|
"precacheOriginals",
|
||||||
|
() => this.originalsOrder,
|
||||||
|
(id) => this.cache!.pathsFor(id).originalPath !== undefined,
|
||||||
|
ORIGINAL_CHUNK,
|
||||||
|
(chunk) =>
|
||||||
|
this.cache!.ensureOriginals({
|
||||||
|
fileIDs: chunk,
|
||||||
|
signal: this.aborter.signal,
|
||||||
|
}),
|
||||||
|
).finally(() => {
|
||||||
|
this.originalsRunning = false;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// One fill sweep: skip files already on disk (one lookup each), fetch the
|
||||||
|
// rest in bounded chunks, and report progress only when there was work.
|
||||||
|
private async sweep(
|
||||||
|
operation: PrecacheEvent["operation"],
|
||||||
|
order: () => number[],
|
||||||
|
present: (fileID: number) => boolean,
|
||||||
|
chunkSize: number,
|
||||||
|
fetch: (chunk: number[]) => Promise<EnsureResult[]>,
|
||||||
|
): Promise<void> {
|
||||||
|
const todo = order().filter((id) => !present(id));
|
||||||
|
if (todo.length === 0) return;
|
||||||
|
this.emit({ operation, status: "started" });
|
||||||
|
let fetched = 0;
|
||||||
|
for (let i = 0; i < todo.length && !this.closed; i += chunkSize) {
|
||||||
|
const results = await fetch(todo.slice(i, i + chunkSize));
|
||||||
|
for (const r of results) if (r.path !== undefined) fetched++;
|
||||||
|
}
|
||||||
|
this.emit({ operation, status: "done", fetched });
|
||||||
|
}
|
||||||
|
|
||||||
|
private emit(event: PrecacheEvent): void {
|
||||||
|
if (!this.onEvent) return;
|
||||||
|
// A misbehaving callback must not break the fill loop.
|
||||||
|
try {
|
||||||
|
this.onEvent(event);
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -193,6 +193,15 @@ export interface TimelineAPI {
|
|||||||
groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[];
|
groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The surface `Library.fresh()` resolves to (issue #75). It is the same three
|
||||||
|
// read namespaces as the default `albums`/`photos`/`timeline`, handed back only
|
||||||
|
// after a forced refresh has brought the local copy current.
|
||||||
|
export interface FreshReads {
|
||||||
|
albums: AlbumsAPI;
|
||||||
|
photos: PhotosAPI;
|
||||||
|
timeline: TimelineAPI;
|
||||||
|
}
|
||||||
|
|
||||||
export const makeAlbumsAPI = (
|
export const makeAlbumsAPI = (
|
||||||
derive: () => DerivedRecords,
|
derive: () => DerivedRecords,
|
||||||
content?: PhotoContent,
|
content?: PhotoContent,
|
||||||
|
|||||||
+32
-25
@@ -1,15 +1,9 @@
|
|||||||
import {
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||||
mkdirSync,
|
|
||||||
mkdtempSync,
|
|
||||||
readFileSync,
|
|
||||||
rmSync,
|
|
||||||
writeFileSync,
|
|
||||||
} from "node:fs";
|
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { tmpdir } from "node:os";
|
|
||||||
import * as jpeg from "jpeg-js";
|
import * as jpeg from "jpeg-js";
|
||||||
import exifReader from "exif-reader";
|
import exifReader from "exif-reader";
|
||||||
import type { Client } from "./client.js";
|
import type { Client } from "./client.js";
|
||||||
|
import type { Library, Photo } from "./library/index.js";
|
||||||
import { fetchMLData } from "./mldata-fetch.js";
|
import { fetchMLData } from "./mldata-fetch.js";
|
||||||
import type { EnteFile } from "./model/types.js";
|
import type { EnteFile } from "./model/types.js";
|
||||||
|
|
||||||
@@ -104,24 +98,29 @@ const extractImageMetadata = (
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Read a file's original bytes through the library's content cache and extract
|
||||||
|
// its embedded image metadata. The bytes come from `photo.original()` — the
|
||||||
|
// same on-disk cache the rest of the library fills — rather than a fresh
|
||||||
|
// per-call download to a throwaway temp file.
|
||||||
const extractExif = async (
|
const extractExif = async (
|
||||||
client: Client,
|
photo: Photo,
|
||||||
file: EnteFile,
|
|
||||||
): Promise<Record<string, unknown> | undefined> => {
|
): Promise<Record<string, unknown> | undefined> => {
|
||||||
const tmpDir = mkdtempSync(join(tmpdir(), "quak-exif-"));
|
|
||||||
try {
|
try {
|
||||||
const origPath = join(tmpDir, "original");
|
const { path } = await photo.original();
|
||||||
await client.downloadFile(file, origPath);
|
const fileBytes = new Uint8Array(readFileSync(path));
|
||||||
const fileBytes = new Uint8Array(readFileSync(origPath));
|
|
||||||
return extractImageMetadata(fileBytes);
|
return extractImageMetadata(fileBytes);
|
||||||
} catch {
|
} catch {
|
||||||
return undefined;
|
return undefined;
|
||||||
} finally {
|
|
||||||
rmSync(tmpDir, { recursive: true, force: true });
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Dump every decrypted metadata layer the account holds into a directory tree
|
||||||
|
// of plain JSON: account, per-collection, and per-file records including the
|
||||||
|
// private and public magic metadata and (by default) the ML data. Collections
|
||||||
|
// and files are enumerated from the library's cache rather than a fresh server
|
||||||
|
// scan; the ML fetch and EXIF extraction are unchanged.
|
||||||
export const runMetadataBackup = async (
|
export const runMetadataBackup = async (
|
||||||
|
lib: Library,
|
||||||
client: Client,
|
client: Client,
|
||||||
outDir: string,
|
outDir: string,
|
||||||
opts?: MetadataBackupOptions,
|
opts?: MetadataBackupOptions,
|
||||||
@@ -139,13 +138,19 @@ export const runMetadataBackup = async (
|
|||||||
);
|
);
|
||||||
|
|
||||||
log("Fetching collections...");
|
log("Fetching collections...");
|
||||||
const collections = await client.listCollections();
|
|
||||||
|
|
||||||
const allFiles: { file: EnteFile; colDirName: string }[] = [];
|
// Enumerate through the library's read surface. Each album carries its
|
||||||
|
// photos, but the full decrypted `Collection`/`EnteFile` records (with the
|
||||||
|
// magic-metadata layers this dump exists to preserve) come from the
|
||||||
|
// library's by-id accessors.
|
||||||
|
const allFiles: { file: EnteFile; photo: Photo; colDirName: string }[] = [];
|
||||||
const fileKeys = new Map<number, Uint8Array>();
|
const fileKeys = new Map<number, Uint8Array>();
|
||||||
const seenFileIDs = new Set<number>();
|
const seenFileIDs = new Set<number>();
|
||||||
|
|
||||||
for (const col of collections) {
|
for (const album of lib.albums.list()) {
|
||||||
|
const col = lib.getCollection(album.collectionID);
|
||||||
|
if (!col) continue;
|
||||||
|
|
||||||
const dirName = `${col.id}-${sanitizePath(col.name || "unnamed")}`;
|
const dirName = `${col.id}-${sanitizePath(col.name || "unnamed")}`;
|
||||||
const colDir = join(outDir, "collections", dirName);
|
const colDir = join(outDir, "collections", dirName);
|
||||||
mkdirSync(colDir, { recursive: true });
|
mkdirSync(colDir, { recursive: true });
|
||||||
@@ -170,11 +175,13 @@ export const runMetadataBackup = async (
|
|||||||
);
|
);
|
||||||
|
|
||||||
log(`[${col.name}] Fetching files...`);
|
log(`[${col.name}] Fetching files...`);
|
||||||
const files = await client.listFiles(col.id, col.key);
|
const photos = album.photos.list();
|
||||||
log(`[${col.name}] ${files.length} file(s)`);
|
log(`[${col.name}] ${photos.length} file(s)`);
|
||||||
|
|
||||||
for (const file of files) {
|
for (const photo of photos) {
|
||||||
allFiles.push({ file, colDirName: dirName });
|
const file = lib.getFile(col.id, photo.fileID);
|
||||||
|
if (!file) continue;
|
||||||
|
allFiles.push({ file, photo, colDirName: dirName });
|
||||||
if (!seenFileIDs.has(file.id)) {
|
if (!seenFileIDs.has(file.id)) {
|
||||||
fileKeys.set(file.id, file.key);
|
fileKeys.set(file.id, file.key);
|
||||||
seenFileIDs.add(file.id);
|
seenFileIDs.add(file.id);
|
||||||
@@ -191,7 +198,7 @@ export const runMetadataBackup = async (
|
|||||||
log(`Got ML data for ${mlDataMap.size} file(s)`);
|
log(`Got ML data for ${mlDataMap.size} file(s)`);
|
||||||
|
|
||||||
const writtenFileIDs = new Set<number>();
|
const writtenFileIDs = new Set<number>();
|
||||||
for (const { file, colDirName } of allFiles) {
|
for (const { file, photo, colDirName } of allFiles) {
|
||||||
const colDir = join(outDir, "collections", colDirName);
|
const colDir = join(outDir, "collections", colDirName);
|
||||||
|
|
||||||
const fileMeta: Record<string, unknown> = {
|
const fileMeta: Record<string, unknown> = {
|
||||||
@@ -210,7 +217,7 @@ export const runMetadataBackup = async (
|
|||||||
|
|
||||||
if (wantExif && !writtenFileIDs.has(file.id)) {
|
if (wantExif && !writtenFileIDs.has(file.id)) {
|
||||||
log(`[${file.metadata.title}] Extracting EXIF...`);
|
log(`[${file.metadata.title}] Extracting EXIF...`);
|
||||||
const exifData = await extractExif(client, file);
|
const exifData = await extractExif(photo);
|
||||||
if (exifData) fileMeta.imageMetadata = exifData;
|
if (exifData) fileMeta.imageMetadata = exifData;
|
||||||
}
|
}
|
||||||
writtenFileIDs.add(file.id);
|
writtenFileIDs.add(file.id);
|
||||||
|
|||||||
+123
-66
@@ -2,13 +2,10 @@ import { createHash } from "node:crypto";
|
|||||||
import { readFileSync } from "node:fs";
|
import { readFileSync } from "node:fs";
|
||||||
import * as jpeg from "jpeg-js";
|
import * as jpeg from "jpeg-js";
|
||||||
import type { Client } from "./client.js";
|
import type { Client } from "./client.js";
|
||||||
|
import type { Library } from "./library/index.js";
|
||||||
import { ApiError } from "./api/client.js";
|
import { ApiError } from "./api/client.js";
|
||||||
import { encryptBlob, toBase64 } from "./crypto/index.js";
|
import { encryptBlob, toBase64 } from "./crypto/index.js";
|
||||||
import { downloadFile } from "./download/index.js";
|
|
||||||
import type { EnteFile } from "./model/types.js";
|
import type { EnteFile } from "./model/types.js";
|
||||||
import { mkdtempSync, rmSync } from "node:fs";
|
|
||||||
import { join } from "node:path";
|
|
||||||
import { tmpdir } from "node:os";
|
|
||||||
|
|
||||||
const THUMB_MAX_DIMENSION = 720;
|
const THUMB_MAX_DIMENSION = 720;
|
||||||
const THUMB_JPEG_QUALITY = 50;
|
const THUMB_JPEG_QUALITY = 50;
|
||||||
@@ -20,34 +17,51 @@ export interface MissingThumbnailInfo {
|
|||||||
reason: string;
|
reason: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Three outcomes, not two. "fixed": a thumbnail was generated and uploaded.
|
||||||
|
// "failed": something went wrong (download, encode, upload) and the file still
|
||||||
|
// has no thumbnail. "skipped": the file is a format this helper cannot
|
||||||
|
// regenerate — a video, or an image that is not a baseline JPEG. Skipped is a
|
||||||
|
// deliberate, expected outcome, not an error (issue #17): the repair path is
|
||||||
|
// JPEG-only because `jpeg-js` is, and a PNG or HEIC is left for a format-aware
|
||||||
|
// tool rather than reported as a failure.
|
||||||
|
export type ThumbnailFixStatus = "fixed" | "skipped" | "failed";
|
||||||
|
|
||||||
export interface ThumbnailFixResult {
|
export interface ThumbnailFixResult {
|
||||||
fileID: number;
|
fileID: number;
|
||||||
title: string;
|
title: string;
|
||||||
collection: string;
|
collection: string;
|
||||||
success: boolean;
|
status: ThumbnailFixStatus;
|
||||||
error?: string;
|
// Why the file was skipped or failed; unset when it was fixed.
|
||||||
|
reason?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type ProgressCallback = (message: string) => void;
|
export type ProgressCallback = (message: string) => void;
|
||||||
|
|
||||||
|
// Enumerate every file the library knows about, newest album first, each file
|
||||||
|
// once, and report those whose server-side thumbnail is missing. "Missing" is
|
||||||
|
// only two answers: an empty body, or a 404. Any other error reaching this
|
||||||
|
// point has already exhausted its retries — a failing server, a dropped
|
||||||
|
// connection, a deadline — and says nothing about whether the thumbnail
|
||||||
|
// exists, so it is logged and the file is left unreported. That distinction is
|
||||||
|
// what stops `fix-missing-thumbnails` from regenerating and uploading over
|
||||||
|
// thumbnails that were fine all along while the CDN was briefly returning 500s.
|
||||||
export const listMissingThumbnails = async (
|
export const listMissingThumbnails = async (
|
||||||
|
lib: Library,
|
||||||
client: Client,
|
client: Client,
|
||||||
onProgress?: ProgressCallback,
|
onProgress?: ProgressCallback,
|
||||||
): Promise<MissingThumbnailInfo[]> => {
|
): Promise<MissingThumbnailInfo[]> => {
|
||||||
const log = onProgress ?? (() => {});
|
const log = onProgress ?? (() => {});
|
||||||
|
const api = client.getApiClient();
|
||||||
const missing: MissingThumbnailInfo[] = [];
|
const missing: MissingThumbnailInfo[] = [];
|
||||||
const seen = new Set<number>();
|
const seen = new Set<number>();
|
||||||
|
|
||||||
const collections = await client.listCollections();
|
for (const album of lib.albums.list()) {
|
||||||
for (const col of collections) {
|
log(`[${album.name}] Checking thumbnails...`);
|
||||||
log(`[${col.name}] Checking thumbnails...`);
|
for (const photo of album.photos.list()) {
|
||||||
const files = await client.listFiles(col.id, col.key);
|
if (seen.has(photo.fileID)) continue;
|
||||||
for (const file of files) {
|
seen.add(photo.fileID);
|
||||||
if (seen.has(file.id)) continue;
|
|
||||||
seen.add(file.id);
|
|
||||||
try {
|
try {
|
||||||
const api = client.getApiClient();
|
const stream = await api.getThumbnailStream(photo.fileID);
|
||||||
const stream = await api.getThumbnailStream(file.id);
|
|
||||||
const reader = stream.getReader();
|
const reader = stream.getReader();
|
||||||
let totalBytes = 0;
|
let totalBytes = 0;
|
||||||
for (;;) {
|
for (;;) {
|
||||||
@@ -57,35 +71,23 @@ export const listMissingThumbnails = async (
|
|||||||
}
|
}
|
||||||
if (totalBytes === 0) {
|
if (totalBytes === 0) {
|
||||||
missing.push({
|
missing.push({
|
||||||
fileID: file.id,
|
fileID: photo.fileID,
|
||||||
title: file.metadata.title,
|
title: photo.title,
|
||||||
collection: col.name,
|
collection: album.name,
|
||||||
reason: "empty thumbnail (0 bytes)",
|
reason: "empty thumbnail (0 bytes)",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// A 404 is the server stating the thumbnail is not there:
|
|
||||||
// that, and an empty body, are the only two answers that mean
|
|
||||||
// "missing". Anything else reaching this point is a failure
|
|
||||||
// that already exhausted its retries — a failing server, a
|
|
||||||
// dropped connection, a deadline — and says nothing about
|
|
||||||
// whether the thumbnail exists.
|
|
||||||
//
|
|
||||||
// The distinction is what stops `helper
|
|
||||||
// fix-missing-thumbnails` from downloading originals,
|
|
||||||
// regenerating thumbnails and uploading them over thumbnails
|
|
||||||
// that were fine all along, because the CDN was briefly
|
|
||||||
// returning 500s while this ran.
|
|
||||||
if (err instanceof ApiError && err.status === 404) {
|
if (err instanceof ApiError && err.status === 404) {
|
||||||
missing.push({
|
missing.push({
|
||||||
fileID: file.id,
|
fileID: photo.fileID,
|
||||||
title: file.metadata.title,
|
title: photo.title,
|
||||||
collection: col.name,
|
collection: album.name,
|
||||||
reason: "thumbnail not found (HTTP 404)",
|
reason: "thumbnail not found (HTTP 404)",
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
log(
|
log(
|
||||||
`[${col.name}] Could not check ${file.metadata.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`,
|
`[${album.name}] Could not check ${photo.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -94,7 +96,7 @@ export const listMissingThumbnails = async (
|
|||||||
return missing;
|
return missing;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Bilinear resize of RGBA pixel buffer
|
// Bilinear resize of an RGBA pixel buffer.
|
||||||
const resizeRGBA = (
|
const resizeRGBA = (
|
||||||
src: Uint8Array,
|
src: Uint8Array,
|
||||||
srcW: number,
|
srcW: number,
|
||||||
@@ -161,7 +163,33 @@ const generateThumbnail = (fileBytes: Uint8Array): Uint8Array => {
|
|||||||
return new Uint8Array(encoded.data);
|
return new Uint8Array(encoded.data);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// A baseline/JFIF JPEG starts with the SOI marker 0xFFD8. `jpeg-js` decodes
|
||||||
|
// only JPEG, so this signature check is what separates a file the helper can
|
||||||
|
// regenerate from one it must skip: a PNG, HEIC, or the odd non-image byte
|
||||||
|
// stream all fail this and are reported as skipped rather than crashing the
|
||||||
|
// decoder into an opaque failure (issue #17).
|
||||||
|
const isJpeg = (bytes: Uint8Array): boolean =>
|
||||||
|
bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xd8;
|
||||||
|
|
||||||
|
// The reason a file cannot have a JPEG thumbnail regenerated for it from its
|
||||||
|
// metadata alone, before any bytes are fetched, or undefined when it might. A
|
||||||
|
// non-image (video, live photo) is unsupported outright; a still image still
|
||||||
|
// has to be checked against its actual bytes once downloaded.
|
||||||
|
const unsupportedByType = (file: EnteFile): string | undefined => {
|
||||||
|
if (file.metadata.fileType !== "image") {
|
||||||
|
return `unsupported file type: ${file.metadata.fileType} (only JPEG images can be regenerated)`;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Regenerate and upload a thumbnail for each requested file. Originals are read
|
||||||
|
// through the library's content cache (`photo.original()`); the generated
|
||||||
|
// thumbnail is JPEG-encoded, encrypted under the file's own key, and registered
|
||||||
|
// with the server — the encrypt-and-upload path is unchanged. Each file is
|
||||||
|
// resolved to one outcome (fixed / skipped / failed) and a failure on one file
|
||||||
|
// never stops the others.
|
||||||
export const fixMissingThumbnails = async (
|
export const fixMissingThumbnails = async (
|
||||||
|
lib: Library,
|
||||||
client: Client,
|
client: Client,
|
||||||
fileIDs: number[],
|
fileIDs: number[],
|
||||||
onProgress?: ProgressCallback,
|
onProgress?: ProgressCallback,
|
||||||
@@ -170,19 +198,24 @@ export const fixMissingThumbnails = async (
|
|||||||
const results: ThumbnailFixResult[] = [];
|
const results: ThumbnailFixResult[] = [];
|
||||||
const api = client.getApiClient();
|
const api = client.getApiClient();
|
||||||
|
|
||||||
const collections = await client.listCollections();
|
// Resolve each requested fileID to its file record and owning album by
|
||||||
|
// enumerating the library, each file taken from the first album that holds
|
||||||
|
// it. The raw `EnteFile` carries the per-file key the thumbnail is
|
||||||
|
// encrypted under, which the projected records deliberately do not.
|
||||||
|
const wanted = new Set(fileIDs);
|
||||||
const fileMap = new Map<
|
const fileMap = new Map<
|
||||||
number,
|
number,
|
||||||
{ file: EnteFile; collectionName: string }
|
{ file: EnteFile; collectionName: string }
|
||||||
>();
|
>();
|
||||||
|
for (const album of lib.albums.list()) {
|
||||||
for (const col of collections) {
|
for (const photo of album.photos.list()) {
|
||||||
const files = await client.listFiles(col.id, col.key);
|
if (!wanted.has(photo.fileID) || fileMap.has(photo.fileID))
|
||||||
for (const file of files) {
|
continue;
|
||||||
if (fileIDs.includes(file.id) && !fileMap.has(file.id)) {
|
const file = lib.getFile(album.collectionID, photo.fileID);
|
||||||
fileMap.set(file.id, {
|
if (file) {
|
||||||
|
fileMap.set(photo.fileID, {
|
||||||
file,
|
file,
|
||||||
collectionName: col.name,
|
collectionName: album.name,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -195,33 +228,61 @@ export const fixMissingThumbnails = async (
|
|||||||
fileID,
|
fileID,
|
||||||
title: "unknown",
|
title: "unknown",
|
||||||
collection: "unknown",
|
collection: "unknown",
|
||||||
success: false,
|
status: "failed",
|
||||||
error: "file not found in any collection",
|
reason: "file not found in any collection",
|
||||||
});
|
});
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const { file, collectionName } = entry;
|
const { file, collectionName } = entry;
|
||||||
const tmpDir = mkdtempSync(join(tmpdir(), "quak-thumb-"));
|
const title = file.metadata.title;
|
||||||
|
|
||||||
|
const typeReason = unsupportedByType(file);
|
||||||
|
if (typeReason) {
|
||||||
|
log(`[${collectionName}] Skipping ${title}: ${typeReason}`);
|
||||||
|
results.push({
|
||||||
|
fileID,
|
||||||
|
title,
|
||||||
|
collection: collectionName,
|
||||||
|
status: "skipped",
|
||||||
|
reason: typeReason,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
log(
|
const photo = lib.photos.byID({ fileID });
|
||||||
`[${collectionName}] Downloading ${file.metadata.title} for thumbnail generation...`,
|
if (!photo) {
|
||||||
);
|
throw new Error("file not present in the library cache");
|
||||||
const origPath = join(tmpDir, "original");
|
}
|
||||||
await downloadFile(api, file, origPath);
|
|
||||||
|
|
||||||
log(
|
log(
|
||||||
`[${collectionName}] Generating thumbnail for ${file.metadata.title}...`,
|
`[${collectionName}] Downloading ${title} for thumbnail generation...`,
|
||||||
);
|
);
|
||||||
const fileBytes = readFileSync(origPath);
|
const { path } = await photo.original();
|
||||||
const thumbJpeg = generateThumbnail(new Uint8Array(fileBytes));
|
const fileBytes = new Uint8Array(readFileSync(path));
|
||||||
|
|
||||||
|
if (!isJpeg(fileBytes)) {
|
||||||
|
const reason =
|
||||||
|
"unsupported image format (only baseline JPEG can be regenerated)";
|
||||||
|
log(`[${collectionName}] Skipping ${title}: ${reason}`);
|
||||||
|
results.push({
|
||||||
|
fileID,
|
||||||
|
title,
|
||||||
|
collection: collectionName,
|
||||||
|
status: "skipped",
|
||||||
|
reason,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
log(`[${collectionName}] Generating thumbnail for ${title}...`);
|
||||||
|
const thumbJpeg = generateThumbnail(fileBytes);
|
||||||
|
|
||||||
log(
|
log(
|
||||||
`[${collectionName}] Encrypting and uploading thumbnail (${thumbJpeg.length} bytes)...`,
|
`[${collectionName}] Encrypting and uploading thumbnail (${thumbJpeg.length} bytes)...`,
|
||||||
);
|
);
|
||||||
const { header, ciphertext } = encryptBlob(thumbJpeg, file.key);
|
const { header, ciphertext } = encryptBlob(thumbJpeg, file.key);
|
||||||
|
|
||||||
const md5 = createHash("md5").update(ciphertext).digest("base64");
|
const md5 = createHash("md5").update(ciphertext).digest("base64");
|
||||||
const { objectKey, url } = await api.getUploadURL(
|
const { objectKey, url } = await api.getUploadURL(
|
||||||
ciphertext.length,
|
ciphertext.length,
|
||||||
@@ -230,28 +291,24 @@ export const fixMissingThumbnails = async (
|
|||||||
await api.putFile(url, ciphertext);
|
await api.putFile(url, ciphertext);
|
||||||
await api.updateThumbnail(file.id, objectKey, toBase64(header));
|
await api.updateThumbnail(file.id, objectKey, toBase64(header));
|
||||||
|
|
||||||
log(
|
log(`[${collectionName}] Thumbnail uploaded for ${title}`);
|
||||||
`[${collectionName}] Thumbnail uploaded for ${file.metadata.title}`,
|
|
||||||
);
|
|
||||||
results.push({
|
results.push({
|
||||||
fileID,
|
fileID,
|
||||||
title: file.metadata.title,
|
title,
|
||||||
collection: collectionName,
|
collection: collectionName,
|
||||||
success: true,
|
status: "fixed",
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
log(
|
log(
|
||||||
`[${collectionName}] FAILED ${file.metadata.title}: ${err instanceof Error ? err.message : err}`,
|
`[${collectionName}] FAILED ${title}: ${err instanceof Error ? err.message : err}`,
|
||||||
);
|
);
|
||||||
results.push({
|
results.push({
|
||||||
fileID,
|
fileID,
|
||||||
title: file.metadata.title,
|
title,
|
||||||
collection: collectionName,
|
collection: collectionName,
|
||||||
success: false,
|
status: "failed",
|
||||||
error: err instanceof Error ? err.message : String(err),
|
reason: err instanceof Error ? err.message : String(err),
|
||||||
});
|
});
|
||||||
} finally {
|
|
||||||
rmSync(tmpDir, { recursive: true, force: true });
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -142,6 +142,10 @@ const openLibrary = (source: ContentSource): Promise<Library> =>
|
|||||||
cacheDirectory: join(root, "cache"),
|
cacheDirectory: join(root, "cache"),
|
||||||
contentSource: source,
|
contentSource: source,
|
||||||
refreshIntervalSeconds: 3600,
|
refreshIntervalSeconds: 3600,
|
||||||
|
// These tests count exact fetches; the background precache (#48) would
|
||||||
|
// add its own, so it is off here (it is covered in precache.test.ts).
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const readLedger = (
|
const readLedger = (
|
||||||
|
|||||||
@@ -2,13 +2,17 @@
|
|||||||
* Tests for `quak backup-metadata <dir>`.
|
* Tests for `quak backup-metadata <dir>`.
|
||||||
*
|
*
|
||||||
* This command dumps all decrypted account metadata into a directory
|
* This command dumps all decrypted account metadata into a directory
|
||||||
* tree of plain JSON files, without downloading any file content. It
|
* tree of plain JSON files, without downloading any file content (unless
|
||||||
* is fast (no multi-megabyte downloads) and produces a complete
|
* `--exif` is given). It is fast and produces a complete plaintext record of
|
||||||
* plaintext record of every collection name, file title, creation
|
* every collection name, file title, creation date, GPS coordinate, camera
|
||||||
* date, GPS coordinate, camera model, caption, face label, and any
|
* model, caption, face label, and any other metadata the Ente clients have
|
||||||
* other metadata the Ente clients have attached.
|
* attached.
|
||||||
*
|
*
|
||||||
* Layout:
|
* As of issue #52 it runs on the library API: `runMetadataBackup(lib, client,
|
||||||
|
* dir)` enumerates collections and files from the library's cache rather than
|
||||||
|
* scanning the client directly, and `--exif` reads each original through the
|
||||||
|
* library's content cache (`photo.original()`). The ML fetch is unchanged. The
|
||||||
|
* output tree is identical:
|
||||||
*
|
*
|
||||||
* <dir>/
|
* <dir>/
|
||||||
* account.json { email, userID }
|
* account.json { email, userID }
|
||||||
@@ -44,7 +48,11 @@ import {
|
|||||||
} from "../../src/crypto/index.js";
|
} from "../../src/crypto/index.js";
|
||||||
import * as jpegJs from "jpeg-js";
|
import * as jpegJs from "jpeg-js";
|
||||||
import { Client } from "../../src/client.js";
|
import { Client } from "../../src/client.js";
|
||||||
import { runMetadataBackup } from "../../src/metadata-backup.js";
|
import { Library } from "../../src/library/index.js";
|
||||||
|
import {
|
||||||
|
runMetadataBackup,
|
||||||
|
type MetadataBackupOptions,
|
||||||
|
} from "../../src/metadata-backup.js";
|
||||||
import type { KeyAttributes } from "../../src/auth/types.js";
|
import type { KeyAttributes } from "../../src/auth/types.js";
|
||||||
|
|
||||||
const TEST_EMAIL = "metabackup@example.com";
|
const TEST_EMAIL = "metabackup@example.com";
|
||||||
@@ -431,16 +439,50 @@ afterAll(() => {
|
|||||||
rmSync(testDir, { recursive: true, force: true });
|
rmSync(testDir, { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Log in against the mock and open a library over its cache. The point commands
|
||||||
|
// open the library with the background precache off and a long refresh interval;
|
||||||
|
// the same here keeps the test deterministic (no thumbnail/original prefetch it
|
||||||
|
// did not ask for, no second refresh mid-test).
|
||||||
|
const openLib = async (client: Client): Promise<Library> =>
|
||||||
|
Library.open({
|
||||||
|
// The library client omits `fetchMLData`, matching how the CLI opens
|
||||||
|
// point commands: `runMetadataBackup` fetches ML data itself through
|
||||||
|
// the client, so the library's background backfill would only be a
|
||||||
|
// redundant second pass over the same endpoint.
|
||||||
|
client: {
|
||||||
|
whoami: () => client.whoami(),
|
||||||
|
collectionsSince: (args) => client.collectionsSince(args),
|
||||||
|
filesSince: (args) => client.filesSince(args),
|
||||||
|
contentSource: () => client.contentSource(),
|
||||||
|
},
|
||||||
|
cacheDirectory: mkdtempSync(join(testDir, "cache-")),
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Run one metadata backup end to end: fresh client, fresh library, then close.
|
||||||
|
const runBackup = async (
|
||||||
|
outDir: string,
|
||||||
|
opts?: MetadataBackupOptions,
|
||||||
|
): Promise<void> => {
|
||||||
|
const client = await Client.login({
|
||||||
|
email: TEST_EMAIL,
|
||||||
|
password: TEST_PASSWORD,
|
||||||
|
apiOptions: { fetch: buildMetaFetch(mock) },
|
||||||
|
});
|
||||||
|
const lib = await openLib(client);
|
||||||
|
try {
|
||||||
|
await runMetadataBackup(lib, client, outDir, opts);
|
||||||
|
} finally {
|
||||||
|
lib.close();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
describe("quak backup-metadata", () => {
|
describe("quak backup-metadata", () => {
|
||||||
it("writes account.json with email and userID", async () => {
|
it("writes account.json with email and userID", async () => {
|
||||||
const outDir = join(testDir, "full");
|
const outDir = join(testDir, "full");
|
||||||
const client = await Client.login({
|
await runBackup(outDir);
|
||||||
email: TEST_EMAIL,
|
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
await runMetadataBackup(client, outDir);
|
|
||||||
|
|
||||||
const account = JSON.parse(
|
const account = JSON.parse(
|
||||||
readFileSync(join(outDir, "account.json"), "utf-8"),
|
readFileSync(join(outDir, "account.json"), "utf-8"),
|
||||||
@@ -451,13 +493,7 @@ describe("quak backup-metadata", () => {
|
|||||||
|
|
||||||
it("creates per-collection directories with _collection.json", async () => {
|
it("creates per-collection directories with _collection.json", async () => {
|
||||||
const outDir = join(testDir, "collections");
|
const outDir = join(testDir, "collections");
|
||||||
const client = await Client.login({
|
await runBackup(outDir);
|
||||||
email: TEST_EMAIL,
|
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
await runMetadataBackup(client, outDir);
|
|
||||||
|
|
||||||
const collDirs = readdirSync(join(outDir, "collections"));
|
const collDirs = readdirSync(join(outDir, "collections"));
|
||||||
expect(collDirs.length).toBe(2);
|
expect(collDirs.length).toBe(2);
|
||||||
@@ -478,13 +514,7 @@ describe("quak backup-metadata", () => {
|
|||||||
|
|
||||||
it("decrypts collection-level pubMagicMetadata", async () => {
|
it("decrypts collection-level pubMagicMetadata", async () => {
|
||||||
const outDir = join(testDir, "coll-magic");
|
const outDir = join(testDir, "coll-magic");
|
||||||
const client = await Client.login({
|
await runBackup(outDir);
|
||||||
email: TEST_EMAIL,
|
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
await runMetadataBackup(client, outDir);
|
|
||||||
|
|
||||||
const collDirs = readdirSync(join(outDir, "collections"));
|
const collDirs = readdirSync(join(outDir, "collections"));
|
||||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||||
@@ -501,13 +531,7 @@ describe("quak backup-metadata", () => {
|
|||||||
|
|
||||||
it("writes per-file JSON with all three metadata layers", async () => {
|
it("writes per-file JSON with all three metadata layers", async () => {
|
||||||
const outDir = join(testDir, "file-meta");
|
const outDir = join(testDir, "file-meta");
|
||||||
const client = await Client.login({
|
await runBackup(outDir);
|
||||||
email: TEST_EMAIL,
|
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
await runMetadataBackup(client, outDir);
|
|
||||||
|
|
||||||
const collDirs = readdirSync(join(outDir, "collections"));
|
const collDirs = readdirSync(join(outDir, "collections"));
|
||||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||||
@@ -526,13 +550,7 @@ describe("quak backup-metadata", () => {
|
|||||||
|
|
||||||
it("handles files with no magic metadata gracefully", async () => {
|
it("handles files with no magic metadata gracefully", async () => {
|
||||||
const outDir = join(testDir, "no-magic");
|
const outDir = join(testDir, "no-magic");
|
||||||
const client = await Client.login({
|
await runBackup(outDir);
|
||||||
email: TEST_EMAIL,
|
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
await runMetadataBackup(client, outDir);
|
|
||||||
|
|
||||||
const collDirs = readdirSync(join(outDir, "collections"));
|
const collDirs = readdirSync(join(outDir, "collections"));
|
||||||
const workDir = collDirs.find((d) => d.includes("Work"))!;
|
const workDir = collDirs.find((d) => d.includes("Work"))!;
|
||||||
@@ -550,14 +568,8 @@ describe("quak backup-metadata", () => {
|
|||||||
|
|
||||||
it("is incremental: second run does not fail", async () => {
|
it("is incremental: second run does not fail", async () => {
|
||||||
const outDir = join(testDir, "incremental");
|
const outDir = join(testDir, "incremental");
|
||||||
const client = await Client.login({
|
await runBackup(outDir);
|
||||||
email: TEST_EMAIL,
|
await runBackup(outDir);
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
await runMetadataBackup(client, outDir);
|
|
||||||
await runMetadataBackup(client, outDir);
|
|
||||||
|
|
||||||
const account = JSON.parse(
|
const account = JSON.parse(
|
||||||
readFileSync(join(outDir, "account.json"), "utf-8"),
|
readFileSync(join(outDir, "account.json"), "utf-8"),
|
||||||
@@ -567,13 +579,7 @@ describe("quak backup-metadata", () => {
|
|||||||
|
|
||||||
it("fetches and decrypts ML data by default", async () => {
|
it("fetches and decrypts ML data by default", async () => {
|
||||||
const outDir = join(testDir, "ml-data");
|
const outDir = join(testDir, "ml-data");
|
||||||
const client = await Client.login({
|
await runBackup(outDir);
|
||||||
email: TEST_EMAIL,
|
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
await runMetadataBackup(client, outDir);
|
|
||||||
|
|
||||||
const collDirs = readdirSync(join(outDir, "collections"));
|
const collDirs = readdirSync(join(outDir, "collections"));
|
||||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||||
@@ -597,13 +603,7 @@ describe("quak backup-metadata", () => {
|
|||||||
|
|
||||||
it("extracts EXIF from downloaded files when --exif is set", async () => {
|
it("extracts EXIF from downloaded files when --exif is set", async () => {
|
||||||
const outDir = join(testDir, "exif-data");
|
const outDir = join(testDir, "exif-data");
|
||||||
const client = await Client.login({
|
await runBackup(outDir, { exif: true });
|
||||||
email: TEST_EMAIL,
|
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildMetaFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
await runMetadataBackup(client, outDir, { exif: true });
|
|
||||||
|
|
||||||
const collDirs = readdirSync(join(outDir, "collections"));
|
const collDirs = readdirSync(join(outDir, "collections"));
|
||||||
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
const vacDir = collDirs.find((d) => d.includes("Vacation"))!;
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
// The CLI presents a file by its own decrypted metadata, not the PhotoRecord
|
||||||
|
// projection (issue #52). For a renamed file the two disagree: the projection
|
||||||
|
// prefers `editedName` and reports `editedTime` in milliseconds, while the CLI
|
||||||
|
// must print the raw `metadata.title` and `metadata.creationTime` (microseconds)
|
||||||
|
// and name downloads after the raw title, byte-identical to the pre-library CLI.
|
||||||
|
//
|
||||||
|
// This locks in that contrast: the shared output helpers emit the raw values,
|
||||||
|
// and the projection of the same file emits the edited ones — so a regression
|
||||||
|
// that re-sourced the CLI from the projection would fail here.
|
||||||
|
|
||||||
|
import { describe, it, expect } from "vitest";
|
||||||
|
|
||||||
|
import {
|
||||||
|
fileListRow,
|
||||||
|
fileListLine,
|
||||||
|
originalName,
|
||||||
|
thumbnailName,
|
||||||
|
} from "../../src/cli-output.js";
|
||||||
|
import { deriveRecords } from "../../src/library/records.js";
|
||||||
|
import type { EnteFile } from "../../src/model/types.js";
|
||||||
|
|
||||||
|
// Microseconds, as Ente stores times.
|
||||||
|
const RAW_CREATION = 1700000000000000;
|
||||||
|
const EDITED_TIME = 1710000000000000;
|
||||||
|
const RAW_TITLE = "IMG_0001.HEIC";
|
||||||
|
const EDITED_NAME = "Sunset.heic";
|
||||||
|
|
||||||
|
// A file the user has renamed and re-dated: basic metadata holds the original
|
||||||
|
// title and capture time; public magic metadata holds the edits.
|
||||||
|
const renamedFile: EnteFile = {
|
||||||
|
id: 100,
|
||||||
|
collectionID: 10,
|
||||||
|
ownerID: 42,
|
||||||
|
key: new Uint8Array(),
|
||||||
|
metadata: {
|
||||||
|
title: RAW_TITLE,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: RAW_CREATION,
|
||||||
|
modificationTime: RAW_CREATION,
|
||||||
|
},
|
||||||
|
pubMagicMetadata: { editedName: EDITED_NAME, editedTime: EDITED_TIME },
|
||||||
|
file: { decryptionHeader: "" },
|
||||||
|
thumbnail: { decryptionHeader: "" },
|
||||||
|
updationTime: RAW_CREATION,
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("CLI file output (issue #52)", () => {
|
||||||
|
it("emits the raw title and microsecond creationTime for --json", () => {
|
||||||
|
expect(fileListRow(renamedFile)).toEqual({
|
||||||
|
id: 100,
|
||||||
|
title: RAW_TITLE,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: RAW_CREATION,
|
||||||
|
collectionID: 10,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits the raw title in the human column", () => {
|
||||||
|
expect(fileListLine(renamedFile)).toBe(`100\timage\t${RAW_TITLE}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("names downloads after the raw title", () => {
|
||||||
|
expect(originalName(renamedFile)).toBe(RAW_TITLE);
|
||||||
|
expect(thumbnailName(renamedFile)).toBe(`thumb_${RAW_TITLE}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not use the editedName/editedTime projection", () => {
|
||||||
|
const record = deriveRecords([], [renamedFile]).photos.get(100);
|
||||||
|
// The projection prefers the edits and reports milliseconds; the CLI
|
||||||
|
// helpers above deliberately do not.
|
||||||
|
expect(record?.title).toBe(EDITED_NAME);
|
||||||
|
expect(record?.takenAt).toBe(Math.floor(EDITED_TIME / 1000));
|
||||||
|
expect(fileListRow(renamedFile).title).not.toBe(record?.title);
|
||||||
|
expect(fileListRow(renamedFile).creationTime).not.toBe(record?.takenAt);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
/**
|
||||||
|
* Tests for the CLI read helpers (`src/cli-read.ts`, owner amendment to
|
||||||
|
* issue #36, issue #52).
|
||||||
|
*
|
||||||
|
* The `collections`, `files`, `get`, and `get-thumb` commands must answer for
|
||||||
|
* current server state, not the local cache, so each helper forces a
|
||||||
|
* `Library.fresh()` round-trip before it reads. The stand-in library below
|
||||||
|
* serves nothing until `fresh()` has been awaited, so a helper that read
|
||||||
|
* without refreshing would come back empty and fail here.
|
||||||
|
*
|
||||||
|
* `collections` and `files` also list in the library's enumeration order
|
||||||
|
* (`listCollections`/`listFiles`) — the order the pre-library CLI printed — not
|
||||||
|
* the albums/photos projection's newest-first order. The fixtures are seeded in
|
||||||
|
* an enumeration order that a newest-first sort would rearrange, so a
|
||||||
|
* regression to the projection order would fail here too. Field values still
|
||||||
|
* come from the raw metadata via `cli-output.ts`.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect } from "vitest";
|
||||||
|
|
||||||
|
import {
|
||||||
|
freshCollections,
|
||||||
|
freshFiles,
|
||||||
|
freshFile,
|
||||||
|
type FreshReadLibrary,
|
||||||
|
} from "../../src/cli-read.js";
|
||||||
|
import { fileListRow } from "../../src/cli-output.js";
|
||||||
|
import type { Photo } from "../../src/library/index.js";
|
||||||
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||||
|
|
||||||
|
const collection = (id: number, updationTime: number): Collection => ({
|
||||||
|
id,
|
||||||
|
ownerID: 42,
|
||||||
|
key: new Uint8Array(),
|
||||||
|
name: `album-${id}`,
|
||||||
|
type: "album",
|
||||||
|
updationTime,
|
||||||
|
isShared: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Microseconds, as Ente stores times.
|
||||||
|
const file = (
|
||||||
|
id: number,
|
||||||
|
collectionID: number,
|
||||||
|
creationTime: number,
|
||||||
|
): EnteFile => ({
|
||||||
|
id,
|
||||||
|
collectionID,
|
||||||
|
ownerID: 42,
|
||||||
|
key: new Uint8Array(),
|
||||||
|
metadata: {
|
||||||
|
title: `file-${id}.jpg`,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime,
|
||||||
|
modificationTime: creationTime,
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: "" },
|
||||||
|
thumbnail: { decryptionHeader: "" },
|
||||||
|
updationTime: creationTime,
|
||||||
|
});
|
||||||
|
|
||||||
|
// A library that reveals its records only after `fresh()` has been awaited, and
|
||||||
|
// serves them in the enumeration order it was given. `photos.byID` returns a
|
||||||
|
// stand-in `Photo` carrying just the fileID the helper passes through.
|
||||||
|
class FakeLibrary implements FreshReadLibrary {
|
||||||
|
freshCalls = 0;
|
||||||
|
private refreshed = false;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly collections: Collection[],
|
||||||
|
private readonly files: EnteFile[],
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async fresh(): Promise<unknown> {
|
||||||
|
this.freshCalls++;
|
||||||
|
this.refreshed = true;
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
listCollections(): Collection[] {
|
||||||
|
return this.refreshed ? this.collections : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
getCollection(id: number): Collection | undefined {
|
||||||
|
return this.listCollections().find((c) => c.id === id);
|
||||||
|
}
|
||||||
|
|
||||||
|
listFiles(collectionID: number): EnteFile[] {
|
||||||
|
return this.refreshed
|
||||||
|
? this.files.filter((f) => f.collectionID === collectionID)
|
||||||
|
: [];
|
||||||
|
}
|
||||||
|
|
||||||
|
getFileByID(fileID: number): EnteFile | undefined {
|
||||||
|
if (!this.refreshed) return undefined;
|
||||||
|
return this.files.find((f) => f.id === fileID);
|
||||||
|
}
|
||||||
|
|
||||||
|
photos = {
|
||||||
|
byID: ({ fileID }: { fileID: number }): Photo | undefined => {
|
||||||
|
if (!this.refreshed) return undefined;
|
||||||
|
if (!this.files.some((f) => f.id === fileID)) return undefined;
|
||||||
|
return { fileID } as unknown as Photo;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("CLI read helpers (issue #36 amendment, issue #52)", () => {
|
||||||
|
it("freshCollections refreshes first, then lists in enumeration order", async () => {
|
||||||
|
// Enumeration order 2, 1, 3; a newest-first sort would be 3, 2, 1.
|
||||||
|
const lib = new FakeLibrary(
|
||||||
|
[collection(2, 200), collection(1, 300), collection(3, 100)],
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
const rows = await freshCollections(lib);
|
||||||
|
|
||||||
|
expect(lib.freshCalls).toBe(1);
|
||||||
|
expect(rows.map((c) => c.id)).toEqual([2, 1, 3]);
|
||||||
|
// The projection's newest-first order is a different sequence, so this
|
||||||
|
// is not accidentally that order.
|
||||||
|
const newestFirst = [...rows]
|
||||||
|
.sort((a, b) => b.updationTime - a.updationTime)
|
||||||
|
.map((c) => c.id);
|
||||||
|
expect(newestFirst).toEqual([1, 2, 3]);
|
||||||
|
expect(rows.map((c) => c.id)).not.toEqual(newestFirst);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("freshFiles refreshes first, lists in enumeration order, keeps raw fields", async () => {
|
||||||
|
// Enumeration order by id 10, 11, 12; creationTimes ascending, so a
|
||||||
|
// newest-first sort would reverse them.
|
||||||
|
const files = [
|
||||||
|
file(10, 1, 1_700_000_000_000_000),
|
||||||
|
file(11, 1, 1_700_000_000_000_001),
|
||||||
|
file(12, 1, 1_700_000_000_000_002),
|
||||||
|
];
|
||||||
|
const lib = new FakeLibrary([collection(1, 100)], files);
|
||||||
|
|
||||||
|
const rows = await freshFiles(lib, 1);
|
||||||
|
expect(lib.freshCalls).toBe(1);
|
||||||
|
expect(rows?.map((f) => f.id)).toEqual([10, 11, 12]);
|
||||||
|
|
||||||
|
// Field values come from raw metadata: microsecond creationTime and the
|
||||||
|
// raw title, unchanged.
|
||||||
|
expect(rows?.map(fileListRow)).toEqual([
|
||||||
|
{
|
||||||
|
id: 10,
|
||||||
|
title: "file-10.jpg",
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: 1_700_000_000_000_000,
|
||||||
|
collectionID: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 11,
|
||||||
|
title: "file-11.jpg",
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: 1_700_000_000_000_001,
|
||||||
|
collectionID: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 12,
|
||||||
|
title: "file-12.jpg",
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: 1_700_000_000_000_002,
|
||||||
|
collectionID: 1,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("freshFiles returns undefined for an unknown collection", async () => {
|
||||||
|
const lib = new FakeLibrary([collection(1, 100)], []);
|
||||||
|
const rows = await freshFiles(lib, 999);
|
||||||
|
expect(lib.freshCalls).toBe(1);
|
||||||
|
expect(rows).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("freshFile refreshes first, then resolves the photo and its raw record", async () => {
|
||||||
|
const f = file(10, 1, 1_700_000_000_000_000);
|
||||||
|
const lib = new FakeLibrary([collection(1, 100)], [f]);
|
||||||
|
|
||||||
|
const resolved = await freshFile(lib, 10);
|
||||||
|
expect(lib.freshCalls).toBe(1);
|
||||||
|
expect(resolved?.photo.fileID).toBe(10);
|
||||||
|
expect(resolved?.file.metadata.title).toBe("file-10.jpg");
|
||||||
|
expect(resolved?.file.metadata.creationTime).toBe(
|
||||||
|
1_700_000_000_000_000,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("freshFile returns undefined for an unknown file", async () => {
|
||||||
|
const lib = new FakeLibrary([collection(1, 100)], []);
|
||||||
|
const resolved = await freshFile(lib, 404);
|
||||||
|
expect(lib.freshCalls).toBe(1);
|
||||||
|
expect(resolved).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,358 @@
|
|||||||
|
/**
|
||||||
|
* Tests for the originals cache size limit and LRU eviction (issue #47),
|
||||||
|
* layered on the on-disk content cache (#46).
|
||||||
|
*
|
||||||
|
* Only `cacheDirectory/originals` is bounded and evicted. Before each original
|
||||||
|
* write the effective limit is
|
||||||
|
* min(cacheOriginalsMaxBytes, bytesUsedByOriginals + bytesFree - freeBelowBytes)
|
||||||
|
* with `bytesFree` read from `statfs` on the volume holding `cacheDirectory`.
|
||||||
|
* The limit therefore falls as the disk fills and rises as space returns. When
|
||||||
|
* a write would cross the limit, least-recently-used originals are removed until
|
||||||
|
* it fits; pinned files are skipped, and if only pinned files remain the write
|
||||||
|
* proceeds over-limit. Last-use is the file `mtime`, bumped whenever a read
|
||||||
|
* returns an original's path, so ordering survives a restart with no ledger.
|
||||||
|
*
|
||||||
|
* `statfs` is injected so the adaptive limit is exercised deterministically:
|
||||||
|
* `bsize` is 1, so `bavail` is the free byte count the formula sees. The source
|
||||||
|
* writes a controllable number of bytes per file, and tests set each stored
|
||||||
|
* file's `mtime` explicitly so LRU order does not depend on wall-clock timing.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync, existsSync, statSync, utimesSync } from "node:fs";
|
||||||
|
import { writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import {
|
||||||
|
ContentCache,
|
||||||
|
type ContentSource,
|
||||||
|
type StatFsFn,
|
||||||
|
} from "../../src/library/content.js";
|
||||||
|
import { RequestPools } from "../../src/library/pools.js";
|
||||||
|
import type { EnteFile } from "../../src/model/types.js";
|
||||||
|
|
||||||
|
const file = (id: number): EnteFile => ({
|
||||||
|
id,
|
||||||
|
collectionID: 1,
|
||||||
|
ownerID: 1,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
metadata: {
|
||||||
|
title: `file-${id}.jpg`,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: 0,
|
||||||
|
modificationTime: 0,
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: "aGVhZGVy" },
|
||||||
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
||||||
|
updationTime: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
// A source that writes a controllable number of bytes per file (default 10).
|
||||||
|
class SizedSource implements ContentSource {
|
||||||
|
sizeFor = new Map<number, number>();
|
||||||
|
|
||||||
|
async original(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
}): Promise<{ bytesWritten: number }> {
|
||||||
|
const n = this.sizeFor.get(args.file.id) ?? 10;
|
||||||
|
await writeFile(args.destination, Buffer.alloc(n, 1));
|
||||||
|
return { bytesWritten: n };
|
||||||
|
}
|
||||||
|
|
||||||
|
async thumbnail(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
}): Promise<{ bytesWritten: number }> {
|
||||||
|
await writeFile(args.destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A source whose original() writes 10 bytes but blocks on a gate before
|
||||||
|
// returning, so two fetches can be held in flight together. `bothStarted`
|
||||||
|
// resolves once both originals have entered, and `release()` lets them finish.
|
||||||
|
class GatedSource implements ContentSource {
|
||||||
|
private openGate!: () => void;
|
||||||
|
private readonly gate = new Promise<void>((r) => (this.openGate = r));
|
||||||
|
private inFlight = 0;
|
||||||
|
private reachedTwo!: () => void;
|
||||||
|
readonly bothStarted = new Promise<void>((r) => (this.reachedTwo = r));
|
||||||
|
|
||||||
|
async original(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
}): Promise<{ bytesWritten: number }> {
|
||||||
|
this.inFlight += 1;
|
||||||
|
if (this.inFlight === 2) this.reachedTwo();
|
||||||
|
await this.gate;
|
||||||
|
await writeFile(args.destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
}
|
||||||
|
|
||||||
|
async thumbnail(args: {
|
||||||
|
file: EnteFile;
|
||||||
|
destination: string;
|
||||||
|
}): Promise<{ bytesWritten: number }> {
|
||||||
|
await writeFile(args.destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
}
|
||||||
|
|
||||||
|
release(): void {
|
||||||
|
this.openGate();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let root: string;
|
||||||
|
let cacheDir: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-eviction-"));
|
||||||
|
cacheDir = join(root, "cache");
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
const originalPath = (id: number): string =>
|
||||||
|
join(cacheDir, "originals", `${id}.jpg`);
|
||||||
|
|
||||||
|
// Pin an original's mtime to a fixed second-resolution instant so LRU order is
|
||||||
|
// deterministic. Lower `seconds` = older = evicted first.
|
||||||
|
const setMtime = (id: number, seconds: number): void => {
|
||||||
|
utimesSync(originalPath(id), seconds, seconds);
|
||||||
|
};
|
||||||
|
|
||||||
|
const buildCache = (args: {
|
||||||
|
source?: ContentSource;
|
||||||
|
files?: EnteFile[];
|
||||||
|
statfs: StatFsFn;
|
||||||
|
cacheOriginalsMaxBytes?: number;
|
||||||
|
freeBelowBytes?: number;
|
||||||
|
isPinned?: (fileID: number) => boolean;
|
||||||
|
}): { cache: ContentCache; source: SizedSource } => {
|
||||||
|
const source = (args.source as SizedSource) ?? new SizedSource();
|
||||||
|
const byID = new Map<number, EnteFile>();
|
||||||
|
for (const f of args.files ?? [file(1), file(2), file(3), file(4), file(5)])
|
||||||
|
byID.set(f.id, f);
|
||||||
|
const cache = new ContentCache({
|
||||||
|
pools: new RequestPools(),
|
||||||
|
source,
|
||||||
|
cacheDirectory: cacheDir,
|
||||||
|
getFile: (id) => byID.get(id),
|
||||||
|
statfs: args.statfs,
|
||||||
|
cacheOriginalsMaxBytes: args.cacheOriginalsMaxBytes,
|
||||||
|
freeBelowBytes: args.freeBelowBytes,
|
||||||
|
isPinned: args.isPinned,
|
||||||
|
});
|
||||||
|
return { cache, source };
|
||||||
|
};
|
||||||
|
|
||||||
|
// Plenty of free space, so the configured max governs the limit unless a test
|
||||||
|
// dials it down. `bsize` of 1 makes `bavail` the free byte count.
|
||||||
|
const abundantFree: StatFsFn = async () => ({
|
||||||
|
bsize: 1,
|
||||||
|
bavail: 1_000_000_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("originals eviction", () => {
|
||||||
|
it("removes least-recently-used originals to fit the limit", async () => {
|
||||||
|
// Each original is 10 bytes; a 25-byte cap holds two.
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 25,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
await cache.original(1);
|
||||||
|
setMtime(1, 1000);
|
||||||
|
await cache.original(2);
|
||||||
|
setMtime(2, 2000);
|
||||||
|
// Writing the third (total 30 > 25) evicts the oldest, file 1.
|
||||||
|
await cache.original(3);
|
||||||
|
|
||||||
|
expect(existsSync(originalPath(1))).toBe(false);
|
||||||
|
expect(existsSync(originalPath(2))).toBe(true);
|
||||||
|
expect(existsSync(originalPath(3))).toBe(true);
|
||||||
|
expect(cache.pathsFor(1).originalPath).toBeUndefined();
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(20);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips pinned files, evicting the oldest unpinned instead", async () => {
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 25,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
isPinned: (id) => id === 1,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
await cache.original(1);
|
||||||
|
setMtime(1, 1000); // oldest, but pinned
|
||||||
|
await cache.original(2);
|
||||||
|
setMtime(2, 2000);
|
||||||
|
await cache.original(3);
|
||||||
|
|
||||||
|
// File 1 is oldest but pinned, so file 2 is evicted instead.
|
||||||
|
expect(existsSync(originalPath(1))).toBe(true);
|
||||||
|
expect(existsSync(originalPath(2))).toBe(false);
|
||||||
|
expect(existsSync(originalPath(3))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("proceeds over-limit when only pinned files remain", async () => {
|
||||||
|
// A 5-byte cap cannot hold even one 10-byte original.
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 5,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
isPinned: () => true,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const result = await cache.original(1);
|
||||||
|
|
||||||
|
expect(existsSync(result.path)).toBe(true);
|
||||||
|
const status = cache.originalsStatus();
|
||||||
|
expect(status.usedBytes).toBe(10);
|
||||||
|
expect(status.usedBytes).toBeGreaterThan(status.limitBytes ?? 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a just-written original larger than the limit, evicting it only on a later write", async () => {
|
||||||
|
// A 5-byte cap cannot hold even one 10-byte original, but a fetch must
|
||||||
|
// never evict the file it just wrote and is about to return.
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 5,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const result = await cache.original(1);
|
||||||
|
|
||||||
|
// The just-written original survives its own over-budget write and the
|
||||||
|
// returned path exists on disk.
|
||||||
|
expect(existsSync(result.path)).toBe(true);
|
||||||
|
expect(existsSync(originalPath(1))).toBe(true);
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(10);
|
||||||
|
setMtime(1, 1000);
|
||||||
|
|
||||||
|
// A later write finds file 1 eligible and evicts it to make room, while
|
||||||
|
// the newly written file 2 is itself kept over-limit.
|
||||||
|
await cache.original(2);
|
||||||
|
expect(existsSync(originalPath(1))).toBe(false);
|
||||||
|
expect(existsSync(originalPath(2))).toBe(true);
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(10);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("adapts the limit down as the disk fills and up as space returns", async () => {
|
||||||
|
// The configured max is generous; free space drives the limit. Eviction
|
||||||
|
// only kicks in once free space falls below the protected reserve.
|
||||||
|
let free = 1_000_000;
|
||||||
|
const statfs: StatFsFn = async () => ({ bsize: 1, bavail: free });
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs,
|
||||||
|
cacheOriginalsMaxBytes: 1000,
|
||||||
|
freeBelowBytes: 100,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
// Ample free space: the limit is the configured max and nothing is
|
||||||
|
// evicted as three 10-byte originals accumulate.
|
||||||
|
await cache.original(1);
|
||||||
|
setMtime(1, 1000);
|
||||||
|
await cache.original(2);
|
||||||
|
setMtime(2, 2000);
|
||||||
|
await cache.original(3);
|
||||||
|
setMtime(3, 3000);
|
||||||
|
expect(cache.originalsStatus().limitBytes).toBe(1000);
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(30);
|
||||||
|
|
||||||
|
// The disk fills: only 95 bytes free, below the 100-byte reserve. The
|
||||||
|
// next write (used 40) sees limit = min(1000, 40 + 95 - 100) = 35 and
|
||||||
|
// evicts the oldest original (file 1) to fit.
|
||||||
|
free = 95;
|
||||||
|
await cache.original(4);
|
||||||
|
expect(cache.originalsStatus().limitBytes).toBe(35);
|
||||||
|
expect(existsSync(originalPath(1))).toBe(false);
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(30);
|
||||||
|
setMtime(4, 4000);
|
||||||
|
|
||||||
|
// Space returns: the limit rises back to the configured max and the
|
||||||
|
// next write is kept without eviction.
|
||||||
|
free = 1_000_000;
|
||||||
|
await cache.original(5);
|
||||||
|
expect(cache.originalsStatus().limitBytes).toBe(1000);
|
||||||
|
expect(existsSync(originalPath(4))).toBe(true);
|
||||||
|
expect(existsSync(originalPath(5))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bumps an original's mtime when a read returns its path", async () => {
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 1000,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
await cache.original(1);
|
||||||
|
// Age the file well into the past.
|
||||||
|
setMtime(1, 1000);
|
||||||
|
expect(statSync(originalPath(1)).mtimeMs).toBeLessThan(2_000_000);
|
||||||
|
|
||||||
|
// A second read is a cache hit that must touch the file.
|
||||||
|
const before = Date.now();
|
||||||
|
await cache.original(1);
|
||||||
|
expect(statSync(originalPath(1)).mtimeMs).toBeGreaterThanOrEqual(
|
||||||
|
before - 2000,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps both originals when two over-budget fetches race", async () => {
|
||||||
|
// A 15-byte cap holds one 10-byte original but not two. Two fetches are
|
||||||
|
// held in flight together; when both stored files cross the limit,
|
||||||
|
// neither eviction pass may delete the sibling whose path has not yet
|
||||||
|
// been returned, so both survive over-limit.
|
||||||
|
const source = new GatedSource();
|
||||||
|
const { cache } = buildCache({
|
||||||
|
source,
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 15,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const p1 = cache.original(1);
|
||||||
|
const p2 = cache.original(2);
|
||||||
|
await source.bothStarted; // both downloads are in flight before either stores
|
||||||
|
source.release();
|
||||||
|
const [r1, r2] = await Promise.all([p1, p2]);
|
||||||
|
|
||||||
|
// Both returned paths exist on disk even though together they exceed the
|
||||||
|
// cap; nothing was evicted out from under a fetch still in progress.
|
||||||
|
expect(existsSync(r1.path)).toBe(true);
|
||||||
|
expect(existsSync(r2.path)).toBe(true);
|
||||||
|
expect(existsSync(originalPath(1))).toBe(true);
|
||||||
|
expect(existsSync(originalPath(2))).toBe(true);
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(20);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never counts or evicts thumbnails", async () => {
|
||||||
|
const { cache } = buildCache({
|
||||||
|
statfs: abundantFree,
|
||||||
|
cacheOriginalsMaxBytes: 5,
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
await cache.thumbnail(1);
|
||||||
|
await cache.thumbnail(2);
|
||||||
|
|
||||||
|
expect(cache.pathsFor(1).thumbnailPath).toBeDefined();
|
||||||
|
expect(cache.pathsFor(2).thumbnailPath).toBeDefined();
|
||||||
|
expect(cache.originalsStatus().usedBytes).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -99,6 +99,10 @@ describe("Library content wiring", () => {
|
|||||||
cacheDirectory: join(root, "cache"),
|
cacheDirectory: join(root, "cache"),
|
||||||
contentSource: source,
|
contentSource: source,
|
||||||
refreshIntervalSeconds: 3600,
|
refreshIntervalSeconds: 3600,
|
||||||
|
// On-demand wiring only; the background precache (#48) is covered
|
||||||
|
// in precache.test.ts and would race the exact-count assertions.
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const photo = lib.photos.byID({ fileID: 1 });
|
const photo = lib.photos.byID({ fileID: 1 });
|
||||||
@@ -122,6 +126,10 @@ describe("Library content wiring", () => {
|
|||||||
cacheDirectory: join(root, "cache"),
|
cacheDirectory: join(root, "cache"),
|
||||||
contentSource: source,
|
contentSource: source,
|
||||||
refreshIntervalSeconds: 3600,
|
refreshIntervalSeconds: 3600,
|
||||||
|
// On-demand wiring only; the background precache (#48) is covered
|
||||||
|
// in precache.test.ts and would race the exact-count assertions.
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
const results = await lib.thumbnails.ensure({
|
const results = await lib.thumbnails.ensure({
|
||||||
|
|||||||
@@ -0,0 +1,272 @@
|
|||||||
|
/**
|
||||||
|
* Tests for fresh reads (issue #75, an owner amendment to design #36).
|
||||||
|
*
|
||||||
|
* The default read namespaces answer from RAM and never touch the network; a
|
||||||
|
* background loop keeps the local copy current. `fresh()` adds an awaited path:
|
||||||
|
* it forces a refresh, waits for it to complete and persist, and only then
|
||||||
|
* hands back the `albums`/`photos`/`timeline` namespaces, guaranteeing the
|
||||||
|
* local copy reflects a completed server round-trip. The contracts here:
|
||||||
|
*
|
||||||
|
* 1. A fresh read observes a server change that a same-instant default read
|
||||||
|
* would miss (the default path has not refreshed yet).
|
||||||
|
* 2. Concurrent fresh reads coalesce onto one in-flight refresh — three
|
||||||
|
* concurrent `fresh()` calls make exactly one collections round-trip, not
|
||||||
|
* three.
|
||||||
|
* 3. A refresh that fails rejects the fresh read (currency was unavailable),
|
||||||
|
* while the default reads stay silent and keep serving the last good copy.
|
||||||
|
*
|
||||||
|
* The client is the same metadata-only mock the background-refresh tests use:
|
||||||
|
* no crypto, no network, scripted pages, and a record of each call. A long
|
||||||
|
* refresh interval keeps the background timer out of the way so each test's
|
||||||
|
* refreshes are exactly the ones it triggers.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { Library } from "../../src/library/index.js";
|
||||||
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
||||||
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||||
|
|
||||||
|
const USER_ID = 42;
|
||||||
|
|
||||||
|
// Long enough that no background tick fires during a test; each test's
|
||||||
|
// refreshes are only the ones its own `fresh()` calls force.
|
||||||
|
const SLOW_INTERVAL = 3600;
|
||||||
|
|
||||||
|
const collection = (id: number, updationTime: number): Collection => ({
|
||||||
|
id,
|
||||||
|
ownerID: USER_ID,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
name: `album-${id}`,
|
||||||
|
type: "album",
|
||||||
|
updationTime,
|
||||||
|
isShared: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const file = (
|
||||||
|
id: number,
|
||||||
|
collectionID: number,
|
||||||
|
updationTime: number,
|
||||||
|
): EnteFile => ({
|
||||||
|
id,
|
||||||
|
collectionID,
|
||||||
|
ownerID: USER_ID,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
metadata: {
|
||||||
|
title: `file-${id}.jpg`,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: updationTime,
|
||||||
|
modificationTime: updationTime,
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: "aGVhZGVy" },
|
||||||
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
||||||
|
updationTime,
|
||||||
|
});
|
||||||
|
|
||||||
|
class MockClient {
|
||||||
|
userID = USER_ID;
|
||||||
|
failCollections = false;
|
||||||
|
collectionsQueue: CollectionsPage[] = [];
|
||||||
|
filesByCollection = new Map<number, FilesPage[]>();
|
||||||
|
|
||||||
|
collectionsSinceTimes: number[] = [];
|
||||||
|
filesCalls: { collectionID: number; sinceTime: number }[] = [];
|
||||||
|
|
||||||
|
whoami(): { email: string; userID: number } {
|
||||||
|
return { email: "user@example.com", userID: this.userID };
|
||||||
|
}
|
||||||
|
|
||||||
|
async collectionsSince(args: {
|
||||||
|
sinceTime: number;
|
||||||
|
}): Promise<CollectionsPage> {
|
||||||
|
this.collectionsSinceTimes.push(args.sinceTime);
|
||||||
|
if (this.failCollections) throw new Error("network down");
|
||||||
|
return (
|
||||||
|
this.collectionsQueue.shift() ?? {
|
||||||
|
collections: [],
|
||||||
|
deleted: [],
|
||||||
|
cursor: args.sinceTime,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async filesSince(args: {
|
||||||
|
collectionID: number;
|
||||||
|
collectionKey: Uint8Array;
|
||||||
|
sinceTime: number;
|
||||||
|
}): Promise<FilesPage> {
|
||||||
|
this.filesCalls.push({
|
||||||
|
collectionID: args.collectionID,
|
||||||
|
sinceTime: args.sinceTime,
|
||||||
|
});
|
||||||
|
const queue = this.filesByCollection.get(args.collectionID);
|
||||||
|
return (
|
||||||
|
queue?.shift() ?? {
|
||||||
|
files: [],
|
||||||
|
deleted: [],
|
||||||
|
cursor: args.sinceTime,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
filesFor(collectionID: number, ...pages: FilesPage[]): void {
|
||||||
|
this.filesByCollection.set(collectionID, pages);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A client seeded with one collection and one file, opened with an empty cache
|
||||||
|
// so the initial refresh is awaited and post-`open()` state is deterministic.
|
||||||
|
const openSeeded = async (
|
||||||
|
cacheDirectory: string,
|
||||||
|
): Promise<{ client: MockClient; lib: Library }> => {
|
||||||
|
const client = new MockClient();
|
||||||
|
client.collectionsQueue.push({
|
||||||
|
collections: [collection(1, 100)],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 100,
|
||||||
|
});
|
||||||
|
client.filesFor(1, {
|
||||||
|
files: [file(1001, 1, 90)],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 90,
|
||||||
|
});
|
||||||
|
const lib = await Library.open({
|
||||||
|
client,
|
||||||
|
cacheDirectory,
|
||||||
|
refreshIntervalSeconds: SLOW_INTERVAL,
|
||||||
|
});
|
||||||
|
return { client, lib };
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("Library.fresh", () => {
|
||||||
|
let dir: string;
|
||||||
|
let cacheDirectory: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
dir = mkdtempSync(join(tmpdir(), "quak-fresh-"));
|
||||||
|
cacheDirectory = join(dir, "cache");
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("observes a server change a same-instant default read would miss", async () => {
|
||||||
|
const { client, lib } = await openSeeded(cacheDirectory);
|
||||||
|
try {
|
||||||
|
// A new file appears on the server after open, advancing its
|
||||||
|
// collection so the next refresh re-enumerates it.
|
||||||
|
client.collectionsQueue.push({
|
||||||
|
collections: [collection(1, 200)],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 200,
|
||||||
|
});
|
||||||
|
client.filesFor(1, {
|
||||||
|
files: [file(1002, 1, 190)],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 190,
|
||||||
|
});
|
||||||
|
|
||||||
|
// A default read at this instant has not refreshed: it misses 1002.
|
||||||
|
expect(lib.photos.byID({ fileID: 1002 })).toBeUndefined();
|
||||||
|
|
||||||
|
// A fresh read forces the round-trip and sees it.
|
||||||
|
const reads = await lib.fresh();
|
||||||
|
expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
||||||
|
// And the change is now live for the default namespaces too.
|
||||||
|
expect(lib.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
||||||
|
} finally {
|
||||||
|
lib.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("coalesces concurrent fresh reads onto one in-flight refresh", async () => {
|
||||||
|
const { client, lib } = await openSeeded(cacheDirectory);
|
||||||
|
try {
|
||||||
|
client.collectionsQueue.push({
|
||||||
|
collections: [collection(1, 200)],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 200,
|
||||||
|
});
|
||||||
|
client.filesFor(1, {
|
||||||
|
files: [file(1002, 1, 190)],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 190,
|
||||||
|
});
|
||||||
|
|
||||||
|
const collectionsBefore = client.collectionsSinceTimes.length;
|
||||||
|
const filesBefore = client.filesCalls.length;
|
||||||
|
|
||||||
|
// Gate the next collections fetch so all three fresh reads are in
|
||||||
|
// flight together before any of them completes.
|
||||||
|
let release: () => void = () => {};
|
||||||
|
const gate = new Promise<void>((r) => {
|
||||||
|
release = r;
|
||||||
|
});
|
||||||
|
const inner = client.collectionsSince.bind(client);
|
||||||
|
client.collectionsSince = async (args: { sinceTime: number }) => {
|
||||||
|
await gate;
|
||||||
|
return inner(args);
|
||||||
|
};
|
||||||
|
|
||||||
|
const all = Promise.all([lib.fresh(), lib.fresh(), lib.fresh()]);
|
||||||
|
release();
|
||||||
|
const [a, b, c] = await all;
|
||||||
|
|
||||||
|
// Exactly one collections round-trip and one file round-trip served
|
||||||
|
// all three fresh reads.
|
||||||
|
expect(client.collectionsSinceTimes.length).toBe(
|
||||||
|
collectionsBefore + 1,
|
||||||
|
);
|
||||||
|
expect(client.filesCalls.length).toBe(filesBefore + 1);
|
||||||
|
|
||||||
|
// All three observed the change.
|
||||||
|
for (const reads of [a, b, c]) {
|
||||||
|
expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
lib.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects the fresh read when the refresh fails, leaving defaults intact", async () => {
|
||||||
|
const { client, lib } = await openSeeded(cacheDirectory);
|
||||||
|
try {
|
||||||
|
client.failCollections = true;
|
||||||
|
|
||||||
|
// Two concurrent fresh reads both reject, and share one failed
|
||||||
|
// round-trip rather than each making its own.
|
||||||
|
const collectionsBefore = client.collectionsSinceTimes.length;
|
||||||
|
const first = lib.fresh();
|
||||||
|
const second = lib.fresh();
|
||||||
|
await expect(first).rejects.toThrow(/network down/);
|
||||||
|
await expect(second).rejects.toThrow(/network down/);
|
||||||
|
expect(client.collectionsSinceTimes.length).toBe(
|
||||||
|
collectionsBefore + 1,
|
||||||
|
);
|
||||||
|
|
||||||
|
// The default reads never rejected: they still serve the last good
|
||||||
|
// copy, and the failure surfaced through status().
|
||||||
|
expect(lib.photos.byID({ fileID: 1001 })?.fileID).toBe(1001);
|
||||||
|
expect(lib.status().lastError).toMatch(/network down/);
|
||||||
|
|
||||||
|
// Recovery: once the server answers, a fresh read resolves current.
|
||||||
|
client.failCollections = false;
|
||||||
|
client.collectionsQueue.push({
|
||||||
|
collections: [collection(2, 300)],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 300,
|
||||||
|
});
|
||||||
|
const reads = await lib.fresh();
|
||||||
|
expect(reads.albums.byID({ collectionID: 2 })?.collectionID).toBe(
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
expect(lib.status().lastError).toBeUndefined();
|
||||||
|
} finally {
|
||||||
|
lib.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
/**
|
||||||
|
* Tests for the content-similarity search surface over the CLIP index
|
||||||
|
* (issue #50).
|
||||||
|
*
|
||||||
|
* The surface is `lib.mldata`: `forFile` reads the full stored payload from
|
||||||
|
* disk, while `similar` and `searchByEmbedding` rank fileIDs by cosine
|
||||||
|
* similarity over the in-RAM `Float32Array` index alone (no disk, no network).
|
||||||
|
* The fixture uses axis-aligned vectors so the correct cosine ranking is
|
||||||
|
* obvious by inspection; cosine ignores magnitude, so `[2, 0, 0]` ranks above
|
||||||
|
* `[0.8, 0.6, 0]` for a `[1, 0, 0]` query.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { MLDataStore } from "../../src/library/mldata.js";
|
||||||
|
import { makeMLDataAPI, type MLDataAPI } from "../../src/library/mlsearch.js";
|
||||||
|
import type { MLData } from "../../src/mldata-fetch.js";
|
||||||
|
|
||||||
|
// A payload shaped like Ente's: a CLIP embedding plus face data that only the
|
||||||
|
// on-disk payload carries (never the RAM index).
|
||||||
|
const payload = (embedding: number[]): MLData => ({
|
||||||
|
face: { faces: [{ faceID: "f", detection: { box: { x: 0.5 } } }] },
|
||||||
|
clip: { embedding },
|
||||||
|
});
|
||||||
|
|
||||||
|
// A small fixture index. Directions are chosen so every cosine ranking below
|
||||||
|
// is unambiguous.
|
||||||
|
const fixture = (): Map<number, MLData> =>
|
||||||
|
new Map([
|
||||||
|
[10, payload([1, 0, 0])],
|
||||||
|
[20, payload([0.8, 0.6, 0])],
|
||||||
|
[30, payload([0, 1, 0])],
|
||||||
|
[40, payload([-1, 0, 0])],
|
||||||
|
[50, payload([2, 0, 0])],
|
||||||
|
]);
|
||||||
|
|
||||||
|
describe("lib.mldata content-similarity search", () => {
|
||||||
|
let dir: string;
|
||||||
|
let store: MLDataStore;
|
||||||
|
let api: MLDataAPI;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
dir = mkdtempSync(join(tmpdir(), "quak-mlsearch-"));
|
||||||
|
store = await MLDataStore.open(dir);
|
||||||
|
const updation = new Map([...fixture().keys()].map((id) => [id, 1]));
|
||||||
|
await store.storeFetched(fixture(), updation);
|
||||||
|
api = makeMLDataAPI(() => store);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forFile returns the whole stored payload, or undefined when uncached", async () => {
|
||||||
|
const full = await api.forFile({ fileID: 20 });
|
||||||
|
expect(full).toBeDefined();
|
||||||
|
// Face data lives only in the payload, never in the RAM index.
|
||||||
|
expect(full?.face).toBeDefined();
|
||||||
|
expect(full?.clip).toEqual({ embedding: [0.8, 0.6, 0] });
|
||||||
|
expect(await api.forFile({ fileID: 999 })).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("similar ranks other files by cosine and excludes the query itself", () => {
|
||||||
|
// Query is file 10 = [1, 0, 0]. By cosine: 50 (1.0) > 20 (0.8) >
|
||||||
|
// 30 (0) > 40 (-1); 10 itself is left out.
|
||||||
|
const ranked = api.similar({ fileID: 10 });
|
||||||
|
expect(ranked.map((r) => r.fileID)).toEqual([50, 20, 30, 40]);
|
||||||
|
// Cosine ignores magnitude: [2,0,0] is a perfect match for [1,0,0].
|
||||||
|
expect(ranked[0]).toMatchObject({ fileID: 50 });
|
||||||
|
expect(ranked[0].score).toBeCloseTo(1, 5);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("similar honours limit and returns [] for an unindexed file", () => {
|
||||||
|
expect(
|
||||||
|
api.similar({ fileID: 10, limit: 2 }).map((r) => r.fileID),
|
||||||
|
).toEqual([50, 20]);
|
||||||
|
expect(api.similar({ fileID: 999 })).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("searchByEmbedding ranks the index by cosine to the query vector", () => {
|
||||||
|
// Query [0, 1, 0]: 30 (1.0) > 20 (0.6) > {10, 40, 50} all 0, broken by
|
||||||
|
// ascending fileID.
|
||||||
|
const ranked = api.searchByEmbedding({ embedding: [0, 1, 0] });
|
||||||
|
expect(ranked.map((r) => r.fileID)).toEqual([30, 20, 10, 40, 50]);
|
||||||
|
expect(ranked[0].score).toBeCloseTo(1, 5);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
api
|
||||||
|
.searchByEmbedding({ embedding: [0, 1, 0], limit: 2 })
|
||||||
|
.map((r) => r.fileID),
|
||||||
|
).toEqual([30, 20]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("searchByEmbedding returns [] for a wrong-length or zero query", () => {
|
||||||
|
expect(api.searchByEmbedding({ embedding: [1, 0] })).toEqual([]);
|
||||||
|
expect(api.searchByEmbedding({ embedding: [0, 0, 0] })).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("degrades to empty results when no ML store is present", async () => {
|
||||||
|
const none = makeMLDataAPI(() => undefined);
|
||||||
|
expect(await none.forFile({ fileID: 10 })).toBeUndefined();
|
||||||
|
expect(none.similar({ fileID: 10 })).toEqual([]);
|
||||||
|
expect(none.searchByEmbedding({ embedding: [1, 0, 0] })).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,436 @@
|
|||||||
|
/**
|
||||||
|
* The aggressive local precache (issue #48), driven from `Library.open`.
|
||||||
|
*
|
||||||
|
* Two background fills start with no caller input: every thumbnail in the
|
||||||
|
* account newest first, and the originals of the pinned set (the favorites
|
||||||
|
* album then the latest `precacheOriginalsDays` window ending at the newest
|
||||||
|
* file). Both run through the shared pools at background priority, so on-demand
|
||||||
|
* work always preempts them; both report through `status()`. The pinned set is
|
||||||
|
* the eviction predicate (#47), so a pinned original is never evicted and a
|
||||||
|
* file that leaves the set becomes an ordinary, evictable original.
|
||||||
|
*
|
||||||
|
* The unit tests drive `Precache` against a fake cache that records what it was
|
||||||
|
* asked to fetch (order and kind) with no pool or network; the integration
|
||||||
|
* tests drive the real wiring through `Library.open` with a stub content
|
||||||
|
* source, and the eviction test drives the real `ContentCache`.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync, existsSync, utimesSync } from "node:fs";
|
||||||
|
import { writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { Precache, type PrecacheCache } from "../../src/library/precache.js";
|
||||||
|
import {
|
||||||
|
deriveRecords,
|
||||||
|
type DerivedRecords,
|
||||||
|
} from "../../src/library/records.js";
|
||||||
|
import {
|
||||||
|
ContentCache,
|
||||||
|
type ContentSource,
|
||||||
|
type EnsureResult,
|
||||||
|
type StatFsFn,
|
||||||
|
} from "../../src/library/content.js";
|
||||||
|
import { RequestPools } from "../../src/library/pools.js";
|
||||||
|
import { Library } from "../../src/library/index.js";
|
||||||
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
||||||
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
||||||
|
|
||||||
|
const DAY_MICROS = 24 * 60 * 60 * 1000 * 1000;
|
||||||
|
|
||||||
|
const collection = (
|
||||||
|
id: number,
|
||||||
|
type: Collection["type"] = "album",
|
||||||
|
): Collection => ({
|
||||||
|
id,
|
||||||
|
ownerID: 1,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
name: `album-${id}`,
|
||||||
|
type,
|
||||||
|
updationTime: 1,
|
||||||
|
isShared: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
// A file whose creationTime (microseconds) places it `daysAgo` days before a
|
||||||
|
// fixed reference instant, so the latest-week window is deterministic.
|
||||||
|
const REFERENCE_MICROS = 1_000 * DAY_MICROS;
|
||||||
|
const file = (id: number, collectionID: number, daysAgo: number): EnteFile => ({
|
||||||
|
id,
|
||||||
|
collectionID,
|
||||||
|
ownerID: 1,
|
||||||
|
key: new Uint8Array([id & 0xff]),
|
||||||
|
metadata: {
|
||||||
|
title: `file-${id}.jpg`,
|
||||||
|
fileType: "image",
|
||||||
|
creationTime: REFERENCE_MICROS - daysAgo * DAY_MICROS,
|
||||||
|
modificationTime: 0,
|
||||||
|
},
|
||||||
|
file: { decryptionHeader: "aGVhZGVy" },
|
||||||
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
||||||
|
updationTime: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
const records = (
|
||||||
|
collections: Collection[],
|
||||||
|
files: EnteFile[],
|
||||||
|
): DerivedRecords => deriveRecords(collections, files);
|
||||||
|
|
||||||
|
// A fake cache: records every fetch (kind + order) and reports the files it has
|
||||||
|
// stored via `pathsFor`. `presentThumbs`/`presentOriginals` seed already-cached
|
||||||
|
// files so the precache skips them with a single lookup.
|
||||||
|
class FakeCache implements PrecacheCache {
|
||||||
|
readonly thumbFetched: number[] = [];
|
||||||
|
readonly originalFetched: number[] = [];
|
||||||
|
readonly presentThumbs = new Set<number>();
|
||||||
|
readonly presentOriginals = new Set<number>();
|
||||||
|
|
||||||
|
pathsFor(fileID: number): {
|
||||||
|
originalPath?: string;
|
||||||
|
thumbnailPath?: string;
|
||||||
|
} {
|
||||||
|
const out: { originalPath?: string; thumbnailPath?: string } = {};
|
||||||
|
if (this.presentThumbs.has(fileID))
|
||||||
|
out.thumbnailPath = `/thumbs/${fileID}`;
|
||||||
|
if (this.presentOriginals.has(fileID))
|
||||||
|
out.originalPath = `/originals/${fileID}`;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensureThumbnails(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
priority: "background";
|
||||||
|
}): Promise<EnsureResult[]> {
|
||||||
|
return args.fileIDs.map((fileID) => {
|
||||||
|
this.thumbFetched.push(fileID);
|
||||||
|
this.presentThumbs.add(fileID);
|
||||||
|
return { fileID, path: `/thumbs/${fileID}` };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensureOriginals(args: {
|
||||||
|
fileIDs: number[];
|
||||||
|
}): Promise<EnsureResult[]> {
|
||||||
|
return args.fileIDs.map((fileID) => {
|
||||||
|
this.originalFetched.push(fileID);
|
||||||
|
this.presentOriginals.add(fileID);
|
||||||
|
return { fileID, path: `/originals/${fileID}` };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve once a predicate holds, polling the microtask queue; fails fast
|
||||||
|
// rather than hanging the suite.
|
||||||
|
const until = async (predicate: () => boolean): Promise<void> => {
|
||||||
|
for (let i = 0; i < 1000; i++) {
|
||||||
|
if (predicate()) return;
|
||||||
|
await new Promise((r) => setTimeout(r, 1));
|
||||||
|
}
|
||||||
|
throw new Error("condition not met in time");
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("Precache unit", () => {
|
||||||
|
it("precaches every thumbnail newest first, skipping present ones", async () => {
|
||||||
|
const cols = [collection(1)];
|
||||||
|
const files = [
|
||||||
|
file(1, 1, 0),
|
||||||
|
file(2, 1, 1),
|
||||||
|
file(3, 1, 2),
|
||||||
|
file(4, 1, 3),
|
||||||
|
];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
cache.presentThumbs.add(3); // already on disk: skipped
|
||||||
|
const pre = new Precache({ originals: false });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
pre.start();
|
||||||
|
|
||||||
|
await until(() => cache.thumbFetched.length === 3);
|
||||||
|
// Newest first (file 1 is newest), file 3 skipped by a lookup.
|
||||||
|
expect(cache.thumbFetched).toEqual([1, 2, 4]);
|
||||||
|
expect(cache.originalFetched).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("pins favorites then the latest-week window and precaches their originals in that order", async () => {
|
||||||
|
const cols = [collection(1), collection(2, "favorites")];
|
||||||
|
// File 10 is an old favorite (30 days old); files 1..3 are within the
|
||||||
|
// 7-day window; file 4 is outside it.
|
||||||
|
const files = [
|
||||||
|
file(1, 1, 0),
|
||||||
|
file(2, 1, 2),
|
||||||
|
file(3, 1, 6),
|
||||||
|
file(4, 1, 20),
|
||||||
|
file(10, 2, 30), // favorite, old
|
||||||
|
];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
pre.start();
|
||||||
|
|
||||||
|
await until(() => cache.originalFetched.length === 4);
|
||||||
|
// Favorite (10) first, then the window newest-first (1, 2, 3). File 4
|
||||||
|
// is outside the window and never pinned.
|
||||||
|
expect(cache.originalFetched).toEqual([10, 1, 2, 3]);
|
||||||
|
expect(pre.isPinned(10)).toBe(true);
|
||||||
|
expect(pre.isPinned(1)).toBe(true);
|
||||||
|
expect(pre.isPinned(4)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops a file from the pinned set when the window moves past it", () => {
|
||||||
|
const cols = [collection(1)];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
pre.bind(cache);
|
||||||
|
|
||||||
|
pre.update(records(cols, [file(1, 1, 0), file(2, 1, 3)]));
|
||||||
|
expect(pre.isPinned(2)).toBe(true);
|
||||||
|
|
||||||
|
// A newer file arrives; the window's newest end moves forward so the
|
||||||
|
// 3-day-old file 2 (now 13 days behind the newest) falls out.
|
||||||
|
pre.update(
|
||||||
|
records(cols, [file(3, 1, -10), file(1, 1, 0), file(2, 1, 3)]),
|
||||||
|
);
|
||||||
|
expect(pre.isPinned(3)).toBe(true);
|
||||||
|
expect(pre.isPinned(2)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports progress through status()", async () => {
|
||||||
|
const cols = [collection(1), collection(2, "favorites")];
|
||||||
|
const files = [file(1, 1, 0), file(2, 1, 1), file(10, 2, 0)];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
|
||||||
|
const before = pre.status();
|
||||||
|
expect(before.thumbnailsTotal).toBe(3);
|
||||||
|
expect(before.thumbnailsCached).toBe(0);
|
||||||
|
expect(before.originalsPinned).toBe(3); // files 1, 2, 10 all in window
|
||||||
|
expect(before.originalsCached).toBe(0);
|
||||||
|
|
||||||
|
pre.start();
|
||||||
|
await until(
|
||||||
|
() =>
|
||||||
|
cache.thumbFetched.length === 3 &&
|
||||||
|
cache.originalFetched.length === 3,
|
||||||
|
);
|
||||||
|
const after = pre.status();
|
||||||
|
expect(after.thumbnailsCached).toBe(3);
|
||||||
|
expect(after.originalsCached).toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("honours the disable flags", async () => {
|
||||||
|
const cols = [collection(1)];
|
||||||
|
const files = [file(1, 1, 0)];
|
||||||
|
const cache = new FakeCache();
|
||||||
|
const pre = new Precache({ thumbnails: false, originals: false });
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
pre.start();
|
||||||
|
|
||||||
|
await new Promise((r) => setTimeout(r, 20));
|
||||||
|
expect(cache.thumbFetched).toEqual([]);
|
||||||
|
expect(cache.originalFetched).toEqual([]);
|
||||||
|
expect(pre.isPinned(1)).toBe(false);
|
||||||
|
expect(pre.status().thumbnailsTotal).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- Integration through the real ContentCache and Library ----
|
||||||
|
|
||||||
|
const enteFile = (id: number, collectionID: number): EnteFile =>
|
||||||
|
file(id, collectionID, 0);
|
||||||
|
|
||||||
|
describe("Precache eviction integration", () => {
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-precache-evict-"));
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never evicts a pinned original the precache put in place", async () => {
|
||||||
|
const cacheDir = join(root, "cache");
|
||||||
|
// File 1 is the favorites album's only file (pinned regardless of age);
|
||||||
|
// files 2 and 3 sit outside the latest-week window, so only file 1 is
|
||||||
|
// pinned. The by-id map serves the bytes for each fetch.
|
||||||
|
const cols = [collection(1), collection(2, "favorites")];
|
||||||
|
const files = [file(1, 2, 30), file(2, 1, 40), file(3, 1, 50)];
|
||||||
|
const byID = new Map<number, EnteFile>(files.map((f) => [f.id, f]));
|
||||||
|
const source: ContentSource = {
|
||||||
|
original: async ({ destination }) => {
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ destination }) => {
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const statfs: StatFsFn = async () => ({
|
||||||
|
bsize: 1,
|
||||||
|
bavail: 1_000_000_000,
|
||||||
|
});
|
||||||
|
const pre = new Precache({ originalsDays: 7 });
|
||||||
|
const cache = new ContentCache({
|
||||||
|
pools: new RequestPools(),
|
||||||
|
source,
|
||||||
|
cacheDirectory: cacheDir,
|
||||||
|
getFile: (id) => byID.get(id),
|
||||||
|
statfs,
|
||||||
|
cacheOriginalsMaxBytes: 25, // holds two 10-byte originals
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
isPinned: (id) => pre.isPinned(id),
|
||||||
|
});
|
||||||
|
pre.bind(cache);
|
||||||
|
pre.update(records(cols, files));
|
||||||
|
expect(pre.isPinned(1)).toBe(true);
|
||||||
|
expect(pre.isPinned(2)).toBe(false);
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
// Fill three originals; the 25-byte cap forces an eviction on the
|
||||||
|
// third, and the pinned file 1 must survive it even though it is the
|
||||||
|
// least-recently-used.
|
||||||
|
await cache.original(1);
|
||||||
|
utimesSync(join(cacheDir, "originals", "1.jpg"), 1000, 1000); // oldest
|
||||||
|
await cache.original(2);
|
||||||
|
utimesSync(join(cacheDir, "originals", "2.jpg"), 2000, 2000);
|
||||||
|
await cache.original(3);
|
||||||
|
|
||||||
|
expect(existsSync(join(cacheDir, "originals", "1.jpg"))).toBe(true);
|
||||||
|
expect(existsSync(join(cacheDir, "originals", "2.jpg"))).toBe(false);
|
||||||
|
expect(existsSync(join(cacheDir, "originals", "3.jpg"))).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Precache preemption", () => {
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-precache-preempt-"));
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lets an on-demand original preempt the background originals fill", async () => {
|
||||||
|
const byID = new Map<number, EnteFile>([
|
||||||
|
[1, enteFile(1, 1)],
|
||||||
|
[2, enteFile(2, 1)],
|
||||||
|
[3, enteFile(3, 1)],
|
||||||
|
]);
|
||||||
|
const finished: number[] = [];
|
||||||
|
let openGate!: () => void;
|
||||||
|
const gate = new Promise<void>((r) => (openGate = r));
|
||||||
|
let sawFirst!: () => void;
|
||||||
|
const firstStarted = new Promise<void>((r) => (sawFirst = r));
|
||||||
|
let started = 0;
|
||||||
|
const source: ContentSource = {
|
||||||
|
original: async ({ file: f, destination }) => {
|
||||||
|
if (++started === 1) sawFirst();
|
||||||
|
await gate;
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
finished.push(f.id);
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ destination }) => {
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
// One content slot, so file 1 holds it while 2 and 3 wait.
|
||||||
|
const cache = new ContentCache({
|
||||||
|
pools: new RequestPools({ contentConcurrency: 1 }),
|
||||||
|
source,
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
getFile: (id) => byID.get(id),
|
||||||
|
statfs: async () => ({ bsize: 1, bavail: 1_000_000_000 }),
|
||||||
|
freeBelowBytes: 0,
|
||||||
|
});
|
||||||
|
await cache.open();
|
||||||
|
|
||||||
|
const pA = cache.ensureOriginals({ fileIDs: [1] }); // background
|
||||||
|
await firstStarted; // file 1 now holds the only slot
|
||||||
|
const pB = cache.original(2); // on-demand, queued behind file 1
|
||||||
|
const pC = cache.ensureOriginals({ fileIDs: [3] }); // background, queued
|
||||||
|
await new Promise((r) => setTimeout(r, 5)); // let both enqueue
|
||||||
|
openGate();
|
||||||
|
await Promise.all([pA, pB, pC]);
|
||||||
|
|
||||||
|
// On-demand file 2 was served before the background file 3.
|
||||||
|
expect(finished).toEqual([1, 2, 3]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Precache through Library.open", () => {
|
||||||
|
let root: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
root = mkdtempSync(join(tmpdir(), "quak-precache-lib-"));
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
if (root && existsSync(root))
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
class MockClient {
|
||||||
|
served = false;
|
||||||
|
whoami(): { email: string; userID: number } {
|
||||||
|
return { email: "u@example.com", userID: 7 };
|
||||||
|
}
|
||||||
|
async collectionsSince(): Promise<CollectionsPage> {
|
||||||
|
if (this.served) return { collections: [], deleted: [], cursor: 1 };
|
||||||
|
this.served = true;
|
||||||
|
return {
|
||||||
|
collections: [collection(1), collection(2, "favorites")],
|
||||||
|
deleted: [],
|
||||||
|
cursor: 1,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
async filesSince(args: { collectionID: number }): Promise<FilesPage> {
|
||||||
|
const files =
|
||||||
|
args.collectionID === 1
|
||||||
|
? [enteFile(1, 1), enteFile(2, 1)]
|
||||||
|
: [enteFile(3, 2)];
|
||||||
|
return { files, deleted: [], cursor: 1 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
it("starts both precaches from open() and reports them in status()", async () => {
|
||||||
|
const thumbFetched = new Set<number>();
|
||||||
|
const origFetched = new Set<number>();
|
||||||
|
const source: ContentSource = {
|
||||||
|
original: async ({ file: f, destination }) => {
|
||||||
|
origFetched.add(f.id);
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
thumbnail: async ({ file: f, destination }) => {
|
||||||
|
thumbFetched.add(f.id);
|
||||||
|
await writeFile(destination, Buffer.alloc(10, 1));
|
||||||
|
return { bytesWritten: 10 };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const lib = await Library.open({
|
||||||
|
client: new MockClient(),
|
||||||
|
cacheDirectory: join(root, "cache"),
|
||||||
|
contentSource: source,
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Every file's thumbnail is precached; the favorite (file 3) and the
|
||||||
|
// week's files (1, 2) all have their originals precached.
|
||||||
|
await until(() => thumbFetched.size === 3 && origFetched.size === 3);
|
||||||
|
const status = lib.status();
|
||||||
|
expect(status.thumbnailsTotal).toBe(3);
|
||||||
|
expect(status.thumbnailsCached).toBe(3);
|
||||||
|
expect(status.originalsPinned).toBe(3);
|
||||||
|
expect(status.originalsCached).toBe(3);
|
||||||
|
lib.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -6,17 +6,30 @@
|
|||||||
* working thumbnails, others return 404 or empty bodies. The tests
|
* working thumbnails, others return 404 or empty bodies. The tests
|
||||||
* verify that the detection and repair logic handles each case correctly.
|
* verify that the detection and repair logic handles each case correctly.
|
||||||
*
|
*
|
||||||
|
* As of issue #52 both helpers take an open `Library` for enumeration and the
|
||||||
|
* `Client` for the API operations that stay unchanged (the thumbnail existence
|
||||||
|
* check, and the encrypt-and-upload path). `fixMissingThumbnails` reads each
|
||||||
|
* original through the library's content cache (`photo.original()`).
|
||||||
|
*
|
||||||
* `fixMissingThumbnails` is the most complex function in quak: it
|
* `fixMissingThumbnails` is the most complex function in quak: it
|
||||||
* downloads the original file, generates a JPEG thumbnail with jpeg-js,
|
* downloads the original file, generates a JPEG thumbnail with jpeg-js,
|
||||||
* encrypts it with secretstream push, gets a presigned upload URL,
|
* encrypts it with secretstream push, gets a presigned upload URL,
|
||||||
* uploads to S3, and registers the new thumbnail with the API. The
|
* uploads to S3, and registers the new thumbnail with the API. The
|
||||||
* test verifies each step actually happened and the uploaded data is
|
* test verifies each step actually happened and the uploaded data is
|
||||||
* a valid encrypted blob that decrypts to a JPEG.
|
* a valid encrypted blob that decrypts to a JPEG.
|
||||||
|
*
|
||||||
|
* It regenerates thumbnails for baseline JPEGs only, because `jpeg-js` decodes
|
||||||
|
* only JPEG. A non-JPEG image (PNG, HEIC) or a video is reported as "skipped
|
||||||
|
* (unsupported)" rather than crashing the decoder into an opaque failure
|
||||||
|
* (issue #17); the mixed test below locks that distinction down.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
import sodium from "libsodium-wrappers-sumo";
|
import sodium from "libsodium-wrappers-sumo";
|
||||||
import * as jpegJs from "jpeg-js";
|
import * as jpegJs from "jpeg-js";
|
||||||
import { beforeAll, describe, expect, it } from "vitest";
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
import {
|
import {
|
||||||
init,
|
init,
|
||||||
toBase64,
|
toBase64,
|
||||||
@@ -27,6 +40,7 @@ import {
|
|||||||
} from "../../src/crypto/index.js";
|
} from "../../src/crypto/index.js";
|
||||||
import { SRP, SrpServer } from "fast-srp-hap";
|
import { SRP, SrpServer } from "fast-srp-hap";
|
||||||
import { Client } from "../../src/client.js";
|
import { Client } from "../../src/client.js";
|
||||||
|
import { Library } from "../../src/library/index.js";
|
||||||
import {
|
import {
|
||||||
listMissingThumbnails,
|
listMissingThumbnails,
|
||||||
fixMissingThumbnails,
|
fixMissingThumbnails,
|
||||||
@@ -42,6 +56,7 @@ const TEST_EMAIL = "thumb@example.com";
|
|||||||
const TEST_PASSWORD = "thumbpass";
|
const TEST_PASSWORD = "thumbpass";
|
||||||
const TEST_OPS = 2;
|
const TEST_OPS = 2;
|
||||||
const TEST_MEM = 64 * 1024 * 1024;
|
const TEST_MEM = 64 * 1024 * 1024;
|
||||||
|
const TEST_TIME = 1700000000000000;
|
||||||
|
|
||||||
interface ThumbMockState {
|
interface ThumbMockState {
|
||||||
verifier: Buffer;
|
verifier: Buffer;
|
||||||
@@ -63,8 +78,17 @@ interface ThumbMockState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mock: ThumbMockState;
|
let mock: ThumbMockState;
|
||||||
|
let tmpRoot: string;
|
||||||
|
|
||||||
const buildThumbMock = async (): Promise<ThumbMockState> => {
|
// PNG signature bytes — enough for `fixMissingThumbnails` to recognise a
|
||||||
|
// non-JPEG image and skip it. It need not be a decodable PNG.
|
||||||
|
const PNG_BYTES = new Uint8Array([
|
||||||
|
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const buildThumbMock = async (opts?: {
|
||||||
|
extraFormats?: boolean;
|
||||||
|
}): Promise<ThumbMockState> => {
|
||||||
const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
|
const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
|
||||||
const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM);
|
const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM);
|
||||||
const loginSubKeyBytes = deriveLoginSubkey(kek);
|
const loginSubKeyBytes = deriveLoginSubkey(kek);
|
||||||
@@ -102,7 +126,6 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
|||||||
opsLimit: TEST_OPS,
|
opsLimit: TEST_OPS,
|
||||||
};
|
};
|
||||||
|
|
||||||
// One collection with 3 files: ok thumbnail, empty thumbnail, 404 thumbnail
|
|
||||||
const collKey = sodium.crypto_secretbox_keygen();
|
const collKey = sodium.crypto_secretbox_keygen();
|
||||||
const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
||||||
const encCK = sodium.crypto_secretbox_easy(collKey, ckN, masterKey);
|
const encCK = sodium.crypto_secretbox_easy(collKey, ckN, masterKey);
|
||||||
@@ -118,10 +141,11 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
|||||||
encryptedName: toBase64(encCN),
|
encryptedName: toBase64(encCN),
|
||||||
nameDecryptionNonce: toBase64(cnN),
|
nameDecryptionNonce: toBase64(cnN),
|
||||||
type: "album",
|
type: "album",
|
||||||
updationTime: 1700000000000000,
|
updationTime: TEST_TIME,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Generate a real tiny JPEG via jpeg-js
|
// Generate a real tiny JPEG via jpeg-js, used as the encrypted body of the
|
||||||
|
// JPEG files so a repair actually decodes and re-encodes real pixels.
|
||||||
const w = 100;
|
const w = 100;
|
||||||
const h = 80;
|
const h = 80;
|
||||||
const pixels = new Uint8Array(w * h * 4);
|
const pixels = new Uint8Array(w * h * 4);
|
||||||
@@ -131,26 +155,32 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
|||||||
pixels[i + 2] = 0; // B
|
pixels[i + 2] = 0; // B
|
||||||
pixels[i + 3] = 255; // A
|
pixels[i + 3] = 255; // A
|
||||||
}
|
}
|
||||||
const tinyJpeg = jpegJs.encode(
|
const tinyJpeg = new Uint8Array(
|
||||||
{ data: pixels, width: w, height: h },
|
jpegJs.encode({ data: pixels, width: w, height: h }, 80).data,
|
||||||
80,
|
);
|
||||||
).data;
|
|
||||||
|
|
||||||
const fileKeys: Record<number, Uint8Array> = {};
|
const fileKeys: Record<number, Uint8Array> = {};
|
||||||
const fileCiphertexts: Record<number, Uint8Array> = {};
|
const fileCiphertexts: Record<number, Uint8Array> = {};
|
||||||
const rawFiles: Record<string, unknown>[] = [];
|
|
||||||
|
|
||||||
for (const fileID of [100, 101, 102]) {
|
// Build one raw file record: encrypt its metadata and its body under a
|
||||||
|
// fresh per-file key, and record the key and ciphertext for the mock to
|
||||||
|
// serve and for the test to verify against.
|
||||||
|
const makeRawFile = (
|
||||||
|
fileID: number,
|
||||||
|
fileType: number,
|
||||||
|
title: string,
|
||||||
|
body: Uint8Array,
|
||||||
|
): Record<string, unknown> => {
|
||||||
const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
fileKeys[fileID] = fk;
|
fileKeys[fileID] = fk;
|
||||||
const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES);
|
||||||
const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey);
|
const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey);
|
||||||
|
|
||||||
const meta = JSON.stringify({
|
const meta = JSON.stringify({
|
||||||
title: `file-${fileID}.jpg`,
|
title,
|
||||||
fileType: 0,
|
fileType,
|
||||||
creationTime: 1700000000000000,
|
creationTime: TEST_TIME,
|
||||||
modificationTime: 1700000000000000,
|
modificationTime: TEST_TIME,
|
||||||
});
|
});
|
||||||
const metaPush =
|
const metaPush =
|
||||||
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
||||||
@@ -161,18 +191,17 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
|||||||
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Encrypt the tiny JPEG as the file body
|
|
||||||
const filePush =
|
const filePush =
|
||||||
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(fk);
|
||||||
const encFile = sodium.crypto_secretstream_xchacha20poly1305_push(
|
const encFile = sodium.crypto_secretstream_xchacha20poly1305_push(
|
||||||
filePush.state,
|
filePush.state,
|
||||||
new Uint8Array(tinyJpeg),
|
body,
|
||||||
null,
|
null,
|
||||||
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
||||||
);
|
);
|
||||||
fileCiphertexts[fileID] = encFile;
|
fileCiphertexts[fileID] = encFile;
|
||||||
|
|
||||||
rawFiles.push({
|
return {
|
||||||
id: fileID,
|
id: fileID,
|
||||||
collectionID: 1,
|
collectionID: 1,
|
||||||
ownerID: 42,
|
ownerID: 42,
|
||||||
@@ -186,8 +215,30 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
|||||||
thumbnail: {
|
thumbnail: {
|
||||||
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
decryptionHeader: toBase64(sodium.randombytes_buf(24)),
|
||||||
},
|
},
|
||||||
updationTime: 1700000000000000,
|
updationTime: TEST_TIME,
|
||||||
});
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// Three JPEG files: ok thumbnail, empty thumbnail, 404 thumbnail.
|
||||||
|
const rawFiles: Record<string, unknown>[] = [];
|
||||||
|
for (const fileID of [100, 101, 102]) {
|
||||||
|
rawFiles.push(makeRawFile(fileID, 0, `file-${fileID}.jpg`, tinyJpeg));
|
||||||
|
}
|
||||||
|
const thumbnailBehavior: Record<number, "ok" | "empty" | "404" | "500"> = {
|
||||||
|
100: "ok",
|
||||||
|
101: "empty",
|
||||||
|
102: "404",
|
||||||
|
};
|
||||||
|
|
||||||
|
// For the issue #17 mixed test: a non-JPEG image and a video, both with a
|
||||||
|
// missing (404) thumbnail so they surface in the missing list too.
|
||||||
|
if (opts?.extraFormats) {
|
||||||
|
rawFiles.push(makeRawFile(103, 0, "file-103.png", PNG_BYTES));
|
||||||
|
rawFiles.push(
|
||||||
|
makeRawFile(104, 1, "file-104.mp4", new Uint8Array([0, 0, 0, 1])),
|
||||||
|
);
|
||||||
|
thumbnailBehavior[103] = "404";
|
||||||
|
thumbnailBehavior[104] = "404";
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -206,11 +257,7 @@ const buildThumbMock = async (): Promise<ThumbMockState> => {
|
|||||||
filesByCollection: { 1: rawFiles },
|
filesByCollection: { 1: rawFiles },
|
||||||
fileCiphertexts,
|
fileCiphertexts,
|
||||||
fileKeys,
|
fileKeys,
|
||||||
thumbnailBehavior: {
|
thumbnailBehavior,
|
||||||
100: "ok",
|
|
||||||
101: "empty",
|
|
||||||
102: "404",
|
|
||||||
},
|
|
||||||
uploadedThumbnails: [],
|
uploadedThumbnails: [],
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
@@ -381,6 +428,31 @@ const countingFetch = (
|
|||||||
return { fetch: fake as typeof globalThis.fetch, matched: () => matched };
|
return { fetch: fake as typeof globalThis.fetch, matched: () => matched };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Open a library over a mock-backed client. As the CLI does for point commands,
|
||||||
|
// the background precache is off and the refresh interval is long, and the
|
||||||
|
// library client omits `fetchMLData` so no background ML fetch runs. The real
|
||||||
|
// `Client` is still used for the API operations the helpers perform directly.
|
||||||
|
const openLib = (client: Client): Promise<Library> =>
|
||||||
|
Library.open({
|
||||||
|
client: {
|
||||||
|
whoami: () => client.whoami(),
|
||||||
|
collectionsSince: (args) => client.collectionsSince(args),
|
||||||
|
filesSince: (args) => client.filesSince(args),
|
||||||
|
contentSource: () => client.contentSource(),
|
||||||
|
},
|
||||||
|
cacheDirectory: mkdtempSync(join(tmpRoot, "cache-")),
|
||||||
|
refreshIntervalSeconds: 3600,
|
||||||
|
precacheThumbnails: false,
|
||||||
|
precacheOriginals: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const login = (fetch: typeof globalThis.fetch, retry?: RetryOptions) =>
|
||||||
|
Client.login({
|
||||||
|
email: TEST_EMAIL,
|
||||||
|
password: TEST_PASSWORD,
|
||||||
|
apiOptions: retry ? { fetch, retry } : { fetch },
|
||||||
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Tests
|
// Tests
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -389,17 +461,21 @@ beforeAll(async () => {
|
|||||||
await init();
|
await init();
|
||||||
await sodium.ready;
|
await sodium.ready;
|
||||||
mock = await buildThumbMock();
|
mock = await buildThumbMock();
|
||||||
|
tmpRoot = mkdtempSync(join(tmpdir(), "quak-thumb-test-"));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
if (tmpRoot && existsSync(tmpRoot))
|
||||||
|
rmSync(tmpRoot, { recursive: true, force: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("listMissingThumbnails", () => {
|
describe("listMissingThumbnails", () => {
|
||||||
it("identifies files with empty and 404 thumbnails, ignores working ones", async () => {
|
it("identifies files with empty and 404 thumbnails, ignores working ones", async () => {
|
||||||
const client = await Client.login({
|
const client = await login(buildThumbFetch(mock));
|
||||||
email: TEST_EMAIL,
|
const lib = await openLib(client);
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildThumbFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
const missing = await listMissingThumbnails(client);
|
const missing = await listMissingThumbnails(lib, client);
|
||||||
|
lib.close();
|
||||||
|
|
||||||
// File 100 has a working thumbnail → not reported
|
// File 100 has a working thumbnail → not reported
|
||||||
// File 101 has an empty thumbnail → reported
|
// File 101 has an empty thumbnail → reported
|
||||||
@@ -436,13 +512,11 @@ describe("listMissingThumbnails", () => {
|
|||||||
buildThumbFetch(failingMock),
|
buildThumbFetch(failingMock),
|
||||||
(url) => url.includes("thumbnails.ente.io") && url.includes("102"),
|
(url) => url.includes("thumbnails.ente.io") && url.includes("102"),
|
||||||
);
|
);
|
||||||
const client = await Client.login({
|
const client = await login(counted.fetch, { ...noWait });
|
||||||
email: TEST_EMAIL,
|
const lib = await openLib(client);
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: counted.fetch, retry: { ...noWait } },
|
|
||||||
});
|
|
||||||
|
|
||||||
const missing = await listMissingThumbnails(client);
|
const missing = await listMissingThumbnails(lib, client);
|
||||||
|
lib.close();
|
||||||
|
|
||||||
// Only the genuinely empty thumbnail is reported.
|
// Only the genuinely empty thumbnail is reported.
|
||||||
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
||||||
@@ -475,13 +549,11 @@ describe("listMissingThumbnails", () => {
|
|||||||
return inner(input, init);
|
return inner(input, init);
|
||||||
}) as typeof globalThis.fetch;
|
}) as typeof globalThis.fetch;
|
||||||
|
|
||||||
const client = await Client.login({
|
const client = await login(fetch, { ...noWait });
|
||||||
email: TEST_EMAIL,
|
const lib = await openLib(client);
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch, retry: { ...noWait } },
|
|
||||||
});
|
|
||||||
|
|
||||||
const missing = await listMissingThumbnails(client);
|
const missing = await listMissingThumbnails(lib, client);
|
||||||
|
lib.close();
|
||||||
|
|
||||||
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
expect(missing.map((m) => m.fileID)).toEqual([101]);
|
||||||
expect(thumbRequests).toBe(4);
|
expect(thumbRequests).toBe(4);
|
||||||
@@ -500,13 +572,11 @@ describe("listMissingThumbnails", () => {
|
|||||||
mockWithDupes.filesByCollection[2] =
|
mockWithDupes.filesByCollection[2] =
|
||||||
mockWithDupes.filesByCollection[1]!;
|
mockWithDupes.filesByCollection[1]!;
|
||||||
|
|
||||||
const client = await Client.login({
|
const client = await login(buildThumbFetch(mockWithDupes));
|
||||||
email: TEST_EMAIL,
|
const lib = await openLib(client);
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildThumbFetch(mockWithDupes) },
|
|
||||||
});
|
|
||||||
|
|
||||||
const missing = await listMissingThumbnails(client);
|
const missing = await listMissingThumbnails(lib, client);
|
||||||
|
lib.close();
|
||||||
|
|
||||||
// Should still be 2, not 4 (each file checked only once)
|
// Should still be 2, not 4 (each file checked only once)
|
||||||
expect(missing.length).toBe(2);
|
expect(missing.length).toBe(2);
|
||||||
@@ -516,16 +586,14 @@ describe("listMissingThumbnails", () => {
|
|||||||
describe("fixMissingThumbnails", () => {
|
describe("fixMissingThumbnails", () => {
|
||||||
it("downloads original, generates thumbnail, encrypts, uploads, and registers", async () => {
|
it("downloads original, generates thumbnail, encrypts, uploads, and registers", async () => {
|
||||||
const fixMock = await buildThumbMock();
|
const fixMock = await buildThumbMock();
|
||||||
const client = await Client.login({
|
const client = await login(buildThumbFetch(fixMock));
|
||||||
email: TEST_EMAIL,
|
const lib = await openLib(client);
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildThumbFetch(fixMock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(client, [101]);
|
const results = await fixMissingThumbnails(lib, client, [101]);
|
||||||
|
lib.close();
|
||||||
|
|
||||||
expect(results.length).toBe(1);
|
expect(results.length).toBe(1);
|
||||||
expect(results[0]!.success).toBe(true);
|
expect(results[0]!.status).toBe("fixed");
|
||||||
expect(results[0]!.fileID).toBe(101);
|
expect(results[0]!.fileID).toBe(101);
|
||||||
expect(results[0]!.title).toBe("file-101.jpg");
|
expect(results[0]!.title).toBe("file-101.jpg");
|
||||||
expect(results[0]!.collection).toBe("Photos");
|
expect(results[0]!.collection).toBe("Photos");
|
||||||
@@ -555,48 +623,76 @@ describe("fixMissingThumbnails", () => {
|
|||||||
|
|
||||||
it("reports failure for nonexistent file IDs without crashing", async () => {
|
it("reports failure for nonexistent file IDs without crashing", async () => {
|
||||||
const fixMock = await buildThumbMock();
|
const fixMock = await buildThumbMock();
|
||||||
const client = await Client.login({
|
const client = await login(buildThumbFetch(fixMock));
|
||||||
email: TEST_EMAIL,
|
const lib = await openLib(client);
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildThumbFetch(fixMock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(client, [999]);
|
const results = await fixMissingThumbnails(lib, client, [999]);
|
||||||
|
lib.close();
|
||||||
|
|
||||||
expect(results.length).toBe(1);
|
expect(results.length).toBe(1);
|
||||||
expect(results[0]!.success).toBe(false);
|
expect(results[0]!.status).toBe("failed");
|
||||||
expect(results[0]!.fileID).toBe(999);
|
expect(results[0]!.fileID).toBe(999);
|
||||||
expect(results[0]!.error).toContain("not found");
|
expect(results[0]!.reason).toContain("not found");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("continues after one file fails and reports mixed results", async () => {
|
it("continues after one file fails and reports mixed results", async () => {
|
||||||
const fixMock = await buildThumbMock();
|
const fixMock = await buildThumbMock();
|
||||||
// Make file 102 fail by removing its ciphertext so download fails
|
// Make file 102 fail by removing its ciphertext so the download 404s.
|
||||||
delete fixMock.fileCiphertexts[102];
|
delete fixMock.fileCiphertexts[102];
|
||||||
|
|
||||||
const client = await Client.login({
|
const client = await login(buildThumbFetch(fixMock));
|
||||||
email: TEST_EMAIL,
|
const lib = await openLib(client);
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildThumbFetch(fixMock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(client, [101, 102]);
|
const results = await fixMissingThumbnails(lib, client, [101, 102]);
|
||||||
|
lib.close();
|
||||||
|
|
||||||
expect(results.length).toBe(2);
|
expect(results.length).toBe(2);
|
||||||
const success = results.find((r) => r.fileID === 101)!;
|
const success = results.find((r) => r.fileID === 101)!;
|
||||||
const failure = results.find((r) => r.fileID === 102)!;
|
const failure = results.find((r) => r.fileID === 102)!;
|
||||||
expect(success.success).toBe(true);
|
expect(success.status).toBe("fixed");
|
||||||
expect(failure.success).toBe(false);
|
expect(failure.status).toBe("failed");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips a non-JPEG image and a video as unsupported, not failed (issue #17)", async () => {
|
||||||
|
// A PNG and a video both throw inside the JPEG decoder. The helper must
|
||||||
|
// recognise them up front and report "skipped", distinct from a genuine
|
||||||
|
// "failed", and must not upload anything for them. The JPEG in the same
|
||||||
|
// batch is still repaired.
|
||||||
|
const fixMock = await buildThumbMock({ extraFormats: true });
|
||||||
|
const client = await login(buildThumbFetch(fixMock));
|
||||||
|
const lib = await openLib(client);
|
||||||
|
|
||||||
|
const results = await fixMissingThumbnails(
|
||||||
|
lib,
|
||||||
|
client,
|
||||||
|
[101, 103, 104],
|
||||||
|
);
|
||||||
|
lib.close();
|
||||||
|
|
||||||
|
const jpeg = results.find((r) => r.fileID === 101)!;
|
||||||
|
const png = results.find((r) => r.fileID === 103)!;
|
||||||
|
const video = results.find((r) => r.fileID === 104)!;
|
||||||
|
|
||||||
|
expect(jpeg.status).toBe("fixed");
|
||||||
|
|
||||||
|
// The PNG is a still image but not a JPEG: skipped only after its bytes
|
||||||
|
// are inspected.
|
||||||
|
expect(png.status).toBe("skipped");
|
||||||
|
expect(png.reason).toContain("JPEG");
|
||||||
|
|
||||||
|
// The video is skipped from its type alone, before any download.
|
||||||
|
expect(video.status).toBe("skipped");
|
||||||
|
expect(video.reason).toContain("video");
|
||||||
|
|
||||||
|
// Only the JPEG was uploaded; the two skipped files touched no upload.
|
||||||
|
expect(fixMock.uploadedThumbnails.length).toBe(1);
|
||||||
|
expect(fixMock.uploadedThumbnails[0]!.fileID).toBe(101);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("Client.getApiClient", () => {
|
describe("Client.getApiClient", () => {
|
||||||
it("returns the ApiClient when logged in", async () => {
|
it("returns the ApiClient when logged in", async () => {
|
||||||
const client = await Client.login({
|
const client = await login(buildThumbFetch(mock));
|
||||||
email: TEST_EMAIL,
|
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildThumbFetch(mock) },
|
|
||||||
});
|
|
||||||
|
|
||||||
const api = client.getApiClient();
|
const api = client.getApiClient();
|
||||||
expect(api).toBeDefined();
|
expect(api).toBeDefined();
|
||||||
@@ -604,11 +700,7 @@ describe("Client.getApiClient", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("throws after logout", async () => {
|
it("throws after logout", async () => {
|
||||||
const client = await Client.login({
|
const client = await login(buildThumbFetch(mock));
|
||||||
email: TEST_EMAIL,
|
|
||||||
password: TEST_PASSWORD,
|
|
||||||
apiOptions: { fetch: buildThumbFetch(mock) },
|
|
||||||
});
|
|
||||||
client.logout();
|
client.logout();
|
||||||
|
|
||||||
expect(() => client.getApiClient()).toThrow(/logged out/);
|
expect(() => client.getApiClient()).toThrow(/logged out/);
|
||||||
|
|||||||
Reference in New Issue
Block a user