Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aeb275e137 | ||
|
|
67794bf5f8 | ||
|
|
b5fcbf4400 | ||
|
|
a1ea47f9a8 | ||
|
|
f65c237e2c |
@@ -26,13 +26,23 @@ declares one.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-26: `quak get` and `quak get-thumb` stop fetching collections' files
|
||||
once they find the file (https://git.eeqj.de/sneak/quak/issues/222). They used
|
||||
to wait for a refresh, which on an empty cache fetched the files of every
|
||||
collection. They now read the collection list, then one collection's files at
|
||||
a time, first those the cache says hold the file. They store nothing they
|
||||
read, and fetch the file through the new `lib.originalOf()` and
|
||||
`lib.thumbnailOf()`.
|
||||
- 2026-10-26: On an empty cache, `quak helper fix-missing-thumbnails --file`
|
||||
stops fetching collections' files once it finds every file it names
|
||||
(https://git.eeqj.de/sneak/quak/issues/223). It used to wait for a refresh,
|
||||
which on an empty cache fetched the files of every collection. Now it reads
|
||||
the collection list, then one collection's files at a time, the most recently
|
||||
updated first. It stores none of the collections or file records it reads. It
|
||||
fetches each original through `lib.originalOf()`, which keeps it in the cache.
|
||||
Once the cache holds the account's collections, it still waits for a refresh.
|
||||
|
||||
- 2026-10-26: On an empty cache, `quak get` and `quak get-thumb` stop fetching
|
||||
collections' files once they find the file
|
||||
(https://git.eeqj.de/sneak/quak/issues/222). They used to wait for a refresh,
|
||||
which on an empty cache fetched the files of every collection. Now they read
|
||||
the collection list, then one collection's files at a time. They store none of
|
||||
the collections or file records they read, and fetch the original or thumbnail
|
||||
through the new `lib.originalOf()` and `lib.thumbnailOf()`. Once the cache
|
||||
holds the account's collections, they still wait for a refresh.
|
||||
|
||||
- 2026-10-26: `quak files` with a collection ID fetches that collection's files
|
||||
alone (https://git.eeqj.de/sneak/quak/issues/217). It used to wait for a
|
||||
|
||||
+26
-13
@@ -37,9 +37,16 @@ import {
|
||||
thumbnailName,
|
||||
} from "./cli-output.js";
|
||||
import { type Log, logRefresh } from "./cli-log.js";
|
||||
import { freshCollections, freshFiles, freshFile } from "./cli-read.js";
|
||||
import {
|
||||
cachedFilesByID,
|
||||
freshCollections,
|
||||
freshFiles,
|
||||
freshFile,
|
||||
freshFilesByID,
|
||||
} from "./cli-read.js";
|
||||
import { withExtension } from "./filename.js";
|
||||
import { runMetadataBackup } from "./metadata-backup.js";
|
||||
import type { Collection, EnteFile } from "./model/types.js";
|
||||
import { listMissingThumbnails, fixMissingThumbnails } from "./thumbnails.js";
|
||||
|
||||
export interface CliContext {
|
||||
@@ -132,10 +139,11 @@ const readLibraryClient = (client: Client): LibraryClient => ({
|
||||
// precache (issue #48) is off, because a one-shot `collections` or `get` must
|
||||
// not start downloading the whole account. No refresh starts as the library
|
||||
// opens or on a timer. Each command asks for the refresh it needs.
|
||||
// `collections` needs none (issue #210). Nor do `get`, `get-thumb` and `files`
|
||||
// with a collection ID (issues #217 and #222). While a refresh runs, each
|
||||
// collection whose files it has fetched is logged, so a large account shows
|
||||
// progress rather than nothing.
|
||||
// `collections` and `files` with a collection ID need none (issues #210 and
|
||||
// #217). Nor do `get` and `get-thumb` on an empty cache (issue #222). Nor does
|
||||
// `helper fix-missing-thumbnails --file` on an empty cache (issue #223). While
|
||||
// a refresh runs, each collection whose files it has fetched is logged, so a
|
||||
// large account shows progress rather than nothing.
|
||||
const openReadLibrary = (ctx: CliContext, client: Client): Promise<Library> =>
|
||||
Library.open({
|
||||
client: readLibraryClient(client),
|
||||
@@ -323,8 +331,8 @@ export const getCommand = async (
|
||||
|
||||
const lib = await openReadLibrary(ctx, client);
|
||||
try {
|
||||
// Ask the server for the file as it is now (issue #36 amendment,
|
||||
// issues #52 and #222).
|
||||
// Force a server round-trip so the file resolves against current state
|
||||
// (issue #36 amendment, issue #52).
|
||||
const file = await freshFile(lib, fileID);
|
||||
if (!file) {
|
||||
ctx.log("fatal", `File ${fileID} not found`);
|
||||
@@ -380,8 +388,8 @@ export const getThumbCommand = async (
|
||||
|
||||
const lib = await openReadLibrary(ctx, client);
|
||||
try {
|
||||
// Ask the server for the file as it is now (issue #36 amendment,
|
||||
// issues #52 and #222).
|
||||
// Force a server round-trip so the file resolves against current state
|
||||
// (issue #36 amendment, issue #52).
|
||||
const file = await freshFile(lib, fileID);
|
||||
if (!file) {
|
||||
ctx.log("fatal", `File ${fileID} not found`);
|
||||
@@ -565,10 +573,13 @@ export const fixMissingThumbnailsCommand = async (
|
||||
}
|
||||
const lib = await openReadLibrary(ctx, client);
|
||||
try {
|
||||
// Refresh first so files added since the cache was written are found;
|
||||
// a failed refresh throws.
|
||||
await lib.fresh();
|
||||
if (fileIDs.length === 0) {
|
||||
let found: Map<number, { file: EnteFile; collection: Collection }>;
|
||||
if (fileIDs.length > 0) {
|
||||
found = await freshFilesByID(lib, fileIDs);
|
||||
} else {
|
||||
// Refresh first so files added since the cache was written are
|
||||
// checked; a failed refresh throws.
|
||||
await lib.fresh();
|
||||
ctx.log("info", "Scanning for missing thumbnails...");
|
||||
const missing = await listMissingThumbnails(
|
||||
lib,
|
||||
@@ -581,12 +592,14 @@ export const fixMissingThumbnailsCommand = async (
|
||||
return 0;
|
||||
}
|
||||
ctx.log("info", `Found ${fileIDs.length} file(s) to fix.`);
|
||||
found = cachedFilesByID(lib, fileIDs);
|
||||
}
|
||||
|
||||
const results = await fixMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
fileIDs,
|
||||
found,
|
||||
(msg, level) => ctx.log(level ?? "info", msg),
|
||||
);
|
||||
|
||||
|
||||
+10
-7
@@ -1,15 +1,18 @@
|
||||
// The CLI's log level, and the one function the commands write stderr lines
|
||||
// through.
|
||||
//
|
||||
// Every line quak prints on stderr is a log line at one of six levels, from
|
||||
// the quietest: fatal, error, warning, info, debug, trace. A run prints the
|
||||
// lines at its own level and at every quieter one. Its level is info unless
|
||||
// `-v` or `-q` asks for another (see "CLI surface" in the README). Every
|
||||
// command that exits non-zero logs a fatal line saying why, so even `-q`, which
|
||||
// prints only fatal lines, never leaves a non-zero exit unexplained. An error
|
||||
// Every line quak logs goes to stderr, at one of six levels, from the
|
||||
// quietest: fatal, error, warning, info, debug, trace. A run prints the lines
|
||||
// at its own level and at every quieter one. Its level is info unless `-v` or
|
||||
// `-q` asks for another (see "What quak prints" in the README). A command that
|
||||
// runs and exits non-zero logs a fatal line saying why, so even `-q`, which
|
||||
// prints only fatal lines, never leaves such an exit unexplained. An error
|
||||
// line reports something that failed, such as a file or an album, and a
|
||||
// warning something that did not fail but needs the user's attention. stdout
|
||||
// carries only a command's output and is not a log.
|
||||
// carries a command's output and the prompts of `login`, and is not a log.
|
||||
// Two things on stderr are not log lines: the `error:` message for a
|
||||
// command line that quak cannot parse, and the usage printed when no command
|
||||
// is given.
|
||||
//
|
||||
// The library itself writes no lines: its layers report through callbacks, and
|
||||
// the CLI logs what they report. `logRequests` gives the API client options
|
||||
|
||||
+79
-24
@@ -2,16 +2,18 @@
|
||||
//
|
||||
// `collections`, `files`, `get`, and `get-thumb` must answer for
|
||||
// the account's state at the moment the command runs, not for whatever the
|
||||
// local cache last happened to hold (owner amendment, issue #36). `files`
|
||||
// without a collection ID therefore forces a server round-trip through
|
||||
// `Library.fresh()` and only then reads, so a change made elsewhere is visible.
|
||||
// `collections` asks the library for the collection list alone through
|
||||
// `Library.freshCollections()` (issue #210): a refresh would also fetch the
|
||||
// files of every collection, which on a large account takes long and which
|
||||
// `collections` does not print. `files` with a collection ID reads that list
|
||||
// too, then that one collection's files through `Library.freshFiles()` (issue
|
||||
// #217). `get` and `get-thumb` read that list, then one collection's files at a
|
||||
// time, until they find the file (issue #222).
|
||||
// local cache last happened to hold (owner amendment, issue #36). `get`,
|
||||
// `get-thumb` and `files` without a collection ID therefore force a server
|
||||
// round-trip through `Library.fresh()` and only then read, so a change made
|
||||
// elsewhere is visible. `collections` asks the library for the collection list
|
||||
// alone through `Library.freshCollections()` (issue #210): a refresh would also
|
||||
// fetch the files of every collection, which on a large account takes long and
|
||||
// which `collections` does not print. `files` with a collection ID reads that
|
||||
// list too, then that one collection's files through `Library.freshFiles()`
|
||||
// (issue #217). On an empty cache, `get` and `get-thumb` read that list, then
|
||||
// one collection's files at a time, until they find the file (issue #222).
|
||||
// `helper fix-missing-thumbnails --file` does the same until it finds every
|
||||
// file it names (issue #223).
|
||||
//
|
||||
// `collections` and `files` with a collection ID list in the server's order.
|
||||
// `files` without one lists in the library's own enumeration order
|
||||
@@ -33,7 +35,7 @@ export interface FreshReadLibrary {
|
||||
freshFiles(collection: Collection): Promise<EnteFile[]>;
|
||||
listCollections(): Collection[];
|
||||
listFiles(collectionID: number): EnteFile[];
|
||||
getFile(collectionID: number, fileID: number): EnteFile | undefined;
|
||||
getFileByID(fileID: number): EnteFile | undefined;
|
||||
}
|
||||
|
||||
// Every live collection, current as of one collections request, in the
|
||||
@@ -70,26 +72,79 @@ export const freshFiles = async (
|
||||
};
|
||||
|
||||
// One file as the server holds it now, or `undefined` when no collection holds
|
||||
// it. It reads the collection list, then one collection's files at a time, and
|
||||
// stops at the first collection that holds the file. It tries first the
|
||||
// collections the cache says hold the file, then the rest in the server's
|
||||
// order. It stores nothing.
|
||||
// it. When the cache holds the account's collections, it refreshes and looks
|
||||
// the file up in the cache. On an empty cache, it reads the collection list,
|
||||
// then one collection's files at a time, and stops at the first collection
|
||||
// that holds the file. That search stores nothing.
|
||||
export const freshFile = async (
|
||||
lib: FreshReadLibrary,
|
||||
fileID: number,
|
||||
): Promise<EnteFile | undefined> => {
|
||||
const collections = await lib.freshCollections();
|
||||
// Whether the cache holds the file in collection `c`.
|
||||
const inCache = (c: Collection): boolean =>
|
||||
lib.getFile(c.id, fileID) !== undefined;
|
||||
const ordered = [
|
||||
...collections.filter(inCache),
|
||||
...collections.filter((c) => !inCache(c)),
|
||||
];
|
||||
for (const collection of ordered) {
|
||||
// Only a refresh stores collections, so a cache holding any holds them all.
|
||||
if (lib.listCollections().length > 0) {
|
||||
await lib.fresh();
|
||||
return lib.getFileByID(fileID);
|
||||
}
|
||||
for (const collection of await lib.freshCollections()) {
|
||||
const files = await lib.freshFiles(collection);
|
||||
const file = files.find((f) => f.id === fileID);
|
||||
if (file) return file;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
// The most recently updated collection first, and the higher ID first on a
|
||||
// tie. `Library.albums.list()` lists albums in this order.
|
||||
const byNewestCollection = (a: Collection, b: Collection): number =>
|
||||
b.updationTime - a.updationTime || b.id - a.id;
|
||||
|
||||
// Each file named by `fileIDs` that the cache holds, mapped to its record and
|
||||
// the collection it was found in. Collections are searched in
|
||||
// `byNewestCollection` order, and a file is found in the first that holds it.
|
||||
// So `helper fix-missing-thumbnails` names a file after the same album as
|
||||
// `helper list-missing-thumbnails`. It reads the cache alone, so the caller
|
||||
// refreshes first.
|
||||
export const cachedFilesByID = (
|
||||
lib: FreshReadLibrary,
|
||||
fileIDs: number[],
|
||||
): Map<number, { file: EnteFile; collection: Collection }> => {
|
||||
const wanted = new Set(fileIDs);
|
||||
const found = new Map<number, { file: EnteFile; collection: Collection }>();
|
||||
const collections = [...lib.listCollections()].sort(byNewestCollection);
|
||||
for (const collection of collections) {
|
||||
for (const file of lib.listFiles(collection.id)) {
|
||||
if (wanted.has(file.id) && !found.has(file.id)) {
|
||||
found.set(file.id, { file, collection });
|
||||
}
|
||||
}
|
||||
}
|
||||
return found;
|
||||
};
|
||||
|
||||
// Each file named by `fileIDs` as the server holds it now, mapped to its record
|
||||
// and collection as `cachedFilesByID` maps it. When the cache holds the
|
||||
// account's collections, it refreshes and looks the files up in the cache. On
|
||||
// an empty cache, it reads the collection list, then one collection's files at
|
||||
// a time, and stops once it has found every file. That search stores nothing.
|
||||
export const freshFilesByID = async (
|
||||
lib: FreshReadLibrary,
|
||||
fileIDs: number[],
|
||||
): Promise<Map<number, { file: EnteFile; collection: Collection }>> => {
|
||||
// Only a refresh stores collections, so a cache holding any holds them all.
|
||||
if (lib.listCollections().length > 0) {
|
||||
await lib.fresh();
|
||||
return cachedFilesByID(lib, fileIDs);
|
||||
}
|
||||
const wanted = new Set(fileIDs);
|
||||
const found = new Map<number, { file: EnteFile; collection: Collection }>();
|
||||
const collections = await lib.freshCollections();
|
||||
for (const collection of [...collections].sort(byNewestCollection)) {
|
||||
if (found.size === wanted.size) break;
|
||||
for (const file of await lib.freshFiles(collection)) {
|
||||
if (wanted.has(file.id) && !found.has(file.id)) {
|
||||
found.set(file.id, { file, collection });
|
||||
}
|
||||
}
|
||||
}
|
||||
return found;
|
||||
};
|
||||
|
||||
@@ -647,8 +647,8 @@ export class Library {
|
||||
}
|
||||
|
||||
// The original of `file`, fetched and cached as `Photo.original()` does.
|
||||
// `file` is used as given, so the store need not hold it. `quak get`
|
||||
// passes a file that `freshFiles()` returned.
|
||||
// `file` is used as given, so the store need not hold it. On an empty
|
||||
// cache, `quak get` passes a file that `freshFiles()` returned.
|
||||
async originalOf(
|
||||
file: EnteFile,
|
||||
opts?: ContentOptions,
|
||||
@@ -662,8 +662,8 @@ export class Library {
|
||||
}
|
||||
|
||||
// The thumbnail of `file`, fetched and cached as `Photo.thumbnail()` does.
|
||||
// `file` is used as given, so the store need not hold it. `quak get-thumb`
|
||||
// passes a file that `freshFiles()` returned.
|
||||
// `file` is used as given, so the store need not hold it. On an empty
|
||||
// cache, `quak get-thumb` passes a file that `freshFiles()` returned.
|
||||
async thumbnailOf(
|
||||
file: EnteFile,
|
||||
opts?: ContentOptions,
|
||||
|
||||
+11
-36
@@ -5,7 +5,7 @@ import type { Client } from "./client.js";
|
||||
import type { Library } from "./library/index.js";
|
||||
import { ApiError } from "./api/client.js";
|
||||
import { encryptBlob, toBase64 } from "./crypto/index.js";
|
||||
import type { EnteFile } from "./model/types.js";
|
||||
import type { Collection, EnteFile } from "./model/types.js";
|
||||
|
||||
// The server refuses a thumbnail larger than the one it already records for the
|
||||
// file (`thumbnail.size`, the encrypted size), so these encodings are tried
|
||||
@@ -226,16 +226,18 @@ const encryptThumbnailWithin = (
|
||||
return undefined;
|
||||
};
|
||||
|
||||
// Regenerate and upload a thumbnail for each requested file. Originals are read
|
||||
// through the library's content cache (`photo.original()`); the generated
|
||||
// thumbnail is JPEG-encoded, encrypted under the file's own key, and registered
|
||||
// with the server — the encrypt-and-upload path is unchanged. Each file is
|
||||
// Regenerate and upload a thumbnail for each requested file. `found` holds the
|
||||
// record of each requested file that a collection holds, and the collection it
|
||||
// was found in. Each original is read through the library's content cache, by
|
||||
// `lib.originalOf(file)`. The new thumbnail is a JPEG, encrypted under the
|
||||
// file's own key. It is uploaded, then registered with the server. Each file is
|
||||
// resolved to one outcome (fixed / skipped / failed) and a failure on one file
|
||||
// never stops the others.
|
||||
export const fixMissingThumbnails = async (
|
||||
lib: Library,
|
||||
client: Client,
|
||||
fileIDs: number[],
|
||||
found: Map<number, { file: EnteFile; collection: Collection }>,
|
||||
onProgress?: ProgressCallback,
|
||||
): Promise<ThumbnailFixResult[]> => {
|
||||
const log = onProgress ?? (() => {});
|
||||
@@ -243,31 +245,8 @@ export const fixMissingThumbnails = async (
|
||||
const api = client.getApiClient();
|
||||
const { userID } = client.whoami();
|
||||
|
||||
// Resolve each requested fileID to its file record and owning album by
|
||||
// enumerating the library, each file taken from the first album that holds
|
||||
// it. The raw `EnteFile` carries the per-file key the thumbnail is
|
||||
// encrypted under, which the projected records deliberately do not.
|
||||
const wanted = new Set(fileIDs);
|
||||
const fileMap = new Map<
|
||||
number,
|
||||
{ file: EnteFile; collectionName: string }
|
||||
>();
|
||||
for (const album of lib.albums.list()) {
|
||||
for (const photo of album.photos.list()) {
|
||||
if (!wanted.has(photo.fileID) || fileMap.has(photo.fileID))
|
||||
continue;
|
||||
const file = lib.getFile(album.collectionID, photo.fileID);
|
||||
if (file) {
|
||||
fileMap.set(photo.fileID, {
|
||||
file,
|
||||
collectionName: album.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const fileID of fileIDs) {
|
||||
const entry = fileMap.get(fileID);
|
||||
const entry = found.get(fileID);
|
||||
if (!entry) {
|
||||
results.push({
|
||||
fileID,
|
||||
@@ -279,7 +258,8 @@ export const fixMissingThumbnails = async (
|
||||
continue;
|
||||
}
|
||||
|
||||
const { file, collectionName } = entry;
|
||||
const { file } = entry;
|
||||
const collectionName = entry.collection.name;
|
||||
const title = file.metadata.title;
|
||||
|
||||
const skipReason = reasonToSkip(file, userID);
|
||||
@@ -297,15 +277,10 @@ export const fixMissingThumbnails = async (
|
||||
const maxSize = file.thumbnail.size!;
|
||||
|
||||
try {
|
||||
const photo = lib.photos.byID({ fileID });
|
||||
if (!photo) {
|
||||
throw new Error("file not present in the library cache");
|
||||
}
|
||||
|
||||
log(
|
||||
`[${collectionName}] Downloading ${title} for thumbnail generation...`,
|
||||
);
|
||||
const { path } = await photo.original();
|
||||
const { path } = await lib.originalOf(file);
|
||||
const fileBytes = new Uint8Array(readFileSync(path));
|
||||
|
||||
if (!isJpeg(fileBytes)) {
|
||||
|
||||
@@ -418,7 +418,7 @@ describe("the first backup of the account", () => {
|
||||
expect(downloads.sort()).toEqual([100, 200, 300]);
|
||||
});
|
||||
|
||||
// The tree "Backup layout" in the README describes. Each original is at
|
||||
// The tree under "The backup directory" in the README. Each original is at
|
||||
// its save path, `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>`, with
|
||||
// the extension of its title; the live photo is its image and its video,
|
||||
// each with the extension it had inside the ZIP, and a `.livephoto.json`
|
||||
|
||||
+112
-13
@@ -1353,6 +1353,23 @@ describe("helper fix-missing-thumbnails", () => {
|
||||
).toBe(1);
|
||||
expect(stderr.text).toBe(last);
|
||||
});
|
||||
|
||||
// beach.jpg (100) is in Vacation (1) and Work (2), both updated at the same
|
||||
// time. The fixer names a file after the first album holding it in the
|
||||
// order of `Library.albums.list()`. On a tie, the album with the higher ID
|
||||
// comes first, so beach.jpg is named after Work.
|
||||
it("names a photo in two albums after the same album on an empty cache as on a cache that holds the account", async () => {
|
||||
const ctx = context(fakeClient({ beachInWork: true }));
|
||||
const opts = { file: ["100"], json: true };
|
||||
const inWork = [{ fileID: 100, collection: "Work", status: "skipped" }];
|
||||
expect(await fixMissingThumbnailsCommand(ctx, opts)).toBe(0);
|
||||
expect(JSON.parse(stdout.text)).toMatchObject(inWork);
|
||||
|
||||
expect(await filesCommand(ctx, undefined, {})).toBe(0);
|
||||
stdout.text = "";
|
||||
expect(await fixMissingThumbnailsCommand(ctx, opts)).toBe(0);
|
||||
expect(JSON.parse(stdout.text)).toMatchObject(inWork);
|
||||
});
|
||||
});
|
||||
|
||||
describe("backup-metadata --exif", () => {
|
||||
@@ -1572,11 +1589,11 @@ describe("an ID given on the command line", () => {
|
||||
// An account with many albums (issues #210, #217 and #222). On an empty cache
|
||||
// `collections`, `files` with a collection ID and `get` used to wait for a
|
||||
// refresh that fetched and decrypted the files of every album, one album after
|
||||
// another. On such an account they seemed to hang. The account here has 40
|
||||
// albums of one photo each, encrypted as Ente's clients encrypt them, on the
|
||||
// fake Ente server of `test/ente-server.ts`. The tests record the path and query
|
||||
// of each request quak sends. The session is the one logging in to that server
|
||||
// saves.
|
||||
// another. So did `helper fix-missing-thumbnails --file` (issue #223). On such
|
||||
// an account they seemed to hang. The account here has 40 albums of one photo
|
||||
// each, encrypted as Ente's clients encrypt them, on the fake Ente server of
|
||||
// `test/ente-server.ts`. The tests record the path and query of each request
|
||||
// quak sends. The session is the one logging in to that server saves.
|
||||
describe("an account with many albums", () => {
|
||||
const ALBUMS = 40;
|
||||
const PASSWORD = "many-albums";
|
||||
@@ -1599,7 +1616,9 @@ describe("an account with many albums", () => {
|
||||
|
||||
// A photo in album `collectionID` as the server sends it, and its original
|
||||
// encrypted as the server stores it. The original's bytes are the title.
|
||||
// These tests fetch no thumbnail.
|
||||
// These tests fetch no thumbnail. The server records a thumbnail size, so
|
||||
// `helper fix-missing-thumbnails` downloads the original before it skips
|
||||
// the photo as not a JPEG.
|
||||
const photoRow = (photo: {
|
||||
id: number;
|
||||
collectionID: number;
|
||||
@@ -1638,6 +1657,7 @@ describe("an account with many albums", () => {
|
||||
},
|
||||
file: { decryptionHeader: toBase64(original.header) },
|
||||
thumbnail: { decryptionHeader: "" },
|
||||
info: { thumbSize: 1000 },
|
||||
updationTime: photo.updationTime,
|
||||
};
|
||||
return { row, original: original.ciphertext };
|
||||
@@ -1816,7 +1836,7 @@ describe("an account with many albums", () => {
|
||||
|
||||
// Photo 200 is in album 2, so `get` asks for no file of album 3 or later.
|
||||
// The last request downloads the photo's original.
|
||||
it("get asks for the albums' files only until one holds the photo", async () => {
|
||||
it("get on an empty cache asks for the albums' files only until one holds the photo", async () => {
|
||||
const out = join(root, "photo.jpg");
|
||||
expect(await getCommand(accountContext(), "200", { out })).toBe(0);
|
||||
|
||||
@@ -1831,9 +1851,9 @@ describe("an account with many albums", () => {
|
||||
expect(existsSync(join(root, "cache", "metadata.json"))).toBe(false);
|
||||
});
|
||||
|
||||
// `files` first fills the cache. The cache then says album 40, the last,
|
||||
// holds photo 4000, so `get` asks for that album's files alone.
|
||||
it("get asks first for the files of the album the cache says holds the photo", async () => {
|
||||
// `files` first fills the cache, so `get` waits for a refresh. No album has
|
||||
// changed since, so the refresh asks for no album's files.
|
||||
it("get on a cache that holds the account asks for no unchanged album's files, also for a photo in no album", async () => {
|
||||
const ctx = accountContext();
|
||||
expect(await filesCommand(ctx, undefined, {})).toBe(0);
|
||||
requests.length = 0;
|
||||
@@ -1841,14 +1861,93 @@ describe("an account with many albums", () => {
|
||||
|
||||
const out = join(root, "photo.jpg");
|
||||
expect(await getCommand(ctx, "4000", { out })).toBe(0);
|
||||
|
||||
expect(readFileSync(out, "utf-8")).toBe("photo-40.jpg");
|
||||
expect(stderr.text).toBe(`12 bytes -> ${out}\n`);
|
||||
expect(requests).toEqual([
|
||||
"/collections/v2?sinceTime=0",
|
||||
"/collections/v2/diff?collectionID=40&sinceTime=0",
|
||||
"/collections/v2?sinceTime=1",
|
||||
"/?fileID=4000",
|
||||
]);
|
||||
|
||||
requests.length = 0;
|
||||
stderr.text = "";
|
||||
expect(await getCommand(ctx, "999", { out })).toBe(1);
|
||||
expect(stderr.text).toBe("File 999 not found\n");
|
||||
expect(requests).toEqual(["/collections/v2?sinceTime=1"]);
|
||||
});
|
||||
|
||||
// The helper searches the most recently updated album first. All 40 were
|
||||
// updated at the same time, and on a tie the higher ID comes first. Photo
|
||||
// 3900 is in album 39, so the helper asks for no file of album 38 or
|
||||
// earlier. The last request downloads the photo's original.
|
||||
it("fix-missing-thumbnails --file on an empty cache asks for the albums' files only until one holds the photo", async () => {
|
||||
const opts = { file: ["3900"], json: true };
|
||||
expect(await fixMissingThumbnailsCommand(accountContext(), opts)).toBe(
|
||||
0,
|
||||
);
|
||||
|
||||
const reason =
|
||||
"unsupported image format (only baseline JPEG can be regenerated)";
|
||||
expect(JSON.parse(stdout.text)).toEqual([
|
||||
{
|
||||
fileID: 3900,
|
||||
title: "photo-39.jpg",
|
||||
collection: "Album 39",
|
||||
status: "skipped",
|
||||
reason,
|
||||
},
|
||||
]);
|
||||
expect(stderr.text).toBe(
|
||||
"[Album 39] Downloading photo-39.jpg for thumbnail generation...\n" +
|
||||
`[Album 39] Skipping photo-39.jpg: ${reason}\n`,
|
||||
);
|
||||
expect(requests).toEqual([
|
||||
"/collections/v2?sinceTime=0",
|
||||
"/collections/v2/diff?collectionID=40&sinceTime=0",
|
||||
"/collections/v2/diff?collectionID=39&sinceTime=0",
|
||||
"/?fileID=3900",
|
||||
]);
|
||||
expect(existsSync(join(root, "cache", "metadata.json"))).toBe(false);
|
||||
});
|
||||
|
||||
// `files` first fills the cache, so the helper waits for a refresh. No
|
||||
// album has changed since, so the refresh asks for no album's files.
|
||||
it("fix-missing-thumbnails --file on a cache that holds the account asks for no unchanged album's files, also for a photo in no album", async () => {
|
||||
const ctx = accountContext();
|
||||
expect(await filesCommand(ctx, undefined, {})).toBe(0);
|
||||
requests.length = 0;
|
||||
stdout.text = "";
|
||||
stderr.text = "";
|
||||
|
||||
const opts = { file: ["3900"], json: true };
|
||||
expect(await fixMissingThumbnailsCommand(ctx, opts)).toBe(0);
|
||||
const reason =
|
||||
"unsupported image format (only baseline JPEG can be regenerated)";
|
||||
expect(JSON.parse(stdout.text)).toEqual([
|
||||
{
|
||||
fileID: 3900,
|
||||
title: "photo-39.jpg",
|
||||
collection: "Album 39",
|
||||
status: "skipped",
|
||||
reason,
|
||||
},
|
||||
]);
|
||||
expect(stderr.text).toBe(
|
||||
"[Album 39] Downloading photo-39.jpg for thumbnail generation...\n" +
|
||||
`[Album 39] Skipping photo-39.jpg: ${reason}\n`,
|
||||
);
|
||||
expect(requests).toEqual([
|
||||
"/collections/v2?sinceTime=1",
|
||||
"/?fileID=3900",
|
||||
]);
|
||||
|
||||
requests.length = 0;
|
||||
stdout.text = "";
|
||||
const notFound = { file: ["999"], json: true };
|
||||
expect(await fixMissingThumbnailsCommand(ctx, notFound)).toBe(1);
|
||||
expect(JSON.parse(stdout.text)).toMatchObject([
|
||||
{ fileID: 999, reason: "file not found in any collection" },
|
||||
]);
|
||||
expect(requests).toEqual(["/collections/v2?sinceTime=1"]);
|
||||
});
|
||||
|
||||
it("files counts on stderr each album whose files its refresh fetched", async () => {
|
||||
|
||||
@@ -234,9 +234,9 @@ describe("-q", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// No album holds file 999, so fixing it fails. The refresh's progress and
|
||||
// the list of failed files are left out. The line saying why the command
|
||||
// failed is the fatal one, and it is printed.
|
||||
// No album holds file 999, so fixing it fails. The summary from
|
||||
// `--- Done ---` to the end of the `Failed files:` list is left out. The
|
||||
// line saying why the command failed is the fatal one, and it is printed.
|
||||
it("prints only the line a failing command exits with", async () => {
|
||||
const fix = ["helper", "fix-missing-thumbnails", "--file", "999"];
|
||||
expect(await quak("-q", ...fix)).toEqual({
|
||||
@@ -328,7 +328,7 @@ describe("what the library reports", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
// Each operation is named in plain words, as the README names it.
|
||||
// Each operation is named in plain words.
|
||||
it("logs a refresh's progress at info, and its start and end at debug", () => {
|
||||
const { log, lines } = collect();
|
||||
const onProgress = logRefresh(log);
|
||||
|
||||
+87
-14
@@ -3,16 +3,19 @@
|
||||
* issue #36, issue #52).
|
||||
*
|
||||
* The `collections`, `files`, `get`, and `get-thumb` commands must answer for
|
||||
* current server state, not the local cache. `files` without a collection ID
|
||||
* forces a `Library.fresh()` round-trip before it reads. The stand-in library
|
||||
* below serves no cached record until `fresh()` has been awaited, so a helper
|
||||
* that read without refreshing would come back empty and fail here.
|
||||
* `collections` asks the library for the collection list alone
|
||||
* current server state, not the local cache. `get`, `get-thumb` and `files`
|
||||
* without a collection ID force a `Library.fresh()` round-trip before they
|
||||
* read. The stand-in library below serves no cached record until `fresh()` has
|
||||
* been awaited, so a helper that read without refreshing would come back empty
|
||||
* and fail here. `collections` asks the library for the collection list alone
|
||||
* (`freshCollections()`, issue #210). `files` with a collection ID asks for
|
||||
* that list, then for that one collection's files (`freshFiles()`, issue
|
||||
* #217). `get` and `get-thumb` ask for that list, then for one collection's
|
||||
* files at a time, until they find the file (issue #222). None of these may
|
||||
* wait for a refresh, which would fetch the files of every collection.
|
||||
* #217). Neither may wait for a refresh, which would fetch the files of every
|
||||
* collection. On an empty cache, `get` and `get-thumb` ask for that list, then
|
||||
* for one collection's files at a time, until they find the file (issue #222).
|
||||
* `helper fix-missing-thumbnails --file` does the same until it finds every
|
||||
* file it names, reading the most recently updated collection first (issue
|
||||
* #223).
|
||||
*
|
||||
* `collections` and `files` also list in the order the library gives them, not
|
||||
* the albums/photos projection's newest-first order. The fixtures are seeded in
|
||||
@@ -27,6 +30,7 @@ import {
|
||||
freshCollections,
|
||||
freshFiles,
|
||||
freshFile,
|
||||
freshFilesByID,
|
||||
type FreshReadLibrary,
|
||||
} from "../../src/cli-read.js";
|
||||
import { fileListRow } from "../../src/cli-output.js";
|
||||
@@ -105,8 +109,9 @@ class FakeLibrary implements FreshReadLibrary {
|
||||
: [];
|
||||
}
|
||||
|
||||
getFile(collectionID: number, fileID: number): EnteFile | undefined {
|
||||
return this.listFiles(collectionID).find((f) => f.id === fileID);
|
||||
getFileByID(fileID: number): EnteFile | undefined {
|
||||
if (!this.refreshed) return undefined;
|
||||
return this.files.find((f) => f.id === fileID);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,7 +226,7 @@ describe("CLI read helpers (issue #36 amendment, issue #52)", () => {
|
||||
],
|
||||
);
|
||||
|
||||
it("freshFile reads each collection's files in the server's order until one holds the file", async () => {
|
||||
it("freshFile on an empty cache reads each collection's files in the server's order until one holds the file", async () => {
|
||||
const lib = threeCollections();
|
||||
|
||||
const found = await freshFile(lib, 20);
|
||||
@@ -233,16 +238,18 @@ describe("CLI read helpers (issue #36 amendment, issue #52)", () => {
|
||||
});
|
||||
|
||||
// The stand-in's cache holds what the server holds once `fresh()` has run.
|
||||
it("freshFile reads first the files of the collection the cache says holds the file", async () => {
|
||||
it("freshFile on a cache that holds the collections refreshes, then looks the file up in the cache", async () => {
|
||||
const lib = threeCollections();
|
||||
await lib.fresh();
|
||||
|
||||
const found = await freshFile(lib, 30);
|
||||
expect(lib.freshFilesCalls).toEqual([3]);
|
||||
expect(lib.freshCalls).toBe(2);
|
||||
expect(lib.freshCollectionsCalls).toBe(0);
|
||||
expect(lib.freshFilesCalls).toEqual([]);
|
||||
expect(found?.collectionID).toBe(3);
|
||||
});
|
||||
|
||||
it("freshFile returns undefined for an unknown file, after reading every collection's files", async () => {
|
||||
it("freshFile on an empty cache returns undefined for an unknown file, after reading every collection's files", async () => {
|
||||
const lib = threeCollections();
|
||||
|
||||
const found = await freshFile(lib, 404);
|
||||
@@ -250,4 +257,70 @@ describe("CLI read helpers (issue #36 amendment, issue #52)", () => {
|
||||
expect(lib.freshFilesCalls).toEqual([1, 2, 3]);
|
||||
expect(found).toBeUndefined();
|
||||
});
|
||||
|
||||
// The server lists collections 1, 2 and 3, and collection 3 was updated
|
||||
// most recently. File 11 is in collections 1 and 2.
|
||||
const fileInTwoCollections = (): FakeLibrary =>
|
||||
new FakeLibrary(
|
||||
[collection(1, 100), collection(2, 200), collection(3, 300)],
|
||||
[
|
||||
file(10, 1, 1_700_000_000_000_000),
|
||||
file(11, 1, 1_700_000_000_000_001),
|
||||
file(11, 2, 1_700_000_000_000_001),
|
||||
file(20, 2, 1_700_000_000_000_002),
|
||||
file(30, 3, 1_700_000_000_000_003),
|
||||
],
|
||||
);
|
||||
|
||||
it("freshFilesByID on an empty cache reads the most recently updated collection's files first, until it has found every file", async () => {
|
||||
const lib = fileInTwoCollections();
|
||||
|
||||
const found = await freshFilesByID(lib, [11, 20]);
|
||||
expect(lib.freshCalls).toBe(0);
|
||||
expect(lib.freshCollectionsCalls).toBe(1);
|
||||
expect(lib.freshFilesCalls).toEqual([3, 2]);
|
||||
expect(found.get(11)?.collection.id).toBe(2);
|
||||
expect(found.get(11)?.file.collectionID).toBe(2);
|
||||
expect(found.get(20)?.collection.id).toBe(2);
|
||||
});
|
||||
|
||||
// Collection 1 was updated last, though collections 2 and 3 have higher
|
||||
// IDs. File 11 is in collections 1 and 2.
|
||||
it("freshFilesByID on an empty cache reads the most recently updated collection first, though its ID is the lowest", async () => {
|
||||
const lib = new FakeLibrary(
|
||||
[collection(1, 400), collection(2, 200), collection(3, 300)],
|
||||
[
|
||||
file(11, 1, 1_700_000_000_000_001),
|
||||
file(11, 2, 1_700_000_000_000_001),
|
||||
],
|
||||
);
|
||||
|
||||
const found = await freshFilesByID(lib, [11]);
|
||||
expect(lib.freshFilesCalls).toEqual([1]);
|
||||
expect(found.get(11)?.collection.id).toBe(1);
|
||||
expect(found.get(11)?.file.collectionID).toBe(1);
|
||||
});
|
||||
|
||||
// The stand-in's cache holds what the server holds once `fresh()` has run.
|
||||
it("freshFilesByID on a cache that holds the collections refreshes, then finds each file in the most recently updated collection that holds it", async () => {
|
||||
const lib = fileInTwoCollections();
|
||||
await lib.fresh();
|
||||
|
||||
const found = await freshFilesByID(lib, [11, 404]);
|
||||
expect(lib.freshCalls).toBe(2);
|
||||
expect(lib.freshCollectionsCalls).toBe(0);
|
||||
expect(lib.freshFilesCalls).toEqual([]);
|
||||
expect(found.get(11)?.collection.id).toBe(2);
|
||||
expect(found.get(11)?.file.collectionID).toBe(2);
|
||||
expect(found.has(404)).toBe(false);
|
||||
});
|
||||
|
||||
it("freshFilesByID on an empty cache reads every collection's files when a file is in none of them", async () => {
|
||||
const lib = fileInTwoCollections();
|
||||
|
||||
const found = await freshFilesByID(lib, [10, 404]);
|
||||
expect(lib.freshFilesCalls).toEqual([3, 2, 1]);
|
||||
expect(found.get(10)?.collection.id).toBe(1);
|
||||
expect(found.has(404)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,10 +6,12 @@
|
||||
* working thumbnails, others return 404 or empty bodies. The tests
|
||||
* verify that the detection and repair logic handles each case correctly.
|
||||
*
|
||||
* As of issue #52 both helpers take an open `Library` for enumeration and the
|
||||
* `Client` for the API operations that stay unchanged (the thumbnail existence
|
||||
* check, and the encrypt-and-upload path). `fixMissingThumbnails` reads each
|
||||
* original through the library's content cache (`photo.original()`).
|
||||
* `listMissingThumbnails` takes an open `Library` to enumerate the files. Both
|
||||
* helpers take the `Client` for the API operations: the thumbnail existence
|
||||
* check, and the encrypt-and-upload path. `fixMissingThumbnails` takes the
|
||||
* record of each file, which these tests look up with `cachedFilesByID`. It
|
||||
* reads each original through the library's content cache, by
|
||||
* `lib.originalOf(file)`.
|
||||
*
|
||||
* `fixMissingThumbnails` is the most complex function in quak: it
|
||||
* downloads the original file, generates a JPEG thumbnail with jpeg-js,
|
||||
@@ -41,6 +43,7 @@ import {
|
||||
import { SRP, SrpServer } from "fast-srp-hap";
|
||||
import { Client } from "../../src/client.js";
|
||||
import { Library } from "../../src/library/index.js";
|
||||
import { cachedFilesByID } from "../../src/cli-read.js";
|
||||
import {
|
||||
listMissingThumbnails,
|
||||
fixMissingThumbnails,
|
||||
@@ -656,7 +659,12 @@ describe("fixMissingThumbnails", () => {
|
||||
const client = await login(buildThumbFetch(fixMock));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(lib, client, [101]);
|
||||
const results = await fixMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
[101],
|
||||
cachedFilesByID(lib, [101]),
|
||||
);
|
||||
lib.close();
|
||||
|
||||
expect(results.length).toBe(1);
|
||||
@@ -693,7 +701,12 @@ describe("fixMissingThumbnails", () => {
|
||||
const client = await login(buildThumbFetch(fixMock));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(lib, client, [999]);
|
||||
const results = await fixMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
[999],
|
||||
cachedFilesByID(lib, [999]),
|
||||
);
|
||||
lib.close();
|
||||
|
||||
expect(results.length).toBe(1);
|
||||
@@ -715,6 +728,7 @@ describe("fixMissingThumbnails", () => {
|
||||
lib,
|
||||
client,
|
||||
[101, 102],
|
||||
cachedFilesByID(lib, [101, 102]),
|
||||
(msg, level) => {
|
||||
if (level === "error") errors.push(msg);
|
||||
},
|
||||
@@ -745,6 +759,7 @@ describe("fixMissingThumbnails", () => {
|
||||
lib,
|
||||
client,
|
||||
[101, 103, 104],
|
||||
cachedFilesByID(lib, [101, 103, 104]),
|
||||
);
|
||||
lib.close();
|
||||
|
||||
@@ -779,7 +794,12 @@ describe("fixMissingThumbnails", () => {
|
||||
const client = await login(counted.fetch);
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(lib, client, [101]);
|
||||
const results = await fixMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
[101],
|
||||
cachedFilesByID(lib, [101]),
|
||||
);
|
||||
lib.close();
|
||||
|
||||
expect(results[0]!.status).toBe("skipped");
|
||||
@@ -800,7 +820,12 @@ describe("fixMissingThumbnails", () => {
|
||||
const client = await login(counted.fetch);
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(lib, client, [101]);
|
||||
const results = await fixMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
[101],
|
||||
cachedFilesByID(lib, [101]),
|
||||
);
|
||||
lib.close();
|
||||
|
||||
expect(results[0]!.status).toBe("skipped");
|
||||
@@ -841,7 +866,12 @@ describe("fixMissingThumbnails", () => {
|
||||
const client = await login(buildThumbFetch(fixMock));
|
||||
const lib = await openLib(client);
|
||||
|
||||
const results = await fixMissingThumbnails(lib, client, [101]);
|
||||
const results = await fixMissingThumbnails(
|
||||
lib,
|
||||
client,
|
||||
[101],
|
||||
cachedFilesByID(lib, [101]),
|
||||
);
|
||||
lib.close();
|
||||
|
||||
expect(results[0]!.status).toBe("fixed");
|
||||
|
||||
Reference in New Issue
Block a user