quak helper fix-missing-thumbnails --file on an empty cache stops fetching collections once it finds every file (closes #223) #227
@@ -689,14 +689,15 @@ request, and neither fetches any collection's files nor changes the cache.
|
|||||||
`files` with a collection ID asks for the same list, then for that one
|
`files` with a collection ID asks for the same list, then for that one
|
||||||
collection's files, and does not change the cache either. On an empty cache,
|
collection's files, and does not change the cache either. On an empty cache,
|
||||||
`get` and `get-thumb` ask for the same list, then for one collection's files at
|
`get` and `get-thumb` ask for the same list, then for one collection's files at
|
||||||
a time, until they find the file. They store none of the collections or file
|
a time, until they find the file. `helper fix-missing-thumbnails --file` does
|
||||||
records they read. The original or thumbnail they fetch is kept in the cache.
|
the same until it finds every file it names. They store none of the collections
|
||||||
Once the cache holds the account's collections, they wait for a refresh instead.
|
or file records they read. The original or thumbnail they fetch is kept in the
|
||||||
The others wait for a refresh, and so does `files` without a collection ID. With
|
cache. Once the cache holds the account's collections, they wait for a refresh
|
||||||
an empty cache, as on a first run, that refresh fetches the files of every
|
instead. The others wait for a refresh, and so does `files` without a collection
|
||||||
collection, one collection after another, and stores nothing until it has them
|
ID. With an empty cache, as on a first run, that refresh fetches the files of
|
||||||
all, which on a large account takes a while. Meanwhile the command logs a line
|
every collection, one collection after another, and stores nothing until it has
|
||||||
on stderr for each collection whose files it has fetched, such as
|
them all, which on a large account takes a while. Meanwhile the command logs a
|
||||||
|
line on stderr for each collection whose files it has fetched, such as
|
||||||
`Fetched the files of 3 of 120 collections`, with `--json` too; `backup` does
|
`Fetched the files of 3 of 120 collections`, with `--json` too; `backup` does
|
||||||
the same while the refresh it starts with runs. stdout holds only the command's
|
the same while the refresh it starts with runs. stdout holds only the command's
|
||||||
output. If that round-trip fails, the command prints the error on one line and
|
output. If that round-trip fails, the command prints the error on one line and
|
||||||
@@ -1192,7 +1193,8 @@ It answers with every live file in that collection that decrypts, in the
|
|||||||
server's order. `collection` is a record that `freshCollections()` returned. The
|
server's order. `collection` is a record that `freshCollections()` returned. The
|
||||||
method fetches no other collection's files and stores nothing. `quak files` with
|
method fetches no other collection's files and stores nothing. `quak files` with
|
||||||
a collection ID uses it, and so do `quak get` and `quak get-thumb` on an empty
|
a collection ID uses it, and so do `quak get` and `quak get-thumb` on an empty
|
||||||
cache.
|
cache. `quak helper fix-missing-thumbnails --file` uses it on an empty cache
|
||||||
|
too.
|
||||||
|
|
||||||
An album or file in the server's answer that quak cannot decrypt, or whose
|
An album or file in the server's answer that quak cannot decrypt, or whose
|
||||||
metadata is not JSON, does not fail the refresh. The refresh leaves it out, an
|
metadata is not JSON, does not fail the refresh. The refresh leaves it out, an
|
||||||
|
|||||||
@@ -26,6 +26,15 @@ declares one.
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-26: On an empty cache, `quak helper fix-missing-thumbnails --file`
|
||||||
|
stops fetching collections' files once it finds every file it names
|
||||||
|
(https://git.eeqj.de/sneak/quak/issues/223). It used to wait for a refresh,
|
||||||
|
which on an empty cache fetched the files of every collection. Now it reads
|
||||||
|
the collection list, then one collection's files at a time, the most recently
|
||||||
|
updated first. It stores none of the collections or file records it reads. It
|
||||||
|
fetches each original through `lib.originalOf()`, which keeps it in the cache.
|
||||||
|
Once the cache holds the account's collections, it still waits for a refresh.
|
||||||
|
|
||||||
- 2026-10-26: On an empty cache, `quak get` and `quak get-thumb` stop fetching
|
- 2026-10-26: On an empty cache, `quak get` and `quak get-thumb` stop fetching
|
||||||
collections' files once they find the file
|
collections' files once they find the file
|
||||||
(https://git.eeqj.de/sneak/quak/issues/222). They used to wait for a refresh,
|
(https://git.eeqj.de/sneak/quak/issues/222). They used to wait for a refresh,
|
||||||
|
|||||||
+20
-7
@@ -37,9 +37,16 @@ import {
|
|||||||
thumbnailName,
|
thumbnailName,
|
||||||
} from "./cli-output.js";
|
} from "./cli-output.js";
|
||||||
import { type Log, logRefresh } from "./cli-log.js";
|
import { type Log, logRefresh } from "./cli-log.js";
|
||||||
import { freshCollections, freshFiles, freshFile } from "./cli-read.js";
|
import {
|
||||||
|
cachedFilesByID,
|
||||||
|
freshCollections,
|
||||||
|
freshFiles,
|
||||||
|
freshFile,
|
||||||
|
freshFilesByID,
|
||||||
|
} from "./cli-read.js";
|
||||||
import { withExtension } from "./filename.js";
|
import { withExtension } from "./filename.js";
|
||||||
import { runMetadataBackup } from "./metadata-backup.js";
|
import { runMetadataBackup } from "./metadata-backup.js";
|
||||||
|
import type { Collection, EnteFile } from "./model/types.js";
|
||||||
import { listMissingThumbnails, fixMissingThumbnails } from "./thumbnails.js";
|
import { listMissingThumbnails, fixMissingThumbnails } from "./thumbnails.js";
|
||||||
|
|
||||||
export interface CliContext {
|
export interface CliContext {
|
||||||
@@ -133,8 +140,9 @@ const readLibraryClient = (client: Client): LibraryClient => ({
|
|||||||
// not start downloading the whole account. No refresh starts as the library
|
// not start downloading the whole account. No refresh starts as the library
|
||||||
// opens or on a timer. Each command asks for the refresh it needs.
|
// opens or on a timer. Each command asks for the refresh it needs.
|
||||||
// `collections` and `files` with a collection ID need none (issues #210 and
|
// `collections` and `files` with a collection ID need none (issues #210 and
|
||||||
// #217). Nor do `get` and `get-thumb` on an empty cache (issue #222). While a
|
// #217). Nor do `get` and `get-thumb` on an empty cache (issue #222). Nor does
|
||||||
// refresh runs, each collection whose files it has fetched is logged, so a
|
// `helper fix-missing-thumbnails --file` on an empty cache (issue #223). While
|
||||||
|
// a refresh runs, each collection whose files it has fetched is logged, so a
|
||||||
// large account shows progress rather than nothing.
|
// large account shows progress rather than nothing.
|
||||||
const openReadLibrary = (ctx: CliContext, client: Client): Promise<Library> =>
|
const openReadLibrary = (ctx: CliContext, client: Client): Promise<Library> =>
|
||||||
Library.open({
|
Library.open({
|
||||||
@@ -565,10 +573,13 @@ export const fixMissingThumbnailsCommand = async (
|
|||||||
}
|
}
|
||||||
const lib = await openReadLibrary(ctx, client);
|
const lib = await openReadLibrary(ctx, client);
|
||||||
try {
|
try {
|
||||||
// Refresh first so files added since the cache was written are found;
|
let found: Map<number, { file: EnteFile; collection: Collection }>;
|
||||||
// a failed refresh throws.
|
if (fileIDs.length > 0) {
|
||||||
await lib.fresh();
|
found = await freshFilesByID(lib, fileIDs);
|
||||||
if (fileIDs.length === 0) {
|
} else {
|
||||||
|
// Refresh first so files added since the cache was written are
|
||||||
|
// checked; a failed refresh throws.
|
||||||
|
await lib.fresh();
|
||||||
ctx.log("info", "Scanning for missing thumbnails...");
|
ctx.log("info", "Scanning for missing thumbnails...");
|
||||||
const missing = await listMissingThumbnails(
|
const missing = await listMissingThumbnails(
|
||||||
lib,
|
lib,
|
||||||
@@ -581,12 +592,14 @@ export const fixMissingThumbnailsCommand = async (
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
ctx.log("info", `Found ${fileIDs.length} file(s) to fix.`);
|
ctx.log("info", `Found ${fileIDs.length} file(s) to fix.`);
|
||||||
|
found = cachedFilesByID(lib, fileIDs);
|
||||||
}
|
}
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(
|
const results = await fixMissingThumbnails(
|
||||||
lib,
|
lib,
|
||||||
client,
|
client,
|
||||||
fileIDs,
|
fileIDs,
|
||||||
|
found,
|
||||||
(msg, level) => ctx.log(level ?? "info", msg),
|
(msg, level) => ctx.log(level ?? "info", msg),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,8 @@
|
|||||||
// list too, then that one collection's files through `Library.freshFiles()`
|
// list too, then that one collection's files through `Library.freshFiles()`
|
||||||
// (issue #217). On an empty cache, `get` and `get-thumb` read that list, then
|
// (issue #217). On an empty cache, `get` and `get-thumb` read that list, then
|
||||||
// one collection's files at a time, until they find the file (issue #222).
|
// one collection's files at a time, until they find the file (issue #222).
|
||||||
|
// `helper fix-missing-thumbnails --file` does the same until it finds every
|
||||||
|
// file it names (issue #223).
|
||||||
//
|
//
|
||||||
// `collections` and `files` with a collection ID list in the server's order.
|
// `collections` and `files` with a collection ID list in the server's order.
|
||||||
// `files` without one lists in the library's own enumeration order
|
// `files` without one lists in the library's own enumeration order
|
||||||
@@ -90,3 +92,59 @@ export const freshFile = async (
|
|||||||
}
|
}
|
||||||
return undefined;
|
return undefined;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// The most recently updated collection first, and the higher ID first on a
|
||||||
|
// tie. `Library.albums.list()` lists albums in this order.
|
||||||
|
const byNewestCollection = (a: Collection, b: Collection): number =>
|
||||||
|
b.updationTime - a.updationTime || b.id - a.id;
|
||||||
|
|
||||||
|
// Each file named by `fileIDs` that the cache holds, mapped to its record and
|
||||||
|
// the collection it was found in. Collections are searched in
|
||||||
|
// `byNewestCollection` order, and a file is found in the first that holds it.
|
||||||
|
// So `helper fix-missing-thumbnails` names a file after the same album as
|
||||||
|
// `helper list-missing-thumbnails`. It reads the cache alone, so the caller
|
||||||
|
// refreshes first.
|
||||||
|
export const cachedFilesByID = (
|
||||||
|
lib: FreshReadLibrary,
|
||||||
|
fileIDs: number[],
|
||||||
|
): Map<number, { file: EnteFile; collection: Collection }> => {
|
||||||
|
const wanted = new Set(fileIDs);
|
||||||
|
const found = new Map<number, { file: EnteFile; collection: Collection }>();
|
||||||
|
const collections = [...lib.listCollections()].sort(byNewestCollection);
|
||||||
|
for (const collection of collections) {
|
||||||
|
for (const file of lib.listFiles(collection.id)) {
|
||||||
|
if (wanted.has(file.id) && !found.has(file.id)) {
|
||||||
|
found.set(file.id, { file, collection });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Each file named by `fileIDs` as the server holds it now, mapped to its record
|
||||||
|
// and collection as `cachedFilesByID` maps it. When the cache holds the
|
||||||
|
// account's collections, it refreshes and looks the files up in the cache. On
|
||||||
|
// an empty cache, it reads the collection list, then one collection's files at
|
||||||
|
// a time, and stops once it has found every file. That search stores nothing.
|
||||||
|
export const freshFilesByID = async (
|
||||||
|
lib: FreshReadLibrary,
|
||||||
|
fileIDs: number[],
|
||||||
|
): Promise<Map<number, { file: EnteFile; collection: Collection }>> => {
|
||||||
|
// Only a refresh stores collections, so a cache holding any holds them all.
|
||||||
|
if (lib.listCollections().length > 0) {
|
||||||
|
await lib.fresh();
|
||||||
|
return cachedFilesByID(lib, fileIDs);
|
||||||
|
}
|
||||||
|
const wanted = new Set(fileIDs);
|
||||||
|
const found = new Map<number, { file: EnteFile; collection: Collection }>();
|
||||||
|
const collections = await lib.freshCollections();
|
||||||
|
for (const collection of [...collections].sort(byNewestCollection)) {
|
||||||
|
if (found.size === wanted.size) break;
|
||||||
|
for (const file of await lib.freshFiles(collection)) {
|
||||||
|
if (wanted.has(file.id) && !found.has(file.id)) {
|
||||||
|
found.set(file.id, { file, collection });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found;
|
||||||
|
};
|
||||||
|
|||||||
+11
-36
@@ -5,7 +5,7 @@ import type { Client } from "./client.js";
|
|||||||
import type { Library } from "./library/index.js";
|
import type { Library } from "./library/index.js";
|
||||||
import { ApiError } from "./api/client.js";
|
import { ApiError } from "./api/client.js";
|
||||||
import { encryptBlob, toBase64 } from "./crypto/index.js";
|
import { encryptBlob, toBase64 } from "./crypto/index.js";
|
||||||
import type { EnteFile } from "./model/types.js";
|
import type { Collection, EnteFile } from "./model/types.js";
|
||||||
|
|
||||||
// The server refuses a thumbnail larger than the one it already records for the
|
// The server refuses a thumbnail larger than the one it already records for the
|
||||||
// file (`thumbnail.size`, the encrypted size), so these encodings are tried
|
// file (`thumbnail.size`, the encrypted size), so these encodings are tried
|
||||||
@@ -226,16 +226,18 @@ const encryptThumbnailWithin = (
|
|||||||
return undefined;
|
return undefined;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Regenerate and upload a thumbnail for each requested file. Originals are read
|
// Regenerate and upload a thumbnail for each requested file. `found` holds the
|
||||||
// through the library's content cache (`photo.original()`); the generated
|
// record of each requested file that a collection holds, and the collection it
|
||||||
// thumbnail is JPEG-encoded, encrypted under the file's own key, and registered
|
// was found in. Each original is read through the library's content cache, by
|
||||||
// with the server — the encrypt-and-upload path is unchanged. Each file is
|
// `lib.originalOf(file)`. The new thumbnail is a JPEG, encrypted under the
|
||||||
|
// file's own key. It is uploaded, then registered with the server. Each file is
|
||||||
// resolved to one outcome (fixed / skipped / failed) and a failure on one file
|
// resolved to one outcome (fixed / skipped / failed) and a failure on one file
|
||||||
// never stops the others.
|
// never stops the others.
|
||||||
export const fixMissingThumbnails = async (
|
export const fixMissingThumbnails = async (
|
||||||
lib: Library,
|
lib: Library,
|
||||||
client: Client,
|
client: Client,
|
||||||
fileIDs: number[],
|
fileIDs: number[],
|
||||||
|
found: Map<number, { file: EnteFile; collection: Collection }>,
|
||||||
onProgress?: ProgressCallback,
|
onProgress?: ProgressCallback,
|
||||||
): Promise<ThumbnailFixResult[]> => {
|
): Promise<ThumbnailFixResult[]> => {
|
||||||
const log = onProgress ?? (() => {});
|
const log = onProgress ?? (() => {});
|
||||||
@@ -243,31 +245,8 @@ export const fixMissingThumbnails = async (
|
|||||||
const api = client.getApiClient();
|
const api = client.getApiClient();
|
||||||
const { userID } = client.whoami();
|
const { userID } = client.whoami();
|
||||||
|
|
||||||
// Resolve each requested fileID to its file record and owning album by
|
|
||||||
// enumerating the library, each file taken from the first album that holds
|
|
||||||
// it. The raw `EnteFile` carries the per-file key the thumbnail is
|
|
||||||
// encrypted under, which the projected records deliberately do not.
|
|
||||||
const wanted = new Set(fileIDs);
|
|
||||||
const fileMap = new Map<
|
|
||||||
number,
|
|
||||||
{ file: EnteFile; collectionName: string }
|
|
||||||
>();
|
|
||||||
for (const album of lib.albums.list()) {
|
|
||||||
for (const photo of album.photos.list()) {
|
|
||||||
if (!wanted.has(photo.fileID) || fileMap.has(photo.fileID))
|
|
||||||
continue;
|
|
||||||
const file = lib.getFile(album.collectionID, photo.fileID);
|
|
||||||
if (file) {
|
|
||||||
fileMap.set(photo.fileID, {
|
|
||||||
file,
|
|
||||||
collectionName: album.name,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const fileID of fileIDs) {
|
for (const fileID of fileIDs) {
|
||||||
const entry = fileMap.get(fileID);
|
const entry = found.get(fileID);
|
||||||
if (!entry) {
|
if (!entry) {
|
||||||
results.push({
|
results.push({
|
||||||
fileID,
|
fileID,
|
||||||
@@ -279,7 +258,8 @@ export const fixMissingThumbnails = async (
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const { file, collectionName } = entry;
|
const { file } = entry;
|
||||||
|
const collectionName = entry.collection.name;
|
||||||
const title = file.metadata.title;
|
const title = file.metadata.title;
|
||||||
|
|
||||||
const skipReason = reasonToSkip(file, userID);
|
const skipReason = reasonToSkip(file, userID);
|
||||||
@@ -297,15 +277,10 @@ export const fixMissingThumbnails = async (
|
|||||||
const maxSize = file.thumbnail.size!;
|
const maxSize = file.thumbnail.size!;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const photo = lib.photos.byID({ fileID });
|
|
||||||
if (!photo) {
|
|
||||||
throw new Error("file not present in the library cache");
|
|
||||||
}
|
|
||||||
|
|
||||||
log(
|
log(
|
||||||
`[${collectionName}] Downloading ${title} for thumbnail generation...`,
|
`[${collectionName}] Downloading ${title} for thumbnail generation...`,
|
||||||
);
|
);
|
||||||
const { path } = await photo.original();
|
const { path } = await lib.originalOf(file);
|
||||||
const fileBytes = new Uint8Array(readFileSync(path));
|
const fileBytes = new Uint8Array(readFileSync(path));
|
||||||
|
|
||||||
if (!isJpeg(fileBytes)) {
|
if (!isJpeg(fileBytes)) {
|
||||||
|
|||||||
+101
-6
@@ -1353,6 +1353,23 @@ describe("helper fix-missing-thumbnails", () => {
|
|||||||
).toBe(1);
|
).toBe(1);
|
||||||
expect(stderr.text).toBe(last);
|
expect(stderr.text).toBe(last);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// beach.jpg (100) is in Vacation (1) and Work (2), both updated at the same
|
||||||
|
// time. The fixer names a file after the first album holding it in the
|
||||||
|
// order of `Library.albums.list()`. On a tie, the album with the higher ID
|
||||||
|
// comes first, so beach.jpg is named after Work.
|
||||||
|
it("names a photo in two albums after the same album on an empty cache as on a cache that holds the account", async () => {
|
||||||
|
const ctx = context(fakeClient({ beachInWork: true }));
|
||||||
|
const opts = { file: ["100"], json: true };
|
||||||
|
const inWork = [{ fileID: 100, collection: "Work", status: "skipped" }];
|
||||||
|
expect(await fixMissingThumbnailsCommand(ctx, opts)).toBe(0);
|
||||||
|
expect(JSON.parse(stdout.text)).toMatchObject(inWork);
|
||||||
|
|
||||||
|
expect(await filesCommand(ctx, undefined, {})).toBe(0);
|
||||||
|
stdout.text = "";
|
||||||
|
expect(await fixMissingThumbnailsCommand(ctx, opts)).toBe(0);
|
||||||
|
expect(JSON.parse(stdout.text)).toMatchObject(inWork);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("backup-metadata --exif", () => {
|
describe("backup-metadata --exif", () => {
|
||||||
@@ -1572,11 +1589,11 @@ describe("an ID given on the command line", () => {
|
|||||||
// An account with many albums (issues #210, #217 and #222). On an empty cache
|
// An account with many albums (issues #210, #217 and #222). On an empty cache
|
||||||
// `collections`, `files` with a collection ID and `get` used to wait for a
|
// `collections`, `files` with a collection ID and `get` used to wait for a
|
||||||
// refresh that fetched and decrypted the files of every album, one album after
|
// refresh that fetched and decrypted the files of every album, one album after
|
||||||
// another. On such an account they seemed to hang. The account here has 40
|
// another. So did `helper fix-missing-thumbnails --file` (issue #223). On such
|
||||||
// albums of one photo each, encrypted as Ente's clients encrypt them, on the
|
// an account they seemed to hang. The account here has 40 albums of one photo
|
||||||
// fake Ente server of `test/ente-server.ts`. The tests record the path and query
|
// each, encrypted as Ente's clients encrypt them, on the fake Ente server of
|
||||||
// of each request quak sends. The session is the one logging in to that server
|
// `test/ente-server.ts`. The tests record the path and query of each request
|
||||||
// saves.
|
// quak sends. The session is the one logging in to that server saves.
|
||||||
describe("an account with many albums", () => {
|
describe("an account with many albums", () => {
|
||||||
const ALBUMS = 40;
|
const ALBUMS = 40;
|
||||||
const PASSWORD = "many-albums";
|
const PASSWORD = "many-albums";
|
||||||
@@ -1599,7 +1616,9 @@ describe("an account with many albums", () => {
|
|||||||
|
|
||||||
// A photo in album `collectionID` as the server sends it, and its original
|
// A photo in album `collectionID` as the server sends it, and its original
|
||||||
// encrypted as the server stores it. The original's bytes are the title.
|
// encrypted as the server stores it. The original's bytes are the title.
|
||||||
// These tests fetch no thumbnail.
|
// These tests fetch no thumbnail. The server records a thumbnail size, so
|
||||||
|
// `helper fix-missing-thumbnails` downloads the original before it skips
|
||||||
|
// the photo as not a JPEG.
|
||||||
const photoRow = (photo: {
|
const photoRow = (photo: {
|
||||||
id: number;
|
id: number;
|
||||||
collectionID: number;
|
collectionID: number;
|
||||||
@@ -1638,6 +1657,7 @@ describe("an account with many albums", () => {
|
|||||||
},
|
},
|
||||||
file: { decryptionHeader: toBase64(original.header) },
|
file: { decryptionHeader: toBase64(original.header) },
|
||||||
thumbnail: { decryptionHeader: "" },
|
thumbnail: { decryptionHeader: "" },
|
||||||
|
info: { thumbSize: 1000 },
|
||||||
updationTime: photo.updationTime,
|
updationTime: photo.updationTime,
|
||||||
};
|
};
|
||||||
return { row, original: original.ciphertext };
|
return { row, original: original.ciphertext };
|
||||||
@@ -1855,6 +1875,81 @@ describe("an account with many albums", () => {
|
|||||||
expect(requests).toEqual(["/collections/v2?sinceTime=1"]);
|
expect(requests).toEqual(["/collections/v2?sinceTime=1"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The helper searches the most recently updated album first. All 40 were
|
||||||
|
// updated at the same time, and on a tie the higher ID comes first. Photo
|
||||||
|
// 3900 is in album 39, so the helper asks for no file of album 38 or
|
||||||
|
// earlier. The last request downloads the photo's original.
|
||||||
|
it("fix-missing-thumbnails --file on an empty cache asks for the albums' files only until one holds the photo", async () => {
|
||||||
|
const opts = { file: ["3900"], json: true };
|
||||||
|
expect(await fixMissingThumbnailsCommand(accountContext(), opts)).toBe(
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
|
||||||
|
const reason =
|
||||||
|
"unsupported image format (only baseline JPEG can be regenerated)";
|
||||||
|
expect(JSON.parse(stdout.text)).toEqual([
|
||||||
|
{
|
||||||
|
fileID: 3900,
|
||||||
|
title: "photo-39.jpg",
|
||||||
|
collection: "Album 39",
|
||||||
|
status: "skipped",
|
||||||
|
reason,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(stderr.text).toBe(
|
||||||
|
"[Album 39] Downloading photo-39.jpg for thumbnail generation...\n" +
|
||||||
|
`[Album 39] Skipping photo-39.jpg: ${reason}\n`,
|
||||||
|
);
|
||||||
|
expect(requests).toEqual([
|
||||||
|
"/collections/v2?sinceTime=0",
|
||||||
|
"/collections/v2/diff?collectionID=40&sinceTime=0",
|
||||||
|
"/collections/v2/diff?collectionID=39&sinceTime=0",
|
||||||
|
"/?fileID=3900",
|
||||||
|
]);
|
||||||
|
expect(existsSync(join(root, "cache", "metadata.json"))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
// `files` first fills the cache, so the helper waits for a refresh. No
|
||||||
|
// album has changed since, so the refresh asks for no album's files.
|
||||||
|
it("fix-missing-thumbnails --file on a cache that holds the account asks for no unchanged album's files, also for a photo in no album", async () => {
|
||||||
|
const ctx = accountContext();
|
||||||
|
expect(await filesCommand(ctx, undefined, {})).toBe(0);
|
||||||
|
requests.length = 0;
|
||||||
|
stdout.text = "";
|
||||||
|
stderr.text = "";
|
||||||
|
|
||||||
|
const opts = { file: ["3900"], json: true };
|
||||||
|
expect(await fixMissingThumbnailsCommand(ctx, opts)).toBe(0);
|
||||||
|
const reason =
|
||||||
|
"unsupported image format (only baseline JPEG can be regenerated)";
|
||||||
|
expect(JSON.parse(stdout.text)).toEqual([
|
||||||
|
{
|
||||||
|
fileID: 3900,
|
||||||
|
title: "photo-39.jpg",
|
||||||
|
collection: "Album 39",
|
||||||
|
status: "skipped",
|
||||||
|
reason,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
expect(stderr.text).toBe(
|
||||||
|
"[Album 39] Downloading photo-39.jpg for thumbnail generation...\n" +
|
||||||
|
`[Album 39] Skipping photo-39.jpg: ${reason}\n`,
|
||||||
|
);
|
||||||
|
expect(requests).toEqual([
|
||||||
|
"/collections/v2?sinceTime=1",
|
||||||
|
"/?fileID=3900",
|
||||||
|
]);
|
||||||
|
|
||||||
|
requests.length = 0;
|
||||||
|
stdout.text = "";
|
||||||
|
const notFound = { file: ["999"], json: true };
|
||||||
|
expect(await fixMissingThumbnailsCommand(ctx, notFound)).toBe(1);
|
||||||
|
expect(JSON.parse(stdout.text)).toMatchObject([
|
||||||
|
{ fileID: 999, reason: "file not found in any collection" },
|
||||||
|
]);
|
||||||
|
expect(requests).toEqual(["/collections/v2?sinceTime=1"]);
|
||||||
|
});
|
||||||
|
|
||||||
it("files counts on stderr each album whose files its refresh fetched", async () => {
|
it("files counts on stderr each album whose files its refresh fetched", async () => {
|
||||||
expect(await filesCommand(accountContext(), undefined, {})).toBe(0);
|
expect(await filesCommand(accountContext(), undefined, {})).toBe(0);
|
||||||
|
|
||||||
|
|||||||
@@ -234,9 +234,9 @@ describe("-q", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// No album holds file 999, so fixing it fails. The refresh's progress and
|
// No album holds file 999, so fixing it fails. The summary from
|
||||||
// the list of failed files are left out. The line saying why the command
|
// `--- Done ---` to the end of the `Failed files:` list is left out. The
|
||||||
// failed is the fatal one, and it is printed.
|
// line saying why the command failed is the fatal one, and it is printed.
|
||||||
it("prints only the line a failing command exits with", async () => {
|
it("prints only the line a failing command exits with", async () => {
|
||||||
const fix = ["helper", "fix-missing-thumbnails", "--file", "999"];
|
const fix = ["helper", "fix-missing-thumbnails", "--file", "999"];
|
||||||
expect(await quak("-q", ...fix)).toEqual({
|
expect(await quak("-q", ...fix)).toEqual({
|
||||||
|
|||||||
@@ -13,6 +13,9 @@
|
|||||||
* #217). Neither may wait for a refresh, which would fetch the files of every
|
* #217). Neither may wait for a refresh, which would fetch the files of every
|
||||||
* collection. On an empty cache, `get` and `get-thumb` ask for that list, then
|
* collection. On an empty cache, `get` and `get-thumb` ask for that list, then
|
||||||
* for one collection's files at a time, until they find the file (issue #222).
|
* for one collection's files at a time, until they find the file (issue #222).
|
||||||
|
* `helper fix-missing-thumbnails --file` does the same until it finds every
|
||||||
|
* file it names, reading the most recently updated collection first (issue
|
||||||
|
* #223).
|
||||||
*
|
*
|
||||||
* `collections` and `files` also list in the order the library gives them, not
|
* `collections` and `files` also list in the order the library gives them, not
|
||||||
* the albums/photos projection's newest-first order. The fixtures are seeded in
|
* the albums/photos projection's newest-first order. The fixtures are seeded in
|
||||||
@@ -27,6 +30,7 @@ import {
|
|||||||
freshCollections,
|
freshCollections,
|
||||||
freshFiles,
|
freshFiles,
|
||||||
freshFile,
|
freshFile,
|
||||||
|
freshFilesByID,
|
||||||
type FreshReadLibrary,
|
type FreshReadLibrary,
|
||||||
} from "../../src/cli-read.js";
|
} from "../../src/cli-read.js";
|
||||||
import { fileListRow } from "../../src/cli-output.js";
|
import { fileListRow } from "../../src/cli-output.js";
|
||||||
@@ -253,4 +257,70 @@ describe("CLI read helpers (issue #36 amendment, issue #52)", () => {
|
|||||||
expect(lib.freshFilesCalls).toEqual([1, 2, 3]);
|
expect(lib.freshFilesCalls).toEqual([1, 2, 3]);
|
||||||
expect(found).toBeUndefined();
|
expect(found).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The server lists collections 1, 2 and 3, and collection 3 was updated
|
||||||
|
// most recently. File 11 is in collections 1 and 2.
|
||||||
|
const fileInTwoCollections = (): FakeLibrary =>
|
||||||
|
new FakeLibrary(
|
||||||
|
[collection(1, 100), collection(2, 200), collection(3, 300)],
|
||||||
|
[
|
||||||
|
file(10, 1, 1_700_000_000_000_000),
|
||||||
|
file(11, 1, 1_700_000_000_000_001),
|
||||||
|
file(11, 2, 1_700_000_000_000_001),
|
||||||
|
file(20, 2, 1_700_000_000_000_002),
|
||||||
|
file(30, 3, 1_700_000_000_000_003),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
it("freshFilesByID on an empty cache reads the most recently updated collection's files first, until it has found every file", async () => {
|
||||||
|
const lib = fileInTwoCollections();
|
||||||
|
|
||||||
|
const found = await freshFilesByID(lib, [11, 20]);
|
||||||
|
expect(lib.freshCalls).toBe(0);
|
||||||
|
expect(lib.freshCollectionsCalls).toBe(1);
|
||||||
|
expect(lib.freshFilesCalls).toEqual([3, 2]);
|
||||||
|
expect(found.get(11)?.collection.id).toBe(2);
|
||||||
|
expect(found.get(11)?.file.collectionID).toBe(2);
|
||||||
|
expect(found.get(20)?.collection.id).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Collection 1 was updated last, though collections 2 and 3 have higher
|
||||||
|
// IDs. File 11 is in collections 1 and 2.
|
||||||
|
it("freshFilesByID on an empty cache reads the most recently updated collection first, though its ID is the lowest", async () => {
|
||||||
|
const lib = new FakeLibrary(
|
||||||
|
[collection(1, 400), collection(2, 200), collection(3, 300)],
|
||||||
|
[
|
||||||
|
file(11, 1, 1_700_000_000_000_001),
|
||||||
|
file(11, 2, 1_700_000_000_000_001),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
const found = await freshFilesByID(lib, [11]);
|
||||||
|
expect(lib.freshFilesCalls).toEqual([1]);
|
||||||
|
expect(found.get(11)?.collection.id).toBe(1);
|
||||||
|
expect(found.get(11)?.file.collectionID).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
// The stand-in's cache holds what the server holds once `fresh()` has run.
|
||||||
|
it("freshFilesByID on a cache that holds the collections refreshes, then finds each file in the most recently updated collection that holds it", async () => {
|
||||||
|
const lib = fileInTwoCollections();
|
||||||
|
await lib.fresh();
|
||||||
|
|
||||||
|
const found = await freshFilesByID(lib, [11, 404]);
|
||||||
|
expect(lib.freshCalls).toBe(2);
|
||||||
|
expect(lib.freshCollectionsCalls).toBe(0);
|
||||||
|
expect(lib.freshFilesCalls).toEqual([]);
|
||||||
|
expect(found.get(11)?.collection.id).toBe(2);
|
||||||
|
expect(found.get(11)?.file.collectionID).toBe(2);
|
||||||
|
expect(found.has(404)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("freshFilesByID on an empty cache reads every collection's files when a file is in none of them", async () => {
|
||||||
|
const lib = fileInTwoCollections();
|
||||||
|
|
||||||
|
const found = await freshFilesByID(lib, [10, 404]);
|
||||||
|
expect(lib.freshFilesCalls).toEqual([3, 2, 1]);
|
||||||
|
expect(found.get(10)?.collection.id).toBe(1);
|
||||||
|
expect(found.has(404)).toBe(false);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -6,10 +6,12 @@
|
|||||||
* working thumbnails, others return 404 or empty bodies. The tests
|
* working thumbnails, others return 404 or empty bodies. The tests
|
||||||
* verify that the detection and repair logic handles each case correctly.
|
* verify that the detection and repair logic handles each case correctly.
|
||||||
*
|
*
|
||||||
* As of issue #52 both helpers take an open `Library` for enumeration and the
|
* `listMissingThumbnails` takes an open `Library` to enumerate the files. Both
|
||||||
* `Client` for the API operations that stay unchanged (the thumbnail existence
|
* helpers take the `Client` for the API operations: the thumbnail existence
|
||||||
* check, and the encrypt-and-upload path). `fixMissingThumbnails` reads each
|
* check, and the encrypt-and-upload path. `fixMissingThumbnails` takes the
|
||||||
* original through the library's content cache (`photo.original()`).
|
* record of each file, which these tests look up with `cachedFilesByID`. It
|
||||||
|
* reads each original through the library's content cache, by
|
||||||
|
* `lib.originalOf(file)`.
|
||||||
*
|
*
|
||||||
* `fixMissingThumbnails` is the most complex function in quak: it
|
* `fixMissingThumbnails` is the most complex function in quak: it
|
||||||
* downloads the original file, generates a JPEG thumbnail with jpeg-js,
|
* downloads the original file, generates a JPEG thumbnail with jpeg-js,
|
||||||
@@ -41,6 +43,7 @@ import {
|
|||||||
import { SRP, SrpServer } from "fast-srp-hap";
|
import { SRP, SrpServer } from "fast-srp-hap";
|
||||||
import { Client } from "../../src/client.js";
|
import { Client } from "../../src/client.js";
|
||||||
import { Library } from "../../src/library/index.js";
|
import { Library } from "../../src/library/index.js";
|
||||||
|
import { cachedFilesByID } from "../../src/cli-read.js";
|
||||||
import {
|
import {
|
||||||
listMissingThumbnails,
|
listMissingThumbnails,
|
||||||
fixMissingThumbnails,
|
fixMissingThumbnails,
|
||||||
@@ -656,7 +659,12 @@ describe("fixMissingThumbnails", () => {
|
|||||||
const client = await login(buildThumbFetch(fixMock));
|
const client = await login(buildThumbFetch(fixMock));
|
||||||
const lib = await openLib(client);
|
const lib = await openLib(client);
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(lib, client, [101]);
|
const results = await fixMissingThumbnails(
|
||||||
|
lib,
|
||||||
|
client,
|
||||||
|
[101],
|
||||||
|
cachedFilesByID(lib, [101]),
|
||||||
|
);
|
||||||
lib.close();
|
lib.close();
|
||||||
|
|
||||||
expect(results.length).toBe(1);
|
expect(results.length).toBe(1);
|
||||||
@@ -693,7 +701,12 @@ describe("fixMissingThumbnails", () => {
|
|||||||
const client = await login(buildThumbFetch(fixMock));
|
const client = await login(buildThumbFetch(fixMock));
|
||||||
const lib = await openLib(client);
|
const lib = await openLib(client);
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(lib, client, [999]);
|
const results = await fixMissingThumbnails(
|
||||||
|
lib,
|
||||||
|
client,
|
||||||
|
[999],
|
||||||
|
cachedFilesByID(lib, [999]),
|
||||||
|
);
|
||||||
lib.close();
|
lib.close();
|
||||||
|
|
||||||
expect(results.length).toBe(1);
|
expect(results.length).toBe(1);
|
||||||
@@ -715,6 +728,7 @@ describe("fixMissingThumbnails", () => {
|
|||||||
lib,
|
lib,
|
||||||
client,
|
client,
|
||||||
[101, 102],
|
[101, 102],
|
||||||
|
cachedFilesByID(lib, [101, 102]),
|
||||||
(msg, level) => {
|
(msg, level) => {
|
||||||
if (level === "error") errors.push(msg);
|
if (level === "error") errors.push(msg);
|
||||||
},
|
},
|
||||||
@@ -745,6 +759,7 @@ describe("fixMissingThumbnails", () => {
|
|||||||
lib,
|
lib,
|
||||||
client,
|
client,
|
||||||
[101, 103, 104],
|
[101, 103, 104],
|
||||||
|
cachedFilesByID(lib, [101, 103, 104]),
|
||||||
);
|
);
|
||||||
lib.close();
|
lib.close();
|
||||||
|
|
||||||
@@ -779,7 +794,12 @@ describe("fixMissingThumbnails", () => {
|
|||||||
const client = await login(counted.fetch);
|
const client = await login(counted.fetch);
|
||||||
const lib = await openLib(client);
|
const lib = await openLib(client);
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(lib, client, [101]);
|
const results = await fixMissingThumbnails(
|
||||||
|
lib,
|
||||||
|
client,
|
||||||
|
[101],
|
||||||
|
cachedFilesByID(lib, [101]),
|
||||||
|
);
|
||||||
lib.close();
|
lib.close();
|
||||||
|
|
||||||
expect(results[0]!.status).toBe("skipped");
|
expect(results[0]!.status).toBe("skipped");
|
||||||
@@ -800,7 +820,12 @@ describe("fixMissingThumbnails", () => {
|
|||||||
const client = await login(counted.fetch);
|
const client = await login(counted.fetch);
|
||||||
const lib = await openLib(client);
|
const lib = await openLib(client);
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(lib, client, [101]);
|
const results = await fixMissingThumbnails(
|
||||||
|
lib,
|
||||||
|
client,
|
||||||
|
[101],
|
||||||
|
cachedFilesByID(lib, [101]),
|
||||||
|
);
|
||||||
lib.close();
|
lib.close();
|
||||||
|
|
||||||
expect(results[0]!.status).toBe("skipped");
|
expect(results[0]!.status).toBe("skipped");
|
||||||
@@ -841,7 +866,12 @@ describe("fixMissingThumbnails", () => {
|
|||||||
const client = await login(buildThumbFetch(fixMock));
|
const client = await login(buildThumbFetch(fixMock));
|
||||||
const lib = await openLib(client);
|
const lib = await openLib(client);
|
||||||
|
|
||||||
const results = await fixMissingThumbnails(lib, client, [101]);
|
const results = await fixMissingThumbnails(
|
||||||
|
lib,
|
||||||
|
client,
|
||||||
|
[101],
|
||||||
|
cachedFilesByID(lib, [101]),
|
||||||
|
);
|
||||||
lib.close();
|
lib.close();
|
||||||
|
|
||||||
expect(results[0]!.status).toBe("fixed");
|
expect(results[0]!.status).toBe("fixed");
|
||||||
|
|||||||
Reference in New Issue
Block a user