Ports the CLI to the library API. collections/files/get/get-thumb use the fresh read variants (Library.fresh(), current server state); backup runs lib.backup; backup-metadata and the missing-thumbnail helpers enumerate via the library. files/get output is byte-identical to the pre-port CLI — raw metadata.title, microsecond creationTime, pre-port row order — exit codes unchanged. Adds --cache-dir; fixes the helper JPEG-only assumption (closes #17). Model: opus-4-8
This commit was merged in pull request #74.
This commit is contained in:
+123
-66
@@ -2,13 +2,10 @@ import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import * as jpeg from "jpeg-js";
|
||||
import type { Client } from "./client.js";
|
||||
import type { Library } from "./library/index.js";
|
||||
import { ApiError } from "./api/client.js";
|
||||
import { encryptBlob, toBase64 } from "./crypto/index.js";
|
||||
import { downloadFile } from "./download/index.js";
|
||||
import type { EnteFile } from "./model/types.js";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
const THUMB_MAX_DIMENSION = 720;
|
||||
const THUMB_JPEG_QUALITY = 50;
|
||||
@@ -20,34 +17,51 @@ export interface MissingThumbnailInfo {
|
||||
reason: string;
|
||||
}
|
||||
|
||||
// Three outcomes, not two. "fixed": a thumbnail was generated and uploaded.
|
||||
// "failed": something went wrong (download, encode, upload) and the file still
|
||||
// has no thumbnail. "skipped": the file is a format this helper cannot
|
||||
// regenerate — a video, or an image that is not a baseline JPEG. Skipped is a
|
||||
// deliberate, expected outcome, not an error (issue #17): the repair path is
|
||||
// JPEG-only because `jpeg-js` is, and a PNG or HEIC is left for a format-aware
|
||||
// tool rather than reported as a failure.
|
||||
export type ThumbnailFixStatus = "fixed" | "skipped" | "failed";
|
||||
|
||||
export interface ThumbnailFixResult {
|
||||
fileID: number;
|
||||
title: string;
|
||||
collection: string;
|
||||
success: boolean;
|
||||
error?: string;
|
||||
status: ThumbnailFixStatus;
|
||||
// Why the file was skipped or failed; unset when it was fixed.
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export type ProgressCallback = (message: string) => void;
|
||||
|
||||
// Enumerate every file the library knows about, newest album first, each file
|
||||
// once, and report those whose server-side thumbnail is missing. "Missing" is
|
||||
// only two answers: an empty body, or a 404. Any other error reaching this
|
||||
// point has already exhausted its retries — a failing server, a dropped
|
||||
// connection, a deadline — and says nothing about whether the thumbnail
|
||||
// exists, so it is logged and the file is left unreported. That distinction is
|
||||
// what stops `fix-missing-thumbnails` from regenerating and uploading over
|
||||
// thumbnails that were fine all along while the CDN was briefly returning 500s.
|
||||
export const listMissingThumbnails = async (
|
||||
lib: Library,
|
||||
client: Client,
|
||||
onProgress?: ProgressCallback,
|
||||
): Promise<MissingThumbnailInfo[]> => {
|
||||
const log = onProgress ?? (() => {});
|
||||
const api = client.getApiClient();
|
||||
const missing: MissingThumbnailInfo[] = [];
|
||||
const seen = new Set<number>();
|
||||
|
||||
const collections = await client.listCollections();
|
||||
for (const col of collections) {
|
||||
log(`[${col.name}] Checking thumbnails...`);
|
||||
const files = await client.listFiles(col.id, col.key);
|
||||
for (const file of files) {
|
||||
if (seen.has(file.id)) continue;
|
||||
seen.add(file.id);
|
||||
for (const album of lib.albums.list()) {
|
||||
log(`[${album.name}] Checking thumbnails...`);
|
||||
for (const photo of album.photos.list()) {
|
||||
if (seen.has(photo.fileID)) continue;
|
||||
seen.add(photo.fileID);
|
||||
try {
|
||||
const api = client.getApiClient();
|
||||
const stream = await api.getThumbnailStream(file.id);
|
||||
const stream = await api.getThumbnailStream(photo.fileID);
|
||||
const reader = stream.getReader();
|
||||
let totalBytes = 0;
|
||||
for (;;) {
|
||||
@@ -57,35 +71,23 @@ export const listMissingThumbnails = async (
|
||||
}
|
||||
if (totalBytes === 0) {
|
||||
missing.push({
|
||||
fileID: file.id,
|
||||
title: file.metadata.title,
|
||||
collection: col.name,
|
||||
fileID: photo.fileID,
|
||||
title: photo.title,
|
||||
collection: album.name,
|
||||
reason: "empty thumbnail (0 bytes)",
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
// A 404 is the server stating the thumbnail is not there:
|
||||
// that, and an empty body, are the only two answers that mean
|
||||
// "missing". Anything else reaching this point is a failure
|
||||
// that already exhausted its retries — a failing server, a
|
||||
// dropped connection, a deadline — and says nothing about
|
||||
// whether the thumbnail exists.
|
||||
//
|
||||
// The distinction is what stops `helper
|
||||
// fix-missing-thumbnails` from downloading originals,
|
||||
// regenerating thumbnails and uploading them over thumbnails
|
||||
// that were fine all along, because the CDN was briefly
|
||||
// returning 500s while this ran.
|
||||
if (err instanceof ApiError && err.status === 404) {
|
||||
missing.push({
|
||||
fileID: file.id,
|
||||
title: file.metadata.title,
|
||||
collection: col.name,
|
||||
fileID: photo.fileID,
|
||||
title: photo.title,
|
||||
collection: album.name,
|
||||
reason: "thumbnail not found (HTTP 404)",
|
||||
});
|
||||
} else {
|
||||
log(
|
||||
`[${col.name}] Could not check ${file.metadata.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`,
|
||||
`[${album.name}] Could not check ${photo.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -94,7 +96,7 @@ export const listMissingThumbnails = async (
|
||||
return missing;
|
||||
};
|
||||
|
||||
// Bilinear resize of RGBA pixel buffer
|
||||
// Bilinear resize of an RGBA pixel buffer.
|
||||
const resizeRGBA = (
|
||||
src: Uint8Array,
|
||||
srcW: number,
|
||||
@@ -161,7 +163,33 @@ const generateThumbnail = (fileBytes: Uint8Array): Uint8Array => {
|
||||
return new Uint8Array(encoded.data);
|
||||
};
|
||||
|
||||
// A baseline/JFIF JPEG starts with the SOI marker 0xFFD8. `jpeg-js` decodes
|
||||
// only JPEG, so this signature check is what separates a file the helper can
|
||||
// regenerate from one it must skip: a PNG, HEIC, or the odd non-image byte
|
||||
// stream all fail this and are reported as skipped rather than crashing the
|
||||
// decoder into an opaque failure (issue #17).
|
||||
const isJpeg = (bytes: Uint8Array): boolean =>
|
||||
bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xd8;
|
||||
|
||||
// The reason a file cannot have a JPEG thumbnail regenerated for it from its
|
||||
// metadata alone, before any bytes are fetched, or undefined when it might. A
|
||||
// non-image (video, live photo) is unsupported outright; a still image still
|
||||
// has to be checked against its actual bytes once downloaded.
|
||||
const unsupportedByType = (file: EnteFile): string | undefined => {
|
||||
if (file.metadata.fileType !== "image") {
|
||||
return `unsupported file type: ${file.metadata.fileType} (only JPEG images can be regenerated)`;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
// Regenerate and upload a thumbnail for each requested file. Originals are read
|
||||
// through the library's content cache (`photo.original()`); the generated
|
||||
// thumbnail is JPEG-encoded, encrypted under the file's own key, and registered
|
||||
// with the server — the encrypt-and-upload path is unchanged. Each file is
|
||||
// resolved to one outcome (fixed / skipped / failed) and a failure on one file
|
||||
// never stops the others.
|
||||
export const fixMissingThumbnails = async (
|
||||
lib: Library,
|
||||
client: Client,
|
||||
fileIDs: number[],
|
||||
onProgress?: ProgressCallback,
|
||||
@@ -170,19 +198,24 @@ export const fixMissingThumbnails = async (
|
||||
const results: ThumbnailFixResult[] = [];
|
||||
const api = client.getApiClient();
|
||||
|
||||
const collections = await client.listCollections();
|
||||
// Resolve each requested fileID to its file record and owning album by
|
||||
// enumerating the library, each file taken from the first album that holds
|
||||
// it. The raw `EnteFile` carries the per-file key the thumbnail is
|
||||
// encrypted under, which the projected records deliberately do not.
|
||||
const wanted = new Set(fileIDs);
|
||||
const fileMap = new Map<
|
||||
number,
|
||||
{ file: EnteFile; collectionName: string }
|
||||
>();
|
||||
|
||||
for (const col of collections) {
|
||||
const files = await client.listFiles(col.id, col.key);
|
||||
for (const file of files) {
|
||||
if (fileIDs.includes(file.id) && !fileMap.has(file.id)) {
|
||||
fileMap.set(file.id, {
|
||||
for (const album of lib.albums.list()) {
|
||||
for (const photo of album.photos.list()) {
|
||||
if (!wanted.has(photo.fileID) || fileMap.has(photo.fileID))
|
||||
continue;
|
||||
const file = lib.getFile(album.collectionID, photo.fileID);
|
||||
if (file) {
|
||||
fileMap.set(photo.fileID, {
|
||||
file,
|
||||
collectionName: col.name,
|
||||
collectionName: album.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -195,33 +228,61 @@ export const fixMissingThumbnails = async (
|
||||
fileID,
|
||||
title: "unknown",
|
||||
collection: "unknown",
|
||||
success: false,
|
||||
error: "file not found in any collection",
|
||||
status: "failed",
|
||||
reason: "file not found in any collection",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const { file, collectionName } = entry;
|
||||
const tmpDir = mkdtempSync(join(tmpdir(), "quak-thumb-"));
|
||||
const title = file.metadata.title;
|
||||
|
||||
const typeReason = unsupportedByType(file);
|
||||
if (typeReason) {
|
||||
log(`[${collectionName}] Skipping ${title}: ${typeReason}`);
|
||||
results.push({
|
||||
fileID,
|
||||
title,
|
||||
collection: collectionName,
|
||||
status: "skipped",
|
||||
reason: typeReason,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
log(
|
||||
`[${collectionName}] Downloading ${file.metadata.title} for thumbnail generation...`,
|
||||
);
|
||||
const origPath = join(tmpDir, "original");
|
||||
await downloadFile(api, file, origPath);
|
||||
const photo = lib.photos.byID({ fileID });
|
||||
if (!photo) {
|
||||
throw new Error("file not present in the library cache");
|
||||
}
|
||||
|
||||
log(
|
||||
`[${collectionName}] Generating thumbnail for ${file.metadata.title}...`,
|
||||
`[${collectionName}] Downloading ${title} for thumbnail generation...`,
|
||||
);
|
||||
const fileBytes = readFileSync(origPath);
|
||||
const thumbJpeg = generateThumbnail(new Uint8Array(fileBytes));
|
||||
const { path } = await photo.original();
|
||||
const fileBytes = new Uint8Array(readFileSync(path));
|
||||
|
||||
if (!isJpeg(fileBytes)) {
|
||||
const reason =
|
||||
"unsupported image format (only baseline JPEG can be regenerated)";
|
||||
log(`[${collectionName}] Skipping ${title}: ${reason}`);
|
||||
results.push({
|
||||
fileID,
|
||||
title,
|
||||
collection: collectionName,
|
||||
status: "skipped",
|
||||
reason,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
log(`[${collectionName}] Generating thumbnail for ${title}...`);
|
||||
const thumbJpeg = generateThumbnail(fileBytes);
|
||||
|
||||
log(
|
||||
`[${collectionName}] Encrypting and uploading thumbnail (${thumbJpeg.length} bytes)...`,
|
||||
);
|
||||
const { header, ciphertext } = encryptBlob(thumbJpeg, file.key);
|
||||
|
||||
const md5 = createHash("md5").update(ciphertext).digest("base64");
|
||||
const { objectKey, url } = await api.getUploadURL(
|
||||
ciphertext.length,
|
||||
@@ -230,28 +291,24 @@ export const fixMissingThumbnails = async (
|
||||
await api.putFile(url, ciphertext);
|
||||
await api.updateThumbnail(file.id, objectKey, toBase64(header));
|
||||
|
||||
log(
|
||||
`[${collectionName}] Thumbnail uploaded for ${file.metadata.title}`,
|
||||
);
|
||||
log(`[${collectionName}] Thumbnail uploaded for ${title}`);
|
||||
results.push({
|
||||
fileID,
|
||||
title: file.metadata.title,
|
||||
title,
|
||||
collection: collectionName,
|
||||
success: true,
|
||||
status: "fixed",
|
||||
});
|
||||
} catch (err) {
|
||||
log(
|
||||
`[${collectionName}] FAILED ${file.metadata.title}: ${err instanceof Error ? err.message : err}`,
|
||||
`[${collectionName}] FAILED ${title}: ${err instanceof Error ? err.message : err}`,
|
||||
);
|
||||
results.push({
|
||||
fileID,
|
||||
title: file.metadata.title,
|
||||
title,
|
||||
collection: collectionName,
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
status: "failed",
|
||||
reason: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
} finally {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user