Add quak helper list-missing-thumbnails and fix-missing-thumbnails
Some checks failed
check / check (push) Failing after 8s
Some checks failed
check / check (push) Failing after 8s
list-missing-thumbnails: iterates all files across all collections, fetches each thumbnail from the CDN, reports any that are missing or empty. Deduplicates by file ID across collections. fix-missing-thumbnails: for each missing thumbnail, downloads the original file, generates a 720px JPEG thumbnail via sharp, encrypts it with secretstream push (encryptBlob), uploads to a presigned URL, and registers the new thumbnail via PUT /files/thumbnail. New crypto: encryptBlob (secretstream push, single chunk TAG_FINAL). New ApiClient methods: getUploadURL, putFile, putJSON, updateThumbnail. New Client method: getApiClient() for modules that need raw API access. Deps: sharp 0.34.5 (image processing), @types/sharp 0.32.0.
This commit is contained in:
183
src/thumbnails.ts
Normal file
183
src/thumbnails.ts
Normal file
@@ -0,0 +1,183 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import sharp from "sharp";
|
||||
import type { Client } from "./client.js";
|
||||
import { encryptBlob, toBase64 } from "./crypto/index.js";
|
||||
import { downloadFile } from "./download/index.js";
|
||||
import type { EnteFile } from "./model/types.js";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
const THUMB_MAX_DIMENSION = 720;
|
||||
const THUMB_JPEG_QUALITY = 50;
|
||||
|
||||
export interface MissingThumbnailInfo {
|
||||
fileID: number;
|
||||
title: string;
|
||||
collection: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface ThumbnailFixResult {
|
||||
fileID: number;
|
||||
title: string;
|
||||
collection: string;
|
||||
success: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export type ProgressCallback = (message: string) => void;
|
||||
|
||||
export const listMissingThumbnails = async (
|
||||
client: Client,
|
||||
onProgress?: ProgressCallback,
|
||||
): Promise<MissingThumbnailInfo[]> => {
|
||||
const log = onProgress ?? (() => {});
|
||||
const missing: MissingThumbnailInfo[] = [];
|
||||
const seen = new Set<number>();
|
||||
|
||||
const collections = await client.listCollections();
|
||||
for (const col of collections) {
|
||||
log(`[${col.name}] Checking thumbnails...`);
|
||||
const files = await client.listFiles(col.id, col.key);
|
||||
for (const file of files) {
|
||||
if (seen.has(file.id)) continue;
|
||||
seen.add(file.id);
|
||||
try {
|
||||
const api = client.getApiClient();
|
||||
const stream = await api.getThumbnailStream(file.id);
|
||||
const reader = stream.getReader();
|
||||
let totalBytes = 0;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (value) totalBytes += value.length;
|
||||
if (done) break;
|
||||
}
|
||||
if (totalBytes === 0) {
|
||||
missing.push({
|
||||
fileID: file.id,
|
||||
title: file.metadata.title,
|
||||
collection: col.name,
|
||||
reason: "empty thumbnail (0 bytes)",
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
missing.push({
|
||||
fileID: file.id,
|
||||
title: file.metadata.title,
|
||||
collection: col.name,
|
||||
reason: "thumbnail fetch failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return missing;
|
||||
};
|
||||
|
||||
const generateThumbnail = async (originalPath: string): Promise<Uint8Array> => {
|
||||
const result = await sharp(originalPath)
|
||||
.rotate()
|
||||
.resize(THUMB_MAX_DIMENSION, THUMB_MAX_DIMENSION, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.jpeg({ quality: THUMB_JPEG_QUALITY })
|
||||
.toBuffer();
|
||||
return new Uint8Array(result);
|
||||
};
|
||||
|
||||
export const fixMissingThumbnails = async (
|
||||
client: Client,
|
||||
fileIDs: number[],
|
||||
onProgress?: ProgressCallback,
|
||||
): Promise<ThumbnailFixResult[]> => {
|
||||
const log = onProgress ?? (() => {});
|
||||
const results: ThumbnailFixResult[] = [];
|
||||
const api = client.getApiClient();
|
||||
|
||||
const collections = await client.listCollections();
|
||||
const fileMap = new Map<
|
||||
number,
|
||||
{ file: EnteFile; collectionName: string }
|
||||
>();
|
||||
|
||||
for (const col of collections) {
|
||||
const files = await client.listFiles(col.id, col.key);
|
||||
for (const file of files) {
|
||||
if (fileIDs.includes(file.id) && !fileMap.has(file.id)) {
|
||||
fileMap.set(file.id, {
|
||||
file,
|
||||
collectionName: col.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const fileID of fileIDs) {
|
||||
const entry = fileMap.get(fileID);
|
||||
if (!entry) {
|
||||
results.push({
|
||||
fileID,
|
||||
title: "unknown",
|
||||
collection: "unknown",
|
||||
success: false,
|
||||
error: "file not found in any collection",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const { file, collectionName } = entry;
|
||||
const tmpDir = mkdtempSync(join(tmpdir(), "quak-thumb-"));
|
||||
|
||||
try {
|
||||
log(
|
||||
`[${collectionName}] Downloading ${file.metadata.title} for thumbnail generation...`,
|
||||
);
|
||||
const origPath = join(tmpDir, "original");
|
||||
await downloadFile(api, file, origPath);
|
||||
|
||||
log(
|
||||
`[${collectionName}] Generating thumbnail for ${file.metadata.title}...`,
|
||||
);
|
||||
const thumbJpeg = await generateThumbnail(origPath);
|
||||
|
||||
log(
|
||||
`[${collectionName}] Encrypting and uploading thumbnail (${thumbJpeg.length} bytes)...`,
|
||||
);
|
||||
const { header, ciphertext } = encryptBlob(thumbJpeg, file.key);
|
||||
|
||||
const md5 = createHash("md5").update(ciphertext).digest("base64");
|
||||
const { objectKey, url } = await api.getUploadURL(
|
||||
ciphertext.length,
|
||||
md5,
|
||||
);
|
||||
await api.putFile(url, ciphertext);
|
||||
await api.updateThumbnail(file.id, objectKey, toBase64(header));
|
||||
|
||||
log(
|
||||
`[${collectionName}] Thumbnail uploaded for ${file.metadata.title}`,
|
||||
);
|
||||
results.push({
|
||||
fileID,
|
||||
title: file.metadata.title,
|
||||
collection: collectionName,
|
||||
success: true,
|
||||
});
|
||||
} catch (err) {
|
||||
log(
|
||||
`[${collectionName}] FAILED ${file.metadata.title}: ${err instanceof Error ? err.message : err}`,
|
||||
);
|
||||
results.push({
|
||||
fileID,
|
||||
title: file.metadata.title,
|
||||
collection: collectionName,
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
} finally {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
};
|
||||
Reference in New Issue
Block a user