sneak/quak
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions 36 Packages Projects Releases Wiki Activity

-v raises the CLI's log level per repeat; -q prints only fatal errors (closes #212)
check / check (push) Canceled after 0s
Details

Browse Source 
Every line quak prints on stderr now goes through one log, in src/cli-log.ts, at one of six levels. The default, info, prints what quak printed before. -v adds each retry and the start and end of each refresh and ML data fetch; -vv adds each request with its result. -q prints only fatal lines, and is refused with -v; both are program options. Every non-zero exit logs one fatal line, so a run that exits 1 because files failed says how many and where to look. Every line reporting a failure is at error. --json changes stdout only, so backup and --json runs log their progress on stderr. withRetry and ApiClient report retries and requests through new callbacks.

Model: opus-5-5
This commit was merged in pull request #216.
This commit is contained in:
clawbot
2026-10-18 21:46:59 +02:00
parent 92e0c6c791
commit 68d22b9927
21 changed files with 1099 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/client.ts
+57 -12
View File
@@ -36,6 +36,15 @@ export interface ApiClientOptions {
retry?: RetryOptions;
requestTimeoutMs?: number;
downloadTimeoutMs?: number;
// Called once for each request sent, with the status of its response or
// the error it failed with, for a caller that logs requests. A request
// that is retried is reported once for each attempt.
onRequest?: (request: {
method: string;
url: string;
status?: number;
error?: unknown;
}) => void;
}
export interface StreamOptions {
@@ -159,6 +168,7 @@ export class ApiClient {
private readonly retry: ResolvedRetryOptions;
private readonly requestTimeoutMs: number;
private readonly downloadTimeoutMs: number;
private readonly onRequest: NonNullable<ApiClientOptions["onRequest"]>;
private token: string | undefined;
constructor(opts?: ApiClientOptions) {
@@ -180,6 +190,7 @@ export class ApiClient {
opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
this.downloadTimeoutMs =
opts?.downloadTimeoutMs ?? DEFAULT_DOWNLOAD_TIMEOUT_MS;
this.onRequest = opts?.onRequest ?? (() => {});
this.token = opts?.authToken;
}
@@ -214,6 +225,33 @@ export class ApiClient {
return h;
}
// Send one request and report it to `onRequest`, with the status of its
// response or the error it failed with. `reportedURL` is the URL given to
// `onRequest`, when that must not be the one sent.
private async send(
url: string,
init: RequestInit & { method: string },
reportedURL = url,
): Promise<Response> {
let resp: Response;
try {
resp = await this._fetch(url, init);
} catch (err) {
this.onRequest({
method: init.method,
url: reportedURL,
error: err,
});
throw err;
}
this.onRequest({
method: init.method,
url: reportedURL,
status: resp.status,
});
return resp;
}
private async throwIfError(resp: Response): Promise<void> {
if (resp.ok) return;
const requestID = resp.headers.get("x-request-id") ?? undefined;
@@ -257,7 +295,7 @@ export class ApiClient {
const url = buildURL(this.apiOrigin, path, query);
// A GET changes nothing, so it is retried under the full policy.
return withRetry(async () => {
const resp = await this._fetch(url, {
const resp = await this.send(url, {
method: "GET",
headers: this.headers(),
signal: AbortSignal.timeout(this.requestTimeoutMs),
@@ -280,7 +318,7 @@ export class ApiClient {
// 3xx status, which is not retried.
return withRetry(
async () => {
const resp = await this._fetch(url, {
const resp = await this.send(url, {
method: "POST",
headers: this.headers({
"Content-Type": "application/json",
@@ -321,16 +359,23 @@ export class ApiClient {
// object at one key in one request, so replaying it either overwrites
// the same bytes or lands them for the first time. There is no partial
// state to protect, hence the full policy rather than the POST rule.
// It is reported without its query, which holds the upload's
// signature: a log keeps what it is given.
const { origin, pathname } = new URL(presignedURL);
await withRetry(async () => {
const resp = await this._fetch(presignedURL, {
method: "PUT",
headers: {
"Content-Type": "application/octet-stream",
"Content-Length": String(data.length),
const resp = await this.send(
presignedURL,
{
method: "PUT",
headers: {
"Content-Type": "application/octet-stream",
"Content-Length": String(data.length),
},
body: data,
signal: AbortSignal.timeout(this.requestTimeoutMs),
},
body: data,
signal: AbortSignal.timeout(this.requestTimeoutMs),
});
origin + pathname,
);
if (!resp.ok) {
// An ApiError, not a bare Error: without the status on the
// error the upload path cannot be classified at all, and a
@@ -348,7 +393,7 @@ export class ApiClient {
// Same replay and redirect rules as `postJSON`, for the same reasons.
return withRetry(
async () => {
const resp = await this._fetch(url, {
const resp = await this.send(url, {
method: "PUT",
headers: this.headers({
"Content-Type": "application/json",
@@ -394,7 +439,7 @@ export class ApiClient {
// headers, when no bytes have arrived either.
const deadline = idleDeadline(this.downloadTimeoutMs);
try {
const resp = await this._fetch(url, {
const resp = await this.send(url, {
method: "GET",
headers: this.headers(),
signal: deadline.signal,
src/backup.ts
+15 -5
View File
@@ -101,7 +101,10 @@ import type { MLData } from "./mldata-fetch.js";
import type { Collection, EnteFile, FileMetadata } from "./model/types.js";
import { hasTransportCode } from "./retry.js";
export type ProgressCallback = (message: string) => void;
// Reports one line of the backup's progress. `level` is "error" when the line
// reports something that failed, such as a file or an album, and is left out
// for every other line.
export type ProgressCallback = (message: string, level?: "error") => void;
// The lock in a backup directory, as `lockBackupDirectory` returns it.
export interface BackupLock {
@@ -654,7 +657,7 @@ const runLockedBackup = async (
for (const record of lib.undecryptable()) {
const error = `could not be decrypted: ${record.error}`;
if (record.fileID === undefined) {
log(`FAILED album ${record.collectionID}: ${error}`);
log(`FAILED album ${record.collectionID}: ${error}`, "error");
undecryptableAlbums.push(record.collectionID);
} else if (scope.has(record.collectionID)) {
const copies = memberships.get(record.fileID);
@@ -699,6 +702,7 @@ const runLockedBackup = async (
} catch (err) {
log(
`FAILED verifying original ${file.metadata.title}: ${describeError(err)}`,
"error",
);
recordFailure(
file.id,
@@ -732,6 +736,7 @@ const runLockedBackup = async (
} catch (err) {
log(
`FAILED original ${file.metadata.title}: ${describeError(err)}`,
"error",
);
recordFailure(
file.id,
@@ -807,7 +812,7 @@ const runLockedBackup = async (
await lib.fetchMLData();
} catch (err) {
mlDataFailure = err;
log(`FAILED ML data: ${describeError(err)}`);
log(`FAILED ML data: ${describeError(err)}`, "error");
}
for (const file of distinct.values()) {
if (lock.lost) throw lock.lost;
@@ -868,7 +873,10 @@ const runLockedBackup = async (
downloadDirectory,
);
} catch (err) {
log(`FAILED removing old album directories: ${describeError(err)}`);
log(
`FAILED removing old album directories: ${describeError(err)}`,
"error",
);
}
// An album whose folder or JSON cannot be written, such as one whose
@@ -897,6 +905,7 @@ const runLockedBackup = async (
} catch (err) {
log(
`FAILED removing old links in ${c.name}: ${describeError(err)}`,
"error",
);
}
@@ -915,6 +924,7 @@ const runLockedBackup = async (
} catch (err) {
log(
`FAILED symlink ${c.name}/${linkName}: ${describeError(err)}`,
"error",
);
recordFailure(
link.file.id,
@@ -931,7 +941,7 @@ const runLockedBackup = async (
metaFiles,
);
} catch (err) {
log(`FAILED album ${c.name}: ${describeError(err)}`);
log(`FAILED album ${c.name}: ${describeError(err)}`, "error");
failedAlbums.push(c.id);
}
}
src/cli-commands.ts
+105 -101
View File
@@ -4,7 +4,10 @@
// code; a thrown error is left to the caller. Nothing here calls
// `process.exit`: `bin/quak.ts` wires these to the command line, and `run` in
// `cli-run.ts` prints a thrown error as one line and exits once output has
// drained. Output must stay byte-identical (see `cli-output.ts`).
// drained. Output must stay byte-identical (see `cli-output.ts`). Each line on
// stderr goes through `ctx.log` at its level (see `cli-log.ts`). Every command
// that exits non-zero logs a fatal line saying why; one that exits 1 because
// files or albums failed logs it last, saying how many and where to look.
import {
copyFileSync,
@@ -33,6 +36,7 @@ import {
originalName,
thumbnailName,
} from "./cli-output.js";
import { type Log, logRefresh } from "./cli-log.js";
import { freshCollections, freshFiles, freshFile } from "./cli-read.js";
import { withExtension } from "./filename.js";
import { runMetadataBackup } from "./metadata-backup.js";
@@ -40,7 +44,8 @@ import { listMissingThumbnails, fixMissingThumbnails } from "./thumbnails.js";
export interface CliContext {
stdout: { write(text: string): unknown };
stderr: { write(text: string): unknown };
// Writes a line on stderr at a log level, if the run's level prints it.
log: Log;
// Directory holding `session.json`.
sessionDir: string;
// The `--cache-dir` global, or undefined to let the library pick its
@@ -82,16 +87,16 @@ const requireSession = (ctx: CliContext): Client | undefined => {
try {
client = ctx.loadSession(sessionPath(ctx));
} catch (err) {
ctx.stderr.write(
`${err instanceof Error ? err.message : err}\n` +
`Run "quak logout" and then "quak login" to replace it.\n`,
ctx.log("fatal", err instanceof Error ? err.message : String(err));
ctx.log(
"fatal",
`Run "quak logout" and then "quak login" to replace it.`,
);
return undefined;
}
if (!client) {
ctx.stderr.write(
`Not logged in. Run "quak login" first.\nSession file: ${sessionPath(ctx)}\n`,
);
ctx.log("fatal", `Not logged in. Run "quak login" first.`);
ctx.log("fatal", `Session file: ${sessionPath(ctx)}`);
return undefined;
}
return client;
@@ -107,7 +112,7 @@ const parseID = (
): number | undefined => {
const id = Number(text);
if (/^[0-9]+$/.test(text) && Number.isSafeInteger(id)) return id;
ctx.stderr.write(`Invalid ${kind} ID: ${text}\n`);
ctx.log("fatal", `Invalid ${kind} ID: ${text}`);
return undefined;
};
@@ -128,13 +133,8 @@ const readLibraryClient = (client: Client): LibraryClient => ({
// downloading the whole account — and no refresh starts as it opens or on a
// timer: each command asks for the one it needs, and `collections` needs none
// (issue #210). While a refresh runs, each collection whose files it has
// fetched is counted on stderr, unless `json` is set, so a large account shows
// progress rather than nothing.
const openReadLibrary = (
ctx: CliContext,
client: Client,
opts: { json?: boolean } = {},
): Promise<Library> =>
// fetched is logged, so a large account shows progress rather than nothing.
const openReadLibrary = (ctx: CliContext, client: Client): Promise<Library> =>
Library.open({
client: readLibraryClient(client),
cacheDirectory: ctx.cacheDir,
@@ -142,12 +142,7 @@ const openReadLibrary = (
refreshIntervalSeconds: Infinity,
precacheThumbnails: false,
precacheOriginals: false,
onProgress: (event) => {
if (opts.json || event.status !== "progress") return;
ctx.stderr.write(
`Fetched the files of ${event.fetched} of ${event.total} collections\n`,
);
},
onProgress: logRefresh(ctx.log),
});
export const loginCommand = async (ctx: CliContext): Promise<number> => {
@@ -156,7 +151,7 @@ export const loginCommand = async (ctx: CliContext): Promise<number> => {
const password =
process.env.QUAK_PASSWORD ?? (await ctx.promptSecret("Password"));
ctx.stderr.write("Authenticating...\n");
ctx.log("info", "Authenticating...");
try {
const client = await ctx.login({
email,
@@ -167,11 +162,12 @@ export const loginCommand = async (ctx: CliContext): Promise<number> => {
saveSession(ctx.sessionDir, client.toJSON());
const info = client.whoami();
ctx.stderr.write(`Logged in as ${info.email} (user ${info.userID})\n`);
ctx.stderr.write(`Session saved to ${sessionPath(ctx)}\n`);
ctx.log("info", `Logged in as ${info.email} (user ${info.userID})`);
ctx.log("info", `Session saved to ${sessionPath(ctx)}`);
} catch (err) {
ctx.stderr.write(
`Login failed: ${err instanceof Error ? err.message : err}\n`,
ctx.log(
"fatal",
`Login failed: ${err instanceof Error ? err.message : err}`,
);
return 1;
}
@@ -193,7 +189,7 @@ export const whoamiCommand = async (ctx: CliContext): Promise<number> => {
export const logoutCommand = async (ctx: CliContext): Promise<number> => {
const path = sessionPath(ctx);
if (!existsSync(path)) {
ctx.stderr.write("No session found.\n");
ctx.log("info", "No session found.");
return 0;
}
await init();
@@ -211,16 +207,15 @@ export const logoutCommand = async (ctx: CliContext): Promise<number> => {
}
unlinkSync(path);
if (failure === undefined) {
ctx.stderr.write("Session ended on the server.\n");
ctx.log("info", "Session ended on the server.");
} else {
ctx.stderr.write(
`Could not end the session on the server: ${failure}\n`,
);
ctx.log("fatal", `Could not end the session on the server: ${failure}`);
}
ctx.stderr.write("Session deleted.\n");
ctx.log("info", "Session deleted.");
if (cacheDir !== undefined) {
ctx.stderr.write(
`Cache directory ${cacheDir} still holds decrypted data; delete it to remove that data.\n`,
ctx.log(
"warning",
`Cache directory ${cacheDir} still holds decrypted data; delete it to remove that data.`,
);
}
return failure === undefined ? 0 : 1;
@@ -285,7 +280,7 @@ export const filesCommand = async (
: parseID(ctx, "collection", collectionIDStr);
if (collectionIDStr !== undefined && collectionID === undefined) return 1;
const lib = await openReadLibrary(ctx, client, opts);
const lib = await openReadLibrary(ctx, client);
try {
// Force a server round-trip and list in enumeration order (issue #36
// amendment, issue #52). Each file prints from its own decrypted
@@ -294,7 +289,7 @@ export const filesCommand = async (
// newest-first order.
const files = await freshFiles(lib, collectionID);
if (!files) {
ctx.stderr.write(`Collection ${collectionID} not found\n`);
ctx.log("fatal", `Collection ${collectionID} not found`);
return 1;
}
@@ -330,7 +325,7 @@ export const getCommand = async (
// (issue #36 amendment, issue #52).
const resolved = await freshFile(lib, fileID);
if (!resolved) {
ctx.stderr.write(`File ${fileID} not found\n`);
ctx.log("fatal", `File ${fileID} not found`);
return 1;
}
const { photo, file } = resolved;
@@ -341,7 +336,7 @@ export const getCommand = async (
if (result.videoPath === undefined) {
const outPath = opts.out ?? originalName(file);
copyFileSync(result.path, outPath);
ctx.stderr.write(`${result.bytes} bytes -> ${outPath}\n`);
ctx.log("info", `${result.bytes} bytes -> ${outPath}`);
return 0;
}
// A live photo is written as its image and its video, each named after
@@ -355,17 +350,16 @@ export const getCommand = async (
? originalName(file, videoExt)
: withExtension(opts.out, videoExt);
if (imageOut.toLowerCase() === videoOut.toLowerCase()) {
ctx.stderr.write(
`File ${fileID} is a live photo, and its video would also be written to ${imageOut}\n`,
ctx.log(
"fatal",
`File ${fileID} is a live photo, and its video would also be written to ${imageOut}`,
);
return 1;
}
copyFileSync(result.path, imageOut);
copyFileSync(result.videoPath, videoOut);
ctx.stderr.write(
`${result.bytes} bytes -> ${imageOut}\n` +
`${statSync(videoOut).size} bytes -> ${videoOut}\n`,
);
ctx.log("info", `${result.bytes} bytes -> ${imageOut}`);
ctx.log("info", `${statSync(videoOut).size} bytes -> ${videoOut}`);
return 0;
} finally {
await lib.close();
@@ -389,7 +383,7 @@ export const getThumbCommand = async (
// (issue #36 amendment, issue #52).
const resolved = await freshFile(lib, fileID);
if (!resolved) {
ctx.stderr.write(`File ${fileID} not found\n`);
ctx.log("fatal", `File ${fileID} not found`);
return 1;
}
const { photo, file } = resolved;
@@ -399,7 +393,7 @@ export const getThumbCommand = async (
// used (not the projection title) (issue #52).
const outPath = opts.out ?? thumbnailName(file);
copyFileSync(result.path, outPath);
ctx.stderr.write(`${result.bytes} bytes -> ${outPath}\n`);
ctx.log("info", `${result.bytes} bytes -> ${outPath}`);
return 0;
} finally {
await lib.close();
@@ -421,9 +415,14 @@ export const backupMetadataCommand = async (
await lib.fresh();
const result = await runMetadataBackup(lib, client, dir, {
exif: opts.exif || opts.all,
onProgress: (msg) => ctx.stderr.write(msg + "\n"),
onProgress: (msg, level) => ctx.log(level ?? "info", msg),
});
return result.filesWithMLDataError > 0 ? 1 : 0;
if (result.filesWithMLDataError === 0) return 0;
ctx.log(
"fatal",
`quak: the ML data of ${result.filesWithMLDataError} file(s) could not be read; see mlDataError in their JSON in ${dir}`,
);
return 1;
} finally {
await lib.close();
}
@@ -438,7 +437,7 @@ export const backupCommand = async (
const client = requireSession(ctx);
if (!client) return 3;
ctx.stderr.write("Starting backup...\n");
ctx.log("info", "Starting backup...");
// The lock is taken before the library opens and starts its refresh, so a
// run refused while another backup of `dir` runs sends no request.
let lock: BackupLock;
@@ -446,7 +445,7 @@ export const backupCommand = async (
lock = await lockBackupDirectory(dir);
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "ELOCKED") throw err;
ctx.stderr.write(`quak: ${(err as Error).message}\n`);
ctx.log("fatal", `quak: ${(err as Error).message}`);
return 2;
}
try {
@@ -461,45 +460,48 @@ export const backupCommand = async (
refreshIntervalSeconds: Infinity,
precacheThumbnails: false,
precacheOriginals: false,
onProgress: logRefresh(ctx.log),
});
try {
const result = await lib.backup({
downloadDirectory: dir,
lock,
verify: opts.verify,
onProgress: (msg) => {
if (!opts.json) ctx.stderr.write(msg + "\n");
},
onProgress: (msg, level) => ctx.log(level ?? "info", msg),
});
if (opts.json) {
ctx.stdout.write(JSON.stringify(result, null, 2) + "\n");
} else {
ctx.stderr.write("\n--- Backup complete ---\n");
ctx.stderr.write(` Total files: ${result.totalFiles}\n`);
ctx.stderr.write(` Downloaded: ${result.downloaded}\n`);
ctx.stderr.write(` Skipped: ${result.skipped}\n`);
ctx.log("info", "\n--- Backup complete ---");
ctx.log("info", ` Total files: ${result.totalFiles}`);
ctx.log("info", ` Downloaded: ${result.downloaded}`);
ctx.log("info", ` Skipped: ${result.skipped}`);
if (opts.verify) {
ctx.stderr.write(` Verified: ${result.verified}\n`);
ctx.stderr.write(` Mismatched: ${result.mismatched}\n`);
ctx.stderr.write(` Unchecked: ${result.unchecked}\n`);
ctx.log("info", ` Verified: ${result.verified}`);
ctx.log("info", ` Mismatched: ${result.mismatched}`);
ctx.log("info", ` Unchecked: ${result.unchecked}`);
}
ctx.stderr.write(` Failed: ${result.failed}\n`);
ctx.log("info", ` Failed: ${result.failed}`);
if (result.errors.length > 0) {
ctx.stderr.write("\nFailed files:\n");
ctx.log("error", "\nFailed files:");
for (const e of result.errors) {
ctx.stderr.write(
` [${e.collection}] ${e.title} (id ${e.fileID}): ${e.error}\n`,
ctx.log(
"error",
` [${e.collection}] ${e.title} (id ${e.fileID}): ${e.error}`,
);
}
}
}
return result.failed > 0 ||
result.undecryptableAlbums.length > 0 ||
result.failedAlbums.length > 0
? 1
: 0;
const albums =
result.undecryptableAlbums.length + result.failedAlbums.length;
if (result.failed === 0 && albums === 0) return 0;
ctx.log(
"fatal",
`quak: ${result.failed} file(s) and ${albums} album(s) failed; see ${join(dir, "failures.json")} and the FAILED lines`,
);
return 1;
} finally {
await lib.close();
}
@@ -515,23 +517,24 @@ export const listMissingThumbnailsCommand = async (
await init();
const client = requireSession(ctx);
if (!client) return 3;
const lib = await openReadLibrary(ctx, client, opts);
const lib = await openReadLibrary(ctx, client);
try {
// Refresh first so files added since the cache was written are
// checked; a failed refresh throws.
await lib.fresh();
const missing = await listMissingThumbnails(lib, client, (msg) => {
if (!opts.json) ctx.stderr.write(msg + "\n");
});
const missing = await listMissingThumbnails(lib, client, (msg, level) =>
ctx.log(level ?? "info", msg),
);
if (opts.json) {
ctx.stdout.write(JSON.stringify(missing, null, 2) + "\n");
} else {
if (missing.length === 0) {
ctx.stderr.write("No missing thumbnails found.\n");
ctx.log("info", "No missing thumbnails found.");
} else {
ctx.stderr.write(
`\n${missing.length} file(s) with missing thumbnails:\n`,
ctx.log(
"info",
`\n${missing.length} file(s) with missing thumbnails:`,
);
for (const m of missing) {
ctx.stdout.write(
@@ -560,33 +563,34 @@ export const fixMissingThumbnailsCommand = async (
if (fileID === undefined) return 1;
fileIDs.push(fileID);
}
const lib = await openReadLibrary(ctx, client, opts);
const lib = await openReadLibrary(ctx, client);
try {
// Refresh first so files added since the cache was written are found;
// a failed refresh throws.
await lib.fresh();
if (fileIDs.length === 0) {
ctx.stderr.write("Scanning for missing thumbnails...\n");
const missing = await listMissingThumbnails(lib, client, (msg) => {
if (!opts.json) ctx.stderr.write(msg + "\n");
});
ctx.log("info", "Scanning for missing thumbnails...");
const missing = await listMissingThumbnails(
lib,
client,
(msg, level) => ctx.log(level ?? "info", msg),
);
fileIDs = missing.map((m) => m.fileID);
if (fileIDs.length === 0) {
ctx.stderr.write("No missing thumbnails found.\n");
ctx.log("info", "No missing thumbnails found.");
return 0;
}
ctx.stderr.write(`Found ${fileIDs.length} file(s) to fix.\n`);
ctx.log("info", `Found ${fileIDs.length} file(s) to fix.`);
}
const results = await fixMissingThumbnails(
lib,
client,
fileIDs,
(msg) => {
if (!opts.json) ctx.stderr.write(msg + "\n");
},
(msg, level) => ctx.log(level ?? "info", msg),
);
const failed = results.filter((r) => r.status === "failed").length;
if (opts.json) {
ctx.stdout.write(JSON.stringify(results, null, 2) + "\n");
} else {
@@ -594,30 +598,30 @@ export const fixMissingThumbnailsCommand = async (
const skipped = results.filter(
(r) => r.status === "skipped",
).length;
const failed = results.filter((r) => r.status === "failed").length;
ctx.stderr.write(`\n--- Done ---\n`);
ctx.stderr.write(` Fixed: ${fixed}\n`);
ctx.stderr.write(` Skipped: ${skipped}\n`);
ctx.stderr.write(` Failed: ${failed}\n`);
ctx.log("info", `\n--- Done ---`);
ctx.log("info", ` Fixed: ${fixed}`);
ctx.log("info", ` Skipped: ${skipped}`);
ctx.log("info", ` Failed: ${failed}`);
if (skipped > 0) {
ctx.stderr.write("\nSkipped:\n");
ctx.log("info", "\nSkipped:");
for (const r of results.filter((r) => r.status === "skipped")) {
ctx.stderr.write(
` ${r.fileID}\t${r.title}\t${r.reason}\n`,
);
ctx.log("info", ` ${r.fileID}\t${r.title}\t${r.reason}`);
}
}
if (failed > 0) {
ctx.stderr.write("\nFailed files:\n");
ctx.log("error", "\nFailed files:");
for (const r of results.filter((r) => r.status === "failed")) {
ctx.stderr.write(
` ${r.fileID}\t${r.title}\t${r.reason}\n`,
);
ctx.log("error", ` ${r.fileID}\t${r.title}\t${r.reason}`);
}
}
}
return results.some((r) => r.status === "failed") ? 1 : 0;
if (failed === 0) return 0;
ctx.log(
"fatal",
`quak: ${failed} file(s) failed; see the "Failed files:" list, or the JSON with --json`,
);
return 1;
} finally {
await lib.close();
}
src/cli-log.ts
+99
View File
@@ -0,0 +1,99 @@
// The CLI's log level, and the one function the commands write stderr lines
// through.
//
// Every line quak prints on stderr is a log line at one of six levels, from
// the quietest: fatal, error, warning, info, debug, trace. A run prints the
// lines at its own level and at every quieter one. Its level is info unless
// `-v` or `-q` asks for another (see "CLI surface" in the README). Every
// command that exits non-zero logs a fatal line saying why, so even `-q`, which
// prints only fatal lines, never leaves a non-zero exit unexplained. An error
// line reports something that failed, such as a file or an album, and a
// warning something that did not fail but needs the user's attention. stdout
// carries only a command's output and is not a log.
//
// The library itself writes no lines: its layers report through callbacks, and
// the CLI logs what they report. `logRequests` gives the API client options
// that log each request and each retry, and `logRefresh` the library's
// `onProgress` that logs what a refresh and an ML data fetch report.
import type { ApiClientOptions } from "./api/client.js";
import { describeError } from "./errors.js";
import type { RefreshEvent } from "./library/index.js";
import type { RetryOptions } from "./retry.js";
const LEVELS = ["fatal", "error", "warning", "info", "debug", "trace"] as const;
export type LogLevel = (typeof LEVELS)[number];
// Writes `line` on stderr when the run's level prints lines at `level`.
export type Log = (level: LogLevel, line: string) => void;
// The level asked for by `-v` given `verbose` times, or by `-q`: info without
// either, debug for `-v`, trace for `-vv` and more, fatal for `-q`.
// `bin/quak.ts` refuses `-q` and `-v` together.
export const logLevel = (verbose: number, quiet: boolean): LogLevel => {
if (quiet) return "fatal";
if (verbose === 0) return "info";
return verbose === 1 ? "debug" : "trace";
};
// The log that writes each line at `level` or quieter to `stderr`, ending it
// with a newline.
export const stderrLog =
(stderr: { write(text: string): unknown }, level: LogLevel): Log =>
(lineLevel, line) => {
if (LEVELS.indexOf(lineLevel) <= LEVELS.indexOf(level)) {
stderr.write(line + "\n");
}
};
// API client options that log each request at trace, with the status of its
// response or the error it failed with, and each retry at debug, with why and
// how long it waits. `retry` is the retry policy, the default when not given.
export const logRequests = (
log: Log,
retry?: RetryOptions,
): ApiClientOptions => ({
retry: {
...retry,
onRetry: (err, attempt, delayMs) =>
log(
"debug",
`Attempt ${attempt} failed, retrying in ${Math.round(delayMs)} ms: ${describeError(err)}`,
),
},
onRequest: ({ method, url, status, error }) =>
log(
"trace",
`${method} ${url}: ${error === undefined ? `HTTP ${status}` : describeError(error)}`,
),
});
// How a log line names each operation the library reports the start and end
// of.
const OPERATION_NAMES: Record<RefreshEvent["operation"], string> = {
refresh: "Refresh",
fetchMLData: "ML data fetch",
precacheThumbnails: "Thumbnail precache",
precacheOriginals: "Original precache",
};
// A library `onProgress` that logs, at info, each collection whose files a
// refresh has fetched, and at debug the start and end of each refresh and ML
// data fetch, with the error of one that failed.
export const logRefresh =
(log: Log) =>
(event: RefreshEvent): void => {
if (event.status === "progress") {
log(
"info",
`Fetched the files of ${event.fetched} of ${event.total} collections`,
);
return;
}
const error = event.error === undefined ? "" : `: ${event.error}`;
log(
"debug",
`${OPERATION_NAMES[event.operation]} ${event.status}${error}`,
);
};
src/cli-run.ts
+12 -9
View File
@@ -2,21 +2,23 @@
import type { Writable } from "node:stream";
import { ApiError } from "./api/client.js";
import type { Log } from "./cli-log.js";
import { describeError } from "./errors.js";
// Run a command and exit with its code once stdout/stderr have drained.
// Exiting before the drain can truncate piped output, and the library can keep
// the event loop alive after a command returns, so a plain return could hang.
// An error the command throws is printed as one `quak: MESSAGE` line, without
// the stack trace, and exits 1. MESSAGE is the error as `describeError` gives
// it, so it carries the reason Node's `fetch` puts on `cause`. A 401 from the
// server means it no longer accepts the saved session: that prints one line
// saying to log in again and exits 3, as a missing or corrupt session file
// does.
// An error the command throws is logged as one fatal `quak: MESSAGE` line,
// without the stack trace, and exits 1. MESSAGE is the error as
// `describeError` gives it, so it carries the reason Node's `fetch` puts on
// `cause`. A 401 from the server means it no longer accepts the saved session:
// that logs one fatal line saying to log in again and exits 3, as a missing or
// corrupt session file does.
export const run = async (
command: Promise<number>,
stdout: Writable,
stderr: Writable,
log: Log,
exit: (code: number) => void,
): Promise<void> => {
let code: number;
@@ -24,12 +26,13 @@ export const run = async (
code = await command;
} catch (err) {
if (err instanceof ApiError && err.status === 401) {
stderr.write(
`quak: the saved session is no longer valid; run "quak login"\n`,
log(
"fatal",
`quak: the saved session is no longer valid; run "quak login"`,
);
code = 3;
} else {
stderr.write(`quak: ${describeError(err)}\n`);
log("fatal", `quak: ${describeError(err)}`);
code = 1;
}
}
src/metadata-backup.ts
+9 -2
View File
@@ -17,7 +17,10 @@ import {
} from "./mldata-fetch.js";
import type { EnteFile } from "./model/types.js";
export type ProgressCallback = (message: string) => void;
// Reports one line of the dump's progress. `level` is "error" when the line
// reports ML data that could not be read, and is left out for every other
// line.
export type ProgressCallback = (message: string, level?: "error") => void;
export interface MetadataBackupOptions {
exif?: boolean;
@@ -195,13 +198,17 @@ export const runMetadataBackup = async (
for (const [id, payload] of payloads) mlDataMap.set(id, payload);
for (const [id, err] of errors) {
const reason = describeError(err);
log(`ML data of file ${id} could not be read: ${reason}`);
log(
`ML data of file ${id} could not be read: ${reason}`,
"error",
);
mlDataErrors.set(id, reason);
}
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
log(
`ML data request for ${batch.length} file(s) failed: ${reason}`,
"error",
);
for (const id of batch) mlDataErrors.set(id, reason);
}
src/retry.ts
+8 -1
View File
@@ -24,6 +24,9 @@ export interface RetryOptions {
sleep?: (ms: number) => Promise<void>;
// Injected so the jitter is reproducible under test.
random?: () => number;
// Called before each wait with the error that failed attempt number
// `attempt` and how long the wait is, for a caller that logs retries.
onRetry?: (err: unknown, attempt: number, delayMs: number) => void;
}
export type ResolvedRetryOptions = Required<RetryOptions>;
@@ -35,6 +38,7 @@ export const DEFAULT_RETRY_OPTIONS: ResolvedRetryOptions = {
sleep: (ms: number): Promise<void> =>
new Promise((resolve) => setTimeout(resolve, ms)),
random: Math.random,
onRetry: () => {},
};
// For a run nobody is watching, such as `quak backup` from cron. A request that
@@ -55,6 +59,7 @@ export const resolveRetryOptions = (
maxDelayMs: opts?.maxDelayMs ?? DEFAULT_RETRY_OPTIONS.maxDelayMs,
sleep: opts?.sleep ?? DEFAULT_RETRY_OPTIONS.sleep,
random: opts?.random ?? DEFAULT_RETRY_OPTIONS.random,
onRetry: opts?.onRetry ?? DEFAULT_RETRY_OPTIONS.onRetry,
});
// Transport failures: the request did not complete, for reasons below HTTP.
@@ -234,7 +239,9 @@ export const withRetry = async <T>(
// and its status intact.
throw err;
}
await policy.sleep(backoffMs(attempt, policy));
const delayMs = backoffMs(attempt, policy);
policy.onRetry(err, attempt, delayMs);
await policy.sleep(delayMs);
}
}
};
src/thumbnails.ts
+6 -1
View File
@@ -49,7 +49,10 @@ export interface ThumbnailFixResult {
reason?: string;
}
export type ProgressCallback = (message: string) => void;
// Reports one line of progress. `level` is "error" when the line reports a
// file that could not be checked or fixed, and is left out for every other
// line.
export type ProgressCallback = (message: string, level?: "error") => void;
// Enumerate every file the library knows about, newest album first, each file
// once, and report those whose server-side thumbnail is missing. "Missing" is
@@ -111,6 +114,7 @@ export const listMissingThumbnails = async (
} else {
log(
`[${album.name}] Could not check ${photo.title}: ${err instanceof Error ? err.message : String(err)} (not reported as missing)`,
"error",
);
}
}
@@ -359,6 +363,7 @@ export const fixMissingThumbnails = async (
} catch (err) {
log(
`[${collectionName}] FAILED ${title}: ${err instanceof Error ? err.message : err}`,
"error",
);
results.push({
fileID,