quak backup writes each file's ML data into its JSON (closes #163)
check / check (push) Successful in 1m37s

lib.backup() now waits for an ML data fetch, joining the one its refresh
started or starting one, before it writes the per-file JSON, and each
file's JSON carries the cached payload as mlData. When the fetch fails,
each file with no cached ML data gets mlDataError and an entry in
failures.json, so the result counts it as failed and quak backup exits 1;
the next run fetches again.

Judgement call: the wait comes after the originals are downloaded, so the fetch runs alongside the downloads.
Judgement call: a failed ML fetch is recorded per file in failures.json, which is how the exit code goes non-zero without changing src/cli-commands.ts.

Model: opus-5-5
This commit was merged in pull request #172.
This commit is contained in:
2026-10-06 05:13:27 +02:00
parent 14ac7d05ef
commit ab4d5d2cc2
6 changed files with 214 additions and 35 deletions
+23 -10
View File
@@ -279,7 +279,8 @@ export class Library {
private cycle?: Promise<void>;
// Guards the ML fetch pass so a slow backfill never runs twice at once; a
// refresh whose pass is still running kicks nothing new. Holds the running
// pass, so `close()` can wait for it.
// pass, so `close()` and `backup()` can wait for it. It rejects when the
// pass fails.
private mlFetch?: Promise<void>;
private closed = false;
private lastRefreshAt?: number;
@@ -569,8 +570,9 @@ export class Library {
// does, joining one already running, and rejects before touching any file
// when it fails. Then puts pending originals at their save paths as
// `Photo.download()` does (and optional thumbnails) through the content
// cache and pools, and rebuilds the derived symlink/JSON views from the
// model. Throws before any network work when no content cache backs the
// cache and pools, waits for an ML data fetch, and rebuilds the derived
// symlink/JSON views from the model, each file's JSON with its ML data.
// Throws before any network work when no content cache backs the
// originals it must fetch.
backup(opts?: BackupOptions): Promise<BackupResult> {
const downloadDirectory =
@@ -593,6 +595,8 @@ export class Library {
original: (fileID, destination) =>
cache!.backupOriginal(fileID, destination),
thumbnail: (fileID) => cache!.thumbnail(fileID),
fetchMLData: () => this.fetchMLDataNow(),
mlData: (fileID) => this.mldata.forFile({ fileID }),
},
{ ...opts, downloadDirectory },
);
@@ -612,7 +616,7 @@ export class Library {
this.timer = undefined;
}
await this.cycle?.catch(() => {});
await this.mlFetch;
await this.mlFetch?.catch(() => {});
await precacheClosed;
}
@@ -676,10 +680,9 @@ export class Library {
// Backfill ML data for the files this refresh knows about. It runs
// outside the refresh's success/failure so a fetch or disk problem
// there never marks the metadata refresh failed, and it is not
// awaited so it never stalls the refresh interval.
this.mlFetch ??= this.runMLFetch().finally(() => {
this.mlFetch = undefined;
});
// awaited so it never stalls the refresh interval. Its failure is
// reported through `status()` and `onProgress`.
void this.fetchMLDataNow().catch(() => {});
} catch (err) {
const error = err instanceof Error ? err.message : String(err);
this.lastError = error;
@@ -784,10 +787,19 @@ export class Library {
}
}
// Join the running ML fetch pass, or start one when none runs. Resolves at
// once when the client cannot fetch ML data; rejects when the pass fails.
private fetchMLDataNow(): Promise<void> {
this.mlFetch ??= this.runMLFetch().finally(() => {
this.mlFetch = undefined;
});
return this.mlFetch;
}
// One ML fetch pass: fetch, decrypt and store the ML data for every file
// the store knows about that is not cached (or whose `updationTime` has
// advanced), through the metadata pool, and update the CLIP index. Guarded
// so passes never overlap; a failure is reported, not thrown.
// advanced), through the metadata pool, and update the CLIP index. A
// failure is reported through `status()` and `onProgress`, then thrown.
private async runMLFetch(): Promise<void> {
const mldata = this.mlStore;
// Bind so the call keeps the client as its receiver when invoked
@@ -831,6 +843,7 @@ export class Library {
const error = err instanceof Error ? err.message : String(err);
this.lastMLError = error;
this.emit({ operation: "fetchMLData", status: "failed", error });
throw err;
}
}