backup-metadata: keep going when an ML data request fails (closes #101)
check / check (push) Successful in 27s

Each ML data request of up to 200 files is now tried on its own. A request
that still fails after its retries is logged, its files are written with the
reason in `mlDataError`, and the command exits 1 once the dump is complete.
`fetchMLData`, used only here, is removed in favour of a per-batch loop over
`fetchMLDataBatch`.

Model: opus-5-5
This commit was merged in pull request #114.
This commit is contained in:
2026-09-23 05:49:38 +02:00
parent d05b53d560
commit bf3b20df2f
6 changed files with 123 additions and 37 deletions
+2 -25
View File
@@ -5,9 +5,8 @@
// comes back encrypted under the file's own key and gzipped; decrypting and
// gunzipping yields the JSON payload
// `{ face: { faces: [...] }, clip: { embedding } }`. Ente caps a request at 200
// ids, so `fetchMLData` batches for callers that want many at once while
// `fetchMLDataBatch` is the single-request unit the library submits to its
// request pool.
// ids, so callers that want many at once split them into batches of
// `MLDATA_BATCH_SIZE` and call `fetchMLDataBatch` once per batch.
import { gunzipSync } from "node:zlib";
@@ -69,25 +68,3 @@ export const fetchMLDataBatch = async (
}
return result;
};
// Fetch ML data for arbitrarily many ids, batching at `MLDATA_BATCH_SIZE`. Used
// by the one-shot metadata backup; the library fetches through its request pool
// with `fetchMLDataBatch` instead.
export const fetchMLData = async (
api: ApiClient,
fileIDs: number[],
fileKeys: Map<number, Uint8Array>,
): Promise<Map<number, MLData>> => {
const result = new Map<number, MLData>();
for (let i = 0; i < fileIDs.length; i += MLDATA_BATCH_SIZE) {
const batch = fileIDs.slice(i, i + MLDATA_BATCH_SIZE);
for (const [id, payload] of await fetchMLDataBatch(
api,
batch,
fileKeys,
)) {
result.set(id, payload);
}
}
return result;
};