backup-metadata: keep going when an ML data request fails (closes #101)
check / check (push) Successful in 27s
check / check (push) Successful in 27s
Each ML data request of up to 200 files is now tried on its own. A request that still fails after its retries is logged, its files are written with the reason in `mlDataError`, and the command exits 1 once the dump is complete. `fetchMLData`, used only here, is removed in favour of a per-batch loop over `fetchMLDataBatch`. Model: opus-5-5
This commit was merged in pull request #114.
This commit is contained in:
+2
-25
@@ -5,9 +5,8 @@
|
||||
// comes back encrypted under the file's own key and gzipped; decrypting and
|
||||
// gunzipping yields the JSON payload
|
||||
// `{ face: { faces: [...] }, clip: { embedding } }`. Ente caps a request at 200
|
||||
// ids, so `fetchMLData` batches for callers that want many at once while
|
||||
// `fetchMLDataBatch` is the single-request unit the library submits to its
|
||||
// request pool.
|
||||
// ids, so callers that want many at once split them into batches of
|
||||
// `MLDATA_BATCH_SIZE` and call `fetchMLDataBatch` once per batch.
|
||||
|
||||
import { gunzipSync } from "node:zlib";
|
||||
|
||||
@@ -69,25 +68,3 @@ export const fetchMLDataBatch = async (
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
// Fetch ML data for arbitrarily many ids, batching at `MLDATA_BATCH_SIZE`. Used
|
||||
// by the one-shot metadata backup; the library fetches through its request pool
|
||||
// with `fetchMLDataBatch` instead.
|
||||
export const fetchMLData = async (
|
||||
api: ApiClient,
|
||||
fileIDs: number[],
|
||||
fileKeys: Map<number, Uint8Array>,
|
||||
): Promise<Map<number, MLData>> => {
|
||||
const result = new Map<number, MLData>();
|
||||
for (let i = 0; i < fileIDs.length; i += MLDATA_BATCH_SIZE) {
|
||||
const batch = fileIDs.slice(i, i + MLDATA_BATCH_SIZE);
|
||||
for (const [id, payload] of await fetchMLDataBatch(
|
||||
api,
|
||||
batch,
|
||||
fileKeys,
|
||||
)) {
|
||||
result.set(id, payload);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user