sneak
703ef25623
check / check (push) Waiting to run
`fetchMLDataBatch` dropped an entry that did not decrypt, gunzip or parse, so the file looked like one with no ML data and the run exited 0. It now returns the reason beside the payloads, also for ML data that is not a JSON object. The library gives the reasons from its last ML data fetch that succeeded in `status().mlDataErrors` and the `fetchMLData` "done" event; a fetch that fails leaves them as they were. `quak backup` writes the reason as `mlDataError` for a file with no ML data cached and records the file in `failures.json`. `backup-metadata` writes `mlDataError` for it and exits 1; `runMetadataBackup` returns `filesWithMLDataError`. Judgement call: a file whose ML data is cached keeps it in its JSON when a newer copy does not decode, as when the fetch fails. Model: opus-5-5
55 lines
1.8 KiB
TypeScript
55 lines
1.8 KiB
TypeScript
// `fetchMLDataBatch`, against a server that sends one file's ML data.
|
|
//
|
|
// ML data that decrypts and gunzips but is not a JSON object is reported with
|
|
// the reason, as ML data that does not parse is, and is never returned as a
|
|
// payload: the library would fail to store it.
|
|
|
|
import { gzipSync } from "node:zlib";
|
|
import sodium from "libsodium-wrappers-sumo";
|
|
import { beforeAll, describe, expect, it } from "vitest";
|
|
|
|
import { ApiClient } from "../../src/api/client.js";
|
|
import { encryptBlob, init, toBase64 } from "../../src/crypto/index.js";
|
|
import { fetchMLDataBatch } from "../../src/mldata-fetch.js";
|
|
|
|
beforeAll(async () => {
|
|
await init();
|
|
await sodium.ready;
|
|
});
|
|
|
|
// Fetch the ML data of file 1 from a server that sends `json`, gzipped and
|
|
// encrypted with the file's key.
|
|
const fetchServing = async (json: string) => {
|
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
|
const { header, ciphertext } = encryptBlob(
|
|
new Uint8Array(gzipSync(json)),
|
|
key,
|
|
);
|
|
const data = [
|
|
{
|
|
fileID: 1,
|
|
encryptedData: toBase64(ciphertext),
|
|
decryptionHeader: toBase64(header),
|
|
},
|
|
];
|
|
const api = new ApiClient({
|
|
fetch: async () =>
|
|
new Response(JSON.stringify({ data }), {
|
|
headers: { "content-type": "application/json" },
|
|
}),
|
|
});
|
|
return fetchMLDataBatch(api, [1], new Map([[1, key]]));
|
|
};
|
|
|
|
describe("fetchMLDataBatch", () => {
|
|
it.each(["null", "5", "[]"])(
|
|
"gives a reason for ML data that is %s",
|
|
async (json) => {
|
|
const { payloads, errors } = await fetchServing(json);
|
|
|
|
expect(payloads.size).toBe(0);
|
|
expect(errors.get(1)).toEqual(new Error("not a JSON object"));
|
|
},
|
|
);
|
|
});
|