Accumulate streamDecrypt reads linearly, not quadratically (closes #21)
check / check (push) Successful in 14s

streamDecrypt no longer recopies the whole accumulation buffer on every network read. Reads are queued with a running byte count and a contiguous buffer is materialised only at each ENC_CHUNK_SIZE boundary, with a straddling read split via a subarray view, so each byte is copied once instead of O(n^2). TAG_FINAL truncation detection, final-chunk handling, retry, and the per-chunk progress hook are unchanged. A new test feeds a multi-chunk body through a ReadableStream that yields many small pieces, exercising the fragmented-read path.

Model: opus-4-8
This commit was merged in pull request #61.
This commit is contained in:
2026-09-22 15:13:09 +02:00
parent 7570055a5b
commit 57e0c69651
2 changed files with 118 additions and 10 deletions
+38 -10
View File
@@ -51,11 +51,41 @@ const streamDecrypt = async (
): Promise<number> => {
const state = initStreamPull(header, key);
const reader = stream.getReader();
let buffer = new Uint8Array(0);
// Incoming reads are held as-is and only stitched into a contiguous chunk
// at each `ENC_CHUNK_SIZE` boundary, so every received byte is copied once.
// Concatenating on each read instead — reallocating the whole accumulator
// per read — is O(n^2) in the bytes buffered, and for a 4 MiB chunk that
// memory churn dwarfs the libsodium decryption itself.
const pending: Uint8Array[] = [];
let pendingBytes = 0;
let totalPlain = 0;
let chunksPulled = 0;
let lastTag = -1;
// Remove the first `size` bytes from `pending` as one contiguous buffer.
// A read that straddles the boundary is split with `subarray` (a view, no
// copy); its tail stays queued for the next chunk. `size` never exceeds
// `pendingBytes`, so the queue always holds enough.
const takeContiguous = (size: number): Uint8Array => {
const out = new Uint8Array(size);
let offset = 0;
while (offset < size) {
const piece = pending[0]!;
const need = size - offset;
if (piece.length <= need) {
out.set(piece, offset);
offset += piece.length;
pending.shift();
} else {
out.set(piece.subarray(0, need), offset);
pending[0] = piece.subarray(need);
offset += need;
}
}
pendingBytes -= size;
return out;
};
const consume = async (
plaintext: Uint8Array,
tag: number,
@@ -69,16 +99,13 @@ const streamDecrypt = async (
for (;;) {
const { done, value } = await reader.read();
if (value) {
const merged = new Uint8Array(buffer.length + value.length);
merged.set(buffer);
merged.set(value, buffer.length);
buffer = merged;
if (value && value.length > 0) {
pending.push(value);
pendingBytes += value.length;
}
while (buffer.length >= ENC_CHUNK_SIZE) {
const encChunk = buffer.slice(0, ENC_CHUNK_SIZE);
buffer = buffer.slice(ENC_CHUNK_SIZE);
while (pendingBytes >= ENC_CHUNK_SIZE) {
const encChunk = takeContiguous(ENC_CHUNK_SIZE);
// A whole chunk that fails to authenticate while the stream carries
// on is corruption, not truncation; that error propagates unchanged.
const { plaintext, tag } = pullStreamChunk(state, encChunk);
@@ -86,7 +113,8 @@ const streamDecrypt = async (
}
if (done) {
if (buffer.length > 0) {
if (pendingBytes > 0) {
const buffer = takeContiguous(pendingBytes);
// Whatever is left over once every whole chunk has been
// consumed must be the stream's final chunk, and a final
// chunk that actually arrived in full authenticates. If it