Accumulate streamDecrypt reads linearly, not quadratically (closes #21)
check / check (push) Successful in 14s
check / check (push) Successful in 14s
streamDecrypt no longer recopies the whole accumulation buffer on every network read. Reads are queued with a running byte count and a contiguous buffer is materialised only at each ENC_CHUNK_SIZE boundary, with a straddling read split via a subarray view, so each byte is copied once instead of O(n^2). TAG_FINAL truncation detection, final-chunk handling, retry, and the per-chunk progress hook are unchanged. A new test feeds a multi-chunk body through a ReadableStream that yields many small pieces, exercising the fragmented-read path. Model: opus-4-8
This commit was merged in pull request #61.
This commit is contained in:
+38
-10
@@ -51,11 +51,41 @@ const streamDecrypt = async (
|
||||
): Promise<number> => {
|
||||
const state = initStreamPull(header, key);
|
||||
const reader = stream.getReader();
|
||||
let buffer = new Uint8Array(0);
|
||||
// Incoming reads are held as-is and only stitched into a contiguous chunk
|
||||
// at each `ENC_CHUNK_SIZE` boundary, so every received byte is copied once.
|
||||
// Concatenating on each read instead — reallocating the whole accumulator
|
||||
// per read — is O(n^2) in the bytes buffered, and for a 4 MiB chunk that
|
||||
// memory churn dwarfs the libsodium decryption itself.
|
||||
const pending: Uint8Array[] = [];
|
||||
let pendingBytes = 0;
|
||||
let totalPlain = 0;
|
||||
let chunksPulled = 0;
|
||||
let lastTag = -1;
|
||||
|
||||
// Remove the first `size` bytes from `pending` as one contiguous buffer.
|
||||
// A read that straddles the boundary is split with `subarray` (a view, no
|
||||
// copy); its tail stays queued for the next chunk. `size` never exceeds
|
||||
// `pendingBytes`, so the queue always holds enough.
|
||||
const takeContiguous = (size: number): Uint8Array => {
|
||||
const out = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
while (offset < size) {
|
||||
const piece = pending[0]!;
|
||||
const need = size - offset;
|
||||
if (piece.length <= need) {
|
||||
out.set(piece, offset);
|
||||
offset += piece.length;
|
||||
pending.shift();
|
||||
} else {
|
||||
out.set(piece.subarray(0, need), offset);
|
||||
pending[0] = piece.subarray(need);
|
||||
offset += need;
|
||||
}
|
||||
}
|
||||
pendingBytes -= size;
|
||||
return out;
|
||||
};
|
||||
|
||||
const consume = async (
|
||||
plaintext: Uint8Array,
|
||||
tag: number,
|
||||
@@ -69,16 +99,13 @@ const streamDecrypt = async (
|
||||
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (value) {
|
||||
const merged = new Uint8Array(buffer.length + value.length);
|
||||
merged.set(buffer);
|
||||
merged.set(value, buffer.length);
|
||||
buffer = merged;
|
||||
if (value && value.length > 0) {
|
||||
pending.push(value);
|
||||
pendingBytes += value.length;
|
||||
}
|
||||
|
||||
while (buffer.length >= ENC_CHUNK_SIZE) {
|
||||
const encChunk = buffer.slice(0, ENC_CHUNK_SIZE);
|
||||
buffer = buffer.slice(ENC_CHUNK_SIZE);
|
||||
while (pendingBytes >= ENC_CHUNK_SIZE) {
|
||||
const encChunk = takeContiguous(ENC_CHUNK_SIZE);
|
||||
// A whole chunk that fails to authenticate while the stream carries
|
||||
// on is corruption, not truncation; that error propagates unchanged.
|
||||
const { plaintext, tag } = pullStreamChunk(state, encChunk);
|
||||
@@ -86,7 +113,8 @@ const streamDecrypt = async (
|
||||
}
|
||||
|
||||
if (done) {
|
||||
if (buffer.length > 0) {
|
||||
if (pendingBytes > 0) {
|
||||
const buffer = takeContiguous(pendingBytes);
|
||||
// Whatever is left over once every whole chunk has been
|
||||
// consumed must be the stream's final chunk, and a final
|
||||
// chunk that actually arrived in full authenticates. If it
|
||||
|
||||
Reference in New Issue
Block a user