sneak/quak
1
0
Fork 0
Code Issues 8 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Make the download deadline an idle deadline and cancel failed bodies (closes #24)
check / check (push) Successful in 17s
Details

Browse Source 
downloadTimeoutMs now aborts a file or thumbnail download only when no
bytes have arrived for that long (default 60 s, was a 600 s cap on the
whole transfer), so a slow download that keeps making progress completes.
The abort reason is still a TimeoutError, so retry classification is
unchanged. A download that fails before reading the whole response body
cancels it, including when the temp file cannot be opened or the header
is malformed, so a failed file no longer holds its connection.

Model: opus-5-5
This commit was merged in pull request #88.
This commit is contained in:
clawbot
2026-09-23 03:14:42 +02:00
parent 2b410c3ed6
commit 28a2beeab8
6 changed files with 331 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/download/index.ts
+63 -47
View File
@@ -98,47 +98,53 @@ const streamDecrypt = async (
onProgress?.(totalPlain);
};
for (;;) {
const { done, value } = await reader.read();
if (value && value.length > 0) {
pending.push(value);
pendingBytes += value.length;
}
try {
for (;;) {
const { done, value } = await reader.read();
if (value && value.length > 0) {
pending.push(value);
pendingBytes += value.length;
}
while (pendingBytes >= ENC_CHUNK_SIZE) {
const encChunk = takeContiguous(ENC_CHUNK_SIZE);
// A whole chunk that fails to authenticate while the stream carries
// on is corruption, not truncation; that error propagates unchanged.
const { plaintext, tag } = pullStreamChunk(state, encChunk);
await consume(plaintext, tag);
}
while (pendingBytes >= ENC_CHUNK_SIZE) {
const encChunk = takeContiguous(ENC_CHUNK_SIZE);
// A whole chunk that fails to authenticate while the stream
// carries on is corruption, not truncation; that error
// propagates unchanged.
const { plaintext, tag } = pullStreamChunk(state, encChunk);
await consume(plaintext, tag);
}
if (done) {
if (pendingBytes > 0) {
const buffer = takeContiguous(pendingBytes);
// Whatever is left over once every whole chunk has been
// consumed must be the stream's final chunk, and a final
// chunk that actually arrived in full authenticates. If it
// does not, the body stopped part-way through a chunk — the
// ordinary shape of a dropped connection. Poly1305 cannot
// tell a partial chunk from a corrupt one, so this is
// reported as the truncation it almost always is, with the
// authentication failure kept as the error's cause. Only the
// pull is guarded: a sink failure on a chunk that did
// authenticate is a disk error, not a truncation.
let pulled;
try {
pulled = pullStreamChunk(state, buffer);
} catch (err) {
throw new TruncatedStreamError(
`download: stream truncated: response body ended with ${buffer.length} trailing bytes that did not authenticate as a final chunk (transfer stopped mid-chunk, or the data is corrupt)`,
{ cause: err },
);
if (done) {
if (pendingBytes > 0) {
const buffer = takeContiguous(pendingBytes);
// Whatever is left over once every whole chunk has been
// consumed must be the stream's final chunk, and a final
// chunk that actually arrived in full authenticates. If
// it does not, the body stopped part-way through a chunk
// — the ordinary shape of a dropped connection. Poly1305
// cannot tell a partial chunk from a corrupt one, so this
// is reported as the truncation it almost always is, with
// the authentication failure kept as the error's cause.
// Only the pull is guarded: a sink failure on a chunk
// that did authenticate is a disk error, not a
// truncation.
let pulled;
try {
pulled = pullStreamChunk(state, buffer);
} catch (err) {
throw new TruncatedStreamError(
`download: stream truncated: response body ended with ${buffer.length} trailing bytes that did not authenticate as a final chunk (transfer stopped mid-chunk, or the data is corrupt)`,
{ cause: err },
);
}
await consume(pulled.plaintext, pulled.tag);
}
await consume(pulled.plaintext, pulled.tag);
break;
}
break;
}
} finally {
reader.releaseLock();
}
// Only the last chunk of a secretstream carries TAG_FINAL. Everything a
@@ -245,17 +251,27 @@ const decryptToTemp = async (
onProgress?: ProgressCallback,
): Promise<number> => {
let bytesWritten = 0;
await stageAtomic(destination, async (handle) => {
bytesWritten = await streamDecrypt(
stream,
header,
key,
async (plaintext) => {
await handle.write(plaintext);
},
onProgress,
);
});
try {
await stageAtomic(destination, async (handle) => {
bytesWritten = await streamDecrypt(
stream,
header,
key,
async (plaintext) => {
await handle.write(plaintext);
},
onProgress,
);
});
} catch (err) {
// Cancel the body so its connection is closed now rather than held
// until the stream is garbage collected. A backup run carries on past
// a failed file, so without this every failure would hold a socket.
// This covers every failure, including a temp file that cannot be
// opened and a header that is rejected before the body is read.
await stream.cancel(err).catch(() => undefined);
throw err;
}
return bytesWritten;
};