Verify secretstream TAG_FINAL and write downloads atomically (closes #1) #20

Merged
clawbot merged 4 commits from download-tag-final-atomic-write into main 2026-08-09 04:59:44 +02:00
5 changed files with 313 additions and 57 deletions
Showing only changes of commit 8a200be8a7 - Show all commits

View File

@@ -14,6 +14,6 @@ export {
pullStreamChunk,
STREAM_CHUNK_OVERHEAD,
STREAM_CHUNK_SIZE,
STREAM_TAG_FINAL,
streamTagFinal,
type StreamPullState,
} from "./stream.js";

View File

@@ -8,17 +8,17 @@ export const STREAM_CHUNK_SIZE = 4 * 1024 * 1024;
// 16 bytes of Poly1305 tag plus 1 byte of secretstream tag.
export const STREAM_CHUNK_OVERHEAD = 17;
// The secretstream tag that marks the last chunk of a stream, i.e.
// libsodium's crypto_secretstream_xchacha20poly1305_TAG_FINAL. Exported so
// callers (the download layer) can detect truncation without importing
// sodium themselves.
// libsodium's crypto_secretstream_xchacha20poly1305_TAG_FINAL: the tag that
// marks the last chunk of a stream. Exported so callers (the download layer)
// can detect truncation without importing sodium themselves.
//
// Written as a literal because libsodium only attaches its constants to the
// module object once `sodium.ready` has resolved, which is long after this
// module is evaluated. The value is fixed at 3 by the secretstream wire
// format; test/crypto/stream.test.ts pins it against libsodium's own
// constant so the two cannot drift apart unnoticed.
export const STREAM_TAG_FINAL = 3;
// This is a function rather than a constant because libsodium attaches its
// constants to the module object only once `sodium.ready` has resolved, which
// is long after this module is evaluated — a module-level read would bind
// `undefined`. Reading it at call time returns the library's own value, so
// there is no second copy of a protocol constant to keep in sync.
export const streamTagFinal = (): number =>
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL;
// Encrypt a small blob as a single secretstream chunk with TAG_FINAL.
// Returns the header and ciphertext. Used for encrypting thumbnails
@@ -59,8 +59,11 @@ export const decryptBlob = (
): Uint8Array => {
const state = initStreamPull(header, key);
const { plaintext, tag } = pullStreamChunk(state, ciphertext);
if (tag !== STREAM_TAG_FINAL) {
throw new Error(`decryptBlob: expected TAG_FINAL (3), got tag ${tag}`);
const tagFinal = streamTagFinal();
if (tag !== tagFinal) {
throw new Error(
`decryptBlob: expected TAG_FINAL (${tagFinal}), got tag ${tag}`,
);
}
return plaintext;
};

View File

@@ -7,7 +7,7 @@ import {
pullStreamChunk,
STREAM_CHUNK_OVERHEAD,
STREAM_CHUNK_SIZE,
STREAM_TAG_FINAL,
streamTagFinal,
} from "../crypto/index.js";
import type { ApiClient } from "../api/client.js";
import type { EnteFile } from "../model/types.js";
@@ -53,11 +53,27 @@ const streamDecrypt = async (
if (done) {
if (buffer.length > 0) {
const { plaintext, tag } = pullStreamChunk(state, buffer);
plainChunks.push(plaintext);
totalPlain += plaintext.length;
// Whatever is left over once every whole chunk has been
// consumed must be the stream's final chunk, and a final
// chunk that actually arrived in full authenticates. If it
// does not, the body stopped part-way through a chunk — the
// ordinary shape of a dropped connection. Poly1305 cannot
// tell a partial chunk from a corrupt one, so this is
// reported as the truncation it almost always is, with the
// authentication failure kept as the error's cause.
let pulled;
try {
pulled = pullStreamChunk(state, buffer);
} catch (err) {
throw new Error(
`download: stream truncated: response body ended with ${buffer.length} trailing bytes that did not authenticate as a final chunk (transfer stopped mid-chunk, or the data is corrupt)`,
{ cause: err },
);
}
plainChunks.push(pulled.plaintext);
totalPlain += pulled.plaintext.length;
chunksPulled++;
lastTag = tag;
lastTag = pulled.tag;
}
break;
}
@@ -73,9 +89,10 @@ const streamDecrypt = async (
"download: stream truncated: response body contained no secretstream chunks",
);
}
if (lastTag !== STREAM_TAG_FINAL) {
const tagFinal = streamTagFinal();
if (lastTag !== tagFinal) {
throw new Error(
`download: stream truncated: last chunk tag ${lastTag}, expected TAG_FINAL (${STREAM_TAG_FINAL})`,
`download: stream truncated: last chunk tag ${lastTag}, expected TAG_FINAL (${tagFinal})`,
);
}

View File

@@ -16,7 +16,8 @@
*
* These tests pin:
* - The chunk-size constants match Ente's expectations, and the
* re-exported `STREAM_TAG_FINAL` matches libsodium's own constant.
* re-exported `streamTagFinal()` matches the tag a real final chunk
* carries.
* - The pull state can decrypt a multi-chunk stream produced by
* sodium.crypto_secretstream_xchacha20poly1305_push, in order.
* - The tag byte is propagated to the caller.
@@ -31,7 +32,7 @@ import {
pullStreamChunk,
STREAM_CHUNK_OVERHEAD,
STREAM_CHUNK_SIZE,
STREAM_TAG_FINAL,
streamTagFinal,
} from "../../src/crypto/index.js";
describe("crypto stream constants", () => {
@@ -49,19 +50,36 @@ describe("crypto stream constants", () => {
});
/**
* `STREAM_TAG_FINAL` is re-exported so callers can detect a truncated
* `streamTagFinal()` is re-exported so callers can detect a truncated
* stream (a body that ended on a non-final chunk) without importing
* libsodium themselves. It has to be declared as a literal, because
* libsodium only attaches its own constants to the module object after
* `sodium.ready` resolves — long after this library's modules are
* evaluated. This test is what keeps the literal honest.
* libsodium themselves.
*
* It is a function, not a constant, and that is load-bearing: libsodium
* attaches its own constants to the module object only after
* `sodium.ready` resolves, which is long after this library's modules are
* evaluated. Reading the value at call time yields libsodium's number;
* reading it at module scope would yield `undefined`, and every
* truncation check downstream would then compare against `undefined` and
* reject good downloads. This test pins the behaviour end to end — it
* compares against the tag observed on a real final chunk pulled back off
* the wire format, so it fails if the accessor is ever made eager.
*/
it("STREAM_TAG_FINAL equals libsodium's TAG_FINAL", async () => {
it("streamTagFinal() is the tag carried by a real final chunk", async () => {
await init();
await sodium.ready;
expect(STREAM_TAG_FINAL).toBe(
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const push =
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push(
push.state,
new TextEncoder().encode("last chunk"),
null,
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
);
const state = initStreamPull(push.header, key);
expect(pullStreamChunk(state, ciphertext).tag).toBe(streamTagFinal());
});
});

View File

@@ -20,6 +20,8 @@
* an explicit `TAG_FINAL` check a truncated body is indistinguishable from
* a complete one. `streamDecrypt` therefore refuses to return unless the
* stream ended on `TAG_FINAL`, and the error says the stream was truncated.
* A transfer that stopped part-way through a chunk is reported the same
* way, since a final chunk that arrived in full always authenticates.
*
* 2. **The destination path is written atomically.** Plaintext goes to a
* temporary sibling file first and is `rename`d into place only after the
@@ -29,7 +31,8 @@
* untouched — whatever was there before is still there, byte for byte, and
* no partial file has appeared. This matters because `runBackup` skips any
* existing non-empty file, so a partial write would be treated as complete
* forever after.
* forever after. The staging file and the rename are observed directly (see
* the `rename` hook below), not inferred from an empty directory.
*
* These tests build synthetic encrypted files using sodium's push API,
* serve them from a mock fetch, and verify the decrypted output on disk.
@@ -43,11 +46,19 @@ import {
mkdtempSync,
writeFileSync,
} from "node:fs";
import { join } from "node:path";
import { dirname, join } from "node:path";
import { tmpdir } from "node:os";
import { createHash } from "node:crypto";
import sodium from "libsodium-wrappers-sumo";
import { beforeAll, afterAll, describe, expect, it } from "vitest";
import {
beforeAll,
beforeEach,
afterAll,
describe,
expect,
it,
vi,
} from "vitest";
import { init, toBase64, STREAM_CHUNK_SIZE } from "../../src/crypto/index.js";
import { ApiClient } from "../../src/api/client.js";
import { downloadFile, downloadThumbnail } from "../../src/download/index.js";
@@ -57,6 +68,52 @@ import type { EnteFile, FileMetadata } from "../../src/model/types.js";
// Test helpers
// ---------------------------------------------------------------------------
/**
* `rename` is intercepted so that the tests can observe — and fail — the
* final step of the atomic write.
*
* Every other failure in this file is injected inside `streamDecrypt`, which
* runs before anything is written to disk. Those tests therefore cannot tell
* an atomic write from a plain `writeFile`: under both, nothing was ever
* created, so an empty directory proves nothing about cleanup. This hook is
* what closes that gap. It records each rename the downloader performs,
* including whether the source existed at that moment (i.e. that the staged
* temp file really was written), and can be told to fail the rename so the
* cleanup path runs with a temp file genuinely on disk.
*
* `vi.hoisted` is required: `vi.mock` factories are hoisted above the imports,
* so a plain module-level `const` would still be in its temporal dead zone by
* the time the factory runs.
*/
const renameHook = vi.hoisted(() => ({
calls: [] as { from: string; to: string; sourceExisted: boolean }[],
failWith: null as Error | null,
}));
vi.mock("node:fs/promises", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs/promises")>();
const { existsSync: sourceExists } = await import("node:fs");
return {
...actual,
rename: async (from: string, to: string): Promise<void> => {
renameHook.calls.push({
from,
to,
sourceExisted: sourceExists(from),
});
if (renameHook.failWith !== null) {
throw renameHook.failWith;
}
await actual.rename(from, to);
},
};
});
beforeEach(() => {
renameHook.calls.length = 0;
renameHook.failWith = null;
});
let testDir: string;
beforeAll(async () => {
@@ -71,6 +128,35 @@ afterAll(() => {
}
});
/**
* Deterministic stand-in for random test payloads.
*
* Fixture *content* is never load-bearing here — the assertions turn on
* length, framing, and the secretstream tag — but it must not be a constant
* fill either, or a downloader that reordered or repeated chunks would still
* produce the expected bytes. A seeded linear congruential generator gives
* both: byte patterns that differ across every offset and seed, reproducible
* on any machine, which is what the README asks of fixtures.
*
* It is also the difference between a fast suite and a broken one.
* `sodium.randombytes_buf` goes through the wasm wrapper a byte at a time and
* costs roughly 20 seconds for the 4 MiB chunk below — about two hundred
* times what it costs to encrypt the same buffer, and on its own enough to
* push `make test` past the 30-second cap in `script/test`. This loop fills
* 4 MiB in a few milliseconds.
*/
const patternBytes = (length: number, seed: number): Uint8Array => {
const out = new Uint8Array(length);
let x = seed >>> 0;
for (let i = 0; i < length; i++) {
// Numerical Recipes' LCG constants; the high byte is used because
// the low bits of an LCG have short periods.
x = (Math.imul(x, 1664525) + 1013904223) >>> 0;
out[i] = (x >>> 24) & 0xff;
}
return out;
};
/**
* Encrypt `plaintext` as a secretstream file body (single chunk with
* TAG_FINAL). Returns the key, header, and ciphertext that the mock CDN
@@ -120,7 +206,7 @@ const encryptMultiChunkBody = (
const plainParts: Uint8Array[] = [];
for (let i = 0; i < leadingChunks; i++) {
const plain = sodium.randombytes_buf(STREAM_CHUNK_SIZE);
const plain = patternBytes(STREAM_CHUNK_SIZE, i + 1);
plainParts.push(plain);
cipherParts.push(
sodium.crypto_secretstream_xchacha20poly1305_push(
@@ -132,7 +218,7 @@ const encryptMultiChunkBody = (
);
}
const finalPlain = sodium.randombytes_buf(finalChunkPlainSize);
const finalPlain = patternBytes(finalChunkPlainSize, leadingChunks + 1);
plainParts.push(finalPlain);
const finalCipher = sodium.crypto_secretstream_xchacha20poly1305_push(
push.state,
@@ -226,9 +312,11 @@ const expectSameBytes = (actual: Uint8Array, expected: Uint8Array): void => {
};
/**
* A multi-chunk fixture shared by the truncation tests. Building it costs a
* few MiB of encryption, so it is built once: one full 4 MiB `TAG_MESSAGE`
* chunk followed by a small `TAG_FINAL` chunk.
* A multi-chunk fixture shared by the truncation tests: one full 4 MiB
* `TAG_MESSAGE` chunk followed by a small `TAG_FINAL` chunk. It is built once
* and shared because encrypting 4 MiB costs about 100ms. Its plaintext is
* generated rather than drawn from the CSPRNG, which is what keeps that
* encryption the whole cost of the fixture.
*/
let multiChunk: ReturnType<typeof encryptMultiChunkBody>;
let multiChunkKey: Uint8Array;
@@ -295,28 +383,33 @@ describe("downloadFile", () => {
});
it("uses metadata.title as filename when outPath is omitted", async () => {
// With no `outPath`, the destination is `metadata.title`, used
// verbatim as a path. The title here is therefore given inside the
// test's temporary directory: a bare relative name would resolve
// against the process working directory, i.e. the repo root, and
// `make check` must not create files in the repo — a failure between
// the write and any cleanup would leave one behind.
const plaintext = new Uint8Array([1, 2, 3]);
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(plaintext, key);
const thumbPush =
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
const file = buildMockEnteFile(key, header, thumbPush.header);
file.metadata.title = "fallback-name.png";
const titlePath = join(testDir, "fallback-name.png");
file.metadata.title = titlePath;
const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) });
const result = await downloadFile(api, file);
expect(result.path).toBe("fallback-name.png");
expect(result.path).toBe(titlePath);
expect(readFileSync(result.path)).toEqual(Buffer.from(plaintext));
// Clean up since it writes to cwd
if (existsSync(result.path)) rmSync(result.path);
});
it("handles a larger single-chunk file (random binary payload)", async () => {
// Most photos are under 4 MiB and therefore a single secretstream
// chunk. This test exercises a non-trivial payload size with
// random binary data (not just ASCII) to verify no encoding bugs.
const plaintext = sodium.randombytes_buf(100_000);
// arbitrary binary data (not just ASCII) to verify no encoding bugs.
const plaintext = patternBytes(100_000, 11);
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(plaintext, key);
const thumbPush =
@@ -414,6 +507,49 @@ describe.each(entryPoints)(
);
});
it("rejects a body whose final chunk arrived only in part", async () => {
// The likelier shape of a dropped connection: the transfer stops
// in the middle of a chunk rather than neatly between two. The
// bytes that arrived are a prefix of a complete chunk, so Poly1305
// rejects them — which is, cryptographically, indistinguishable
// from corruption of a whole chunk.
//
// It is still reported as truncation, because that is what it
// almost always is and because this library's entire reason for
// checking TAG_FINAL is to make a short transfer visible. Calling
// a short transfer "authentication failed" would send a user
// hunting for a corrupt file when their network is at fault. The
// underlying authentication failure is kept as the error's
// `cause`, so the real diagnosis is never lost.
const shortBody = multiChunk.body.slice(
0,
multiChunk.body.length - 8,
);
const { api, file } = fixtureFor(
multiChunkKey,
multiChunk.header,
shortBody,
);
const dir = freshDir();
const outPath = join(dir, "partial-final.bin");
const err = await download(api, file, outPath).catch(
(e: unknown) => e,
);
expect(err).toBeInstanceOf(Error);
expect((err as Error).message).toMatch(/truncated/i);
expect((err as Error).cause).toBeInstanceOf(Error);
expect(((err as Error).cause as Error).message).toMatch(
/authentication failed/i,
);
// And, as with every other failure, the destination is untouched
// and no staged temp file survives.
expect(existsSync(outPath)).toBe(false);
expect(readdirSync(dir)).toEqual([]);
});
it("rejects an empty body instead of writing a zero-byte file", async () => {
// Ente always emits at least one chunk, even for empty content:
// `encryptBlob` shows that a zero-length plaintext still produces a
@@ -445,7 +581,7 @@ describe.each(entryPoints)(
// the rejection.
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptNonFinalBody(
sodium.randombytes_buf(256),
patternBytes(256, 21),
key,
);
const { api, file } = fixtureFor(key, header, ciphertext);
@@ -463,19 +599,25 @@ describe.each(entryPoints)(
expect(readdirSync(dir)).toEqual([]);
});
it("leaves no file at the destination when a chunk fails authentication", async () => {
// The same guarantee has to hold for every failure mode, not just
// truncation. Here a byte of ciphertext is flipped, so Poly1305
// verification fails inside `pullStreamChunk`. The error must
// propagate unchanged (it is the real diagnosis) and the destination
// must still be untouched.
const plaintext = sodium.randombytes_buf(256);
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(plaintext, key);
const corrupted = Uint8Array.from(ciphertext);
it("reports a corrupt whole chunk as an authentication failure, not truncation", async () => {
// The counterpart to the partial-final-chunk case above, and the
// reason the two are distinguishable at all. A byte is flipped
// inside the first chunk of a multi-chunk body: that chunk arrives
// complete — the stream goes on past it — so its failure cannot be
// a short transfer. It is corruption, and the caller is told so,
// with `pullStreamChunk`'s error propagated unchanged because it is
// the real diagnosis.
//
// The same on-disk guarantee holds for this failure mode as for
// every other: nothing at the destination, nothing left over.
const corrupted = Uint8Array.from(multiChunk.body);
corrupted[10] ^= 0xff;
const { api, file } = fixtureFor(key, header, corrupted);
const { api, file } = fixtureFor(
multiChunkKey,
multiChunk.header,
corrupted,
);
const dir = freshDir();
const outPath = join(dir, "corrupt.bin");
@@ -498,7 +640,7 @@ describe.each(entryPoints)(
);
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptNonFinalBody(
sodium.randombytes_buf(256),
patternBytes(256, 22),
key,
);
const { api, file } = fixtureFor(key, header, ciphertext);
@@ -518,7 +660,7 @@ describe.each(entryPoints)(
// The mirror image of the previous test: a complete download does
// overwrite whatever was at the destination, atomically, via rename.
const existing = new TextEncoder().encode("stale contents");
const plaintext = sodium.randombytes_buf(512);
const plaintext = patternBytes(512, 23);
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(plaintext, key);
@@ -535,5 +677,81 @@ describe.each(entryPoints)(
// successful download leaves exactly one file behind.
expect(readdirSync(dir)).toEqual(["replaced.bin"]);
});
it("stages the plaintext in a sibling temp file and renames it into place", async () => {
// The atomic write, observed directly rather than inferred from an
// empty directory. Every other failure in this file is injected
// inside `streamDecrypt`, which runs before anything is written —
// so under those tests a plain `writeFile` to the destination would
// look identical. This one watches the rename itself.
//
// Two properties are load-bearing. The staging file must exist on
// disk when the rename happens: that is what makes the destination
// appear complete or not at all, instead of filling up as bytes
// land. And it must be a sibling of the destination, because
// `rename` is only atomic within one filesystem — staging in
// `/tmp` and renaming across a mount point would silently become a
// copy, reintroducing the partial-file window this exists to close.
const plaintext = patternBytes(512, 31);
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(plaintext, key);
const { api, file } = fixtureFor(key, header, ciphertext);
const dir = freshDir();
const outPath = join(dir, "staged.bin");
await download(api, file, outPath);
expect(renameHook.calls).toHaveLength(1);
const staged = renameHook.calls[0]!;
expect(staged.to).toBe(outPath);
expect(staged.from).not.toBe(outPath);
expect(dirname(staged.from)).toBe(dir);
expect(staged.sourceExisted).toBe(true);
// Afterwards the temp file is gone and only the destination is
// left, holding the complete plaintext.
expect(existsSync(staged.from)).toBe(false);
expect(readFileSync(outPath)).toEqual(Buffer.from(plaintext));
expect(readdirSync(dir)).toEqual(["staged.bin"]);
});
it("removes the staged temp file when the rename itself fails", async () => {
// The cleanup path. It can only run when something fails at or
// after the write, which no amount of bad network data can
// produce: by the time anything is written the whole stream has
// already decrypted and verified. Failing the rename is what
// reaches it — a real possibility on a full disk, a read-only
// mount, or a destination that has become a directory.
//
// This is the only case in which the temp file is on disk at the
// moment of failure, so it is the only one that can show it is
// actually removed rather than merely never created. It also pins
// that the caller sees the original failure: a cleanup that threw
// over the top of it would hide why the download failed.
const existing = new TextEncoder().encode("known-good contents");
const plaintext = patternBytes(512, 32);
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const { header, ciphertext } = encryptFileBody(plaintext, key);
const { api, file } = fixtureFor(key, header, ciphertext);
const dir = freshDir();
const outPath = join(dir, "rename-fails.bin");
writeFileSync(outPath, existing);
renameHook.failWith = new Error("simulated rename failure");
await expect(download(api, file, outPath)).rejects.toThrow(
"simulated rename failure",
);
expect(renameHook.calls).toHaveLength(1);
const staged = renameHook.calls[0]!;
expect(staged.sourceExisted).toBe(true);
expect(existsSync(staged.from)).toBe(false);
// The previous contents are still there, untouched, and the
// directory holds nothing else.
expect(readFileSync(outPath)).toEqual(Buffer.from(existing));
expect(readdirSync(dir)).toEqual(["rename-fails.bin"]);
});
},
);