diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 3cd569a..4e3e53b 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -14,6 +14,6 @@ export { pullStreamChunk, STREAM_CHUNK_OVERHEAD, STREAM_CHUNK_SIZE, - STREAM_TAG_FINAL, + streamTagFinal, type StreamPullState, } from "./stream.js"; diff --git a/src/crypto/stream.ts b/src/crypto/stream.ts index 7799cdd..cb48818 100644 --- a/src/crypto/stream.ts +++ b/src/crypto/stream.ts @@ -8,17 +8,17 @@ export const STREAM_CHUNK_SIZE = 4 * 1024 * 1024; // 16 bytes of Poly1305 tag plus 1 byte of secretstream tag. export const STREAM_CHUNK_OVERHEAD = 17; -// The secretstream tag that marks the last chunk of a stream, i.e. -// libsodium's crypto_secretstream_xchacha20poly1305_TAG_FINAL. Exported so -// callers (the download layer) can detect truncation without importing -// sodium themselves. +// libsodium's crypto_secretstream_xchacha20poly1305_TAG_FINAL: the tag that +// marks the last chunk of a stream. Exported so callers (the download layer) +// can detect truncation without importing sodium themselves. // -// Written as a literal because libsodium only attaches its constants to the -// module object once `sodium.ready` has resolved, which is long after this -// module is evaluated. The value is fixed at 3 by the secretstream wire -// format; test/crypto/stream.test.ts pins it against libsodium's own -// constant so the two cannot drift apart unnoticed. -export const STREAM_TAG_FINAL = 3; +// This is a function rather than a constant because libsodium attaches its +// constants to the module object only once `sodium.ready` has resolved, which +// is long after this module is evaluated — a module-level read would bind +// `undefined`. Reading it at call time returns the library's own value, so +// there is no second copy of a protocol constant to keep in sync. +export const streamTagFinal = (): number => + sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL; // Encrypt a small blob as a single secretstream chunk with TAG_FINAL. // Returns the header and ciphertext. Used for encrypting thumbnails @@ -59,8 +59,11 @@ export const decryptBlob = ( ): Uint8Array => { const state = initStreamPull(header, key); const { plaintext, tag } = pullStreamChunk(state, ciphertext); - if (tag !== STREAM_TAG_FINAL) { - throw new Error(`decryptBlob: expected TAG_FINAL (3), got tag ${tag}`); + const tagFinal = streamTagFinal(); + if (tag !== tagFinal) { + throw new Error( + `decryptBlob: expected TAG_FINAL (${tagFinal}), got tag ${tag}`, + ); } return plaintext; }; diff --git a/src/download/index.ts b/src/download/index.ts index 0d6dd30..6e7a6b1 100644 --- a/src/download/index.ts +++ b/src/download/index.ts @@ -7,7 +7,7 @@ import { pullStreamChunk, STREAM_CHUNK_OVERHEAD, STREAM_CHUNK_SIZE, - STREAM_TAG_FINAL, + streamTagFinal, } from "../crypto/index.js"; import type { ApiClient } from "../api/client.js"; import type { EnteFile } from "../model/types.js"; @@ -53,11 +53,27 @@ const streamDecrypt = async ( if (done) { if (buffer.length > 0) { - const { plaintext, tag } = pullStreamChunk(state, buffer); - plainChunks.push(plaintext); - totalPlain += plaintext.length; + // Whatever is left over once every whole chunk has been + // consumed must be the stream's final chunk, and a final + // chunk that actually arrived in full authenticates. If it + // does not, the body stopped part-way through a chunk — the + // ordinary shape of a dropped connection. Poly1305 cannot + // tell a partial chunk from a corrupt one, so this is + // reported as the truncation it almost always is, with the + // authentication failure kept as the error's cause. + let pulled; + try { + pulled = pullStreamChunk(state, buffer); + } catch (err) { + throw new Error( + `download: stream truncated: response body ended with ${buffer.length} trailing bytes that did not authenticate as a final chunk (transfer stopped mid-chunk, or the data is corrupt)`, + { cause: err }, + ); + } + plainChunks.push(pulled.plaintext); + totalPlain += pulled.plaintext.length; chunksPulled++; - lastTag = tag; + lastTag = pulled.tag; } break; } @@ -73,9 +89,10 @@ const streamDecrypt = async ( "download: stream truncated: response body contained no secretstream chunks", ); } - if (lastTag !== STREAM_TAG_FINAL) { + const tagFinal = streamTagFinal(); + if (lastTag !== tagFinal) { throw new Error( - `download: stream truncated: last chunk tag ${lastTag}, expected TAG_FINAL (${STREAM_TAG_FINAL})`, + `download: stream truncated: last chunk tag ${lastTag}, expected TAG_FINAL (${tagFinal})`, ); } diff --git a/test/crypto/stream.test.ts b/test/crypto/stream.test.ts index b3544d3..f4b2707 100644 --- a/test/crypto/stream.test.ts +++ b/test/crypto/stream.test.ts @@ -16,7 +16,8 @@ * * These tests pin: * - The chunk-size constants match Ente's expectations, and the - * re-exported `STREAM_TAG_FINAL` matches libsodium's own constant. + * re-exported `streamTagFinal()` matches the tag a real final chunk + * carries. * - The pull state can decrypt a multi-chunk stream produced by * sodium.crypto_secretstream_xchacha20poly1305_push, in order. * - The tag byte is propagated to the caller. @@ -31,7 +32,7 @@ import { pullStreamChunk, STREAM_CHUNK_OVERHEAD, STREAM_CHUNK_SIZE, - STREAM_TAG_FINAL, + streamTagFinal, } from "../../src/crypto/index.js"; describe("crypto stream constants", () => { @@ -49,19 +50,36 @@ describe("crypto stream constants", () => { }); /** - * `STREAM_TAG_FINAL` is re-exported so callers can detect a truncated + * `streamTagFinal()` is re-exported so callers can detect a truncated * stream (a body that ended on a non-final chunk) without importing - * libsodium themselves. It has to be declared as a literal, because - * libsodium only attaches its own constants to the module object after - * `sodium.ready` resolves — long after this library's modules are - * evaluated. This test is what keeps the literal honest. + * libsodium themselves. + * + * It is a function, not a constant, and that is load-bearing: libsodium + * attaches its own constants to the module object only after + * `sodium.ready` resolves, which is long after this library's modules are + * evaluated. Reading the value at call time yields libsodium's number; + * reading it at module scope would yield `undefined`, and every + * truncation check downstream would then compare against `undefined` and + * reject good downloads. This test pins the behaviour end to end — it + * compares against the tag observed on a real final chunk pulled back off + * the wire format, so it fails if the accessor is ever made eager. */ - it("STREAM_TAG_FINAL equals libsodium's TAG_FINAL", async () => { + it("streamTagFinal() is the tag carried by a real final chunk", async () => { await init(); await sodium.ready; - expect(STREAM_TAG_FINAL).toBe( + + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const push = + sodium.crypto_secretstream_xchacha20poly1305_init_push(key); + const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push( + push.state, + new TextEncoder().encode("last chunk"), + null, sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, ); + + const state = initStreamPull(push.header, key); + expect(pullStreamChunk(state, ciphertext).tag).toBe(streamTagFinal()); }); }); diff --git a/test/download/download.test.ts b/test/download/download.test.ts index 579991a..db50a9d 100644 --- a/test/download/download.test.ts +++ b/test/download/download.test.ts @@ -20,6 +20,8 @@ * an explicit `TAG_FINAL` check a truncated body is indistinguishable from * a complete one. `streamDecrypt` therefore refuses to return unless the * stream ended on `TAG_FINAL`, and the error says the stream was truncated. + * A transfer that stopped part-way through a chunk is reported the same + * way, since a final chunk that arrived in full always authenticates. * * 2. **The destination path is written atomically.** Plaintext goes to a * temporary sibling file first and is `rename`d into place only after the @@ -29,7 +31,8 @@ * untouched — whatever was there before is still there, byte for byte, and * no partial file has appeared. This matters because `runBackup` skips any * existing non-empty file, so a partial write would be treated as complete - * forever after. + * forever after. The staging file and the rename are observed directly (see + * the `rename` hook below), not inferred from an empty directory. * * These tests build synthetic encrypted files using sodium's push API, * serve them from a mock fetch, and verify the decrypted output on disk. @@ -43,11 +46,19 @@ import { mkdtempSync, writeFileSync, } from "node:fs"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { tmpdir } from "node:os"; import { createHash } from "node:crypto"; import sodium from "libsodium-wrappers-sumo"; -import { beforeAll, afterAll, describe, expect, it } from "vitest"; +import { + beforeAll, + beforeEach, + afterAll, + describe, + expect, + it, + vi, +} from "vitest"; import { init, toBase64, STREAM_CHUNK_SIZE } from "../../src/crypto/index.js"; import { ApiClient } from "../../src/api/client.js"; import { downloadFile, downloadThumbnail } from "../../src/download/index.js"; @@ -57,6 +68,52 @@ import type { EnteFile, FileMetadata } from "../../src/model/types.js"; // Test helpers // --------------------------------------------------------------------------- +/** + * `rename` is intercepted so that the tests can observe — and fail — the + * final step of the atomic write. + * + * Every other failure in this file is injected inside `streamDecrypt`, which + * runs before anything is written to disk. Those tests therefore cannot tell + * an atomic write from a plain `writeFile`: under both, nothing was ever + * created, so an empty directory proves nothing about cleanup. This hook is + * what closes that gap. It records each rename the downloader performs, + * including whether the source existed at that moment (i.e. that the staged + * temp file really was written), and can be told to fail the rename so the + * cleanup path runs with a temp file genuinely on disk. + * + * `vi.hoisted` is required: `vi.mock` factories are hoisted above the imports, + * so a plain module-level `const` would still be in its temporal dead zone by + * the time the factory runs. + */ +const renameHook = vi.hoisted(() => ({ + calls: [] as { from: string; to: string; sourceExisted: boolean }[], + failWith: null as Error | null, +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + const { existsSync: sourceExists } = await import("node:fs"); + return { + ...actual, + rename: async (from: string, to: string): Promise => { + renameHook.calls.push({ + from, + to, + sourceExisted: sourceExists(from), + }); + if (renameHook.failWith !== null) { + throw renameHook.failWith; + } + await actual.rename(from, to); + }, + }; +}); + +beforeEach(() => { + renameHook.calls.length = 0; + renameHook.failWith = null; +}); + let testDir: string; beforeAll(async () => { @@ -71,6 +128,35 @@ afterAll(() => { } }); +/** + * Deterministic stand-in for random test payloads. + * + * Fixture *content* is never load-bearing here — the assertions turn on + * length, framing, and the secretstream tag — but it must not be a constant + * fill either, or a downloader that reordered or repeated chunks would still + * produce the expected bytes. A seeded linear congruential generator gives + * both: byte patterns that differ across every offset and seed, reproducible + * on any machine, which is what the README asks of fixtures. + * + * It is also the difference between a fast suite and a broken one. + * `sodium.randombytes_buf` goes through the wasm wrapper a byte at a time and + * costs roughly 20 seconds for the 4 MiB chunk below — about two hundred + * times what it costs to encrypt the same buffer, and on its own enough to + * push `make test` past the 30-second cap in `script/test`. This loop fills + * 4 MiB in a few milliseconds. + */ +const patternBytes = (length: number, seed: number): Uint8Array => { + const out = new Uint8Array(length); + let x = seed >>> 0; + for (let i = 0; i < length; i++) { + // Numerical Recipes' LCG constants; the high byte is used because + // the low bits of an LCG have short periods. + x = (Math.imul(x, 1664525) + 1013904223) >>> 0; + out[i] = (x >>> 24) & 0xff; + } + return out; +}; + /** * Encrypt `plaintext` as a secretstream file body (single chunk with * TAG_FINAL). Returns the key, header, and ciphertext that the mock CDN @@ -120,7 +206,7 @@ const encryptMultiChunkBody = ( const plainParts: Uint8Array[] = []; for (let i = 0; i < leadingChunks; i++) { - const plain = sodium.randombytes_buf(STREAM_CHUNK_SIZE); + const plain = patternBytes(STREAM_CHUNK_SIZE, i + 1); plainParts.push(plain); cipherParts.push( sodium.crypto_secretstream_xchacha20poly1305_push( @@ -132,7 +218,7 @@ const encryptMultiChunkBody = ( ); } - const finalPlain = sodium.randombytes_buf(finalChunkPlainSize); + const finalPlain = patternBytes(finalChunkPlainSize, leadingChunks + 1); plainParts.push(finalPlain); const finalCipher = sodium.crypto_secretstream_xchacha20poly1305_push( push.state, @@ -226,9 +312,11 @@ const expectSameBytes = (actual: Uint8Array, expected: Uint8Array): void => { }; /** - * A multi-chunk fixture shared by the truncation tests. Building it costs a - * few MiB of encryption, so it is built once: one full 4 MiB `TAG_MESSAGE` - * chunk followed by a small `TAG_FINAL` chunk. + * A multi-chunk fixture shared by the truncation tests: one full 4 MiB + * `TAG_MESSAGE` chunk followed by a small `TAG_FINAL` chunk. It is built once + * and shared because encrypting 4 MiB costs about 100ms. Its plaintext is + * generated rather than drawn from the CSPRNG, which is what keeps that + * encryption the whole cost of the fixture. */ let multiChunk: ReturnType; let multiChunkKey: Uint8Array; @@ -295,28 +383,33 @@ describe("downloadFile", () => { }); it("uses metadata.title as filename when outPath is omitted", async () => { + // With no `outPath`, the destination is `metadata.title`, used + // verbatim as a path. The title here is therefore given inside the + // test's temporary directory: a bare relative name would resolve + // against the process working directory, i.e. the repo root, and + // `make check` must not create files in the repo — a failure between + // the write and any cleanup would leave one behind. const plaintext = new Uint8Array([1, 2, 3]); const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); const { header, ciphertext } = encryptFileBody(plaintext, key); const thumbPush = sodium.crypto_secretstream_xchacha20poly1305_init_push(key); const file = buildMockEnteFile(key, header, thumbPush.header); - file.metadata.title = "fallback-name.png"; + const titlePath = join(testDir, "fallback-name.png"); + file.metadata.title = titlePath; const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) }); const result = await downloadFile(api, file); - expect(result.path).toBe("fallback-name.png"); + expect(result.path).toBe(titlePath); expect(readFileSync(result.path)).toEqual(Buffer.from(plaintext)); - // Clean up since it writes to cwd - if (existsSync(result.path)) rmSync(result.path); }); it("handles a larger single-chunk file (random binary payload)", async () => { // Most photos are under 4 MiB and therefore a single secretstream // chunk. This test exercises a non-trivial payload size with - // random binary data (not just ASCII) to verify no encoding bugs. - const plaintext = sodium.randombytes_buf(100_000); + // arbitrary binary data (not just ASCII) to verify no encoding bugs. + const plaintext = patternBytes(100_000, 11); const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); const { header, ciphertext } = encryptFileBody(plaintext, key); const thumbPush = @@ -414,6 +507,49 @@ describe.each(entryPoints)( ); }); + it("rejects a body whose final chunk arrived only in part", async () => { + // The likelier shape of a dropped connection: the transfer stops + // in the middle of a chunk rather than neatly between two. The + // bytes that arrived are a prefix of a complete chunk, so Poly1305 + // rejects them — which is, cryptographically, indistinguishable + // from corruption of a whole chunk. + // + // It is still reported as truncation, because that is what it + // almost always is and because this library's entire reason for + // checking TAG_FINAL is to make a short transfer visible. Calling + // a short transfer "authentication failed" would send a user + // hunting for a corrupt file when their network is at fault. The + // underlying authentication failure is kept as the error's + // `cause`, so the real diagnosis is never lost. + const shortBody = multiChunk.body.slice( + 0, + multiChunk.body.length - 8, + ); + const { api, file } = fixtureFor( + multiChunkKey, + multiChunk.header, + shortBody, + ); + const dir = freshDir(); + const outPath = join(dir, "partial-final.bin"); + + const err = await download(api, file, outPath).catch( + (e: unknown) => e, + ); + + expect(err).toBeInstanceOf(Error); + expect((err as Error).message).toMatch(/truncated/i); + expect((err as Error).cause).toBeInstanceOf(Error); + expect(((err as Error).cause as Error).message).toMatch( + /authentication failed/i, + ); + + // And, as with every other failure, the destination is untouched + // and no staged temp file survives. + expect(existsSync(outPath)).toBe(false); + expect(readdirSync(dir)).toEqual([]); + }); + it("rejects an empty body instead of writing a zero-byte file", async () => { // Ente always emits at least one chunk, even for empty content: // `encryptBlob` shows that a zero-length plaintext still produces a @@ -445,7 +581,7 @@ describe.each(entryPoints)( // the rejection. const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); const { header, ciphertext } = encryptNonFinalBody( - sodium.randombytes_buf(256), + patternBytes(256, 21), key, ); const { api, file } = fixtureFor(key, header, ciphertext); @@ -463,19 +599,25 @@ describe.each(entryPoints)( expect(readdirSync(dir)).toEqual([]); }); - it("leaves no file at the destination when a chunk fails authentication", async () => { - // The same guarantee has to hold for every failure mode, not just - // truncation. Here a byte of ciphertext is flipped, so Poly1305 - // verification fails inside `pullStreamChunk`. The error must - // propagate unchanged (it is the real diagnosis) and the destination - // must still be untouched. - const plaintext = sodium.randombytes_buf(256); - const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); - const { header, ciphertext } = encryptFileBody(plaintext, key); - const corrupted = Uint8Array.from(ciphertext); + it("reports a corrupt whole chunk as an authentication failure, not truncation", async () => { + // The counterpart to the partial-final-chunk case above, and the + // reason the two are distinguishable at all. A byte is flipped + // inside the first chunk of a multi-chunk body: that chunk arrives + // complete — the stream goes on past it — so its failure cannot be + // a short transfer. It is corruption, and the caller is told so, + // with `pullStreamChunk`'s error propagated unchanged because it is + // the real diagnosis. + // + // The same on-disk guarantee holds for this failure mode as for + // every other: nothing at the destination, nothing left over. + const corrupted = Uint8Array.from(multiChunk.body); corrupted[10] ^= 0xff; - const { api, file } = fixtureFor(key, header, corrupted); + const { api, file } = fixtureFor( + multiChunkKey, + multiChunk.header, + corrupted, + ); const dir = freshDir(); const outPath = join(dir, "corrupt.bin"); @@ -498,7 +640,7 @@ describe.each(entryPoints)( ); const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); const { header, ciphertext } = encryptNonFinalBody( - sodium.randombytes_buf(256), + patternBytes(256, 22), key, ); const { api, file } = fixtureFor(key, header, ciphertext); @@ -518,7 +660,7 @@ describe.each(entryPoints)( // The mirror image of the previous test: a complete download does // overwrite whatever was at the destination, atomically, via rename. const existing = new TextEncoder().encode("stale contents"); - const plaintext = sodium.randombytes_buf(512); + const plaintext = patternBytes(512, 23); const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); const { header, ciphertext } = encryptFileBody(plaintext, key); @@ -535,5 +677,81 @@ describe.each(entryPoints)( // successful download leaves exactly one file behind. expect(readdirSync(dir)).toEqual(["replaced.bin"]); }); + + it("stages the plaintext in a sibling temp file and renames it into place", async () => { + // The atomic write, observed directly rather than inferred from an + // empty directory. Every other failure in this file is injected + // inside `streamDecrypt`, which runs before anything is written — + // so under those tests a plain `writeFile` to the destination would + // look identical. This one watches the rename itself. + // + // Two properties are load-bearing. The staging file must exist on + // disk when the rename happens: that is what makes the destination + // appear complete or not at all, instead of filling up as bytes + // land. And it must be a sibling of the destination, because + // `rename` is only atomic within one filesystem — staging in + // `/tmp` and renaming across a mount point would silently become a + // copy, reintroducing the partial-file window this exists to close. + const plaintext = patternBytes(512, 31); + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptFileBody(plaintext, key); + const { api, file } = fixtureFor(key, header, ciphertext); + const dir = freshDir(); + const outPath = join(dir, "staged.bin"); + + await download(api, file, outPath); + + expect(renameHook.calls).toHaveLength(1); + const staged = renameHook.calls[0]!; + expect(staged.to).toBe(outPath); + expect(staged.from).not.toBe(outPath); + expect(dirname(staged.from)).toBe(dir); + expect(staged.sourceExisted).toBe(true); + + // Afterwards the temp file is gone and only the destination is + // left, holding the complete plaintext. + expect(existsSync(staged.from)).toBe(false); + expect(readFileSync(outPath)).toEqual(Buffer.from(plaintext)); + expect(readdirSync(dir)).toEqual(["staged.bin"]); + }); + + it("removes the staged temp file when the rename itself fails", async () => { + // The cleanup path. It can only run when something fails at or + // after the write, which no amount of bad network data can + // produce: by the time anything is written the whole stream has + // already decrypted and verified. Failing the rename is what + // reaches it — a real possibility on a full disk, a read-only + // mount, or a destination that has become a directory. + // + // This is the only case in which the temp file is on disk at the + // moment of failure, so it is the only one that can show it is + // actually removed rather than merely never created. It also pins + // that the caller sees the original failure: a cleanup that threw + // over the top of it would hide why the download failed. + const existing = new TextEncoder().encode("known-good contents"); + const plaintext = patternBytes(512, 32); + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptFileBody(plaintext, key); + const { api, file } = fixtureFor(key, header, ciphertext); + const dir = freshDir(); + const outPath = join(dir, "rename-fails.bin"); + writeFileSync(outPath, existing); + + renameHook.failWith = new Error("simulated rename failure"); + + await expect(download(api, file, outPath)).rejects.toThrow( + "simulated rename failure", + ); + + expect(renameHook.calls).toHaveLength(1); + const staged = renameHook.calls[0]!; + expect(staged.sourceExisted).toBe(true); + expect(existsSync(staged.from)).toBe(false); + + // The previous contents are still there, untouched, and the + // directory holds nothing else. + expect(readFileSync(outPath)).toEqual(Buffer.from(existing)); + expect(readdirSync(dir)).toEqual(["rename-fails.bin"]); + }); }, );