Verify secretstream TAG_FINAL and write downloads atomically (closes #1) #20
@@ -14,6 +14,6 @@ export {
|
|||||||
pullStreamChunk,
|
pullStreamChunk,
|
||||||
STREAM_CHUNK_OVERHEAD,
|
STREAM_CHUNK_OVERHEAD,
|
||||||
STREAM_CHUNK_SIZE,
|
STREAM_CHUNK_SIZE,
|
||||||
STREAM_TAG_FINAL,
|
streamTagFinal,
|
||||||
type StreamPullState,
|
type StreamPullState,
|
||||||
} from "./stream.js";
|
} from "./stream.js";
|
||||||
|
|||||||
@@ -8,17 +8,17 @@ export const STREAM_CHUNK_SIZE = 4 * 1024 * 1024;
|
|||||||
// 16 bytes of Poly1305 tag plus 1 byte of secretstream tag.
|
// 16 bytes of Poly1305 tag plus 1 byte of secretstream tag.
|
||||||
export const STREAM_CHUNK_OVERHEAD = 17;
|
export const STREAM_CHUNK_OVERHEAD = 17;
|
||||||
|
|
||||||
// The secretstream tag that marks the last chunk of a stream, i.e.
|
// libsodium's crypto_secretstream_xchacha20poly1305_TAG_FINAL: the tag that
|
||||||
// libsodium's crypto_secretstream_xchacha20poly1305_TAG_FINAL. Exported so
|
// marks the last chunk of a stream. Exported so callers (the download layer)
|
||||||
// callers (the download layer) can detect truncation without importing
|
// can detect truncation without importing sodium themselves.
|
||||||
// sodium themselves.
|
|
||||||
//
|
//
|
||||||
// Written as a literal because libsodium only attaches its constants to the
|
// This is a function rather than a constant because libsodium attaches its
|
||||||
// module object once `sodium.ready` has resolved, which is long after this
|
// constants to the module object only once `sodium.ready` has resolved, which
|
||||||
// module is evaluated. The value is fixed at 3 by the secretstream wire
|
// is long after this module is evaluated — a module-level read would bind
|
||||||
// format; test/crypto/stream.test.ts pins it against libsodium's own
|
// `undefined`. Reading it at call time returns the library's own value, so
|
||||||
// constant so the two cannot drift apart unnoticed.
|
// there is no second copy of a protocol constant to keep in sync.
|
||||||
export const STREAM_TAG_FINAL = 3;
|
export const streamTagFinal = (): number =>
|
||||||
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL;
|
||||||
|
|
||||||
// Encrypt a small blob as a single secretstream chunk with TAG_FINAL.
|
// Encrypt a small blob as a single secretstream chunk with TAG_FINAL.
|
||||||
// Returns the header and ciphertext. Used for encrypting thumbnails
|
// Returns the header and ciphertext. Used for encrypting thumbnails
|
||||||
@@ -59,8 +59,11 @@ export const decryptBlob = (
|
|||||||
): Uint8Array => {
|
): Uint8Array => {
|
||||||
const state = initStreamPull(header, key);
|
const state = initStreamPull(header, key);
|
||||||
const { plaintext, tag } = pullStreamChunk(state, ciphertext);
|
const { plaintext, tag } = pullStreamChunk(state, ciphertext);
|
||||||
if (tag !== STREAM_TAG_FINAL) {
|
const tagFinal = streamTagFinal();
|
||||||
throw new Error(`decryptBlob: expected TAG_FINAL (3), got tag ${tag}`);
|
if (tag !== tagFinal) {
|
||||||
|
throw new Error(
|
||||||
|
`decryptBlob: expected TAG_FINAL (${tagFinal}), got tag ${tag}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
return plaintext;
|
return plaintext;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import {
|
|||||||
pullStreamChunk,
|
pullStreamChunk,
|
||||||
STREAM_CHUNK_OVERHEAD,
|
STREAM_CHUNK_OVERHEAD,
|
||||||
STREAM_CHUNK_SIZE,
|
STREAM_CHUNK_SIZE,
|
||||||
STREAM_TAG_FINAL,
|
streamTagFinal,
|
||||||
} from "../crypto/index.js";
|
} from "../crypto/index.js";
|
||||||
import type { ApiClient } from "../api/client.js";
|
import type { ApiClient } from "../api/client.js";
|
||||||
import type { EnteFile } from "../model/types.js";
|
import type { EnteFile } from "../model/types.js";
|
||||||
@@ -53,11 +53,27 @@ const streamDecrypt = async (
|
|||||||
|
|
||||||
if (done) {
|
if (done) {
|
||||||
if (buffer.length > 0) {
|
if (buffer.length > 0) {
|
||||||
const { plaintext, tag } = pullStreamChunk(state, buffer);
|
// Whatever is left over once every whole chunk has been
|
||||||
plainChunks.push(plaintext);
|
// consumed must be the stream's final chunk, and a final
|
||||||
totalPlain += plaintext.length;
|
// chunk that actually arrived in full authenticates. If it
|
||||||
|
// does not, the body stopped part-way through a chunk — the
|
||||||
|
// ordinary shape of a dropped connection. Poly1305 cannot
|
||||||
|
// tell a partial chunk from a corrupt one, so this is
|
||||||
|
// reported as the truncation it almost always is, with the
|
||||||
|
// authentication failure kept as the error's cause.
|
||||||
|
let pulled;
|
||||||
|
try {
|
||||||
|
pulled = pullStreamChunk(state, buffer);
|
||||||
|
} catch (err) {
|
||||||
|
throw new Error(
|
||||||
|
`download: stream truncated: response body ended with ${buffer.length} trailing bytes that did not authenticate as a final chunk (transfer stopped mid-chunk, or the data is corrupt)`,
|
||||||
|
{ cause: err },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
plainChunks.push(pulled.plaintext);
|
||||||
|
totalPlain += pulled.plaintext.length;
|
||||||
chunksPulled++;
|
chunksPulled++;
|
||||||
lastTag = tag;
|
lastTag = pulled.tag;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -73,9 +89,10 @@ const streamDecrypt = async (
|
|||||||
"download: stream truncated: response body contained no secretstream chunks",
|
"download: stream truncated: response body contained no secretstream chunks",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (lastTag !== STREAM_TAG_FINAL) {
|
const tagFinal = streamTagFinal();
|
||||||
|
if (lastTag !== tagFinal) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`download: stream truncated: last chunk tag ${lastTag}, expected TAG_FINAL (${STREAM_TAG_FINAL})`,
|
`download: stream truncated: last chunk tag ${lastTag}, expected TAG_FINAL (${tagFinal})`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,8 @@
|
|||||||
*
|
*
|
||||||
* These tests pin:
|
* These tests pin:
|
||||||
* - The chunk-size constants match Ente's expectations, and the
|
* - The chunk-size constants match Ente's expectations, and the
|
||||||
* re-exported `STREAM_TAG_FINAL` matches libsodium's own constant.
|
* re-exported `streamTagFinal()` matches the tag a real final chunk
|
||||||
|
* carries.
|
||||||
* - The pull state can decrypt a multi-chunk stream produced by
|
* - The pull state can decrypt a multi-chunk stream produced by
|
||||||
* sodium.crypto_secretstream_xchacha20poly1305_push, in order.
|
* sodium.crypto_secretstream_xchacha20poly1305_push, in order.
|
||||||
* - The tag byte is propagated to the caller.
|
* - The tag byte is propagated to the caller.
|
||||||
@@ -31,7 +32,7 @@ import {
|
|||||||
pullStreamChunk,
|
pullStreamChunk,
|
||||||
STREAM_CHUNK_OVERHEAD,
|
STREAM_CHUNK_OVERHEAD,
|
||||||
STREAM_CHUNK_SIZE,
|
STREAM_CHUNK_SIZE,
|
||||||
STREAM_TAG_FINAL,
|
streamTagFinal,
|
||||||
} from "../../src/crypto/index.js";
|
} from "../../src/crypto/index.js";
|
||||||
|
|
||||||
describe("crypto stream constants", () => {
|
describe("crypto stream constants", () => {
|
||||||
@@ -49,19 +50,36 @@ describe("crypto stream constants", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* `STREAM_TAG_FINAL` is re-exported so callers can detect a truncated
|
* `streamTagFinal()` is re-exported so callers can detect a truncated
|
||||||
* stream (a body that ended on a non-final chunk) without importing
|
* stream (a body that ended on a non-final chunk) without importing
|
||||||
* libsodium themselves. It has to be declared as a literal, because
|
* libsodium themselves.
|
||||||
* libsodium only attaches its own constants to the module object after
|
*
|
||||||
* `sodium.ready` resolves — long after this library's modules are
|
* It is a function, not a constant, and that is load-bearing: libsodium
|
||||||
* evaluated. This test is what keeps the literal honest.
|
* attaches its own constants to the module object only after
|
||||||
|
* `sodium.ready` resolves, which is long after this library's modules are
|
||||||
|
* evaluated. Reading the value at call time yields libsodium's number;
|
||||||
|
* reading it at module scope would yield `undefined`, and every
|
||||||
|
* truncation check downstream would then compare against `undefined` and
|
||||||
|
* reject good downloads. This test pins the behaviour end to end — it
|
||||||
|
* compares against the tag observed on a real final chunk pulled back off
|
||||||
|
* the wire format, so it fails if the accessor is ever made eager.
|
||||||
*/
|
*/
|
||||||
it("STREAM_TAG_FINAL equals libsodium's TAG_FINAL", async () => {
|
it("streamTagFinal() is the tag carried by a real final chunk", async () => {
|
||||||
await init();
|
await init();
|
||||||
await sodium.ready;
|
await sodium.ready;
|
||||||
expect(STREAM_TAG_FINAL).toBe(
|
|
||||||
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
|
const push =
|
||||||
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
||||||
|
const ciphertext = sodium.crypto_secretstream_xchacha20poly1305_push(
|
||||||
|
push.state,
|
||||||
|
new TextEncoder().encode("last chunk"),
|
||||||
|
null,
|
||||||
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const state = initStreamPull(push.header, key);
|
||||||
|
expect(pullStreamChunk(state, ciphertext).tag).toBe(streamTagFinal());
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,8 @@
|
|||||||
* an explicit `TAG_FINAL` check a truncated body is indistinguishable from
|
* an explicit `TAG_FINAL` check a truncated body is indistinguishable from
|
||||||
* a complete one. `streamDecrypt` therefore refuses to return unless the
|
* a complete one. `streamDecrypt` therefore refuses to return unless the
|
||||||
* stream ended on `TAG_FINAL`, and the error says the stream was truncated.
|
* stream ended on `TAG_FINAL`, and the error says the stream was truncated.
|
||||||
|
* A transfer that stopped part-way through a chunk is reported the same
|
||||||
|
* way, since a final chunk that arrived in full always authenticates.
|
||||||
*
|
*
|
||||||
* 2. **The destination path is written atomically.** Plaintext goes to a
|
* 2. **The destination path is written atomically.** Plaintext goes to a
|
||||||
* temporary sibling file first and is `rename`d into place only after the
|
* temporary sibling file first and is `rename`d into place only after the
|
||||||
@@ -29,7 +31,8 @@
|
|||||||
* untouched — whatever was there before is still there, byte for byte, and
|
* untouched — whatever was there before is still there, byte for byte, and
|
||||||
* no partial file has appeared. This matters because `runBackup` skips any
|
* no partial file has appeared. This matters because `runBackup` skips any
|
||||||
* existing non-empty file, so a partial write would be treated as complete
|
* existing non-empty file, so a partial write would be treated as complete
|
||||||
* forever after.
|
* forever after. The staging file and the rename are observed directly (see
|
||||||
|
* the `rename` hook below), not inferred from an empty directory.
|
||||||
*
|
*
|
||||||
* These tests build synthetic encrypted files using sodium's push API,
|
* These tests build synthetic encrypted files using sodium's push API,
|
||||||
* serve them from a mock fetch, and verify the decrypted output on disk.
|
* serve them from a mock fetch, and verify the decrypted output on disk.
|
||||||
@@ -43,11 +46,19 @@ import {
|
|||||||
mkdtempSync,
|
mkdtempSync,
|
||||||
writeFileSync,
|
writeFileSync,
|
||||||
} from "node:fs";
|
} from "node:fs";
|
||||||
import { join } from "node:path";
|
import { dirname, join } from "node:path";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { createHash } from "node:crypto";
|
import { createHash } from "node:crypto";
|
||||||
import sodium from "libsodium-wrappers-sumo";
|
import sodium from "libsodium-wrappers-sumo";
|
||||||
import { beforeAll, afterAll, describe, expect, it } from "vitest";
|
import {
|
||||||
|
beforeAll,
|
||||||
|
beforeEach,
|
||||||
|
afterAll,
|
||||||
|
describe,
|
||||||
|
expect,
|
||||||
|
it,
|
||||||
|
vi,
|
||||||
|
} from "vitest";
|
||||||
import { init, toBase64, STREAM_CHUNK_SIZE } from "../../src/crypto/index.js";
|
import { init, toBase64, STREAM_CHUNK_SIZE } from "../../src/crypto/index.js";
|
||||||
import { ApiClient } from "../../src/api/client.js";
|
import { ApiClient } from "../../src/api/client.js";
|
||||||
import { downloadFile, downloadThumbnail } from "../../src/download/index.js";
|
import { downloadFile, downloadThumbnail } from "../../src/download/index.js";
|
||||||
@@ -57,6 +68,52 @@ import type { EnteFile, FileMetadata } from "../../src/model/types.js";
|
|||||||
// Test helpers
|
// Test helpers
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* `rename` is intercepted so that the tests can observe — and fail — the
|
||||||
|
* final step of the atomic write.
|
||||||
|
*
|
||||||
|
* Every other failure in this file is injected inside `streamDecrypt`, which
|
||||||
|
* runs before anything is written to disk. Those tests therefore cannot tell
|
||||||
|
* an atomic write from a plain `writeFile`: under both, nothing was ever
|
||||||
|
* created, so an empty directory proves nothing about cleanup. This hook is
|
||||||
|
* what closes that gap. It records each rename the downloader performs,
|
||||||
|
* including whether the source existed at that moment (i.e. that the staged
|
||||||
|
* temp file really was written), and can be told to fail the rename so the
|
||||||
|
* cleanup path runs with a temp file genuinely on disk.
|
||||||
|
*
|
||||||
|
* `vi.hoisted` is required: `vi.mock` factories are hoisted above the imports,
|
||||||
|
* so a plain module-level `const` would still be in its temporal dead zone by
|
||||||
|
* the time the factory runs.
|
||||||
|
*/
|
||||||
|
const renameHook = vi.hoisted(() => ({
|
||||||
|
calls: [] as { from: string; to: string; sourceExisted: boolean }[],
|
||||||
|
failWith: null as Error | null,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("node:fs/promises", async (importOriginal) => {
|
||||||
|
const actual = await importOriginal<typeof import("node:fs/promises")>();
|
||||||
|
const { existsSync: sourceExists } = await import("node:fs");
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
rename: async (from: string, to: string): Promise<void> => {
|
||||||
|
renameHook.calls.push({
|
||||||
|
from,
|
||||||
|
to,
|
||||||
|
sourceExisted: sourceExists(from),
|
||||||
|
});
|
||||||
|
if (renameHook.failWith !== null) {
|
||||||
|
throw renameHook.failWith;
|
||||||
|
}
|
||||||
|
await actual.rename(from, to);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
renameHook.calls.length = 0;
|
||||||
|
renameHook.failWith = null;
|
||||||
|
});
|
||||||
|
|
||||||
let testDir: string;
|
let testDir: string;
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
@@ -71,6 +128,35 @@ afterAll(() => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deterministic stand-in for random test payloads.
|
||||||
|
*
|
||||||
|
* Fixture *content* is never load-bearing here — the assertions turn on
|
||||||
|
* length, framing, and the secretstream tag — but it must not be a constant
|
||||||
|
* fill either, or a downloader that reordered or repeated chunks would still
|
||||||
|
* produce the expected bytes. A seeded linear congruential generator gives
|
||||||
|
* both: byte patterns that differ across every offset and seed, reproducible
|
||||||
|
* on any machine, which is what the README asks of fixtures.
|
||||||
|
*
|
||||||
|
* It is also the difference between a fast suite and a broken one.
|
||||||
|
* `sodium.randombytes_buf` goes through the wasm wrapper a byte at a time and
|
||||||
|
* costs roughly 20 seconds for the 4 MiB chunk below — about two hundred
|
||||||
|
* times what it costs to encrypt the same buffer, and on its own enough to
|
||||||
|
* push `make test` past the 30-second cap in `script/test`. This loop fills
|
||||||
|
* 4 MiB in a few milliseconds.
|
||||||
|
*/
|
||||||
|
const patternBytes = (length: number, seed: number): Uint8Array => {
|
||||||
|
const out = new Uint8Array(length);
|
||||||
|
let x = seed >>> 0;
|
||||||
|
for (let i = 0; i < length; i++) {
|
||||||
|
// Numerical Recipes' LCG constants; the high byte is used because
|
||||||
|
// the low bits of an LCG have short periods.
|
||||||
|
x = (Math.imul(x, 1664525) + 1013904223) >>> 0;
|
||||||
|
out[i] = (x >>> 24) & 0xff;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Encrypt `plaintext` as a secretstream file body (single chunk with
|
* Encrypt `plaintext` as a secretstream file body (single chunk with
|
||||||
* TAG_FINAL). Returns the key, header, and ciphertext that the mock CDN
|
* TAG_FINAL). Returns the key, header, and ciphertext that the mock CDN
|
||||||
@@ -120,7 +206,7 @@ const encryptMultiChunkBody = (
|
|||||||
const plainParts: Uint8Array[] = [];
|
const plainParts: Uint8Array[] = [];
|
||||||
|
|
||||||
for (let i = 0; i < leadingChunks; i++) {
|
for (let i = 0; i < leadingChunks; i++) {
|
||||||
const plain = sodium.randombytes_buf(STREAM_CHUNK_SIZE);
|
const plain = patternBytes(STREAM_CHUNK_SIZE, i + 1);
|
||||||
plainParts.push(plain);
|
plainParts.push(plain);
|
||||||
cipherParts.push(
|
cipherParts.push(
|
||||||
sodium.crypto_secretstream_xchacha20poly1305_push(
|
sodium.crypto_secretstream_xchacha20poly1305_push(
|
||||||
@@ -132,7 +218,7 @@ const encryptMultiChunkBody = (
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const finalPlain = sodium.randombytes_buf(finalChunkPlainSize);
|
const finalPlain = patternBytes(finalChunkPlainSize, leadingChunks + 1);
|
||||||
plainParts.push(finalPlain);
|
plainParts.push(finalPlain);
|
||||||
const finalCipher = sodium.crypto_secretstream_xchacha20poly1305_push(
|
const finalCipher = sodium.crypto_secretstream_xchacha20poly1305_push(
|
||||||
push.state,
|
push.state,
|
||||||
@@ -226,9 +312,11 @@ const expectSameBytes = (actual: Uint8Array, expected: Uint8Array): void => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A multi-chunk fixture shared by the truncation tests. Building it costs a
|
* A multi-chunk fixture shared by the truncation tests: one full 4 MiB
|
||||||
* few MiB of encryption, so it is built once: one full 4 MiB `TAG_MESSAGE`
|
* `TAG_MESSAGE` chunk followed by a small `TAG_FINAL` chunk. It is built once
|
||||||
* chunk followed by a small `TAG_FINAL` chunk.
|
* and shared because encrypting 4 MiB costs about 100ms. Its plaintext is
|
||||||
|
* generated rather than drawn from the CSPRNG, which is what keeps that
|
||||||
|
* encryption the whole cost of the fixture.
|
||||||
*/
|
*/
|
||||||
let multiChunk: ReturnType<typeof encryptMultiChunkBody>;
|
let multiChunk: ReturnType<typeof encryptMultiChunkBody>;
|
||||||
let multiChunkKey: Uint8Array;
|
let multiChunkKey: Uint8Array;
|
||||||
@@ -295,28 +383,33 @@ describe("downloadFile", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("uses metadata.title as filename when outPath is omitted", async () => {
|
it("uses metadata.title as filename when outPath is omitted", async () => {
|
||||||
|
// With no `outPath`, the destination is `metadata.title`, used
|
||||||
|
// verbatim as a path. The title here is therefore given inside the
|
||||||
|
// test's temporary directory: a bare relative name would resolve
|
||||||
|
// against the process working directory, i.e. the repo root, and
|
||||||
|
// `make check` must not create files in the repo — a failure between
|
||||||
|
// the write and any cleanup would leave one behind.
|
||||||
const plaintext = new Uint8Array([1, 2, 3]);
|
const plaintext = new Uint8Array([1, 2, 3]);
|
||||||
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
||||||
const thumbPush =
|
const thumbPush =
|
||||||
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
|
||||||
const file = buildMockEnteFile(key, header, thumbPush.header);
|
const file = buildMockEnteFile(key, header, thumbPush.header);
|
||||||
file.metadata.title = "fallback-name.png";
|
const titlePath = join(testDir, "fallback-name.png");
|
||||||
|
file.metadata.title = titlePath;
|
||||||
|
|
||||||
const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) });
|
const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) });
|
||||||
const result = await downloadFile(api, file);
|
const result = await downloadFile(api, file);
|
||||||
|
|
||||||
expect(result.path).toBe("fallback-name.png");
|
expect(result.path).toBe(titlePath);
|
||||||
expect(readFileSync(result.path)).toEqual(Buffer.from(plaintext));
|
expect(readFileSync(result.path)).toEqual(Buffer.from(plaintext));
|
||||||
// Clean up since it writes to cwd
|
|
||||||
if (existsSync(result.path)) rmSync(result.path);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("handles a larger single-chunk file (random binary payload)", async () => {
|
it("handles a larger single-chunk file (random binary payload)", async () => {
|
||||||
// Most photos are under 4 MiB and therefore a single secretstream
|
// Most photos are under 4 MiB and therefore a single secretstream
|
||||||
// chunk. This test exercises a non-trivial payload size with
|
// chunk. This test exercises a non-trivial payload size with
|
||||||
// random binary data (not just ASCII) to verify no encoding bugs.
|
// arbitrary binary data (not just ASCII) to verify no encoding bugs.
|
||||||
const plaintext = sodium.randombytes_buf(100_000);
|
const plaintext = patternBytes(100_000, 11);
|
||||||
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
||||||
const thumbPush =
|
const thumbPush =
|
||||||
@@ -414,6 +507,49 @@ describe.each(entryPoints)(
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("rejects a body whose final chunk arrived only in part", async () => {
|
||||||
|
// The likelier shape of a dropped connection: the transfer stops
|
||||||
|
// in the middle of a chunk rather than neatly between two. The
|
||||||
|
// bytes that arrived are a prefix of a complete chunk, so Poly1305
|
||||||
|
// rejects them — which is, cryptographically, indistinguishable
|
||||||
|
// from corruption of a whole chunk.
|
||||||
|
//
|
||||||
|
// It is still reported as truncation, because that is what it
|
||||||
|
// almost always is and because this library's entire reason for
|
||||||
|
// checking TAG_FINAL is to make a short transfer visible. Calling
|
||||||
|
// a short transfer "authentication failed" would send a user
|
||||||
|
// hunting for a corrupt file when their network is at fault. The
|
||||||
|
// underlying authentication failure is kept as the error's
|
||||||
|
// `cause`, so the real diagnosis is never lost.
|
||||||
|
const shortBody = multiChunk.body.slice(
|
||||||
|
0,
|
||||||
|
multiChunk.body.length - 8,
|
||||||
|
);
|
||||||
|
const { api, file } = fixtureFor(
|
||||||
|
multiChunkKey,
|
||||||
|
multiChunk.header,
|
||||||
|
shortBody,
|
||||||
|
);
|
||||||
|
const dir = freshDir();
|
||||||
|
const outPath = join(dir, "partial-final.bin");
|
||||||
|
|
||||||
|
const err = await download(api, file, outPath).catch(
|
||||||
|
(e: unknown) => e,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(err).toBeInstanceOf(Error);
|
||||||
|
expect((err as Error).message).toMatch(/truncated/i);
|
||||||
|
expect((err as Error).cause).toBeInstanceOf(Error);
|
||||||
|
expect(((err as Error).cause as Error).message).toMatch(
|
||||||
|
/authentication failed/i,
|
||||||
|
);
|
||||||
|
|
||||||
|
// And, as with every other failure, the destination is untouched
|
||||||
|
// and no staged temp file survives.
|
||||||
|
expect(existsSync(outPath)).toBe(false);
|
||||||
|
expect(readdirSync(dir)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
it("rejects an empty body instead of writing a zero-byte file", async () => {
|
it("rejects an empty body instead of writing a zero-byte file", async () => {
|
||||||
// Ente always emits at least one chunk, even for empty content:
|
// Ente always emits at least one chunk, even for empty content:
|
||||||
// `encryptBlob` shows that a zero-length plaintext still produces a
|
// `encryptBlob` shows that a zero-length plaintext still produces a
|
||||||
@@ -445,7 +581,7 @@ describe.each(entryPoints)(
|
|||||||
// the rejection.
|
// the rejection.
|
||||||
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
const { header, ciphertext } = encryptNonFinalBody(
|
const { header, ciphertext } = encryptNonFinalBody(
|
||||||
sodium.randombytes_buf(256),
|
patternBytes(256, 21),
|
||||||
key,
|
key,
|
||||||
);
|
);
|
||||||
const { api, file } = fixtureFor(key, header, ciphertext);
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
||||||
@@ -463,19 +599,25 @@ describe.each(entryPoints)(
|
|||||||
expect(readdirSync(dir)).toEqual([]);
|
expect(readdirSync(dir)).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("leaves no file at the destination when a chunk fails authentication", async () => {
|
it("reports a corrupt whole chunk as an authentication failure, not truncation", async () => {
|
||||||
// The same guarantee has to hold for every failure mode, not just
|
// The counterpart to the partial-final-chunk case above, and the
|
||||||
// truncation. Here a byte of ciphertext is flipped, so Poly1305
|
// reason the two are distinguishable at all. A byte is flipped
|
||||||
// verification fails inside `pullStreamChunk`. The error must
|
// inside the first chunk of a multi-chunk body: that chunk arrives
|
||||||
// propagate unchanged (it is the real diagnosis) and the destination
|
// complete — the stream goes on past it — so its failure cannot be
|
||||||
// must still be untouched.
|
// a short transfer. It is corruption, and the caller is told so,
|
||||||
const plaintext = sodium.randombytes_buf(256);
|
// with `pullStreamChunk`'s error propagated unchanged because it is
|
||||||
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
// the real diagnosis.
|
||||||
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
//
|
||||||
const corrupted = Uint8Array.from(ciphertext);
|
// The same on-disk guarantee holds for this failure mode as for
|
||||||
|
// every other: nothing at the destination, nothing left over.
|
||||||
|
const corrupted = Uint8Array.from(multiChunk.body);
|
||||||
corrupted[10] ^= 0xff;
|
corrupted[10] ^= 0xff;
|
||||||
|
|
||||||
const { api, file } = fixtureFor(key, header, corrupted);
|
const { api, file } = fixtureFor(
|
||||||
|
multiChunkKey,
|
||||||
|
multiChunk.header,
|
||||||
|
corrupted,
|
||||||
|
);
|
||||||
const dir = freshDir();
|
const dir = freshDir();
|
||||||
const outPath = join(dir, "corrupt.bin");
|
const outPath = join(dir, "corrupt.bin");
|
||||||
|
|
||||||
@@ -498,7 +640,7 @@ describe.each(entryPoints)(
|
|||||||
);
|
);
|
||||||
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
const { header, ciphertext } = encryptNonFinalBody(
|
const { header, ciphertext } = encryptNonFinalBody(
|
||||||
sodium.randombytes_buf(256),
|
patternBytes(256, 22),
|
||||||
key,
|
key,
|
||||||
);
|
);
|
||||||
const { api, file } = fixtureFor(key, header, ciphertext);
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
||||||
@@ -518,7 +660,7 @@ describe.each(entryPoints)(
|
|||||||
// The mirror image of the previous test: a complete download does
|
// The mirror image of the previous test: a complete download does
|
||||||
// overwrite whatever was at the destination, atomically, via rename.
|
// overwrite whatever was at the destination, atomically, via rename.
|
||||||
const existing = new TextEncoder().encode("stale contents");
|
const existing = new TextEncoder().encode("stale contents");
|
||||||
const plaintext = sodium.randombytes_buf(512);
|
const plaintext = patternBytes(512, 23);
|
||||||
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
||||||
|
|
||||||
@@ -535,5 +677,81 @@ describe.each(entryPoints)(
|
|||||||
// successful download leaves exactly one file behind.
|
// successful download leaves exactly one file behind.
|
||||||
expect(readdirSync(dir)).toEqual(["replaced.bin"]);
|
expect(readdirSync(dir)).toEqual(["replaced.bin"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("stages the plaintext in a sibling temp file and renames it into place", async () => {
|
||||||
|
// The atomic write, observed directly rather than inferred from an
|
||||||
|
// empty directory. Every other failure in this file is injected
|
||||||
|
// inside `streamDecrypt`, which runs before anything is written —
|
||||||
|
// so under those tests a plain `writeFile` to the destination would
|
||||||
|
// look identical. This one watches the rename itself.
|
||||||
|
//
|
||||||
|
// Two properties are load-bearing. The staging file must exist on
|
||||||
|
// disk when the rename happens: that is what makes the destination
|
||||||
|
// appear complete or not at all, instead of filling up as bytes
|
||||||
|
// land. And it must be a sibling of the destination, because
|
||||||
|
// `rename` is only atomic within one filesystem — staging in
|
||||||
|
// `/tmp` and renaming across a mount point would silently become a
|
||||||
|
// copy, reintroducing the partial-file window this exists to close.
|
||||||
|
const plaintext = patternBytes(512, 31);
|
||||||
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
||||||
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
||||||
|
const dir = freshDir();
|
||||||
|
const outPath = join(dir, "staged.bin");
|
||||||
|
|
||||||
|
await download(api, file, outPath);
|
||||||
|
|
||||||
|
expect(renameHook.calls).toHaveLength(1);
|
||||||
|
const staged = renameHook.calls[0]!;
|
||||||
|
expect(staged.to).toBe(outPath);
|
||||||
|
expect(staged.from).not.toBe(outPath);
|
||||||
|
expect(dirname(staged.from)).toBe(dir);
|
||||||
|
expect(staged.sourceExisted).toBe(true);
|
||||||
|
|
||||||
|
// Afterwards the temp file is gone and only the destination is
|
||||||
|
// left, holding the complete plaintext.
|
||||||
|
expect(existsSync(staged.from)).toBe(false);
|
||||||
|
expect(readFileSync(outPath)).toEqual(Buffer.from(plaintext));
|
||||||
|
expect(readdirSync(dir)).toEqual(["staged.bin"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("removes the staged temp file when the rename itself fails", async () => {
|
||||||
|
// The cleanup path. It can only run when something fails at or
|
||||||
|
// after the write, which no amount of bad network data can
|
||||||
|
// produce: by the time anything is written the whole stream has
|
||||||
|
// already decrypted and verified. Failing the rename is what
|
||||||
|
// reaches it — a real possibility on a full disk, a read-only
|
||||||
|
// mount, or a destination that has become a directory.
|
||||||
|
//
|
||||||
|
// This is the only case in which the temp file is on disk at the
|
||||||
|
// moment of failure, so it is the only one that can show it is
|
||||||
|
// actually removed rather than merely never created. It also pins
|
||||||
|
// that the caller sees the original failure: a cleanup that threw
|
||||||
|
// over the top of it would hide why the download failed.
|
||||||
|
const existing = new TextEncoder().encode("known-good contents");
|
||||||
|
const plaintext = patternBytes(512, 32);
|
||||||
|
const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
|
||||||
|
const { header, ciphertext } = encryptFileBody(plaintext, key);
|
||||||
|
const { api, file } = fixtureFor(key, header, ciphertext);
|
||||||
|
const dir = freshDir();
|
||||||
|
const outPath = join(dir, "rename-fails.bin");
|
||||||
|
writeFileSync(outPath, existing);
|
||||||
|
|
||||||
|
renameHook.failWith = new Error("simulated rename failure");
|
||||||
|
|
||||||
|
await expect(download(api, file, outPath)).rejects.toThrow(
|
||||||
|
"simulated rename failure",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(renameHook.calls).toHaveLength(1);
|
||||||
|
const staged = renameHook.calls[0]!;
|
||||||
|
expect(staged.sourceExisted).toBe(true);
|
||||||
|
expect(existsSync(staged.from)).toBe(false);
|
||||||
|
|
||||||
|
// The previous contents are still there, untouched, and the
|
||||||
|
// directory holds nothing else.
|
||||||
|
expect(readFileSync(outPath)).toEqual(Buffer.from(existing));
|
||||||
|
expect(readdirSync(dir)).toEqual(["rename-fails.bin"]);
|
||||||
|
});
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|||||||
Reference in New Issue
Block a user