Names quak builds from server data fit in 255 bytes (closes #205)

The default names of `quak get`, `quak get-thumb`, `downloadFile` and `downloadThumbnail`, and the album folders of `backup-metadata`, could be over 255 bytes, which a file system refuses. `defaultFileName` and `shortenKeepingExtension` in `src/filename.ts` now cut such a name short, keeping its extension when it fits; a live photo's image and video get their own extension before the cut. A cut name never starts with a dot and is never a Windows device name. `--out` and `outPath` are kept as given. A `backup-metadata` folder keeps its `<id>-`. `safeExtension` gives `.bin` for an extension of more than 16 letters and digits, so a save path's name always fits.

Model: opus-5-5

test/cli/backup.test.ts

@@ -1795,12 +1795,11 @@ describe("backup album folders", () => {
         expect(readLedger(outDir).files["5"]).toMatchObject({ attempts: 1 });
     });
 
-    // The extensions of these titles leave no room for the ID, so each is cut
-    // short and the name before the ID keeps only the bytes that cut leaves
-    // over. File 1's extension is 303 bytes and is cut to 251, which leaves
-    // none. File 2's is 271 bytes and is cut to 250, since "€" takes three
-    // bytes, which leaves one, for the "a". Neither extension is only letters
-    // and digits, so both originals are saved with `.bin`, a name that fits.
+    // The extensions of these titles leave no room for the ID and the "a"
+    // before them, so each is cut short to 250 bytes, which leaves one, for
+    // the "a". File 1's extension is 303 bytes and file 2's 271. Neither
+    // extension is only letters and digits, so both originals are saved with
+    // `.bin`, a name that fits.
     it("cuts the extension of a title short when it leaves no room for the ID", async () => {
         const outDir = join(root, "backup");
         const lib = libraryOf([
@@ -1815,7 +1814,7 @@ describe("backup album folders", () => {
 
         const result = await runBackup(lib, { downloadDirectory: outDir });
 
-        const link = ` (1).${"L".repeat(250)}`;
+        const link = `a (1).${"L".repeat(249)}`;
         const link2 = `a (2).${"€".repeat(83)}`;
         expect(Buffer.byteLength(link)).toBe(255);
         expect(Buffer.byteLength(link2)).toBe(255);
@@ -1830,9 +1829,9 @@ describe("backup album folders", () => {
 
     // Files 1 and 3 share a title, so each gets its ID before the
     // extension. File 2's title is too long, and its extension leaves no
-    // room for the ID. Were the ID put at the end of file 2's title cut
-    // short, its link would have file 1's name, and the link made second
-    // would replace the first.
+    // room for the ID and the "x" before it. Were the ID put at the end of
+    // file 2's title cut short, its link would have file 1's name, and the
+    // link made second would replace the first.
     it("keeps names unique when a title's extension leaves no room for the ID", async () => {
         const outDir = join(root, "backup");
         const shared = `x.${"Q".repeat(245)} (2)`;
@@ -1852,8 +1851,8 @@ describe("backup album folders", () => {
         expect(result.failed).toBe(0);
         expect(tree(outDir)).toEqual([
             "Trip/",
-            `Trip/ (2).${"Q".repeat(250)} -> ${linkTo("2.bin")}`,
             `Trip/x (1).${"Q".repeat(245)} (2) -> ${linkTo("1.bin")}`,
+            `Trip/x (2).${"Q".repeat(249)} -> ${linkTo("2.bin")}`,
             `Trip/x (3).${"Q".repeat(245)} (2) -> ${linkTo("3.bin")}`,
             "Trip.json",
         ]);

test/cli/commands.test.ts

@@ -21,7 +21,7 @@ import {
     utimesSync,
     writeFileSync,
 } from "node:fs";
-import { join } from "node:path";
+import { basename, join } from "node:path";
 import { tmpdir } from "node:os";
 import { PassThrough } from "node:stream";
 import * as jpegJs from "jpeg-js";
@@ -187,6 +187,9 @@ const FETCHED_BOTH_ALBUMS =
     "Fetched the files of 1 of 2 collections\n" +
     "Fetched the files of 2 of 2 collections\n";
 
+// The same for an account with one album.
+const FETCHED_ONE_ALBUM = "Fetched the files of 1 of 1 collections\n";
+
 // Collects everything written to it.
 class Output {
     text = "";
@@ -562,17 +565,109 @@ describe("get and get-thumb", () => {
     });
 });
 
+// A file system refuses a file name over 255 bytes. The account here is one
+// album, whose name is 300 bytes, holding one file, 700, with the title given.
+describe("names from an album name or a title over 255 bytes", () => {
+    const longNames = (title: string): Client =>
+        ({
+            ...fakeClient(),
+            collectionsSince: async () => ({
+                collections: [collection(7, "A".repeat(300))],
+                deleted: [],
+                cursor: 1,
+            }),
+            filesSince: async () => ({
+                files: [file(700, 7, title)],
+                deleted: [],
+                cursor: 1,
+            }),
+        }) as unknown as Client;
+    const longTitle = `${"t".repeat(296)}.jpg`;
+
+    // Run `command` in a new directory, where `get` and `get-thumb` without
+    // `--out` write, and return its exit code and the directory.
+    const inNewDirectory = async (
+        command: () => Promise<number>,
+    ): Promise<{ code: number; dir: string }> => {
+        const dir = mkdtempSync(join(root, "cwd-"));
+        const previous = process.cwd();
+        process.chdir(dir);
+        try {
+            return { code: await command(), dir };
+        } finally {
+            process.chdir(previous);
+        }
+    };
+
+    it("get cuts the title short to 255 bytes, keeping its extension, and exits 0", async () => {
+        const { code, dir } = await inNewDirectory(() =>
+            getCommand(context(longNames(longTitle)), "700", {}),
+        );
+
+        const name = `${"t".repeat(251)}.jpg`;
+        expect(Buffer.byteLength(name)).toBe(255);
+        expect(code).toBe(0);
+        expect(readdirSync(dir)).toEqual([name]);
+        expect(stderr.text).toBe(FETCHED_ONE_ALBUM + `7 bytes -> ${name}\n`);
+    });
+
+    it("get-thumb cuts the title short so that thumb_ and it fit in 255 bytes, and exits 0", async () => {
+        const { code, dir } = await inNewDirectory(() =>
+            getThumbCommand(context(longNames(longTitle)), "700", {}),
+        );
+
+        const name = `thumb_${"t".repeat(245)}.jpg`;
+        expect(Buffer.byteLength(name)).toBe(255);
+        expect(code).toBe(0);
+        expect(readdirSync(dir)).toEqual([name]);
+        expect(stderr.text).toBe(FETCHED_ONE_ALBUM + `3 bytes -> ${name}\n`);
+    });
+
+    it("backup-metadata cuts the album name short so that its folder's name fits in 255 bytes, and exits 0", async () => {
+        const dir = join(root, "dump");
+
+        const code = await backupMetadataCommand(
+            context(longNames("beach.jpg")),
+            dir,
+            {},
+        );
+
+        const folder = `7-${"A".repeat(253)}`;
+        expect(Buffer.byteLength(folder)).toBe(255);
+        expect(code).toBe(0);
+        expect(readdirSync(join(dir, "collections"))).toEqual([folder]);
+        expect(readdirSync(join(dir, "collections", folder)).sort()).toEqual([
+            "700.json",
+            "_collection.json",
+        ]);
+    });
+
+    // The title's extension is 300 letters, too long for the save path's
+    // name, so the original is saved with `.bin`.
+    it("backup saves a title with a 300-letter extension as .bin, and exits 0", async () => {
+        const dir = join(root, "backup");
+        const title = `b.${"e".repeat(300)}`;
+
+        const code = await backupCommand(context(longNames(title)), dir, {});
+
+        const path = savePath(dir, file(700, 7, title));
+        expect(code).toBe(0);
+        expect(basename(path)).toMatch(/^\d{4}-\d{2}-\d{2}\.700\.bin$/);
+        expect(readFileSync(path)).toEqual(Buffer.alloc(7, 700 & 0xff));
+    });
+});
+
 // A live photo, which Ente stores as one ZIP, is written as its image and its
 // video, which a photo viewer can open. The account here is Vacation holding
 // one live photo, 300, downloaded through the real download layer
 // (test/live-photo.ts).
 describe("a live photo", () => {
-    // What `get` prints on stderr while its refresh fetches Vacation's files.
-    const FETCHED_ONE_ALBUM = "Fetched the files of 1 of 1 collections\n";
-
-    const livePhotoClient = async (image = IMAGE): Promise<Client> => {
+    const livePhotoClient = async (
+        image = IMAGE,
+        title = "IMG_0300.HEIC",
+    ): Promise<Client> => {
         const { file: live, body } = await asLivePhoto(
-            file(300, 1, "IMG_0300.HEIC"),
+            file(300, 1, title),
             livePhotoZip({ "image.heic": image, "video.mov": VIDEO }),
             livePhotoHash(image, VIDEO),
         );
@@ -620,6 +715,44 @@ describe("a live photo", () => {
         );
     });
 
+    // Run `get` without `--out` in a new directory for a live photo titled
+    // `title`, and return its exit code and the names it wrote.
+    const getInNewDirectory = async (
+        title: string,
+    ): Promise<{ code: number; names: string[] }> => {
+        const ctx = context(await livePhotoClient(IMAGE, title));
+        const dir = mkdtempSync(join(root, "cwd-"));
+        const previous = process.cwd();
+        process.chdir(dir);
+        try {
+            const code = await getCommand(ctx, "300", {});
+            return { code, names: readdirSync(dir).sort() };
+        } finally {
+            process.chdir(previous);
+        }
+    };
+    const imageName = `${"t".repeat(250)}.heic`;
+    const videoName = `${"t".repeat(251)}.mov`;
+
+    // The title fits, but ".heic" is one byte longer than ".jpg", so the
+    // image's name is cut short once its extension is in it.
+    it("get cuts its image's name short for a 255-byte title ending in .jpg, and exits 0", async () => {
+        const title = `${"t".repeat(251)}.jpg`;
+
+        const result = await getInNewDirectory(title);
+
+        expect(Buffer.byteLength(title)).toBe(255);
+        expect(Buffer.byteLength(imageName)).toBe(255);
+        expect(Buffer.byteLength(videoName)).toBe(255);
+        expect(result).toEqual({ code: 0, names: [imageName, videoName] });
+    });
+
+    it("get cuts the names of its image and video short for a 300-byte title with no extension, and exits 0", async () => {
+        const result = await getInNewDirectory("t".repeat(300));
+
+        expect(result).toEqual({ code: 0, names: [imageName, videoName] });
+    });
+
     it("get --out writes the image there and the video beside it", async () => {
         const out = join(root, "photo.jpg");
         const video = join(root, "photo.mov");

test/download/download.test.ts

@@ -658,6 +658,26 @@ describe("downloadFile", () => {
         expect(readdirSync(parent)).toEqual(["cwd"]);
     });
 
+    // A file system refuses a file name over 255 bytes.
+    it("cuts a title too long for a file name short, keeping its extension", async () => {
+        const plaintext = new Uint8Array([1, 2, 3]);
+        const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
+        const { header, ciphertext } = encryptFileBody(plaintext, key);
+        const thumbPush =
+            sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
+        const file = buildMockEnteFile(key, header, thumbPush.header);
+        file.metadata.title = `${"t".repeat(296)}.jpg`;
+        const dir = mkdtempSync(join(testDir, "long-title-"));
+
+        const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) });
+        const result = await inDirectory(dir, () => downloadFile(api, file));
+
+        const name = `${"t".repeat(251)}.jpg`;
+        expect(Buffer.byteLength(name)).toBe(255);
+        expect(result.path).toBe(name);
+        expect(readdirSync(dir)).toEqual([name]);
+    });
+
     it("uses an explicit outPath verbatim, even one with ..", async () => {
         // The caller is trusted: its path is not sanitized.
         const { api, file } = fixtureFor(
@@ -751,6 +771,28 @@ describe("downloadThumbnail", () => {
         expect(result.path).toBe("thumb__etc_passwd");
         expect(readdirSync(dir)).toEqual(["thumb__etc_passwd"]);
     });
+
+    it("leaves room for thumb_ when it cuts a title too long for a file name short", async () => {
+        const plaintext = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]);
+        const key = sodium.crypto_secretstream_xchacha20poly1305_keygen();
+        const filePush =
+            sodium.crypto_secretstream_xchacha20poly1305_init_push(key);
+        const { header: thumbHeader, ciphertext: thumbCipher } =
+            encryptFileBody(plaintext, key);
+        const file = buildMockEnteFile(key, filePush.header, thumbHeader);
+        file.metadata.title = `${"t".repeat(296)}.jpg`;
+        const dir = mkdtempSync(join(testDir, "long-thumb-title-"));
+
+        const api = new ApiClient({ fetch: mockFetchForBody(thumbCipher) });
+        const result = await inDirectory(dir, () =>
+            downloadThumbnail(api, file),
+        );
+
+        const name = `thumb_${"t".repeat(245)}.jpg`;
+        expect(Buffer.byteLength(name)).toBe(255);
+        expect(result.path).toBe(name);
+        expect(readdirSync(dir)).toEqual([name]);
+    });
 });
 
 // ---------------------------------------------------------------------------
@@ -1881,6 +1923,53 @@ describe("downloadFile live photos", () => {
         ]);
     });
 
+    // A file system refuses a file name over 255 bytes. Each name is cut
+    // short once its own extension is in it.
+    const imageName = `${"t".repeat(250)}.heic`;
+    const videoName = `${"t".repeat(251)}.mov`;
+
+    // The title fits, but ".heic" is one byte longer than ".jpg".
+    it("cuts the image's name short for a 255-byte title ending in .jpg when no outPath is given", async () => {
+        const title = `${"t".repeat(251)}.jpg`;
+        const t = setup(livePhotoZip(), { ...livePhoto, title });
+
+        const result = await inDirectory(t.dir, () =>
+            downloadFile(t.api, t.file),
+        );
+
+        expect(Buffer.byteLength(title)).toBe(255);
+        expect(Buffer.byteLength(imageName)).toBe(255);
+        expect(Buffer.byteLength(videoName)).toBe(255);
+        expect(result.path).toBe(imageName);
+        expect(result.videoPath).toBe(videoName);
+        expect(readdirSync(t.dir).sort()).toEqual([imageName, videoName]);
+    });
+
+    it("cuts the names of the image and the video short for a 300-byte title with no extension when no outPath is given", async () => {
+        const t = setup(livePhotoZip(), {
+            ...livePhoto,
+            title: "t".repeat(300),
+        });
+
+        const result = await inDirectory(t.dir, () =>
+            downloadFile(t.api, t.file),
+        );
+
+        expect(result.path).toBe(imageName);
+        expect(result.videoPath).toBe(videoName);
+        expect(readdirSync(t.dir).sort()).toEqual([imageName, videoName]);
+    });
+
+    // The caller's path is used as is; only the extension is each part's own.
+    it("keeps an outPath too long for a file name as given", async () => {
+        const t = setup(livePhotoZip(), livePhoto);
+        const outPath = join(t.dir, `${"t".repeat(251)}.jpg`);
+
+        await expect(downloadFile(t.api, t.file, outPath)).rejects.toThrow(
+            /ENAMETOOLONG/,
+        );
+    });
+
     it("gives each part its own extension from the ZIP, letters and digits only", async () => {
         const zip = livePhotoZip({ "image.JPG": IMAGE, "video.m-4v": VIDEO });
         const t = setup(zip, livePhoto);

test/filename/filename.test.ts

@@ -15,8 +15,10 @@
 import { describe, expect, it } from "vitest";
 
 import {
+    defaultFileName,
     safeExtension,
     sanitizeFileName,
+    shortenKeepingExtension,
     shortenToBytes,
 } from "../../src/filename.js";
 
@@ -94,6 +96,12 @@ describe("safeExtension", () => {
         expect(safeExtension("x.jp g")).toBe(".bin");
         expect(safeExtension("x.jpg\0")).toBe(".bin");
     });
+
+    it("keeps an extension of 16 letters and digits, and uses .bin for a longer one", () => {
+        expect(safeExtension(`x.${"e".repeat(16)}`)).toBe(`.${"e".repeat(16)}`);
+        expect(safeExtension(`x.${"e".repeat(17)}`)).toBe(".bin");
+        expect(safeExtension(`b.${"e".repeat(300)}`)).toBe(".bin");
+    });
 });
 
 describe("shortenToBytes", () => {
@@ -118,3 +126,103 @@ describe("shortenToBytes", () => {
         );
     });
 });
+
+describe("shortenKeepingExtension", () => {
+    it("returns a name that fits unchanged", () => {
+        expect(shortenKeepingExtension("beach.jpg", 9)).toBe("beach.jpg");
+        expect(shortenKeepingExtension("beach", 5)).toBe("beach");
+    });
+
+    it("cuts the part before the extension short and keeps the extension", () => {
+        expect(shortenKeepingExtension(`${"t".repeat(296)}.jpg`, 255)).toBe(
+            `${"t".repeat(251)}.jpg`,
+        );
+        expect(shortenKeepingExtension("é".repeat(150), 255)).toBe(
+            "é".repeat(127),
+        );
+    });
+
+    // A name cut down to its extension would start with a dot, and so be
+    // hidden.
+    it("keeps the first character before the extension, cutting the extension short to leave room for it", () => {
+        expect(shortenKeepingExtension(`b.${"e".repeat(300)}`, 255)).toBe(
+            `b.${"e".repeat(253)}`,
+        );
+        expect(shortenKeepingExtension(`ab.${"e".repeat(300)}`, 255)).toBe(
+            `a.${"e".repeat(253)}`,
+        );
+        expect(shortenKeepingExtension(`é.${"e".repeat(300)}`, 255)).toBe(
+            `é.${"e".repeat(252)}`,
+        );
+        // "€" takes three bytes, so the extension is cut to 253 and the
+        // part before it keeps the two bytes left over, for the "a".
+        expect(shortenKeepingExtension(`a.${"€".repeat(90)}`, 255)).toBe(
+            `a.${"€".repeat(84)}`,
+        );
+    });
+
+    it("puts `insert` right before the extension and leaves room for it", () => {
+        expect(shortenKeepingExtension("beach.jpg", 255, " (7)")).toBe(
+            "beach (7).jpg",
+        );
+        expect(
+            shortenKeepingExtension(`${"t".repeat(296)}.jpg`, 255, " (7)"),
+        ).toBe(`${"t".repeat(247)} (7).jpg`);
+        expect(
+            shortenKeepingExtension(`b.${"e".repeat(300)}`, 255, " (7)"),
+        ).toBe(`b (7).${"e".repeat(249)}`);
+    });
+});
+
+describe("defaultFileName", () => {
+    it("is the title when it is safe and fits", () => {
+        expect(defaultFileName("IMG_0001.HEIC", 42)).toBe("IMG_0001.HEIC");
+    });
+
+    it("replaces unsafe characters, and is file-<id> for an empty title", () => {
+        expect(defaultFileName("../x.jpg", 42)).toBe("__x.jpg");
+        expect(defaultFileName("", 42)).toBe("file-42");
+    });
+
+    it("cuts a title over 255 bytes short, keeping its extension", () => {
+        expect(defaultFileName(`${"t".repeat(296)}.jpg`, 42)).toBe(
+            `${"t".repeat(251)}.jpg`,
+        );
+        expect(defaultFileName(`b.${"e".repeat(300)}`, 42)).toBe(
+            `b.${"e".repeat(253)}`,
+        );
+    });
+
+    it("puts `prefix` before the title and leaves room for it", () => {
+        expect(defaultFileName("beach.jpg", 42, { prefix: "thumb_" })).toBe(
+            "thumb_beach.jpg",
+        );
+        expect(
+            defaultFileName(`${"t".repeat(296)}.jpg`, 42, { prefix: "thumb_" }),
+        ).toBe(`thumb_${"t".repeat(245)}.jpg`);
+    });
+
+    // A live photo's image and video are each named with their own
+    // extension, and the name is cut short once that extension is in it.
+    it("puts `ext` in place of the title's extension before cutting the name short", () => {
+        expect(defaultFileName("IMG_0001.HEIC", 42, { ext: ".mov" })).toBe(
+            "IMG_0001.mov",
+        );
+        expect(defaultFileName("", 42, { ext: ".mov" })).toBe("file-42.mov");
+        expect(
+            defaultFileName(`${"t".repeat(251)}.jpg`, 42, { ext: ".heic" }),
+        ).toBe(`${"t".repeat(250)}.heic`);
+        expect(defaultFileName("t".repeat(300), 42, { ext: ".heic" })).toBe(
+            `${"t".repeat(250)}.heic`,
+        );
+    });
+
+    // Cutting `CONSOLE` or `NULL` short before a long extension leaves `CON`
+    // or `NUL`, so the name gets the `_` `sanitizeFileName` gives a device
+    // name, and the part before the extension gives up a byte for it.
+    it("never returns a Windows device name, even once cut short", () => {
+        const ext = `.${"e".repeat(251)}`;
+        expect(defaultFileName(`CONSOLE${ext}`, 42)).toBe(`_CO${ext}`);
+        expect(defaultFileName(`NULL${ext}`, 42)).toBe(`_NU${ext}`);
+    });
+});