An album whose folder cannot be made no longer stops the backup (closes #197)

Each album's folder, links and JSON are written inside one try. A failure
prints `FAILED album <name>: <reason>`, lists the album's ID in the result's
new `failedAlbums`, and so in `--json`, and makes `quak backup` exit 1. The
other albums and `failures.json` are still written.

An album whose folder would have another album's JSON name, ignoring case
(`x.json` beside `x`), gets its ID added. A folder or link name over 255
bytes gets the ID and is cut short between two characters by
`shortenToBytes` in `src/filename.ts`; a folder name leaves room for
`.json`. A link's ID stays right before its extension, which is cut short
only once nothing is left before the ID, so no two names with IDs match.

Apart from its call to `uniqueNames`, the album loop is only indented into
the try.

Model: opus-5-5

test/cli/backup-end-to-end.test.ts

@@ -274,15 +274,15 @@ afterAll(() => {
 
 // Run `quak backup <dir>` as `bin/quak.ts` runs it, sending its requests
 // through `fetch`, and return its exit code and everything it printed on
-// stderr. Its requests are retried as those of `quak backup` are, except that
-// a retry does not wait. Its cache is beside `dir`, so each run of one
-// directory finds the cache the run before it left, as cron's runs find the
-// user's.
+// stderr and on stdout. Its requests are retried as those of `quak backup`
+// are, except that a retry does not wait. Its cache is beside `dir`, so each
+// run of one directory finds the cache the run before it left, as cron's runs
+// find the user's.
 const backup = async (
     dir: string,
-    opts: { verify?: boolean } = {},
+    opts: { verify?: boolean; json?: boolean } = {},
     fetch = server,
-): Promise<{ code: number; stderr: string }> => {
+): Promise<{ code: number; stderr: string; stdout: string }> => {
     let printed = "";
     const stderr = new Writable({
         write(chunk: Buffer, _encoding, done) {
@@ -290,8 +290,10 @@ const backup = async (
             done();
         },
     });
+    let output = "";
     const stdout = new Writable({
-        write(_chunk, _encoding, done) {
+        write(chunk: Buffer, _encoding, done) {
+            output += chunk.toString();
             done();
         },
     });
@@ -316,7 +318,7 @@ const backup = async (
     const code = await new Promise<number>((resolve) => {
         void run(backupCommand(ctx, dir, opts), stdout, stderr, resolve);
     });
-    return { code, stderr: printed };
+    return { code, stderr: printed, stdout: output };
 };
 
 // The ID of the file whose original a request downloads, or undefined for any
@@ -627,6 +629,40 @@ describe("quak backup", () => {
         ).toEqual([]);
     });
 
+    // The user has put a file in collections/ with the name of the Trips
+    // folder, so quak cannot make that folder. It says so on one line, writes
+    // the two albums after it, and exits 1, and --json lists the album's ID
+    // in `failedAlbums`. No file failed, so there is no failures.json.
+    it("exits 1 when an album's folder cannot be made, writes the other albums, and lists it in --json", async () => {
+        const dir = mkdtempSync(join(root, "backup-"));
+        mkdirSync(join(dir, "collections"));
+        writeFileSync(join(dir, "collections", "Trips"), "mine");
+
+        const first = await backup(dir);
+        const { code, stdout } = await backup(dir, { json: true });
+
+        expect(first.code).toBe(1);
+        expect(first.stderr).toMatch(/^FAILED album Trips: EEXIST: /m);
+        expect(code).toBe(1);
+        expect(JSON.parse(stdout)).toMatchObject({
+            failed: 0,
+            undecryptableAlbums: [],
+            failedAlbums: [10],
+        });
+        const collections = join(dir, "collections");
+        expect(readJSON(join(collections, "Family.json")).id).toBe(20);
+        expect(readdirSync(join(collections, "Family")).sort()).toEqual([
+            "IMG_0002.heic",
+            "IMG_0002.mov",
+            "beach.jpg",
+        ]);
+        expect(readJSON(join(collections, "Friend's Wedding.json")).id).toBe(
+            30,
+        );
+        expect(existsSync(join(collections, "Trips.json"))).toBe(false);
+        expect(existsSync(join(dir, "failures.json"))).toBe(false);
+    });
+
     // The server no longer accepts the saved session's token, as after
     // `quak logout` elsewhere, and answers every request with 401. The run
     // stops at the refresh it starts with, says to log in again, and exits 3.

test/cli/backup.test.ts

@@ -1498,13 +1498,15 @@ describe("backup album folders", () => {
         files: EnteFile[];
     }
 
-    const libraryOf = (albums: Album[]): BackupLibrary => ({
+    // The original of the file `notFound`, if given, always fails to download.
+    const libraryOf = (albums: Album[], notFound?: number): BackupLibrary => ({
         refresh: async () => {},
         listCollections: () => albums.map((a) => a.collection),
         listFiles: (id) =>
             albums.find((a) => a.collection.id === id)?.files ?? [],
         undecryptable: () => [],
         original: async (fileID) => {
+            if (fileID === notFound) throw new Error("HTTP 404");
             const path = join(root, `source-${fileID}`);
             writeFileSync(path, `original ${fileID}`);
             return { path };
@@ -1600,6 +1602,197 @@ describe("backup album folders", () => {
         expect(albumID(outDir, "Trip (11) (12).json")).toBe(12);
     });
 
+    // The next three tests end with the album "ok" (14), whose one file
+    // always fails to download, so each run has a failure to record in
+    // failures.json after the album folders are written.
+    const ok: Album = {
+        collection: collection(14, "ok"),
+        files: [file(5, 14, "e.jpg")],
+    };
+
+    // The folder of an album named "x.json" would be the JSON file of the
+    // album "x", and once ended every run there. Two runs, as on two nights,
+    // write the same tree, and the failed file's attempt count goes up.
+    it("adds the ID to an album whose folder would have another album's JSON name", async () => {
+        const outDir = join(root, "backup");
+        const lib = libraryOf(
+            [
+                {
+                    collection: collection(10, "x"),
+                    files: [file(1, 10, "a.jpg")],
+                },
+                {
+                    collection: collection(11, "x.json"),
+                    files: [file(2, 11, "b.jpg")],
+                },
+                {
+                    collection: collection(12, "Trip"),
+                    files: [file(3, 12, "c.jpg")],
+                },
+                {
+                    collection: collection(13, "TRIP.json"),
+                    files: [file(4, 13, "d.jpg")],
+                },
+                ok,
+            ],
+            5,
+        );
+
+        await runBackup(lib, { downloadDirectory: outDir });
+        const result = await runBackup(lib, { downloadDirectory: outDir });
+
+        expect(result.failedAlbums).toEqual([]);
+        expect(tree(outDir)).toEqual([
+            "TRIP.json (13)/",
+            `TRIP.json (13)/d.jpg -> ${linkTo("4.jpg")}`,
+            "TRIP.json (13).json",
+            "Trip/",
+            `Trip/c.jpg -> ${linkTo("3.jpg")}`,
+            "Trip.json",
+            "ok/",
+            "ok.json",
+            "x/",
+            `x/a.jpg -> ${linkTo("1.jpg")}`,
+            "x.json",
+            "x.json (11)/",
+            `x.json (11)/b.jpg -> ${linkTo("2.jpg")}`,
+            "x.json (11).json",
+        ]);
+        expect(albumID(outDir, "x.json")).toBe(10);
+        expect(albumID(outDir, "x.json (11).json")).toBe(11);
+        expect(albumID(outDir, "TRIP.json (13).json")).toBe(13);
+        expect(albumID(outDir, "ok.json")).toBe(14);
+        expect(result.failed).toBe(1);
+        expect(readLedger(outDir).files["5"]).toMatchObject({ attempts: 2 });
+    });
+
+    // A name over 255 bytes is refused as a file name, and an album folder's
+    // name leaves room for `.json` in its JSON's. "é" takes two bytes, and a
+    // name is cut between two characters.
+    it("cuts an album name and a title too long for a file name short and adds the ID", async () => {
+        const outDir = join(root, "backup");
+        const lib = libraryOf(
+            [
+                {
+                    collection: collection(10, "L".repeat(300)),
+                    files: [file(1, 10, `${"é".repeat(150)}.jpg`)],
+                },
+                ok,
+            ],
+            5,
+        );
+
+        await runBackup(lib, { downloadDirectory: outDir });
+        const result = await runBackup(lib, { downloadDirectory: outDir });
+
+        const folder = `${"L".repeat(245)} (10)`;
+        const link = `${"é".repeat(123)} (1).jpg`;
+        expect(Buffer.byteLength(`${folder}.json`)).toBe(255);
+        expect(Buffer.byteLength(link)).toBe(254);
+        expect(result.failedAlbums).toEqual([]);
+        expect(tree(outDir)).toEqual([
+            `${folder}/`,
+            `${folder}/${link} -> ${linkTo("1.jpg")}`,
+            `${folder}.json`,
+            "ok/",
+            "ok.json",
+        ]);
+        expect(albumID(outDir, `${folder}.json`)).toBe(10);
+        expect(albumID(outDir, "ok.json")).toBe(14);
+        expect(result.failed).toBe(1);
+        expect(readLedger(outDir).files["5"]).toMatchObject({ attempts: 2 });
+    });
+
+    // The user has put a file in collections/ with the name of the album
+    // Trip's folder, so the folder cannot be made.
+    it("goes on to the next album when an album's folder cannot be made, and lists it as failed", async () => {
+        const outDir = join(root, "backup");
+        const lib = libraryOf(
+            [
+                {
+                    collection: collection(10, "Trip"),
+                    files: [file(1, 10, "a.jpg")],
+                },
+                ok,
+            ],
+            5,
+        );
+        mkdirSync(join(outDir, "collections"), { recursive: true });
+        writeFileSync(join(outDir, "collections", "Trip"), "mine");
+        const logged: string[] = [];
+
+        const result = await runBackup(lib, {
+            downloadDirectory: outDir,
+            onProgress: (message) => logged.push(message),
+        });
+
+        expect(result.failedAlbums).toEqual([10]);
+        expect(logged).toContainEqual(
+            expect.stringMatching(/^FAILED album Trip: EEXIST: /),
+        );
+        expect(tree(outDir)).toEqual(["Trip", "ok/", "ok.json"]);
+        expect(albumID(outDir, "ok.json")).toBe(14);
+        expect(existsSync(saved(outDir, "1.jpg"))).toBe(true);
+        expect(result.failed).toBe(1);
+        expect(readLedger(outDir).files["5"]).toMatchObject({ attempts: 1 });
+    });
+
+    // The extension of this title is 303 bytes, so it leaves no room for the
+    // ID: nothing is left of the name before the ID, and the extension is cut
+    // short after it. The extension holds a "-", so the original is saved
+    // with `.bin`, a name that fits.
+    it("cuts the extension of a title short when it leaves no room for the ID", async () => {
+        const outDir = join(root, "backup");
+        const lib = libraryOf([
+            {
+                collection: collection(10, "Trip"),
+                files: [file(1, 10, `a.${"L".repeat(300)}-x`)],
+            },
+        ]);
+
+        const result = await runBackup(lib, { downloadDirectory: outDir });
+
+        const link = ` (1).${"L".repeat(250)}`;
+        expect(Buffer.byteLength(link)).toBe(255);
+        expect(result.failed).toBe(0);
+        expect(tree(outDir)).toEqual([
+            "Trip/",
+            `Trip/${link} -> ${linkTo("1.bin")}`,
+            "Trip.json",
+        ]);
+    });
+
+    // Files 1 and 3 share a title, so each gets its ID before the
+    // extension. File 2's title is too long, and its extension leaves no
+    // room for the ID. Were the ID put at the end of file 2's title cut
+    // short, its link would have file 1's name, and the link made second
+    // would replace the first.
+    it("keeps names unique when a title's extension leaves no room for the ID", async () => {
+        const outDir = join(root, "backup");
+        const shared = `x.${"Q".repeat(245)} (2)`;
+        const lib = libraryOf([
+            {
+                collection: collection(10, "Trip"),
+                files: [
+                    file(1, 10, shared),
+                    file(2, 10, `x (1).${"Q".repeat(250)}-`),
+                    file(3, 10, shared),
+                ],
+            },
+        ]);
+
+        const result = await runBackup(lib, { downloadDirectory: outDir });
+
+        expect(result.failed).toBe(0);
+        expect(tree(outDir)).toEqual([
+            "Trip/",
+            `Trip/ (2).${"Q".repeat(250)} -> ${linkTo("2.bin")}`,
+            `Trip/x (1).${"Q".repeat(245)} (2) -> ${linkTo("1.bin")}`,
+            `Trip/x (3).${"Q".repeat(245)} (2) -> ${linkTo("3.bin")}`,
+            "Trip.json",
+        ]);
+    });
+
     it("changes nothing on a second run over an unchanged account", async () => {
         const outDir = join(root, "backup");
         const lib = libraryOf([

test/filename/filename.test.ts

@@ -14,7 +14,11 @@
 
 import { describe, expect, it } from "vitest";
 
-import { safeExtension, sanitizeFileName } from "../../src/filename.js";
+import {
+    safeExtension,
+    sanitizeFileName,
+    shortenToBytes,
+} from "../../src/filename.js";
 
 const FALLBACK = "file-42";
 
@@ -91,3 +95,26 @@ describe("safeExtension", () => {
         expect(safeExtension("x.jpg\0")).toBe(".bin");
     });
 });
+
+describe("shortenToBytes", () => {
+    it("returns a name that fits unchanged", () => {
+        expect(shortenToBytes("beach", 5)).toBe("beach");
+        expect(shortenToBytes("café", 5)).toBe("café");
+        expect(shortenToBytes("", 0)).toBe("");
+    });
+
+    it("cuts a name to the number of bytes given", () => {
+        expect(shortenToBytes("L".repeat(300), 255)).toBe("L".repeat(255));
+    });
+
+    // In UTF-8 "é" takes two bytes and an emoji four, which a JavaScript
+    // string holds as two halves.
+    it("cuts between two characters, never inside one", () => {
+        expect(shortenToBytes("ééé", 5)).toBe("éé");
+        expect(shortenToBytes("a\u{1F600}b", 4)).toBe("a");
+        expect(shortenToBytes("a\u{1F600}b", 5)).toBe("a\u{1F600}");
+        expect(Buffer.byteLength(shortenToBytes("é".repeat(200), 255))).toBe(
+            254,
+        );
+    });
+});