An album whose folder cannot be made no longer stops the backup (closes #197)

Each album's folder, links and JSON are written inside one try. A failure
prints `FAILED album <name>: <reason>`, lists the album's ID in the new
`failedAlbums` of the result and of `--json`, and makes `quak backup` exit 1;
the other albums and `failures.json` are still written.

An album whose folder would have another album's JSON name, ignoring case,
gets its ID. A name over 255 bytes (250 for a folder, which leaves room for
`.json`) gets the ID and is cut short between two characters by
`shortenToBytes` in `src/filename.ts`. A link's ID goes right before its
extension.

Apart from its call to `uniqueNames`, the album loop is only indented into
the try.

Model: opus-5-5

src/backup.ts

@@ -50,7 +50,9 @@
 // running attempt count, and the last-tried time, and the run continues.
 // A file the library could not decrypt is recorded there too, unless a copy of
 // it in another album decrypted; an album it could not decrypt is logged by ID
-// and listed in `result.undecryptableAlbums`, as its name is not known.
+// and listed in `result.undecryptableAlbums`, as its name is not known. An
+// album whose folder or JSON cannot be written is logged by name and
+// listed in `result.failedAlbums`, and the run goes on to the next album.
 // `result.failed` — and thus the CLI's exit code — stays non-zero while any
 // failure remains unresolved and clears once every one succeeds. Each run
 // reconciles the ledger against the files it attempted, so an entry for a file
@@ -83,7 +85,12 @@ import {
 } from "./crypto/index.js";
 import { removeLeftoverTempFiles } from "./download/index.js";
 import { describeError } from "./errors.js";
-import { sanitizeFileName, withExtension } from "./filename.js";
+import {
+    MAX_NAME_BYTES,
+    sanitizeFileName,
+    shortenToBytes,
+    withExtension,
+} from "./filename.js";
 import {
     copyAtomic,
     placeOriginal,
@@ -160,6 +167,9 @@ export interface BackupResult {
     // The IDs of the albums the library could not decrypt, whose files it
     // could not list; the CLI exits non-zero while there are any.
     undecryptableAlbums: number[];
+    // The IDs of the albums whose folder or JSON this run could not write;
+    // the CLI exits non-zero while there are any.
+    failedAlbums: number[];
 }
 
 // The slice of the library that backup drives. `Library` implements it; a test
@@ -267,31 +277,47 @@ const rebuildSymlink = (linkPath: string, target: string): void => {
 // The on-disk names for the entries of one directory, in entry order. Each
 // name is used as is unless another entry would get the same name, ignoring
 // case (two names that differ only in case are one entry on a case-insensitive
-// file system); then every entry sharing it gets ` (<id>)`, before the
-// extension when `beforeExtension` is set. A name with an ID added can match
-// another entry's own name (`IMG (6).JPG`), so this repeats until no name is
-// shared. IDs are stable, so the names are too.
+// file system), or it is too long for a file name; then it gets ` (<id>)`,
+// right before the extension when `beforeExtension` is set. To fit, the part
+// before the ID is cut short as far as it must be. When the extension alone
+// leaves no room for the ID, the extension is cut short and the part before
+// the ID keeps only the bytes that cut leaves over. With `withJSON`, each
+// entry also has the JSON file `<name>.json` beside it, as an album does: a
+// name then leaves room for `.json`, and one that is another entry's JSON name
+// gets its ID added too.
+// Two names with different IDs added never match, but one can match another
+// entry's own name (`IMG (6).JPG`), so this repeats until no name is shared.
+// IDs are stable, so the names are too.
 const uniqueNames = (
     entries: { id: number; name: string }[],
-    beforeExtension: boolean,
+    opts: { beforeExtension?: boolean; withJSON?: boolean },
 ): string[] => {
+    const maxBytes = MAX_NAME_BYTES - (opts.withJSON ? ".json".length : 0);
     const withID = (id: number, name: string): string => {
-        const ext = beforeExtension ? extname(name) : "";
-        const stem = name.slice(0, name.length - ext.length);
-        return `${stem} (${id})${ext}`;
+        const tag = ` (${id})`;
+        const fullExt = opts.beforeExtension ? extname(name) : "";
+        const stem = name.slice(0, name.length - fullExt.length);
+        const ext = shortenToBytes(fullExt, maxBytes - Buffer.byteLength(tag));
+        const room = maxBytes - Buffer.byteLength(tag + ext);
+        return shortenToBytes(stem, room) + tag + ext;
     };
     const names = entries.map((e) => e.name);
     const suffixed = new Set<number>();
     for (;;) {
         const counts = new Map<string, number>();
-        for (const name of names) {
+        const count = (name: string): void => {
             const key = name.toLowerCase();
             counts.set(key, (counts.get(key) ?? 0) + 1);
+        };
+        for (const name of names) {
+            count(name);
+            if (opts.withJSON) count(`${name}.json`);
         }
         let changed = false;
         for (const [i, { id, name }] of entries.entries()) {
             if (suffixed.has(i)) continue;
-            if (counts.get(name.toLowerCase()) === 1) continue;
+            const fits = Buffer.byteLength(name) <= maxBytes;
+            if (fits && counts.get(name.toLowerCase()) === 1) continue;
             names[i] = withID(id, name);
             suffixed.add(i);
             changed = true;
@@ -841,7 +867,7 @@ const runLockedBackup = async (
             id: c.id,
             name: sanitizeFileName(c.name, `collection-${c.id}`),
         })),
-        false,
+        { withJSON: true },
     );
     const albumDirNames = new Map(
         allCollections.map((c, i) => [c.id, dirNames[i]!]),
@@ -857,60 +883,69 @@ const runLockedBackup = async (
         log(`FAILED removing old album directories: ${describeError(err)}`);
     }
 
+    // An album whose folder or JSON cannot be written, such as one whose
+    // folder name is taken by a file the user put there, is logged and listed
+    // in the result, and the run goes on to the next album.
+    const failedAlbums: number[] = [];
     for (const c of collections) {
         if (lock.lost) throw lock.lost;
-        const colDirName = albumDirNames.get(c.id)!;
-        const colDir = join(collectionsDir, colDirName);
-        mkdirSync(colDir, { recursive: true });
-
-        // Every album links the one original, saved from the file's entry in
-        // `distinct`.
-        const files = filesByCollection.get(c.id) ?? [];
-        const links = files.flatMap((f) =>
-            linksFor(
-                f,
-                storedAtSavePath(downloadDirectory, distinct.get(f.id)!),
-            ),
-        );
-        const linkNames = uniqueNames(links, true);
         try {
-            removeStaleLinks(colDir, new Set(linkNames), downloadDirectory);
-        } catch (err) {
-            log(
-                `FAILED removing old links in ${c.name}: ${describeError(err)}`,
-            );
-        }
+            const colDirName = albumDirNames.get(c.id)!;
+            const colDir = join(collectionsDir, colDirName);
+            mkdirSync(colDir, { recursive: true });
 
-        const metaFiles = files.map((f) => ({
-            id: f.id,
-            metadata: f.metadata,
-        }));
-        for (const [i, link] of links.entries()) {
-            if (!includeOriginals || link.target === undefined) continue;
-            const linkName = linkNames[i]!;
+            // Every album links the one original, saved from the file's entry
+            // in `distinct`.
+            const files = filesByCollection.get(c.id) ?? [];
+            const links = files.flatMap((f) =>
+                linksFor(
+                    f,
+                    storedAtSavePath(downloadDirectory, distinct.get(f.id)!),
+                ),
+            );
+            const linkNames = uniqueNames(links, { beforeExtension: true });
             try {
-                rebuildSymlink(
-                    join(colDir, linkName),
-                    relative(colDir, link.target),
-                );
+                removeStaleLinks(colDir, new Set(linkNames), downloadDirectory);
             } catch (err) {
                 log(
-                    `FAILED symlink ${c.name}/${linkName}: ${describeError(err)}`,
-                );
-                recordFailure(
-                    link.file.id,
-                    link.file.metadata.title,
-                    c.name,
-                    err,
+                    `FAILED removing old links in ${c.name}: ${describeError(err)}`,
                 );
             }
-        }
 
-        writeAlbumJSON(
-            join(collectionsDir, `${colDirName}.json`),
-            c,
-            metaFiles,
-        );
+            const metaFiles = files.map((f) => ({
+                id: f.id,
+                metadata: f.metadata,
+            }));
+            for (const [i, link] of links.entries()) {
+                if (!includeOriginals || link.target === undefined) continue;
+                const linkName = linkNames[i]!;
+                try {
+                    rebuildSymlink(
+                        join(colDir, linkName),
+                        relative(colDir, link.target),
+                    );
+                } catch (err) {
+                    log(
+                        `FAILED symlink ${c.name}/${linkName}: ${describeError(err)}`,
+                    );
+                    recordFailure(
+                        link.file.id,
+                        link.file.metadata.title,
+                        c.name,
+                        err,
+                    );
+                }
+            }
+
+            writeAlbumJSON(
+                join(collectionsDir, `${colDirName}.json`),
+                c,
+                metaFiles,
+            );
+        } catch (err) {
+            log(`FAILED album ${c.name}: ${describeError(err)}`);
+            failedAlbums.push(c.id);
+        }
     }
 
     // Reconcile the ledger against what this run actually attempted: an entry
@@ -936,6 +971,7 @@ const runLockedBackup = async (
         failed: ledger.size,
         errors,
         undecryptableAlbums,
+        failedAlbums,
     };
 };
 

src/cli-commands.ts

@@ -470,7 +470,9 @@ export const backupCommand = async (
                 }
             }
 
-            return result.failed > 0 || result.undecryptableAlbums.length > 0
+            return result.failed > 0 ||
+                result.undecryptableAlbums.length > 0 ||
+                result.failedAlbums.length > 0
                 ? 1
                 : 0;
         } finally {

src/filename.ts

@@ -4,7 +4,8 @@
 // hands us, and quak does not trust the server. Any name taken from them and
 // used on disk goes through here, so it can only ever name one file inside the
 // directory the caller chose: never a path, never `..`, never hidden, never a
-// Windows device name.
+// Windows device name. Such a name can still be too long for a file name, and
+// `shortenToBytes` cuts one short.
 //
 // A path the user typed (`--out`, `outPath`) is not passed through here: the
 // caller is trusted, the server is not.
@@ -39,3 +40,20 @@ export const safeExtension = (title: string): string => {
 // `name` with its extension, if it has one, replaced by `ext` (".mov").
 export const withExtension = (name: string, ext: string): string =>
     name.slice(0, name.length - extname(name).length) + ext;
+
+// The longest file name, in bytes, that ext4 and most other file systems allow.
+export const MAX_NAME_BYTES = 255;
+
+// `name` cut short to at most `maxBytes` bytes of UTF-8, the encoding a file
+// name is written to disk in, between two characters, never inside one. A name
+// that fits comes back unchanged.
+export const shortenToBytes = (name: string, maxBytes: number): string => {
+    let bytes = 0;
+    let end = 0;
+    for (const character of name) {
+        bytes += Buffer.byteLength(character);
+        if (bytes > maxBytes) break;
+        end += character.length;
+    }
+    return name.slice(0, end);
+};