From 3cabeba24997499ebe70ef0f143589b7257f14a2 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 15 Oct 2026 23:53:26 +0000 Subject: [PATCH] An album whose folder cannot be made no longer stops the backup (closes #197) Each album's folder, links and JSON are written inside one try. A failure prints `FAILED album : `, lists the album's ID in the new `failedAlbums` of the result and of `--json`, and makes `quak backup` exit 1; the other albums and `failures.json` are still written. An album whose folder would have another album's JSON name, ignoring case, gets its ID. A name over 255 bytes (250 for a folder, which leaves room for `.json`) gets the ID and is cut short between two characters by `shortenToBytes` in `src/filename.ts`. A link's ID goes right before its extension. Apart from its call to `uniqueNames`, the album loop is only indented into the try. Model: opus-5-5 --- README.md | 38 +++-- TODO.md | 12 +- src/backup.ts | 150 ++++++++++------- src/cli-commands.ts | 4 +- src/filename.ts | 20 ++- test/cli/backup-end-to-end.test.ts | 52 +++++- test/cli/backup.test.ts | 250 ++++++++++++++++++++++++++++- test/filename/filename.test.ts | 29 +++- 8 files changed, 473 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 3fc1506..731d590 100644 --- a/README.md +++ b/README.md @@ -134,12 +134,12 @@ matters. ### Exit codes -| Code | Meaning | -| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `0` | The backup is complete: every original is at its save path, and no file failed. | -| `1` | The run finished with files in `failures.json` or an album it could not decrypt, or it stopped on another error, printed as one line `quak: `. The next run tries again. | -| `2` | Another backup of the same directory is running. This run sent no request and changed nothing. | -| `3` | There is no usable session: none is saved, the saved one is corrupt, or the server no longer accepts it. Run `quak login` as the user the cron job runs as. | +| Code | Meaning | +| ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | The backup is complete: every original is at its save path, and no file failed. | +| `1` | The run finished with files in `failures.json` or albums it could not decrypt or write (see "Backup layout"), or it stopped on another error, printed as one line `quak: `. The next run tries again. | +| `2` | Another backup of the same directory is running. This run sent no request and changed nothing. | +| `3` | There is no usable session: none is saved, the saved one is corrupt, or the server no longer accepts it. Run `quak login` as the user the cron job runs as. | `` is the error's message, followed by the message of each error that caused it. Under Node, as `yarn quak` runs quak, `fetch` reports every failure @@ -810,8 +810,23 @@ after the file's title, both with unsafe characters replaced. When two collections would get the same name, or two symlinks in one collection the same name (ignoring case in both), each of them gets its ID added: two albums named `Trip` become `Trip (10)/` and `Trip (11)/`, and two files titled `IMG_0001.JPG` -become `IMG_0001 (12345).JPG` and `IMG_0001 (12346).JPG`. IDs never change, so a -name stays the same from run to run until such a clash appears or goes away. +become `IMG_0001 (12345).JPG` and `IMG_0001 (12346).JPG`. A collection whose +directory would have the name of another collection's JSON, again ignoring case, +gets its ID added as well: beside an album `x`, whose JSON is `x.json`, an album +named `x.json` gets the directory `x.json (12)/`. A name too long for a file +name, which most file systems limit to 255 bytes, gets its ID added too. A name +with its ID added is cut short between two characters in the part before the ID, +just far enough that it fits. When a symlink's extension alone leaves no room +for the ID, the extension is cut short and the part before the ID keeps only the +bytes that cut leaves over. A collection's directory name also leaves room for +the `.json` its JSON adds. IDs never change, so a name stays the same from run +to run until such a clash appears or goes away. + +A collection whose directory or JSON cannot be written, such as one whose +directory name is taken by a file you put in `collections/`, does not stop the +run. A run without `--json` prints `FAILED album : ` for it, +`--json` and the `BackupResult` list its ID in `failedAlbums`, and the exit code +is 1. The other collections and `failures.json` are still written. A live photo, which Ente stores as one ZIP of its image and its video, is stored as those two files, which a photo viewer can open: each is @@ -1105,9 +1120,10 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange` `downloadDirectory` (falls back to the library's), `includeOriginals` (default `true`), `includeThumbnails` (default `false`), `onlyAlbumNames`, `verify` (default `false`), `onProgress`, and `lock`. See Backup layout above for the - tree it writes. Whether the backup took the lock or its caller did, once the - lock is lost the backup rejects with `lost the lock on : ` before - it starts another file. + tree it writes, and for the albums `BackupResult` lists in + `undecryptableAlbums` and `failedAlbums`. Whether the backup took the lock or + its caller did, once the lock is lost the backup rejects with + `lost the lock on : ` before it starts another file. ### Request pools diff --git a/TODO.md b/TODO.md index 6ef08dc..da48290 100644 --- a/TODO.md +++ b/TODO.md @@ -17,8 +17,7 @@ pre-1.0 # Next Step -Five bugs are open: https://git.eeqj.de/sneak/quak/issues/197, -https://git.eeqj.de/sneak/quak/issues/198, +Four bugs are open: https://git.eeqj.de/sneak/quak/issues/198, https://git.eeqj.de/sneak/quak/issues/199, https://git.eeqj.de/sneak/quak/issues/205 and https://git.eeqj.de/sneak/quak/issues/206. The check of quak against a live Ente @@ -30,6 +29,15 @@ declares one. # Completed Steps +- 2026-10-15: An album whose folder or JSON `quak backup` cannot write no longer + ends the run before the other albums and `failures.json` are written (issue + 197). The run prints `FAILED album : `, lists the album's ID in + `failedAlbums` in the result and in `--json`, goes on, and exits 1. An album + whose folder would have the name of another album's JSON, as `x.json` beside + `x` would, gets its ID added. A folder or link name over 255 bytes is cut + short between two characters, by `shortenToBytes` in `src/filename.ts`, and + gets the ID. + - 2026-10-15: A network failure is printed and recorded with its reason, not as just `fetch failed` (issue 196). `describeError` in `src/errors.ts` gives an error's message followed by the message of each error on its `cause` chain, diff --git a/src/backup.ts b/src/backup.ts index 16d2da9..925d05e 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -50,7 +50,9 @@ // running attempt count, and the last-tried time, and the run continues. // A file the library could not decrypt is recorded there too, unless a copy of // it in another album decrypted; an album it could not decrypt is logged by ID -// and listed in `result.undecryptableAlbums`, as its name is not known. +// and listed in `result.undecryptableAlbums`, as its name is not known. An +// album whose folder or JSON cannot be written is logged by name and +// listed in `result.failedAlbums`, and the run goes on to the next album. // `result.failed` — and thus the CLI's exit code — stays non-zero while any // failure remains unresolved and clears once every one succeeds. Each run // reconciles the ledger against the files it attempted, so an entry for a file @@ -83,7 +85,12 @@ import { } from "./crypto/index.js"; import { removeLeftoverTempFiles } from "./download/index.js"; import { describeError } from "./errors.js"; -import { sanitizeFileName, withExtension } from "./filename.js"; +import { + MAX_NAME_BYTES, + sanitizeFileName, + shortenToBytes, + withExtension, +} from "./filename.js"; import { copyAtomic, placeOriginal, @@ -160,6 +167,9 @@ export interface BackupResult { // The IDs of the albums the library could not decrypt, whose files it // could not list; the CLI exits non-zero while there are any. undecryptableAlbums: number[]; + // The IDs of the albums whose folder or JSON this run could not write; + // the CLI exits non-zero while there are any. + failedAlbums: number[]; } // The slice of the library that backup drives. `Library` implements it; a test @@ -264,31 +274,47 @@ const rebuildSymlink = (linkPath: string, target: string): void => { // The on-disk names for the entries of one directory, in entry order. Each // name is used as is unless another entry would get the same name, ignoring // case (two names that differ only in case are one entry on a case-insensitive -// file system); then every entry sharing it gets ` ()`, before the -// extension when `beforeExtension` is set. A name with an ID added can match -// another entry's own name (`IMG (6).JPG`), so this repeats until no name is -// shared. IDs are stable, so the names are too. +// file system), or it is too long for a file name; then it gets ` ()`, +// right before the extension when `beforeExtension` is set. To fit, the part +// before the ID is cut short as far as it must be. When the extension alone +// leaves no room for the ID, the extension is cut short and the part before +// the ID keeps only the bytes that cut leaves over. With `withJSON`, each +// entry also has the JSON file `.json` beside it, as an album does: a +// name then leaves room for `.json`, and one that is another entry's JSON name +// gets its ID added too. +// Two names with different IDs added never match, but one can match another +// entry's own name (`IMG (6).JPG`), so this repeats until no name is shared. +// IDs are stable, so the names are too. const uniqueNames = ( entries: { id: number; name: string }[], - beforeExtension: boolean, + opts: { beforeExtension?: boolean; withJSON?: boolean }, ): string[] => { + const maxBytes = MAX_NAME_BYTES - (opts.withJSON ? ".json".length : 0); const withID = (id: number, name: string): string => { - const ext = beforeExtension ? extname(name) : ""; - const stem = name.slice(0, name.length - ext.length); - return `${stem} (${id})${ext}`; + const tag = ` (${id})`; + const fullExt = opts.beforeExtension ? extname(name) : ""; + const stem = name.slice(0, name.length - fullExt.length); + const ext = shortenToBytes(fullExt, maxBytes - Buffer.byteLength(tag)); + const room = maxBytes - Buffer.byteLength(tag + ext); + return shortenToBytes(stem, room) + tag + ext; }; const names = entries.map((e) => e.name); const suffixed = new Set(); for (;;) { const counts = new Map(); - for (const name of names) { + const count = (name: string): void => { const key = name.toLowerCase(); counts.set(key, (counts.get(key) ?? 0) + 1); + }; + for (const name of names) { + count(name); + if (opts.withJSON) count(`${name}.json`); } let changed = false; for (const [i, { id, name }] of entries.entries()) { if (suffixed.has(i)) continue; - if (counts.get(name.toLowerCase()) === 1) continue; + const fits = Buffer.byteLength(name) <= maxBytes; + if (fits && counts.get(name.toLowerCase()) === 1) continue; names[i] = withID(id, name); suffixed.add(i); changed = true; @@ -838,7 +864,7 @@ const runLockedBackup = async ( id: c.id, name: sanitizeFileName(c.name, `collection-${c.id}`), })), - false, + { withJSON: true }, ); const albumDirNames = new Map( allCollections.map((c, i) => [c.id, dirNames[i]!]), @@ -854,60 +880,69 @@ const runLockedBackup = async ( log(`FAILED removing old album directories: ${describeError(err)}`); } + // An album whose folder or JSON cannot be written, such as one whose + // folder name is taken by a file the user put there, is logged and listed + // in the result, and the run goes on to the next album. + const failedAlbums: number[] = []; for (const c of collections) { if (lock.lost) throw lock.lost; - const colDirName = albumDirNames.get(c.id)!; - const colDir = join(collectionsDir, colDirName); - mkdirSync(colDir, { recursive: true }); - - // Every album links the one original, saved from the file's entry in - // `distinct`. - const files = filesByCollection.get(c.id) ?? []; - const links = files.flatMap((f) => - linksFor( - f, - storedAtSavePath(downloadDirectory, distinct.get(f.id)!), - ), - ); - const linkNames = uniqueNames(links, true); try { - removeStaleLinks(colDir, new Set(linkNames), downloadDirectory); - } catch (err) { - log( - `FAILED removing old links in ${c.name}: ${describeError(err)}`, - ); - } + const colDirName = albumDirNames.get(c.id)!; + const colDir = join(collectionsDir, colDirName); + mkdirSync(colDir, { recursive: true }); - const metaFiles = files.map((f) => ({ - id: f.id, - metadata: f.metadata, - })); - for (const [i, link] of links.entries()) { - if (!includeOriginals || link.target === undefined) continue; - const linkName = linkNames[i]!; + // Every album links the one original, saved from the file's entry + // in `distinct`. + const files = filesByCollection.get(c.id) ?? []; + const links = files.flatMap((f) => + linksFor( + f, + storedAtSavePath(downloadDirectory, distinct.get(f.id)!), + ), + ); + const linkNames = uniqueNames(links, { beforeExtension: true }); try { - rebuildSymlink( - join(colDir, linkName), - relative(colDir, link.target), - ); + removeStaleLinks(colDir, new Set(linkNames), downloadDirectory); } catch (err) { log( - `FAILED symlink ${c.name}/${linkName}: ${describeError(err)}`, - ); - recordFailure( - link.file.id, - link.file.metadata.title, - c.name, - err, + `FAILED removing old links in ${c.name}: ${describeError(err)}`, ); } - } - writeAlbumJSON( - join(collectionsDir, `${colDirName}.json`), - c, - metaFiles, - ); + const metaFiles = files.map((f) => ({ + id: f.id, + metadata: f.metadata, + })); + for (const [i, link] of links.entries()) { + if (!includeOriginals || link.target === undefined) continue; + const linkName = linkNames[i]!; + try { + rebuildSymlink( + join(colDir, linkName), + relative(colDir, link.target), + ); + } catch (err) { + log( + `FAILED symlink ${c.name}/${linkName}: ${describeError(err)}`, + ); + recordFailure( + link.file.id, + link.file.metadata.title, + c.name, + err, + ); + } + } + + writeAlbumJSON( + join(collectionsDir, `${colDirName}.json`), + c, + metaFiles, + ); + } catch (err) { + log(`FAILED album ${c.name}: ${describeError(err)}`); + failedAlbums.push(c.id); + } } // Reconcile the ledger against what this run actually attempted: an entry @@ -933,6 +968,7 @@ const runLockedBackup = async ( failed: ledger.size, errors, undecryptableAlbums, + failedAlbums, }; }; diff --git a/src/cli-commands.ts b/src/cli-commands.ts index 7a0c505..d545cf1 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -470,7 +470,9 @@ export const backupCommand = async ( } } - return result.failed > 0 || result.undecryptableAlbums.length > 0 + return result.failed > 0 || + result.undecryptableAlbums.length > 0 || + result.failedAlbums.length > 0 ? 1 : 0; } finally { diff --git a/src/filename.ts b/src/filename.ts index 1e6250b..e9781ca 100644 --- a/src/filename.ts +++ b/src/filename.ts @@ -4,7 +4,8 @@ // hands us, and quak does not trust the server. Any name taken from them and // used on disk goes through here, so it can only ever name one file inside the // directory the caller chose: never a path, never `..`, never hidden, never a -// Windows device name. +// Windows device name. Such a name can still be too long for a file name, and +// `shortenToBytes` cuts one short. // // A path the user typed (`--out`, `outPath`) is not passed through here: the // caller is trusted, the server is not. @@ -39,3 +40,20 @@ export const safeExtension = (title: string): string => { // `name` with its extension, if it has one, replaced by `ext` (".mov"). export const withExtension = (name: string, ext: string): string => name.slice(0, name.length - extname(name).length) + ext; + +// The longest file name, in bytes, that ext4 and most other file systems allow. +export const MAX_NAME_BYTES = 255; + +// `name` cut short to at most `maxBytes` bytes of UTF-8, the encoding a file +// name is written to disk in, between two characters, never inside one. A name +// that fits comes back unchanged. +export const shortenToBytes = (name: string, maxBytes: number): string => { + let bytes = 0; + let end = 0; + for (const character of name) { + bytes += Buffer.byteLength(character); + if (bytes > maxBytes) break; + end += character.length; + } + return name.slice(0, end); +}; diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index 367251e..e541229 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -274,15 +274,15 @@ afterAll(() => { // Run `quak backup ` as `bin/quak.ts` runs it, sending its requests // through `fetch`, and return its exit code and everything it printed on -// stderr. Its requests are retried as those of `quak backup` are, except that -// a retry does not wait. Its cache is beside `dir`, so each run of one -// directory finds the cache the run before it left, as cron's runs find the -// user's. +// stderr and on stdout. Its requests are retried as those of `quak backup` +// are, except that a retry does not wait. Its cache is beside `dir`, so each +// run of one directory finds the cache the run before it left, as cron's runs +// find the user's. const backup = async ( dir: string, - opts: { verify?: boolean } = {}, + opts: { verify?: boolean; json?: boolean } = {}, fetch = server, -): Promise<{ code: number; stderr: string }> => { +): Promise<{ code: number; stderr: string; stdout: string }> => { let printed = ""; const stderr = new Writable({ write(chunk: Buffer, _encoding, done) { @@ -290,8 +290,10 @@ const backup = async ( done(); }, }); + let output = ""; const stdout = new Writable({ - write(_chunk, _encoding, done) { + write(chunk: Buffer, _encoding, done) { + output += chunk.toString(); done(); }, }); @@ -316,7 +318,7 @@ const backup = async ( const code = await new Promise((resolve) => { void run(backupCommand(ctx, dir, opts), stdout, stderr, resolve); }); - return { code, stderr: printed }; + return { code, stderr: printed, stdout: output }; }; // The ID of the file whose original a request downloads, or undefined for any @@ -627,6 +629,40 @@ describe("quak backup", () => { ).toEqual([]); }); + // The user has put a file in collections/ with the name of the Trips + // folder, so quak cannot make that folder. It says so on one line, writes + // the two albums after it, and exits 1, and --json lists the album's ID + // in `failedAlbums`. No file failed, so there is no failures.json. + it("exits 1 when an album's folder cannot be made, writes the other albums, and lists it in --json", async () => { + const dir = mkdtempSync(join(root, "backup-")); + mkdirSync(join(dir, "collections")); + writeFileSync(join(dir, "collections", "Trips"), "mine"); + + const first = await backup(dir); + const { code, stdout } = await backup(dir, { json: true }); + + expect(first.code).toBe(1); + expect(first.stderr).toMatch(/^FAILED album Trips: EEXIST: /m); + expect(code).toBe(1); + expect(JSON.parse(stdout)).toMatchObject({ + failed: 0, + undecryptableAlbums: [], + failedAlbums: [10], + }); + const collections = join(dir, "collections"); + expect(readJSON(join(collections, "Family.json")).id).toBe(20); + expect(readdirSync(join(collections, "Family")).sort()).toEqual([ + "IMG_0002.heic", + "IMG_0002.mov", + "beach.jpg", + ]); + expect(readJSON(join(collections, "Friend's Wedding.json")).id).toBe( + 30, + ); + expect(existsSync(join(collections, "Trips.json"))).toBe(false); + expect(existsSync(join(dir, "failures.json"))).toBe(false); + }); + // The server no longer accepts the saved session's token, as after // `quak logout` elsewhere, and answers every request with 401. The run // stops at the refresh it starts with, says to log in again, and exits 3. diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 1938a4b..09bedc6 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -1498,13 +1498,15 @@ describe("backup album folders", () => { files: EnteFile[]; } - const libraryOf = (albums: Album[]): BackupLibrary => ({ + // The original of the file `notFound`, if given, always fails to download. + const libraryOf = (albums: Album[], notFound?: number): BackupLibrary => ({ refresh: async () => {}, listCollections: () => albums.map((a) => a.collection), listFiles: (id) => albums.find((a) => a.collection.id === id)?.files ?? [], undecryptable: () => [], original: async (fileID) => { + if (fileID === notFound) throw new Error("HTTP 404"); const path = join(root, `source-${fileID}`); writeFileSync(path, `original ${fileID}`); return { path }; @@ -1600,6 +1602,252 @@ describe("backup album folders", () => { expect(albumID(outDir, "Trip (11) (12).json")).toBe(12); }); + // The next three tests end with the album "ok" (14), whose one file + // always fails to download, so each run has a failure to record in + // failures.json after the album folders are written. + const ok: Album = { + collection: collection(14, "ok"), + files: [file(5, 14, "e.jpg")], + }; + + // The folder of an album named "x.json" would be the JSON file of the + // album "x", and once ended every run there. Two runs, as on two nights, + // write the same tree, and the failed file's attempt count goes up. + it("adds the ID to an album whose folder would have another album's JSON name", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf( + [ + { + collection: collection(10, "x"), + files: [file(1, 10, "a.jpg")], + }, + { + collection: collection(11, "x.json"), + files: [file(2, 11, "b.jpg")], + }, + { + collection: collection(12, "Trip"), + files: [file(3, 12, "c.jpg")], + }, + { + collection: collection(13, "TRIP.json"), + files: [file(4, 13, "d.jpg")], + }, + ok, + ], + 5, + ); + + await runBackup(lib, { downloadDirectory: outDir }); + const result = await runBackup(lib, { downloadDirectory: outDir }); + + expect(result.failedAlbums).toEqual([]); + expect(tree(outDir)).toEqual([ + "TRIP.json (13)/", + `TRIP.json (13)/d.jpg -> ${linkTo("4.jpg")}`, + "TRIP.json (13).json", + "Trip/", + `Trip/c.jpg -> ${linkTo("3.jpg")}`, + "Trip.json", + "ok/", + "ok.json", + "x/", + `x/a.jpg -> ${linkTo("1.jpg")}`, + "x.json", + "x.json (11)/", + `x.json (11)/b.jpg -> ${linkTo("2.jpg")}`, + "x.json (11).json", + ]); + expect(albumID(outDir, "x.json")).toBe(10); + expect(albumID(outDir, "x.json (11).json")).toBe(11); + expect(albumID(outDir, "TRIP.json (13).json")).toBe(13); + expect(albumID(outDir, "ok.json")).toBe(14); + expect(result.failed).toBe(1); + expect(readLedger(outDir).files["5"]).toMatchObject({ attempts: 2 }); + }); + + // A name over 255 bytes is refused as a file name, and an album folder's + // name leaves room for `.json` in its JSON's. "é" takes two bytes, and a + // name is cut between two characters. + it("cuts an album name and a title too long for a file name short and adds the ID", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf( + [ + { + collection: collection(10, "L".repeat(300)), + files: [file(1, 10, `${"é".repeat(150)}.jpg`)], + }, + ok, + ], + 5, + ); + + await runBackup(lib, { downloadDirectory: outDir }); + const result = await runBackup(lib, { downloadDirectory: outDir }); + + const folder = `${"L".repeat(245)} (10)`; + const link = `${"é".repeat(123)} (1).jpg`; + expect(Buffer.byteLength(`${folder}.json`)).toBe(255); + expect(Buffer.byteLength(link)).toBe(254); + expect(result.failedAlbums).toEqual([]); + expect(tree(outDir)).toEqual([ + `${folder}/`, + `${folder}/${link} -> ${linkTo("1.jpg")}`, + `${folder}.json`, + "ok/", + "ok.json", + ]); + expect(albumID(outDir, `${folder}.json`)).toBe(10); + expect(albumID(outDir, "ok.json")).toBe(14); + expect(result.failed).toBe(1); + expect(readLedger(outDir).files["5"]).toMatchObject({ attempts: 2 }); + }); + + // The user has put a file in collections/ with the name of the album + // Trip's folder, so the folder cannot be made. + it("goes on to the next album when an album's folder cannot be made, and lists it as failed", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf( + [ + { + collection: collection(10, "Trip"), + files: [file(1, 10, "a.jpg")], + }, + ok, + ], + 5, + ); + mkdirSync(join(outDir, "collections"), { recursive: true }); + writeFileSync(join(outDir, "collections", "Trip"), "mine"); + const logged: string[] = []; + + const result = await runBackup(lib, { + downloadDirectory: outDir, + onProgress: (message) => logged.push(message), + }); + + expect(result.failedAlbums).toEqual([10]); + expect(logged).toContainEqual( + expect.stringMatching(/^FAILED album Trip: EEXIST: /), + ); + expect(tree(outDir)).toEqual(["Trip", "ok/", "ok.json"]); + expect(albumID(outDir, "ok.json")).toBe(14); + expect(existsSync(saved(outDir, "1.jpg"))).toBe(true); + expect(result.failed).toBe(1); + expect(readLedger(outDir).files["5"]).toMatchObject({ attempts: 1 }); + }); + + // The extensions of these titles leave no room for the ID, so each is cut + // short and the name before the ID keeps only the bytes that cut leaves + // over. File 1's extension is 303 bytes and is cut to 251, which leaves + // none. File 2's is 271 bytes and is cut to 250, since "€" takes three + // bytes, which leaves one, for the "a". Neither extension is only letters + // and digits, so both originals are saved with `.bin`, a name that fits. + it("cuts the extension of a title short when it leaves no room for the ID", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [ + file(1, 10, `a.${"L".repeat(300)}-x`), + file(2, 10, `a.${"€".repeat(90)}`), + ], + }, + ]); + + const result = await runBackup(lib, { downloadDirectory: outDir }); + + const link = ` (1).${"L".repeat(250)}`; + const link2 = `a (2).${"€".repeat(83)}`; + expect(Buffer.byteLength(link)).toBe(255); + expect(Buffer.byteLength(link2)).toBe(255); + expect(result.failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Trip/", + `Trip/${link} -> ${linkTo("1.bin")}`, + `Trip/${link2} -> ${linkTo("2.bin")}`, + "Trip.json", + ]); + }); + + // Files 1 and 3 share a title, so each gets its ID before the + // extension. File 2's title is too long, and its extension leaves no + // room for the ID. Were the ID put at the end of file 2's title cut + // short, its link would have file 1's name, and the link made second + // would replace the first. + it("keeps names unique when a title's extension leaves no room for the ID", async () => { + const outDir = join(root, "backup"); + const shared = `x.${"Q".repeat(245)} (2)`; + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [ + file(1, 10, shared), + file(2, 10, `x (1).${"Q".repeat(250)}-`), + file(3, 10, shared), + ], + }, + ]); + + const result = await runBackup(lib, { downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Trip/", + `Trip/ (2).${"Q".repeat(250)} -> ${linkTo("2.bin")}`, + `Trip/x (1).${"Q".repeat(245)} (2) -> ${linkTo("1.bin")}`, + `Trip/x (3).${"Q".repeat(245)} (2) -> ${linkTo("3.bin")}`, + "Trip.json", + ]); + }); + + // A title of 255 bytes is the longest file name allowed, and an album + // name of 250 bytes the longest whose JSON name fits; both are kept as + // they are. A name one byte longer gets the ID and is cut short. "é" + // takes two bytes, and in both names the room left ends inside one, so + // the cut falls before it, one byte short of the room. + it("keeps a name at the byte limit and adds the ID to one a byte over it", async () => { + const outDir = join(root, "backup"); + const album = "A".repeat(250); + const title = `${"t".repeat(251)}.jpg`; + const longAlbum = `${"é".repeat(125)}B`; + const longTitle = `${"é".repeat(126)}.jpg`; + const lib = libraryOf([ + { + collection: collection(10, album), + files: [file(1, 10, title), file(2, 10, longTitle)], + }, + { + collection: collection(11, longAlbum), + files: [file(3, 11, "c.jpg")], + }, + ]); + + const result = await runBackup(lib, { downloadDirectory: outDir }); + + const folder = `${"é".repeat(122)} (11)`; + const link = `${"é".repeat(123)} (2).jpg`; + expect(Buffer.byteLength(`${album}.json`)).toBe(255); + expect(Buffer.byteLength(title)).toBe(255); + expect(Buffer.byteLength(longAlbum)).toBe(251); + expect(Buffer.byteLength(longTitle)).toBe(256); + expect(Buffer.byteLength(`${folder}.json`)).toBe(254); + expect(Buffer.byteLength(link)).toBe(254); + expect(result.failed).toBe(0); + expect(result.failedAlbums).toEqual([]); + expect(tree(outDir)).toEqual([ + `${album}/`, + `${album}/${title} -> ${linkTo("1.jpg")}`, + `${album}/${link} -> ${linkTo("2.jpg")}`, + `${album}.json`, + `${folder}/`, + `${folder}/c.jpg -> ${linkTo("3.jpg")}`, + `${folder}.json`, + ]); + expect(albumID(outDir, `${album}.json`)).toBe(10); + expect(albumID(outDir, `${folder}.json`)).toBe(11); + }); + it("changes nothing on a second run over an unchanged account", async () => { const outDir = join(root, "backup"); const lib = libraryOf([ diff --git a/test/filename/filename.test.ts b/test/filename/filename.test.ts index 81e7c56..4cfce05 100644 --- a/test/filename/filename.test.ts +++ b/test/filename/filename.test.ts @@ -14,7 +14,11 @@ import { describe, expect, it } from "vitest"; -import { safeExtension, sanitizeFileName } from "../../src/filename.js"; +import { + safeExtension, + sanitizeFileName, + shortenToBytes, +} from "../../src/filename.js"; const FALLBACK = "file-42"; @@ -91,3 +95,26 @@ describe("safeExtension", () => { expect(safeExtension("x.jpg\0")).toBe(".bin"); }); }); + +describe("shortenToBytes", () => { + it("returns a name that fits unchanged", () => { + expect(shortenToBytes("beach", 5)).toBe("beach"); + expect(shortenToBytes("café", 5)).toBe("café"); + expect(shortenToBytes("", 0)).toBe(""); + }); + + it("cuts a name to the number of bytes given", () => { + expect(shortenToBytes("L".repeat(300), 255)).toBe("L".repeat(255)); + }); + + // In UTF-8 "é" takes two bytes and an emoji four, which a JavaScript + // string holds as two halves. + it("cuts between two characters, never inside one", () => { + expect(shortenToBytes("ééé", 5)).toBe("éé"); + expect(shortenToBytes("a\u{1F600}b", 4)).toBe("a"); + expect(shortenToBytes("a\u{1F600}b", 5)).toBe("a\u{1F600}"); + expect(Buffer.byteLength(shortenToBytes("é".repeat(200), 255))).toBe( + 254, + ); + }); +});