From d3722839009bf3ab4fdd817002bc83151d89944e Mon Sep 17 00:00:00 2001 From: sneak Date: Sun, 18 Oct 2026 16:26:46 +0000 Subject: [PATCH] Names quak builds from server data fit in 255 bytes (closes #205) The default names of `quak get`, `quak get-thumb`, `downloadFile` and `downloadThumbnail`, and the album folders of `backup-metadata`, could be over 255 bytes, which a file system refuses. `shortenKeepingExtension` in `src/filename.ts` cuts a name short, keeping its extension when it fits, and can put text right before the extension; the backup's `withID` now calls it instead of doing the same itself. The `get` defaults use it with `thumb_` counted. A `backup-metadata` folder keeps its `-` and has the name after it cut short. `safeExtension` gives `.bin` for an extension of more than 16 letters and digits, so a save path's name always fits. Model: opus-5-5 --- README.md | 19 +++++-- TODO.md | 17 ++++-- src/backup.ts | 9 ++-- src/cli-output.ts | 23 +++++++-- src/download/index.ts | 24 +++++++-- src/filename.ts | 27 ++++++++-- src/metadata-backup.ts | 16 +++++- test/cli/commands.test.ts | 94 +++++++++++++++++++++++++++++++++- test/download/download.test.ts | 42 +++++++++++++++ test/filename/filename.test.ts | 46 +++++++++++++++++ 10 files changed, 290 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 4ff51c2..11c570b 100644 --- a/README.md +++ b/README.md @@ -675,6 +675,12 @@ before any request with one line, `Invalid file ID: ` or `Invalid collection ID: `, and exit status 1. One bad `--file` ID refuses the whole command, so nothing is fixed. +Without `--out`, `get` writes the file to the working directory under its title, +and `get-thumb` writes the thumbnail there under `thumb_` followed by the title, +each with unsafe characters replaced. A name over 255 bytes, which most file +systems refuse, is cut short between two characters before its extension, so the +extension is kept whenever it fits. + `get` and `get-thumb` resolve the file by ID directly, so `--collection` is accepted for backward compatibility but ignored. For a live photo, `get` writes its image and its video, each named after the title with its own extension, as @@ -696,6 +702,12 @@ and `helper fix-missing-thumbnails` take `--json` for machine-readable output. any that do not match the content hash Ente records; one it cannot replace stays as it was and goes into `failures.json` (see "Backup layout"). +`backup-metadata ` writes `account.json` and, for each collection, a +directory `collections/-/` holding `_collection.json` and a +`.json` for each of its files. The collection's name has unsafe +characters replaced and is cut short between two characters so that the +directory's name fits in 255 bytes; the ID keeps the directories apart. + `backup-metadata` fetches ML data in requests of up to 200 files. When a request fails, the error is logged, each of its files is written with the reason in an `mlDataError` field instead of `mlData`, and the dump goes on. A file whose ML @@ -748,9 +760,10 @@ Each original is saved at its save path, the same path `photo.savePath` gives and `photo.download()` writes (see Read surface below). The date is the photo's `takenAt` (the date set in Ente if it was edited, else its creation time) in the time zone of the machine running quak. The extension is the one in the file's -name as uploaded, case kept, or `.bin` when it has none or it holds anything but -letters and digits. When the date or the time zone changes, the next run saves -the original at its new path and leaves the old copy where it is. +name as uploaded, case kept, or `.bin` when it has none, it holds anything but +letters and digits, or it is more than 16 of them long. When the date or the +time zone changes, the next run saves the original at its new path and leaves +the old copy where it is. `account.json` holds the account's `email` and `userID`, as `backup-metadata` writes it. A collection's JSON holds its `id`, `name`, `type`, `ownerID`, diff --git a/TODO.md b/TODO.md index 840b2bb..fbf5f30 100644 --- a/TODO.md +++ b/TODO.md @@ -17,15 +17,26 @@ pre-1.0 # Next Step -One bug is open: https://git.eeqj.de/sneak/quak/issues/205. The check of quak -against a live Ente account, https://git.eeqj.de/sneak/quak/issues/187, needs an -account only sneak has, so it waits on him. +No bug is open. The check of quak against a live Ente account, +https://git.eeqj.de/sneak/quak/issues/187, needs an account only sneak has, so +it waits on him. Tagging and releases are decided by sneak alone, and happen only when he declares one. # Completed Steps +- 2026-10-18: Names quak builds from an album name or a title fit in a file name + (issue 205). `shortenKeepingExtension` in `src/filename.ts` cuts a name short + to a number of bytes, keeping its extension when it fits, and can put text + right before the extension; the backup's `withID` uses it for an ID. The + default names of `quak get`, `quak get-thumb`, `downloadFile` and + `downloadThumbnail` are cut short with it to 255 bytes, `thumb_` included. + `backup-metadata` cuts an album's name short so that its folder, + `-`, fits in 255 bytes. `safeExtension` gives `.bin` for an + extension of more than 16 letters and digits, so a save path's name always + fits. + - 2026-10-16: ML data the server sends that does not decrypt, gunzip or parse is no longer taken for no ML data (issue 198). `fetchMLDataBatch` returns why for each such file beside the payloads; ML data that parses to something other diff --git a/src/backup.ts b/src/backup.ts index 32a9412..16fd802 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -82,6 +82,7 @@ import { describeError } from "./errors.js"; import { MAX_NAME_BYTES, sanitizeFileName, + shortenKeepingExtension, shortenToBytes, withExtension, } from "./filename.js"; @@ -305,11 +306,9 @@ const uniqueNames = ( const maxBytes = MAX_NAME_BYTES - (opts.withJSON ? ".json".length : 0); const withID = (id: number, name: string): string => { const tag = ` (${id})`; - const fullExt = opts.beforeExtension ? extname(name) : ""; - const stem = name.slice(0, name.length - fullExt.length); - const ext = shortenToBytes(fullExt, maxBytes - Buffer.byteLength(tag)); - const room = maxBytes - Buffer.byteLength(tag + ext); - return shortenToBytes(stem, room) + tag + ext; + return opts.beforeExtension + ? shortenKeepingExtension(name, maxBytes, tag) + : shortenToBytes(name, maxBytes - Buffer.byteLength(tag)) + tag; }; const names = entries.map((e) => e.name); const suffixed = new Set(); diff --git a/src/cli-output.ts b/src/cli-output.ts index 2bf9b21..582e374 100644 --- a/src/cli-output.ts +++ b/src/cli-output.ts @@ -9,7 +9,11 @@ // `metadata.title`, and issue #52 requires that output stay byte-identical, so // the commands shape their output from the raw `EnteFile` through here. -import { sanitizeFileName } from "./filename.js"; +import { + MAX_NAME_BYTES, + sanitizeFileName, + shortenKeepingExtension, +} from "./filename.js"; import type { EnteFile, FileType, Microseconds } from "./model/types.js"; // One row of `quak files --json`. @@ -34,10 +38,19 @@ export const fileListLine = (file: EnteFile): string => `${file.id}\t${file.metadata.fileType}\t${file.metadata.title}`; // Default output path for `quak get` when `--out` is not given. The title comes -// from the server, so it is sanitized; `--out` is the user's and is used as is. +// from the server, so it is sanitized, and cut short to fit in a file name, +// keeping its extension; `--out` is the user's and is used as is. export const originalName = (file: EnteFile): string => - sanitizeFileName(file.metadata.title, `file-${file.id}`); + shortenKeepingExtension( + sanitizeFileName(file.metadata.title, `file-${file.id}`), + MAX_NAME_BYTES, + ); -// Default output path for `quak get-thumb` when `--out` is not given. +// Default output path for `quak get-thumb` when `--out` is not given: `thumb_` +// and the title, cut short so that the two fit in a file name. export const thumbnailName = (file: EnteFile): string => - `thumb_${originalName(file)}`; + "thumb_" + + shortenKeepingExtension( + sanitizeFileName(file.metadata.title, `file-${file.id}`), + MAX_NAME_BYTES - "thumb_".length, + ); diff --git a/src/download/index.ts b/src/download/index.ts index 59b510a..ac360a6 100644 --- a/src/download/index.ts +++ b/src/download/index.ts @@ -16,7 +16,13 @@ import { streamTagFinal, } from "../crypto/index.js"; import { TruncatedStreamError } from "../errors.js"; -import { safeExtension, sanitizeFileName, withExtension } from "../filename.js"; +import { + MAX_NAME_BYTES, + safeExtension, + sanitizeFileName, + shortenKeepingExtension, + withExtension, +} from "../filename.js"; import { withRetry } from "../retry.js"; import type { ApiClient } from "../api/client.js"; import type { EnteFile } from "../model/types.js"; @@ -583,9 +589,14 @@ export const downloadFile = async ( onProgress?: ProgressCallback, ): Promise => { // `outPath` is the caller's and is used as is; the title is the server's - // and is sanitized so it can only name a file in the current directory. + // and is sanitized so it can only name a file in the current directory, + // and cut short to fit in a file name, keeping its extension. const resolvedPath = - outPath ?? sanitizeFileName(file.metadata.title, `file-${file.id}`); + outPath ?? + shortenKeepingExtension( + sanitizeFileName(file.metadata.title, `file-${file.id}`), + MAX_NAME_BYTES, + ); const header = fromBase64(file.file.decryptionHeader); return fetchAndDecrypt( api, @@ -604,9 +615,14 @@ export const downloadThumbnail = async ( outPath?: string, onProgress?: ProgressCallback, ): Promise => { + // As in `downloadFile`, with `thumb_` before the title. const resolvedPath = outPath ?? - `thumb_${sanitizeFileName(file.metadata.title, `file-${file.id}`)}`; + "thumb_" + + shortenKeepingExtension( + sanitizeFileName(file.metadata.title, `file-${file.id}`), + MAX_NAME_BYTES - "thumb_".length, + ); const header = fromBase64(file.thumbnail.decryptionHeader); return fetchAndDecrypt( api, diff --git a/src/filename.ts b/src/filename.ts index e9781ca..4ef21db 100644 --- a/src/filename.ts +++ b/src/filename.ts @@ -5,7 +5,7 @@ // used on disk goes through here, so it can only ever name one file inside the // directory the caller chose: never a path, never `..`, never hidden, never a // Windows device name. Such a name can still be too long for a file name, and -// `shortenToBytes` cuts one short. +// `shortenToBytes` or `shortenKeepingExtension` cuts one short. // // A path the user typed (`--out`, `outPath`) is not passed through here: the // caller is trusted, the server is not. @@ -30,11 +30,11 @@ export const sanitizeFileName = (name: string, fallback: string): string => { return RESERVED_DEVICE_NAME.test(cleaned) ? `_${cleaned}` : cleaned; }; -// The extension of `title` (".jpg"), or ".bin" when it has none or it holds -// anything but letters and digits. +// The extension of `title` (".jpg"), or ".bin" when it has none, it holds +// anything but letters and digits, or it is more than 16 of them long. export const safeExtension = (title: string): string => { const ext = extname(title); - return /^\.[A-Za-z0-9]+$/.test(ext) ? ext : ".bin"; + return /^\.[A-Za-z0-9]{1,16}$/.test(ext) ? ext : ".bin"; }; // `name` with its extension, if it has one, replaced by `ext` (".mov"). @@ -57,3 +57,22 @@ export const shortenToBytes = (name: string, maxBytes: number): string => { } return name.slice(0, end); }; + +// `name` cut short to fit in `maxBytes` bytes with `insert` put right before +// its extension (`IMG_0001 (12).jpg`), each cut made as `shortenToBytes` +// makes it. The part before the extension is cut first, so the extension is +// kept whenever it fits. When the extension alone leaves no room for +// `insert`, the extension is cut short too, and the part before it keeps only +// the bytes that cut leaves over. `insert` is never cut. With no `insert`, a +// name that fits comes back unchanged. +export const shortenKeepingExtension = ( + name: string, + maxBytes: number, + insert = "", +): string => { + const fullExt = extname(name); + const stem = name.slice(0, name.length - fullExt.length); + const ext = shortenToBytes(fullExt, maxBytes - Buffer.byteLength(insert)); + const room = maxBytes - Buffer.byteLength(insert + ext); + return shortenToBytes(stem, room) + insert + ext; +}; diff --git a/src/metadata-backup.ts b/src/metadata-backup.ts index 79fc4ed..9620efe 100644 --- a/src/metadata-backup.ts +++ b/src/metadata-backup.ts @@ -5,7 +5,11 @@ import type { Client } from "./client.js"; import { describeError } from "./errors.js"; import { readExifTags } from "./exif.js"; import type { Library, Photo } from "./library/index.js"; -import { sanitizeFileName } from "./filename.js"; +import { + MAX_NAME_BYTES, + sanitizeFileName, + shortenToBytes, +} from "./filename.js"; import { fetchMLDataBatch, MLDATA_BATCH_SIZE, @@ -127,7 +131,15 @@ export const runMetadataBackup = async ( const col = lib.getCollection(album.collectionID); if (!col) continue; - const dirName = `${col.id}-${sanitizeFileName(col.name, "unnamed")}`; + // The album's ID keeps the folders apart, so the name after it can be + // cut short to fit in a file name. + const prefix = `${col.id}-`; + const dirName = + prefix + + shortenToBytes( + sanitizeFileName(col.name, "unnamed"), + MAX_NAME_BYTES - prefix.length, + ); const colDir = join(outDir, "collections", dirName); mkdirSync(colDir, { recursive: true }); diff --git a/test/cli/commands.test.ts b/test/cli/commands.test.ts index 241e259..8a4fcf7 100644 --- a/test/cli/commands.test.ts +++ b/test/cli/commands.test.ts @@ -21,7 +21,7 @@ import { utimesSync, writeFileSync, } from "node:fs"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { tmpdir } from "node:os"; import { PassThrough } from "node:stream"; import * as jpegJs from "jpeg-js"; @@ -544,6 +544,98 @@ describe("get and get-thumb", () => { }); }); +// A file system refuses a file name over 255 bytes. The account here is one +// album, whose name is 300 bytes, holding one file, 700, with the title given. +describe("names from an album name or a title over 255 bytes", () => { + const longNames = (title: string): Client => + ({ + ...fakeClient(), + collectionsSince: async () => ({ + collections: [collection(7, "A".repeat(300))], + deleted: [], + cursor: 1, + }), + filesSince: async () => ({ + files: [file(700, 7, title)], + deleted: [], + cursor: 1, + }), + }) as unknown as Client; + const longTitle = `${"t".repeat(296)}.jpg`; + + // Run `command` in a new directory, where `get` and `get-thumb` without + // `--out` write, and return its exit code and the directory. + const inNewDirectory = async ( + command: () => Promise, + ): Promise<{ code: number; dir: string }> => { + const dir = mkdtempSync(join(root, "cwd-")); + const previous = process.cwd(); + process.chdir(dir); + try { + return { code: await command(), dir }; + } finally { + process.chdir(previous); + } + }; + + it("get cuts the title short to 255 bytes, keeping its extension, and exits 0", async () => { + const { code, dir } = await inNewDirectory(() => + getCommand(context(longNames(longTitle)), "700", {}), + ); + + const name = `${"t".repeat(251)}.jpg`; + expect(Buffer.byteLength(name)).toBe(255); + expect(code).toBe(0); + expect(readdirSync(dir)).toEqual([name]); + expect(stderr.text).toBe(`7 bytes -> ${name}\n`); + }); + + it("get-thumb cuts the title short so that thumb_ and it fit in 255 bytes, and exits 0", async () => { + const { code, dir } = await inNewDirectory(() => + getThumbCommand(context(longNames(longTitle)), "700", {}), + ); + + const name = `thumb_${"t".repeat(245)}.jpg`; + expect(Buffer.byteLength(name)).toBe(255); + expect(code).toBe(0); + expect(readdirSync(dir)).toEqual([name]); + expect(stderr.text).toBe(`3 bytes -> ${name}\n`); + }); + + it("backup-metadata cuts the album name short so that its folder's name fits in 255 bytes, and exits 0", async () => { + const dir = join(root, "dump"); + + const code = await backupMetadataCommand( + context(longNames("beach.jpg")), + dir, + {}, + ); + + const folder = `7-${"A".repeat(253)}`; + expect(Buffer.byteLength(folder)).toBe(255); + expect(code).toBe(0); + expect(readdirSync(join(dir, "collections"))).toEqual([folder]); + expect(readdirSync(join(dir, "collections", folder)).sort()).toEqual([ + "700.json", + "_collection.json", + ]); + }); + + // The title's extension is 300 letters, too long for the save path's + // name, so the original is saved with `.bin`. + it("backup saves a title with a 300-letter extension as .bin, and exits 0", async () => { + const dir = join(root, "backup"); + const title = `b.${"e".repeat(300)}`; + + const code = await backupCommand(context(longNames(title)), dir, {}); + + const path = savePath(dir, file(700, 7, title)); + expect(code).toBe(0); + expect(basename(path)).toMatch(/^\d{4}-\d{2}-\d{2}\.700\.bin$/); + expect(readFileSync(path)).toEqual(Buffer.alloc(7, 700 & 0xff)); + }); +}); + // A live photo, which Ente stores as one ZIP, is written as its image and its // video, which a photo viewer can open. The account here is Vacation holding // one live photo, 300, downloaded through the real download layer diff --git a/test/download/download.test.ts b/test/download/download.test.ts index 7333b36..9a1d38a 100644 --- a/test/download/download.test.ts +++ b/test/download/download.test.ts @@ -658,6 +658,26 @@ describe("downloadFile", () => { expect(readdirSync(parent)).toEqual(["cwd"]); }); + // A file system refuses a file name over 255 bytes. + it("cuts a title too long for a file name short, keeping its extension", async () => { + const plaintext = new Uint8Array([1, 2, 3]); + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const { header, ciphertext } = encryptFileBody(plaintext, key); + const thumbPush = + sodium.crypto_secretstream_xchacha20poly1305_init_push(key); + const file = buildMockEnteFile(key, header, thumbPush.header); + file.metadata.title = `${"t".repeat(296)}.jpg`; + const dir = mkdtempSync(join(testDir, "long-title-")); + + const api = new ApiClient({ fetch: mockFetchForBody(ciphertext) }); + const result = await inDirectory(dir, () => downloadFile(api, file)); + + const name = `${"t".repeat(251)}.jpg`; + expect(Buffer.byteLength(name)).toBe(255); + expect(result.path).toBe(name); + expect(readdirSync(dir)).toEqual([name]); + }); + it("uses an explicit outPath verbatim, even one with ..", async () => { // The caller is trusted: its path is not sanitized. const { api, file } = fixtureFor( @@ -751,6 +771,28 @@ describe("downloadThumbnail", () => { expect(result.path).toBe("thumb__etc_passwd"); expect(readdirSync(dir)).toEqual(["thumb__etc_passwd"]); }); + + it("leaves room for thumb_ when it cuts a title too long for a file name short", async () => { + const plaintext = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]); + const key = sodium.crypto_secretstream_xchacha20poly1305_keygen(); + const filePush = + sodium.crypto_secretstream_xchacha20poly1305_init_push(key); + const { header: thumbHeader, ciphertext: thumbCipher } = + encryptFileBody(plaintext, key); + const file = buildMockEnteFile(key, filePush.header, thumbHeader); + file.metadata.title = `${"t".repeat(296)}.jpg`; + const dir = mkdtempSync(join(testDir, "long-thumb-title-")); + + const api = new ApiClient({ fetch: mockFetchForBody(thumbCipher) }); + const result = await inDirectory(dir, () => + downloadThumbnail(api, file), + ); + + const name = `thumb_${"t".repeat(245)}.jpg`; + expect(Buffer.byteLength(name)).toBe(255); + expect(result.path).toBe(name); + expect(readdirSync(dir)).toEqual([name]); + }); }); // --------------------------------------------------------------------------- diff --git a/test/filename/filename.test.ts b/test/filename/filename.test.ts index 4cfce05..0840270 100644 --- a/test/filename/filename.test.ts +++ b/test/filename/filename.test.ts @@ -17,6 +17,7 @@ import { describe, expect, it } from "vitest"; import { safeExtension, sanitizeFileName, + shortenKeepingExtension, shortenToBytes, } from "../../src/filename.js"; @@ -94,6 +95,12 @@ describe("safeExtension", () => { expect(safeExtension("x.jp g")).toBe(".bin"); expect(safeExtension("x.jpg\0")).toBe(".bin"); }); + + it("keeps an extension of 16 letters and digits, and uses .bin for a longer one", () => { + expect(safeExtension(`x.${"e".repeat(16)}`)).toBe(`.${"e".repeat(16)}`); + expect(safeExtension(`x.${"e".repeat(17)}`)).toBe(".bin"); + expect(safeExtension(`b.${"e".repeat(300)}`)).toBe(".bin"); + }); }); describe("shortenToBytes", () => { @@ -118,3 +125,42 @@ describe("shortenToBytes", () => { ); }); }); + +describe("shortenKeepingExtension", () => { + it("returns a name that fits unchanged", () => { + expect(shortenKeepingExtension("beach.jpg", 9)).toBe("beach.jpg"); + expect(shortenKeepingExtension("beach", 5)).toBe("beach"); + }); + + it("cuts the part before the extension short and keeps the extension", () => { + expect(shortenKeepingExtension(`${"t".repeat(296)}.jpg`, 255)).toBe( + `${"t".repeat(251)}.jpg`, + ); + expect(shortenKeepingExtension("é".repeat(150), 255)).toBe( + "é".repeat(127), + ); + }); + + it("cuts the extension short when it alone does not fit", () => { + expect(shortenKeepingExtension(`b.${"e".repeat(300)}`, 255)).toBe( + `.${"e".repeat(254)}`, + ); + // "€" takes three bytes, so the extension is cut to 253 and the + // part before it keeps the two bytes left over, for the "a". + expect(shortenKeepingExtension(`a.${"€".repeat(90)}`, 255)).toBe( + `a.${"€".repeat(84)}`, + ); + }); + + it("puts `insert` right before the extension and leaves room for it", () => { + expect(shortenKeepingExtension("beach.jpg", 255, " (7)")).toBe( + "beach (7).jpg", + ); + expect( + shortenKeepingExtension(`${"t".repeat(296)}.jpg`, 255, " (7)"), + ).toBe(`${"t".repeat(247)} (7).jpg`); + expect( + shortenKeepingExtension(`b.${"e".repeat(300)}`, 255, " (7)"), + ).toBe(` (7).${"e".repeat(250)}`); + }); +});