diff --git a/README.md b/README.md index 9ada598..14e69c6 100644 --- a/README.md +++ b/README.md @@ -497,6 +497,20 @@ failure still exits non-zero. .json collection metadata + file list ``` +A collection's directory and JSON are named after the collection, and a symlink +after the file's title, both with unsafe characters replaced. When two +collections would get the same name, or two files in one collection the same +title (ignoring case in both), each of them gets its ID added: two albums named +`Trip` become `Trip (10)/` and `Trip (11)/`, and two files titled `IMG_0001.JPG` +become `IMG_0001 (12345).JPG` and `IMG_0001 (12346).JPG`. IDs never change, so a +name stays the same from run to run until such a clash appears or goes away. + +Each run removes the symlinks into `originals/` that no longer belong in their +collection's directory, and the directories (and JSON) of collections that were +deleted or renamed. Nothing else in `collections/` is touched: a file or a +symlink you put there stays, and a directory that still holds one after its +symlinks are removed stays too, with its JSON. + Each file is downloaded exactly once regardless of how many collections it appears in. On subsequent runs, existing originals are skipped. If a download fails, the error is logged and the backup continues with the next file. The exit diff --git a/TODO.md b/TODO.md index 172c0d7..9335d2c 100644 --- a/TODO.md +++ b/TODO.md @@ -18,6 +18,13 @@ Tag v1.0.0. # Completed Steps +- 2026-09-23: Fixed the backup's per-collection folders (issue 103). Two files + in one collection with the same title, and two collections with the same name, + each get their ID added to the name (`IMG_0001 (12345).JPG`, `Trip (10)/`), so + none replaces another's symlink or JSON. Each run removes symlinks into + `originals/` for files no longer in the collection, and the folders of deleted + or renamed collections, leaving anything else in `collections/` alone. The + README backup layout states the naming rule. - 2026-09-23: Checked downloaded originals against their recorded content hash (issue 68). `downloadFile`, which `quak get`, the content cache and backup all use, hashes the decrypted bytes (unkeyed BLAKE2b-512, standard base64) and diff --git a/src/backup.ts b/src/backup.ts index 71565f0..78df900 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -17,7 +17,9 @@ // temp-then-rename, so a file that exists is whole and is never re-fetched — an // interrupted run resumes by listing the directory. The derived views hold no // unique state, so they are rebuilt every run; that repairs stale sidecars and -// missing or broken symlinks left by an earlier crash. +// missing or broken symlinks left by an earlier crash. A rebuild also removes +// the symlinks into originals/ that no longer belong to an album, and the +// directories of albums that no longer exist. // // Resilience (issue #8): no per-file condition aborts the run. A failed // download or a failed symlink is caught, recorded in `failures.json` with a @@ -34,13 +36,14 @@ import { readdirSync, readFileSync, readlinkSync, + rmdirSync, rmSync, statSync, symlinkSync, writeFileSync, } from "node:fs"; import { copyFile, rename, rm } from "node:fs/promises"; -import { basename, dirname, join, relative } from "node:path"; +import { basename, dirname, extname, join, relative } from "node:path"; import { fsyncPath } from "./download/index.js"; import { safeExtension, sanitizeFileName } from "./filename.js"; @@ -227,6 +230,93 @@ const rebuildSymlink = (linkPath: string, target: string): void => { symlinkSync(target, linkPath); }; +// The on-disk names for the entries of one directory, keyed by ID. Each name +// is used as is unless another entry would get the same name, ignoring case +// (two names that differ only in case are one entry on a case-insensitive +// file system); then every entry sharing it gets ` ()`, before the +// extension when `beforeExtension` is set. A name with an ID added can match +// another entry's own name (`IMG (6).JPG`), so this repeats until no name is +// shared. IDs are stable, so the names are too. +const namesByID = ( + entries: { id: number; name: string }[], + beforeExtension: boolean, +): Map => { + const withID = (id: number, name: string): string => { + const ext = beforeExtension ? extname(name) : ""; + const stem = name.slice(0, name.length - ext.length); + return `${stem} (${id})${ext}`; + }; + const names = new Map(); + for (const { id, name } of entries) names.set(id, name); + const suffixed = new Set(); + for (;;) { + const counts = new Map(); + for (const name of names.values()) { + const key = name.toLowerCase(); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + let changed = false; + for (const { id, name } of entries) { + if (suffixed.has(id)) continue; + if (counts.get(name.toLowerCase()) === 1) continue; + names.set(id, withID(id, name)); + suffixed.add(id); + changed = true; + } + if (!changed) return names; + } +}; + +// Remove the symlinks in the album directory `dir` that point into +// `originalsDir` and are not named in `keep`. Nothing else in the directory +// is touched: anything else there was put there by the user. +const removeStaleLinks = ( + dir: string, + keep: Set, + originalsDir: string, +): void => { + const target = relative(dir, originalsDir); + for (const name of readdirSync(dir)) { + if (keep.has(name)) continue; + const path = join(dir, name); + if ( + lstatSync(path).isSymbolicLink() && + dirname(readlinkSync(path)) === target + ) { + rmSync(path); + } + } +}; + +// Remove the directories under `collectionsDir` that an earlier run wrote for +// an album that is gone or renamed: a directory not named in `current` with a +// `.json` beside it holding an album ID, which is what a run writes. Its +// symlinks into originals/ are removed; if that leaves it empty, it and its +// JSON are deleted, otherwise both stay for what the user put there. +const removeStaleAlbumDirs = ( + collectionsDir: string, + current: Set, + originalsDir: string, +): void => { + for (const entry of readdirSync(collectionsDir, { withFileTypes: true })) { + if (!entry.isDirectory() || current.has(entry.name)) continue; + const jsonPath = join(collectionsDir, `${entry.name}.json`); + try { + const album = JSON.parse(readFileSync(jsonPath, "utf-8")) as { + id?: unknown; + }; + if (typeof album.id !== "number") continue; + } catch { + continue; + } + const dir = join(collectionsDir, entry.name); + removeStaleLinks(dir, new Set(), originalsDir); + if (readdirSync(dir).length > 0) continue; + rmdirSync(dir); + rmSync(jsonPath); + } +}; + const loadLedger = (path: string): Map => { const ledger = new Map(); try { @@ -303,9 +393,10 @@ export const runBackup = async ( // Collections in scope, and the distinct files across them (a file shared // by two albums is one original). - const collections = lib - .listCollections() - .filter((c) => (only ? only.has(c.name) : true)); + const allCollections = lib.listCollections(); + const collections = allCollections.filter((c) => + only ? only.has(c.name) : true, + ); const collectionName = new Map(); for (const c of collections) collectionName.set(c.id, c.name); @@ -406,23 +497,55 @@ export const runBackup = async ( } } - // Then the per-collection symlink trees and JSON. + // Then the per-collection symlink trees and JSON. Directory names are + // chosen across every album, not just those in scope, so a scoped run + // names an album the same as a full one and never takes the directory of + // an album it skipped. Stale entries are removed before anything is + // rebuilt, so on a case-insensitive file system removing an old name can + // never remove the new one. + const albumDirNames = namesByID( + allCollections.map((c) => ({ + id: c.id, + name: sanitizeFileName(c.name, `collection-${c.id}`), + })), + false, + ); + try { + removeStaleAlbumDirs( + collectionsDir, + new Set(albumDirNames.values()), + originalsDir, + ); + } catch (err) { + log(`FAILED removing old album directories: ${errorMessage(err)}`); + } + for (const c of collections) { - const colDirName = sanitizeFileName(c.name, `collection-${c.id}`); + const colDirName = albumDirNames.get(c.id)!; const colDir = join(collectionsDir, colDirName); mkdirSync(colDir, { recursive: true }); const files = filesByCollection.get(c.id) ?? []; + const linkNames = namesByID( + files.map((f) => ({ + id: f.id, + name: sanitizeFileName(f.metadata.title, `file-${f.id}`), + })), + true, + ); + try { + removeStaleLinks(colDir, new Set(linkNames.values()), originalsDir); + } catch (err) { + log(`FAILED removing old links in ${c.name}: ${errorMessage(err)}`); + } + const metaFiles: { id: number; metadata: EnteFile["metadata"] }[] = []; for (const file of files) { metaFiles.push({ id: file.id, metadata: file.metadata }); if (!includeOriginals) continue; const orig = join(originalsDir, originalName(file)); if (!isPresent(orig)) continue; - const linkName = sanitizeFileName( - file.metadata.title, - `file-${file.id}`, - ); + const linkName = linkNames.get(file.id)!; const linkPath = join(colDir, linkName); try { rebuildSymlink(linkPath, relative(colDir, orig)); diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 55ac01e..6f4ed3d 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -40,6 +40,7 @@ import { readFileSync, readlinkSync, rmSync, + symlinkSync, writeFileSync, } from "node:fs"; import { spawnSync } from "node:child_process"; @@ -47,6 +48,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { runBackup, type BackupLibrary } from "../../src/backup.js"; import { Library } from "../../src/library/index.js"; import type { ContentSource } from "../../src/library/content.js"; import type { CollectionsPage, FilesPage } from "../../src/client.js"; @@ -622,3 +624,318 @@ describe("lib.backup", () => { lib.close(); }); }); + +// The album folders under collections/, driven through `runBackup` with a +// stand-in library whose albums a test changes between runs. +describe("backup album folders", () => { + interface Album { + collection: Collection; + files: EnteFile[]; + } + + const libraryOf = (albums: Album[]): BackupLibrary => ({ + refresh: async () => {}, + listCollections: () => albums.map((a) => a.collection), + listFiles: (id) => + albums.find((a) => a.collection.id === id)?.files ?? [], + original: async (fileID) => { + const path = join(root, `source-${fileID}`); + writeFileSync(path, `original ${fileID}`); + return { path }; + }, + thumbnail: async () => { + throw new Error("no thumbnails in this stand-in"); + }, + }); + + // Every entry under collections/, one level of directories deep, with each + // symlink's target. + const tree = (outDir: string): string[] => { + const lines: string[] = []; + const list = (dir: string, prefix: string): void => { + for (const name of readdirSync(dir).sort()) { + const path = join(dir, name); + const st = lstatSync(path); + if (st.isSymbolicLink()) { + lines.push(`${prefix}${name} -> ${readlinkSync(path)}`); + } else if (st.isDirectory() && prefix === "") { + lines.push(`${name}/`); + list(path, `${name}/`); + } else { + lines.push(`${prefix}${name}`); + } + } + }; + list(join(outDir, "collections"), ""); + return lines; + }; + + const albumID = (outDir: string, jsonName: string): number => + JSON.parse(readFileSync(join(outDir, "collections", jsonName), "utf-8")) + .id; + + it("gives every file and every album its own name when names repeat", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [ + file(1, 10, "IMG_0001.JPG"), + file(2, 10, "IMG_0001.JPG"), + file(4, 10, "img_0001.jpg"), + file(3, 10, "other.jpg"), + ], + }, + { + collection: collection(11, "Trip"), + files: [file(3, 11, "other.jpg")], + }, + ]); + + const result = await runBackup(lib, { downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Trip (10)/", + "Trip (10)/IMG_0001 (1).JPG -> ../../originals/1.JPG", + "Trip (10)/IMG_0001 (2).JPG -> ../../originals/2.JPG", + "Trip (10)/img_0001 (4).jpg -> ../../originals/4.jpg", + "Trip (10)/other.jpg -> ../../originals/3.jpg", + "Trip (10).json", + "Trip (11)/", + "Trip (11)/other.jpg -> ../../originals/3.jpg", + "Trip (11).json", + ]); + expect(albumID(outDir, "Trip (10).json")).toBe(10); + expect(albumID(outDir, "Trip (11).json")).toBe(11); + }); + + it("keeps names unique when a name with an ID added is another entry's own name", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [ + file(5, 10, "IMG (6).JPG"), + file(6, 10, "IMG.JPG"), + file(7, 10, "IMG.JPG"), + ], + }, + { + collection: collection(11, "Trip"), + files: [file(8, 11, "a.jpg")], + }, + { + collection: collection(12, "Trip (11)"), + files: [file(9, 12, "b.jpg")], + }, + ]); + + const result = await runBackup(lib, { downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Trip (10)/", + "Trip (10)/IMG (6) (5).JPG -> ../../originals/5.JPG", + "Trip (10)/IMG (6).JPG -> ../../originals/6.JPG", + "Trip (10)/IMG (7).JPG -> ../../originals/7.JPG", + "Trip (10).json", + "Trip (11)/", + "Trip (11)/a.jpg -> ../../originals/8.jpg", + "Trip (11) (12)/", + "Trip (11) (12)/b.jpg -> ../../originals/9.jpg", + "Trip (11) (12).json", + "Trip (11).json", + ]); + expect(albumID(outDir, "Trip (10).json")).toBe(10); + expect(albumID(outDir, "Trip (11).json")).toBe(11); + expect(albumID(outDir, "Trip (11) (12).json")).toBe(12); + }); + + it("changes nothing on a second run over an unchanged account", async () => { + const outDir = join(root, "backup"); + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [ + file(1, 10, "IMG_0001.JPG"), + file(2, 10, "IMG_0001.JPG"), + ], + }, + { + collection: collection(11, "Trip"), + files: [file(3, 11, "other.jpg")], + }, + ]); + + await runBackup(lib, { downloadDirectory: outDir }); + const before = tree(outDir); + const second = await runBackup(lib, { downloadDirectory: outDir }); + + expect(second.downloaded).toBe(0); + expect(second.failed).toBe(0); + expect(tree(outDir)).toEqual(before); + }); + + it("leaves the albums an onlyAlbumNames run skips as they were", async () => { + const outDir = join(root, "backup"); + // "trip" is skipped by the scoped run but its name clashes with the + // in-scope "Trip", so "Trip" must keep its ID suffix. + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [file(1, 10, "a.jpg")], + }, + { + collection: collection(11, "trip"), + files: [file(2, 11, "b.jpg")], + }, + { + collection: collection(12, "Work"), + files: [file(3, 12, "c.jpg")], + }, + ]); + const json = (name: string): string => + readFileSync(join(outDir, "collections", name), "utf-8"); + + await runBackup(lib, { downloadDirectory: outDir }); + const before = tree(outDir); + const skippedJSON = [json("trip (11).json"), json("Work.json")]; + const scoped = await runBackup(lib, { + downloadDirectory: outDir, + onlyAlbumNames: ["Trip"], + }); + + expect(scoped.failed).toBe(0); + expect(before).toEqual([ + "Trip (10)/", + "Trip (10)/a.jpg -> ../../originals/1.jpg", + "Trip (10).json", + "Work/", + "Work/c.jpg -> ../../originals/3.jpg", + "Work.json", + "trip (11)/", + "trip (11)/b.jpg -> ../../originals/2.jpg", + "trip (11).json", + ]); + expect(tree(outDir)).toEqual(before); + expect([json("trip (11).json"), json("Work.json")]).toEqual( + skippedJSON, + ); + }); + + it("removes links and album folders that are gone, and nothing the user added", async () => { + const outDir = join(root, "backup"); + const albums: Album[] = [ + { + collection: collection(10, "Trip"), + files: [ + file(1, 10, "IMG_0001.JPG"), + file(2, 10, "IMG_0001.JPG"), + file(3, 10, "other.jpg"), + ], + }, + { + collection: collection(12, "Work"), + files: [file(5, 12, "a.jpg")], + }, + { + collection: collection(13, "Old"), + files: [file(5, 13, "a.jpg")], + }, + ]; + const lib = libraryOf(albums); + await runBackup(lib, { downloadDirectory: outDir }); + + // What the user put in the tree: a note and a symlink of their own in + // an album, a note in an album about to be renamed, and a folder quak + // did not create. + const collectionsDir = join(outDir, "collections"); + writeFileSync(join(collectionsDir, "Trip", "notes.txt"), "mine"); + symlinkSync("../elsewhere", join(collectionsDir, "Trip", "mine")); + writeFileSync(join(collectionsDir, "Work", "keep.txt"), "mine"); + mkdirSync(join(collectionsDir, "Mine")); + writeFileSync(join(collectionsDir, "Mine", "keep.txt"), "mine"); + + // File 2 leaves Trip, Work is renamed Office, Old is deleted. + albums[0]!.files.splice(1, 1); + albums[1]!.collection = collection(12, "Office"); + albums.splice(2, 1); + const result = await runBackup(lib, { downloadDirectory: outDir }); + + expect(result.failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Mine/", + "Mine/keep.txt", + "Office/", + "Office/a.jpg -> ../../originals/5.jpg", + "Office.json", + "Trip/", + "Trip/IMG_0001.JPG -> ../../originals/1.JPG", + "Trip/mine -> ../elsewhere", + "Trip/notes.txt", + "Trip/other.jpg -> ../../originals/3.jpg", + "Trip.json", + "Work/", + "Work/keep.txt", + "Work.json", + ]); + }); + + // One album backed up, then a folder the user made beside it holding a + // symlink into originals/, with `json` (if given) as its sibling JSON. + const backupWithUserFolder = async ( + json: string | undefined, + ): Promise<{ outDir: string; failed: number }> => { + const outDir = join(root, "backup"); + const lib = libraryOf([ + { + collection: collection(10, "Trip"), + files: [file(1, 10, "a.jpg")], + }, + ]); + await runBackup(lib, { downloadDirectory: outDir }); + const collectionsDir = join(outDir, "collections"); + mkdirSync(join(collectionsDir, "Mine")); + symlinkSync( + "../../originals/1.jpg", + join(collectionsDir, "Mine", "a.jpg"), + ); + if (json !== undefined) { + writeFileSync(join(collectionsDir, "Mine.json"), json); + } + const result = await runBackup(lib, { downloadDirectory: outDir }); + return { outDir, failed: result.failed }; + }; + + it("leaves a user folder with no JSON beside it as it was", async () => { + const { outDir, failed } = await backupWithUserFolder(undefined); + + expect(failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Mine/", + "Mine/a.jpg -> ../../originals/1.jpg", + "Trip/", + "Trip/a.jpg -> ../../originals/1.jpg", + "Trip.json", + ]); + }); + + it("leaves a user folder whose JSON has no album ID as it was", async () => { + const json = '{"name":"Mine"}'; + const { outDir, failed } = await backupWithUserFolder(json); + + expect(failed).toBe(0); + expect(tree(outDir)).toEqual([ + "Mine/", + "Mine/a.jpg -> ../../originals/1.jpg", + "Mine.json", + "Trip/", + "Trip/a.jpg -> ../../originals/1.jpg", + "Trip.json", + ]); + expect( + readFileSync(join(outDir, "collections", "Mine.json"), "utf-8"), + ).toBe(json); + }); +});