From 97c4b1185b45ee48e70a8db94a4eb29534ed8369 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 22 Sep 2026 16:55:32 +0000 Subject: [PATCH] Rebuild backup on the library API with a durable failure ledger (closes #51) backup() is a Library method: it refreshes, fetches each pending original (and optional thumbnails) through the content cache and pools, then rebuilds the sidecar, symlink, and per-collection JSON views from the model. The downloadDirectory layout and exit-code contract are unchanged. An original already on disk is complete and never re-fetched, so runs are idempotent and an interrupted run resumes. Per-file download and symlink failures are recorded in a durable failures.json and no longer abort the run, subsuming #8. Each run reconciles the ledger against the files it attempted: an entry survives only for a file that failed this run, so a failure for a since-deleted or out-of-scope file clears instead of failing every future scheduled backup. Model: opus-4-8 --- bin/quak.ts | 11 +- src/backup.ts | 463 +++++++++++++++++++------ src/index.ts | 4 + src/library/index.ts | 47 +++ test/cli/backup.test.ts | 750 ++++++++++++++++++++-------------------- 5 files changed, 793 insertions(+), 482 deletions(-) diff --git a/bin/quak.ts b/bin/quak.ts index 68ab983..6789287 100644 --- a/bin/quak.ts +++ b/bin/quak.ts @@ -8,7 +8,7 @@ import { Command } from "commander"; import envPaths from "env-paths"; import { Client, type ClientSnapshot } from "../src/client.js"; import { init } from "../src/crypto/index.js"; -import { runBackup } from "../src/backup.js"; +import { Library } from "../src/library/index.js"; import { runMetadataBackup } from "../src/metadata-backup.js"; import { listMissingThumbnails, @@ -321,9 +321,14 @@ program const client = requireSession(); stderr.write("Starting backup...\n"); - const result = await runBackup(client, dir, (msg) => { - if (!opts.json) stderr.write(msg + "\n"); + const lib = await Library.open({ client, downloadDirectory: dir }); + const result = await lib.backup({ + downloadDirectory: dir, + onProgress: (msg) => { + if (!opts.json) stderr.write(msg + "\n"); + }, }); + lib.close(); if (opts.json) { stdout.write(JSON.stringify(result, null, 2) + "\n"); diff --git a/src/backup.ts b/src/backup.ts index b6cece3..ef921b0 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -1,13 +1,65 @@ +// The backup command, rebuilt on the library API (issue #51). +// +// `lib.backup()` refreshes the library, then, for every file in scope, gets its +// original bytes onto disk under `downloadDirectory` and rebuilds the derived +// views (per-file sidecars, per-collection symlink trees, per-collection JSON) +// from the model. The on-disk layout is the historical one, unchanged: +// +// / +// originals/. the decrypted bytes +// originals/.json per-file metadata sidecar +// collections// symlink into ../../originals +// collections/<name>.json per-collection metadata +// failures.json durable ledger of unresolved failures +// +// Crash-safety rests on two properties. Bytes are present-means-complete: an +// original appears under `originals/` only via the content layer's atomic +// temp-then-rename, so a file that exists is whole and is never re-fetched — an +// interrupted run resumes by listing the directory. The derived views hold no +// unique state, so they are rebuilt every run; that repairs stale sidecars and +// missing or broken symlinks left by an earlier crash. +// +// Resilience (issue #8): no per-file condition aborts the run. A failed +// download or a failed symlink is caught, recorded in `failures.json` with a +// classification, a running attempt count, and the last-tried time, and the run +// continues. `result.failed` — and thus the CLI's exit code — stays non-zero +// while any failure remains unresolved and clears once every one succeeds. Each +// run reconciles the ledger against the files it attempted, so an entry for a +// file that has since left the library (deleted) or this run's scope is dropped +// rather than counted forever, which would poison a scheduled backup's exit code. + import { - existsSync, + copyFileSync, + lstatSync, mkdirSync, + readFileSync, + readlinkSync, + renameSync, + rmSync, statSync, symlinkSync, writeFileSync, } from "node:fs"; -import { join, relative, extname } from "node:path"; -import type { Client } from "./client.js"; -import type { EnteFile } from "./model/types.js"; +import { basename, dirname, extname, join, relative } from "node:path"; + +import type { Collection, EnteFile } from "./model/types.js"; + +export type ProgressCallback = (message: string) => void; + +export interface BackupOptions { + // Where the backup tree lives. Required: with none, `backup()` throws + // before any network traffic. A library opened with a `downloadDirectory` + // supplies the default. + downloadDirectory?: string; + // Fetch and store full-resolution originals. Default true. + includeOriginals?: boolean; + // Also fetch and store thumbnails under `thumbnails/<fileID>.jpg`. Default + // false. + includeThumbnails?: boolean; + // Restrict the backup to albums with these names; others are left untouched. + onlyAlbumNames?: string[]; + onProgress?: ProgressCallback; +} export interface BackupError { fileID: number; @@ -17,139 +69,354 @@ export interface BackupError { } export interface BackupResult { + // Distinct files in scope this run. totalFiles: number; + // Originals fetched (or copied from the cache) this run. downloaded: number; + // Originals already present and left untouched. skipped: number; + // Files with an unresolved failure after this run (the ledger size); the + // CLI exits non-zero while this is above zero. A file can be both + // downloaded and failed if its bytes landed but its symlink did not. failed: number; + // This run's per-file errors, in encounter order. errors: BackupError[]; } -export type ProgressCallback = (message: string) => void; +// The slice of the library that backup drives. `Library` implements it; a test +// can drive backup with a stand-in. +export interface BackupLibrary { + refresh(): Promise<void>; + listCollections(): Collection[]; + listFiles(collectionID: number): EnteFile[]; + // Get an original's bytes onto disk through the content cache/pools, + // returning where they landed (the cache, or a prior backup). + original(fileID: number): Promise<{ path: string }>; + thumbnail(fileID: number): Promise<{ path: string }>; +} + +type FailureClass = "transient" | "permanent" | "unknown"; + +interface FailureEntry { + fileID: number; + title: string; + classification: FailureClass; + attempts: number; + lastTriedAt: number; + error: string; +} + +const LEDGER_VERSION = 1; const sanitizePath = (name: string): string => name.replace(/[/\\:*?"<>|]/g, "_").replace(/^\.+/, "_"); -const originalFileName = (file: EnteFile): string => { +// The originals/ filename for a file: `<id><ext>`, the extension taken from the +// title (or `.bin`). Matches the content cache's own naming so a present check +// lines up with what a fetch would write. +const originalName = (file: EnteFile): string => { const ext = extname(file.metadata.title || "") || ".bin"; return `${file.id}${ext}`; }; +// A regular file with content is treated as complete. A zero-byte file is not: +// it is the shape an aborted write leaves and must be re-fetched. +const isPresent = (path: string): boolean => { + try { + const s = statSync(path); + return s.isFile() && s.size > 0; + } catch { + return false; + } +}; + +// Best-effort classification for the ledger. Retryable server/network problems +// are transient; refusals and local filesystem/decrypt errors are permanent; +// anything else is unknown. Both the error code and message are inspected. +const classify = (err: unknown): FailureClass => { + const e = err as NodeJS.ErrnoException; + const text = + `${e?.code ?? ""} ${err instanceof Error ? err.message : String(err)}`.toLowerCase(); + if ( + /timeout|timed out|econnreset|econnrefused|econnaborted|network|socket|eai_again|throttl|temporarily|429|500|502|503|504/.test( + text, + ) + ) { + return "transient"; + } + if ( + /enoent|eacces|eperm|eexist|eisdir|enotempty|erofs|enospc|not found|forbidden|unauthor|decrypt|truncat|401|403|404/.test( + text, + ) + ) { + return "permanent"; + } + return "unknown"; +}; + +const errorMessage = (err: unknown): string => + err instanceof Error ? err.message : String(err); + +// Copy bytes into `dest` via a temp file in the same directory plus rename, so +// `dest` appears only once it is whole ("present means complete"). +const copyAtomic = (src: string, dest: string): void => { + if (src === dest) return; + const tmp = join( + dirname(dest), + `.quak-backup-${basename(dest)}-${process.pid}-${Math.random() + .toString(36) + .slice(2)}.tmp`, + ); + try { + copyFileSync(src, tmp); + renameSync(tmp, dest); + } finally { + rmSync(tmp, { force: true }); + } +}; + +// Ensure `linkPath` is a symlink to `target`, rebuilding a missing, wrong, or +// non-symlink entry. Throws on failure (a directory in the way, no permission) +// so the caller records it and moves on rather than aborting the run. +const rebuildSymlink = (linkPath: string, target: string): void => { + try { + const st = lstatSync(linkPath); + if (st.isSymbolicLink() && readlinkSync(linkPath) === target) return; + } catch { + // Nothing there (or unreadable): fall through to create it. + } + // Remove a wrong symlink or stray file. `force` ignores a missing path but + // still refuses a directory (no `recursive`), which surfaces as a failure. + rmSync(linkPath, { force: true }); + symlinkSync(target, linkPath); +}; + +const loadLedger = (path: string): Map<number, FailureEntry> => { + const ledger = new Map<number, FailureEntry>(); + try { + const parsed = JSON.parse(readFileSync(path, "utf-8")) as { + files?: Record<string, FailureEntry>; + }; + for (const entry of Object.values(parsed.files ?? {})) { + if (entry && typeof entry.fileID === "number") { + ledger.set(entry.fileID, entry); + } + } + } catch { + // No ledger yet, or an unreadable one: start clean. + } + return ledger; +}; + +const saveLedger = (path: string, ledger: Map<number, FailureEntry>): void => { + if (ledger.size === 0) { + rmSync(path, { force: true }); + return; + } + const files: Record<string, FailureEntry> = {}; + for (const [fileID, entry] of ledger) files[String(fileID)] = entry; + writeFileSync( + path, + JSON.stringify({ version: LEDGER_VERSION, files }, null, 2), + ); +}; + +const writeSidecar = (path: string, file: EnteFile): void => { + const meta: Record<string, unknown> = { + id: file.id, + collectionID: file.collectionID, + ownerID: file.ownerID, + metadata: file.metadata, + }; + if (file.magicMetadata) meta.magicMetadata = file.magicMetadata; + if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata; + writeFileSync(path, JSON.stringify(meta, null, 2)); +}; + export const runBackup = async ( - client: Client, - outDir: string, - onProgress?: ProgressCallback, + lib: BackupLibrary, + opts: BackupOptions, ): Promise<BackupResult> => { - const log = onProgress ?? (() => {}); + const downloadDirectory = opts.downloadDirectory; + if (!downloadDirectory) { + throw new Error( + "backup requires a downloadDirectory (pass one to backup() or " + + "open the library with one)", + ); + } + const includeOriginals = opts.includeOriginals ?? true; + const includeThumbnails = opts.includeThumbnails ?? false; + const log = opts.onProgress ?? (() => {}); + const only = opts.onlyAlbumNames ? new Set(opts.onlyAlbumNames) : undefined; - mkdirSync(outDir, { recursive: true }); - const originalsDir = join(outDir, "originals"); + log("Refreshing library..."); + await lib.refresh(); + + const originalsDir = join(downloadDirectory, "originals"); + const collectionsDir = join(downloadDirectory, "collections"); + const thumbnailsDir = join(downloadDirectory, "thumbnails"); mkdirSync(originalsDir, { recursive: true }); - const collectionsDir = join(outDir, "collections"); mkdirSync(collectionsDir, { recursive: true }); + if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true }); - log("Fetching collections..."); - const collections = await client.listCollections(); - const downloadedIDs = new Set<number>(); + const ledgerPath = join(downloadDirectory, "failures.json"); + const ledger = loadLedger(ledgerPath); + const now = Date.now(); - let totalFiles = 0; + // Collections in scope, and the distinct files across them (a file shared + // by two albums is one original). + const collections = lib + .listCollections() + .filter((c) => (only ? only.has(c.name) : true)); + const collectionName = new Map<number, string>(); + for (const c of collections) collectionName.set(c.id, c.name); + + const distinct = new Map<number, EnteFile>(); + const filesByCollection = new Map<number, EnteFile[]>(); + for (const c of collections) { + const files = lib.listFiles(c.id); + filesByCollection.set(c.id, files); + for (const f of files) if (!distinct.has(f.id)) distinct.set(f.id, f); + } + + const errors: BackupError[] = []; + const failedThisRun = new Set<number>(); let downloaded = 0; let skipped = 0; - let failed = 0; - const errors: BackupError[] = []; - for (const col of collections) { - const colDirName = sanitizePath(col.name || `collection-${col.id}`); + const recordFailure = ( + file: EnteFile, + collection: string, + err: unknown, + ): void => { + const error = errorMessage(err); + errors.push({ + fileID: file.id, + title: file.metadata.title, + collection, + error, + }); + const prior = ledger.get(file.id); + ledger.set(file.id, { + fileID: file.id, + title: file.metadata.title, + classification: classify(err), + attempts: (prior?.attempts ?? 0) + 1, + lastTriedAt: now, + error, + }); + failedThisRun.add(file.id); + }; + + // Phase 1: get the bytes. Fetch each pending original (and optional + // thumbnail) through the content cache/pools and place it under the backup + // tree; a present file is left as is. + if (includeOriginals) { + for (const [fileID, file] of distinct) { + const dest = join(originalsDir, originalName(file)); + if (isPresent(dest)) { + skipped++; + continue; + } + try { + log(`Fetching original ${file.metadata.title} (${fileID})...`); + const { path } = await lib.original(fileID); + copyAtomic(path, dest); + downloaded++; + } catch (err) { + log( + `FAILED original ${file.metadata.title}: ${errorMessage(err)}`, + ); + recordFailure( + file, + collectionName.get(file.collectionID) ?? "", + err, + ); + } + } + } + + if (includeThumbnails) { + for (const [fileID, file] of distinct) { + const dest = join(thumbnailsDir, `${fileID}.jpg`); + if (isPresent(dest)) continue; + try { + const { path } = await lib.thumbnail(fileID); + copyAtomic(path, dest); + } catch (err) { + recordFailure( + file, + collectionName.get(file.collectionID) ?? "", + err, + ); + } + } + } + + // Phase 2: rebuild the derived views from the model. Sidecars first, for + // every present original (this repairs stale ones). + if (includeOriginals) { + for (const [fileID, file] of distinct) { + const orig = join(originalsDir, originalName(file)); + if (isPresent(orig)) { + writeSidecar(join(originalsDir, `${fileID}.json`), file); + } + } + } + + // Then the per-collection symlink trees and JSON. + for (const c of collections) { + const colDirName = sanitizePath(c.name || `collection-${c.id}`); const colDir = join(collectionsDir, colDirName); mkdirSync(colDir, { recursive: true }); - log(`[${col.name}] Fetching file list...`); - const files = await client.listFiles(col.id, col.key); - log(`[${col.name}] ${files.length} file(s)`); - - const collectionMeta: { - id: number; - name: string; - type: string; - files: { id: number; metadata: EnteFile["metadata"] }[]; - } = { - id: col.id, - name: col.name, - type: col.type, - files: [], - }; - + const files = filesByCollection.get(c.id) ?? []; + const metaFiles: { id: number; metadata: EnteFile["metadata"] }[] = []; for (const file of files) { - totalFiles++; - const origName = originalFileName(file); - const origPath = join(originalsDir, origName); + metaFiles.push({ id: file.id, metadata: file.metadata }); + if (!includeOriginals) continue; + const orig = join(originalsDir, originalName(file)); + if (!isPresent(orig)) continue; const linkName = sanitizePath( file.metadata.title || `file-${file.id}`, ); const linkPath = join(colDir, linkName); - - if (!downloadedIDs.has(file.id)) { - if (existsSync(origPath) && statSync(origPath).size > 0) { - skipped++; - downloadedIDs.add(file.id); - } else { - try { - log(`[${col.name}] Downloading ${linkName}...`); - await client.downloadFile(file, origPath); - downloaded++; - downloadedIDs.add(file.id); - } catch (err) { - log( - `[${col.name}] FAILED ${linkName}: ${err instanceof Error ? err.message : err}`, - ); - failed++; - errors.push({ - fileID: file.id, - title: file.metadata.title, - collection: col.name, - error: - err instanceof Error - ? err.message - : String(err), - }); - continue; - } - } + try { + rebuildSymlink(linkPath, relative(colDir, orig)); + } catch (err) { + log( + `FAILED symlink ${c.name}/${linkName}: ${errorMessage(err)}`, + ); + recordFailure(file, c.name, err); } - - // Write per-file metadata JSON alongside the original - const metaJsonPath = join(originalsDir, `${file.id}.json`); - if (!existsSync(metaJsonPath)) { - const fileMeta: Record<string, unknown> = { - id: file.id, - collectionID: file.collectionID, - ownerID: file.ownerID, - metadata: file.metadata, - }; - if (file.magicMetadata) { - fileMeta.magicMetadata = file.magicMetadata; - } - if (file.pubMagicMetadata) { - fileMeta.pubMagicMetadata = file.pubMagicMetadata; - } - writeFileSync(metaJsonPath, JSON.stringify(fileMeta, null, 2)); - } - - if (!existsSync(linkPath) && existsSync(origPath)) { - const target = relative(colDir, origPath); - symlinkSync(target, linkPath); - } - - collectionMeta.files.push({ - id: file.id, - metadata: file.metadata, - }); } writeFileSync( join(collectionsDir, `${colDirName}.json`), - JSON.stringify(collectionMeta, null, 2), + JSON.stringify( + { id: c.id, name: c.name, type: c.type, files: metaFiles }, + null, + 2, + ), ); } - return { totalFiles, downloaded, skipped, failed, errors }; + // Reconcile the ledger against what this run actually attempted: an entry + // survives only for a file that failed this run. A file that succeeded had + // its failure resolved; a file gone from the library (deleted) or outside + // this run's scope is not something this run can resolve, so keeping its + // stale entry would keep the exit code non-zero forever — a single + // since-deleted photo would fail every future scheduled backup. + for (const fileID of [...ledger.keys()]) { + if (!failedThisRun.has(fileID)) ledger.delete(fileID); + } + saveLedger(ledgerPath, ledger); + + return { + totalFiles: distinct.size, + downloaded, + skipped, + failed: ledger.size, + errors, + }; }; diff --git a/src/index.ts b/src/index.ts index 4bb1439..438e60e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -59,6 +59,10 @@ export { type EnsureOptions, type EnsureResult, type EnsureEvent, + runBackup, + type BackupOptions, + type BackupResult, + type BackupError, } from "./library/index.js"; export { RequestPools, diff --git a/src/library/index.ts b/src/library/index.ts index 04e3fcd..911bf88 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -74,6 +74,14 @@ export { import type { CollectionsPage, FilesPage } from "../client.js"; import { MLDATA_BATCH_SIZE, type MLData } from "../mldata-fetch.js"; import type { Collection, EnteFile } from "../model/types.js"; +import { runBackup, type BackupOptions, type BackupResult } from "../backup.js"; + +export { + runBackup, + type BackupOptions, + type BackupResult, + type BackupError, +} from "../backup.js"; export const DEFAULT_REFRESH_INTERVAL_SECONDS = 3; @@ -378,6 +386,45 @@ export class Library { }; } + // Back up every in-scope file to `downloadDirectory` in the historical + // on-disk layout, with a durable failure ledger (issue #51). Refreshes + // first, fetches pending originals (and optional thumbnails) through the + // content cache and pools, then rebuilds the derived symlink/JSON views + // from the model. Throws before any network work when no download directory + // is available or no content cache backs the originals it must fetch. + backup(opts?: BackupOptions): Promise<BackupResult> { + const downloadDirectory = + opts?.downloadDirectory ?? this.downloadDirectory; + const includeOriginals = opts?.includeOriginals ?? true; + const includeThumbnails = opts?.includeThumbnails ?? false; + if (!downloadDirectory) { + return Promise.reject( + new Error( + "backup requires a downloadDirectory (pass one to " + + "backup() or open the library with one)", + ), + ); + } + if ((includeOriginals || includeThumbnails) && !this.cache) { + return Promise.reject( + new Error( + "backup requires a library opened with a content cache", + ), + ); + } + const cache = this.cache; + return runBackup( + { + refresh: () => this.runRefresh(), + listCollections: () => this.store.listCollections(), + listFiles: (id) => this.store.listFiles(id), + original: (fileID) => cache!.original(fileID), + thumbnail: (fileID) => cache!.thumbnail(fileID), + }, + { ...opts, downloadDirectory }, + ); + } + // Stop the background timer. Idempotent. An in-flight refresh is left to // finish; it will not schedule another cycle once closed. close(): void { diff --git a/test/cli/backup.test.ts b/test/cli/backup.test.ts index 5f2efab..cc5eeb5 100644 --- a/test/cli/backup.test.ts +++ b/test/cli/backup.test.ts @@ -1,368 +1,198 @@ /** - * Tests for the `quak backup` command's core logic. + * Tests for the `quak backup` logic, now built on the library API (issue #51). * - * `quak backup <dir>` downloads every file from every collection into - * a local directory tree: + * `lib.backup({ downloadDirectory })` refreshes the library, fetches each + * pending file's original through the content cache/pools, and materialises the + * unchanged on-disk layout: * - * <dir>/ - * <collection-name>/ - * <file-title> - * <file-title> - * <collection-name>/ - * ... - * metadata.json (all decrypted collection + file metadata) + * <downloadDirectory>/ + * originals/ + * <fileID>.<ext> the decrypted bytes ("present means complete") + * <fileID>.json per-file metadata sidecar (rebuilt each run) + * collections/ + * <name>/<title> symlink into ../originals (rebuilt each run) + * <name>.json per-collection metadata (rebuilt each run) + * failures.json durable ledger of unresolved failures * - * The backup command has two properties that distinguish it from a naive - * "download everything" loop: + * The properties that distinguish backup from a naive download loop, and that + * these tests lock down: * - * 1. **Skip existing files.** If `<dir>/<collection>/<title>` already - * exists on disk and its size matches the decrypted content length - * recorded in metadata.json from a prior run, the file is not - * re-downloaded. This makes interrupted backups resumable and - * incremental runs fast. + * 1. Present-means-complete: an original already on disk is not re-fetched, so + * runs are idempotent and interrupted runs resume. + * 2. Per-file resilience: a download failure or a symlink failure is recorded + * and the run continues (issue #8); the derived symlink/JSON views are + * rebuilt from the model every run. + * 3. A durable `failures.json` records each unresolved failure's classification, + * attempt count, and last-tried time; the exit code (result.failed) is + * non-zero while any failure remains and clears once every one is resolved. * - * 2. **Never crash on a single file failure.** If a file download or - * decryption fails, the error is logged and the backup continues - * with the next file. At the end, the exit code is non-zero if any - * files failed, and the summary lists them. The Ente first-party - * CLI crashes entirely when a single file can't be retrieved, - * which defeats the purpose of a backup tool. - * - * These tests exercise the backup logic (in src/backup.ts) using the - * same mock server from the Client usage tests. The CLI binary itself - * is a thin wrapper around this module. + * The cache and download layers are covered elsewhere (content.test.ts, + * download tests); here a mock library client and a stand-in content source + * drive the backup logic with no crypto and no network. */ import { existsSync, lstatSync, + mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, + writeFileSync, } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import sodium from "libsodium-wrappers-sumo"; -import { SRP, SrpServer } from "fast-srp-hap"; -import { beforeAll, afterAll, describe, expect, it } from "vitest"; -import { - init, - toBase64, - deriveKEK, - deriveLoginSubkey, -} from "../../src/crypto/index.js"; -import { Client } from "../../src/client.js"; -import { runBackup } from "../../src/backup.js"; -import type { KeyAttributes } from "../../src/auth/types.js"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; -// --------------------------------------------------------------------------- -// Mock server (condensed from usage.test.ts) -// --------------------------------------------------------------------------- +import { Library } from "../../src/library/index.js"; +import type { ContentSource } from "../../src/library/content.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; -const TEST_EMAIL = "backup@example.com"; -const TEST_PASSWORD = "backuppass"; -const TEST_OPS = 2; -const TEST_MEM = 64 * 1024 * 1024; +const USER_ID = 42; -interface MockState { - verifier: Buffer; - srpAttributes: Record<string, unknown>; - keyAttributes: KeyAttributes; - encryptedToken: string; - collections: Record<string, unknown>[]; - files: Record< - number, - { raw: Record<string, unknown>; plaintext: Uint8Array } - >; +// Decrypted-byte length each stub original writes, keyed by fileID. +const SIZE_BY_ID: Record<number, number> = { 100: 3000, 101: 2000, 200: 1500 }; + +const collection = (id: number, name: string): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + name, + type: "album", + updationTime: 1, + isShared: false, +}); + +const file = (id: number, collectionID: number, title: string): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id & 0xff]), + metadata: { + title, + fileType: "image", + creationTime: 1, + modificationTime: 1, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 1, +}); + +// A metadata-only client: two albums, three files, served once. No ML. +class MockClient { + private served = false; + whoami(): { email: string; userID: number } { + return { email: "backup@example.com", userID: USER_ID }; + } + async collectionsSince(): Promise<CollectionsPage> { + if (this.served) return { collections: [], deleted: [], cursor: 1 }; + this.served = true; + return { + collections: [collection(1, "Vacation"), collection(2, "Work")], + deleted: [], + cursor: 1, + }; + } + async filesSince(args: { collectionID: number }): Promise<FilesPage> { + const files = + args.collectionID === 1 + ? [file(100, 1, "beach.jpg"), file(101, 1, "sunset.jpg")] + : args.collectionID === 2 + ? [file(200, 2, "diagram.png")] + : []; + return { files, deleted: [], cursor: 1 }; + } } -let mock: MockState; -let testDir: string; +// A content source that writes byte buffers of the expected length and can be +// told to fail one fileID's original, to exercise per-file resilience. +interface StubSource extends ContentSource { + failID?: number; + originalCalls: number; +} -const buildMock = async (): Promise<MockState> => { - const kekSalt = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES); - const kek = await deriveKEK(TEST_PASSWORD, kekSalt, TEST_OPS, TEST_MEM); - const loginSubKeyBytes = deriveLoginSubkey(kek); - - const srpUserID = "backup-srp"; - const srpSalt = sodium.randombytes_buf(16); - const verifier = SRP.computeVerifier( - SRP.params["4096"], - Buffer.from(srpSalt), - Buffer.from(srpUserID), - Buffer.from(loginSubKeyBytes), - ); - - const masterKey = sodium.randombytes_buf(32); - const keyNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encryptedKey = sodium.crypto_secretbox_easy(masterKey, keyNonce, kek); - const kp = sodium.crypto_box_keypair(); - const skNonce = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encSK = sodium.crypto_secretbox_easy( - kp.privateKey, - skNonce, - masterKey, - ); - const tokenBytes = sodium.randombytes_buf(32); - const encToken = sodium.crypto_box_seal(tokenBytes, kp.publicKey); - - const keyAttributes: KeyAttributes = { - kekSalt: toBase64(kekSalt), - encryptedKey: toBase64(encryptedKey), - keyDecryptionNonce: toBase64(keyNonce), - publicKey: toBase64(kp.publicKey), - encryptedSecretKey: toBase64(encSK), - secretKeyDecryptionNonce: toBase64(skNonce), - memLimit: TEST_MEM, - opsLimit: TEST_OPS, - }; - - const makeCollection = (id: number, name: string) => { - const ck = sodium.crypto_secretbox_keygen(); - const ckN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encCK = sodium.crypto_secretbox_easy(ck, ckN, masterKey); - const nameBytes = new TextEncoder().encode(name); - const cnN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encCN = sodium.crypto_secretbox_easy(nameBytes, cnN, ck); - return { - raw: { - id, - owner: { id: 42 }, - encryptedKey: toBase64(encCK), - keyDecryptionNonce: toBase64(ckN), - encryptedName: toBase64(encCN), - nameDecryptionNonce: toBase64(cnN), - type: "album", - updationTime: 1700000000000000, - }, - key: ck, - }; - }; - - const makeFile = ( - id: number, - collKey: Uint8Array, - title: string, - plaintext: Uint8Array, - collID: number, - ) => { - const fk = sodium.crypto_secretstream_xchacha20poly1305_keygen(); - const fkN = sodium.randombytes_buf(sodium.crypto_secretbox_NONCEBYTES); - const encFK = sodium.crypto_secretbox_easy(fk, fkN, collKey); - - const meta = JSON.stringify({ - title, - fileType: 0, - creationTime: 1700000000000000, - modificationTime: 1700000000000000, - }); - const metaPush = - sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); - const encMeta = sodium.crypto_secretstream_xchacha20poly1305_push( - metaPush.state, - new TextEncoder().encode(meta), - null, - sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, - ); - - const filePush = - sodium.crypto_secretstream_xchacha20poly1305_init_push(fk); - const encFile = sodium.crypto_secretstream_xchacha20poly1305_push( - filePush.state, - plaintext, - null, - sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL, - ); - - return { - raw: { - id, - collectionID: collID, - ownerID: 42, - encryptedKey: toBase64(encFK), - keyDecryptionNonce: toBase64(fkN), - metadata: { - encryptedData: toBase64(encMeta), - decryptionHeader: toBase64(metaPush.header), - }, - file: { decryptionHeader: toBase64(filePush.header) }, - thumbnail: { - decryptionHeader: toBase64(sodium.randombytes_buf(24)), - }, - updationTime: 1700000000000000, - }, - plaintext, - ciphertext: encFile, - }; - }; - - const col1 = makeCollection(1, "Vacation"); - const col2 = makeCollection(2, "Work"); - const file1 = makeFile( - 100, - col1.key, - "beach.jpg", - sodium.randombytes_buf(3000), - 1, - ); - const file2 = makeFile( - 101, - col1.key, - "sunset.jpg", - sodium.randombytes_buf(2000), - 1, - ); - const file3 = makeFile( - 200, - col2.key, - "diagram.png", - sodium.randombytes_buf(1500), - 2, - ); - - return { - verifier, - srpAttributes: { - srpUserID, - srpSalt: toBase64(srpSalt), - memLimit: TEST_MEM, - opsLimit: TEST_OPS, - kekSalt: toBase64(kekSalt), - isEmailMFAEnabled: false, +const stubSource = (): StubSource => { + const s: StubSource = { + originalCalls: 0, + original: async ({ file: f, destination }) => { + s.originalCalls++; + if (s.failID === f.id) throw new Error("HTTP 500 from server"); + const size = SIZE_BY_ID[f.id] ?? 10; + writeFileSync(destination, Buffer.alloc(size)); + return { bytesWritten: size }; + }, + thumbnail: async ({ destination }) => { + writeFileSync(destination, Buffer.alloc(5)); + return { bytesWritten: 5 }; }, - keyAttributes, - encryptedToken: toBase64(encToken), - collections: [col1.raw, col2.raw], - files: { - 1: { - raw: [file1.raw, file2.raw], - ciphertexts: { 100: file1.ciphertext, 101: file2.ciphertext }, - }, - 2: { raw: [file3.raw], ciphertexts: { 200: file3.ciphertext } }, - 100: { plaintext: file1.plaintext }, - 101: { plaintext: file2.plaintext }, - 200: { plaintext: file3.plaintext }, - } as Record<number, unknown>, }; + return s; }; -const buildMockFetch = (m: MockState, opts?: { failFileID?: number }) => { - let srpServer: SrpServer; +let root: string; - return (async ( - input: RequestInfo | URL, - init?: RequestInit, - ): Promise<Response> => { - const url = - typeof input === "string" - ? input - : input instanceof URL - ? input.href - : input.url; - const parsed = new URL(url); - const path = parsed.pathname; - const json = (body: unknown) => - new Response(JSON.stringify(body), { - status: 200, - headers: { "content-type": "application/json" }, - }); +const openLibrary = (source: ContentSource): Promise<Library> => + Library.open({ + client: new MockClient(), + cacheDirectory: join(root, "cache"), + contentSource: source, + refreshIntervalSeconds: 3600, + }); - if (path === "/users/srp/attributes") - return json({ attributes: m.srpAttributes }); +const readLedger = ( + outDir: string, +): { files: Record<string, Record<string, unknown>> } => + JSON.parse(readFileSync(join(outDir, "failures.json"), "utf-8")); - if (path === "/users/srp/create-session") { - const body = JSON.parse(init?.body as string); - const serverKey = await SRP.genKey(); - srpServer = new SrpServer( - SRP.params["4096"], - m.verifier, - serverKey, - ); - const B = srpServer.computeB(); - srpServer.setA(Buffer.from(body.srpA, "base64")); - return json({ sessionID: "s1", srpB: B.toString("base64") }); - } - - if (path === "/users/srp/verify-session") { - const body = JSON.parse(init?.body as string); - srpServer.checkM1(Buffer.from(body.srpM1, "base64")); - return json({ - srpM2: srpServer.computeM2().toString("base64"), - id: 42, - keyAttributes: m.keyAttributes, - encryptedToken: m.encryptedToken, - }); - } - - if (path === "/collections/v2") - return json({ collections: m.collections }); - - if (path === "/collections/v2/diff") { - const collID = Number(parsed.searchParams.get("collectionID")); - const collData = m.files[collID] as { raw: unknown[] } | undefined; - return json({ diff: collData?.raw ?? [], hasMore: false }); - } - - // File download - if (url.includes("fileID=") || path.startsWith("/files/download/")) { - const fileID = Number( - parsed.searchParams.get("fileID") ?? path.split("/").pop(), - ); - if (opts?.failFileID === fileID) { - return new Response("Internal Server Error", { status: 500 }); - } - const collData = Object.values(m.files).find( - (v: unknown) => - v && - typeof v === "object" && - "ciphertexts" in (v as Record<string, unknown>) && - fileID in - (v as Record<string, Record<number, unknown>>) - .ciphertexts, - ) as { ciphertexts: Record<number, Uint8Array> } | undefined; - if (collData) { - return new Response(collData.ciphertexts[fileID], { - status: 200, - }); - } - return new Response("not found", { status: 404 }); - } - - return new Response("not found", { status: 404 }); - }) as typeof globalThis.fetch; +// Write a durable ledger holding one prior failure, to exercise pruning of +// entries the current run cannot resolve. +const seedLedger = (outDir: string, fileID: number, title: string): void => { + mkdirSync(outDir, { recursive: true }); + writeFileSync( + join(outDir, "failures.json"), + JSON.stringify({ + version: 1, + files: { + [String(fileID)]: { + fileID, + title, + classification: "transient", + attempts: 1, + lastTriedAt: Date.now(), + error: "HTTP 500 from server", + }, + }, + }), + ); }; -// --------------------------------------------------------------------------- -// Setup / teardown -// --------------------------------------------------------------------------- - -beforeAll(async () => { - await init(); - await sodium.ready; - mock = await buildMock(); - testDir = mkdtempSync(join(tmpdir(), "quak-backup-test-")); +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-backup-test-")); }); -afterAll(() => { - if (testDir && existsSync(testDir)) - rmSync(testDir, { recursive: true, force: true }); +afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); }); -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- +describe("lib.backup", () => { + it("throws before any network when no downloadDirectory is given", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + await expect(lib.backup()).rejects.toThrow(/downloadDirectory/i); + expect(source.originalCalls).toBe(0); + lib.close(); + }); -describe("quak backup", () => { - it("downloads all files organized by collection name", async () => { - const outDir = join(testDir, "full-backup"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMockFetch(mock) }, - }); + it("writes the expected on-disk layout for every file", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); - const result = await runBackup(client, outDir); + const result = await lib.backup({ downloadDirectory: outDir }); expect(result.totalFiles).toBe(3); expect(result.downloaded).toBe(3); @@ -370,7 +200,7 @@ describe("quak backup", () => { expect(result.failed).toBe(0); expect(result.errors).toEqual([]); - // Originals are under <outDir>/originals/<fileID>.<ext> + // Originals under originals/<fileID>.<ext>. expect(readFileSync(join(outDir, "originals", "100.jpg")).length).toBe( 3000, ); @@ -381,55 +211,57 @@ describe("quak backup", () => { 1500, ); - // Collection dirs under collections/ contain symlinks to originals - const beachLink = join(outDir, "collections", "Vacation", "beach.jpg"); - expect(lstatSync(beachLink).isSymbolicLink()).toBe(true); - expect(readlinkSync(beachLink)).toContain("originals"); - expect(readFileSync(beachLink).length).toBe(3000); + // Per-file metadata sidecar. + const sidecar = JSON.parse( + readFileSync(join(outDir, "originals", "100.json"), "utf-8"), + ); + expect(sidecar.id).toBe(100); + expect(sidecar.metadata.title).toBe("beach.jpg"); + + // Collection dirs contain symlinks into ../originals. + const beach = join(outDir, "collections", "Vacation", "beach.jpg"); + expect(lstatSync(beach).isSymbolicLink()).toBe(true); + expect(readlinkSync(beach)).toContain("originals"); + expect(readFileSync(beach).length).toBe(3000); + + // Per-collection metadata JSON. + const vacation = JSON.parse( + readFileSync(join(outDir, "collections", "Vacation.json"), "utf-8"), + ); + expect(vacation.name).toBe("Vacation"); + expect(vacation.files.length).toBe(2); + expect(vacation.files[0].metadata.title).toBeDefined(); + + // A clean run leaves no failure ledger behind. + expect(existsSync(join(outDir, "failures.json"))).toBe(false); + lib.close(); }); - it("skips files that already exist on disk with matching size", async () => { - const outDir = join(testDir, "incremental"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMockFetch(mock) }, - }); + it("is an idempotent no-op when every original is already present", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); - // First run: download everything - const first = await runBackup(client, outDir); + const first = await lib.backup({ downloadDirectory: outDir }); expect(first.downloaded).toBe(3); + const callsAfterFirst = source.originalCalls; - // Second run: everything should be skipped - const second = await runBackup(client, outDir); + const second = await lib.backup({ downloadDirectory: outDir }); expect(second.downloaded).toBe(0); expect(second.skipped).toBe(3); expect(second.failed).toBe(0); + // A present original is neither fetched nor copied again. + expect(source.originalCalls).toBe(callsAfterFirst); + lib.close(); }); - it("continues after a single file download failure", async () => { - // File 101 (sunset.jpg) will return HTTP 500. The other two - // files must still download. The result must report the failure - // without throwing. - // - // A 500 is retryable, so this file now costs several requests before - // it is given up on — that is the point of the retry policy, and - // `runBackup`'s own resilience is unchanged by it: the retry lives - // strictly below this loop, and an exhausted file is still logged, - // counted, and stepped over rather than aborting the run. The - // injected `sleep` is what keeps the suite from actually waiting out - // the backoff. - const outDir = join(testDir, "partial-failure"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { - fetch: buildMockFetch(mock, { failFileID: 101 }), - retry: { sleep: () => Promise.resolve(), random: () => 0 }, - }, - }); + it("continues past a download failure and records it in failures.json", async () => { + const source = stubSource(); + source.failID = 101; + const lib = await openLibrary(source); + const outDir = join(root, "backup"); - const result = await runBackup(client, outDir); + const result = await lib.backup({ downloadDirectory: outDir }); expect(result.totalFiles).toBe(3); expect(result.downloaded).toBe(2); @@ -438,32 +270,188 @@ describe("quak backup", () => { expect(result.errors[0]!.fileID).toBe(101); expect(result.errors[0]!.title).toBe("sunset.jpg"); - // The two successful originals are on disk + // The two good files are on disk; the failed one is not. expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true); expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); - // The failed file has no original and no symlink expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(false); expect( existsSync(join(outDir, "collections", "Vacation", "sunset.jpg")), ).toBe(false); + + // Durable ledger with classification, attempts, last-tried. + const ledger = readLedger(outDir); + const entry = ledger.files["101"]!; + expect(entry.attempts).toBe(1); + expect(entry.classification).toBeDefined(); + expect(typeof entry.lastTriedAt).toBe("number"); + lib.close(); }); - it("writes per-collection JSON metadata", async () => { - const outDir = join(testDir, "metadata-check"); - const client = await Client.login({ - email: TEST_EMAIL, - password: TEST_PASSWORD, - apiOptions: { fetch: buildMockFetch(mock) }, + it("increments the attempt count across runs and clears the ledger once resolved", async () => { + const source = stubSource(); + source.failID = 101; + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + const r1 = await lib.backup({ downloadDirectory: outDir }); + expect(r1.failed).toBe(1); + expect(readLedger(outDir).files["101"]!.attempts).toBe(1); + + // Second run: the two good files are present, only 101 is retried. + const r2 = await lib.backup({ downloadDirectory: outDir }); + expect(r2.failed).toBe(1); + expect(r2.skipped).toBe(2); + expect(readLedger(outDir).files["101"]!.attempts).toBe(2); + + // Resume with a healthy source: 101 downloads, the rest are skipped. + source.failID = undefined; + const r3 = await lib.backup({ downloadDirectory: outDir }); + expect(r3.failed).toBe(0); + expect(r3.skipped).toBe(2); + expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(true); + // A ledger with no remaining failures is removed. + expect(existsSync(join(outDir, "failures.json"))).toBe(false); + lib.close(); + }); + + it("does not abort when a symlink cannot be created (issue #8)", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + // Occupy beach.jpg's symlink path with a directory so symlink creation + // fails for that one file. + mkdirSync(join(outDir, "collections", "Vacation", "beach.jpg"), { + recursive: true, }); - await runBackup(client, outDir); + const result = await lib.backup({ downloadDirectory: outDir }); - // Each collection gets a <name>.json next to its image dir - const vacationMeta = join(outDir, "collections", "Vacation.json"); - expect(existsSync(vacationMeta)).toBe(true); - const meta = JSON.parse(readFileSync(vacationMeta, "utf-8")); - expect(meta.name).toBe("Vacation"); - expect(meta.files.length).toBeGreaterThan(0); - expect(meta.files[0].metadata.title).toBeDefined(); + // Every original still downloads despite the symlink failure. + expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true); + expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); + // The other symlinks are still built. + expect( + lstatSync( + join(outDir, "collections", "Vacation", "sunset.jpg"), + ).isSymbolicLink(), + ).toBe(true); + expect( + lstatSync( + join(outDir, "collections", "Work", "diagram.png"), + ).isSymbolicLink(), + ).toBe(true); + + // The symlink failure is recorded, not thrown. + expect(result.failed).toBeGreaterThanOrEqual(1); + const err = result.errors.find((e) => e.fileID === 100); + expect(err).toBeDefined(); + expect(err!.collection).toBe("Vacation"); + expect(readLedger(outDir).files["100"]).toBeDefined(); + lib.close(); + }); + + it("rebuilds a stale sidecar and a missing symlink on a later run", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + await lib.backup({ downloadDirectory: outDir }); + + // Corrupt a sidecar and delete a symlink between runs. + writeFileSync(join(outDir, "originals", "100.json"), "not json"); + rmSync(join(outDir, "collections", "Vacation", "beach.jpg")); + + const result = await lib.backup({ downloadDirectory: outDir }); + expect(result.failed).toBe(0); + + // The derived views are repaired from the model. + const sidecar = JSON.parse( + readFileSync(join(outDir, "originals", "100.json"), "utf-8"), + ); + expect(sidecar.metadata.title).toBe("beach.jpg"); + expect( + lstatSync( + join(outDir, "collections", "Vacation", "beach.jpg"), + ).isSymbolicLink(), + ).toBe(true); + lib.close(); + }); + + it("backs up only the named albums when onlyAlbumNames is given", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + const result = await lib.backup({ + downloadDirectory: outDir, + onlyAlbumNames: ["Work"], + }); + + expect(result.totalFiles).toBe(1); + expect(result.downloaded).toBe(1); + expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); + expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(false); + expect(existsSync(join(outDir, "collections", "Work.json"))).toBe(true); + expect(existsSync(join(outDir, "collections", "Vacation.json"))).toBe( + false, + ); + lib.close(); + }); + + it("prunes a ledger entry for a file no longer in the library and exits zero", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + // A prior failure for a file that has since left the library (deleted + // from the account). This run has no way to resolve it, so it must not + // keep the exit code non-zero forever. + seedLedger(outDir, 999, "gone.jpg"); + + const result = await lib.backup({ downloadDirectory: outDir }); + + // Everything still present is backed up cleanly, and the stale entry is + // dropped rather than counted. + expect(result.downloaded).toBe(3); + expect(result.failed).toBe(0); + expect(existsSync(join(outDir, "failures.json"))).toBe(false); + lib.close(); + }); + + it("prunes an out-of-scope ledger entry on a scoped run and exits zero", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + // A prior failure for a Vacation file; this run is scoped to Work and + // never attempts it, so it must not poison the scoped run's exit code. + seedLedger(outDir, 100, "beach.jpg"); + + const result = await lib.backup({ + downloadDirectory: outDir, + onlyAlbumNames: ["Work"], + }); + + expect(result.totalFiles).toBe(1); + expect(result.failed).toBe(0); + expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true); + expect(existsSync(join(outDir, "failures.json"))).toBe(false); + lib.close(); + }); + + it("also stores thumbnails when includeThumbnails is set", async () => { + const source = stubSource(); + const lib = await openLibrary(source); + const outDir = join(root, "backup"); + + await lib.backup({ + downloadDirectory: outDir, + includeThumbnails: true, + }); + + expect(existsSync(join(outDir, "thumbnails", "100.jpg"))).toBe(true); + expect(existsSync(join(outDir, "thumbnails", "200.jpg"))).toBe(true); + lib.close(); }); });