From 9b07e8d7b10f6ecb47d6b79d81a5328f7563bb65 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 14 Oct 2026 03:12:21 +0000 Subject: [PATCH 1/4] Test a backup whose lock is lost while it runs The fake Ente server holds the download of the first file open while the test removes backup.lock, or puts another backup's lock in its place, and fires proper-lockfile's timer on a faked clock. quak backup must then stop before its next file and print one line, and lib.backup() must reject with the reason. A backup whose lock stays must still run to the end. These fail until the lock handles being lost. Model: opus-5-5 --- test/cli/backup-end-to-end.test.ts | 141 ++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 2 deletions(-) diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index 123c00a..c69884a 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -9,7 +9,8 @@ * data this file encrypts with libsodium, as Ente's clients encrypt it. So * quak's own code fetches each original, decrypts it, checks it against the * hash its uploader recorded and unpacks a live photo, and quak's own retries - * repeat a failed request. + * repeat a failed request. One test runs `lib.backup()` instead, as a program + * using quak as a library does. * * The account, user 42, has three albums: * @@ -24,6 +25,7 @@ import { existsSync, lstatSync, + mkdirSync, mkdtempSync, readdirSync, readFileSync, @@ -36,7 +38,16 @@ import { join } from "node:path"; import { Writable } from "node:stream"; import { gzipSync } from "node:zlib"; import sodium from "libsodium-wrappers-sumo"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; import { backupCommand, @@ -47,6 +58,7 @@ import { run } from "../../src/cli-run.js"; import { loadSession } from "../../src/cli-session.js"; import { Client } from "../../src/client.js"; import { init, toBase64 } from "../../src/crypto/index.js"; +import { Library } from "../../src/library/index.js"; import { UNATTENDED_RETRY_OPTIONS } from "../../src/retry.js"; import { buildAccount, @@ -633,3 +645,128 @@ describe("quak backup", () => { expect(stderr).toContain(" Verified: 2\n Mismatched: 1\n"); }); }); + +// `backup.lock` is a directory that proper-lockfile touches every 5 seconds, +// from a timer, while the backup runs. Each time, it first looks at the lock +// on disk. The lock is lost when it is gone, or when another backup has made +// its own in its place, as one does with a lock nobody touched for 10 +// seconds. Another backup may then be writing to the directory, so the backup +// stops before it starts another file. +// +// The fake server holds the download of beach.jpg, the first file backed up, +// open until the test lets it go, so the lock can be lost while the backup +// runs. Only `setTimeout` and `clearTimeout` are faked, so a test can fire +// proper-lockfile's timer without waiting 5 seconds. +describe("a backup whose lock is lost", () => { + let dir: string; + let lock: string; + let downloads: number[]; + let beachRequested: Promise; + let releaseBeach: () => void; + let fetch: typeof globalThis.fetch; + + beforeEach(() => { + dir = mkdtempSync(join(root, "backup-")); + lock = join(dir, "backup.lock"); + downloads = []; + let requested!: () => void; + beachRequested = new Promise((resolve) => { + requested = resolve; + }); + const released = new Promise((resolve) => { + releaseBeach = resolve; + }); + const recording = recordingDownloads(downloads); + fetch = async (input, init) => { + if (downloadOf(input) === BEACH.id) { + requested(); + await released; + } + return recording(input, init); + }; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + // Fire proper-lockfile's timer, and wait for the look at the lock on disk + // that it starts to finish: Node lists a file system call in progress as + // an `FSReqCallback`. + const fireLockTimer = async (): Promise => { + vi.advanceTimersByTime(5_000); + while (process.getActiveResourcesInfo().includes("FSReqCallback")) { + await new Promise((resolve) => setImmediate(resolve)); + } + }; + + // beach.jpg, whose download was under way, is stored; no other file is + // started. `quak backup` prints why as one line, with no stack trace, and + // exits 1. + it("stops quak backup before its next file when backup.lock is removed, and exits 1", async () => { + const result = backup(dir, {}, fetch); + await beachRequested; + rmSync(lock, { recursive: true }); + await fireLockTimer(); + releaseBeach(); + const { code, stderr } = await result; + + expect(code).toBe(1); + expect(stderr).toBe( + "Starting backup...\n" + + "Refreshing library...\n" + + "Fetching original beach.jpg (100)...\n" + + `quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lock}'\n`, + ); + expect(downloads).toEqual([BEACH.id]); + expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual( + Buffer.from(JPEG_WITH_EXIF), + ); + }); + + // Nothing touches the lock but quak, which finds it as it left it each + // time it looks, so the backup runs to the end and removes its lock. + it("lets quak backup run to the end while its lock stays", async () => { + const result = backup(dir, {}, fetch); + await beachRequested; + await fireLockTimer(); + await fireLockTimer(); + releaseBeach(); + const { code } = await result; + + expect(code).toBe(0); + expect(downloads.sort()).toEqual([100, 200, 300]); + expect(existsSync(lock)).toBe(false); + }); + + // A program using quak as a library sees the same: `lib.backup()` rejects + // with the reason. Here another backup has made its own lock in place of + // quak's, and that lock is left where it is. + it("rejects lib.backup() when another backup has taken the lock over", async () => { + const lib = await Library.open({ + client: loadSession(join(sessionDir, "session.json"), { fetch })!, + cacheDirectory: `${dir}-cache`, + downloadDirectory: dir, + refreshIntervalSeconds: Infinity, + precacheThumbnails: false, + precacheOriginals: false, + }); + try { + const result = lib.backup(); + await beachRequested; + rmSync(lock, { recursive: true }); + mkdirSync(lock); + await fireLockTimer(); + releaseBeach(); + + await expect(result).rejects.toThrow( + `lost the lock on ${dir}: Unable to update lock within the stale threshold`, + ); + expect(downloads).toEqual([BEACH.id]); + expect(existsSync(lock)).toBe(true); + } finally { + await lib.close(); + } + }); +}); -- 2.54.0 From 1b1bcc1f3e500c775799bf593b8fdfffebb4d25d Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 14 Oct 2026 03:21:14 +0000 Subject: [PATCH 2/4] A backup whose lock is lost stops with one line, not a stack trace (closes #184) lockBackupDirectory now passes proper-lockfile an onCompromised that records why the lock was lost, where the default threw from a timer and ended the process. The backup checks its lock before each original and before it writes failures.json, and fails with "lost the lock on DIR: REASON", which quak backup prints through run as one line before exiting 1. Releasing a lost lock leaves backup.lock alone, as it may be another backup's. The lock of each directory is kept by path so that a backup run with lockHeld, as quak backup runs it, finds the lock its caller took. Model: opus-5-5 --- README.md | 20 ++++++++++++++++---- TODO.md | 9 +++++++++ src/backup.ts | 41 +++++++++++++++++++++++++++++++++++++---- src/library/index.ts | 3 ++- 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 0a7e7a3..0095718 100644 --- a/README.md +++ b/README.md @@ -761,9 +761,9 @@ drops it. `backup.lock` keeps two backups of the same directory from running at once, such as a cron run that starts while the previous one is still going. A backup creates `` if it is missing and takes the lock before its refresh, and -removes the lock when it ends, whether it succeeds or fails; `quak backup` takes -it before it opens its library, so a refused run sends no request. The lock is a -directory that +removes the lock when it ends, whether it succeeds or fails, unless it lost the +lock (see below); `quak backup` takes it before it opens its library, so a +refused run sends no request. The lock is a directory that [proper-lockfile](https://github.com/moxystudio/node-proper-lockfile) creates and keeps touching while the backup runs. A second backup of the directory, from another process or the same one, fails at once: `quak backup` prints @@ -775,6 +775,16 @@ over, so nobody has to remove it. The lock is outside the date folders and `collections/`, so it is never taken for an original, and the removal of old album directories never touches it. +A running backup loses its lock when someone removes `backup.lock`, or when +another backup takes it over because the running one left it untouched for 10 +seconds, as a stopped or stalled process does. The backup finds this out the +next time it goes to touch the lock, which it does every 5 seconds, and then +stops before it starts another file. The original it is downloading at that +moment is still stored. It leaves `failures.json` and `backup.lock` as they are, +since another backup may be using them. `quak backup` prints +`quak: lost the lock on : ` as one line and exits 1, and +`lib.backup()` rejects with `lost the lock on : `. + A collection's directory and JSON are named after the collection, and a symlink after the file's title, both with unsafe characters replaced. When two collections would get the same name, or two symlinks in one collection the same @@ -1074,7 +1084,9 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange` back to the library's), `includeOriginals` (default `true`), `includeThumbnails` (default `false`), `onlyAlbumNames`, `verify` (default `false`), `onProgress`, and `lockHeld` (default `false`). See Backup layout - above for the tree it writes. + above for the tree it writes. Whether the backup took the lock or its caller + did, once the lock is lost the backup rejects with + `lost the lock on : ` before it starts another file. ### Request pools diff --git a/TODO.md b/TODO.md index e170323..7013286 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,15 @@ declares one. # Completed Steps +- 2026-10-14: A backup whose lock is lost no longer dies with a stack trace + (issue 184). `lockBackupDirectory` gives proper-lockfile an `onCompromised` + that records why the lock was lost instead of throwing from its timer. The + backup then stops before it starts another original, and before it writes + `failures.json`, with `lost the lock on DIR: REASON`, which `quak backup` + prints as one line before it exits 1. Releasing a lost lock leaves + `backup.lock` alone. Tests in `test/cli/backup-end-to-end.test.ts` remove the + lock, or put another backup's in its place, while a download is held open. + - 2026-10-14: An album or file quak cannot decrypt, or whose metadata is not JSON, no longer fails the refresh, so one bad record no longer stops `quak backup` from storing anything (issue 181). The refresh leaves it out, an diff --git a/src/backup.ts b/src/backup.ts index 5deae14..5eb88ed 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -2,7 +2,8 @@ // // `lib.backup()` takes the lock in `downloadDirectory` (unless its caller // already holds it), and fails at once when another backup of it holds the -// lock. It waits for a completed refresh of the library (a failed one fails the +// lock; when the lock is lost while it runs, it fails before it starts another +// file. It waits for a completed refresh of the library (a failed one fails the // backup before any file is touched), then, for every file in scope, puts its // original at its save path under `downloadDirectory`, as `Photo.download()` // does, waits for an ML data fetch, and rebuilds the derived views (per-file @@ -114,7 +115,8 @@ export interface BackupOptions { onProgress?: ProgressCallback; // The caller already holds the lock in `downloadDirectory`, taken with // `lockBackupDirectory`, and releases it itself, so the backup does not - // take it. `quak backup` takes it before it opens its library. + // take it, but still fails once that lock is lost. `quak backup` takes it + // before it opens its library. lockHeld?: boolean; } @@ -521,6 +523,9 @@ const runLockedBackup = async ( const log = opts.onProgress ?? (() => {}); const only = opts.onlyAlbumNames ? new Set(opts.onlyAlbumNames) : undefined; const verify = opts.verify ?? false; + // The lock `lockBackupDirectory` took for this backup. Once it is lost, + // another backup may be writing to `downloadDirectory`. + const lock = heldLocks.get(resolve(downloadDirectory)); log("Refreshing library..."); await lib.refresh(); @@ -642,6 +647,7 @@ const runLockedBackup = async ( // that cannot be read is recorded as failed and left where it is. if (includeOriginals) { for (const [fileID, file] of distinct) { + if (lock?.lost) throw lock.lost; let stored = storedAtSavePath(downloadDirectory, file); let mismatch = false; if (stored !== undefined && verify) { @@ -864,7 +870,9 @@ const runLockedBackup = async ( // its failure resolved; a file gone from the library (deleted) or outside // this run's scope is not something this run can resolve, so keeping its // stale entry would keep the exit code non-zero forever — a single - // since-deleted photo would fail every future scheduled backup. + // since-deleted photo would fail every future scheduled backup. A backup + // whose lock was lost leaves the ledger as it is. + if (lock?.lost) throw lock.lost; for (const fileID of [...ledger.keys()]) { if (!failedThisRun.has(fileID)) ledger.delete(fileID); } @@ -883,6 +891,10 @@ const runLockedBackup = async ( }; }; +// The lock this process holds on each backup directory, by the directory's +// absolute path, until it is released. `lost` is set once the lock is lost. +const heldLocks = new Map(); + // Only one backup of a directory runs at a time, in this process or another. // Creates `downloadDirectory` if it is missing, takes the lock in it and // returns the function that releases it; while another backup holds the lock, @@ -890,13 +902,26 @@ const runLockedBackup = async ( // directory `backup.lock`, whose modification time proper-lockfile keeps // current while it is held. One it has not touched for 10 seconds was left by // a run that could not remove it, and is taken over. +// +// proper-lockfile finds the lock lost when `backup.lock` is gone or is another +// backup's, and calls `onCompromised` from a timer, where throwing would end +// the process. The backup holding the lock stops before it starts another file +// instead (see `runLockedBackup`), and releasing a lost lock leaves +// `backup.lock` alone. export const lockBackupDirectory = async ( downloadDirectory: string, ): Promise<() => Promise> => { mkdirSync(downloadDirectory, { recursive: true }); + const lock: { lost?: Error } = {}; + let release: () => Promise; try { - return await lockfile.lock(downloadDirectory, { + release = await lockfile.lock(downloadDirectory, { lockfilePath: join(downloadDirectory, "backup.lock"), + onCompromised: (err) => { + lock.lost = new Error( + `lost the lock on ${downloadDirectory}: ${err.message}`, + ); + }, }); } catch (err) { if ((err as NodeJS.ErrnoException).code !== "ELOCKED") throw err; @@ -905,6 +930,14 @@ export const lockBackupDirectory = async ( { code: "ELOCKED" }, ); } + const path = resolve(downloadDirectory); + heldLocks.set(path, lock); + return async () => { + // Once this lock was lost, another backup in this process may have + // taken the directory. + if (heldLocks.get(path) === lock) heldLocks.delete(path); + if (lock.lost === undefined) await release(); + }; }; // Runs the backup holding the lock, which it takes and releases itself unless diff --git a/src/library/index.ts b/src/library/index.ts index 443f4d4..294f56f 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -572,7 +572,8 @@ export class Library { // library's, each original at its save path, with a durable failure // ledger (issue #51). Takes the lock in that directory first, unless // `opts.lockHeld` says the caller holds it, failing at once while another - // backup of it runs (see `runBackup`). Waits for a + // backup of it runs, and before it starts another file once the lock is + // lost (see `lockBackupDirectory`). Waits for a // completed refresh, as `fresh()` does, joining one already running, and // rejects before touching any file but the lock when it fails. Then puts // pending originals at their save paths as `Photo.download()` does (and -- 2.54.0 From dc1f483a3da38439d8a5791ddda320d745fab7e6 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 14 Oct 2026 04:23:07 +0000 Subject: [PATCH 3/4] Test a lock lost during the last original, and a caller's lock One test loses the lock while toast.mp4, the last original, downloads: quak backup must then write no file's JSON and no album. Another takes the lock with lockBackupDirectory, passes it to lib.backup() as lock, and loses it: the backup must reject, and the lock's lost must hold the reason. The fake server now holds whichever download a test names. Both fail until the backup checks its lock before each file and takes the caller's lock itself. Model: opus-5-5 --- test/cli/backup-end-to-end.test.ts | 119 +++++++++++++++++++++-------- 1 file changed, 89 insertions(+), 30 deletions(-) diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index c69884a..c53eaa7 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -9,7 +9,7 @@ * data this file encrypts with libsodium, as Ente's clients encrypt it. So * quak's own code fetches each original, decrypts it, checks it against the * hash its uploader recorded and unpacks a live photo, and quak's own retries - * repeat a failed request. One test runs `lib.backup()` instead, as a program + * repeat a failed request. Two tests run `lib.backup()` instead, as a program * using quak as a library does. * * The account, user 42, has three albums: @@ -58,7 +58,7 @@ import { run } from "../../src/cli-run.js"; import { loadSession } from "../../src/cli-session.js"; import { Client } from "../../src/client.js"; import { init, toBase64 } from "../../src/crypto/index.js"; -import { Library } from "../../src/library/index.js"; +import { Library, lockBackupDirectory } from "../../src/library/index.js"; import { UNATTENDED_RETRY_OPTIONS } from "../../src/retry.js"; import { buildAccount, @@ -653,32 +653,35 @@ describe("quak backup", () => { // seconds. Another backup may then be writing to the directory, so the backup // stops before it starts another file. // -// The fake server holds the download of beach.jpg, the first file backed up, -// open until the test lets it go, so the lock can be lost while the backup -// runs. Only `setTimeout` and `clearTimeout` are faked, so a test can fire -// proper-lockfile's timer without waiting 5 seconds. +// The fake server holds one file's download open until the test lets it go, +// so the lock can be lost while the backup runs: beach.jpg's, the first file +// backed up, unless the test names another. Only `setTimeout` and +// `clearTimeout` are faked, so a test can fire proper-lockfile's timer without +// waiting 5 seconds. describe("a backup whose lock is lost", () => { let dir: string; - let lock: string; + let lockPath: string; let downloads: number[]; - let beachRequested: Promise; - let releaseBeach: () => void; + let heldFile: number; + let heldRequested: Promise; + let releaseHeld: () => void; let fetch: typeof globalThis.fetch; beforeEach(() => { dir = mkdtempSync(join(root, "backup-")); - lock = join(dir, "backup.lock"); + lockPath = join(dir, "backup.lock"); downloads = []; + heldFile = BEACH.id; let requested!: () => void; - beachRequested = new Promise((resolve) => { + heldRequested = new Promise((resolve) => { requested = resolve; }); const released = new Promise((resolve) => { - releaseBeach = resolve; + releaseHeld = resolve; }); const recording = recordingDownloads(downloads); fetch = async (input, init) => { - if (downloadOf(input) === BEACH.id) { + if (downloadOf(input) === heldFile) { requested(); await released; } @@ -706,10 +709,10 @@ describe("a backup whose lock is lost", () => { // exits 1. it("stops quak backup before its next file when backup.lock is removed, and exits 1", async () => { const result = backup(dir, {}, fetch); - await beachRequested; - rmSync(lock, { recursive: true }); + await heldRequested; + rmSync(lockPath, { recursive: true }); await fireLockTimer(); - releaseBeach(); + releaseHeld(); const { code, stderr } = await result; expect(code).toBe(1); @@ -717,7 +720,7 @@ describe("a backup whose lock is lost", () => { "Starting backup...\n" + "Refreshing library...\n" + "Fetching original beach.jpg (100)...\n" + - `quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lock}'\n`, + `quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'\n`, ); expect(downloads).toEqual([BEACH.id]); expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual( @@ -725,26 +728,52 @@ describe("a backup whose lock is lost", () => { ); }); + // Here the lock is lost while toast.mp4, the last original, downloads. + // All three originals are stored, beside what the backup wrote before it + // started them, but no file's JSON and no album is written. + it("stops quak backup before any file's JSON when the lock is lost during the last original", async () => { + heldFile = TOAST_VIDEO.id; + const result = backup(dir, {}, fetch); + await heldRequested; + rmSync(lockPath, { recursive: true }); + await fireLockTimer(); + releaseHeld(); + const { code, stderr } = await result; + + expect(code).toBe(1); + expect(stderr).toContain(`quak: lost the lock on ${dir}: `); + expect(listTree(dir)).toEqual([ + "2024", + "2024/2024-03", + "2024/2024-03/2024-03-01", + "2024/2024-03/2024-03-01/2024-03-01.100.jpg", + "2024/2024-03/2024-03-01/2024-03-01.200.mp4", + "2024/2024-03/2024-03-01/2024-03-01.300.heic", + "2024/2024-03/2024-03-01/2024-03-01.300.livephoto.json", + "2024/2024-03/2024-03-01/2024-03-01.300.mov", + "account.json", + "collections", + ]); + }); + // Nothing touches the lock but quak, which finds it as it left it each // time it looks, so the backup runs to the end and removes its lock. it("lets quak backup run to the end while its lock stays", async () => { const result = backup(dir, {}, fetch); - await beachRequested; + await heldRequested; await fireLockTimer(); await fireLockTimer(); - releaseBeach(); + releaseHeld(); const { code } = await result; expect(code).toBe(0); expect(downloads.sort()).toEqual([100, 200, 300]); - expect(existsSync(lock)).toBe(false); + expect(existsSync(lockPath)).toBe(false); }); - // A program using quak as a library sees the same: `lib.backup()` rejects - // with the reason. Here another backup has made its own lock in place of - // quak's, and that lock is left where it is. - it("rejects lib.backup() when another backup has taken the lock over", async () => { - const lib = await Library.open({ + // The library a program using quak opens to back `dir` up. + const openLibrary = () => + Library.open({ client: loadSession(join(sessionDir, "session.json"), { fetch })!, cacheDirectory: `${dir}-cache`, downloadDirectory: dir, @@ -752,21 +781,51 @@ describe("a backup whose lock is lost", () => { precacheThumbnails: false, precacheOriginals: false, }); + + // A program using quak as a library sees the same: `lib.backup()` rejects + // with the reason. Here another backup has made its own lock in place of + // quak's, and that lock is left where it is. + it("rejects lib.backup() when another backup has taken the lock over", async () => { + const lib = await openLibrary(); try { const result = lib.backup(); - await beachRequested; - rmSync(lock, { recursive: true }); - mkdirSync(lock); + await heldRequested; + rmSync(lockPath, { recursive: true }); + mkdirSync(lockPath); await fireLockTimer(); - releaseBeach(); + releaseHeld(); await expect(result).rejects.toThrow( `lost the lock on ${dir}: Unable to update lock within the stale threshold`, ); expect(downloads).toEqual([BEACH.id]); - expect(existsSync(lock)).toBe(true); + expect(existsSync(lockPath)).toBe(true); } finally { await lib.close(); } }); + + // A program can take the lock before it opens its library, as `quak + // backup` does. `lockBackupDirectory` returns the lock; the program passes + // it to `lib.backup()`, which checks it as it checks a lock it took, and + // releases it once the backup ends. The lock's `lost` says why it was lost. + it("rejects lib.backup() given its caller's lock once that lock is lost", async () => { + const lock = await lockBackupDirectory(dir); + const lib = await openLibrary(); + try { + const result = lib.backup({ lock }); + await heldRequested; + rmSync(lockPath, { recursive: true }); + await fireLockTimer(); + releaseHeld(); + + const lost = `lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lockPath}'`; + await expect(result).rejects.toThrow(lost); + expect(lock.lost?.message).toBe(lost); + expect(downloads).toEqual([BEACH.id]); + } finally { + await lib.close(); + await lock.release(); + } + }); }); -- 2.54.0 From 9c921e95dc544fa7180c6ff9adbfc1e0333c1e00 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 14 Oct 2026 04:30:17 +0000 Subject: [PATCH 4/4] Check the lock before every file, and pass the lock itself to the backup The backup now checks its lock after the refresh and before each file of the thumbnail, JSON and album passes, as well as before each original and before failures.json. lockBackupDirectory returns the lock, with release() and lost, in place of a release function, and lib.backup() takes it as lock in place of lockHeld: true; quak backup passes it. This drops the process-wide map from directory path to lock, which missed a lock taken through another path to the same directory. Model: opus-5-5 --- README.md | 37 ++++++------- TODO.md | 13 ++--- src/backup.ts | 85 ++++++++++++++++-------------- src/cli-commands.ts | 10 ++-- src/index.ts | 1 + src/library/index.ts | 9 ++-- test/cli/backup-end-to-end.test.ts | 7 +-- 7 files changed, 87 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 0095718..bc74537 100644 --- a/README.md +++ b/README.md @@ -1070,23 +1070,24 @@ photos newest first). `lib.subscribe({ onChange })` delivers a `LibraryChange` directory, and fails at once with an error whose `code` is `ELOCKED` while another backup of it runs. A caller can instead take the lock itself, before it opens its library, as `quak backup` does: `await lockBackupDirectory(dir)` - takes it, failing the same way, and returns the function that releases it, and - the caller passes `lockHeld: true` to the backup. It waits for a refresh as - `fresh()` does, puts every in-scope original not already at its save path - there as `photo.download()` does (and, with `includeThumbnails`, fetches - thumbnails) through the content cache, waits for an ML data fetch, and - rebuilds the on-disk backup tree with a durable failure ledger. Each file's - JSON holds its ML data and, for an image (for a live photo, its image), its - original's EXIF and XMP, with its dimensions for a JPEG only; a video's JSON - holds none of these three. A fetched original is written straight to its save - path and not into the cache, which then counts it as present; one the cache - already held is copied from there. `BackupOptions`: `downloadDirectory` (falls - back to the library's), `includeOriginals` (default `true`), - `includeThumbnails` (default `false`), `onlyAlbumNames`, `verify` (default - `false`), `onProgress`, and `lockHeld` (default `false`). See Backup layout - above for the tree it writes. Whether the backup took the lock or its caller - did, once the lock is lost the backup rejects with - `lost the lock on : ` before it starts another file. + takes it, failing the same way, and returns a `BackupLock`, whose `release()` + removes the lock unless it was lost and whose `lost` holds why it was lost. + The caller passes it to the backup as `lock`, and releases it once the backup + ends. It waits for a refresh as `fresh()` does, puts every in-scope original + not already at its save path there as `photo.download()` does (and, with + `includeThumbnails`, fetches thumbnails) through the content cache, waits for + an ML data fetch, and rebuilds the on-disk backup tree with a durable failure + ledger. Each file's JSON holds its ML data and, for an image (for a live + photo, its image), its original's EXIF and XMP, with its dimensions for a JPEG + only; a video's JSON holds none of these three. A fetched original is written + straight to its save path and not into the cache, which then counts it as + present; one the cache already held is copied from there. `BackupOptions`: + `downloadDirectory` (falls back to the library's), `includeOriginals` (default + `true`), `includeThumbnails` (default `false`), `onlyAlbumNames`, `verify` + (default `false`), `onProgress`, and `lock`. See Backup layout above for the + tree it writes. Whether the backup took the lock or its caller did, once the + lock is lost the backup rejects with `lost the lock on : ` before + it starts another file. ### Request pools @@ -1146,7 +1147,7 @@ from a very old client, is stored unchecked. - `src/library/mlsearch.ts`: `MLDataAPI`, `SimilarResult` - `src/exif.ts`: `ExifTags`, `PhotoExif` - `src/library/pools.ts`: `RequestPools`, `RequestPoolsOptions`, `BoundedPool` -- `src/backup.ts`: `BackupOptions`, `BackupResult`, `BackupError` +- `src/backup.ts`: `BackupLock`, `BackupOptions`, `BackupResult`, `BackupError` - `src/client.ts`: `Client`, `LoginOptions`, `ClientSnapshot` - `src/api/client.ts`: `ApiClient`, `ApiClientOptions`, `StreamOptions` - `src/errors.ts`: `ApiError`, `TruncatedStreamError` diff --git a/TODO.md b/TODO.md index 7013286..5f555ea 100644 --- a/TODO.md +++ b/TODO.md @@ -27,12 +27,13 @@ declares one. - 2026-10-14: A backup whose lock is lost no longer dies with a stack trace (issue 184). `lockBackupDirectory` gives proper-lockfile an `onCompromised` - that records why the lock was lost instead of throwing from its timer. The - backup then stops before it starts another original, and before it writes - `failures.json`, with `lost the lock on DIR: REASON`, which `quak backup` - prints as one line before it exits 1. Releasing a lost lock leaves - `backup.lock` alone. Tests in `test/cli/backup-end-to-end.test.ts` remove the - lock, or put another backup's in its place, while a download is held open. + that records why the lock was lost instead of throwing from its timer, and + returns the lock, which `quak backup` passes to `lib.backup()` as `lock`. The + backup then stops before it starts another file with + `lost the lock on DIR: REASON`, which `quak backup` prints as one line before + it exits 1. Releasing a lost lock leaves `backup.lock` alone. Tests in + `test/cli/backup-end-to-end.test.ts` remove the lock, or put another backup's + in its place, while a download is held open. - 2026-10-14: An album or file quak cannot decrypt, or whose metadata is not JSON, no longer fails the refresh, so one bad record no longer stops diff --git a/src/backup.ts b/src/backup.ts index 5eb88ed..bd76d2f 100644 --- a/src/backup.ts +++ b/src/backup.ts @@ -1,7 +1,7 @@ // The backup command, rebuilt on the library API (issue #51). // // `lib.backup()` takes the lock in `downloadDirectory` (unless its caller -// already holds it), and fails at once when another backup of it holds the +// passes one in), and fails at once when another backup of it holds the // lock; when the lock is lost while it runs, it fails before it starts another // file. It waits for a completed refresh of the library (a failed one fails the // backup before any file is touched), then, for every file in scope, puts its @@ -97,6 +97,14 @@ import type { Collection, EnteFile, FileMetadata } from "./model/types.js"; export type ProgressCallback = (message: string) => void; +// The lock in a backup directory, as `lockBackupDirectory` returns it. +export interface BackupLock { + // Removes `backup.lock`, unless the lock was lost. + release(): Promise; + // Why the lock was lost, once it is. + lost?: Error; +} + export interface BackupOptions { // Where the backup tree lives. `lib.backup()` defaults it to the library's // download directory; `runBackup` with none throws before any network @@ -113,11 +121,11 @@ export interface BackupOptions { // bytes do not match the content hash its metadata records. Default false. verify?: boolean; onProgress?: ProgressCallback; - // The caller already holds the lock in `downloadDirectory`, taken with - // `lockBackupDirectory`, and releases it itself, so the backup does not - // take it, but still fails once that lock is lost. `quak backup` takes it + // The lock in `downloadDirectory`, when the caller took it with + // `lockBackupDirectory` and releases it itself. The backup then does not + // take the lock, and fails once this one is lost. `quak backup` takes it // before it opens its library. - lockHeld?: boolean; + lock?: BackupLock; } export interface BackupError { @@ -512,23 +520,24 @@ const writeAlbumJSON = ( writeFileSync(path, JSON.stringify(album, null, 2)); }; -// The backup itself, which `runBackup` below runs while the lock is held. +// The backup itself, which `runBackup` below runs while `lock` is held. Once +// the lock is lost, another backup may be writing to `downloadDirectory`, so +// this one fails before it starts another file. const runLockedBackup = async ( lib: BackupLibrary, opts: BackupOptions, downloadDirectory: string, + lock: BackupLock, ): Promise => { const includeOriginals = opts.includeOriginals ?? true; const includeThumbnails = opts.includeThumbnails ?? false; const log = opts.onProgress ?? (() => {}); const only = opts.onlyAlbumNames ? new Set(opts.onlyAlbumNames) : undefined; const verify = opts.verify ?? false; - // The lock `lockBackupDirectory` took for this backup. Once it is lost, - // another backup may be writing to `downloadDirectory`. - const lock = heldLocks.get(resolve(downloadDirectory)); log("Refreshing library..."); await lib.refresh(); + if (lock.lost) throw lock.lost; const collectionsDir = join(downloadDirectory, "collections"); const thumbnailsDir = join(downloadDirectory, "thumbnails"); @@ -647,7 +656,7 @@ const runLockedBackup = async ( // that cannot be read is recorded as failed and left where it is. if (includeOriginals) { for (const [fileID, file] of distinct) { - if (lock?.lost) throw lock.lost; + if (lock.lost) throw lock.lost; let stored = storedAtSavePath(downloadDirectory, file); let mismatch = false; if (stored !== undefined && verify) { @@ -727,6 +736,7 @@ const runLockedBackup = async ( if (includeThumbnails) { for (const [fileID, file] of distinct) { + if (lock.lost) throw lock.lost; const dest = join(thumbnailsDir, `${fileID}.jpg`); if (isPresent(dest)) continue; try { @@ -759,6 +769,7 @@ const runLockedBackup = async ( log(`FAILED ML data: ${mlDataError}`); } for (const file of distinct.values()) { + if (lock.lost) throw lock.lost; const stored = storedAtSavePath(downloadDirectory, file); if (stored === undefined) continue; const path = withExtension( @@ -802,6 +813,7 @@ const runLockedBackup = async ( const albumDirNames = new Map( allCollections.map((c, i) => [c.id, dirNames[i]!]), ); + if (lock.lost) throw lock.lost; try { removeStaleAlbumDirs( collectionsDir, @@ -813,6 +825,7 @@ const runLockedBackup = async ( } for (const c of collections) { + if (lock.lost) throw lock.lost; const colDirName = albumDirNames.get(c.id)!; const colDir = join(collectionsDir, colDirName); mkdirSync(colDir, { recursive: true }); @@ -872,7 +885,7 @@ const runLockedBackup = async ( // stale entry would keep the exit code non-zero forever — a single // since-deleted photo would fail every future scheduled backup. A backup // whose lock was lost leaves the ledger as it is. - if (lock?.lost) throw lock.lost; + if (lock.lost) throw lock.lost; for (const fileID of [...ledger.keys()]) { if (!failedThisRun.has(fileID)) ledger.delete(fileID); } @@ -891,29 +904,30 @@ const runLockedBackup = async ( }; }; -// The lock this process holds on each backup directory, by the directory's -// absolute path, until it is released. `lost` is set once the lock is lost. -const heldLocks = new Map(); - // Only one backup of a directory runs at a time, in this process or another. // Creates `downloadDirectory` if it is missing, takes the lock in it and -// returns the function that releases it; while another backup holds the lock, -// fails at once with an error whose `code` is `ELOCKED`. The lock is the -// directory `backup.lock`, whose modification time proper-lockfile keeps -// current while it is held. One it has not touched for 10 seconds was left by -// a run that could not remove it, and is taken over. +// returns it; while another backup holds the lock, fails at once with an error +// whose `code` is `ELOCKED`. The lock is the directory `backup.lock`, whose +// modification time proper-lockfile keeps current while it is held. One it has +// not touched for 10 seconds was left by a run that could not remove it, and +// is taken over. // // proper-lockfile finds the lock lost when `backup.lock` is gone or is another // backup's, and calls `onCompromised` from a timer, where throwing would end -// the process. The backup holding the lock stops before it starts another file -// instead (see `runLockedBackup`), and releasing a lost lock leaves -// `backup.lock` alone. +// the process. The `onCompromised` given here sets the lock's `lost` instead, +// and the backup holding the lock stops before it starts another file (see +// `runLockedBackup`). Releasing a lost lock leaves `backup.lock` alone, since +// it may be another backup's. export const lockBackupDirectory = async ( downloadDirectory: string, -): Promise<() => Promise> => { +): Promise => { mkdirSync(downloadDirectory, { recursive: true }); - const lock: { lost?: Error } = {}; let release: () => Promise; + const lock: BackupLock = { + release: async () => { + if (lock.lost === undefined) await release(); + }, + }; try { release = await lockfile.lock(downloadDirectory, { lockfilePath: join(downloadDirectory, "backup.lock"), @@ -930,18 +944,11 @@ export const lockBackupDirectory = async ( { code: "ELOCKED" }, ); } - const path = resolve(downloadDirectory); - heldLocks.set(path, lock); - return async () => { - // Once this lock was lost, another backup in this process may have - // taken the directory. - if (heldLocks.get(path) === lock) heldLocks.delete(path); - if (lock.lost === undefined) await release(); - }; + return lock; }; // Runs the backup holding the lock, which it takes and releases itself unless -// the caller already holds it (`opts.lockHeld`). +// the caller passes the lock it took (`opts.lock`). export const runBackup = async ( lib: BackupLibrary, opts: BackupOptions, @@ -953,13 +960,13 @@ export const runBackup = async ( "open the library with one)", ); } - if (opts.lockHeld) { - return runLockedBackup(lib, opts, downloadDirectory); + if (opts.lock) { + return runLockedBackup(lib, opts, downloadDirectory, opts.lock); } - const release = await lockBackupDirectory(downloadDirectory); + const lock = await lockBackupDirectory(downloadDirectory); try { - return await runLockedBackup(lib, opts, downloadDirectory); + return await runLockedBackup(lib, opts, downloadDirectory, lock); } finally { - await release(); + await lock.release(); } }; diff --git a/src/cli-commands.ts b/src/cli-commands.ts index 076d1ef..7a0c505 100644 --- a/src/cli-commands.ts +++ b/src/cli-commands.ts @@ -20,7 +20,7 @@ import { type ClientSnapshot, type LoginOptions, } from "./client.js"; -import { lockBackupDirectory } from "./backup.js"; +import { lockBackupDirectory, type BackupLock } from "./backup.js"; import { init } from "./crypto/index.js"; import { defaultCacheDirectory, @@ -416,9 +416,9 @@ export const backupCommand = async ( ctx.stderr.write("Starting backup...\n"); // The lock is taken before the library opens and starts its refresh, so a // run refused while another backup of `dir` runs sends no request. - let release: () => Promise; + let lock: BackupLock; try { - release = await lockBackupDirectory(dir); + lock = await lockBackupDirectory(dir); } catch (err) { if ((err as NodeJS.ErrnoException).code !== "ELOCKED") throw err; ctx.stderr.write(`quak: ${(err as Error).message}\n`); @@ -440,7 +440,7 @@ export const backupCommand = async ( try { const result = await lib.backup({ downloadDirectory: dir, - lockHeld: true, + lock, verify: opts.verify, onProgress: (msg) => { if (!opts.json) ctx.stderr.write(msg + "\n"); @@ -477,7 +477,7 @@ export const backupCommand = async ( await lib.close(); } } finally { - await release(); + await lock.release(); } }; diff --git a/src/index.ts b/src/index.ts index 1328893..686dbef 100644 --- a/src/index.ts +++ b/src/index.ts @@ -69,6 +69,7 @@ export { type EnsureEvent, lockBackupDirectory, runBackup, + type BackupLock, type BackupOptions, type BackupResult, type BackupError, diff --git a/src/library/index.ts b/src/library/index.ts index 294f56f..eb36410 100644 --- a/src/library/index.ts +++ b/src/library/index.ts @@ -96,6 +96,7 @@ import { runBackup, type BackupOptions, type BackupResult } from "../backup.js"; export { lockBackupDirectory, runBackup, + type BackupLock, type BackupOptions, type BackupResult, type BackupError, @@ -570,10 +571,10 @@ export class Library { // Back up every in-scope file to `opts.downloadDirectory`, or else the // library's, each original at its save path, with a durable failure - // ledger (issue #51). Takes the lock in that directory first, unless - // `opts.lockHeld` says the caller holds it, failing at once while another - // backup of it runs, and before it starts another file once the lock is - // lost (see `lockBackupDirectory`). Waits for a + // ledger (issue #51). Takes the lock in that directory first, unless the + // caller passes the lock it took as `opts.lock`, failing at once while + // another backup of it runs, and before it starts another file once the + // lock is lost (see `lockBackupDirectory`). Waits for a // completed refresh, as `fresh()` does, joining one already running, and // rejects before touching any file but the lock when it fails. Then puts // pending originals at their save paths as `Photo.download()` does (and diff --git a/test/cli/backup-end-to-end.test.ts b/test/cli/backup-end-to-end.test.ts index c53eaa7..886a1fc 100644 --- a/test/cli/backup-end-to-end.test.ts +++ b/test/cli/backup-end-to-end.test.ts @@ -806,9 +806,10 @@ describe("a backup whose lock is lost", () => { }); // A program can take the lock before it opens its library, as `quak - // backup` does. `lockBackupDirectory` returns the lock; the program passes - // it to `lib.backup()`, which checks it as it checks a lock it took, and - // releases it once the backup ends. The lock's `lost` says why it was lost. + // backup` does: `lockBackupDirectory` returns the lock, which the program + // passes to `lib.backup()` and releases once the backup ends. The backup + // checks it as it checks a lock it took, and the lock's `lost` says why it + // was lost. it("rejects lib.backup() given its caller's lock once that lock is lost", async () => { const lock = await lockBackupDirectory(dir); const lib = await openLibrary(); -- 2.54.0