A backup whose lock is lost stops with one line, not a stack trace (closes #184)

lockBackupDirectory now passes proper-lockfile an onCompromised that records
why the lock was lost, where the default threw from a timer and ended the
process. The backup checks its lock before each original and before it
writes failures.json, and fails with "lost the lock on DIR: REASON", which
quak backup prints through run as one line before exiting 1. Releasing a
lost lock leaves backup.lock alone, as it may be another backup's. The lock
of each directory is kept by path so that a backup run with lockHeld, as
quak backup runs it, finds the lock its caller took.

Model: opus-5-5

src/backup.ts

@@ -2,7 +2,8 @@
 //
 // `lib.backup()` takes the lock in `downloadDirectory` (unless its caller
 // already holds it), and fails at once when another backup of it holds the
-// lock. It waits for a completed refresh of the library (a failed one fails the
+// lock; when the lock is lost while it runs, it fails before it starts another
+// file. It waits for a completed refresh of the library (a failed one fails the
 // backup before any file is touched), then, for every file in scope, puts its
 // original at its save path under `downloadDirectory`, as `Photo.download()`
 // does, waits for an ML data fetch, and rebuilds the derived views (per-file
@@ -114,7 +115,8 @@ export interface BackupOptions {
     onProgress?: ProgressCallback;
     // The caller already holds the lock in `downloadDirectory`, taken with
     // `lockBackupDirectory`, and releases it itself, so the backup does not
-    // take it. `quak backup` takes it before it opens its library.
+    // take it, but still fails once that lock is lost. `quak backup` takes it
+    // before it opens its library.
     lockHeld?: boolean;
 }
 
@@ -521,6 +523,9 @@ const runLockedBackup = async (
     const log = opts.onProgress ?? (() => {});
     const only = opts.onlyAlbumNames ? new Set(opts.onlyAlbumNames) : undefined;
     const verify = opts.verify ?? false;
+    // The lock `lockBackupDirectory` took for this backup. Once it is lost,
+    // another backup may be writing to `downloadDirectory`.
+    const lock = heldLocks.get(resolve(downloadDirectory));
 
     log("Refreshing library...");
     await lib.refresh();
@@ -642,6 +647,7 @@ const runLockedBackup = async (
     // that cannot be read is recorded as failed and left where it is.
     if (includeOriginals) {
         for (const [fileID, file] of distinct) {
+            if (lock?.lost) throw lock.lost;
             let stored = storedAtSavePath(downloadDirectory, file);
             let mismatch = false;
             if (stored !== undefined && verify) {
@@ -864,7 +870,9 @@ const runLockedBackup = async (
     // its failure resolved; a file gone from the library (deleted) or outside
     // this run's scope is not something this run can resolve, so keeping its
     // stale entry would keep the exit code non-zero forever — a single
-    // since-deleted photo would fail every future scheduled backup.
+    // since-deleted photo would fail every future scheduled backup. A backup
+    // whose lock was lost leaves the ledger as it is.
+    if (lock?.lost) throw lock.lost;
     for (const fileID of [...ledger.keys()]) {
         if (!failedThisRun.has(fileID)) ledger.delete(fileID);
     }
@@ -883,6 +891,10 @@ const runLockedBackup = async (
     };
 };
 
+// The lock this process holds on each backup directory, by the directory's
+// absolute path, until it is released. `lost` is set once the lock is lost.
+const heldLocks = new Map<string, { lost?: Error }>();
+
 // Only one backup of a directory runs at a time, in this process or another.
 // Creates `downloadDirectory` if it is missing, takes the lock in it and
 // returns the function that releases it; while another backup holds the lock,
@@ -890,13 +902,26 @@ const runLockedBackup = async (
 // directory `backup.lock`, whose modification time proper-lockfile keeps
 // current while it is held. One it has not touched for 10 seconds was left by
 // a run that could not remove it, and is taken over.
+//
+// proper-lockfile finds the lock lost when `backup.lock` is gone or is another
+// backup's, and calls `onCompromised` from a timer, where throwing would end
+// the process. The backup holding the lock stops before it starts another file
+// instead (see `runLockedBackup`), and releasing a lost lock leaves
+// `backup.lock` alone.
 export const lockBackupDirectory = async (
     downloadDirectory: string,
 ): Promise<() => Promise<void>> => {
     mkdirSync(downloadDirectory, { recursive: true });
+    const lock: { lost?: Error } = {};
+    let release: () => Promise<void>;
     try {
-        return await lockfile.lock(downloadDirectory, {
+        release = await lockfile.lock(downloadDirectory, {
             lockfilePath: join(downloadDirectory, "backup.lock"),
+            onCompromised: (err) => {
+                lock.lost = new Error(
+                    `lost the lock on ${downloadDirectory}: ${err.message}`,
+                );
+            },
         });
     } catch (err) {
         if ((err as NodeJS.ErrnoException).code !== "ELOCKED") throw err;
@@ -905,6 +930,14 @@ export const lockBackupDirectory = async (
             { code: "ELOCKED" },
         );
     }
+    const path = resolve(downloadDirectory);
+    heldLocks.set(path, lock);
+    return async () => {
+        // Once this lock was lost, another backup in this process may have
+        // taken the directory.
+        if (heldLocks.get(path) === lock) heldLocks.delete(path);
+        if (lock.lost === undefined) await release();
+    };
 };
 
 // Runs the backup holding the lock, which it takes and releases itself unless

src/library/index.ts

@@ -572,7 +572,8 @@ export class Library {
     // library's, each original at its save path, with a durable failure
     // ledger (issue #51). Takes the lock in that directory first, unless
     // `opts.lockHeld` says the caller holds it, failing at once while another
-    // backup of it runs (see `runBackup`). Waits for a
+    // backup of it runs, and before it starts another file once the lock is
+    // lost (see `lockBackupDirectory`). Waits for a
     // completed refresh, as `fresh()` does, joining one already running, and
     // rejects before touching any file but the lock when it fails. Then puts
     // pending originals at their save paths as `Photo.download()` does (and