Test a backup whose lock is lost while it runs

The fake Ente server holds the download of the first file open while the
test removes backup.lock, or puts another backup's lock in its place, and
fires proper-lockfile's timer on a faked clock. quak backup must then stop
before its next file and print one line, and lib.backup() must reject with
the reason. A backup whose lock stays must still run to the end. These fail
until the lock handles being lost.

Model: opus-5-5

test/cli/backup-end-to-end.test.ts

@@ -9,7 +9,8 @@
  * data this file encrypts with libsodium, as Ente's clients encrypt it. So
  * quak's own code fetches each original, decrypts it, checks it against the
  * hash its uploader recorded and unpacks a live photo, and quak's own retries
- * repeat a failed request.
+ * repeat a failed request. One test runs `lib.backup()` instead, as a program
+ * using quak as a library does.
  *
  * The account, user 42, has three albums:
  *
@@ -24,6 +25,7 @@
 import {
     existsSync,
     lstatSync,
+    mkdirSync,
     mkdtempSync,
     readdirSync,
     readFileSync,
@@ -36,7 +38,16 @@ import { join } from "node:path";
 import { Writable } from "node:stream";
 import { gzipSync } from "node:zlib";
 import sodium from "libsodium-wrappers-sumo";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+    afterAll,
+    afterEach,
+    beforeAll,
+    beforeEach,
+    describe,
+    expect,
+    it,
+    vi,
+} from "vitest";
 
 import {
     backupCommand,
@@ -47,6 +58,7 @@ import { run } from "../../src/cli-run.js";
 import { loadSession } from "../../src/cli-session.js";
 import { Client } from "../../src/client.js";
 import { init, toBase64 } from "../../src/crypto/index.js";
+import { Library } from "../../src/library/index.js";
 import { UNATTENDED_RETRY_OPTIONS } from "../../src/retry.js";
 import {
     buildAccount,
@@ -633,3 +645,128 @@ describe("quak backup", () => {
         expect(stderr).toContain("  Verified:    2\n  Mismatched:  1\n");
     });
 });
+
+// `backup.lock` is a directory that proper-lockfile touches every 5 seconds,
+// from a timer, while the backup runs. Each time, it first looks at the lock
+// on disk. The lock is lost when it is gone, or when another backup has made
+// its own in its place, as one does with a lock nobody touched for 10
+// seconds. Another backup may then be writing to the directory, so the backup
+// stops before it starts another file.
+//
+// The fake server holds the download of beach.jpg, the first file backed up,
+// open until the test lets it go, so the lock can be lost while the backup
+// runs. Only `setTimeout` and `clearTimeout` are faked, so a test can fire
+// proper-lockfile's timer without waiting 5 seconds.
+describe("a backup whose lock is lost", () => {
+    let dir: string;
+    let lock: string;
+    let downloads: number[];
+    let beachRequested: Promise<void>;
+    let releaseBeach: () => void;
+    let fetch: typeof globalThis.fetch;
+
+    beforeEach(() => {
+        dir = mkdtempSync(join(root, "backup-"));
+        lock = join(dir, "backup.lock");
+        downloads = [];
+        let requested!: () => void;
+        beachRequested = new Promise((resolve) => {
+            requested = resolve;
+        });
+        const released = new Promise<void>((resolve) => {
+            releaseBeach = resolve;
+        });
+        const recording = recordingDownloads(downloads);
+        fetch = async (input, init) => {
+            if (downloadOf(input) === BEACH.id) {
+                requested();
+                await released;
+            }
+            return recording(input, init);
+        };
+        vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+    });
+
+    afterEach(() => {
+        vi.useRealTimers();
+    });
+
+    // Fire proper-lockfile's timer, and wait for the look at the lock on disk
+    // that it starts to finish: Node lists a file system call in progress as
+    // an `FSReqCallback`.
+    const fireLockTimer = async (): Promise<void> => {
+        vi.advanceTimersByTime(5_000);
+        while (process.getActiveResourcesInfo().includes("FSReqCallback")) {
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+    };
+
+    // beach.jpg, whose download was under way, is stored; no other file is
+    // started. `quak backup` prints why as one line, with no stack trace, and
+    // exits 1.
+    it("stops quak backup before its next file when backup.lock is removed, and exits 1", async () => {
+        const result = backup(dir, {}, fetch);
+        await beachRequested;
+        rmSync(lock, { recursive: true });
+        await fireLockTimer();
+        releaseBeach();
+        const { code, stderr } = await result;
+
+        expect(code).toBe(1);
+        expect(stderr).toBe(
+            "Starting backup...\n" +
+                "Refreshing library...\n" +
+                "Fetching original beach.jpg (100)...\n" +
+                `quak: lost the lock on ${dir}: ENOENT: no such file or directory, stat '${lock}'\n`,
+        );
+        expect(downloads).toEqual([BEACH.id]);
+        expect(readFileSync(join(dir, DAY, "2024-03-01.100.jpg"))).toEqual(
+            Buffer.from(JPEG_WITH_EXIF),
+        );
+    });
+
+    // Nothing touches the lock but quak, which finds it as it left it each
+    // time it looks, so the backup runs to the end and removes its lock.
+    it("lets quak backup run to the end while its lock stays", async () => {
+        const result = backup(dir, {}, fetch);
+        await beachRequested;
+        await fireLockTimer();
+        await fireLockTimer();
+        releaseBeach();
+        const { code } = await result;
+
+        expect(code).toBe(0);
+        expect(downloads.sort()).toEqual([100, 200, 300]);
+        expect(existsSync(lock)).toBe(false);
+    });
+
+    // A program using quak as a library sees the same: `lib.backup()` rejects
+    // with the reason. Here another backup has made its own lock in place of
+    // quak's, and that lock is left where it is.
+    it("rejects lib.backup() when another backup has taken the lock over", async () => {
+        const lib = await Library.open({
+            client: loadSession(join(sessionDir, "session.json"), { fetch })!,
+            cacheDirectory: `${dir}-cache`,
+            downloadDirectory: dir,
+            refreshIntervalSeconds: Infinity,
+            precacheThumbnails: false,
+            precacheOriginals: false,
+        });
+        try {
+            const result = lib.backup();
+            await beachRequested;
+            rmSync(lock, { recursive: true });
+            mkdirSync(lock);
+            await fireLockTimer();
+            releaseBeach();
+
+            await expect(result).rejects.toThrow(
+                `lost the lock on ${dir}: Unable to update lock within the stale threshold`,
+            );
+            expect(downloads).toEqual([BEACH.id]);
+            expect(existsSync(lock)).toBe(true);
+        } finally {
+            await lib.close();
+        }
+    });
+});