sneak
c6ab2de940
check / check (push) Successful in 1m59s
Within a collection's folder, files whose sanitized titles match (ignoring case) each get their file ID added before the extension, and collections whose sanitized names match get their ID added, so no symlink or JSON replaces another. Names are chosen across all collections, so a scoped run names folders the same as a full one. Each run first removes symlinks into originals/ that no longer belong to a collection, and the folders quak wrote (a sibling JSON with an album ID) for collections that are gone or renamed. Anything else is left alone; a folder still holding user files keeps its JSON. Model: opus-5-5
843 lines
31 KiB
TypeScript
843 lines
31 KiB
TypeScript
/**
|
|
* Tests for the `quak backup` logic, now built on the library API (issue #51).
|
|
*
|
|
* `lib.backup({ downloadDirectory })` refreshes the library, fetches each
|
|
* pending file's original through the content cache/pools, and materialises the
|
|
* unchanged on-disk layout:
|
|
*
|
|
* <downloadDirectory>/
|
|
* originals/
|
|
* <fileID>.<ext> the decrypted bytes ("present means complete")
|
|
* <fileID>.json per-file metadata sidecar (rebuilt each run)
|
|
* collections/
|
|
* <name>/<title> symlink into ../originals (rebuilt each run)
|
|
* <name>.json per-collection metadata (rebuilt each run)
|
|
* failures.json durable ledger of unresolved failures
|
|
*
|
|
* The properties that distinguish backup from a naive download loop, and that
|
|
* these tests lock down:
|
|
*
|
|
* 1. Present-means-complete: an original already on disk is not re-fetched, so
|
|
* runs are idempotent and interrupted runs resume.
|
|
* 2. Per-file resilience: a download failure or a symlink failure is recorded
|
|
* and the run continues (issue #8); the derived symlink/JSON views are
|
|
* rebuilt from the model every run.
|
|
* 3. A durable `failures.json` records each unresolved failure's classification,
|
|
* attempt count, and last-tried time; the exit code (result.failed) is
|
|
* non-zero while any failure remains and clears once every one is resolved.
|
|
*
|
|
* The cache and download layers are covered elsewhere (content.test.ts,
|
|
* download tests); here a mock library client and a stand-in content source
|
|
* drive the backup logic with no crypto and no network.
|
|
*/
|
|
|
|
import {
|
|
existsSync,
|
|
lstatSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readdirSync,
|
|
readFileSync,
|
|
readlinkSync,
|
|
rmSync,
|
|
symlinkSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { spawnSync } from "node:child_process";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
|
|
|
import { runBackup, type BackupLibrary } from "../../src/backup.js";
|
|
import { Library } from "../../src/library/index.js";
|
|
import type { ContentSource } from "../../src/library/content.js";
|
|
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
|
import type { Collection, EnteFile } from "../../src/model/types.js";
|
|
|
|
// `open` and `rename` are wrapped to record, in order, every fsync and rename,
|
|
// so a test can pin the sequence "fsync the temp file, rename, fsync the
|
|
// directory" that makes a copied original survive a power cut. `vi.hoisted`
|
|
// because `vi.mock` factories run before module-level constants exist.
|
|
const fsEvents = vi.hoisted(() => [] as string[]);
|
|
|
|
vi.mock("node:fs/promises", async (importOriginal) => {
|
|
const actual = await importOriginal<typeof import("node:fs/promises")>();
|
|
return {
|
|
...actual,
|
|
open: async (
|
|
...args: Parameters<typeof actual.open>
|
|
): Promise<Awaited<ReturnType<typeof actual.open>>> => {
|
|
const handle = await actual.open(...args);
|
|
const realSync = handle.sync.bind(handle);
|
|
handle.sync = async (): Promise<void> => {
|
|
fsEvents.push(`sync:${String(args[0])}`);
|
|
await realSync();
|
|
};
|
|
return handle;
|
|
},
|
|
rename: async (from: string, to: string): Promise<void> => {
|
|
fsEvents.push(`rename:${to}`);
|
|
await actual.rename(from, to);
|
|
},
|
|
};
|
|
});
|
|
|
|
const USER_ID = 42;
|
|
|
|
// Decrypted-byte length each stub original writes, keyed by fileID.
|
|
const SIZE_BY_ID: Record<number, number> = { 100: 3000, 101: 2000, 200: 1500 };
|
|
|
|
const collection = (id: number, name: string): Collection => ({
|
|
id,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
name,
|
|
type: "album",
|
|
updationTime: 1,
|
|
isShared: false,
|
|
});
|
|
|
|
const file = (id: number, collectionID: number, title: string): EnteFile => ({
|
|
id,
|
|
collectionID,
|
|
ownerID: USER_ID,
|
|
key: new Uint8Array([id & 0xff]),
|
|
metadata: {
|
|
title,
|
|
fileType: "image",
|
|
creationTime: 1,
|
|
modificationTime: 1,
|
|
},
|
|
file: { decryptionHeader: "aGVhZGVy" },
|
|
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
|
updationTime: 1,
|
|
});
|
|
|
|
// A metadata-only client: two albums, three files, served once. No ML.
|
|
class MockClient {
|
|
private served = false;
|
|
whoami(): { email: string; userID: number } {
|
|
return { email: "backup@example.com", userID: USER_ID };
|
|
}
|
|
async collectionsSince(): Promise<CollectionsPage> {
|
|
if (this.served) return { collections: [], deleted: [], cursor: 1 };
|
|
this.served = true;
|
|
return {
|
|
collections: [collection(1, "Vacation"), collection(2, "Work")],
|
|
deleted: [],
|
|
cursor: 1,
|
|
};
|
|
}
|
|
async filesSince(args: { collectionID: number }): Promise<FilesPage> {
|
|
const files =
|
|
args.collectionID === 1
|
|
? [file(100, 1, "beach.jpg"), file(101, 1, "sunset.jpg")]
|
|
: args.collectionID === 2
|
|
? [file(200, 2, "diagram.png")]
|
|
: [];
|
|
return { files, deleted: [], cursor: 1 };
|
|
}
|
|
}
|
|
|
|
// A server that names an album and a file so as to climb out of the backup
|
|
// directory.
|
|
class HostileClient extends MockClient {
|
|
override async collectionsSince(): Promise<CollectionsPage> {
|
|
const page = await super.collectionsSince();
|
|
return {
|
|
...page,
|
|
collections: page.collections.length
|
|
? [collection(3, "../escape")]
|
|
: [],
|
|
};
|
|
}
|
|
override async filesSince(args: {
|
|
collectionID: number;
|
|
}): Promise<FilesPage> {
|
|
const files =
|
|
args.collectionID === 3
|
|
? [file(300, 3, "../../.ssh/authorized_keys")]
|
|
: [];
|
|
return { files, deleted: [], cursor: 1 };
|
|
}
|
|
}
|
|
|
|
// A content source that writes byte buffers of the expected length and can be
|
|
// told to fail one fileID's original, to exercise per-file resilience.
|
|
interface StubSource extends ContentSource {
|
|
failID?: number;
|
|
failThumbID?: number;
|
|
originalCalls: number;
|
|
}
|
|
|
|
const stubSource = (): StubSource => {
|
|
const s: StubSource = {
|
|
originalCalls: 0,
|
|
original: async ({ file: f, destination }) => {
|
|
s.originalCalls++;
|
|
if (s.failID === f.id) throw new Error("HTTP 500 from server");
|
|
const size = SIZE_BY_ID[f.id] ?? 10;
|
|
writeFileSync(destination, Buffer.alloc(size));
|
|
return { bytesWritten: size };
|
|
},
|
|
thumbnail: async ({ file: f, destination }) => {
|
|
if (s.failThumbID === f.id) throw new Error("HTTP 500 from server");
|
|
writeFileSync(destination, Buffer.alloc(5));
|
|
return { bytesWritten: 5 };
|
|
},
|
|
};
|
|
return s;
|
|
};
|
|
|
|
let root: string;
|
|
|
|
const openLibrary = (
|
|
source: ContentSource,
|
|
client: MockClient = new MockClient(),
|
|
): Promise<Library> =>
|
|
Library.open({
|
|
client,
|
|
cacheDirectory: join(root, "cache"),
|
|
contentSource: source,
|
|
refreshIntervalSeconds: 3600,
|
|
// These tests count exact fetches; the background precache (#48) would
|
|
// add its own, so it is off here (it is covered in precache.test.ts).
|
|
precacheThumbnails: false,
|
|
precacheOriginals: false,
|
|
});
|
|
|
|
const readLedger = (
|
|
outDir: string,
|
|
): { files: Record<string, Record<string, unknown>> } =>
|
|
JSON.parse(readFileSync(join(outDir, "failures.json"), "utf-8"));
|
|
|
|
// Write a durable ledger holding one prior failure, to exercise pruning of
|
|
// entries the current run cannot resolve.
|
|
const seedLedger = (outDir: string, fileID: number, title: string): void => {
|
|
mkdirSync(outDir, { recursive: true });
|
|
writeFileSync(
|
|
join(outDir, "failures.json"),
|
|
JSON.stringify({
|
|
version: 1,
|
|
files: {
|
|
[String(fileID)]: {
|
|
fileID,
|
|
title,
|
|
classification: "transient",
|
|
attempts: 1,
|
|
lastTriedAt: Date.now(),
|
|
error: "HTTP 500 from server",
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
};
|
|
|
|
beforeEach(() => {
|
|
root = mkdtempSync(join(tmpdir(), "quak-backup-test-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
if (root && existsSync(root))
|
|
rmSync(root, { recursive: true, force: true });
|
|
});
|
|
|
|
describe("lib.backup", () => {
|
|
it("throws before any network when no downloadDirectory is given", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
await expect(lib.backup()).rejects.toThrow(/downloadDirectory/i);
|
|
expect(source.originalCalls).toBe(0);
|
|
lib.close();
|
|
});
|
|
|
|
it("writes the expected on-disk layout for every file", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.totalFiles).toBe(3);
|
|
expect(result.downloaded).toBe(3);
|
|
expect(result.skipped).toBe(0);
|
|
expect(result.failed).toBe(0);
|
|
expect(result.errors).toEqual([]);
|
|
|
|
// Originals under originals/<fileID>.<ext>.
|
|
expect(readFileSync(join(outDir, "originals", "100.jpg")).length).toBe(
|
|
3000,
|
|
);
|
|
expect(readFileSync(join(outDir, "originals", "101.jpg")).length).toBe(
|
|
2000,
|
|
);
|
|
expect(readFileSync(join(outDir, "originals", "200.png")).length).toBe(
|
|
1500,
|
|
);
|
|
|
|
// Per-file metadata sidecar.
|
|
const sidecar = JSON.parse(
|
|
readFileSync(join(outDir, "originals", "100.json"), "utf-8"),
|
|
);
|
|
expect(sidecar.id).toBe(100);
|
|
expect(sidecar.metadata.title).toBe("beach.jpg");
|
|
|
|
// Collection dirs contain symlinks into ../originals.
|
|
const beach = join(outDir, "collections", "Vacation", "beach.jpg");
|
|
expect(lstatSync(beach).isSymbolicLink()).toBe(true);
|
|
expect(readlinkSync(beach)).toContain("originals");
|
|
expect(readFileSync(beach).length).toBe(3000);
|
|
|
|
// Per-collection metadata JSON.
|
|
const vacation = JSON.parse(
|
|
readFileSync(join(outDir, "collections", "Vacation.json"), "utf-8"),
|
|
);
|
|
expect(vacation.name).toBe("Vacation");
|
|
expect(vacation.files.length).toBe(2);
|
|
expect(vacation.files[0].metadata.title).toBeDefined();
|
|
|
|
// A clean run leaves no failure ledger behind.
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
lib.close();
|
|
});
|
|
|
|
it("keeps server-supplied album and file names inside the backup", async () => {
|
|
const lib = await openLibrary(stubSource(), new HostileClient());
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.failed).toBe(0);
|
|
// The title has no usable extension, so the original is `.bin`.
|
|
expect(existsSync(join(outDir, "originals", "300.bin"))).toBe(true);
|
|
const link = join(
|
|
outDir,
|
|
"collections",
|
|
"__escape",
|
|
"__.._.ssh_authorized_keys",
|
|
);
|
|
expect(lstatSync(link).isSymbolicLink()).toBe(true);
|
|
expect(existsSync(join(outDir, "collections", "__escape.json"))).toBe(
|
|
true,
|
|
);
|
|
// Nothing landed beside or above the backup directory.
|
|
expect(readdirSync(root).sort()).toEqual(["backup", "cache"]);
|
|
lib.close();
|
|
});
|
|
|
|
it("is an idempotent no-op when every original is already present", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const first = await lib.backup({ downloadDirectory: outDir });
|
|
expect(first.downloaded).toBe(3);
|
|
const callsAfterFirst = source.originalCalls;
|
|
|
|
const second = await lib.backup({ downloadDirectory: outDir });
|
|
expect(second.downloaded).toBe(0);
|
|
expect(second.skipped).toBe(3);
|
|
expect(second.failed).toBe(0);
|
|
// A present original is neither fetched nor copied again.
|
|
expect(source.originalCalls).toBe(callsAfterFirst);
|
|
lib.close();
|
|
});
|
|
|
|
it("continues past a download failure and records it in failures.json", async () => {
|
|
const source = stubSource();
|
|
source.failID = 101;
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
expect(result.totalFiles).toBe(3);
|
|
expect(result.downloaded).toBe(2);
|
|
expect(result.failed).toBe(1);
|
|
expect(result.errors.length).toBe(1);
|
|
expect(result.errors[0]!.fileID).toBe(101);
|
|
expect(result.errors[0]!.title).toBe("sunset.jpg");
|
|
|
|
// The two good files are on disk; the failed one is not.
|
|
expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true);
|
|
expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true);
|
|
expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(false);
|
|
expect(
|
|
existsSync(join(outDir, "collections", "Vacation", "sunset.jpg")),
|
|
).toBe(false);
|
|
|
|
// Durable ledger with classification, attempts, last-tried.
|
|
const ledger = readLedger(outDir);
|
|
const entry = ledger.files["101"]!;
|
|
expect(entry.attempts).toBe(1);
|
|
expect(entry.classification).toBeDefined();
|
|
expect(typeof entry.lastTriedAt).toBe("number");
|
|
lib.close();
|
|
});
|
|
|
|
it("increments the attempt count across runs and clears the ledger once resolved", async () => {
|
|
const source = stubSource();
|
|
source.failID = 101;
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const r1 = await lib.backup({ downloadDirectory: outDir });
|
|
expect(r1.failed).toBe(1);
|
|
expect(readLedger(outDir).files["101"]!.attempts).toBe(1);
|
|
|
|
// Second run: the two good files are present, only 101 is retried.
|
|
const r2 = await lib.backup({ downloadDirectory: outDir });
|
|
expect(r2.failed).toBe(1);
|
|
expect(r2.skipped).toBe(2);
|
|
expect(readLedger(outDir).files["101"]!.attempts).toBe(2);
|
|
|
|
// Resume with a healthy source: 101 downloads, the rest are skipped.
|
|
source.failID = undefined;
|
|
const r3 = await lib.backup({ downloadDirectory: outDir });
|
|
expect(r3.failed).toBe(0);
|
|
expect(r3.skipped).toBe(2);
|
|
expect(existsSync(join(outDir, "originals", "101.jpg"))).toBe(true);
|
|
// A ledger with no remaining failures is removed.
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
lib.close();
|
|
});
|
|
|
|
it("does not abort when a symlink cannot be created (issue #8)", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
// Occupy beach.jpg's symlink path with a directory so symlink creation
|
|
// fails for that one file.
|
|
mkdirSync(join(outDir, "collections", "Vacation", "beach.jpg"), {
|
|
recursive: true,
|
|
});
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
// Every original still downloads despite the symlink failure.
|
|
expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(true);
|
|
expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true);
|
|
// The other symlinks are still built.
|
|
expect(
|
|
lstatSync(
|
|
join(outDir, "collections", "Vacation", "sunset.jpg"),
|
|
).isSymbolicLink(),
|
|
).toBe(true);
|
|
expect(
|
|
lstatSync(
|
|
join(outDir, "collections", "Work", "diagram.png"),
|
|
).isSymbolicLink(),
|
|
).toBe(true);
|
|
|
|
// The symlink failure is recorded, not thrown.
|
|
expect(result.failed).toBeGreaterThanOrEqual(1);
|
|
const err = result.errors.find((e) => e.fileID === 100);
|
|
expect(err).toBeDefined();
|
|
expect(err!.collection).toBe("Vacation");
|
|
expect(readLedger(outDir).files["100"]).toBeDefined();
|
|
lib.close();
|
|
});
|
|
|
|
it("rebuilds a stale sidecar and a missing symlink on a later run", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
// Corrupt a sidecar and delete a symlink between runs.
|
|
writeFileSync(join(outDir, "originals", "100.json"), "not json");
|
|
rmSync(join(outDir, "collections", "Vacation", "beach.jpg"));
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
expect(result.failed).toBe(0);
|
|
|
|
// The derived views are repaired from the model.
|
|
const sidecar = JSON.parse(
|
|
readFileSync(join(outDir, "originals", "100.json"), "utf-8"),
|
|
);
|
|
expect(sidecar.metadata.title).toBe("beach.jpg");
|
|
expect(
|
|
lstatSync(
|
|
join(outDir, "collections", "Vacation", "beach.jpg"),
|
|
).isSymbolicLink(),
|
|
).toBe(true);
|
|
lib.close();
|
|
});
|
|
|
|
it("backs up only the named albums when onlyAlbumNames is given", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
onlyAlbumNames: ["Work"],
|
|
});
|
|
|
|
expect(result.totalFiles).toBe(1);
|
|
expect(result.downloaded).toBe(1);
|
|
expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true);
|
|
expect(existsSync(join(outDir, "originals", "100.jpg"))).toBe(false);
|
|
expect(existsSync(join(outDir, "collections", "Work.json"))).toBe(true);
|
|
expect(existsSync(join(outDir, "collections", "Vacation.json"))).toBe(
|
|
false,
|
|
);
|
|
lib.close();
|
|
});
|
|
|
|
it("prunes a ledger entry for a file no longer in the library and exits zero", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
// A prior failure for a file that has since left the library (deleted
|
|
// from the account). This run has no way to resolve it, so it must not
|
|
// keep the exit code non-zero forever.
|
|
seedLedger(outDir, 999, "gone.jpg");
|
|
|
|
const result = await lib.backup({ downloadDirectory: outDir });
|
|
|
|
// Everything still present is backed up cleanly, and the stale entry is
|
|
// dropped rather than counted.
|
|
expect(result.downloaded).toBe(3);
|
|
expect(result.failed).toBe(0);
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
lib.close();
|
|
});
|
|
|
|
it("prunes an out-of-scope ledger entry on a scoped run and exits zero", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
// A prior failure for a Vacation file; this run is scoped to Work and
|
|
// never attempts it, so it must not poison the scoped run's exit code.
|
|
seedLedger(outDir, 100, "beach.jpg");
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
onlyAlbumNames: ["Work"],
|
|
});
|
|
|
|
expect(result.totalFiles).toBe(1);
|
|
expect(result.failed).toBe(0);
|
|
expect(existsSync(join(outDir, "originals", "200.png"))).toBe(true);
|
|
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
|
|
lib.close();
|
|
});
|
|
|
|
it("also stores thumbnails when includeThumbnails is set", async () => {
|
|
const source = stubSource();
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
await lib.backup({
|
|
downloadDirectory: outDir,
|
|
includeThumbnails: true,
|
|
});
|
|
|
|
expect(existsSync(join(outDir, "thumbnails", "100.jpg"))).toBe(true);
|
|
expect(existsSync(join(outDir, "thumbnails", "200.jpg"))).toBe(true);
|
|
lib.close();
|
|
});
|
|
|
|
it("counts one attempt when a file fails both its original and thumbnail in a run", async () => {
|
|
const source = stubSource();
|
|
source.failID = 101;
|
|
source.failThumbID = 101;
|
|
const lib = await openLibrary(source);
|
|
const outDir = join(root, "backup");
|
|
|
|
const result = await lib.backup({
|
|
downloadDirectory: outDir,
|
|
includeThumbnails: true,
|
|
});
|
|
|
|
// Both kinds fail for 101, but the run counts it once.
|
|
const errs = result.errors.filter((e) => e.fileID === 101);
|
|
expect(errs.length).toBe(1);
|
|
expect(readLedger(outDir).files["101"]!.attempts).toBe(1);
|
|
lib.close();
|
|
});
|
|
|
|
it("fsyncs a copied original before the rename and its directory after", async () => {
|
|
const lib = await openLibrary(stubSource());
|
|
const outDir = join(root, "backup");
|
|
const originals = join(outDir, "originals");
|
|
const dest = join(originals, "100.jpg");
|
|
fsEvents.length = 0;
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
const at = fsEvents.indexOf(`rename:${dest}`);
|
|
expect(at).toBeGreaterThan(0);
|
|
expect(fsEvents[at - 1]).toMatch(
|
|
/^sync:.*\/\.quak-backup-100\.jpg-\d+-[0-9a-z]*\.tmp$/,
|
|
);
|
|
expect(fsEvents[at + 1]).toBe(`sync:${originals}`);
|
|
lib.close();
|
|
});
|
|
|
|
it("removes temp files left by a killed backup but not those of one still running", async () => {
|
|
const outDir = join(root, "backup");
|
|
const originals = join(outDir, "originals");
|
|
mkdirSync(originals, { recursive: true });
|
|
// A child that has already exited: its process ID is not running.
|
|
const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid;
|
|
const leftover = `.quak-backup-100.jpg-${exitedPID}-abc123.tmp`;
|
|
// This test's own process stands in for a backup running at the same
|
|
// time.
|
|
const inProgress = `.quak-backup-101.jpg-${process.pid}-def456.tmp`;
|
|
writeFileSync(join(originals, leftover), "partial");
|
|
writeFileSync(join(originals, inProgress), "partial");
|
|
const lib = await openLibrary(stubSource());
|
|
|
|
await lib.backup({ downloadDirectory: outDir });
|
|
|
|
const names = readdirSync(originals);
|
|
expect(names).not.toContain(leftover);
|
|
expect(names).toContain(inProgress);
|
|
lib.close();
|
|
});
|
|
|
|
it("removes leftover temp files in thumbnails/ but not those of a backup still running", async () => {
|
|
const outDir = join(root, "backup");
|
|
const thumbnails = join(outDir, "thumbnails");
|
|
mkdirSync(thumbnails, { recursive: true });
|
|
const exitedPID = spawnSync(process.execPath, ["-e", ""]).pid;
|
|
const leftover = `.quak-backup-100.jpg-${exitedPID}-abc123.tmp`;
|
|
const inProgress = `.quak-backup-101.jpg-${process.pid}-def456.tmp`;
|
|
writeFileSync(join(thumbnails, leftover), "partial");
|
|
writeFileSync(join(thumbnails, inProgress), "partial");
|
|
const lib = await openLibrary(stubSource());
|
|
|
|
await lib.backup({
|
|
downloadDirectory: outDir,
|
|
includeThumbnails: true,
|
|
});
|
|
|
|
const names = readdirSync(thumbnails);
|
|
expect(names).not.toContain(leftover);
|
|
expect(names).toContain(inProgress);
|
|
lib.close();
|
|
});
|
|
});
|
|
|
|
// The album folders under collections/, driven through `runBackup` with a
|
|
// stand-in library whose albums a test changes between runs.
|
|
describe("backup album folders", () => {
|
|
interface Album {
|
|
collection: Collection;
|
|
files: EnteFile[];
|
|
}
|
|
|
|
const libraryOf = (albums: Album[]): BackupLibrary => ({
|
|
refresh: async () => {},
|
|
listCollections: () => albums.map((a) => a.collection),
|
|
listFiles: (id) =>
|
|
albums.find((a) => a.collection.id === id)?.files ?? [],
|
|
original: async (fileID) => {
|
|
const path = join(root, `source-${fileID}`);
|
|
writeFileSync(path, `original ${fileID}`);
|
|
return { path };
|
|
},
|
|
thumbnail: async () => {
|
|
throw new Error("no thumbnails in this stand-in");
|
|
},
|
|
});
|
|
|
|
// Every entry under collections/, one level of directories deep, with each
|
|
// symlink's target.
|
|
const tree = (outDir: string): string[] => {
|
|
const lines: string[] = [];
|
|
const list = (dir: string, prefix: string): void => {
|
|
for (const name of readdirSync(dir).sort()) {
|
|
const path = join(dir, name);
|
|
const st = lstatSync(path);
|
|
if (st.isSymbolicLink()) {
|
|
lines.push(`${prefix}${name} -> ${readlinkSync(path)}`);
|
|
} else if (st.isDirectory() && prefix === "") {
|
|
lines.push(`${name}/`);
|
|
list(path, `${name}/`);
|
|
} else {
|
|
lines.push(`${prefix}${name}`);
|
|
}
|
|
}
|
|
};
|
|
list(join(outDir, "collections"), "");
|
|
return lines;
|
|
};
|
|
|
|
const albumID = (outDir: string, jsonName: string): number =>
|
|
JSON.parse(readFileSync(join(outDir, "collections", jsonName), "utf-8"))
|
|
.id;
|
|
|
|
it("gives every file and every album its own name when names repeat", async () => {
|
|
const outDir = join(root, "backup");
|
|
const lib = libraryOf([
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [
|
|
file(1, 10, "IMG_0001.JPG"),
|
|
file(2, 10, "IMG_0001.JPG"),
|
|
file(4, 10, "img_0001.jpg"),
|
|
file(3, 10, "other.jpg"),
|
|
],
|
|
},
|
|
{
|
|
collection: collection(11, "Trip"),
|
|
files: [file(3, 11, "other.jpg")],
|
|
},
|
|
]);
|
|
|
|
const result = await runBackup(lib, { downloadDirectory: outDir });
|
|
|
|
expect(result.failed).toBe(0);
|
|
expect(tree(outDir)).toEqual([
|
|
"Trip (10)/",
|
|
"Trip (10)/IMG_0001 (1).JPG -> ../../originals/1.JPG",
|
|
"Trip (10)/IMG_0001 (2).JPG -> ../../originals/2.JPG",
|
|
"Trip (10)/img_0001 (4).jpg -> ../../originals/4.jpg",
|
|
"Trip (10)/other.jpg -> ../../originals/3.jpg",
|
|
"Trip (10).json",
|
|
"Trip (11)/",
|
|
"Trip (11)/other.jpg -> ../../originals/3.jpg",
|
|
"Trip (11).json",
|
|
]);
|
|
expect(albumID(outDir, "Trip (10).json")).toBe(10);
|
|
expect(albumID(outDir, "Trip (11).json")).toBe(11);
|
|
});
|
|
|
|
it("changes nothing on a second run over an unchanged account", async () => {
|
|
const outDir = join(root, "backup");
|
|
const lib = libraryOf([
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [
|
|
file(1, 10, "IMG_0001.JPG"),
|
|
file(2, 10, "IMG_0001.JPG"),
|
|
],
|
|
},
|
|
{
|
|
collection: collection(11, "Trip"),
|
|
files: [file(3, 11, "other.jpg")],
|
|
},
|
|
]);
|
|
|
|
await runBackup(lib, { downloadDirectory: outDir });
|
|
const before = tree(outDir);
|
|
const second = await runBackup(lib, { downloadDirectory: outDir });
|
|
|
|
expect(second.downloaded).toBe(0);
|
|
expect(second.failed).toBe(0);
|
|
expect(tree(outDir)).toEqual(before);
|
|
});
|
|
|
|
it("leaves the albums an onlyAlbumNames run skips as they were", async () => {
|
|
const outDir = join(root, "backup");
|
|
// "trip" is skipped by the scoped run but its name clashes with the
|
|
// in-scope "Trip", so "Trip" must keep its ID suffix.
|
|
const lib = libraryOf([
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [file(1, 10, "a.jpg")],
|
|
},
|
|
{
|
|
collection: collection(11, "trip"),
|
|
files: [file(2, 11, "b.jpg")],
|
|
},
|
|
{
|
|
collection: collection(12, "Work"),
|
|
files: [file(3, 12, "c.jpg")],
|
|
},
|
|
]);
|
|
const json = (name: string): string =>
|
|
readFileSync(join(outDir, "collections", name), "utf-8");
|
|
|
|
await runBackup(lib, { downloadDirectory: outDir });
|
|
const before = tree(outDir);
|
|
const skippedJSON = [json("trip (11).json"), json("Work.json")];
|
|
const scoped = await runBackup(lib, {
|
|
downloadDirectory: outDir,
|
|
onlyAlbumNames: ["Trip"],
|
|
});
|
|
|
|
expect(scoped.failed).toBe(0);
|
|
expect(before).toEqual([
|
|
"Trip (10)/",
|
|
"Trip (10)/a.jpg -> ../../originals/1.jpg",
|
|
"Trip (10).json",
|
|
"Work/",
|
|
"Work/c.jpg -> ../../originals/3.jpg",
|
|
"Work.json",
|
|
"trip (11)/",
|
|
"trip (11)/b.jpg -> ../../originals/2.jpg",
|
|
"trip (11).json",
|
|
]);
|
|
expect(tree(outDir)).toEqual(before);
|
|
expect([json("trip (11).json"), json("Work.json")]).toEqual(
|
|
skippedJSON,
|
|
);
|
|
});
|
|
|
|
it("removes links and album folders that are gone, and nothing the user added", async () => {
|
|
const outDir = join(root, "backup");
|
|
const albums: Album[] = [
|
|
{
|
|
collection: collection(10, "Trip"),
|
|
files: [
|
|
file(1, 10, "IMG_0001.JPG"),
|
|
file(2, 10, "IMG_0001.JPG"),
|
|
file(3, 10, "other.jpg"),
|
|
],
|
|
},
|
|
{
|
|
collection: collection(12, "Work"),
|
|
files: [file(5, 12, "a.jpg")],
|
|
},
|
|
{
|
|
collection: collection(13, "Old"),
|
|
files: [file(5, 13, "a.jpg")],
|
|
},
|
|
];
|
|
const lib = libraryOf(albums);
|
|
await runBackup(lib, { downloadDirectory: outDir });
|
|
|
|
// What the user put in the tree: a note and a symlink of their own in
|
|
// an album, a note in an album about to be renamed, and a folder quak
|
|
// did not create.
|
|
const collectionsDir = join(outDir, "collections");
|
|
writeFileSync(join(collectionsDir, "Trip", "notes.txt"), "mine");
|
|
symlinkSync("../elsewhere", join(collectionsDir, "Trip", "mine"));
|
|
writeFileSync(join(collectionsDir, "Work", "keep.txt"), "mine");
|
|
mkdirSync(join(collectionsDir, "Mine"));
|
|
writeFileSync(join(collectionsDir, "Mine", "keep.txt"), "mine");
|
|
|
|
// File 2 leaves Trip, Work is renamed Office, Old is deleted.
|
|
albums[0]!.files.splice(1, 1);
|
|
albums[1]!.collection = collection(12, "Office");
|
|
albums.splice(2, 1);
|
|
const result = await runBackup(lib, { downloadDirectory: outDir });
|
|
|
|
expect(result.failed).toBe(0);
|
|
expect(tree(outDir)).toEqual([
|
|
"Mine/",
|
|
"Mine/keep.txt",
|
|
"Office/",
|
|
"Office/a.jpg -> ../../originals/5.jpg",
|
|
"Office.json",
|
|
"Trip/",
|
|
"Trip/IMG_0001.JPG -> ../../originals/1.JPG",
|
|
"Trip/mine -> ../elsewhere",
|
|
"Trip/notes.txt",
|
|
"Trip/other.jpg -> ../../originals/3.jpg",
|
|
"Trip.json",
|
|
"Work/",
|
|
"Work/keep.txt",
|
|
"Work.json",
|
|
]);
|
|
});
|
|
});
|