One album or file quak cannot decrypt no longer stops a refresh or a backup (closes #181)

A file or album whose key, metadata or name does not decrypt used to
throw out of the refresh, so quak backup stopped before storing
anything, every run. Client.collectionsSince and filesSince now return
such a row in undecryptable with the reason, the refresh leaves it out,
and the store keeps the list in metadata.json until a later refresh
decrypts or deletes the record. lib.backup() records each such file in
failures.json unless a copy in another album decrypted, in which case it
stores the original from that copy. Each such album is listed in
BackupResult.undecryptableAlbums and printed as one FAILED album line.
quak backup exits 1 while either remains. An album or file that no
longer decrypts loses its album folder, JSON and links; its original
stays.

Model: opus-5-5

src/backup.ts

@@ -47,6 +47,9 @@
 // download, a failed symlink, or ML data missing because the ML data fetch
 // failed is caught, recorded in `failures.json` with a classification, a
 // running attempt count, and the last-tried time, and the run continues.
+// A file the library could not decrypt is recorded there too, unless a copy of
+// it in another album decrypted; an album it could not decrypt is logged by ID
+// and listed in `result.undecryptableAlbums`, as its name is not known.
 // `result.failed` — and thus the CLI's exit code — stays non-zero while any
 // failure remains unresolved and clears once every one succeeds. Each run
 // reconciles the ledger against the files it attempted, so an entry for a file
@@ -86,6 +89,7 @@ import {
     storedAtSavePath,
 } from "./library/content.js";
 import { representative } from "./library/records.js";
+import type { Undecryptable } from "./library/store.js";
 import { extractImageMetadata } from "./metadata-backup.js";
 import type { MLData } from "./mldata-fetch.js";
 import type { Collection, EnteFile, FileMetadata } from "./model/types.js";
@@ -141,6 +145,9 @@ export interface BackupResult {
     failed: number;
     // This run's per-file errors, in encounter order.
     errors: BackupError[];
+    // The IDs of the albums the library could not decrypt, whose files it
+    // could not list; the CLI exits non-zero while there are any.
+    undecryptableAlbums: number[];
 }
 
 // The slice of the library that backup drives. `Library` implements it; a test
@@ -151,6 +158,9 @@ export interface BackupLibrary {
     refresh(): Promise<void>;
     listCollections(): Collection[];
     listFiles(collectionID: number): EnteFile[];
+    // The albums, and files in albums, that the library could not decrypt,
+    // which `listCollections` and `listFiles` leave out.
+    undecryptable(): Undecryptable[];
     // Get an original's bytes onto disk through the content cache/pools,
     // returning where they landed: `destination` when they were fetched now,
     // otherwise wherever they already were (the cache, or the library's save
@@ -574,33 +584,56 @@ const runLockedBackup = async (
     let unchecked = 0;
 
     const recordFailure = (
-        file: EnteFile,
+        fileID: number,
+        title: string,
         collection: string,
         err: unknown,
     ): void => {
         // Count at most one attempt per file per run: a file whose original
         // and thumbnail both fail this run must not double its attempt count
         // or appear twice in errors.
-        if (failedThisRun.has(file.id)) return;
+        if (failedThisRun.has(fileID)) return;
         const error = errorMessage(err);
-        errors.push({
-            fileID: file.id,
-            title: file.metadata.title,
-            collection,
-            error,
-        });
-        const prior = ledger.get(file.id);
-        ledger.set(file.id, {
-            fileID: file.id,
-            title: file.metadata.title,
+        errors.push({ fileID, title, collection, error });
+        const prior = ledger.get(fileID);
+        ledger.set(fileID, {
+            fileID,
+            title,
             classification: classify(err),
             attempts: (prior?.attempts ?? 0) + 1,
             lastTriedAt: now,
             error,
         });
-        failedThisRun.add(file.id);
+        failedThisRun.add(fileID);
     };
 
+    // What the library could not decrypt. An album is logged by its ID, as
+    // its name is not known, so a run limited to some albums logs it too. A
+    // file in an album in scope is backed up from a copy of it in another
+    // album that decrypted, even an album out of scope. With no such copy it
+    // is recorded as failed, with an empty title as its metadata is not known.
+    const scope = new Set(collections.map((c) => c.id));
+    const undecryptableAlbums: number[] = [];
+    for (const record of lib.undecryptable()) {
+        const error = `could not be decrypted: ${record.error}`;
+        if (record.fileID === undefined) {
+            log(`FAILED album ${record.collectionID}: ${error}`);
+            undecryptableAlbums.push(record.collectionID);
+        } else if (scope.has(record.collectionID)) {
+            const copies = memberships.get(record.fileID);
+            if (copies) {
+                distinct.set(record.fileID, representative(copies));
+            } else {
+                recordFailure(
+                    record.fileID,
+                    "",
+                    collectionName.get(record.collectionID) ?? "",
+                    new Error(error),
+                );
+            }
+        }
+    }
+
     // Phase 1: get the bytes. Put each pending original at its save path
     // through the content cache/pools, as `Photo.download()` does, and fetch
     // the optional thumbnails; a present file is left as is. With `verify`, a
@@ -636,7 +669,8 @@ const runLockedBackup = async (
                         `FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
                     );
                     recordFailure(
-                        file,
+                        file.id,
+                        file.metadata.title,
                         collectionName.get(file.collectionID) ?? "",
                         err,
                     );
@@ -676,7 +710,8 @@ const runLockedBackup = async (
                     `FAILED original ${file.metadata.title}: ${errorMessage(err)}`,
                 );
                 recordFailure(
-                    file,
+                    file.id,
+                    file.metadata.title,
                     collectionName.get(file.collectionID) ?? "",
                     err,
                 );
@@ -693,7 +728,8 @@ const runLockedBackup = async (
                 await copyAtomic(path, dest);
             } catch (err) {
                 recordFailure(
-                    file,
+                    file.id,
+                    file.metadata.title,
                     collectionName.get(file.collectionID) ?? "",
                     err,
                 );
@@ -733,7 +769,8 @@ const runLockedBackup = async (
             if (mlData === undefined && mlDataError !== undefined) {
                 writeSidecar(path, file, { mlDataError }, image);
                 recordFailure(
-                    file,
+                    file.id,
+                    file.metadata.title,
                     collectionName.get(file.collectionID) ?? "",
                     new Error(`ML data: ${mlDataError}`),
                 );
@@ -806,7 +843,12 @@ const runLockedBackup = async (
                 log(
                     `FAILED symlink ${c.name}/${linkName}: ${errorMessage(err)}`,
                 );
-                recordFailure(link.file, c.name, err);
+                recordFailure(
+                    link.file.id,
+                    link.file.metadata.title,
+                    c.name,
+                    err,
+                );
             }
         }
 
@@ -837,6 +879,7 @@ const runLockedBackup = async (
         unchecked,
         failed: ledger.size,
         errors,
+        undecryptableAlbums,
     };
 };
 

src/cli-commands.ts

@@ -470,7 +470,9 @@ export const backupCommand = async (
                 }
             }
 
-            return result.failed > 0 ? 1 : 0;
+            return result.failed > 0 || result.undecryptableAlbums.length > 0
+                ? 1
+                : 0;
         } finally {
             await lib.close();
         }

src/client.ts

@@ -44,20 +44,28 @@ export interface ClientSnapshot {
 
 // The result of a resumable enumeration. Live decrypted records and deleted
 // ids are kept apart on purpose: a tombstone carries no key or metadata to
-// decrypt, so it is a bare id rather than a hollowed-out record. `cursor` is
-// the max `updationTime` seen, to pass back into the next call.
+// decrypt, so it is a bare id rather than a hollowed-out record. A live row
+// that cannot be decrypted, or whose metadata cannot be parsed, is in
+// `undecryptable` with the reason instead, so one bad row does not fail the
+// call; absent means there was none. `cursor` is the max `updationTime` seen,
+// those rows included, to pass back into the next call.
 export interface CollectionsPage {
     collections: Collection[];
     deleted: number[];
+    undecryptable?: { id: number; error: string }[];
     cursor: number;
 }
 
 export interface FilesPage {
     files: EnteFile[];
     deleted: number[];
+    undecryptable?: { id: number; error: string }[];
     cursor: number;
 }
 
+const errorMessage = (err: unknown): string =>
+    err instanceof Error ? err.message : String(err);
+
 export class Client {
     private readonly api: ApiClient;
     private readonly email: string;
@@ -237,11 +245,12 @@ export class Client {
     }
 
     // Enumerate collections changed since `sinceTime`. Live collections are
-    // decrypted; tombstoned ones (isDeleted) are surfaced as bare ids. The
-    // returned cursor is the max `updationTime` seen — including tombstones, so
-    // the next sync resumes past them — falling back to `sinceTime` when the
-    // response is empty. `/collections/v2` returns the whole changed set in one
-    // response, so there is no pagination here.
+    // decrypted; tombstoned ones (isDeleted) are surfaced as bare ids, and
+    // ones that do not decrypt as ids with the reason. The returned cursor is
+    // the max `updationTime` seen — including tombstones, so the next sync
+    // resumes past them — falling back to `sinceTime` when the response is
+    // empty. `/collections/v2` returns the whole changed set in one response,
+    // so there is no pagination here.
     async collectionsSince(args: {
         sinceTime: number;
     }): Promise<CollectionsPage> {
@@ -254,31 +263,40 @@ export class Client {
 
         const collections: Collection[] = [];
         const deleted: number[] = [];
+        const undecryptable: { id: number; error: string }[] = [];
         let cursor = args.sinceTime;
         for (const raw of raws) {
             if (raw.isDeleted) {
                 deleted.push(raw.id);
             } else {
-                collections.push(
-                    decryptCollection(
-                        raw,
-                        {
-                            masterKey: this.masterKey,
-                            publicKey: this.publicKey,
-                            secretKey: this.secretKey,
-                        },
-                        this.userID,
-                    ),
-                );
+                try {
+                    collections.push(
+                        decryptCollection(
+                            raw,
+                            {
+                                masterKey: this.masterKey,
+                                publicKey: this.publicKey,
+                                secretKey: this.secretKey,
+                            },
+                            this.userID,
+                        ),
+                    );
+                } catch (err) {
+                    undecryptable.push({
+                        id: raw.id,
+                        error: errorMessage(err),
+                    });
+                }
             }
             if (raw.updationTime > cursor) cursor = raw.updationTime;
         }
-        return { collections, deleted, cursor };
+        return { collections, deleted, undecryptable, cursor };
     }
 
     // Enumerate a collection's files changed since `sinceTime`, paginating the
     // diff from that cursor. Live rows are decrypted; tombstoned ones are
-    // surfaced as bare ids. Returns the final cursor to resume from.
+    // surfaced as bare ids, and ones that do not decrypt or parse as ids with
+    // the reason. Returns the final cursor to resume from.
     async filesSince(args: {
         collectionID: number;
         collectionKey: Uint8Array;
@@ -288,6 +306,7 @@ export class Client {
         const { collectionID, collectionKey } = args;
         const files: EnteFile[] = [];
         const deleted: number[] = [];
+        const undecryptable: { id: number; error: string }[] = [];
         let cursor = args.sinceTime;
         for (;;) {
             const { diff, hasMore } = await this.api.getJSON<{
@@ -300,7 +319,14 @@ export class Client {
                 if (raw.isDeleted) {
                     deleted.push(raw.id);
                 } else {
-                    files.push(decryptFile(raw, collectionKey));
+                    try {
+                        files.push(decryptFile(raw, collectionKey));
+                    } catch (err) {
+                        undecryptable.push({
+                            id: raw.id,
+                            error: errorMessage(err),
+                        });
+                    }
                 }
                 if (raw.updationTime > pageMax) pageMax = raw.updationTime;
             }
@@ -321,18 +347,19 @@ export class Client {
             }
             cursor = pageMax;
         }
-        return { files, deleted, cursor };
+        return { files, deleted, undecryptable, cursor };
     }
 
-    // Whole-account listing: every live collection, deletions hidden. A thin
-    // wrapper over `collectionsSince` from the beginning of time.
+    // Whole-account listing: every live collection that decrypts, deletions
+    // hidden. A thin wrapper over `collectionsSince` from the beginning of
+    // time.
     async listCollections(): Promise<Collection[]> {
         const { collections } = await this.collectionsSince({ sinceTime: 0 });
         return collections;
     }
 
-    // Every live file in a collection, deletions hidden. A thin wrapper over
-    // `filesSince` from the beginning of time.
+    // Every live file in a collection that decrypts, deletions hidden. A thin
+    // wrapper over `filesSince` from the beginning of time.
     async listFiles(
         collectionID: number,
         collectionKey: Uint8Array,

src/library/index.ts

@@ -601,6 +601,7 @@ export class Library {
                 refresh: () => this.refreshNow(),
                 listCollections: () => this.store.listCollections(),
                 listFiles: (id) => this.store.listFiles(id),
+                undecryptable: () => this.store.listUndecryptable(),
                 original: (fileID, destination) =>
                     cache!.backupOriginal(fileID, destination),
                 thumbnail: (fileID) => cache!.thumbnail(fileID),
@@ -703,7 +704,10 @@ export class Library {
 
     // Fetch every change since the stored cursor, then commit. All network
     // reads happen before any store mutation, so a fetch that throws leaves the
-    // store untouched and the previous snapshot intact.
+    // store untouched and the previous snapshot intact. A collection or file
+    // that does not decrypt is left out instead, as if deleted, with the reason
+    // kept in the store: the cursor moves past it, so no later refresh sees it
+    // again until the server's copy changes.
     private async refreshOnce(): Promise<void> {
         const page = await this.client.collectionsSince({
             sinceTime: this.store.collectionsSinceTime,
@@ -740,9 +744,17 @@ export class Library {
                 this.store.deleteCollection(id);
                 changed = true;
             }
+            if (this.store.deleteUndecryptable(id)) changed = true;
+        }
+        // A collection left out takes its files with it.
+        for (const { id, error } of page.undecryptable ?? []) {
+            this.store.deleteCollection(id);
+            this.store.putUndecryptable({ collectionID: id, error });
+            changed = true;
         }
         for (const collection of page.collections) {
             this.store.putCollection(collection);
+            this.store.deleteUndecryptable(collection.id);
             changed = true;
         }
 
@@ -752,9 +764,22 @@ export class Library {
                     this.store.deleteFile(collectionID, id);
                     changed = true;
                 }
+                if (this.store.deleteUndecryptable(collectionID, id)) {
+                    changed = true;
+                }
+            }
+            for (const { id, error } of filePage.undecryptable ?? []) {
+                this.store.deleteFile(collectionID, id);
+                this.store.putUndecryptable({
+                    collectionID,
+                    fileID: id,
+                    error,
+                });
+                changed = true;
             }
             for (const f of filePage.files) {
                 this.store.putFile(f);
+                this.store.deleteUndecryptable(collectionID, f.id);
                 changed = true;
             }
         }

src/library/store.ts

@@ -2,10 +2,11 @@
 //
 // The store keeps one `metadata.json` file holding the account's server
 // state: the user id, a schema version, the cursor for the incremental
-// collections listing, and the decrypted collection and file records. The
-// whole file is read into RAM on load and rewritten as a whole on save; there
-// is no partial update and no lock file. A separate refresh unit populates the
-// store from the server — this module only stores what it is given.
+// collections listing, the decrypted collection and file records, and the
+// collections and files the refresh could not decrypt. The whole file is read
+// into RAM on load and rewritten as a whole on save; there is no partial
+// update and no lock file. A separate refresh unit populates the store from
+// the server — this module only stores what it is given.
 //
 // The file is a cache, so it is never trusted to exist or to be intact: a
 // missing or unreadable file loads as an empty store rather than an error, and
@@ -33,12 +34,22 @@ const FILE_MODE = 0o600;
 type StoredCollection = Omit<Collection, "key"> & { key: string };
 type StoredFile = Omit<EnteFile, "key"> & { key: string };
 
+// A collection, or with `fileID` a file in it, that the last refresh to fetch
+// it could not decrypt or parse, and why. It is kept until a later refresh
+// fetches it again, because a refresh fetches only what changed.
+export interface Undecryptable {
+    collectionID: number;
+    fileID?: number;
+    error: string;
+}
+
 interface StoredMetadata {
     schemaVersion: number;
     userID: number;
     collectionsSinceTime: Microseconds;
     collections: StoredCollection[];
     files: StoredFile[];
+    undecryptable: Undecryptable[];
 }
 
 const encodeKey = (key: Uint8Array): string =>
@@ -67,6 +78,7 @@ export class MetadataStore {
 
     private readonly collections = new Map<number, Collection>();
     private readonly files = new Map<string, EnteFile>();
+    private undecryptable: Undecryptable[] = [];
 
     private constructor(path: string) {
         this.path = path;
@@ -107,6 +119,12 @@ export class MetadataStore {
                 };
                 store.files.set(fileKey(file.collectionID, file.id), file);
             }
+            for (const record of parsed.undecryptable) {
+                if (typeof record.collectionID !== "number") {
+                    throw new Error("damaged record of what did not decrypt");
+                }
+                store.undecryptable.push(record);
+            }
         } catch {
             // Any corruption discards the partial result: a half-read cache is
             // worse than an empty one, since the refresh unit will rebuild it.
@@ -134,6 +152,7 @@ export class MetadataStore {
                 ...f,
                 key: encodeKey(f.key),
             })),
+            undecryptable: this.undecryptable,
         };
 
         const dir = dirname(this.path);
@@ -165,7 +184,8 @@ export class MetadataStore {
         this.collections.set(collection.id, collection);
     }
 
-    // Removing a collection also drops its file memberships: a file record is
+    // Removing a collection also drops its file memberships, and any record
+    // that it or one of its files could not be decrypted: a file record is
     // only meaningful as part of a collection the cache still knows about.
     deleteCollection(id: number): void {
         this.collections.delete(id);
@@ -174,6 +194,9 @@ export class MetadataStore {
                 this.files.delete(key);
             }
         }
+        this.undecryptable = this.undecryptable.filter(
+            (record) => record.collectionID !== id,
+        );
     }
 
     getFile(collectionID: number, fileID: number): EnteFile | undefined {
@@ -205,4 +228,26 @@ export class MetadataStore {
     deleteFile(collectionID: number, fileID: number): void {
         this.files.delete(fileKey(collectionID, fileID));
     }
+
+    listUndecryptable(): Undecryptable[] {
+        return [...this.undecryptable];
+    }
+
+    // Replaces any earlier record of the same collection or file.
+    putUndecryptable(record: Undecryptable): void {
+        this.deleteUndecryptable(record.collectionID, record.fileID);
+        this.undecryptable.push(record);
+    }
+
+    // Forgets the record of the collection, or with `fileID` of that file in
+    // it. Returns whether there was one.
+    deleteUndecryptable(collectionID: number, fileID?: number): boolean {
+        const before = this.undecryptable.length;
+        this.undecryptable = this.undecryptable.filter(
+            (record) =>
+                record.collectionID !== collectionID ||
+                record.fileID !== fileID,
+        );
+        return this.undecryptable.length < before;
+    }
 }