One album or file quak cannot decrypt no longer stops a refresh or a backup (closes #181)

collectionsSince and filesSince put a row that does not decrypt or parse in
an undecryptable list with the reason instead of throwing. The refresh
leaves it out, an album with its files, and the store keeps the list in
metadata.json until the server sends a copy that decrypts or deletes it,
since the next refresh asks only for changes. lib.backup() records each
such file in failures.json as "could not be decrypted: <reason>", unless a
copy in another album decrypted, logs one line per such album and lists
their IDs in undecryptableAlbums; quak backup exits 1 while any remain.
Tests use a stand-in Ente server with real encryption.

Model: opus-5-5

src/library/store.ts

@@ -2,10 +2,11 @@
 //
 // The store keeps one `metadata.json` file holding the account's server
 // state: the user id, a schema version, the cursor for the incremental
-// collections listing, and the decrypted collection and file records. The
-// whole file is read into RAM on load and rewritten as a whole on save; there
-// is no partial update and no lock file. A separate refresh unit populates the
-// store from the server — this module only stores what it is given.
+// collections listing, the decrypted collection and file records, and the
+// collections and files the refresh could not decrypt. The whole file is read
+// into RAM on load and rewritten as a whole on save; there is no partial
+// update and no lock file. A separate refresh unit populates the store from
+// the server — this module only stores what it is given.
 //
 // The file is a cache, so it is never trusted to exist or to be intact: a
 // missing or unreadable file loads as an empty store rather than an error, and
@@ -33,12 +34,22 @@ const FILE_MODE = 0o600;
 type StoredCollection = Omit<Collection, "key"> & { key: string };
 type StoredFile = Omit<EnteFile, "key"> & { key: string };
 
+// A collection, or with `fileID` a file in it, that the last refresh to fetch
+// it could not decrypt or parse, and why. It is kept until a later refresh
+// fetches it again, because a refresh fetches only what changed.
+export interface Undecryptable {
+    collectionID: number;
+    fileID?: number;
+    error: string;
+}
+
 interface StoredMetadata {
     schemaVersion: number;
     userID: number;
     collectionsSinceTime: Microseconds;
     collections: StoredCollection[];
     files: StoredFile[];
+    undecryptable?: Undecryptable[];
 }
 
 const encodeKey = (key: Uint8Array): string =>
@@ -67,6 +78,7 @@ export class MetadataStore {
 
     private readonly collections = new Map<number, Collection>();
     private readonly files = new Map<string, EnteFile>();
+    private undecryptable: Undecryptable[] = [];
 
     private constructor(path: string) {
         this.path = path;
@@ -107,6 +119,7 @@ export class MetadataStore {
                 };
                 store.files.set(fileKey(file.collectionID, file.id), file);
             }
+            store.undecryptable = parsed.undecryptable ?? [];
         } catch {
             // Any corruption discards the partial result: a half-read cache is
             // worse than an empty one, since the refresh unit will rebuild it.
@@ -134,6 +147,7 @@ export class MetadataStore {
                 ...f,
                 key: encodeKey(f.key),
             })),
+            undecryptable: this.undecryptable,
         };
 
         const dir = dirname(this.path);
@@ -165,7 +179,8 @@ export class MetadataStore {
         this.collections.set(collection.id, collection);
     }
 
-    // Removing a collection also drops its file memberships: a file record is
+    // Removing a collection also drops its file memberships, and any record
+    // that it or one of its files could not be decrypted: a file record is
     // only meaningful as part of a collection the cache still knows about.
     deleteCollection(id: number): void {
         this.collections.delete(id);
@@ -174,6 +189,9 @@ export class MetadataStore {
                 this.files.delete(key);
             }
         }
+        this.undecryptable = this.undecryptable.filter(
+            (record) => record.collectionID !== id,
+        );
     }
 
     getFile(collectionID: number, fileID: number): EnteFile | undefined {
@@ -205,4 +223,26 @@ export class MetadataStore {
     deleteFile(collectionID: number, fileID: number): void {
         this.files.delete(fileKey(collectionID, fileID));
     }
+
+    listUndecryptable(): Undecryptable[] {
+        return [...this.undecryptable];
+    }
+
+    // Replaces any earlier record of the same collection or file.
+    putUndecryptable(record: Undecryptable): void {
+        this.deleteUndecryptable(record.collectionID, record.fileID);
+        this.undecryptable.push(record);
+    }
+
+    // Forgets the record of the collection, or with `fileID` of that file in
+    // it. Returns whether there was one.
+    deleteUndecryptable(collectionID: number, fileID?: number): boolean {
+        const before = this.undecryptable.length;
+        this.undecryptable = this.undecryptable.filter(
+            (record) =>
+                record.collectionID !== collectionID ||
+                record.fileID !== fileID,
+        );
+        return this.undecryptable.length < before;
+    }
 }