quak backup works on up to the content pool's limit of originals at once (closes #182) #192
@@ -1099,8 +1099,7 @@ in-flight fetches by key, and an idle pool never lends its slots to a busy one.
|
||||
`lib.backup()`, and so `quak backup`, works on up to the content pool's limit of
|
||||
originals at once, 5 unless the library was opened with other `pools`. The limit
|
||||
covers everything it does for one original, hashing it for `verify` included, so
|
||||
however large the account is, no more originals than that are open at once. With
|
||||
`includeThumbnails`, it fetches the thumbnails one at a time.
|
||||
however large the account is, no more originals than that are open at once.
|
||||
|
||||
### On-disk cache layout
|
||||
|
||||
|
||||
+8
-4
@@ -657,8 +657,7 @@ const runLockedBackup = async (
|
||||
// present original is hashed first, and one that does not match the hash
|
||||
// its metadata records is removed and fetched like a missing one. One
|
||||
// that cannot be read is recorded as failed and left where it is.
|
||||
// `storeOriginal` does this for one original, and records its failure
|
||||
// rather than throwing.
|
||||
// `storeOriginal` does this for one original, and records its failure.
|
||||
const storeOriginal = async (file: EnteFile): Promise<void> => {
|
||||
let stored = storedAtSavePath(downloadDirectory, file);
|
||||
let mismatch = false;
|
||||
@@ -731,15 +730,20 @@ const runLockedBackup = async (
|
||||
// Up to the content pool's limit of originals are worked on at once,
|
||||
// their `verify` hashing included, so however large the account is, no
|
||||
// more than that many are open. Each worker takes the next file from the
|
||||
// iterator they all share until none is left.
|
||||
// iterator they all share until none is left. A worker that throws, as
|
||||
// one does when `onProgress` throws, fails the backup only once every
|
||||
// worker has stopped, so none writes into the backup after it has ended.
|
||||
if (includeOriginals) {
|
||||
const pending = distinct.values();
|
||||
const worker = async (): Promise<void> => {
|
||||
for (const file of pending) await storeOriginal(file);
|
||||
};
|
||||
await Promise.all(
|
||||
const workers = await Promise.allSettled(
|
||||
Array.from({ length: lib.contentConcurrency }, worker),
|
||||
);
|
||||
for (const w of workers) {
|
||||
if (w.status === "rejected") throw w.reason;
|
||||
}
|
||||
}
|
||||
|
||||
if (includeThumbnails) {
|
||||
|
||||
Reference in New Issue
Block a user