quak backup works on up to the content pool's limit of originals at once (closes #182) #192
@@ -1099,8 +1099,7 @@ in-flight fetches by key, and an idle pool never lends its slots to a busy one.
|
|||||||
`lib.backup()`, and so `quak backup`, works on up to the content pool's limit of
|
`lib.backup()`, and so `quak backup`, works on up to the content pool's limit of
|
||||||
originals at once, 5 unless the library was opened with other `pools`. The limit
|
originals at once, 5 unless the library was opened with other `pools`. The limit
|
||||||
covers everything it does for one original, hashing it for `verify` included, so
|
covers everything it does for one original, hashing it for `verify` included, so
|
||||||
however large the account is, no more originals than that are open at once. With
|
however large the account is, no more originals than that are open at once.
|
||||||
`includeThumbnails`, it fetches the thumbnails one at a time.
|
|
||||||
|
|
||||||
### On-disk cache layout
|
### On-disk cache layout
|
||||||
|
|
||||||
|
|||||||
+8
-4
@@ -657,8 +657,7 @@ const runLockedBackup = async (
|
|||||||
// present original is hashed first, and one that does not match the hash
|
// present original is hashed first, and one that does not match the hash
|
||||||
// its metadata records is removed and fetched like a missing one. One
|
// its metadata records is removed and fetched like a missing one. One
|
||||||
// that cannot be read is recorded as failed and left where it is.
|
// that cannot be read is recorded as failed and left where it is.
|
||||||
// `storeOriginal` does this for one original, and records its failure
|
// `storeOriginal` does this for one original, and records its failure.
|
||||||
// rather than throwing.
|
|
||||||
const storeOriginal = async (file: EnteFile): Promise<void> => {
|
const storeOriginal = async (file: EnteFile): Promise<void> => {
|
||||||
let stored = storedAtSavePath(downloadDirectory, file);
|
let stored = storedAtSavePath(downloadDirectory, file);
|
||||||
let mismatch = false;
|
let mismatch = false;
|
||||||
@@ -731,15 +730,20 @@ const runLockedBackup = async (
|
|||||||
// Up to the content pool's limit of originals are worked on at once,
|
// Up to the content pool's limit of originals are worked on at once,
|
||||||
// their `verify` hashing included, so however large the account is, no
|
// their `verify` hashing included, so however large the account is, no
|
||||||
// more than that many are open. Each worker takes the next file from the
|
// more than that many are open. Each worker takes the next file from the
|
||||||
// iterator they all share until none is left.
|
// iterator they all share until none is left. A worker that throws, as
|
||||||
|
// one does when `onProgress` throws, fails the backup only once every
|
||||||
|
// worker has stopped, so none writes into the backup after it has ended.
|
||||||
if (includeOriginals) {
|
if (includeOriginals) {
|
||||||
const pending = distinct.values();
|
const pending = distinct.values();
|
||||||
const worker = async (): Promise<void> => {
|
const worker = async (): Promise<void> => {
|
||||||
for (const file of pending) await storeOriginal(file);
|
for (const file of pending) await storeOriginal(file);
|
||||||
};
|
};
|
||||||
await Promise.all(
|
const workers = await Promise.allSettled(
|
||||||
Array.from({ length: lib.contentConcurrency }, worker),
|
Array.from({ length: lib.contentConcurrency }, worker),
|
||||||
);
|
);
|
||||||
|
for (const w of workers) {
|
||||||
|
if (w.status === "rejected") throw w.reason;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (includeThumbnails) {
|
if (includeThumbnails) {
|
||||||
|
|||||||
Reference in New Issue
Block a user