quak backup works on up to the content pool's limit of originals at once (closes #182)
lib.backup() ran its originals one at a time, so the content pool's five slots, or the contentConcurrency of the pools a library was opened with, went unused. The work for one original, its verify hashing included, is now storeOriginal, and that many workers take files from one iterator over the backup's files, so no more originals than the limit are open at once. BackupLibrary gains contentConcurrency, which the library sets from its content pool. The README's "Request pools" says so. The tests of a backup whose lock is lost now expect all three originals downloaded, since all three are under way at once. Model: opus-5-5
This commit is contained in:
+75
-67
@@ -6,9 +6,9 @@
|
||||
// file. It waits for a completed refresh of the library (a failed one fails the
|
||||
// backup before any file is touched), then, for every file in scope, puts its
|
||||
// original at its save path under `downloadDirectory`, as `Photo.download()`
|
||||
// does, waits for an ML data fetch, and rebuilds the derived views (per-file
|
||||
// sidecars, per-collection symlink trees, per-collection JSON) from the model.
|
||||
// The on-disk layout:
|
||||
// does, up to the content pool's limit of them at once, waits for an ML data
|
||||
// fetch, and rebuilds the derived views (per-file sidecars, per-collection
|
||||
// symlink trees, per-collection JSON) from the model. The on-disk layout:
|
||||
//
|
||||
// <downloadDirectory>/
|
||||
// YYYY/YYYY-MM/YYYY-MM-DD/
|
||||
@@ -181,6 +181,9 @@ export interface BackupLibrary {
|
||||
destination: string,
|
||||
): Promise<{ path: string; videoPath?: string }>;
|
||||
thumbnail(fileID: number): Promise<{ path: string }>;
|
||||
// How many originals the backup works on at once: the limit of the
|
||||
// library's content pool.
|
||||
contentConcurrency: number;
|
||||
// Wait for an ML data fetch to complete, joining one already running or
|
||||
// starting one. Rejects with the reason when it fails; resolves at once
|
||||
// when the library cannot fetch ML data.
|
||||
@@ -654,75 +657,32 @@ const runLockedBackup = async (
|
||||
// present original is hashed first, and one that does not match the hash
|
||||
// its metadata records is removed and fetched like a missing one. One
|
||||
// that cannot be read is recorded as failed and left where it is.
|
||||
if (includeOriginals) {
|
||||
for (const [fileID, file] of distinct) {
|
||||
if (lock.lost) throw lock.lost;
|
||||
let stored = storedAtSavePath(downloadDirectory, file);
|
||||
let mismatch = false;
|
||||
if (stored !== undefined && verify) {
|
||||
try {
|
||||
if (file.metadata.hash === undefined) {
|
||||
unchecked++;
|
||||
} else if (
|
||||
(await storedHash(stored)) === file.metadata.hash
|
||||
) {
|
||||
verified++;
|
||||
} else {
|
||||
log(
|
||||
`MISMATCH original ${file.metadata.title} (${fileID}): its bytes do not match its content hash`,
|
||||
);
|
||||
mismatched++;
|
||||
mismatch = true;
|
||||
rmSync(stored.path);
|
||||
if (stored.videoPath !== undefined) {
|
||||
rmSync(stored.videoPath);
|
||||
}
|
||||
stored = undefined;
|
||||
}
|
||||
} catch (err) {
|
||||
log(
|
||||
`FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
|
||||
);
|
||||
recordFailure(
|
||||
file.id,
|
||||
file.metadata.title,
|
||||
collectionName.get(file.collectionID) ?? "",
|
||||
err,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (stored !== undefined) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
// `storeOriginal` does this for one original, and records its failure
|
||||
// rather than throwing.
|
||||
const storeOriginal = async (file: EnteFile): Promise<void> => {
|
||||
let stored = storedAtSavePath(downloadDirectory, file);
|
||||
let mismatch = false;
|
||||
if (stored !== undefined && verify) {
|
||||
try {
|
||||
log(`Fetching original ${file.metadata.title} (${fileID})...`);
|
||||
// A fetched original is written straight to its save path (a
|
||||
// live photo beside it); only one that was already cached
|
||||
// elsewhere is copied.
|
||||
const placed = await placeOriginal(
|
||||
downloadDirectory,
|
||||
file,
|
||||
(dest) => lib.original(fileID, dest),
|
||||
);
|
||||
storedThisRun.add(fileID);
|
||||
// A copy from the cache is not checked as a download is and can
|
||||
// hold the same bad bytes, so what is put back after a
|
||||
// mismatch is hashed too. A bad copy stays where it is and the
|
||||
// file fails.
|
||||
if (
|
||||
mismatch &&
|
||||
(await storedHash(placed)) !== file.metadata.hash
|
||||
) {
|
||||
throw new Error(
|
||||
"the original put back does not match its content hash either",
|
||||
if (file.metadata.hash === undefined) {
|
||||
unchecked++;
|
||||
} else if ((await storedHash(stored)) === file.metadata.hash) {
|
||||
verified++;
|
||||
} else {
|
||||
log(
|
||||
`MISMATCH original ${file.metadata.title} (${file.id}): its bytes do not match its content hash`,
|
||||
);
|
||||
mismatched++;
|
||||
mismatch = true;
|
||||
rmSync(stored.path);
|
||||
if (stored.videoPath !== undefined) {
|
||||
rmSync(stored.videoPath);
|
||||
}
|
||||
stored = undefined;
|
||||
}
|
||||
downloaded++;
|
||||
} catch (err) {
|
||||
log(
|
||||
`FAILED original ${file.metadata.title}: ${errorMessage(err)}`,
|
||||
`FAILED verifying original ${file.metadata.title}: ${errorMessage(err)}`,
|
||||
);
|
||||
recordFailure(
|
||||
file.id,
|
||||
@@ -730,8 +690,56 @@ const runLockedBackup = async (
|
||||
collectionName.get(file.collectionID) ?? "",
|
||||
err,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (stored !== undefined) {
|
||||
skipped++;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
log(`Fetching original ${file.metadata.title} (${file.id})...`);
|
||||
// A fetched original is written straight to its save path (a
|
||||
// live photo beside it); only one that was already cached
|
||||
// elsewhere is copied.
|
||||
const placed = await placeOriginal(
|
||||
downloadDirectory,
|
||||
file,
|
||||
(dest) => lib.original(file.id, dest),
|
||||
);
|
||||
storedThisRun.add(file.id);
|
||||
// A copy from the cache is not checked as a download is and can
|
||||
// hold the same bad bytes, so what is put back after a mismatch
|
||||
// is hashed too. A bad copy stays where it is and the file fails.
|
||||
if (mismatch && (await storedHash(placed)) !== file.metadata.hash) {
|
||||
throw new Error(
|
||||
"the original put back does not match its content hash either",
|
||||
);
|
||||
}
|
||||
downloaded++;
|
||||
} catch (err) {
|
||||
log(`FAILED original ${file.metadata.title}: ${errorMessage(err)}`);
|
||||
recordFailure(
|
||||
file.id,
|
||||
file.metadata.title,
|
||||
collectionName.get(file.collectionID) ?? "",
|
||||
err,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Up to the content pool's limit of originals are worked on at once,
|
||||
// their `verify` hashing included, so however large the account is, no
|
||||
// more than that many are open. Each worker takes the next file from the
|
||||
// iterator they all share until none is left.
|
||||
if (includeOriginals) {
|
||||
const pending = distinct.values();
|
||||
const worker = async (): Promise<void> => {
|
||||
for (const file of pending) await storeOriginal(file);
|
||||
};
|
||||
await Promise.all(
|
||||
Array.from({ length: lib.contentConcurrency }, worker),
|
||||
);
|
||||
}
|
||||
|
||||
if (includeThumbnails) {
|
||||
|
||||
@@ -607,6 +607,7 @@ export class Library {
|
||||
original: (fileID, destination) =>
|
||||
cache!.backupOriginal(fileID, destination),
|
||||
thumbnail: (fileID) => cache!.thumbnail(fileID),
|
||||
contentConcurrency: this.pools.content.concurrency,
|
||||
fetchMLData: () => this.fetchMLDataNow(),
|
||||
mlData: (fileID) => this.mldata.forFile({ fileID }),
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user