Write each backed-up original once and skip the precache (closes #106)
check / check (push) Successful in 49s
check / check (push) Successful in 49s
An original fetched for a backup is now written by the download writer straight into the backup's originals/, and the content cache records it there instead of keeping its own copy. One the cache already held is still copied. `quak backup` opens its library with the thumbnail and originals precache off, as the one-shot commands do. Model: opus-5-5
This commit was merged in pull request #124.
This commit is contained in:
+8
-4
@@ -4,7 +4,8 @@
|
||||
// fails the backup before any file is touched), then, for every file in scope,
|
||||
// gets its original bytes onto disk under `downloadDirectory` and rebuilds the
|
||||
// derived views (per-file sidecars, per-collection symlink trees,
|
||||
// per-collection JSON) from the model. The on-disk layout is the historical one, unchanged:
|
||||
// per-collection JSON) from the model. The on-disk layout is the historical
|
||||
// one, unchanged:
|
||||
//
|
||||
// <downloadDirectory>/
|
||||
// originals/<fileID>.<ext> the decrypted bytes
|
||||
@@ -96,8 +97,9 @@ export interface BackupLibrary {
|
||||
listCollections(): Collection[];
|
||||
listFiles(collectionID: number): EnteFile[];
|
||||
// Get an original's bytes onto disk through the content cache/pools,
|
||||
// returning where they landed (the cache, or a prior backup).
|
||||
original(fileID: number): Promise<{ path: string }>;
|
||||
// returning where they landed: `destination` when they were fetched now,
|
||||
// otherwise wherever they already were (the cache, or a prior backup).
|
||||
original(fileID: number, destination: string): Promise<{ path: string }>;
|
||||
thumbnail(fileID: number): Promise<{ path: string }>;
|
||||
}
|
||||
|
||||
@@ -424,7 +426,9 @@ export const runBackup = async (
|
||||
}
|
||||
try {
|
||||
log(`Fetching original ${file.metadata.title} (${fileID})...`);
|
||||
const { path } = await lib.original(fileID);
|
||||
// A fetched original is written straight to `dest`; only one
|
||||
// that was already cached elsewhere is copied.
|
||||
const { path } = await lib.original(fileID, dest);
|
||||
await copyAtomic(path, dest);
|
||||
downloaded++;
|
||||
} catch (err) {
|
||||
|
||||
@@ -380,10 +380,14 @@ export const backupCommand = async (
|
||||
if (!client) return 1;
|
||||
|
||||
ctx.stderr.write("Starting backup...\n");
|
||||
// The precache is off: the backup fetches what it needs, and must not
|
||||
// also fill the cache with every thumbnail and the recent originals.
|
||||
const lib = await Library.open({
|
||||
client,
|
||||
downloadDirectory: dir,
|
||||
cacheDirectory: ctx.cacheDir,
|
||||
precacheThumbnails: false,
|
||||
precacheOriginals: false,
|
||||
});
|
||||
try {
|
||||
const result = await lib.backup({
|
||||
|
||||
+21
-3
@@ -321,6 +321,23 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
return this.get(fileID, "thumbnail", "on-demand", opts?.onProgress);
|
||||
}
|
||||
|
||||
// Get an original for a backup. One not present anywhere is written
|
||||
// straight to `destination` and recorded there, so no second copy lands
|
||||
// in the cache; one already present is returned where it is.
|
||||
async backupOriginal(
|
||||
fileID: number,
|
||||
destination: string,
|
||||
): Promise<ContentResult> {
|
||||
const result = await this.acquire(
|
||||
fileID,
|
||||
"original",
|
||||
"on-demand",
|
||||
undefined,
|
||||
{ destination },
|
||||
);
|
||||
return { path: result.path, bytes: result.bytes };
|
||||
}
|
||||
|
||||
async ensure(args: EnsureOptions): Promise<EnsureResult[]> {
|
||||
return this.ensureThumbnails(args);
|
||||
}
|
||||
@@ -422,13 +439,14 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
|
||||
// The core: return the cached path if present, else fetch through the pool,
|
||||
// store, and return it. `cached` distinguishes a present hit (no network,
|
||||
// no download event) from a fresh fetch.
|
||||
// no download event) from a fresh fetch. A fetched original is stored at
|
||||
// `opts.destination` when given, instead of in `originalsDir`.
|
||||
private async acquire(
|
||||
fileID: number,
|
||||
kind: Kind,
|
||||
priority: Priority,
|
||||
signal: AbortSignal | undefined,
|
||||
opts?: { onByte?: ProgressCallback },
|
||||
opts?: { onByte?: ProgressCallback; destination?: string },
|
||||
): Promise<{ path: string; bytes: number; cached: boolean }> {
|
||||
const file = this.getFile(fileID);
|
||||
if (!file) throw new Error(`content cache: unknown file ${fileID}`);
|
||||
@@ -469,7 +487,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
kind === "original" ? this.originalsDir : this.thumbnailsDir;
|
||||
const dest =
|
||||
kind === "original"
|
||||
? join(dir, originalName(file))
|
||||
? (opts?.destination ?? join(dir, originalName(file)))
|
||||
: join(dir, `${fileID}${THUMBNAIL_EXT}`);
|
||||
const pool =
|
||||
kind === "original" ? this.pools.content : this.pools.thumbnails;
|
||||
|
||||
@@ -571,7 +571,8 @@ export class Library {
|
||||
refresh: () => this.refreshNow(),
|
||||
listCollections: () => this.store.listCollections(),
|
||||
listFiles: (id) => this.store.listFiles(id),
|
||||
original: (fileID) => cache!.original(fileID),
|
||||
original: (fileID, destination) =>
|
||||
cache!.backupOriginal(fileID, destination),
|
||||
thumbnail: (fileID) => cache!.thumbnail(fileID),
|
||||
},
|
||||
{ ...opts, downloadDirectory },
|
||||
|
||||
Reference in New Issue
Block a user