sneak/quak
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

Store live photos as their image and their video (closes #107)
check / check (push) Successful in 41s
Details

Browse Source 
A live photo, which Ente stores as one ZIP, is unpacked as it downloads
into its image and its video, each `<fileID>.<ext>` with its extension
from the ZIP, beside `<fileID>.livephoto.json`, which names the two.
Both are checked against the recorded hash and renamed into place only
when both are complete. A ZIP with a second image or video, or whose
parts come to more than 20 times its size plus 16 MiB, is refused. The
backup and the content cache count a live photo as stored only with both
files, album folders link both, `quak get` writes both, and the content
result gives the video as `videoPath`. A ZIP an earlier version stored
is replaced.

Model: opus-5-5
This commit was merged in pull request #128.
This commit is contained in:
clawbot
2026-09-28 18:06:01 +02:00
parent 04094a8cfb
commit 9e6deb21eb
17 changed files with 1746 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backup.ts
+105 -53
View File
@@ -5,7 +5,7 @@
// gets its original bytes onto disk under `downloadDirectory` and rebuilds the
// derived views (per-file sidecars, per-collection symlink trees,
// per-collection JSON) from the model. The on-disk layout is the historical
// one, unchanged:
// one:
//
// <downloadDirectory>/
// originals/<fileID>.<ext> the decrypted bytes
@@ -14,6 +14,10 @@
// collections/<name>.json per-collection metadata
// failures.json durable ledger of unresolved failures
//
// A live photo's original is its image and its video, `<fileID>.<ext>` each
// with its own extension, and `originals/<fileID>.livephoto.json` naming them;
// its album folders link both.
//
// Crash-safety rests on two properties. Bytes are present-means-complete: an
// original appears under `originals/` only via the content layer's atomic
// temp-then-rename, so a file that exists is whole and is never re-fetched — an
@@ -48,7 +52,12 @@ import { copyFile, rename, rm } from "node:fs/promises";
import { basename, dirname, extname, join, relative } from "node:path";
import { fsyncPath, removeLeftoverTempFiles } from "./download/index.js";
import { safeExtension, sanitizeFileName } from "./filename.js";
import { sanitizeFileName, withExtension } from "./filename.js";
import {
nameInOriginals,
storedOriginal,
writeLivePhotoJSON,
} from "./library/content.js";
import type { Collection, EnteFile } from "./model/types.js";
export type ProgressCallback = (message: string) => void;
@@ -98,8 +107,13 @@ export interface BackupLibrary {
listFiles(collectionID: number): EnteFile[];
// Get an original's bytes onto disk through the content cache/pools,
// returning where they landed: `destination` when they were fetched now,
// otherwise wherever they already were (the cache, or a prior backup).
original(fileID: number, destination: string): Promise<{ path: string }>;
// otherwise wherever they already were (the cache, or a prior backup). A
// live photo lands as its image and its video, fetched now beside
// `destination`.
original(
fileID: number,
destination: string,
): Promise<{ path: string; videoPath?: string }>;
thumbnail(fileID: number): Promise<{ path: string }>;
}
@@ -116,12 +130,6 @@ interface FailureEntry {
const LEDGER_VERSION = 1;
// The originals/ filename for a file: `<id><ext>`, the extension taken from the
// title (or `.bin`). Matches the content cache's own naming so a present check
// lines up with what a fetch would write.
const originalName = (file: EnteFile): string =>
`${file.id}${safeExtension(file.metadata.title)}`;
// A regular file with content is treated as complete. A zero-byte file is not:
// it is the shape an aborted write leaves and must be re-fetched.
const isPresent = (path: string): boolean => {
@@ -187,6 +195,31 @@ const copyAtomic = async (src: string, dest: string): Promise<void> => {
}
};
// Put an original the library returned at `dest` in originals/, where a fresh
// fetch already wrote it. A live photo's image and video go beside `dest`: when
// they came from the cache they are copied, after removing whatever was at
// `dest` (an earlier version's ZIP of the two). Then the JSON file naming them
// is written, which is what makes the live photo count as stored.
const placeOriginal = async (
file: EnteFile,
dest: string,
got: { path: string; videoPath?: string },
): Promise<void> => {
if (got.videoPath === undefined) {
await copyAtomic(got.path, dest);
return;
}
const originalsDir = dirname(dest);
const path = join(originalsDir, basename(got.path));
const videoPath = join(originalsDir, basename(got.videoPath));
if (got.path !== path) {
await rm(dest, { force: true });
await copyAtomic(got.path, path);
await copyAtomic(got.videoPath, videoPath);
}
await writeLivePhotoJSON(originalsDir, file.id, { path, videoPath });
};
// Ensure `linkPath` is a symlink to `target`, rebuilding a missing, wrong, or
// non-symlink entry. Throws on failure (a directory in the way, no permission)
// so the caller records it and moves on rather than aborting the run.
@@ -203,43 +236,61 @@ const rebuildSymlink = (linkPath: string, target: string): void => {
symlinkSync(target, linkPath);
};
// The on-disk names for the entries of one directory, keyed by ID. Each name
// is used as is unless another entry would get the same name, ignoring case
// (two names that differ only in case are one entry on a case-insensitive
// The on-disk names for the entries of one directory, in entry order. Each
// name is used as is unless another entry would get the same name, ignoring
// case (two names that differ only in case are one entry on a case-insensitive
// file system); then every entry sharing it gets ` (<id>)`, before the
// extension when `beforeExtension` is set. A name with an ID added can match
// another entry's own name (`IMG (6).JPG`), so this repeats until no name is
// shared. IDs are stable, so the names are too.
const namesByID = (
const uniqueNames = (
entries: { id: number; name: string }[],
beforeExtension: boolean,
): Map<number, string> => {
): string[] => {
const withID = (id: number, name: string): string => {
const ext = beforeExtension ? extname(name) : "";
const stem = name.slice(0, name.length - ext.length);
return `${stem} (${id})${ext}`;
};
const names = new Map<number, string>();
for (const { id, name } of entries) names.set(id, name);
const names = entries.map((e) => e.name);
const suffixed = new Set<number>();
for (;;) {
const counts = new Map<string, number>();
for (const name of names.values()) {
for (const name of names) {
const key = name.toLowerCase();
counts.set(key, (counts.get(key) ?? 0) + 1);
}
let changed = false;
for (const { id, name } of entries) {
if (suffixed.has(id)) continue;
for (const [i, { id, name }] of entries.entries()) {
if (suffixed.has(i)) continue;
if (counts.get(name.toLowerCase()) === 1) continue;
names.set(id, withID(id, name));
suffixed.add(id);
names[i] = withID(id, name);
suffixed.add(i);
changed = true;
}
if (!changed) return names;
}
};
// The links a file gets in its album's folder: one named after its title, to
// its original if that is stored. A stored live photo gets two, to its image
// and its video, each named after the title with that file's extension.
const linksFor = (
file: EnteFile,
stored: { path: string; videoPath?: string } | undefined,
): { id: number; name: string; file: EnteFile; target?: string }[] => {
const name = sanitizeFileName(file.metadata.title, `file-${file.id}`);
if (stored?.videoPath === undefined) {
return [{ id: file.id, name, file, target: stored?.path }];
}
return [stored.path, stored.videoPath].map((target) => ({
id: file.id,
name: withExtension(name, extname(target)),
file,
target,
}));
};
// Remove the symlinks in the album directory `dir` that point into
// `originalsDir` and are not named in `keep`. Nothing else in the directory
// is touched: anything else there was put there by the user.
@@ -419,17 +470,21 @@ export const runBackup = async (
// tree; a present file is left as is.
if (includeOriginals) {
for (const [fileID, file] of distinct) {
const dest = join(originalsDir, originalName(file));
if (isPresent(dest)) {
if (storedOriginal(originalsDir, file) !== undefined) {
skipped++;
continue;
}
const dest = join(originalsDir, nameInOriginals(file));
try {
log(`Fetching original ${file.metadata.title} (${fileID})...`);
// A fetched original is written straight to `dest`; only one
// that was already cached elsewhere is copied.
const { path } = await lib.original(fileID, dest);
await copyAtomic(path, dest);
// A fetched original is written straight to `dest` (a live
// photo beside it); only one that was already cached elsewhere
// is copied.
await placeOriginal(
file,
dest,
await lib.original(fileID, dest),
);
downloaded++;
} catch (err) {
log(
@@ -465,8 +520,7 @@ export const runBackup = async (
// every present original (this repairs stale ones).
if (includeOriginals) {
for (const [fileID, file] of distinct) {
const orig = join(originalsDir, originalName(file));
if (isPresent(orig)) {
if (storedOriginal(originalsDir, file) !== undefined) {
writeSidecar(join(originalsDir, `${fileID}.json`), file);
}
}
@@ -478,19 +532,18 @@ export const runBackup = async (
// an album it skipped. Stale entries are removed before anything is
// rebuilt, so on a case-insensitive file system removing an old name can
// never remove the new one.
const albumDirNames = namesByID(
const dirNames = uniqueNames(
allCollections.map((c) => ({
id: c.id,
name: sanitizeFileName(c.name, `collection-${c.id}`),
})),
false,
);
const albumDirNames = new Map(
allCollections.map((c, i) => [c.id, dirNames[i]!]),
);
try {
removeStaleAlbumDirs(
collectionsDir,
new Set(albumDirNames.values()),
originalsDir,
);
removeStaleAlbumDirs(collectionsDir, new Set(dirNames), originalsDir);
} catch (err) {
log(`FAILED removing old album directories: ${errorMessage(err)}`);
}
@@ -501,34 +554,33 @@ export const runBackup = async (
mkdirSync(colDir, { recursive: true });
const files = filesByCollection.get(c.id) ?? [];
const linkNames = namesByID(
files.map((f) => ({
id: f.id,
name: sanitizeFileName(f.metadata.title, `file-${f.id}`),
})),
true,
const links = files.flatMap((f) =>
linksFor(f, storedOriginal(originalsDir, f)),
);
const linkNames = uniqueNames(links, true);
try {
removeStaleLinks(colDir, new Set(linkNames.values()), originalsDir);
removeStaleLinks(colDir, new Set(linkNames), originalsDir);
} catch (err) {
log(`FAILED removing old links in ${c.name}: ${errorMessage(err)}`);
}
const metaFiles: { id: number; metadata: EnteFile["metadata"] }[] = [];
for (const file of files) {
metaFiles.push({ id: file.id, metadata: file.metadata });
if (!includeOriginals) continue;
const orig = join(originalsDir, originalName(file));
if (!isPresent(orig)) continue;
const linkName = linkNames.get(file.id)!;
const linkPath = join(colDir, linkName);
const metaFiles = files.map((f) => ({
id: f.id,
metadata: f.metadata,
}));
for (const [i, link] of links.entries()) {
if (!includeOriginals || link.target === undefined) continue;
const linkName = linkNames[i]!;
try {
rebuildSymlink(linkPath, relative(colDir, orig));
rebuildSymlink(
join(colDir, linkName),
relative(colDir, link.target),
);
} catch (err) {
log(
`FAILED symlink ${c.name}/${linkName}: ${errorMessage(err)}`,
);
recordFailure(file, c.name, err);
recordFailure(link.file, c.name, err);
}
}