quak backup writes the account and album records backup-metadata writes (closes #166)
check / check (push) Successful in 2m14s
check / check (push) Successful in 2m14s
quak backup now writes account.json with the account's email and user ID, adds each album's ownerID, isShared, updationTime and, when present, its magicMetadata, pubMagicMetadata and sharedMagicMetadata to the album's JSON, and adds updationTime to each file's JSON. The fields and their order match backup-metadata's account.json, _collection.json and per-file JSON. The interface runBackup drives gains whoami, which the library passes through from its client. Model: opus-5-5
This commit is contained in:
@@ -599,8 +599,8 @@ the smallest does not.
|
|||||||
per unique file, two for a live photo: see
|
per unique file, two for a live photo: see
|
||||||
below)
|
below)
|
||||||
YYYY-MM-DD.<fileID>.json the file's basic metadata fields quak
|
YYYY-MM-DD.<fileID>.json the file's basic metadata fields quak
|
||||||
keeps, its private and public magic
|
keeps, its update time, its private and
|
||||||
metadata, and its ML data
|
public magic metadata, and its ML data
|
||||||
YYYY-MM-DD.<fileID>.livephoto.json
|
YYYY-MM-DD.<fileID>.livephoto.json
|
||||||
which of a live photo's two files is which
|
which of a live photo's two files is which
|
||||||
collections/
|
collections/
|
||||||
@@ -608,6 +608,7 @@ the smallest does not.
|
|||||||
<title> -> ../../YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>
|
<title> -> ../../YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>
|
||||||
(symlink)
|
(symlink)
|
||||||
<name>.json collection metadata + file list
|
<name>.json collection metadata + file list
|
||||||
|
account.json the account's email and user ID
|
||||||
failures.json files that failed and have not yet succeeded
|
failures.json files that failed and have not yet succeeded
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -619,6 +620,15 @@ name as uploaded, case kept, or `.bin` when it has none or it holds anything but
|
|||||||
letters and digits. When the date or the time zone changes, the next run saves
|
letters and digits. When the date or the time zone changes, the next run saves
|
||||||
the original at its new path and leaves the old copy where it is.
|
the original at its new path and leaves the old copy where it is.
|
||||||
|
|
||||||
|
`account.json` holds the account's `email` and `userID`, as `backup-metadata`
|
||||||
|
writes it. A collection's JSON holds its `id`, `name`, `type`, `ownerID`,
|
||||||
|
`isShared` and `updationTime`, its `magicMetadata`, `pubMagicMetadata` and
|
||||||
|
`sharedMagicMetadata` when it has them, as `backup-metadata`'s
|
||||||
|
`_collection.json` does, and `files`, each file's `id` and `metadata`. A file's
|
||||||
|
JSON holds its `id`, `collectionID`, `ownerID`, `metadata` and `updationTime`,
|
||||||
|
and its `magicMetadata` and `pubMagicMetadata` when it has them. Update times
|
||||||
|
are in microseconds, as Ente records them.
|
||||||
|
|
||||||
A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as
|
A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as
|
||||||
`mlData`, the same payload `backup-metadata` writes; a file Ente has no ML data
|
`mlData`, the same payload `backup-metadata` writes; a file Ente has no ML data
|
||||||
for has no `mlData`. The backup waits for the library's ML data fetch to finish
|
for has no `mlData`. The backup waits for the library's ML data fetch to finish
|
||||||
|
|||||||
@@ -25,6 +25,12 @@ declares one.
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-06: `quak backup` writes the account and album records
|
||||||
|
`backup-metadata` writes (issue 166): `account.json` with the account's
|
||||||
|
`email` and `userID`, and in each album's JSON its `ownerID`, `isShared`,
|
||||||
|
`updationTime` and, when present, its three layers of magic metadata. Each
|
||||||
|
file's JSON gains its `updationTime`.
|
||||||
|
|
||||||
- 2026-10-05: `quak backup` writes each file's ML data (its faces and CLIP
|
- 2026-10-05: `quak backup` writes each file's ML data (its faces and CLIP
|
||||||
embedding) into the file's JSON as `mlData`, the payload
|
embedding) into the file's JSON as `mlData`, the payload
|
||||||
`lib.mldata.forFile()` returns (issue 163). `lib.backup()` waits for an ML
|
`lib.mldata.forFile()` returns (issue 163). `lib.backup()` waits for an ML
|
||||||
|
|||||||
+36
-7
@@ -14,6 +14,7 @@
|
|||||||
// the file's ML data
|
// the file's ML data
|
||||||
// collections/<name>/<title> symlink to the original
|
// collections/<name>/<title> symlink to the original
|
||||||
// collections/<name>.json per-collection metadata
|
// collections/<name>.json per-collection metadata
|
||||||
|
// account.json the account's email and user ID
|
||||||
// failures.json durable ledger of unresolved failures
|
// failures.json durable ledger of unresolved failures
|
||||||
//
|
//
|
||||||
// A live photo's original is its image and its video, each with its own
|
// A live photo's original is its image and its video, each with its own
|
||||||
@@ -64,7 +65,7 @@ import {
|
|||||||
} from "./library/content.js";
|
} from "./library/content.js";
|
||||||
import { representative } from "./library/records.js";
|
import { representative } from "./library/records.js";
|
||||||
import type { MLData } from "./mldata-fetch.js";
|
import type { MLData } from "./mldata-fetch.js";
|
||||||
import type { Collection, EnteFile } from "./model/types.js";
|
import type { Collection, EnteFile, FileMetadata } from "./model/types.js";
|
||||||
|
|
||||||
export type ProgressCallback = (message: string) => void;
|
export type ProgressCallback = (message: string) => void;
|
||||||
|
|
||||||
@@ -108,6 +109,8 @@ export interface BackupResult {
|
|||||||
// The slice of the library that backup drives. `Library` implements it; a test
|
// The slice of the library that backup drives. `Library` implements it; a test
|
||||||
// can drive backup with a stand-in.
|
// can drive backup with a stand-in.
|
||||||
export interface BackupLibrary {
|
export interface BackupLibrary {
|
||||||
|
// The account the library belongs to.
|
||||||
|
whoami(): { email: string; userID: number };
|
||||||
refresh(): Promise<void>;
|
refresh(): Promise<void>;
|
||||||
listCollections(): Collection[];
|
listCollections(): Collection[];
|
||||||
listFiles(collectionID: number): EnteFile[];
|
listFiles(collectionID: number): EnteFile[];
|
||||||
@@ -363,6 +366,7 @@ const writeSidecar = (
|
|||||||
collectionID: file.collectionID,
|
collectionID: file.collectionID,
|
||||||
ownerID: file.ownerID,
|
ownerID: file.ownerID,
|
||||||
metadata: file.metadata,
|
metadata: file.metadata,
|
||||||
|
updationTime: file.updationTime,
|
||||||
};
|
};
|
||||||
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
||||||
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
||||||
@@ -371,6 +375,29 @@ const writeSidecar = (
|
|||||||
writeFileSync(path, JSON.stringify(meta, null, 2));
|
writeFileSync(path, JSON.stringify(meta, null, 2));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// The album's JSON: its basic fields, its magic metadata, and its files.
|
||||||
|
const writeAlbumJSON = (
|
||||||
|
path: string,
|
||||||
|
c: Collection,
|
||||||
|
files: { id: number; metadata: FileMetadata }[],
|
||||||
|
): void => {
|
||||||
|
const album: Record<string, unknown> = {
|
||||||
|
id: c.id,
|
||||||
|
name: c.name,
|
||||||
|
type: c.type,
|
||||||
|
ownerID: c.ownerID,
|
||||||
|
isShared: c.isShared,
|
||||||
|
updationTime: c.updationTime,
|
||||||
|
};
|
||||||
|
if (c.magicMetadata) album.magicMetadata = c.magicMetadata;
|
||||||
|
if (c.pubMagicMetadata) album.pubMagicMetadata = c.pubMagicMetadata;
|
||||||
|
if (c.sharedMagicMetadata) {
|
||||||
|
album.sharedMagicMetadata = c.sharedMagicMetadata;
|
||||||
|
}
|
||||||
|
album.files = files;
|
||||||
|
writeFileSync(path, JSON.stringify(album, null, 2));
|
||||||
|
};
|
||||||
|
|
||||||
export const runBackup = async (
|
export const runBackup = async (
|
||||||
lib: BackupLibrary,
|
lib: BackupLibrary,
|
||||||
opts: BackupOptions,
|
opts: BackupOptions,
|
||||||
@@ -393,6 +420,11 @@ export const runBackup = async (
|
|||||||
const collectionsDir = join(downloadDirectory, "collections");
|
const collectionsDir = join(downloadDirectory, "collections");
|
||||||
const thumbnailsDir = join(downloadDirectory, "thumbnails");
|
const thumbnailsDir = join(downloadDirectory, "thumbnails");
|
||||||
mkdirSync(collectionsDir, { recursive: true });
|
mkdirSync(collectionsDir, { recursive: true });
|
||||||
|
const { email, userID } = lib.whoami();
|
||||||
|
writeFileSync(
|
||||||
|
join(downloadDirectory, "account.json"),
|
||||||
|
JSON.stringify({ email, userID }, null, 2),
|
||||||
|
);
|
||||||
if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true });
|
if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true });
|
||||||
removeLeftoverTempFiles(thumbnailsDir);
|
removeLeftoverTempFiles(thumbnailsDir);
|
||||||
for (const dir of dateFolders(downloadDirectory)) {
|
for (const dir of dateFolders(downloadDirectory)) {
|
||||||
@@ -618,13 +650,10 @@ export const runBackup = async (
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
writeFileSync(
|
writeAlbumJSON(
|
||||||
join(collectionsDir, `${colDirName}.json`),
|
join(collectionsDir, `${colDirName}.json`),
|
||||||
JSON.stringify(
|
c,
|
||||||
{ id: c.id, name: c.name, type: c.type, files: metaFiles },
|
metaFiles,
|
||||||
null,
|
|
||||||
2,
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -571,7 +571,8 @@ export class Library {
|
|||||||
// when it fails. Then puts pending originals at their save paths as
|
// when it fails. Then puts pending originals at their save paths as
|
||||||
// `Photo.download()` does (and optional thumbnails) through the content
|
// `Photo.download()` does (and optional thumbnails) through the content
|
||||||
// cache and pools, waits for an ML data fetch, and rebuilds the derived
|
// cache and pools, waits for an ML data fetch, and rebuilds the derived
|
||||||
// symlink/JSON views from the model, each file's JSON with its ML data.
|
// symlink/JSON views from the model, each file's JSON with its ML data,
|
||||||
|
// beside an `account.json` with the account's email and user ID.
|
||||||
// Throws before any network work when no content cache backs the
|
// Throws before any network work when no content cache backs the
|
||||||
// originals it must fetch.
|
// originals it must fetch.
|
||||||
backup(opts?: BackupOptions): Promise<BackupResult> {
|
backup(opts?: BackupOptions): Promise<BackupResult> {
|
||||||
@@ -589,6 +590,7 @@ export class Library {
|
|||||||
const cache = this.cache;
|
const cache = this.cache;
|
||||||
return runBackup(
|
return runBackup(
|
||||||
{
|
{
|
||||||
|
whoami: () => this.client.whoami(),
|
||||||
refresh: () => this.refreshNow(),
|
refresh: () => this.refreshNow(),
|
||||||
listCollections: () => this.store.listCollections(),
|
listCollections: () => this.store.listCollections(),
|
||||||
listFiles: (id) => this.store.listFiles(id),
|
listFiles: (id) => this.store.listFiles(id),
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
* collections/
|
* collections/
|
||||||
* <name>/<title> symlink to the original (rebuilt each run)
|
* <name>/<title> symlink to the original (rebuilt each run)
|
||||||
* <name>.json per-collection metadata (rebuilt each run)
|
* <name>.json per-collection metadata (rebuilt each run)
|
||||||
|
* account.json the account's email and user ID
|
||||||
* failures.json durable ledger of unresolved failures
|
* failures.json durable ledger of unresolved failures
|
||||||
*
|
*
|
||||||
* The properties that distinguish backup from a naive download loop, and that
|
* The properties that distinguish backup from a naive download loop, and that
|
||||||
@@ -917,6 +918,106 @@ describe("ML data in each file's JSON", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The fields `RecordsClient` gives the Vacation album: shared by another
|
||||||
|
// account, with all three layers of magic metadata.
|
||||||
|
const SHARED_ALBUM = {
|
||||||
|
ownerID: 7,
|
||||||
|
isShared: true,
|
||||||
|
updationTime: 1700,
|
||||||
|
magicMetadata: { visibility: 0 },
|
||||||
|
pubMagicMetadata: { coverID: 100 },
|
||||||
|
sharedMagicMetadata: { visibility: 2 },
|
||||||
|
};
|
||||||
|
|
||||||
|
// Serves the Vacation album with `SHARED_ALBUM`'s fields, and each file with
|
||||||
|
// the update time 5000 + its ID.
|
||||||
|
class RecordsClient extends MockClient {
|
||||||
|
override async collectionsSince(): Promise<CollectionsPage> {
|
||||||
|
const page = await super.collectionsSince();
|
||||||
|
return {
|
||||||
|
...page,
|
||||||
|
collections: page.collections.map((c) =>
|
||||||
|
c.id === 1 ? { ...c, ...SHARED_ALBUM } : c,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
override async filesSince(args: {
|
||||||
|
collectionID: number;
|
||||||
|
}): Promise<FilesPage> {
|
||||||
|
const page = await super.filesSince(args);
|
||||||
|
return {
|
||||||
|
...page,
|
||||||
|
files: page.files.map((f) => ({
|
||||||
|
...f,
|
||||||
|
updationTime: 5000 + f.id,
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The JSON the backup in `outDir` wrote for the album named `name`.
|
||||||
|
const albumJSON = (outDir: string, name: string): Record<string, unknown> =>
|
||||||
|
JSON.parse(
|
||||||
|
readFileSync(join(outDir, "collections", `${name}.json`), "utf-8"),
|
||||||
|
);
|
||||||
|
|
||||||
|
describe("account and album records", () => {
|
||||||
|
it("writes account.json with the account's email and user ID", async () => {
|
||||||
|
const lib = await openLibrary(stubSource());
|
||||||
|
const outDir = join(root, "backup");
|
||||||
|
|
||||||
|
await lib.backup({ downloadDirectory: outDir });
|
||||||
|
|
||||||
|
expect(
|
||||||
|
JSON.parse(readFileSync(join(outDir, "account.json"), "utf-8")),
|
||||||
|
).toEqual({ email: "backup@example.com", userID: USER_ID });
|
||||||
|
await lib.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("writes each album's owner, sharing, update time and magic metadata into its JSON", async () => {
|
||||||
|
const lib = await openLibrary(stubSource(), new RecordsClient());
|
||||||
|
const outDir = join(root, "backup");
|
||||||
|
|
||||||
|
await lib.backup({ downloadDirectory: outDir });
|
||||||
|
|
||||||
|
expect(albumJSON(outDir, "Vacation")).toEqual({
|
||||||
|
id: 1,
|
||||||
|
name: "Vacation",
|
||||||
|
type: "album",
|
||||||
|
...SHARED_ALBUM,
|
||||||
|
files: [
|
||||||
|
{ id: 100, metadata: file(100, 1, "beach.jpg").metadata },
|
||||||
|
{ id: 101, metadata: file(101, 1, "sunset.jpg").metadata },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
// An album with no magic metadata gets no magic metadata fields.
|
||||||
|
expect(albumJSON(outDir, "Work")).toEqual({
|
||||||
|
id: 2,
|
||||||
|
name: "Work",
|
||||||
|
type: "album",
|
||||||
|
ownerID: USER_ID,
|
||||||
|
isShared: false,
|
||||||
|
updationTime: 1,
|
||||||
|
files: [
|
||||||
|
{ id: 200, metadata: file(200, 2, "diagram.png").metadata },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
await lib.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("writes each file's update time into its JSON", async () => {
|
||||||
|
const lib = await openLibrary(stubSource(), new RecordsClient());
|
||||||
|
const outDir = join(root, "backup");
|
||||||
|
|
||||||
|
await lib.backup({ downloadDirectory: outDir });
|
||||||
|
|
||||||
|
for (const fileID of [100, 101, 200]) {
|
||||||
|
expect(fileJSON(outDir, fileID).updationTime).toBe(5000 + fileID);
|
||||||
|
}
|
||||||
|
await lib.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// Every entry under collections/, one level of directories deep, with each
|
// Every entry under collections/, one level of directories deep, with each
|
||||||
// symlink's target.
|
// symlink's target.
|
||||||
const tree = (outDir: string): string[] => {
|
const tree = (outDir: string): string[] => {
|
||||||
@@ -962,6 +1063,7 @@ describe("backup album folders", () => {
|
|||||||
},
|
},
|
||||||
fetchMLData: async () => {},
|
fetchMLData: async () => {},
|
||||||
mlData: async () => undefined,
|
mlData: async () => undefined,
|
||||||
|
whoami: () => ({ email: "backup@example.com", userID: USER_ID }),
|
||||||
});
|
});
|
||||||
|
|
||||||
const albumID = (outDir: string, jsonName: string): number =>
|
const albumID = (outDir: string, jsonName: string): number =>
|
||||||
|
|||||||
Reference in New Issue
Block a user