quak backup writes the account and album records backup-metadata writes #175
@@ -599,8 +599,8 @@ the smallest does not.
|
|||||||
per unique file, two for a live photo: see
|
per unique file, two for a live photo: see
|
||||||
below)
|
below)
|
||||||
YYYY-MM-DD.<fileID>.json the file's basic metadata fields quak
|
YYYY-MM-DD.<fileID>.json the file's basic metadata fields quak
|
||||||
keeps, its private and public magic
|
keeps, its update time, its private and
|
||||||
metadata, and its ML data
|
public magic metadata, and its ML data
|
||||||
YYYY-MM-DD.<fileID>.livephoto.json
|
YYYY-MM-DD.<fileID>.livephoto.json
|
||||||
which of a live photo's two files is which
|
which of a live photo's two files is which
|
||||||
collections/
|
collections/
|
||||||
@@ -608,6 +608,7 @@ the smallest does not.
|
|||||||
<title> -> ../../YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>
|
<title> -> ../../YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>
|
||||||
(symlink)
|
(symlink)
|
||||||
<name>.json collection metadata + file list
|
<name>.json collection metadata + file list
|
||||||
|
account.json the account's email and user ID
|
||||||
failures.json files that failed and have not yet succeeded
|
failures.json files that failed and have not yet succeeded
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -619,6 +620,15 @@ name as uploaded, case kept, or `.bin` when it has none or it holds anything but
|
|||||||
letters and digits. When the date or the time zone changes, the next run saves
|
letters and digits. When the date or the time zone changes, the next run saves
|
||||||
the original at its new path and leaves the old copy where it is.
|
the original at its new path and leaves the old copy where it is.
|
||||||
|
|
||||||
|
`account.json` holds the account's `email` and `userID`, as `backup-metadata`
|
||||||
|
writes it. A collection's JSON holds its `id`, `name`, `type`, `ownerID`,
|
||||||
|
`isShared` and `updationTime`, its `magicMetadata`, `pubMagicMetadata` and
|
||||||
|
`sharedMagicMetadata` when it has them, as `backup-metadata`'s
|
||||||
|
`_collection.json` does, and `files`, each file's `id` and `metadata`. A file's
|
||||||
|
JSON holds its `id`, `collectionID`, `ownerID`, `metadata` and `updationTime`,
|
||||||
|
and its `magicMetadata` and `pubMagicMetadata` when it has them. Update times
|
||||||
|
are in microseconds, as Ente records them.
|
||||||
|
|
||||||
A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as
|
A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as
|
||||||
`mlData`, the same payload `backup-metadata` writes; a file Ente has no ML data
|
`mlData`, the same payload `backup-metadata` writes; a file Ente has no ML data
|
||||||
for has no `mlData`. The backup waits for the library's ML data fetch to finish
|
for has no `mlData`. The backup waits for the library's ML data fetch to finish
|
||||||
|
|||||||
@@ -25,6 +25,12 @@ declares one.
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-06: `quak backup` writes the account and album records
|
||||||
|
`backup-metadata` writes (issue 166): `account.json` with the account's
|
||||||
|
`email` and `userID`, and in each album's JSON its `ownerID`, `isShared`,
|
||||||
|
`updationTime` and, when present, its three layers of magic metadata. Each
|
||||||
|
file's JSON gains its `updationTime`.
|
||||||
|
|
||||||
- 2026-10-05: `quak backup` writes each file's ML data (its faces and CLIP
|
- 2026-10-05: `quak backup` writes each file's ML data (its faces and CLIP
|
||||||
embedding) into the file's JSON as `mlData`, the payload
|
embedding) into the file's JSON as `mlData`, the payload
|
||||||
`lib.mldata.forFile()` returns (issue 163). `lib.backup()` waits for an ML
|
`lib.mldata.forFile()` returns (issue 163). `lib.backup()` waits for an ML
|
||||||
|
|||||||
+36
-7
@@ -14,6 +14,7 @@
|
|||||||
// the file's ML data
|
// the file's ML data
|
||||||
// collections/<name>/<title> symlink to the original
|
// collections/<name>/<title> symlink to the original
|
||||||
// collections/<name>.json per-collection metadata
|
// collections/<name>.json per-collection metadata
|
||||||
|
// account.json the account's email and user ID
|
||||||
// failures.json durable ledger of unresolved failures
|
// failures.json durable ledger of unresolved failures
|
||||||
//
|
//
|
||||||
// A live photo's original is its image and its video, each with its own
|
// A live photo's original is its image and its video, each with its own
|
||||||
@@ -64,7 +65,7 @@ import {
|
|||||||
} from "./library/content.js";
|
} from "./library/content.js";
|
||||||
import { representative } from "./library/records.js";
|
import { representative } from "./library/records.js";
|
||||||
import type { MLData } from "./mldata-fetch.js";
|
import type { MLData } from "./mldata-fetch.js";
|
||||||
import type { Collection, EnteFile } from "./model/types.js";
|
import type { Collection, EnteFile, FileMetadata } from "./model/types.js";
|
||||||
|
|
||||||
export type ProgressCallback = (message: string) => void;
|
export type ProgressCallback = (message: string) => void;
|
||||||
|
|
||||||
@@ -108,6 +109,8 @@ export interface BackupResult {
|
|||||||
// The slice of the library that backup drives. `Library` implements it; a test
|
// The slice of the library that backup drives. `Library` implements it; a test
|
||||||
// can drive backup with a stand-in.
|
// can drive backup with a stand-in.
|
||||||
export interface BackupLibrary {
|
export interface BackupLibrary {
|
||||||
|
// The account the library belongs to.
|
||||||
|
whoami(): { email: string; userID: number };
|
||||||
refresh(): Promise<void>;
|
refresh(): Promise<void>;
|
||||||
listCollections(): Collection[];
|
listCollections(): Collection[];
|
||||||
listFiles(collectionID: number): EnteFile[];
|
listFiles(collectionID: number): EnteFile[];
|
||||||
@@ -363,6 +366,7 @@ const writeSidecar = (
|
|||||||
collectionID: file.collectionID,
|
collectionID: file.collectionID,
|
||||||
ownerID: file.ownerID,
|
ownerID: file.ownerID,
|
||||||
metadata: file.metadata,
|
metadata: file.metadata,
|
||||||
|
updationTime: file.updationTime,
|
||||||
};
|
};
|
||||||
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
||||||
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
||||||
@@ -371,6 +375,29 @@ const writeSidecar = (
|
|||||||
writeFileSync(path, JSON.stringify(meta, null, 2));
|
writeFileSync(path, JSON.stringify(meta, null, 2));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// The album's JSON: its basic fields, its magic metadata, and its files.
|
||||||
|
const writeAlbumJSON = (
|
||||||
|
path: string,
|
||||||
|
c: Collection,
|
||||||
|
files: { id: number; metadata: FileMetadata }[],
|
||||||
|
): void => {
|
||||||
|
const album: Record<string, unknown> = {
|
||||||
|
id: c.id,
|
||||||
|
name: c.name,
|
||||||
|
type: c.type,
|
||||||
|
ownerID: c.ownerID,
|
||||||
|
isShared: c.isShared,
|
||||||
|
updationTime: c.updationTime,
|
||||||
|
};
|
||||||
|
if (c.magicMetadata) album.magicMetadata = c.magicMetadata;
|
||||||
|
if (c.pubMagicMetadata) album.pubMagicMetadata = c.pubMagicMetadata;
|
||||||
|
if (c.sharedMagicMetadata) {
|
||||||
|
album.sharedMagicMetadata = c.sharedMagicMetadata;
|
||||||
|
}
|
||||||
|
album.files = files;
|
||||||
|
writeFileSync(path, JSON.stringify(album, null, 2));
|
||||||
|
};
|
||||||
|
|
||||||
export const runBackup = async (
|
export const runBackup = async (
|
||||||
lib: BackupLibrary,
|
lib: BackupLibrary,
|
||||||
opts: BackupOptions,
|
opts: BackupOptions,
|
||||||
@@ -393,6 +420,11 @@ export const runBackup = async (
|
|||||||
const collectionsDir = join(downloadDirectory, "collections");
|
const collectionsDir = join(downloadDirectory, "collections");
|
||||||
const thumbnailsDir = join(downloadDirectory, "thumbnails");
|
const thumbnailsDir = join(downloadDirectory, "thumbnails");
|
||||||
mkdirSync(collectionsDir, { recursive: true });
|
mkdirSync(collectionsDir, { recursive: true });
|
||||||
|
const { email, userID } = lib.whoami();
|
||||||
|
writeFileSync(
|
||||||
|
join(downloadDirectory, "account.json"),
|
||||||
|
JSON.stringify({ email, userID }, null, 2),
|
||||||
|
);
|
||||||
if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true });
|
if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true });
|
||||||
removeLeftoverTempFiles(thumbnailsDir);
|
removeLeftoverTempFiles(thumbnailsDir);
|
||||||
for (const dir of dateFolders(downloadDirectory)) {
|
for (const dir of dateFolders(downloadDirectory)) {
|
||||||
@@ -618,13 +650,10 @@ export const runBackup = async (
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
writeFileSync(
|
writeAlbumJSON(
|
||||||
join(collectionsDir, `${colDirName}.json`),
|
join(collectionsDir, `${colDirName}.json`),
|
||||||
JSON.stringify(
|
c,
|
||||||
{ id: c.id, name: c.name, type: c.type, files: metaFiles },
|
metaFiles,
|
||||||
null,
|
|
||||||
2,
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -571,7 +571,8 @@ export class Library {
|
|||||||
// when it fails. Then puts pending originals at their save paths as
|
// when it fails. Then puts pending originals at their save paths as
|
||||||
// `Photo.download()` does (and optional thumbnails) through the content
|
// `Photo.download()` does (and optional thumbnails) through the content
|
||||||
// cache and pools, waits for an ML data fetch, and rebuilds the derived
|
// cache and pools, waits for an ML data fetch, and rebuilds the derived
|
||||||
// symlink/JSON views from the model, each file's JSON with its ML data.
|
// symlink/JSON views from the model, each file's JSON with its ML data,
|
||||||
|
// beside an `account.json` with the account's email and user ID.
|
||||||
// Throws before any network work when no content cache backs the
|
// Throws before any network work when no content cache backs the
|
||||||
// originals it must fetch.
|
// originals it must fetch.
|
||||||
backup(opts?: BackupOptions): Promise<BackupResult> {
|
backup(opts?: BackupOptions): Promise<BackupResult> {
|
||||||
@@ -589,6 +590,7 @@ export class Library {
|
|||||||
const cache = this.cache;
|
const cache = this.cache;
|
||||||
return runBackup(
|
return runBackup(
|
||||||
{
|
{
|
||||||
|
whoami: () => this.client.whoami(),
|
||||||
refresh: () => this.refreshNow(),
|
refresh: () => this.refreshNow(),
|
||||||
listCollections: () => this.store.listCollections(),
|
listCollections: () => this.store.listCollections(),
|
||||||
listFiles: (id) => this.store.listFiles(id),
|
listFiles: (id) => this.store.listFiles(id),
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
* collections/
|
* collections/
|
||||||
* <name>/<title> symlink to the original (rebuilt each run)
|
* <name>/<title> symlink to the original (rebuilt each run)
|
||||||
* <name>.json per-collection metadata (rebuilt each run)
|
* <name>.json per-collection metadata (rebuilt each run)
|
||||||
|
* account.json the account's email and user ID
|
||||||
* failures.json durable ledger of unresolved failures
|
* failures.json durable ledger of unresolved failures
|
||||||
*
|
*
|
||||||
* The properties that distinguish backup from a naive download loop, and that
|
* The properties that distinguish backup from a naive download loop, and that
|
||||||
@@ -917,6 +918,106 @@ describe("ML data in each file's JSON", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The fields `RecordsClient` gives the Vacation album: shared by another
|
||||||
|
// account, with all three layers of magic metadata.
|
||||||
|
const SHARED_ALBUM = {
|
||||||
|
ownerID: 7,
|
||||||
|
isShared: true,
|
||||||
|
updationTime: 1700,
|
||||||
|
magicMetadata: { visibility: 0 },
|
||||||
|
pubMagicMetadata: { coverID: 100 },
|
||||||
|
sharedMagicMetadata: { visibility: 2 },
|
||||||
|
};
|
||||||
|
|
||||||
|
// Serves the Vacation album with `SHARED_ALBUM`'s fields, and each file with
|
||||||
|
// the update time 5000 + its ID.
|
||||||
|
class RecordsClient extends MockClient {
|
||||||
|
override async collectionsSince(): Promise<CollectionsPage> {
|
||||||
|
const page = await super.collectionsSince();
|
||||||
|
return {
|
||||||
|
...page,
|
||||||
|
collections: page.collections.map((c) =>
|
||||||
|
c.id === 1 ? { ...c, ...SHARED_ALBUM } : c,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
override async filesSince(args: {
|
||||||
|
collectionID: number;
|
||||||
|
}): Promise<FilesPage> {
|
||||||
|
const page = await super.filesSince(args);
|
||||||
|
return {
|
||||||
|
...page,
|
||||||
|
files: page.files.map((f) => ({
|
||||||
|
...f,
|
||||||
|
updationTime: 5000 + f.id,
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The JSON the backup in `outDir` wrote for the album named `name`.
|
||||||
|
const albumJSON = (outDir: string, name: string): Record<string, unknown> =>
|
||||||
|
JSON.parse(
|
||||||
|
readFileSync(join(outDir, "collections", `${name}.json`), "utf-8"),
|
||||||
|
);
|
||||||
|
|
||||||
|
describe("account and album records", () => {
|
||||||
|
it("writes account.json with the account's email and user ID", async () => {
|
||||||
|
const lib = await openLibrary(stubSource());
|
||||||
|
const outDir = join(root, "backup");
|
||||||
|
|
||||||
|
await lib.backup({ downloadDirectory: outDir });
|
||||||
|
|
||||||
|
expect(
|
||||||
|
JSON.parse(readFileSync(join(outDir, "account.json"), "utf-8")),
|
||||||
|
).toEqual({ email: "backup@example.com", userID: USER_ID });
|
||||||
|
await lib.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("writes each album's owner, sharing, update time and magic metadata into its JSON", async () => {
|
||||||
|
const lib = await openLibrary(stubSource(), new RecordsClient());
|
||||||
|
const outDir = join(root, "backup");
|
||||||
|
|
||||||
|
await lib.backup({ downloadDirectory: outDir });
|
||||||
|
|
||||||
|
expect(albumJSON(outDir, "Vacation")).toEqual({
|
||||||
|
id: 1,
|
||||||
|
name: "Vacation",
|
||||||
|
type: "album",
|
||||||
|
...SHARED_ALBUM,
|
||||||
|
files: [
|
||||||
|
{ id: 100, metadata: file(100, 1, "beach.jpg").metadata },
|
||||||
|
{ id: 101, metadata: file(101, 1, "sunset.jpg").metadata },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
// An album with no magic metadata gets no magic metadata fields.
|
||||||
|
expect(albumJSON(outDir, "Work")).toEqual({
|
||||||
|
id: 2,
|
||||||
|
name: "Work",
|
||||||
|
type: "album",
|
||||||
|
ownerID: USER_ID,
|
||||||
|
isShared: false,
|
||||||
|
updationTime: 1,
|
||||||
|
files: [
|
||||||
|
{ id: 200, metadata: file(200, 2, "diagram.png").metadata },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
await lib.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("writes each file's update time into its JSON", async () => {
|
||||||
|
const lib = await openLibrary(stubSource(), new RecordsClient());
|
||||||
|
const outDir = join(root, "backup");
|
||||||
|
|
||||||
|
await lib.backup({ downloadDirectory: outDir });
|
||||||
|
|
||||||
|
for (const fileID of [100, 101, 200]) {
|
||||||
|
expect(fileJSON(outDir, fileID).updationTime).toBe(5000 + fileID);
|
||||||
|
}
|
||||||
|
await lib.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// Every entry under collections/, one level of directories deep, with each
|
// Every entry under collections/, one level of directories deep, with each
|
||||||
// symlink's target.
|
// symlink's target.
|
||||||
const tree = (outDir: string): string[] => {
|
const tree = (outDir: string): string[] => {
|
||||||
@@ -962,6 +1063,7 @@ describe("backup album folders", () => {
|
|||||||
},
|
},
|
||||||
fetchMLData: async () => {},
|
fetchMLData: async () => {},
|
||||||
mlData: async () => undefined,
|
mlData: async () => undefined,
|
||||||
|
whoami: () => ({ email: "backup@example.com", userID: USER_ID }),
|
||||||
});
|
});
|
||||||
|
|
||||||
const albumID = (outDir: string, jsonName: string): number =>
|
const albumID = (outDir: string, jsonName: string): number =>
|
||||||
|
|||||||
Reference in New Issue
Block a user