Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
41304fb7db | ||
|
|
2483c85321 |
@@ -609,8 +609,8 @@ the smallest does not.
|
||||
per unique file, two for a live photo: see
|
||||
below)
|
||||
YYYY-MM-DD.<fileID>.json the file's basic metadata fields quak
|
||||
keeps, its private and public magic
|
||||
metadata, and its ML data
|
||||
keeps, its update time, its private and
|
||||
public magic metadata, and its ML data
|
||||
YYYY-MM-DD.<fileID>.livephoto.json
|
||||
which of a live photo's two files is which
|
||||
collections/
|
||||
@@ -618,6 +618,7 @@ the smallest does not.
|
||||
<title> -> ../../YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>
|
||||
(symlink)
|
||||
<name>.json collection metadata + file list
|
||||
account.json the account's email and user ID
|
||||
failures.json files that failed and have not yet succeeded
|
||||
```
|
||||
|
||||
@@ -629,6 +630,15 @@ name as uploaded, case kept, or `.bin` when it has none or it holds anything but
|
||||
letters and digits. When the date or the time zone changes, the next run saves
|
||||
the original at its new path and leaves the old copy where it is.
|
||||
|
||||
`account.json` holds the account's `email` and `userID`, as `backup-metadata`
|
||||
writes it. A collection's JSON holds its `id`, `name`, `type`, `ownerID`,
|
||||
`isShared` and `updationTime`, its `magicMetadata`, `pubMagicMetadata` and
|
||||
`sharedMagicMetadata` when it has them, as `backup-metadata`'s
|
||||
`_collection.json` does, and `files`, each file's `id` and `metadata`. A file's
|
||||
JSON holds its `id`, `collectionID`, `ownerID`, `metadata` and `updationTime`,
|
||||
and its `magicMetadata` and `pubMagicMetadata` when it has them. Update times
|
||||
are in microseconds, as Ente records them.
|
||||
|
||||
A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as
|
||||
`mlData`, the same payload `backup-metadata` writes; a file Ente has no ML data
|
||||
for has no `mlData`. The backup waits for the library's ML data fetch to finish
|
||||
|
||||
@@ -25,7 +25,7 @@ declares one.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-05: When the refresh a command starts with gets HTTP 401, because the
|
||||
- 2026-10-06: When the refresh a command starts with gets HTTP 401, because the
|
||||
server no longer accepts the saved session's token, the command prints one
|
||||
line, `quak: the saved session is no longer valid; run "quak login"`, and
|
||||
exits 3 (issue 164). A missing or corrupt session file keeps its message and
|
||||
@@ -34,6 +34,12 @@ declares one.
|
||||
the refresh, so `quak backup` stops there before it touches any file. Every
|
||||
other error still exits 1.
|
||||
|
||||
- 2026-10-06: `quak backup` writes the account and album records
|
||||
`backup-metadata` writes (issue 166): `account.json` with the account's
|
||||
`email` and `userID`, and in each album's JSON its `ownerID`, `isShared`,
|
||||
`updationTime` and, when present, its three layers of magic metadata. Each
|
||||
file's JSON gains its `updationTime`.
|
||||
|
||||
- 2026-10-05: `quak backup` writes each file's ML data (its faces and CLIP
|
||||
embedding) into the file's JSON as `mlData`, the payload
|
||||
`lib.mldata.forFile()` returns (issue 163). `lib.backup()` waits for an ML
|
||||
|
||||
+36
-7
@@ -14,6 +14,7 @@
|
||||
// the file's ML data
|
||||
// collections/<name>/<title> symlink to the original
|
||||
// collections/<name>.json per-collection metadata
|
||||
// account.json the account's email and user ID
|
||||
// failures.json durable ledger of unresolved failures
|
||||
//
|
||||
// A live photo's original is its image and its video, each with its own
|
||||
@@ -64,7 +65,7 @@ import {
|
||||
} from "./library/content.js";
|
||||
import { representative } from "./library/records.js";
|
||||
import type { MLData } from "./mldata-fetch.js";
|
||||
import type { Collection, EnteFile } from "./model/types.js";
|
||||
import type { Collection, EnteFile, FileMetadata } from "./model/types.js";
|
||||
|
||||
export type ProgressCallback = (message: string) => void;
|
||||
|
||||
@@ -108,6 +109,8 @@ export interface BackupResult {
|
||||
// The slice of the library that backup drives. `Library` implements it; a test
|
||||
// can drive backup with a stand-in.
|
||||
export interface BackupLibrary {
|
||||
// The account the library belongs to.
|
||||
whoami(): { email: string; userID: number };
|
||||
refresh(): Promise<void>;
|
||||
listCollections(): Collection[];
|
||||
listFiles(collectionID: number): EnteFile[];
|
||||
@@ -363,6 +366,7 @@ const writeSidecar = (
|
||||
collectionID: file.collectionID,
|
||||
ownerID: file.ownerID,
|
||||
metadata: file.metadata,
|
||||
updationTime: file.updationTime,
|
||||
};
|
||||
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
||||
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
||||
@@ -371,6 +375,29 @@ const writeSidecar = (
|
||||
writeFileSync(path, JSON.stringify(meta, null, 2));
|
||||
};
|
||||
|
||||
// The album's JSON: its basic fields, its magic metadata, and its files.
|
||||
const writeAlbumJSON = (
|
||||
path: string,
|
||||
c: Collection,
|
||||
files: { id: number; metadata: FileMetadata }[],
|
||||
): void => {
|
||||
const album: Record<string, unknown> = {
|
||||
id: c.id,
|
||||
name: c.name,
|
||||
type: c.type,
|
||||
ownerID: c.ownerID,
|
||||
isShared: c.isShared,
|
||||
updationTime: c.updationTime,
|
||||
};
|
||||
if (c.magicMetadata) album.magicMetadata = c.magicMetadata;
|
||||
if (c.pubMagicMetadata) album.pubMagicMetadata = c.pubMagicMetadata;
|
||||
if (c.sharedMagicMetadata) {
|
||||
album.sharedMagicMetadata = c.sharedMagicMetadata;
|
||||
}
|
||||
album.files = files;
|
||||
writeFileSync(path, JSON.stringify(album, null, 2));
|
||||
};
|
||||
|
||||
export const runBackup = async (
|
||||
lib: BackupLibrary,
|
||||
opts: BackupOptions,
|
||||
@@ -393,6 +420,11 @@ export const runBackup = async (
|
||||
const collectionsDir = join(downloadDirectory, "collections");
|
||||
const thumbnailsDir = join(downloadDirectory, "thumbnails");
|
||||
mkdirSync(collectionsDir, { recursive: true });
|
||||
const { email, userID } = lib.whoami();
|
||||
writeFileSync(
|
||||
join(downloadDirectory, "account.json"),
|
||||
JSON.stringify({ email, userID }, null, 2),
|
||||
);
|
||||
if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true });
|
||||
removeLeftoverTempFiles(thumbnailsDir);
|
||||
for (const dir of dateFolders(downloadDirectory)) {
|
||||
@@ -618,13 +650,10 @@ export const runBackup = async (
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync(
|
||||
writeAlbumJSON(
|
||||
join(collectionsDir, `${colDirName}.json`),
|
||||
JSON.stringify(
|
||||
{ id: c.id, name: c.name, type: c.type, files: metaFiles },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
c,
|
||||
metaFiles,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -571,7 +571,8 @@ export class Library {
|
||||
// when it fails. Then puts pending originals at their save paths as
|
||||
// `Photo.download()` does (and optional thumbnails) through the content
|
||||
// cache and pools, waits for an ML data fetch, and rebuilds the derived
|
||||
// symlink/JSON views from the model, each file's JSON with its ML data.
|
||||
// symlink/JSON views from the model, each file's JSON with its ML data,
|
||||
// beside an `account.json` with the account's email and user ID.
|
||||
// Throws before any network work when no content cache backs the
|
||||
// originals it must fetch.
|
||||
backup(opts?: BackupOptions): Promise<BackupResult> {
|
||||
@@ -589,6 +590,7 @@ export class Library {
|
||||
const cache = this.cache;
|
||||
return runBackup(
|
||||
{
|
||||
whoami: () => this.client.whoami(),
|
||||
refresh: () => this.refreshNow(),
|
||||
listCollections: () => this.store.listCollections(),
|
||||
listFiles: (id) => this.store.listFiles(id),
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
* collections/
|
||||
* <name>/<title> symlink to the original (rebuilt each run)
|
||||
* <name>.json per-collection metadata (rebuilt each run)
|
||||
* account.json the account's email and user ID
|
||||
* failures.json durable ledger of unresolved failures
|
||||
*
|
||||
* The properties that distinguish backup from a naive download loop, and that
|
||||
@@ -917,6 +918,106 @@ describe("ML data in each file's JSON", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// The fields `RecordsClient` gives the Vacation album: shared by another
|
||||
// account, with all three layers of magic metadata.
|
||||
const SHARED_ALBUM = {
|
||||
ownerID: 7,
|
||||
isShared: true,
|
||||
updationTime: 1700,
|
||||
magicMetadata: { visibility: 0 },
|
||||
pubMagicMetadata: { coverID: 100 },
|
||||
sharedMagicMetadata: { visibility: 2 },
|
||||
};
|
||||
|
||||
// Serves the Vacation album with `SHARED_ALBUM`'s fields, and each file with
|
||||
// the update time 5000 + its ID.
|
||||
class RecordsClient extends MockClient {
|
||||
override async collectionsSince(): Promise<CollectionsPage> {
|
||||
const page = await super.collectionsSince();
|
||||
return {
|
||||
...page,
|
||||
collections: page.collections.map((c) =>
|
||||
c.id === 1 ? { ...c, ...SHARED_ALBUM } : c,
|
||||
),
|
||||
};
|
||||
}
|
||||
override async filesSince(args: {
|
||||
collectionID: number;
|
||||
}): Promise<FilesPage> {
|
||||
const page = await super.filesSince(args);
|
||||
return {
|
||||
...page,
|
||||
files: page.files.map((f) => ({
|
||||
...f,
|
||||
updationTime: 5000 + f.id,
|
||||
})),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// The JSON the backup in `outDir` wrote for the album named `name`.
|
||||
const albumJSON = (outDir: string, name: string): Record<string, unknown> =>
|
||||
JSON.parse(
|
||||
readFileSync(join(outDir, "collections", `${name}.json`), "utf-8"),
|
||||
);
|
||||
|
||||
describe("account and album records", () => {
|
||||
it("writes account.json with the account's email and user ID", async () => {
|
||||
const lib = await openLibrary(stubSource());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(
|
||||
JSON.parse(readFileSync(join(outDir, "account.json"), "utf-8")),
|
||||
).toEqual({ email: "backup@example.com", userID: USER_ID });
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("writes each album's owner, sharing, update time and magic metadata into its JSON", async () => {
|
||||
const lib = await openLibrary(stubSource(), new RecordsClient());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(albumJSON(outDir, "Vacation")).toEqual({
|
||||
id: 1,
|
||||
name: "Vacation",
|
||||
type: "album",
|
||||
...SHARED_ALBUM,
|
||||
files: [
|
||||
{ id: 100, metadata: file(100, 1, "beach.jpg").metadata },
|
||||
{ id: 101, metadata: file(101, 1, "sunset.jpg").metadata },
|
||||
],
|
||||
});
|
||||
// An album with no magic metadata gets no magic metadata fields.
|
||||
expect(albumJSON(outDir, "Work")).toEqual({
|
||||
id: 2,
|
||||
name: "Work",
|
||||
type: "album",
|
||||
ownerID: USER_ID,
|
||||
isShared: false,
|
||||
updationTime: 1,
|
||||
files: [
|
||||
{ id: 200, metadata: file(200, 2, "diagram.png").metadata },
|
||||
],
|
||||
});
|
||||
await lib.close();
|
||||
});
|
||||
|
||||
it("writes each file's update time into its JSON", async () => {
|
||||
const lib = await openLibrary(stubSource(), new RecordsClient());
|
||||
const outDir = join(root, "backup");
|
||||
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
for (const fileID of [100, 101, 200]) {
|
||||
expect(fileJSON(outDir, fileID).updationTime).toBe(5000 + fileID);
|
||||
}
|
||||
await lib.close();
|
||||
});
|
||||
});
|
||||
|
||||
// Every entry under collections/, one level of directories deep, with each
|
||||
// symlink's target.
|
||||
const tree = (outDir: string): string[] => {
|
||||
@@ -962,6 +1063,7 @@ describe("backup album folders", () => {
|
||||
},
|
||||
fetchMLData: async () => {},
|
||||
mlData: async () => undefined,
|
||||
whoami: () => ({ email: "backup@example.com", userID: USER_ID }),
|
||||
});
|
||||
|
||||
const albumID = (outDir: string, jsonName: string): number =>
|
||||
|
||||
Reference in New Issue
Block a user