Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6967117cee |
@@ -609,8 +609,8 @@ the smallest does not.
|
|||||||
per unique file, two for a live photo: see
|
per unique file, two for a live photo: see
|
||||||
below)
|
below)
|
||||||
YYYY-MM-DD.<fileID>.json the file's basic metadata fields quak
|
YYYY-MM-DD.<fileID>.json the file's basic metadata fields quak
|
||||||
keeps, its update time, its private and
|
keeps, its private and public magic
|
||||||
public magic metadata, and its ML data
|
metadata, and its ML data
|
||||||
YYYY-MM-DD.<fileID>.livephoto.json
|
YYYY-MM-DD.<fileID>.livephoto.json
|
||||||
which of a live photo's two files is which
|
which of a live photo's two files is which
|
||||||
collections/
|
collections/
|
||||||
@@ -618,7 +618,6 @@ the smallest does not.
|
|||||||
<title> -> ../../YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>
|
<title> -> ../../YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>
|
||||||
(symlink)
|
(symlink)
|
||||||
<name>.json collection metadata + file list
|
<name>.json collection metadata + file list
|
||||||
account.json the account's email and user ID
|
|
||||||
failures.json files that failed and have not yet succeeded
|
failures.json files that failed and have not yet succeeded
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -630,15 +629,6 @@ name as uploaded, case kept, or `.bin` when it has none or it holds anything but
|
|||||||
letters and digits. When the date or the time zone changes, the next run saves
|
letters and digits. When the date or the time zone changes, the next run saves
|
||||||
the original at its new path and leaves the old copy where it is.
|
the original at its new path and leaves the old copy where it is.
|
||||||
|
|
||||||
`account.json` holds the account's `email` and `userID`, as `backup-metadata`
|
|
||||||
writes it. A collection's JSON holds its `id`, `name`, `type`, `ownerID`,
|
|
||||||
`isShared` and `updationTime`, its `magicMetadata`, `pubMagicMetadata` and
|
|
||||||
`sharedMagicMetadata` when it has them, as `backup-metadata`'s
|
|
||||||
`_collection.json` does, and `files`, each file's `id` and `metadata`. A file's
|
|
||||||
JSON holds its `id`, `collectionID`, `ownerID`, `metadata` and `updationTime`,
|
|
||||||
and its `magicMetadata` and `pubMagicMetadata` when it has them. Update times
|
|
||||||
are in microseconds, as Ente records them.
|
|
||||||
|
|
||||||
A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as
|
A file's JSON holds Ente's ML data for it (its faces and its CLIP embedding) as
|
||||||
`mlData`, the same payload `backup-metadata` writes; a file Ente has no ML data
|
`mlData`, the same payload `backup-metadata` writes; a file Ente has no ML data
|
||||||
for has no `mlData`. The backup waits for the library's ML data fetch to finish
|
for has no `mlData`. The backup waits for the library's ML data fetch to finish
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ declares one.
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-10-06: When the refresh a command starts with gets HTTP 401, because the
|
- 2026-10-05: When the refresh a command starts with gets HTTP 401, because the
|
||||||
server no longer accepts the saved session's token, the command prints one
|
server no longer accepts the saved session's token, the command prints one
|
||||||
line, `quak: the saved session is no longer valid; run "quak login"`, and
|
line, `quak: the saved session is no longer valid; run "quak login"`, and
|
||||||
exits 3 (issue 164). A missing or corrupt session file keeps its message and
|
exits 3 (issue 164). A missing or corrupt session file keeps its message and
|
||||||
@@ -34,12 +34,6 @@ declares one.
|
|||||||
the refresh, so `quak backup` stops there before it touches any file. Every
|
the refresh, so `quak backup` stops there before it touches any file. Every
|
||||||
other error still exits 1.
|
other error still exits 1.
|
||||||
|
|
||||||
- 2026-10-06: `quak backup` writes the account and album records
|
|
||||||
`backup-metadata` writes (issue 166): `account.json` with the account's
|
|
||||||
`email` and `userID`, and in each album's JSON its `ownerID`, `isShared`,
|
|
||||||
`updationTime` and, when present, its three layers of magic metadata. Each
|
|
||||||
file's JSON gains its `updationTime`.
|
|
||||||
|
|
||||||
- 2026-10-05: `quak backup` writes each file's ML data (its faces and CLIP
|
- 2026-10-05: `quak backup` writes each file's ML data (its faces and CLIP
|
||||||
embedding) into the file's JSON as `mlData`, the payload
|
embedding) into the file's JSON as `mlData`, the payload
|
||||||
`lib.mldata.forFile()` returns (issue 163). `lib.backup()` waits for an ML
|
`lib.mldata.forFile()` returns (issue 163). `lib.backup()` waits for an ML
|
||||||
|
|||||||
+7
-36
@@ -14,7 +14,6 @@
|
|||||||
// the file's ML data
|
// the file's ML data
|
||||||
// collections/<name>/<title> symlink to the original
|
// collections/<name>/<title> symlink to the original
|
||||||
// collections/<name>.json per-collection metadata
|
// collections/<name>.json per-collection metadata
|
||||||
// account.json the account's email and user ID
|
|
||||||
// failures.json durable ledger of unresolved failures
|
// failures.json durable ledger of unresolved failures
|
||||||
//
|
//
|
||||||
// A live photo's original is its image and its video, each with its own
|
// A live photo's original is its image and its video, each with its own
|
||||||
@@ -65,7 +64,7 @@ import {
|
|||||||
} from "./library/content.js";
|
} from "./library/content.js";
|
||||||
import { representative } from "./library/records.js";
|
import { representative } from "./library/records.js";
|
||||||
import type { MLData } from "./mldata-fetch.js";
|
import type { MLData } from "./mldata-fetch.js";
|
||||||
import type { Collection, EnteFile, FileMetadata } from "./model/types.js";
|
import type { Collection, EnteFile } from "./model/types.js";
|
||||||
|
|
||||||
export type ProgressCallback = (message: string) => void;
|
export type ProgressCallback = (message: string) => void;
|
||||||
|
|
||||||
@@ -109,8 +108,6 @@ export interface BackupResult {
|
|||||||
// The slice of the library that backup drives. `Library` implements it; a test
|
// The slice of the library that backup drives. `Library` implements it; a test
|
||||||
// can drive backup with a stand-in.
|
// can drive backup with a stand-in.
|
||||||
export interface BackupLibrary {
|
export interface BackupLibrary {
|
||||||
// The account the library belongs to.
|
|
||||||
whoami(): { email: string; userID: number };
|
|
||||||
refresh(): Promise<void>;
|
refresh(): Promise<void>;
|
||||||
listCollections(): Collection[];
|
listCollections(): Collection[];
|
||||||
listFiles(collectionID: number): EnteFile[];
|
listFiles(collectionID: number): EnteFile[];
|
||||||
@@ -366,7 +363,6 @@ const writeSidecar = (
|
|||||||
collectionID: file.collectionID,
|
collectionID: file.collectionID,
|
||||||
ownerID: file.ownerID,
|
ownerID: file.ownerID,
|
||||||
metadata: file.metadata,
|
metadata: file.metadata,
|
||||||
updationTime: file.updationTime,
|
|
||||||
};
|
};
|
||||||
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
if (file.magicMetadata) meta.magicMetadata = file.magicMetadata;
|
||||||
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
if (file.pubMagicMetadata) meta.pubMagicMetadata = file.pubMagicMetadata;
|
||||||
@@ -375,29 +371,6 @@ const writeSidecar = (
|
|||||||
writeFileSync(path, JSON.stringify(meta, null, 2));
|
writeFileSync(path, JSON.stringify(meta, null, 2));
|
||||||
};
|
};
|
||||||
|
|
||||||
// The album's JSON: its basic fields, its magic metadata, and its files.
|
|
||||||
const writeAlbumJSON = (
|
|
||||||
path: string,
|
|
||||||
c: Collection,
|
|
||||||
files: { id: number; metadata: FileMetadata }[],
|
|
||||||
): void => {
|
|
||||||
const album: Record<string, unknown> = {
|
|
||||||
id: c.id,
|
|
||||||
name: c.name,
|
|
||||||
type: c.type,
|
|
||||||
ownerID: c.ownerID,
|
|
||||||
isShared: c.isShared,
|
|
||||||
updationTime: c.updationTime,
|
|
||||||
};
|
|
||||||
if (c.magicMetadata) album.magicMetadata = c.magicMetadata;
|
|
||||||
if (c.pubMagicMetadata) album.pubMagicMetadata = c.pubMagicMetadata;
|
|
||||||
if (c.sharedMagicMetadata) {
|
|
||||||
album.sharedMagicMetadata = c.sharedMagicMetadata;
|
|
||||||
}
|
|
||||||
album.files = files;
|
|
||||||
writeFileSync(path, JSON.stringify(album, null, 2));
|
|
||||||
};
|
|
||||||
|
|
||||||
export const runBackup = async (
|
export const runBackup = async (
|
||||||
lib: BackupLibrary,
|
lib: BackupLibrary,
|
||||||
opts: BackupOptions,
|
opts: BackupOptions,
|
||||||
@@ -420,11 +393,6 @@ export const runBackup = async (
|
|||||||
const collectionsDir = join(downloadDirectory, "collections");
|
const collectionsDir = join(downloadDirectory, "collections");
|
||||||
const thumbnailsDir = join(downloadDirectory, "thumbnails");
|
const thumbnailsDir = join(downloadDirectory, "thumbnails");
|
||||||
mkdirSync(collectionsDir, { recursive: true });
|
mkdirSync(collectionsDir, { recursive: true });
|
||||||
const { email, userID } = lib.whoami();
|
|
||||||
writeFileSync(
|
|
||||||
join(downloadDirectory, "account.json"),
|
|
||||||
JSON.stringify({ email, userID }, null, 2),
|
|
||||||
);
|
|
||||||
if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true });
|
if (includeThumbnails) mkdirSync(thumbnailsDir, { recursive: true });
|
||||||
removeLeftoverTempFiles(thumbnailsDir);
|
removeLeftoverTempFiles(thumbnailsDir);
|
||||||
for (const dir of dateFolders(downloadDirectory)) {
|
for (const dir of dateFolders(downloadDirectory)) {
|
||||||
@@ -650,10 +618,13 @@ export const runBackup = async (
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
writeAlbumJSON(
|
writeFileSync(
|
||||||
join(collectionsDir, `${colDirName}.json`),
|
join(collectionsDir, `${colDirName}.json`),
|
||||||
c,
|
JSON.stringify(
|
||||||
metaFiles,
|
{ id: c.id, name: c.name, type: c.type, files: metaFiles },
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -571,8 +571,7 @@ export class Library {
|
|||||||
// when it fails. Then puts pending originals at their save paths as
|
// when it fails. Then puts pending originals at their save paths as
|
||||||
// `Photo.download()` does (and optional thumbnails) through the content
|
// `Photo.download()` does (and optional thumbnails) through the content
|
||||||
// cache and pools, waits for an ML data fetch, and rebuilds the derived
|
// cache and pools, waits for an ML data fetch, and rebuilds the derived
|
||||||
// symlink/JSON views from the model, each file's JSON with its ML data,
|
// symlink/JSON views from the model, each file's JSON with its ML data.
|
||||||
// beside an `account.json` with the account's email and user ID.
|
|
||||||
// Throws before any network work when no content cache backs the
|
// Throws before any network work when no content cache backs the
|
||||||
// originals it must fetch.
|
// originals it must fetch.
|
||||||
backup(opts?: BackupOptions): Promise<BackupResult> {
|
backup(opts?: BackupOptions): Promise<BackupResult> {
|
||||||
@@ -590,7 +589,6 @@ export class Library {
|
|||||||
const cache = this.cache;
|
const cache = this.cache;
|
||||||
return runBackup(
|
return runBackup(
|
||||||
{
|
{
|
||||||
whoami: () => this.client.whoami(),
|
|
||||||
refresh: () => this.refreshNow(),
|
refresh: () => this.refreshNow(),
|
||||||
listCollections: () => this.store.listCollections(),
|
listCollections: () => this.store.listCollections(),
|
||||||
listFiles: (id) => this.store.listFiles(id),
|
listFiles: (id) => this.store.listFiles(id),
|
||||||
|
|||||||
@@ -12,7 +12,6 @@
|
|||||||
* collections/
|
* collections/
|
||||||
* <name>/<title> symlink to the original (rebuilt each run)
|
* <name>/<title> symlink to the original (rebuilt each run)
|
||||||
* <name>.json per-collection metadata (rebuilt each run)
|
* <name>.json per-collection metadata (rebuilt each run)
|
||||||
* account.json the account's email and user ID
|
|
||||||
* failures.json durable ledger of unresolved failures
|
* failures.json durable ledger of unresolved failures
|
||||||
*
|
*
|
||||||
* The properties that distinguish backup from a naive download loop, and that
|
* The properties that distinguish backup from a naive download loop, and that
|
||||||
@@ -918,106 +917,6 @@ describe("ML data in each file's JSON", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// The fields `RecordsClient` gives the Vacation album: shared by another
|
|
||||||
// account, with all three layers of magic metadata.
|
|
||||||
const SHARED_ALBUM = {
|
|
||||||
ownerID: 7,
|
|
||||||
isShared: true,
|
|
||||||
updationTime: 1700,
|
|
||||||
magicMetadata: { visibility: 0 },
|
|
||||||
pubMagicMetadata: { coverID: 100 },
|
|
||||||
sharedMagicMetadata: { visibility: 2 },
|
|
||||||
};
|
|
||||||
|
|
||||||
// Serves the Vacation album with `SHARED_ALBUM`'s fields, and each file with
|
|
||||||
// the update time 5000 + its ID.
|
|
||||||
class RecordsClient extends MockClient {
|
|
||||||
override async collectionsSince(): Promise<CollectionsPage> {
|
|
||||||
const page = await super.collectionsSince();
|
|
||||||
return {
|
|
||||||
...page,
|
|
||||||
collections: page.collections.map((c) =>
|
|
||||||
c.id === 1 ? { ...c, ...SHARED_ALBUM } : c,
|
|
||||||
),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
override async filesSince(args: {
|
|
||||||
collectionID: number;
|
|
||||||
}): Promise<FilesPage> {
|
|
||||||
const page = await super.filesSince(args);
|
|
||||||
return {
|
|
||||||
...page,
|
|
||||||
files: page.files.map((f) => ({
|
|
||||||
...f,
|
|
||||||
updationTime: 5000 + f.id,
|
|
||||||
})),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The JSON the backup in `outDir` wrote for the album named `name`.
|
|
||||||
const albumJSON = (outDir: string, name: string): Record<string, unknown> =>
|
|
||||||
JSON.parse(
|
|
||||||
readFileSync(join(outDir, "collections", `${name}.json`), "utf-8"),
|
|
||||||
);
|
|
||||||
|
|
||||||
describe("account and album records", () => {
|
|
||||||
it("writes account.json with the account's email and user ID", async () => {
|
|
||||||
const lib = await openLibrary(stubSource());
|
|
||||||
const outDir = join(root, "backup");
|
|
||||||
|
|
||||||
await lib.backup({ downloadDirectory: outDir });
|
|
||||||
|
|
||||||
expect(
|
|
||||||
JSON.parse(readFileSync(join(outDir, "account.json"), "utf-8")),
|
|
||||||
).toEqual({ email: "backup@example.com", userID: USER_ID });
|
|
||||||
await lib.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("writes each album's owner, sharing, update time and magic metadata into its JSON", async () => {
|
|
||||||
const lib = await openLibrary(stubSource(), new RecordsClient());
|
|
||||||
const outDir = join(root, "backup");
|
|
||||||
|
|
||||||
await lib.backup({ downloadDirectory: outDir });
|
|
||||||
|
|
||||||
expect(albumJSON(outDir, "Vacation")).toEqual({
|
|
||||||
id: 1,
|
|
||||||
name: "Vacation",
|
|
||||||
type: "album",
|
|
||||||
...SHARED_ALBUM,
|
|
||||||
files: [
|
|
||||||
{ id: 100, metadata: file(100, 1, "beach.jpg").metadata },
|
|
||||||
{ id: 101, metadata: file(101, 1, "sunset.jpg").metadata },
|
|
||||||
],
|
|
||||||
});
|
|
||||||
// An album with no magic metadata gets no magic metadata fields.
|
|
||||||
expect(albumJSON(outDir, "Work")).toEqual({
|
|
||||||
id: 2,
|
|
||||||
name: "Work",
|
|
||||||
type: "album",
|
|
||||||
ownerID: USER_ID,
|
|
||||||
isShared: false,
|
|
||||||
updationTime: 1,
|
|
||||||
files: [
|
|
||||||
{ id: 200, metadata: file(200, 2, "diagram.png").metadata },
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await lib.close();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("writes each file's update time into its JSON", async () => {
|
|
||||||
const lib = await openLibrary(stubSource(), new RecordsClient());
|
|
||||||
const outDir = join(root, "backup");
|
|
||||||
|
|
||||||
await lib.backup({ downloadDirectory: outDir });
|
|
||||||
|
|
||||||
for (const fileID of [100, 101, 200]) {
|
|
||||||
expect(fileJSON(outDir, fileID).updationTime).toBe(5000 + fileID);
|
|
||||||
}
|
|
||||||
await lib.close();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// Every entry under collections/, one level of directories deep, with each
|
// Every entry under collections/, one level of directories deep, with each
|
||||||
// symlink's target.
|
// symlink's target.
|
||||||
const tree = (outDir: string): string[] => {
|
const tree = (outDir: string): string[] => {
|
||||||
@@ -1063,7 +962,6 @@ describe("backup album folders", () => {
|
|||||||
},
|
},
|
||||||
fetchMLData: async () => {},
|
fetchMLData: async () => {},
|
||||||
mlData: async () => undefined,
|
mlData: async () => undefined,
|
||||||
whoami: () => ({ email: "backup@example.com", userID: USER_ID }),
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const albumID = (outDir: string, jsonName: string): number =>
|
const albumID = (outDir: string, jsonName: string): number =>
|
||||||
|
|||||||
Reference in New Issue
Block a user