Example script: download every album's photos and metadata (closes #144) #150

Merged
clawbot merged 2 commits from issue-144-download-albums-example into next 2026-10-02 00:33:01 +02:00
5 changed files with 456 additions and 5 deletions
Showing only changes of commit 56641c6785 - Show all commits
+36 -4
View File
@@ -80,6 +80,34 @@ await lib.close();
The lower-level `Client` (login, session serialization, and the raw The lower-level `Client` (login, session serialization, and the raw
enumeration/download calls) is exported too and documented under Design below. enumeration/download calls) is exported too and documented under Design below.
## Examples
`examples/download-albums.ts` downloads every album's photos and their metadata
into a directory, `photos` in the working directory unless you name another. The
build compiles it; run it after `yarn install`:
```bash
yarn build
QUAK_EMAIL=… QUAK_PASSWORD=… node dist/examples/download-albums.js [dir]
```
It asks on the terminal for a two-factor or email code when the account requires
one, and writes:
- each photo's original at its save path under `dir`, as `photo.download()`
writes it: `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD.<fileID>.<ext>`, and for a live
photo its image, its video and the `.livephoto.json` file naming them
- beside each original, a JSON file named after it with `.json` added, for
example `2026-03-01.12345.jpg.json`: the photo's record (`photo.record()`)
without its cache paths, and its EXIF fields (`photo.exif()`) under `exif`
- `albums/<collectionID>.json` for each album: its `collectionID`, its `name`,
and under `savePaths` the save paths of its photos relative to `dir`, newest
first
A photo in several albums is downloaded once. A second run downloads nothing and
rewrites only the JSON files whose content changed. A failed download stops the
run; running it again carries on, since every photo already saved is skipped.
## Entrypoints ## Entrypoints
This repository adheres to the This repository adheres to the
@@ -239,6 +267,9 @@ quak/
index.ts public library exports index.ts public library exports
bin/ bin/
quak.ts CLI entrypoint (commander.js) quak.ts CLI entrypoint (commander.js)
examples/
download-albums.ts
download every album's photos and metadata
test/ unit + integration tests (vitest) test/ unit + integration tests (vitest)
Makefile Makefile
Dockerfile lint phase, test phase, compile Dockerfile lint phase, test phase, compile
@@ -247,10 +278,11 @@ quak/
``` ```
`make build` compiles that tree into `dist/`, preserving its shape: the library `make build` compiles that tree into `dist/`, preserving its shape: the library
lands in `dist/src/` and the CLI in `dist/bin/quak.js`, which is what lands in `dist/src/`, the examples in `dist/examples/`, and the CLI in
`package.json` points `main`, `types` and `bin` at. The compiler's `rootDir` is `dist/bin/quak.js`, which is what `package.json` points `main`, `types` and
the repository root rather than `src/`, because `bin/` is compiled too and `bin` at. The compiler's `rootDir` is the repository root rather than `src/`,
`rootDir` has to contain everything that is compiled. because `bin/` is compiled too and `rootDir` has to contain everything that is
compiled.
### Cryptography ### Cryptography
+6
View File
@@ -25,6 +25,12 @@ declares one.
# Completed Steps # Completed Steps
- 2026-10-01: `examples/download-albums.ts` logs in, opens the library, and for
every album downloads each photo to its save path, writes the photo's record
and EXIF fields to a JSON file beside it, and writes the album's photos to
`albums/<collectionID>.json` (issue 144). The build compiles it to
`dist/examples/`; the README's "Examples" section says how to run it.
- 2026-10-01: A `Photo` has one async method for each field of `exif()`, named - 2026-10-01: A `Photo` has one async method for each field of `exif()`, named
and typed as in `PhotoExif`: `make()`, `model()`, `lensModel()`, and typed as in `PhotoExif`: `make()`, `model()`, `lensModel()`,
`dateTimeOriginal()`, `offsetTimeOriginal()`, `exposureTime()`, `fNumber()`, `dateTimeOriginal()`, `offsetTimeOriginal()`, `exposureTime()`, `fNumber()`,
+126
View File
@@ -0,0 +1,126 @@
// Download every album's photos to a directory, with each photo's metadata
// beside it, using only quak's public API. The README's "Examples" section
// describes the files it writes.
//
// yarn build
// QUAK_EMAIL=… QUAK_PASSWORD=… node dist/examples/download-albums.js [dir]
import { realpathSync } from "node:fs";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join, relative } from "node:path";
import { stdin, stdout } from "node:process";
import { createInterface } from "node:readline/promises";
import { pathToFileURL } from "node:url";
import { Client, Library } from "../src/index.js";
// Write `text` to `path` unless the file already holds exactly that, so a
// second run rewrites nothing.
async function writeIfChanged(path: string, text: string): Promise<void> {
const current = await readFile(path, "utf-8").catch(() => undefined);
if (current !== text) await writeFile(path, text);
}
const pretty = (value: unknown): string =>
JSON.stringify(value, null, 2) + "\n";
// For every album in `lib`, download each photo to its save path, write the
// photo's metadata to `{savePath}.json`, and write the album's photos to
// `{dir}/albums/{collectionID}.json`. Returns how many photos it downloaded
// and how many were already at their save paths.
export async function downloadAlbums(
lib: Library,
dir: string,
): Promise<{ downloaded: number; alreadyLocal: number }> {
let downloaded = 0;
let alreadyLocal = 0;
// A photo in several albums is handled once.
const done = new Set<number>();
for (const album of lib.albums.list()) {
const savePaths: string[] = [];
for (const photo of album.photos.list()) {
if (!done.has(photo.fileID)) {
done.add(photo.fileID);
if (photo.isLocal) alreadyLocal++;
else downloaded++;
await photo.download();
// The cache paths say where quak's cache keeps copies, not
// anything about the photo.
const record = { ...photo.record() };
delete record.thumbnailPath;
delete record.originalPath;
const exif = await photo.exif();
// savePath is read after download(): a live photo's names its
// image only once the image is stored.
await writeIfChanged(
`${photo.savePath}.json`,
pretty({ ...record, exif }),
);
}
savePaths.push(relative(dir, photo.savePath));
}
await mkdir(join(dir, "albums"), { recursive: true });
await writeIfChanged(
join(dir, "albums", `${album.collectionID}.json`),
pretty({
collectionID: album.collectionID,
name: album.name,
savePaths,
}),
);
}
return { downloaded, alreadyLocal };
}
// Ask for a login code on the terminal. Client.login calls this only when the
// account requires a code.
async function ask(question: string): Promise<string> {
const terminal = createInterface({ input: stdin, output: stdout });
try {
return await terminal.question(question);
} finally {
terminal.close();
}
}
async function main(): Promise<void> {
const email = process.env.QUAK_EMAIL;
const password = process.env.QUAK_PASSWORD;
if (!email || !password) {
console.error(
"Set QUAK_EMAIL and QUAK_PASSWORD to the account's email and password.",
);
process.exit(1);
}
const dir = process.argv[2] ?? "photos";
const client = await Client.login({
email,
password,
totp: () => ask("Two-factor code: "),
emailOTP: () => ask("Code sent to your email: "),
});
const lib = await Library.open({
client,
downloadDirectory: dir,
// As in `quak backup`: fetch only the originals this script saves,
// not every thumbnail and the recent originals into the cache too.
precacheThumbnails: false,
precacheOriginals: false,
});
try {
const { downloaded, alreadyLocal } = await downloadAlbums(lib, dir);
console.log(
`${downloaded} photos downloaded, ${alreadyLocal} already local, in ${dir}`,
);
} finally {
await lib.close();
}
}
// Run main() when node runs this file, not when a test imports it. argv[1] is
// the path as given, and import.meta.url has symlinks resolved.
const script = process.argv[1];
if (script && pathToFileURL(realpathSync(script)).href === import.meta.url) {
await main();
}
+287
View File
@@ -0,0 +1,287 @@
/**
* The example script `examples/download-albums.ts` (issue #144), run twice
* against a stand-in account, as a user would run it twice.
*
* The account has two albums sharing one photo, and one of its photos is a
* live photo whose image is a HEIC with EXIF. The first run puts every
* original at its save path, writes each photo's metadata beside it and each
* album's photos under `albums/`. The second run fetches nothing and changes
* no file.
*/
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import {
existsSync,
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
statSync,
utimesSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { downloadAlbums } from "../../examples/download-albums.js";
import { Library, type ContentSource } from "../../src/index.js";
import type { CollectionsPage, FilesPage } from "../../src/client.js";
import type { Collection, EnteFile } from "../../src/model/types.js";
import { HEIC_WITH_EXIF } from "../exif-heic.js";
import {
asLivePhoto,
cdnSource,
livePhotoHash,
livePhotoZip,
VIDEO,
} from "../live-photo.js";
const USER_ID = 7;
// Every photo is taken at noon local time on 2026-03-01, so the machine's time
// zone cannot move it to another day; it is saved in the folder `DAY`. Ente
// stores times in microseconds.
const TAKEN_MS = new Date(2026, 2, 1, 12).getTime();
const DAY = join("2026", "2026-03", "2026-03-01");
const collection = (id: number, name: string): Collection => ({
id,
ownerID: USER_ID,
key: new Uint8Array([id]),
name,
type: "album",
updationTime: 1,
isShared: false,
});
const file = (id: number, collectionID: number): EnteFile => ({
id,
collectionID,
ownerID: USER_ID,
key: new Uint8Array([id]),
metadata: {
title: `file-${id}.jpg`,
fileType: "image",
creationTime: TAKEN_MS * 1000,
modificationTime: TAKEN_MS * 1000,
},
file: { decryptionHeader: "aGVhZGVy" },
thumbnail: { decryptionHeader: "dGh1bWI=" },
updationTime: 1,
});
let root: string;
beforeEach(() => {
root = mkdtempSync(join(tmpdir(), "quak-download-albums-"));
});
afterEach(() => {
if (root && existsSync(root))
rmSync(root, { recursive: true, force: true });
});
// Every file and directory under `dir`, by path.
const entries = (dir: string): string[] =>
readdirSync(dir, { recursive: true, encoding: "utf-8" });
// Set the modification time of everything under `dir` to the epoch, so that
// anything written there afterwards has a later one, however soon it comes.
const backdate = (dir: string): void => {
for (const name of entries(dir)) utimesSync(join(dir, name), 0, 0);
};
// The modification time of everything under `dir`, by path.
const mtimes = (dir: string): Map<string, number> =>
new Map(
entries(dir).map((name) => [name, statSync(join(dir, name)).mtimeMs]),
);
const readJSON = (path: string): unknown =>
JSON.parse(readFileSync(path, "utf-8"));
describe("examples/download-albums.ts", () => {
it("downloads every album's photos with their metadata, and nothing on a second run", async () => {
// Album 1, "Trip", holds photos 1 and 2. Album 2, "Family", holds
// photo 2 and photo 3, a live photo.
const live = await asLivePhoto(
file(3, 2),
livePhotoZip({ "image.heic": HEIC_WITH_EXIF, "video.mov": VIDEO }),
livePhotoHash(HEIC_WITH_EXIF, VIDEO),
);
const client = {
whoami: () => ({ email: "u@example.com", userID: USER_ID }),
collectionsSince: async (): Promise<CollectionsPage> => ({
collections: [collection(1, "Trip"), collection(2, "Family")],
deleted: [],
cursor: 1,
}),
filesSince: async (args: {
collectionID: number;
}): Promise<FilesPage> => ({
files:
args.collectionID === 1
? [file(1, 1), file(2, 1)]
: [file(2, 2), live.file],
deleted: [],
cursor: 1,
}),
};
// Photo 3 comes from a stand-in server, encrypted as Ente serves a
// live photo. Any other original is a few bytes naming its photo.
// `calls` counts every fetch, thumbnails included.
const server = cdnSource(new Map([[3, live.body]]));
let calls = 0;
const source: ContentSource = {
original: async (args) => {
calls++;
if (args.file.id === 3) return server.original(args);
const bytes = `original-${args.file.id}`;
writeFileSync(args.destination, bytes);
return { bytesWritten: bytes.length };
},
thumbnail: async (args) => {
calls++;
return server.thumbnail(args);
},
};
const dir = join(root, "photos");
const open = (): Promise<Library> =>
Library.open({
client,
cacheDirectory: join(root, "cache"),
downloadDirectory: dir,
contentSource: source,
refreshIntervalSeconds: 3600,
precacheThumbnails: false,
precacheOriginals: false,
});
const first = await open();
// The cache already holds photo 1's original, so its record names a
// cache path, which the metadata leaves out. download() copies it
// from the cache rather than fetching it again.
await first.photos.byID({ fileID: 1 })!.original();
expect(await downloadAlbums(first, dir)).toEqual({
downloaded: 3,
alreadyLocal: 0,
});
await first.close();
expect(calls).toBe(3);
// Each original at its save path, a live photo as its image, its video
// and the file naming them, and each photo's metadata beside it.
const day = join(dir, DAY);
expect(readdirSync(day).sort()).toEqual([
"2026-03-01.1.jpg",
"2026-03-01.1.jpg.json",
"2026-03-01.2.jpg",
"2026-03-01.2.jpg.json",
"2026-03-01.3.heic",
"2026-03-01.3.heic.json",
"2026-03-01.3.livephoto.json",
"2026-03-01.3.mov",
]);
expect(readFileSync(join(day, "2026-03-01.1.jpg"), "utf-8")).toBe(
"original-1",
);
expect(readFileSync(join(day, "2026-03-01.2.jpg"), "utf-8")).toBe(
"original-2",
);
expect(readFileSync(join(day, "2026-03-01.3.heic"))).toEqual(
Buffer.from(HEIC_WITH_EXIF),
);
expect(readFileSync(join(day, "2026-03-01.3.mov"))).toEqual(
Buffer.from(VIDEO),
);
// The metadata is the photo's record and its EXIF fields. The
// originals of photos 1 and 2 are not image data, so they have no
// EXIF fields. Photo 3's image holds a camera, an exposure and a
// position, and the date it was taken is written as an ISO 8601
// string. `record` holds the fields the three records share.
const record = {
takenAt: TAKEN_MS,
modifiedAt: TAKEN_MS,
fileType: "image",
isArchived: false,
isHidden: false,
};
expect(readJSON(join(day, "2026-03-01.1.jpg.json"))).toEqual({
...record,
fileID: 1,
albumIDs: [1],
title: "file-1.jpg",
exif: {},
});
expect(readJSON(join(day, "2026-03-01.2.jpg.json"))).toEqual({
...record,
fileID: 2,
albumIDs: [1, 2],
title: "file-2.jpg",
exif: {},
});
expect(readJSON(join(day, "2026-03-01.3.heic.json"))).toEqual({
...record,
fileID: 3,
albumIDs: [2],
title: "file-3.jpg",
fileType: "livePhoto",
hash: livePhotoHash(HEIC_WITH_EXIF, VIDEO),
exif: {
make: "Canon",
model: "EOS R5",
lensModel: "RF50mm F1.8 STM",
dateTimeOriginal: "2021-07-15T14:30:00.000Z",
offsetTimeOriginal: "+02:00",
exposureTime: 1 / 250,
fNumber: 2.8,
iso: 400,
focalLength: 50,
orientation: 6,
gpsLatitude: 40 + 26 / 60 + 46 / 3600,
gpsLongitude: -(79 + 58 / 60 + 56 / 3600),
gpsAltitude: -12.5,
},
});
// Each album's photos, newest first, by save path relative to `dir`.
// Photo 2 is in both.
expect(readdirSync(join(dir, "albums")).sort()).toEqual([
"1.json",
"2.json",
]);
expect(readJSON(join(dir, "albums", "1.json"))).toEqual({
collectionID: 1,
name: "Trip",
savePaths: [
join(DAY, "2026-03-01.2.jpg"),
join(DAY, "2026-03-01.1.jpg"),
],
});
expect(readJSON(join(dir, "albums", "2.json"))).toEqual({
collectionID: 2,
name: "Family",
savePaths: [
join(DAY, "2026-03-01.3.heic"),
join(DAY, "2026-03-01.2.jpg"),
],
});
// The second run, with a newly opened library, finds every photo
// already local, fetches nothing, and writes, renames or adds no
// file.
backdate(dir);
const before = mtimes(dir);
const second = await open();
expect(await downloadAlbums(second, dir)).toEqual({
downloaded: 0,
alreadyLocal: 3,
});
await second.close();
expect(calls).toBe(3);
expect(mtimes(dir)).toEqual(before);
});
});
+1 -1
View File
@@ -18,5 +18,5 @@
"sourceMap": true, "sourceMap": true,
"resolveJsonModule": true "resolveJsonModule": true
}, },
"include": ["src/**/*", "bin/**/*"] "include": ["src/**/*", "bin/**/*", "examples/**/*"]
} }