From 56641c6785d36fbe3844a18744d9c9d32ce17ccb Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 1 Oct 2026 21:37:15 +0000 Subject: [PATCH 1/2] Example script: download every album's photos and metadata (closes #144) `examples/download-albums.ts` logs in with `QUAK_EMAIL` and `QUAK_PASSWORD`, opens the library, and for every album downloads each photo to its save path, writes `{savePath}.json` with the photo's record (cache paths left out) and its EXIF fields, and writes `albums/{collectionID}.json` with the album's save paths. A JSON file is written only when its content changed, so a second run downloads and rewrites nothing. `tsconfig.json` includes `examples/`, so the build type-checks it. A test runs it twice against a stand-in account. Model: opus-5-5 --- README.md | 40 +++- TODO.md | 6 + examples/download-albums.ts | 126 +++++++++++ test/examples/download-albums.test.ts | 287 ++++++++++++++++++++++++++ tsconfig.json | 2 +- 5 files changed, 456 insertions(+), 5 deletions(-) create mode 100644 examples/download-albums.ts create mode 100644 test/examples/download-albums.test.ts diff --git a/README.md b/README.md index 1515a67..f184a5a 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,34 @@ await lib.close(); The lower-level `Client` (login, session serialization, and the raw enumeration/download calls) is exported too and documented under Design below. +## Examples + +`examples/download-albums.ts` downloads every album's photos and their metadata +into a directory, `photos` in the working directory unless you name another. The +build compiles it; run it after `yarn install`: + +```bash +yarn build +QUAK_EMAIL=… QUAK_PASSWORD=… node dist/examples/download-albums.js [dir] +``` + +It asks on the terminal for a two-factor or email code when the account requires +one, and writes: + +- each photo's original at its save path under `dir`, as `photo.download()` + writes it: `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD..`, and for a live + photo its image, its video and the `.livephoto.json` file naming them +- beside each original, a JSON file named after it with `.json` added, for + example `2026-03-01.12345.jpg.json`: the photo's record (`photo.record()`) + without its cache paths, and its EXIF fields (`photo.exif()`) under `exif` +- `albums/.json` for each album: its `collectionID`, its `name`, + and under `savePaths` the save paths of its photos relative to `dir`, newest + first + +A photo in several albums is downloaded once. A second run downloads nothing and +rewrites only the JSON files whose content changed. A failed download stops the +run; running it again carries on, since every photo already saved is skipped. + ## Entrypoints This repository adheres to the @@ -239,6 +267,9 @@ quak/ index.ts public library exports bin/ quak.ts CLI entrypoint (commander.js) + examples/ + download-albums.ts + download every album's photos and metadata test/ unit + integration tests (vitest) Makefile Dockerfile lint phase, test phase, compile @@ -247,10 +278,11 @@ quak/ ``` `make build` compiles that tree into `dist/`, preserving its shape: the library -lands in `dist/src/` and the CLI in `dist/bin/quak.js`, which is what -`package.json` points `main`, `types` and `bin` at. The compiler's `rootDir` is -the repository root rather than `src/`, because `bin/` is compiled too and -`rootDir` has to contain everything that is compiled. +lands in `dist/src/`, the examples in `dist/examples/`, and the CLI in +`dist/bin/quak.js`, which is what `package.json` points `main`, `types` and +`bin` at. The compiler's `rootDir` is the repository root rather than `src/`, +because `bin/` is compiled too and `rootDir` has to contain everything that is +compiled. ### Cryptography diff --git a/TODO.md b/TODO.md index 3d1af8a..29cfd56 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,12 @@ declares one. # Completed Steps +- 2026-10-01: `examples/download-albums.ts` logs in, opens the library, and for + every album downloads each photo to its save path, writes the photo's record + and EXIF fields to a JSON file beside it, and writes the album's photos to + `albums/.json` (issue 144). The build compiles it to + `dist/examples/`; the README's "Examples" section says how to run it. + - 2026-10-01: A `Photo` has one async method for each field of `exif()`, named and typed as in `PhotoExif`: `make()`, `model()`, `lensModel()`, `dateTimeOriginal()`, `offsetTimeOriginal()`, `exposureTime()`, `fNumber()`, diff --git a/examples/download-albums.ts b/examples/download-albums.ts new file mode 100644 index 0000000..4ab712a --- /dev/null +++ b/examples/download-albums.ts @@ -0,0 +1,126 @@ +// Download every album's photos to a directory, with each photo's metadata +// beside it, using only quak's public API. The README's "Examples" section +// describes the files it writes. +// +// yarn build +// QUAK_EMAIL=… QUAK_PASSWORD=… node dist/examples/download-albums.js [dir] + +import { realpathSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, relative } from "node:path"; +import { stdin, stdout } from "node:process"; +import { createInterface } from "node:readline/promises"; +import { pathToFileURL } from "node:url"; + +import { Client, Library } from "../src/index.js"; + +// Write `text` to `path` unless the file already holds exactly that, so a +// second run rewrites nothing. +async function writeIfChanged(path: string, text: string): Promise { + const current = await readFile(path, "utf-8").catch(() => undefined); + if (current !== text) await writeFile(path, text); +} + +const pretty = (value: unknown): string => + JSON.stringify(value, null, 2) + "\n"; + +// For every album in `lib`, download each photo to its save path, write the +// photo's metadata to `{savePath}.json`, and write the album's photos to +// `{dir}/albums/{collectionID}.json`. Returns how many photos it downloaded +// and how many were already at their save paths. +export async function downloadAlbums( + lib: Library, + dir: string, +): Promise<{ downloaded: number; alreadyLocal: number }> { + let downloaded = 0; + let alreadyLocal = 0; + // A photo in several albums is handled once. + const done = new Set(); + for (const album of lib.albums.list()) { + const savePaths: string[] = []; + for (const photo of album.photos.list()) { + if (!done.has(photo.fileID)) { + done.add(photo.fileID); + if (photo.isLocal) alreadyLocal++; + else downloaded++; + await photo.download(); + // The cache paths say where quak's cache keeps copies, not + // anything about the photo. + const record = { ...photo.record() }; + delete record.thumbnailPath; + delete record.originalPath; + const exif = await photo.exif(); + // savePath is read after download(): a live photo's names its + // image only once the image is stored. + await writeIfChanged( + `${photo.savePath}.json`, + pretty({ ...record, exif }), + ); + } + savePaths.push(relative(dir, photo.savePath)); + } + await mkdir(join(dir, "albums"), { recursive: true }); + await writeIfChanged( + join(dir, "albums", `${album.collectionID}.json`), + pretty({ + collectionID: album.collectionID, + name: album.name, + savePaths, + }), + ); + } + return { downloaded, alreadyLocal }; +} + +// Ask for a login code on the terminal. Client.login calls this only when the +// account requires a code. +async function ask(question: string): Promise { + const terminal = createInterface({ input: stdin, output: stdout }); + try { + return await terminal.question(question); + } finally { + terminal.close(); + } +} + +async function main(): Promise { + const email = process.env.QUAK_EMAIL; + const password = process.env.QUAK_PASSWORD; + if (!email || !password) { + console.error( + "Set QUAK_EMAIL and QUAK_PASSWORD to the account's email and password.", + ); + process.exit(1); + } + const dir = process.argv[2] ?? "photos"; + + const client = await Client.login({ + email, + password, + totp: () => ask("Two-factor code: "), + emailOTP: () => ask("Code sent to your email: "), + }); + const lib = await Library.open({ + client, + downloadDirectory: dir, + // As in `quak backup`: fetch only the originals this script saves, + // not every thumbnail and the recent originals into the cache too. + precacheThumbnails: false, + precacheOriginals: false, + }); + try { + const { downloaded, alreadyLocal } = await downloadAlbums(lib, dir); + console.log( + `${downloaded} photos downloaded, ${alreadyLocal} already local, in ${dir}`, + ); + } finally { + await lib.close(); + } +} + +// Run main() when node runs this file, not when a test imports it. argv[1] is +// the path as given, and import.meta.url has symlinks resolved. +const script = process.argv[1]; +if (script && pathToFileURL(realpathSync(script)).href === import.meta.url) { + await main(); +} diff --git a/test/examples/download-albums.test.ts b/test/examples/download-albums.test.ts new file mode 100644 index 0000000..114bca6 --- /dev/null +++ b/test/examples/download-albums.test.ts @@ -0,0 +1,287 @@ +/** + * The example script `examples/download-albums.ts` (issue #144), run twice + * against a stand-in account, as a user would run it twice. + * + * The account has two albums sharing one photo, and one of its photos is a + * live photo whose image is a HEIC with EXIF. The first run puts every + * original at its save path, writes each photo's metadata beside it and each + * album's photos under `albums/`. The second run fetches nothing and changes + * no file. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + existsSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { downloadAlbums } from "../../examples/download-albums.js"; +import { Library, type ContentSource } from "../../src/index.js"; +import type { CollectionsPage, FilesPage } from "../../src/client.js"; +import type { Collection, EnteFile } from "../../src/model/types.js"; +import { HEIC_WITH_EXIF } from "../exif-heic.js"; +import { + asLivePhoto, + cdnSource, + livePhotoHash, + livePhotoZip, + VIDEO, +} from "../live-photo.js"; + +const USER_ID = 7; + +// Every photo is taken at noon local time on 2026-03-01, so the machine's time +// zone cannot move it to another day; it is saved in the folder `DAY`. Ente +// stores times in microseconds. +const TAKEN_MS = new Date(2026, 2, 1, 12).getTime(); +const DAY = join("2026", "2026-03", "2026-03-01"); + +const collection = (id: number, name: string): Collection => ({ + id, + ownerID: USER_ID, + key: new Uint8Array([id]), + name, + type: "album", + updationTime: 1, + isShared: false, +}); + +const file = (id: number, collectionID: number): EnteFile => ({ + id, + collectionID, + ownerID: USER_ID, + key: new Uint8Array([id]), + metadata: { + title: `file-${id}.jpg`, + fileType: "image", + creationTime: TAKEN_MS * 1000, + modificationTime: TAKEN_MS * 1000, + }, + file: { decryptionHeader: "aGVhZGVy" }, + thumbnail: { decryptionHeader: "dGh1bWI=" }, + updationTime: 1, +}); + +let root: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "quak-download-albums-")); +}); + +afterEach(() => { + if (root && existsSync(root)) + rmSync(root, { recursive: true, force: true }); +}); + +// Every file and directory under `dir`, by path. +const entries = (dir: string): string[] => + readdirSync(dir, { recursive: true, encoding: "utf-8" }); + +// Set the modification time of everything under `dir` to the epoch, so that +// anything written there afterwards has a later one, however soon it comes. +const backdate = (dir: string): void => { + for (const name of entries(dir)) utimesSync(join(dir, name), 0, 0); +}; + +// The modification time of everything under `dir`, by path. +const mtimes = (dir: string): Map => + new Map( + entries(dir).map((name) => [name, statSync(join(dir, name)).mtimeMs]), + ); + +const readJSON = (path: string): unknown => + JSON.parse(readFileSync(path, "utf-8")); + +describe("examples/download-albums.ts", () => { + it("downloads every album's photos with their metadata, and nothing on a second run", async () => { + // Album 1, "Trip", holds photos 1 and 2. Album 2, "Family", holds + // photo 2 and photo 3, a live photo. + const live = await asLivePhoto( + file(3, 2), + livePhotoZip({ "image.heic": HEIC_WITH_EXIF, "video.mov": VIDEO }), + livePhotoHash(HEIC_WITH_EXIF, VIDEO), + ); + const client = { + whoami: () => ({ email: "u@example.com", userID: USER_ID }), + collectionsSince: async (): Promise => ({ + collections: [collection(1, "Trip"), collection(2, "Family")], + deleted: [], + cursor: 1, + }), + filesSince: async (args: { + collectionID: number; + }): Promise => ({ + files: + args.collectionID === 1 + ? [file(1, 1), file(2, 1)] + : [file(2, 2), live.file], + deleted: [], + cursor: 1, + }), + }; + + // Photo 3 comes from a stand-in server, encrypted as Ente serves a + // live photo. Any other original is a few bytes naming its photo. + // `calls` counts every fetch, thumbnails included. + const server = cdnSource(new Map([[3, live.body]])); + let calls = 0; + const source: ContentSource = { + original: async (args) => { + calls++; + if (args.file.id === 3) return server.original(args); + const bytes = `original-${args.file.id}`; + writeFileSync(args.destination, bytes); + return { bytesWritten: bytes.length }; + }, + thumbnail: async (args) => { + calls++; + return server.thumbnail(args); + }, + }; + + const dir = join(root, "photos"); + const open = (): Promise => + Library.open({ + client, + cacheDirectory: join(root, "cache"), + downloadDirectory: dir, + contentSource: source, + refreshIntervalSeconds: 3600, + precacheThumbnails: false, + precacheOriginals: false, + }); + + const first = await open(); + // The cache already holds photo 1's original, so its record names a + // cache path, which the metadata leaves out. download() copies it + // from the cache rather than fetching it again. + await first.photos.byID({ fileID: 1 })!.original(); + expect(await downloadAlbums(first, dir)).toEqual({ + downloaded: 3, + alreadyLocal: 0, + }); + await first.close(); + expect(calls).toBe(3); + + // Each original at its save path, a live photo as its image, its video + // and the file naming them, and each photo's metadata beside it. + const day = join(dir, DAY); + expect(readdirSync(day).sort()).toEqual([ + "2026-03-01.1.jpg", + "2026-03-01.1.jpg.json", + "2026-03-01.2.jpg", + "2026-03-01.2.jpg.json", + "2026-03-01.3.heic", + "2026-03-01.3.heic.json", + "2026-03-01.3.livephoto.json", + "2026-03-01.3.mov", + ]); + expect(readFileSync(join(day, "2026-03-01.1.jpg"), "utf-8")).toBe( + "original-1", + ); + expect(readFileSync(join(day, "2026-03-01.2.jpg"), "utf-8")).toBe( + "original-2", + ); + expect(readFileSync(join(day, "2026-03-01.3.heic"))).toEqual( + Buffer.from(HEIC_WITH_EXIF), + ); + expect(readFileSync(join(day, "2026-03-01.3.mov"))).toEqual( + Buffer.from(VIDEO), + ); + + // The metadata is the photo's record and its EXIF fields. The + // originals of photos 1 and 2 are not image data, so they have no + // EXIF fields. Photo 3's image holds a camera, an exposure and a + // position, and the date it was taken is written as an ISO 8601 + // string. `record` holds the fields the three records share. + const record = { + takenAt: TAKEN_MS, + modifiedAt: TAKEN_MS, + fileType: "image", + isArchived: false, + isHidden: false, + }; + expect(readJSON(join(day, "2026-03-01.1.jpg.json"))).toEqual({ + ...record, + fileID: 1, + albumIDs: [1], + title: "file-1.jpg", + exif: {}, + }); + expect(readJSON(join(day, "2026-03-01.2.jpg.json"))).toEqual({ + ...record, + fileID: 2, + albumIDs: [1, 2], + title: "file-2.jpg", + exif: {}, + }); + expect(readJSON(join(day, "2026-03-01.3.heic.json"))).toEqual({ + ...record, + fileID: 3, + albumIDs: [2], + title: "file-3.jpg", + fileType: "livePhoto", + hash: livePhotoHash(HEIC_WITH_EXIF, VIDEO), + exif: { + make: "Canon", + model: "EOS R5", + lensModel: "RF50mm F1.8 STM", + dateTimeOriginal: "2021-07-15T14:30:00.000Z", + offsetTimeOriginal: "+02:00", + exposureTime: 1 / 250, + fNumber: 2.8, + iso: 400, + focalLength: 50, + orientation: 6, + gpsLatitude: 40 + 26 / 60 + 46 / 3600, + gpsLongitude: -(79 + 58 / 60 + 56 / 3600), + gpsAltitude: -12.5, + }, + }); + + // Each album's photos, newest first, by save path relative to `dir`. + // Photo 2 is in both. + expect(readdirSync(join(dir, "albums")).sort()).toEqual([ + "1.json", + "2.json", + ]); + expect(readJSON(join(dir, "albums", "1.json"))).toEqual({ + collectionID: 1, + name: "Trip", + savePaths: [ + join(DAY, "2026-03-01.2.jpg"), + join(DAY, "2026-03-01.1.jpg"), + ], + }); + expect(readJSON(join(dir, "albums", "2.json"))).toEqual({ + collectionID: 2, + name: "Family", + savePaths: [ + join(DAY, "2026-03-01.3.heic"), + join(DAY, "2026-03-01.2.jpg"), + ], + }); + + // The second run, with a newly opened library, finds every photo + // already local, fetches nothing, and writes, renames or adds no + // file. + backdate(dir); + const before = mtimes(dir); + const second = await open(); + expect(await downloadAlbums(second, dir)).toEqual({ + downloaded: 0, + alreadyLocal: 3, + }); + await second.close(); + expect(calls).toBe(3); + expect(mtimes(dir)).toEqual(before); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 1cab40d..4d5ce89 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,5 +18,5 @@ "sourceMap": true, "resolveJsonModule": true }, - "include": ["src/**/*", "bin/**/*"] + "include": ["src/**/*", "bin/**/*", "examples/**/*"] } -- 2.54.0 From b42c5b0e0def88baca09cd1250d1e62064be52e2 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 1 Oct 2026 22:10:08 +0000 Subject: [PATCH 2/2] download-albums example: walk the albums from lib.fresh() downloadAlbums now waits for a refresh from the server through lib.fresh() before walking the albums, so albums and photos added since the cache was last written are downloaded, and a failed refresh throws instead of reporting an empty or stale library as done. The test's stand-in account gains an album holding a new photo before the second run, and that run must download it. The README's "Examples" section says the script opens the library with the thumbnail and originals precache off, as `quak backup` does. Model: opus-5-5 --- README.md | 6 ++- examples/download-albums.ts | 5 ++- test/examples/download-albums.test.ts | 58 ++++++++++++++++++++------- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f184a5a..12c99e2 100644 --- a/README.md +++ b/README.md @@ -91,8 +91,10 @@ yarn build QUAK_EMAIL=… QUAK_PASSWORD=… node dist/examples/download-albums.js [dir] ``` -It asks on the terminal for a two-factor or email code when the account requires -one, and writes: +It opens the library with `precacheThumbnails` and `precacheOriginals` off, as +`quak backup` does, so the only file content it fetches is the originals it +saves. It asks on the terminal for a two-factor or email code when the account +requires one, and writes: - each photo's original at its save path under `dir`, as `photo.download()` writes it: `YYYY/YYYY-MM/YYYY-MM-DD/YYYY-MM-DD..`, and for a live diff --git a/examples/download-albums.ts b/examples/download-albums.ts index 4ab712a..7175f4f 100644 --- a/examples/download-albums.ts +++ b/examples/download-albums.ts @@ -36,7 +36,10 @@ export async function downloadAlbums( let alreadyLocal = 0; // A photo in several albums is handled once. const done = new Set(); - for (const album of lib.albums.list()) { + // fresh() waits for a refresh from the server and throws if it fails, so + // albums and photos added since the cache was last written are included. + const { albums } = await lib.fresh(); + for (const album of albums.list()) { const savePaths: string[] = []; for (const photo of album.photos.list()) { if (!done.has(photo.fileID)) { diff --git a/test/examples/download-albums.test.ts b/test/examples/download-albums.test.ts index 114bca6..8607b03 100644 --- a/test/examples/download-albums.test.ts +++ b/test/examples/download-albums.test.ts @@ -5,8 +5,10 @@ * The account has two albums sharing one photo, and one of its photos is a * live photo whose image is a HEIC with EXIF. The first run puts every * original at its save path, writes each photo's metadata beside it and each - * album's photos under `albums/`. The second run fetches nothing and changes - * no file. + * album's photos under `albums/`. Before the second run the account gains an + * album holding a new photo. The second run downloads that photo and writes + * its files and the new album's, and fetches nothing else and changes no + * other file. */ import { describe, it, expect, beforeEach, afterEach } from "vitest"; @@ -101,7 +103,7 @@ const readJSON = (path: string): unknown => JSON.parse(readFileSync(path, "utf-8")); describe("examples/download-albums.ts", () => { - it("downloads every album's photos with their metadata, and nothing on a second run", async () => { + it("downloads every album's photos with their metadata, and on a second run only what the account gained", async () => { // Album 1, "Trip", holds photos 1 and 2. Album 2, "Family", holds // photo 2 and photo 3, a live photo. const live = await asLivePhoto( @@ -109,20 +111,22 @@ describe("examples/download-albums.ts", () => { livePhotoZip({ "image.heic": HEIC_WITH_EXIF, "video.mov": VIDEO }), livePhotoHash(HEIC_WITH_EXIF, VIDEO), ); + const collections = [collection(1, "Trip"), collection(2, "Family")]; + const filesByAlbum = new Map([ + [1, [file(1, 1), file(2, 1)]], + [2, [file(2, 2), live.file]], + ]); const client = { whoami: () => ({ email: "u@example.com", userID: USER_ID }), collectionsSince: async (): Promise => ({ - collections: [collection(1, "Trip"), collection(2, "Family")], + collections: [...collections], deleted: [], cursor: 1, }), filesSince: async (args: { collectionID: number; }): Promise => ({ - files: - args.collectionID === 1 - ? [file(1, 1), file(2, 1)] - : [file(2, 2), live.file], + files: filesByAlbum.get(args.collectionID) ?? [], deleted: [], cursor: 1, }), @@ -270,18 +274,44 @@ describe("examples/download-albums.ts", () => { ], }); - // The second run, with a newly opened library, finds every photo - // already local, fetches nothing, and writes, renames or adds no - // file. + // Before the second run the account gains album 3, "Garden", holding + // a new photo 4. The second run, with a newly opened library, opens + // the cache written by the first and still downloads photo 4. It finds + // the other photos already local and fetches nothing else. + collections.push(collection(3, "Garden")); + filesByAlbum.set(3, [file(4, 3)]); backdate(dir); const before = mtimes(dir); const second = await open(); expect(await downloadAlbums(second, dir)).toEqual({ - downloaded: 0, + downloaded: 1, alreadyLocal: 3, }); await second.close(); - expect(calls).toBe(3); - expect(mtimes(dir)).toEqual(before); + expect(calls).toBe(4); + expect(readFileSync(join(day, "2026-03-01.4.jpg"), "utf-8")).toBe( + "original-4", + ); + expect(readJSON(join(dir, "albums", "3.json"))).toEqual({ + collectionID: 3, + name: "Garden", + savePaths: [join(DAY, "2026-03-01.4.jpg")], + }); + + // The second run adds only photo 4's files and album 3's, and + // rewrites, renames or removes no file from the first run. The two + // directories that gain a file are the only other changes. + const after = mtimes(dir); + expect( + [...after.keys()].filter((name) => !before.has(name)).sort(), + ).toEqual([ + join(DAY, "2026-03-01.4.jpg"), + join(DAY, "2026-03-01.4.jpg.json"), + join("albums", "3.json"), + ]); + for (const [name, mtime] of before) { + if (name === DAY || name === "albums") continue; + expect(after.get(name), name).toBe(mtime); + } }); }); -- 2.54.0