From 44b2859877da361716f31421100462b6d67ef321 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 1 Oct 2026 22:44:37 +0000 Subject: [PATCH 1/3] Remove the content cache's handling of an earlier version's live-photo ZIP (closes #151) quak keeps no handling of old data. The content cache no longer checks whether a live photo's file with no JSON file naming it is a ZIP an earlier version cached, and no longer removes it; such a file is left alone, as any other one already was. The README sentences and the tests about that ZIP are gone, and one download test no longer names it in its title. How a live photo is stored and served today is unchanged. Model: opus-5-5 --- README.md | 5 +-- TODO.md | 4 +++ src/library/content.ts | 43 +++----------------------- test/download/download.test.ts | 2 +- test/library/content-library.test.ts | 46 ---------------------------- test/library/content.test.ts | 37 ---------------------- 6 files changed, 11 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index 12c99e2..2c8b6a7 100644 --- a/README.md +++ b/README.md @@ -879,10 +879,7 @@ current account's records name. A live photo's original is cached as at its save path: its image and its video, each `originals/.` with its own extension, and -`originals/.livephoto.json` naming them; the two are evicted together. A -live photo that an earlier version cached as its ZIP is not served: the library -removes the ZIP when it opens the cache, and fetches the two files when the -photo is next read or precached. +`originals/.livephoto.json` naming them; the two are evicted together. A stored file appears only via an atomic temp-then-rename, so its presence means it is complete. Every downloaded original (by `quak get`, the cache, or diff --git a/TODO.md b/TODO.md index 29cfd56..f37e1cb 100644 --- a/TODO.md +++ b/TODO.md @@ -25,6 +25,10 @@ declares one. # Completed Steps +- 2026-10-01: The content cache no longer looks for a live photo that an earlier + version cached as one ZIP (issue 151). When the cache opens, a live photo's + file that no JSON file names is now always left alone. + - 2026-10-01: `examples/download-albums.ts` logs in, opens the library, and for every album downloads each photo to its save path, writes the photo's record and EXIF fields to a JSON file beside it, and writes the album's photos to diff --git a/src/library/content.ts b/src/library/content.ts index 396fdb8..213e4b8 100644 --- a/src/library/content.ts +++ b/src/library/content.ts @@ -29,14 +29,7 @@ // the cache does not count as saved there, but is copied there rather than // fetched again. -import { - closeSync, - existsSync, - openSync, - readFileSync, - readSync, - statSync, -} from "node:fs"; +import { existsSync, readFileSync, statSync } from "node:fs"; import { chmod, copyFile, @@ -281,25 +274,6 @@ const fileSize = (path: string): number | undefined => { const hasContent = (path: string | undefined): boolean => path !== undefined && (fileSize(path) ?? 0) > 0; -// Whether the file at `path` begins as a ZIP does, with `PK\x03\x04`. False -// when it cannot be read. -const isZip = (path: string): boolean => { - try { - const fd = openSync(path, "r"); - try { - const head = Buffer.alloc(4); - return ( - readSync(fd, head, 0, 4, 0) === 4 && - head.toString("latin1") === "PK\x03\x04" - ); - } finally { - closeSync(fd); - } - } catch { - return false; - } -}; - // A live photo's image and video are named with the extensions from inside its // ZIP, so their names alone do not say which is which. Wherever the cache or a // save path stores one, a JSON file of this name beside them names both. @@ -713,8 +687,7 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { await this.touch(cached.path); return { ...cached, bytes: size, cached: true }; } - // A recorded file that has since gone, or a live photo an earlier - // version stored as one ZIP, re-fetches below. + // A recorded file that has since gone re-fetches below. known.delete(fileID); } @@ -983,20 +956,14 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { if (id === undefined || !existsSync(path)) continue; // A live photo's image and video are one entry, as the JSON file // beside them names them. A live photo's file with no such JSON - // file is not its original. If it is a ZIP, it is the one an - // earlier version stored under the image's name, and is removed. - // Any other is left alone: another process may have just stored - // it and not yet written the JSON file. + // file is not its original and is left alone: another process may + // have just stored it and not yet written the JSON file. const livePhoto = names.has(livePhotoJSONName(String(id))) ? readLivePhotoJSON(dir, String(id)) : undefined; if (livePhoto !== undefined) { into.set(id, livePhoto); - } else if (isLivePhoto(id)) { - if (isZip(path)) { - await rm(path, { force: true }).catch(() => undefined); - } - } else { + } else if (!isLivePhoto(id)) { into.set(id, { path }); } } diff --git a/test/download/download.test.ts b/test/download/download.test.ts index e08a948..8a9d612 100644 --- a/test/download/download.test.ts +++ b/test/download/download.test.ts @@ -1883,7 +1883,7 @@ describe("downloadFile live photos", () => { expect(readdirSync(t.dir).sort()).toEqual(["f.JPG", "f.bin"]); }); - it("replaces what was at the destination, such as an earlier ZIP of the two", async () => { + it("replaces what was at the destination", async () => { const t = setup(livePhotoZip(), livePhoto); writeFileSync(t.outPath, livePhotoZip()); diff --git a/test/library/content-library.test.ts b/test/library/content-library.test.ts index ffa0cc7..a57722b 100644 --- a/test/library/content-library.test.ts +++ b/test/library/content-library.test.ts @@ -176,52 +176,6 @@ describe("Library content wiring", () => { await lib.close(); }); - it("removes a live photo's ZIP an earlier version cached when it opens, and precaches its image and video", async () => { - const { file: live, body } = await asLivePhoto(file(1, 1)); - class LiveClient extends MockClient { - override async filesSince(): Promise { - return { files: [live], deleted: [], cursor: 1 }; - } - } - const originals = join(root, "cache", "originals"); - const open = (opts: Partial): Promise => - Library.open({ - client: new LiveClient(), - cacheDirectory: join(root, "cache"), - contentSource: cdnSource(new Map([[1, body]])), - refreshIntervalSeconds: 3600, - precacheThumbnails: false, - precacheOriginals: false, - ...opts, - }); - // A first run records the library, so the next one knows that file 1 - // is a live photo when it opens the cache. - await (await open({})).close(); - writeFileSync(join(originals, "1.jpg"), livePhotoZip()); - - let precached!: () => void; - const done = new Promise((r) => (precached = r)); - const lib = await open({ - precacheOriginals: true, - onProgress: (e) => { - if (e.operation === "precacheOriginals" && e.status === "done") - precached(); - }, - }); - expect(existsSync(join(originals, "1.jpg"))).toBe(false); - await done; - - expect(readdirSync(originals).sort()).toEqual([ - "1.heic", - "1.livephoto.json", - "1.mov", - ]); - expect(lib.photos.byID({ fileID: 1 })!.record().originalPath).toBe( - join(originals, "1.heic"), - ); - await lib.close(); - }); - it("throws from content methods when opened without a content source", async () => { const lib = await Library.open({ client: new MockClient(), diff --git a/test/library/content.test.ts b/test/library/content.test.ts index f209359..b2c644a 100644 --- a/test/library/content.test.ts +++ b/test/library/content.test.ts @@ -566,43 +566,6 @@ describe("ContentCache live photos", () => { expect(events).toEqual(["skipped"]); }); - it("replaces a live photo an earlier version stored as a ZIP under the image's name", async () => { - const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); - mkdirSync(originals(), { recursive: true }); - writeFileSync(join(originals(), "5.HEIC"), livePhotoZip()); - const cache = cacheOf([live], new Map([[5, body]])); - // Opened without being told that file 5 is a live photo, the cache - // records the ZIP, and does not serve it. - await cache.open(); - - const result = await cache.original(5); - - expect(result.videoPath).toBe(join(originals(), "5.mov")); - expect(readdirSync(originals()).sort()).toEqual([ - "5.heic", - "5.livephoto.json", - "5.mov", - ]); - }); - - it("removes a live photo's ZIP an earlier version stored when it opens, so the precache fetches the image and video", async () => { - const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); - mkdirSync(originals(), { recursive: true }); - writeFileSync(join(originals(), "5.HEIC"), livePhotoZip()); - const cache = cacheOf([live], new Map([[5, body]])); - - await cache.open((fileID) => fileID === 5); - - expect(readdirSync(originals())).toEqual([]); - expect(cache.pathsFor(5)).toEqual({}); - const [fetched] = await cache.ensureOriginals({ fileIDs: [5] }); - expect(fetched).toEqual({ - fileID: 5, - path: join(originals(), "5.heic"), - }); - expect(cache.pathsFor(5)).toEqual({ originalPath: fetched!.path }); - }); - it("leaves the image and video another process has just stored when it opens before their JSON file is written", async () => { const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); const server = cdnSource(new Map([[5, body]])); -- 2.54.0 From 5d11889e1a016e0d94cf4a16d84deacf3c3595a6 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 1 Oct 2026 23:18:16 +0000 Subject: [PATCH 2/3] Live photo download no longer removes what was at its destination The image and video are renamed into place, and the rename already replaces the image's name, so the removal and the test that planted a ZIP there are gone, along with the comments describing it. Adds a test for a live photo the content cache recorded with no video, because it opened before knowing the file is a live photo while no JSON file named its image and video: original() fetches both again and writes the JSON file. The comment on that check now names this case. Model: opus-5-5 --- src/download/index.ts | 8 +++----- src/library/content.ts | 5 ++++- test/download/download.test.ts | 9 --------- test/library/content.test.ts | 30 ++++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/src/download/index.ts b/src/download/index.ts index 7729890..e27184b 100644 --- a/src/download/index.ts +++ b/src/download/index.ts @@ -361,8 +361,8 @@ const openPart = async ( // written unpacked: each part is named `destination` with the extension // replaced by its own entry's, and the two must differ ignoring case. When the // file records a hash, `:` must match it, each over that -// part's own bytes. Only then is whatever was at `destination` removed and the -// image, then the video, renamed into place; on any failure neither is stored. +// part's own bytes. Only then are the image, then the video, renamed into +// place; on any failure neither is stored. // // The ZIP is chosen by its uploader and may expand enormously, so each part is // written as it decompresses and never held, and the ZIP is refused once the @@ -486,7 +486,6 @@ const decryptLivePhoto = async ( await part.handle.sync(); await part.handle.close(); } - await rm(destination, { force: true }); await rename(image.tmpPath, path); try { await rename(video.tmpPath, videoPath); @@ -570,8 +569,7 @@ const fetchAndDecrypt = async ( }, api.getRetryOptions()); // Write `file`'s original to `outPath`. A live photo is written as its image -// and its video beside `outPath` instead, and whatever was at `outPath` is -// removed (see `decryptLivePhoto`). +// and its video beside `outPath` instead (see `decryptLivePhoto`). export const downloadFile = async ( api: ApiClient, file: EnteFile, diff --git a/src/library/content.ts b/src/library/content.ts index 213e4b8..383319b 100644 --- a/src/library/content.ts +++ b/src/library/content.ts @@ -687,7 +687,10 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI { await this.touch(cached.path); return { ...cached, bytes: size, cached: true }; } - // A recorded file that has since gone re-fetches below. + // A recorded file that has since gone re-fetches below. So does a + // live photo recorded with no video: the cache opened before the + // library's records said it is a live photo, while its image and + // video had no JSON file beside them yet. known.delete(fileID); } diff --git a/test/download/download.test.ts b/test/download/download.test.ts index 8a9d612..06788fd 100644 --- a/test/download/download.test.ts +++ b/test/download/download.test.ts @@ -1883,15 +1883,6 @@ describe("downloadFile live photos", () => { expect(readdirSync(t.dir).sort()).toEqual(["f.JPG", "f.bin"]); }); - it("replaces what was at the destination", async () => { - const t = setup(livePhotoZip(), livePhoto); - writeFileSync(t.outPath, livePhotoZip()); - - await t.run(); - - expect(readdirSync(t.dir).sort()).toEqual(["f.heic", "f.mov"]); - }); - it("renames the image and then the video into place, each from its own temp file", async () => { const t = setup(livePhotoZip(), livePhoto); diff --git a/test/library/content.test.ts b/test/library/content.test.ts index b2c644a..b461742 100644 --- a/test/library/content.test.ts +++ b/test/library/content.test.ts @@ -596,6 +596,36 @@ describe("ContentCache live photos", () => { expect(second!.pathsFor(5)).toEqual({}); }); + it("fetches a live photo's image and video again when the cache opened before knowing it is a live photo and no JSON file names them", async () => { + const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC")); + mkdirSync(originals(), { recursive: true }); + writeFileSync(join(originals(), "5.heic"), "an image"); + writeFileSync(join(originals(), "5.mov"), "a video"); + const cache = cacheOf([live], new Map([[5, body]])); + // Opened without being told that file 5 is a live photo, the cache + // records one of the two files as its original, with no video. + await cache.open(); + + const events: string[] = []; + const result = await cache.original(5, { + onProgress: (e) => events.push(e.status), + }); + + expect(events.at(-1)).toBe("done"); + expect(result).toEqual({ + path: join(originals(), "5.heic"), + videoPath: join(originals(), "5.mov"), + bytes: IMAGE.length, + }); + expect(readFileSync(result.path)).toEqual(Buffer.from(IMAGE)); + expect(readFileSync(result.videoPath!)).toEqual(Buffer.from(VIDEO)); + expect( + JSON.parse( + readFileSync(join(originals(), "5.livephoto.json"), "utf-8"), + ), + ).toEqual({ image: "5.heic", video: "5.mov" }); + }); + it.each(["missing", "empty"])( "fetches a live photo again when the video its JSON file names is %s", async (state) => { -- 2.54.0 From 02afdd6d09fbd3950409ccc767883409ab2d7b0e Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 2 Oct 2026 01:16:55 +0000 Subject: [PATCH 3/3] Test that a reopened library precaches a live photo with no JSON file Reopening the library over a live photo's image and video in the cache with no JSON file beside them records no original for the photo; the precache then fetches both and writes the JSON file. Model: opus-5-5 --- test/library/content-library.test.ts | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/test/library/content-library.test.ts b/test/library/content-library.test.ts index a57722b..d5976a4 100644 --- a/test/library/content-library.test.ts +++ b/test/library/content-library.test.ts @@ -176,6 +176,66 @@ describe("Library content wiring", () => { await lib.close(); }); + it("does not take a live photo's image or video with no JSON file as its original when it opens, and precaches both", async () => { + const { file: live, body } = await asLivePhoto(file(1, 1)); + class LiveClient extends MockClient { + override async filesSince(): Promise { + return { files: [live], deleted: [], cursor: 1 }; + } + } + const originals = join(root, "cache", "originals"); + const open = (opts: Partial): Promise => + Library.open({ + client: new LiveClient(), + cacheDirectory: join(root, "cache"), + contentSource: cdnSource(new Map([[1, body]])), + refreshIntervalSeconds: 3600, + precacheThumbnails: false, + precacheOriginals: false, + ...opts, + }); + // A first run records the library, so the next one knows that file 1 + // is a live photo when it opens the cache. + await (await open({})).close(); + writeFileSync(join(originals, "1.heic"), "an image"); + writeFileSync(join(originals, "1.mov"), "a video"); + + let precached!: () => void; + const done = new Promise((r) => (precached = r)); + const lib = await open({ + precacheOriginals: true, + onProgress: (e) => { + if (e.operation === "precacheOriginals" && e.status === "done") + precached(); + }, + }); + expect( + lib.photos.byID({ fileID: 1 })!.record().originalPath, + ).toBeUndefined(); + await done; + + expect(readdirSync(originals).sort()).toEqual([ + "1.heic", + "1.livephoto.json", + "1.mov", + ]); + expect(readFileSync(join(originals, "1.heic"))).toEqual( + Buffer.from(IMAGE), + ); + expect(readFileSync(join(originals, "1.mov"))).toEqual( + Buffer.from(VIDEO), + ); + expect( + JSON.parse( + readFileSync(join(originals, "1.livephoto.json"), "utf-8"), + ), + ).toEqual({ image: "1.heic", video: "1.mov" }); + expect(lib.photos.byID({ fileID: 1 })!.record().originalPath).toBe( + join(originals, "1.heic"), + ); + await lib.close(); + }); + it("throws from content methods when opened without a content source", async () => { const lib = await Library.open({ client: new MockClient(), -- 2.54.0