exif(): read HEIF/HEIC originals with exifreader (closes #145)

`photo.exif()` and `quak backup-metadata --exif` now read EXIF through `exifreader`. HEIC/HEIF originals get EXIF, including a live photo's image, as do the other formats `exifreader` reads. It replaces `exif-reader` and the hand-written JPEG scan. `PhotoExif` is unchanged.

The `backup-metadata` dump now holds `exifreader`'s tag output, with unnamed tags keyed `undefined-` plus their number. GPS altitude without a reference counts as above sea level. A latitude or longitude without its hemisphere tag, an unreadable text tag, and a date the parser rejects each give no field.

Licence: `exifreader` is MPL-2.0, used unmodified.

Model: opus-5-5

test/cli/commands.test.ts

@@ -56,6 +56,7 @@ import type { ContentSource } from "../../src/library/content.js";
 import type { Collection, EnteFile } from "../../src/model/types.js";
 import { init, toBase64 } from "../../src/crypto/index.js";
 import { defaultCacheDirectory } from "../../src/library/index.js";
+import { HEIC_WITH_EXIF } from "../exif-heic.js";
 import {
     asLivePhoto,
     cdnSource,
@@ -653,6 +654,31 @@ describe("a live photo", () => {
             height: 4,
         });
     });
+
+    it("backup-metadata --exif records the EXIF of a HEIC image", async () => {
+        const dir = join(root, "dump");
+
+        expect(
+            await backupMetadataCommand(
+                context(await livePhotoClient(HEIC_WITH_EXIF)),
+                dir,
+                { exif: true },
+            ),
+        ).toBe(0);
+
+        const record = JSON.parse(
+            readFileSync(
+                join(dir, "collections", "1-Vacation", "300.json"),
+                "utf-8",
+            ),
+        );
+        expect(record.imageMetadata.exifError).toBeUndefined();
+        expect(record.imageMetadata.exif).toMatchObject({
+            Make: { value: ["Canon"] },
+            Model: { value: ["EOS R5"] },
+            DateTimeOriginal: { value: ["2021:07:15 14:30:00"] },
+        });
+    });
 });
 
 describe("backup", () => {

test/cli/metadata-exif.test.ts

@@ -1,19 +1,24 @@
 /**
- * Tests for the JPEG EXIF scan behind `quak backup-metadata --exif`.
+ * Tests for reading EXIF (`src/exif.ts`) and the image metadata
+ * `quak backup-metadata --exif` records.
  *
- * The originals come from users' libraries, so a truncated or corrupt JPEG
- * must neither hang the scan nor throw out of it, and a malformed file must be
- * told apart from one that simply has no EXIF: the record carries the reason in
- * `exifError`. Each input below is a short hand-built byte array.
+ * The originals come from users' libraries, so a truncated or corrupt file
+ * must neither hang the read nor throw out of it: `readPhotoExif` gives `{}`,
+ * and `backup-metadata` tells an EXIF block it cannot read apart from a file
+ * that simply has no EXIF, carrying the reason in `exifError`. Each JPEG below
+ * is a short hand-built byte array; the HEIC is a real file.
  */
 
 import { describe, expect, it } from "vitest";
-import { extractExifFromJpeg } from "../../src/exif.js";
+import { readPhotoExif } from "../../src/exif.js";
 import { extractImageMetadata } from "../../src/metadata-backup.js";
+import { HEIC_WITH_EXIF } from "../exif-heic.js";
 
 const SOI = [0xff, 0xd8]; // start of image
-const SOS = [0xff, 0xda, 0x00, 0x02]; // start of scan, where the scan stops
+const SOS = [0xff, 0xda, 0x00, 0x02]; // start of scan
 const EXIF_HEADER = [0x45, 0x78, 0x69, 0x66, 0x00, 0x00]; // "Exif\0\0"
+const APP0 = [0xff, 0xe0, 0x00, 0x04, 0x00, 0x00];
+const ZERO_LENGTH_APP0 = [0xff, 0xe0, 0x00, 0x00];
 
 // A big-endian TIFF block with one IFD entry: Orientation (0x0112), SHORT, 6.
 const TIFF_ORIENTATION_6 = [
@@ -22,6 +27,66 @@ const TIFF_ORIENTATION_6 = [
     0x00, 0x00,
 ];
 
+// A big-endian TIFF block holding Orientation 6 and DateTimeOriginal
+// "0000:00:00 00:00:00", which a camera with an unset clock writes.
+const TIFF_UNSET_DATE = [
+    ...[0x4d, 0x4d, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08], // the first IFD at 8
+    // The first IFD, at 8: two entries, then no next IFD.
+    ...[0x00, 0x02],
+    // Orientation (0x0112), SHORT, 6.
+    ...[0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x06, 0x00, 0x00],
+    // The Exif IFD's offset (0x8769), LONG, 38.
+    ...[0x87, 0x69, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x26],
+    ...[0x00, 0x00, 0x00, 0x00],
+    // The Exif IFD, at 38: one entry, then no next IFD.
+    ...[0x00, 0x01],
+    // DateTimeOriginal (0x9003), 20 ASCII bytes at 56.
+    ...[0x90, 0x03, 0x00, 0x02, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x38],
+    ...[0x00, 0x00, 0x00, 0x00],
+    ...new TextEncoder().encode("0000:00:00 00:00:00\0"), // at 56
+];
+
+// A big-endian TIFF block holding a GPSAltitude of 12.5 m and no
+// GPSAltitudeRef.
+const TIFF_ALTITUDE_WITHOUT_REF = [
+    ...[0x4d, 0x4d, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08], // the first IFD at 8
+    // The first IFD, at 8: one entry, then no next IFD.
+    ...[0x00, 0x01],
+    // The GPS IFD's offset (0x8825), LONG, 26.
+    ...[0x88, 0x25, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x1a],
+    ...[0x00, 0x00, 0x00, 0x00],
+    // The GPS IFD, at 26: one entry, then no next IFD.
+    ...[0x00, 0x01],
+    // GPSAltitude (0x0006), one RATIONAL at 44.
+    ...[0x00, 0x06, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x2c],
+    ...[0x00, 0x00, 0x00, 0x00],
+    ...[0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x02], // 25/2, at 44
+];
+
+// A big-endian TIFF block holding Orientation 6 and a Make whose value lies
+// past the end of the file.
+const TIFF_MAKE_PAST_END = [
+    ...[0x4d, 0x4d, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08], // the first IFD at 8
+    // The first IFD, at 8: two entries, then no next IFD.
+    ...[0x00, 0x02],
+    // Make (0x010f), 6 ASCII bytes at 4096.
+    ...[0x01, 0x0f, 0x00, 0x02, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x10, 0x00],
+    // Orientation (0x0112), SHORT, 6.
+    ...[0x01, 0x12, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x06, 0x00, 0x00],
+    ...[0x00, 0x00, 0x00, 0x00],
+];
+
+// A big-endian TIFF block with one IFD entry that exifreader has no name for:
+// tag 0xc000 (49152), SHORT, 7.
+const TIFF_UNNAMED_TAG = [
+    ...[0x4d, 0x4d, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x08], // the first IFD at 8
+    // The first IFD, at 8: one entry, then no next IFD.
+    ...[0x00, 0x01],
+    // Tag 0xc000, SHORT, 7.
+    ...[0xc0, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x07, 0x00, 0x00],
+    ...[0x00, 0x00, 0x00, 0x00],
+];
+
 // An APP1 segment whose length field matches its data.
 const app1 = (data: number[]): number[] => {
     const len = data.length + 2;
@@ -31,74 +96,87 @@ const app1 = (data: number[]): number[] => {
 const bytes = (...parts: number[][]): Uint8Array =>
     new Uint8Array(parts.flat());
 
-describe("extractExifFromJpeg", () => {
-    it("returns the EXIF segment of a valid JPEG", () => {
+describe("readPhotoExif", () => {
+    it("reads the common fields of a valid JPEG", () => {
         const data = [...EXIF_HEADER, ...TIFF_ORIENTATION_6];
-        const scan = extractExifFromJpeg(bytes(SOI, app1(data), SOS));
-        expect(scan.error).toBeUndefined();
-        expect([...scan.exif!]).toEqual(data);
+        expect(readPhotoExif(bytes(SOI, app1(data), SOS))).toStrictEqual({
+            orientation: 6,
+        });
     });
 
-    it("returns nothing for a file that is not a JPEG", () => {
-        const png = bytes([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
-        expect(extractExifFromJpeg(png)).toEqual({});
+    it("gives no dateTimeOriginal for a DateTimeOriginal of 0000:00:00 00:00:00", () => {
+        const data = [...EXIF_HEADER, ...TIFF_UNSET_DATE];
+        expect(readPhotoExif(bytes(SOI, app1(data), SOS))).toStrictEqual({
+            orientation: 6,
+        });
     });
 
-    it("returns nothing for a JPEG without EXIF", () => {
-        const app0 = [0xff, 0xe0, 0x00, 0x04, 0x00, 0x00];
-        expect(extractExifFromJpeg(bytes(SOI, app0, SOS))).toEqual({});
+    it("reads a GPSAltitude without GPSAltitudeRef as above sea level", () => {
+        const data = [...EXIF_HEADER, ...TIFF_ALTITUDE_WITHOUT_REF];
+        expect(readPhotoExif(bytes(SOI, app1(data), SOS))).toStrictEqual({
+            gpsAltitude: 12.5,
+        });
     });
 
-    it("ignores an APP1 segment too short to hold the Exif header", () => {
-        // A length under 8 cannot hold the six-byte "Exif\0\0" header, so the
-        // segment is not EXIF. This one has length 7 and holds only "Exif\0",
-        // which the old code, lacking the length check, returned as EXIF.
-        const short = app1(EXIF_HEADER.slice(0, 5));
-        expect(extractExifFromJpeg(bytes(SOI, short, SOS))).toEqual({});
+    it("gives no make for a Make whose value lies past the end of the file", () => {
+        const data = [...EXIF_HEADER, ...TIFF_MAKE_PAST_END];
+        expect(readPhotoExif(bytes(SOI, app1(data), SOS))).toStrictEqual({
+            orientation: 6,
+        });
     });
 
-    it("accepts an APP1 segment of length 8 holding just the Exif header", () => {
-        const scan = extractExifFromJpeg(bytes(SOI, app1(EXIF_HEADER), SOS));
-        expect(scan.error).toBeUndefined();
-        expect([...scan.exif!]).toEqual(EXIF_HEADER);
-    });
-
-    it("reports a JPEG truncated inside a segment header", () => {
-        const scan = extractExifFromJpeg(bytes(SOI, [0xff, 0xe1, 0x00]));
-        expect(scan.exif).toBeUndefined();
-        expect(scan.error).toMatch(/truncated segment length/);
-    });
-
-    it("reports a JPEG that ends before the image data", () => {
-        const app0 = [0xff, 0xe0, 0x00, 0x04, 0x00, 0x00];
-        const scan = extractExifFromJpeg(bytes(SOI, app0));
-        expect(scan.error).toMatch(/ends before the image data/);
-    });
-
-    it("stops on a zero-length segment instead of looping", () => {
-        // A length of 0 would otherwise step the scan by 2 bytes at a time
-        // through the rest of the file, reading garbage as markers.
-        const zero = [0xff, 0xe0, 0x00, 0x00];
-        const scan = extractExifFromJpeg(
-            bytes(SOI, zero, zero, zero, zero, SOS),
-        );
-        expect(scan.error).toMatch(/segment length 0 at byte 2 is too small/);
-    });
-
-    it("stops on a segment length of 1", () => {
-        const scan = extractExifFromJpeg(
-            bytes(SOI, [0xff, 0xe0, 0x00, 0x01], SOS),
-        );
-        expect(scan.error).toMatch(/segment length 1 at byte 2 is too small/);
-    });
-
-    it("reports a segment length that runs past the end of the file", () => {
+    it.each([
+        [
+            "a file that is not an image",
+            new TextEncoder().encode("just some text, not an image"),
+        ],
+        [
+            "a PNG without EXIF",
+            bytes([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+        ],
+        ["a JPEG without EXIF", bytes(SOI, APP0, SOS)],
+        // A length under 8 cannot hold the six-byte "Exif\0\0" header. This one
+        // has length 7 and holds only "Exif\0", so a read past its end would
+        // take the next segment's bytes as EXIF.
+        [
+            "an APP1 segment too short to hold the Exif header",
+            bytes(SOI, app1(EXIF_HEADER.slice(0, 5)), SOS),
+        ],
+        [
+            "an APP1 segment holding just the Exif header",
+            bytes(SOI, app1(EXIF_HEADER), SOS),
+        ],
+        [
+            "a JPEG truncated inside a segment header",
+            bytes(SOI, [0xff, 0xe1, 0x00]),
+        ],
+        ["a JPEG that ends before the image data", bytes(SOI, APP0)],
+        // A length of 0 would step a scan by 2 bytes at a time through the
+        // rest of the file, reading garbage as markers.
+        [
+            "a zero-length segment",
+            bytes(
+                SOI,
+                ZERO_LENGTH_APP0,
+                ZERO_LENGTH_APP0,
+                ZERO_LENGTH_APP0,
+                ZERO_LENGTH_APP0,
+                SOS,
+            ),
+        ],
+        ["a segment length of 1", bytes(SOI, [0xff, 0xe0, 0x00, 0x01], SOS)],
         // APP1 claims 0x4000 bytes but only the "Exif\0\0" header follows.
-        const scan = extractExifFromJpeg(
+        [
+            "a segment length that runs past the end of the file",
             bytes(SOI, [0xff, 0xe1, 0x40, 0x00], EXIF_HEADER),
-        );
-        expect(scan.exif).toBeUndefined();
-        expect(scan.error).toMatch(/runs past the end of the file/);
+        ],
+        // "XX" where the TIFF byte order belongs.
+        [
+            "an EXIF block that cannot be parsed",
+            bytes(SOI, app1([...EXIF_HEADER, 0x58, 0x58]), SOS),
+        ],
+    ])("returns no fields for %s", (_, input) => {
+        expect(readPhotoExif(input)).toStrictEqual({});
     });
 });
 
@@ -108,10 +186,30 @@ describe("extractImageMetadata", () => {
             bytes(SOI, app1([...EXIF_HEADER, ...TIFF_ORIENTATION_6]), SOS),
         );
         expect(meta?.exifError).toBeUndefined();
-        expect(meta?.exif).toMatchObject({ Image: { Orientation: 6 } });
+        expect(meta?.exif).toMatchObject({ Orientation: { value: 6 } });
     });
 
-    it("returns nothing for a file that is not a JPEG", () => {
+    it("parses EXIF from a HEIC", () => {
+        const meta = extractImageMetadata(HEIC_WITH_EXIF);
+        expect(meta?.exifError).toBeUndefined();
+        expect(meta?.exif).toMatchObject({
+            Make: { value: ["Canon"] },
+            Model: { value: ["EOS R5"] },
+            DateTimeOriginal: { value: ["2021:07:15 14:30:00"] },
+            Orientation: { value: 6 },
+            GPSLatitudeRef: { value: ["N"] },
+        });
+    });
+
+    it("keys a tag exifreader has no name for by its number", () => {
+        const meta = extractImageMetadata(
+            bytes(SOI, app1([...EXIF_HEADER, ...TIFF_UNNAMED_TAG]), SOS),
+        );
+        expect(meta?.exifError).toBeUndefined();
+        expect(meta?.exif).toMatchObject({ "undefined-49152": { value: 7 } });
+    });
+
+    it("returns nothing for a file that is not an image", () => {
         const text = new TextEncoder().encode("just some text, not an image");
         expect(extractImageMetadata(text)).toBeUndefined();
     });
@@ -121,14 +219,20 @@ describe("extractImageMetadata", () => {
             bytes(SOI, [0xff, 0xe1, 0x40, 0x00], EXIF_HEADER),
         );
         expect(meta?.exif).toBeUndefined();
-        expect(meta?.exifError).toMatch(/runs past the end of the file/);
+        expect(meta?.exifError).toBe(
+            "no tag could be read from the EXIF block",
+        );
     });
 
     it("keeps the raw bytes and the reason when EXIF cannot be parsed", () => {
-        const data = [...EXIF_HEADER, 0x58, 0x58];
-        const meta = extractImageMetadata(bytes(SOI, app1(data), SOS));
+        // The raw bytes are the whole EXIF block as exifreader finds it: for a
+        // JPEG, the APP1 segment, marker and length included.
+        const segment = app1([...EXIF_HEADER, 0x58, 0x58]);
+        const meta = extractImageMetadata(bytes(SOI, segment, SOS));
         expect(meta?.exif).toBeUndefined();
-        expect(meta?.exifRaw).toBe(Buffer.from(data).toString("base64"));
-        expect(meta?.exifError).toEqual(expect.any(String));
+        expect(meta?.exifRaw).toBe(Buffer.from(segment).toString("base64"));
+        expect(meta?.exifError).toBe(
+            "no tag could be read from the EXIF block",
+        );
     });
 });

test/exif-heic.ts

@@ -0,0 +1,27 @@
+/**
+ * `exif.heic`, beside this file: a real 64x64 HEIC whose EXIF holds the same
+ * values as the hand-built JPEG in `library/content-library.test.ts`, for the
+ * tests of `exif()` and `backup-metadata --exif`.
+ *
+ * It was made once, in a throwaway node:22-alpine container (Alpine 3.23.3),
+ * with libheif 1.23.0 and exiftool 13.55:
+ *
+ *     apk add libheif-tools exiftool imagemagick
+ *     magick -size 64x64 gradient:red-blue -depth 8 in.png
+ *     heif-enc -q 30 -o exif.heic in.png
+ *     exiftool -overwrite_original \
+ *         -Make=Canon -Model="EOS R5" -LensModel="RF50mm F1.8 STM" \
+ *         -DateTimeOriginal="2021:07:15 14:30:00" -OffsetTimeOriginal="+02:00" \
+ *         -ExposureTime=1/250 -FNumber=2.8 -ISO=400 -FocalLength=50 \
+ *         -Orientation#=6 \
+ *         -GPSLatitude="40 26 46" -GPSLatitudeRef=N \
+ *         -GPSLongitude="79 58 56" -GPSLongitudeRef=W \
+ *         -GPSAltitude=12.5 -GPSAltitudeRef#=1 \
+ *         exif.heic
+ */
+
+import { readFileSync } from "node:fs";
+
+export const HEIC_WITH_EXIF = new Uint8Array(
+    readFileSync(new URL("exif.heic", import.meta.url)),
+);

test/library/content-library.test.ts

@@ -25,7 +25,16 @@ import { Library, type LibraryOptions } from "../../src/library/index.js";
 import type { ContentSource } from "../../src/library/content.js";
 import type { CollectionsPage, FilesPage } from "../../src/client.js";
 import type { Collection, EnteFile } from "../../src/model/types.js";
-import { asLivePhoto, cdnSource, IMAGE, livePhotoZip } from "../live-photo.js";
+import type { PhotoExif } from "../../src/exif.js";
+import { HEIC_WITH_EXIF } from "../exif-heic.js";
+import {
+    asLivePhoto,
+    cdnSource,
+    IMAGE,
+    livePhotoHash,
+    livePhotoZip,
+    VIDEO,
+} from "../live-photo.js";
 
 const USER_ID = 7;
 
@@ -297,7 +306,7 @@ const JPEG_WITH_EXIF = new Uint8Array([
 ]);
 
 // A JPEG whose EXIF segment is laid out correctly but holds "XX" where the TIFF
-// byte order belongs, so exif-reader cannot parse it.
+// byte order belongs, so exifreader cannot parse it.
 const JPEG_WITH_BAD_EXIF = new Uint8Array([
     ...[0xff, 0xd8], // start of image
     ...[0xff, 0xe1, ...u16(2 + 6 + 2)], // APP1 and its length
@@ -411,7 +420,49 @@ describe("Photo save path, local copy, content and EXIF", () => {
         await lib.close();
     });
 
-    it("returns no EXIF fields for an original that is not a JPEG", async () => {
+    // What exif() returns for HEIC_WITH_EXIF, which holds the same values as
+    // JPEG_WITH_EXIF.
+    const heicFields: PhotoExif = {
+        make: "Canon",
+        model: "EOS R5",
+        lensModel: "RF50mm F1.8 STM",
+        dateTimeOriginal: new Date(Date.UTC(2021, 6, 15, 14, 30)),
+        offsetTimeOriginal: "+02:00",
+        exposureTime: 1 / 250,
+        fNumber: 2.8,
+        iso: 400,
+        focalLength: 50,
+        orientation: 6,
+        gpsLatitude: 40 + 26 / 60 + 46 / 3600,
+        gpsLongitude: -(79 + 58 / 60 + 56 / 3600),
+        gpsAltitude: -12.5,
+    };
+
+    it("reads the same common EXIF fields from a HEIC original", async () => {
+        const lib = await open({ contentSource: stubSource(HEIC_WITH_EXIF) });
+        expect(await lib.photos.byID({ fileID: 1 })!.exif()).toStrictEqual(
+            heicFields,
+        );
+        await lib.close();
+    });
+
+    it("reads the EXIF of a live photo whose image is a HEIC", async () => {
+        const { file: live, body } = await asLivePhoto(
+            file(1, 1),
+            livePhotoZip({ "image.heic": HEIC_WITH_EXIF, "video.mov": VIDEO }),
+            livePhotoHash(HEIC_WITH_EXIF, VIDEO),
+        );
+        const lib = await open({
+            client: new FilesClient([live]),
+            contentSource: cdnSource(new Map([[1, body]])),
+        });
+        expect(await lib.photos.byID({ fileID: 1 })!.exif()).toStrictEqual(
+            heicFields,
+        );
+        await lib.close();
+    });
+
+    it("returns no EXIF fields for an original that is not an image", async () => {
         const lib = await open();
         expect(await lib.photos.byID({ fileID: 1 })!.exif()).toStrictEqual({});
         await lib.close();