sneak/quak
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

quak backup writes each original's EXIF, XMP and dimensions into its JSON (closes #167)
check / check (push) Successful in 3m11s
Details

Browse Source 
Each file's JSON gains imageMetadata, what extractImageMetadata finds in
the stored original (for a live photo, its image), or the reason the
read failed in imageMetadataError; a failed read fails neither the file
nor the run. A video is not read. An original is read when the run
stores it or when its JSON has neither field; otherwise the field is
carried over from that JSON, so a run does not read every original
again. The hand-built JPEG fixtures move to test/exif-jpeg.ts so the
backup tests can use them.

Judgement call: an original with nothing to record gets imageMetadata
{} instead of no field, so it is not read again on every run.

Model: opus-5-5
This commit is contained in:
clawbot
2026-10-06 10:48:12 +00:00
parent ad0e11407f
commit df34735989
7 changed files with 371 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/cli/backup.test.ts
+178
View File
@@ -33,6 +33,7 @@
*/
import {
chmodSync,
existsSync,
lstatSync,
mkdirSync,
@@ -53,12 +54,16 @@ import { runBackup, type BackupLibrary } from "../../src/backup.js";
import { Library } from "../../src/library/index.js";
import type { ContentSource } from "../../src/library/content.js";
import type { CollectionsPage, FilesPage } from "../../src/client.js";
import { extractImageMetadata } from "../../src/metadata-backup.js";
import type { MLData } from "../../src/mldata-fetch.js";
import type { Collection, EnteFile } from "../../src/model/types.js";
import { HEIC_WITH_EXIF } from "../exif-heic.js";
import { JPEG_WITH_EXIF } from "../exif-jpeg.js";
import {
asLivePhoto,
cdnSource,
IMAGE,
livePhotoHash,
livePhotoZip,
VIDEO,
} from "../live-photo.js";
@@ -1018,6 +1023,179 @@ describe("account and album records", () => {
});
});
// What a file's JSON holds as `imageMetadata` for an original of `bytes`.
const imageMetadataOf = (bytes: Uint8Array): unknown =>
JSON.parse(JSON.stringify(extractImageMetadata(bytes)));
describe("image metadata in each file's JSON", () => {
// Serves `files` in the Vacation album and nothing in Work.
class FilesClient extends MockClient {
constructor(private readonly files: EnteFile[]) {
super();
}
override async filesSince(args: {
collectionID: number;
}): Promise<FilesPage> {
return {
files: args.collectionID === 1 ? this.files : [],
deleted: [],
cursor: 1,
};
}
}
// Writes `originals.get(fileID)` as each file's original, looked up at
// each fetch, so a test can change it between runs.
const bytesSource = (
originals: Map<number, Uint8Array>,
): ContentSource => ({
original: async ({ file: f, destination }) => {
const bytes = originals.get(f.id)!;
writeFileSync(destination, bytes);
return { bytesWritten: bytes.length };
},
thumbnail: async () => {
throw new Error("no thumbnails in this source");
},
});
// A JPEG, a HEIC, a video, and a file holding no image metadata. The
// video's bytes are the JPEG's, so reading it would find EXIF.
const clip = file(602, 1, "clip.mov");
const files = [
file(600, 1, "photo.jpg"),
file(601, 1, "photo.heic"),
{ ...clip, metadata: { ...clip.metadata, fileType: "video" as const } },
file(603, 1, "notes.png"),
];
const originals = (): Map<number, Uint8Array> =>
new Map([
[600, JPEG_WITH_EXIF],
[601, HEIC_WITH_EXIF],
[602, JPEG_WITH_EXIF],
[603, new TextEncoder().encode("not an image")],
]);
const open = (bytes = originals()): Promise<Library> =>
openLibrary(bytesSource(bytes), new FilesClient(files));
// Rewrite the JSON of `fileID` without its image metadata, as an earlier
// version of quak wrote it.
const dropImageMetadata = (outDir: string, fileID: number): void => {
const json = fileJSON(outDir, fileID);
delete json.imageMetadata;
writeFileSync(saved(outDir, `${fileID}.json`), JSON.stringify(json));
};
it("writes each new original's image metadata, and none for a video", async () => {
const lib = await open();
const outDir = join(root, "backup");
const result = await lib.backup({ downloadDirectory: outDir });
expect(result).toMatchObject({ downloaded: 4, failed: 0 });
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
imageMetadataOf(JPEG_WITH_EXIF),
);
expect(fileJSON(outDir, 601).imageMetadata).toEqual(
imageMetadataOf(HEIC_WITH_EXIF),
);
expect(fileJSON(outDir, 602)).not.toHaveProperty("imageMetadata");
expect(fileJSON(outDir, 602)).not.toHaveProperty("imageMetadataError");
// Empty, so that the next run does not read it again.
expect(fileJSON(outDir, 603).imageMetadata).toEqual({});
await lib.close();
});
it("reads a live photo's image", async () => {
const { file: live, body } = await asLivePhoto(
file(500, 1, "IMG_0500.HEIC"),
livePhotoZip({ "image.heic": HEIC_WITH_EXIF, "video.mov": VIDEO }),
livePhotoHash(HEIC_WITH_EXIF, VIDEO),
);
const lib = await openLibrary(
cdnSource(new Map([[500, body]])),
new FilesClient([live]),
);
const outDir = join(root, "backup");
await lib.backup({ downloadDirectory: outDir });
expect(fileJSON(outDir, 500).imageMetadata).toEqual(
imageMetadataOf(HEIC_WITH_EXIF),
);
await lib.close();
});
it("keeps the image metadata of an original stored by an earlier run without reading it again", async () => {
const lib = await open();
const outDir = join(root, "backup");
await lib.backup({ downloadDirectory: outDir });
// A read of the original now would give the HEIC's.
writeFileSync(saved(outDir, "600.jpg"), HEIC_WITH_EXIF);
const second = await lib.backup({ downloadDirectory: outDir });
expect(second).toMatchObject({ downloaded: 0, failed: 0 });
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
imageMetadataOf(JPEG_WITH_EXIF),
);
await lib.close();
});
it("reads an original stored by an earlier version, whose JSON has no image metadata", async () => {
const lib = await open();
const outDir = join(root, "backup");
await lib.backup({ downloadDirectory: outDir });
dropImageMetadata(outDir, 600);
const second = await lib.backup({ downloadDirectory: outDir });
expect(second).toMatchObject({ downloaded: 0, failed: 0 });
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
imageMetadataOf(JPEG_WITH_EXIF),
);
await lib.close();
});
it("reads an original the run stores again, though its JSON has image metadata", async () => {
const bytes = originals();
const lib = await open(bytes);
const outDir = join(root, "backup");
await lib.backup({ downloadDirectory: outDir });
rmSync(saved(outDir, "600.jpg"));
bytes.set(600, HEIC_WITH_EXIF);
const second = await lib.backup({ downloadDirectory: outDir });
expect(second).toMatchObject({ downloaded: 1, failed: 0 });
expect(fileJSON(outDir, 600).imageMetadata).toEqual(
imageMetadataOf(HEIC_WITH_EXIF),
);
await lib.close();
});
// Root ignores file permissions, so this fails when run as root. The
// `test` phase of the `Dockerfile` runs as the `node` user.
it("gives the reason an original could not be read, failing neither the file nor the run", async () => {
const lib = await open();
const outDir = join(root, "backup");
await lib.backup({ downloadDirectory: outDir });
dropImageMetadata(outDir, 600);
const original = saved(outDir, "600.jpg");
chmodSync(original, 0o000);
const second = await lib
.backup({ downloadDirectory: outDir })
.finally(() => chmodSync(original, 0o600));
expect(second).toMatchObject({ failed: 0, errors: [] });
expect(existsSync(join(outDir, "failures.json"))).toBe(false);
expect(fileJSON(outDir, 600)).not.toHaveProperty("imageMetadata");
expect(fileJSON(outDir, 600).imageMetadataError).toMatch(/EACCES/);
await lib.close();
});
});
// Every entry under collections/, one level of directories deep, with each
// symlink's target.
const tree = (outDir: string): string[] => {
test/exif-heic.ts
+2 -2
View File
@@ -1,7 +1,7 @@
/**
* `exif.heic`, beside this file: a real 64x64 HEIC whose EXIF holds the same
* values as the hand-built JPEG in `library/content-library.test.ts`, for the
* tests of `exif()` and `backup-metadata --exif`.
* values as the hand-built JPEG in `exif-jpeg.ts`, for the tests of `exif()`,
* `backup-metadata --exif` and the image metadata `quak backup` records.
*
* It was made once, in a throwaway node:22-alpine container (Alpine 3.23.3),
* with libheif 1.23.0 and exiftool 13.55:
test/exif-jpeg.ts
+89
View File
@@ -0,0 +1,89 @@
/**
* Hand-built JPEGs for the tests of `exif()` and of the image metadata
* `quak backup` records: one whose EXIF holds the same values as `exif.heic`
* (see `exif-heic.ts`), and one whose EXIF cannot be parsed.
*/
// Big-endian bytes for the hand-built JPEG below.
const u16 = (n: number): number[] => [n >> 8, n & 0xff];
const u32 = (n: number): number[] => [...u16(n >>> 16), ...u16(n & 0xffff)];
const ascii = (s: string): number[] => [...new TextEncoder().encode(s), 0];
const rational = (num: number, den: number): number[] => [
...u32(num),
...u32(den),
];
// One IFD entry: tag, type (1 BYTE, 2 ASCII, 3 SHORT, 4 LONG, 5 RATIONAL),
// count, then the value when it fits in 4 bytes, else its offset.
const entry = (
tag: number,
type: number,
count: number,
value: number[],
): number[] => [...u16(tag), ...u16(type), ...u32(count), ...value];
// The TIFF block of a JPEG's EXIF segment, holding every field `Photo`'s typed
// methods return: the camera in the first IFD, the exposure in the Exif IFD,
// and a GPS position of 40°26'46" N, 79°58'56" W, 12.5 m below sea level.
// Offsets count from the start of this block.
const TIFF = [
...[0x4d, 0x4d, 0x00, 0x2a], // big-endian TIFF
...u32(8), // the first IFD's offset
// The first IFD, at 8: five entries, then no next IFD.
...u16(5),
...entry(0x010f, 2, 6, u32(74)), // Make
...entry(0x0110, 2, 7, u32(80)), // Model
...entry(0x0112, 3, 1, [...u16(6), 0, 0]), // Orientation
...entry(0x8769, 4, 1, u32(88)), // the Exif IFD's offset
...entry(0x8825, 4, 1, u32(246)), // the GPS IFD's offset
...u32(0),
...ascii("Canon"), // at 74
...ascii("EOS R5"), // at 80
0, // a pad byte
// The Exif IFD, at 88: seven entries, then no next IFD.
...u16(7),
...entry(0x829a, 5, 1, u32(178)), // ExposureTime
...entry(0x829d, 5, 1, u32(186)), // FNumber
...entry(0x8827, 3, 1, [...u16(400), 0, 0]), // ISOSpeedRatings
...entry(0x9003, 2, 20, u32(194)), // DateTimeOriginal
...entry(0x9011, 2, 7, u32(214)), // OffsetTimeOriginal
...entry(0x920a, 5, 1, u32(222)), // FocalLength
...entry(0xa434, 2, 16, u32(230)), // LensModel
...u32(0),
...rational(1, 250), // at 178
...rational(28, 10), // at 186
...ascii("2021:07:15 14:30:00"), // at 194
...ascii("+02:00"), // at 214
0, // a pad byte
...rational(50, 1), // at 222
...ascii("RF50mm F1.8 STM"), // at 230
// The GPS IFD, at 246: six entries, then no next IFD.
...u16(6),
...entry(0x0001, 2, 2, [...ascii("N"), 0, 0]), // GPSLatitudeRef
...entry(0x0002, 5, 3, u32(324)), // GPSLatitude
...entry(0x0003, 2, 2, [...ascii("W"), 0, 0]), // GPSLongitudeRef
...entry(0x0004, 5, 3, u32(348)), // GPSLongitude
...entry(0x0005, 1, 1, [1, 0, 0, 0]), // GPSAltitudeRef: below sea level
...entry(0x0006, 5, 1, u32(372)), // GPSAltitude
...u32(0),
...[...rational(40, 1), ...rational(26, 1), ...rational(46, 1)], // at 324
...[...rational(79, 1), ...rational(58, 1), ...rational(56, 1)], // at 348
...rational(25, 2), // at 372
];
export const JPEG_WITH_EXIF = new Uint8Array([
...[0xff, 0xd8], // start of image
...[0xff, 0xe1, ...u16(2 + 6 + TIFF.length)], // APP1 and its length
...[...ascii("Exif"), 0], // "Exif\0\0"
...TIFF,
...[0xff, 0xda, 0x00, 0x02], // start of scan
]);
// A JPEG whose EXIF segment is laid out correctly but holds "XX" where the TIFF
// byte order belongs, so exifreader cannot parse it.
export const JPEG_WITH_BAD_EXIF = new Uint8Array([
...[0xff, 0xd8], // start of image
...[0xff, 0xe1, ...u16(2 + 6 + 2)], // APP1 and its length
...[...ascii("Exif"), 0], // "Exif\0\0"
...[0x58, 0x58], // "XX"
...[0xff, 0xda, 0x00, 0x02], // start of scan
]);
test/library/content-library.test.ts
+1 -84
View File
@@ -29,6 +29,7 @@ import type { CollectionsPage, FilesPage } from "../../src/client.js";
import type { Collection, EnteFile } from "../../src/model/types.js";
import { readPhotoExif, type PhotoExif } from "../../src/exif.js";
import { HEIC_WITH_EXIF } from "../exif-heic.js";
import { JPEG_WITH_BAD_EXIF, JPEG_WITH_EXIF } from "../exif-jpeg.js";
import {
asLivePhoto,
cdnSource,
@@ -256,90 +257,6 @@ describe("Library content wiring", () => {
});
});
// Big-endian bytes for the hand-built JPEG below.
const u16 = (n: number): number[] => [n >> 8, n & 0xff];
const u32 = (n: number): number[] => [...u16(n >>> 16), ...u16(n & 0xffff)];
const ascii = (s: string): number[] => [...new TextEncoder().encode(s), 0];
const rational = (num: number, den: number): number[] => [
...u32(num),
...u32(den),
];
// One IFD entry: tag, type (1 BYTE, 2 ASCII, 3 SHORT, 4 LONG, 5 RATIONAL),
// count, then the value when it fits in 4 bytes, else its offset.
const entry = (
tag: number,
type: number,
count: number,
value: number[],
): number[] => [...u16(tag), ...u16(type), ...u32(count), ...value];
// The TIFF block of a JPEG's EXIF segment, holding every field `Photo`'s typed
// methods return: the camera in the first IFD, the exposure in the Exif IFD,
// and a GPS position of 40°26'46" N, 79°58'56" W, 12.5 m below sea level.
// Offsets count from the start of this block.
const TIFF = [
...[0x4d, 0x4d, 0x00, 0x2a], // big-endian TIFF
...u32(8), // the first IFD's offset
// The first IFD, at 8: five entries, then no next IFD.
...u16(5),
...entry(0x010f, 2, 6, u32(74)), // Make
...entry(0x0110, 2, 7, u32(80)), // Model
...entry(0x0112, 3, 1, [...u16(6), 0, 0]), // Orientation
...entry(0x8769, 4, 1, u32(88)), // the Exif IFD's offset
...entry(0x8825, 4, 1, u32(246)), // the GPS IFD's offset
...u32(0),
...ascii("Canon"), // at 74
...ascii("EOS R5"), // at 80
0, // a pad byte
// The Exif IFD, at 88: seven entries, then no next IFD.
...u16(7),
...entry(0x829a, 5, 1, u32(178)), // ExposureTime
...entry(0x829d, 5, 1, u32(186)), // FNumber
...entry(0x8827, 3, 1, [...u16(400), 0, 0]), // ISOSpeedRatings
...entry(0x9003, 2, 20, u32(194)), // DateTimeOriginal
...entry(0x9011, 2, 7, u32(214)), // OffsetTimeOriginal
...entry(0x920a, 5, 1, u32(222)), // FocalLength
...entry(0xa434, 2, 16, u32(230)), // LensModel
...u32(0),
...rational(1, 250), // at 178
...rational(28, 10), // at 186
...ascii("2021:07:15 14:30:00"), // at 194
...ascii("+02:00"), // at 214
0, // a pad byte
...rational(50, 1), // at 222
...ascii("RF50mm F1.8 STM"), // at 230
// The GPS IFD, at 246: six entries, then no next IFD.
...u16(6),
...entry(0x0001, 2, 2, [...ascii("N"), 0, 0]), // GPSLatitudeRef
...entry(0x0002, 5, 3, u32(324)), // GPSLatitude
...entry(0x0003, 2, 2, [...ascii("W"), 0, 0]), // GPSLongitudeRef
...entry(0x0004, 5, 3, u32(348)), // GPSLongitude
...entry(0x0005, 1, 1, [1, 0, 0, 0]), // GPSAltitudeRef: below sea level
...entry(0x0006, 5, 1, u32(372)), // GPSAltitude
...u32(0),
...[...rational(40, 1), ...rational(26, 1), ...rational(46, 1)], // at 324
...[...rational(79, 1), ...rational(58, 1), ...rational(56, 1)], // at 348
...rational(25, 2), // at 372
];
const JPEG_WITH_EXIF = new Uint8Array([
...[0xff, 0xd8], // start of image
...[0xff, 0xe1, ...u16(2 + 6 + TIFF.length)], // APP1 and its length
...[...ascii("Exif"), 0], // "Exif\0\0"
...TIFF,
...[0xff, 0xda, 0x00, 0x02], // start of scan
]);
// A JPEG whose EXIF segment is laid out correctly but holds "XX" where the TIFF
// byte order belongs, so exifreader cannot parse it.
const JPEG_WITH_BAD_EXIF = new Uint8Array([
...[0xff, 0xd8], // start of image
...[0xff, 0xe1, ...u16(2 + 6 + 2)], // APP1 and its length
...[...ascii("Exif"), 0], // "Exif\0\0"
...[0x58, 0x58], // "XX"
...[0xff, 0xda, 0x00, 0x02], // start of scan
]);
describe("Photo save path, local copy, content and EXIF", () => {
// The same account, with `files` in its album instead.
class FilesClient extends MockClient {