Compare commits
1
Commits
main
..
6215a887e1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6215a887e1 |
@@ -14,18 +14,14 @@ pre-1.0
|
||||
|
||||
# Next Step
|
||||
|
||||
None: every issue still open is done on `next` and waits for it to reach `main`.
|
||||
None: every issue still open is done on `next` or `next2` and waits for it to
|
||||
reach `main`.
|
||||
|
||||
Tagging and releases are decided by sneak alone, and happen only when he
|
||||
declares one.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-09-28: Tested the live-photo writer's fsyncs (issue 130). A test checks
|
||||
that the image's and the video's temp files are fsynced before either is
|
||||
renamed into place, and the directory after both renames, as the `writeAtomic`
|
||||
test does for one file.
|
||||
|
||||
- 2026-09-23: Live photos are stored as their image and their video (issue 107).
|
||||
A live photo, which Ente stores as one ZIP, is unpacked as it downloads into
|
||||
`<fileID>.<ext>` for the image and for the video, each with its extension from
|
||||
|
||||
+4
-34
@@ -24,14 +24,7 @@
|
||||
// does; thumbnails have none. On top of that this module refuses to record a
|
||||
// stored file that came out empty.
|
||||
|
||||
import {
|
||||
closeSync,
|
||||
existsSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
readSync,
|
||||
statSync,
|
||||
} from "node:fs";
|
||||
import { existsSync, readFileSync, statSync } from "node:fs";
|
||||
import {
|
||||
chmod,
|
||||
mkdir,
|
||||
@@ -253,25 +246,6 @@ const fileSize = (path: string): number | undefined => {
|
||||
const hasContent = (path: string | undefined): boolean =>
|
||||
path !== undefined && (fileSize(path) ?? 0) > 0;
|
||||
|
||||
// Whether the file at `path` begins as a ZIP does, with `PK\x03\x04`. False
|
||||
// when it cannot be read.
|
||||
const isZip = (path: string): boolean => {
|
||||
try {
|
||||
const fd = openSync(path, "r");
|
||||
try {
|
||||
const head = Buffer.alloc(4);
|
||||
return (
|
||||
readSync(fd, head, 0, 4, 0) === 4 &&
|
||||
head.toString("latin1") === "PK\x03\x04"
|
||||
);
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// A live photo's image and video are named with the extensions from inside its
|
||||
// ZIP, so their names alone do not say which is which. Wherever the cache or a
|
||||
// backup stores one, a JSON file of this name beside them names both.
|
||||
@@ -873,19 +847,15 @@ export class ContentCache implements PhotoContent, ThumbnailsAPI {
|
||||
if (id === undefined || !existsSync(path)) continue;
|
||||
// A live photo's image and video are one entry, as the JSON file
|
||||
// beside them names them. A live photo's file with no such JSON
|
||||
// file is not its original. If it is a ZIP, it is the one an
|
||||
// earlier version stored under the image's name, and is removed.
|
||||
// Any other is left alone: another process may have just stored
|
||||
// it and not yet written the JSON file.
|
||||
// file, such as the ZIP an earlier version stored under the
|
||||
// image's name, is not its original and is removed.
|
||||
const livePhoto = names.has(livePhotoJSONName(id))
|
||||
? readLivePhotoJSON(dir, id)
|
||||
: undefined;
|
||||
if (livePhoto !== undefined) {
|
||||
into.set(id, livePhoto);
|
||||
} else if (isLivePhoto(id)) {
|
||||
if (isZip(path)) {
|
||||
await rm(path, { force: true }).catch(() => undefined);
|
||||
}
|
||||
await rm(path, { force: true }).catch(() => undefined);
|
||||
} else {
|
||||
into.set(id, { path });
|
||||
}
|
||||
|
||||
@@ -1262,32 +1262,6 @@ describe("backup of live photos", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["missing", "empty"])(
|
||||
"fetches a live photo again when the image its JSON file names is %s",
|
||||
async (state) => {
|
||||
const { file: live, body } = await asLivePhoto(
|
||||
file(500, 10, "IMG_0500.HEIC"),
|
||||
);
|
||||
const lib = await open([live], new Map([[500, body]]));
|
||||
const outDir = join(root, "backup");
|
||||
await lib.backup({ downloadDirectory: outDir });
|
||||
const image = join(outDir, "originals", "500.heic");
|
||||
if (state === "missing") rmSync(image);
|
||||
else writeFileSync(image, "");
|
||||
|
||||
const result = await lib.backup({ downloadDirectory: outDir });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
downloaded: 1,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
});
|
||||
expect(readFileSync(image)).toEqual(Buffer.from(IMAGE));
|
||||
expect(tree(outDir)).toEqual(linked);
|
||||
await lib.close();
|
||||
},
|
||||
);
|
||||
|
||||
it("serves a live photo the backup stored to a library reading the backup", async () => {
|
||||
const { file: live, body } = await asLivePhoto(
|
||||
file(500, 10, "IMG_0500.HEIC"),
|
||||
|
||||
@@ -1909,24 +1909,6 @@ describe("downloadFile live photos", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("fsyncs each part's temp file before the renames and the directory after", async () => {
|
||||
const t = setup(livePhotoZip(), livePhoto);
|
||||
|
||||
await t.run();
|
||||
|
||||
// The same durability contract as `writeAtomic`, for two files: both
|
||||
// temp files are on disk before either is renamed into place, and the
|
||||
// directory is fsynced once both renames are done.
|
||||
const [imageTmp, videoTmp] = renameHook.calls.map((c) => c.from);
|
||||
expect(durabilityHook.events).toEqual([
|
||||
`sync:w:${imageTmp}`,
|
||||
`sync:w:${videoTmp}`,
|
||||
`rename:${join(t.dir, "f.heic")}`,
|
||||
`rename:${join(t.dir, "f.mov")}`,
|
||||
`sync:r:${t.dir}`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("stores neither part when the video cannot be renamed into place", async () => {
|
||||
const t = setup(livePhotoZip(), livePhoto);
|
||||
renameHook.failWith = new Error("simulated rename failure");
|
||||
|
||||
@@ -560,36 +560,6 @@ describe("ContentCache live photos", () => {
|
||||
expect(cache.pathsFor(5)).toEqual({ originalPath: fetched!.path });
|
||||
});
|
||||
|
||||
it("leaves the image and video another process has just stored when it opens before their JSON file is written", async () => {
|
||||
const { file: live, body } = await asLivePhoto(file(5, "IMG_5.HEIC"));
|
||||
const server = cdnSource(new Map([[5, body]]));
|
||||
let second: ContentCache | undefined;
|
||||
// Once the image and video are in place, and before the first cache
|
||||
// writes the JSON file naming them, a second cache opens.
|
||||
const source: ContentSource = {
|
||||
original: async (args) => {
|
||||
const stored = await server.original(args);
|
||||
second = cacheOf([live], new Map());
|
||||
await second.open((fileID) => fileID === 5);
|
||||
return stored;
|
||||
},
|
||||
thumbnail: (args) => server.thumbnail(args),
|
||||
};
|
||||
const first = buildCache({ files: [live], source }).cache;
|
||||
await first.open((fileID) => fileID === 5);
|
||||
|
||||
const result = await first.original(5);
|
||||
|
||||
expect(readdirSync(originals()).sort()).toEqual([
|
||||
"5.heic",
|
||||
"5.livephoto.json",
|
||||
"5.mov",
|
||||
]);
|
||||
expect(readFileSync(result.path)).toEqual(Buffer.from(IMAGE));
|
||||
expect(readFileSync(result.videoPath!)).toEqual(Buffer.from(VIDEO));
|
||||
expect(second!.pathsFor(5)).toEqual({});
|
||||
});
|
||||
|
||||
it.each(["missing", "empty"])(
|
||||
"fetches a live photo again when the video its JSON file names is %s",
|
||||
async (state) => {
|
||||
|
||||
@@ -787,13 +787,12 @@ describe("fixMissingThumbnails", () => {
|
||||
});
|
||||
|
||||
it("re-encodes smaller until the thumbnail fits the recorded size", async () => {
|
||||
// A noisy 64x48 JPEG, which the default encoding (quality 50, not
|
||||
// A noisy 400x300 JPEG, which the default encoding (quality 50, not
|
||||
// resized because it is under 720 px) cannot compress below the size
|
||||
// recorded here: one byte less than that encoding's ciphertext. It is
|
||||
// small so that each encode is quick even on a busy host.
|
||||
// recorded here: one byte less than that encoding's ciphertext.
|
||||
const fixMock = await buildThumbMock();
|
||||
const w = 64;
|
||||
const h = 48;
|
||||
const w = 400;
|
||||
const h = 300;
|
||||
const noisy = new Uint8Array(
|
||||
jpegJs.encode(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user