Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2269b413a4 |
+26
-31
@@ -21,7 +21,6 @@ import {
|
|||||||
originalName,
|
originalName,
|
||||||
thumbnailName,
|
thumbnailName,
|
||||||
} from "../src/cli-output.js";
|
} from "../src/cli-output.js";
|
||||||
import { freshCollections, freshFiles, freshFile } from "../src/cli-read.js";
|
|
||||||
import { runMetadataBackup } from "../src/metadata-backup.js";
|
import { runMetadataBackup } from "../src/metadata-backup.js";
|
||||||
import {
|
import {
|
||||||
listMissingThumbnails,
|
listMissingThumbnails,
|
||||||
@@ -184,30 +183,27 @@ program
|
|||||||
await init();
|
await init();
|
||||||
const client = requireSession();
|
const client = requireSession();
|
||||||
const lib = await openReadLibrary(client);
|
const lib = await openReadLibrary(client);
|
||||||
// Force a server round-trip and list in enumeration order (issue #36
|
const albums = lib.albums.list();
|
||||||
// amendment, issue #52): the pre-library CLI printed current state in
|
|
||||||
// this order, not the albums projection's newest-first order.
|
|
||||||
const collections = await freshCollections(lib);
|
|
||||||
|
|
||||||
if (opts.json) {
|
if (opts.json) {
|
||||||
stdout.write(
|
stdout.write(
|
||||||
JSON.stringify(
|
JSON.stringify(
|
||||||
collections.map((c) => ({
|
albums.map((a) => ({
|
||||||
id: c.id,
|
id: a.collectionID,
|
||||||
name: c.name,
|
name: a.name,
|
||||||
type: c.type,
|
type: a.type,
|
||||||
ownerID: c.ownerID,
|
ownerID: lib.getCollection(a.collectionID)?.ownerID,
|
||||||
isShared: c.isShared,
|
isShared: a.isShared,
|
||||||
updationTime: c.updationTime,
|
updationTime: a.updationTime,
|
||||||
})),
|
})),
|
||||||
null,
|
null,
|
||||||
2,
|
2,
|
||||||
) + "\n",
|
) + "\n",
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
for (const c of collections) {
|
for (const a of albums) {
|
||||||
stdout.write(
|
stdout.write(
|
||||||
`${c.id}\t${c.type}\t${c.name}${c.isShared ? " (shared)" : ""}\n`,
|
`${a.collectionID}\t${a.type}\t${a.name}${a.isShared ? " (shared)" : ""}\n`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -232,18 +228,21 @@ program
|
|||||||
}
|
}
|
||||||
|
|
||||||
const lib = await openReadLibrary(client);
|
const lib = await openReadLibrary(client);
|
||||||
// Force a server round-trip and list in enumeration order (issue #36
|
const album = lib.albums.byID({ collectionID });
|
||||||
// amendment, issue #52). Each file prints from its own decrypted
|
if (!album) {
|
||||||
// metadata (raw title, microsecond creationTime) via cli-output, and in
|
|
||||||
// the pre-library CLI's enumeration order, not the projection's
|
|
||||||
// newest-first order.
|
|
||||||
const files = await freshFiles(lib, collectionID);
|
|
||||||
if (!files) {
|
|
||||||
stderr.write(`Collection ${collectionID} not found\n`);
|
stderr.write(`Collection ${collectionID} not found\n`);
|
||||||
finish(lib, 1);
|
finish(lib, 1);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Present each file from its own decrypted metadata, not the
|
||||||
|
// PhotoRecord projection, so the raw title and the microsecond
|
||||||
|
// creationTime print as the pre-library CLI did (issue #52). The
|
||||||
|
// album's photo order is kept; only the field source changes.
|
||||||
|
const files = album.photos.list().flatMap((p) => {
|
||||||
|
const file = lib.getFile(collectionID, p.fileID);
|
||||||
|
return file ? [file] : [];
|
||||||
|
});
|
||||||
if (opts.json) {
|
if (opts.json) {
|
||||||
stdout.write(
|
stdout.write(
|
||||||
JSON.stringify(files.map(fileListRow), null, 2) + "\n",
|
JSON.stringify(files.map(fileListRow), null, 2) + "\n",
|
||||||
@@ -272,15 +271,13 @@ program
|
|||||||
}
|
}
|
||||||
|
|
||||||
const lib = await openReadLibrary(client);
|
const lib = await openReadLibrary(client);
|
||||||
// Force a server round-trip so the file resolves against current state
|
const photo = lib.photos.byID({ fileID });
|
||||||
// (issue #36 amendment, issue #52).
|
const file = lib.getFileByID(fileID);
|
||||||
const resolved = await freshFile(lib, fileID);
|
if (!photo || !file) {
|
||||||
if (!resolved) {
|
|
||||||
stderr.write(`File ${fileID} not found\n`);
|
stderr.write(`File ${fileID} not found\n`);
|
||||||
finish(lib, 1);
|
finish(lib, 1);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const { photo, file } = resolved;
|
|
||||||
|
|
||||||
const result = await photo.original();
|
const result = await photo.original();
|
||||||
// Default name is the file's own title, as the pre-library CLI used
|
// Default name is the file's own title, as the pre-library CLI used
|
||||||
@@ -307,15 +304,13 @@ program
|
|||||||
}
|
}
|
||||||
|
|
||||||
const lib = await openReadLibrary(client);
|
const lib = await openReadLibrary(client);
|
||||||
// Force a server round-trip so the file resolves against current state
|
const photo = lib.photos.byID({ fileID });
|
||||||
// (issue #36 amendment, issue #52).
|
const file = lib.getFileByID(fileID);
|
||||||
const resolved = await freshFile(lib, fileID);
|
if (!photo || !file) {
|
||||||
if (!resolved) {
|
|
||||||
stderr.write(`File ${fileID} not found\n`);
|
stderr.write(`File ${fileID} not found\n`);
|
||||||
finish(lib, 1);
|
finish(lib, 1);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const { photo, file } = resolved;
|
|
||||||
|
|
||||||
const result = await photo.thumbnail();
|
const result = await photo.thumbnail();
|
||||||
// Default name is thumb_<file's own title>, as the pre-library CLI
|
// Default name is thumb_<file's own title>, as the pre-library CLI
|
||||||
|
|||||||
@@ -1,62 +0,0 @@
|
|||||||
// How the CLI's read commands obtain current data.
|
|
||||||
//
|
|
||||||
// `collections`, `files --collection`, `get`, and `get-thumb` must answer for
|
|
||||||
// the account's state at the moment the command runs, not for whatever the
|
|
||||||
// local cache last happened to hold (owner amendment, issue #36). Each helper
|
|
||||||
// therefore forces a server round-trip through `Library.fresh()` and only then
|
|
||||||
// reads — so a collection, file, or metadata change made elsewhere is visible.
|
|
||||||
//
|
|
||||||
// `collections` and `files` also list in the library's own enumeration order —
|
|
||||||
// `listCollections()`/`listFiles()`, the order the pre-library CLI printed —
|
|
||||||
// rather than the `albums`/`photos` projection's newest-first order, which
|
|
||||||
// re-sorts the rows. The field values still come from each record's raw
|
|
||||||
// metadata via `cli-output.ts`.
|
|
||||||
|
|
||||||
import type { Collection, EnteFile } from "./model/types.js";
|
|
||||||
import type { Photo, PhotosAPI } from "./library/index.js";
|
|
||||||
|
|
||||||
// The slice of `Library` these helpers read. `Library` satisfies it
|
|
||||||
// structurally; a test can drive them with a stand-in that records the
|
|
||||||
// `fresh()` call and serves records in a known enumeration order.
|
|
||||||
export interface FreshReadLibrary {
|
|
||||||
fresh(): Promise<unknown>;
|
|
||||||
listCollections(): Collection[];
|
|
||||||
getCollection(id: number): Collection | undefined;
|
|
||||||
listFiles(collectionID: number): EnteFile[];
|
|
||||||
getFileByID(fileID: number): EnteFile | undefined;
|
|
||||||
photos: Pick<PhotosAPI, "byID">;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every live collection, current as of a forced refresh, in enumeration order.
|
|
||||||
export const freshCollections = async (
|
|
||||||
lib: FreshReadLibrary,
|
|
||||||
): Promise<Collection[]> => {
|
|
||||||
await lib.fresh();
|
|
||||||
return lib.listCollections();
|
|
||||||
};
|
|
||||||
|
|
||||||
// The files of one collection, current as of a forced refresh, in enumeration
|
|
||||||
// order. `undefined` (not an empty list) when the collection does not exist, so
|
|
||||||
// the caller can tell "no such collection" from "an empty collection".
|
|
||||||
export const freshFiles = async (
|
|
||||||
lib: FreshReadLibrary,
|
|
||||||
collectionID: number,
|
|
||||||
): Promise<EnteFile[] | undefined> => {
|
|
||||||
await lib.fresh();
|
|
||||||
if (!lib.getCollection(collectionID)) return undefined;
|
|
||||||
return lib.listFiles(collectionID);
|
|
||||||
};
|
|
||||||
|
|
||||||
// One file, current as of a forced refresh, resolved to both its content
|
|
||||||
// handle (`Photo`, for fetching bytes) and its raw record (`EnteFile`, for the
|
|
||||||
// default output name and field values). `undefined` when the file is unknown.
|
|
||||||
export const freshFile = async (
|
|
||||||
lib: FreshReadLibrary,
|
|
||||||
fileID: number,
|
|
||||||
): Promise<{ photo: Photo; file: EnteFile } | undefined> => {
|
|
||||||
await lib.fresh();
|
|
||||||
const photo = lib.photos.byID({ fileID });
|
|
||||||
const file = lib.getFileByID(fileID);
|
|
||||||
if (!photo || !file) return undefined;
|
|
||||||
return { photo, file };
|
|
||||||
};
|
|
||||||
+12
-77
@@ -6,17 +6,10 @@
|
|||||||
// existing copy loaded, that first refresh runs in the background and `open()`
|
// existing copy loaded, that first refresh runs in the background and `open()`
|
||||||
// returns as soon as the cached data is ready to serve — a slow or unreachable
|
// returns as soon as the cached data is ready to serve — a slow or unreachable
|
||||||
// server no longer stalls opening. A background timer then refreshes every
|
// server no longer stalls opening. A background timer then refreshes every
|
||||||
// `refreshIntervalSeconds`. Every default read is answered from RAM — no
|
// `refreshIntervalSeconds`. Every read is answered from RAM — no read touches
|
||||||
// default read touches the network. There is deliberately no `sync()`, no
|
// the network. There is deliberately no `sync()`, no `refresh()`, no
|
||||||
// `refresh()`, no `serverReachable` flag, and no "before each read" mode
|
// `serverReachable` flag, and no "before each read" mode (design #36): the
|
||||||
// (design #36).
|
// only ways state changes are the refreshes above.
|
||||||
//
|
|
||||||
// `fresh()` is the one exception (issue #75, an owner amendment to #36): it
|
|
||||||
// forces a refresh, awaits it, and only then hands back the read namespaces, so
|
|
||||||
// a caller that needs server-current data can ask for it. Concurrent `fresh()`
|
|
||||||
// calls coalesce onto one in-flight refresh, and a refresh that fails rejects
|
|
||||||
// the caller (the default reads stay silent and serve the last good copy). The
|
|
||||||
// default methods and the background loop are unchanged.
|
|
||||||
//
|
//
|
||||||
// A refresh stages all of its network work first and only mutates the store
|
// A refresh stages all of its network work first and only mutates the store
|
||||||
// once every fetch has succeeded. A refresh that fails partway therefore never
|
// once every fetch has succeeded. A refresh that fails partway therefore never
|
||||||
@@ -47,7 +40,6 @@ import {
|
|||||||
type AlbumsAPI,
|
type AlbumsAPI,
|
||||||
type PhotosAPI,
|
type PhotosAPI,
|
||||||
type TimelineAPI,
|
type TimelineAPI,
|
||||||
type FreshReads,
|
|
||||||
} from "./read.js";
|
} from "./read.js";
|
||||||
import {
|
import {
|
||||||
ContentCache,
|
ContentCache,
|
||||||
@@ -65,7 +57,6 @@ export {
|
|||||||
type AlbumsAPI,
|
type AlbumsAPI,
|
||||||
type PhotosAPI,
|
type PhotosAPI,
|
||||||
type TimelineAPI,
|
type TimelineAPI,
|
||||||
type FreshReads,
|
|
||||||
type PhotoFilter,
|
type PhotoFilter,
|
||||||
type TimelineGroup,
|
type TimelineGroup,
|
||||||
type GroupBy,
|
type GroupBy,
|
||||||
@@ -259,12 +250,7 @@ export class Library {
|
|||||||
private readonly precache?: Precache;
|
private readonly precache?: Precache;
|
||||||
|
|
||||||
private timer?: ReturnType<typeof setTimeout>;
|
private timer?: ReturnType<typeof setTimeout>;
|
||||||
// The in-flight refresh cycle, or undefined when none runs. One slot serves
|
private refreshing = false;
|
||||||
// both paths: the background loop skips when it is set, and a fresh read
|
|
||||||
// (issue #75) coalesces onto it or starts one. The promise carries the
|
|
||||||
// cycle's real outcome (it rejects on failure); the background loop ignores
|
|
||||||
// that, a fresh read propagates it.
|
|
||||||
private cycle?: Promise<void>;
|
|
||||||
// Guards the ML fetch pass so a slow backfill never runs twice at once; a
|
// Guards the ML fetch pass so a slow backfill never runs twice at once; a
|
||||||
// refresh whose pass is still running kicks nothing new.
|
// refresh whose pass is still running kicks nothing new.
|
||||||
private mlFetching = false;
|
private mlFetching = false;
|
||||||
@@ -505,21 +491,6 @@ export class Library {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fresh reads (issue #75, owner amendment to design #36). Force a refresh,
|
|
||||||
// wait for it to complete and persist, then hand back the same
|
|
||||||
// `albums`/`photos`/`timeline` namespaces — now guaranteed to reflect a
|
|
||||||
// completed server round-trip. Concurrent calls coalesce onto one refresh;
|
|
||||||
// a refresh that fails rejects here, where the default namespaces would
|
|
||||||
// instead stay silent and serve the last good copy.
|
|
||||||
async fresh(): Promise<FreshReads> {
|
|
||||||
await this.refreshNow();
|
|
||||||
return {
|
|
||||||
albums: this.albums,
|
|
||||||
photos: this.photos,
|
|
||||||
timeline: this.timeline,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Back up every in-scope file to `downloadDirectory` in the historical
|
// Back up every in-scope file to `downloadDirectory` in the historical
|
||||||
// on-disk layout, with a durable failure ledger (issue #51). Refreshes
|
// on-disk layout, with a durable failure ledger (issue #51). Refreshes
|
||||||
// first, fetches pending originals (and optional thumbnails) through the
|
// first, fetches pending originals (and optional thumbnails) through the
|
||||||
@@ -579,48 +550,11 @@ export class Library {
|
|||||||
this.timer.unref?.();
|
this.timer.unref?.();
|
||||||
}
|
}
|
||||||
|
|
||||||
// The background loop's refresh: run a cycle unless one is already in flight
|
// One refresh cycle, guarded so a failure never escapes and overlapping
|
||||||
// (or the library is closed), and never let a failure escape — the
|
// cycles never run. Errors are reported, not thrown.
|
||||||
// background path reports errors through `status()`/`onProgress`, it does
|
private async runRefresh(): Promise<void> {
|
||||||
// not throw. Resolves once the cycle it started (or skipped past) settles.
|
if (this.closed || this.refreshing) return;
|
||||||
private runRefresh(): Promise<void> {
|
this.refreshing = true;
|
||||||
if (this.closed || this.cycle) return Promise.resolve();
|
|
||||||
return this.startCycle().catch(() => {});
|
|
||||||
}
|
|
||||||
|
|
||||||
// A fresh read's refresh (issue #75): force a cycle and await it, rejecting
|
|
||||||
// if it fails. Concurrent fresh reads coalesce onto the one in-flight cycle
|
|
||||||
// — the background loop's included — so they never fan out into redundant
|
|
||||||
// server round-trips.
|
|
||||||
private refreshNow(): Promise<void> {
|
|
||||||
if (this.closed) {
|
|
||||||
return Promise.reject(new Error("the library is closed"));
|
|
||||||
}
|
|
||||||
return this.cycle ?? this.startCycle();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start one refresh cycle and record it as the in-flight cycle so every
|
|
||||||
// caller coalesces onto it. The returned promise carries the cycle's real
|
|
||||||
// outcome; each caller attaches the handling its own path needs, and the
|
|
||||||
// slot is cleared once the cycle settles.
|
|
||||||
private startCycle(): Promise<void> {
|
|
||||||
const cycle = this.refreshCycle();
|
|
||||||
this.cycle = cycle;
|
|
||||||
void cycle.then(
|
|
||||||
() => {
|
|
||||||
if (this.cycle === cycle) this.cycle = undefined;
|
|
||||||
},
|
|
||||||
() => {
|
|
||||||
if (this.cycle === cycle) this.cycle = undefined;
|
|
||||||
},
|
|
||||||
);
|
|
||||||
return cycle;
|
|
||||||
}
|
|
||||||
|
|
||||||
// One refresh cycle: the network fetch and commit, wrapped in the progress
|
|
||||||
// events and status bookkeeping. Throws when the refresh fails so a fresh
|
|
||||||
// read can reject; `runRefresh` swallows that throw for the background loop.
|
|
||||||
private async refreshCycle(): Promise<void> {
|
|
||||||
this.emit({ operation: "refresh", status: "started" });
|
this.emit({ operation: "refresh", status: "started" });
|
||||||
try {
|
try {
|
||||||
await this.refreshOnce();
|
await this.refreshOnce();
|
||||||
@@ -636,7 +570,8 @@ export class Library {
|
|||||||
const error = err instanceof Error ? err.message : String(err);
|
const error = err instanceof Error ? err.message : String(err);
|
||||||
this.lastError = error;
|
this.lastError = error;
|
||||||
this.emit({ operation: "refresh", status: "failed", error });
|
this.emit({ operation: "refresh", status: "failed", error });
|
||||||
throw err;
|
} finally {
|
||||||
|
this.refreshing = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -193,15 +193,6 @@ export interface TimelineAPI {
|
|||||||
groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[];
|
groups(args: { groupBy: GroupBy; filter?: PhotoFilter }): TimelineGroup[];
|
||||||
}
|
}
|
||||||
|
|
||||||
// The surface `Library.fresh()` resolves to (issue #75). It is the same three
|
|
||||||
// read namespaces as the default `albums`/`photos`/`timeline`, handed back only
|
|
||||||
// after a forced refresh has brought the local copy current.
|
|
||||||
export interface FreshReads {
|
|
||||||
albums: AlbumsAPI;
|
|
||||||
photos: PhotosAPI;
|
|
||||||
timeline: TimelineAPI;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const makeAlbumsAPI = (
|
export const makeAlbumsAPI = (
|
||||||
derive: () => DerivedRecords,
|
derive: () => DerivedRecords,
|
||||||
content?: PhotoContent,
|
content?: PhotoContent,
|
||||||
|
|||||||
@@ -1,195 +0,0 @@
|
|||||||
/**
|
|
||||||
* Tests for the CLI read helpers (`src/cli-read.ts`, owner amendment to
|
|
||||||
* issue #36, issue #52).
|
|
||||||
*
|
|
||||||
* The `collections`, `files`, `get`, and `get-thumb` commands must answer for
|
|
||||||
* current server state, not the local cache, so each helper forces a
|
|
||||||
* `Library.fresh()` round-trip before it reads. The stand-in library below
|
|
||||||
* serves nothing until `fresh()` has been awaited, so a helper that read
|
|
||||||
* without refreshing would come back empty and fail here.
|
|
||||||
*
|
|
||||||
* `collections` and `files` also list in the library's enumeration order
|
|
||||||
* (`listCollections`/`listFiles`) — the order the pre-library CLI printed — not
|
|
||||||
* the albums/photos projection's newest-first order. The fixtures are seeded in
|
|
||||||
* an enumeration order that a newest-first sort would rearrange, so a
|
|
||||||
* regression to the projection order would fail here too. Field values still
|
|
||||||
* come from the raw metadata via `cli-output.ts`.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { describe, it, expect } from "vitest";
|
|
||||||
|
|
||||||
import {
|
|
||||||
freshCollections,
|
|
||||||
freshFiles,
|
|
||||||
freshFile,
|
|
||||||
type FreshReadLibrary,
|
|
||||||
} from "../../src/cli-read.js";
|
|
||||||
import { fileListRow } from "../../src/cli-output.js";
|
|
||||||
import type { Photo } from "../../src/library/index.js";
|
|
||||||
import type { Collection, EnteFile } from "../../src/model/types.js";
|
|
||||||
|
|
||||||
const collection = (id: number, updationTime: number): Collection => ({
|
|
||||||
id,
|
|
||||||
ownerID: 42,
|
|
||||||
key: new Uint8Array(),
|
|
||||||
name: `album-${id}`,
|
|
||||||
type: "album",
|
|
||||||
updationTime,
|
|
||||||
isShared: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Microseconds, as Ente stores times.
|
|
||||||
const file = (
|
|
||||||
id: number,
|
|
||||||
collectionID: number,
|
|
||||||
creationTime: number,
|
|
||||||
): EnteFile => ({
|
|
||||||
id,
|
|
||||||
collectionID,
|
|
||||||
ownerID: 42,
|
|
||||||
key: new Uint8Array(),
|
|
||||||
metadata: {
|
|
||||||
title: `file-${id}.jpg`,
|
|
||||||
fileType: "image",
|
|
||||||
creationTime,
|
|
||||||
modificationTime: creationTime,
|
|
||||||
},
|
|
||||||
file: { decryptionHeader: "" },
|
|
||||||
thumbnail: { decryptionHeader: "" },
|
|
||||||
updationTime: creationTime,
|
|
||||||
});
|
|
||||||
|
|
||||||
// A library that reveals its records only after `fresh()` has been awaited, and
|
|
||||||
// serves them in the enumeration order it was given. `photos.byID` returns a
|
|
||||||
// stand-in `Photo` carrying just the fileID the helper passes through.
|
|
||||||
class FakeLibrary implements FreshReadLibrary {
|
|
||||||
freshCalls = 0;
|
|
||||||
private refreshed = false;
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
private readonly collections: Collection[],
|
|
||||||
private readonly files: EnteFile[],
|
|
||||||
) {}
|
|
||||||
|
|
||||||
async fresh(): Promise<unknown> {
|
|
||||||
this.freshCalls++;
|
|
||||||
this.refreshed = true;
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
|
|
||||||
listCollections(): Collection[] {
|
|
||||||
return this.refreshed ? this.collections : [];
|
|
||||||
}
|
|
||||||
|
|
||||||
getCollection(id: number): Collection | undefined {
|
|
||||||
return this.listCollections().find((c) => c.id === id);
|
|
||||||
}
|
|
||||||
|
|
||||||
listFiles(collectionID: number): EnteFile[] {
|
|
||||||
return this.refreshed
|
|
||||||
? this.files.filter((f) => f.collectionID === collectionID)
|
|
||||||
: [];
|
|
||||||
}
|
|
||||||
|
|
||||||
getFileByID(fileID: number): EnteFile | undefined {
|
|
||||||
if (!this.refreshed) return undefined;
|
|
||||||
return this.files.find((f) => f.id === fileID);
|
|
||||||
}
|
|
||||||
|
|
||||||
photos = {
|
|
||||||
byID: ({ fileID }: { fileID: number }): Photo | undefined => {
|
|
||||||
if (!this.refreshed) return undefined;
|
|
||||||
if (!this.files.some((f) => f.id === fileID)) return undefined;
|
|
||||||
return { fileID } as unknown as Photo;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("CLI read helpers (issue #36 amendment, issue #52)", () => {
|
|
||||||
it("freshCollections refreshes first, then lists in enumeration order", async () => {
|
|
||||||
// Enumeration order 2, 1, 3; a newest-first sort would be 3, 2, 1.
|
|
||||||
const lib = new FakeLibrary(
|
|
||||||
[collection(2, 200), collection(1, 300), collection(3, 100)],
|
|
||||||
[],
|
|
||||||
);
|
|
||||||
const rows = await freshCollections(lib);
|
|
||||||
|
|
||||||
expect(lib.freshCalls).toBe(1);
|
|
||||||
expect(rows.map((c) => c.id)).toEqual([2, 1, 3]);
|
|
||||||
// The projection's newest-first order is a different sequence, so this
|
|
||||||
// is not accidentally that order.
|
|
||||||
const newestFirst = [...rows]
|
|
||||||
.sort((a, b) => b.updationTime - a.updationTime)
|
|
||||||
.map((c) => c.id);
|
|
||||||
expect(newestFirst).toEqual([1, 2, 3]);
|
|
||||||
expect(rows.map((c) => c.id)).not.toEqual(newestFirst);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("freshFiles refreshes first, lists in enumeration order, keeps raw fields", async () => {
|
|
||||||
// Enumeration order by id 10, 11, 12; creationTimes ascending, so a
|
|
||||||
// newest-first sort would reverse them.
|
|
||||||
const files = [
|
|
||||||
file(10, 1, 1_700_000_000_000_000),
|
|
||||||
file(11, 1, 1_700_000_000_000_001),
|
|
||||||
file(12, 1, 1_700_000_000_000_002),
|
|
||||||
];
|
|
||||||
const lib = new FakeLibrary([collection(1, 100)], files);
|
|
||||||
|
|
||||||
const rows = await freshFiles(lib, 1);
|
|
||||||
expect(lib.freshCalls).toBe(1);
|
|
||||||
expect(rows?.map((f) => f.id)).toEqual([10, 11, 12]);
|
|
||||||
|
|
||||||
// Field values come from raw metadata: microsecond creationTime and the
|
|
||||||
// raw title, unchanged.
|
|
||||||
expect(rows?.map(fileListRow)).toEqual([
|
|
||||||
{
|
|
||||||
id: 10,
|
|
||||||
title: "file-10.jpg",
|
|
||||||
fileType: "image",
|
|
||||||
creationTime: 1_700_000_000_000_000,
|
|
||||||
collectionID: 1,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 11,
|
|
||||||
title: "file-11.jpg",
|
|
||||||
fileType: "image",
|
|
||||||
creationTime: 1_700_000_000_000_001,
|
|
||||||
collectionID: 1,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 12,
|
|
||||||
title: "file-12.jpg",
|
|
||||||
fileType: "image",
|
|
||||||
creationTime: 1_700_000_000_000_002,
|
|
||||||
collectionID: 1,
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("freshFiles returns undefined for an unknown collection", async () => {
|
|
||||||
const lib = new FakeLibrary([collection(1, 100)], []);
|
|
||||||
const rows = await freshFiles(lib, 999);
|
|
||||||
expect(lib.freshCalls).toBe(1);
|
|
||||||
expect(rows).toBeUndefined();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("freshFile refreshes first, then resolves the photo and its raw record", async () => {
|
|
||||||
const f = file(10, 1, 1_700_000_000_000_000);
|
|
||||||
const lib = new FakeLibrary([collection(1, 100)], [f]);
|
|
||||||
|
|
||||||
const resolved = await freshFile(lib, 10);
|
|
||||||
expect(lib.freshCalls).toBe(1);
|
|
||||||
expect(resolved?.photo.fileID).toBe(10);
|
|
||||||
expect(resolved?.file.metadata.title).toBe("file-10.jpg");
|
|
||||||
expect(resolved?.file.metadata.creationTime).toBe(
|
|
||||||
1_700_000_000_000_000,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("freshFile returns undefined for an unknown file", async () => {
|
|
||||||
const lib = new FakeLibrary([collection(1, 100)], []);
|
|
||||||
const resolved = await freshFile(lib, 404);
|
|
||||||
expect(lib.freshCalls).toBe(1);
|
|
||||||
expect(resolved).toBeUndefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,272 +0,0 @@
|
|||||||
/**
|
|
||||||
* Tests for fresh reads (issue #75, an owner amendment to design #36).
|
|
||||||
*
|
|
||||||
* The default read namespaces answer from RAM and never touch the network; a
|
|
||||||
* background loop keeps the local copy current. `fresh()` adds an awaited path:
|
|
||||||
* it forces a refresh, waits for it to complete and persist, and only then
|
|
||||||
* hands back the `albums`/`photos`/`timeline` namespaces, guaranteeing the
|
|
||||||
* local copy reflects a completed server round-trip. The contracts here:
|
|
||||||
*
|
|
||||||
* 1. A fresh read observes a server change that a same-instant default read
|
|
||||||
* would miss (the default path has not refreshed yet).
|
|
||||||
* 2. Concurrent fresh reads coalesce onto one in-flight refresh — three
|
|
||||||
* concurrent `fresh()` calls make exactly one collections round-trip, not
|
|
||||||
* three.
|
|
||||||
* 3. A refresh that fails rejects the fresh read (currency was unavailable),
|
|
||||||
* while the default reads stay silent and keep serving the last good copy.
|
|
||||||
*
|
|
||||||
* The client is the same metadata-only mock the background-refresh tests use:
|
|
||||||
* no crypto, no network, scripted pages, and a record of each call. A long
|
|
||||||
* refresh interval keeps the background timer out of the way so each test's
|
|
||||||
* refreshes are exactly the ones it triggers.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
||||||
import { mkdtempSync, rmSync } from "node:fs";
|
|
||||||
import { tmpdir } from "node:os";
|
|
||||||
import { join } from "node:path";
|
|
||||||
|
|
||||||
import { Library } from "../../src/library/index.js";
|
|
||||||
import type { CollectionsPage, FilesPage } from "../../src/client.js";
|
|
||||||
import type { Collection, EnteFile } from "../../src/model/types.js";
|
|
||||||
|
|
||||||
const USER_ID = 42;
|
|
||||||
|
|
||||||
// Long enough that no background tick fires during a test; each test's
|
|
||||||
// refreshes are only the ones its own `fresh()` calls force.
|
|
||||||
const SLOW_INTERVAL = 3600;
|
|
||||||
|
|
||||||
const collection = (id: number, updationTime: number): Collection => ({
|
|
||||||
id,
|
|
||||||
ownerID: USER_ID,
|
|
||||||
key: new Uint8Array([id & 0xff]),
|
|
||||||
name: `album-${id}`,
|
|
||||||
type: "album",
|
|
||||||
updationTime,
|
|
||||||
isShared: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
const file = (
|
|
||||||
id: number,
|
|
||||||
collectionID: number,
|
|
||||||
updationTime: number,
|
|
||||||
): EnteFile => ({
|
|
||||||
id,
|
|
||||||
collectionID,
|
|
||||||
ownerID: USER_ID,
|
|
||||||
key: new Uint8Array([id & 0xff]),
|
|
||||||
metadata: {
|
|
||||||
title: `file-${id}.jpg`,
|
|
||||||
fileType: "image",
|
|
||||||
creationTime: updationTime,
|
|
||||||
modificationTime: updationTime,
|
|
||||||
},
|
|
||||||
file: { decryptionHeader: "aGVhZGVy" },
|
|
||||||
thumbnail: { decryptionHeader: "dGh1bWI=" },
|
|
||||||
updationTime,
|
|
||||||
});
|
|
||||||
|
|
||||||
class MockClient {
|
|
||||||
userID = USER_ID;
|
|
||||||
failCollections = false;
|
|
||||||
collectionsQueue: CollectionsPage[] = [];
|
|
||||||
filesByCollection = new Map<number, FilesPage[]>();
|
|
||||||
|
|
||||||
collectionsSinceTimes: number[] = [];
|
|
||||||
filesCalls: { collectionID: number; sinceTime: number }[] = [];
|
|
||||||
|
|
||||||
whoami(): { email: string; userID: number } {
|
|
||||||
return { email: "user@example.com", userID: this.userID };
|
|
||||||
}
|
|
||||||
|
|
||||||
async collectionsSince(args: {
|
|
||||||
sinceTime: number;
|
|
||||||
}): Promise<CollectionsPage> {
|
|
||||||
this.collectionsSinceTimes.push(args.sinceTime);
|
|
||||||
if (this.failCollections) throw new Error("network down");
|
|
||||||
return (
|
|
||||||
this.collectionsQueue.shift() ?? {
|
|
||||||
collections: [],
|
|
||||||
deleted: [],
|
|
||||||
cursor: args.sinceTime,
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async filesSince(args: {
|
|
||||||
collectionID: number;
|
|
||||||
collectionKey: Uint8Array;
|
|
||||||
sinceTime: number;
|
|
||||||
}): Promise<FilesPage> {
|
|
||||||
this.filesCalls.push({
|
|
||||||
collectionID: args.collectionID,
|
|
||||||
sinceTime: args.sinceTime,
|
|
||||||
});
|
|
||||||
const queue = this.filesByCollection.get(args.collectionID);
|
|
||||||
return (
|
|
||||||
queue?.shift() ?? {
|
|
||||||
files: [],
|
|
||||||
deleted: [],
|
|
||||||
cursor: args.sinceTime,
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
filesFor(collectionID: number, ...pages: FilesPage[]): void {
|
|
||||||
this.filesByCollection.set(collectionID, pages);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A client seeded with one collection and one file, opened with an empty cache
|
|
||||||
// so the initial refresh is awaited and post-`open()` state is deterministic.
|
|
||||||
const openSeeded = async (
|
|
||||||
cacheDirectory: string,
|
|
||||||
): Promise<{ client: MockClient; lib: Library }> => {
|
|
||||||
const client = new MockClient();
|
|
||||||
client.collectionsQueue.push({
|
|
||||||
collections: [collection(1, 100)],
|
|
||||||
deleted: [],
|
|
||||||
cursor: 100,
|
|
||||||
});
|
|
||||||
client.filesFor(1, {
|
|
||||||
files: [file(1001, 1, 90)],
|
|
||||||
deleted: [],
|
|
||||||
cursor: 90,
|
|
||||||
});
|
|
||||||
const lib = await Library.open({
|
|
||||||
client,
|
|
||||||
cacheDirectory,
|
|
||||||
refreshIntervalSeconds: SLOW_INTERVAL,
|
|
||||||
});
|
|
||||||
return { client, lib };
|
|
||||||
};
|
|
||||||
|
|
||||||
describe("Library.fresh", () => {
|
|
||||||
let dir: string;
|
|
||||||
let cacheDirectory: string;
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
dir = mkdtempSync(join(tmpdir(), "quak-fresh-"));
|
|
||||||
cacheDirectory = join(dir, "cache");
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
rmSync(dir, { recursive: true, force: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("observes a server change a same-instant default read would miss", async () => {
|
|
||||||
const { client, lib } = await openSeeded(cacheDirectory);
|
|
||||||
try {
|
|
||||||
// A new file appears on the server after open, advancing its
|
|
||||||
// collection so the next refresh re-enumerates it.
|
|
||||||
client.collectionsQueue.push({
|
|
||||||
collections: [collection(1, 200)],
|
|
||||||
deleted: [],
|
|
||||||
cursor: 200,
|
|
||||||
});
|
|
||||||
client.filesFor(1, {
|
|
||||||
files: [file(1002, 1, 190)],
|
|
||||||
deleted: [],
|
|
||||||
cursor: 190,
|
|
||||||
});
|
|
||||||
|
|
||||||
// A default read at this instant has not refreshed: it misses 1002.
|
|
||||||
expect(lib.photos.byID({ fileID: 1002 })).toBeUndefined();
|
|
||||||
|
|
||||||
// A fresh read forces the round-trip and sees it.
|
|
||||||
const reads = await lib.fresh();
|
|
||||||
expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
|
||||||
// And the change is now live for the default namespaces too.
|
|
||||||
expect(lib.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
|
||||||
} finally {
|
|
||||||
lib.close();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it("coalesces concurrent fresh reads onto one in-flight refresh", async () => {
|
|
||||||
const { client, lib } = await openSeeded(cacheDirectory);
|
|
||||||
try {
|
|
||||||
client.collectionsQueue.push({
|
|
||||||
collections: [collection(1, 200)],
|
|
||||||
deleted: [],
|
|
||||||
cursor: 200,
|
|
||||||
});
|
|
||||||
client.filesFor(1, {
|
|
||||||
files: [file(1002, 1, 190)],
|
|
||||||
deleted: [],
|
|
||||||
cursor: 190,
|
|
||||||
});
|
|
||||||
|
|
||||||
const collectionsBefore = client.collectionsSinceTimes.length;
|
|
||||||
const filesBefore = client.filesCalls.length;
|
|
||||||
|
|
||||||
// Gate the next collections fetch so all three fresh reads are in
|
|
||||||
// flight together before any of them completes.
|
|
||||||
let release: () => void = () => {};
|
|
||||||
const gate = new Promise<void>((r) => {
|
|
||||||
release = r;
|
|
||||||
});
|
|
||||||
const inner = client.collectionsSince.bind(client);
|
|
||||||
client.collectionsSince = async (args: { sinceTime: number }) => {
|
|
||||||
await gate;
|
|
||||||
return inner(args);
|
|
||||||
};
|
|
||||||
|
|
||||||
const all = Promise.all([lib.fresh(), lib.fresh(), lib.fresh()]);
|
|
||||||
release();
|
|
||||||
const [a, b, c] = await all;
|
|
||||||
|
|
||||||
// Exactly one collections round-trip and one file round-trip served
|
|
||||||
// all three fresh reads.
|
|
||||||
expect(client.collectionsSinceTimes.length).toBe(
|
|
||||||
collectionsBefore + 1,
|
|
||||||
);
|
|
||||||
expect(client.filesCalls.length).toBe(filesBefore + 1);
|
|
||||||
|
|
||||||
// All three observed the change.
|
|
||||||
for (const reads of [a, b, c]) {
|
|
||||||
expect(reads.photos.byID({ fileID: 1002 })?.fileID).toBe(1002);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
lib.close();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it("rejects the fresh read when the refresh fails, leaving defaults intact", async () => {
|
|
||||||
const { client, lib } = await openSeeded(cacheDirectory);
|
|
||||||
try {
|
|
||||||
client.failCollections = true;
|
|
||||||
|
|
||||||
// Two concurrent fresh reads both reject, and share one failed
|
|
||||||
// round-trip rather than each making its own.
|
|
||||||
const collectionsBefore = client.collectionsSinceTimes.length;
|
|
||||||
const first = lib.fresh();
|
|
||||||
const second = lib.fresh();
|
|
||||||
await expect(first).rejects.toThrow(/network down/);
|
|
||||||
await expect(second).rejects.toThrow(/network down/);
|
|
||||||
expect(client.collectionsSinceTimes.length).toBe(
|
|
||||||
collectionsBefore + 1,
|
|
||||||
);
|
|
||||||
|
|
||||||
// The default reads never rejected: they still serve the last good
|
|
||||||
// copy, and the failure surfaced through status().
|
|
||||||
expect(lib.photos.byID({ fileID: 1001 })?.fileID).toBe(1001);
|
|
||||||
expect(lib.status().lastError).toMatch(/network down/);
|
|
||||||
|
|
||||||
// Recovery: once the server answers, a fresh read resolves current.
|
|
||||||
client.failCollections = false;
|
|
||||||
client.collectionsQueue.push({
|
|
||||||
collections: [collection(2, 300)],
|
|
||||||
deleted: [],
|
|
||||||
cursor: 300,
|
|
||||||
});
|
|
||||||
const reads = await lib.fresh();
|
|
||||||
expect(reads.albums.byID({ collectionID: 2 })?.collectionID).toBe(
|
|
||||||
2,
|
|
||||||
);
|
|
||||||
expect(lib.status().lastError).toBeUndefined();
|
|
||||||
} finally {
|
|
||||||
lib.close();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
Reference in New Issue
Block a user