sneak/quak
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions 27 Packages Projects Releases Wiki Activity

quak get and get-thumb on an empty cache stop fetching collections once they find the file (closes #222)
check / check (push) Waiting to run
Details

Browse Source 
`quak get` and `quak get-thumb` used to wait for a refresh, which on an
empty cache fetched the files of every collection. Now, on an empty
cache, `freshFile` reads the collection list, then one collection's
files at a time, and stops at the first that holds the file. That search
stores nothing. Once the cache holds the account's collections,
`freshFile` refreshes as before and looks the file up in the cache. The
commands fetch the content through the new `Library.originalOf()` and
`Library.thumbnailOf()`, which take the file record itself, since the
store may not hold it.

On an empty cache they no longer print a refresh's progress lines. The
`-q` test that relied on `get` refreshing now runs
`helper fix-missing-thumbnails`.

Model: opus-5-5
This commit was merged in pull request #226.
This commit is contained in:
clawbot
2026-10-26 10:18:28 +01:00
parent af1491966c
commit f65c237e2c
11 changed files with 278 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/cli/commands.test.ts
+82 -39
View File
@@ -194,9 +194,6 @@ const FETCHED_BOTH_ALBUMS =
"Fetched the files of 1 of 2 collections\n" +
"Fetched the files of 2 of 2 collections\n";
// The same for an account with one album.
const FETCHED_ONE_ALBUM = "Fetched the files of 1 of 1 collections\n";
// Collects everything written to it.
class Output {
text = "";
@@ -576,27 +573,27 @@ describe("get and get-thumb", () => {
const out = join(root, "diagram.png");
expect(await getCommand(context(), "200", { out })).toBe(0);
expect(readFileSync(out)).toEqual(Buffer.alloc(7, 200));
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS + `7 bytes -> ${out}\n`);
expect(stderr.text).toBe(`7 bytes -> ${out}\n`);
});
it("get-thumb finds a file in any album without --collection", async () => {
const out = join(root, "thumb.jpg");
expect(await getThumbCommand(context(), "200", { out })).toBe(0);
expect(readFileSync(out)).toEqual(Buffer.alloc(3, 200));
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS + `3 bytes -> ${out}\n`);
expect(stderr.text).toBe(`3 bytes -> ${out}\n`);
});
it("get exits 1 when no album has the file", async () => {
const out = join(root, "x");
expect(await getCommand(context(), "999", { out })).toBe(1);
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS + "File 999 not found\n");
expect(stderr.text).toBe("File 999 not found\n");
expect(existsSync(out)).toBe(false);
});
it("get-thumb exits 1 when no album has the file", async () => {
const out = join(root, "x");
expect(await getThumbCommand(context(), "999", { out })).toBe(1);
expect(stderr.text).toBe(FETCHED_BOTH_ALBUMS + "File 999 not found\n");
expect(stderr.text).toBe("File 999 not found\n");
expect(existsSync(out)).toBe(false);
});
});
@@ -644,7 +641,7 @@ describe("names from an album name or a title over 255 bytes", () => {
expect(Buffer.byteLength(name)).toBe(255);
expect(code).toBe(0);
expect(readdirSync(dir)).toEqual([name]);
expect(stderr.text).toBe(FETCHED_ONE_ALBUM + `7 bytes -> ${name}\n`);
expect(stderr.text).toBe(`7 bytes -> ${name}\n`);
});
it("get-thumb cuts the title short so that thumb_ and it fit in 255 bytes, and exits 0", async () => {
@@ -656,7 +653,7 @@ describe("names from an album name or a title over 255 bytes", () => {
expect(Buffer.byteLength(name)).toBe(255);
expect(code).toBe(0);
expect(readdirSync(dir)).toEqual([name]);
expect(stderr.text).toBe(FETCHED_ONE_ALBUM + `3 bytes -> ${name}\n`);
expect(stderr.text).toBe(`3 bytes -> ${name}\n`);
});
it("backup-metadata cuts the album name short so that its folder's name fits in 255 bytes, and exits 0", async () => {
@@ -745,8 +742,7 @@ describe("a live photo", () => {
Buffer.from(VIDEO),
);
expect(stderr.text).toBe(
FETCHED_ONE_ALBUM +
`${IMAGE.length} bytes -> IMG_0300.heic\n` +
`${IMAGE.length} bytes -> IMG_0300.heic\n` +
`${VIDEO.length} bytes -> IMG_0300.mov\n`,
);
});
@@ -800,8 +796,7 @@ describe("a live photo", () => {
expect(readFileSync(out)).toEqual(Buffer.from(IMAGE));
expect(readFileSync(video)).toEqual(Buffer.from(VIDEO));
expect(stderr.text).toBe(
FETCHED_ONE_ALBUM +
`${IMAGE.length} bytes -> ${out}\n${VIDEO.length} bytes -> ${video}\n`,
`${IMAGE.length} bytes -> ${out}\n${VIDEO.length} bytes -> ${video}\n`,
);
});
@@ -815,8 +810,7 @@ describe("a live photo", () => {
expect(existsSync(out)).toBe(false);
expect(existsSync(join(root, "photo.mov"))).toBe(false);
expect(stderr.text).toBe(
FETCHED_ONE_ALBUM +
`File 300 is a live photo, and its video would also be written to ${out}\n`,
`File 300 is a live photo, and its video would also be written to ${out}\n`,
);
});
@@ -1575,13 +1569,14 @@ describe("an ID given on the command line", () => {
);
});
// An account with many albums (issues #210 and #217). On an empty cache
// `collections` and `files` with a collection ID used to wait for a refresh
// that fetched and decrypted the files of every album, one album after another.
// On such an account they seemed to hang. The account here has 40 albums of one
// photo each, encrypted as Ente's clients encrypt them, on the fake Ente server
// of `test/ente-server.ts`. The tests record the path and query of each
// request quak sends. The session is the one logging in to that server saves.
// An account with many albums (issues #210, #217 and #222). On an empty cache
// `collections`, `files` with a collection ID and `get` used to wait for a
// refresh that fetched and decrypted the files of every album, one album after
// another. On such an account they seemed to hang. The account here has 40
// albums of one photo each, encrypted as Ente's clients encrypt them, on the
// fake Ente server of `test/ente-server.ts`. The tests record the path and query
// of each request quak sends. The session is the one logging in to that server
// saves.
describe("an account with many albums", () => {
const ALBUMS = 40;
const PASSWORD = "many-albums";
@@ -1602,14 +1597,15 @@ describe("an account with many albums", () => {
// The key of each album, by album ID.
const albumKeys = new Map<number, Uint8Array>();
// A photo in album `collectionID` as the server sends it. These tests
// fetch no original and no thumbnail.
// A photo in album `collectionID` as the server sends it, and its original
// encrypted as the server stores it. The original's bytes are the title.
// These tests fetch no thumbnail.
const photoRow = (photo: {
id: number;
collectionID: number;
title: string;
updationTime: number;
}): Record<string, unknown> => {
}): { row: Record<string, unknown>; original: Uint8Array } => {
const photoKey = sodium.crypto_secretstream_xchacha20poly1305_keygen();
const photoKeyBox = encryptSecretbox(
photoKey,
@@ -1626,7 +1622,11 @@ describe("an account with many albums", () => {
),
photoKey,
);
return {
const original = encryptStreamBlob(
new TextEncoder().encode(photo.title),
photoKey,
);
const row = {
id: photo.id,
collectionID: photo.collectionID,
ownerID: USER_ID,
@@ -1636,10 +1636,11 @@ describe("an account with many albums", () => {
encryptedData: toBase64(metadata.ciphertext),
decryptionHeader: toBase64(metadata.header),
},
file: { decryptionHeader: "" },
file: { decryptionHeader: toBase64(original.header) },
thumbnail: { decryptionHeader: "" },
updationTime: photo.updationTime,
};
return { row, original: original.ciphertext };
};
// Album n is named `Album n` and holds one photo, n * 100, titled
@@ -1649,6 +1650,7 @@ describe("an account with many albums", () => {
const account = await buildAccount(PASSWORD);
const collections: Record<string, unknown>[] = [];
const filesByCollection: Record<number, Record<string, unknown>[]> = {};
const fileCiphertexts: Record<number, Uint8Array> = {};
for (let id = 1; id <= ALBUMS; id++) {
const albumKey = sodium.crypto_secretbox_keygen();
albumKeys.set(id, albumKey);
@@ -1667,14 +1669,14 @@ describe("an account with many albums", () => {
type: "album",
updationTime: 1,
});
filesByCollection[id] = [
photoRow({
id: id * 100,
collectionID: id,
title: `photo-${id}.jpg`,
updationTime: 1,
}),
];
const { row, original } = photoRow({
id: id * 100,
collectionID: id,
title: `photo-${id}.jpg`,
updationTime: 1,
});
filesByCollection[id] = [row];
fileCiphertexts[id * 100] = original;
}
const ente = buildMetaFetch({
verifier: account.verifier,
@@ -1684,7 +1686,7 @@ describe("an account with many albums", () => {
collections,
filesByCollection,
encryptedMLData: {},
fileCiphertexts: {},
fileCiphertexts,
});
server = (async (input: RequestInfo | URL, init?: RequestInit) => {
const url =
@@ -1758,13 +1760,13 @@ describe("an account with many albums", () => {
collectionID: 7,
title: "photo-7.jpg",
updationTime: 1,
}),
}).row,
photoRow({
id: 701,
collectionID: 7,
title: "photo-701.jpg",
updationTime: 1,
}),
}).row,
];
const secondPage = [
photoRow({
@@ -1772,7 +1774,7 @@ describe("an account with many albums", () => {
collectionID: 7,
title: "photo-7-edited.jpg",
updationTime: 2,
}),
}).row,
];
const paged = (async (input: RequestInfo | URL, init?: RequestInit) => {
const url = new URL(input instanceof Request ? input.url : input);
@@ -1812,6 +1814,47 @@ describe("an account with many albums", () => {
expect(stderr.text).toBe(progress);
});
// Photo 200 is in album 2, so `get` asks for no file of album 3 or later.
// The last request downloads the photo's original.
it("get on an empty cache asks for the albums' files only until one holds the photo", async () => {
const out = join(root, "photo.jpg");
expect(await getCommand(accountContext(), "200", { out })).toBe(0);
expect(readFileSync(out, "utf-8")).toBe("photo-2.jpg");
expect(stderr.text).toBe(`11 bytes -> ${out}\n`);
expect(requests).toEqual([
"/collections/v2?sinceTime=0",
"/collections/v2/diff?collectionID=1&sinceTime=0",
"/collections/v2/diff?collectionID=2&sinceTime=0",
"/?fileID=200",
]);
expect(existsSync(join(root, "cache", "metadata.json"))).toBe(false);
});
// `files` first fills the cache, so `get` waits for a refresh. No album has
// changed since, so the refresh asks for no album's files.
it("get on a cache that holds the account asks for no unchanged album's files, also for a photo in no album", async () => {
const ctx = accountContext();
expect(await filesCommand(ctx, undefined, {})).toBe(0);
requests.length = 0;
stderr.text = "";
const out = join(root, "photo.jpg");
expect(await getCommand(ctx, "4000", { out })).toBe(0);
expect(readFileSync(out, "utf-8")).toBe("photo-40.jpg");
expect(stderr.text).toBe(`12 bytes -> ${out}\n`);
expect(requests).toEqual([
"/collections/v2?sinceTime=1",
"/?fileID=4000",
]);
requests.length = 0;
stderr.text = "";
expect(await getCommand(ctx, "999", { out })).toBe(1);
expect(stderr.text).toBe("File 999 not found\n");
expect(requests).toEqual(["/collections/v2?sinceTime=1"]);
});
it("files counts on stderr each album whose files its refresh fetched", async () => {
expect(await filesCommand(accountContext(), undefined, {})).toBe(0);