test: containerized Firefox end-to-end harness (closes #184)
All checks were successful
check / check (push) Successful in 34s
All checks were successful
check / check (push) Successful in 34s
Drives the real popup in a real Firefox with dist/firefox/ installed as an unpacked MV2 temporary add-on via geckodriver. make test-e2e-firefox, outside make check like the Chrome suite. Zero npm dependencies: plain fetch and child_process against geckodriver's HTTP API. Base image, Firefox tarball and geckodriver are each pinned by digest and verified at build time. Error capture reads the privileged console service through Marionette's chrome context, not WebDriver BiDi. BiDi delivers nothing at all for extension pages, so a BiDi-based harness would observe zero events and report success -- the vacuous-check shape this repo has shipped twice. Both the driver and the README say so where someone would be tempted to simplify. Demonstrated to discriminate: a background page that throws at the top of the file, a missing import, and an async throw where every UI assertion still passes each fail the run. Three limits are measured and documented rather than papered over: capture is poll-based so an error is attributed to a step, not a moment; the console ring buffer holds 250 messages and evicts the oldest, measured against a clean-run peak of 4; and the drained window ends roughly 1.5s after the last step, with observed jitter rather than a hard boundary. Content-script capture is marked unverified because --network none leaves no page to inject into, and that same choice inverts coverage of network-dependent code.
This commit was merged in pull request #256.
This commit is contained in:
288
tests/e2e/firefox/run.js
Normal file
288
tests/e2e/firefox/run.js
Normal file
@@ -0,0 +1,288 @@
|
||||
// Firefox end-to-end suite: drives the real popup in a real Firefox with
|
||||
// the unpacked MV2 build installed as a temporary add-on, and fails the run
|
||||
// on any uncaught error coming from an extension source.
|
||||
//
|
||||
// Run via script/test-e2e-firefox, which builds dist/firefox/ and the pinned
|
||||
// container. The extension directory is the one argument.
|
||||
//
|
||||
// node tests/e2e/firefox/run.js [dist/firefox]
|
||||
//
|
||||
// Deliberately not part of script/check, and deliberately not named
|
||||
// *.test.js: REPO_POLICIES.md caps make test at 20 seconds and a browser
|
||||
// suite does not fit.
|
||||
//
|
||||
// This shares no driver layer with the Chrome suite in tests/e2e/, and the
|
||||
// UI steps below are written twice on purpose. Chrome runs on Playwright,
|
||||
// which cannot see extension-page errors in Firefox at all (see the BiDi
|
||||
// note in driver.js), so the two backends have no common substrate to
|
||||
// abstract over. Three duplicated steps do not pay for a shim; revisit if
|
||||
// this suite grows to where they do.
|
||||
//
|
||||
// LIMITATION, and the difference from the Chrome suite worth knowing: error
|
||||
// capture here is POLL-BASED, not event-streamed. The console service is
|
||||
// drained at each step boundary, so an error is attributed to the step it
|
||||
// was drained after, never to a moment within that step. What is drained
|
||||
// covers the whole run from add-on install to the last drain below, which
|
||||
// lands ~1.5s after the last step returns (500ms settle + 1000ms sleep +
|
||||
// two drain round trips). That cut-off jitters run to run: three runs of
|
||||
// throws at fixed offsets reported everything to +1.5s and one of them
|
||||
// also +1.6s, and past it the browser is torn down first. Inside the
|
||||
// window there is no race — the drain reads and clears in one chrome
|
||||
// round trip — but there is a capacity limit: nsIConsoleService keeps
|
||||
// only the newest 250 messages, so 400 throws in one step report as
|
||||
// exactly 250. A clean run peaks at 4 of 250, so that is headroom today
|
||||
// and not a guarantee for a step that logs heavily. The Chrome harness
|
||||
// receives pageerror events as they happen and can say more. Do not read
|
||||
// a green Firefox run as the same claim.
|
||||
|
||||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const { ConsoleErrors, EXTENSION_ORIGIN, start, sleep } = require("./driver");
|
||||
|
||||
const REPO_ROOT = path.resolve(__dirname, "..", "..", "..");
|
||||
const POPUP_URL = EXTENSION_ORIGIN + "/src/popup/index.html";
|
||||
const PASSWORD = "e2e-harness-password";
|
||||
|
||||
// Firefox installs the add-on and starts its background page asynchronously
|
||||
// after the install call returns. Nothing observable marks the end of that,
|
||||
// so the popup's own first render is the signal we wait on instead.
|
||||
const STEP_TIMEOUT_MS = 120000;
|
||||
|
||||
const steps = [];
|
||||
|
||||
function step(name, fn) {
|
||||
steps.push({ name, fn });
|
||||
}
|
||||
|
||||
function assert(cond, message) {
|
||||
if (!cond) throw new Error(message);
|
||||
}
|
||||
|
||||
function withTimeout(promise, name) {
|
||||
let timer;
|
||||
const timeout = new Promise((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
new Error(
|
||||
name + " timed out after " + STEP_TIMEOUT_MS + "ms",
|
||||
),
|
||||
),
|
||||
STEP_TIMEOUT_MS,
|
||||
);
|
||||
});
|
||||
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------- steps
|
||||
|
||||
step("popup loads and reaches the welcome view", async (env) => {
|
||||
const d = env.driver;
|
||||
await d.navigate(POPUP_URL);
|
||||
await d.waitVisible("#view-welcome", STEP_TIMEOUT_MS);
|
||||
const title = await d.title();
|
||||
assert(title === "AutistMask", "unexpected popup title: " + title);
|
||||
});
|
||||
|
||||
step("wallet creation through the UI reaches the main view", async (env) => {
|
||||
const d = env.driver;
|
||||
await d.click("#btn-welcome-add");
|
||||
await d.waitVisible("#view-add-wallet");
|
||||
await d.click("#btn-generate-phrase");
|
||||
await d.waitFor(
|
||||
"a generated recovery phrase of at least 12 words",
|
||||
`const el = document.getElementById("wallet-mnemonic");
|
||||
return !!el && el.value.trim().split(/\\s+/).length >= 12;`,
|
||||
);
|
||||
env.phrase = (await d.value("#wallet-mnemonic")).trim();
|
||||
|
||||
await d.fill("#add-wallet-password", PASSWORD);
|
||||
await d.fill("#add-wallet-password-confirm", PASSWORD);
|
||||
await d.click("#btn-add-wallet-confirm");
|
||||
// Argon2id under libsodium, for real, so this is the slow one.
|
||||
await d.waitVisible("#view-main", STEP_TIMEOUT_MS);
|
||||
|
||||
assert(
|
||||
env.phrase.split(/\s+/).length >= 12,
|
||||
"wallet creation did not yield a recovery phrase",
|
||||
);
|
||||
const addrs = await d.count("#wallet-list .btn-addr-info");
|
||||
assert(addrs > 0, "no addresses rendered in the wallet list");
|
||||
});
|
||||
|
||||
step("add token screen opens from address detail", async (env) => {
|
||||
const d = env.driver;
|
||||
if (!(await d.isVisible("#view-address"))) {
|
||||
await d.waitVisible("#view-main");
|
||||
await d.click("#wallet-list .btn-addr-info");
|
||||
}
|
||||
await d.waitVisible("#view-address");
|
||||
|
||||
await d.click("#btn-add-token");
|
||||
// Reported with the view it actually stayed on: a screen that does
|
||||
// not change is the symptom a missing import produces, and naming
|
||||
// the screen is what makes that diagnosable.
|
||||
try {
|
||||
await d.waitVisible("#view-add-token");
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
e.message + "; current view is " + (await d.currentView()),
|
||||
);
|
||||
}
|
||||
|
||||
const picks = await d.count("#common-token-list .common-token");
|
||||
assert(picks > 0, "no common-token quick-pick buttons rendered");
|
||||
});
|
||||
|
||||
// ------------------------------------------------------------- runner
|
||||
|
||||
function formatError(e) {
|
||||
return (
|
||||
e.msg + " (" + e.src + ":" + e.line + (e.cat ? ", " + e.cat : "") + ")"
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// A suite that runs nothing must never report success.
|
||||
if (steps.length === 0) {
|
||||
console.log("1..0");
|
||||
console.log("# FAILED: the Firefox e2e suite registered no steps");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const extDir = path.resolve(REPO_ROOT, process.argv[2] || "dist/firefox");
|
||||
if (!fs.existsSync(path.join(extDir, "manifest.json"))) {
|
||||
console.error(
|
||||
"e2e-firefox: no unpacked build at " +
|
||||
extDir +
|
||||
" — run make build first",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
let driver;
|
||||
try {
|
||||
driver = await start();
|
||||
await driver.newSession();
|
||||
await driver.installAddon(extDir);
|
||||
} catch (e) {
|
||||
// A browser we cannot start is a failure of the suite, not an
|
||||
// absent suite. Never skip and report success.
|
||||
console.error("e2e-firefox: cannot run the suite: " + e.message);
|
||||
if (driver) await driver.quit().catch(() => {});
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const errors = new ConsoleErrors(driver, EXTENSION_ORIGIN);
|
||||
const env = { driver, phrase: null };
|
||||
|
||||
console.log("# extension origin: " + EXTENSION_ORIGIN);
|
||||
console.log("1.." + steps.length);
|
||||
|
||||
let failed = 0;
|
||||
let n = 0;
|
||||
try {
|
||||
// Drain, never reset: anything the add-on logged while installing
|
||||
// and starting its background page has no earlier step to belong
|
||||
// to, so it is folded into step 1 below. Services.console.reset()
|
||||
// here would DELETE it instead, and a background page that throws
|
||||
// at the top of the file — a dead background page — would then
|
||||
// produce a fully green run.
|
||||
let installErrors = [];
|
||||
let installFailure = null;
|
||||
try {
|
||||
installErrors = await errors.take();
|
||||
} catch (e) {
|
||||
installFailure =
|
||||
"could not read the console after install: " + e.message;
|
||||
}
|
||||
|
||||
for (const s of steps) {
|
||||
n += 1;
|
||||
let failure = null;
|
||||
try {
|
||||
await withTimeout(s.fn(env), s.name);
|
||||
} catch (e) {
|
||||
failure = e.message;
|
||||
}
|
||||
|
||||
// Let anything the step provoked reach the console service
|
||||
// before draining it. Without this a failure logged on the
|
||||
// way out of the step lands in the next step's drain, which
|
||||
// still fails the run but blames the wrong step.
|
||||
await sleep(500);
|
||||
|
||||
let found = [];
|
||||
try {
|
||||
found = await errors.take();
|
||||
} catch (e) {
|
||||
failure = failure || "could not read the console: " + e.message;
|
||||
}
|
||||
|
||||
if (n === 1) {
|
||||
found = installErrors.concat(found);
|
||||
installErrors = [];
|
||||
failure = failure || installFailure;
|
||||
installFailure = null;
|
||||
}
|
||||
|
||||
// Any uncaught error from an extension source fails the step
|
||||
// that provoked it, whether or not its assertions passed.
|
||||
if (!failure && found.length > 0) {
|
||||
failure =
|
||||
n === 1
|
||||
? "uncaught extension errors during add-on install, " +
|
||||
"background startup or this step"
|
||||
: "uncaught extension errors during this step";
|
||||
}
|
||||
|
||||
if (failure) {
|
||||
failed += 1;
|
||||
console.log("not ok " + n + " - " + s.name);
|
||||
console.log(" " + failure);
|
||||
for (const e of found) console.log(" " + formatError(e));
|
||||
} else {
|
||||
console.log("ok " + n + " - " + s.name);
|
||||
}
|
||||
}
|
||||
|
||||
// The tail: errors logged after the last step returned cannot be
|
||||
// blamed on any one step, but they are still reported and they
|
||||
// still fail the run.
|
||||
await sleep(1000);
|
||||
const trailing = await errors.take();
|
||||
console.log(
|
||||
"# " +
|
||||
(steps.length - failed) +
|
||||
"/" +
|
||||
steps.length +
|
||||
" steps passed",
|
||||
);
|
||||
if (trailing.length > 0) {
|
||||
console.log(
|
||||
"# " +
|
||||
trailing.length +
|
||||
" extension error(s) recorded after the last step, not " +
|
||||
"attributable to any single step:",
|
||||
);
|
||||
for (const e of trailing) console.log("# " + formatError(e));
|
||||
}
|
||||
if (failed > 0 || trailing.length > 0) {
|
||||
console.log("# FAILED");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} finally {
|
||||
await driver.quit().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error("e2e-firefox: " + (e && e.stack ? e.stack : e));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Reference in New Issue
Block a user