test: containerized Firefox end-to-end harness (closes #184)
All checks were successful
check / check (push) Successful in 34s
All checks were successful
check / check (push) Successful in 34s
Drives the real popup in a real Firefox with dist/firefox/ installed as an unpacked MV2 temporary add-on via geckodriver. make test-e2e-firefox, outside make check like the Chrome suite. Zero npm dependencies: plain fetch and child_process against geckodriver's HTTP API. Base image, Firefox tarball and geckodriver are each pinned by digest and verified at build time. Error capture reads the privileged console service through Marionette's chrome context, not WebDriver BiDi. BiDi delivers nothing at all for extension pages, so a BiDi-based harness would observe zero events and report success -- the vacuous-check shape this repo has shipped twice. Both the driver and the README say so where someone would be tempted to simplify. Demonstrated to discriminate: a background page that throws at the top of the file, a missing import, and an async throw where every UI assertion still passes each fail the run. Three limits are measured and documented rather than papered over: capture is poll-based so an error is attributed to a step, not a moment; the console ring buffer holds 250 messages and evicts the oldest, measured against a clean-run peak of 4; and the drained window ends roughly 1.5s after the last step, with observed jitter rather than a hard boundary. Content-script capture is marked unverified because --network none leaves no page to inject into, and that same choice inverts coverage of network-dependent code.
This commit was merged in pull request #256.
This commit is contained in:
464
tests/e2e/firefox/driver.js
Normal file
464
tests/e2e/firefox/driver.js
Normal file
@@ -0,0 +1,464 @@
|
||||
// A minimal WebDriver client for geckodriver, plus the privileged console
|
||||
// reader the error assertions are built on. No npm dependencies: global
|
||||
// fetch and child_process against geckodriver's HTTP API is less code than
|
||||
// a driver library and keeps the harness at zero packages.
|
||||
//
|
||||
// Run through script/test-e2e-firefox, which builds dist/firefox/ and the
|
||||
// pinned container around this. FIREFOX_BIN and GECKODRIVER locate the two
|
||||
// binaries; the image sets both.
|
||||
|
||||
"use strict";
|
||||
|
||||
const { spawn } = require("child_process");
|
||||
const net = require("net");
|
||||
|
||||
const FIREFOX_BIN = process.env.FIREFOX_BIN || "firefox";
|
||||
const GECKODRIVER = process.env.GECKODRIVER || "geckodriver";
|
||||
|
||||
// The extension id declared in manifest/firefox.json, and the uuid the
|
||||
// popup is served from. Firefox normally assigns that uuid randomly per
|
||||
// profile, which would make the popup URL undiscoverable without querying
|
||||
// privileged state; setting extensions.webextensions.uuids before launch
|
||||
// pins it instead. This only works because the manifest declares a fixed
|
||||
// browser_specific_settings.gecko.id — without one the mapping has no key.
|
||||
const EXTENSION_ID = "autistmask@sneak.berlin";
|
||||
const EXTENSION_UUID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
|
||||
const EXTENSION_ORIGIN = "moz-extension://" + EXTENSION_UUID;
|
||||
|
||||
// The W3C web element identifier. Getting the last character wrong yields
|
||||
// an element reference of "undefined" and a bewildering "element with the
|
||||
// reference undefined is not known" from geckodriver, so findElement()
|
||||
// below checks for the key rather than indexing blindly.
|
||||
const WEB_ELEMENT_KEY = "element-6066-11e4-a52e-4f735466cecf";
|
||||
|
||||
const SCRIPT_TIMEOUT_MS = 120000;
|
||||
const DEFAULT_WAIT_MS = 20000;
|
||||
const POLL_INTERVAL_MS = 100;
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
// An ephemeral port picked by the kernel, then handed to geckodriver.
|
||||
// There is a race between closing this listener and geckodriver binding,
|
||||
// but this host runs many sessions at once and a fixed 4444 is a
|
||||
// guaranteed collision rather than a possible one.
|
||||
function freePort() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const srv = net.createServer();
|
||||
srv.on("error", reject);
|
||||
srv.listen(0, "127.0.0.1", () => {
|
||||
const { port } = srv.address();
|
||||
srv.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
class WebDriverError extends Error {
|
||||
constructor(command, body) {
|
||||
const v = (body && body.value) || {};
|
||||
super(
|
||||
command +
|
||||
" failed: " +
|
||||
(v.error || "unknown error") +
|
||||
": " +
|
||||
(v.message || JSON.stringify(body)),
|
||||
);
|
||||
this.name = "WebDriverError";
|
||||
this.error = v.error;
|
||||
}
|
||||
}
|
||||
|
||||
class Driver {
|
||||
constructor(proc, base) {
|
||||
this.proc = proc;
|
||||
this.base = base;
|
||||
this.sessionId = null;
|
||||
this.context = "content";
|
||||
}
|
||||
|
||||
async send(method, path, body) {
|
||||
const url = this.base + path;
|
||||
const res = await fetch(url, {
|
||||
method,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
const text = await res.text();
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(text);
|
||||
} catch (_) {
|
||||
throw new Error(
|
||||
method + " " + path + ": non-JSON response: " + text,
|
||||
);
|
||||
}
|
||||
if (!res.ok) throw new WebDriverError(method + " " + path, parsed);
|
||||
return parsed.value;
|
||||
}
|
||||
|
||||
session(method, path, body) {
|
||||
return this.send(method, "/session/" + this.sessionId + path, body);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ setup
|
||||
|
||||
async newSession() {
|
||||
const prefs = {
|
||||
// See EXTENSION_UUID above. The pref is a string pref whose
|
||||
// value is itself JSON.
|
||||
"extensions.webextensions.uuids": JSON.stringify({
|
||||
[EXTENSION_ID]: EXTENSION_UUID,
|
||||
}),
|
||||
};
|
||||
|
||||
const value = await this.send("POST", "/session", {
|
||||
capabilities: {
|
||||
alwaysMatch: {
|
||||
browserName: "firefox",
|
||||
"moz:firefoxOptions": {
|
||||
binary: FIREFOX_BIN,
|
||||
args: [
|
||||
"-headless",
|
||||
// Mandatory on Firefox 153: without it,
|
||||
// navigating to moz-extension:// and running
|
||||
// chrome-context script both fail with
|
||||
// "unsupported operation".
|
||||
//
|
||||
// It grants the driver FULL CHROME PRIVILEGES
|
||||
// over this browser. Acceptable only because
|
||||
// the browser is a throwaway in a CI
|
||||
// container; never point a session with this
|
||||
// flag at anything you care about.
|
||||
"-remote-allow-system-access",
|
||||
],
|
||||
prefs,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
this.sessionId = value.sessionId;
|
||||
await this.session("POST", "/timeouts", { script: SCRIPT_TIMEOUT_MS });
|
||||
return value;
|
||||
}
|
||||
|
||||
// Installs the unpacked MV2 build straight from a directory.
|
||||
// temporary:true bypasses signature checks, so no XPI and no signing
|
||||
// are involved, and the add-on dies with the profile.
|
||||
async installAddon(dir) {
|
||||
return this.session("POST", "/moz/addon/install", {
|
||||
path: dir,
|
||||
temporary: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Classic navigation on purpose. BiDi's browsingContext.navigate
|
||||
// refuses moz-extension:// URLs outright.
|
||||
async navigate(url) {
|
||||
await this.session("POST", "/url", { url });
|
||||
}
|
||||
|
||||
async quit() {
|
||||
if (this.sessionId) {
|
||||
await this.session("DELETE", "").catch(() => {});
|
||||
this.sessionId = null;
|
||||
}
|
||||
this.proc.kill("SIGTERM");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------- scripts
|
||||
|
||||
async setContext(context) {
|
||||
if (this.context === context) return;
|
||||
await this.session("POST", "/moz/context", { context });
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
async execute(script, args = []) {
|
||||
await this.setContext("content");
|
||||
return this.session("POST", "/execute/sync", { script, args });
|
||||
}
|
||||
|
||||
// Runs in the privileged chrome scope, where Services and Ci exist.
|
||||
async executeChrome(script, args = []) {
|
||||
await this.setContext("chrome");
|
||||
try {
|
||||
return await this.session("POST", "/execute/sync", {
|
||||
script,
|
||||
args,
|
||||
});
|
||||
} finally {
|
||||
await this.setContext("content");
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------- page waits
|
||||
|
||||
// Polls a content-context expression until it returns truthy. Every
|
||||
// wait in the suite goes through here so a timeout always says which
|
||||
// condition it was waiting on rather than "timed out".
|
||||
async waitFor(what, script, args = [], timeout = DEFAULT_WAIT_MS) {
|
||||
const deadline = Date.now() + timeout;
|
||||
let last = null;
|
||||
for (;;) {
|
||||
try {
|
||||
const v = await this.execute(script, args);
|
||||
if (v) return v;
|
||||
last = null;
|
||||
} catch (e) {
|
||||
// A navigation or view swap in flight makes execute
|
||||
// throw; that is a not-yet, not a failure, until the
|
||||
// deadline says otherwise.
|
||||
last = e.message;
|
||||
}
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error(
|
||||
"timed out after " +
|
||||
timeout +
|
||||
"ms waiting for " +
|
||||
what +
|
||||
(last ? " (last error: " + last + ")" : ""),
|
||||
);
|
||||
}
|
||||
await sleep(POLL_INTERVAL_MS);
|
||||
}
|
||||
}
|
||||
|
||||
// Shown means shown: in the popup a view is switched by toggling a
|
||||
// "hidden" class, and an element that is present but collapsed is not
|
||||
// the thing a test means by visible.
|
||||
async waitVisible(selector, timeout = DEFAULT_WAIT_MS) {
|
||||
return this.waitFor(
|
||||
"selector " + selector + " to be visible",
|
||||
`const el = document.querySelector(arguments[0]);
|
||||
if (!el) return false;
|
||||
const r = el.getBoundingClientRect();
|
||||
return r.width > 0 && r.height > 0;`,
|
||||
[selector],
|
||||
timeout,
|
||||
);
|
||||
}
|
||||
|
||||
async isVisible(selector) {
|
||||
return this.execute(
|
||||
`const el = document.querySelector(arguments[0]);
|
||||
if (!el) return false;
|
||||
const r = el.getBoundingClientRect();
|
||||
return r.width > 0 && r.height > 0;`,
|
||||
[selector],
|
||||
);
|
||||
}
|
||||
|
||||
async count(selector) {
|
||||
return this.execute(
|
||||
"return document.querySelectorAll(arguments[0]).length;",
|
||||
[selector],
|
||||
);
|
||||
}
|
||||
|
||||
async text(selector) {
|
||||
return this.execute(
|
||||
`const el = document.querySelector(arguments[0]);
|
||||
return el ? el.textContent : null;`,
|
||||
[selector],
|
||||
);
|
||||
}
|
||||
|
||||
async title() {
|
||||
return this.session("GET", "/title");
|
||||
}
|
||||
|
||||
// The id of the view element currently on top, which is what a
|
||||
// failing step needs to report: "the screen did not change" is only
|
||||
// useful if it says which screen it stayed on.
|
||||
async currentView() {
|
||||
return this.execute(
|
||||
`const views = document.querySelectorAll('[id^="view-"]');
|
||||
for (const v of views) {
|
||||
const r = v.getBoundingClientRect();
|
||||
if (r.width > 0 && r.height > 0) return v.id;
|
||||
}
|
||||
return null;`,
|
||||
);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------- interactions
|
||||
|
||||
async findElement(selector) {
|
||||
const value = await this.session("POST", "/element", {
|
||||
using: "css selector",
|
||||
value: selector,
|
||||
});
|
||||
const ref = value && value[WEB_ELEMENT_KEY];
|
||||
if (typeof ref !== "string") {
|
||||
throw new Error(
|
||||
"no " +
|
||||
WEB_ELEMENT_KEY +
|
||||
" in the element response for " +
|
||||
selector +
|
||||
": " +
|
||||
JSON.stringify(value),
|
||||
);
|
||||
}
|
||||
return ref;
|
||||
}
|
||||
|
||||
// Real WebDriver clicks and real key events rather than in-page
|
||||
// .click() and value assignment: the popup's handlers are wired to
|
||||
// events, and synthesising them from inside the page would test the
|
||||
// harness's idea of the UI instead of the UI.
|
||||
async click(selector) {
|
||||
await this.waitVisible(selector);
|
||||
const id = await this.findElement(selector);
|
||||
await this.session("POST", "/element/" + id + "/click", {});
|
||||
}
|
||||
|
||||
async fill(selector, value) {
|
||||
await this.waitVisible(selector);
|
||||
const id = await this.findElement(selector);
|
||||
await this.session("POST", "/element/" + id + "/clear", {});
|
||||
await this.session("POST", "/element/" + id + "/value", {
|
||||
text: String(value),
|
||||
});
|
||||
}
|
||||
|
||||
async value(selector) {
|
||||
return this.execute(
|
||||
`const el = document.querySelector(arguments[0]);
|
||||
return el ? el.value : null;`,
|
||||
[selector],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------- error capture
|
||||
|
||||
// Uncaught errors from extension code, read out of the privileged console
|
||||
// service.
|
||||
//
|
||||
// This is not the obvious mechanism, and the obvious one does not work:
|
||||
// WebDriver BiDi's log.entryAdded delivers NOTHING for extension pages.
|
||||
// Verified on Firefox 142 and 153 against a same-session control — a plain
|
||||
// http:// page yields uncaught errors with stack traces, the
|
||||
// moz-extension:// popup yields zero events, because the remote agent
|
||||
// excludes extension browsing contexts from BiDi observation. A harness
|
||||
// built on Playwright-BiDi or Puppeteer-BiDi therefore sees nothing and
|
||||
// reports success. Do not "simplify" this back to BiDi.
|
||||
//
|
||||
// nsIConsoleService is not per-page: it also carries errors from the
|
||||
// background page, which BiDi would not have covered even if it worked.
|
||||
// Background-page capture is verified by probe — a throw at the top of
|
||||
// src/background/index.js, which kills the background page outright, fails
|
||||
// the run. Content-script errors should arrive by the same route, but that
|
||||
// is UNVERIFIED here and must not be claimed: the container runs with
|
||||
// --network none, so there is no http:// page for a content script to be
|
||||
// injected into and this suite never exercises one.
|
||||
//
|
||||
// Warnings are excluded so the semantics match Playwright's pageerror:
|
||||
// uncaught errors only.
|
||||
//
|
||||
// The read and the clear are ONE chrome script on purpose. Splitting them
|
||||
// into two round trips leaves a blind window between them in which an
|
||||
// error is logged into a buffer that is about to be discarded, and is
|
||||
// destroyed unread rather than deferred to the next drain. That was not
|
||||
// theoretical: with a separate reset() call, a probe of 100 sequenced
|
||||
// throws at 20ms spacing lost one of them outright.
|
||||
const DRAIN_ERRORS_SCRIPT = `
|
||||
const origin = arguments[0];
|
||||
const out = [];
|
||||
for (const raw of Services.console.getMessageArray() || []) {
|
||||
let e;
|
||||
try {
|
||||
e = raw.QueryInterface(Ci.nsIScriptError);
|
||||
} catch (_) {
|
||||
continue;
|
||||
}
|
||||
if (e.flags & Ci.nsIScriptError.warningFlag) continue;
|
||||
const src = e.sourceName || "";
|
||||
if (!src.startsWith(origin)) continue;
|
||||
out.push({
|
||||
msg: e.errorMessage,
|
||||
src: src,
|
||||
line: e.lineNumber,
|
||||
cat: e.category,
|
||||
});
|
||||
}
|
||||
Services.console.reset();
|
||||
return out;
|
||||
`;
|
||||
|
||||
class ConsoleErrors {
|
||||
constructor(driver, originPrefix) {
|
||||
this.driver = driver;
|
||||
this.originPrefix = originPrefix;
|
||||
}
|
||||
|
||||
// Everything logged since the last take, read and cleared atomically
|
||||
// in a single chrome round trip. Poll-based, so an error is attributed
|
||||
// to the step that was running when it was drained, not to the moment
|
||||
// inside that step at which it happened — see the limitation note in
|
||||
// run.js. An error that arrives mid-drain is not lost — it makes this
|
||||
// batch or the next one — but the console service ring buffer holds
|
||||
// only 250 messages, so more than that between two takes evicts the
|
||||
// oldest unread. A clean run peaks at 4.
|
||||
async take() {
|
||||
const found = await this.driver.executeChrome(DRAIN_ERRORS_SCRIPT, [
|
||||
this.originPrefix,
|
||||
]);
|
||||
return found || [];
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------- startup
|
||||
|
||||
async function waitForDriverReady(base, timeoutMs) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
for (;;) {
|
||||
try {
|
||||
const res = await fetch(base + "/status");
|
||||
if (res.ok) {
|
||||
const body = await res.json();
|
||||
if (body && body.value && body.value.ready !== false) return;
|
||||
}
|
||||
} catch (_) {
|
||||
// not listening yet
|
||||
}
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error(
|
||||
"geckodriver did not become ready within " + timeoutMs + "ms",
|
||||
);
|
||||
}
|
||||
await sleep(POLL_INTERVAL_MS);
|
||||
}
|
||||
}
|
||||
|
||||
async function start() {
|
||||
const port = await freePort();
|
||||
const proc = spawn(
|
||||
GECKODRIVER,
|
||||
["--port", String(port), "--host", "127.0.0.1"],
|
||||
{ stdio: ["ignore", "inherit", "inherit"] },
|
||||
);
|
||||
proc.on("error", (e) => {
|
||||
console.error("geckodriver failed to spawn: " + e.message);
|
||||
});
|
||||
|
||||
const base = "http://127.0.0.1:" + port;
|
||||
try {
|
||||
await waitForDriverReady(base, 30000);
|
||||
} catch (e) {
|
||||
proc.kill("SIGKILL");
|
||||
throw e;
|
||||
}
|
||||
return new Driver(proc, base);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ConsoleErrors,
|
||||
Driver,
|
||||
EXTENSION_ID,
|
||||
EXTENSION_ORIGIN,
|
||||
EXTENSION_UUID,
|
||||
start,
|
||||
sleep,
|
||||
};
|
||||
Reference in New Issue
Block a user