// The transform half of script/vendor-blocklist: upstream's config.json in, // src/shared/phishingBlocklist.json out. Build-time repo tooling; nothing here // is shipped to users. // // Usage: node script/lib/build-blocklist.js // // What it does, and why each step is here: // // - only the blacklist is carried over. The extension matches a hostname and // its parent domains against that one list; upstream's whitelist, fuzzylist // and version metadata are read by nothing here, so shipping them would add // megabytes of dead weight to every install. // - entries are lowercased and de-duplicated, because that is the form // isPhishingDomain() compares against. // - entries that cannot be a hostname are dropped and counted. Upstream // carries the odd URL-shaped entry (a path, a scheme); hostname matching can // never match one, and once the artifact is hashes nobody can see that it is // in there, so it is reported at vendoring time instead. // - entries are hashed (see src/shared/domainHash.js) and sorted, and the // digests are concatenated into one fixed-width string. Sorted is what makes // the runtime lookup a binary search over that string, with no set to build // on every service-worker wake; one string rather than an array of 100k+ is // what keeps the file, the bundle and the JSON parse small. // // Deterministic by construction: same input bytes, same output bytes. "use strict"; const fs = require("fs"); const { HASH_ALGORITHM, HASH_HEX_CHARS, hashDomain, } = require("../../src/shared/domainHash"); // A blocklist that has collapsed to a handful of entries is a broken fetch or a // changed upstream shape, not a quiet day in phishing. Vendoring it would // disarm the feature, so it fails instead and a human decides. const MIN_ENTRIES = 10000; function fail(message) { process.stderr.write("build-blocklist: " + message + "\n"); process.exit(1); } // A hostname, as the matcher understands one: dot-separated labels of letters, // digits, hyphens and underscores. Anything else — a path, a scheme, a space, // an empty string, a non-ASCII label a browser would have punycoded before it // ever reached isPhishingDomain() — cannot be produced by the hostname variants // the extension looks up, so it could only ever sit in the artifact unused. // // Underscores are deliberate. They are not legal in a hostname per RFC 1123, // but DNS carries them and browsers resolve them, and upstream lists 141 entries // that use one — real phishing sites on shared subdomain hosts. A stricter // pattern silently drops every one of them. const HOSTNAME_RE = /^[a-z0-9_]([a-z0-9_-]*[a-z0-9_])?(\.[a-z0-9_]([a-z0-9_-]*[a-z0-9_])?)+$/; function main(argv) { const [source, output] = argv; if (!source || !output) { fail("usage: build-blocklist.js "); } let config; try { config = JSON.parse(fs.readFileSync(source, "utf8")); } catch (e) { fail("could not read " + source + " as JSON: " + e.message); } if (!Array.isArray(config.blacklist)) { fail( "the source has no blacklist array, so its shape is not the one " + "this transform understands. Refusing to write an artifact.", ); } const seen = new Set(); let dropped = 0; for (const raw of config.blacklist) { if (typeof raw !== "string") { dropped++; continue; } const domain = raw.trim().toLowerCase(); if (!HOSTNAME_RE.test(domain)) { dropped++; continue; } seen.add(domain); } if (seen.size < MIN_ENTRIES) { fail( "the source yielded " + seen.size + " usable entries, below the " + MIN_ENTRIES + " floor. That is a broken source or a changed upstream " + "shape, and vendoring it would disarm phishing detection. " + "Refusing to write an artifact.", ); } const hashes = []; for (const domain of seen) hashes.push(hashDomain(domain)); hashes.sort(); // Truncation makes collisions possible; they are harmless (both entries are // blocked either way) but they must not inflate the count the artifact // claims, which the runtime cross-checks against the string length. const unique = []; for (const hash of hashes) { if (unique.length === 0 || unique[unique.length - 1] !== hash) { unique.push(hash); } } const artifact = { algorithm: HASH_ALGORITHM, hashHexChars: HASH_HEX_CHARS, count: unique.length, hashes: unique.join(""), }; // Four-space JSON with a trailing newline: what prettier emits for this // shape, so a vendored artifact passes make fmt-check untouched. fs.writeFileSync(output, JSON.stringify(artifact, null, 4) + "\n"); process.stdout.write( "build-blocklist: " + config.blacklist.length + " source entries -> " + seen.size + " usable domains -> " + unique.length + " digests (" + dropped + " not hostnames, " + (seen.size - unique.length) + " digest collisions)\n", ); } main(process.argv.slice(2));