AutistMask/script/lib/build-blocklist.js

// The transform half of script/vendor-blocklist: upstream's config.json in,
// src/shared/phishingBlocklist.json out. Build-time repo tooling; nothing here
// is shipped to users.
//
// Usage: node script/lib/build-blocklist.js <source.json> <output.json>
//
// What it does, and why each step is here:
//
//   - only the blacklist is carried over. The extension matches a hostname and
//     its parent domains against that one list; upstream's whitelist, fuzzylist
//     and version metadata are read by nothing here, so shipping them would add
//     megabytes of dead weight to every install.
//   - entries are lowercased and de-duplicated, because that is the form
//     isPhishingDomain() compares against.
//   - entries that cannot be a hostname are dropped and counted. Upstream
//     carries the odd URL-shaped entry (a path, a scheme); hostname matching can
//     never match one, and once the artifact is hashes nobody can see that it is
//     in there, so it is reported at vendoring time instead.
//   - entries are hashed (see src/shared/domainHash.js) and sorted, and the
//     digests are concatenated into one fixed-width string. Sorted is what makes
//     the runtime lookup a binary search over that string, with no set to build
//     on every service-worker wake; one string rather than an array of 100k+ is
//     what keeps the file, the bundle and the JSON parse small.
//
// Deterministic by construction: same input bytes, same output bytes.

"use strict";

const fs = require("fs");

const {
    HASH_ALGORITHM,
    HASH_HEX_CHARS,
    hashDomain,
} = require("../../src/shared/domainHash");

// A blocklist that has collapsed to a handful of entries is a broken fetch or a
// changed upstream shape, not a quiet day in phishing. Vendoring it would
// disarm the feature, so it fails instead and a human decides.
const MIN_ENTRIES = 10000;

function fail(message) {
    process.stderr.write("build-blocklist: " + message + "\n");
    process.exit(1);
}

// A hostname, as the matcher understands one: dot-separated labels of letters,
// digits, hyphens and underscores. Anything else — a path, a scheme, a space,
// an empty string, a non-ASCII label a browser would have punycoded before it
// ever reached isPhishingDomain() — cannot be produced by the hostname variants
// the extension looks up, so it could only ever sit in the artifact unused.
//
// Underscores are deliberate. They are not legal in a hostname per RFC 1123,
// but DNS carries them and browsers resolve them, and upstream lists 141 entries
// that use one — real phishing sites on shared subdomain hosts. A stricter
// pattern silently drops every one of them.
const HOSTNAME_RE =
    /^[a-z0-9_]([a-z0-9_-]*[a-z0-9_])?(\.[a-z0-9_]([a-z0-9_-]*[a-z0-9_])?)+$/;

function main(argv) {
    const [source, output] = argv;
    if (!source || !output) {
        fail("usage: build-blocklist.js <source.json> <output.json>");
    }

    let config;
    try {
        config = JSON.parse(fs.readFileSync(source, "utf8"));
    } catch (e) {
        fail("could not read " + source + " as JSON: " + e.message);
    }

    if (!Array.isArray(config.blacklist)) {
        fail(
            "the source has no blacklist array, so its shape is not the one " +
                "this transform understands. Refusing to write an artifact.",
        );
    }

    const seen = new Set();
    let dropped = 0;
    for (const raw of config.blacklist) {
        if (typeof raw !== "string") {
            dropped++;
            continue;
        }
        const domain = raw.trim().toLowerCase();
        if (!HOSTNAME_RE.test(domain)) {
            dropped++;
            continue;
        }
        seen.add(domain);
    }

    if (seen.size < MIN_ENTRIES) {
        fail(
            "the source yielded " +
                seen.size +
                " usable entries, below the " +
                MIN_ENTRIES +
                " floor. That is a broken source or a changed upstream " +
                "shape, and vendoring it would disarm phishing detection. " +
                "Refusing to write an artifact.",
        );
    }

    const hashes = [];
    for (const domain of seen) hashes.push(hashDomain(domain));
    hashes.sort();

    // Truncation makes collisions possible; they are harmless (both entries are
    // blocked either way) but they must not inflate the count the artifact
    // claims, which the runtime cross-checks against the string length.
    const unique = [];
    for (const hash of hashes) {
        if (unique.length === 0 || unique[unique.length - 1] !== hash) {
            unique.push(hash);
        }
    }

    const artifact = {
        algorithm: HASH_ALGORITHM,
        hashHexChars: HASH_HEX_CHARS,
        count: unique.length,
        hashes: unique.join(""),
    };

    // Four-space JSON with a trailing newline: what prettier emits for this
    // shape, so a vendored artifact passes make fmt-check untouched.
    fs.writeFileSync(output, JSON.stringify(artifact, null, 4) + "\n");

    process.stdout.write(
        "build-blocklist: " +
            config.blacklist.length +
            " source entries -> " +
            seen.size +
            " usable domains -> " +
            unique.length +
            " digests (" +
            dropped +
            " not hostnames, " +
            (seen.size - unique.length) +
            " digest collisions)\n",
    );
}

main(process.argv.slice(2));