refactor: vendor phishing blocklist, delta-only memory model
All checks were successful
check / check (push) Successful in 25s
All checks were successful
check / check (push) Successful in 25s
- Vendor community-maintained phishing domain blocklist into src/shared/phishingBlocklist.json (bundled at build time by esbuild) - Refactor phishingDomains.js: build vendored Sets at module load, fetch live list periodically, keep only delta (new entries not in vendored) in memory for small runtime footprint - Domain checker checks delta first (fresh scam sites), then vendored - Persist delta to localStorage if under 256 KiB - Load delta from localStorage on startup for instant coverage - Add startPeriodicRefresh() with 24h setInterval in background script - Remove dead code: popup's local isPhishingDomain() re-check was inert (popup never called updatePhishingList so its blacklistSet was always empty); now relies solely on background's authoritative flag - Remove all competitor name mentions from UI warning text and comments - Update README: document phishing domain protection architecture, update external services list - Update tests: cover vendored blocklist loading, delta computation, localStorage persistence, delta+vendored interaction Closes #114
This commit is contained in:
@@ -1,37 +1,106 @@
|
||||
// Domain-based phishing detection using MetaMask's eth-phishing-detect blocklist.
|
||||
// Fetches the blocklist at runtime, caches it in memory, and checks hostnames.
|
||||
// Domain-based phishing detection using a vendored blocklist with delta updates.
|
||||
//
|
||||
// The blocklist source:
|
||||
// https://github.com/MetaMask/eth-phishing-detect (src/config.json)
|
||||
// A community-maintained phishing domain blocklist is vendored in
|
||||
// phishingBlocklist.json and bundled at build time. At runtime, we fetch
|
||||
// the live list periodically and keep only the delta (new entries not in
|
||||
// the vendored list) in memory. This keeps runtime memory usage small.
|
||||
//
|
||||
// The config uses { blacklist: [...], whitelist: [...], fuzzylist: [...] }.
|
||||
// We check exact hostname and parent-domain matches against the blacklist,
|
||||
// with whitelist overrides.
|
||||
// The domain-checker checks the in-memory delta first (fresh/recent scam
|
||||
// sites), then falls back to the vendored list.
|
||||
//
|
||||
// If the delta is under 256 KiB it is persisted to localStorage so it
|
||||
// survives extension/service-worker restarts.
|
||||
|
||||
const vendoredConfig = require("./phishingBlocklist.json");
|
||||
|
||||
const BLOCKLIST_URL =
|
||||
"https://raw.githubusercontent.com/MetaMask/eth-phishing-detect/main/src/config.json";
|
||||
|
||||
const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
const REFRESH_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
const DELTA_STORAGE_KEY = "phishing-delta";
|
||||
const MAX_DELTA_BYTES = 256 * 1024; // 256 KiB
|
||||
|
||||
let blacklistSet = new Set();
|
||||
let whitelistSet = new Set();
|
||||
// Vendored sets — built once from the bundled JSON.
|
||||
const vendoredBlacklist = new Set(
|
||||
(vendoredConfig.blacklist || []).map((d) => d.toLowerCase()),
|
||||
);
|
||||
const vendoredWhitelist = new Set(
|
||||
(vendoredConfig.whitelist || []).map((d) => d.toLowerCase()),
|
||||
);
|
||||
|
||||
// Delta sets — only entries from live list that are NOT in vendored.
|
||||
let deltaBlacklist = new Set();
|
||||
let deltaWhitelist = new Set();
|
||||
let lastFetchTime = 0;
|
||||
let fetchPromise = null;
|
||||
let refreshTimer = null;
|
||||
|
||||
/**
|
||||
* Load a pre-parsed config into the in-memory sets.
|
||||
* Used for testing and for loading from cache.
|
||||
* Load delta entries from localStorage on startup.
|
||||
* Called once during module initialization in the background script.
|
||||
*/
|
||||
function loadDeltaFromStorage() {
|
||||
try {
|
||||
const raw = localStorage.getItem(DELTA_STORAGE_KEY);
|
||||
if (!raw) return;
|
||||
const data = JSON.parse(raw);
|
||||
if (data.blacklist && Array.isArray(data.blacklist)) {
|
||||
deltaBlacklist = new Set(
|
||||
data.blacklist.map((d) => d.toLowerCase()),
|
||||
);
|
||||
}
|
||||
if (data.whitelist && Array.isArray(data.whitelist)) {
|
||||
deltaWhitelist = new Set(
|
||||
data.whitelist.map((d) => d.toLowerCase()),
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// localStorage unavailable or corrupt — start empty
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist delta to localStorage if it fits within MAX_DELTA_BYTES.
|
||||
*/
|
||||
function saveDeltaToStorage() {
|
||||
try {
|
||||
const data = {
|
||||
blacklist: Array.from(deltaBlacklist),
|
||||
whitelist: Array.from(deltaWhitelist),
|
||||
};
|
||||
const json = JSON.stringify(data);
|
||||
if (json.length < MAX_DELTA_BYTES) {
|
||||
localStorage.setItem(DELTA_STORAGE_KEY, json);
|
||||
} else {
|
||||
// Too large — remove stale key if present
|
||||
localStorage.removeItem(DELTA_STORAGE_KEY);
|
||||
}
|
||||
} catch {
|
||||
// localStorage unavailable — skip silently
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a pre-parsed config and compute the delta against the vendored list.
|
||||
* Used for both live fetches and testing.
|
||||
*
|
||||
* @param {{ blacklist?: string[], whitelist?: string[] }} config
|
||||
*/
|
||||
function loadConfig(config) {
|
||||
blacklistSet = new Set(
|
||||
(config.blacklist || []).map((d) => d.toLowerCase()),
|
||||
const liveBlacklist = (config.blacklist || []).map((d) => d.toLowerCase());
|
||||
const liveWhitelist = (config.whitelist || []).map((d) => d.toLowerCase());
|
||||
|
||||
// Delta = entries in the live list that are NOT in the vendored list
|
||||
deltaBlacklist = new Set(
|
||||
liveBlacklist.filter((d) => !vendoredBlacklist.has(d)),
|
||||
);
|
||||
whitelistSet = new Set(
|
||||
(config.whitelist || []).map((d) => d.toLowerCase()),
|
||||
deltaWhitelist = new Set(
|
||||
liveWhitelist.filter((d) => !vendoredWhitelist.has(d)),
|
||||
);
|
||||
|
||||
lastFetchTime = Date.now();
|
||||
saveDeltaToStorage();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -54,8 +123,8 @@ function hostnameVariants(hostname) {
|
||||
|
||||
/**
|
||||
* Check if a hostname is on the phishing blocklist.
|
||||
* Checks exact hostname and all parent domains.
|
||||
* Whitelisted domains are never flagged.
|
||||
* Checks delta first (fresh/recent scam sites), then vendored list.
|
||||
* Whitelisted domains (delta + vendored) are never flagged.
|
||||
*
|
||||
* @param {string} hostname - The hostname to check.
|
||||
* @returns {boolean}
|
||||
@@ -63,25 +132,28 @@ function hostnameVariants(hostname) {
|
||||
function isPhishingDomain(hostname) {
|
||||
if (!hostname) return false;
|
||||
const variants = hostnameVariants(hostname);
|
||||
// Whitelist takes priority
|
||||
|
||||
// Whitelist takes priority — check delta whitelist first, then vendored
|
||||
for (const v of variants) {
|
||||
if (whitelistSet.has(v)) return false;
|
||||
if (deltaWhitelist.has(v) || vendoredWhitelist.has(v)) return false;
|
||||
}
|
||||
|
||||
// Check delta blacklist first (fresh/recent scam sites), then vendored
|
||||
for (const v of variants) {
|
||||
if (blacklistSet.has(v)) return true;
|
||||
if (deltaBlacklist.has(v) || vendoredBlacklist.has(v)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the latest blocklist from the MetaMask repo.
|
||||
* Fetch the latest blocklist and compute delta against vendored data.
|
||||
* De-duplicates concurrent fetches. Results are cached for CACHE_TTL_MS.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function updatePhishingList() {
|
||||
// Skip if recently fetched
|
||||
if (Date.now() - lastFetchTime < CACHE_TTL_MS && blacklistSet.size > 0) {
|
||||
if (Date.now() - lastFetchTime < CACHE_TTL_MS && lastFetchTime > 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -95,7 +167,8 @@ async function updatePhishingList() {
|
||||
const config = await resp.json();
|
||||
loadConfig(config);
|
||||
} catch {
|
||||
// Silently fail — we'll retry next time.
|
||||
// Silently fail — vendored list still provides coverage.
|
||||
// We'll retry next time.
|
||||
} finally {
|
||||
fetchPromise = null;
|
||||
}
|
||||
@@ -105,29 +178,61 @@ async function updatePhishingList() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the current blocklist size (for diagnostics).
|
||||
* Start periodic refresh of the phishing list.
|
||||
* Should be called once from the background script on startup.
|
||||
*/
|
||||
function startPeriodicRefresh() {
|
||||
if (refreshTimer) return;
|
||||
refreshTimer = setInterval(updatePhishingList, REFRESH_INTERVAL_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the total blocklist size (vendored + delta) for diagnostics.
|
||||
*
|
||||
* @returns {number}
|
||||
*/
|
||||
function getBlocklistSize() {
|
||||
return blacklistSet.size;
|
||||
return vendoredBlacklist.size + deltaBlacklist.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the delta blocklist size for diagnostics.
|
||||
*
|
||||
* @returns {number}
|
||||
*/
|
||||
function getDeltaSize() {
|
||||
return deltaBlacklist.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset internal state (for testing).
|
||||
*/
|
||||
function _reset() {
|
||||
blacklistSet = new Set();
|
||||
whitelistSet = new Set();
|
||||
deltaBlacklist = new Set();
|
||||
deltaWhitelist = new Set();
|
||||
lastFetchTime = 0;
|
||||
fetchPromise = null;
|
||||
if (refreshTimer) {
|
||||
clearInterval(refreshTimer);
|
||||
refreshTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Load persisted delta on module initialization
|
||||
loadDeltaFromStorage();
|
||||
|
||||
module.exports = {
|
||||
isPhishingDomain,
|
||||
updatePhishingList,
|
||||
startPeriodicRefresh,
|
||||
loadConfig,
|
||||
getBlocklistSize,
|
||||
getDeltaSize,
|
||||
hostnameVariants,
|
||||
_reset,
|
||||
// Exposed for testing only
|
||||
_getVendoredBlacklistSize: () => vendoredBlacklist.size,
|
||||
_getVendoredWhitelistSize: () => vendoredWhitelist.size,
|
||||
_getDeltaBlacklist: () => deltaBlacklist,
|
||||
_getDeltaWhitelist: () => deltaWhitelist,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user