AutistMask/src/shared/balances.js

// Balance fetching: ETH balances and the balances of the tokens the user
// tracks via RPC, the other ERC-20 token balances and the holder counts via
// Blockscout while the user has "Use Blockscout" on, ENS reverse lookup via
// RPC.

const {
    JsonRpcProvider,
    Network,
    Contract,
    formatEther,
    formatUnits,
} = require("ethers");
const { ERC20_ABI } = require("./constants");
const { NETWORKS } = require("./networks");
const { log, debugFetch, urlOrigin } = require("./log");
const { deriveAddressFromXpub } = require("./wallet");
const { TOKEN_BY_ADDRESS } = require("./tokenList");
const { LOW_HOLDER_THRESHOLD, parseHoldersCount } = require("./holders");
const { isSpoofedSymbol } = require("./symbolSpoof");
const { toDecimals } = require("./transferAmount");
const { resolveTokenDecimals } = require("./approvalAmount");

// Use a static network to skip auto-detection (which can fail and cause
// "could not coalesce error" on some RPC endpoints like Cloudflare).
//
// `networkId` is REQUIRED, and is one of the ids in networks.js. It used to be
// optional, falling back to currentNetwork() — the module-level `state`
// singleton, which the MV3 service worker never populates. The endpoint then
// came out right and the static hint came out mainnet, so ethers fixed
// `chainId` at 0x1 and every non-mainnet dApp send was prepared for the wrong
// chain and then refused by the wallet's own verifier
// (https://git.eeqj.de/sneak/AutistMask/issues/320). Requiring it is what
// stops that from coming back: a caller that has no network to name has no
// business constructing a provider, and there is no longer a default for it
// to get silently wrong.
//
// Validated against NETWORKS rather than passed straight to Network.from():
// ethers knows chains this wallet does not, so an id that is not one of ours
// is a caller bug and must not resolve to a working provider for some other
// chain.
//
// `disableCcipRead` keeps every ENS lookup on the RPC endpoint. Left on, as
// ethers has it, a resolver that says a name is resolved off-chain makes ethers
// fetch the web address the resolver gives, straight from the browser, with the
// name being looked up; for the reverse lookup of the user's addresses that is
// the user's own name (https://git.eeqj.de/sneak/AutistMask/issues/668). Such a
// name does not resolve.
function getProvider(rpcUrl, networkId) {
    const net = Network.from(requireNetworkId(networkId).id);
    const provider = new JsonRpcProvider(rpcUrl, net, { staticNetwork: net });
    provider.disableCcipRead = true;
    return provider;
}

function requireNetworkId(networkId) {
    const net = NETWORKS[networkId];
    if (!net) {
        throw new Error(
            "getProvider requires the id of a supported network; got " +
                JSON.stringify(networkId),
        );
    }
    return net;
}

// A token balance as an exact decimal string, never cut: a cut stores a small
// nonzero holding as zero. fetchTokenBalances() stores every nonzero holding of
// a token it admits, however small; the screens that leave out one below
// 0.000001 decide that themselves, through isBelowOneMillionth() in
// src/shared/amountDisplay.js.
function formatTokenBalance(raw, decimals) {
    const val = formatUnits(raw, decimals);
    const parts = val.split(".");
    if (parts.length === 1) return val + ".0";
    const dec = parts[1].replace(/0+$/, "") || "0";
    return parts[0] + "." + dec;
}

// The explorer's reported holding as an exact base-unit integer, or null when
// it reported nothing usable. Base units carry no scale, so this value is
// meaningful before the scale is known — which is what lets a holding of zero
// be recognised as zero without guessing a scale to divide it by.
function rawUnits(value) {
    if (typeof value === "bigint") return value >= 0n ? value : null;
    if (typeof value === "number") {
        return Number.isSafeInteger(value) && value >= 0 ? BigInt(value) : null;
    }
    if (typeof value !== "string" || !/^[0-9]+$/.test(value)) return null;
    return BigInt(value);
}

// Fetch token balances for a single address from Blockscout.
// Returns [{ address, name, symbol, decimals, balance, holders }].
//
// `useBlockscout` is the user's "Use Blockscout" setting. While it is off,
// Blockscout is not asked and the answer is an empty list: no token the user
// has not tracked is listed.
//
// Filters out spam: only shows tokens that are in the known token list,
// explicitly tracked by the user, or have >= 1000 holders. A tracked token's
// balance is read through the RPC endpoint instead, and its row here gives that
// entry its holder count and `decimals` (fetchTrackedTokenBalances()).
//
// `decimals` and `balance` are each null when the answer is unknown, the same
// way `holders` already is. Absence is never filled in here: this is the
// upstream of every screen that displays a token amount, so a value invented
// at this point is indistinguishable from a real one everywhere below it.
//
// `signal`, passed by the popup, is aborted when the popup closes; a request
// that fails after that was cancelled by the closing and is not logged.
async function fetchTokenBalances(
    address,
    blockscoutUrl,
    useBlockscout,
    trackedTokens,
    signal,
) {
    if (!useBlockscout) return [];
    try {
        const resp = await debugFetch(
            blockscoutUrl + "/addresses/" + address + "/token-balances",
        );
        if (!resp.ok) {
            log.errorf("blockscout token-balances:", resp.status);
            return null;
        }
        const items = await resp.json();
        if (!Array.isArray(items)) return null;

        const trackedSet = new Set(
            (trackedTokens || []).map((t) => t.address.toLowerCase()),
        );

        const balances = [];
        for (const item of items) {
            // Case-insensitive: the token type is an explorer's label, not a
            // protocol value, and an exact comparison silently drops a real
            // holding if one ever writes "erc-20".  Which types are admitted
            // is unchanged.
            const type = String(item.token?.type || "").toUpperCase();
            if (type !== "ERC-20") continue;

            const tokenAddr = (item.token.address_hash || "").toLowerCase();

            // What the explorer reported, or null. NEVER a default: this
            // value is written to state and every later reader — the approval
            // screen's amount line, the swap lines, the Send screen — takes it
            // as the token's resolved scale. A fabricated 18 reads exactly
            // like a real 18 at that point, so it does not merely display the
            // wrong quantity, it walks straight past the refusal those screens
            // already have for a scale nobody knows
            // (https://git.eeqj.de/sneak/AutistMask/issues/349).
            const decimals = toDecimals(item.token.decimals);

            const raw = rawUnits(item.value);
            // No usable amount at all is nothing to list, exactly as a
            // formatted "0.0" was before. Checked on the base-unit integer so
            // it does not depend on knowing the scale: zero base units is zero
            // tokens at every scale, and a value the explorer did not report
            // as an integer is not a holding.
            if (raw === null || raw === 0n) continue;

            // The scale this row's balance is DISPLAYED at, which is not the
            // same question as what the explorer said. The bundled list and
            // the tokens the user tracks both outrank the explorer already
            // (resolveTokenDecimals), so a token they know keeps showing its
            // real quantity even when the explorer's entry omits decimals.
            // Only what neither of them nor the explorer knows is unknown.
            // The stored `decimals` above stays the explorer's own answer
            // either way: copying another source into it would make
            // explorerDecimals()'s disagreement check compare something other
            // than explorer values.
            const known = resolveTokenDecimals(tokenAddr, { trackedTokens });
            const scale = known !== null ? known : decimals;
            // null is a holding of an amount that cannot be stated, which is
            // not the same as a holding of zero, and must never render as one.
            const bal = scale === null ? null : formatTokenBalance(raw, scale);
            // null means the explorer reported no readable count, which is
            // not the same as a count of zero.  This gate is not the
            // low-holder display filter: it has no user-facing off switch and
            // governs the whole balance list, so it stays strict and admits a
            // token only on a reported count — an unreported one is no
            // evidence, and `null >= LOW_HOLDER_THRESHOLD` is false.  A
            // legitimate token still reaches the list through the known
            // token list or by the user tracking it, and the null is carried
            // through to the views, where the two low-holder filters treat
            // an unknown count as "do not judge" rather than as zero.
            const holders = parseHoldersCount(item.token.holders_count);
            const isKnown = TOKEN_BY_ADDRESS.has(tokenAddr);
            const isTracked = trackedSet.has(tokenAddr);
            const hasEnoughHolders = holders >= LOW_HOLDER_THRESHOLD;

            // Skip spam tokens the user never asked to see
            if (!isKnown && !isTracked && !hasEnoughHolders) continue;

            // Skip tokens spoofing a known symbol from a different address.
            // Every row here is an ERC-20 the explorer reported, so it has a
            // contract address; the native ETH balance is fetched over RPC in
            // refreshBalances and never passes through this loop.
            if (isSpoofedSymbol(item.token.symbol, tokenAddr)) continue;

            balances.push({
                address: item.token.address_hash,
                name: item.token.name || "",
                symbol: item.token.symbol || "???",
                // null means the explorer reported no usable scale — unknown,
                // not 18. Distinguishable from a real 18 at read time is the
                // entire point: resolveTokenDecimals() falls through a null to
                // its refusal, and takes an 18 as the answer.
                decimals: decimals,
                // null means nothing anywhere knows the scale, so there is no
                // token quantity to state. Not "0.0": a nonzero holding shown
                // as zero is the same lie in the balance list that the
                // approval screens refuse to tell.
                balance: bal,
                holders: holders,
            });
        }
        return balances;
    } catch (e) {
        if (!signal?.aborted) {
            log.errorf("fetchTokenBalances failed:", e.message);
        }
        return null;
    }
}

// The entries at `address` of the tokens the user tracks, shaped as
// fetchTokenBalances() returns them, for the tokens the address holds.
//
// The balance is read with balanceOf through the RPC endpoint. The name and
// symbol are the tracked token's. `decimals` and `holders` are those of
// Blockscout's entry for the token in `explorerRequest`, this refresh's
// fetchTokenBalances() for the address; while Blockscout fails they are kept
// from the token's entry in `previous`, the address's entries before this
// refresh. The balance is stated at the scale the screens read the token at
// (resolveTokenDecimals()): the bundled list's, then the one the tracked token
// records, then Blockscout's, as in fetchTokenBalances().
//
// A token whose balanceOf fails keeps its entry in `previous`, as a failed ETH
// balance keeps its figure, or takes Blockscout's entry from this refresh when
// it has none. The failure is logged. A balance of zero gets no entry, as a
// zero Blockscout reports gets none: the balance lists take a tracked token's
// zero from the tracked list. A token whose symbol the spoof rule refuses gets
// no entry either, as in fetchTokenBalances().
//
// `signal` is as for fetchTokenBalances().
async function fetchTrackedTokenBalances(
    provider,
    address,
    trackedTokens,
    previous,
    explorerRequest,
    signal,
) {
    const listed = trackedTokens.filter(
        (token) => !isSpoofedSymbol(token.symbol, token.address),
    );
    // Each token's balanceOf answer, or null where it failed.
    const balances = await Promise.all(
        listed.map(async (token) => {
            try {
                const contract = new Contract(
                    token.address,
                    ERC20_ABI,
                    provider,
                );
                return await contract.balanceOf(address);
            } catch (e) {
                if (!signal?.aborted) {
                    log.errorf(
                        "token balance failed",
                        address,
                        token.address,
                        e.shortMessage || e.message,
                    );
                }
                return null;
            }
        }),
    );
    const explorerRows = await explorerRequest;
    const reported = explorerRows ?? previous;

    const rows = [];
    listed.forEach((token, i) => {
        const lower = token.address.toLowerCase();
        const entryIn = (list) =>
            list.find((row) => String(row.address).toLowerCase() === lower);
        const raw = balances[i];
        if (raw === null) {
            const kept = entryIn(previous) || entryIn(explorerRows || []);
            if (kept) rows.push(kept);
            return;
        }
        if (raw === 0n) return;
        const entry = entryIn(reported);
        const decimals = entry ? entry.decimals : null;
        const known = resolveTokenDecimals(lower, { trackedTokens });
        const scale = known !== null ? known : decimals;
        rows.push({
            address: token.address,
            name: token.name || "",
            symbol: token.symbol || "???",
            decimals: decimals,
            // null when nothing knows the scale: a holding whose amount
            // cannot be stated, never a zero.
            balance: scale === null ? null : formatTokenBalance(raw, scale),
            holders: entry ? entry.holders : null,
        });
    });
    return rows;
}

// Fetch ETH balances, ENS names, and ERC-20 token balances for all addresses.
// `trackedTokens` are the tokens tracked on the active network.
// `useBlockscout` and `signal` are as for fetchTokenBalances().
async function refreshBalances(
    wallets,
    rpcUrl,
    blockscoutUrl,
    useBlockscout,
    trackedTokens,
    networkId,
    signal,
) {
    log.debugf("refreshBalances start, rpc:", urlOrigin(rpcUrl));
    const provider = getProvider(rpcUrl, networkId);
    const tracked = trackedTokens || [];
    const trackedSet = new Set(tracked.map((t) => t.address.toLowerCase()));
    const updates = [];

    for (const wallet of wallets) {
        for (const addr of wallet.addresses) {
            // ETH balance
            updates.push(
                provider
                    .getBalance(addr.address)
                    .then((bal) => {
                        // Exact, never cut: a cut here stores a small nonzero
                        // balance as zero.
                        addr.balance = formatEther(bal);
                        log.debugf("ETH balance", addr.address, addr.balance);
                    })
                    .catch((e) => {
                        if (signal?.aborted) return;
                        log.errorf(
                            "ETH balance failed",
                            addr.address,
                            e.shortMessage || e.message,
                        );
                    }),
            );

            // ENS reverse lookup — only overwrite on success so that
            // transient RPC errors don't wipe a previously resolved name.
            updates.push(
                provider
                    .lookupAddress(addr.address)
                    .then((name) => {
                        addr.ensName = name || null;
                        log.debugf(
                            "ENS reverse",
                            addr.address,
                            "->",
                            addr.ensName,
                        );
                    })
                    .catch((e) => {
                        if (signal?.aborted) return;
                        log.errorf(
                            "ENS reverse failed",
                            addr.address,
                            e.shortMessage || e.message,
                        );
                        // Keep existing addr.ensName if we had one
                    }),
            );

            // ERC-20 token balances: the tracked tokens' through the RPC
            // endpoint, and from Blockscout the tokens the user has not
            // tracked. When Blockscout fails, the untracked rows from before
            // this refresh stay; while "Use Blockscout" is off there are none.
            // The Blockscout request is started first and handed to
            // fetchTrackedTokenBalances() too, which waits for it after
            // sending its balanceOf reads, so the two run side by side.
            const previous = addr.tokenBalances || [];
            const explorerRequest = fetchTokenBalances(
                addr.address,
                blockscoutUrl,
                useBlockscout,
                tracked,
                signal,
            );
            updates.push(
                Promise.all([
                    fetchTrackedTokenBalances(
                        provider,
                        addr.address,
                        tracked,
                        previous,
                        explorerRequest,
                        signal,
                    ),
                    explorerRequest,
                ]).then(([trackedRows, explorerRows]) => {
                    const untrackedRows = (explorerRows ?? previous).filter(
                        (row) =>
                            !trackedSet.has(String(row.address).toLowerCase()),
                    );
                    addr.tokenBalances = trackedRows.concat(untrackedRows);
                    log.debugf(
                        "Token balances",
                        addr.address,
                        addr.tokenBalances.length,
                        "tokens",
                    );
                }),
            );
        }
    }

    await Promise.all(updates);
    log.debugf("refreshBalances done");
}

// Look up token metadata from its contract.
// Calls symbol() and decimals() to verify it implements ERC-20.
// `signal` is as for fetchTokenBalances().
async function lookupTokenInfo(contractAddress, rpcUrl, networkId, signal) {
    log.debugf("lookupTokenInfo", contractAddress, "rpc:", urlOrigin(rpcUrl));
    const provider = getProvider(rpcUrl, networkId);
    const contract = new Contract(contractAddress, ERC20_ABI, provider);

    let name, symbol, decimals;
    try {
        symbol = await contract.symbol();
        log.debugf("symbol() =", symbol);
    } catch (e) {
        if (!signal?.aborted) {
            log.errorf("symbol() failed:", e.shortMessage || e.message);
        }
        throw new Error("Not a valid ERC-20 token (symbol() failed).");
    }

    try {
        decimals = await contract.decimals();
        log.debugf("decimals() =", decimals);
    } catch (e) {
        if (!signal?.aborted) {
            log.errorf("decimals() failed:", e.shortMessage || e.message);
        }
        throw new Error("Not a valid ERC-20 token (decimals() failed).");
    }

    try {
        name = await contract.name();
        log.debugf("name() =", name);
    } catch (e) {
        if (!signal?.aborted) {
            log.warnf(
                "name() failed, using symbol as name:",
                e.shortMessage || e.message,
            );
        }
        name = symbol;
    }

    // Truncate to prevent storage of excessively long values from RPC.
    // Counted in code points, as displaySymbol() counts them, so the cut never
    // falls between the two halves of an emoji and stores a half that renders
    // as U+FFFD.
    name = Array.from(String(name)).slice(0, 64).join("");
    symbol = Array.from(String(symbol)).slice(0, 12).join("");

    log.infof("Token resolved:", symbol, "decimals", Number(decimals));
    return { name, symbol, decimals: Number(decimals) };
}

// Derive HD addresses starting from index 0 and check for on-chain activity.
// Checks gapLimit addresses in parallel per batch.  Stops when an entire
// batch has no used addresses (i.e. gapLimit consecutive empty addresses).
// Returns { addresses: [{ address, index }], nextIndex }.
// `signal` is as for fetchTokenBalances().
async function scanForAddresses(xpub, rpcUrl, networkId, signal, gapLimit = 5) {
    log.debugf("scanForAddresses start, gapLimit:", gapLimit);
    const provider = getProvider(rpcUrl, networkId);
    const used = [];
    let checked = 0;
    let checkUpTo = gapLimit;

    while (checked < checkUpTo) {
        const batch = [];
        for (let i = checked; i < checkUpTo; i++) {
            const addr = deriveAddressFromXpub(xpub, i);
            batch.push({ addr, index: i });
        }

        const results = await Promise.all(
            batch.map(async ({ addr, index }) => {
                try {
                    const [balance, txCount] = await Promise.all([
                        provider.getBalance(addr),
                        provider.getTransactionCount(addr),
                    ]);
                    return { addr, index, isUsed: balance > 0n || txCount > 0 };
                } catch (e) {
                    if (!signal?.aborted) {
                        log.errorf(
                            "scanForAddresses check failed",
                            addr,
                            e.shortMessage || e.message,
                        );
                    }
                    return { addr, index, isUsed: false };
                }
            }),
        );

        checked = checkUpTo;

        for (const r of results) {
            if (r.isUsed) {
                used.push({ address: r.addr, index: r.index });
                log.debugf("scanForAddresses used", r.addr, "index:", r.index);
                checkUpTo = Math.max(checkUpTo, r.index + 1 + gapLimit);
            }
        }
    }

    used.sort((a, b) => a.index - b.index);
    const nextIndex = used.length > 0 ? used[used.length - 1].index + 1 : 1;
    log.infof(
        "scanForAddresses done, found:",
        used.length,
        "nextIndex:",
        nextIndex,
    );
    return { addresses: used, nextIndex };
}

module.exports = {
    fetchTokenBalances,
    refreshBalances,
    lookupTokenInfo,
    getProvider,
    scanForAddresses,
};