/** * Scores scrape outputs against firecrawl/scrape-content-dataset-v1. * * node scripts/benchmarks/scrape-content/score.mts [more.jsonl ...] * * Every provider is scored by this same code. Text is compared as tokens: * words for most scripts, single characters for Chinese, Japanese and Korean. * - content found: share of the reference main-content tokens present in the * output (multiset recall, like ROUGE-1 recall). * - coverage: content found >= 10%, Firecrawl's published definition. * - clutter found: share of the reference clutter tokens (navigation, footer, * ads) present in the output. Lower is better. * A failed or empty scrape scores 0 for content and clutter. */ import { readFileSync, writeFileSync } from "node:fs"; import { benchmarkRows, loadDataset } from "./dataset.mts"; export type ScrapeResult = { id: string; url: string; ok: boolean; markdown: string; ms: number; error?: string; }; const TOKEN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]|[\p{L}\p{N}]+/gu; export function tokens(text: string): string[] { return ( text .normalize("NFKC") .toLowerCase() // Markdown links and images keep their visible text only. .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") .match(TOKEN) ?? [] ); } /** Share of `reference` tokens found in `output`, counting repeats. */ export function recall(reference: string, output: string): number | null { const want = tokens(reference); if (!want.length) return null; const have = new Map(); for (const token of tokens(output)) have.set(token, (have.get(token) ?? 0) + 1); let found = 0; for (const token of want) { const count = have.get(token) ?? 0; if (count > 0) { found++; have.set(token, count - 1); } } return found / want.length; } const mean = (values: number[]) => values.length ? values.reduce((a, b) => a + b, 0) / values.length : 0; const percentile = (values: number[], p: number) => { if (!values.length) return 0; const sorted = [...values].sort((a, b) => a - b); return sorted[Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1)]; }; export function score(datasetPath: string, resultsPath: string) { const rows = benchmarkRows(loadDataset(datasetPath)); const results = new Map( readFileSync(resultsPath, "utf8") .split("\n") .filter(Boolean) .map((line) => JSON.parse(line) as ScrapeResult) .map((result) => [result.id, result]), ); const perUrl = rows.map((row) => { const result = results.get(row.id); const output = result?.ok ? result.markdown : ""; const content = recall(row.truth_text, output) ?? 0; const clutter = recall(row.lie_text, output); return { id: row.id, url: row.url, ran: Boolean(result), ok: Boolean(result?.ok && output.trim()), content, covered: content >= 0.1, clutter, ms: result?.ms ?? null, error: result?.error ?? "", }; }); const ran = perUrl.filter((row) => row.ran); const clutterRows = ran.filter((row) => row.clutter !== null); const latencies = ran .filter((row) => row.ok && row.ms !== null) .map((row) => row.ms as number); return { pages: rows.length, ran: ran.length, succeeded: ran.filter((row) => row.ok).length, coverage: ran.filter((row) => row.covered).length / (ran.length || 1), contentFound: mean(ran.map((row) => row.content)), clutterFound: mean(clutterRows.map((row) => row.clutter as number)), latencyP50Ms: percentile(latencies, 0.5), latencyP95Ms: percentile(latencies, 0.95), perUrl, }; } if (import.meta.url === `file://${process.argv[1]}`) { const [datasetPath, ...resultPaths] = process.argv.slice(2); for (const resultsPath of resultPaths) { const { perUrl, ...summary } = score(datasetPath, resultsPath); writeFileSync( resultsPath.replace(/\.jsonl$/, ".scores.json"), `${JSON.stringify({ summary, perUrl }, null, 2)}\n`, ); console.log(resultsPath, JSON.stringify(summary)); } }