/** * Runs one provider over the benchmark set and writes JSON lines. * * node scripts/benchmarks/scrape-content/run.mts novagentum * FIRECRAWL_API_KEY=... node scripts/benchmarks/scrape-content/run.mts firecrawl * * Both providers use their default Scrape settings and return Markdown of the * main content. Novagentum runs the production scrapePage() with plain HTML * fetching (a free Scrape: no JavaScript rendering, which paid Scrapes add). * Rows already in are skipped, so a run can be resumed. */ import { appendFileSync, existsSync, readFileSync } from "node:fs"; import { fetchPublicWebsite } from "../../../src/lib/public-website.ts"; import { scrapePage } from "../../../src/lib/scrape.ts"; import { benchmarkRows, type DatasetRow, loadDataset } from "./dataset.mts"; import type { ScrapeResult } from "./score.mts"; const [provider, datasetPath, outPath] = process.argv.slice(2); const CONCURRENCY = Number(process.env.CONCURRENCY ?? 8); const TIMEOUT_MS = 60_000; async function novagentum(row: DatasetRow) { const result = await scrapePage( { url: row.url }, { fetchWebsite: fetchPublicWebsite }, ); return { ok: result.fetched && Boolean(result.content?.markdown), markdown: result.content?.markdown ?? "", error: result.error?.code, rateLimited: false, }; } /** Plan rate limits are not scrape failures: wait and retry on 429. */ async function firecrawl(row: DatasetRow) { for (let attempt = 0; ; attempt++) { const result = await firecrawlOnce(row); if (!result.rateLimited || attempt >= 10) return result; await new Promise((resolve) => setTimeout(resolve, 7_000)); } } async function firecrawlOnce(row: DatasetRow) { const started = performance.now(); const response = await fetch("https://api.firecrawl.dev/v2/scrape", { method: "POST", headers: { Authorization: `Bearer ${process.env.FIRECRAWL_API_KEY ?? ""}`, "Content-Type": "application/json", }, body: JSON.stringify({ url: row.url, formats: ["markdown"] }), signal: AbortSignal.timeout(TIMEOUT_MS), }); const body = (await response.json().catch(() => ({}))) as { success?: boolean; data?: { markdown?: string }; error?: string; }; return { ok: response.ok && Boolean(body.success && body.data?.markdown), markdown: body.data?.markdown ?? "", error: response.ok ? body.error : `${response.status} ${body.error ?? ""}`, rateLimited: response.status === 429, // Time of this request only, so rate-limit waits are not counted. ms: Math.round(performance.now() - started), }; } const providers = { novagentum, firecrawl } as const; const run = providers[provider as keyof typeof providers]; if (!run || !datasetPath || !outPath) throw new Error( "Usage: run.mts ", ); const done = new Set( existsSync(outPath) ? readFileSync(outPath, "utf8") .split("\n") .filter(Boolean) .map((line) => (JSON.parse(line) as ScrapeResult).id) : [], ); const queue = benchmarkRows(loadDataset(datasetPath)) .filter((row) => !done.has(row.id)) .slice(0, Number(process.env.LIMIT ?? Number.POSITIVE_INFINITY)); console.error(`${provider}: ${queue.length} pages to scrape`); let finished = 0; async function worker() { for (let row = queue.shift(); row; row = queue.shift()) { const started = performance.now(); let result: Omit & { ms?: number }; try { let timer: ReturnType | undefined; try { result = await Promise.race([ run(row).then(({ rateLimited: _, ...rest }) => rest), new Promise((_, reject) => { timer = setTimeout( () => reject(new Error("timeout")), provider === "firecrawl" ? TIMEOUT_MS * 12 : TIMEOUT_MS, ); }), ]); } finally { clearTimeout(timer); } } catch (error) { result = { ok: false, markdown: "", error: error instanceof Error ? error.message : String(error), }; } const line: ScrapeResult = { id: row.id, url: row.url, ms: Math.round(performance.now() - started), ...result, }; appendFileSync(outPath, `${JSON.stringify(line)}\n`); if (++finished % 50 === 0) console.error(`${provider}: ${finished} done`); } } await Promise.all(Array.from({ length: CONCURRENCY }, worker)); console.error(`${provider}: finished ${finished}`);