import { readFileSync } from "node:fs"; /** One row of firecrawl/scrape-content-dataset-v1 (MIT licence). */ export type DatasetRow = { id: string; url: string; truth_text: string; lie_text: string; error: string; }; /** Minimal RFC 4180 parser: quoted fields, doubled quotes, newlines in quotes. */ export function parseCsv(text: string): string[][] { const rows: string[][] = []; let row: string[] = []; let field = ""; let quoted = false; for (let i = 0; i < text.length; i++) { const char = text[i]; if (quoted) { if (char === '"' && text[i + 1] === '"') { field += '"'; i++; } else if (char === '"') quoted = false; else field += char; } else if (char === '"') quoted = true; else if (char === ",") { row.push(field); field = ""; } else if (char === "\n" || char === "\r") { if (char === "\r" && text[i + 1] === "\n") i++; row.push(field); rows.push(row); row = []; field = ""; } else field += char; } if (field || row.length) { row.push(field); rows.push(row); } return rows.filter((r) => r.some((cell) => cell !== "")); } export function loadDataset(path: string): DatasetRow[] { const [header, ...rows] = parseCsv(readFileSync(path, "utf8")); return rows.map( (cells) => Object.fromEntries( header.map((name, i) => [name, cells[i] ?? ""]), ) as DatasetRow, ); } /** * The benchmark set: rows the dataset authors fetched without error, that * have reference text, and that are HTML pages. PDFs are left out because a * free Scrape does not convert PDFs and Firecrawl bills PDFs per page. */ export function benchmarkRows(rows: DatasetRow[]): DatasetRow[] { return rows.filter( (row) => !row.error.trim() && row.truth_text.trim() && !/\.pdf(?:$|[?#])/i.test(row.url), ); }