/** * Backfill cards.embedding from cards.image_url via Vercel AI Gateway. * * Idempotent: skips rows where embedded_at is set unless FORCE=1. * * Usage: * POSTGRES_URL= AI_GATEWAY_API_KEY= node scripts/backfill-card-embeddings.js * * Options (env): * BATCH_SIZE — rows per fetch (default 25) * SLEEP_MS — delay between embed calls (default 250) * FORCE — "1" to re-embed rows that already have embedded_at * LIMIT — max rows to process (default unlimited) * DRY_RUN — "true" to list candidates only */ import { sql } from '../lib/sql.js'; import { embedCardImage, formatEmbeddingForPg } from '../lib/card-embed.js'; if (!process.env.POSTGRES_URL) { console.error('POSTGRES_URL is required'); process.exit(1); } if (!process.env.AI_GATEWAY_API_KEY) { console.error('AI_GATEWAY_API_KEY is required'); process.exit(1); } const BATCH_SIZE = Number(process.env.BATCH_SIZE || 25); const SLEEP_MS = Number(process.env.SLEEP_MS || 250); const FORCE = process.env.FORCE === '1'; const LIMIT = process.env.LIMIT ? Number(process.env.LIMIT) : null; const DRY_RUN = process.env.DRY_RUN === 'true'; function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } async function fetchCandidates(lastId) { if (FORCE) { const { rows } = await sql` SELECT id, name, image_url FROM cards WHERE id > ${lastId} AND image_url IS NOT NULL AND image_url <> '' ORDER BY id LIMIT ${BATCH_SIZE} `; return rows; } const { rows } = await sql` SELECT id, name, image_url FROM cards WHERE id > ${lastId} AND image_url IS NOT NULL AND image_url <> '' AND embedding IS NULL ORDER BY id LIMIT ${BATCH_SIZE} `; return rows; } async function main() { let lastId = 0; let processed = 0; let updated = 0; console.log(`Backfill starting (force=${FORCE}, dryRun=${DRY_RUN})`); while (true) { if (LIMIT != null && processed >= LIMIT) break; const rows = await fetchCandidates(lastId); if (!rows.length) break; for (const row of rows) { if (LIMIT != null && processed >= LIMIT) break; processed++; lastId = row.id; if (DRY_RUN) { console.log(`[dry-run] would embed card ${row.id}: ${row.name}`); continue; } try { const embedding = await embedCardImage({ imageUrl: row.image_url }); const vectorLiteral = formatEmbeddingForPg(embedding); await sql` UPDATE cards SET embedding = ${vectorLiteral}::vector, embedded_at = CURRENT_TIMESTAMP WHERE id = ${row.id} `; updated++; console.log(`Embedded card ${row.id}: ${row.name}`); } catch (error) { console.error(`Failed card ${row.id} (${row.name}):`, error.message); } if (SLEEP_MS > 0) { await sleep(SLEEP_MS); } } } console.log(`Done. processed=${processed} updated=${updated}`); } main().catch((error) => { console.error(error); process.exit(1); });