deckhearth/scripts/backfill-card-embeddings.js

/**
 * Backfill cards.embedding from cards.image_url via Vercel AI Gateway.
 *
 * Idempotent: skips rows where embedded_at is set unless FORCE=1.
 *
 * Usage:
 *   POSTGRES_URL=<url> AI_GATEWAY_API_KEY=<key> node scripts/backfill-card-embeddings.js
 *
 * Options (env):
 *   BATCH_SIZE   — rows per fetch (default 25)
 *   SLEEP_MS     — delay between embed calls (default 250)
 *   FORCE        — "1" to re-embed rows that already have embedded_at
 *   LIMIT        — max rows to process (default unlimited)
 *   DRY_RUN      — "true" to list candidates only
 */

import { neon } from '@neondatabase/serverless';

import { embedCardImage, formatEmbeddingForPg } from '../lib/card-embed.js';

if (!process.env.POSTGRES_URL) {
  console.error('POSTGRES_URL is required');
  process.exit(1);
}

if (!process.env.AI_GATEWAY_API_KEY) {
  console.error('AI_GATEWAY_API_KEY is required');
  process.exit(1);
}

const sql = neon(process.env.POSTGRES_URL, { fullResults: false });
const BATCH_SIZE = Number(process.env.BATCH_SIZE || 25);
const SLEEP_MS = Number(process.env.SLEEP_MS || 250);
const FORCE = process.env.FORCE === '1';
const LIMIT = process.env.LIMIT ? Number(process.env.LIMIT) : null;
const DRY_RUN = process.env.DRY_RUN === 'true';

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function fetchCandidates(lastId) {
  if (FORCE) {
    return sql`
      SELECT id, name, image_url
      FROM cards
      WHERE id > ${lastId}
        AND image_url IS NOT NULL
        AND image_url <> ''
      ORDER BY id
      LIMIT ${BATCH_SIZE}
    `;
  }

  return sql`
    SELECT id, name, image_url
    FROM cards
    WHERE id > ${lastId}
      AND image_url IS NOT NULL
      AND image_url <> ''
      AND embedding IS NULL
    ORDER BY id
    LIMIT ${BATCH_SIZE}
  `;
}

async function main() {
  let lastId = 0;
  let processed = 0;
  let updated = 0;

  console.log(`Backfill starting (force=${FORCE}, dryRun=${DRY_RUN})`);

  while (true) {
    if (LIMIT != null && processed >= LIMIT) break;

    const rows = await fetchCandidates(lastId);
    if (!rows.length) break;

    for (const row of rows) {
      if (LIMIT != null && processed >= LIMIT) break;
      processed++;
      lastId = row.id;

      if (DRY_RUN) {
        console.log(`[dry-run] would embed card ${row.id}: ${row.name}`);
        continue;
      }

      try {
        const embedding = await embedCardImage({ imageUrl: row.image_url });
        const vectorLiteral = formatEmbeddingForPg(embedding);
        await sql`
          UPDATE cards
          SET embedding = ${vectorLiteral}::vector,
              embedded_at = CURRENT_TIMESTAMP
          WHERE id = ${row.id}
        `;
        updated++;
        console.log(`Embedded card ${row.id}: ${row.name}`);
      } catch (error) {
        console.error(`Failed card ${row.id} (${row.name}):`, error.message);
      }

      if (SLEEP_MS > 0) {
        await sleep(SLEEP_MS);
      }
    }
  }

  console.log(`Done. processed=${processed} updated=${updated}`);
}

main().catch((error) => {
  console.error(error);
  process.exit(1);
});