deckhearth/lib/card-import/bulk-sync.js

/**
 * Bulk catalog sync using Scryfall's bulk-data endpoint.
 *
 * Downloads the Oracle Cards file (~168 MB) and upserts all MTG cards in one pass.
 * Much faster and more complete than the per-set discovery approach — catches all
 * missing sets in one shot without rate-limit concerns.
 *
 * Also refreshes prices, legalities, and edhrec_rank on existing cards.
 */

import { sql } from '@vercel/postgres';

const BATCH_SIZE = 100;
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

async function fetchBulkDownloadUrl() {
  const response = await fetch('https://api.scryfall.com/bulk-data/oracle_cards', {
    headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
  });
  if (!response.ok) {
    throw new Error(`Scryfall bulk-data API error: ${response.status}`);
  }
  return response.json();
}

function mapCard(card) {
  let currentPrice = null;
  let marketPrice = null;
  if (card.prices) {
    currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null;
    marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null;
  }

  return {
    scryfallId: card.id,
    oracleId: card.oracle_id || null,
    illustrationId: card.illustration_id || null,
    name: card.name,
    setName: card.set_name,
    setCode: card.set,
    cardNumber: card.collector_number,
    rarity: card.rarity,
    manaCost: card.mana_cost || null,
    cmc: card.cmc != null ? Math.round(card.cmc) : null,
    cardType: card.type_line || null,
    colors: JSON.stringify(card.colors || []),
    colorIdentity: JSON.stringify(card.color_identity || []),
    oracleText: card.oracle_text || null,
    power: card.power || null,
    toughness: card.toughness || null,
    imageUrl: card.image_uris?.normal || null,
    stockImageUrl: card.image_uris?.art_crop || null,
    currentPrice,
    marketPrice,
    keywords: JSON.stringify(card.keywords || []),
    legalities: JSON.stringify(card.legalities || {}),
    flavorText: card.flavor_text || null,
    artist: card.artist || null,
    releasedAt: card.released_at || null,
    layout: card.layout || null,
    edhrecRank: card.edhrec_rank || null,
    reserved: card.reserved || false,
    reprint: card.reprint || false,
    finishes: JSON.stringify(card.finishes || []),
  };
}

async function upsertBatch(cards) {
  if (cards.length === 0) return 0;

  const columns = [
    'scryfall_id', 'oracle_id', 'illustration_id', 'name', 'set_name', 'set_code',
    'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors',
    'color_identity', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url',
    'current_price', 'market_price', 'keywords', 'legalities', 'flavor_text', 'artist',
    'released_at', 'layout', 'edhrec_rank', 'reserved', 'reprint', 'finishes', 'verified',
  ];

  const values = [];
  const placeholders = [];
  let paramIdx = 1;

  for (const row of cards) {
    const rowValues = [
      row.scryfallId, row.oracleId, row.illustrationId, row.name, row.setName, row.setCode,
      row.cardNumber, row.rarity, 'MTG', row.manaCost, row.cmc, row.cardType, row.colors,
      row.colorIdentity, row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl,
      row.currentPrice, row.marketPrice, row.keywords, row.legalities, row.flavorText, row.artist,
      row.releasedAt, row.layout, row.edhrecRank, row.reserved, row.reprint, row.finishes, true,
    ];
    const rowP = rowValues.map(() => `$${paramIdx++}`);
    placeholders.push(`(${rowP.join(',')})`);
    values.push(...rowValues);
  }

  const updateCols = columns
    .filter((c) => c !== 'scryfall_id')
    .map((c) => `${c} = EXCLUDED.${c}`)
    .join(', ');

  await sql.query(
    `INSERT INTO cards (${columns.join(', ')})
     VALUES ${placeholders.join(',\n')}
     ON CONFLICT (scryfall_id) DO UPDATE SET ${updateCols}, updated_at = CURRENT_TIMESTAMP`,
    values
  );

  return cards.length;
}

/**
 * Run a full bulk MTG sync. Returns a summary compatible with the catalog sync API shape.
 */
export async function runBulkMtgSync() {
  const meta = await fetchBulkDownloadUrl();
  console.log(`[bulk-sync] Oracle Cards updated: ${meta.updated_at}, size: ${(meta.size / 1024 / 1024).toFixed(0)} MB`);

  const response = await fetch(meta.download_uri, {
    headers: { 'User-Agent': 'DeckHearth/1.0' },
  });
  if (!response.ok) {
    throw new Error(`Bulk download failed: ${response.status}`);
  }

  const allCards = await response.json();
  const mtgCards = allCards.filter((c) => c.lang === 'en' && !c.digital);

  let upserted = 0;
  let errors = 0;
  let batch = [];

  for (const card of mtgCards) {
    batch.push(mapCard(card));
    if (batch.length >= BATCH_SIZE) {
      try {
        upserted += await upsertBatch(batch);
      } catch (err) {
        console.error('[bulk-sync] Batch error:', err.message);
        errors += batch.length;
      }
      batch = [];
    }
  }

  if (batch.length > 0) {
    try {
      upserted += await upsertBatch(batch);
    } catch (err) {
      console.error('[bulk-sync] Final batch error:', err.message);
      errors += batch.length;
    }
  }

  // Backfill oracle_id on older per-set imported cards
  const backfill = await sql`
    UPDATE cards target
    SET oracle_id = source.oracle_id,
        color_identity = COALESCE(target.color_identity, source.color_identity),
        keywords = COALESCE(target.keywords, source.keywords),
        legalities = COALESCE(target.legalities, source.legalities),
        layout = COALESCE(target.layout, source.layout),
        edhrec_rank = COALESCE(target.edhrec_rank, source.edhrec_rank)
    FROM (
      SELECT DISTINCT ON (name) name, oracle_id, color_identity, keywords, legalities, layout, edhrec_rank
      FROM cards WHERE oracle_id IS NOT NULL AND game = 'MTG'
    ) source
    WHERE target.name = source.name AND target.game = 'MTG' AND target.oracle_id IS NULL
  `;

  const summary = {
    mode: 'bulk',
    totalInFile: mtgCards.length,
    upserted,
    errors,
    backfilledOracleId: backfill.rowCount || 0,
    updatedAt: meta.updated_at,
  };

  console.log('[bulk-sync]', JSON.stringify(summary));
  return summary;
}