deckhearth/lib/card-import/bulk-sync.js

/**
 * Bulk catalog sync using Scryfall's bulk-data endpoint.
 *
 * Downloads the Oracle Cards file (~168 MB) and upserts all MTG cards in one pass.
 * Much faster and more complete than the per-set discovery approach — catches all
 * missing sets in one shot without rate-limit concerns.
 *
 * Also refreshes prices, legalities, and edhrec_rank on existing cards.
 */

import { gunzipSync } from 'node:zlib';

import { sql } from '../sql.js';
import { recordPriceSnapshots } from '../pricing/price-history.js';

const BATCH_SIZE = 100;
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

export function resolveBulkDownloadUri(meta) {
  return meta?.download_uri ?? meta?.jsonl_download_uri ?? null;
}

async function fetchBulkDownloadUrl() {
  const response = await fetch('https://api.scryfall.com/bulk-data/oracle_cards', {
    headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
  });
  if (!response.ok) {
    throw new Error(`Scryfall bulk-data API error: ${response.status}`);
  }
  return response.json();
}

/**
 * Download and parse Scryfall oracle bulk (JSON array or JSONL, optionally gzip).
 */
export async function downloadBulkOracleCards(meta) {
  const downloadUri = resolveBulkDownloadUri(meta);
  if (!downloadUri) {
    throw new Error('Scryfall bulk meta missing download_uri and jsonl_download_uri');
  }

  const response = await fetch(downloadUri, {
    headers: { 'User-Agent': 'DeckHearth/1.0' },
  });
  if (!response.ok) {
    throw new Error(`Bulk download failed: ${response.status}`);
  }

  const raw = Buffer.from(await response.arrayBuffer());
  const text = downloadUri.endsWith('.gz')
    ? gunzipSync(raw).toString('utf8')
    : raw.toString('utf8');

  if (downloadUri.includes('.jsonl')) {
    return text
      .split('\n')
      .map((line) => line.trim())
      .filter(Boolean)
      .map((line) => JSON.parse(line));
  }

  return JSON.parse(text);
}

export function mapCard(card) {
  let currentPrice = null;
  let marketPrice = null;
  if (card.prices) {
    currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null;
    marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null;
  }

  return {
    scryfallId: card.id,
    oracleId: card.oracle_id || null,
    illustrationId: card.illustration_id || null,
    name: card.name,
    setName: card.set_name,
    setCode: card.set,
    cardNumber: card.collector_number,
    rarity: card.rarity,
    manaCost: card.mana_cost || null,
    cmc: card.cmc != null ? Math.round(card.cmc) : null,
    cardType: card.type_line || null,
    colors: JSON.stringify(card.colors || []),
    colorIdentity: JSON.stringify(card.color_identity || []),
    oracleText: card.oracle_text || null,
    power: card.power || null,
    toughness: card.toughness || null,
    imageUrl: card.image_uris?.normal || null,
    stockImageUrl: card.image_uris?.art_crop || null,
    currentPrice,
    marketPrice,
    keywords: JSON.stringify(card.keywords || []),
    legalities: JSON.stringify(card.legalities || {}),
    flavorText: card.flavor_text || null,
    artist: card.artist || null,
    releasedAt: card.released_at || null,
    layout: card.layout || null,
    edhrecRank: card.edhrec_rank || null,
    reserved: card.reserved || false,
    reprint: card.reprint || false,
    finishes: JSON.stringify(card.finishes || []),
  };
}

function hasPrice(row) {
  return row.currentPrice != null || row.marketPrice != null;
}

async function resolveCardIds(scryfallIds) {
  if (scryfallIds.length === 0) return new Map();

  const result = await sql.query(
    'SELECT id, scryfall_id FROM cards WHERE scryfall_id = ANY($1)',
    [scryfallIds]
  );

  return new Map(result.rows.map((row) => [row.scryfall_id, row.id]));
}

async function recordBatchSnapshots(cards) {
  const pricedCards = cards.filter(hasPrice);
  if (pricedCards.length === 0) return 0;

  const idByScryfall = await resolveCardIds(pricedCards.map((row) => row.scryfallId));
  const snapshots = [];

  for (const row of pricedCards) {
    const cardId = idByScryfall.get(row.scryfallId);
    if (cardId == null) continue;

    if (row.currentPrice != null) {
      snapshots.push({
        cardId,
        source: 'scryfall',
        priceType: 'market',
        currency: 'USD',
        amount: row.currentPrice,
      });
    }

    if (row.marketPrice != null) {
      snapshots.push({
        cardId,
        source: 'scryfall',
        priceType: 'foil',
        currency: 'USD',
        amount: row.marketPrice,
      });
    }
  }

  return recordPriceSnapshots(snapshots);
}

async function upsertBatch(cards) {
  if (cards.length === 0) return { upserted: 0, snapshotsWritten: 0 };

  const columns = [
    'scryfall_id', 'oracle_id', 'illustration_id', 'name', 'set_name', 'set_code',
    'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors',
    'color_identity', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url',
    'current_price', 'market_price', 'prices_updated_at', 'keywords', 'legalities', 'flavor_text', 'artist',
    'released_at', 'layout', 'edhrec_rank', 'reserved', 'reprint', 'finishes', 'verified',
  ];

  const values = [];
  const placeholders = [];
  let paramIdx = 1;

  for (const row of cards) {
    const pricesUpdatedAt = hasPrice(row) ? new Date() : null;
    const rowValues = [
      row.scryfallId, row.oracleId, row.illustrationId, row.name, row.setName, row.setCode,
      row.cardNumber, row.rarity, 'MTG', row.manaCost, row.cmc, row.cardType, row.colors,
      row.colorIdentity, row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl,
      row.currentPrice, row.marketPrice, pricesUpdatedAt, row.keywords, row.legalities, row.flavorText, row.artist,
      row.releasedAt, row.layout, row.edhrecRank, row.reserved, row.reprint, row.finishes, true,
    ];
    const rowP = rowValues.map(() => `$${paramIdx++}`);
    placeholders.push(`(${rowP.join(',')})`);
    values.push(...rowValues);
  }

  const updateCols = columns
    .filter((c) => c !== 'scryfall_id')
    .map((c) => `${c} = EXCLUDED.${c}`)
    .join(', ');

  await sql.query(
    `INSERT INTO cards (${columns.join(', ')})
     VALUES ${placeholders.join(',\n')}
     ON CONFLICT (scryfall_id) DO UPDATE SET ${updateCols}, updated_at = CURRENT_TIMESTAMP`,
    values
  );

  const snapshotsWritten = await recordBatchSnapshots(cards);

  return { upserted: cards.length, snapshotsWritten };
}

/**
 * Run a full bulk MTG sync. Returns a summary compatible with the catalog sync API shape.
 */
export async function runBulkMtgSync() {
  const meta = await fetchBulkDownloadUrl();
  const sizeMb = ((meta.compressed_size ?? meta.size ?? 0) / 1024 / 1024).toFixed(0);
  console.log(`[bulk-sync] Oracle Cards updated: ${meta.updated_at}, size: ~${sizeMb} MB`);

  const allCards = await downloadBulkOracleCards(meta);
  const mtgCards = allCards.filter((c) => c.lang === 'en' && !c.digital);

  let upserted = 0;
  let snapshotsWritten = 0;
  let errors = 0;
  let batch = [];

  for (const card of mtgCards) {
    batch.push(mapCard(card));
    if (batch.length >= BATCH_SIZE) {
      try {
        const result = await upsertBatch(batch);
        upserted += result.upserted;
        snapshotsWritten += result.snapshotsWritten;
      } catch (err) {
        console.error('[bulk-sync] Batch error:', err.message);
        errors += batch.length;
      }
      batch = [];
    }
  }

  if (batch.length > 0) {
    try {
      const result = await upsertBatch(batch);
      upserted += result.upserted;
      snapshotsWritten += result.snapshotsWritten;
    } catch (err) {
      console.error('[bulk-sync] Final batch error:', err.message);
      errors += batch.length;
    }
  }

  // Backfill oracle_id on older per-set imported cards
  const backfill = await sql`
    UPDATE cards target
    SET oracle_id = source.oracle_id,
        color_identity = COALESCE(target.color_identity, source.color_identity),
        keywords = COALESCE(target.keywords, source.keywords),
        legalities = COALESCE(target.legalities, source.legalities),
        layout = COALESCE(target.layout, source.layout),
        edhrec_rank = COALESCE(target.edhrec_rank, source.edhrec_rank)
    FROM (
      SELECT DISTINCT ON (name) name, oracle_id, color_identity, keywords, legalities, layout, edhrec_rank
      FROM cards WHERE oracle_id IS NOT NULL AND game = 'MTG'
    ) source
    WHERE target.name = source.name AND target.game = 'MTG' AND target.oracle_id IS NULL
  `;

  const summary = {
    mode: 'bulk',
    totalInFile: mtgCards.length,
    upserted,
    snapshotsWritten,
    errors,
    backfilledOracleId: backfill.rowCount || 0,
    updatedAt: meta.updated_at,
  };

  console.log('[bulk-sync]', JSON.stringify(summary));
  return summary;
}