deckhearth/lib/card-import/lorcana-bulk.js

/**
 * Bulk catalog sync for Disney Lorcana via lorcana-api.com /bulk/cards.
 *
 * Single fetch returns all cards (~2.3k). Upserts keyed on scryfall_id (stores Unique_ID).
 * Enriches prices from Foilio hourly feed when available.
 * Pricing: also fetches individually via api.lorcast.com/v0/cards/search (free, 10 req/sec rate limit).
 */

import { sql as vercelSql } from '../sql.js';
import { recordPriceSnapshots } from '../pricing/price-history.js';
import { fetchFolioPriceIndex, matchFolioPrice } from '../pricing/foolio-client.js';

const BATCH_SIZE = 100;
const BULK_URL = 'https://api.lorcana-api.com/bulk/cards';
const LORCAST_SEARCH_URL = 'https://api.lorcast.com/v0/cards/search';
const LORCAST_RATE_LIMIT_MS = 110; // 10 req/sec, per lorcast.com/docs

function hasPrice(row) {
  return row.currentPrice != null || row.marketPrice != null;
}

/**
 * Map a Lorcana bulk API card object to cards-table insert shape.
 */
export function mapLorcanaCard(card, pricing = null) {
  const colors = card.Color
    ? card.Color.split(',').map((entry) => entry.trim()).filter(Boolean)
    : [];

  const keywords = [];
  if (card.Classifications) {
    keywords.push(
      ...card.Classifications.split(',').map((entry) => entry.trim()).filter(Boolean)
    );
  }
  if (card.Abilities) {
    keywords.push(...card.Abilities.split(',').map((entry) => entry.trim()).filter(Boolean));
  }
  if (card.Inkable === true) {
    keywords.push('Inkable');
  }

  const cost = card.Cost != null ? Number(card.Cost) : null;

  const row = {
    scryfallId: card.Unique_ID,
    name: card.Name,
    setName: card.Set_Name,
    setCode: card.Set_ID,
    cardNumber: card.Card_Num != null ? String(card.Card_Num) : null,
    rarity: card.Rarity || null,
    manaCost: cost != null ? String(cost) : null,
    cmc: cost,
    cardType: card.Type || null,
    colors: JSON.stringify(colors),
    colorIdentity: JSON.stringify(colors),
    oracleText: card.Body_Text || null,
    power: card.Strength != null ? String(card.Strength) : null,
    toughness: card.Willpower != null ? String(card.Willpower) : null,
    imageUrl: card.Image || null,
    stockImageUrl: card.Image || null,
    flavorText: card.Flavor_Text || null,
    artist: card.Artist || null,
    edhrecRank: card.Lore != null ? Number(card.Lore) : null,
    keywords: JSON.stringify(keywords),
    tcgplayerId: card.tcgplayer_id || null,
    currentPrice: card.prices?.usd != null ? parseFloat(card.prices.usd) : null,
    marketPrice: card.prices?.usd_foil != null ? parseFloat(card.prices.usd_foil) : null,
  };

  if (pricing) {
    row.currentPrice = pricing.currentPrice;
    row.marketPrice = pricing.marketPrice;
    row.priceCurrency = pricing.currency ?? 'USD';
    if (hasPrice(row)) {
      row.pricesUpdatedAt = new Date();
    }
  }

  return row;
}

export function enrichLorcanaRowWithFolio(row, folioIndex, stats) {
  const pricing = matchFolioPrice(folioIndex, row);
  if (!pricing || !hasPrice(pricing)) {
    if (stats) stats.unmatchedCount += 1;
    return row;
  }

  if (stats) stats.pricesMatched += 1;
  return {
    ...row,
    currentPrice: pricing.currentPrice,
    marketPrice: pricing.marketPrice,
    priceCurrency: pricing.currency,
    pricesUpdatedAt: new Date(),
  };
}

export function buildUpsertQuery(batch) {
  const columns = [
    'scryfall_id', 'name', 'set_name', 'set_code', 'card_number', 'rarity', 'game',
    'mana_cost', 'cmc', 'card_type', 'colors', 'color_identity', 'oracle_text',
    'power', 'toughness', 'image_url', 'stock_image_url', 'current_price', 'market_price',
    'prices_updated_at', 'keywords', 'flavor_text', 'artist', 'edhrec_rank', 'verified',
  ];

  const placeholders = [];
  const values = [];
  let paramIdx = 1;

  for (const row of batch) {
    const rowPlaceholders = [];
    const rowValues = [
      row.scryfallId, row.name, row.setName, row.setCode, row.cardNumber, row.rarity,
      'Lorcana', row.manaCost, row.cmc, row.cardType, row.colors, row.colorIdentity,
      row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl,
      row.currentPrice ?? null, row.marketPrice ?? null, row.pricesUpdatedAt ?? null,
      row.keywords, row.flavorText, row.artist, row.edhrecRank, true,
    ];

    for (let i = 0; i < rowValues.length; i += 1) {
      rowPlaceholders.push(`$${paramIdx}`);
      paramIdx += 1;
    }
    placeholders.push(`(${rowPlaceholders.join(', ')})`);
    values.push(...rowValues);
  }

  const updateCols = columns
    .filter((col) => col !== 'scryfall_id')
    .map((col) => `${col} = EXCLUDED.${col}`)
    .join(', ');

  const query = `
    INSERT INTO cards (${columns.join(', ')})
    VALUES ${placeholders.join(',\n')}
    ON CONFLICT (scryfall_id) DO UPDATE SET
      ${updateCols},
      updated_at = CURRENT_TIMESTAMP
  `;

  return { query, values };
}

async function resolveCardIds(sql, scryfallIds) {
  if (scryfallIds.length === 0) return new Map();

  const result = await sql.query(
    'SELECT id, scryfall_id FROM cards WHERE scryfall_id = ANY($1)',
    [scryfallIds]
  );

  return new Map(result.rows.map((row) => [row.scryfall_id, row.id]));
}

async function recordBatchSnapshots(sql, batch) {
  const pricedRows = batch.filter(hasPrice);
  if (pricedRows.length === 0) return 0;

  const idByScryfall = await resolveCardIds(sql, pricedRows.map((row) => row.scryfallId));
  const snapshots = [];

  for (const row of pricedRows) {
    const cardId = idByScryfall.get(row.scryfallId);
    if (cardId == null) continue;

    const currency = row.priceCurrency ?? 'USD';

    if (row.currentPrice != null) {
      snapshots.push({
        cardId,
        source: 'foilio',
        priceType: 'market',
        currency,
        amount: row.currentPrice,
      });
    }

    if (row.marketPrice != null && row.marketPrice !== row.currentPrice) {
      snapshots.push({
        cardId,
        source: 'foilio',
        priceType: 'market',
        currency,
        amount: row.marketPrice,
      });
    } else if (row.marketPrice != null && row.currentPrice == null) {
      snapshots.push({
        cardId,
        source: 'foilio',
        priceType: 'market',
        currency,
        amount: row.marketPrice,
      });
    }
  }

  return recordPriceSnapshots(snapshots);
}

async function upsertBatch(sql, batch) {
  if (batch.length === 0) return { upserted: 0, snapshotsWritten: 0 };

  const { query, values } = buildUpsertQuery(batch);
  await sql.query(query, values);
  const snapshotsWritten = await recordBatchSnapshots(sql, batch);

  return { upserted: batch.length, snapshotsWritten };
}

export async function fetchLorcanaBulkCards() {
  const response = await fetch(BULK_URL, {
    headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
  });
  if (!response.ok) {
    throw new Error(`Lorcana bulk API error: ${response.status}`);
  }
  return response.json();
}

const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

/**
 * Fetch pricing for a card via Lorcast search API.
 * Returns { currentPrice, marketPrice, tcgplayerId, imageUrl, stockImageUrl } or null.
 */
export async function fetchLorcastPricing(name, setCode) {
  const query = encodeURIComponent(`${name} set:${setCode}`);
  const url = `${LORCAST_SEARCH_URL}?q=${query}`;
  
  const response = await fetch(url, {
    headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
  });
  
  if (!response.ok) {
    if (response.status === 429) {
      console.warn('[lorcast] Rate limited, waiting...');
      await delay(5000);
      return fetchLorcastPricing(name, setCode); // retry once
    }
    return null;
  }
  
  const data = await response.json();
  if (!data?.results?.length) return null;
  
  const card = data.results[0];
  return {
    currentPrice: card.prices?.usd != null ? parseFloat(card.prices.usd) : null,
    marketPrice: card.prices?.usd_foil != null ? parseFloat(card.prices.usd_foil) : null,
    tcgplayerId: card.tcgplayer_id || null,
    imageUrl: card.image_uris?.digital?.normal || null,
    stockImageUrl: card.image_uris?.digital?.large || null,
  };
}

/**
 * Update card pricing in database using Lorcast search.
 */
export async function updateLorcastPricing(sql, cards, options = {}) {
  const { dryRun = false, concurrency = 5 } = options;
  let updated = 0;
  let errors = 0;
  
  for (let i = 0; i < cards.length; i += concurrency) {
    const chunk = cards.slice(i, i + concurrency);
    await Promise.all(chunk.map(async (card) => {
      try {
        const pricing = await fetchLorcastPricing(card.name, card.setCode);
        if (pricing && (pricing.currentPrice != null || pricing.marketPrice != null)) {
          if (!dryRun) {
            await sql`
              UPDATE cards
              SET current_price = COALESCE(${pricing.currentPrice}, current_price),
                  market_price = COALESCE(${pricing.marketPrice}, market_price),
                  tcgplayer_id = COALESCE(${pricing.tcgplayerId}, tcgplayer_id),
                  image_url = COALESCE(${pricing.imageUrl}, image_url),
                  stock_image_url = COALESCE(${pricing.stockImageUrl}, stock_image_url),
                  updated_at = CURRENT_TIMESTAMP
              WHERE scryfall_id = ${card.scryfallId}
            `;
          }
          updated++;
        }
        await delay(LORCAST_RATE_LIMIT_MS);
      } catch (err) {
        console.error('[lorcast-pricing] Error for', card.name, err.message);
        errors++;
      }
    }));
  }
  
  return { updated, errors };
}

/**
 * Run a full Lorcana bulk sync. Returns summary compatible with catalog sync reporting.
 *
 * @param {{ sql?: { query: (q: string, v: unknown[]) => Promise<unknown> }, dryRun?: boolean, fetchFolioIndex?: () => Promise<{ byCardId: Map<string, unknown>, byNameSet: Map<string, unknown> }> }} options
 */
export async function runBulkLorcanaSync(options = {}) {
  const dryRun = options.dryRun ?? false;
  const sql = options.sql ?? vercelSql;
  const fetchFolioIndex = options.fetchFolioIndex ?? fetchFolioPriceIndex;

  let folioIndex = { byCardId: new Map(), byNameSet: new Map() };
  if (!dryRun) {
    try {
      folioIndex = await fetchFolioIndex();
    } catch (error) {
      console.warn('[lorcana-bulk] Foilio feed fetch failed:', error.message);
    }
  }

  const allCards = await fetchLorcanaBulkCards();
  const lorcanaCards = allCards.filter(
    (card) => !card.Gamemode || card.Gamemode === 'Lorcana'
  );

  let upserted = 0;
  let snapshotsWritten = 0;
  let errors = 0;
  let pricesMatched = 0;
  let unmatchedCount = 0;
  let batch = [];

  const matchStats = {
    pricesMatched: 0,
    unmatchedCount: 0,
  };

  for (const card of lorcanaCards) {
    if (!card.Unique_ID) {
      errors += 1;
      continue;
    }

    let mapped = mapLorcanaCard(card);
    mapped = enrichLorcanaRowWithFolio(mapped, folioIndex, matchStats);
    batch.push(mapped);

    if (batch.length >= BATCH_SIZE) {
      try {
        if (!dryRun) {
          const result = await upsertBatch(sql, batch);
          upserted += result.upserted;
          snapshotsWritten += result.snapshotsWritten;
        } else {
          upserted += batch.length;
        }
      } catch (err) {
        console.error('[lorcana-bulk] Batch error:', err.message);
        errors += batch.length;
      }
      batch = [];
    }
  }

  if (batch.length > 0) {
    try {
      if (!dryRun) {
        const result = await upsertBatch(sql, batch);
        upserted += result.upserted;
        snapshotsWritten += result.snapshotsWritten;
      } else {
        upserted += batch.length;
      }
    } catch (err) {
      console.error('[lorcana-bulk] Final batch error:', err.message);
      errors += batch.length;
    }
  }

  pricesMatched = matchStats.pricesMatched;
  unmatchedCount = matchStats.unmatchedCount;

  // Fetch and update pricing from Lorcast (free, 10 req/sec)
  let lorcastUpdated = 0;
  let lorcastErrors = 0;
  if (!dryRun && lorcanaCards.length > 0) {
    console.log('[lorcana-bulk] Fetching pricing from Lorcast...');
    const pricingSummary = await updateLorcastPricing(
      sql,
      lorcanaCards.map(c => ({
        scryfallId: c.Unique_ID,
        name: c.Name,
        setCode: c.Set_ID,
      })),
      { dryRun }
    );
    lorcastUpdated = pricingSummary.updated;
    lorcastErrors = pricingSummary.errors;
  }

  const summary = {
    mode: 'bulk',
    game: 'lorcana',
    totalInFile: lorcanaCards.length,
    upserted,
    snapshotsWritten,
    errors,
    pricesMatched,
    unmatchedCount,
    lorcastUpdated,
    lorcastErrors,
  };

  console.log('[lorcana-bulk]', JSON.stringify(summary));
  return summary;
}