/** * Bulk catalog sync for Disney Lorcana via lorcana-api.com /bulk/cards. * * Single fetch returns all cards (~2.3k). Upserts keyed on scryfall_id (stores Unique_ID). * Enriches prices from Foilio hourly feed when available. * Pricing: also fetches individually via api.lorcast.com/v0/cards/search (free, 10 req/sec rate limit). */ import { sql as vercelSql } from '../sql.js'; import { recordPriceSnapshots } from '../pricing/price-history.js'; import { fetchFolioPriceIndex, matchFolioPrice } from '../pricing/foolio-client.js'; const BATCH_SIZE = 100; const BULK_URL = 'https://api.lorcana-api.com/bulk/cards'; const LORCAST_SEARCH_URL = 'https://api.lorcast.com/v0/cards/search'; const LORCAST_RATE_LIMIT_MS = 110; // 10 req/sec, per lorcast.com/docs function hasPrice(row) { return row.currentPrice != null || row.marketPrice != null; } /** * Map a Lorcana bulk API card object to cards-table insert shape. */ export function mapLorcanaCard(card, pricing = null) { const colors = card.Color ? card.Color.split(',').map((entry) => entry.trim()).filter(Boolean) : []; const keywords = []; if (card.Classifications) { keywords.push( ...card.Classifications.split(',').map((entry) => entry.trim()).filter(Boolean) ); } if (card.Abilities) { keywords.push(...card.Abilities.split(',').map((entry) => entry.trim()).filter(Boolean)); } if (card.Inkable === true) { keywords.push('Inkable'); } const cost = card.Cost != null ? Number(card.Cost) : null; const row = { scryfallId: card.Unique_ID, name: card.Name, setName: card.Set_Name, setCode: card.Set_ID, cardNumber: card.Card_Num != null ? String(card.Card_Num) : null, rarity: card.Rarity || null, manaCost: cost != null ? String(cost) : null, cmc: cost, cardType: card.Type || null, colors: JSON.stringify(colors), colorIdentity: JSON.stringify(colors), oracleText: card.Body_Text || null, power: card.Strength != null ? String(card.Strength) : null, toughness: card.Willpower != null ? String(card.Willpower) : null, imageUrl: card.Image || null, stockImageUrl: card.Image || null, flavorText: card.Flavor_Text || null, artist: card.Artist || null, edhrecRank: card.Lore != null ? Number(card.Lore) : null, keywords: JSON.stringify(keywords), tcgplayerId: card.tcgplayer_id || null, currentPrice: card.prices?.usd != null ? parseFloat(card.prices.usd) : null, marketPrice: card.prices?.usd_foil != null ? parseFloat(card.prices.usd_foil) : null, }; if (pricing) { row.currentPrice = pricing.currentPrice; row.marketPrice = pricing.marketPrice; row.priceCurrency = pricing.currency ?? 'USD'; if (hasPrice(row)) { row.pricesUpdatedAt = new Date(); } } return row; } export function enrichLorcanaRowWithFolio(row, folioIndex, stats) { const pricing = matchFolioPrice(folioIndex, row); if (!pricing || !hasPrice(pricing)) { if (stats) stats.unmatchedCount += 1; return row; } if (stats) stats.pricesMatched += 1; return { ...row, currentPrice: pricing.currentPrice, marketPrice: pricing.marketPrice, priceCurrency: pricing.currency, pricesUpdatedAt: new Date(), }; } export function buildUpsertQuery(batch) { const columns = [ 'scryfall_id', 'name', 'set_name', 'set_code', 'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors', 'color_identity', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url', 'current_price', 'market_price', 'prices_updated_at', 'keywords', 'flavor_text', 'artist', 'edhrec_rank', 'verified', ]; const placeholders = []; const values = []; let paramIdx = 1; for (const row of batch) { const rowPlaceholders = []; const rowValues = [ row.scryfallId, row.name, row.setName, row.setCode, row.cardNumber, row.rarity, 'Lorcana', row.manaCost, row.cmc, row.cardType, row.colors, row.colorIdentity, row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl, row.currentPrice ?? null, row.marketPrice ?? null, row.pricesUpdatedAt ?? null, row.keywords, row.flavorText, row.artist, row.edhrecRank, true, ]; for (let i = 0; i < rowValues.length; i += 1) { rowPlaceholders.push(`$${paramIdx}`); paramIdx += 1; } placeholders.push(`(${rowPlaceholders.join(', ')})`); values.push(...rowValues); } const updateCols = columns .filter((col) => col !== 'scryfall_id') .map((col) => `${col} = EXCLUDED.${col}`) .join(', '); const query = ` INSERT INTO cards (${columns.join(', ')}) VALUES ${placeholders.join(',\n')} ON CONFLICT (scryfall_id) DO UPDATE SET ${updateCols}, updated_at = CURRENT_TIMESTAMP `; return { query, values }; } async function resolveCardIds(sql, scryfallIds) { if (scryfallIds.length === 0) return new Map(); const result = await sql.query( 'SELECT id, scryfall_id FROM cards WHERE scryfall_id = ANY($1)', [scryfallIds] ); return new Map(result.rows.map((row) => [row.scryfall_id, row.id])); } async function recordBatchSnapshots(sql, batch) { const pricedRows = batch.filter(hasPrice); if (pricedRows.length === 0) return 0; const idByScryfall = await resolveCardIds(sql, pricedRows.map((row) => row.scryfallId)); const snapshots = []; for (const row of pricedRows) { const cardId = idByScryfall.get(row.scryfallId); if (cardId == null) continue; const currency = row.priceCurrency ?? 'USD'; if (row.currentPrice != null) { snapshots.push({ cardId, source: 'foilio', priceType: 'market', currency, amount: row.currentPrice, }); } if (row.marketPrice != null && row.marketPrice !== row.currentPrice) { snapshots.push({ cardId, source: 'foilio', priceType: 'market', currency, amount: row.marketPrice, }); } else if (row.marketPrice != null && row.currentPrice == null) { snapshots.push({ cardId, source: 'foilio', priceType: 'market', currency, amount: row.marketPrice, }); } } return recordPriceSnapshots(snapshots); } async function upsertBatch(sql, batch) { if (batch.length === 0) return { upserted: 0, snapshotsWritten: 0 }; const { query, values } = buildUpsertQuery(batch); await sql.query(query, values); const snapshotsWritten = await recordBatchSnapshots(sql, batch); return { upserted: batch.length, snapshotsWritten }; } export async function fetchLorcanaBulkCards() { const response = await fetch(BULK_URL, { headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' }, }); if (!response.ok) { throw new Error(`Lorcana bulk API error: ${response.status}`); } return response.json(); } const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); /** * Fetch pricing for a card via Lorcast search API. * Returns { currentPrice, marketPrice, tcgplayerId, imageUrl, stockImageUrl } or null. */ export async function fetchLorcastPricing(name, setCode) { const query = encodeURIComponent(`${name} set:${setCode}`); const url = `${LORCAST_SEARCH_URL}?q=${query}`; const response = await fetch(url, { headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' }, }); if (!response.ok) { if (response.status === 429) { console.warn('[lorcast] Rate limited, waiting...'); await delay(5000); return fetchLorcastPricing(name, setCode); // retry once } return null; } const data = await response.json(); if (!data?.results?.length) return null; const card = data.results[0]; return { currentPrice: card.prices?.usd != null ? parseFloat(card.prices.usd) : null, marketPrice: card.prices?.usd_foil != null ? parseFloat(card.prices.usd_foil) : null, tcgplayerId: card.tcgplayer_id || null, imageUrl: card.image_uris?.digital?.normal || null, stockImageUrl: card.image_uris?.digital?.large || null, }; } /** * Update card pricing in database using Lorcast search. */ export async function updateLorcastPricing(sql, cards, options = {}) { const { dryRun = false, concurrency = 5 } = options; let updated = 0; let errors = 0; for (let i = 0; i < cards.length; i += concurrency) { const chunk = cards.slice(i, i + concurrency); await Promise.all(chunk.map(async (card) => { try { const pricing = await fetchLorcastPricing(card.name, card.setCode); if (pricing && (pricing.currentPrice != null || pricing.marketPrice != null)) { if (!dryRun) { await sql` UPDATE cards SET current_price = COALESCE(${pricing.currentPrice}, current_price), market_price = COALESCE(${pricing.marketPrice}, market_price), tcgplayer_id = COALESCE(${pricing.tcgplayerId}, tcgplayer_id), image_url = COALESCE(${pricing.imageUrl}, image_url), stock_image_url = COALESCE(${pricing.stockImageUrl}, stock_image_url), updated_at = CURRENT_TIMESTAMP WHERE scryfall_id = ${card.scryfallId} `; } updated++; } await delay(LORCAST_RATE_LIMIT_MS); } catch (err) { console.error('[lorcast-pricing] Error for', card.name, err.message); errors++; } })); } return { updated, errors }; } /** * Run a full Lorcana bulk sync. Returns summary compatible with catalog sync reporting. * * @param {{ sql?: { query: (q: string, v: unknown[]) => Promise }, dryRun?: boolean, fetchFolioIndex?: () => Promise<{ byCardId: Map, byNameSet: Map }> }} options */ export async function runBulkLorcanaSync(options = {}) { const dryRun = options.dryRun ?? false; const sql = options.sql ?? vercelSql; const fetchFolioIndex = options.fetchFolioIndex ?? fetchFolioPriceIndex; let folioIndex = { byCardId: new Map(), byNameSet: new Map() }; if (!dryRun) { try { folioIndex = await fetchFolioIndex(); } catch (error) { console.warn('[lorcana-bulk] Foilio feed fetch failed:', error.message); } } const allCards = await fetchLorcanaBulkCards(); const lorcanaCards = allCards.filter( (card) => !card.Gamemode || card.Gamemode === 'Lorcana' ); let upserted = 0; let snapshotsWritten = 0; let errors = 0; let pricesMatched = 0; let unmatchedCount = 0; let batch = []; const matchStats = { pricesMatched: 0, unmatchedCount: 0, }; for (const card of lorcanaCards) { if (!card.Unique_ID) { errors += 1; continue; } let mapped = mapLorcanaCard(card); mapped = enrichLorcanaRowWithFolio(mapped, folioIndex, matchStats); batch.push(mapped); if (batch.length >= BATCH_SIZE) { try { if (!dryRun) { const result = await upsertBatch(sql, batch); upserted += result.upserted; snapshotsWritten += result.snapshotsWritten; } else { upserted += batch.length; } } catch (err) { console.error('[lorcana-bulk] Batch error:', err.message); errors += batch.length; } batch = []; } } if (batch.length > 0) { try { if (!dryRun) { const result = await upsertBatch(sql, batch); upserted += result.upserted; snapshotsWritten += result.snapshotsWritten; } else { upserted += batch.length; } } catch (err) { console.error('[lorcana-bulk] Final batch error:', err.message); errors += batch.length; } } pricesMatched = matchStats.pricesMatched; unmatchedCount = matchStats.unmatchedCount; // Fetch and update pricing from Lorcast (free, 10 req/sec) let lorcastUpdated = 0; let lorcastErrors = 0; if (!dryRun && lorcanaCards.length > 0) { console.log('[lorcana-bulk] Fetching pricing from Lorcast...'); const pricingSummary = await updateLorcastPricing( sql, lorcanaCards.map(c => ({ scryfallId: c.Unique_ID, name: c.Name, setCode: c.Set_ID, })), { dryRun } ); lorcastUpdated = pricingSummary.updated; lorcastErrors = pricingSummary.errors; } const summary = { mode: 'bulk', game: 'lorcana', totalInFile: lorcanaCards.length, upserted, snapshotsWritten, errors, pricesMatched, unmatchedCount, lorcastUpdated, lorcastErrors, }; console.log('[lorcana-bulk]', JSON.stringify(summary)); return summary; }