/** * Bulk catalog sync using Scryfall's bulk-data endpoint. * * Downloads the Oracle Cards file (~168 MB) and upserts all MTG cards in one pass. * Much faster and more complete than the per-set discovery approach — catches all * missing sets in one shot without rate-limit concerns. * * Also refreshes prices, legalities, and edhrec_rank on existing cards. */ import { sql } from '../sql.js'; const BATCH_SIZE = 100; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); async function fetchBulkDownloadUrl() { const response = await fetch('https://api.scryfall.com/bulk-data/oracle_cards', { headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' }, }); if (!response.ok) { throw new Error(`Scryfall bulk-data API error: ${response.status}`); } return response.json(); } function mapCard(card) { let currentPrice = null; let marketPrice = null; if (card.prices) { currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null; marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null; } return { scryfallId: card.id, oracleId: card.oracle_id || null, illustrationId: card.illustration_id || null, name: card.name, setName: card.set_name, setCode: card.set, cardNumber: card.collector_number, rarity: card.rarity, manaCost: card.mana_cost || null, cmc: card.cmc != null ? Math.round(card.cmc) : null, cardType: card.type_line || null, colors: JSON.stringify(card.colors || []), colorIdentity: JSON.stringify(card.color_identity || []), oracleText: card.oracle_text || null, power: card.power || null, toughness: card.toughness || null, imageUrl: card.image_uris?.normal || null, stockImageUrl: card.image_uris?.art_crop || null, currentPrice, marketPrice, keywords: JSON.stringify(card.keywords || []), legalities: JSON.stringify(card.legalities || {}), flavorText: card.flavor_text || null, artist: card.artist || null, releasedAt: card.released_at || null, layout: card.layout || null, edhrecRank: card.edhrec_rank || null, reserved: card.reserved || false, reprint: card.reprint || false, finishes: JSON.stringify(card.finishes || []), }; } async function upsertBatch(cards) { if (cards.length === 0) return 0; const columns = [ 'scryfall_id', 'oracle_id', 'illustration_id', 'name', 'set_name', 'set_code', 'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors', 'color_identity', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url', 'current_price', 'market_price', 'keywords', 'legalities', 'flavor_text', 'artist', 'released_at', 'layout', 'edhrec_rank', 'reserved', 'reprint', 'finishes', 'verified', ]; const values = []; const placeholders = []; let paramIdx = 1; for (const row of cards) { const rowValues = [ row.scryfallId, row.oracleId, row.illustrationId, row.name, row.setName, row.setCode, row.cardNumber, row.rarity, 'MTG', row.manaCost, row.cmc, row.cardType, row.colors, row.colorIdentity, row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl, row.currentPrice, row.marketPrice, row.keywords, row.legalities, row.flavorText, row.artist, row.releasedAt, row.layout, row.edhrecRank, row.reserved, row.reprint, row.finishes, true, ]; const rowP = rowValues.map(() => `$${paramIdx++}`); placeholders.push(`(${rowP.join(',')})`); values.push(...rowValues); } const updateCols = columns .filter((c) => c !== 'scryfall_id') .map((c) => `${c} = EXCLUDED.${c}`) .join(', '); await sql.query( `INSERT INTO cards (${columns.join(', ')}) VALUES ${placeholders.join(',\n')} ON CONFLICT (scryfall_id) DO UPDATE SET ${updateCols}, updated_at = CURRENT_TIMESTAMP`, values ); return cards.length; } /** * Run a full bulk MTG sync. Returns a summary compatible with the catalog sync API shape. */ export async function runBulkMtgSync() { const meta = await fetchBulkDownloadUrl(); console.log(`[bulk-sync] Oracle Cards updated: ${meta.updated_at}, size: ${(meta.size / 1024 / 1024).toFixed(0)} MB`); const response = await fetch(meta.download_uri, { headers: { 'User-Agent': 'DeckHearth/1.0' }, }); if (!response.ok) { throw new Error(`Bulk download failed: ${response.status}`); } const allCards = await response.json(); const mtgCards = allCards.filter((c) => c.lang === 'en' && !c.digital); let upserted = 0; let errors = 0; let batch = []; for (const card of mtgCards) { batch.push(mapCard(card)); if (batch.length >= BATCH_SIZE) { try { upserted += await upsertBatch(batch); } catch (err) { console.error('[bulk-sync] Batch error:', err.message); errors += batch.length; } batch = []; } } if (batch.length > 0) { try { upserted += await upsertBatch(batch); } catch (err) { console.error('[bulk-sync] Final batch error:', err.message); errors += batch.length; } } // Backfill oracle_id on older per-set imported cards const backfill = await sql` UPDATE cards target SET oracle_id = source.oracle_id, color_identity = COALESCE(target.color_identity, source.color_identity), keywords = COALESCE(target.keywords, source.keywords), legalities = COALESCE(target.legalities, source.legalities), layout = COALESCE(target.layout, source.layout), edhrec_rank = COALESCE(target.edhrec_rank, source.edhrec_rank) FROM ( SELECT DISTINCT ON (name) name, oracle_id, color_identity, keywords, legalities, layout, edhrec_rank FROM cards WHERE oracle_id IS NOT NULL AND game = 'MTG' ) source WHERE target.name = source.name AND target.game = 'MTG' AND target.oracle_id IS NULL `; const summary = { mode: 'bulk', totalInFile: mtgCards.length, upserted, errors, backfilledOracleId: backfill.rowCount || 0, updatedAt: meta.updated_at, }; console.log('[bulk-sync]', JSON.stringify(summary)); return summary; }