/** * Bulk Pokémon catalog sync from pokemon-tcg-data GitHub JSON. * * Fetches all sets from sets/en.json, then upserts every card in each set * (keyed on scryfall_id = external card id). Populates legalities from set metadata. * Enriches prices from TCGdex when available. */ import { sql } from '../sql.js'; import { recordPriceSnapshots } from '../pricing/price-history.js'; import { buildTcgdexPriceIndex, fetchSetCards, matchTcgdexCard, parseTcgdexPricing, } from '../pricing/tcgdex-client.js'; import { fetchPokemonSetCards, fetchPokemonSets, mapGithubCardForInsert, } from './pokemon-github.js'; const DEFAULT_BATCH_SIZE = 100; const DEFAULT_SET_DELAY_MS = 100; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); function getSetLegalities(setMeta) { if (setMeta?.legalities && typeof setMeta.legalities === 'object') { return setMeta.legalities; } return {}; } function hasPrice(row) { return row.currentPrice != null || row.marketPrice != null; } export function mapPokemonBulkRow(mapped, setLegalities, pricing = null) { const row = { scryfallId: mapped.externalId, name: mapped.name, setName: mapped.setName, setCode: mapped.setCode, cardNumber: mapped.cardNumber, rarity: mapped.rarity, cardType: mapped.cardType, colors: JSON.stringify(mapped.types || []), oracleText: mapped.flavorText, power: mapped.hp, imageUrl: mapped.imageSmall, stockImageUrl: mapped.imageLarge, legalities: JSON.stringify(setLegalities), currentPrice: null, marketPrice: null, pricesUpdatedAt: null, priceCurrency: 'USD', }; if (pricing) { row.currentPrice = pricing.currentPrice; row.marketPrice = pricing.marketPrice; row.priceCurrency = pricing.currency ?? 'USD'; if (hasPrice(row)) { row.pricesUpdatedAt = new Date(); } } return row; } function buildUpsertQuery(batch) { const columns = [ 'scryfall_id', 'name', 'set_name', 'set_code', 'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url', 'current_price', 'market_price', 'prices_updated_at', 'legalities', 'verified', ]; const placeholders = []; const values = []; let paramIdx = 1; for (const row of batch) { const rowPlaceholders = []; const rowValues = [ row.scryfallId, row.name, row.setName, row.setCode, row.cardNumber, row.rarity, 'Pokemon', null, null, row.cardType, row.colors, row.oracleText, row.power, null, row.imageUrl, row.stockImageUrl, row.currentPrice, row.marketPrice, row.pricesUpdatedAt, row.legalities, true, ]; for (let i = 0; i < rowValues.length; i += 1) { rowPlaceholders.push(`$${paramIdx}`); paramIdx += 1; } placeholders.push(`(${rowPlaceholders.join(', ')})`); values.push(...rowValues); } const updateCols = columns .filter((c) => c !== 'scryfall_id') .map((c) => `${c} = EXCLUDED.${c}`) .join(', '); const query = ` INSERT INTO cards (${columns.join(', ')}) VALUES ${placeholders.join(',\n')} ON CONFLICT (scryfall_id) DO UPDATE SET ${updateCols}, updated_at = CURRENT_TIMESTAMP `; return { query, values }; } async function resolveCardIds(scryfallIds, queryFn) { if (scryfallIds.length === 0) return new Map(); const result = await queryFn( 'SELECT id, scryfall_id FROM cards WHERE scryfall_id = ANY($1)', [scryfallIds] ); return new Map(result.rows.map((row) => [row.scryfall_id, row.id])); } async function recordBatchSnapshots(batch, queryFn) { const pricedRows = batch.filter(hasPrice); if (pricedRows.length === 0) return 0; const idByScryfall = await resolveCardIds(pricedRows.map((row) => row.scryfallId), queryFn); const snapshots = []; for (const row of pricedRows) { const cardId = idByScryfall.get(row.scryfallId); if (cardId == null) continue; const currency = row.priceCurrency ?? 'USD'; if (row.currentPrice != null) { snapshots.push({ cardId, source: 'tcgdex', priceType: 'market', currency, amount: row.currentPrice, }); } if (row.marketPrice != null && row.marketPrice !== row.currentPrice) { snapshots.push({ cardId, source: 'tcgdex', priceType: 'market', currency, amount: row.marketPrice, }); } else if (row.marketPrice != null && row.currentPrice == null) { snapshots.push({ cardId, source: 'tcgdex', priceType: 'market', currency, amount: row.marketPrice, }); } } return recordPriceSnapshots(snapshots); } async function upsertBatch(batch, queryFn, stats) { if (batch.length === 0) return; if (stats.dryRun) { stats.skipped += batch.length; return; } const { query, values } = buildUpsertQuery(batch); try { await queryFn(query, values); stats.upserted += batch.length; stats.snapshotsWritten += await recordBatchSnapshots(batch, queryFn); } catch (error) { console.error(`[pokemon-bulk] Batch failed (${batch.length} rows):`, error.message); for (const row of batch) { try { const { query: singleQ, values: singleV } = buildUpsertQuery([row]); await queryFn(singleQ, singleV); stats.upserted += 1; stats.snapshotsWritten += await recordBatchSnapshots([row], queryFn); } catch (singleErr) { console.error(`[pokemon-bulk] Failed: ${row.name} (${row.scryfallId}):`, singleErr.message); stats.errors += 1; } } } } function enrichRowWithTcgdex(row, tcgIndex, stats) { const tcgCard = matchTcgdexCard(tcgIndex, row); if (!tcgCard) { stats.unmatchedCount += 1; return row; } const pricing = parseTcgdexPricing(tcgCard); if (!hasPrice({ currentPrice: pricing.currentPrice, marketPrice: pricing.marketPrice })) { stats.unmatchedCount += 1; return row; } stats.pricesMatched += 1; return { ...row, currentPrice: pricing.currentPrice, marketPrice: pricing.marketPrice, priceCurrency: pricing.currency, pricesUpdatedAt: new Date(), }; } async function upsertSetCards(setMeta, cards, options, stats) { const setLegalities = getSetLegalities(setMeta); let tcgIndex = { byNumber: new Map(), byName: new Map() }; if (!stats.dryRun && options.fetchTcgdexSetCards) { try { const tcgCards = await options.fetchTcgdexSetCards(setMeta.id); tcgIndex = buildTcgdexPriceIndex(tcgCards); } catch (error) { console.warn(`[pokemon-bulk] TCGdex fetch failed for ${setMeta.id}:`, error.message); } } let batch = []; for (const card of cards) { const mapped = mapGithubCardForInsert(card, setMeta); let row = mapPokemonBulkRow(mapped, setLegalities); row = enrichRowWithTcgdex(row, tcgIndex, stats); batch.push(row); if (batch.length >= options.batchSize) { await upsertBatch(batch, options.queryFn, stats); batch = []; } } if (batch.length > 0) { await upsertBatch(batch, options.queryFn, stats); } } /** * Run a full Pokémon bulk sync from pokemon-tcg-data GitHub. * Returns a summary compatible with multi-game catalog sync orchestration. */ export async function runBulkPokemonSync(options = {}) { const batchSize = options.batchSize ?? parseInt(process.env.BATCH_SIZE || String(DEFAULT_BATCH_SIZE), 10); const dryRun = options.dryRun ?? process.env.DRY_RUN === 'true'; const setDelayMs = options.setDelayMs ?? DEFAULT_SET_DELAY_MS; const queryFn = options.query ?? ((query, values) => sql.query(query, values)); const fetchTcgdexSetCards = options.fetchTcgdexSetCards ?? fetchSetCards; const sets = await fetchPokemonSets(); console.log(`[pokemon-bulk] Found ${sets.length} sets in pokemon-tcg-data`); const stats = { dryRun, upserted: 0, skipped: 0, errors: 0, setsProcessed: 0, pricesMatched: 0, unmatchedCount: 0, snapshotsWritten: 0, }; for (let index = 0; index < sets.length; index += 1) { const setMeta = sets[index]; const setCode = setMeta.id; if (!setCode) { console.warn('[pokemon-bulk] Skipping set with no id'); continue; } try { const cards = await fetchPokemonSetCards(setCode); stats.setsProcessed += 1; if (cards.length === 0) { console.log(`[pokemon-bulk] Set ${setCode}: no cards (empty or missing file)`); } else { await upsertSetCards(setMeta, cards, { batchSize, queryFn, fetchTcgdexSetCards }, stats); console.log(`[pokemon-bulk] Set ${setCode}: processed ${cards.length} cards`); } } catch (error) { console.error(`[pokemon-bulk] Set ${setCode} failed:`, error.message); stats.errors += 1; } if (index < sets.length - 1) { await delay(setDelayMs); } } const summary = { mode: 'bulk', game: 'pokemon', setsProcessed: stats.setsProcessed, totalSets: sets.length, upserted: stats.upserted, skipped: stats.skipped, errors: stats.errors, pricesMatched: stats.pricesMatched, unmatchedCount: stats.unmatchedCount, snapshotsWritten: stats.snapshotsWritten, dryRun, }; console.log('[pokemon-bulk]', JSON.stringify(summary)); return summary; }