From 7982486f861de42881ee4be4cdc60d8239c5162c Mon Sep 17 00:00:00 2001 From: Randall Stillwell Date: Thu, 28 May 2026 11:44:49 -0500 Subject: [PATCH] =?UTF-8?q?Switch=20Pok=C3=A9mon=20catalog=20import=20to?= =?UTF-8?q?=20pokemon-tcg-data=20on=20GitHub.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace pokemontcg.io API discovery and import with raw JSON from PokemonTCG/pokemon-tcg-data; format collector numbers as number/printedTotal and drop the API key dependency for catalog sync. Co-authored-by: Cursor --- lib/card-import/discover.js | 7 +- lib/card-import/pokemon-github.js | 111 ++++++++++++++++++++ lib/card-import/pokemon.js | 102 +++++------------- scripts/README.md | 2 +- test/lib/card-import-pokemon-github.test.js | 56 ++++++++++ 5 files changed, 196 insertions(+), 82 deletions(-) create mode 100644 lib/card-import/pokemon-github.js create mode 100644 test/lib/card-import-pokemon-github.test.js diff --git a/lib/card-import/discover.js b/lib/card-import/discover.js index fe0cae0..b739e4f 100644 --- a/lib/card-import/discover.js +++ b/lib/card-import/discover.js @@ -87,9 +87,8 @@ export async function discoverMissingMtgSets() { } export async function discoverMissingPokemonSets() { - const { fetchWithRetry, pokemonHeaders } = await import('./pokemon.js'); - const response = await fetchWithRetry('https://api.pokemontcg.io/v2/sets'); - const data = await response.json(); + const { fetchPokemonSets } = await import('./pokemon-github.js'); + const sets = await fetchPokemonSets(); const knownCodes = await getKnownPokemonSetCodes(); - return filterMissingPokemonSets(data.data || [], knownCodes); + return filterMissingPokemonSets(sets, knownCodes); } diff --git a/lib/card-import/pokemon-github.js b/lib/card-import/pokemon-github.js new file mode 100644 index 0000000..ec0155f --- /dev/null +++ b/lib/card-import/pokemon-github.js @@ -0,0 +1,111 @@ +const DEFAULT_POKEMON_TCG_DATA_BASE = + 'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master'; + +export function getPokemonTcgDataBaseUrl() { + return process.env.POKEMON_TCG_DATA_BASE_URL || DEFAULT_POKEMON_TCG_DATA_BASE; +} + +export function pokemonSetsUrl() { + return `${getPokemonTcgDataBaseUrl()}/sets/en.json`; +} + +export function pokemonSetCardsUrl(setCode) { + return `${getPokemonTcgDataBaseUrl()}/cards/en/${encodeURIComponent(setCode)}.json`; +} + +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +function githubHeaders() { + return { + 'User-Agent': 'Deck-Hearth/1.0', + Accept: 'application/json', + }; +} + +export async function fetchWithRetry(url, maxRetries = 3, delayMs = 2000) { + for (let attempt = 1; attempt <= maxRetries; attempt += 1) { + try { + const response = await fetch(url, { headers: githubHeaders() }); + + if (response.ok) { + return response; + } + + if (response.status === 404) { + throw new Error(`Resource not found: ${response.status}`); + } + + if (response.status === 504 || response.status === 503 || response.status === 429) { + console.log( + `[fetchWithRetry] Attempt ${attempt}: ${response.status}, retrying in ${delayMs * attempt}ms` + ); + await delay(delayMs * attempt); + continue; + } + + throw new Error(`Pokemon TCG data fetch error: ${response.status}`); + } catch (error) { + if (attempt === maxRetries) { + throw error; + } + console.log(`[fetchWithRetry] Attempt ${attempt} failed:`, error.message); + await delay(delayMs * attempt); + } + } + + throw new Error('fetchWithRetry exhausted retries'); +} + +export async function fetchPokemonSets() { + const response = await fetchWithRetry(pokemonSetsUrl()); + const sets = await response.json(); + return Array.isArray(sets) ? sets : []; +} + +export async function fetchPokemonSetCards(setCode) { + try { + const response = await fetchWithRetry(pokemonSetCardsUrl(setCode)); + const cards = await response.json(); + return Array.isArray(cards) ? cards : []; + } catch (error) { + if (error.message.includes('404')) { + console.warn(`[fetchPokemonSetCards] No card file for set ${setCode} in pokemon-tcg-data`); + return []; + } + throw error; + } +} + +export function normalizePokemonRarity(rarity) { + const value = rarity || 'Unknown'; + if (value.includes('Holo')) return 'Holographic'; + if (value.includes('Secret')) return 'Secret Rare'; + if (value.includes('Ultra')) return 'Ultra Rare'; + return value; +} + +/** + * Map a pokemon-tcg-data card JSON object into DB insert fields. + */ +export function mapGithubCardForInsert(card, setMeta) { + const setCode = setMeta?.id || null; + const setName = setMeta?.name || setCode; + const printedTotal = setMeta?.printedTotal || setMeta?.total || null; + const cardNumber = + card.number && printedTotal ? `${card.number}/${printedTotal}` : card.number || null; + + return { + externalId: card.id, + name: card.name, + setName, + setCode, + cardNumber, + rarity: normalizePokemonRarity(card.rarity), + cardType: card.supertype || 'Pokemon', + types: card.types || [], + flavorText: card.flavorText || null, + hp: card.hp || card.attacks?.[0]?.damage || null, + imageSmall: card.images?.small || null, + imageLarge: card.images?.large || null, + }; +} diff --git a/lib/card-import/pokemon.js b/lib/card-import/pokemon.js index d6eb5b2..53ad6b1 100644 --- a/lib/card-import/pokemon.js +++ b/lib/card-import/pokemon.js @@ -1,63 +1,27 @@ import { sql } from '@vercel/postgres'; -const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +import { + fetchPokemonSetCards, + fetchPokemonSets, + mapGithubCardForInsert, +} from './pokemon-github.js'; -function pokemonHeaders() { - const headers = { - 'User-Agent': 'Deck-Hearth/1.0', - Accept: 'application/json', - }; - if (process.env.POKEMON_TCG_API_KEY) { - headers['X-Api-Key'] = process.env.POKEMON_TCG_API_KEY; - } - return headers; -} - -export async function fetchWithRetry(url, maxRetries = 3, delayMs = 2000) { - for (let attempt = 1; attempt <= maxRetries; attempt += 1) { - try { - const response = await fetch(url, { headers: pokemonHeaders() }); - - if (response.ok) { - return response; - } - - if (response.status === 404) { - throw new Error(`Resource not found: ${response.status}`); - } - - if (response.status === 504 || response.status === 503) { - console.log( - `[fetchWithRetry] Attempt ${attempt}: ${response.status}, retrying in ${delayMs * attempt}ms` - ); - await delay(delayMs * attempt); - continue; - } - - throw new Error(`Pokemon TCG API error: ${response.status}`); - } catch (error) { - if (attempt === maxRetries) { - throw error; - } - console.log(`[fetchWithRetry] Attempt ${attempt} failed:`, error.message); - await delay(delayMs * attempt); - } - } - - throw new Error('fetchWithRetry exhausted retries'); -} +export { fetchWithRetry, fetchPokemonSets } from './pokemon-github.js'; /** - * Import all cards for a Pokémon TCG set id. Skips rows already present by scryfall_id - * (legacy column name stores Pokémon TCG API card ids too). + * Import all cards for a Pokémon set id from the pokemon-tcg-data GitHub repo. + * Skips rows already present by scryfall_id (legacy column name stores external card ids). */ export async function importPokemonSet(setCode) { - const response = await fetchWithRetry( - `https://api.pokemontcg.io/v2/cards?q=set.id:${setCode}&pageSize=250` - ); + const [sets, cards] = await Promise.all([ + fetchPokemonSets(), + fetchPokemonSetCards(setCode), + ]); - const data = await response.json(); - const cards = data.data || []; + const setMeta = sets.find((set) => set.id?.toLowerCase() === setCode.toLowerCase()) || { + id: setCode, + name: setCode, + }; if (cards.length === 0) { return { setCode, imported: 0, skipped: 0, total: 0 }; @@ -68,8 +32,10 @@ export async function importPokemonSet(setCode) { for (const card of cards) { try { + const mapped = mapGithubCardForInsert(card, setMeta); + const existingCard = await sql` - SELECT id FROM cards WHERE scryfall_id = ${card.id} + SELECT id FROM cards WHERE scryfall_id = ${mapped.externalId} `; if (existingCard.rows.length > 0) { @@ -77,22 +43,6 @@ export async function importPokemonSet(setCode) { continue; } - let currentPrice = null; - if (card.tcgplayer?.prices?.normal?.market) { - currentPrice = parseFloat(card.tcgplayer.prices.normal.market); - } else if (card.tcgplayer?.prices?.holofoil?.market) { - currentPrice = parseFloat(card.tcgplayer.prices.holofoil.market); - } - - let rarity = card.rarity || 'Unknown'; - if (rarity.includes('Holo')) { - rarity = 'Holographic'; - } else if (rarity.includes('Secret')) { - rarity = 'Secret Rare'; - } else if (rarity.includes('Ultra')) { - rarity = 'Ultra Rare'; - } - await sql` INSERT INTO cards ( name, set_name, set_code, card_number, rarity, game, @@ -100,12 +50,12 @@ export async function importPokemonSet(setCode) { power, toughness, image_url, stock_image_url, current_price, market_price, scryfall_id, verified ) VALUES ( - ${card.name}, ${card.set.name}, ${card.set.id}, ${card.number}, - ${rarity}, 'Pokemon', null, null, ${card.supertype || 'Pokemon'}, - ${JSON.stringify(card.types || [])}, ${card.flavorText || null}, - ${card.attacks?.[0]?.damage || null}, null, - ${card.images?.small || null}, ${card.images?.large || null}, - ${currentPrice}, null, ${card.id}, true + ${mapped.name}, ${mapped.setName}, ${mapped.setCode}, ${mapped.cardNumber}, + ${mapped.rarity}, 'Pokemon', null, null, ${mapped.cardType}, + ${JSON.stringify(mapped.types)}, ${mapped.flavorText}, + ${mapped.hp}, null, + ${mapped.imageSmall}, ${mapped.imageLarge}, + null, null, ${mapped.externalId}, true ) `; @@ -118,5 +68,3 @@ export async function importPokemonSet(setCode) { return { setCode, imported, skipped, total: cards.length }; } - -export { pokemonHeaders }; diff --git a/scripts/README.md b/scripts/README.md index 24bad20..2a1f0a6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -176,7 +176,7 @@ You can stop the script with `Ctrl+C` and restart it later. The scripts will sta ## Data Sources - **Magic: The Gathering**: Scryfall API -- **Pokemon**: Pokemon TCG API +- **Pokemon**: [PokemonTCG/pokemon-tcg-data](https://github.com/PokemonTCG/pokemon-tcg-data) on GitHub (sets + per-set JSON; images from linked CDNs). Optional override: `POKEMON_TCG_DATA_BASE_URL` (defaults to `master` branch raw URLs). The legacy Pokemon TCG API is no longer used by catalog sync. - **Lorcana**: Lorcana API (limited availability) ## Performance Notes diff --git a/test/lib/card-import-pokemon-github.test.js b/test/lib/card-import-pokemon-github.test.js new file mode 100644 index 0000000..1de3f61 --- /dev/null +++ b/test/lib/card-import-pokemon-github.test.js @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; + +import { + mapGithubCardForInsert, + pokemonSetCardsUrl, + pokemonSetsUrl, +} from '../../lib/card-import/pokemon-github.js'; + +describe('pokemon-github URLs', () => { + it('builds default pokemon-tcg-data raw GitHub paths', () => { + expect(pokemonSetsUrl()).toBe( + 'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master/sets/en.json' + ); + expect(pokemonSetCardsUrl('me3')).toBe( + 'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master/cards/en/me3.json' + ); + }); +}); + +describe('mapGithubCardForInsert', () => { + it('maps card fields and formats collector number with printed total', () => { + const mapped = mapGithubCardForInsert( + { + id: 'me3-18', + name: 'Seel', + supertype: 'Pokémon', + number: '18', + rarity: 'Common', + types: ['Water'], + flavorText: 'The horn on its head is sharp.', + hp: '80', + images: { + small: 'https://images.scrydex.com/pokemon/me3-18/small', + large: 'https://images.scrydex.com/pokemon/me3-18/large', + }, + attacks: [{ damage: '10' }], + }, + { id: 'me3', name: 'Perfect Order', printedTotal: 88 } + ); + + expect(mapped).toEqual({ + externalId: 'me3-18', + name: 'Seel', + setName: 'Perfect Order', + setCode: 'me3', + cardNumber: '18/88', + rarity: 'Common', + cardType: 'Pokémon', + types: ['Water'], + flavorText: 'The horn on its head is sharp.', + hp: '80', + imageSmall: 'https://images.scrydex.com/pokemon/me3-18/small', + imageLarge: 'https://images.scrydex.com/pokemon/me3-18/large', + }); + }); +});