Switch Pokémon catalog import to pokemon-tcg-data on GitHub. (#52)

Replace pokemontcg.io API discovery and import with raw JSON from PokemonTCG/pokemon-tcg-data; format collector numbers as number/printedTotal and drop the API key dependency for catalog sync.

Co-authored-by: Cursor <cursoragent@cursor.com>

lib/card-import/discover.js

@@ -87,9 +87,8 @@ export async function discoverMissingMtgSets() {
 }
 
 export async function discoverMissingPokemonSets() {
-  const { fetchWithRetry, pokemonHeaders } = await import('./pokemon.js');
+  const { fetchPokemonSets } = await import('./pokemon-github.js');
-  const response = await fetchWithRetry('https://api.pokemontcg.io/v2/sets');
+  const sets = await fetchPokemonSets();
-  const data = await response.json();
   const knownCodes = await getKnownPokemonSetCodes();
-  return filterMissingPokemonSets(data.data || [], knownCodes);
+  return filterMissingPokemonSets(sets, knownCodes);
 }

lib/card-import/pokemon-github.js

@@ -0,0 +1,111 @@
+const DEFAULT_POKEMON_TCG_DATA_BASE =
+  'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master';
+
+export function getPokemonTcgDataBaseUrl() {
+  return process.env.POKEMON_TCG_DATA_BASE_URL || DEFAULT_POKEMON_TCG_DATA_BASE;
+}
+
+export function pokemonSetsUrl() {
+  return `${getPokemonTcgDataBaseUrl()}/sets/en.json`;
+}
+
+export function pokemonSetCardsUrl(setCode) {
+  return `${getPokemonTcgDataBaseUrl()}/cards/en/${encodeURIComponent(setCode)}.json`;
+}
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+function githubHeaders() {
+  return {
+    'User-Agent': 'Deck-Hearth/1.0',
+    Accept: 'application/json',
+  };
+}
+
+export async function fetchWithRetry(url, maxRetries = 3, delayMs = 2000) {
+  for (let attempt = 1; attempt <= maxRetries; attempt += 1) {
+    try {
+      const response = await fetch(url, { headers: githubHeaders() });
+
+      if (response.ok) {
+        return response;
+      }
+
+      if (response.status === 404) {
+        throw new Error(`Resource not found: ${response.status}`);
+      }
+
+      if (response.status === 504 || response.status === 503 || response.status === 429) {
+        console.log(
+          `[fetchWithRetry] Attempt ${attempt}: ${response.status}, retrying in ${delayMs * attempt}ms`
+        );
+        await delay(delayMs * attempt);
+        continue;
+      }
+
+      throw new Error(`Pokemon TCG data fetch error: ${response.status}`);
+    } catch (error) {
+      if (attempt === maxRetries) {
+        throw error;
+      }
+      console.log(`[fetchWithRetry] Attempt ${attempt} failed:`, error.message);
+      await delay(delayMs * attempt);
+    }
+  }
+
+  throw new Error('fetchWithRetry exhausted retries');
+}
+
+export async function fetchPokemonSets() {
+  const response = await fetchWithRetry(pokemonSetsUrl());
+  const sets = await response.json();
+  return Array.isArray(sets) ? sets : [];
+}
+
+export async function fetchPokemonSetCards(setCode) {
+  try {
+    const response = await fetchWithRetry(pokemonSetCardsUrl(setCode));
+    const cards = await response.json();
+    return Array.isArray(cards) ? cards : [];
+  } catch (error) {
+    if (error.message.includes('404')) {
+      console.warn(`[fetchPokemonSetCards] No card file for set ${setCode} in pokemon-tcg-data`);
+      return [];
+    }
+    throw error;
+  }
+}
+
+export function normalizePokemonRarity(rarity) {
+  const value = rarity || 'Unknown';
+  if (value.includes('Holo')) return 'Holographic';
+  if (value.includes('Secret')) return 'Secret Rare';
+  if (value.includes('Ultra')) return 'Ultra Rare';
+  return value;
+}
+
+/**
+ * Map a pokemon-tcg-data card JSON object into DB insert fields.
+ */
+export function mapGithubCardForInsert(card, setMeta) {
+  const setCode = setMeta?.id || null;
+  const setName = setMeta?.name || setCode;
+  const printedTotal = setMeta?.printedTotal || setMeta?.total || null;
+  const cardNumber =
+    card.number && printedTotal ? `${card.number}/${printedTotal}` : card.number || null;
+
+  return {
+    externalId: card.id,
+    name: card.name,
+    setName,
+    setCode,
+    cardNumber,
+    rarity: normalizePokemonRarity(card.rarity),
+    cardType: card.supertype || 'Pokemon',
+    types: card.types || [],
+    flavorText: card.flavorText || null,
+    hp: card.hp || card.attacks?.[0]?.damage || null,
+    imageSmall: card.images?.small || null,
+    imageLarge: card.images?.large || null,
+  };
+}

lib/card-import/pokemon.js

@@ -1,63 +1,27 @@
 import { sql } from '@vercel/postgres';
 
-const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+import {
+  fetchPokemonSetCards,
+  fetchPokemonSets,
+  mapGithubCardForInsert,
+} from './pokemon-github.js';
 
-function pokemonHeaders() {
+export { fetchWithRetry, fetchPokemonSets } from './pokemon-github.js';
-  const headers = {
-    'User-Agent': 'Deck-Hearth/1.0',
-    Accept: 'application/json',
-  };
-  if (process.env.POKEMON_TCG_API_KEY) {
-    headers['X-Api-Key'] = process.env.POKEMON_TCG_API_KEY;
-  }
-  return headers;
-}
-
-export async function fetchWithRetry(url, maxRetries = 3, delayMs = 2000) {
-  for (let attempt = 1; attempt <= maxRetries; attempt += 1) {
-    try {
-      const response = await fetch(url, { headers: pokemonHeaders() });
-
-      if (response.ok) {
-        return response;
-      }
-
-      if (response.status === 404) {
-        throw new Error(`Resource not found: ${response.status}`);
-      }
-
-      if (response.status === 504 || response.status === 503) {
-        console.log(
-          `[fetchWithRetry] Attempt ${attempt}: ${response.status}, retrying in ${delayMs * attempt}ms`
-        );
-        await delay(delayMs * attempt);
-        continue;
-      }
-
-      throw new Error(`Pokemon TCG API error: ${response.status}`);
-    } catch (error) {
-      if (attempt === maxRetries) {
-        throw error;
-      }
-      console.log(`[fetchWithRetry] Attempt ${attempt} failed:`, error.message);
-      await delay(delayMs * attempt);
-    }
-  }
-
-  throw new Error('fetchWithRetry exhausted retries');
-}
 
 /**
- * Import all cards for a Pokémon TCG set id. Skips rows already present by scryfall_id
+ * Import all cards for a Pokémon set id from the pokemon-tcg-data GitHub repo.
- * (legacy column name stores Pokémon TCG API card ids too).
+ * Skips rows already present by scryfall_id (legacy column name stores external card ids).
  */
 export async function importPokemonSet(setCode) {
-  const response = await fetchWithRetry(
+  const [sets, cards] = await Promise.all([
-    `https://api.pokemontcg.io/v2/cards?q=set.id:${setCode}&pageSize=250`
+    fetchPokemonSets(),
-  );
+    fetchPokemonSetCards(setCode),
+  ]);
 
-  const data = await response.json();
+  const setMeta = sets.find((set) => set.id?.toLowerCase() === setCode.toLowerCase()) || {
-  const cards = data.data || [];
+    id: setCode,
+    name: setCode,
+  };
 
   if (cards.length === 0) {
     return { setCode, imported: 0, skipped: 0, total: 0 };
@@ -68,8 +32,10 @@ export async function importPokemonSet(setCode) {
 
   for (const card of cards) {
     try {
+      const mapped = mapGithubCardForInsert(card, setMeta);
+
       const existingCard = await sql`
-        SELECT id FROM cards WHERE scryfall_id = ${card.id}
+        SELECT id FROM cards WHERE scryfall_id = ${mapped.externalId}
       `;
 
       if (existingCard.rows.length > 0) {
@@ -77,22 +43,6 @@ export async function importPokemonSet(setCode) {
         continue;
       }
 
-      let currentPrice = null;
-      if (card.tcgplayer?.prices?.normal?.market) {
-        currentPrice = parseFloat(card.tcgplayer.prices.normal.market);
-      } else if (card.tcgplayer?.prices?.holofoil?.market) {
-        currentPrice = parseFloat(card.tcgplayer.prices.holofoil.market);
-      }
-
-      let rarity = card.rarity || 'Unknown';
-      if (rarity.includes('Holo')) {
-        rarity = 'Holographic';
-      } else if (rarity.includes('Secret')) {
-        rarity = 'Secret Rare';
-      } else if (rarity.includes('Ultra')) {
-        rarity = 'Ultra Rare';
-      }
-
       await sql`
         INSERT INTO cards (
           name, set_name, set_code, card_number, rarity, game,
@@ -100,12 +50,12 @@ export async function importPokemonSet(setCode) {
           power, toughness, image_url, stock_image_url,
           current_price, market_price, scryfall_id, verified
         ) VALUES (
-          ${card.name}, ${card.set.name}, ${card.set.id}, ${card.number},
+          ${mapped.name}, ${mapped.setName}, ${mapped.setCode}, ${mapped.cardNumber},
-          ${rarity}, 'Pokemon', null, null, ${card.supertype || 'Pokemon'},
+          ${mapped.rarity}, 'Pokemon', null, null, ${mapped.cardType},
-          ${JSON.stringify(card.types || [])}, ${card.flavorText || null},
+          ${JSON.stringify(mapped.types)}, ${mapped.flavorText},
-          ${card.attacks?.[0]?.damage || null}, null,
+          ${mapped.hp}, null,
-          ${card.images?.small || null}, ${card.images?.large || null},
+          ${mapped.imageSmall}, ${mapped.imageLarge},
-          ${currentPrice}, null, ${card.id}, true
+          null, null, ${mapped.externalId}, true
         )
       `;
 
@@ -118,5 +68,3 @@ export async function importPokemonSet(setCode) {
 
   return { setCode, imported, skipped, total: cards.length };
 }
-
-export { pokemonHeaders };

scripts/README.md

@@ -176,7 +176,7 @@ You can stop the script with `Ctrl+C` and restart it later. The scripts will sta
 ## Data Sources
 
 - **Magic: The Gathering**: Scryfall API
-- **Pokemon**: Pokemon TCG API
+- **Pokemon**: [PokemonTCG/pokemon-tcg-data](https://github.com/PokemonTCG/pokemon-tcg-data) on GitHub (sets + per-set JSON; images from linked CDNs). Optional override: `POKEMON_TCG_DATA_BASE_URL` (defaults to `master` branch raw URLs). The legacy Pokemon TCG API is no longer used by catalog sync.
 - **Lorcana**: Lorcana API (limited availability)
 
 ## Performance Notes

test/lib/card-import-pokemon-github.test.js

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  mapGithubCardForInsert,
+  pokemonSetCardsUrl,
+  pokemonSetsUrl,
+} from '../../lib/card-import/pokemon-github.js';
+
+describe('pokemon-github URLs', () => {
+  it('builds default pokemon-tcg-data raw GitHub paths', () => {
+    expect(pokemonSetsUrl()).toBe(
+      'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master/sets/en.json'
+    );
+    expect(pokemonSetCardsUrl('me3')).toBe(
+      'https://raw.githubusercontent.com/PokemonTCG/pokemon-tcg-data/master/cards/en/me3.json'
+    );
+  });
+});
+
+describe('mapGithubCardForInsert', () => {
+  it('maps card fields and formats collector number with printed total', () => {
+    const mapped = mapGithubCardForInsert(
+      {
+        id: 'me3-18',
+        name: 'Seel',
+        supertype: 'Pokémon',
+        number: '18',
+        rarity: 'Common',
+        types: ['Water'],
+        flavorText: 'The horn on its head is sharp.',
+        hp: '80',
+        images: {
+          small: 'https://images.scrydex.com/pokemon/me3-18/small',
+          large: 'https://images.scrydex.com/pokemon/me3-18/large',
+        },
+        attacks: [{ damage: '10' }],
+      },
+      { id: 'me3', name: 'Perfect Order', printedTotal: 88 }
+    );
+
+    expect(mapped).toEqual({
+      externalId: 'me3-18',
+      name: 'Seel',
+      setName: 'Perfect Order',
+      setCode: 'me3',
+      cardNumber: '18/88',
+      rarity: 'Common',
+      cardType: 'Pokémon',
+      types: ['Water'],
+      flavorText: 'The horn on its head is sharp.',
+      hp: '80',
+      imageSmall: 'https://images.scrydex.com/pokemon/me3-18/small',
+      imageLarge: 'https://images.scrydex.com/pokemon/me3-18/large',
+    });
+  });
+});