From 59e2ca4ce4e64e0dd24173b183c4f809f7279ad5 Mon Sep 17 00:00:00 2001
From: Randall Stillwell <me@randallstillwell.com>
Date: Sat, 13 Jun 2026 10:49:50 -0500
Subject: [PATCH] fix(catalog-sync): add retry logic and pagination for
 Scryfall API

Scryfall was returning 429/503 transiently, causing catalog sync to
fail immediately with no recovery. Adds exponential-backoff retry
(3 attempts) for rate-limit and service-unavailable responses in both
the set discovery and card import paths. Also adds proper pagination
support for sets with 175+ cards and URL-encodes set codes.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 lib/card-import/discover.js | 19 ++++++++--
 lib/card-import/mtg.js      | 71 ++++++++++++++++++++++++++++++++++---
 2 files changed, 83 insertions(+), 7 deletions(-)

diff --git a/lib/card-import/discover.js b/lib/card-import/discover.js
index b739e4f..ddc5bbd 100644
--- a/lib/card-import/discover.js
+++ b/lib/card-import/discover.js
@@ -75,12 +75,27 @@ export async function getKnownPokemonSetCodes() {
   return new Set(rows.map((row) => row.set_code));
 }
 
+const SCRYFALL_RETRY_DELAY_MS = 2000;
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
 export async function discoverMissingMtgSets() {
-  const response = await fetch('https://api.scryfall.com/sets');
-  if (!response.ok) {
+  let response;
+  for (let attempt = 1; attempt <= 3; attempt += 1) {
+    response = await fetch('https://api.scryfall.com/sets', {
+      headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
+    });
+    if (response.ok) break;
+    if (response.status === 429 || response.status === 503) {
+      await delay(SCRYFALL_RETRY_DELAY_MS * attempt);
+      continue;
+    }
     throw new Error(`Scryfall sets API error: ${response.status}`);
   }
 
+  if (!response.ok) {
+    throw new Error(`Scryfall sets API error after retries: ${response.status}`);
+  }
+
   const data = await response.json();
   const knownCodes = await getKnownMtgSetCodes();
   return filterMissingMtgSets(data.data || [], knownCodes);
diff --git a/lib/card-import/mtg.js b/lib/card-import/mtg.js
index 0a923bf..8a49a6c 100644
--- a/lib/card-import/mtg.js
+++ b/lib/card-import/mtg.js
@@ -1,17 +1,78 @@
 import { sql } from '@vercel/postgres';
 
+const SCRYFALL_DELAY_MS = 100;
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+async function fetchScryfallWithRetry(url, maxRetries = 3) {
+  for (let attempt = 1; attempt <= maxRetries; attempt += 1) {
+    const response = await fetch(url, {
+      headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
+    });
+
+    if (response.ok) return response;
+
+    if (response.status === 429 || response.status === 503) {
+      const retryAfter = parseInt(response.headers.get('retry-after') || '2', 10);
+      const waitMs = Math.max(retryAfter * 1000, 1000 * attempt);
+      console.warn(`[fetchScryfallWithRetry] ${response.status} on attempt ${attempt}, waiting ${waitMs}ms`);
+      await delay(waitMs);
+      continue;
+    }
+
+    if (response.status === 404) {
+      return response;
+    }
+
+    throw new Error(`Scryfall API error: ${response.status} for ${url}`);
+  }
+  throw new Error(`Scryfall API: exhausted ${maxRetries} retries for ${url}`);
+}
+
+async function fetchAllScryfallPages(initialUrl) {
+  const allCards = [];
+  let url = initialUrl;
+
+  while (url) {
+    const response = await fetchScryfallWithRetry(url);
+    if (!response.ok) break;
+
+    const data = await response.json();
+    allCards.push(...(data.data || []));
+
+    if (data.has_more && data.next_page) {
+      url = data.next_page;
+      await delay(SCRYFALL_DELAY_MS);
+    } else {
+      url = null;
+    }
+  }
+
+  return allCards;
+}
+
 /**
  * Import all cards for a Scryfall set code. Skips rows already present by scryfall_id.
  */
 export async function importMtgSet(setCode) {
-  const response = await fetch(`https://api.scryfall.com/cards/search?q=set:${setCode}`);
+  const initialUrl = `https://api.scryfall.com/cards/search?q=set:${encodeURIComponent(setCode)}`;
+  const firstResponse = await fetchScryfallWithRetry(initialUrl);
 
-  if (!response.ok) {
-    throw new Error(`Scryfall API error: ${response.status}`);
+  if (firstResponse.status === 404) {
+    return { setCode, imported: 0, skipped: 0, total: 0 };
   }
 
-  const data = await response.json();
-  const cards = data.data || [];
+  if (!firstResponse.ok) {
+    throw new Error(`Scryfall API error: ${firstResponse.status} for set ${setCode}`);
+  }
+
+  const firstPage = await firstResponse.json();
+  let cards = firstPage.data || [];
+
+  if (firstPage.has_more && firstPage.next_page) {
+    await delay(SCRYFALL_DELAY_MS);
+    const remaining = await fetchAllScryfallPages(firstPage.next_page);
+    cards = cards.concat(remaining);
+  }
 
   let imported = 0;
   let skipped = 0;