59e2ca4ce4
Scryfall was returning 429/503 transiently, causing catalog sync to fail immediately with no recovery. Adds exponential-backoff retry (3 attempts) for rate-limit and service-unavailable responses in both the set discovery and card import paths. Also adds proper pagination support for sets with 175+ cards and URL-encodes set codes. Co-authored-by: Cursor <cursoragent@cursor.com>
109 lines
3.1 KiB
JavaScript
109 lines
3.1 KiB
JavaScript
import { sql } from '@vercel/postgres';
|
|
|
|
const EXCLUDED_MTG_SET_TYPES = new Set(['token', 'memorabilia', 'funny', 'treasure_chest']);
|
|
|
|
/**
|
|
* Pure filter: Scryfall set objects not yet present in knownCodes (lowercase set codes).
|
|
*/
|
|
export function filterMissingMtgSets(scryfallSets, knownCodes, now = new Date()) {
|
|
const missing = [];
|
|
|
|
for (const set of scryfallSets) {
|
|
if (!set?.code || set.digital) continue;
|
|
if (!set.released_at) continue;
|
|
if (new Date(set.released_at) > now) continue;
|
|
if (EXCLUDED_MTG_SET_TYPES.has(set.set_type)) continue;
|
|
|
|
const code = set.code.toLowerCase();
|
|
if (knownCodes.has(code)) continue;
|
|
|
|
missing.push({
|
|
code: set.code,
|
|
name: set.name,
|
|
releasedAt: set.released_at,
|
|
});
|
|
}
|
|
|
|
missing.sort((a, b) => b.releasedAt.localeCompare(a.releasedAt));
|
|
return missing;
|
|
}
|
|
|
|
/**
|
|
* Pure filter: Pokémon TCG set objects not yet present in knownCodes (lowercase set ids).
|
|
*/
|
|
export function filterMissingPokemonSets(pokemonSets, knownCodes) {
|
|
const missing = [];
|
|
|
|
for (const set of pokemonSets) {
|
|
if (!set?.id) continue;
|
|
const code = set.id.toLowerCase();
|
|
if (knownCodes.has(code)) continue;
|
|
|
|
missing.push({
|
|
id: set.id,
|
|
name: set.name,
|
|
releasedAt: set.releaseDate || null,
|
|
});
|
|
}
|
|
|
|
missing.sort((a, b) => {
|
|
const dateA = a.releasedAt || '';
|
|
const dateB = b.releasedAt || '';
|
|
if (!dateA && !dateB) return 0;
|
|
if (!dateA) return 1;
|
|
if (!dateB) return -1;
|
|
return dateB.localeCompare(dateA);
|
|
});
|
|
return missing;
|
|
}
|
|
|
|
export async function getKnownMtgSetCodes() {
|
|
const { rows } = await sql`
|
|
SELECT DISTINCT LOWER(set_code) AS set_code
|
|
FROM cards
|
|
WHERE game = 'MTG' AND set_code IS NOT NULL
|
|
`;
|
|
return new Set(rows.map((row) => row.set_code));
|
|
}
|
|
|
|
export async function getKnownPokemonSetCodes() {
|
|
const { rows } = await sql`
|
|
SELECT DISTINCT LOWER(set_code) AS set_code
|
|
FROM cards
|
|
WHERE game = 'Pokemon' AND set_code IS NOT NULL
|
|
`;
|
|
return new Set(rows.map((row) => row.set_code));
|
|
}
|
|
|
|
const SCRYFALL_RETRY_DELAY_MS = 2000;
|
|
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
export async function discoverMissingMtgSets() {
|
|
let response;
|
|
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
|
response = await fetch('https://api.scryfall.com/sets', {
|
|
headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
|
|
});
|
|
if (response.ok) break;
|
|
if (response.status === 429 || response.status === 503) {
|
|
await delay(SCRYFALL_RETRY_DELAY_MS * attempt);
|
|
continue;
|
|
}
|
|
throw new Error(`Scryfall sets API error: ${response.status}`);
|
|
}
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`Scryfall sets API error after retries: ${response.status}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
const knownCodes = await getKnownMtgSetCodes();
|
|
return filterMissingMtgSets(data.data || [], knownCodes);
|
|
}
|
|
|
|
export async function discoverMissingPokemonSets() {
|
|
const { fetchPokemonSets } = await import('./pokemon-github.js');
|
|
const sets = await fetchPokemonSets();
|
|
const knownCodes = await getKnownPokemonSetCodes();
|
|
return filterMissingPokemonSets(sets, knownCodes);
|
|
}
|