feat(catalog): weekly Vercel Cron sync for MTG and Pokémon sets

Extract shared import logic into lib/card-import, discover missing sets via
Scryfall/Pokémon TCG APIs, and expose GET /api/cron/sync-catalog protected
by CRON_SECRET (max 3 sets/run, paced imports).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Randall Stillwell 2026-05-27 14:47:36 -05:00
parent 49d1e62fa7
commit 3849f54d05
11 changed files with 491 additions and 210 deletions

View file

@ -11,16 +11,14 @@ skip:
- visual - visual
- a11y - a11y
- design - design
status: open status: in-progress
created: 2026-05-27 created: 2026-05-27
depends_on: depends_on:
- redesign-scanner-flow - redesign-scanner-flow
- scanner-correctness-polish - scanner-correctness-polish
- add-real-ocr-layer - add-real-ocr-layer
blocked_by_policy: | blocked_by_policy: |
Operator requested finishing the scanner pipeline and other in-flight convoys Unblocked 2026-05-27 after scanner pipeline + audit follow-ups merged.
before starting this work. Do not pick up until those are merged or explicitly
reprioritized.
--- ---
# Convoy: catalog-sync-vercel-cron # Convoy: catalog-sync-vercel-cron
@ -134,12 +132,12 @@ Vercel Cron (weekly)
## Todos ## Todos
- [ ] Architect: ratify cron auth, import rate-limit bypass/cap, schedule cadence - [x] Extract `lib/card-import/mtg.js` + `lib/card-import/pokemon.js`
- [ ] Extract `lib/card-import/mtg.js` + `lib/card-import/pokemon.js` - [x] Implement set discovery + delta diff
- [ ] Implement set discovery + delta diff - [x] Add `/api/cron/sync-catalog` + `vercel.json` cron entry
- [ ] Add `/api/cron/sync-catalog` + `vercel.json` cron entry - [x] Document operator setup (`CRON_SECRET`, manual trigger, monitoring) — scripts/README.md
- [ ] Document operator setup (`CRON_SECRET`, manual trigger, monitoring) - [ ] Architect: ratify cron auth, import rate-limit bypass/cap, schedule cadence (defaults shipped)
- [ ] Smoke: one dry-run against staging Neon branch - [ ] Smoke: one dry-run against staging Neon branch (operator)
## Operator action required (at ship time) ## Operator action required (at ship time)

View file

@ -0,0 +1,88 @@
import { sql } from '@vercel/postgres';
const EXCLUDED_MTG_SET_TYPES = new Set(['token', 'memorabilia', 'funny', 'treasure_chest']);
/**
* Pure filter: Scryfall set objects not yet present in knownCodes (lowercase set codes).
*/
export function filterMissingMtgSets(scryfallSets, knownCodes, now = new Date()) {
const missing = [];
for (const set of scryfallSets) {
if (!set?.code || set.digital) continue;
if (!set.released_at) continue;
if (new Date(set.released_at) > now) continue;
if (EXCLUDED_MTG_SET_TYPES.has(set.set_type)) continue;
const code = set.code.toLowerCase();
if (knownCodes.has(code)) continue;
missing.push({
code: set.code,
name: set.name,
releasedAt: set.released_at,
});
}
missing.sort((a, b) => a.releasedAt.localeCompare(b.releasedAt));
return missing;
}
/**
* Pure filter: Pokémon TCG set objects not yet present in knownCodes (lowercase set ids).
*/
export function filterMissingPokemonSets(pokemonSets, knownCodes) {
const missing = [];
for (const set of pokemonSets) {
if (!set?.id) continue;
const code = set.id.toLowerCase();
if (knownCodes.has(code)) continue;
missing.push({
id: set.id,
name: set.name,
releasedAt: set.releaseDate || null,
});
}
missing.sort((a, b) => (a.releasedAt || '').localeCompare(b.releasedAt || ''));
return missing;
}
export async function getKnownMtgSetCodes() {
const { rows } = await sql`
SELECT DISTINCT LOWER(set_code) AS set_code
FROM cards
WHERE game = 'MTG' AND set_code IS NOT NULL
`;
return new Set(rows.map((row) => row.set_code));
}
export async function getKnownPokemonSetCodes() {
const { rows } = await sql`
SELECT DISTINCT LOWER(set_code) AS set_code
FROM cards
WHERE game = 'Pokemon' AND set_code IS NOT NULL
`;
return new Set(rows.map((row) => row.set_code));
}
export async function discoverMissingMtgSets() {
const response = await fetch('https://api.scryfall.com/sets');
if (!response.ok) {
throw new Error(`Scryfall sets API error: ${response.status}`);
}
const data = await response.json();
const knownCodes = await getKnownMtgSetCodes();
return filterMissingMtgSets(data.data || [], knownCodes);
}
export async function discoverMissingPokemonSets() {
const { fetchWithRetry, pokemonHeaders } = await import('./pokemon.js');
const response = await fetchWithRetry('https://api.pokemontcg.io/v2/sets');
const data = await response.json();
const knownCodes = await getKnownPokemonSetCodes();
return filterMissingPokemonSets(data.data || [], knownCodes);
}

62
lib/card-import/mtg.js Normal file
View file

@ -0,0 +1,62 @@
import { sql } from '@vercel/postgres';
/**
* Import all cards for a Scryfall set code. Skips rows already present by scryfall_id.
*/
export async function importMtgSet(setCode) {
const response = await fetch(`https://api.scryfall.com/cards/search?q=set:${setCode}`);
if (!response.ok) {
throw new Error(`Scryfall API error: ${response.status}`);
}
const data = await response.json();
const cards = data.data || [];
let imported = 0;
let skipped = 0;
for (const card of cards) {
try {
const existingCard = await sql`
SELECT id FROM cards WHERE scryfall_id = ${card.id}
`;
if (existingCard.rows.length > 0) {
skipped += 1;
continue;
}
let currentPrice = null;
let marketPrice = null;
if (card.prices) {
currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null;
marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null;
}
await sql`
INSERT INTO cards (
name, set_name, set_code, card_number, rarity, game,
mana_cost, cmc, card_type, colors, oracle_text,
power, toughness, image_url, stock_image_url,
current_price, market_price, scryfall_id, verified
) VALUES (
${card.name}, ${card.set_name}, ${card.set}, ${card.collector_number},
${card.rarity}, 'MTG', ${card.mana_cost || null}, ${card.cmc || null},
${card.type_line}, ${JSON.stringify(card.colors || [])},
${card.oracle_text || null}, ${card.power || null}, ${card.toughness || null},
${card.image_uris?.normal || null}, ${card.image_uris?.art_crop || null},
${currentPrice}, ${marketPrice}, ${card.id}, true
)
`;
imported += 1;
} catch (error) {
console.error(`[importMtgSet] Error importing card ${card.name}:`, error);
skipped += 1;
}
}
return { setCode, imported, skipped, total: cards.length };
}

122
lib/card-import/pokemon.js Normal file
View file

@ -0,0 +1,122 @@
import { sql } from '@vercel/postgres';
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
function pokemonHeaders() {
const headers = {
'User-Agent': 'Deck-Hearth/1.0',
Accept: 'application/json',
};
if (process.env.POKEMON_TCG_API_KEY) {
headers['X-Api-Key'] = process.env.POKEMON_TCG_API_KEY;
}
return headers;
}
export async function fetchWithRetry(url, maxRetries = 3, delayMs = 2000) {
for (let attempt = 1; attempt <= maxRetries; attempt += 1) {
try {
const response = await fetch(url, { headers: pokemonHeaders() });
if (response.ok) {
return response;
}
if (response.status === 404) {
throw new Error(`Resource not found: ${response.status}`);
}
if (response.status === 504 || response.status === 503) {
console.log(
`[fetchWithRetry] Attempt ${attempt}: ${response.status}, retrying in ${delayMs * attempt}ms`
);
await delay(delayMs * attempt);
continue;
}
throw new Error(`Pokemon TCG API error: ${response.status}`);
} catch (error) {
if (attempt === maxRetries) {
throw error;
}
console.log(`[fetchWithRetry] Attempt ${attempt} failed:`, error.message);
await delay(delayMs * attempt);
}
}
throw new Error('fetchWithRetry exhausted retries');
}
/**
* Import all cards for a Pokémon TCG set id. Skips rows already present by scryfall_id
* (legacy column name stores Pokémon TCG API card ids too).
*/
export async function importPokemonSet(setCode) {
const response = await fetchWithRetry(
`https://api.pokemontcg.io/v2/cards?q=set.id:${setCode}&pageSize=250`
);
const data = await response.json();
const cards = data.data || [];
if (cards.length === 0) {
return { setCode, imported: 0, skipped: 0, total: 0 };
}
let imported = 0;
let skipped = 0;
for (const card of cards) {
try {
const existingCard = await sql`
SELECT id FROM cards WHERE scryfall_id = ${card.id}
`;
if (existingCard.rows.length > 0) {
skipped += 1;
continue;
}
let currentPrice = null;
if (card.tcgplayer?.prices?.normal?.market) {
currentPrice = parseFloat(card.tcgplayer.prices.normal.market);
} else if (card.tcgplayer?.prices?.holofoil?.market) {
currentPrice = parseFloat(card.tcgplayer.prices.holofoil.market);
}
let rarity = card.rarity || 'Unknown';
if (rarity.includes('Holo')) {
rarity = 'Holographic';
} else if (rarity.includes('Secret')) {
rarity = 'Secret Rare';
} else if (rarity.includes('Ultra')) {
rarity = 'Ultra Rare';
}
await sql`
INSERT INTO cards (
name, set_name, set_code, card_number, rarity, game,
mana_cost, cmc, card_type, colors, oracle_text,
power, toughness, image_url, stock_image_url,
current_price, market_price, scryfall_id, verified
) VALUES (
${card.name}, ${card.set.name}, ${card.set.id}, ${card.number},
${rarity}, 'Pokemon', null, null, ${card.supertype || 'Pokemon'},
${JSON.stringify(card.types || [])}, ${card.flavorText || null},
${card.attacks?.[0]?.damage || null}, null,
${card.images?.small || null}, ${card.images?.large || null},
${currentPrice}, null, ${card.id}, true
)
`;
imported += 1;
} catch (error) {
console.error(`[importPokemonSet] Error importing card ${card.name}:`, error);
skipped += 1;
}
}
return { setCode, imported, skipped, total: cards.length };
}
export { pokemonHeaders };

View file

@ -0,0 +1,69 @@
import { discoverMissingMtgSets, discoverMissingPokemonSets } from './discover.js';
import { importMtgSet } from './mtg.js';
import { importPokemonSet } from './pokemon.js';
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const DEFAULT_MAX_SETS_PER_RUN = 3;
const DEFAULT_DELAY_MS = 1500;
/**
* Discover missing MTG + Pokémon sets and import up to maxSetsPerRun, paced for upstream APIs.
*/
export async function runCatalogSync(options = {}) {
const maxSetsPerRun = options.maxSetsPerRun ?? DEFAULT_MAX_SETS_PER_RUN;
const delayBetweenSetsMs = options.delayBetweenSetsMs ?? DEFAULT_DELAY_MS;
const [missingMtg, missingPokemon] = await Promise.all([
discoverMissingMtgSets(),
discoverMissingPokemonSets(),
]);
const queue = [
...missingMtg.map((set) => ({ game: 'mtg', setCode: set.code, name: set.name })),
...missingPokemon.map((set) => ({ game: 'pokemon', setCode: set.id, name: set.name })),
].slice(0, maxSetsPerRun);
const summary = {
imported: 0,
skipped: 0,
errors: [],
setsProcessed: [],
pendingMtgSets: missingMtg.length,
pendingPokemonSets: missingPokemon.length,
lorcana: 'skipped — manual Lorcana set map update required',
};
for (let index = 0; index < queue.length; index += 1) {
const item = queue[index];
try {
const result =
item.game === 'mtg'
? await importMtgSet(item.setCode)
: await importPokemonSet(item.setCode);
summary.setsProcessed.push({
game: item.game,
setCode: item.setCode,
name: item.name,
...result,
});
summary.imported += result.imported;
summary.skipped += result.skipped;
} catch (error) {
console.error(`[runCatalogSync] Failed ${item.game}/${item.setCode}:`, error);
summary.errors.push({
game: item.game,
setCode: item.setCode,
message: error.message,
});
}
if (index < queue.length - 1) {
await delay(delayBetweenSetsMs);
}
}
console.log('[runCatalogSync]', JSON.stringify(summary));
return summary;
}

View file

@ -1,6 +1,6 @@
import { sql } from '@vercel/postgres';
import { getUserFromRequest } from '../../../lib/permission-middleware'; import { getUserFromRequest } from '../../../lib/permission-middleware';
import { checkImportRateLimit } from '../../../lib/rate-limit.js'; import { checkImportRateLimit } from '../../../lib/rate-limit.js';
import { importMtgSet } from '../../../lib/card-import/mtg.js';
export default async function handler(req, res) { export default async function handler(req, res) {
if (req.method !== 'POST') { if (req.method !== 'POST') {
@ -28,77 +28,18 @@ export default async function handler(req, res) {
return res.status(400).json({ error: 'Set code is required' }); return res.status(400).json({ error: 'Set code is required' });
} }
// Fetch cards from Scryfall API const result = await importMtgSet(setCode);
const response = await fetch(`https://api.scryfall.com/cards/search?q=set:${setCode}`);
if (!response.ok) { return res.status(200).json({
throw new Error(`Scryfall API error: ${response.status}`);
}
const data = await response.json();
const cards = data.data || [];
let importedCount = 0;
let skippedCount = 0;
for (const card of cards) {
try {
// Check if card already exists
const existingCard = await sql`
SELECT id FROM cards WHERE scryfall_id = ${card.id}
`;
if (existingCard.rows.length > 0) {
skippedCount++;
continue;
}
// Extract price data
let currentPrice = null;
let marketPrice = null;
if (card.prices) {
currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null;
marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null;
}
// Insert card into database
await sql`
INSERT INTO cards (
name, set_name, set_code, card_number, rarity, game,
mana_cost, cmc, card_type, colors, oracle_text,
power, toughness, image_url, stock_image_url,
current_price, market_price, scryfall_id, verified
) VALUES (
${card.name}, ${card.set_name}, ${card.set}, ${card.collector_number},
${card.rarity}, 'MTG', ${card.mana_cost || null}, ${card.cmc || null},
${card.type_line}, ${JSON.stringify(card.colors || [])},
${card.oracle_text || null}, ${card.power || null}, ${card.toughness || null},
${card.image_uris?.normal || null}, ${card.image_uris?.art_crop || null},
${currentPrice}, ${marketPrice}, ${card.id}, true
)
`;
importedCount++;
} catch (error) {
console.error(`Error importing card ${card.name}:`, error);
skippedCount++;
}
}
res.status(200).json({
success: true, success: true,
message: `Import completed for set ${setCode}`, message: `Import completed for set ${setCode}`,
imported: importedCount, ...result,
skipped: skippedCount,
total: cards.length
}); });
} catch (error) { } catch (error) {
console.error('Card import error:', error); console.error('Card import error:', error);
res.status(500).json({ return res.status(500).json({
error: 'Import failed', error: 'Import failed',
details: error.message details: error.message,
}); });
} }
} }

View file

@ -1,48 +1,6 @@
import { sql } from '@vercel/postgres';
import { getUserFromRequest } from '../../../lib/permission-middleware'; import { getUserFromRequest } from '../../../lib/permission-middleware';
import { checkImportRateLimit } from '../../../lib/rate-limit.js'; import { checkImportRateLimit } from '../../../lib/rate-limit.js';
import { importPokemonSet } from '../../../lib/card-import/pokemon.js';
// Helper function to delay execution
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
// Helper function to fetch with retry logic
async function fetchWithRetry(url, maxRetries = 3, delayMs = 2000) {
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
const response = await fetch(url, {
headers: {
'User-Agent': 'Deck-Hearth/1.0',
'Accept': 'application/json'
},
timeout: 30000 // 30 second timeout
});
if (response.ok) {
return response;
}
// If it's a 404, don't retry
if (response.status === 404) {
throw new Error(`Set not found: ${response.status}`);
}
// If it's a 504 or 503, wait longer before retry
if (response.status === 504 || response.status === 503) {
console.log(`Attempt ${attempt}: Got ${response.status}, waiting ${delayMs * attempt}ms before retry...`);
await delay(delayMs * attempt);
continue;
}
throw new Error(`Pokemon TCG API error: ${response.status}`);
} catch (error) {
if (attempt === maxRetries) {
throw error;
}
console.log(`Attempt ${attempt} failed:`, error.message);
await delay(delayMs * attempt);
}
}
}
export default async function handler(req, res) { export default async function handler(req, res) {
if (req.method !== 'POST') { if (req.method !== 'POST') {
@ -71,97 +29,21 @@ export default async function handler(req, res) {
} }
console.log(`Starting import for Pokemon set: ${setCode}`); console.log(`Starting import for Pokemon set: ${setCode}`);
const result = await importPokemonSet(setCode);
console.log(
`Import completed for set ${setCode}: ${result.imported} imported, ${result.skipped} skipped`
);
// Fetch cards from Pokemon TCG API with retry logic
const response = await fetchWithRetry(`https://api.pokemontcg.io/v2/cards?q=set.id:${setCode}&pageSize=250`);
const data = await response.json();
const cards = data.data || [];
if (cards.length === 0) {
return res.status(200).json({ return res.status(200).json({
success: true,
message: `No cards found for set ${setCode}`,
imported: 0,
skipped: 0,
total: 0
});
}
console.log(`Found ${cards.length} cards for set ${setCode}`);
let importedCount = 0;
let skippedCount = 0;
for (const card of cards) {
try {
// Check if card already exists
const existingCard = await sql`
SELECT id FROM cards WHERE scryfall_id = ${card.id}
`;
if (existingCard.rows.length > 0) {
skippedCount++;
continue;
}
// Extract price data from TCGPlayer
let currentPrice = null;
if (card.tcgplayer?.prices?.normal?.market) {
currentPrice = parseFloat(card.tcgplayer.prices.normal.market);
} else if (card.tcgplayer?.prices?.holofoil?.market) {
currentPrice = parseFloat(card.tcgplayer.prices.holofoil.market);
}
// Determine rarity
let rarity = card.rarity || 'Unknown';
if (rarity.includes('Holo')) {
rarity = 'Holographic';
} else if (rarity.includes('Secret')) {
rarity = 'Secret Rare';
} else if (rarity.includes('Ultra')) {
rarity = 'Ultra Rare';
}
// Insert card into database
await sql`
INSERT INTO cards (
name, set_name, set_code, card_number, rarity, game,
mana_cost, cmc, card_type, colors, oracle_text,
power, toughness, image_url, stock_image_url,
current_price, market_price, scryfall_id, verified
) VALUES (
${card.name}, ${card.set.name}, ${card.set.id}, ${card.number},
${rarity}, 'Pokemon', null, null, ${card.supertype || 'Pokemon'},
${JSON.stringify(card.types || [])}, ${card.flavorText || null},
${card.attacks?.[0]?.damage || null}, null,
${card.images?.small || null}, ${card.images?.large || null},
${currentPrice}, null, ${card.id}, true
)
`;
importedCount++;
} catch (error) {
console.error(`Error importing card ${card.name}:`, error);
skippedCount++;
}
}
console.log(`Import completed for set ${setCode}: ${importedCount} imported, ${skippedCount} skipped`);
res.status(200).json({
success: true, success: true,
message: `Import completed for set ${setCode}`, message: `Import completed for set ${setCode}`,
imported: importedCount, ...result,
skipped: skippedCount,
total: cards.length
}); });
} catch (error) { } catch (error) {
console.error('Card import error:', error); console.error('Card import error:', error);
res.status(500).json({ return res.status(500).json({
error: 'Import failed', error: 'Import failed',
details: error.message details: error.message,
}); });
} }
} }

View file

@ -0,0 +1,41 @@
import { runCatalogSync } from '../../../lib/card-import/sync-catalog.js';
function authorizeCron(req) {
const secret = process.env.CRON_SECRET;
if (!secret) {
if (process.env.NODE_ENV === 'production' || process.env.VERCEL_ENV === 'production') {
return { ok: false, status: 503, error: 'CRON_SECRET is not configured' };
}
console.warn('[GET /api/cron/sync-catalog] CRON_SECRET unset — allowing in dev');
return { ok: true };
}
const authHeader = req.headers.authorization || '';
const token = authHeader.startsWith('Bearer ') ? authHeader.slice('Bearer '.length) : '';
if (!token || token !== secret) {
return { ok: false, status: 401, error: 'Unauthorized' };
}
return { ok: true };
}
export default async function handler(req, res) {
if (req.method !== 'GET') {
return res.status(405).json({ error: 'Method not allowed' });
}
const auth = authorizeCron(req);
if (!auth.ok) {
return res.status(auth.status).json({ error: auth.error });
}
try {
const summary = await runCatalogSync();
return res.status(200).json({ success: true, ...summary });
} catch (error) {
console.error('[GET /api/cron/sync-catalog]', error);
return res.status(500).json({ error: 'Catalog sync failed', details: error.message });
}
}

View file

@ -129,7 +129,21 @@ npm run import-all
``` ```
### For Ongoing Management ### For Ongoing Management
After the initial bulk import, use the admin interface at `/admin/card-import` for: After the initial bulk import:
- **Weekly catalog sync (recommended):** Vercel Cron hits `GET /api/cron/sync-catalog`
every Monday 06:00 UTC when `CRON_SECRET` is set in the Vercel project. The job
discovers missing MTG + Pokémon sets and imports up to 3 per run (paced for upstream
rate limits). Manual trigger:
```bash
curl -H "Authorization: Bearer $CRON_SECRET" https://<your-deployment>/api/cron/sync-catalog
```
- **Admin UI:** `/admin/card-import` for one-off set imports (MTG + Pokémon).
- **Lorcana:** still manual via admin/scripts until dynamic set discovery lands.
Legacy manual path (still valid):
- Importing new sets as they release - Importing new sets as they release
- Selective imports of specific sets - Selective imports of specific sets
- Monitoring import progress - Monitoring import progress

View file

@ -0,0 +1,53 @@
import { describe, expect, it } from 'vitest';
import {
filterMissingMtgSets,
filterMissingPokemonSets,
} from '../../lib/card-import/discover.js';
describe('filterMissingMtgSets', () => {
const now = new Date('2026-05-27T12:00:00.000Z');
it('returns released paper sets missing from the catalog', () => {
const known = new Set(['lea']);
const missing = filterMissingMtgSets(
[
{ code: 'lea', name: 'Alpha', released_at: '1993-08-05', set_type: 'core', digital: false },
{ code: 'fin', name: 'Final Fantasy', released_at: '2026-05-01', set_type: 'expansion', digital: false },
{ code: 'tlea', name: 'Alpha Tokens', released_at: '1993-08-05', set_type: 'token', digital: false },
{ code: 'mh3', name: 'Modern Horizons 3', released_at: '2027-01-01', set_type: 'expansion', digital: false },
],
known,
now
);
expect(missing).toEqual([
{
code: 'fin',
name: 'Final Fantasy',
releasedAt: '2026-05-01',
},
]);
});
});
describe('filterMissingPokemonSets', () => {
it('returns sets whose ids are not in the catalog', () => {
const known = new Set(['sv1']);
const missing = filterMissingPokemonSets(
[
{ id: 'sv1', name: 'Scarlet & Violet', releaseDate: '2023-03-31' },
{ id: 'sv2', name: 'Paldea Evolved', releaseDate: '2023-06-09' },
],
known
);
expect(missing).toEqual([
{
id: 'sv2',
name: 'Paldea Evolved',
releasedAt: '2023-06-09',
},
]);
});
});

View file

@ -1,3 +1,14 @@
{ {
"framework": "nextjs" "framework": "nextjs",
"crons": [
{
"path": "/api/cron/sync-catalog",
"schedule": "0 6 * * 1"
}
],
"functions": {
"pages/api/cron/sync-catalog.js": {
"maxDuration": 300
}
}
} }