feat(catalog): weekly Vercel Cron sync for MTG and Pokémon sets (#48)
Extract shared import logic into lib/card-import, discover missing sets via Scryfall/Pokémon TCG APIs, and expose GET /api/cron/sync-catalog protected by CRON_SECRET (max 3 sets/run, paced imports). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
49d1e62fa7
commit
0a47362103
11 changed files with 491 additions and 210 deletions
|
|
@ -11,16 +11,14 @@ skip:
|
|||
- visual
|
||||
- a11y
|
||||
- design
|
||||
status: open
|
||||
status: in-progress
|
||||
created: 2026-05-27
|
||||
depends_on:
|
||||
- redesign-scanner-flow
|
||||
- scanner-correctness-polish
|
||||
- add-real-ocr-layer
|
||||
blocked_by_policy: |
|
||||
Operator requested finishing the scanner pipeline and other in-flight convoys
|
||||
before starting this work. Do not pick up until those are merged or explicitly
|
||||
reprioritized.
|
||||
Unblocked 2026-05-27 after scanner pipeline + audit follow-ups merged.
|
||||
---
|
||||
|
||||
# Convoy: catalog-sync-vercel-cron
|
||||
|
|
@ -134,12 +132,12 @@ Vercel Cron (weekly)
|
|||
|
||||
## Todos
|
||||
|
||||
- [ ] Architect: ratify cron auth, import rate-limit bypass/cap, schedule cadence
|
||||
- [ ] Extract `lib/card-import/mtg.js` + `lib/card-import/pokemon.js`
|
||||
- [ ] Implement set discovery + delta diff
|
||||
- [ ] Add `/api/cron/sync-catalog` + `vercel.json` cron entry
|
||||
- [ ] Document operator setup (`CRON_SECRET`, manual trigger, monitoring)
|
||||
- [ ] Smoke: one dry-run against staging Neon branch
|
||||
- [x] Extract `lib/card-import/mtg.js` + `lib/card-import/pokemon.js`
|
||||
- [x] Implement set discovery + delta diff
|
||||
- [x] Add `/api/cron/sync-catalog` + `vercel.json` cron entry
|
||||
- [x] Document operator setup (`CRON_SECRET`, manual trigger, monitoring) — scripts/README.md
|
||||
- [ ] Architect: ratify cron auth, import rate-limit bypass/cap, schedule cadence (defaults shipped)
|
||||
- [ ] Smoke: one dry-run against staging Neon branch (operator)
|
||||
|
||||
## Operator action required (at ship time)
|
||||
|
||||
|
|
|
|||
88
lib/card-import/discover.js
Normal file
88
lib/card-import/discover.js
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import { sql } from '@vercel/postgres';
|
||||
|
||||
const EXCLUDED_MTG_SET_TYPES = new Set(['token', 'memorabilia', 'funny', 'treasure_chest']);
|
||||
|
||||
/**
|
||||
* Pure filter: Scryfall set objects not yet present in knownCodes (lowercase set codes).
|
||||
*/
|
||||
export function filterMissingMtgSets(scryfallSets, knownCodes, now = new Date()) {
|
||||
const missing = [];
|
||||
|
||||
for (const set of scryfallSets) {
|
||||
if (!set?.code || set.digital) continue;
|
||||
if (!set.released_at) continue;
|
||||
if (new Date(set.released_at) > now) continue;
|
||||
if (EXCLUDED_MTG_SET_TYPES.has(set.set_type)) continue;
|
||||
|
||||
const code = set.code.toLowerCase();
|
||||
if (knownCodes.has(code)) continue;
|
||||
|
||||
missing.push({
|
||||
code: set.code,
|
||||
name: set.name,
|
||||
releasedAt: set.released_at,
|
||||
});
|
||||
}
|
||||
|
||||
missing.sort((a, b) => a.releasedAt.localeCompare(b.releasedAt));
|
||||
return missing;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure filter: Pokémon TCG set objects not yet present in knownCodes (lowercase set ids).
|
||||
*/
|
||||
export function filterMissingPokemonSets(pokemonSets, knownCodes) {
|
||||
const missing = [];
|
||||
|
||||
for (const set of pokemonSets) {
|
||||
if (!set?.id) continue;
|
||||
const code = set.id.toLowerCase();
|
||||
if (knownCodes.has(code)) continue;
|
||||
|
||||
missing.push({
|
||||
id: set.id,
|
||||
name: set.name,
|
||||
releasedAt: set.releaseDate || null,
|
||||
});
|
||||
}
|
||||
|
||||
missing.sort((a, b) => (a.releasedAt || '').localeCompare(b.releasedAt || ''));
|
||||
return missing;
|
||||
}
|
||||
|
||||
export async function getKnownMtgSetCodes() {
|
||||
const { rows } = await sql`
|
||||
SELECT DISTINCT LOWER(set_code) AS set_code
|
||||
FROM cards
|
||||
WHERE game = 'MTG' AND set_code IS NOT NULL
|
||||
`;
|
||||
return new Set(rows.map((row) => row.set_code));
|
||||
}
|
||||
|
||||
export async function getKnownPokemonSetCodes() {
|
||||
const { rows } = await sql`
|
||||
SELECT DISTINCT LOWER(set_code) AS set_code
|
||||
FROM cards
|
||||
WHERE game = 'Pokemon' AND set_code IS NOT NULL
|
||||
`;
|
||||
return new Set(rows.map((row) => row.set_code));
|
||||
}
|
||||
|
||||
export async function discoverMissingMtgSets() {
|
||||
const response = await fetch('https://api.scryfall.com/sets');
|
||||
if (!response.ok) {
|
||||
throw new Error(`Scryfall sets API error: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const knownCodes = await getKnownMtgSetCodes();
|
||||
return filterMissingMtgSets(data.data || [], knownCodes);
|
||||
}
|
||||
|
||||
export async function discoverMissingPokemonSets() {
|
||||
const { fetchWithRetry, pokemonHeaders } = await import('./pokemon.js');
|
||||
const response = await fetchWithRetry('https://api.pokemontcg.io/v2/sets');
|
||||
const data = await response.json();
|
||||
const knownCodes = await getKnownPokemonSetCodes();
|
||||
return filterMissingPokemonSets(data.data || [], knownCodes);
|
||||
}
|
||||
62
lib/card-import/mtg.js
Normal file
62
lib/card-import/mtg.js
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { sql } from '@vercel/postgres';
|
||||
|
||||
/**
|
||||
* Import all cards for a Scryfall set code. Skips rows already present by scryfall_id.
|
||||
*/
|
||||
export async function importMtgSet(setCode) {
|
||||
const response = await fetch(`https://api.scryfall.com/cards/search?q=set:${setCode}`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Scryfall API error: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const cards = data.data || [];
|
||||
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const card of cards) {
|
||||
try {
|
||||
const existingCard = await sql`
|
||||
SELECT id FROM cards WHERE scryfall_id = ${card.id}
|
||||
`;
|
||||
|
||||
if (existingCard.rows.length > 0) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let currentPrice = null;
|
||||
let marketPrice = null;
|
||||
|
||||
if (card.prices) {
|
||||
currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null;
|
||||
marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null;
|
||||
}
|
||||
|
||||
await sql`
|
||||
INSERT INTO cards (
|
||||
name, set_name, set_code, card_number, rarity, game,
|
||||
mana_cost, cmc, card_type, colors, oracle_text,
|
||||
power, toughness, image_url, stock_image_url,
|
||||
current_price, market_price, scryfall_id, verified
|
||||
) VALUES (
|
||||
${card.name}, ${card.set_name}, ${card.set}, ${card.collector_number},
|
||||
${card.rarity}, 'MTG', ${card.mana_cost || null}, ${card.cmc || null},
|
||||
${card.type_line}, ${JSON.stringify(card.colors || [])},
|
||||
${card.oracle_text || null}, ${card.power || null}, ${card.toughness || null},
|
||||
${card.image_uris?.normal || null}, ${card.image_uris?.art_crop || null},
|
||||
${currentPrice}, ${marketPrice}, ${card.id}, true
|
||||
)
|
||||
`;
|
||||
|
||||
imported += 1;
|
||||
} catch (error) {
|
||||
console.error(`[importMtgSet] Error importing card ${card.name}:`, error);
|
||||
skipped += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return { setCode, imported, skipped, total: cards.length };
|
||||
}
|
||||
122
lib/card-import/pokemon.js
Normal file
122
lib/card-import/pokemon.js
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import { sql } from '@vercel/postgres';
|
||||
|
||||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function pokemonHeaders() {
|
||||
const headers = {
|
||||
'User-Agent': 'Deck-Hearth/1.0',
|
||||
Accept: 'application/json',
|
||||
};
|
||||
if (process.env.POKEMON_TCG_API_KEY) {
|
||||
headers['X-Api-Key'] = process.env.POKEMON_TCG_API_KEY;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
export async function fetchWithRetry(url, maxRetries = 3, delayMs = 2000) {
|
||||
for (let attempt = 1; attempt <= maxRetries; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(url, { headers: pokemonHeaders() });
|
||||
|
||||
if (response.ok) {
|
||||
return response;
|
||||
}
|
||||
|
||||
if (response.status === 404) {
|
||||
throw new Error(`Resource not found: ${response.status}`);
|
||||
}
|
||||
|
||||
if (response.status === 504 || response.status === 503) {
|
||||
console.log(
|
||||
`[fetchWithRetry] Attempt ${attempt}: ${response.status}, retrying in ${delayMs * attempt}ms`
|
||||
);
|
||||
await delay(delayMs * attempt);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Pokemon TCG API error: ${response.status}`);
|
||||
} catch (error) {
|
||||
if (attempt === maxRetries) {
|
||||
throw error;
|
||||
}
|
||||
console.log(`[fetchWithRetry] Attempt ${attempt} failed:`, error.message);
|
||||
await delay(delayMs * attempt);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('fetchWithRetry exhausted retries');
|
||||
}
|
||||
|
||||
/**
|
||||
* Import all cards for a Pokémon TCG set id. Skips rows already present by scryfall_id
|
||||
* (legacy column name stores Pokémon TCG API card ids too).
|
||||
*/
|
||||
export async function importPokemonSet(setCode) {
|
||||
const response = await fetchWithRetry(
|
||||
`https://api.pokemontcg.io/v2/cards?q=set.id:${setCode}&pageSize=250`
|
||||
);
|
||||
|
||||
const data = await response.json();
|
||||
const cards = data.data || [];
|
||||
|
||||
if (cards.length === 0) {
|
||||
return { setCode, imported: 0, skipped: 0, total: 0 };
|
||||
}
|
||||
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const card of cards) {
|
||||
try {
|
||||
const existingCard = await sql`
|
||||
SELECT id FROM cards WHERE scryfall_id = ${card.id}
|
||||
`;
|
||||
|
||||
if (existingCard.rows.length > 0) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let currentPrice = null;
|
||||
if (card.tcgplayer?.prices?.normal?.market) {
|
||||
currentPrice = parseFloat(card.tcgplayer.prices.normal.market);
|
||||
} else if (card.tcgplayer?.prices?.holofoil?.market) {
|
||||
currentPrice = parseFloat(card.tcgplayer.prices.holofoil.market);
|
||||
}
|
||||
|
||||
let rarity = card.rarity || 'Unknown';
|
||||
if (rarity.includes('Holo')) {
|
||||
rarity = 'Holographic';
|
||||
} else if (rarity.includes('Secret')) {
|
||||
rarity = 'Secret Rare';
|
||||
} else if (rarity.includes('Ultra')) {
|
||||
rarity = 'Ultra Rare';
|
||||
}
|
||||
|
||||
await sql`
|
||||
INSERT INTO cards (
|
||||
name, set_name, set_code, card_number, rarity, game,
|
||||
mana_cost, cmc, card_type, colors, oracle_text,
|
||||
power, toughness, image_url, stock_image_url,
|
||||
current_price, market_price, scryfall_id, verified
|
||||
) VALUES (
|
||||
${card.name}, ${card.set.name}, ${card.set.id}, ${card.number},
|
||||
${rarity}, 'Pokemon', null, null, ${card.supertype || 'Pokemon'},
|
||||
${JSON.stringify(card.types || [])}, ${card.flavorText || null},
|
||||
${card.attacks?.[0]?.damage || null}, null,
|
||||
${card.images?.small || null}, ${card.images?.large || null},
|
||||
${currentPrice}, null, ${card.id}, true
|
||||
)
|
||||
`;
|
||||
|
||||
imported += 1;
|
||||
} catch (error) {
|
||||
console.error(`[importPokemonSet] Error importing card ${card.name}:`, error);
|
||||
skipped += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return { setCode, imported, skipped, total: cards.length };
|
||||
}
|
||||
|
||||
export { pokemonHeaders };
|
||||
69
lib/card-import/sync-catalog.js
Normal file
69
lib/card-import/sync-catalog.js
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
import { discoverMissingMtgSets, discoverMissingPokemonSets } from './discover.js';
|
||||
import { importMtgSet } from './mtg.js';
|
||||
import { importPokemonSet } from './pokemon.js';
|
||||
|
||||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
const DEFAULT_MAX_SETS_PER_RUN = 3;
|
||||
const DEFAULT_DELAY_MS = 1500;
|
||||
|
||||
/**
|
||||
* Discover missing MTG + Pokémon sets and import up to maxSetsPerRun, paced for upstream APIs.
|
||||
*/
|
||||
export async function runCatalogSync(options = {}) {
|
||||
const maxSetsPerRun = options.maxSetsPerRun ?? DEFAULT_MAX_SETS_PER_RUN;
|
||||
const delayBetweenSetsMs = options.delayBetweenSetsMs ?? DEFAULT_DELAY_MS;
|
||||
|
||||
const [missingMtg, missingPokemon] = await Promise.all([
|
||||
discoverMissingMtgSets(),
|
||||
discoverMissingPokemonSets(),
|
||||
]);
|
||||
|
||||
const queue = [
|
||||
...missingMtg.map((set) => ({ game: 'mtg', setCode: set.code, name: set.name })),
|
||||
...missingPokemon.map((set) => ({ game: 'pokemon', setCode: set.id, name: set.name })),
|
||||
].slice(0, maxSetsPerRun);
|
||||
|
||||
const summary = {
|
||||
imported: 0,
|
||||
skipped: 0,
|
||||
errors: [],
|
||||
setsProcessed: [],
|
||||
pendingMtgSets: missingMtg.length,
|
||||
pendingPokemonSets: missingPokemon.length,
|
||||
lorcana: 'skipped — manual Lorcana set map update required',
|
||||
};
|
||||
|
||||
for (let index = 0; index < queue.length; index += 1) {
|
||||
const item = queue[index];
|
||||
try {
|
||||
const result =
|
||||
item.game === 'mtg'
|
||||
? await importMtgSet(item.setCode)
|
||||
: await importPokemonSet(item.setCode);
|
||||
|
||||
summary.setsProcessed.push({
|
||||
game: item.game,
|
||||
setCode: item.setCode,
|
||||
name: item.name,
|
||||
...result,
|
||||
});
|
||||
summary.imported += result.imported;
|
||||
summary.skipped += result.skipped;
|
||||
} catch (error) {
|
||||
console.error(`[runCatalogSync] Failed ${item.game}/${item.setCode}:`, error);
|
||||
summary.errors.push({
|
||||
game: item.game,
|
||||
setCode: item.setCode,
|
||||
message: error.message,
|
||||
});
|
||||
}
|
||||
|
||||
if (index < queue.length - 1) {
|
||||
await delay(delayBetweenSetsMs);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[runCatalogSync]', JSON.stringify(summary));
|
||||
return summary;
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { sql } from '@vercel/postgres';
|
||||
import { getUserFromRequest } from '../../../lib/permission-middleware';
|
||||
import { checkImportRateLimit } from '../../../lib/rate-limit.js';
|
||||
import { importMtgSet } from '../../../lib/card-import/mtg.js';
|
||||
|
||||
export default async function handler(req, res) {
|
||||
if (req.method !== 'POST') {
|
||||
|
|
@ -23,82 +23,23 @@ export default async function handler(req, res) {
|
|||
|
||||
try {
|
||||
const { setCode } = req.body;
|
||||
|
||||
|
||||
if (!setCode) {
|
||||
return res.status(400).json({ error: 'Set code is required' });
|
||||
}
|
||||
|
||||
// Fetch cards from Scryfall API
|
||||
const response = await fetch(`https://api.scryfall.com/cards/search?q=set:${setCode}`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Scryfall API error: ${response.status}`);
|
||||
}
|
||||
const result = await importMtgSet(setCode);
|
||||
|
||||
const data = await response.json();
|
||||
const cards = data.data || [];
|
||||
|
||||
let importedCount = 0;
|
||||
let skippedCount = 0;
|
||||
|
||||
for (const card of cards) {
|
||||
try {
|
||||
// Check if card already exists
|
||||
const existingCard = await sql`
|
||||
SELECT id FROM cards WHERE scryfall_id = ${card.id}
|
||||
`;
|
||||
|
||||
if (existingCard.rows.length > 0) {
|
||||
skippedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract price data
|
||||
let currentPrice = null;
|
||||
let marketPrice = null;
|
||||
|
||||
if (card.prices) {
|
||||
currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null;
|
||||
marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null;
|
||||
}
|
||||
|
||||
// Insert card into database
|
||||
await sql`
|
||||
INSERT INTO cards (
|
||||
name, set_name, set_code, card_number, rarity, game,
|
||||
mana_cost, cmc, card_type, colors, oracle_text,
|
||||
power, toughness, image_url, stock_image_url,
|
||||
current_price, market_price, scryfall_id, verified
|
||||
) VALUES (
|
||||
${card.name}, ${card.set_name}, ${card.set}, ${card.collector_number},
|
||||
${card.rarity}, 'MTG', ${card.mana_cost || null}, ${card.cmc || null},
|
||||
${card.type_line}, ${JSON.stringify(card.colors || [])},
|
||||
${card.oracle_text || null}, ${card.power || null}, ${card.toughness || null},
|
||||
${card.image_uris?.normal || null}, ${card.image_uris?.art_crop || null},
|
||||
${currentPrice}, ${marketPrice}, ${card.id}, true
|
||||
)
|
||||
`;
|
||||
|
||||
importedCount++;
|
||||
} catch (error) {
|
||||
console.error(`Error importing card ${card.name}:`, error);
|
||||
skippedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
res.status(200).json({
|
||||
return res.status(200).json({
|
||||
success: true,
|
||||
message: `Import completed for set ${setCode}`,
|
||||
imported: importedCount,
|
||||
skipped: skippedCount,
|
||||
total: cards.length
|
||||
...result,
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Card import error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Import failed',
|
||||
details: error.message
|
||||
return res.status(500).json({
|
||||
error: 'Import failed',
|
||||
details: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,48 +1,6 @@
|
|||
import { sql } from '@vercel/postgres';
|
||||
import { getUserFromRequest } from '../../../lib/permission-middleware';
|
||||
import { checkImportRateLimit } from '../../../lib/rate-limit.js';
|
||||
|
||||
// Helper function to delay execution
|
||||
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
// Helper function to fetch with retry logic
|
||||
async function fetchWithRetry(url, maxRetries = 3, delayMs = 2000) {
|
||||
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Deck-Hearth/1.0',
|
||||
'Accept': 'application/json'
|
||||
},
|
||||
timeout: 30000 // 30 second timeout
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
return response;
|
||||
}
|
||||
|
||||
// If it's a 404, don't retry
|
||||
if (response.status === 404) {
|
||||
throw new Error(`Set not found: ${response.status}`);
|
||||
}
|
||||
|
||||
// If it's a 504 or 503, wait longer before retry
|
||||
if (response.status === 504 || response.status === 503) {
|
||||
console.log(`Attempt ${attempt}: Got ${response.status}, waiting ${delayMs * attempt}ms before retry...`);
|
||||
await delay(delayMs * attempt);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Pokemon TCG API error: ${response.status}`);
|
||||
} catch (error) {
|
||||
if (attempt === maxRetries) {
|
||||
throw error;
|
||||
}
|
||||
console.log(`Attempt ${attempt} failed:`, error.message);
|
||||
await delay(delayMs * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
import { importPokemonSet } from '../../../lib/card-import/pokemon.js';
|
||||
|
||||
export default async function handler(req, res) {
|
||||
if (req.method !== 'POST') {
|
||||
|
|
@ -65,103 +23,27 @@ export default async function handler(req, res) {
|
|||
|
||||
try {
|
||||
const { setCode } = req.body;
|
||||
|
||||
|
||||
if (!setCode) {
|
||||
return res.status(400).json({ error: 'Set code is required' });
|
||||
}
|
||||
|
||||
console.log(`Starting import for Pokemon set: ${setCode}`);
|
||||
const result = await importPokemonSet(setCode);
|
||||
console.log(
|
||||
`Import completed for set ${setCode}: ${result.imported} imported, ${result.skipped} skipped`
|
||||
);
|
||||
|
||||
// Fetch cards from Pokemon TCG API with retry logic
|
||||
const response = await fetchWithRetry(`https://api.pokemontcg.io/v2/cards?q=set.id:${setCode}&pageSize=250`);
|
||||
|
||||
const data = await response.json();
|
||||
const cards = data.data || [];
|
||||
|
||||
if (cards.length === 0) {
|
||||
return res.status(200).json({
|
||||
success: true,
|
||||
message: `No cards found for set ${setCode}`,
|
||||
imported: 0,
|
||||
skipped: 0,
|
||||
total: 0
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Found ${cards.length} cards for set ${setCode}`);
|
||||
|
||||
let importedCount = 0;
|
||||
let skippedCount = 0;
|
||||
|
||||
for (const card of cards) {
|
||||
try {
|
||||
// Check if card already exists
|
||||
const existingCard = await sql`
|
||||
SELECT id FROM cards WHERE scryfall_id = ${card.id}
|
||||
`;
|
||||
|
||||
if (existingCard.rows.length > 0) {
|
||||
skippedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract price data from TCGPlayer
|
||||
let currentPrice = null;
|
||||
if (card.tcgplayer?.prices?.normal?.market) {
|
||||
currentPrice = parseFloat(card.tcgplayer.prices.normal.market);
|
||||
} else if (card.tcgplayer?.prices?.holofoil?.market) {
|
||||
currentPrice = parseFloat(card.tcgplayer.prices.holofoil.market);
|
||||
}
|
||||
|
||||
// Determine rarity
|
||||
let rarity = card.rarity || 'Unknown';
|
||||
if (rarity.includes('Holo')) {
|
||||
rarity = 'Holographic';
|
||||
} else if (rarity.includes('Secret')) {
|
||||
rarity = 'Secret Rare';
|
||||
} else if (rarity.includes('Ultra')) {
|
||||
rarity = 'Ultra Rare';
|
||||
}
|
||||
|
||||
// Insert card into database
|
||||
await sql`
|
||||
INSERT INTO cards (
|
||||
name, set_name, set_code, card_number, rarity, game,
|
||||
mana_cost, cmc, card_type, colors, oracle_text,
|
||||
power, toughness, image_url, stock_image_url,
|
||||
current_price, market_price, scryfall_id, verified
|
||||
) VALUES (
|
||||
${card.name}, ${card.set.name}, ${card.set.id}, ${card.number},
|
||||
${rarity}, 'Pokemon', null, null, ${card.supertype || 'Pokemon'},
|
||||
${JSON.stringify(card.types || [])}, ${card.flavorText || null},
|
||||
${card.attacks?.[0]?.damage || null}, null,
|
||||
${card.images?.small || null}, ${card.images?.large || null},
|
||||
${currentPrice}, null, ${card.id}, true
|
||||
)
|
||||
`;
|
||||
|
||||
importedCount++;
|
||||
} catch (error) {
|
||||
console.error(`Error importing card ${card.name}:`, error);
|
||||
skippedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Import completed for set ${setCode}: ${importedCount} imported, ${skippedCount} skipped`);
|
||||
|
||||
res.status(200).json({
|
||||
return res.status(200).json({
|
||||
success: true,
|
||||
message: `Import completed for set ${setCode}`,
|
||||
imported: importedCount,
|
||||
skipped: skippedCount,
|
||||
total: cards.length
|
||||
...result,
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Card import error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Import failed',
|
||||
details: error.message
|
||||
return res.status(500).json({
|
||||
error: 'Import failed',
|
||||
details: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
41
pages/api/cron/sync-catalog.js
Normal file
41
pages/api/cron/sync-catalog.js
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import { runCatalogSync } from '../../../lib/card-import/sync-catalog.js';
|
||||
|
||||
function authorizeCron(req) {
|
||||
const secret = process.env.CRON_SECRET;
|
||||
|
||||
if (!secret) {
|
||||
if (process.env.NODE_ENV === 'production' || process.env.VERCEL_ENV === 'production') {
|
||||
return { ok: false, status: 503, error: 'CRON_SECRET is not configured' };
|
||||
}
|
||||
console.warn('[GET /api/cron/sync-catalog] CRON_SECRET unset — allowing in dev');
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
const authHeader = req.headers.authorization || '';
|
||||
const token = authHeader.startsWith('Bearer ') ? authHeader.slice('Bearer '.length) : '';
|
||||
|
||||
if (!token || token !== secret) {
|
||||
return { ok: false, status: 401, error: 'Unauthorized' };
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export default async function handler(req, res) {
|
||||
if (req.method !== 'GET') {
|
||||
return res.status(405).json({ error: 'Method not allowed' });
|
||||
}
|
||||
|
||||
const auth = authorizeCron(req);
|
||||
if (!auth.ok) {
|
||||
return res.status(auth.status).json({ error: auth.error });
|
||||
}
|
||||
|
||||
try {
|
||||
const summary = await runCatalogSync();
|
||||
return res.status(200).json({ success: true, ...summary });
|
||||
} catch (error) {
|
||||
console.error('[GET /api/cron/sync-catalog]', error);
|
||||
return res.status(500).json({ error: 'Catalog sync failed', details: error.message });
|
||||
}
|
||||
}
|
||||
|
|
@ -129,7 +129,21 @@ npm run import-all
|
|||
```
|
||||
|
||||
### For Ongoing Management
|
||||
After the initial bulk import, use the admin interface at `/admin/card-import` for:
|
||||
After the initial bulk import:
|
||||
|
||||
- **Weekly catalog sync (recommended):** Vercel Cron hits `GET /api/cron/sync-catalog`
|
||||
every Monday 06:00 UTC when `CRON_SECRET` is set in the Vercel project. The job
|
||||
discovers missing MTG + Pokémon sets and imports up to 3 per run (paced for upstream
|
||||
rate limits). Manual trigger:
|
||||
|
||||
```bash
|
||||
curl -H "Authorization: Bearer $CRON_SECRET" https://<your-deployment>/api/cron/sync-catalog
|
||||
```
|
||||
|
||||
- **Admin UI:** `/admin/card-import` for one-off set imports (MTG + Pokémon).
|
||||
- **Lorcana:** still manual via admin/scripts until dynamic set discovery lands.
|
||||
|
||||
Legacy manual path (still valid):
|
||||
- Importing new sets as they release
|
||||
- Selective imports of specific sets
|
||||
- Monitoring import progress
|
||||
|
|
|
|||
53
test/lib/card-import-discover.test.js
Normal file
53
test/lib/card-import-discover.test.js
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
filterMissingMtgSets,
|
||||
filterMissingPokemonSets,
|
||||
} from '../../lib/card-import/discover.js';
|
||||
|
||||
describe('filterMissingMtgSets', () => {
|
||||
const now = new Date('2026-05-27T12:00:00.000Z');
|
||||
|
||||
it('returns released paper sets missing from the catalog', () => {
|
||||
const known = new Set(['lea']);
|
||||
const missing = filterMissingMtgSets(
|
||||
[
|
||||
{ code: 'lea', name: 'Alpha', released_at: '1993-08-05', set_type: 'core', digital: false },
|
||||
{ code: 'fin', name: 'Final Fantasy', released_at: '2026-05-01', set_type: 'expansion', digital: false },
|
||||
{ code: 'tlea', name: 'Alpha Tokens', released_at: '1993-08-05', set_type: 'token', digital: false },
|
||||
{ code: 'mh3', name: 'Modern Horizons 3', released_at: '2027-01-01', set_type: 'expansion', digital: false },
|
||||
],
|
||||
known,
|
||||
now
|
||||
);
|
||||
|
||||
expect(missing).toEqual([
|
||||
{
|
||||
code: 'fin',
|
||||
name: 'Final Fantasy',
|
||||
releasedAt: '2026-05-01',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('filterMissingPokemonSets', () => {
|
||||
it('returns sets whose ids are not in the catalog', () => {
|
||||
const known = new Set(['sv1']);
|
||||
const missing = filterMissingPokemonSets(
|
||||
[
|
||||
{ id: 'sv1', name: 'Scarlet & Violet', releaseDate: '2023-03-31' },
|
||||
{ id: 'sv2', name: 'Paldea Evolved', releaseDate: '2023-06-09' },
|
||||
],
|
||||
known
|
||||
);
|
||||
|
||||
expect(missing).toEqual([
|
||||
{
|
||||
id: 'sv2',
|
||||
name: 'Paldea Evolved',
|
||||
releasedAt: '2023-06-09',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
13
vercel.json
13
vercel.json
|
|
@ -1,3 +1,14 @@
|
|||
{
|
||||
"framework": "nextjs"
|
||||
"framework": "nextjs",
|
||||
"crons": [
|
||||
{
|
||||
"path": "/api/cron/sync-catalog",
|
||||
"schedule": "0 6 * * 1"
|
||||
}
|
||||
],
|
||||
"functions": {
|
||||
"pages/api/cron/sync-catalog.js": {
|
||||
"maxDuration": 300
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue