67073aab7f
Add full Scryfall bulk data pipeline:
- Migration: 13 new columns on `cards` (oracle_id, illustration_id,
color_identity, keywords, legalities, flavor_text, artist, released_at,
layout, edhrec_rank, reserved, reprint, finishes) with GIN indexes
for JSONB search.
- Migration: `tags` + `card_tags` tables for Tagger community data.
- Script: `bulk-import-scryfall.js` — downloads Oracle Cards bulk file
(168 MB) and upserts all 36k+ MTG cards with rich metadata.
- Script: `import-scryfall-tags.js` — imports oracle tags (4.5k tags,
227k taggings) and art tags (11k tags, 458k taggings).
- Lib: `bulk-sync.js` — runtime bulk sync callable from the admin API.
- Admin UI: mode toggle (incremental vs bulk) on catalog sync panel.
Enables Commander deck validation (color_identity), format legality
checks, keyword search, EDHREC popularity ranking, and functional
card tagging ("removal", "ramp", "draw") for deck building assistance.
Co-authored-by: Cursor <cursoragent@cursor.com>
456 lines
No EOL
18 KiB
JavaScript
456 lines
No EOL
18 KiB
JavaScript
import { useState, useEffect } from 'react';
|
||
import { useRouter } from 'next/router';
|
||
import dynamic from 'next/dynamic';
|
||
import Layout from '../../components/Layout';
|
||
import AdminProtected from '../../components/AdminProtected';
|
||
import { Button } from '../../components/ui';
|
||
|
||
const CardImport = () => {
|
||
const router = useRouter();
|
||
const [importType, setImportType] = useState('mtg');
|
||
const [setCode, setSetCode] = useState('');
|
||
const [isImporting, setIsImporting] = useState(false);
|
||
const [importResult, setImportResult] = useState(null);
|
||
const [isSyncing, setIsSyncing] = useState(false);
|
||
const [syncResult, setSyncResult] = useState(null);
|
||
|
||
const [syncMode, setSyncMode] = useState('incremental');
|
||
|
||
const handleCatalogSync = async () => {
|
||
setIsSyncing(true);
|
||
setSyncResult(null);
|
||
|
||
try {
|
||
const response = await fetch('/api/admin/sync-catalog', {
|
||
method: 'POST',
|
||
headers: {
|
||
'Content-Type': 'application/json',
|
||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
||
},
|
||
body: JSON.stringify({ mode: syncMode }),
|
||
});
|
||
|
||
const result = await response.json();
|
||
|
||
if (response.ok) {
|
||
setSyncResult({ success: true, ...result });
|
||
} else {
|
||
setSyncResult({
|
||
success: false,
|
||
message: result.error || 'Catalog sync failed',
|
||
});
|
||
}
|
||
} catch (error) {
|
||
setSyncResult({
|
||
success: false,
|
||
message: `Network error: ${error.message}`,
|
||
});
|
||
} finally {
|
||
setIsSyncing(false);
|
||
}
|
||
};
|
||
|
||
const handleImport = async () => {
|
||
if (!setCode.trim()) {
|
||
alert('Please enter a set code');
|
||
return;
|
||
}
|
||
|
||
setIsImporting(true);
|
||
setImportResult(null);
|
||
|
||
try {
|
||
const endpoint = importType === 'mtg'
|
||
? '/api/cards/import-mtg'
|
||
: '/api/cards/import-pokemon';
|
||
|
||
const response = await fetch(endpoint, {
|
||
method: 'POST',
|
||
headers: {
|
||
'Content-Type': 'application/json',
|
||
'Authorization': `Bearer ${localStorage.getItem('auth_token')}`,
|
||
},
|
||
body: JSON.stringify({ setCode: setCode.trim() }),
|
||
});
|
||
|
||
const result = await response.json();
|
||
|
||
if (response.ok) {
|
||
setImportResult({
|
||
success: true,
|
||
message: result.message,
|
||
imported: result.imported,
|
||
skipped: result.skipped,
|
||
total: result.total
|
||
});
|
||
} else {
|
||
setImportResult({
|
||
success: false,
|
||
message: result.error || 'Import failed'
|
||
});
|
||
}
|
||
} catch (error) {
|
||
setImportResult({
|
||
success: false,
|
||
message: 'Network error: ' + error.message
|
||
});
|
||
} finally {
|
||
setIsImporting(false);
|
||
}
|
||
};
|
||
|
||
const popularSets = {
|
||
mtg: [
|
||
{ code: 'neo', name: 'Kamigawa: Neon Dynasty' },
|
||
{ code: 'vow', name: 'Innistrad: Crimson Vow' },
|
||
{ code: 'mid', name: 'Innistrad: Midnight Hunt' },
|
||
{ code: 'afr', name: 'Adventures in the Forgotten Realms' },
|
||
{ code: 'stx', name: 'Strixhaven: School of Mages' },
|
||
{ code: 'khm', name: 'Kaldheim' },
|
||
{ code: 'znr', name: 'Zendikar Rising' },
|
||
{ code: 'iko', name: 'Ikoria: Lair of Behemoths' },
|
||
{ code: 'thb', name: 'Theros Beyond Death' },
|
||
{ code: 'eld', name: 'Throne of Eldraine' }
|
||
],
|
||
pokemon: [
|
||
{ code: 'swsh1', name: 'Sword & Shield' },
|
||
{ code: 'swsh2', name: 'Rebel Clash' },
|
||
{ code: 'swsh3', name: 'Darkness Ablaze' },
|
||
{ code: 'swsh4', name: 'Vivid Voltage' },
|
||
{ code: 'swsh5', name: 'Battle Styles' },
|
||
{ code: 'swsh6', name: 'Chilling Reign' },
|
||
{ code: 'swsh7', name: 'Evolving Skies' },
|
||
{ code: 'swsh8', name: 'Fusion Strike' },
|
||
{ code: 'swsh9', name: 'Brilliant Stars' },
|
||
{ code: 'swsh10', name: 'Astral Radiance' }
|
||
]
|
||
};
|
||
|
||
return (
|
||
<AdminProtected>
|
||
{(user) => (
|
||
<Layout user={user}>
|
||
<div className="container mx-auto px-6 py-8">
|
||
{/* Admin Navigation */}
|
||
<div className="glass-panel mb-8 p-4 rounded-xl">
|
||
<div className="flex items-center justify-between">
|
||
<h1 className="text-2xl font-bold" style={{ color: 'var(--text-primary)' }}>
|
||
Admin Tools
|
||
</h1>
|
||
<div className="flex gap-4">
|
||
<Button
|
||
variant="secondary"
|
||
onClick={() => router.push('/admin/card-editor')}
|
||
leadingIcon={<span aria-hidden="true">🖊️</span>}
|
||
>
|
||
Card Editor
|
||
</Button>
|
||
<Button
|
||
variant="primary"
|
||
onClick={() => router.push('/admin/card-import')}
|
||
leadingIcon={<span aria-hidden="true">📥</span>}
|
||
>
|
||
Card Import
|
||
</Button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Card Import Section */}
|
||
<div className="mb-6">
|
||
<h2 className="text-3xl font-bold mb-2" style={{ color: 'var(--text-primary)' }}>
|
||
Card Import Manager
|
||
</h2>
|
||
<p className="text-lg" style={{ color: 'var(--text-secondary)' }}>
|
||
Import cards from external APIs into your database
|
||
</p>
|
||
</div>
|
||
|
||
{/* Catalog sync */}
|
||
<div className="glass-panel mb-8 p-6 rounded-xl">
|
||
<div className="flex flex-col gap-4 lg:flex-row lg:items-start lg:justify-between">
|
||
<div>
|
||
<h2 className="text-xl font-semibold mb-2" style={{ color: 'var(--text-primary)' }}>
|
||
Catalog sync
|
||
</h2>
|
||
<p className="text-sm max-w-2xl" style={{ color: 'var(--text-secondary)' }}>
|
||
{syncMode === 'bulk'
|
||
? 'Download Scryfall\'s full Oracle Cards bulk file and upsert all 36k+ MTG cards with rich metadata (legalities, keywords, color identity, tags). Takes ~30s.'
|
||
: 'Import up to three newest missing MTG and Pokémon sets — the same job the weekly Vercel cron runs. Pending scan submissions are auto-linked when a unique match exists.'}
|
||
</p>
|
||
<div className="flex gap-2 mt-3">
|
||
<button
|
||
onClick={() => setSyncMode('incremental')}
|
||
className="px-3 py-1.5 rounded-lg text-xs font-medium transition-all"
|
||
style={{
|
||
backgroundColor: syncMode === 'incremental' ? 'var(--accent-ember)' : 'var(--bg-tertiary)',
|
||
color: syncMode === 'incremental' ? '#fff' : 'var(--text-secondary)',
|
||
}}
|
||
>
|
||
Incremental (new sets)
|
||
</button>
|
||
<button
|
||
onClick={() => setSyncMode('bulk')}
|
||
className="px-3 py-1.5 rounded-lg text-xs font-medium transition-all"
|
||
style={{
|
||
backgroundColor: syncMode === 'bulk' ? 'var(--accent-ember)' : 'var(--bg-tertiary)',
|
||
color: syncMode === 'bulk' ? '#fff' : 'var(--text-secondary)',
|
||
}}
|
||
>
|
||
Bulk (full MTG refresh)
|
||
</button>
|
||
</div>
|
||
</div>
|
||
<Button
|
||
variant="primary"
|
||
size="lg"
|
||
onClick={handleCatalogSync}
|
||
disabled={isSyncing}
|
||
loading={isSyncing}
|
||
className="shrink-0"
|
||
>
|
||
{isSyncing ? 'Syncing catalog…' : syncMode === 'bulk' ? 'Run bulk sync' : 'Run catalog sync'}
|
||
</Button>
|
||
</div>
|
||
|
||
{syncResult && (
|
||
<div
|
||
className={`mt-4 p-4 rounded-xl ${
|
||
syncResult.success
|
||
? 'bg-green-100 border border-green-300'
|
||
: 'bg-red-100 border border-red-300'
|
||
}`}
|
||
>
|
||
<h3
|
||
className={`font-semibold mb-2 ${
|
||
syncResult.success ? 'text-green-800' : 'text-red-800'
|
||
}`}
|
||
>
|
||
{syncResult.success ? 'Catalog sync complete' : 'Catalog sync failed'}
|
||
</h3>
|
||
{syncResult.success ? (
|
||
<div className="text-sm space-y-2 text-green-900">
|
||
{syncResult.mode === 'bulk' ? (
|
||
<p>
|
||
Upserted {syncResult.upserted} MTG cards from Scryfall bulk data.
|
||
{syncResult.backfilledOracleId > 0 && ` Backfilled oracle_id on ${syncResult.backfilledOracleId} existing rows.`}
|
||
{syncResult.errors > 0 && ` (${syncResult.errors} errors)`}
|
||
</p>
|
||
) : (
|
||
<p>
|
||
Imported {syncResult.imported} cards ({syncResult.skipped} skipped).
|
||
</p>
|
||
)}
|
||
{syncResult.submissionsReconciled > 0 && (
|
||
<p>
|
||
Linked {syncResult.submissionsReconciled} pending scan submission
|
||
{syncResult.submissionsReconciled === 1 ? '' : 's'} to catalog cards.
|
||
</p>
|
||
)}
|
||
{(syncResult.pendingMtgSets != null || syncResult.pendingPokemonSets != null) && (
|
||
<p>
|
||
Still pending: {syncResult.pendingMtgSets} MTG sets,{' '}
|
||
{syncResult.pendingPokemonSets} Pokémon sets.
|
||
</p>
|
||
)}
|
||
{syncResult.lorcana && <p>{syncResult.lorcana}</p>}
|
||
{Array.isArray(syncResult.setsProcessed) && syncResult.setsProcessed.length > 0 && (
|
||
<ul className="list-disc pl-5 space-y-1">
|
||
{syncResult.setsProcessed.map((set) => (
|
||
<li key={`${set.game}-${set.setCode}`}>
|
||
{set.name} ({set.game}/{set.setCode}) — {set.imported} imported
|
||
</li>
|
||
))}
|
||
</ul>
|
||
)}
|
||
{Array.isArray(syncResult.reconciliation) && syncResult.reconciliation.length > 0 && (
|
||
<ul className="list-disc pl-5 space-y-1">
|
||
{syncResult.reconciliation.flatMap((entry) =>
|
||
entry.details.map((detail) => (
|
||
<li key={`submission-${detail.submissionId}`}>
|
||
Submission #{detail.submissionId}: {detail.name}
|
||
{detail.cardNumber ? ` (${detail.cardNumber})` : ''} → card #{detail.cardId}
|
||
</li>
|
||
))
|
||
)}
|
||
</ul>
|
||
)}
|
||
{Array.isArray(syncResult.errors) && syncResult.errors.length > 0 && (
|
||
<ul className="list-disc pl-5 space-y-1 text-red-800">
|
||
{syncResult.errors.map((entry) => (
|
||
<li key={`${entry.game}-${entry.setCode}`}>
|
||
{entry.game}/{entry.setCode}: {entry.message}
|
||
</li>
|
||
))}
|
||
</ul>
|
||
)}
|
||
</div>
|
||
) : (
|
||
<p className="text-sm text-red-800">{syncResult.message}</p>
|
||
)}
|
||
</div>
|
||
)}
|
||
</div>
|
||
|
||
<div className="grid grid-cols-1 lg:grid-cols-2 gap-8">
|
||
{/* Import Form */}
|
||
<div className="space-y-6">
|
||
<div className="glass-panel p-6 rounded-xl">
|
||
<h2 className="text-xl font-semibold mb-4" style={{ color: 'var(--text-primary)' }}>
|
||
Import Settings
|
||
</h2>
|
||
|
||
<div className="space-y-4">
|
||
<div>
|
||
<label className="block text-sm font-medium mb-2" style={{ color: 'var(--text-primary)' }}>
|
||
TCG Type
|
||
</label>
|
||
<select
|
||
value={importType}
|
||
onChange={(e) => setImportType(e.target.value)}
|
||
className="input-field w-full"
|
||
>
|
||
<option value="mtg">Magic: The Gathering</option>
|
||
<option value="pokemon">Pokemon TCG</option>
|
||
</select>
|
||
</div>
|
||
|
||
<div>
|
||
<label className="block text-sm font-medium mb-2" style={{ color: 'var(--text-primary)' }}>
|
||
Set Code
|
||
</label>
|
||
<input
|
||
type="text"
|
||
value={setCode}
|
||
onChange={(e) => setSetCode(e.target.value)}
|
||
placeholder="e.g., neo, vow, swsh1"
|
||
className="input-field w-full"
|
||
/>
|
||
<p className="text-xs mt-1" style={{ color: 'var(--text-secondary)' }}>
|
||
Enter the set code (usually 3-4 characters)
|
||
</p>
|
||
</div>
|
||
|
||
<Button
|
||
variant="primary"
|
||
size="lg"
|
||
onClick={handleImport}
|
||
disabled={isImporting || !setCode.trim()}
|
||
loading={isImporting}
|
||
fullWidth
|
||
>
|
||
{isImporting ? 'Importing...' : 'Import Cards'}
|
||
</Button>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Import Result */}
|
||
{importResult && (
|
||
<div className={`p-6 rounded-xl ${
|
||
importResult.success
|
||
? 'bg-green-100 border border-green-300'
|
||
: 'bg-red-100 border border-red-300'
|
||
}`}>
|
||
<h3 className={`font-semibold mb-2 ${
|
||
importResult.success ? 'text-green-800' : 'text-red-800'
|
||
}`}>
|
||
{importResult.success ? 'Import Successful' : 'Import Failed'}
|
||
</h3>
|
||
<p className="text-sm mb-2">
|
||
{importResult.message}
|
||
</p>
|
||
{importResult.success && (
|
||
<div className="text-sm space-y-1">
|
||
<p>Imported: {importResult.imported}</p>
|
||
<p>Skipped: {importResult.skipped}</p>
|
||
<p>Total: {importResult.total}</p>
|
||
</div>
|
||
)}
|
||
</div>
|
||
)}
|
||
</div>
|
||
|
||
{/* Popular Sets */}
|
||
<div className="space-y-6">
|
||
<div className="glass-panel p-6 rounded-xl">
|
||
<h2 className="text-xl font-semibold mb-4" style={{ color: 'var(--text-primary)' }}>
|
||
Popular Sets
|
||
</h2>
|
||
|
||
<div className="space-y-4">
|
||
<div>
|
||
<h3 className="font-medium mb-2" style={{ color: 'var(--text-primary)' }}>
|
||
Magic: The Gathering
|
||
</h3>
|
||
<div className="grid grid-cols-2 gap-2">
|
||
{popularSets.mtg.map((set) => (
|
||
<button
|
||
key={set.code}
|
||
onClick={() => {
|
||
setImportType('mtg');
|
||
setSetCode(set.code);
|
||
}}
|
||
className="text-left p-2 rounded-lg text-sm hover:bg-opacity-20 transition-all duration-200"
|
||
style={{
|
||
backgroundColor: 'var(--bg-tertiary)',
|
||
color: 'var(--text-secondary)'
|
||
}}
|
||
>
|
||
<div className="font-medium">{set.code.toUpperCase()}</div>
|
||
<div className="text-xs opacity-75">{set.name}</div>
|
||
</button>
|
||
))}
|
||
</div>
|
||
</div>
|
||
|
||
<div>
|
||
<h3 className="font-medium mb-2" style={{ color: 'var(--text-primary)' }}>
|
||
Pokemon TCG
|
||
</h3>
|
||
<div className="grid grid-cols-2 gap-2">
|
||
{popularSets.pokemon.map((set) => (
|
||
<button
|
||
key={set.code}
|
||
onClick={() => {
|
||
setImportType('pokemon');
|
||
setSetCode(set.code);
|
||
}}
|
||
className="text-left p-2 rounded-lg text-sm hover:bg-opacity-20 transition-all duration-200"
|
||
style={{
|
||
backgroundColor: 'var(--bg-tertiary)',
|
||
color: 'var(--text-secondary)'
|
||
}}
|
||
>
|
||
<div className="font-medium">{set.code.toUpperCase()}</div>
|
||
<div className="text-xs opacity-75">{set.name}</div>
|
||
</button>
|
||
))}
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Import Tips */}
|
||
<div className="glass-panel p-6 rounded-xl">
|
||
<h2 className="text-xl font-semibold mb-4" style={{ color: 'var(--text-primary)' }}>
|
||
Import Tips
|
||
</h2>
|
||
<ul className="space-y-2 text-sm" style={{ color: 'var(--text-secondary)' }}>
|
||
<li>• Set codes are case-insensitive</li>
|
||
<li>• Duplicate cards will be skipped automatically</li>
|
||
<li>• Import may take several minutes for large sets</li>
|
||
<li>• Prices are fetched from TCGPlayer when available</li>
|
||
<li>• Images are stored as URLs to external sources</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</Layout>
|
||
)}
|
||
</AdminProtected>
|
||
);
|
||
};
|
||
|
||
// Export with dynamic import to disable SSR
|
||
export default dynamic(() => Promise.resolve(CardImport), { ssr: false }); |