- Import up to three newest missing MTG and Pokémon sets — the same job the weekly Vercel cron runs.
- Pending scan submissions for those sets are auto-linked to the catalog when a unique match exists.
+ {syncMode === 'bulk'
+ ? 'Download Scryfall\'s full Oracle Cards bulk file and upsert all 36k+ MTG cards with rich metadata (legalities, keywords, color identity, tags). Takes ~30s.'
+ : 'Import up to three newest missing MTG and Pokémon sets — the same job the weekly Vercel cron runs. Pending scan submissions are auto-linked when a unique match exists.'}
+
+ {syncResult.mode === 'bulk' ? (
+
+ Upserted {syncResult.upserted} MTG cards from Scryfall bulk data.
+ {syncResult.backfilledOracleId > 0 && ` Backfilled oracle_id on ${syncResult.backfilledOracleId} existing rows.`}
+ {syncResult.errors > 0 && ` (${syncResult.errors} errors)`}
+
+ ) : (
Imported {syncResult.imported} cards ({syncResult.skipped} skipped).
+ )}
{syncResult.submissionsReconciled > 0 && (
Linked {syncResult.submissionsReconciled} pending scan submission
diff --git a/pages/api/admin/sync-catalog.js b/pages/api/admin/sync-catalog.js
index d74b855..e982aa5 100644
--- a/pages/api/admin/sync-catalog.js
+++ b/pages/api/admin/sync-catalog.js
@@ -1,6 +1,7 @@
import { withAdmin } from '../../../lib/permission-middleware';
import { checkImportRateLimit } from '../../../lib/rate-limit.js';
import { runCatalogSync } from '../../../lib/card-import/sync-catalog.js';
+import { runBulkMtgSync } from '../../../lib/card-import/bulk-sync.js';
export default withAdmin(async function handler(req, res, user) {
if (req.method !== 'POST') {
@@ -13,7 +14,14 @@ export default withAdmin(async function handler(req, res, user) {
return res.status(429).json({ error: 'Too many attempts. Try again later.' });
}
+ const mode = req.body?.mode || 'incremental';
+
try {
+ if (mode === 'bulk') {
+ const summary = await runBulkMtgSync();
+ return res.status(200).json({ success: true, ...summary });
+ }
+
const summary = await runCatalogSync();
return res.status(200).json({ success: true, ...summary });
} catch (error) {
diff --git a/scripts/bulk-import-scryfall.js b/scripts/bulk-import-scryfall.js
new file mode 100644
index 0000000..12dd9a3
--- /dev/null
+++ b/scripts/bulk-import-scryfall.js
@@ -0,0 +1,217 @@
+/**
+ * Bulk import MTG cards from Scryfall's Oracle Cards bulk data file.
+ *
+ * Downloads the ~168 MB Oracle Cards JSON (one entry per unique card/oracle_id),
+ * then upserts every card into the `cards` table keyed on `scryfall_id`.
+ *
+ * Populates all new bulk-data columns (oracle_id, illustration_id, color_identity,
+ * keywords, legalities, flavor_text, artist, released_at, layout, edhrec_rank,
+ * reserved, reprint, finishes) in addition to the original columns.
+ *
+ * Usage:
+ * POSTGRES_URL= node scripts/bulk-import-scryfall.js
+ *
+ * Options (env vars):
+ * BULK_TYPE — "oracle_cards" (default) | "default_cards" | "unique_artwork"
+ * BATCH_SIZE — rows per INSERT batch (default 100)
+ * DRY_RUN — "true" to fetch and count without writing to DB
+ */
+
+import { neon } from '@neondatabase/serverless';
+
+if (!process.env.POSTGRES_URL) {
+ console.error('POSTGRES_URL is required');
+ process.exit(1);
+}
+
+const sql = neon(process.env.POSTGRES_URL, { fullResults: false });
+
+const BULK_TYPE = process.env.BULK_TYPE || 'oracle_cards';
+const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '100', 10);
+const DRY_RUN = process.env.DRY_RUN === 'true';
+
+async function fetchBulkDownloadUrl(type) {
+ const response = await fetch(`https://api.scryfall.com/bulk-data/${type}`, {
+ headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
+ });
+ if (!response.ok) {
+ throw new Error(`Failed to fetch bulk-data metadata: ${response.status}`);
+ }
+ const data = await response.json();
+ console.log(`[bulk-import] File: ${data.name}`);
+ console.log(`[bulk-import] Updated: ${data.updated_at}`);
+ console.log(`[bulk-import] Size: ${(data.size / 1024 / 1024).toFixed(1)} MB`);
+ return data.download_uri;
+}
+
+function mapCard(card) {
+ let currentPrice = null;
+ let marketPrice = null;
+ if (card.prices) {
+ currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null;
+ marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null;
+ }
+
+ return {
+ scryfallId: card.id,
+ oracleId: card.oracle_id || null,
+ illustrationId: card.illustration_id || null,
+ name: card.name,
+ setName: card.set_name,
+ setCode: card.set,
+ cardNumber: card.collector_number,
+ rarity: card.rarity,
+ manaCost: card.mana_cost || null,
+ cmc: card.cmc != null ? Math.round(card.cmc) : null,
+ cardType: card.type_line || null,
+ colors: JSON.stringify(card.colors || []),
+ colorIdentity: JSON.stringify(card.color_identity || []),
+ oracleText: card.oracle_text || null,
+ power: card.power || null,
+ toughness: card.toughness || null,
+ imageUrl: card.image_uris?.normal || null,
+ stockImageUrl: card.image_uris?.art_crop || null,
+ currentPrice,
+ marketPrice,
+ keywords: JSON.stringify(card.keywords || []),
+ legalities: JSON.stringify(card.legalities || {}),
+ flavorText: card.flavor_text || null,
+ artist: card.artist || null,
+ releasedAt: card.released_at || null,
+ layout: card.layout || null,
+ edhrecRank: card.edhrec_rank || null,
+ reserved: card.reserved || false,
+ reprint: card.reprint || false,
+ finishes: JSON.stringify(card.finishes || []),
+ };
+}
+
+function buildUpsertQuery(batch) {
+ const columns = [
+ 'scryfall_id', 'oracle_id', 'illustration_id', 'name', 'set_name', 'set_code',
+ 'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors',
+ 'color_identity', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url',
+ 'current_price', 'market_price', 'keywords', 'legalities', 'flavor_text', 'artist',
+ 'released_at', 'layout', 'edhrec_rank', 'reserved', 'reprint', 'finishes', 'verified',
+ ];
+
+ const placeholders = [];
+ const values = [];
+ let paramIdx = 1;
+
+ for (const row of batch) {
+ const rowPlaceholders = [];
+ const rowValues = [
+ row.scryfallId, row.oracleId, row.illustrationId, row.name, row.setName, row.setCode,
+ row.cardNumber, row.rarity, 'MTG', row.manaCost, row.cmc, row.cardType, row.colors,
+ row.colorIdentity, row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl,
+ row.currentPrice, row.marketPrice, row.keywords, row.legalities, row.flavorText, row.artist,
+ row.releasedAt, row.layout, row.edhrecRank, row.reserved, row.reprint, row.finishes, true,
+ ];
+
+ for (let i = 0; i < rowValues.length; i++) {
+ rowPlaceholders.push(`$${paramIdx}`);
+ paramIdx += 1;
+ }
+ placeholders.push(`(${rowPlaceholders.join(', ')})`);
+ values.push(...rowValues);
+ }
+
+ const updateCols = columns
+ .filter((c) => c !== 'scryfall_id')
+ .map((c) => `${c} = EXCLUDED.${c}`)
+ .join(', ');
+
+ const query = `
+ INSERT INTO cards (${columns.join(', ')})
+ VALUES ${placeholders.join(',\n')}
+ ON CONFLICT (scryfall_id) DO UPDATE SET
+ ${updateCols},
+ updated_at = CURRENT_TIMESTAMP
+ `;
+
+ return { query, values };
+}
+
+async function processBatch(batch, stats) {
+ if (DRY_RUN) {
+ stats.skipped += batch.length;
+ return;
+ }
+
+ const { query, values } = buildUpsertQuery(batch);
+ try {
+ await sql.query(query, values);
+ stats.upserted += batch.length;
+ } catch (error) {
+ console.error(`[bulk-import] Batch failed (${batch.length} rows):`, error.message);
+ for (const row of batch) {
+ try {
+ const { query: singleQ, values: singleV } = buildUpsertQuery([row]);
+ await sql.query(singleQ, singleV);
+ stats.upserted += 1;
+ } catch (singleErr) {
+ console.error(`[bulk-import] Failed: ${row.name} (${row.scryfallId}):`, singleErr.message);
+ stats.errors += 1;
+ }
+ }
+ }
+}
+
+async function run() {
+ console.log(`[bulk-import] Starting Scryfall bulk import (type=${BULK_TYPE}, batch=${BATCH_SIZE}, dryRun=${DRY_RUN})`);
+
+ const downloadUrl = await fetchBulkDownloadUrl(BULK_TYPE);
+ console.log(`[bulk-import] Downloading...`);
+
+ const response = await fetch(downloadUrl, {
+ headers: { 'User-Agent': 'DeckHearth/1.0' },
+ });
+
+ if (!response.ok) {
+ throw new Error(`Download failed: ${response.status}`);
+ }
+
+ const text = await response.text();
+ console.log(`[bulk-import] Downloaded ${(text.length / 1024 / 1024).toFixed(1)} MB, parsing...`);
+
+ const cards = JSON.parse(text);
+ console.log(`[bulk-import] Total cards in bulk file: ${cards.length}`);
+
+ const mtgCards = cards.filter((c) => c.lang === 'en' && !c.digital);
+ console.log(`[bulk-import] MTG cards (English, paper): ${mtgCards.length}`);
+
+ const stats = { upserted: 0, skipped: 0, errors: 0 };
+ let batch = [];
+ let processed = 0;
+
+ for (const card of mtgCards) {
+ batch.push(mapCard(card));
+
+ if (batch.length >= BATCH_SIZE) {
+ await processBatch(batch, stats);
+ processed += batch.length;
+ batch = [];
+
+ if (processed % 5000 === 0) {
+ console.log(`[bulk-import] Progress: ${processed}/${mtgCards.length} (${((processed / mtgCards.length) * 100).toFixed(1)}%)`);
+ }
+ }
+ }
+
+ if (batch.length > 0) {
+ await processBatch(batch, stats);
+ processed += batch.length;
+ }
+
+ console.log(`[bulk-import] Complete!`);
+ console.log(`[bulk-import] Upserted: ${stats.upserted}`);
+ console.log(`[bulk-import] Skipped (dry run): ${stats.skipped}`);
+ console.log(`[bulk-import] Errors: ${stats.errors}`);
+ console.log(`[bulk-import] Total processed: ${processed}`);
+}
+
+run().catch((err) => {
+ console.error('[bulk-import] Fatal error:', err);
+ process.exit(1);
+});
diff --git a/scripts/import-scryfall-tags.js b/scripts/import-scryfall-tags.js
new file mode 100644
index 0000000..06c5961
--- /dev/null
+++ b/scripts/import-scryfall-tags.js
@@ -0,0 +1,247 @@
+/**
+ * Import Scryfall Tagger community tags (Oracle Tags + Art Tags) into the
+ * `tags` and `card_tags` tables.
+ *
+ * Oracle tags describe card functionality (removal, ramp, draw, etc.)
+ * Art tags describe what's depicted in the artwork (dragon, forest, battle, etc.)
+ *
+ * Tags join to cards via oracle_id (oracle tags) or illustration_id (art tags).
+ * The card_tags table stores the raw join IDs so cards added later are automatically
+ * matched without re-importing tags.
+ *
+ * Usage:
+ * POSTGRES_URL= node scripts/import-scryfall-tags.js
+ *
+ * Options (env vars):
+ * TAG_TYPE — "both" (default) | "oracle" | "art"
+ * BATCH_SIZE — rows per INSERT batch (default 200)
+ * DRY_RUN — "true" to fetch and count without writing to DB
+ */
+
+import { neon } from '@neondatabase/serverless';
+
+if (!process.env.POSTGRES_URL) {
+ console.error('POSTGRES_URL is required');
+ process.exit(1);
+}
+
+const sql = neon(process.env.POSTGRES_URL, { fullResults: false });
+
+const TAG_TYPE = process.env.TAG_TYPE || 'both';
+const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '200', 10);
+const DRY_RUN = process.env.DRY_RUN === 'true';
+
+async function fetchBulkDownloadUrl(type) {
+ const response = await fetch(`https://api.scryfall.com/bulk-data/${type}`, {
+ headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
+ });
+ if (!response.ok) {
+ throw new Error(`Failed to fetch bulk-data metadata for ${type}: ${response.status}`);
+ }
+ const data = await response.json();
+ console.log(`[tag-import] File: ${data.name} (${(data.size / 1024 / 1024).toFixed(1)} MB)`);
+ return data.download_uri;
+}
+
+async function downloadJson(url) {
+ console.log(`[tag-import] Downloading...`);
+ const response = await fetch(url, {
+ headers: { 'User-Agent': 'DeckHearth/1.0' },
+ });
+ if (!response.ok) {
+ throw new Error(`Download failed: ${response.status}`);
+ }
+ const text = await response.text();
+ console.log(`[tag-import] Downloaded ${(text.length / 1024 / 1024).toFixed(1)} MB`);
+ return JSON.parse(text);
+}
+
+async function upsertTags(tags, stats) {
+ const batch = [];
+
+ for (const tag of tags) {
+ batch.push({
+ id: tag.id,
+ slug: tag.slug,
+ label: tag.label,
+ type: tag.type,
+ description: tag.description || null,
+ parentIds: JSON.stringify(tag.parent_ids || []),
+ childIds: JSON.stringify(tag.child_ids || []),
+ aliases: JSON.stringify(tag.aliases || []),
+ });
+
+ if (batch.length >= BATCH_SIZE) {
+ await flushTagBatch(batch, stats);
+ batch.length = 0;
+ }
+ }
+
+ if (batch.length > 0) {
+ await flushTagBatch(batch, stats);
+ }
+}
+
+async function flushTagBatch(batch, stats) {
+ if (DRY_RUN) {
+ stats.tags += batch.length;
+ return;
+ }
+
+ const placeholders = [];
+ const values = [];
+ let paramIdx = 1;
+
+ for (const row of batch) {
+ const rowP = [];
+ const rowV = [row.id, row.slug, row.label, row.type, row.description, row.parentIds, row.childIds, row.aliases];
+ for (let i = 0; i < rowV.length; i++) {
+ rowP.push(`$${paramIdx}`);
+ paramIdx += 1;
+ }
+ placeholders.push(`(${rowP.join(', ')})`);
+ values.push(...rowV);
+ }
+
+ const query = `
+ INSERT INTO tags (id, slug, label, type, description, parent_ids, child_ids, aliases)
+ VALUES ${placeholders.join(',\n')}
+ ON CONFLICT (id) DO UPDATE SET
+ slug = EXCLUDED.slug,
+ label = EXCLUDED.label,
+ description = EXCLUDED.description,
+ parent_ids = EXCLUDED.parent_ids,
+ child_ids = EXCLUDED.child_ids,
+ aliases = EXCLUDED.aliases,
+ updated_at = CURRENT_TIMESTAMP
+ `;
+
+ try {
+ await sql.query(query, values);
+ stats.tags += batch.length;
+ } catch (error) {
+ console.error(`[tag-import] Tag batch failed:`, error.message);
+ stats.tagErrors += batch.length;
+ }
+}
+
+async function importTaggings(tags, stats) {
+ let batch = [];
+ let processed = 0;
+ const total = tags.reduce((sum, t) => sum + (t.taggings?.length || 0), 0);
+ console.log(`[tag-import] Total taggings to import: ${total}`);
+
+ for (const tag of tags) {
+ if (!tag.taggings || tag.taggings.length === 0) continue;
+
+ for (const tagging of tag.taggings) {
+ batch.push({
+ tagId: tag.id,
+ oracleId: tagging.oracle_id || null,
+ illustrationId: tagging.illustration_id || null,
+ weight: tagging.weight || 'median',
+ annotation: tagging.annotation || null,
+ });
+
+ if (batch.length >= BATCH_SIZE) {
+ await flushTaggingBatch(batch, stats);
+ processed += batch.length;
+ batch = [];
+
+ if (processed % 10000 === 0) {
+ console.log(`[tag-import] Tagging progress: ${processed}/${total} (${((processed / total) * 100).toFixed(1)}%)`);
+ }
+ }
+ }
+ }
+
+ if (batch.length > 0) {
+ await flushTaggingBatch(batch, stats);
+ processed += batch.length;
+ }
+
+ console.log(`[tag-import] Taggings processed: ${processed}`);
+}
+
+async function flushTaggingBatch(batch, stats) {
+ if (DRY_RUN) {
+ stats.taggings += batch.length;
+ return;
+ }
+
+ const placeholders = [];
+ const values = [];
+ let paramIdx = 1;
+
+ for (const row of batch) {
+ const rowP = [];
+ const rowV = [row.tagId, row.oracleId, row.illustrationId, row.weight, row.annotation];
+ for (let i = 0; i < rowV.length; i++) {
+ rowP.push(`$${paramIdx}`);
+ paramIdx += 1;
+ }
+ placeholders.push(`(${rowP.join(', ')})`);
+ values.push(...rowV);
+ }
+
+ const uniqueCol = batch[0].oracleId ? 'card_tags_unique_tag_oracle' : 'card_tags_unique_tag_illustration';
+
+ const query = `
+ INSERT INTO card_tags (tag_id, oracle_id, illustration_id, weight, annotation)
+ VALUES ${placeholders.join(',\n')}
+ ON CONFLICT ON CONSTRAINT ${uniqueCol} DO UPDATE SET
+ weight = EXCLUDED.weight,
+ annotation = EXCLUDED.annotation
+ `;
+
+ try {
+ await sql.query(query, values);
+ stats.taggings += batch.length;
+ } catch (error) {
+ console.error(`[tag-import] Tagging batch failed:`, error.message);
+ stats.taggingErrors += batch.length;
+ }
+}
+
+async function importFile(type) {
+ const bulkType = type === 'oracle' ? 'oracle_tags' : 'art_tags';
+ console.log(`\n[tag-import] === Importing ${type} tags (${bulkType}) ===`);
+
+ const url = await fetchBulkDownloadUrl(bulkType);
+ const tags = await downloadJson(url);
+ console.log(`[tag-import] Tags in file: ${tags.length}`);
+
+ const stats = { tags: 0, tagErrors: 0, taggings: 0, taggingErrors: 0 };
+
+ console.log(`[tag-import] Upserting tag definitions...`);
+ await upsertTags(tags, stats);
+ console.log(`[tag-import] Tags upserted: ${stats.tags}, errors: ${stats.tagErrors}`);
+
+ console.log(`[tag-import] Importing taggings...`);
+ await importTaggings(tags, stats);
+ console.log(`[tag-import] Taggings upserted: ${stats.taggings}, errors: ${stats.taggingErrors}`);
+
+ return stats;
+}
+
+async function run() {
+ console.log(`[tag-import] Starting Scryfall Tagger import (type=${TAG_TYPE}, batch=${BATCH_SIZE}, dryRun=${DRY_RUN})`);
+
+ const results = {};
+
+ if (TAG_TYPE === 'oracle' || TAG_TYPE === 'both') {
+ results.oracle = await importFile('oracle');
+ }
+
+ if (TAG_TYPE === 'art' || TAG_TYPE === 'both') {
+ results.art = await importFile('art');
+ }
+
+ console.log(`\n[tag-import] === Summary ===`);
+ console.log(JSON.stringify(results, null, 2));
+}
+
+run().catch((err) => {
+ console.error('[tag-import] Fatal error:', err);
+ process.exit(1);
+});