From 67073aab7f231f0390f7813d23189df0afd474fb Mon Sep 17 00:00:00 2001 From: Randall Stillwell Date: Sun, 14 Jun 2026 07:51:39 -0500 Subject: [PATCH] feat(catalog): Scryfall bulk data import + Tagger community tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add full Scryfall bulk data pipeline: - Migration: 13 new columns on `cards` (oracle_id, illustration_id, color_identity, keywords, legalities, flavor_text, artist, released_at, layout, edhrec_rank, reserved, reprint, finishes) with GIN indexes for JSONB search. - Migration: `tags` + `card_tags` tables for Tagger community data. - Script: `bulk-import-scryfall.js` — downloads Oracle Cards bulk file (168 MB) and upserts all 36k+ MTG cards with rich metadata. - Script: `import-scryfall-tags.js` — imports oracle tags (4.5k tags, 227k taggings) and art tags (11k tags, 458k taggings). - Lib: `bulk-sync.js` — runtime bulk sync callable from the admin API. - Admin UI: mode toggle (incremental vs bulk) on catalog sync panel. Enables Commander deck validation (color_identity), format legality checks, keyword search, EDHREC popularity ranking, and functional card tagging ("removal", "ramp", "draw") for deck building assistance. Co-authored-by: Cursor --- lib/card-import/bulk-sync.js | 181 +++++++++++++ ...1781440700404_add-scryfall-bulk-columns.js | 51 ++++ migrations/1781440721350_add-tagger-tables.js | 53 ++++ package-lock.json | 10 + package.json | 3 + pages/admin/card-import.js | 40 ++- pages/api/admin/sync-catalog.js | 8 + scripts/bulk-import-scryfall.js | 217 +++++++++++++++ scripts/import-scryfall-tags.js | 247 ++++++++++++++++++ 9 files changed, 807 insertions(+), 3 deletions(-) create mode 100644 lib/card-import/bulk-sync.js create mode 100644 migrations/1781440700404_add-scryfall-bulk-columns.js create mode 100644 migrations/1781440721350_add-tagger-tables.js create mode 100644 scripts/bulk-import-scryfall.js create mode 100644 scripts/import-scryfall-tags.js diff --git a/lib/card-import/bulk-sync.js b/lib/card-import/bulk-sync.js new file mode 100644 index 0000000..6374e6e --- /dev/null +++ b/lib/card-import/bulk-sync.js @@ -0,0 +1,181 @@ +/** + * Bulk catalog sync using Scryfall's bulk-data endpoint. + * + * Downloads the Oracle Cards file (~168 MB) and upserts all MTG cards in one pass. + * Much faster and more complete than the per-set discovery approach — catches all + * missing sets in one shot without rate-limit concerns. + * + * Also refreshes prices, legalities, and edhrec_rank on existing cards. + */ + +import { sql } from '@vercel/postgres'; + +const BATCH_SIZE = 100; +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function fetchBulkDownloadUrl() { + const response = await fetch('https://api.scryfall.com/bulk-data/oracle_cards', { + headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' }, + }); + if (!response.ok) { + throw new Error(`Scryfall bulk-data API error: ${response.status}`); + } + return response.json(); +} + +function mapCard(card) { + let currentPrice = null; + let marketPrice = null; + if (card.prices) { + currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null; + marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null; + } + + return { + scryfallId: card.id, + oracleId: card.oracle_id || null, + illustrationId: card.illustration_id || null, + name: card.name, + setName: card.set_name, + setCode: card.set, + cardNumber: card.collector_number, + rarity: card.rarity, + manaCost: card.mana_cost || null, + cmc: card.cmc != null ? Math.round(card.cmc) : null, + cardType: card.type_line || null, + colors: JSON.stringify(card.colors || []), + colorIdentity: JSON.stringify(card.color_identity || []), + oracleText: card.oracle_text || null, + power: card.power || null, + toughness: card.toughness || null, + imageUrl: card.image_uris?.normal || null, + stockImageUrl: card.image_uris?.art_crop || null, + currentPrice, + marketPrice, + keywords: JSON.stringify(card.keywords || []), + legalities: JSON.stringify(card.legalities || {}), + flavorText: card.flavor_text || null, + artist: card.artist || null, + releasedAt: card.released_at || null, + layout: card.layout || null, + edhrecRank: card.edhrec_rank || null, + reserved: card.reserved || false, + reprint: card.reprint || false, + finishes: JSON.stringify(card.finishes || []), + }; +} + +async function upsertBatch(cards) { + if (cards.length === 0) return 0; + + const columns = [ + 'scryfall_id', 'oracle_id', 'illustration_id', 'name', 'set_name', 'set_code', + 'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors', + 'color_identity', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url', + 'current_price', 'market_price', 'keywords', 'legalities', 'flavor_text', 'artist', + 'released_at', 'layout', 'edhrec_rank', 'reserved', 'reprint', 'finishes', 'verified', + ]; + + const values = []; + const placeholders = []; + let paramIdx = 1; + + for (const row of cards) { + const rowValues = [ + row.scryfallId, row.oracleId, row.illustrationId, row.name, row.setName, row.setCode, + row.cardNumber, row.rarity, 'MTG', row.manaCost, row.cmc, row.cardType, row.colors, + row.colorIdentity, row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl, + row.currentPrice, row.marketPrice, row.keywords, row.legalities, row.flavorText, row.artist, + row.releasedAt, row.layout, row.edhrecRank, row.reserved, row.reprint, row.finishes, true, + ]; + const rowP = rowValues.map(() => `$${paramIdx++}`); + placeholders.push(`(${rowP.join(',')})`); + values.push(...rowValues); + } + + const updateCols = columns + .filter((c) => c !== 'scryfall_id') + .map((c) => `${c} = EXCLUDED.${c}`) + .join(', '); + + await sql.query( + `INSERT INTO cards (${columns.join(', ')}) + VALUES ${placeholders.join(',\n')} + ON CONFLICT (scryfall_id) DO UPDATE SET ${updateCols}, updated_at = CURRENT_TIMESTAMP`, + values + ); + + return cards.length; +} + +/** + * Run a full bulk MTG sync. Returns a summary compatible with the catalog sync API shape. + */ +export async function runBulkMtgSync() { + const meta = await fetchBulkDownloadUrl(); + console.log(`[bulk-sync] Oracle Cards updated: ${meta.updated_at}, size: ${(meta.size / 1024 / 1024).toFixed(0)} MB`); + + const response = await fetch(meta.download_uri, { + headers: { 'User-Agent': 'DeckHearth/1.0' }, + }); + if (!response.ok) { + throw new Error(`Bulk download failed: ${response.status}`); + } + + const allCards = await response.json(); + const mtgCards = allCards.filter((c) => c.lang === 'en' && !c.digital); + + let upserted = 0; + let errors = 0; + let batch = []; + + for (const card of mtgCards) { + batch.push(mapCard(card)); + if (batch.length >= BATCH_SIZE) { + try { + upserted += await upsertBatch(batch); + } catch (err) { + console.error('[bulk-sync] Batch error:', err.message); + errors += batch.length; + } + batch = []; + } + } + + if (batch.length > 0) { + try { + upserted += await upsertBatch(batch); + } catch (err) { + console.error('[bulk-sync] Final batch error:', err.message); + errors += batch.length; + } + } + + // Backfill oracle_id on older per-set imported cards + const backfill = await sql` + UPDATE cards target + SET oracle_id = source.oracle_id, + color_identity = COALESCE(target.color_identity, source.color_identity), + keywords = COALESCE(target.keywords, source.keywords), + legalities = COALESCE(target.legalities, source.legalities), + layout = COALESCE(target.layout, source.layout), + edhrec_rank = COALESCE(target.edhrec_rank, source.edhrec_rank) + FROM ( + SELECT DISTINCT ON (name) name, oracle_id, color_identity, keywords, legalities, layout, edhrec_rank + FROM cards WHERE oracle_id IS NOT NULL AND game = 'MTG' + ) source + WHERE target.name = source.name AND target.game = 'MTG' AND target.oracle_id IS NULL + `; + + const summary = { + mode: 'bulk', + totalInFile: mtgCards.length, + upserted, + errors, + backfilledOracleId: backfill.rowCount || 0, + updatedAt: meta.updated_at, + }; + + console.log('[bulk-sync]', JSON.stringify(summary)); + return summary; +} diff --git a/migrations/1781440700404_add-scryfall-bulk-columns.js b/migrations/1781440700404_add-scryfall-bulk-columns.js new file mode 100644 index 0000000..45b6e2c --- /dev/null +++ b/migrations/1781440700404_add-scryfall-bulk-columns.js @@ -0,0 +1,51 @@ +/** + * Add Scryfall bulk-data columns to `cards` for richer MTG metadata. + * These enable Tagger tag joins (oracle_id, illustration_id), Commander + * deck validation (color_identity), format legality checks, keyword + * search, and better card detail pages. + */ +export const shorthands = undefined; + +export const up = (pgm) => { + pgm.addColumns('cards', { + oracle_id: { type: 'varchar(36)', comment: 'Stable across printings; joins oracle tags' }, + illustration_id: { type: 'varchar(36)', comment: 'Stable per artwork; joins art tags' }, + color_identity: { type: 'jsonb', comment: '["W","U","B","R","G"] — includes symbols in rules text' }, + keywords: { type: 'jsonb', comment: '["Flying","Trample",...] — searchable mechanics' }, + legalities: { type: 'jsonb', comment: '{standard:"legal", modern:"not_legal",...}' }, + flavor_text: { type: 'text' }, + artist: { type: 'varchar(255)' }, + released_at: { type: 'date' }, + layout: { type: 'varchar(50)', comment: 'normal, transform, split, mdfc, adventure, etc.' }, + edhrec_rank: { type: 'integer', comment: 'Commander popularity — lower is more popular' }, + reserved: { type: 'boolean', default: false }, + reprint: { type: 'boolean', default: false }, + finishes: { type: 'jsonb', comment: '["nonfoil","foil","etched"]' }, + }); + + pgm.createIndex('cards', 'oracle_id'); + pgm.createIndex('cards', 'illustration_id'); + pgm.createIndex('cards', 'color_identity', { method: 'gin' }); + pgm.createIndex('cards', 'keywords', { method: 'gin' }); + pgm.createIndex('cards', 'legalities', { method: 'gin' }); + pgm.createIndex('cards', 'artist'); + pgm.createIndex('cards', 'edhrec_rank'); +}; + +export const down = (pgm) => { + pgm.dropColumns('cards', [ + 'oracle_id', + 'illustration_id', + 'color_identity', + 'keywords', + 'legalities', + 'flavor_text', + 'artist', + 'released_at', + 'layout', + 'edhrec_rank', + 'reserved', + 'reprint', + 'finishes', + ]); +}; diff --git a/migrations/1781440721350_add-tagger-tables.js b/migrations/1781440721350_add-tagger-tables.js new file mode 100644 index 0000000..c2f2486 --- /dev/null +++ b/migrations/1781440721350_add-tagger-tables.js @@ -0,0 +1,53 @@ +/** + * Create tables for Scryfall Tagger community tags (oracle + art). + * Tags join to cards via oracle_id (oracle tags) or illustration_id (art tags). + */ +export const shorthands = undefined; + +export const up = (pgm) => { + pgm.createTable('tags', { + id: { type: 'uuid', primaryKey: true }, + slug: { type: 'varchar(255)', notNull: true }, + label: { type: 'varchar(255)', notNull: true }, + type: { type: 'varchar(20)', notNull: true, comment: 'oracle or illustration' }, + description: { type: 'text' }, + parent_ids: { type: 'jsonb' }, + child_ids: { type: 'jsonb' }, + aliases: { type: 'jsonb' }, + created_at: { type: 'timestamp', default: pgm.func('CURRENT_TIMESTAMP') }, + updated_at: { type: 'timestamp', default: pgm.func('CURRENT_TIMESTAMP') }, + }); + + pgm.createIndex('tags', 'type'); + pgm.createIndex('tags', 'slug'); + pgm.createIndex('tags', 'label'); + + pgm.createTable('card_tags', { + id: { type: 'serial', primaryKey: true }, + tag_id: { type: 'uuid', notNull: true, references: 'tags(id)', onDelete: 'CASCADE' }, + card_id: { type: 'integer', references: 'cards(id)', onDelete: 'CASCADE' }, + oracle_id: { type: 'varchar(36)', comment: 'For oracle tags — matches cards.oracle_id' }, + illustration_id: { type: 'varchar(36)', comment: 'For art tags — matches cards.illustration_id' }, + weight: { type: 'varchar(20)', default: 'median' }, + annotation: { type: 'text' }, + created_at: { type: 'timestamp', default: pgm.func('CURRENT_TIMESTAMP') }, + }); + + pgm.createIndex('card_tags', 'tag_id'); + pgm.createIndex('card_tags', 'card_id'); + pgm.createIndex('card_tags', 'oracle_id'); + pgm.createIndex('card_tags', 'illustration_id'); + pgm.addConstraint('card_tags', 'card_tags_unique_tag_oracle', { + unique: ['tag_id', 'oracle_id'], + where: 'oracle_id IS NOT NULL', + }); + pgm.addConstraint('card_tags', 'card_tags_unique_tag_illustration', { + unique: ['tag_id', 'illustration_id'], + where: 'illustration_id IS NOT NULL', + }); +}; + +export const down = (pgm) => { + pgm.dropTable('card_tags'); + pgm.dropTable('tags'); +}; diff --git a/package-lock.json b/package-lock.json index eb83de5..846be6d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "deck-hearth", "version": "0.1.0", "dependencies": { + "@neondatabase/serverless": "^1.1.0", "@upstash/ratelimit": "^2.0.8", "@upstash/redis": "^1.38.0", "@vercel/blob": "^1.1.1", @@ -1849,6 +1850,15 @@ "@tybys/wasm-util": "^0.10.0" } }, + "node_modules/@neondatabase/serverless": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-1.1.0.tgz", + "integrity": "sha512-r3ZZhRjEcfEdKIZnoB1RusNgvHuaBRqfCzV4Gi+5A9yUX0S4HTws/ASWqt13wL4y4I+0rqsWGdA2w7EQXHi3+Q==", + "license": "MIT", + "engines": { + "node": ">=19.0.0" + } + }, "node_modules/@next/env": { "version": "16.2.6", "resolved": "https://registry.npmjs.org/@next/env/-/env-16.2.6.tgz", diff --git a/package.json b/package.json index aec3b1e..5ba8ecc 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "setup-db": "node scripts/setup-neon-db.js", "import-popular": "node scripts/import-popular-sets.js", "import-all": "node scripts/bulk-import-all.js", + "bulk-import": "node --env-file=.env.local scripts/bulk-import-scryfall.js", + "import-tags": "node --env-file=.env.local scripts/import-scryfall-tags.js", "test": "vitest", "test:run": "vitest run", "test:smoke": "playwright test --project=smoke", @@ -19,6 +21,7 @@ "test:visual:update": "playwright test --project=visual --update-snapshots" }, "dependencies": { + "@neondatabase/serverless": "^1.1.0", "@upstash/ratelimit": "^2.0.8", "@upstash/redis": "^1.38.0", "@vercel/blob": "^1.1.1", diff --git a/pages/admin/card-import.js b/pages/admin/card-import.js index 0de285c..d95a543 100644 --- a/pages/admin/card-import.js +++ b/pages/admin/card-import.js @@ -14,6 +14,8 @@ const CardImport = () => { const [isSyncing, setIsSyncing] = useState(false); const [syncResult, setSyncResult] = useState(null); + const [syncMode, setSyncMode] = useState('incremental'); + const handleCatalogSync = async () => { setIsSyncing(true); setSyncResult(null); @@ -25,6 +27,7 @@ const CardImport = () => { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('auth_token')}`, }, + body: JSON.stringify({ mode: syncMode }), }); const result = await response.json(); @@ -171,9 +174,32 @@ const CardImport = () => { Catalog sync

- Import up to three newest missing MTG and Pokémon sets — the same job the weekly Vercel cron runs. - Pending scan submissions for those sets are auto-linked to the catalog when a unique match exists. + {syncMode === 'bulk' + ? 'Download Scryfall\'s full Oracle Cards bulk file and upsert all 36k+ MTG cards with rich metadata (legalities, keywords, color identity, tags). Takes ~30s.' + : 'Import up to three newest missing MTG and Pokémon sets — the same job the weekly Vercel cron runs. Pending scan submissions are auto-linked when a unique match exists.'}

+
+ + +
@@ -204,9 +230,17 @@ const CardImport = () => { {syncResult.success ? (
+ {syncResult.mode === 'bulk' ? ( +

+ Upserted {syncResult.upserted} MTG cards from Scryfall bulk data. + {syncResult.backfilledOracleId > 0 && ` Backfilled oracle_id on ${syncResult.backfilledOracleId} existing rows.`} + {syncResult.errors > 0 && ` (${syncResult.errors} errors)`} +

+ ) : (

Imported {syncResult.imported} cards ({syncResult.skipped} skipped).

+ )} {syncResult.submissionsReconciled > 0 && (

Linked {syncResult.submissionsReconciled} pending scan submission diff --git a/pages/api/admin/sync-catalog.js b/pages/api/admin/sync-catalog.js index d74b855..e982aa5 100644 --- a/pages/api/admin/sync-catalog.js +++ b/pages/api/admin/sync-catalog.js @@ -1,6 +1,7 @@ import { withAdmin } from '../../../lib/permission-middleware'; import { checkImportRateLimit } from '../../../lib/rate-limit.js'; import { runCatalogSync } from '../../../lib/card-import/sync-catalog.js'; +import { runBulkMtgSync } from '../../../lib/card-import/bulk-sync.js'; export default withAdmin(async function handler(req, res, user) { if (req.method !== 'POST') { @@ -13,7 +14,14 @@ export default withAdmin(async function handler(req, res, user) { return res.status(429).json({ error: 'Too many attempts. Try again later.' }); } + const mode = req.body?.mode || 'incremental'; + try { + if (mode === 'bulk') { + const summary = await runBulkMtgSync(); + return res.status(200).json({ success: true, ...summary }); + } + const summary = await runCatalogSync(); return res.status(200).json({ success: true, ...summary }); } catch (error) { diff --git a/scripts/bulk-import-scryfall.js b/scripts/bulk-import-scryfall.js new file mode 100644 index 0000000..12dd9a3 --- /dev/null +++ b/scripts/bulk-import-scryfall.js @@ -0,0 +1,217 @@ +/** + * Bulk import MTG cards from Scryfall's Oracle Cards bulk data file. + * + * Downloads the ~168 MB Oracle Cards JSON (one entry per unique card/oracle_id), + * then upserts every card into the `cards` table keyed on `scryfall_id`. + * + * Populates all new bulk-data columns (oracle_id, illustration_id, color_identity, + * keywords, legalities, flavor_text, artist, released_at, layout, edhrec_rank, + * reserved, reprint, finishes) in addition to the original columns. + * + * Usage: + * POSTGRES_URL= node scripts/bulk-import-scryfall.js + * + * Options (env vars): + * BULK_TYPE — "oracle_cards" (default) | "default_cards" | "unique_artwork" + * BATCH_SIZE — rows per INSERT batch (default 100) + * DRY_RUN — "true" to fetch and count without writing to DB + */ + +import { neon } from '@neondatabase/serverless'; + +if (!process.env.POSTGRES_URL) { + console.error('POSTGRES_URL is required'); + process.exit(1); +} + +const sql = neon(process.env.POSTGRES_URL, { fullResults: false }); + +const BULK_TYPE = process.env.BULK_TYPE || 'oracle_cards'; +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '100', 10); +const DRY_RUN = process.env.DRY_RUN === 'true'; + +async function fetchBulkDownloadUrl(type) { + const response = await fetch(`https://api.scryfall.com/bulk-data/${type}`, { + headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch bulk-data metadata: ${response.status}`); + } + const data = await response.json(); + console.log(`[bulk-import] File: ${data.name}`); + console.log(`[bulk-import] Updated: ${data.updated_at}`); + console.log(`[bulk-import] Size: ${(data.size / 1024 / 1024).toFixed(1)} MB`); + return data.download_uri; +} + +function mapCard(card) { + let currentPrice = null; + let marketPrice = null; + if (card.prices) { + currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null; + marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null; + } + + return { + scryfallId: card.id, + oracleId: card.oracle_id || null, + illustrationId: card.illustration_id || null, + name: card.name, + setName: card.set_name, + setCode: card.set, + cardNumber: card.collector_number, + rarity: card.rarity, + manaCost: card.mana_cost || null, + cmc: card.cmc != null ? Math.round(card.cmc) : null, + cardType: card.type_line || null, + colors: JSON.stringify(card.colors || []), + colorIdentity: JSON.stringify(card.color_identity || []), + oracleText: card.oracle_text || null, + power: card.power || null, + toughness: card.toughness || null, + imageUrl: card.image_uris?.normal || null, + stockImageUrl: card.image_uris?.art_crop || null, + currentPrice, + marketPrice, + keywords: JSON.stringify(card.keywords || []), + legalities: JSON.stringify(card.legalities || {}), + flavorText: card.flavor_text || null, + artist: card.artist || null, + releasedAt: card.released_at || null, + layout: card.layout || null, + edhrecRank: card.edhrec_rank || null, + reserved: card.reserved || false, + reprint: card.reprint || false, + finishes: JSON.stringify(card.finishes || []), + }; +} + +function buildUpsertQuery(batch) { + const columns = [ + 'scryfall_id', 'oracle_id', 'illustration_id', 'name', 'set_name', 'set_code', + 'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors', + 'color_identity', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url', + 'current_price', 'market_price', 'keywords', 'legalities', 'flavor_text', 'artist', + 'released_at', 'layout', 'edhrec_rank', 'reserved', 'reprint', 'finishes', 'verified', + ]; + + const placeholders = []; + const values = []; + let paramIdx = 1; + + for (const row of batch) { + const rowPlaceholders = []; + const rowValues = [ + row.scryfallId, row.oracleId, row.illustrationId, row.name, row.setName, row.setCode, + row.cardNumber, row.rarity, 'MTG', row.manaCost, row.cmc, row.cardType, row.colors, + row.colorIdentity, row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl, + row.currentPrice, row.marketPrice, row.keywords, row.legalities, row.flavorText, row.artist, + row.releasedAt, row.layout, row.edhrecRank, row.reserved, row.reprint, row.finishes, true, + ]; + + for (let i = 0; i < rowValues.length; i++) { + rowPlaceholders.push(`$${paramIdx}`); + paramIdx += 1; + } + placeholders.push(`(${rowPlaceholders.join(', ')})`); + values.push(...rowValues); + } + + const updateCols = columns + .filter((c) => c !== 'scryfall_id') + .map((c) => `${c} = EXCLUDED.${c}`) + .join(', '); + + const query = ` + INSERT INTO cards (${columns.join(', ')}) + VALUES ${placeholders.join(',\n')} + ON CONFLICT (scryfall_id) DO UPDATE SET + ${updateCols}, + updated_at = CURRENT_TIMESTAMP + `; + + return { query, values }; +} + +async function processBatch(batch, stats) { + if (DRY_RUN) { + stats.skipped += batch.length; + return; + } + + const { query, values } = buildUpsertQuery(batch); + try { + await sql.query(query, values); + stats.upserted += batch.length; + } catch (error) { + console.error(`[bulk-import] Batch failed (${batch.length} rows):`, error.message); + for (const row of batch) { + try { + const { query: singleQ, values: singleV } = buildUpsertQuery([row]); + await sql.query(singleQ, singleV); + stats.upserted += 1; + } catch (singleErr) { + console.error(`[bulk-import] Failed: ${row.name} (${row.scryfallId}):`, singleErr.message); + stats.errors += 1; + } + } + } +} + +async function run() { + console.log(`[bulk-import] Starting Scryfall bulk import (type=${BULK_TYPE}, batch=${BATCH_SIZE}, dryRun=${DRY_RUN})`); + + const downloadUrl = await fetchBulkDownloadUrl(BULK_TYPE); + console.log(`[bulk-import] Downloading...`); + + const response = await fetch(downloadUrl, { + headers: { 'User-Agent': 'DeckHearth/1.0' }, + }); + + if (!response.ok) { + throw new Error(`Download failed: ${response.status}`); + } + + const text = await response.text(); + console.log(`[bulk-import] Downloaded ${(text.length / 1024 / 1024).toFixed(1)} MB, parsing...`); + + const cards = JSON.parse(text); + console.log(`[bulk-import] Total cards in bulk file: ${cards.length}`); + + const mtgCards = cards.filter((c) => c.lang === 'en' && !c.digital); + console.log(`[bulk-import] MTG cards (English, paper): ${mtgCards.length}`); + + const stats = { upserted: 0, skipped: 0, errors: 0 }; + let batch = []; + let processed = 0; + + for (const card of mtgCards) { + batch.push(mapCard(card)); + + if (batch.length >= BATCH_SIZE) { + await processBatch(batch, stats); + processed += batch.length; + batch = []; + + if (processed % 5000 === 0) { + console.log(`[bulk-import] Progress: ${processed}/${mtgCards.length} (${((processed / mtgCards.length) * 100).toFixed(1)}%)`); + } + } + } + + if (batch.length > 0) { + await processBatch(batch, stats); + processed += batch.length; + } + + console.log(`[bulk-import] Complete!`); + console.log(`[bulk-import] Upserted: ${stats.upserted}`); + console.log(`[bulk-import] Skipped (dry run): ${stats.skipped}`); + console.log(`[bulk-import] Errors: ${stats.errors}`); + console.log(`[bulk-import] Total processed: ${processed}`); +} + +run().catch((err) => { + console.error('[bulk-import] Fatal error:', err); + process.exit(1); +}); diff --git a/scripts/import-scryfall-tags.js b/scripts/import-scryfall-tags.js new file mode 100644 index 0000000..06c5961 --- /dev/null +++ b/scripts/import-scryfall-tags.js @@ -0,0 +1,247 @@ +/** + * Import Scryfall Tagger community tags (Oracle Tags + Art Tags) into the + * `tags` and `card_tags` tables. + * + * Oracle tags describe card functionality (removal, ramp, draw, etc.) + * Art tags describe what's depicted in the artwork (dragon, forest, battle, etc.) + * + * Tags join to cards via oracle_id (oracle tags) or illustration_id (art tags). + * The card_tags table stores the raw join IDs so cards added later are automatically + * matched without re-importing tags. + * + * Usage: + * POSTGRES_URL= node scripts/import-scryfall-tags.js + * + * Options (env vars): + * TAG_TYPE — "both" (default) | "oracle" | "art" + * BATCH_SIZE — rows per INSERT batch (default 200) + * DRY_RUN — "true" to fetch and count without writing to DB + */ + +import { neon } from '@neondatabase/serverless'; + +if (!process.env.POSTGRES_URL) { + console.error('POSTGRES_URL is required'); + process.exit(1); +} + +const sql = neon(process.env.POSTGRES_URL, { fullResults: false }); + +const TAG_TYPE = process.env.TAG_TYPE || 'both'; +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '200', 10); +const DRY_RUN = process.env.DRY_RUN === 'true'; + +async function fetchBulkDownloadUrl(type) { + const response = await fetch(`https://api.scryfall.com/bulk-data/${type}`, { + headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch bulk-data metadata for ${type}: ${response.status}`); + } + const data = await response.json(); + console.log(`[tag-import] File: ${data.name} (${(data.size / 1024 / 1024).toFixed(1)} MB)`); + return data.download_uri; +} + +async function downloadJson(url) { + console.log(`[tag-import] Downloading...`); + const response = await fetch(url, { + headers: { 'User-Agent': 'DeckHearth/1.0' }, + }); + if (!response.ok) { + throw new Error(`Download failed: ${response.status}`); + } + const text = await response.text(); + console.log(`[tag-import] Downloaded ${(text.length / 1024 / 1024).toFixed(1)} MB`); + return JSON.parse(text); +} + +async function upsertTags(tags, stats) { + const batch = []; + + for (const tag of tags) { + batch.push({ + id: tag.id, + slug: tag.slug, + label: tag.label, + type: tag.type, + description: tag.description || null, + parentIds: JSON.stringify(tag.parent_ids || []), + childIds: JSON.stringify(tag.child_ids || []), + aliases: JSON.stringify(tag.aliases || []), + }); + + if (batch.length >= BATCH_SIZE) { + await flushTagBatch(batch, stats); + batch.length = 0; + } + } + + if (batch.length > 0) { + await flushTagBatch(batch, stats); + } +} + +async function flushTagBatch(batch, stats) { + if (DRY_RUN) { + stats.tags += batch.length; + return; + } + + const placeholders = []; + const values = []; + let paramIdx = 1; + + for (const row of batch) { + const rowP = []; + const rowV = [row.id, row.slug, row.label, row.type, row.description, row.parentIds, row.childIds, row.aliases]; + for (let i = 0; i < rowV.length; i++) { + rowP.push(`$${paramIdx}`); + paramIdx += 1; + } + placeholders.push(`(${rowP.join(', ')})`); + values.push(...rowV); + } + + const query = ` + INSERT INTO tags (id, slug, label, type, description, parent_ids, child_ids, aliases) + VALUES ${placeholders.join(',\n')} + ON CONFLICT (id) DO UPDATE SET + slug = EXCLUDED.slug, + label = EXCLUDED.label, + description = EXCLUDED.description, + parent_ids = EXCLUDED.parent_ids, + child_ids = EXCLUDED.child_ids, + aliases = EXCLUDED.aliases, + updated_at = CURRENT_TIMESTAMP + `; + + try { + await sql.query(query, values); + stats.tags += batch.length; + } catch (error) { + console.error(`[tag-import] Tag batch failed:`, error.message); + stats.tagErrors += batch.length; + } +} + +async function importTaggings(tags, stats) { + let batch = []; + let processed = 0; + const total = tags.reduce((sum, t) => sum + (t.taggings?.length || 0), 0); + console.log(`[tag-import] Total taggings to import: ${total}`); + + for (const tag of tags) { + if (!tag.taggings || tag.taggings.length === 0) continue; + + for (const tagging of tag.taggings) { + batch.push({ + tagId: tag.id, + oracleId: tagging.oracle_id || null, + illustrationId: tagging.illustration_id || null, + weight: tagging.weight || 'median', + annotation: tagging.annotation || null, + }); + + if (batch.length >= BATCH_SIZE) { + await flushTaggingBatch(batch, stats); + processed += batch.length; + batch = []; + + if (processed % 10000 === 0) { + console.log(`[tag-import] Tagging progress: ${processed}/${total} (${((processed / total) * 100).toFixed(1)}%)`); + } + } + } + } + + if (batch.length > 0) { + await flushTaggingBatch(batch, stats); + processed += batch.length; + } + + console.log(`[tag-import] Taggings processed: ${processed}`); +} + +async function flushTaggingBatch(batch, stats) { + if (DRY_RUN) { + stats.taggings += batch.length; + return; + } + + const placeholders = []; + const values = []; + let paramIdx = 1; + + for (const row of batch) { + const rowP = []; + const rowV = [row.tagId, row.oracleId, row.illustrationId, row.weight, row.annotation]; + for (let i = 0; i < rowV.length; i++) { + rowP.push(`$${paramIdx}`); + paramIdx += 1; + } + placeholders.push(`(${rowP.join(', ')})`); + values.push(...rowV); + } + + const uniqueCol = batch[0].oracleId ? 'card_tags_unique_tag_oracle' : 'card_tags_unique_tag_illustration'; + + const query = ` + INSERT INTO card_tags (tag_id, oracle_id, illustration_id, weight, annotation) + VALUES ${placeholders.join(',\n')} + ON CONFLICT ON CONSTRAINT ${uniqueCol} DO UPDATE SET + weight = EXCLUDED.weight, + annotation = EXCLUDED.annotation + `; + + try { + await sql.query(query, values); + stats.taggings += batch.length; + } catch (error) { + console.error(`[tag-import] Tagging batch failed:`, error.message); + stats.taggingErrors += batch.length; + } +} + +async function importFile(type) { + const bulkType = type === 'oracle' ? 'oracle_tags' : 'art_tags'; + console.log(`\n[tag-import] === Importing ${type} tags (${bulkType}) ===`); + + const url = await fetchBulkDownloadUrl(bulkType); + const tags = await downloadJson(url); + console.log(`[tag-import] Tags in file: ${tags.length}`); + + const stats = { tags: 0, tagErrors: 0, taggings: 0, taggingErrors: 0 }; + + console.log(`[tag-import] Upserting tag definitions...`); + await upsertTags(tags, stats); + console.log(`[tag-import] Tags upserted: ${stats.tags}, errors: ${stats.tagErrors}`); + + console.log(`[tag-import] Importing taggings...`); + await importTaggings(tags, stats); + console.log(`[tag-import] Taggings upserted: ${stats.taggings}, errors: ${stats.taggingErrors}`); + + return stats; +} + +async function run() { + console.log(`[tag-import] Starting Scryfall Tagger import (type=${TAG_TYPE}, batch=${BATCH_SIZE}, dryRun=${DRY_RUN})`); + + const results = {}; + + if (TAG_TYPE === 'oracle' || TAG_TYPE === 'both') { + results.oracle = await importFile('oracle'); + } + + if (TAG_TYPE === 'art' || TAG_TYPE === 'both') { + results.art = await importFile('art'); + } + + console.log(`\n[tag-import] === Summary ===`); + console.log(JSON.stringify(results, null, 2)); +} + +run().catch((err) => { + console.error('[tag-import] Fatal error:', err); + process.exit(1); +});