rstillwell/deckhearth
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions 10

feat(catalog): Scryfall bulk data import + Tagger community tags

Browse source 
Add full Scryfall bulk data pipeline:

- Migration: 13 new columns on `cards` (oracle_id, illustration_id,
  color_identity, keywords, legalities, flavor_text, artist, released_at,
  layout, edhrec_rank, reserved, reprint, finishes) with GIN indexes
  for JSONB search.
- Migration: `tags` + `card_tags` tables for Tagger community data.
- Script: `bulk-import-scryfall.js` — downloads Oracle Cards bulk file
  (168 MB) and upserts all 36k+ MTG cards with rich metadata.
- Script: `import-scryfall-tags.js` — imports oracle tags (4.5k tags,
  227k taggings) and art tags (11k tags, 458k taggings).
- Lib: `bulk-sync.js` — runtime bulk sync callable from the admin API.
- Admin UI: mode toggle (incremental vs bulk) on catalog sync panel.

Enables Commander deck validation (color_identity), format legality
checks, keyword search, EDHREC popularity ranking, and functional
card tagging ("removal", "ramp", "draw") for deck building assistance.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Randall Stillwell 2026-06-14 07:51:39 -05:00
parent 59e2ca4ce4
commit 67073aab7f
9 changed files with 807 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

181
lib/card-import/bulk-sync.js Normal file
View file

@ -0,0 +1,181 @@
/**
* Bulk catalog sync using Scryfall's bulk-data endpoint.
*
* Downloads the Oracle Cards file (~168 MB) and upserts all MTG cards in one pass.
* Much faster and more complete than the per-set discovery approach — catches all
* missing sets in one shot without rate-limit concerns.
*
* Also refreshes prices, legalities, and edhrec_rank on existing cards.
*/
import { sql } from '@vercel/postgres';
const BATCH_SIZE = 100;
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function fetchBulkDownloadUrl() {
const response = await fetch('https://api.scryfall.com/bulk-data/oracle_cards', {
headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
});
if (!response.ok) {
throw new Error(`Scryfall bulk-data API error: ${response.status}`);
}
return response.json();
}
function mapCard(card) {
let currentPrice = null;
let marketPrice = null;
if (card.prices) {
currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null;
marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null;
}
return {
scryfallId: card.id,
oracleId: card.oracle_id || null,
illustrationId: card.illustration_id || null,
name: card.name,
setName: card.set_name,
setCode: card.set,
cardNumber: card.collector_number,
rarity: card.rarity,
manaCost: card.mana_cost || null,
cmc: card.cmc != null ? Math.round(card.cmc) : null,
cardType: card.type_line || null,
colors: JSON.stringify(card.colors || []),
colorIdentity: JSON.stringify(card.color_identity || []),
oracleText: card.oracle_text || null,
power: card.power || null,
toughness: card.toughness || null,
imageUrl: card.image_uris?.normal || null,
stockImageUrl: card.image_uris?.art_crop || null,
currentPrice,
marketPrice,
keywords: JSON.stringify(card.keywords || []),
legalities: JSON.stringify(card.legalities || {}),
flavorText: card.flavor_text || null,
artist: card.artist || null,
releasedAt: card.released_at || null,
layout: card.layout || null,
edhrecRank: card.edhrec_rank || null,
reserved: card.reserved || false,
reprint: card.reprint || false,
finishes: JSON.stringify(card.finishes || []),
};
}
async function upsertBatch(cards) {
if (cards.length === 0) return 0;
const columns = [
'scryfall_id', 'oracle_id', 'illustration_id', 'name', 'set_name', 'set_code',
'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors',
'color_identity', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url',
'current_price', 'market_price', 'keywords', 'legalities', 'flavor_text', 'artist',
'released_at', 'layout', 'edhrec_rank', 'reserved', 'reprint', 'finishes', 'verified',
];
const values = [];
const placeholders = [];
let paramIdx = 1;
for (const row of cards) {
const rowValues = [
row.scryfallId, row.oracleId, row.illustrationId, row.name, row.setName, row.setCode,
row.cardNumber, row.rarity, 'MTG', row.manaCost, row.cmc, row.cardType, row.colors,
row.colorIdentity, row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl,
row.currentPrice, row.marketPrice, row.keywords, row.legalities, row.flavorText, row.artist,
row.releasedAt, row.layout, row.edhrecRank, row.reserved, row.reprint, row.finishes, true,
];
const rowP = rowValues.map(() => `$${paramIdx++}`);
placeholders.push(`(${rowP.join(',')})`);
values.push(...rowValues);
}
const updateCols = columns
.filter((c) => c !== 'scryfall_id')
.map((c) => `${c} = EXCLUDED.${c}`)
.join(', ');
await sql.query(
`INSERT INTO cards (${columns.join(', ')})
VALUES ${placeholders.join(',\n')}
ON CONFLICT (scryfall_id) DO UPDATE SET ${updateCols}, updated_at = CURRENT_TIMESTAMP`,
values
);
return cards.length;
}
/**
* Run a full bulk MTG sync. Returns a summary compatible with the catalog sync API shape.
*/
export async function runBulkMtgSync() {
const meta = await fetchBulkDownloadUrl();
console.log(`[bulk-sync] Oracle Cards updated: ${meta.updated_at}, size: ${(meta.size / 1024 / 1024).toFixed(0)} MB`);
const response = await fetch(meta.download_uri, {
headers: { 'User-Agent': 'DeckHearth/1.0' },
});
if (!response.ok) {
throw new Error(`Bulk download failed: ${response.status}`);
}
const allCards = await response.json();
const mtgCards = allCards.filter((c) => c.lang === 'en' && !c.digital);
let upserted = 0;
let errors = 0;
let batch = [];
for (const card of mtgCards) {
batch.push(mapCard(card));
if (batch.length >= BATCH_SIZE) {
try {
upserted += await upsertBatch(batch);
} catch (err) {
console.error('[bulk-sync] Batch error:', err.message);
errors += batch.length;
}
batch = [];
}
}
if (batch.length > 0) {
try {
upserted += await upsertBatch(batch);
} catch (err) {
console.error('[bulk-sync] Final batch error:', err.message);
errors += batch.length;
}
}
// Backfill oracle_id on older per-set imported cards
const backfill = await sql`
UPDATE cards target
SET oracle_id = source.oracle_id,
color_identity = COALESCE(target.color_identity, source.color_identity),
keywords = COALESCE(target.keywords, source.keywords),
legalities = COALESCE(target.legalities, source.legalities),
layout = COALESCE(target.layout, source.layout),
edhrec_rank = COALESCE(target.edhrec_rank, source.edhrec_rank)
FROM (
SELECT DISTINCT ON (name) name, oracle_id, color_identity, keywords, legalities, layout, edhrec_rank
FROM cards WHERE oracle_id IS NOT NULL AND game = 'MTG'
) source
WHERE target.name = source.name AND target.game = 'MTG' AND target.oracle_id IS NULL
`;
const summary = {
mode: 'bulk',
totalInFile: mtgCards.length,
upserted,
errors,
backfilledOracleId: backfill.rowCount || 0,
updatedAt: meta.updated_at,
};
console.log('[bulk-sync]', JSON.stringify(summary));
return summary;
}

51
migrations/1781440700404_add-scryfall-bulk-columns.js Normal file
View file

@ -0,0 +1,51 @@
/**
* Add Scryfall bulk-data columns to `cards` for richer MTG metadata.
* These enable Tagger tag joins (oracle_id, illustration_id), Commander
* deck validation (color_identity), format legality checks, keyword
* search, and better card detail pages.
*/
export const shorthands = undefined;
export const up = (pgm) => {
pgm.addColumns('cards', {
oracle_id: { type: 'varchar(36)', comment: 'Stable across printings; joins oracle tags' },
illustration_id: { type: 'varchar(36)', comment: 'Stable per artwork; joins art tags' },
color_identity: { type: 'jsonb', comment: '["W","U","B","R","G"] — includes symbols in rules text' },
keywords: { type: 'jsonb', comment: '["Flying","Trample",...] — searchable mechanics' },
legalities: { type: 'jsonb', comment: '{standard:"legal", modern:"not_legal",...}' },
flavor_text: { type: 'text' },
artist: { type: 'varchar(255)' },
released_at: { type: 'date' },
layout: { type: 'varchar(50)', comment: 'normal, transform, split, mdfc, adventure, etc.' },
edhrec_rank: { type: 'integer', comment: 'Commander popularity — lower is more popular' },
reserved: { type: 'boolean', default: false },
reprint: { type: 'boolean', default: false },
finishes: { type: 'jsonb', comment: '["nonfoil","foil","etched"]' },
});
pgm.createIndex('cards', 'oracle_id');
pgm.createIndex('cards', 'illustration_id');
pgm.createIndex('cards', 'color_identity', { method: 'gin' });
pgm.createIndex('cards', 'keywords', { method: 'gin' });
pgm.createIndex('cards', 'legalities', { method: 'gin' });
pgm.createIndex('cards', 'artist');
pgm.createIndex('cards', 'edhrec_rank');
};
export const down = (pgm) => {
pgm.dropColumns('cards', [
'oracle_id',
'illustration_id',
'color_identity',
'keywords',
'legalities',
'flavor_text',
'artist',
'released_at',
'layout',
'edhrec_rank',
'reserved',
'reprint',
'finishes',
]);
};

53
migrations/1781440721350_add-tagger-tables.js Normal file
View file

@ -0,0 +1,53 @@
/**
* Create tables for Scryfall Tagger community tags (oracle + art).
* Tags join to cards via oracle_id (oracle tags) or illustration_id (art tags).
*/
export const shorthands = undefined;
export const up = (pgm) => {
pgm.createTable('tags', {
id: { type: 'uuid', primaryKey: true },
slug: { type: 'varchar(255)', notNull: true },
label: { type: 'varchar(255)', notNull: true },
type: { type: 'varchar(20)', notNull: true, comment: 'oracle or illustration' },
description: { type: 'text' },
parent_ids: { type: 'jsonb' },
child_ids: { type: 'jsonb' },
aliases: { type: 'jsonb' },
created_at: { type: 'timestamp', default: pgm.func('CURRENT_TIMESTAMP') },
updated_at: { type: 'timestamp', default: pgm.func('CURRENT_TIMESTAMP') },
});
pgm.createIndex('tags', 'type');
pgm.createIndex('tags', 'slug');
pgm.createIndex('tags', 'label');
pgm.createTable('card_tags', {
id: { type: 'serial', primaryKey: true },
tag_id: { type: 'uuid', notNull: true, references: 'tags(id)', onDelete: 'CASCADE' },
card_id: { type: 'integer', references: 'cards(id)', onDelete: 'CASCADE' },
oracle_id: { type: 'varchar(36)', comment: 'For oracle tags — matches cards.oracle_id' },
illustration_id: { type: 'varchar(36)', comment: 'For art tags — matches cards.illustration_id' },
weight: { type: 'varchar(20)', default: 'median' },
annotation: { type: 'text' },
created_at: { type: 'timestamp', default: pgm.func('CURRENT_TIMESTAMP') },
});
pgm.createIndex('card_tags', 'tag_id');
pgm.createIndex('card_tags', 'card_id');
pgm.createIndex('card_tags', 'oracle_id');
pgm.createIndex('card_tags', 'illustration_id');
pgm.addConstraint('card_tags', 'card_tags_unique_tag_oracle', {
unique: ['tag_id', 'oracle_id'],
where: 'oracle_id IS NOT NULL',
});
pgm.addConstraint('card_tags', 'card_tags_unique_tag_illustration', {
unique: ['tag_id', 'illustration_id'],
where: 'illustration_id IS NOT NULL',
});
};
export const down = (pgm) => {
pgm.dropTable('card_tags');
pgm.dropTable('tags');
};

10
package-lock.json generated
View file

@ -8,6 +8,7 @@
"name": "deck-hearth",
"version": "0.1.0",
"dependencies": {
"@neondatabase/serverless": "^1.1.0",
"@upstash/ratelimit": "^2.0.8",
"@upstash/redis": "^1.38.0",
"@vercel/blob": "^1.1.1",
@ -1849,6 +1850,15 @@
"@tybys/wasm-util": "^0.10.0"
}
},
"node_modules/@neondatabase/serverless": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-1.1.0.tgz",
"integrity": "sha512-r3ZZhRjEcfEdKIZnoB1RusNgvHuaBRqfCzV4Gi+5A9yUX0S4HTws/ASWqt13wL4y4I+0rqsWGdA2w7EQXHi3+Q==",
"license": "MIT",
"engines": {
"node": ">=19.0.0"
}
},
"node_modules/@next/env": {
"version": "16.2.6",
"resolved": "https://registry.npmjs.org/@next/env/-/env-16.2.6.tgz",

3
package.json
View file

@ -12,6 +12,8 @@
"setup-db": "node scripts/setup-neon-db.js",
"import-popular": "node scripts/import-popular-sets.js",
"import-all": "node scripts/bulk-import-all.js",
"bulk-import": "node --env-file=.env.local scripts/bulk-import-scryfall.js",
"import-tags": "node --env-file=.env.local scripts/import-scryfall-tags.js",
"test": "vitest",
"test:run": "vitest run",
"test:smoke": "playwright test --project=smoke",
@ -19,6 +21,7 @@
"test:visual:update": "playwright test --project=visual --update-snapshots"
},
"dependencies": {
"@neondatabase/serverless": "^1.1.0",
"@upstash/ratelimit": "^2.0.8",
"@upstash/redis": "^1.38.0",
"@vercel/blob": "^1.1.1",

40
pages/admin/card-import.js
View file

@ -14,6 +14,8 @@ const CardImport = () => {
const [isSyncing, setIsSyncing] = useState(false);
const [syncResult, setSyncResult] = useState(null);
const [syncMode, setSyncMode] = useState('incremental');
const handleCatalogSync = async () => {
setIsSyncing(true);
setSyncResult(null);
@ -25,6 +27,7 @@ const CardImport = () => {
'Content-Type': 'application/json',
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
},
body: JSON.stringify({ mode: syncMode }),
});
const result = await response.json();
@ -171,9 +174,32 @@ const CardImport = () => {
Catalog sync
</h2>
<p className="text-sm max-w-2xl" style={{ color: 'var(--text-secondary)' }}>
Import up to three newest missing MTG and Pokémon sets — the same job the weekly Vercel cron runs.
Pending scan submissions for those sets are auto-linked to the catalog when a unique match exists.
{syncMode === 'bulk'
? 'Download Scryfall\'s full Oracle Cards bulk file and upsert all 36k+ MTG cards with rich metadata (legalities, keywords, color identity, tags). Takes ~30s.'
: 'Import up to three newest missing MTG and Pokémon sets — the same job the weekly Vercel cron runs. Pending scan submissions are auto-linked when a unique match exists.'}
</p>
<div className="flex gap-2 mt-3">
<button
onClick={() => setSyncMode('incremental')}
className="px-3 py-1.5 rounded-lg text-xs font-medium transition-all"
style={{
backgroundColor: syncMode === 'incremental' ? 'var(--accent-ember)' : 'var(--bg-tertiary)',
color: syncMode === 'incremental' ? '#fff' : 'var(--text-secondary)',
}}
>
Incremental (new sets)
</button>
<button
onClick={() => setSyncMode('bulk')}
className="px-3 py-1.5 rounded-lg text-xs font-medium transition-all"
style={{
backgroundColor: syncMode === 'bulk' ? 'var(--accent-ember)' : 'var(--bg-tertiary)',
color: syncMode === 'bulk' ? '#fff' : 'var(--text-secondary)',
}}
>
Bulk (full MTG refresh)
</button>
</div>
</div>
<Button
variant="primary"
@ -183,7 +209,7 @@ const CardImport = () => {
loading={isSyncing}
className="shrink-0"
>
{isSyncing ? 'Syncing catalog…' : 'Run catalog sync'}
{isSyncing ? 'Syncing catalog…' : syncMode === 'bulk' ? 'Run bulk sync' : 'Run catalog sync'}
</Button>
</div>
@ -204,9 +230,17 @@ const CardImport = () => {
</h3>
{syncResult.success ? (
<div className="text-sm space-y-2 text-green-900">
{syncResult.mode === 'bulk' ? (
<p>
Upserted {syncResult.upserted} MTG cards from Scryfall bulk data.
{syncResult.backfilledOracleId > 0 && ` Backfilled oracle_id on ${syncResult.backfilledOracleId} existing rows.`}
{syncResult.errors > 0 && ` (${syncResult.errors} errors)`}
</p>
) : (
<p>
Imported {syncResult.imported} cards ({syncResult.skipped} skipped).
</p>
)}
{syncResult.submissionsReconciled > 0 && (
<p>
Linked {syncResult.submissionsReconciled} pending scan submission

8
pages/api/admin/sync-catalog.js
View file

@ -1,6 +1,7 @@
import { withAdmin } from '../../../lib/permission-middleware';
import { checkImportRateLimit } from '../../../lib/rate-limit.js';
import { runCatalogSync } from '../../../lib/card-import/sync-catalog.js';
import { runBulkMtgSync } from '../../../lib/card-import/bulk-sync.js';
export default withAdmin(async function handler(req, res, user) {
if (req.method !== 'POST') {
@ -13,7 +14,14 @@ export default withAdmin(async function handler(req, res, user) {
return res.status(429).json({ error: 'Too many attempts. Try again later.' });
}
const mode = req.body?.mode || 'incremental';
try {
if (mode === 'bulk') {
const summary = await runBulkMtgSync();
return res.status(200).json({ success: true, ...summary });
}
const summary = await runCatalogSync();
return res.status(200).json({ success: true, ...summary });
} catch (error) {

217
scripts/bulk-import-scryfall.js Normal file
View file

@ -0,0 +1,217 @@
/**
* Bulk import MTG cards from Scryfall's Oracle Cards bulk data file.
*
* Downloads the ~168 MB Oracle Cards JSON (one entry per unique card/oracle_id),
* then upserts every card into the `cards` table keyed on `scryfall_id`.
*
* Populates all new bulk-data columns (oracle_id, illustration_id, color_identity,
* keywords, legalities, flavor_text, artist, released_at, layout, edhrec_rank,
* reserved, reprint, finishes) in addition to the original columns.
*
* Usage:
* POSTGRES_URL=<url> node scripts/bulk-import-scryfall.js
*
* Options (env vars):
* BULK_TYPE — "oracle_cards" (default) | "default_cards" | "unique_artwork"
* BATCH_SIZE — rows per INSERT batch (default 100)
* DRY_RUN — "true" to fetch and count without writing to DB
*/
import { neon } from '@neondatabase/serverless';
if (!process.env.POSTGRES_URL) {
console.error('POSTGRES_URL is required');
process.exit(1);
}
const sql = neon(process.env.POSTGRES_URL, { fullResults: false });
const BULK_TYPE = process.env.BULK_TYPE || 'oracle_cards';
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '100', 10);
const DRY_RUN = process.env.DRY_RUN === 'true';
async function fetchBulkDownloadUrl(type) {
const response = await fetch(`https://api.scryfall.com/bulk-data/${type}`, {
headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
});
if (!response.ok) {
throw new Error(`Failed to fetch bulk-data metadata: ${response.status}`);
}
const data = await response.json();
console.log(`[bulk-import] File: ${data.name}`);
console.log(`[bulk-import] Updated: ${data.updated_at}`);
console.log(`[bulk-import] Size: ${(data.size / 1024 / 1024).toFixed(1)} MB`);
return data.download_uri;
}
function mapCard(card) {
let currentPrice = null;
let marketPrice = null;
if (card.prices) {
currentPrice = card.prices.usd ? parseFloat(card.prices.usd) : null;
marketPrice = card.prices.usd_foil ? parseFloat(card.prices.usd_foil) : null;
}
return {
scryfallId: card.id,
oracleId: card.oracle_id || null,
illustrationId: card.illustration_id || null,
name: card.name,
setName: card.set_name,
setCode: card.set,
cardNumber: card.collector_number,
rarity: card.rarity,
manaCost: card.mana_cost || null,
cmc: card.cmc != null ? Math.round(card.cmc) : null,
cardType: card.type_line || null,
colors: JSON.stringify(card.colors || []),
colorIdentity: JSON.stringify(card.color_identity || []),
oracleText: card.oracle_text || null,
power: card.power || null,
toughness: card.toughness || null,
imageUrl: card.image_uris?.normal || null,
stockImageUrl: card.image_uris?.art_crop || null,
currentPrice,
marketPrice,
keywords: JSON.stringify(card.keywords || []),
legalities: JSON.stringify(card.legalities || {}),
flavorText: card.flavor_text || null,
artist: card.artist || null,
releasedAt: card.released_at || null,
layout: card.layout || null,
edhrecRank: card.edhrec_rank || null,
reserved: card.reserved || false,
reprint: card.reprint || false,
finishes: JSON.stringify(card.finishes || []),
};
}
function buildUpsertQuery(batch) {
const columns = [
'scryfall_id', 'oracle_id', 'illustration_id', 'name', 'set_name', 'set_code',
'card_number', 'rarity', 'game', 'mana_cost', 'cmc', 'card_type', 'colors',
'color_identity', 'oracle_text', 'power', 'toughness', 'image_url', 'stock_image_url',
'current_price', 'market_price', 'keywords', 'legalities', 'flavor_text', 'artist',
'released_at', 'layout', 'edhrec_rank', 'reserved', 'reprint', 'finishes', 'verified',
];
const placeholders = [];
const values = [];
let paramIdx = 1;
for (const row of batch) {
const rowPlaceholders = [];
const rowValues = [
row.scryfallId, row.oracleId, row.illustrationId, row.name, row.setName, row.setCode,
row.cardNumber, row.rarity, 'MTG', row.manaCost, row.cmc, row.cardType, row.colors,
row.colorIdentity, row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl,
row.currentPrice, row.marketPrice, row.keywords, row.legalities, row.flavorText, row.artist,
row.releasedAt, row.layout, row.edhrecRank, row.reserved, row.reprint, row.finishes, true,
];
for (let i = 0; i < rowValues.length; i++) {
rowPlaceholders.push(`$${paramIdx}`);
paramIdx += 1;
}
placeholders.push(`(${rowPlaceholders.join(', ')})`);
values.push(...rowValues);
}
const updateCols = columns
.filter((c) => c !== 'scryfall_id')
.map((c) => `${c} = EXCLUDED.${c}`)
.join(', ');
const query = `
INSERT INTO cards (${columns.join(', ')})
VALUES ${placeholders.join(',\n')}
ON CONFLICT (scryfall_id) DO UPDATE SET
${updateCols},
updated_at = CURRENT_TIMESTAMP
`;
return { query, values };
}
async function processBatch(batch, stats) {
if (DRY_RUN) {
stats.skipped += batch.length;
return;
}
const { query, values } = buildUpsertQuery(batch);
try {
await sql.query(query, values);
stats.upserted += batch.length;
} catch (error) {
console.error(`[bulk-import] Batch failed (${batch.length} rows):`, error.message);
for (const row of batch) {
try {
const { query: singleQ, values: singleV } = buildUpsertQuery([row]);
await sql.query(singleQ, singleV);
stats.upserted += 1;
} catch (singleErr) {
console.error(`[bulk-import] Failed: ${row.name} (${row.scryfallId}):`, singleErr.message);
stats.errors += 1;
}
}
}
}
async function run() {
console.log(`[bulk-import] Starting Scryfall bulk import (type=${BULK_TYPE}, batch=${BATCH_SIZE}, dryRun=${DRY_RUN})`);
const downloadUrl = await fetchBulkDownloadUrl(BULK_TYPE);
console.log(`[bulk-import] Downloading...`);
const response = await fetch(downloadUrl, {
headers: { 'User-Agent': 'DeckHearth/1.0' },
});
if (!response.ok) {
throw new Error(`Download failed: ${response.status}`);
}
const text = await response.text();
console.log(`[bulk-import] Downloaded ${(text.length / 1024 / 1024).toFixed(1)} MB, parsing...`);
const cards = JSON.parse(text);
console.log(`[bulk-import] Total cards in bulk file: ${cards.length}`);
const mtgCards = cards.filter((c) => c.lang === 'en' && !c.digital);
console.log(`[bulk-import] MTG cards (English, paper): ${mtgCards.length}`);
const stats = { upserted: 0, skipped: 0, errors: 0 };
let batch = [];
let processed = 0;
for (const card of mtgCards) {
batch.push(mapCard(card));
if (batch.length >= BATCH_SIZE) {
await processBatch(batch, stats);
processed += batch.length;
batch = [];
if (processed % 5000 === 0) {
console.log(`[bulk-import] Progress: ${processed}/${mtgCards.length} (${((processed / mtgCards.length) * 100).toFixed(1)}%)`);
}
}
}
if (batch.length > 0) {
await processBatch(batch, stats);
processed += batch.length;
}
console.log(`[bulk-import] Complete!`);
console.log(`[bulk-import] Upserted: ${stats.upserted}`);
console.log(`[bulk-import] Skipped (dry run): ${stats.skipped}`);
console.log(`[bulk-import] Errors: ${stats.errors}`);
console.log(`[bulk-import] Total processed: ${processed}`);
}
run().catch((err) => {
console.error('[bulk-import] Fatal error:', err);
process.exit(1);
});

247
scripts/import-scryfall-tags.js Normal file
View file

@ -0,0 +1,247 @@
/**
* Import Scryfall Tagger community tags (Oracle Tags + Art Tags) into the
* `tags` and `card_tags` tables.
*
* Oracle tags describe card functionality (removal, ramp, draw, etc.)
* Art tags describe what's depicted in the artwork (dragon, forest, battle, etc.)
*
* Tags join to cards via oracle_id (oracle tags) or illustration_id (art tags).
* The card_tags table stores the raw join IDs so cards added later are automatically
* matched without re-importing tags.
*
* Usage:
* POSTGRES_URL=<url> node scripts/import-scryfall-tags.js
*
* Options (env vars):
* TAG_TYPE — "both" (default) | "oracle" | "art"
* BATCH_SIZE — rows per INSERT batch (default 200)
* DRY_RUN — "true" to fetch and count without writing to DB
*/
import { neon } from '@neondatabase/serverless';
if (!process.env.POSTGRES_URL) {
console.error('POSTGRES_URL is required');
process.exit(1);
}
const sql = neon(process.env.POSTGRES_URL, { fullResults: false });
const TAG_TYPE = process.env.TAG_TYPE || 'both';
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '200', 10);
const DRY_RUN = process.env.DRY_RUN === 'true';
async function fetchBulkDownloadUrl(type) {
const response = await fetch(`https://api.scryfall.com/bulk-data/${type}`, {
headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
});
if (!response.ok) {
throw new Error(`Failed to fetch bulk-data metadata for ${type}: ${response.status}`);
}
const data = await response.json();
console.log(`[tag-import] File: ${data.name} (${(data.size / 1024 / 1024).toFixed(1)} MB)`);
return data.download_uri;
}
async function downloadJson(url) {
console.log(`[tag-import] Downloading...`);
const response = await fetch(url, {
headers: { 'User-Agent': 'DeckHearth/1.0' },
});
if (!response.ok) {
throw new Error(`Download failed: ${response.status}`);
}
const text = await response.text();
console.log(`[tag-import] Downloaded ${(text.length / 1024 / 1024).toFixed(1)} MB`);
return JSON.parse(text);
}
async function upsertTags(tags, stats) {
const batch = [];
for (const tag of tags) {
batch.push({
id: tag.id,
slug: tag.slug,
label: tag.label,
type: tag.type,
description: tag.description || null,
parentIds: JSON.stringify(tag.parent_ids || []),
childIds: JSON.stringify(tag.child_ids || []),
aliases: JSON.stringify(tag.aliases || []),
});
if (batch.length >= BATCH_SIZE) {
await flushTagBatch(batch, stats);
batch.length = 0;
}
}
if (batch.length > 0) {
await flushTagBatch(batch, stats);
}
}
async function flushTagBatch(batch, stats) {
if (DRY_RUN) {
stats.tags += batch.length;
return;
}
const placeholders = [];
const values = [];
let paramIdx = 1;
for (const row of batch) {
const rowP = [];
const rowV = [row.id, row.slug, row.label, row.type, row.description, row.parentIds, row.childIds, row.aliases];
for (let i = 0; i < rowV.length; i++) {
rowP.push(`$${paramIdx}`);
paramIdx += 1;
}
placeholders.push(`(${rowP.join(', ')})`);
values.push(...rowV);
}
const query = `
INSERT INTO tags (id, slug, label, type, description, parent_ids, child_ids, aliases)
VALUES ${placeholders.join(',\n')}
ON CONFLICT (id) DO UPDATE SET
slug = EXCLUDED.slug,
label = EXCLUDED.label,
description = EXCLUDED.description,
parent_ids = EXCLUDED.parent_ids,
child_ids = EXCLUDED.child_ids,
aliases = EXCLUDED.aliases,
updated_at = CURRENT_TIMESTAMP
`;
try {
await sql.query(query, values);
stats.tags += batch.length;
} catch (error) {
console.error(`[tag-import] Tag batch failed:`, error.message);
stats.tagErrors += batch.length;
}
}
async function importTaggings(tags, stats) {
let batch = [];
let processed = 0;
const total = tags.reduce((sum, t) => sum + (t.taggings?.length || 0), 0);
console.log(`[tag-import] Total taggings to import: ${total}`);
for (const tag of tags) {
if (!tag.taggings || tag.taggings.length === 0) continue;
for (const tagging of tag.taggings) {
batch.push({
tagId: tag.id,
oracleId: tagging.oracle_id || null,
illustrationId: tagging.illustration_id || null,
weight: tagging.weight || 'median',
annotation: tagging.annotation || null,
});
if (batch.length >= BATCH_SIZE) {
await flushTaggingBatch(batch, stats);
processed += batch.length;
batch = [];
if (processed % 10000 === 0) {
console.log(`[tag-import] Tagging progress: ${processed}/${total} (${((processed / total) * 100).toFixed(1)}%)`);
}
}
}
}
if (batch.length > 0) {
await flushTaggingBatch(batch, stats);
processed += batch.length;
}
console.log(`[tag-import] Taggings processed: ${processed}`);
}
async function flushTaggingBatch(batch, stats) {
if (DRY_RUN) {
stats.taggings += batch.length;
return;
}
const placeholders = [];
const values = [];
let paramIdx = 1;
for (const row of batch) {
const rowP = [];
const rowV = [row.tagId, row.oracleId, row.illustrationId, row.weight, row.annotation];
for (let i = 0; i < rowV.length; i++) {
rowP.push(`$${paramIdx}`);
paramIdx += 1;
}
placeholders.push(`(${rowP.join(', ')})`);
values.push(...rowV);
}
const uniqueCol = batch[0].oracleId ? 'card_tags_unique_tag_oracle' : 'card_tags_unique_tag_illustration';
const query = `
INSERT INTO card_tags (tag_id, oracle_id, illustration_id, weight, annotation)
VALUES ${placeholders.join(',\n')}
ON CONFLICT ON CONSTRAINT ${uniqueCol} DO UPDATE SET
weight = EXCLUDED.weight,
annotation = EXCLUDED.annotation
`;
try {
await sql.query(query, values);
stats.taggings += batch.length;
} catch (error) {
console.error(`[tag-import] Tagging batch failed:`, error.message);
stats.taggingErrors += batch.length;
}
}
async function importFile(type) {
const bulkType = type === 'oracle' ? 'oracle_tags' : 'art_tags';
console.log(`\n[tag-import] === Importing ${type} tags (${bulkType}) ===`);
const url = await fetchBulkDownloadUrl(bulkType);
const tags = await downloadJson(url);
console.log(`[tag-import] Tags in file: ${tags.length}`);
const stats = { tags: 0, tagErrors: 0, taggings: 0, taggingErrors: 0 };
console.log(`[tag-import] Upserting tag definitions...`);
await upsertTags(tags, stats);
console.log(`[tag-import] Tags upserted: ${stats.tags}, errors: ${stats.tagErrors}`);
console.log(`[tag-import] Importing taggings...`);
await importTaggings(tags, stats);
console.log(`[tag-import] Taggings upserted: ${stats.taggings}, errors: ${stats.taggingErrors}`);
return stats;
}
async function run() {
console.log(`[tag-import] Starting Scryfall Tagger import (type=${TAG_TYPE}, batch=${BATCH_SIZE}, dryRun=${DRY_RUN})`);
const results = {};
if (TAG_TYPE === 'oracle' || TAG_TYPE === 'both') {
results.oracle = await importFile('oracle');
}
if (TAG_TYPE === 'art' || TAG_TYPE === 'both') {
results.art = await importFile('art');
}
console.log(`\n[tag-import] === Summary ===`);
console.log(JSON.stringify(results, null, 2));
}
run().catch((err) => {
console.error('[tag-import] Fatal error:', err);
process.exit(1);
});