diff --git a/docs/SCHEMA_MAP.md b/docs/SCHEMA_MAP.md index 9a46511..bf6a9a1 100644 --- a/docs/SCHEMA_MAP.md +++ b/docs/SCHEMA_MAP.md @@ -15,14 +15,16 @@ > historical effects into the migration history; until then this file > remains the curated reference for the full prod shape. > -> **Last reviewed:** 2026-05-22 against `scripts/setup-neon-db.js` + every `scripts/add-*.js` and `scripts/fix-*.js` in repo HEAD. +> **Last reviewed:** 2026-06-14 against `migrations/` HEAD (through +> `1781442340000_cleanup-lorcana-orphans.js`). ## Quick model groups | Group | Tables | Purpose | | --- | --- | --- | | **Identity** | `users`, `user_settings`, `user_avatars` | Accounts, profile, preferences | -| **Catalog** | `cards` | Master card list across MTG / Pokémon / Lorcana | +| **Catalog** | `cards`, `tags`, `card_tags` | Master card list + Scryfall Tagger community tags | +| **Sync telemetry** | `catalog_sync_log` | Per-game bulk/incremental catalog sync history | | **Ownership** | `user_cards`, `user_favorites` | What a user owns / has favorited (UI: **My Collection**) | | **Collections** | `collections`, `collection_cards`, `collection_permissions`, `collection_activity` | Curated card lists with sharing (UI: **Lists**) | | **Decks** | `decks`, `deck_cards` | Playable deck definitions | @@ -73,15 +75,30 @@ | `power`, `toughness` | `VARCHAR(10)` | MTG creatures | | `image_url`, `stock_image_url` | `TEXT` | | | `current_price`, `market_price` | `DECIMAL(10,2)` | | -| `scryfall_id` | `VARCHAR(255) UNIQUE` | Use for dedupe on MTG import | +| `scryfall_id` | `VARCHAR(255) UNIQUE` | External dedupe key (Scryfall UUID, Pokémon TCG id, Lorcana `Unique_ID`) | | `verified` | `BOOLEAN` default `false` | Admin-edited cards | -| `quantity` | `INTEGER` default `0` | **Unused; consider dropping — quantity lives in `user_cards`** | -| `favorited` | `BOOLEAN` default `false` | **Unused; favorites live in `user_favorites`** | +| `quantity` | `INTEGER` default `0` | **Unused; consider dropping — quantity lives in `user_cards`** (`1781442330001_reconcile-cards-columns`) | +| `favorited` | `BOOLEAN` default `false` | **Unused; favorites live in `user_favorites`** (`1781442330001_reconcile-cards-columns`) | +| `oracle_id` | `VARCHAR(36)` | Stable across MTG printings; joins oracle tags (`1781440700404`) | +| `illustration_id` | `VARCHAR(36)` | Stable per artwork; joins art tags (`1781440700404`) | +| `color_identity` | `JSONB` | MTG Commander identity; Lorcana ink colors (`1781440700404`) | +| `keywords` | `JSONB` | MTG mechanics / Lorcana classifications (`1781440700404`) | +| `legalities` | `JSONB` | Format legality map (`1781440700404`) | +| `flavor_text` | `TEXT` | (`1781440700404`) | +| `artist` | `VARCHAR(255)` | (`1781440700404`) | +| `released_at` | `DATE` | (`1781440700404`) | +| `layout` | `VARCHAR(50)` | MTG card layout (`1781440700404`) | +| `edhrec_rank` | `INTEGER` | MTG Commander popularity; Lorcana lore value repurposed (`1781440700404`) | +| `reserved` | `BOOLEAN` default `false` | MTG Reserved List (`1781440700404`) | +| `reprint` | `BOOLEAN` default `false` | (`1781440700404`) | +| `finishes` | `JSONB` | `["nonfoil","foil","etched"]` (`1781440700404`) | | `created_at`, `updated_at` | `TIMESTAMP` default now | | -**Indexes (post `1779853647565_add-pg-trgm-card-name-index`):** +**Indexes (post `1779853647565_add-pg-trgm-card-name-index` + `1781440700404`):** - `idx_cards_name_trgm` — GIN on `name` using `gin_trgm_ops` for Layer-1 OCR fuzzy match (`similarity()` / `pg_trgm`). +- `cards_oracle_id_index`, `cards_illustration_id_index`, `cards_artist_index`, `cards_edhrec_rank_index` +- GIN on `color_identity`, `keywords`, `legalities` **Extensions used by scan pipeline:** @@ -99,7 +116,53 @@ | `is_foil` | `BOOLEAN` default `false` | | | `notes` | `TEXT` | | | `scan_image_url` | `TEXT` | Vercel Blob URL of scanner capture (`1779908094455_add-user-cards-scan-image-url`) | -| | | **UNIQUE(user_id, card_id, is_foil)** | +| | | **UNIQUE(user_id, card_id, is_foil)** — canonical 3-column constraint per `1781442330006_reconcile-user-cards-unique` (foil and non-foil are separate rows). The historical 2-column `user_cards_user_card_unique` constraint is dropped if present. | + +### tags + +Scryfall Tagger community tags (`1781440721350_add-tagger-tables`). + +| Column | Type | Notes | +| --- | --- | --- | +| `id` | `UUID PK` | Stable tag id from Scryfall bulk | +| `slug` | `VARCHAR(255) NOT NULL` | URL-safe identifier | +| `label` | `VARCHAR(255) NOT NULL` | Human-readable name | +| `type` | `VARCHAR(20) NOT NULL` | `'oracle'` (functional) or `'illustration'` (art) | +| `description` | `TEXT` | Optional | +| `parent_ids`, `child_ids`, `aliases` | `JSONB` | Tag hierarchy | +| `created_at`, `updated_at` | `TIMESTAMP` | | + +### card_tags + +Joins tags to cards via `oracle_id` (oracle tags) or `illustration_id` (art tags). + +| Column | Type | Notes | +| --- | --- | --- | +| `id` | `SERIAL PK` | | +| `tag_id` | `UUID FK tags(id) ON DELETE CASCADE` | | +| `card_id` | `INTEGER FK cards(id) ON DELETE CASCADE` | Optional resolved link | +| `oracle_id` | `VARCHAR(36)` | For oracle tags | +| `illustration_id` | `VARCHAR(36)` | For art tags | +| `weight` | `VARCHAR(20)` default `'median'` | `very_strong` / `strong` / `median` / `weak` | +| `annotation` | `TEXT` | Optional per-tagging note | +| `created_at` | `TIMESTAMP` | | +| | | **UNIQUE(tag_id, oracle_id)** where `oracle_id IS NOT NULL` | +| | | **UNIQUE(tag_id, illustration_id)** where `illustration_id IS NOT NULL` | + +### catalog_sync_log + +Per-game catalog sync telemetry (`1781442329511_add-catalog-sync-log`). + +| Column | Type | Notes | +| --- | --- | --- | +| `id` | `SERIAL PK` | | +| `game` | `VARCHAR(20) NOT NULL` | `'mtg'`, `'pokemon'`, `'lorcana'` | +| `mode` | `VARCHAR(20) NOT NULL` | `'unified'`, `'bulk'`, `'incremental'` | +| `upserted` | `INTEGER` default `0` | Cards written this run | +| `errors` | `INTEGER` default `0` | Failed rows | +| `source_updated_at` | `TIMESTAMPTZ` | Upstream bulk file timestamp when available | +| `ran_at` | `TIMESTAMPTZ` default now | | +| `details` | `JSONB` | Per-run summary payload | ### user_favorites diff --git a/lib/card-import/lorcana-bulk.js b/lib/card-import/lorcana-bulk.js new file mode 100644 index 0000000..b1aa169 --- /dev/null +++ b/lib/card-import/lorcana-bulk.js @@ -0,0 +1,185 @@ +/** + * Bulk catalog sync for Disney Lorcana via lorcana-api.com /bulk/cards. + * + * Single fetch returns all cards (~2.3k). Upserts keyed on scryfall_id (stores Unique_ID). + */ + +import { sql as vercelSql } from '@vercel/postgres'; + +const BATCH_SIZE = 100; +const BULK_URL = 'https://api.lorcana-api.com/bulk/cards'; + +/** + * Map a Lorcana bulk API card object to cards-table insert shape. + */ +export function mapLorcanaCard(card) { + const colors = card.Color + ? card.Color.split(',').map((entry) => entry.trim()).filter(Boolean) + : []; + + const keywords = []; + if (card.Classifications) { + keywords.push( + ...card.Classifications.split(',').map((entry) => entry.trim()).filter(Boolean) + ); + } + if (card.Abilities) { + keywords.push(...card.Abilities.split(',').map((entry) => entry.trim()).filter(Boolean)); + } + if (card.Inkable === true) { + keywords.push('Inkable'); + } + + const cost = card.Cost != null ? Number(card.Cost) : null; + + return { + scryfallId: card.Unique_ID, + name: card.Name, + setName: card.Set_Name, + setCode: card.Set_ID, + cardNumber: card.Card_Num != null ? String(card.Card_Num) : null, + rarity: card.Rarity || null, + manaCost: cost != null ? String(cost) : null, + cmc: cost, + cardType: card.Type || null, + colors: JSON.stringify(colors), + colorIdentity: JSON.stringify(colors), + oracleText: card.Body_Text || null, + power: card.Strength != null ? String(card.Strength) : null, + toughness: card.Willpower != null ? String(card.Willpower) : null, + imageUrl: card.Image || null, + stockImageUrl: card.Image || null, + flavorText: card.Flavor_Text || null, + artist: card.Artist || null, + edhrecRank: card.Lore != null ? Number(card.Lore) : null, + keywords: JSON.stringify(keywords), + }; +} + +export function buildUpsertQuery(batch) { + const columns = [ + 'scryfall_id', 'name', 'set_name', 'set_code', 'card_number', 'rarity', 'game', + 'mana_cost', 'cmc', 'card_type', 'colors', 'color_identity', 'oracle_text', + 'power', 'toughness', 'image_url', 'stock_image_url', 'keywords', 'flavor_text', + 'artist', 'edhrec_rank', 'verified', + ]; + + const placeholders = []; + const values = []; + let paramIdx = 1; + + for (const row of batch) { + const rowPlaceholders = []; + const rowValues = [ + row.scryfallId, row.name, row.setName, row.setCode, row.cardNumber, row.rarity, + 'Lorcana', row.manaCost, row.cmc, row.cardType, row.colors, row.colorIdentity, + row.oracleText, row.power, row.toughness, row.imageUrl, row.stockImageUrl, + row.keywords, row.flavorText, row.artist, row.edhrecRank, true, + ]; + + for (let i = 0; i < rowValues.length; i += 1) { + rowPlaceholders.push(`$${paramIdx}`); + paramIdx += 1; + } + placeholders.push(`(${rowPlaceholders.join(', ')})`); + values.push(...rowValues); + } + + const updateCols = columns + .filter((col) => col !== 'scryfall_id') + .map((col) => `${col} = EXCLUDED.${col}`) + .join(', '); + + const query = ` + INSERT INTO cards (${columns.join(', ')}) + VALUES ${placeholders.join(',\n')} + ON CONFLICT (scryfall_id) DO UPDATE SET + ${updateCols}, + updated_at = CURRENT_TIMESTAMP + `; + + return { query, values }; +} + +async function upsertBatch(sql, batch) { + if (batch.length === 0) return 0; + const { query, values } = buildUpsertQuery(batch); + await sql.query(query, values); + return batch.length; +} + +export async function fetchLorcanaBulkCards() { + const response = await fetch(BULK_URL, { + headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' }, + }); + if (!response.ok) { + throw new Error(`Lorcana bulk API error: ${response.status}`); + } + return response.json(); +} + +/** + * Run a full Lorcana bulk sync. Returns summary compatible with catalog sync reporting. + * + * @param {{ sql?: { query: (q: string, v: unknown[]) => Promise }, dryRun?: boolean }} options + */ +export async function runBulkLorcanaSync(options = {}) { + const dryRun = options.dryRun ?? false; + const sql = options.sql ?? vercelSql; + + const allCards = await fetchLorcanaBulkCards(); + const lorcanaCards = allCards.filter( + (card) => !card.Gamemode || card.Gamemode === 'Lorcana' + ); + + let upserted = 0; + let errors = 0; + let batch = []; + + for (const card of lorcanaCards) { + if (!card.Unique_ID) { + errors += 1; + continue; + } + + batch.push(mapLorcanaCard(card)); + + if (batch.length >= BATCH_SIZE) { + try { + if (!dryRun) { + upserted += await upsertBatch(sql, batch); + } else { + upserted += batch.length; + } + } catch (err) { + console.error('[lorcana-bulk] Batch error:', err.message); + errors += batch.length; + } + batch = []; + } + } + + if (batch.length > 0) { + try { + if (!dryRun) { + upserted += await upsertBatch(sql, batch); + } else { + upserted += batch.length; + } + } catch (err) { + console.error('[lorcana-bulk] Final batch error:', err.message); + errors += batch.length; + } + } + + const summary = { + mode: 'bulk', + game: 'lorcana', + totalInFile: lorcanaCards.length, + upserted, + errors, + }; + + console.log('[lorcana-bulk]', JSON.stringify(summary)); + return summary; +} diff --git a/lib/card-import/pokemon-bulk.js b/lib/card-import/pokemon-bulk.js new file mode 100644 index 0000000..0608352 --- /dev/null +++ b/lib/card-import/pokemon-bulk.js @@ -0,0 +1,215 @@ +/** + * Bulk Pokémon catalog sync from pokemon-tcg-data GitHub JSON. + * + * Fetches all sets from sets/en.json, then upserts every card in each set + * (keyed on scryfall_id = external card id). Populates legalities from set metadata. + */ + +import { sql } from '@vercel/postgres'; + +import { + fetchPokemonSetCards, + fetchPokemonSets, + mapGithubCardForInsert, +} from './pokemon-github.js'; + +const DEFAULT_BATCH_SIZE = 100; +const DEFAULT_SET_DELAY_MS = 100; + +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +function getSetLegalities(setMeta) { + if (setMeta?.legalities && typeof setMeta.legalities === 'object') { + return setMeta.legalities; + } + return {}; +} + +export function mapPokemonBulkRow(mapped, setLegalities) { + return { + scryfallId: mapped.externalId, + name: mapped.name, + setName: mapped.setName, + setCode: mapped.setCode, + cardNumber: mapped.cardNumber, + rarity: mapped.rarity, + cardType: mapped.cardType, + colors: JSON.stringify(mapped.types || []), + oracleText: mapped.flavorText, + power: mapped.hp, + imageUrl: mapped.imageSmall, + stockImageUrl: mapped.imageLarge, + legalities: JSON.stringify(setLegalities), + }; +} + +function buildUpsertQuery(batch) { + const columns = [ + 'scryfall_id', 'name', 'set_name', 'set_code', 'card_number', 'rarity', 'game', + 'mana_cost', 'cmc', 'card_type', 'colors', 'oracle_text', 'power', 'toughness', + 'image_url', 'stock_image_url', 'current_price', 'market_price', 'legalities', 'verified', + ]; + + const placeholders = []; + const values = []; + let paramIdx = 1; + + for (const row of batch) { + const rowPlaceholders = []; + const rowValues = [ + row.scryfallId, + row.name, + row.setName, + row.setCode, + row.cardNumber, + row.rarity, + 'Pokemon', + null, + null, + row.cardType, + row.colors, + row.oracleText, + row.power, + null, + row.imageUrl, + row.stockImageUrl, + null, + null, + row.legalities, + true, + ]; + + for (let i = 0; i < rowValues.length; i += 1) { + rowPlaceholders.push(`$${paramIdx}`); + paramIdx += 1; + } + placeholders.push(`(${rowPlaceholders.join(', ')})`); + values.push(...rowValues); + } + + const updateCols = columns + .filter((c) => c !== 'scryfall_id') + .map((c) => `${c} = EXCLUDED.${c}`) + .join(', '); + + const query = ` + INSERT INTO cards (${columns.join(', ')}) + VALUES ${placeholders.join(',\n')} + ON CONFLICT (scryfall_id) DO UPDATE SET + ${updateCols}, + updated_at = CURRENT_TIMESTAMP + `; + + return { query, values }; +} + +async function upsertBatch(batch, queryFn, stats) { + if (batch.length === 0) return; + + if (stats.dryRun) { + stats.skipped += batch.length; + return; + } + + const { query, values } = buildUpsertQuery(batch); + try { + await queryFn(query, values); + stats.upserted += batch.length; + } catch (error) { + console.error(`[pokemon-bulk] Batch failed (${batch.length} rows):`, error.message); + for (const row of batch) { + try { + const { query: singleQ, values: singleV } = buildUpsertQuery([row]); + await queryFn(singleQ, singleV); + stats.upserted += 1; + } catch (singleErr) { + console.error(`[pokemon-bulk] Failed: ${row.name} (${row.scryfallId}):`, singleErr.message); + stats.errors += 1; + } + } + } +} + +async function upsertSetCards(setMeta, cards, options, stats) { + const setLegalities = getSetLegalities(setMeta); + let batch = []; + + for (const card of cards) { + const mapped = mapGithubCardForInsert(card, setMeta); + batch.push(mapPokemonBulkRow(mapped, setLegalities)); + + if (batch.length >= options.batchSize) { + await upsertBatch(batch, options.queryFn, stats); + batch = []; + } + } + + if (batch.length > 0) { + await upsertBatch(batch, options.queryFn, stats); + } +} + +/** + * Run a full Pokémon bulk sync from pokemon-tcg-data GitHub. + * Returns a summary compatible with multi-game catalog sync orchestration. + */ +export async function runBulkPokemonSync(options = {}) { + const batchSize = options.batchSize ?? parseInt(process.env.BATCH_SIZE || String(DEFAULT_BATCH_SIZE), 10); + const dryRun = options.dryRun ?? process.env.DRY_RUN === 'true'; + const setDelayMs = options.setDelayMs ?? DEFAULT_SET_DELAY_MS; + const queryFn = options.query ?? ((query, values) => sql.query(query, values)); + + const sets = await fetchPokemonSets(); + console.log(`[pokemon-bulk] Found ${sets.length} sets in pokemon-tcg-data`); + + const stats = { + dryRun, + upserted: 0, + skipped: 0, + errors: 0, + setsProcessed: 0, + }; + + for (let index = 0; index < sets.length; index += 1) { + const setMeta = sets[index]; + const setCode = setMeta.id; + + if (!setCode) { + console.warn('[pokemon-bulk] Skipping set with no id'); + continue; + } + + try { + const cards = await fetchPokemonSetCards(setCode); + stats.setsProcessed += 1; + + if (cards.length === 0) { + console.log(`[pokemon-bulk] Set ${setCode}: no cards (empty or missing file)`); + } else { + await upsertSetCards(setMeta, cards, { batchSize, queryFn }, stats); + console.log(`[pokemon-bulk] Set ${setCode}: processed ${cards.length} cards`); + } + } catch (error) { + console.error(`[pokemon-bulk] Set ${setCode} failed:`, error.message); + stats.errors += 1; + } + + if (index < sets.length - 1) { + await delay(setDelayMs); + } + } + + const summary = { + mode: 'bulk', + game: 'pokemon', + setsProcessed: stats.setsProcessed, + totalSets: sets.length, + upserted: stats.upserted, + skipped: stats.skipped, + errors: stats.errors, + dryRun, + }; + + console.log('[pokemon-bulk]', JSON.stringify(summary)); + return summary; +} diff --git a/lib/card-import/sync-catalog.js b/lib/card-import/sync-catalog.js index 19f051c..196b29c 100644 --- a/lib/card-import/sync-catalog.js +++ b/lib/card-import/sync-catalog.js @@ -1,3 +1,8 @@ +import { sql } from '@vercel/postgres'; + +import { runBulkMtgSync } from './bulk-sync.js'; +import { runBulkLorcanaSync } from './lorcana-bulk.js'; +import { runBulkPokemonSync } from './pokemon-bulk.js'; import { discoverMissingMtgSets, discoverMissingPokemonSets } from './discover.js'; import { importMtgSet } from './mtg.js'; import { importPokemonSet } from './pokemon.js'; @@ -39,6 +44,95 @@ export function buildCatalogSyncQueue(missingMtg, missingPokemon, maxSetsPerRun return queue.slice(0, maxSetsPerRun); } +function countGameErrors(result) { + if (!result) return 0; + if (result.error) return 1; + if (typeof result.errors === 'number') return result.errors; + if (Array.isArray(result.errors)) return result.errors.length; + return 0; +} + +/** + * Persist one game's sync result to catalog_sync_log. + */ +export async function logCatalogSyncRun(game, result) { + if (!result) return; + + const upserted = result.upserted ?? result.imported ?? 0; + const errorCount = countGameErrors(result); + const mode = result.mode ?? 'bulk'; + const sourceUpdatedAt = result.updatedAt ?? null; + + try { + await sql` + INSERT INTO catalog_sync_log (game, mode, upserted, errors, source_updated_at, ran_at, details) + VALUES ( + ${game}, + ${mode}, + ${upserted}, + ${errorCount}, + ${sourceUpdatedAt}, + CURRENT_TIMESTAMP, + ${JSON.stringify(result)} + ) + `; + } catch (error) { + console.error(`[logCatalogSyncRun] Failed to log ${game} sync:`, error.message); + } +} + +async function runGameSync(game, syncFn, options) { + try { + return await syncFn(options); + } catch (error) { + console.error(`[runUnifiedCatalogSync] ${game} failed:`, error); + return { mode: 'bulk', error: error.message }; + } +} + +/** + * Run MTG, Pokémon, and Lorcana bulk syncs in sequence. + * Per-game failures are isolated; results are logged to catalog_sync_log. + */ +export async function runUnifiedCatalogSync(options = {}) { + const startedAt = Date.now(); + + const summary = { + mode: 'unified', + mtg: null, + pokemon: null, + lorcana: null, + errors: [], + durationMs: 0, + }; + + summary.mtg = await runGameSync('mtg', runBulkMtgSync, options); + if (summary.mtg.error) { + summary.errors.push({ game: 'mtg', message: summary.mtg.error }); + } + + summary.pokemon = await runGameSync('pokemon', runBulkPokemonSync, options); + if (summary.pokemon.error) { + summary.errors.push({ game: 'pokemon', message: summary.pokemon.error }); + } + + summary.lorcana = await runGameSync('lorcana', runBulkLorcanaSync, options); + if (summary.lorcana.error) { + summary.errors.push({ game: 'lorcana', message: summary.lorcana.error }); + } + + summary.durationMs = Date.now() - startedAt; + + await Promise.all([ + logCatalogSyncRun('mtg', summary.mtg), + logCatalogSyncRun('pokemon', summary.pokemon), + logCatalogSyncRun('lorcana', summary.lorcana), + ]); + + console.log('[runUnifiedCatalogSync]', JSON.stringify(summary)); + return summary; +} + /** * Discover missing MTG + Pokémon sets and import up to maxSetsPerRun, paced for upstream APIs. */ diff --git a/migrations/1781442175729_normalize-lorcana-set-codes.js b/migrations/1781442175729_normalize-lorcana-set-codes.js new file mode 100644 index 0000000..875d867 --- /dev/null +++ b/migrations/1781442175729_normalize-lorcana-set-codes.js @@ -0,0 +1,119 @@ +/** + * Normalize legacy Lorcana set_code values and scryfall_id (external id) shapes + * so bulk import from lorcana-api.com upserts cleanly instead of inserting duplicates. + * + * Legacy imports used numeric set codes (1/2/3) or lowercase abbreviations (tfc/rotf/ink). + * The bulk API uses Set_ID values TFC, ROF, INK, etc., and Unique_ID like TFC-001. + */ +export const shorthands = undefined; + +export const up = (pgm) => { + pgm.sql(` + UPDATE cards + SET game = 'Lorcana' + WHERE game ILIKE 'lorcana' AND game <> 'Lorcana'; + + UPDATE cards + SET set_code = 'TFC', set_name = 'The First Chapter' + WHERE game = 'Lorcana' AND set_code IN ('1', 'tfc', 'TFC'); + + UPDATE cards + SET set_code = 'ROF', set_name = 'Rise of the Floodborn' + WHERE game = 'Lorcana' AND set_code IN ('2', 'rotf', 'ROF'); + + UPDATE cards + SET set_code = 'INK', set_name = 'Into the Inklands' + WHERE game = 'Lorcana' AND set_code IN ('3', 'ink', 'INK', 'ITI'); + + UPDATE cards + SET set_code = 'TFC', set_name = 'The First Chapter' + WHERE game = 'Lorcana' + AND set_code IS NULL + AND set_name ILIKE '%first chapter%'; + + UPDATE cards + SET set_code = 'ROF', set_name = 'Rise of the Floodborn' + WHERE game = 'Lorcana' + AND set_code IS NULL + AND set_name ILIKE '%rise of the floodborn%'; + + UPDATE cards + SET set_code = 'INK', set_name = 'Into the Inklands' + WHERE game = 'Lorcana' + AND set_code IS NULL + AND set_name ILIKE '%into the inklands%'; + + `); + + // Drop legacy duplicate rows when the canonical Unique_ID row already exists. + pgm.sql(` + DELETE FROM cards AS dup + WHERE dup.game = 'Lorcana' + AND dup.set_code IS NOT NULL + AND TRIM(dup.card_number) <> '' + AND dup.scryfall_id IS DISTINCT FROM ( + dup.set_code || '-' || LPAD( + REGEXP_REPLACE(TRIM(dup.card_number), '/.*$', ''), + 3, + '0' + ) + ) + AND EXISTS ( + SELECT 1 + FROM cards AS keeper + WHERE keeper.scryfall_id = dup.set_code || '-' || LPAD( + REGEXP_REPLACE(TRIM(dup.card_number), '/.*$', ''), + 3, + '0' + ) + ); + `); + + // Keep one row per set + collector number before scryfall_id unification. + pgm.sql(` + DELETE FROM cards AS dup + WHERE dup.game = 'Lorcana' + AND dup.set_code IS NOT NULL + AND TRIM(dup.card_number) <> '' + AND dup.id <> ( + SELECT MIN(peer.id) + FROM cards AS peer + WHERE peer.game = 'Lorcana' + AND peer.set_code = dup.set_code + AND REGEXP_REPLACE(TRIM(peer.card_number), '/.*$', '') = REGEXP_REPLACE(TRIM(dup.card_number), '/.*$', '') + ); + `); + + // Unify remaining legacy scryfall_id values to Unique_ID shape (SET-NNN). + pgm.sql(` + UPDATE cards AS c + SET scryfall_id = c.set_code || '-' || LPAD( + REGEXP_REPLACE(TRIM(c.card_number), '/.*$', ''), + 3, + '0' + ) + WHERE c.game = 'Lorcana' + AND c.set_code IS NOT NULL + AND TRIM(c.card_number) <> '' + AND c.scryfall_id IS DISTINCT FROM ( + c.set_code || '-' || LPAD( + REGEXP_REPLACE(TRIM(c.card_number), '/.*$', ''), + 3, + '0' + ) + ) + AND NOT EXISTS ( + SELECT 1 + FROM cards AS other + WHERE other.scryfall_id = c.set_code || '-' || LPAD( + REGEXP_REPLACE(TRIM(c.card_number), '/.*$', ''), + 3, + '0' + ) + ); + `); +}; + +export const down = () => { + throw new Error('Irreversible data migration'); +}; diff --git a/migrations/1781442329511_add-catalog-sync-log.js b/migrations/1781442329511_add-catalog-sync-log.js new file mode 100644 index 0000000..91a4b9e --- /dev/null +++ b/migrations/1781442329511_add-catalog-sync-log.js @@ -0,0 +1,24 @@ +/** + * Track per-game catalog sync runs (staleness, upsert counts, errors). + */ +export const shorthands = undefined; + +export const up = (pgm) => { + pgm.createTable('catalog_sync_log', { + id: 'id', + game: { type: 'varchar(20)', notNull: true }, + mode: { type: 'varchar(20)', notNull: true }, + upserted: { type: 'integer', notNull: true, default: 0 }, + errors: { type: 'integer', notNull: true, default: 0 }, + source_updated_at: { type: 'timestamptz' }, + ran_at: { type: 'timestamptz', notNull: true, default: pgm.func('CURRENT_TIMESTAMP') }, + details: { type: 'jsonb' }, + }); + + pgm.createIndex('catalog_sync_log', 'game'); + pgm.createIndex('catalog_sync_log', 'ran_at'); +}; + +export const down = (pgm) => { + pgm.dropTable('catalog_sync_log'); +}; diff --git a/migrations/1781000000001_reconcile-cards-columns.js b/migrations/1781442330001_reconcile-cards-columns.js similarity index 100% rename from migrations/1781000000001_reconcile-cards-columns.js rename to migrations/1781442330001_reconcile-cards-columns.js diff --git a/migrations/1781000000002_reconcile-collections-columns.js b/migrations/1781442330002_reconcile-collections-columns.js similarity index 100% rename from migrations/1781000000002_reconcile-collections-columns.js rename to migrations/1781442330002_reconcile-collections-columns.js diff --git a/migrations/1781000000004_reconcile-favorites-system.js b/migrations/1781442330004_reconcile-favorites-system.js similarity index 100% rename from migrations/1781000000004_reconcile-favorites-system.js rename to migrations/1781442330004_reconcile-favorites-system.js diff --git a/migrations/1781000000005_reconcile-user-profile.js b/migrations/1781442330005_reconcile-user-profile.js similarity index 100% rename from migrations/1781000000005_reconcile-user-profile.js rename to migrations/1781442330005_reconcile-user-profile.js diff --git a/migrations/1781000000006_reconcile-user-cards-unique.js b/migrations/1781442330006_reconcile-user-cards-unique.js similarity index 100% rename from migrations/1781000000006_reconcile-user-cards-unique.js rename to migrations/1781442330006_reconcile-user-cards-unique.js diff --git a/migrations/1781442340000_cleanup-lorcana-orphans.js b/migrations/1781442340000_cleanup-lorcana-orphans.js new file mode 100644 index 0000000..dd56617 --- /dev/null +++ b/migrations/1781442340000_cleanup-lorcana-orphans.js @@ -0,0 +1,49 @@ +/** + * Remove legacy Lorcana catalog rows that predate bulk import normalization. + * Remaps collection_cards to canonical TFC printings where possible. + */ +export const shorthands = undefined; + +export const up = (pgm) => { + // Remap collection_cards from orphan Elsa/Mickey rows to canonical TFC printings. + pgm.sql(` + UPDATE collection_cards cc + SET card_id = canonical.id + FROM cards orphan + JOIN cards canonical ON canonical.game = 'Lorcana' + AND canonical.name = orphan.name + AND canonical.set_code = orphan.set_code + AND canonical.scryfall_id ~ '^[A-Za-z]{2,4}-[0-9]{3}$' + WHERE cc.card_id = orphan.id + AND orphan.game = 'Lorcana' + AND (orphan.scryfall_id IS NULL OR orphan.scryfall_id !~ '^[A-Za-z]{2,4}-[0-9]{3}$') + AND NOT EXISTS ( + SELECT 1 FROM collection_cards existing + WHERE existing.collection_id = cc.collection_id + AND existing.card_id = canonical.id + ); + `); + + // Drop collection refs that would duplicate after remap. + pgm.sql(` + DELETE FROM collection_cards cc + USING cards orphan + WHERE cc.card_id = orphan.id + AND orphan.game = 'Lorcana' + AND (orphan.scryfall_id IS NULL OR orphan.scryfall_id !~ '^[A-Za-z]{2,4}-[0-9]{3}$'); + `); + + // Delete unreferenced legacy rows (variant collector numbers, null external ids). + pgm.sql(` + DELETE FROM cards c + WHERE c.game = 'Lorcana' + AND (c.scryfall_id IS NULL OR c.scryfall_id !~ '^[A-Za-z]{2,4}-[0-9]{3}$') + AND NOT EXISTS (SELECT 1 FROM user_cards uc WHERE uc.card_id = c.id) + AND NOT EXISTS (SELECT 1 FROM deck_cards dc WHERE dc.card_id = c.id) + AND NOT EXISTS (SELECT 1 FROM collection_cards cc WHERE cc.card_id = c.id); + `); +}; + +export const down = () => { + throw new Error('Irreversible data cleanup'); +}; diff --git a/package.json b/package.json index 5ba8ecc..5d0e023 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "import-popular": "node scripts/import-popular-sets.js", "import-all": "node scripts/bulk-import-all.js", "bulk-import": "node --env-file=.env.local scripts/bulk-import-scryfall.js", + "bulk-import-lorcana": "node --env-file=.env.local scripts/bulk-import-lorcana.js", + "bulk-import-pokemon": "node --env-file=.env.local scripts/bulk-import-pokemon.js", "import-tags": "node --env-file=.env.local scripts/import-scryfall-tags.js", "test": "vitest", "test:run": "vitest run", diff --git a/pages/admin/card-import.js b/pages/admin/card-import.js index d95a543..04f83f2 100644 --- a/pages/admin/card-import.js +++ b/pages/admin/card-import.js @@ -14,7 +14,7 @@ const CardImport = () => { const [isSyncing, setIsSyncing] = useState(false); const [syncResult, setSyncResult] = useState(null); - const [syncMode, setSyncMode] = useState('incremental'); + const [syncMode, setSyncMode] = useState('unified'); const handleCatalogSync = async () => { setIsSyncing(true); @@ -174,11 +174,23 @@ const CardImport = () => { Catalog sync

- {syncMode === 'bulk' + {syncMode === 'unified' + ? 'Run bulk sync for MTG (Scryfall), Pokémon (GitHub), and Lorcana (lorcana-api.com) in one job — the same weekly Vercel cron path. Per-game failures are isolated.' + : syncMode === 'bulk' ? 'Download Scryfall\'s full Oracle Cards bulk file and upsert all 36k+ MTG cards with rich metadata (legalities, keywords, color identity, tags). Takes ~30s.' - : 'Import up to three newest missing MTG and Pokémon sets — the same job the weekly Vercel cron runs. Pending scan submissions are auto-linked when a unique match exists.'} + : 'Import up to three newest missing MTG and Pokémon sets. Pending scan submissions are auto-linked when a unique match exists.'}

-
+
+
@@ -209,7 +221,7 @@ const CardImport = () => { loading={isSyncing} className="shrink-0" > - {isSyncing ? 'Syncing catalog…' : syncMode === 'bulk' ? 'Run bulk sync' : 'Run catalog sync'} + {isSyncing ? 'Syncing catalog…' : syncMode === 'unified' ? 'Run unified sync' : syncMode === 'bulk' ? 'Run bulk MTG sync' : 'Run incremental sync'} @@ -230,7 +242,39 @@ const CardImport = () => { {syncResult.success ? (
- {syncResult.mode === 'bulk' ? ( + {syncResult.mode === 'unified' ? ( + <> +

+ Unified sync finished in {(syncResult.durationMs / 1000).toFixed(1)}s. +

+ + + ) : syncResult.mode === 'bulk' ? (

Upserted {syncResult.upserted} MTG cards from Scryfall bulk data. {syncResult.backfilledOracleId > 0 && ` Backfilled oracle_id on ${syncResult.backfilledOracleId} existing rows.`} @@ -278,8 +322,10 @@ const CardImport = () => { {Array.isArray(syncResult.errors) && syncResult.errors.length > 0 && (

diff --git a/pages/api/admin/sync-catalog.js b/pages/api/admin/sync-catalog.js index e982aa5..4a455c7 100644 --- a/pages/api/admin/sync-catalog.js +++ b/pages/api/admin/sync-catalog.js @@ -1,6 +1,9 @@ import { withAdmin } from '../../../lib/permission-middleware'; import { checkImportRateLimit } from '../../../lib/rate-limit.js'; -import { runCatalogSync } from '../../../lib/card-import/sync-catalog.js'; +import { + runCatalogSync, + runUnifiedCatalogSync, +} from '../../../lib/card-import/sync-catalog.js'; import { runBulkMtgSync } from '../../../lib/card-import/bulk-sync.js'; export default withAdmin(async function handler(req, res, user) { @@ -14,7 +17,7 @@ export default withAdmin(async function handler(req, res, user) { return res.status(429).json({ error: 'Too many attempts. Try again later.' }); } - const mode = req.body?.mode || 'incremental'; + const mode = req.body?.mode || 'unified'; try { if (mode === 'bulk') { @@ -22,6 +25,11 @@ export default withAdmin(async function handler(req, res, user) { return res.status(200).json({ success: true, ...summary }); } + if (mode === 'unified') { + const summary = await runUnifiedCatalogSync(); + return res.status(200).json({ success: true, ...summary }); + } + const summary = await runCatalogSync(); return res.status(200).json({ success: true, ...summary }); } catch (error) { diff --git a/pages/api/cron/sync-catalog.js b/pages/api/cron/sync-catalog.js index 379e0a2..6766324 100644 --- a/pages/api/cron/sync-catalog.js +++ b/pages/api/cron/sync-catalog.js @@ -1,4 +1,4 @@ -import { runCatalogSync } from '../../../lib/card-import/sync-catalog.js'; +import { runUnifiedCatalogSync } from '../../../lib/card-import/sync-catalog.js'; function authorizeCron(req) { const secret = process.env.CRON_SECRET; @@ -32,7 +32,7 @@ export default async function handler(req, res) { } try { - const summary = await runCatalogSync(); + const summary = await runUnifiedCatalogSync(); return res.status(200).json({ success: true, ...summary }); } catch (error) { console.error('[GET /api/cron/sync-catalog]', error); diff --git a/scripts/bulk-import-lorcana.js b/scripts/bulk-import-lorcana.js new file mode 100644 index 0000000..e8020c9 --- /dev/null +++ b/scripts/bulk-import-lorcana.js @@ -0,0 +1,37 @@ +/** + * Bulk import Disney Lorcana cards from lorcana-api.com /bulk/cards. + * + * Usage: + * node --env-file=.env.local scripts/bulk-import-lorcana.js + * + * Options (env vars): + * DRY_RUN — "true" to fetch and count without writing to DB + */ + +import { neon } from '@neondatabase/serverless'; + +import { runBulkLorcanaSync } from '../lib/card-import/lorcana-bulk.js'; + +if (!process.env.POSTGRES_URL) { + console.error('POSTGRES_URL is required'); + process.exit(1); +} + +const DRY_RUN = process.env.DRY_RUN === 'true'; +const sql = neon(process.env.POSTGRES_URL, { fullResults: false }); + +async function run() { + console.log(`[bulk-import-lorcana] Starting (dryRun=${DRY_RUN})`); + + const summary = await runBulkLorcanaSync({ sql, dryRun: DRY_RUN }); + + console.log('[bulk-import-lorcana] Complete!'); + console.log(`[bulk-import-lorcana] Total in file: ${summary.totalInFile}`); + console.log(`[bulk-import-lorcana] Upserted: ${summary.upserted}`); + console.log(`[bulk-import-lorcana] Errors: ${summary.errors}`); +} + +run().catch((err) => { + console.error('[bulk-import-lorcana] Fatal error:', err); + process.exit(1); +}); diff --git a/scripts/bulk-import-pokemon.js b/scripts/bulk-import-pokemon.js new file mode 100644 index 0000000..390391c --- /dev/null +++ b/scripts/bulk-import-pokemon.js @@ -0,0 +1,51 @@ +/** + * Bulk import all Pokémon cards from pokemon-tcg-data GitHub JSON. + * + * Fetches sets/en.json, then upserts every card in each set (keyed on scryfall_id). + * Populates legalities from set metadata where available. + * + * Usage: + * POSTGRES_URL= node scripts/bulk-import-pokemon.js + * node --env-file=.env.local scripts/bulk-import-pokemon.js + * + * Options (env vars): + * BATCH_SIZE — rows per INSERT batch (default 100) + * DRY_RUN — "true" to fetch and count without writing to DB + */ + +import { neon } from '@neondatabase/serverless'; + +import { runBulkPokemonSync } from '../lib/card-import/pokemon-bulk.js'; + +if (!process.env.POSTGRES_URL) { + console.error('POSTGRES_URL is required'); + process.exit(1); +} + +const sql = neon(process.env.POSTGRES_URL, { fullResults: false }); + +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '100', 10); +const DRY_RUN = process.env.DRY_RUN === 'true'; + +async function run() { + console.log( + `[bulk-import-pokemon] Starting (batch=${BATCH_SIZE}, dryRun=${DRY_RUN})` + ); + + const summary = await runBulkPokemonSync({ + batchSize: BATCH_SIZE, + dryRun: DRY_RUN, + query: (query, values) => sql.query(query, values), + }); + + console.log('[bulk-import-pokemon] Complete!'); + console.log(`[bulk-import-pokemon] Sets processed: ${summary.setsProcessed}/${summary.totalSets}`); + console.log(`[bulk-import-pokemon] Upserted: ${summary.upserted}`); + console.log(`[bulk-import-pokemon] Skipped (dry run): ${summary.skipped}`); + console.log(`[bulk-import-pokemon] Errors: ${summary.errors}`); +} + +run().catch((err) => { + console.error('[bulk-import-pokemon] Fatal error:', err); + process.exit(1); +}); diff --git a/test/lib/card-import-lorcana-bulk.test.js b/test/lib/card-import-lorcana-bulk.test.js new file mode 100644 index 0000000..1f979d0 --- /dev/null +++ b/test/lib/card-import-lorcana-bulk.test.js @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest'; + +import { mapLorcanaCard } from '../../lib/card-import/lorcana-bulk.js'; + +describe('mapLorcanaCard', () => { + it('maps Lorcana bulk API fields to cards table shape', () => { + const mapped = mapLorcanaCard({ + Unique_ID: 'TFC-001', + Name: 'Ariel - On Human Legs', + Set_Name: 'The First Chapter', + Set_ID: 'TFC', + Card_Num: 1, + Rarity: 'Uncommon', + Cost: 4, + Type: 'Character', + Color: 'Amber, Sapphire', + Classifications: 'Storyborn, Hero', + Body_Text: 'Voiceless: This character cannot {e} to sing songs.', + Strength: 3, + Willpower: 4, + Image: 'https://lorcana-api.com/images/ariel/on_human_legs/ariel-on_human_legs-large.png', + Artist: 'Koni', + Flavor_Text: '...', + Lore: 2, + Inkable: true, + Abilities: 'Shift 3', + }); + + expect(mapped).toEqual({ + scryfallId: 'TFC-001', + name: 'Ariel - On Human Legs', + setName: 'The First Chapter', + setCode: 'TFC', + cardNumber: '1', + rarity: 'Uncommon', + manaCost: '4', + cmc: 4, + cardType: 'Character', + colors: JSON.stringify(['Amber', 'Sapphire']), + colorIdentity: JSON.stringify(['Amber', 'Sapphire']), + oracleText: 'Voiceless: This character cannot {e} to sing songs.', + power: '3', + toughness: '4', + imageUrl: 'https://lorcana-api.com/images/ariel/on_human_legs/ariel-on_human_legs-large.png', + stockImageUrl: 'https://lorcana-api.com/images/ariel/on_human_legs/ariel-on_human_legs-large.png', + flavorText: '...', + artist: 'Koni', + edhrecRank: 2, + keywords: JSON.stringify(['Storyborn', 'Hero', 'Shift 3', 'Inkable']), + }); + }); + + it('omits Inkable keyword when card is not inkable', () => { + const mapped = mapLorcanaCard({ + Unique_ID: 'ARI-001', + Name: 'Rhino - Motivational Speaker', + Set_Name: 'Archazia\'s Island', + Set_ID: 'ARI', + Card_Num: 1, + Inkable: false, + Classifications: 'Storyborn, Ally', + }); + + expect(JSON.parse(mapped.keywords)).toEqual(['Storyborn', 'Ally']); + }); +}); diff --git a/test/lib/card-import-sync-catalog.test.js b/test/lib/card-import-sync-catalog.test.js index b37c269..8420839 100644 --- a/test/lib/card-import-sync-catalog.test.js +++ b/test/lib/card-import-sync-catalog.test.js @@ -1,6 +1,25 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi, beforeEach } from 'vitest'; -import { buildCatalogSyncQueue } from '../../lib/card-import/sync-catalog.js'; +import { buildCatalogSyncQueue, runUnifiedCatalogSync } from '../../lib/card-import/sync-catalog.js'; +import { runBulkMtgSync } from '../../lib/card-import/bulk-sync.js'; +import { runBulkPokemonSync } from '../../lib/card-import/pokemon-bulk.js'; +import { runBulkLorcanaSync } from '../../lib/card-import/lorcana-bulk.js'; + +vi.mock('../../lib/card-import/bulk-sync.js', () => ({ + runBulkMtgSync: vi.fn(), +})); + +vi.mock('../../lib/card-import/pokemon-bulk.js', () => ({ + runBulkPokemonSync: vi.fn(), +})); + +vi.mock('../../lib/card-import/lorcana-bulk.js', () => ({ + runBulkLorcanaSync: vi.fn(), +})); + +vi.mock('@vercel/postgres', () => ({ + sql: vi.fn(() => Promise.resolve({ rows: [] })), +})); describe('buildCatalogSyncQueue', () => { it('merges MTG and Pokémon missing sets by release date, newest first', () => { @@ -38,3 +57,27 @@ describe('buildCatalogSyncQueue', () => { ]); }); }); + +describe('runUnifiedCatalogSync', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('runs all three games and isolates per-game failures', async () => { + runBulkMtgSync.mockResolvedValue({ mode: 'bulk', upserted: 36000, errors: 0 }); + runBulkPokemonSync.mockRejectedValue(new Error('Pokemon API down')); + runBulkLorcanaSync.mockResolvedValue({ mode: 'bulk', upserted: 2283, errors: 0 }); + + const summary = await runUnifiedCatalogSync(); + + expect(summary.mode).toBe('unified'); + expect(summary.mtg.upserted).toBe(36000); + expect(summary.pokemon.error).toBe('Pokemon API down'); + expect(summary.lorcana.upserted).toBe(2283); + expect(summary.errors).toEqual([{ game: 'pokemon', message: 'Pokemon API down' }]); + expect(summary.durationMs).toBeGreaterThanOrEqual(0); + expect(runBulkMtgSync).toHaveBeenCalledOnce(); + expect(runBulkPokemonSync).toHaveBeenCalledOnce(); + expect(runBulkLorcanaSync).toHaveBeenCalledOnce(); + }); +});