diff --git a/lib/card-import/bulk-sync.js b/lib/card-import/bulk-sync.js index bbb1d3d..45aac14 100644 --- a/lib/card-import/bulk-sync.js +++ b/lib/card-import/bulk-sync.js @@ -8,12 +8,18 @@ * Also refreshes prices, legalities, and edhrec_rank on existing cards. */ +import { gunzipSync } from 'node:zlib'; + import { sql } from '../sql.js'; import { recordPriceSnapshots } from '../pricing/price-history.js'; const BATCH_SIZE = 100; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +export function resolveBulkDownloadUri(meta) { + return meta?.download_uri ?? meta?.jsonl_download_uri ?? null; +} + async function fetchBulkDownloadUrl() { const response = await fetch('https://api.scryfall.com/bulk-data/oracle_cards', { headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' }, @@ -24,6 +30,38 @@ async function fetchBulkDownloadUrl() { return response.json(); } +/** + * Download and parse Scryfall oracle bulk (JSON array or JSONL, optionally gzip). + */ +export async function downloadBulkOracleCards(meta) { + const downloadUri = resolveBulkDownloadUri(meta); + if (!downloadUri) { + throw new Error('Scryfall bulk meta missing download_uri and jsonl_download_uri'); + } + + const response = await fetch(downloadUri, { + headers: { 'User-Agent': 'DeckHearth/1.0' }, + }); + if (!response.ok) { + throw new Error(`Bulk download failed: ${response.status}`); + } + + const raw = Buffer.from(await response.arrayBuffer()); + const text = downloadUri.endsWith('.gz') + ? gunzipSync(raw).toString('utf8') + : raw.toString('utf8'); + + if (downloadUri.includes('.jsonl')) { + return text + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line)); + } + + return JSON.parse(text); +} + export function mapCard(card) { let currentPrice = null; let marketPrice = null; @@ -167,16 +205,10 @@ async function upsertBatch(cards) { */ export async function runBulkMtgSync() { const meta = await fetchBulkDownloadUrl(); - console.log(`[bulk-sync] Oracle Cards updated: ${meta.updated_at}, size: ${(meta.size / 1024 / 1024).toFixed(0)} MB`); + const sizeMb = ((meta.compressed_size ?? meta.size ?? 0) / 1024 / 1024).toFixed(0); + console.log(`[bulk-sync] Oracle Cards updated: ${meta.updated_at}, size: ~${sizeMb} MB`); - const response = await fetch(meta.download_uri, { - headers: { 'User-Agent': 'DeckHearth/1.0' }, - }); - if (!response.ok) { - throw new Error(`Bulk download failed: ${response.status}`); - } - - const allCards = await response.json(); + const allCards = await downloadBulkOracleCards(meta); const mtgCards = allCards.filter((c) => c.lang === 'en' && !c.digital); let upserted = 0; diff --git a/test/lib/card-import/bulk-sync.test.js b/test/lib/card-import/bulk-sync.test.js index a44466b..7e891ba 100644 --- a/test/lib/card-import/bulk-sync.test.js +++ b/test/lib/card-import/bulk-sync.test.js @@ -1,6 +1,14 @@ import { describe, expect, it } from 'vitest'; -import { mapCard } from '../../../lib/card-import/bulk-sync.js'; +import { mapCard, resolveBulkDownloadUri } from '../../../lib/card-import/bulk-sync.js'; + +describe('resolveBulkDownloadUri', () => { + it('prefers download_uri then jsonl_download_uri', () => { + expect(resolveBulkDownloadUri({ download_uri: 'https://a.test/json' })).toBe('https://a.test/json'); + expect(resolveBulkDownloadUri({ jsonl_download_uri: 'https://a.test/file.jsonl.gz' })) + .toBe('https://a.test/file.jsonl.gz'); + }); +}); describe('mapCard', () => { it('maps Scryfall usd to current_price and usd_foil to market_price', () => {