fix(mtg): support Scryfall oracle bulk JSONL.gz download

Co-authored-by: Cursor <cursoragent@cursor.com>

lib/card-import/bulk-sync.js

@@ -8,12 +8,18 @@
  * Also refreshes prices, legalities, and edhrec_rank on existing cards.
  */
 
+import { gunzipSync } from 'node:zlib';
+
 import { sql } from '../sql.js';
 import { recordPriceSnapshots } from '../pricing/price-history.js';
 
 const BATCH_SIZE = 100;
 const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
+export function resolveBulkDownloadUri(meta) {
+  return meta?.download_uri ?? meta?.jsonl_download_uri ?? null;
+}
+
 async function fetchBulkDownloadUrl() {
   const response = await fetch('https://api.scryfall.com/bulk-data/oracle_cards', {
     headers: { 'User-Agent': 'DeckHearth/1.0', Accept: 'application/json' },
@@ -24,6 +30,38 @@ async function fetchBulkDownloadUrl() {
   return response.json();
 }
 
+/**
+ * Download and parse Scryfall oracle bulk (JSON array or JSONL, optionally gzip).
+ */
+export async function downloadBulkOracleCards(meta) {
+  const downloadUri = resolveBulkDownloadUri(meta);
+  if (!downloadUri) {
+    throw new Error('Scryfall bulk meta missing download_uri and jsonl_download_uri');
+  }
+
+  const response = await fetch(downloadUri, {
+    headers: { 'User-Agent': 'DeckHearth/1.0' },
+  });
+  if (!response.ok) {
+    throw new Error(`Bulk download failed: ${response.status}`);
+  }
+
+  const raw = Buffer.from(await response.arrayBuffer());
+  const text = downloadUri.endsWith('.gz')
+    ? gunzipSync(raw).toString('utf8')
+    : raw.toString('utf8');
+
+  if (downloadUri.includes('.jsonl')) {
+    return text
+      .split('\n')
+      .map((line) => line.trim())
+      .filter(Boolean)
+      .map((line) => JSON.parse(line));
+  }
+
+  return JSON.parse(text);
+}
+
 export function mapCard(card) {
   let currentPrice = null;
   let marketPrice = null;
@@ -167,16 +205,10 @@ async function upsertBatch(cards) {
  */
 export async function runBulkMtgSync() {
   const meta = await fetchBulkDownloadUrl();
-  console.log(`[bulk-sync] Oracle Cards updated: ${meta.updated_at}, size: ${(meta.size / 1024 / 1024).toFixed(0)} MB`);
+  const sizeMb = ((meta.compressed_size ?? meta.size ?? 0) / 1024 / 1024).toFixed(0);
+  console.log(`[bulk-sync] Oracle Cards updated: ${meta.updated_at}, size: ~${sizeMb} MB`);
 
-  const response = await fetch(meta.download_uri, {
-    headers: { 'User-Agent': 'DeckHearth/1.0' },
-  });
-  if (!response.ok) {
-    throw new Error(`Bulk download failed: ${response.status}`);
-  }
-
-  const allCards = await response.json();
+  const allCards = await downloadBulkOracleCards(meta);
   const mtgCards = allCards.filter((c) => c.lang === 'en' && !c.digital);
 
   let upserted = 0;

test/lib/card-import/bulk-sync.test.js

@@ -1,6 +1,14 @@
 import { describe, expect, it } from 'vitest';
 
-import { mapCard } from '../../../lib/card-import/bulk-sync.js';
+import { mapCard, resolveBulkDownloadUri } from '../../../lib/card-import/bulk-sync.js';
+
+describe('resolveBulkDownloadUri', () => {
+  it('prefers download_uri then jsonl_download_uri', () => {
+    expect(resolveBulkDownloadUri({ download_uri: 'https://a.test/json' })).toBe('https://a.test/json');
+    expect(resolveBulkDownloadUri({ jsonl_download_uri: 'https://a.test/file.jsonl.gz' }))
+      .toBe('https://a.test/file.jsonl.gz');
+  });
+});
 
 describe('mapCard', () => {
   it('maps Scryfall usd to current_price and usd_foil to market_price', () => {