Replace @vercel/postgres, Blob, and Upstash with lib/sql.js, MinIO object storage, and CT 102 Redis rate limits. Add Dockerfile for Dokploy deploy, homelab runbooks, Neon data-copy helper, and point CI smoke/visual at the homelab URL instead of Vercel previews. Co-authored-by: Cursor <cursoragent@cursor.com>
204 lines
5.8 KiB
JavaScript
204 lines
5.8 KiB
JavaScript
import { sql } from '../sql.js';
|
|
|
|
import { runBulkMtgSync } from './bulk-sync.js';
|
|
import { runBulkLorcanaSync } from './lorcana-bulk.js';
|
|
import { runBulkPokemonSync } from './pokemon-bulk.js';
|
|
import { discoverMissingMtgSets, discoverMissingPokemonSets } from './discover.js';
|
|
import { importMtgSet } from './mtg.js';
|
|
import { importPokemonSet } from './pokemon.js';
|
|
import { reconcileSubmissionsForImportedSet } from './reconcile-submissions.js';
|
|
|
|
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
const DEFAULT_MAX_SETS_PER_RUN = 3;
|
|
const DEFAULT_DELAY_MS = 1500;
|
|
|
|
/**
|
|
* Merge missing MTG + Pokémon sets and pick the newest releases first.
|
|
*/
|
|
export function buildCatalogSyncQueue(missingMtg, missingPokemon, maxSetsPerRun = DEFAULT_MAX_SETS_PER_RUN) {
|
|
const queue = [
|
|
...missingMtg.map((set) => ({
|
|
game: 'mtg',
|
|
setCode: set.code,
|
|
name: set.name,
|
|
releasedAt: set.releasedAt,
|
|
})),
|
|
...missingPokemon.map((set) => ({
|
|
game: 'pokemon',
|
|
setCode: set.id,
|
|
name: set.name,
|
|
releasedAt: set.releasedAt,
|
|
})),
|
|
];
|
|
|
|
queue.sort((a, b) => {
|
|
const dateA = a.releasedAt || '';
|
|
const dateB = b.releasedAt || '';
|
|
if (!dateA && !dateB) return 0;
|
|
if (!dateA) return 1;
|
|
if (!dateB) return -1;
|
|
return dateB.localeCompare(dateA);
|
|
});
|
|
|
|
return queue.slice(0, maxSetsPerRun);
|
|
}
|
|
|
|
function countGameErrors(result) {
|
|
if (!result) return 0;
|
|
if (result.error) return 1;
|
|
if (typeof result.errors === 'number') return result.errors;
|
|
if (Array.isArray(result.errors)) return result.errors.length;
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* Persist one game's sync result to catalog_sync_log.
|
|
*/
|
|
export async function logCatalogSyncRun(game, result) {
|
|
if (!result) return;
|
|
|
|
const upserted = result.upserted ?? result.imported ?? 0;
|
|
const errorCount = countGameErrors(result);
|
|
const mode = result.mode ?? 'bulk';
|
|
const sourceUpdatedAt = result.updatedAt ?? null;
|
|
|
|
try {
|
|
await sql`
|
|
INSERT INTO catalog_sync_log (game, mode, upserted, errors, source_updated_at, ran_at, details)
|
|
VALUES (
|
|
${game},
|
|
${mode},
|
|
${upserted},
|
|
${errorCount},
|
|
${sourceUpdatedAt},
|
|
CURRENT_TIMESTAMP,
|
|
${JSON.stringify(result)}
|
|
)
|
|
`;
|
|
} catch (error) {
|
|
console.error(`[logCatalogSyncRun] Failed to log ${game} sync:`, error.message);
|
|
}
|
|
}
|
|
|
|
async function runGameSync(game, syncFn, options) {
|
|
try {
|
|
return await syncFn(options);
|
|
} catch (error) {
|
|
console.error(`[runUnifiedCatalogSync] ${game} failed:`, error);
|
|
return { mode: 'bulk', error: error.message };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Run MTG, Pokémon, and Lorcana bulk syncs in sequence.
|
|
* Per-game failures are isolated; results are logged to catalog_sync_log.
|
|
*/
|
|
export async function runUnifiedCatalogSync(options = {}) {
|
|
const startedAt = Date.now();
|
|
|
|
const summary = {
|
|
mode: 'unified',
|
|
mtg: null,
|
|
pokemon: null,
|
|
lorcana: null,
|
|
errors: [],
|
|
durationMs: 0,
|
|
};
|
|
|
|
summary.mtg = await runGameSync('mtg', runBulkMtgSync, options);
|
|
if (summary.mtg.error) {
|
|
summary.errors.push({ game: 'mtg', message: summary.mtg.error });
|
|
}
|
|
|
|
summary.pokemon = await runGameSync('pokemon', runBulkPokemonSync, options);
|
|
if (summary.pokemon.error) {
|
|
summary.errors.push({ game: 'pokemon', message: summary.pokemon.error });
|
|
}
|
|
|
|
summary.lorcana = await runGameSync('lorcana', runBulkLorcanaSync, options);
|
|
if (summary.lorcana.error) {
|
|
summary.errors.push({ game: 'lorcana', message: summary.lorcana.error });
|
|
}
|
|
|
|
summary.durationMs = Date.now() - startedAt;
|
|
|
|
await Promise.all([
|
|
logCatalogSyncRun('mtg', summary.mtg),
|
|
logCatalogSyncRun('pokemon', summary.pokemon),
|
|
logCatalogSyncRun('lorcana', summary.lorcana),
|
|
]);
|
|
|
|
console.log('[runUnifiedCatalogSync]', JSON.stringify(summary));
|
|
return summary;
|
|
}
|
|
|
|
/**
|
|
* Discover missing MTG + Pokémon sets and import up to maxSetsPerRun, paced for upstream APIs.
|
|
*/
|
|
export async function runCatalogSync(options = {}) {
|
|
const maxSetsPerRun = options.maxSetsPerRun ?? DEFAULT_MAX_SETS_PER_RUN;
|
|
const delayBetweenSetsMs = options.delayBetweenSetsMs ?? DEFAULT_DELAY_MS;
|
|
|
|
const [missingMtg, missingPokemon] = await Promise.all([
|
|
discoverMissingMtgSets(),
|
|
discoverMissingPokemonSets(),
|
|
]);
|
|
|
|
const queue = buildCatalogSyncQueue(missingMtg, missingPokemon, maxSetsPerRun);
|
|
|
|
const summary = {
|
|
imported: 0,
|
|
skipped: 0,
|
|
errors: [],
|
|
setsProcessed: [],
|
|
pendingMtgSets: missingMtg.length,
|
|
pendingPokemonSets: missingPokemon.length,
|
|
lorcana: 'skipped — manual Lorcana set map update required',
|
|
submissionsReconciled: 0,
|
|
reconciliation: [],
|
|
};
|
|
|
|
for (let index = 0; index < queue.length; index += 1) {
|
|
const item = queue[index];
|
|
try {
|
|
const result =
|
|
item.game === 'mtg'
|
|
? await importMtgSet(item.setCode)
|
|
: await importPokemonSet(item.setCode);
|
|
|
|
summary.setsProcessed.push({
|
|
game: item.game,
|
|
setCode: item.setCode,
|
|
name: item.name,
|
|
...result,
|
|
});
|
|
summary.imported += result.imported;
|
|
summary.skipped += result.skipped;
|
|
|
|
const reconcileResult = await reconcileSubmissionsForImportedSet({
|
|
game: item.game,
|
|
setCode: item.setCode,
|
|
setName: item.name,
|
|
});
|
|
summary.submissionsReconciled += reconcileResult.matched;
|
|
if (reconcileResult.matched > 0 || reconcileResult.skipped > 0) {
|
|
summary.reconciliation.push(reconcileResult);
|
|
}
|
|
} catch (error) {
|
|
console.error(`[runCatalogSync] Failed ${item.game}/${item.setCode}:`, error);
|
|
summary.errors.push({
|
|
game: item.game,
|
|
setCode: item.setCode,
|
|
message: error.message,
|
|
});
|
|
}
|
|
|
|
if (index < queue.length - 1) {
|
|
await delay(delayBetweenSetsMs);
|
|
}
|
|
}
|
|
|
|
console.log('[runCatalogSync]', JSON.stringify(summary));
|
|
return summary;
|
|
}
|