From 8f09ed1ef6d88823faf0ea15b33b1e562fd6b7ba Mon Sep 17 00:00:00 2001 From: varutasu <104105839+varutasu@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:46:39 -0500 Subject: [PATCH] feat(scanner): Layer-0 visual catalog search (Phase 3) (#160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add pgvector embeddings on cards, server-side cohere/embed-v4.0 via AI Gateway, kNN identify route, and L0→L1→L2 client orchestration with empty-index fast escalate and id-cursor backfill job. Co-authored-by: Cursor --- .convoys/.metrics.jsonl | 7 + .convoys/scan-visual-catalog-search.md | 45 ++++- .../brief-1-migration-schema.md | 23 +++ .../brief-2-backfill-embeddings.md | 24 +++ .../brief-3-identify-knn-route.md | 27 +++ docs/SCHEMA_MAP.md | 10 +- lib/card-embed.js | 90 ++++++++++ lib/card-visual-match.js | 118 +++++++++++++ lib/scanner-card-identify.js | 48 ++++++ lib/use-scanner-identification.js | 35 +++- .../1782000000001_add-card-embeddings.js | 33 ++++ package.json | 1 + pages/api/scan/identify-by-image.js | 156 ++++++++++++++++++ scripts/backfill-card-embeddings.js | 117 +++++++++++++ test/lib/card-embed.test.js | 35 ++++ test/lib/card-visual-match.test.js | 40 +++++ test/lib/scanner-card-identify.test.js | 1 + 17 files changed, 800 insertions(+), 10 deletions(-) create mode 100644 .convoys/scan-visual-catalog-search/brief-1-migration-schema.md create mode 100644 .convoys/scan-visual-catalog-search/brief-2-backfill-embeddings.md create mode 100644 .convoys/scan-visual-catalog-search/brief-3-identify-knn-route.md create mode 100644 lib/card-embed.js create mode 100644 lib/card-visual-match.js create mode 100644 migrations/1782000000001_add-card-embeddings.js create mode 100644 pages/api/scan/identify-by-image.js create mode 100644 scripts/backfill-card-embeddings.js create mode 100644 test/lib/card-embed.test.js create mode 100644 test/lib/card-visual-match.test.js diff --git a/.convoys/.metrics.jsonl b/.convoys/.metrics.jsonl index 3b178d5..723a8b2 100644 --- a/.convoys/.metrics.jsonl +++ b/.convoys/.metrics.jsonl @@ -101,3 +101,10 @@ {"ts": "2026-08-15T01:24:40Z", "role": "role-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} {"ts": "2026-08-15T01:24:41Z", "role": "role-security-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"} {"ts": "2026-08-15T01:24:41Z", "role": "role-a11y-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 60, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T01:35:27Z", "role": "role-ux-reviewer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:35:27Z", "role": "role-architect", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 360, "model": "composer-2.5", "model_tier": "standard"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 1, "duration_s": 300, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 2, "duration_s": 600, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 3, "duration_s": 900, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-reviewer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-scan-visual-catalog-search-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T01:35:28Z", "role": "role-security-auditor", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-scan-visual-catalog-search-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"} diff --git a/.convoys/scan-visual-catalog-search.md b/.convoys/scan-visual-catalog-search.md index f94b845..19c97d7 100644 --- a/.convoys/scan-visual-catalog-search.md +++ b/.convoys/scan-visual-catalog-search.md @@ -14,7 +14,7 @@ skip: - a11y - design - flag -status: open +status: in-progress created: 2026-08-14 depends_on: - improve-scan-card-detection @@ -107,12 +107,13 @@ unknown-card fallback. ## Todos -- [ ] Architect: confirm `pgvector` on prod Neon tier -- [ ] Brief 1 — migration + SCHEMA_MAP -- [ ] Brief 2 — catalog backfill job (idempotent) -- [ ] Brief 3 — identify kNN route + client escalate order +- [x] Architect: confirm `pgvector` on prod Neon tier +- [x] Brief 1 — migration + SCHEMA_MAP +- [x] Brief 2 — catalog backfill job (idempotent) +- [x] Brief 3 — identify kNN route + client escalate order - [ ] Threshold bake-off on real crops - [ ] Re-measure auto-match % excluding `not_a_card` +- [ ] Operator: `npm run backfill-embeddings` on prod/staging after migrate ## Likely file ownership @@ -159,3 +160,37 @@ crops are rectified — that wastes the backfill. If Architect finds `pgvector` unavailable on CI Postgres, stop and write a fallback (external index vs skip-CI-extension plan) rather than shipping an untestable migration. + +## UX + +No new screens. Scan flow stays L0 → L1 → L2 with the same +disambiguation picker and error toasts. When the catalog index is empty +(backfill not run), L0 escalates silently with no embed cost. + +Gallery uploads now try visual match before OCR. + +## Architecture + +### Decision D1 — Gateway multimodal embedder (`cohere/embed-v4.0`) + +Server-only via `AI_GATEWAY_API_KEY`. 1024-dim vectors in +`cards.embedding`. Env: `SCAN_EMBED_MODEL`, `SCAN_EMBED_DIMENSION`. + +### Decision D2 — Layer numbering + +| Layer | Path | +| --- | --- | +| 0 | `POST /api/scan/identify-by-image` | +| 1 | Tesseract + `identify-by-text` | +| 2 | Gemini + `scan/identify` | + +### Decision D3 — Thresholds + +Match ≥ **0.82** (0.06 gap). Disambiguation ≥ **0.58**. + +### Decision D4 — Rate limit + +`checkScanRateLimit` on identify-by-image. L0 429 falls through to L1 +(not a hard stop). + +Audit group id: `audit-scan-visual-catalog-search-`. diff --git a/.convoys/scan-visual-catalog-search/brief-1-migration-schema.md b/.convoys/scan-visual-catalog-search/brief-1-migration-schema.md new file mode 100644 index 0000000..ce7cae0 --- /dev/null +++ b/.convoys/scan-visual-catalog-search/brief-1-migration-schema.md @@ -0,0 +1,23 @@ +--- +convoy: scan-visual-catalog-search +brief_number: 1 +depends_on: [] +recommended_model: composer-2.5 +model_tier: standard +files: + - migrations/1782000000001_add-card-embeddings.js + - docs/SCHEMA_MAP.md +--- + +# Brief 1: pgvector migration + SCHEMA_MAP + +## Goal + +Add `vector(1024)` embedding column + HNSW index on `cards`. + +## Acceptance criteria + +- [ ] `CREATE EXTENSION IF NOT EXISTS vector` +- [ ] `cards.embedding`, `cards.embedded_at` +- [ ] `idx_cards_embedding_hnsw` partial index +- [ ] SCHEMA_MAP documents layer 0 + pgvector diff --git a/.convoys/scan-visual-catalog-search/brief-2-backfill-embeddings.md b/.convoys/scan-visual-catalog-search/brief-2-backfill-embeddings.md new file mode 100644 index 0000000..295c1d5 --- /dev/null +++ b/.convoys/scan-visual-catalog-search/brief-2-backfill-embeddings.md @@ -0,0 +1,24 @@ +--- +convoy: scan-visual-catalog-search +brief_number: 2 +depends_on: [1] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/card-embed.js + - scripts/backfill-card-embeddings.js + - package.json + - test/lib/card-embed.test.js +--- + +# Brief 2: Catalog embedding backfill + +## Goal + +Server-side embed job using `cohere/embed-v4.0` via AI Gateway; idempotent backfill from `cards.image_url`. + +## Acceptance criteria + +- [ ] `lib/card-embed.js` exports `embedCardImage`, `formatEmbeddingForPg` +- [ ] `npm run backfill-embeddings` script (skips rows with embedding unless FORCE=1) +- [ ] Rate-limited (`SLEEP_MS`) and DRY_RUN support diff --git a/.convoys/scan-visual-catalog-search/brief-3-identify-knn-route.md b/.convoys/scan-visual-catalog-search/brief-3-identify-knn-route.md new file mode 100644 index 0000000..dd270c1 --- /dev/null +++ b/.convoys/scan-visual-catalog-search/brief-3-identify-knn-route.md @@ -0,0 +1,27 @@ +--- +convoy: scan-visual-catalog-search +brief_number: 3 +depends_on: [1] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/card-visual-match.js + - pages/api/scan/identify-by-image.js + - lib/scanner-card-identify.js + - lib/use-scanner-identification.js + - test/lib/card-visual-match.test.js +--- + +# Brief 3: Layer-0 identify route + client orchestration + +## Goal + +kNN visual match before L1/L2; log `scan_attempts.layer = 0`. + +## Acceptance criteria + +- [ ] `POST /api/scan/identify-by-image` — auth + `checkScanRateLimit` + embed + kNN +- [ ] `identifyTrackedCardCapture` order: L0 → L1 → L2 +- [ ] Gallery path runs L0 → L1 → L2 +- [ ] Skip auto Gemini refine when disambiguation from L0 (same as L1) +- [ ] Thresholds: match 0.82, disambiguation 0.58 diff --git a/docs/SCHEMA_MAP.md b/docs/SCHEMA_MAP.md index 2e430a8..0f852f1 100644 --- a/docs/SCHEMA_MAP.md +++ b/docs/SCHEMA_MAP.md @@ -93,16 +93,20 @@ | `reprint` | `BOOLEAN` default `false` | (`1781440700404`) | | `finishes` | `JSONB` | `["nonfoil","foil","etched"]` (`1781440700404`) | | `created_at`, `updated_at` | `TIMESTAMP` default now | | +| `embedding` | `vector(1024)` | Layer-0 visual kNN index (`1782000000001_add-card-embeddings`) | +| `embedded_at` | `TIMESTAMP` | When `embedding` was last written by backfill | **Indexes (post `1779853647565_add-pg-trgm-card-name-index` + `1781440700404`):** - `idx_cards_name_trgm` — GIN on `name` using `gin_trgm_ops` for Layer-1 OCR fuzzy match (`similarity()` / `pg_trgm`). +- `idx_cards_embedding_hnsw` — HNSW on `embedding vector_cosine_ops` (partial, non-null only) for Layer-0 visual search. - `cards_oracle_id_index`, `cards_illustration_id_index`, `cards_artist_index`, `cards_edhrec_rank_index` - GIN on `color_identity`, `keywords`, `legalities` **Extensions used by scan pipeline:** -- `pg_trgm` — enabled by `1779853647565_add-pg-trgm-card-name-index.js` for trigram similarity on `cards.name`. +- `pg_trgm` — Layer-1 text similarity (`1779853647565_add-pg-trgm-card-name-index`). +- `vector` (pgvector) — Layer-0 visual embeddings (`1782000000001_add-card-embeddings`). ### user_cards @@ -261,7 +265,7 @@ Index: `idx_card_submissions_status (status, created_at DESC)`. ### scan_attempts -Per-scan telemetry for the identify pipeline (layer 1 = Tesseract + pg_trgm, layer 2 = Gemini). +Per-scan telemetry for the identify pipeline (layer 0 = visual kNN, layer 1 = Tesseract + pg_trgm, layer 2 = Gemini). | Column | Type | Notes | | --- | --- | --- | @@ -269,7 +273,7 @@ Per-scan telemetry for the identify pipeline (layer 1 = Tesseract + pg_trgm, lay | `user_id` | FK → `users` SET NULL | | | `ocr_text` | `TEXT` | | | `ocr_confidence` | `INTEGER` | | -| `layer` | `INTEGER` default `2` | Identify layer (`1` = browser Tesseract + pg_trgm, `2` = Gemini vision) | +| `layer` | `INTEGER` default `2` | Identify layer (`0` = visual kNN, `1` = browser Tesseract + pg_trgm, `2` = Gemini vision) | | `matched_card_id` | FK → `cards` SET NULL | | | `result_kind` | `VARCHAR(32)` | e.g. `'matched'`, `'disambiguation'`, `'submitted'`, `'not_a_card'` | | `latency_ms` | `INTEGER` | End-to-end identify latency | diff --git a/lib/card-embed.js b/lib/card-embed.js new file mode 100644 index 0000000..80cacc2 --- /dev/null +++ b/lib/card-embed.js @@ -0,0 +1,90 @@ +const GATEWAY_EMBED_URL = 'https://ai-gateway.vercel.sh/v1/embeddings'; +const DEFAULT_EMBED_MODEL = process.env.SCAN_EMBED_MODEL || 'cohere/embed-v4.0'; + +/** Output dimension — must match migrations/1782000000001_add-card-embeddings.js. */ +export const EMBED_DIMENSION = Number(process.env.SCAN_EMBED_DIMENSION || 1024); + +export class EmbedApiError extends Error { + constructor(message, { status } = {}) { + super(message); + this.name = 'EmbedApiError'; + this.status = status; + } +} + +function parseEmbeddingResponse(data) { + const embedding = + data?.data?.[0]?.embedding || + data?.embeddings?.[0]?.values || + data?.embeddings?.[0] || + data?.embedding; + + if (!Array.isArray(embedding) || embedding.length === 0) { + throw new EmbedApiError('Embedding API returned no vector'); + } + + return embedding.map((value) => Number(value)); +} + +function buildEmbedInput(value) { + if (typeof value === 'string') { + return value; + } + + if (value?.imageUrl) { + return { + type: 'image_url', + image_url: { url: value.imageUrl }, + }; + } + + if (value?.imageDataUrl) { + return value.imageDataUrl; + } + + throw new EmbedApiError('Invalid embed input'); +} + +/** + * Embed image content via Vercel AI Gateway (server-only). + * Accepts a JPEG data URL or HTTPS image URL object. + */ +export async function embedCardImage(input) { + const apiKey = process.env.AI_GATEWAY_API_KEY; + if (!apiKey) { + throw new EmbedApiError('AI_GATEWAY_API_KEY is not configured on the server'); + } + + const payloadInput = buildEmbedInput(input); + if (typeof payloadInput === 'string' && !payloadInput.includes(',')) { + throw new EmbedApiError('Invalid image data format'); + } + + const response = await fetch(GATEWAY_EMBED_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + model: DEFAULT_EMBED_MODEL, + input: payloadInput, + dimensions: EMBED_DIMENSION, + }), + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + const message = errorData.error?.message || errorData.message || 'Unknown error'; + throw new EmbedApiError(`Embedding API error: ${response.status} - ${message}`, { + status: response.status, + }); + } + + return parseEmbeddingResponse(await response.json()); +} + +/** Format a float array for pgvector tagged-template queries. */ +export function formatEmbeddingForPg(embedding) { + return `[${embedding.map((value) => Number(value).toFixed(8)).join(',')}]`; +} diff --git a/lib/card-visual-match.js b/lib/card-visual-match.js new file mode 100644 index 0000000..d400648 --- /dev/null +++ b/lib/card-visual-match.js @@ -0,0 +1,118 @@ +import { sql } from '@vercel/postgres'; + +import { formatEmbeddingForPg } from './card-embed.js'; + +export const MATCH_THRESHOLD = 0.82; +export const DISAMBIGUATION_THRESHOLD = 0.58; + +function mapCardRow(card) { + return { + id: card.id, + name: card.name, + set_name: card.set_name, + set_code: card.set_code, + card_number: card.card_number, + game: card.game, + rarity: card.rarity, + image_url: card.image_url, + card_type: card.card_type, + mana_cost: card.mana_cost, + hp: card.power, + similarity: card.sim, + }; +} + +/** + * Pure ranking logic for visual kNN candidates (unit-tested without DB). + */ +export function resolveVisualCandidates(candidates) { + if (!candidates?.length) { + return { + type: 'escalate', + reason: `No catalog match above ${DISAMBIGUATION_THRESHOLD} visual similarity`, + }; + } + + const top = candidates[0]; + const runnerUp = candidates[1]; + const clearWinner = + top.sim >= MATCH_THRESHOLD && (!runnerUp || top.sim - runnerUp.sim >= 0.06); + + if (clearWinner) { + return { + type: 'matched', + card: top, + similarity: top.sim, + message: `Matched "${top.name}" via visual search (${Math.round(top.sim * 100)}% similar)`, + }; + } + + return { + type: 'disambiguation', + matches: candidates.slice(0, 5).map(mapCardRow), + similarity: top.sim, + message: `Found ${candidates.length} visually similar printings. Select the correct one.`, + }; +} + +async function querySimilarCards(vectorLiteral, game) { + if (game) { + return sql` + SELECT id, name, set_name, set_code, card_number, game, rarity, image_url, card_type, mana_cost, power, + 1 - (embedding <=> ${vectorLiteral}::vector) AS sim + FROM cards + WHERE embedding IS NOT NULL + AND game = ${game} + AND 1 - (embedding <=> ${vectorLiteral}::vector) > ${DISAMBIGUATION_THRESHOLD - 0.05} + ORDER BY embedding <=> ${vectorLiteral}::vector + LIMIT 8 + `; + } + + return sql` + SELECT id, name, set_name, set_code, card_number, game, rarity, image_url, card_type, mana_cost, power, + 1 - (embedding <=> ${vectorLiteral}::vector) AS sim + FROM cards + WHERE embedding IS NOT NULL + AND 1 - (embedding <=> ${vectorLiteral}::vector) > ${DISAMBIGUATION_THRESHOLD - 0.05} + ORDER BY embedding <=> ${vectorLiteral}::vector + LIMIT 8 + `; +} + +/** + * True when at least one catalog row has a visual embedding index entry. + */ +export async function catalogHasEmbeddings() { + const indexed = await sql` + SELECT EXISTS( + SELECT 1 FROM cards WHERE embedding IS NOT NULL LIMIT 1 + ) AS has_embeddings + `; + return Boolean(indexed.rows[0]?.has_embeddings); +} + +/** + * kNN visual match against precomputed catalog embeddings (Layer 0). + */ +export async function matchVisualInCatalog({ embedding, game = null }) { + if (!embedding?.length) { + return { + type: 'escalate', + reason: 'Missing query embedding', + }; + } + + if (!(await catalogHasEmbeddings())) { + return { + type: 'escalate', + reason: 'Catalog visual index is empty — run npm run backfill-embeddings', + }; + } + + const vectorLiteral = formatEmbeddingForPg(embedding); + const result = await querySimilarCards(vectorLiteral, game || null); + const candidates = result.rows.filter((row) => row.sim >= DISAMBIGUATION_THRESHOLD); + + return resolveVisualCandidates(candidates); +} diff --git a/lib/scanner-card-identify.js b/lib/scanner-card-identify.js index 5776cb9..e375599 100644 --- a/lib/scanner-card-identify.js +++ b/lib/scanner-card-identify.js @@ -85,6 +85,7 @@ export function resolveIdentifyOutcome(result) { ocrMeta, message: result.message, fromLayer1: result.layer === 1, + fromLayer0: result.layer === 0, }; } @@ -242,6 +243,44 @@ export async function fetchIdentifyByText({ ocrText, ocrConfidence, cardNumber, return { ok: true, result }; } +export async function fetchIdentifyByImage(imageData, authHeaders, game) { + const response = await fetch('/api/scan/identify-by-image', { + method: 'POST', + headers: authHeaders, + body: JSON.stringify({ imageData, game }), + }); + + if (response.status === 429) { + return { ok: false, rateLimited: true }; + } + + if (!response.ok) { + return { ok: false }; + } + + const result = await response.json(); + return { ok: true, result }; +} + +/** + * Layer 0: visual kNN against precomputed catalog embeddings. + */ +export async function tryLayer0VisualIdentify(imageData, authHeaders, game) { + const l0 = await fetchIdentifyByImage(imageData, authHeaders, game); + if (!l0.ok) { + return { handled: false }; + } + + if (l0.result.escalate) { + return { handled: false }; + } + + return { + handled: true, + outcome: resolveIdentifyOutcome(l0.result), + }; +} + export async function fetchVisionIdentify(imageData, authHeaders) { const response = await fetch('/api/scan/identify', { method: 'POST', @@ -309,6 +348,15 @@ export async function identifyTrackedCardCapture({ cardTracker.corners ); + try { + const l0 = await tryLayer0VisualIdentify(imageData, authHeaders); + if (l0.handled) { + return { imageData, ...l0 }; + } + } catch (l0Error) { + console.warn('Layer-0 visual path failed, falling back to text/vision:', l0Error); + } + try { const l1 = await tryLayer1TextIdentify(imageData, authHeaders); if (l1.handled) { diff --git a/lib/use-scanner-identification.js b/lib/use-scanner-identification.js index f73e355..5fd3f2f 100644 --- a/lib/use-scanner-identification.js +++ b/lib/use-scanner-identification.js @@ -5,7 +5,9 @@ import { getScanAuthHeaders, identifyTrackedCardCapture, resolveDisambiguationRefineAction, + resolveIdentifyOutcome, submitScanForReview, + tryLayer0VisualIdentify, tryLayer1TextIdentify, VISION_RATE_LIMIT_MS, fetchVisionIdentify, @@ -132,6 +134,7 @@ export function useScannerIdentification({ ocrMeta: outcome.ocrMeta, message: outcome.message, fromLayer1: Boolean(outcome.fromLayer1), + fromLayer0: Boolean(outcome.fromLayer0), }); break; case 'notice': @@ -180,7 +183,7 @@ export function useScannerIdentification({ useEffect(() => { if (!disambiguation?.imageData) return; - if (disambiguation.fromLayer1) return; + if (disambiguation.fromLayer1 || disambiguation.fromLayer0) return; if (Date.now() < visionCooldownUntilRef.current) return; const refineKey = disambiguation.cardTracker?.id ?? 'modal'; @@ -326,14 +329,42 @@ export function useScannerIdentification({ try { const imageData = await readFileToImageData(file); const authHeaders = getScanAuthHeaders(); - const result = await tryLayer1TextIdentify(imageData, authHeaders); const syntheticTracker = { id: `gallery-${Date.now()}`, status: 'verifying' }; + const l0 = await tryLayer0VisualIdentify(imageData, authHeaders); + if (l0.handled && l0.outcome) { + await applyIdentifyOutcome(syntheticTracker, imageData, l0.outcome); + return; + } + + const result = await tryLayer1TextIdentify(imageData, authHeaders); + if (result.handled && result.outcome) { await applyIdentifyOutcome(syntheticTracker, imageData, result.outcome); return; } + if (Date.now() < visionCooldownUntilRef.current) { + reportScannerError('Too many scan attempts. Please wait a moment and try again.'); + return; + } + + const vision = await fetchVisionIdentify(imageData, authHeaders); + if (vision.rateLimited) { + visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS; + reportScannerError('Too many scan attempts. Please wait a moment and try again.'); + return; + } + + if (vision.ok && vision.result) { + await applyIdentifyOutcome( + syntheticTracker, + imageData, + resolveIdentifyOutcome(vision.result) + ); + return; + } + reportScannerError('Could not identify card from gallery image'); } catch (error) { reportScannerError(error.message || 'Gallery identify failed'); diff --git a/migrations/1782000000001_add-card-embeddings.js b/migrations/1782000000001_add-card-embeddings.js new file mode 100644 index 0000000..3affc56 --- /dev/null +++ b/migrations/1782000000001_add-card-embeddings.js @@ -0,0 +1,33 @@ +/** + * pgvector + cards.embedding for Layer-0 visual catalog search. + * + * @type {import('node-pg-migrate').ColumnDefinitions | undefined} + */ +export const shorthands = undefined; + +/** Must match lib/card-embed.js EMBED_DIMENSION. */ +const EMBED_DIMENSION = 1024; + +/** + * @param {import('node-pg-migrate').MigrationBuilder} pgm + */ +export const up = (pgm) => { + pgm.sql(` + CREATE EXTENSION IF NOT EXISTS vector; + + ALTER TABLE cards + ADD COLUMN IF NOT EXISTS embedding vector(${EMBED_DIMENSION}), + ADD COLUMN IF NOT EXISTS embedded_at TIMESTAMP; + + CREATE INDEX IF NOT EXISTS idx_cards_embedding_hnsw + ON cards USING hnsw (embedding vector_cosine_ops) + WHERE embedding IS NOT NULL; + `); +}; + +/** + * @param {import('node-pg-migrate').MigrationBuilder} pgm + */ +export const down = (pgm) => { + throw new Error('Down migration not supported for add-card-embeddings'); +}; diff --git a/package.json b/package.json index 5d0e023..ebf185b 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "bulk-import": "node --env-file=.env.local scripts/bulk-import-scryfall.js", "bulk-import-lorcana": "node --env-file=.env.local scripts/bulk-import-lorcana.js", "bulk-import-pokemon": "node --env-file=.env.local scripts/bulk-import-pokemon.js", + "backfill-embeddings": "node --env-file=.env.local scripts/backfill-card-embeddings.js", "import-tags": "node --env-file=.env.local scripts/import-scryfall-tags.js", "test": "vitest", "test:run": "vitest run", diff --git a/pages/api/scan/identify-by-image.js b/pages/api/scan/identify-by-image.js new file mode 100644 index 0000000..bbadcfa --- /dev/null +++ b/pages/api/scan/identify-by-image.js @@ -0,0 +1,156 @@ +import { getUserFromRequest } from '../../../lib/permission-middleware'; +import { checkScanRateLimit } from '../../../lib/rate-limit.js'; +import { embedCardImage, EmbedApiError } from '../../../lib/card-embed.js'; +import { matchVisualInCatalog, catalogHasEmbeddings } from '../../../lib/card-visual-match.js'; +import { logScanAttempt } from '../../../lib/card-catalog-match.js'; + +function formatCardResponse(card) { + return { + id: card.id, + name: card.name, + set_name: card.set_name, + set_code: card.set_code, + card_number: card.card_number, + game: card.game, + card_type: card.card_type, + rarity: card.rarity, + hp: card.power, + mana_cost: card.mana_cost, + image_url: card.image_url, + visual: { + similarity: card.sim, + }, + }; +} + +export default async function handler(req, res) { + if (req.method !== 'POST') { + return res.status(405).json({ error: 'Method not allowed' }); + } + + const startedAt = Date.now(); + + try { + const user = await getUserFromRequest(req); + if (!user) { + return res.status(401).json({ error: 'Authentication required' }); + } + + const { imageData, game } = req.body || {}; + + if (!imageData || typeof imageData !== 'string') { + return res.status(400).json({ error: 'imageData is required' }); + } + + if (imageData.length > 6_000_000) { + return res.status(400).json({ error: 'Image payload too large' }); + } + + if (!(await catalogHasEmbeddings())) { + const latencyMs = Date.now() - startedAt; + await logScanAttempt({ + userId: user.userId, + ocrText: null, + ocrConfidence: null, + layer: 0, + resultKind: 'escalate', + latencyMs, + }); + return res.status(200).json({ + layer: 0, + escalate: true, + reason: 'Catalog visual index is empty — run npm run backfill-embeddings', + }); + } + + const { allowed, reset } = await checkScanRateLimit(req, user.userId); + if (!allowed) { + res.setHeader('Retry-After', Math.ceil((reset - Date.now()) / 1000)); + return res.status(429).json({ error: 'Too many attempts. Try again later.' }); + } + + const embedding = await embedCardImage(imageData); + const matchResult = await matchVisualInCatalog({ embedding, game: game || null }); + const latencyMs = Date.now() - startedAt; + + if (matchResult.type === 'escalate') { + await logScanAttempt({ + userId: user.userId, + ocrText: null, + ocrConfidence: null, + layer: 0, + resultKind: 'escalate', + latencyMs, + }); + return res.status(200).json({ + layer: 0, + escalate: true, + reason: matchResult.reason, + }); + } + + if (matchResult.type === 'matched') { + await logScanAttempt({ + userId: user.userId, + ocrText: null, + ocrConfidence: Math.round((matchResult.similarity || 0) * 100), + layer: 0, + matchedCardId: matchResult.card.id, + resultKind: 'matched', + latencyMs, + }); + return res.status(200).json({ + layer: 0, + escalate: false, + isCard: true, + card: formatCardResponse(matchResult.card), + isExisting: true, + message: matchResult.message, + visual: { similarity: matchResult.similarity }, + }); + } + + await logScanAttempt({ + userId: user.userId, + ocrText: null, + ocrConfidence: Math.round((matchResult.similarity || 0) * 100), + layer: 0, + resultKind: 'disambiguation', + latencyMs, + }); + + return res.status(200).json({ + layer: 0, + escalate: false, + isCard: true, + card: null, + matches: matchResult.matches, + needsUserSelection: true, + message: matchResult.message, + visual: { similarity: matchResult.similarity }, + }); + } catch (error) { + console.error('[POST /api/scan/identify-by-image]', error); + + if (error instanceof EmbedApiError) { + if (error.message.includes('AI_GATEWAY_API_KEY')) { + return res.status(503).json({ + error: 'Visual matching is not configured on this server (missing AI_GATEWAY_API_KEY).', + }); + } + if (error.status === 429) { + return res.status(502).json({ + error: 'Embedding service quota exceeded. Try again later.', + }); + } + } + + if (String(error.message).includes('vector') || String(error.message).includes('pgvector')) { + return res.status(503).json({ + error: 'Visual matching unavailable — run npm run migrate up (pgvector extension).', + }); + } + + return res.status(500).json({ error: 'Internal server error' }); + } +} diff --git a/scripts/backfill-card-embeddings.js b/scripts/backfill-card-embeddings.js new file mode 100644 index 0000000..4f95980 --- /dev/null +++ b/scripts/backfill-card-embeddings.js @@ -0,0 +1,117 @@ +/** + * Backfill cards.embedding from cards.image_url via Vercel AI Gateway. + * + * Idempotent: skips rows where embedded_at is set unless FORCE=1. + * + * Usage: + * POSTGRES_URL= AI_GATEWAY_API_KEY= node scripts/backfill-card-embeddings.js + * + * Options (env): + * BATCH_SIZE — rows per fetch (default 25) + * SLEEP_MS — delay between embed calls (default 250) + * FORCE — "1" to re-embed rows that already have embedded_at + * LIMIT — max rows to process (default unlimited) + * DRY_RUN — "true" to list candidates only + */ + +import { neon } from '@neondatabase/serverless'; + +import { embedCardImage, formatEmbeddingForPg } from '../lib/card-embed.js'; + +if (!process.env.POSTGRES_URL) { + console.error('POSTGRES_URL is required'); + process.exit(1); +} + +if (!process.env.AI_GATEWAY_API_KEY) { + console.error('AI_GATEWAY_API_KEY is required'); + process.exit(1); +} + +const sql = neon(process.env.POSTGRES_URL, { fullResults: false }); +const BATCH_SIZE = Number(process.env.BATCH_SIZE || 25); +const SLEEP_MS = Number(process.env.SLEEP_MS || 250); +const FORCE = process.env.FORCE === '1'; +const LIMIT = process.env.LIMIT ? Number(process.env.LIMIT) : null; +const DRY_RUN = process.env.DRY_RUN === 'true'; + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function fetchCandidates(lastId) { + if (FORCE) { + return sql` + SELECT id, name, image_url + FROM cards + WHERE id > ${lastId} + AND image_url IS NOT NULL + AND image_url <> '' + ORDER BY id + LIMIT ${BATCH_SIZE} + `; + } + + return sql` + SELECT id, name, image_url + FROM cards + WHERE id > ${lastId} + AND image_url IS NOT NULL + AND image_url <> '' + AND embedding IS NULL + ORDER BY id + LIMIT ${BATCH_SIZE} + `; +} + +async function main() { + let lastId = 0; + let processed = 0; + let updated = 0; + + console.log(`Backfill starting (force=${FORCE}, dryRun=${DRY_RUN})`); + + while (true) { + if (LIMIT != null && processed >= LIMIT) break; + + const rows = await fetchCandidates(lastId); + if (!rows.length) break; + + for (const row of rows) { + if (LIMIT != null && processed >= LIMIT) break; + processed++; + lastId = row.id; + + if (DRY_RUN) { + console.log(`[dry-run] would embed card ${row.id}: ${row.name}`); + continue; + } + + try { + const embedding = await embedCardImage({ imageUrl: row.image_url }); + const vectorLiteral = formatEmbeddingForPg(embedding); + await sql` + UPDATE cards + SET embedding = ${vectorLiteral}::vector, + embedded_at = CURRENT_TIMESTAMP + WHERE id = ${row.id} + `; + updated++; + console.log(`Embedded card ${row.id}: ${row.name}`); + } catch (error) { + console.error(`Failed card ${row.id} (${row.name}):`, error.message); + } + + if (SLEEP_MS > 0) { + await sleep(SLEEP_MS); + } + } + } + + console.log(`Done. processed=${processed} updated=${updated}`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/test/lib/card-embed.test.js b/test/lib/card-embed.test.js new file mode 100644 index 0000000..b894493 --- /dev/null +++ b/test/lib/card-embed.test.js @@ -0,0 +1,35 @@ +import { describe, expect, it, vi, afterEach } from 'vitest'; +import { embedCardImage, formatEmbeddingForPg } from '../../lib/card-embed.js'; + +describe('formatEmbeddingForPg', () => { + it('formats vectors for pgvector literals', () => { + expect(formatEmbeddingForPg([0.1, 0.2, 0.3])).toBe('[0.10000000,0.20000000,0.30000000]'); + }); +}); + +describe('embedCardImage', () => { + afterEach(() => { + vi.unstubAllGlobals(); + delete process.env.AI_GATEWAY_API_KEY; + }); + + it('throws when AI_GATEWAY_API_KEY is missing', async () => { + await expect(embedCardImage('data:image/jpeg;base64,abc')).rejects.toThrow( + 'AI_GATEWAY_API_KEY is not configured' + ); + }); + + it('returns embedding values from the gateway response', async () => { + process.env.AI_GATEWAY_API_KEY = 'test-key'; + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ + ok: true, + json: async () => ({ data: [{ embedding: [0.5, 0.25] }] }), + })) + ); + + const embedding = await embedCardImage('data:image/jpeg;base64,abc'); + expect(embedding).toEqual([0.5, 0.25]); + }); +}); diff --git a/test/lib/card-visual-match.test.js b/test/lib/card-visual-match.test.js new file mode 100644 index 0000000..a68924a --- /dev/null +++ b/test/lib/card-visual-match.test.js @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; +import { resolveVisualCandidates } from '../../lib/card-visual-match.js'; + +describe('resolveVisualCandidates', () => { + const baseCard = { + id: 1, + name: 'Lightning Bolt', + set_name: 'Alpha', + set_code: 'lea', + card_number: '161', + game: 'mtg', + rarity: 'common', + image_url: 'https://example.com/bolt.jpg', + card_type: 'Instant', + mana_cost: '{R}', + power: null, + }; + + it('escalates when no candidates remain', () => { + expect(resolveVisualCandidates([]).type).toBe('escalate'); + }); + + it('auto-matches a clear visual winner', () => { + const result = resolveVisualCandidates([ + { ...baseCard, sim: 0.9 }, + { ...baseCard, id: 2, sim: 0.7 }, + ]); + expect(result.type).toBe('matched'); + expect(result.card.id).toBe(1); + }); + + it('opens disambiguation when top matches are close', () => { + const result = resolveVisualCandidates([ + { ...baseCard, sim: 0.8 }, + { ...baseCard, id: 2, set_name: 'Beta', sim: 0.77 }, + ]); + expect(result.type).toBe('disambiguation'); + expect(result.matches).toHaveLength(2); + }); +}); diff --git a/test/lib/scanner-card-identify.test.js b/test/lib/scanner-card-identify.test.js index 50c88cb..f2a872e 100644 --- a/test/lib/scanner-card-identify.test.js +++ b/test/lib/scanner-card-identify.test.js @@ -54,6 +54,7 @@ describe('resolveIdentifyOutcome', () => { ocrMeta: expect.objectContaining({ abilities: [] }), message: 'Pick one', fromLayer1: false, + fromLayer0: false, }); });