.convoys/.metrics.jsonl

@@ -101,3 +101,10 @@
 {"ts": "2026-08-15T01:24:40Z", "role": "role-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
 {"ts": "2026-08-15T01:24:41Z", "role": "role-security-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"}
 {"ts": "2026-08-15T01:24:41Z", "role": "role-a11y-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 60, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
+{"ts": "2026-08-15T01:35:27Z", "role": "role-ux-reviewer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"}
+{"ts": "2026-08-15T01:35:27Z", "role": "role-architect", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 360, "model": "composer-2.5", "model_tier": "standard"}
+{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 1, "duration_s": 300, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
+{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 2, "duration_s": 600, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
+{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 3, "duration_s": 900, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
+{"ts": "2026-08-15T01:35:28Z", "role": "role-reviewer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-scan-visual-catalog-search-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
+{"ts": "2026-08-15T01:35:28Z", "role": "role-security-auditor", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-scan-visual-catalog-search-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"}

.convoys/scan-visual-catalog-search.md

@@ -14,7 +14,7 @@ skip:
   - a11y
   - design
   - flag
-status: open
+status: in-progress
 created: 2026-08-14
 depends_on:
   - improve-scan-card-detection
@@ -107,12 +107,13 @@ unknown-card fallback.
 
 ## Todos
 
-- [ ] Architect: confirm `pgvector` on prod Neon tier
+- [x] Architect: confirm `pgvector` on prod Neon tier
-- [ ] Brief 1 — migration + SCHEMA_MAP
+- [x] Brief 1 — migration + SCHEMA_MAP
-- [ ] Brief 2 — catalog backfill job (idempotent)
+- [x] Brief 2 — catalog backfill job (idempotent)
-- [ ] Brief 3 — identify kNN route + client escalate order
+- [x] Brief 3 — identify kNN route + client escalate order
 - [ ] Threshold bake-off on real crops
 - [ ] Re-measure auto-match % excluding `not_a_card`
+- [ ] Operator: `npm run backfill-embeddings` on prod/staging after migrate
 
 ## Likely file ownership
 
@@ -159,3 +160,37 @@ crops are rectified — that wastes the backfill. If Architect finds
 `pgvector` unavailable on CI Postgres, stop and write a fallback
 (external index vs skip-CI-extension plan) rather than shipping an
 untestable migration.
+
+## UX
+
+No new screens. Scan flow stays L0 → L1 → L2 with the same
+disambiguation picker and error toasts. When the catalog index is empty
+(backfill not run), L0 escalates silently with no embed cost.
+
+Gallery uploads now try visual match before OCR.
+
+## Architecture
+
+### Decision D1 — Gateway multimodal embedder (`cohere/embed-v4.0`)
+
+Server-only via `AI_GATEWAY_API_KEY`. 1024-dim vectors in
+`cards.embedding`. Env: `SCAN_EMBED_MODEL`, `SCAN_EMBED_DIMENSION`.
+
+### Decision D2 — Layer numbering
+
+| Layer | Path |
+| --- | --- |
+| 0 | `POST /api/scan/identify-by-image` |
+| 1 | Tesseract + `identify-by-text` |
+| 2 | Gemini + `scan/identify` |
+
+### Decision D3 — Thresholds
+
+Match ≥ **0.82** (0.06 gap). Disambiguation ≥ **0.58**.
+
+### Decision D4 — Rate limit
+
+`checkScanRateLimit` on identify-by-image. L0 429 falls through to L1
+(not a hard stop).
+
+Audit group id: `audit-scan-visual-catalog-search-<pr>`.

.convoys/scan-visual-catalog-search/brief-1-migration-schema.md

@@ -0,0 +1,23 @@
+---
+convoy: scan-visual-catalog-search
+brief_number: 1
+depends_on: []
+recommended_model: composer-2.5
+model_tier: standard
+files:
+  - migrations/1782000000001_add-card-embeddings.js
+  - docs/SCHEMA_MAP.md
+---
+
+# Brief 1: pgvector migration + SCHEMA_MAP
+
+## Goal
+
+Add `vector(1024)` embedding column + HNSW index on `cards`.
+
+## Acceptance criteria
+
+- [ ] `CREATE EXTENSION IF NOT EXISTS vector`
+- [ ] `cards.embedding`, `cards.embedded_at`
+- [ ] `idx_cards_embedding_hnsw` partial index
+- [ ] SCHEMA_MAP documents layer 0 + pgvector

.convoys/scan-visual-catalog-search/brief-2-backfill-embeddings.md

@@ -0,0 +1,24 @@
+---
+convoy: scan-visual-catalog-search
+brief_number: 2
+depends_on: [1]
+recommended_model: composer-2.5-fast
+model_tier: fast
+files:
+  - lib/card-embed.js
+  - scripts/backfill-card-embeddings.js
+  - package.json
+  - test/lib/card-embed.test.js
+---
+
+# Brief 2: Catalog embedding backfill
+
+## Goal
+
+Server-side embed job using `cohere/embed-v4.0` via AI Gateway; idempotent backfill from `cards.image_url`.
+
+## Acceptance criteria
+
+- [ ] `lib/card-embed.js` exports `embedCardImage`, `formatEmbeddingForPg`
+- [ ] `npm run backfill-embeddings` script (skips rows with embedding unless FORCE=1)
+- [ ] Rate-limited (`SLEEP_MS`) and DRY_RUN support

.convoys/scan-visual-catalog-search/brief-3-identify-knn-route.md

@@ -0,0 +1,27 @@
+---
+convoy: scan-visual-catalog-search
+brief_number: 3
+depends_on: [1]
+recommended_model: composer-2.5-fast
+model_tier: fast
+files:
+  - lib/card-visual-match.js
+  - pages/api/scan/identify-by-image.js
+  - lib/scanner-card-identify.js
+  - lib/use-scanner-identification.js
+  - test/lib/card-visual-match.test.js
+---
+
+# Brief 3: Layer-0 identify route + client orchestration
+
+## Goal
+
+kNN visual match before L1/L2; log `scan_attempts.layer = 0`.
+
+## Acceptance criteria
+
+- [ ] `POST /api/scan/identify-by-image` — auth + `checkScanRateLimit` + embed + kNN
+- [ ] `identifyTrackedCardCapture` order: L0 → L1 → L2
+- [ ] Gallery path runs L0 → L1 → L2
+- [ ] Skip auto Gemini refine when disambiguation from L0 (same as L1)
+- [ ] Thresholds: match 0.82, disambiguation 0.58

docs/SCHEMA_MAP.md

@@ -93,16 +93,20 @@
 | `reprint` | `BOOLEAN` default `false` | (`1781440700404`) |
 | `finishes` | `JSONB` | `["nonfoil","foil","etched"]` (`1781440700404`) |
 | `created_at`, `updated_at` | `TIMESTAMP` default now | |
+| `embedding` | `vector(1024)` | Layer-0 visual kNN index (`1782000000001_add-card-embeddings`) |
+| `embedded_at` | `TIMESTAMP` | When `embedding` was last written by backfill |
 
 **Indexes (post `1779853647565_add-pg-trgm-card-name-index` + `1781440700404`):**
 
 - `idx_cards_name_trgm` — GIN on `name` using `gin_trgm_ops` for Layer-1 OCR fuzzy match (`similarity()` / `pg_trgm`).
+- `idx_cards_embedding_hnsw` — HNSW on `embedding vector_cosine_ops` (partial, non-null only) for Layer-0 visual search.
 - `cards_oracle_id_index`, `cards_illustration_id_index`, `cards_artist_index`, `cards_edhrec_rank_index`
 - GIN on `color_identity`, `keywords`, `legalities`
 
 **Extensions used by scan pipeline:**
 
-- `pg_trgm` — enabled by `1779853647565_add-pg-trgm-card-name-index.js` for trigram similarity on `cards.name`.
+- `pg_trgm` — Layer-1 text similarity (`1779853647565_add-pg-trgm-card-name-index`).
+- `vector` (pgvector) — Layer-0 visual embeddings (`1782000000001_add-card-embeddings`).
 
 ### user_cards
 
@@ -261,7 +265,7 @@ Index: `idx_card_submissions_status (status, created_at DESC)`.
 
 ### scan_attempts
 
-Per-scan telemetry for the identify pipeline (layer 1 = Tesseract + pg_trgm, layer 2 = Gemini).
+Per-scan telemetry for the identify pipeline (layer 0 = visual kNN, layer 1 = Tesseract + pg_trgm, layer 2 = Gemini).
 
 | Column | Type | Notes |
 | --- | --- | --- |
@@ -269,7 +273,7 @@ Per-scan telemetry for the identify pipeline (layer 1 = Tesseract + pg_trgm, lay
 | `user_id` | FK → `users` SET NULL | |
 | `ocr_text` | `TEXT` | |
 | `ocr_confidence` | `INTEGER` | |
-| `layer` | `INTEGER` default `2` | Identify layer (`1` = browser Tesseract + pg_trgm, `2` = Gemini vision) |
+| `layer` | `INTEGER` default `2` | Identify layer (`0` = visual kNN, `1` = browser Tesseract + pg_trgm, `2` = Gemini vision) |
 | `matched_card_id` | FK → `cards` SET NULL | |
 | `result_kind` | `VARCHAR(32)` | e.g. `'matched'`, `'disambiguation'`, `'submitted'`, `'not_a_card'` |
 | `latency_ms` | `INTEGER` | End-to-end identify latency |

lib/card-embed.js

@@ -0,0 +1,90 @@
+const GATEWAY_EMBED_URL = 'https://ai-gateway.vercel.sh/v1/embeddings';
+const DEFAULT_EMBED_MODEL = process.env.SCAN_EMBED_MODEL || 'cohere/embed-v4.0';
+
+/** Output dimension — must match migrations/1782000000001_add-card-embeddings.js. */
+export const EMBED_DIMENSION = Number(process.env.SCAN_EMBED_DIMENSION || 1024);
+
+export class EmbedApiError extends Error {
+  constructor(message, { status } = {}) {
+    super(message);
+    this.name = 'EmbedApiError';
+    this.status = status;
+  }
+}
+
+function parseEmbeddingResponse(data) {
+  const embedding =
+    data?.data?.[0]?.embedding ||
+    data?.embeddings?.[0]?.values ||
+    data?.embeddings?.[0] ||
+    data?.embedding;
+
+  if (!Array.isArray(embedding) || embedding.length === 0) {
+    throw new EmbedApiError('Embedding API returned no vector');
+  }
+
+  return embedding.map((value) => Number(value));
+}
+
+function buildEmbedInput(value) {
+  if (typeof value === 'string') {
+    return value;
+  }
+
+  if (value?.imageUrl) {
+    return {
+      type: 'image_url',
+      image_url: { url: value.imageUrl },
+    };
+  }
+
+  if (value?.imageDataUrl) {
+    return value.imageDataUrl;
+  }
+
+  throw new EmbedApiError('Invalid embed input');
+}
+
+/**
+ * Embed image content via Vercel AI Gateway (server-only).
+ * Accepts a JPEG data URL or HTTPS image URL object.
+ */
+export async function embedCardImage(input) {
+  const apiKey = process.env.AI_GATEWAY_API_KEY;
+  if (!apiKey) {
+    throw new EmbedApiError('AI_GATEWAY_API_KEY is not configured on the server');
+  }
+
+  const payloadInput = buildEmbedInput(input);
+  if (typeof payloadInput === 'string' && !payloadInput.includes(',')) {
+    throw new EmbedApiError('Invalid image data format');
+  }
+
+  const response = await fetch(GATEWAY_EMBED_URL, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      model: DEFAULT_EMBED_MODEL,
+      input: payloadInput,
+      dimensions: EMBED_DIMENSION,
+    }),
+  });
+
+  if (!response.ok) {
+    const errorData = await response.json().catch(() => ({}));
+    const message = errorData.error?.message || errorData.message || 'Unknown error';
+    throw new EmbedApiError(`Embedding API error: ${response.status} - ${message}`, {
+      status: response.status,
+    });
+  }
+
+  return parseEmbeddingResponse(await response.json());
+}
+
+/** Format a float array for pgvector tagged-template queries. */
+export function formatEmbeddingForPg(embedding) {
+  return `[${embedding.map((value) => Number(value).toFixed(8)).join(',')}]`;
+}

lib/card-visual-match.js

@@ -0,0 +1,118 @@
+import { sql } from '@vercel/postgres';
+
+import { formatEmbeddingForPg } from './card-embed.js';
+
+export const MATCH_THRESHOLD = 0.82;
+export const DISAMBIGUATION_THRESHOLD = 0.58;
+
+function mapCardRow(card) {
+  return {
+    id: card.id,
+    name: card.name,
+    set_name: card.set_name,
+    set_code: card.set_code,
+    card_number: card.card_number,
+    game: card.game,
+    rarity: card.rarity,
+    image_url: card.image_url,
+    card_type: card.card_type,
+    mana_cost: card.mana_cost,
+    hp: card.power,
+    similarity: card.sim,
+  };
+}
+
+/**
+ * Pure ranking logic for visual kNN candidates (unit-tested without DB).
+ */
+export function resolveVisualCandidates(candidates) {
+  if (!candidates?.length) {
+    return {
+      type: 'escalate',
+      reason: `No catalog match above ${DISAMBIGUATION_THRESHOLD} visual similarity`,
+    };
+  }
+
+  const top = candidates[0];
+  const runnerUp = candidates[1];
+  const clearWinner =
+    top.sim >= MATCH_THRESHOLD && (!runnerUp || top.sim - runnerUp.sim >= 0.06);
+
+  if (clearWinner) {
+    return {
+      type: 'matched',
+      card: top,
+      similarity: top.sim,
+      message: `Matched "${top.name}" via visual search (${Math.round(top.sim * 100)}% similar)`,
+    };
+  }
+
+  return {
+    type: 'disambiguation',
+    matches: candidates.slice(0, 5).map(mapCardRow),
+    similarity: top.sim,
+    message: `Found ${candidates.length} visually similar printings. Select the correct one.`,
+  };
+}
+
+async function querySimilarCards(vectorLiteral, game) {
+  if (game) {
+    return sql`
+      SELECT id, name, set_name, set_code, card_number, game, rarity, image_url, card_type, mana_cost, power,
+             1 - (embedding <=> ${vectorLiteral}::vector) AS sim
+      FROM cards
+      WHERE embedding IS NOT NULL
+        AND game = ${game}
+        AND 1 - (embedding <=> ${vectorLiteral}::vector) > ${DISAMBIGUATION_THRESHOLD - 0.05}
+      ORDER BY embedding <=> ${vectorLiteral}::vector
+      LIMIT 8
+    `;
+  }
+
+  return sql`
+    SELECT id, name, set_name, set_code, card_number, game, rarity, image_url, card_type, mana_cost, power,
+           1 - (embedding <=> ${vectorLiteral}::vector) AS sim
+    FROM cards
+    WHERE embedding IS NOT NULL
+      AND 1 - (embedding <=> ${vectorLiteral}::vector) > ${DISAMBIGUATION_THRESHOLD - 0.05}
+    ORDER BY embedding <=> ${vectorLiteral}::vector
+    LIMIT 8
+  `;
+}
+
+/**
+ * True when at least one catalog row has a visual embedding index entry.
+ */
+export async function catalogHasEmbeddings() {
+  const indexed = await sql`
+    SELECT EXISTS(
+      SELECT 1 FROM cards WHERE embedding IS NOT NULL LIMIT 1
+    ) AS has_embeddings
+  `;
+  return Boolean(indexed.rows[0]?.has_embeddings);
+}
+
+/**
+ * kNN visual match against precomputed catalog embeddings (Layer 0).
+ */
+export async function matchVisualInCatalog({ embedding, game = null }) {
+  if (!embedding?.length) {
+    return {
+      type: 'escalate',
+      reason: 'Missing query embedding',
+    };
+  }
+
+  if (!(await catalogHasEmbeddings())) {
+    return {
+      type: 'escalate',
+      reason: 'Catalog visual index is empty — run npm run backfill-embeddings',
+    };
+  }
+
+  const vectorLiteral = formatEmbeddingForPg(embedding);
+  const result = await querySimilarCards(vectorLiteral, game || null);
+  const candidates = result.rows.filter((row) => row.sim >= DISAMBIGUATION_THRESHOLD);
+
+  return resolveVisualCandidates(candidates);
+}

lib/scanner-card-identify.js

@@ -85,6 +85,7 @@ export function resolveIdentifyOutcome(result) {
       ocrMeta,
       message: result.message,
       fromLayer1: result.layer === 1,
+      fromLayer0: result.layer === 0,
     };
   }
 
@@ -242,6 +243,44 @@ export async function fetchIdentifyByText({ ocrText, ocrConfidence, cardNumber,
   return { ok: true, result };
 }
 
+export async function fetchIdentifyByImage(imageData, authHeaders, game) {
+  const response = await fetch('/api/scan/identify-by-image', {
+    method: 'POST',
+    headers: authHeaders,
+    body: JSON.stringify({ imageData, game }),
+  });
+
+  if (response.status === 429) {
+    return { ok: false, rateLimited: true };
+  }
+
+  if (!response.ok) {
+    return { ok: false };
+  }
+
+  const result = await response.json();
+  return { ok: true, result };
+}
+
+/**
+ * Layer 0: visual kNN against precomputed catalog embeddings.
+ */
+export async function tryLayer0VisualIdentify(imageData, authHeaders, game) {
+  const l0 = await fetchIdentifyByImage(imageData, authHeaders, game);
+  if (!l0.ok) {
+    return { handled: false };
+  }
+
+  if (l0.result.escalate) {
+    return { handled: false };
+  }
+
+  return {
+    handled: true,
+    outcome: resolveIdentifyOutcome(l0.result),
+  };
+}
+
 export async function fetchVisionIdentify(imageData, authHeaders) {
   const response = await fetch('/api/scan/identify', {
     method: 'POST',
@@ -309,6 +348,15 @@ export async function identifyTrackedCardCapture({
     cardTracker.corners
   );
 
+  try {
+    const l0 = await tryLayer0VisualIdentify(imageData, authHeaders);
+    if (l0.handled) {
+      return { imageData, ...l0 };
+    }
+  } catch (l0Error) {
+    console.warn('Layer-0 visual path failed, falling back to text/vision:', l0Error);
+  }
+
   try {
     const l1 = await tryLayer1TextIdentify(imageData, authHeaders);
     if (l1.handled) {

lib/use-scanner-identification.js

@@ -5,7 +5,9 @@ import {
   getScanAuthHeaders,
   identifyTrackedCardCapture,
   resolveDisambiguationRefineAction,
+  resolveIdentifyOutcome,
   submitScanForReview,
+  tryLayer0VisualIdentify,
   tryLayer1TextIdentify,
   VISION_RATE_LIMIT_MS,
   fetchVisionIdentify,
@@ -132,6 +134,7 @@ export function useScannerIdentification({
           ocrMeta: outcome.ocrMeta,
           message: outcome.message,
           fromLayer1: Boolean(outcome.fromLayer1),
+          fromLayer0: Boolean(outcome.fromLayer0),
         });
         break;
       case 'notice':
@@ -180,7 +183,7 @@ export function useScannerIdentification({
 
   useEffect(() => {
     if (!disambiguation?.imageData) return;
-    if (disambiguation.fromLayer1) return;
+    if (disambiguation.fromLayer1 || disambiguation.fromLayer0) return;
     if (Date.now() < visionCooldownUntilRef.current) return;
 
     const refineKey = disambiguation.cardTracker?.id ?? 'modal';
@@ -326,14 +329,42 @@ export function useScannerIdentification({
     try {
       const imageData = await readFileToImageData(file);
       const authHeaders = getScanAuthHeaders();
-      const result = await tryLayer1TextIdentify(imageData, authHeaders);
       const syntheticTracker = { id: `gallery-${Date.now()}`, status: 'verifying' };
 
+      const l0 = await tryLayer0VisualIdentify(imageData, authHeaders);
+      if (l0.handled && l0.outcome) {
+        await applyIdentifyOutcome(syntheticTracker, imageData, l0.outcome);
+        return;
+      }
+
+      const result = await tryLayer1TextIdentify(imageData, authHeaders);
+
       if (result.handled && result.outcome) {
         await applyIdentifyOutcome(syntheticTracker, imageData, result.outcome);
         return;
       }
 
+      if (Date.now() < visionCooldownUntilRef.current) {
+        reportScannerError('Too many scan attempts. Please wait a moment and try again.');
+        return;
+      }
+
+      const vision = await fetchVisionIdentify(imageData, authHeaders);
+      if (vision.rateLimited) {
+        visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
+        reportScannerError('Too many scan attempts. Please wait a moment and try again.');
+        return;
+      }
+
+      if (vision.ok && vision.result) {
+        await applyIdentifyOutcome(
+          syntheticTracker,
+          imageData,
+          resolveIdentifyOutcome(vision.result)
+        );
+        return;
+      }
+
       reportScannerError('Could not identify card from gallery image');
     } catch (error) {
       reportScannerError(error.message || 'Gallery identify failed');

migrations/1782000000001_add-card-embeddings.js

@@ -0,0 +1,33 @@
+/**
+ * pgvector + cards.embedding for Layer-0 visual catalog search.
+ *
+ * @type {import('node-pg-migrate').ColumnDefinitions | undefined}
+ */
+export const shorthands = undefined;
+
+/** Must match lib/card-embed.js EMBED_DIMENSION. */
+const EMBED_DIMENSION = 1024;
+
+/**
+ * @param {import('node-pg-migrate').MigrationBuilder} pgm
+ */
+export const up = (pgm) => {
+  pgm.sql(`
+    CREATE EXTENSION IF NOT EXISTS vector;
+
+    ALTER TABLE cards
+      ADD COLUMN IF NOT EXISTS embedding vector(${EMBED_DIMENSION}),
+      ADD COLUMN IF NOT EXISTS embedded_at TIMESTAMP;
+
+    CREATE INDEX IF NOT EXISTS idx_cards_embedding_hnsw
+      ON cards USING hnsw (embedding vector_cosine_ops)
+      WHERE embedding IS NOT NULL;
+  `);
+};
+
+/**
+ * @param {import('node-pg-migrate').MigrationBuilder} pgm
+ */
+export const down = (pgm) => {
+  throw new Error('Down migration not supported for add-card-embeddings');
+};

package.json

@@ -15,6 +15,7 @@
     "bulk-import": "node --env-file=.env.local scripts/bulk-import-scryfall.js",
     "bulk-import-lorcana": "node --env-file=.env.local scripts/bulk-import-lorcana.js",
     "bulk-import-pokemon": "node --env-file=.env.local scripts/bulk-import-pokemon.js",
+    "backfill-embeddings": "node --env-file=.env.local scripts/backfill-card-embeddings.js",
     "import-tags": "node --env-file=.env.local scripts/import-scryfall-tags.js",
     "test": "vitest",
     "test:run": "vitest run",

pages/api/scan/identify-by-image.js

@@ -0,0 +1,156 @@
+import { getUserFromRequest } from '../../../lib/permission-middleware';
+import { checkScanRateLimit } from '../../../lib/rate-limit.js';
+import { embedCardImage, EmbedApiError } from '../../../lib/card-embed.js';
+import { matchVisualInCatalog, catalogHasEmbeddings } from '../../../lib/card-visual-match.js';
+import { logScanAttempt } from '../../../lib/card-catalog-match.js';
+
+function formatCardResponse(card) {
+  return {
+    id: card.id,
+    name: card.name,
+    set_name: card.set_name,
+    set_code: card.set_code,
+    card_number: card.card_number,
+    game: card.game,
+    card_type: card.card_type,
+    rarity: card.rarity,
+    hp: card.power,
+    mana_cost: card.mana_cost,
+    image_url: card.image_url,
+    visual: {
+      similarity: card.sim,
+    },
+  };
+}
+
+export default async function handler(req, res) {
+  if (req.method !== 'POST') {
+    return res.status(405).json({ error: 'Method not allowed' });
+  }
+
+  const startedAt = Date.now();
+
+  try {
+    const user = await getUserFromRequest(req);
+    if (!user) {
+      return res.status(401).json({ error: 'Authentication required' });
+    }
+
+    const { imageData, game } = req.body || {};
+
+    if (!imageData || typeof imageData !== 'string') {
+      return res.status(400).json({ error: 'imageData is required' });
+    }
+
+    if (imageData.length > 6_000_000) {
+      return res.status(400).json({ error: 'Image payload too large' });
+    }
+
+    if (!(await catalogHasEmbeddings())) {
+      const latencyMs = Date.now() - startedAt;
+      await logScanAttempt({
+        userId: user.userId,
+        ocrText: null,
+        ocrConfidence: null,
+        layer: 0,
+        resultKind: 'escalate',
+        latencyMs,
+      });
+      return res.status(200).json({
+        layer: 0,
+        escalate: true,
+        reason: 'Catalog visual index is empty — run npm run backfill-embeddings',
+      });
+    }
+
+    const { allowed, reset } = await checkScanRateLimit(req, user.userId);
+    if (!allowed) {
+      res.setHeader('Retry-After', Math.ceil((reset - Date.now()) / 1000));
+      return res.status(429).json({ error: 'Too many attempts. Try again later.' });
+    }
+
+    const embedding = await embedCardImage(imageData);
+    const matchResult = await matchVisualInCatalog({ embedding, game: game || null });
+    const latencyMs = Date.now() - startedAt;
+
+    if (matchResult.type === 'escalate') {
+      await logScanAttempt({
+        userId: user.userId,
+        ocrText: null,
+        ocrConfidence: null,
+        layer: 0,
+        resultKind: 'escalate',
+        latencyMs,
+      });
+      return res.status(200).json({
+        layer: 0,
+        escalate: true,
+        reason: matchResult.reason,
+      });
+    }
+
+    if (matchResult.type === 'matched') {
+      await logScanAttempt({
+        userId: user.userId,
+        ocrText: null,
+        ocrConfidence: Math.round((matchResult.similarity || 0) * 100),
+        layer: 0,
+        matchedCardId: matchResult.card.id,
+        resultKind: 'matched',
+        latencyMs,
+      });
+      return res.status(200).json({
+        layer: 0,
+        escalate: false,
+        isCard: true,
+        card: formatCardResponse(matchResult.card),
+        isExisting: true,
+        message: matchResult.message,
+        visual: { similarity: matchResult.similarity },
+      });
+    }
+
+    await logScanAttempt({
+      userId: user.userId,
+      ocrText: null,
+      ocrConfidence: Math.round((matchResult.similarity || 0) * 100),
+      layer: 0,
+      resultKind: 'disambiguation',
+      latencyMs,
+    });
+
+    return res.status(200).json({
+      layer: 0,
+      escalate: false,
+      isCard: true,
+      card: null,
+      matches: matchResult.matches,
+      needsUserSelection: true,
+      message: matchResult.message,
+      visual: { similarity: matchResult.similarity },
+    });
+  } catch (error) {
+    console.error('[POST /api/scan/identify-by-image]', error);
+
+    if (error instanceof EmbedApiError) {
+      if (error.message.includes('AI_GATEWAY_API_KEY')) {
+        return res.status(503).json({
+          error: 'Visual matching is not configured on this server (missing AI_GATEWAY_API_KEY).',
+        });
+      }
+      if (error.status === 429) {
+        return res.status(502).json({
+          error: 'Embedding service quota exceeded. Try again later.',
+        });
+      }
+    }
+
+    if (String(error.message).includes('vector') || String(error.message).includes('pgvector')) {
+      return res.status(503).json({
+        error: 'Visual matching unavailable — run npm run migrate up (pgvector extension).',
+      });
+    }
+
+    return res.status(500).json({ error: 'Internal server error' });
+  }
+}

scripts/backfill-card-embeddings.js

@@ -0,0 +1,117 @@
+/**
+ * Backfill cards.embedding from cards.image_url via Vercel AI Gateway.
+ *
+ * Idempotent: skips rows where embedded_at is set unless FORCE=1.
+ *
+ * Usage:
+ *   POSTGRES_URL=<url> AI_GATEWAY_API_KEY=<key> node scripts/backfill-card-embeddings.js
+ *
+ * Options (env):
+ *   BATCH_SIZE   — rows per fetch (default 25)
+ *   SLEEP_MS     — delay between embed calls (default 250)
+ *   FORCE        — "1" to re-embed rows that already have embedded_at
+ *   LIMIT        — max rows to process (default unlimited)
+ *   DRY_RUN      — "true" to list candidates only
+ */
+
+import { neon } from '@neondatabase/serverless';
+
+import { embedCardImage, formatEmbeddingForPg } from '../lib/card-embed.js';
+
+if (!process.env.POSTGRES_URL) {
+  console.error('POSTGRES_URL is required');
+  process.exit(1);
+}
+
+if (!process.env.AI_GATEWAY_API_KEY) {
+  console.error('AI_GATEWAY_API_KEY is required');
+  process.exit(1);
+}
+
+const sql = neon(process.env.POSTGRES_URL, { fullResults: false });
+const BATCH_SIZE = Number(process.env.BATCH_SIZE || 25);
+const SLEEP_MS = Number(process.env.SLEEP_MS || 250);
+const FORCE = process.env.FORCE === '1';
+const LIMIT = process.env.LIMIT ? Number(process.env.LIMIT) : null;
+const DRY_RUN = process.env.DRY_RUN === 'true';
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function fetchCandidates(lastId) {
+  if (FORCE) {
+    return sql`
+      SELECT id, name, image_url
+      FROM cards
+      WHERE id > ${lastId}
+        AND image_url IS NOT NULL
+        AND image_url <> ''
+      ORDER BY id
+      LIMIT ${BATCH_SIZE}
+    `;
+  }
+
+  return sql`
+    SELECT id, name, image_url
+    FROM cards
+    WHERE id > ${lastId}
+      AND image_url IS NOT NULL
+      AND image_url <> ''
+      AND embedding IS NULL
+    ORDER BY id
+    LIMIT ${BATCH_SIZE}
+  `;
+}
+
+async function main() {
+  let lastId = 0;
+  let processed = 0;
+  let updated = 0;
+
+  console.log(`Backfill starting (force=${FORCE}, dryRun=${DRY_RUN})`);
+
+  while (true) {
+    if (LIMIT != null && processed >= LIMIT) break;
+
+    const rows = await fetchCandidates(lastId);
+    if (!rows.length) break;
+
+    for (const row of rows) {
+      if (LIMIT != null && processed >= LIMIT) break;
+      processed++;
+      lastId = row.id;
+
+      if (DRY_RUN) {
+        console.log(`[dry-run] would embed card ${row.id}: ${row.name}`);
+        continue;
+      }
+
+      try {
+        const embedding = await embedCardImage({ imageUrl: row.image_url });
+        const vectorLiteral = formatEmbeddingForPg(embedding);
+        await sql`
+          UPDATE cards
+          SET embedding = ${vectorLiteral}::vector,
+              embedded_at = CURRENT_TIMESTAMP
+          WHERE id = ${row.id}
+        `;
+        updated++;
+        console.log(`Embedded card ${row.id}: ${row.name}`);
+      } catch (error) {
+        console.error(`Failed card ${row.id} (${row.name}):`, error.message);
+      }
+
+      if (SLEEP_MS > 0) {
+        await sleep(SLEEP_MS);
+      }
+    }
+  }
+
+  console.log(`Done. processed=${processed} updated=${updated}`);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

test/lib/card-embed.test.js

@@ -0,0 +1,35 @@
+import { describe, expect, it, vi, afterEach } from 'vitest';
+import { embedCardImage, formatEmbeddingForPg } from '../../lib/card-embed.js';
+
+describe('formatEmbeddingForPg', () => {
+  it('formats vectors for pgvector literals', () => {
+    expect(formatEmbeddingForPg([0.1, 0.2, 0.3])).toBe('[0.10000000,0.20000000,0.30000000]');
+  });
+});
+
+describe('embedCardImage', () => {
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    delete process.env.AI_GATEWAY_API_KEY;
+  });
+
+  it('throws when AI_GATEWAY_API_KEY is missing', async () => {
+    await expect(embedCardImage('data:image/jpeg;base64,abc')).rejects.toThrow(
+      'AI_GATEWAY_API_KEY is not configured'
+    );
+  });
+
+  it('returns embedding values from the gateway response', async () => {
+    process.env.AI_GATEWAY_API_KEY = 'test-key';
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async () => ({
+        ok: true,
+        json: async () => ({ data: [{ embedding: [0.5, 0.25] }] }),
+      }))
+    );
+
+    const embedding = await embedCardImage('data:image/jpeg;base64,abc');
+    expect(embedding).toEqual([0.5, 0.25]);
+  });
+});

test/lib/card-visual-match.test.js

@@ -0,0 +1,40 @@
+import { describe, expect, it } from 'vitest';
+import { resolveVisualCandidates } from '../../lib/card-visual-match.js';
+
+describe('resolveVisualCandidates', () => {
+  const baseCard = {
+    id: 1,
+    name: 'Lightning Bolt',
+    set_name: 'Alpha',
+    set_code: 'lea',
+    card_number: '161',
+    game: 'mtg',
+    rarity: 'common',
+    image_url: 'https://example.com/bolt.jpg',
+    card_type: 'Instant',
+    mana_cost: '{R}',
+    power: null,
+  };
+
+  it('escalates when no candidates remain', () => {
+    expect(resolveVisualCandidates([]).type).toBe('escalate');
+  });
+
+  it('auto-matches a clear visual winner', () => {
+    const result = resolveVisualCandidates([
+      { ...baseCard, sim: 0.9 },
+      { ...baseCard, id: 2, sim: 0.7 },
+    ]);
+    expect(result.type).toBe('matched');
+    expect(result.card.id).toBe(1);
+  });
+
+  it('opens disambiguation when top matches are close', () => {
+    const result = resolveVisualCandidates([
+      { ...baseCard, sim: 0.8 },
+      { ...baseCard, id: 2, set_name: 'Beta', sim: 0.77 },
+    ]);
+    expect(result.type).toBe('disambiguation');
+    expect(result.matches).toHaveLength(2);
+  });
+});

test/lib/scanner-card-identify.test.js

@@ -54,6 +54,7 @@ describe('resolveIdentifyOutcome', () => {
       ocrMeta: expect.objectContaining({ abilities: [] }),
       message: 'Pick one',
       fromLayer1: false,
+      fromLayer0: false,
     });
   });