feat(scanner): Layer-0 visual catalog search (Phase 3) (#160)
Add pgvector embeddings on cards, server-side cohere/embed-v4.0 via AI Gateway, kNN identify route, and L0→L1→L2 client orchestration with empty-index fast escalate and id-cursor backfill job. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
varutasu
GitHub
parent
484bd02f9a
commit
8f09ed1ef6
17 changed files with 800 additions and 10 deletions
|
|
@ -101,3 +101,10 @@
|
|||
{"ts": "2026-08-15T01:24:40Z", "role": "role-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||
{"ts": "2026-08-15T01:24:41Z", "role": "role-security-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"}
|
||||
{"ts": "2026-08-15T01:24:41Z", "role": "role-a11y-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 60, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||
{"ts": "2026-08-15T01:35:27Z", "role": "role-ux-reviewer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T01:35:27Z", "role": "role-architect", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 360, "model": "composer-2.5", "model_tier": "standard"}
|
||||
{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 1, "duration_s": 300, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 2, "duration_s": 600, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T01:35:28Z", "role": "role-implementer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 3, "duration_s": 900, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T01:35:28Z", "role": "role-reviewer", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-scan-visual-catalog-search-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||
{"ts": "2026-08-15T01:35:28Z", "role": "role-security-auditor", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-scan-visual-catalog-search-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ skip:
|
|||
- a11y
|
||||
- design
|
||||
- flag
|
||||
status: open
|
||||
status: in-progress
|
||||
created: 2026-08-14
|
||||
depends_on:
|
||||
- improve-scan-card-detection
|
||||
|
|
@ -107,12 +107,13 @@ unknown-card fallback.
|
|||
|
||||
## Todos
|
||||
|
||||
- [ ] Architect: confirm `pgvector` on prod Neon tier
|
||||
- [ ] Brief 1 — migration + SCHEMA_MAP
|
||||
- [ ] Brief 2 — catalog backfill job (idempotent)
|
||||
- [ ] Brief 3 — identify kNN route + client escalate order
|
||||
- [x] Architect: confirm `pgvector` on prod Neon tier
|
||||
- [x] Brief 1 — migration + SCHEMA_MAP
|
||||
- [x] Brief 2 — catalog backfill job (idempotent)
|
||||
- [x] Brief 3 — identify kNN route + client escalate order
|
||||
- [ ] Threshold bake-off on real crops
|
||||
- [ ] Re-measure auto-match % excluding `not_a_card`
|
||||
- [ ] Operator: `npm run backfill-embeddings` on prod/staging after migrate
|
||||
|
||||
## Likely file ownership
|
||||
|
||||
|
|
@ -159,3 +160,37 @@ crops are rectified — that wastes the backfill. If Architect finds
|
|||
`pgvector` unavailable on CI Postgres, stop and write a fallback
|
||||
(external index vs skip-CI-extension plan) rather than shipping an
|
||||
untestable migration.
|
||||
|
||||
## UX
|
||||
|
||||
No new screens. Scan flow stays L0 → L1 → L2 with the same
|
||||
disambiguation picker and error toasts. When the catalog index is empty
|
||||
(backfill not run), L0 escalates silently with no embed cost.
|
||||
|
||||
Gallery uploads now try visual match before OCR.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Decision D1 — Gateway multimodal embedder (`cohere/embed-v4.0`)
|
||||
|
||||
Server-only via `AI_GATEWAY_API_KEY`. 1024-dim vectors in
|
||||
`cards.embedding`. Env: `SCAN_EMBED_MODEL`, `SCAN_EMBED_DIMENSION`.
|
||||
|
||||
### Decision D2 — Layer numbering
|
||||
|
||||
| Layer | Path |
|
||||
| --- | --- |
|
||||
| 0 | `POST /api/scan/identify-by-image` |
|
||||
| 1 | Tesseract + `identify-by-text` |
|
||||
| 2 | Gemini + `scan/identify` |
|
||||
|
||||
### Decision D3 — Thresholds
|
||||
|
||||
Match ≥ **0.82** (0.06 gap). Disambiguation ≥ **0.58**.
|
||||
|
||||
### Decision D4 — Rate limit
|
||||
|
||||
`checkScanRateLimit` on identify-by-image. L0 429 falls through to L1
|
||||
(not a hard stop).
|
||||
|
||||
Audit group id: `audit-scan-visual-catalog-search-<pr>`.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
convoy: scan-visual-catalog-search
|
||||
brief_number: 1
|
||||
depends_on: []
|
||||
recommended_model: composer-2.5
|
||||
model_tier: standard
|
||||
files:
|
||||
- migrations/1782000000001_add-card-embeddings.js
|
||||
- docs/SCHEMA_MAP.md
|
||||
---
|
||||
|
||||
# Brief 1: pgvector migration + SCHEMA_MAP
|
||||
|
||||
## Goal
|
||||
|
||||
Add `vector(1024)` embedding column + HNSW index on `cards`.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [ ] `CREATE EXTENSION IF NOT EXISTS vector`
|
||||
- [ ] `cards.embedding`, `cards.embedded_at`
|
||||
- [ ] `idx_cards_embedding_hnsw` partial index
|
||||
- [ ] SCHEMA_MAP documents layer 0 + pgvector
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
---
|
||||
convoy: scan-visual-catalog-search
|
||||
brief_number: 2
|
||||
depends_on: [1]
|
||||
recommended_model: composer-2.5-fast
|
||||
model_tier: fast
|
||||
files:
|
||||
- lib/card-embed.js
|
||||
- scripts/backfill-card-embeddings.js
|
||||
- package.json
|
||||
- test/lib/card-embed.test.js
|
||||
---
|
||||
|
||||
# Brief 2: Catalog embedding backfill
|
||||
|
||||
## Goal
|
||||
|
||||
Server-side embed job using `cohere/embed-v4.0` via AI Gateway; idempotent backfill from `cards.image_url`.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [ ] `lib/card-embed.js` exports `embedCardImage`, `formatEmbeddingForPg`
|
||||
- [ ] `npm run backfill-embeddings` script (skips rows with embedding unless FORCE=1)
|
||||
- [ ] Rate-limited (`SLEEP_MS`) and DRY_RUN support
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
---
|
||||
convoy: scan-visual-catalog-search
|
||||
brief_number: 3
|
||||
depends_on: [1]
|
||||
recommended_model: composer-2.5-fast
|
||||
model_tier: fast
|
||||
files:
|
||||
- lib/card-visual-match.js
|
||||
- pages/api/scan/identify-by-image.js
|
||||
- lib/scanner-card-identify.js
|
||||
- lib/use-scanner-identification.js
|
||||
- test/lib/card-visual-match.test.js
|
||||
---
|
||||
|
||||
# Brief 3: Layer-0 identify route + client orchestration
|
||||
|
||||
## Goal
|
||||
|
||||
kNN visual match before L1/L2; log `scan_attempts.layer = 0`.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [ ] `POST /api/scan/identify-by-image` — auth + `checkScanRateLimit` + embed + kNN
|
||||
- [ ] `identifyTrackedCardCapture` order: L0 → L1 → L2
|
||||
- [ ] Gallery path runs L0 → L1 → L2
|
||||
- [ ] Skip auto Gemini refine when disambiguation from L0 (same as L1)
|
||||
- [ ] Thresholds: match 0.82, disambiguation 0.58
|
||||
|
|
@ -93,16 +93,20 @@
|
|||
| `reprint` | `BOOLEAN` default `false` | (`1781440700404`) |
|
||||
| `finishes` | `JSONB` | `["nonfoil","foil","etched"]` (`1781440700404`) |
|
||||
| `created_at`, `updated_at` | `TIMESTAMP` default now | |
|
||||
| `embedding` | `vector(1024)` | Layer-0 visual kNN index (`1782000000001_add-card-embeddings`) |
|
||||
| `embedded_at` | `TIMESTAMP` | When `embedding` was last written by backfill |
|
||||
|
||||
**Indexes (post `1779853647565_add-pg-trgm-card-name-index` + `1781440700404`):**
|
||||
|
||||
- `idx_cards_name_trgm` — GIN on `name` using `gin_trgm_ops` for Layer-1 OCR fuzzy match (`similarity()` / `pg_trgm`).
|
||||
- `idx_cards_embedding_hnsw` — HNSW on `embedding vector_cosine_ops` (partial, non-null only) for Layer-0 visual search.
|
||||
- `cards_oracle_id_index`, `cards_illustration_id_index`, `cards_artist_index`, `cards_edhrec_rank_index`
|
||||
- GIN on `color_identity`, `keywords`, `legalities`
|
||||
|
||||
**Extensions used by scan pipeline:**
|
||||
|
||||
- `pg_trgm` — enabled by `1779853647565_add-pg-trgm-card-name-index.js` for trigram similarity on `cards.name`.
|
||||
- `pg_trgm` — Layer-1 text similarity (`1779853647565_add-pg-trgm-card-name-index`).
|
||||
- `vector` (pgvector) — Layer-0 visual embeddings (`1782000000001_add-card-embeddings`).
|
||||
|
||||
### user_cards
|
||||
|
||||
|
|
@ -261,7 +265,7 @@ Index: `idx_card_submissions_status (status, created_at DESC)`.
|
|||
|
||||
### scan_attempts
|
||||
|
||||
Per-scan telemetry for the identify pipeline (layer 1 = Tesseract + pg_trgm, layer 2 = Gemini).
|
||||
Per-scan telemetry for the identify pipeline (layer 0 = visual kNN, layer 1 = Tesseract + pg_trgm, layer 2 = Gemini).
|
||||
|
||||
| Column | Type | Notes |
|
||||
| --- | --- | --- |
|
||||
|
|
@ -269,7 +273,7 @@ Per-scan telemetry for the identify pipeline (layer 1 = Tesseract + pg_trgm, lay
|
|||
| `user_id` | FK → `users` SET NULL | |
|
||||
| `ocr_text` | `TEXT` | |
|
||||
| `ocr_confidence` | `INTEGER` | |
|
||||
| `layer` | `INTEGER` default `2` | Identify layer (`1` = browser Tesseract + pg_trgm, `2` = Gemini vision) |
|
||||
| `layer` | `INTEGER` default `2` | Identify layer (`0` = visual kNN, `1` = browser Tesseract + pg_trgm, `2` = Gemini vision) |
|
||||
| `matched_card_id` | FK → `cards` SET NULL | |
|
||||
| `result_kind` | `VARCHAR(32)` | e.g. `'matched'`, `'disambiguation'`, `'submitted'`, `'not_a_card'` |
|
||||
| `latency_ms` | `INTEGER` | End-to-end identify latency |
|
||||
|
|
|
|||
90
lib/card-embed.js
Normal file
90
lib/card-embed.js
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
const GATEWAY_EMBED_URL = 'https://ai-gateway.vercel.sh/v1/embeddings';
|
||||
const DEFAULT_EMBED_MODEL = process.env.SCAN_EMBED_MODEL || 'cohere/embed-v4.0';
|
||||
|
||||
/** Output dimension — must match migrations/1782000000001_add-card-embeddings.js. */
|
||||
export const EMBED_DIMENSION = Number(process.env.SCAN_EMBED_DIMENSION || 1024);
|
||||
|
||||
export class EmbedApiError extends Error {
|
||||
constructor(message, { status } = {}) {
|
||||
super(message);
|
||||
this.name = 'EmbedApiError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
function parseEmbeddingResponse(data) {
|
||||
const embedding =
|
||||
data?.data?.[0]?.embedding ||
|
||||
data?.embeddings?.[0]?.values ||
|
||||
data?.embeddings?.[0] ||
|
||||
data?.embedding;
|
||||
|
||||
if (!Array.isArray(embedding) || embedding.length === 0) {
|
||||
throw new EmbedApiError('Embedding API returned no vector');
|
||||
}
|
||||
|
||||
return embedding.map((value) => Number(value));
|
||||
}
|
||||
|
||||
function buildEmbedInput(value) {
|
||||
if (typeof value === 'string') {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (value?.imageUrl) {
|
||||
return {
|
||||
type: 'image_url',
|
||||
image_url: { url: value.imageUrl },
|
||||
};
|
||||
}
|
||||
|
||||
if (value?.imageDataUrl) {
|
||||
return value.imageDataUrl;
|
||||
}
|
||||
|
||||
throw new EmbedApiError('Invalid embed input');
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed image content via Vercel AI Gateway (server-only).
|
||||
* Accepts a JPEG data URL or HTTPS image URL object.
|
||||
*/
|
||||
export async function embedCardImage(input) {
|
||||
const apiKey = process.env.AI_GATEWAY_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new EmbedApiError('AI_GATEWAY_API_KEY is not configured on the server');
|
||||
}
|
||||
|
||||
const payloadInput = buildEmbedInput(input);
|
||||
if (typeof payloadInput === 'string' && !payloadInput.includes(',')) {
|
||||
throw new EmbedApiError('Invalid image data format');
|
||||
}
|
||||
|
||||
const response = await fetch(GATEWAY_EMBED_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: DEFAULT_EMBED_MODEL,
|
||||
input: payloadInput,
|
||||
dimensions: EMBED_DIMENSION,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}));
|
||||
const message = errorData.error?.message || errorData.message || 'Unknown error';
|
||||
throw new EmbedApiError(`Embedding API error: ${response.status} - ${message}`, {
|
||||
status: response.status,
|
||||
});
|
||||
}
|
||||
|
||||
return parseEmbeddingResponse(await response.json());
|
||||
}
|
||||
|
||||
/** Format a float array for pgvector tagged-template queries. */
|
||||
export function formatEmbeddingForPg(embedding) {
|
||||
return `[${embedding.map((value) => Number(value).toFixed(8)).join(',')}]`;
|
||||
}
|
||||
118
lib/card-visual-match.js
Normal file
118
lib/card-visual-match.js
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import { sql } from '@vercel/postgres';
|
||||
|
||||
import { formatEmbeddingForPg } from './card-embed.js';
|
||||
|
||||
export const MATCH_THRESHOLD = 0.82;
|
||||
export const DISAMBIGUATION_THRESHOLD = 0.58;
|
||||
|
||||
function mapCardRow(card) {
|
||||
return {
|
||||
id: card.id,
|
||||
name: card.name,
|
||||
set_name: card.set_name,
|
||||
set_code: card.set_code,
|
||||
card_number: card.card_number,
|
||||
game: card.game,
|
||||
rarity: card.rarity,
|
||||
image_url: card.image_url,
|
||||
card_type: card.card_type,
|
||||
mana_cost: card.mana_cost,
|
||||
hp: card.power,
|
||||
similarity: card.sim,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure ranking logic for visual kNN candidates (unit-tested without DB).
|
||||
*/
|
||||
export function resolveVisualCandidates(candidates) {
|
||||
if (!candidates?.length) {
|
||||
return {
|
||||
type: 'escalate',
|
||||
reason: `No catalog match above ${DISAMBIGUATION_THRESHOLD} visual similarity`,
|
||||
};
|
||||
}
|
||||
|
||||
const top = candidates[0];
|
||||
const runnerUp = candidates[1];
|
||||
const clearWinner =
|
||||
top.sim >= MATCH_THRESHOLD && (!runnerUp || top.sim - runnerUp.sim >= 0.06);
|
||||
|
||||
if (clearWinner) {
|
||||
return {
|
||||
type: 'matched',
|
||||
card: top,
|
||||
similarity: top.sim,
|
||||
message: `Matched "${top.name}" via visual search (${Math.round(top.sim * 100)}% similar)`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'disambiguation',
|
||||
matches: candidates.slice(0, 5).map(mapCardRow),
|
||||
similarity: top.sim,
|
||||
message: `Found ${candidates.length} visually similar printings. Select the correct one.`,
|
||||
};
|
||||
}
|
||||
|
||||
async function querySimilarCards(vectorLiteral, game) {
|
||||
if (game) {
|
||||
return sql`
|
||||
SELECT id, name, set_name, set_code, card_number, game, rarity, image_url, card_type, mana_cost, power,
|
||||
1 - (embedding <=> ${vectorLiteral}::vector) AS sim
|
||||
FROM cards
|
||||
WHERE embedding IS NOT NULL
|
||||
AND game = ${game}
|
||||
AND 1 - (embedding <=> ${vectorLiteral}::vector) > ${DISAMBIGUATION_THRESHOLD - 0.05}
|
||||
ORDER BY embedding <=> ${vectorLiteral}::vector
|
||||
LIMIT 8
|
||||
`;
|
||||
}
|
||||
|
||||
return sql`
|
||||
SELECT id, name, set_name, set_code, card_number, game, rarity, image_url, card_type, mana_cost, power,
|
||||
1 - (embedding <=> ${vectorLiteral}::vector) AS sim
|
||||
FROM cards
|
||||
WHERE embedding IS NOT NULL
|
||||
AND 1 - (embedding <=> ${vectorLiteral}::vector) > ${DISAMBIGUATION_THRESHOLD - 0.05}
|
||||
ORDER BY embedding <=> ${vectorLiteral}::vector
|
||||
LIMIT 8
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when at least one catalog row has a visual embedding index entry.
|
||||
*/
|
||||
export async function catalogHasEmbeddings() {
|
||||
const indexed = await sql`
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM cards WHERE embedding IS NOT NULL LIMIT 1
|
||||
) AS has_embeddings
|
||||
`;
|
||||
return Boolean(indexed.rows[0]?.has_embeddings);
|
||||
}
|
||||
|
||||
/**
|
||||
* kNN visual match against precomputed catalog embeddings (Layer 0).
|
||||
*/
|
||||
export async function matchVisualInCatalog({ embedding, game = null }) {
|
||||
if (!embedding?.length) {
|
||||
return {
|
||||
type: 'escalate',
|
||||
reason: 'Missing query embedding',
|
||||
};
|
||||
}
|
||||
|
||||
if (!(await catalogHasEmbeddings())) {
|
||||
return {
|
||||
type: 'escalate',
|
||||
reason: 'Catalog visual index is empty — run npm run backfill-embeddings',
|
||||
};
|
||||
}
|
||||
|
||||
const vectorLiteral = formatEmbeddingForPg(embedding);
|
||||
const result = await querySimilarCards(vectorLiteral, game || null);
|
||||
const candidates = result.rows.filter((row) => row.sim >= DISAMBIGUATION_THRESHOLD);
|
||||
|
||||
return resolveVisualCandidates(candidates);
|
||||
}
|
||||
|
|
@ -85,6 +85,7 @@ export function resolveIdentifyOutcome(result) {
|
|||
ocrMeta,
|
||||
message: result.message,
|
||||
fromLayer1: result.layer === 1,
|
||||
fromLayer0: result.layer === 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -242,6 +243,44 @@ export async function fetchIdentifyByText({ ocrText, ocrConfidence, cardNumber,
|
|||
return { ok: true, result };
|
||||
}
|
||||
|
||||
export async function fetchIdentifyByImage(imageData, authHeaders, game) {
|
||||
const response = await fetch('/api/scan/identify-by-image', {
|
||||
method: 'POST',
|
||||
headers: authHeaders,
|
||||
body: JSON.stringify({ imageData, game }),
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
return { ok: false, rateLimited: true };
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
return { ok: true, result };
|
||||
}
|
||||
|
||||
/**
|
||||
* Layer 0: visual kNN against precomputed catalog embeddings.
|
||||
*/
|
||||
export async function tryLayer0VisualIdentify(imageData, authHeaders, game) {
|
||||
const l0 = await fetchIdentifyByImage(imageData, authHeaders, game);
|
||||
if (!l0.ok) {
|
||||
return { handled: false };
|
||||
}
|
||||
|
||||
if (l0.result.escalate) {
|
||||
return { handled: false };
|
||||
}
|
||||
|
||||
return {
|
||||
handled: true,
|
||||
outcome: resolveIdentifyOutcome(l0.result),
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchVisionIdentify(imageData, authHeaders) {
|
||||
const response = await fetch('/api/scan/identify', {
|
||||
method: 'POST',
|
||||
|
|
@ -309,6 +348,15 @@ export async function identifyTrackedCardCapture({
|
|||
cardTracker.corners
|
||||
);
|
||||
|
||||
try {
|
||||
const l0 = await tryLayer0VisualIdentify(imageData, authHeaders);
|
||||
if (l0.handled) {
|
||||
return { imageData, ...l0 };
|
||||
}
|
||||
} catch (l0Error) {
|
||||
console.warn('Layer-0 visual path failed, falling back to text/vision:', l0Error);
|
||||
}
|
||||
|
||||
try {
|
||||
const l1 = await tryLayer1TextIdentify(imageData, authHeaders);
|
||||
if (l1.handled) {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ import {
|
|||
getScanAuthHeaders,
|
||||
identifyTrackedCardCapture,
|
||||
resolveDisambiguationRefineAction,
|
||||
resolveIdentifyOutcome,
|
||||
submitScanForReview,
|
||||
tryLayer0VisualIdentify,
|
||||
tryLayer1TextIdentify,
|
||||
VISION_RATE_LIMIT_MS,
|
||||
fetchVisionIdentify,
|
||||
|
|
@ -132,6 +134,7 @@ export function useScannerIdentification({
|
|||
ocrMeta: outcome.ocrMeta,
|
||||
message: outcome.message,
|
||||
fromLayer1: Boolean(outcome.fromLayer1),
|
||||
fromLayer0: Boolean(outcome.fromLayer0),
|
||||
});
|
||||
break;
|
||||
case 'notice':
|
||||
|
|
@ -180,7 +183,7 @@ export function useScannerIdentification({
|
|||
|
||||
useEffect(() => {
|
||||
if (!disambiguation?.imageData) return;
|
||||
if (disambiguation.fromLayer1) return;
|
||||
if (disambiguation.fromLayer1 || disambiguation.fromLayer0) return;
|
||||
if (Date.now() < visionCooldownUntilRef.current) return;
|
||||
|
||||
const refineKey = disambiguation.cardTracker?.id ?? 'modal';
|
||||
|
|
@ -326,14 +329,42 @@ export function useScannerIdentification({
|
|||
try {
|
||||
const imageData = await readFileToImageData(file);
|
||||
const authHeaders = getScanAuthHeaders();
|
||||
const result = await tryLayer1TextIdentify(imageData, authHeaders);
|
||||
const syntheticTracker = { id: `gallery-${Date.now()}`, status: 'verifying' };
|
||||
|
||||
const l0 = await tryLayer0VisualIdentify(imageData, authHeaders);
|
||||
if (l0.handled && l0.outcome) {
|
||||
await applyIdentifyOutcome(syntheticTracker, imageData, l0.outcome);
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await tryLayer1TextIdentify(imageData, authHeaders);
|
||||
|
||||
if (result.handled && result.outcome) {
|
||||
await applyIdentifyOutcome(syntheticTracker, imageData, result.outcome);
|
||||
return;
|
||||
}
|
||||
|
||||
if (Date.now() < visionCooldownUntilRef.current) {
|
||||
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
||||
return;
|
||||
}
|
||||
|
||||
const vision = await fetchVisionIdentify(imageData, authHeaders);
|
||||
if (vision.rateLimited) {
|
||||
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
|
||||
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
||||
return;
|
||||
}
|
||||
|
||||
if (vision.ok && vision.result) {
|
||||
await applyIdentifyOutcome(
|
||||
syntheticTracker,
|
||||
imageData,
|
||||
resolveIdentifyOutcome(vision.result)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
reportScannerError('Could not identify card from gallery image');
|
||||
} catch (error) {
|
||||
reportScannerError(error.message || 'Gallery identify failed');
|
||||
|
|
|
|||
33
migrations/1782000000001_add-card-embeddings.js
Normal file
33
migrations/1782000000001_add-card-embeddings.js
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
/**
|
||||
* pgvector + cards.embedding for Layer-0 visual catalog search.
|
||||
*
|
||||
* @type {import('node-pg-migrate').ColumnDefinitions | undefined}
|
||||
*/
|
||||
export const shorthands = undefined;
|
||||
|
||||
/** Must match lib/card-embed.js EMBED_DIMENSION. */
|
||||
const EMBED_DIMENSION = 1024;
|
||||
|
||||
/**
|
||||
* @param {import('node-pg-migrate').MigrationBuilder} pgm
|
||||
*/
|
||||
export const up = (pgm) => {
|
||||
pgm.sql(`
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
|
||||
ALTER TABLE cards
|
||||
ADD COLUMN IF NOT EXISTS embedding vector(${EMBED_DIMENSION}),
|
||||
ADD COLUMN IF NOT EXISTS embedded_at TIMESTAMP;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cards_embedding_hnsw
|
||||
ON cards USING hnsw (embedding vector_cosine_ops)
|
||||
WHERE embedding IS NOT NULL;
|
||||
`);
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {import('node-pg-migrate').MigrationBuilder} pgm
|
||||
*/
|
||||
export const down = (pgm) => {
|
||||
throw new Error('Down migration not supported for add-card-embeddings');
|
||||
};
|
||||
|
|
@ -15,6 +15,7 @@
|
|||
"bulk-import": "node --env-file=.env.local scripts/bulk-import-scryfall.js",
|
||||
"bulk-import-lorcana": "node --env-file=.env.local scripts/bulk-import-lorcana.js",
|
||||
"bulk-import-pokemon": "node --env-file=.env.local scripts/bulk-import-pokemon.js",
|
||||
"backfill-embeddings": "node --env-file=.env.local scripts/backfill-card-embeddings.js",
|
||||
"import-tags": "node --env-file=.env.local scripts/import-scryfall-tags.js",
|
||||
"test": "vitest",
|
||||
"test:run": "vitest run",
|
||||
|
|
|
|||
156
pages/api/scan/identify-by-image.js
Normal file
156
pages/api/scan/identify-by-image.js
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
import { getUserFromRequest } from '../../../lib/permission-middleware';
|
||||
import { checkScanRateLimit } from '../../../lib/rate-limit.js';
|
||||
import { embedCardImage, EmbedApiError } from '../../../lib/card-embed.js';
|
||||
import { matchVisualInCatalog, catalogHasEmbeddings } from '../../../lib/card-visual-match.js';
|
||||
import { logScanAttempt } from '../../../lib/card-catalog-match.js';
|
||||
|
||||
function formatCardResponse(card) {
|
||||
return {
|
||||
id: card.id,
|
||||
name: card.name,
|
||||
set_name: card.set_name,
|
||||
set_code: card.set_code,
|
||||
card_number: card.card_number,
|
||||
game: card.game,
|
||||
card_type: card.card_type,
|
||||
rarity: card.rarity,
|
||||
hp: card.power,
|
||||
mana_cost: card.mana_cost,
|
||||
image_url: card.image_url,
|
||||
visual: {
|
||||
similarity: card.sim,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export default async function handler(req, res) {
|
||||
if (req.method !== 'POST') {
|
||||
return res.status(405).json({ error: 'Method not allowed' });
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
|
||||
try {
|
||||
const user = await getUserFromRequest(req);
|
||||
if (!user) {
|
||||
return res.status(401).json({ error: 'Authentication required' });
|
||||
}
|
||||
|
||||
const { imageData, game } = req.body || {};
|
||||
|
||||
if (!imageData || typeof imageData !== 'string') {
|
||||
return res.status(400).json({ error: 'imageData is required' });
|
||||
}
|
||||
|
||||
if (imageData.length > 6_000_000) {
|
||||
return res.status(400).json({ error: 'Image payload too large' });
|
||||
}
|
||||
|
||||
if (!(await catalogHasEmbeddings())) {
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
await logScanAttempt({
|
||||
userId: user.userId,
|
||||
ocrText: null,
|
||||
ocrConfidence: null,
|
||||
layer: 0,
|
||||
resultKind: 'escalate',
|
||||
latencyMs,
|
||||
});
|
||||
return res.status(200).json({
|
||||
layer: 0,
|
||||
escalate: true,
|
||||
reason: 'Catalog visual index is empty — run npm run backfill-embeddings',
|
||||
});
|
||||
}
|
||||
|
||||
const { allowed, reset } = await checkScanRateLimit(req, user.userId);
|
||||
if (!allowed) {
|
||||
res.setHeader('Retry-After', Math.ceil((reset - Date.now()) / 1000));
|
||||
return res.status(429).json({ error: 'Too many attempts. Try again later.' });
|
||||
}
|
||||
|
||||
const embedding = await embedCardImage(imageData);
|
||||
const matchResult = await matchVisualInCatalog({ embedding, game: game || null });
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
|
||||
if (matchResult.type === 'escalate') {
|
||||
await logScanAttempt({
|
||||
userId: user.userId,
|
||||
ocrText: null,
|
||||
ocrConfidence: null,
|
||||
layer: 0,
|
||||
resultKind: 'escalate',
|
||||
latencyMs,
|
||||
});
|
||||
return res.status(200).json({
|
||||
layer: 0,
|
||||
escalate: true,
|
||||
reason: matchResult.reason,
|
||||
});
|
||||
}
|
||||
|
||||
if (matchResult.type === 'matched') {
|
||||
await logScanAttempt({
|
||||
userId: user.userId,
|
||||
ocrText: null,
|
||||
ocrConfidence: Math.round((matchResult.similarity || 0) * 100),
|
||||
layer: 0,
|
||||
matchedCardId: matchResult.card.id,
|
||||
resultKind: 'matched',
|
||||
latencyMs,
|
||||
});
|
||||
return res.status(200).json({
|
||||
layer: 0,
|
||||
escalate: false,
|
||||
isCard: true,
|
||||
card: formatCardResponse(matchResult.card),
|
||||
isExisting: true,
|
||||
message: matchResult.message,
|
||||
visual: { similarity: matchResult.similarity },
|
||||
});
|
||||
}
|
||||
|
||||
await logScanAttempt({
|
||||
userId: user.userId,
|
||||
ocrText: null,
|
||||
ocrConfidence: Math.round((matchResult.similarity || 0) * 100),
|
||||
layer: 0,
|
||||
resultKind: 'disambiguation',
|
||||
latencyMs,
|
||||
});
|
||||
|
||||
return res.status(200).json({
|
||||
layer: 0,
|
||||
escalate: false,
|
||||
isCard: true,
|
||||
card: null,
|
||||
matches: matchResult.matches,
|
||||
needsUserSelection: true,
|
||||
message: matchResult.message,
|
||||
visual: { similarity: matchResult.similarity },
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[POST /api/scan/identify-by-image]', error);
|
||||
|
||||
if (error instanceof EmbedApiError) {
|
||||
if (error.message.includes('AI_GATEWAY_API_KEY')) {
|
||||
return res.status(503).json({
|
||||
error: 'Visual matching is not configured on this server (missing AI_GATEWAY_API_KEY).',
|
||||
});
|
||||
}
|
||||
if (error.status === 429) {
|
||||
return res.status(502).json({
|
||||
error: 'Embedding service quota exceeded. Try again later.',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (String(error.message).includes('vector') || String(error.message).includes('pgvector')) {
|
||||
return res.status(503).json({
|
||||
error: 'Visual matching unavailable — run npm run migrate up (pgvector extension).',
|
||||
});
|
||||
}
|
||||
|
||||
return res.status(500).json({ error: 'Internal server error' });
|
||||
}
|
||||
}
|
||||
117
scripts/backfill-card-embeddings.js
Normal file
117
scripts/backfill-card-embeddings.js
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
/**
|
||||
* Backfill cards.embedding from cards.image_url via Vercel AI Gateway.
|
||||
*
|
||||
* Idempotent: skips rows where embedded_at is set unless FORCE=1.
|
||||
*
|
||||
* Usage:
|
||||
* POSTGRES_URL=<url> AI_GATEWAY_API_KEY=<key> node scripts/backfill-card-embeddings.js
|
||||
*
|
||||
* Options (env):
|
||||
* BATCH_SIZE — rows per fetch (default 25)
|
||||
* SLEEP_MS — delay between embed calls (default 250)
|
||||
* FORCE — "1" to re-embed rows that already have embedded_at
|
||||
* LIMIT — max rows to process (default unlimited)
|
||||
* DRY_RUN — "true" to list candidates only
|
||||
*/
|
||||
|
||||
import { neon } from '@neondatabase/serverless';
|
||||
|
||||
import { embedCardImage, formatEmbeddingForPg } from '../lib/card-embed.js';
|
||||
|
||||
if (!process.env.POSTGRES_URL) {
|
||||
console.error('POSTGRES_URL is required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!process.env.AI_GATEWAY_API_KEY) {
|
||||
console.error('AI_GATEWAY_API_KEY is required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const sql = neon(process.env.POSTGRES_URL, { fullResults: false });
|
||||
const BATCH_SIZE = Number(process.env.BATCH_SIZE || 25);
|
||||
const SLEEP_MS = Number(process.env.SLEEP_MS || 250);
|
||||
const FORCE = process.env.FORCE === '1';
|
||||
const LIMIT = process.env.LIMIT ? Number(process.env.LIMIT) : null;
|
||||
const DRY_RUN = process.env.DRY_RUN === 'true';
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function fetchCandidates(lastId) {
|
||||
if (FORCE) {
|
||||
return sql`
|
||||
SELECT id, name, image_url
|
||||
FROM cards
|
||||
WHERE id > ${lastId}
|
||||
AND image_url IS NOT NULL
|
||||
AND image_url <> ''
|
||||
ORDER BY id
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`;
|
||||
}
|
||||
|
||||
return sql`
|
||||
SELECT id, name, image_url
|
||||
FROM cards
|
||||
WHERE id > ${lastId}
|
||||
AND image_url IS NOT NULL
|
||||
AND image_url <> ''
|
||||
AND embedding IS NULL
|
||||
ORDER BY id
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
let lastId = 0;
|
||||
let processed = 0;
|
||||
let updated = 0;
|
||||
|
||||
console.log(`Backfill starting (force=${FORCE}, dryRun=${DRY_RUN})`);
|
||||
|
||||
while (true) {
|
||||
if (LIMIT != null && processed >= LIMIT) break;
|
||||
|
||||
const rows = await fetchCandidates(lastId);
|
||||
if (!rows.length) break;
|
||||
|
||||
for (const row of rows) {
|
||||
if (LIMIT != null && processed >= LIMIT) break;
|
||||
processed++;
|
||||
lastId = row.id;
|
||||
|
||||
if (DRY_RUN) {
|
||||
console.log(`[dry-run] would embed card ${row.id}: ${row.name}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const embedding = await embedCardImage({ imageUrl: row.image_url });
|
||||
const vectorLiteral = formatEmbeddingForPg(embedding);
|
||||
await sql`
|
||||
UPDATE cards
|
||||
SET embedding = ${vectorLiteral}::vector,
|
||||
embedded_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ${row.id}
|
||||
`;
|
||||
updated++;
|
||||
console.log(`Embedded card ${row.id}: ${row.name}`);
|
||||
} catch (error) {
|
||||
console.error(`Failed card ${row.id} (${row.name}):`, error.message);
|
||||
}
|
||||
|
||||
if (SLEEP_MS > 0) {
|
||||
await sleep(SLEEP_MS);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Done. processed=${processed} updated=${updated}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
35
test/lib/card-embed.test.js
Normal file
35
test/lib/card-embed.test.js
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { describe, expect, it, vi, afterEach } from 'vitest';
|
||||
import { embedCardImage, formatEmbeddingForPg } from '../../lib/card-embed.js';
|
||||
|
||||
describe('formatEmbeddingForPg', () => {
|
||||
it('formats vectors for pgvector literals', () => {
|
||||
expect(formatEmbeddingForPg([0.1, 0.2, 0.3])).toBe('[0.10000000,0.20000000,0.30000000]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('embedCardImage', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
delete process.env.AI_GATEWAY_API_KEY;
|
||||
});
|
||||
|
||||
it('throws when AI_GATEWAY_API_KEY is missing', async () => {
|
||||
await expect(embedCardImage('data:image/jpeg;base64,abc')).rejects.toThrow(
|
||||
'AI_GATEWAY_API_KEY is not configured'
|
||||
);
|
||||
});
|
||||
|
||||
it('returns embedding values from the gateway response', async () => {
|
||||
process.env.AI_GATEWAY_API_KEY = 'test-key';
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(async () => ({
|
||||
ok: true,
|
||||
json: async () => ({ data: [{ embedding: [0.5, 0.25] }] }),
|
||||
}))
|
||||
);
|
||||
|
||||
const embedding = await embedCardImage('data:image/jpeg;base64,abc');
|
||||
expect(embedding).toEqual([0.5, 0.25]);
|
||||
});
|
||||
});
|
||||
40
test/lib/card-visual-match.test.js
Normal file
40
test/lib/card-visual-match.test.js
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { resolveVisualCandidates } from '../../lib/card-visual-match.js';
|
||||
|
||||
describe('resolveVisualCandidates', () => {
|
||||
const baseCard = {
|
||||
id: 1,
|
||||
name: 'Lightning Bolt',
|
||||
set_name: 'Alpha',
|
||||
set_code: 'lea',
|
||||
card_number: '161',
|
||||
game: 'mtg',
|
||||
rarity: 'common',
|
||||
image_url: 'https://example.com/bolt.jpg',
|
||||
card_type: 'Instant',
|
||||
mana_cost: '{R}',
|
||||
power: null,
|
||||
};
|
||||
|
||||
it('escalates when no candidates remain', () => {
|
||||
expect(resolveVisualCandidates([]).type).toBe('escalate');
|
||||
});
|
||||
|
||||
it('auto-matches a clear visual winner', () => {
|
||||
const result = resolveVisualCandidates([
|
||||
{ ...baseCard, sim: 0.9 },
|
||||
{ ...baseCard, id: 2, sim: 0.7 },
|
||||
]);
|
||||
expect(result.type).toBe('matched');
|
||||
expect(result.card.id).toBe(1);
|
||||
});
|
||||
|
||||
it('opens disambiguation when top matches are close', () => {
|
||||
const result = resolveVisualCandidates([
|
||||
{ ...baseCard, sim: 0.8 },
|
||||
{ ...baseCard, id: 2, set_name: 'Beta', sim: 0.77 },
|
||||
]);
|
||||
expect(result.type).toBe('disambiguation');
|
||||
expect(result.matches).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
|
@ -54,6 +54,7 @@ describe('resolveIdentifyOutcome', () => {
|
|||
ocrMeta: expect.objectContaining({ abilities: [] }),
|
||||
message: 'Pick one',
|
||||
fromLayer1: false,
|
||||
fromLayer0: false,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue