Scanner identify upgrade — Phase 1 hot path (#156)
* docs(convoy): seed scanner identify upgrade epic and sub-convoys Baseline scan_attempts telemetry and three-phase plan for faster, more accurate card identification without touching scanner chrome. Co-authored-by: Cursor <cursoragent@cursor.com> * feat(scanner): tighten Layer-1 identify hot path (Phase 1) Cut verify hold-still gates, OCR collector numbers on Layer 1, request structured Gemini JSON, and skip automatic L2 refine when L1 opens the printing picker. Includes convoy UX/architecture briefs and unit tests. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
varutasu
GitHub
parent
73424aae59
commit
0b4f419f49
22 changed files with 1263 additions and 74 deletions
|
|
@ -79,6 +79,10 @@
|
||||||
{"ts": "2026-08-15T00:14:27Z", "role": "role-ia-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 95, "model": "composer-2.5-fast", "model_tier": "fast"}
|
{"ts": "2026-08-15T00:14:27Z", "role": "role-ia-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 95, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
{"ts": "2026-08-15T00:15:31Z", "role": "role-ui-designer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"}
|
{"ts": "2026-08-15T00:15:31Z", "role": "role-ui-designer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
{"ts": "2026-08-15T00:16:36Z", "role": "role-ux-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 180, "model": "composer-2.5-fast", "model_tier": "fast"}
|
{"ts": "2026-08-15T00:16:36Z", "role": "role-ux-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 180, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
|
{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "scanner-identify-upgrade", "repo": "scanner-identify-upgrade", "skip_flags": [], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
|
{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "visual", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
|
{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
|
{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "visual", "a11y", "design", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
{"ts": "2026-08-15T00:18:40Z", "role": "role-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 420, "model": "composer-2.5", "model_tier": "standard"}
|
{"ts": "2026-08-15T00:18:40Z", "role": "role-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 420, "model": "composer-2.5", "model_tier": "standard"}
|
||||||
{"ts": "2026-08-15T00:19:56Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
{"ts": "2026-08-15T00:19:56Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
{"ts": "2026-08-15T00:20:23Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 2, "duration_s": 120, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
{"ts": "2026-08-15T00:20:23Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 2, "duration_s": 120, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
|
|
@ -88,4 +92,5 @@
|
||||||
{"ts": "2026-08-15T00:25:54Z", "role": "role-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 210, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
{"ts": "2026-08-15T00:25:54Z", "role": "role-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 210, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||||
{"ts": "2026-08-15T00:26:05Z", "role": "role-a11y-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 78, "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
{"ts": "2026-08-15T00:26:05Z", "role": "role-a11y-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 78, "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||||
{"ts": "2026-08-15T00:26:08Z", "role": "role-design-system-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 82, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
{"ts": "2026-08-15T00:26:08Z", "role": "role-design-system-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 82, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||||
|
{"ts": "2026-08-15T00:55:56Z", "role": "role-reviewer", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 0, "duration_s": 180, "multitask_group": "audit-tighten-scan-identify-hot-path-uncommitted", "model": "cursor-grok-4.5-high", "model_tier": "fast"}
|
||||||
{"ts": "2026-08-15T00:56:39Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
{"ts": "2026-08-15T00:56:39Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||||
|
|
|
||||||
151
.convoys/improve-scan-card-detection.md
Normal file
151
.convoys/improve-scan-card-detection.md
Normal file
|
|
@ -0,0 +1,151 @@
|
||||||
|
---
|
||||||
|
name: improve-scan-card-detection
|
||||||
|
classification: feature
|
||||||
|
success_metric: |
|
||||||
|
After ship, L2 result_kind=not_a_card ≤15% of L2 (was 44.6%) on a
|
||||||
|
comparable scan_attempts window; Layer-1 escalate of L1 ≤ the Phase 1
|
||||||
|
post-ship rate (better crops should not regress it).
|
||||||
|
skip:
|
||||||
|
- ia
|
||||||
|
- ui-design
|
||||||
|
- flag
|
||||||
|
status: open
|
||||||
|
created: 2026-08-14
|
||||||
|
depends_on:
|
||||||
|
- tighten-scan-identify-hot-path
|
||||||
|
umbrella: scanner-identify-upgrade
|
||||||
|
model_policy:
|
||||||
|
default_session: auto
|
||||||
|
roles:
|
||||||
|
role-conductor: composer-2.5-fast
|
||||||
|
role-architect: composer-2.5
|
||||||
|
role-ia-architect: composer-2.5-fast
|
||||||
|
role-ux-reviewer: composer-2.5-fast
|
||||||
|
role-ui-designer: composer-2.5-fast
|
||||||
|
role-implementer: composer-2.5-fast
|
||||||
|
role-reviewer: cursor-grok-4.5-high
|
||||||
|
role-security-auditor: gpt-5.6-terra-medium
|
||||||
|
role-design-system-auditor: cursor-grok-4.5-high
|
||||||
|
role-a11y-auditor: cursor-grok-4.5-high
|
||||||
|
role-doc-writer: auto
|
||||||
|
escalate_to: claude-sonnet-5-thinking-medium
|
||||||
|
escalate_to_premium: claude-4.6-opus-high-thinking
|
||||||
|
never_premium:
|
||||||
|
- role-reviewer
|
||||||
|
- role-security-auditor
|
||||||
|
- role-design-system-auditor
|
||||||
|
- role-a11y-auditor
|
||||||
|
- role-ui-designer
|
||||||
|
- role-doc-writer
|
||||||
|
---
|
||||||
|
|
||||||
|
# Convoy: improve-scan-card-detection
|
||||||
|
|
||||||
|
Phase 2 of `scanner-identify-upgrade`. Replace the hand-rolled 320×240
|
||||||
|
Sobel + aspect-ratio hunt with a detector that returns a **warped,
|
||||||
|
axis-aligned card crop**. Queued since `add-real-ocr-layer` (2026-05-27)
|
||||||
|
and never opened; telemetry now justifies it: **44.6% of Gemini calls
|
||||||
|
are `not_a_card`**.
|
||||||
|
|
||||||
|
`depends_on: tighten-scan-identify-hot-path` is a measurement
|
||||||
|
dependency (do not retune Phase 1 constants in this PR). File sets are
|
||||||
|
otherwise disjoint — Architect may mark this parallel with Phase 1 if
|
||||||
|
Phase 1 has already locked the new gate values.
|
||||||
|
|
||||||
|
## Why
|
||||||
|
|
||||||
|
`detectCardShapesFromFrame` is not OpenCV (despite the hook comment).
|
||||||
|
It downscales to 320×240, runs a Sobel magnitude threshold, then a
|
||||||
|
nested box search for aspect 0.63–0.77. There is no four-corner
|
||||||
|
homography. Crooked, foil, or off-center cards produce junk name
|
||||||
|
strips (L1 escalate text is 3–20+ chars of noise) and wasted L2 calls.
|
||||||
|
|
||||||
|
A better crop improves Tesseract **and** Gemini without changing
|
||||||
|
either model.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
### In scope
|
||||||
|
|
||||||
|
- Replace or wrap `detectCardShapesFromFrame` so tracked bounds are
|
||||||
|
card-shaped **and** a perspective-corrected JPEG can be produced for
|
||||||
|
identify.
|
||||||
|
- Architect picks **one** in-browser approach (do not land both):
|
||||||
|
1. **OpenCV.js contours + approxPolyDP + warpPerspective** — no
|
||||||
|
training, larger WASM.
|
||||||
|
2. **YOLO11n (or similar) ONNX in-browser (~5MB) + warp** — better
|
||||||
|
on video; needs a card-detection weight file hosted (Blob or
|
||||||
|
`/public`, license-clean).
|
||||||
|
- Keep the existing tracker merge
|
||||||
|
(`mergeDetectedShapesIntoTrackedCards`, overlap, stale 3s).
|
||||||
|
- Overlay brackets in `components/scanner/ScannerCamera.js` keep
|
||||||
|
consuming `{x,y,width,height}` (or four corners mapped to a rect).
|
||||||
|
Do not redesign the overlay.
|
||||||
|
- Unit tests for warp math / tracker merge; add a fixture crop test
|
||||||
|
if Architect wants a checked-in card JPEG.
|
||||||
|
|
||||||
|
### Out of scope
|
||||||
|
|
||||||
|
- Identify / Gemini / Tesseract / catalog match (Phase 1 + 3).
|
||||||
|
- Python/CUDA microservice, PaddleOCR server, Roboflow-hosted detect.
|
||||||
|
- Scanner chrome / cart (`scanner-mobile-checkout`).
|
||||||
|
- Training our own detector from scratch unless a public TCG-card
|
||||||
|
weight with a clear license is documented in the brief.
|
||||||
|
|
||||||
|
## Roles invoked
|
||||||
|
|
||||||
|
1. `role-ux-reviewer` — time-to-bracket, false-positive boxes, dual-card
|
||||||
|
frames.
|
||||||
|
2. `role-architect` — OpenCV.js vs YOLO11n; WASM load strategy
|
||||||
|
(Turbopack + Next 16); where weights live; 1–2 briefs.
|
||||||
|
3. `role-implementer`.
|
||||||
|
4. Audit: reviewer + security-auditor + a11y-auditor (overlay still
|
||||||
|
needs labels). Design-system only if overlay styling changes.
|
||||||
|
|
||||||
|
## Todos
|
||||||
|
|
||||||
|
- [ ] Architect: pick detector; document WASM / weight budget
|
||||||
|
- [ ] Brief 1 — detect + warp library + tests
|
||||||
|
- [ ] Brief 2 — wire `use-camera-scanner.js` + crop used by
|
||||||
|
`identifyTrackedCardCapture`
|
||||||
|
- [ ] Confirm `scanner-mobile-checkout` overlay still maps bounds
|
||||||
|
- [ ] Re-measure L2 `not_a_card` share
|
||||||
|
|
||||||
|
## Likely file ownership
|
||||||
|
|
||||||
|
| Area | Files |
|
||||||
|
| --- | --- |
|
||||||
|
| Detect | `lib/scanner-card-detection.js`, `test/lib/scanner-card-detection.test.js` |
|
||||||
|
| Warp helper | new `lib/scanner-card-warp.js` (if Architect splits) |
|
||||||
|
| Camera loop | `lib/use-camera-scanner.js` |
|
||||||
|
| Crop consumer | `lib/scanner-card-identify.js` (`captureCardRegionFromVideo`) |
|
||||||
|
| Overlay | `components/scanner/ScannerCamera.js` only if bounds shape changes |
|
||||||
|
|
||||||
|
Do not edit `lib/ocr-worker.js` or `lib/scan-vision.js` here.
|
||||||
|
|
||||||
|
## Multitask dispatch
|
||||||
|
|
||||||
|
Serial unless Architect splits detect-lib vs camera-wire with disjoint
|
||||||
|
files.
|
||||||
|
|
||||||
|
Audit group id: `audit-improve-scan-card-detection-<pr>`.
|
||||||
|
|
||||||
|
## CI impact
|
||||||
|
|
||||||
|
| Workflow / job | Behavior |
|
||||||
|
| --- | --- |
|
||||||
|
| `visual-diff.yml` | **May fire** if `ScannerCamera.js` changes |
|
||||||
|
| `preview-smoke.yml` | Fires |
|
||||||
|
| `ci.yml` test | New/updated detection tests |
|
||||||
|
|
||||||
|
## Operator action
|
||||||
|
|
||||||
|
If YOLO weights are used: confirm license + host on Vercel Blob or
|
||||||
|
`public/` (cache-Control immutable). No new secrets.
|
||||||
|
|
||||||
|
## Conductor notes
|
||||||
|
|
||||||
|
Do not drop OpenCV.js in only to reimplement the current rectangle
|
||||||
|
hunt. Success is a **rectified card image**, not a prettier box.
|
||||||
|
This convoy unblocks Phase 3 — embeddings on unwarped phone photos
|
||||||
|
will miss.
|
||||||
161
.convoys/scan-visual-catalog-search.md
Normal file
161
.convoys/scan-visual-catalog-search.md
Normal file
|
|
@ -0,0 +1,161 @@
|
||||||
|
---
|
||||||
|
name: scan-visual-catalog-search
|
||||||
|
classification: feature
|
||||||
|
success_metric: |
|
||||||
|
After catalog embeddings exist and the new kNN path is on, ≥50% of
|
||||||
|
legitimate card scans (scan_attempts excluding not_a_card) auto-match
|
||||||
|
a printing with no picker — the original add-real-ocr-layer 70%
|
||||||
|
target, measured honestly. Gemini L2 share of attempts falls vs the
|
||||||
|
Phase 1 post-ship baseline.
|
||||||
|
skip:
|
||||||
|
- ia
|
||||||
|
- ui-design
|
||||||
|
- visual
|
||||||
|
- a11y
|
||||||
|
- design
|
||||||
|
- flag
|
||||||
|
status: open
|
||||||
|
created: 2026-08-14
|
||||||
|
depends_on:
|
||||||
|
- improve-scan-card-detection
|
||||||
|
umbrella: scanner-identify-upgrade
|
||||||
|
model_policy:
|
||||||
|
default_session: auto
|
||||||
|
roles:
|
||||||
|
role-conductor: composer-2.5-fast
|
||||||
|
role-architect: composer-2.5
|
||||||
|
role-ia-architect: composer-2.5-fast
|
||||||
|
role-ux-reviewer: composer-2.5-fast
|
||||||
|
role-ui-designer: composer-2.5-fast
|
||||||
|
role-implementer: composer-2.5-fast
|
||||||
|
role-reviewer: cursor-grok-4.5-high
|
||||||
|
role-security-auditor: gpt-5.6-terra-medium
|
||||||
|
role-design-system-auditor: cursor-grok-4.5-high
|
||||||
|
role-a11y-auditor: cursor-grok-4.5-high
|
||||||
|
role-doc-writer: auto
|
||||||
|
escalate_to: claude-sonnet-5-thinking-medium
|
||||||
|
escalate_to_premium: claude-4.6-opus-high-thinking
|
||||||
|
never_premium:
|
||||||
|
- role-reviewer
|
||||||
|
- role-security-auditor
|
||||||
|
- role-design-system-auditor
|
||||||
|
- role-a11y-auditor
|
||||||
|
- role-ui-designer
|
||||||
|
- role-doc-writer
|
||||||
|
---
|
||||||
|
|
||||||
|
# Convoy: scan-visual-catalog-search
|
||||||
|
|
||||||
|
Phase 3 of `scanner-identify-upgrade`. Identify cards by **what they
|
||||||
|
look like**, not by reading the name. This is how Manabox / Delver-style
|
||||||
|
scanners get printing-accurate matches on foil and alt-art.
|
||||||
|
|
||||||
|
Blocked on `improve-scan-card-detection`: embeddings of unwarped
|
||||||
|
phone crops will not match catalog `image_url`s reliably.
|
||||||
|
|
||||||
|
## Why
|
||||||
|
|
||||||
|
Name-only `pg_trgm` cannot distinguish printings. Tesseract fails on
|
||||||
|
foil and stylized type even when it emits 8–20 characters (76.7% L1
|
||||||
|
escalate with non-empty text). Gemini can read a name but still
|
||||||
|
returns a picker when set/number are missing, and costs 5/min.
|
||||||
|
|
||||||
|
The catalog already stores `cards.image_url` for every imported
|
||||||
|
printing. Precompute an embedding per row; at scan time embed the
|
||||||
|
warped crop and take top-k cosine. OCR / Gemini become hints and
|
||||||
|
unknown-card fallback.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
### In scope
|
||||||
|
|
||||||
|
- **Migration:** `pgvector` (confirm Neon availability) +
|
||||||
|
`cards.embedding` (or a side table `card_embeddings`) + ANN index.
|
||||||
|
Update `docs/SCHEMA_MAP.md`.
|
||||||
|
- **Offline embed job:** new dated script or `npm run` task that
|
||||||
|
reads `image_url`, writes vectors. Idempotent. Rate-limit the
|
||||||
|
embedding provider. Do **not** edit historical `scripts/add-*.js`.
|
||||||
|
- **Architect picks the embedder** (one):
|
||||||
|
1. Gateway embedding model (same `AI_GATEWAY_API_KEY`, server-only).
|
||||||
|
2. In-browser MobileCLIP-S2 / SigLIP ONNX for the *query* crop, with
|
||||||
|
catalog vectors baked or fetched — only if weight license + size
|
||||||
|
are acceptable.
|
||||||
|
- **Identify path:** new Layer-0 (or replace L1) — kNN then
|
||||||
|
auto-match / disambiguate / escalate to existing L2. Log
|
||||||
|
`scan_attempts.layer = 0` (or Architect-ratified value).
|
||||||
|
- **Similarity thresholds** analogous to 0.85 / 0.60, tuned on a
|
||||||
|
held-out set of scan captures if any exist in Blob.
|
||||||
|
- Auth + rate-limit on any new route. No client-side API keys
|
||||||
|
(`forbidden-client-side-llm-keys` must stay green).
|
||||||
|
|
||||||
|
### Out of scope
|
||||||
|
|
||||||
|
- Replacing Gemini entirely on day one — keep L2 for catalog misses
|
||||||
|
and low similarity.
|
||||||
|
- Training a custom card CNN.
|
||||||
|
- Python GPU service.
|
||||||
|
- Changing scanner chrome / cart.
|
||||||
|
- Auto-approving the 27 pending `card_submissions`.
|
||||||
|
|
||||||
|
## Roles invoked
|
||||||
|
|
||||||
|
1. `role-architect` — embedder, schema, layer numbering, thresholds,
|
||||||
|
brief split (migration / backfill / route). Security-sensitive:
|
||||||
|
escalate to Sonnet if schema + new route land together.
|
||||||
|
2. `role-implementer`.
|
||||||
|
3. Audit: `role-reviewer` + `role-security-auditor` (required).
|
||||||
|
|
||||||
|
## Todos
|
||||||
|
|
||||||
|
- [ ] Architect: confirm `pgvector` on prod Neon tier
|
||||||
|
- [ ] Brief 1 — migration + SCHEMA_MAP
|
||||||
|
- [ ] Brief 2 — catalog backfill job (idempotent)
|
||||||
|
- [ ] Brief 3 — identify kNN route + client escalate order
|
||||||
|
- [ ] Threshold bake-off on real crops
|
||||||
|
- [ ] Re-measure auto-match % excluding `not_a_card`
|
||||||
|
|
||||||
|
## Likely file ownership
|
||||||
|
|
||||||
|
| Area | Files |
|
||||||
|
| --- | --- |
|
||||||
|
| Schema | `migrations/*_card-embeddings.js`, `docs/SCHEMA_MAP.md` |
|
||||||
|
| Backfill | new `scripts/` job (dated) or `lib/card-embed-backfill.js` |
|
||||||
|
| Query | new `lib/card-visual-match.js`, `pages/api/scan/identify-by-image.js` **or** fold into existing identify |
|
||||||
|
| Client | `lib/scanner-card-identify.js` (`tryLayer1TextIdentify` sibling) |
|
||||||
|
|
||||||
|
Do not rewrite `lib/scanner-card-detection.js` here.
|
||||||
|
|
||||||
|
## Multitask dispatch
|
||||||
|
|
||||||
|
Brief 1 first. Brief 2 after 1. Brief 3 after 1 (can overlap 2 if the
|
||||||
|
route degrades to escalate-when-empty-index).
|
||||||
|
|
||||||
|
```
|
||||||
|
/multitask role-reviewer + role-security-auditor
|
||||||
|
```
|
||||||
|
|
||||||
|
Group id: `audit-scan-visual-catalog-search-<pr>`.
|
||||||
|
|
||||||
|
## CI impact
|
||||||
|
|
||||||
|
| Workflow / job | Behavior |
|
||||||
|
| --- | --- |
|
||||||
|
| `schema-map-fresh` | **Fires** — migration + SCHEMA_MAP |
|
||||||
|
| `ci.yml` migrate | Must apply `pgvector` on CT 102 CI Postgres — Architect must verify the extension is available there or gate the migration |
|
||||||
|
| `forbidden-client-side-llm-keys` | Blocking |
|
||||||
|
|
||||||
|
## Operator action
|
||||||
|
|
||||||
|
- Confirm Neon `pgvector` (or Neon’s equivalent) on the prod project.
|
||||||
|
- Budget: one embedding per catalog image, plus one per live scan if
|
||||||
|
the query embed is server-side. Architect publishes a cost note
|
||||||
|
before Brief 2 runs against prod images.
|
||||||
|
- No new browser secrets.
|
||||||
|
|
||||||
|
## Conductor notes
|
||||||
|
|
||||||
|
This is the accuracy leap. Do not start it to "try CLIP" before Phase 2
|
||||||
|
crops are rectified — that wastes the backfill. If Architect finds
|
||||||
|
`pgvector` unavailable on CI Postgres, stop and write a fallback
|
||||||
|
(external index vs skip-CI-extension plan) rather than shipping an
|
||||||
|
untestable migration.
|
||||||
207
.convoys/scanner-identify-upgrade.md
Normal file
207
.convoys/scanner-identify-upgrade.md
Normal file
|
|
@ -0,0 +1,207 @@
|
||||||
|
---
|
||||||
|
name: scanner-identify-upgrade
|
||||||
|
classification: feature
|
||||||
|
success_metric: |
|
||||||
|
Of legitimate card scans (excluding not_a_card), ≥50% auto-match a
|
||||||
|
catalog printing with no picker; Layer-1 escalate rate falls from 76.7%
|
||||||
|
to ≤40%; median /api/scan/identify latency stays near today's 1.7s p50
|
||||||
|
or improves. Measured on scan_attempts after Phase 1 ships.
|
||||||
|
skip: []
|
||||||
|
status: open
|
||||||
|
created: 2026-08-14
|
||||||
|
depends_on:
|
||||||
|
- add-real-ocr-layer
|
||||||
|
- server-side-scan-pipeline
|
||||||
|
model_policy:
|
||||||
|
default_session: auto
|
||||||
|
roles:
|
||||||
|
role-conductor: composer-2.5-fast
|
||||||
|
role-architect: composer-2.5
|
||||||
|
role-ia-architect: composer-2.5-fast
|
||||||
|
role-ux-reviewer: composer-2.5-fast
|
||||||
|
role-ui-designer: composer-2.5-fast
|
||||||
|
role-implementer: composer-2.5-fast
|
||||||
|
role-reviewer: cursor-grok-4.5-high
|
||||||
|
role-security-auditor: gpt-5.6-terra-medium
|
||||||
|
role-design-system-auditor: cursor-grok-4.5-high
|
||||||
|
role-a11y-auditor: cursor-grok-4.5-high
|
||||||
|
role-doc-writer: auto
|
||||||
|
escalate_to: claude-sonnet-5-thinking-medium
|
||||||
|
escalate_to_premium: claude-4.6-opus-high-thinking
|
||||||
|
never_premium:
|
||||||
|
- role-reviewer
|
||||||
|
- role-security-auditor
|
||||||
|
- role-design-system-auditor
|
||||||
|
- role-a11y-auditor
|
||||||
|
- role-ui-designer
|
||||||
|
- role-doc-writer
|
||||||
|
---
|
||||||
|
|
||||||
|
# Scanner identify upgrade — epic
|
||||||
|
|
||||||
|
Umbrella for making the camera scanner **faster and more accurate**
|
||||||
|
without replacing the two-layer shape (cheap local path, then
|
||||||
|
server-owned Gemini). Planning-only in this file. Each numbered
|
||||||
|
sub-convoy is its own gated PR stream.
|
||||||
|
|
||||||
|
Worktree: `tcg-vault-worktrees/scanner-identify-upgrade` on
|
||||||
|
`convoy/scanner-identify-upgrade` (branched from `main` @ `c6c1364`).
|
||||||
|
Do **not** land these docs on `feat/scanner-mobile-checkout` — that
|
||||||
|
convoy owns scanner chrome/cart and explicitly leaves identify out of
|
||||||
|
scope.
|
||||||
|
|
||||||
|
## Why
|
||||||
|
|
||||||
|
`add-real-ocr-layer` (PR #38) targeted ≥70% of scans resolving at
|
||||||
|
Layer-1 (Tesseract + `pg_trgm`) with zero Gemini calls. Live
|
||||||
|
`scan_attempts` from the local Neon (2026-05-27 → 2026-08-12, n=224,
|
||||||
|
1 user) shows that target was missed by a wide margin:
|
||||||
|
|
||||||
|
| Signal | Value | Target / note |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| L1 auto-match (`result_kind=matched`) | **5.8% of L1** / 2.7% of all | ≥70% L1 resolve |
|
||||||
|
| L1 escalate | **76.7% of L1** (79/103) | Should be the minority path |
|
||||||
|
| L1 disambiguation | 17.5% of L1 (18/103) | Name-only matcher cannot pick printings |
|
||||||
|
| End-to-end auto-match | **10.3%** (23/224) | User still picks or retries most cards |
|
||||||
|
| L2 `not_a_card` | **44.6% of L2** (54/121) | Detector fires on non-cards |
|
||||||
|
| L2 p50 / p90 latency | **1655 / 2349 ms** | Plus 3.5s hold-still *before* identify |
|
||||||
|
| L1 p50 latency | **158 ms** | Fast, but almost never uniquely matches |
|
||||||
|
| `card_submissions` | 27 pending, 0 reviewed | Catalog-gap queue is unread |
|
||||||
|
|
||||||
|
The six L1 "matches" include noisy Tesseract strips (`J ——`, `Rock
|
||||||
|
Jockey ©`) at confidence 36–68. L1 is a cheap filter that rarely
|
||||||
|
identifies a printing. Meanwhile every card waits
|
||||||
|
`DETECTION_START_DELAY_MS` (1s) + `MIN_FIRST_SEEN_MS_FOR_VERIFY` (2.5s)
|
||||||
|
before OCR starts, and L2 is capped at 5 Gemini calls / user / minute.
|
||||||
|
|
||||||
|
Users feel this as: hold still forever, then pick from a list, or get
|
||||||
|
"not a card" / "saved for review."
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
### In scope (this epic)
|
||||||
|
|
||||||
|
1. **`tighten-scan-identify-hot-path`** — Phase 1. Cut the hold-still
|
||||||
|
gate, OCR collector number on L1, structured Gemini JSON, optional
|
||||||
|
`SCAN_VISION_MODEL` bump, stop the automatic second Gemini refine.
|
||||||
|
2. **`improve-scan-card-detection`** — Phase 2. Replace the 320×240
|
||||||
|
Sobel brute-force detector with a real card crop + perspective warp.
|
||||||
|
This was queued in `add-real-ocr-layer` and never opened.
|
||||||
|
3. **`scan-visual-catalog-search`** — Phase 3. Embed catalog
|
||||||
|
`image_url`s; nearest-neighbor the warped crop. Gemini becomes
|
||||||
|
fallback. This is the accuracy leap.
|
||||||
|
|
||||||
|
### Out of scope (this epic)
|
||||||
|
|
||||||
|
- Scanner chrome, cart, checkout (`scanner-mobile-checkout` /
|
||||||
|
`scanner-rebuild`). Identify libs only.
|
||||||
|
- Training a custom card CNN or standing up a Python/CUDA OCR service.
|
||||||
|
- Swapping Tesseract for EasyOCR / PaddleOCR as the *primary* identifier.
|
||||||
|
- Raising the L2 rate limit until Phase 1 hit-rate is re-measured.
|
||||||
|
- Admin review of the 27 pending `card_submissions` (ops, not this epic).
|
||||||
|
|
||||||
|
## Baseline (do not re-query to "start" Phase 1)
|
||||||
|
|
||||||
|
Pulled 2026-08-14 from local `.env.local` → Neon `scan_attempts`.
|
||||||
|
|
||||||
|
```
|
||||||
|
layer | result_kind | n | pct
|
||||||
|
1 | escalate | 79 | 35.3
|
||||||
|
1 | disambiguation | 18 | 8.0
|
||||||
|
1 | matched | 6 | 2.7
|
||||||
|
2 | not_a_card | 54 | 24.1
|
||||||
|
2 | submitted | 26 | 11.6
|
||||||
|
2 | disambiguation | 22 | 9.8
|
||||||
|
2 | matched | 17 | 7.6
|
||||||
|
2 | needs_input | 2 | 0.9
|
||||||
|
```
|
||||||
|
|
||||||
|
L1 escalate text-length buckets (all had ≥3 chars — Tesseract is
|
||||||
|
emitting text that `pg_trgm` cannot match): 3–7 chars n=28; 8–19 n=30;
|
||||||
|
20+ n=21.
|
||||||
|
|
||||||
|
Re-measure with the same grouping after each sub-convoy ships.
|
||||||
|
|
||||||
|
## Dependency graph
|
||||||
|
|
||||||
|
```
|
||||||
|
[baseline pulled 2026-08-14]
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌───────────────────────────────────┐
|
||||||
|
│ 1. tighten-scan-identify-hot-path │
|
||||||
|
│ gates, collector #, schema JSON│
|
||||||
|
└───────────────┬───────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌───────────────────────────────────┐
|
||||||
|
│ 2. improve-scan-card-detection │
|
||||||
|
│ detect + homography crop │
|
||||||
|
└───────────────┬───────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌───────────────────────────────────┐
|
||||||
|
│ 3. scan-visual-catalog-search │
|
||||||
|
│ embeddings + pgvector kNN │
|
||||||
|
└───────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**Strict-blockers:** #1 before #2 only if #2 would retune the same
|
||||||
|
stability constants — otherwise #1 (identify files) and #2 (detection
|
||||||
|
files) are file-disjoint and may run in parallel after Architect
|
||||||
|
confirms. #3 needs a stable crop (#2) to be worth the embedding job;
|
||||||
|
do not start #3 until #2 has a warped JPEG.
|
||||||
|
|
||||||
|
**Sibling:** `scanner-mobile-checkout` (other worktree / branch) must
|
||||||
|
not edit `lib/ocr-worker.js`, `lib/scan-vision.js`,
|
||||||
|
`lib/card-text-match.js`, `lib/scanner-card-identify.js`,
|
||||||
|
`lib/scanner-card-detection.js`, or `/api/scan/identify`.
|
||||||
|
|
||||||
|
## Roles invoked
|
||||||
|
|
||||||
|
Umbrella is planning-only. Each sub-convoy lists its own roles.
|
||||||
|
Default for #1: Architect → Implementer → audit fan-out (reviewer +
|
||||||
|
security-auditor). UX reviewer on #1 and #2. Skip IA and UI Designer
|
||||||
|
on all three (no new routes or visual language).
|
||||||
|
|
||||||
|
## Todos
|
||||||
|
|
||||||
|
- [x] Pull `scan_attempts` baseline (2026-08-14)
|
||||||
|
- [x] Open worktree `scanner-identify-upgrade` from `main`
|
||||||
|
- [x] Seed sub-convoys #1–#3
|
||||||
|
- [ ] Architect: pick up `tighten-scan-identify-hot-path` first
|
||||||
|
- [ ] Re-measure `scan_attempts` after #1 ships
|
||||||
|
- [ ] Architect: `improve-scan-card-detection` (or parallel if files stay disjoint)
|
||||||
|
- [ ] Architect: `scan-visual-catalog-search` after warped crops exist
|
||||||
|
|
||||||
|
## Worktree
|
||||||
|
|
||||||
|
| Checkout | Branch | Purpose |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `tcg-vault-worktrees/scanner-identify-upgrade` | `convoy/scanner-identify-upgrade` | These convoy docs |
|
||||||
|
| (later) Agents Window worktree per brief | `brief/scanner-identify-upgrade/<n>-<title>` | Implementer slices |
|
||||||
|
|
||||||
|
`scripts/wt.sh` is a deprecation stub. Create implementer worktrees from
|
||||||
|
the Agents Window after Architect writes `slice_dependencies:`.
|
||||||
|
|
||||||
|
## Multitask dispatch
|
||||||
|
|
||||||
|
No implementer fan-out from this umbrella. After #1 Architect marks
|
||||||
|
parallel-safe briefs (`depends_on: []` + disjoint `files:`):
|
||||||
|
|
||||||
|
```
|
||||||
|
/multitask role-implementer briefs <ids>
|
||||||
|
```
|
||||||
|
|
||||||
|
After each PR draft:
|
||||||
|
|
||||||
|
```
|
||||||
|
/multitask role-reviewer + role-security-auditor
|
||||||
|
```
|
||||||
|
|
||||||
|
Add design-system + a11y auditors only if the PR touches
|
||||||
|
`components/` or `pages/scanner.js` (not expected in #1).
|
||||||
|
|
||||||
|
Group ids: `audit-tighten-scan-identify-hot-path-<pr>`,
|
||||||
|
`audit-improve-scan-card-detection-<pr>`,
|
||||||
|
`audit-scan-visual-catalog-search-<pr>`.
|
||||||
|
|
@ -415,6 +415,7 @@ Total: ~14 convoys to get from current state to public-launch-ready. Estimate 4-
|
||||||
|
|
||||||
Follow-ups surfaced mid-convoy or mid-PR that didn't fit the original launch sequence but need to land before public traffic. Listed in priority order; not all will be P0/P1 — most are CI / DX / hygiene polish.
|
Follow-ups surfaced mid-convoy or mid-PR that didn't fit the original launch sequence but need to land before public traffic. Listed in priority order; not all will be P0/P1 — most are CI / DX / hygiene polish.
|
||||||
|
|
||||||
|
- **`scanner-identify-upgrade`** (priority: P1 scanner accuracy; opened 2026-08-14). Umbrella after measuring `scan_attempts` (n=224, 2026-05-27–2026-08-12): L1 auto-match 5.8% of L1 vs the `add-real-ocr-layer` ≥70% target; L1 escalate 76.7%; L2 `not_a_card` 44.6% of Gemini calls; end-to-end auto-match 10.3%. Sub-convoys: `tighten-scan-identify-hot-path` → `improve-scan-card-detection` (queued since PR #38, never opened) → `scan-visual-catalog-search`. Worktree `tcg-vault-worktrees/scanner-identify-upgrade` on `convoy/scanner-identify-upgrade`. File-disjoint from `scanner-mobile-checkout` (chrome/cart). See `.convoys/scanner-identify-upgrade.md`.
|
||||||
- **`rotate-default-admin`** — **RESOLVED 2026-06-13** by PR #141 (`scripts/rotate-admin-password.js`). The convoy chose option B from the architect's three-option menu (close as no-op / build script / build forced-rotation flow): a parameterized one-shot rotation script that's safer than "manually change via app" (audit-trail-preserving via `updated_at`) and lighter than building a first-login forced-rotation flow in the app (that heavier option is the deferred `force-admin-password-reset-flow` convoy). Script reads `POSTGRES_URL` + `ADMIN_NEW_PASSWORD` env vars, validates target row exists + has `role='admin'`, refuses to rotate non-admin rows, verifies the new hash matches the supplied plaintext via `bcrypt.compare` post-update, never echoes the password. AGENTS.md Gotcha #4 documents the rotation workflow. Sibling test users (alice/bob in `scripts/create-test-users.js`) intentionally NOT rotated (dev fixtures, not real auth surfaces).
|
- **`rotate-default-admin`** — **RESOLVED 2026-06-13** by PR #141 (`scripts/rotate-admin-password.js`). The convoy chose option B from the architect's three-option menu (close as no-op / build script / build forced-rotation flow): a parameterized one-shot rotation script that's safer than "manually change via app" (audit-trail-preserving via `updated_at`) and lighter than building a first-login forced-rotation flow in the app (that heavier option is the deferred `force-admin-password-reset-flow` convoy). Script reads `POSTGRES_URL` + `ADMIN_NEW_PASSWORD` env vars, validates target row exists + has `role='admin'`, refuses to rotate non-admin rows, verifies the new hash matches the supplied plaintext via `bcrypt.compare` post-update, never echoes the password. AGENTS.md Gotcha #4 documents the rotation workflow. Sibling test users (alice/bob in `scripts/create-test-users.js`) intentionally NOT rotated (dev fixtures, not real auth surfaces).
|
||||||
- **`delete-dead-lorcana-import`** — **RESOLVED 2026-06-02** by PR #59 (`8262fec`). Deleted `pages/api/cards/import-lorcana.js` and `scripts/import-lorcana.js`; no dedicated convoy file (cleanup tracked here only). Entry kept for audit trail.
|
- **`delete-dead-lorcana-import`** — **RESOLVED 2026-06-02** by PR #59 (`8262fec`). Deleted `pages/api/cards/import-lorcana.js` and `scripts/import-lorcana.js`; no dedicated convoy file (cleanup tracked here only). Entry kept for audit trail.
|
||||||
- **`tighten-visual-diff-path-filter`** — **RESOLVED 2026-05-26** by `tighten-visual-diff-path-filter` convoy, squash commit `ba95462` (PR #26). Single-edit `paths:` filter change in `.github/workflows/visual-diff.yml`: inserted `'!pages/api/**'` immediately after `'pages/**'` (order-sensitive per GitHub Actions' minimatch path-filter semantics — exclusions only fire after a prior include matches). Verified the YAML deserialization order at gate time (`['pages/**', '!pages/api/**', 'components/**', 'styles/**', 'tailwind.config.js', 'postcss.config.js']`). `preview-smoke.yml` left untouched (no `paths:` filter; intentionally fires on every PR). Diff: 2 files, +279 / -0 (1 YAML entry + inline comment block + the planning convoy file). **Post-merge verification still pending** — the only true verification is that the next API-only PR after this merges does NOT trigger `Screenshot diff`. PR #30 (`single-sql-client`, squash `c403ea4`) was the **first API-only PR post-merge** and its CI Checks tab showed `Screenshot diff: not triggered` — empirical confirmation that the `!pages/api/**` exclusion fires correctly. The next-API-only-PR success line was originally specified in the convoy file's § Verification plan as the deferred-to-post-merge gate; this is that confirmation. Entry kept (not removed) to preserve the audit trail. See `.convoys/tighten-visual-diff-path-filter.md` § As-shipped.
|
- **`tighten-visual-diff-path-filter`** — **RESOLVED 2026-05-26** by `tighten-visual-diff-path-filter` convoy, squash commit `ba95462` (PR #26). Single-edit `paths:` filter change in `.github/workflows/visual-diff.yml`: inserted `'!pages/api/**'` immediately after `'pages/**'` (order-sensitive per GitHub Actions' minimatch path-filter semantics — exclusions only fire after a prior include matches). Verified the YAML deserialization order at gate time (`['pages/**', '!pages/api/**', 'components/**', 'styles/**', 'tailwind.config.js', 'postcss.config.js']`). `preview-smoke.yml` left untouched (no `paths:` filter; intentionally fires on every PR). Diff: 2 files, +279 / -0 (1 YAML entry + inline comment block + the planning convoy file). **Post-merge verification still pending** — the only true verification is that the next API-only PR after this merges does NOT trigger `Screenshot diff`. PR #30 (`single-sql-client`, squash `c403ea4`) was the **first API-only PR post-merge** and its CI Checks tab showed `Screenshot diff: not triggered` — empirical confirmation that the `!pages/api/**` exclusion fires correctly. The next-API-only-PR success line was originally specified in the convoy file's § Verification plan as the deferred-to-post-merge gate; this is that confirmation. Entry kept (not removed) to preserve the audit trail. See `.convoys/tighten-visual-diff-path-filter.md` § As-shipped.
|
||||||
|
|
|
||||||
328
.convoys/tighten-scan-identify-hot-path.md
Normal file
328
.convoys/tighten-scan-identify-hot-path.md
Normal file
|
|
@ -0,0 +1,328 @@
|
||||||
|
---
|
||||||
|
name: tighten-scan-identify-hot-path
|
||||||
|
classification: feature
|
||||||
|
success_metric: |
|
||||||
|
After ship, a new scan_attempts window shows L1 escalate ≤40% of L1
|
||||||
|
(was 76.7%), L1 matched ≥25% of L1 (was 5.8%), and first identify
|
||||||
|
attempt starts in ≤1.2s of tracking (was 3.5s). L2 p50 stays ≤1.8s.
|
||||||
|
skip:
|
||||||
|
- ia
|
||||||
|
- ui-design
|
||||||
|
- visual
|
||||||
|
- flag
|
||||||
|
status: in-progress
|
||||||
|
created: 2026-08-14
|
||||||
|
depends_on:
|
||||||
|
- add-real-ocr-layer
|
||||||
|
umbrella: scanner-identify-upgrade
|
||||||
|
model_policy:
|
||||||
|
default_session: auto
|
||||||
|
roles:
|
||||||
|
role-conductor: composer-2.5-fast
|
||||||
|
role-architect: composer-2.5
|
||||||
|
role-ia-architect: composer-2.5-fast
|
||||||
|
role-ux-reviewer: composer-2.5-fast
|
||||||
|
role-ui-designer: composer-2.5-fast
|
||||||
|
role-implementer: composer-2.5-fast
|
||||||
|
role-reviewer: cursor-grok-4.5-high
|
||||||
|
role-security-auditor: gpt-5.6-terra-medium
|
||||||
|
role-design-system-auditor: cursor-grok-4.5-high
|
||||||
|
role-a11y-auditor: cursor-grok-4.5-high
|
||||||
|
role-doc-writer: auto
|
||||||
|
escalate_to: claude-sonnet-5-thinking-medium
|
||||||
|
escalate_to_premium: claude-4.6-opus-high-thinking
|
||||||
|
never_premium:
|
||||||
|
- role-reviewer
|
||||||
|
- role-security-auditor
|
||||||
|
- role-design-system-auditor
|
||||||
|
- role-a11y-auditor
|
||||||
|
- role-ui-designer
|
||||||
|
- role-doc-writer
|
||||||
|
---
|
||||||
|
|
||||||
|
# Convoy: tighten-scan-identify-hot-path
|
||||||
|
|
||||||
|
Phase 1 of `scanner-identify-upgrade`. Make the **existing** two-layer
|
||||||
|
pipeline faster and more unique-match capable. No new vendors, no
|
||||||
|
detection rewrite, no embeddings.
|
||||||
|
|
||||||
|
## Why
|
||||||
|
|
||||||
|
Live telemetry (see umbrella) says Layer-1 is fast (p50 158ms) and
|
||||||
|
almost useless as an identifier (5.8% auto-match, 76.7% escalate). The
|
||||||
|
user still waits 3.5s of hold-still before that cheap path runs. L2
|
||||||
|
Gemini is 1.7s p50 and spends a second token on every disambiguation
|
||||||
|
refine. Collector number — the unique printing key L2 already uses —
|
||||||
|
is never read on L1.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
### In scope
|
||||||
|
|
||||||
|
- **Hold-still constants** in `lib/scanner-card-detection.js`:
|
||||||
|
`MIN_FIRST_SEEN_MS_FOR_VERIFY` 2500 → ~800;
|
||||||
|
`MIN_STABLE_COUNT_FOR_VERIFY` 6 → 3. Keep tests in
|
||||||
|
`test/lib/scanner-card-detection.test.js` in lockstep.
|
||||||
|
- **Name-strip OCR mode:** Tesseract PSM 7 (single line), JPEG quality
|
||||||
|
0.92 on the crop used for OCR (`lib/ocr-worker.js`,
|
||||||
|
`captureCardRegionFromVideo` quality if shared).
|
||||||
|
- **Collector-number strip:** second crop of the bottom ~18% of the
|
||||||
|
card; pass number (+ optional set hint) into `matchTextInCatalog`.
|
||||||
|
When name + number uniquely match a `cards` row, auto-match even if
|
||||||
|
many printings share the name.
|
||||||
|
- **Structured Gemini output** in `lib/scan-vision.js`:
|
||||||
|
`response_format` JSON schema (or Gateway equivalent). Delete the
|
||||||
|
confidence-30 regex fallback as the success path; keep a hard parse
|
||||||
|
error → `needs_input`.
|
||||||
|
- **Model A/B:** default stays `google/gemini-2.5-flash-lite`. Document
|
||||||
|
and allow `SCAN_VISION_MODEL=google/gemini-3.1-flash-lite` (GA on
|
||||||
|
Vercel AI Gateway, ~2.5× input price). Do not flip prod default in
|
||||||
|
this convoy unless Architect + a short identify bake-off say so.
|
||||||
|
- **Stop automatic L2 refine** in `use-scanner-identification.js` when
|
||||||
|
L1 already returned a printing list. User picks; Gemini refine is
|
||||||
|
opt-in or only when L1 had no set/number hint.
|
||||||
|
- **`docs/SCHEMA_MAP.md`:** `scan_attempts.layer` is 1 = Tesseract
|
||||||
|
(shipped), not "Tesseract future".
|
||||||
|
|
||||||
|
### Out of scope
|
||||||
|
|
||||||
|
- Detector / OpenCV / YOLO / perspective warp →
|
||||||
|
`improve-scan-card-detection`.
|
||||||
|
- pgvector / CLIP / catalog embeddings →
|
||||||
|
`scan-visual-catalog-search`.
|
||||||
|
- Changing `checkScanRateLimit` (5/min). Revisit after re-measure.
|
||||||
|
- Scanner chrome, cart, checkout (`scanner-mobile-checkout`).
|
||||||
|
- Replacing `tesseract.js`.
|
||||||
|
|
||||||
|
## Roles invoked
|
||||||
|
|
||||||
|
1. `role-ux-reviewer` — hold-still feel; when the picker still appears;
|
||||||
|
no new screens.
|
||||||
|
2. `role-architect` — ratify constant values, L1 match rules when a
|
||||||
|
collector number is present, JSON schema shape, whether the model
|
||||||
|
default flips. Write briefs + `slice_dependencies`.
|
||||||
|
3. `role-implementer` — per brief.
|
||||||
|
4. Audit: `role-reviewer` + `role-security-auditor`. Skip design-system
|
||||||
|
+ a11y unless a brief unexpectedly touches `components/`.
|
||||||
|
|
||||||
|
## Todos
|
||||||
|
|
||||||
|
- [x] UX: confirm 800ms / 3-frame gate does not cause double-scans
|
||||||
|
- [x] Architect: briefs + match-rule when number is present
|
||||||
|
- [x] Brief A — detection constants + tests
|
||||||
|
- [x] Brief B — OCR strips (name PSM 7 + collector number) +
|
||||||
|
`card-text-match.js` / `identify-by-text`
|
||||||
|
- [x] Brief C — `scan-vision.js` JSON schema + optional model id
|
||||||
|
- [x] Brief D — skip automatic disambiguation Gemini refine
|
||||||
|
- [x] SCHEMA_MAP layer-1 wording
|
||||||
|
- [ ] Re-query `scan_attempts` after preview traffic
|
||||||
|
|
||||||
|
## Likely file ownership (Architect will lock)
|
||||||
|
|
||||||
|
| Area | Files |
|
||||||
|
| --- | --- |
|
||||||
|
| Gates | `lib/scanner-card-detection.js`, `test/lib/scanner-card-detection.test.js` |
|
||||||
|
| OCR | `lib/ocr-worker.js` |
|
||||||
|
| L1 match | `lib/card-text-match.js`, `pages/api/cards/identify-by-text.js` |
|
||||||
|
| L2 vision | `lib/scan-vision.js`, `pages/api/scan/identify.js` |
|
||||||
|
| Refine | `lib/use-scanner-identification.js`, `lib/scanner-card-identify.js` |
|
||||||
|
| Docs | `docs/SCHEMA_MAP.md` |
|
||||||
|
|
||||||
|
Briefs A and C are file-disjoint and can run in parallel. B depends on
|
||||||
|
nothing if it does not retouch detection constants. D depends on B
|
||||||
|
(needs L1 payload shape).
|
||||||
|
|
||||||
|
## Multitask dispatch
|
||||||
|
|
||||||
|
Planning: UX → Architect (serial).
|
||||||
|
|
||||||
|
After Architect, if `slice_dependencies` marks A+C `depends_on: []`:
|
||||||
|
|
||||||
|
```
|
||||||
|
/multitask role-implementer briefs A, C
|
||||||
|
```
|
||||||
|
|
||||||
|
Then B, then D.
|
||||||
|
|
||||||
|
Audit group id: `audit-tighten-scan-identify-hot-path-<pr>`.
|
||||||
|
|
||||||
|
## CI impact
|
||||||
|
|
||||||
|
| Workflow / job | Behavior |
|
||||||
|
| --- | --- |
|
||||||
|
| `ci.yml` test | Fires — detection + identify unit tests change |
|
||||||
|
| `visual-diff.yml` | Should **not** fire if `components/` / `pages/` / `styles/` untouched |
|
||||||
|
| `preview-smoke.yml` | Fires |
|
||||||
|
| `forbidden-client-side-llm-keys` | Must stay green — no client Gemini URLs |
|
||||||
|
|
||||||
|
## Operator action
|
||||||
|
|
||||||
|
Optional: set `SCAN_VISION_MODEL=google/gemini-3.1-flash-lite` on a
|
||||||
|
preview to bake off against 2.5 Flash Lite. Leave prod on 2.5 until
|
||||||
|
the bake-off.
|
||||||
|
|
||||||
|
## Conductor notes
|
||||||
|
|
||||||
|
Do not treat Tesseract replacement as in-scope if L1 escalate stays
|
||||||
|
high after collector-number matching — that is a crop/detection
|
||||||
|
problem (#2) or a visual-search problem (#3), not an OCR-engine
|
||||||
|
problem. The six historical L1 matches already show Tesseract can
|
||||||
|
read a name when the crop is clean; it cannot pick a printing.
|
||||||
|
|
||||||
|
## UX
|
||||||
|
|
||||||
|
No new routes or screens. Changes are timing + fewer automatic Gemini
|
||||||
|
calls during disambiguation.
|
||||||
|
|
||||||
|
### Existing components to reuse
|
||||||
|
|
||||||
|
- `ScannerDisambiguation` (`components/scanner/ScannerDisambiguation.js`) —
|
||||||
|
printing picker when L1/L2 cannot unique-match.
|
||||||
|
- `ScannerToast` / status copy around tracked card (`components/scanner/ScannerCamera.js`)
|
||||||
|
— continue showing detecting → verifying → found without new chrome.
|
||||||
|
- `ScanDisambiguationDialog` pattern if still referenced — do not fork a
|
||||||
|
second picker.
|
||||||
|
|
||||||
|
### Design direction alignment
|
||||||
|
|
||||||
|
Liquid Glass tokens unchanged. No new overlays. Faster identify should
|
||||||
|
feel like the camera "wakes up" sooner (H1 visibility of system status).
|
||||||
|
|
||||||
|
### Patterns to follow
|
||||||
|
|
||||||
|
- Disambiguation stays a bottom sheet / modal pause (`verificationPausedRef`
|
||||||
|
already wired in `use-scanner-identification.js`).
|
||||||
|
- Error toasts debounced via `reportScannerError` (4s gap) — keep that
|
||||||
|
when tightening gates so double-failures do not spam.
|
||||||
|
|
||||||
|
### A11y constraints
|
||||||
|
|
||||||
|
- No new interactive controls in this convoy.
|
||||||
|
- Existing disambiguation list must remain keyboard-selectable (hand off
|
||||||
|
unchanged to a11y auditor if Brief D touches the sheet).
|
||||||
|
|
||||||
|
### Interaction patterns
|
||||||
|
|
||||||
|
| Pattern | Requirement | Heuristic |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Hold-still gate | **required** — 800ms + 3 stable frames; user sees brackets sooner | H1 feedback |
|
||||||
|
| Auto-match | **required** — collector number should skip picker for multi-printing names | H6 recognition vs recall |
|
||||||
|
| Disambiguation refine | **required** — stop silent second Gemini call when L1 already opened picker; user tap only | H3 user control |
|
||||||
|
| Rate-limit toast | **required** — unchanged copy when L2 throttled | H9 error recovery |
|
||||||
|
| Loading during verify | **nice-to-have** — optional subtle "Reading…" on card status if trivial | H1 |
|
||||||
|
|
||||||
|
### Anti-patterns
|
||||||
|
|
||||||
|
- Adding a setup screen or settings toggle for gate timing (H4 consistency).
|
||||||
|
- Auto-picking a printing without number evidence when multiple exact-name
|
||||||
|
rows exist (H6).
|
||||||
|
- Blocking the camera on L1 OCR worker load (H1 — worker stays async).
|
||||||
|
|
||||||
|
### Mobile / responsive
|
||||||
|
|
||||||
|
Gate timing applies equally on phone and desktop; no viewport-specific
|
||||||
|
branches in this convoy.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
### File plan
|
||||||
|
|
||||||
|
| File | Action | Purpose |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `lib/scanner-card-detection.js` | modified | 800ms / 3-frame verify gates |
|
||||||
|
| `test/lib/scanner-card-detection.test.js` | modified | Lock new constants |
|
||||||
|
| `lib/ocr-worker.js` | modified | PSM 7 name strip + bottom collector strip |
|
||||||
|
| `lib/card-text-match.js` | modified | Name + collector number unique match |
|
||||||
|
| `pages/api/cards/identify-by-text.js` | modified | Accept `cardNumber` body field |
|
||||||
|
| `test/lib/card-text-match.test.js` | new | Unit tests for number path (mock-free helpers) |
|
||||||
|
| `lib/scanner-card-identify.js` | modified | Pass number to L1; JPEG 0.92; `fromLayer1` flag |
|
||||||
|
| `lib/scan-vision.js` | modified | JSON schema response; stricter parse |
|
||||||
|
| `pages/api/scan/identify.js` | modified | Handle vision parse failures → needs_input |
|
||||||
|
| `lib/use-scanner-identification.js` | modified | Skip auto Gemini refine when `fromLayer1` |
|
||||||
|
| `docs/SCHEMA_MAP.md` | modified | Layer 1 = Tesseract (shipped) |
|
||||||
|
|
||||||
|
### API surface
|
||||||
|
|
||||||
|
**POST `/api/cards/identify-by-text`** (modified)
|
||||||
|
|
||||||
|
- Auth: `getUserFromRequest` → 401
|
||||||
|
- Body: `{ ocrText, ocrConfidence?, cardNumber?, game? }`
|
||||||
|
- Response: unchanged envelope + optional `cardNumber` echo in `ocr` meta
|
||||||
|
- Rate limit: none (unchanged)
|
||||||
|
|
||||||
|
**POST `/api/scan/identify`** (modified behavior only)
|
||||||
|
|
||||||
|
- On vision JSON parse failure: 200 with `needsUserInput: true` instead of
|
||||||
|
silent regex fallback at confidence 30
|
||||||
|
|
||||||
|
### Schema diff
|
||||||
|
|
||||||
|
No migration. `scan_attempts.layer` semantics documented only.
|
||||||
|
|
||||||
|
### Test plan
|
||||||
|
|
||||||
|
- Update `scanner-card-detection.test.js` thresholds (800ms, stable 3).
|
||||||
|
- New `card-text-match.test.js` for `extractCollectorNumberCandidate` and
|
||||||
|
collector-aware disambiguation resolution (pure functions exported for test).
|
||||||
|
- Extend `scanner-card-identify.test.js` for `fromLayer1` disambiguation
|
||||||
|
payload if added to pure helpers.
|
||||||
|
- Run `npm run test:run` + `npm run lint`.
|
||||||
|
|
||||||
|
### Risks
|
||||||
|
|
||||||
|
| Risk | Mitigation |
|
||||||
|
| --- | --- |
|
||||||
|
| Faster gate double-scans same card | `scanAttempts < 1` + tracker negative state unchanged |
|
||||||
|
| OCR number strip reads set symbol garbage | Normalize with `extractCollectorNumberCandidate`; fall back to name-only |
|
||||||
|
| Gateway rejects `response_format` | Catch 400, log once, fall back to prose prompt (Architect: implementer may feature-detect) |
|
||||||
|
| L1 skip-refine hides catalog-gap auto-submit | Refine still runs for L2 disambiguation only |
|
||||||
|
|
||||||
|
### Decomposition
|
||||||
|
|
||||||
|
| Brief # | Title | Files | Depends on | Size |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| 1 | Faster verify gates | detection + test | — | S |
|
||||||
|
| 2 | Collector number L1 | ocr-worker, card-text-match, identify-by-text, scanner-card-identify, test | — | M |
|
||||||
|
| 3 | Structured vision JSON | scan-vision, identify API | — | S |
|
||||||
|
| 4 | Skip L1 disambiguation refine | use-scanner-identification, SCHEMA_MAP | 2 | S |
|
||||||
|
|
||||||
|
### Slice dependencies
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
slice_dependencies:
|
||||||
|
- brief: 1
|
||||||
|
depends_on: []
|
||||||
|
files:
|
||||||
|
- lib/scanner-card-detection.js
|
||||||
|
- test/lib/scanner-card-detection.test.js
|
||||||
|
- brief: 2
|
||||||
|
depends_on: []
|
||||||
|
files:
|
||||||
|
- lib/ocr-worker.js
|
||||||
|
- lib/card-text-match.js
|
||||||
|
- pages/api/cards/identify-by-text.js
|
||||||
|
- lib/scanner-card-identify.js
|
||||||
|
- test/lib/card-text-match.test.js
|
||||||
|
- brief: 3
|
||||||
|
depends_on: []
|
||||||
|
files:
|
||||||
|
- lib/scan-vision.js
|
||||||
|
- pages/api/scan/identify.js
|
||||||
|
- brief: 4
|
||||||
|
depends_on: [2]
|
||||||
|
files:
|
||||||
|
- lib/use-scanner-identification.js
|
||||||
|
- docs/SCHEMA_MAP.md
|
||||||
|
```
|
||||||
|
|
||||||
|
### Decisions (post-UX)
|
||||||
|
|
||||||
|
| # | Decision |
|
||||||
|
| --- | --- |
|
||||||
|
| D1 | Gate values: `MIN_FIRST_SEEN_MS_FOR_VERIFY = 800`, `MIN_STABLE_COUNT_FOR_VERIFY = 3`. |
|
||||||
|
| D2 | Collector strip: bottom 18% of crop; PSM 7 for both strips. |
|
||||||
|
| D3 | When exact-name multiple printings AND parsed collector number matches exactly one row → auto-match. |
|
||||||
|
| D4 | `SCAN_VISION_MODEL` default stays `google/gemini-2.5-flash-lite`; structured JSON either way. |
|
||||||
|
| D5 | Auto Gemini refine skipped when disambiguation opened from L1 (`result.layer === 1`). |
|
||||||
|
|
||||||
|
Human gate 1: **approved** (operator requested full pipeline run 2026-08-14).
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
---
|
||||||
|
convoy: tighten-scan-identify-hot-path
|
||||||
|
brief_number: 1
|
||||||
|
depends_on: []
|
||||||
|
recommended_model: composer-2.5-fast
|
||||||
|
model_tier: fast
|
||||||
|
files:
|
||||||
|
- lib/scanner-card-detection.js
|
||||||
|
- test/lib/scanner-card-detection.test.js
|
||||||
|
---
|
||||||
|
|
||||||
|
# Brief 1: Faster verify gates
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Cut hold-still wait from 3.5s to ~1.4s by lowering verify gate constants.
|
||||||
|
|
||||||
|
## Acceptance criteria
|
||||||
|
|
||||||
|
- [ ] `MIN_FIRST_SEEN_MS_FOR_VERIFY = 800`
|
||||||
|
- [ ] `MIN_STABLE_COUNT_FOR_VERIFY = 3`
|
||||||
|
- [ ] Tests updated and green
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
---
|
||||||
|
convoy: tighten-scan-identify-hot-path
|
||||||
|
brief_number: 2
|
||||||
|
depends_on: []
|
||||||
|
recommended_model: composer-2.5-fast
|
||||||
|
model_tier: fast
|
||||||
|
files:
|
||||||
|
- lib/ocr-worker.js
|
||||||
|
- lib/card-text-match.js
|
||||||
|
- pages/api/cards/identify-by-text.js
|
||||||
|
- lib/scanner-card-identify.js
|
||||||
|
- test/lib/card-text-match.test.js
|
||||||
|
---
|
||||||
|
|
||||||
|
# Brief 2: Collector number on Layer 1
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
OCR name + collector number strips; unique-match printings when number resolves one row.
|
||||||
|
|
||||||
|
## Acceptance criteria
|
||||||
|
|
||||||
|
- [ ] PSM 7 on name strip; bottom 18% number strip
|
||||||
|
- [ ] `matchTextInCatalog` accepts `cardNumber`
|
||||||
|
- [ ] identify-by-text passes `cardNumber`
|
||||||
|
- [ ] JPEG capture quality 0.92 for OCR path
|
||||||
|
- [ ] Unit tests for number extraction helper
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
---
|
||||||
|
convoy: tighten-scan-identify-hot-path
|
||||||
|
brief_number: 3
|
||||||
|
depends_on: []
|
||||||
|
recommended_model: composer-2.5-fast
|
||||||
|
model_tier: fast
|
||||||
|
files:
|
||||||
|
- lib/scan-vision.js
|
||||||
|
- pages/api/scan/identify.js
|
||||||
|
---
|
||||||
|
|
||||||
|
# Brief 3: Structured Gemini JSON
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Request JSON schema from AI Gateway; fail loud on parse errors.
|
||||||
|
|
||||||
|
## Acceptance criteria
|
||||||
|
|
||||||
|
- [ ] `response_format` json_schema on gateway call
|
||||||
|
- [ ] No confidence-30 regex success path
|
||||||
|
- [ ] identify route returns needsUserInput on parse failure
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
---
|
||||||
|
convoy: tighten-scan-identify-hot-path
|
||||||
|
brief_number: 4
|
||||||
|
depends_on: [2]
|
||||||
|
recommended_model: composer-2.5-fast
|
||||||
|
model_tier: fast
|
||||||
|
files:
|
||||||
|
- lib/use-scanner-identification.js
|
||||||
|
- docs/SCHEMA_MAP.md
|
||||||
|
---
|
||||||
|
|
||||||
|
# Brief 4: Skip L1 disambiguation refine
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Do not auto-call Gemini when L1 already opened the printing picker.
|
||||||
|
|
||||||
|
## Acceptance criteria
|
||||||
|
|
||||||
|
- [ ] `fromLayer1` on disambiguation state when `result.layer === 1`
|
||||||
|
- [ ] useEffect refine skipped when `fromLayer1`
|
||||||
|
- [ ] SCHEMA_MAP documents layer 1 = Tesseract
|
||||||
|
|
@ -261,7 +261,7 @@ Index: `idx_card_submissions_status (status, created_at DESC)`.
|
||||||
|
|
||||||
### scan_attempts
|
### scan_attempts
|
||||||
|
|
||||||
Per-scan telemetry for the identify pipeline (layer 2 = Gemini today).
|
Per-scan telemetry for the identify pipeline (layer 1 = Tesseract + pg_trgm, layer 2 = Gemini).
|
||||||
|
|
||||||
| Column | Type | Notes |
|
| Column | Type | Notes |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
|
|
@ -269,7 +269,7 @@ Per-scan telemetry for the identify pipeline (layer 2 = Gemini today).
|
||||||
| `user_id` | FK → `users` SET NULL | |
|
| `user_id` | FK → `users` SET NULL | |
|
||||||
| `ocr_text` | `TEXT` | |
|
| `ocr_text` | `TEXT` | |
|
||||||
| `ocr_confidence` | `INTEGER` | |
|
| `ocr_confidence` | `INTEGER` | |
|
||||||
| `layer` | `INTEGER` default `2` | OCR layer (1 = Tesseract future) |
|
| `layer` | `INTEGER` default `2` | Identify layer (`1` = browser Tesseract + pg_trgm, `2` = Gemini vision) |
|
||||||
| `matched_card_id` | FK → `cards` SET NULL | |
|
| `matched_card_id` | FK → `cards` SET NULL | |
|
||||||
| `result_kind` | `VARCHAR(32)` | e.g. `'matched'`, `'disambiguation'`, `'submitted'`, `'not_a_card'` |
|
| `result_kind` | `VARCHAR(32)` | e.g. `'matched'`, `'disambiguation'`, `'submitted'`, `'not_a_card'` |
|
||||||
| `latency_ms` | `INTEGER` | End-to-end identify latency |
|
| `latency_ms` | `INTEGER` | End-to-end identify latency |
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
import { sql } from '@vercel/postgres';
|
import { sql } from '@vercel/postgres';
|
||||||
|
|
||||||
|
import { findPrintingByCollectorNumber } from './card-number-utils.js';
|
||||||
|
|
||||||
const MATCH_THRESHOLD = 0.85;
|
const MATCH_THRESHOLD = 0.85;
|
||||||
const DISAMBIGUATION_THRESHOLD = 0.6;
|
const DISAMBIGUATION_THRESHOLD = 0.6;
|
||||||
const MIN_QUERY_LENGTH = 3;
|
const MIN_QUERY_LENGTH = 3;
|
||||||
|
|
@ -64,11 +66,53 @@ async function querySimilarCards(query, game) {
|
||||||
`;
|
`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolveExactNameMatches(exactNameMatches, cardNumber, query) {
|
||||||
|
if (exactNameMatches.length === 1) {
|
||||||
|
return {
|
||||||
|
type: 'matched',
|
||||||
|
card: exactNameMatches[0],
|
||||||
|
query,
|
||||||
|
similarity: exactNameMatches[0].sim,
|
||||||
|
message: `Matched "${exactNameMatches[0].name}" via text search`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (exactNameMatches.length > 1 && cardNumber) {
|
||||||
|
const uniqueByNumber = findPrintingByCollectorNumber(exactNameMatches, cardNumber);
|
||||||
|
if (uniqueByNumber) {
|
||||||
|
return {
|
||||||
|
type: 'matched',
|
||||||
|
card: uniqueByNumber,
|
||||||
|
query,
|
||||||
|
similarity: uniqueByNumber.sim,
|
||||||
|
message: `Matched "${uniqueByNumber.name}" (${uniqueByNumber.card_number}) via collector number`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (exactNameMatches.length > 1) {
|
||||||
|
return {
|
||||||
|
type: 'disambiguation',
|
||||||
|
matches: exactNameMatches.slice(0, 8).map(mapCardRow),
|
||||||
|
query,
|
||||||
|
message: `Found ${exactNameMatches.length} printings of "${query}". Select the correct one.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fuzzy match OCR text against cards.name using pg_trgm similarity.
|
* Fuzzy match OCR text against cards.name using pg_trgm similarity.
|
||||||
*/
|
*/
|
||||||
export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence = null }) {
|
export async function matchTextInCatalog({
|
||||||
|
ocrText,
|
||||||
|
cardNumber = null,
|
||||||
|
game = null,
|
||||||
|
ocrConfidence = null,
|
||||||
|
}) {
|
||||||
const query = extractNameCandidate(ocrText);
|
const query = extractNameCandidate(ocrText);
|
||||||
|
const trimmedNumber = cardNumber && String(cardNumber).trim() ? String(cardNumber).trim() : null;
|
||||||
|
|
||||||
if (!query || query.length < MIN_QUERY_LENGTH) {
|
if (!query || query.length < MIN_QUERY_LENGTH) {
|
||||||
return {
|
return {
|
||||||
|
|
@ -76,6 +120,7 @@ export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence =
|
||||||
reason: 'OCR text too short for catalog match',
|
reason: 'OCR text too short for catalog match',
|
||||||
query,
|
query,
|
||||||
ocrConfidence,
|
ocrConfidence,
|
||||||
|
cardNumber: trimmedNumber,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -88,6 +133,7 @@ export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence =
|
||||||
reason: `No catalog match above ${DISAMBIGUATION_THRESHOLD} similarity for "${query}"`,
|
reason: `No catalog match above ${DISAMBIGUATION_THRESHOLD} similarity for "${query}"`,
|
||||||
query,
|
query,
|
||||||
ocrConfidence,
|
ocrConfidence,
|
||||||
|
cardNumber: trimmedNumber,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -96,14 +142,9 @@ export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence =
|
||||||
(row) => row.name?.toLowerCase() === normalizedQuery
|
(row) => row.name?.toLowerCase() === normalizedQuery
|
||||||
);
|
);
|
||||||
|
|
||||||
// Same card name, multiple printings — always ask the user.
|
const exactResolution = resolveExactNameMatches(exactNameMatches, trimmedNumber, query);
|
||||||
if (exactNameMatches.length > 1) {
|
if (exactResolution) {
|
||||||
return {
|
return exactResolution;
|
||||||
type: 'disambiguation',
|
|
||||||
matches: exactNameMatches.slice(0, 8).map(mapCardRow),
|
|
||||||
query,
|
|
||||||
message: `Found ${exactNameMatches.length} printings of "${query}". Select the correct one.`,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const top = candidates[0];
|
const top = candidates[0];
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,12 @@
|
||||||
|
|
||||||
let workerPromise = null;
|
let workerPromise = null;
|
||||||
|
|
||||||
|
const NAME_STRIP_PSM = '7';
|
||||||
|
const NUMBER_STRIP_PSM = '7';
|
||||||
|
const NAME_STRIP_HEIGHT_RATIO = 0.35;
|
||||||
|
const NUMBER_STRIP_HEIGHT_RATIO = 0.18;
|
||||||
|
const OCR_JPEG_QUALITY = 0.92;
|
||||||
|
|
||||||
async function getWorker() {
|
async function getWorker() {
|
||||||
if (!workerPromise) {
|
if (!workerPromise) {
|
||||||
workerPromise = (async () => {
|
workerPromise = (async () => {
|
||||||
|
|
@ -12,53 +18,116 @@ async function getWorker() {
|
||||||
const worker = await createWorker('eng', 1, {
|
const worker = await createWorker('eng', 1, {
|
||||||
logger: () => {},
|
logger: () => {},
|
||||||
});
|
});
|
||||||
await worker.setParameters({
|
|
||||||
tessedit_pageseg_mode: '6',
|
|
||||||
});
|
|
||||||
return worker;
|
return worker;
|
||||||
})();
|
})();
|
||||||
}
|
}
|
||||||
return workerPromise;
|
return workerPromise;
|
||||||
}
|
}
|
||||||
|
|
||||||
function cropNameStrip(imageDataUrl) {
|
function cropCardStrip(imageDataUrl, { fromTop = true, heightRatio }) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const img = new Image();
|
const img = new Image();
|
||||||
img.onload = () => {
|
img.onload = () => {
|
||||||
const stripHeight = Math.max(1, Math.floor(img.height * 0.35));
|
const stripHeight = Math.max(1, Math.floor(img.height * heightRatio));
|
||||||
const canvas = document.createElement('canvas');
|
const canvas = document.createElement('canvas');
|
||||||
canvas.width = img.width;
|
canvas.width = img.width;
|
||||||
canvas.height = stripHeight;
|
canvas.height = stripHeight;
|
||||||
const ctx = canvas.getContext('2d');
|
const ctx = canvas.getContext('2d');
|
||||||
ctx.drawImage(img, 0, 0, img.width, stripHeight, 0, 0, img.width, stripHeight);
|
const sourceY = fromTop ? 0 : Math.max(0, img.height - stripHeight);
|
||||||
resolve(canvas.toDataURL('image/jpeg', 0.92));
|
ctx.drawImage(
|
||||||
|
img,
|
||||||
|
0,
|
||||||
|
sourceY,
|
||||||
|
img.width,
|
||||||
|
stripHeight,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
canvas.width,
|
||||||
|
canvas.height
|
||||||
|
);
|
||||||
|
resolve(canvas.toDataURL('image/jpeg', OCR_JPEG_QUALITY));
|
||||||
};
|
};
|
||||||
img.onerror = () => reject(new Error('Failed to load image for OCR'));
|
img.onerror = () => reject(new Error('Failed to load image for OCR'));
|
||||||
img.src = imageDataUrl;
|
img.src = imageDataUrl;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pick the most likely collector number token from OCR of the bottom strip.
|
||||||
|
* @param {string} rawText
|
||||||
|
* @returns {string}
|
||||||
|
*/
|
||||||
|
export function extractCollectorNumberCandidate(rawText) {
|
||||||
|
if (!rawText || typeof rawText !== 'string') return '';
|
||||||
|
|
||||||
|
const lines = rawText
|
||||||
|
.split(/\r?\n/)
|
||||||
|
.map((line) => line.replace(/\s+/g, ' ').trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
const joined = lines.join(' ');
|
||||||
|
const slashMatch = joined.match(/\b(\d{1,4}\s*\/\s*\d{1,4}[A-Za-z]?)\b/);
|
||||||
|
if (slashMatch) {
|
||||||
|
return slashMatch[1].replace(/\s+/g, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
||||||
|
const line = lines[i];
|
||||||
|
if (/\d/.test(line) && line.length <= 12) {
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* OCR the top name strip of a card crop.
|
* OCR the top name strip of a card crop.
|
||||||
* @param {string} imageDataUrl - full card crop data URL
|
* @param {string} imageDataUrl - full card crop data URL
|
||||||
* @returns {Promise<{ text: string, confidence: number }>}
|
* @returns {Promise<{ text: string, confidence: number }>}
|
||||||
*/
|
*/
|
||||||
export async function recognizeCardNameStrip(imageDataUrl) {
|
export async function recognizeCardNameStrip(imageDataUrl) {
|
||||||
|
const fields = await recognizeCardFields(imageDataUrl);
|
||||||
|
return {
|
||||||
|
text: fields.nameText,
|
||||||
|
confidence: fields.nameConfidence,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OCR name strip + bottom collector-number strip.
|
||||||
|
* @param {string} imageDataUrl
|
||||||
|
* @returns {Promise<{ nameText: string, nameConfidence: number, cardNumber: string, numberConfidence: number }>}
|
||||||
|
*/
|
||||||
|
export async function recognizeCardFields(imageDataUrl) {
|
||||||
if (typeof window === 'undefined') {
|
if (typeof window === 'undefined') {
|
||||||
return { text: '', confidence: 0 };
|
return { nameText: '', nameConfidence: 0, cardNumber: '', numberConfidence: 0 };
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const stripUrl = await cropNameStrip(imageDataUrl);
|
const [nameStripUrl, numberStripUrl] = await Promise.all([
|
||||||
|
cropCardStrip(imageDataUrl, { fromTop: true, heightRatio: NAME_STRIP_HEIGHT_RATIO }),
|
||||||
|
cropCardStrip(imageDataUrl, { fromTop: false, heightRatio: NUMBER_STRIP_HEIGHT_RATIO }),
|
||||||
|
]);
|
||||||
|
|
||||||
const worker = await getWorker();
|
const worker = await getWorker();
|
||||||
const { data } = await worker.recognize(stripUrl);
|
await worker.setParameters({ tessedit_pageseg_mode: NAME_STRIP_PSM });
|
||||||
|
const nameResult = await worker.recognize(nameStripUrl);
|
||||||
|
|
||||||
|
await worker.setParameters({ tessedit_pageseg_mode: NUMBER_STRIP_PSM });
|
||||||
|
const numberResult = await worker.recognize(numberStripUrl);
|
||||||
|
|
||||||
|
const cardNumber = extractCollectorNumberCandidate(numberResult.data.text || '');
|
||||||
|
|
||||||
return {
|
return {
|
||||||
text: (data.text || '').trim(),
|
nameText: (nameResult.data.text || '').trim(),
|
||||||
confidence: Math.round(data.confidence || 0),
|
nameConfidence: Math.round(nameResult.data.confidence || 0),
|
||||||
|
cardNumber,
|
||||||
|
numberConfidence: Math.round(numberResult.data.confidence || 0),
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn('[ocr-worker] recognizeCardNameStrip failed:', error);
|
console.warn('[ocr-worker] recognizeCardFields failed:', error);
|
||||||
return { text: '', confidence: 0 };
|
return { nameText: '', nameConfidence: 0, cardNumber: '', numberConfidence: 0 };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -8,45 +8,105 @@ HOLOGRAPHIC / FOIL CARDS: Many cards have reflective foil surfaces with glare or
|
||||||
|
|
||||||
CRITICAL: Only respond with card data if you can clearly identify a TRADING CARD in the image. Ignore random objects, books, papers, phone screens, screenshots, and non-card gaming items. Blurry images with no readable card frame should be rejected.
|
CRITICAL: Only respond with card data if you can clearly identify a TRADING CARD in the image. Ignore random objects, books, papers, phone screens, screenshots, and non-card gaming items. Blurry images with no readable card frame should be rejected.
|
||||||
|
|
||||||
If you detect a trading card, extract information in this JSON format:
|
Respond with JSON only (no markdown fences) using this shape:
|
||||||
{
|
{
|
||||||
"isCard": true,
|
"isCard": true,
|
||||||
"cardName": "exact card name as printed on the card",
|
"cardName": "exact card name",
|
||||||
"setName": "set name if visible",
|
"setName": "set name or null",
|
||||||
"setCode": "set code/symbol if visible",
|
"setCode": "set code or null",
|
||||||
"cardNumber": "collector number if visible",
|
"cardNumber": "collector number or null",
|
||||||
"game": "mtg, pokemon, or lorcana (lowercase)",
|
"game": "mtg, pokemon, or lorcana",
|
||||||
"cardType": "creature, instant, sorcery, trainer, etc.",
|
"cardType": "creature, instant, etc.",
|
||||||
"rarity": "common, uncommon, rare, mythic, etc.",
|
"rarity": "common, uncommon, rare, mythic, etc.",
|
||||||
"manaCost": "mana cost if visible",
|
"manaCost": "mana cost or null",
|
||||||
"hp": "HP or power if visible",
|
"hp": "HP or power or null",
|
||||||
"abilities": ["list of abilities or attacks if clearly readable"],
|
"abilities": ["ability strings"],
|
||||||
"confidence": 85,
|
"confidence": 85,
|
||||||
"rawText": "all text visible on the card"
|
"rawText": "all visible text",
|
||||||
|
"reason": null
|
||||||
}
|
}
|
||||||
|
|
||||||
If NO trading card is clearly visible, respond with:
|
If NO trading card is clearly visible, respond with:
|
||||||
{
|
{
|
||||||
"isCard": false,
|
"isCard": false,
|
||||||
|
"cardName": null,
|
||||||
|
"setName": null,
|
||||||
|
"setCode": null,
|
||||||
|
"cardNumber": null,
|
||||||
|
"game": null,
|
||||||
|
"cardType": null,
|
||||||
|
"rarity": null,
|
||||||
|
"manaCost": null,
|
||||||
|
"hp": null,
|
||||||
|
"abilities": [],
|
||||||
"confidence": 0,
|
"confidence": 0,
|
||||||
|
"rawText": null,
|
||||||
"reason": "No trading card detected in image"
|
"reason": "No trading card detected in image"
|
||||||
}
|
}
|
||||||
|
|
||||||
Be conservative — only extract data you can clearly read. Quality over quantity.`;
|
Be conservative — only extract data you can clearly read. Quality over quantity.`;
|
||||||
|
|
||||||
|
const CARD_VISION_SCHEMA = {
|
||||||
|
name: 'card_scan_result',
|
||||||
|
strict: true,
|
||||||
|
schema: {
|
||||||
|
type: 'object',
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
isCard: { type: 'boolean' },
|
||||||
|
cardName: { type: ['string', 'null'] },
|
||||||
|
setName: { type: ['string', 'null'] },
|
||||||
|
setCode: { type: ['string', 'null'] },
|
||||||
|
cardNumber: { type: ['string', 'null'] },
|
||||||
|
game: { type: ['string', 'null'] },
|
||||||
|
cardType: { type: ['string', 'null'] },
|
||||||
|
rarity: { type: ['string', 'null'] },
|
||||||
|
manaCost: { type: ['string', 'null'] },
|
||||||
|
hp: { type: ['string', 'null'] },
|
||||||
|
abilities: {
|
||||||
|
type: 'array',
|
||||||
|
items: { type: 'string' },
|
||||||
|
},
|
||||||
|
confidence: { type: 'number' },
|
||||||
|
rawText: { type: ['string', 'null'] },
|
||||||
|
reason: { type: ['string', 'null'] },
|
||||||
|
},
|
||||||
|
required: [
|
||||||
|
'isCard',
|
||||||
|
'cardName',
|
||||||
|
'setName',
|
||||||
|
'setCode',
|
||||||
|
'cardNumber',
|
||||||
|
'game',
|
||||||
|
'cardType',
|
||||||
|
'rarity',
|
||||||
|
'manaCost',
|
||||||
|
'hp',
|
||||||
|
'abilities',
|
||||||
|
'confidence',
|
||||||
|
'rawText',
|
||||||
|
'reason',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export class VisionParseError extends Error {
|
||||||
|
constructor(message) {
|
||||||
|
super(message);
|
||||||
|
this.name = 'VisionParseError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function parseVisionJson(content) {
|
function parseVisionJson(content) {
|
||||||
|
if (!content || typeof content !== 'string') {
|
||||||
|
throw new VisionParseError('Empty vision model response');
|
||||||
|
}
|
||||||
|
|
||||||
const cleanContent = content.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
|
const cleanContent = content.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
|
||||||
try {
|
try {
|
||||||
return JSON.parse(cleanContent);
|
return JSON.parse(cleanContent);
|
||||||
} catch {
|
} catch {
|
||||||
const cardNameMatch = content.match(/card.*?name.*?[:"]\s*([^"'\n,}]+)/i);
|
throw new VisionParseError('Failed to parse structured vision response');
|
||||||
return {
|
|
||||||
isCard: !!cardNameMatch,
|
|
||||||
cardName: cardNameMatch ? cardNameMatch[1].trim() : null,
|
|
||||||
confidence: 30,
|
|
||||||
rawText: content,
|
|
||||||
reason: 'Failed to parse structured response',
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -59,6 +119,29 @@ function normalizeGame(game) {
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildGatewayBody(imageDataUrl, { structured = true } = {}) {
|
||||||
|
const body = {
|
||||||
|
model: DEFAULT_VISION_MODEL,
|
||||||
|
temperature: 0.1,
|
||||||
|
messages: [{
|
||||||
|
role: 'user',
|
||||||
|
content: [
|
||||||
|
{ type: 'text', text: CARD_PROMPT },
|
||||||
|
{ type: 'image_url', image_url: { url: imageDataUrl } },
|
||||||
|
],
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
|
||||||
|
if (structured) {
|
||||||
|
body.response_format = {
|
||||||
|
type: 'json_schema',
|
||||||
|
json_schema: CARD_VISION_SCHEMA,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Server-side vision analysis via Vercel AI Gateway. Requires AI_GATEWAY_API_KEY.
|
* Server-side vision analysis via Vercel AI Gateway. Requires AI_GATEWAY_API_KEY.
|
||||||
* @param {string} imageDataUrl - data:image/jpeg;base64,... capture from scanner
|
* @param {string} imageDataUrl - data:image/jpeg;base64,... capture from scanner
|
||||||
|
|
@ -73,25 +156,26 @@ export async function analyzeCardImage(imageDataUrl) {
|
||||||
throw new Error('Invalid image data format');
|
throw new Error('Invalid image data format');
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = await fetch(GATEWAY_URL, {
|
let response = await fetch(GATEWAY_URL, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
Authorization: `Bearer ${apiKey}`,
|
Authorization: `Bearer ${apiKey}`,
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify(buildGatewayBody(imageDataUrl)),
|
||||||
model: DEFAULT_VISION_MODEL,
|
|
||||||
temperature: 0.1,
|
|
||||||
messages: [{
|
|
||||||
role: 'user',
|
|
||||||
content: [
|
|
||||||
{ type: 'text', text: CARD_PROMPT },
|
|
||||||
{ type: 'image_url', image_url: { url: imageDataUrl } },
|
|
||||||
],
|
|
||||||
}],
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (!response.ok && response.status === 400) {
|
||||||
|
response = await fetch(GATEWAY_URL, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
Authorization: `Bearer ${apiKey}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify(buildGatewayBody(imageDataUrl, { structured: false })),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const errorData = await response.json().catch(() => ({}));
|
const errorData = await response.json().catch(() => ({}));
|
||||||
const message = errorData.error?.message || errorData.message || 'Unknown error';
|
const message = errorData.error?.message || errorData.message || 'Unknown error';
|
||||||
|
|
@ -101,7 +185,7 @@ export async function analyzeCardImage(imageDataUrl) {
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
const content = data.choices?.[0]?.message?.content;
|
const content = data.choices?.[0]?.message?.content;
|
||||||
if (!content) {
|
if (!content) {
|
||||||
throw new Error('No response from vision model');
|
throw new VisionParseError('No response from vision model');
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = parseVisionJson(content);
|
const result = parseVisionJson(content);
|
||||||
|
|
|
||||||
|
|
@ -12,10 +12,10 @@ export const SHAPE_DETECTION_INTERVAL_MS = 200;
|
||||||
export const VERIFICATION_INTERVAL_MS = 1000;
|
export const VERIFICATION_INTERVAL_MS = 1000;
|
||||||
|
|
||||||
/** Minimum stable frames before a tracked card is sent for identification. */
|
/** Minimum stable frames before a tracked card is sent for identification. */
|
||||||
export const MIN_STABLE_COUNT_FOR_VERIFY = 6;
|
export const MIN_STABLE_COUNT_FOR_VERIFY = 3;
|
||||||
|
|
||||||
/** Minimum time (ms) a card must be tracked before verification. */
|
/** Minimum time (ms) a card must be tracked before verification. */
|
||||||
export const MIN_FIRST_SEEN_MS_FOR_VERIFY = 2500;
|
export const MIN_FIRST_SEEN_MS_FOR_VERIFY = 800;
|
||||||
|
|
||||||
/** Delay after camera start before detection loops begin. */
|
/** Delay after camera start before detection loops begin. */
|
||||||
export const DETECTION_START_DELAY_MS = 1000;
|
export const DETECTION_START_DELAY_MS = 1000;
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,9 @@
|
||||||
/** Default margin (px) around tracked bounds when cropping a card capture. */
|
/** Default margin (px) around tracked bounds when cropping a card capture. */
|
||||||
export const CAPTURE_MARGIN_PX = 20;
|
export const CAPTURE_MARGIN_PX = 20;
|
||||||
|
|
||||||
|
/** JPEG quality for card crops sent to OCR / vision. */
|
||||||
|
export const OCR_CAPTURE_JPEG_QUALITY = 0.92;
|
||||||
|
|
||||||
/** Vision rate-limit backoff duration (ms). */
|
/** Vision rate-limit backoff duration (ms). */
|
||||||
export const VISION_RATE_LIMIT_MS = 60_000;
|
export const VISION_RATE_LIMIT_MS = 60_000;
|
||||||
|
|
||||||
|
|
@ -79,6 +82,7 @@ export function resolveIdentifyOutcome(result) {
|
||||||
matches: result.matches,
|
matches: result.matches,
|
||||||
ocrMeta,
|
ocrMeta,
|
||||||
message: result.message,
|
message: result.message,
|
||||||
|
fromLayer1: result.layer === 1,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -182,7 +186,7 @@ export function captureCardRegionFromVideo(video, canvas, bounds, margin = CAPTU
|
||||||
canvas.height
|
canvas.height
|
||||||
);
|
);
|
||||||
|
|
||||||
return canvas.toDataURL('image/jpeg', 0.8);
|
return canvas.toDataURL('image/jpeg', OCR_CAPTURE_JPEG_QUALITY);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function submitScanForReview({ imageData, name, candidateCardIds, authHeaders }) {
|
export async function submitScanForReview({ imageData, name, candidateCardIds, authHeaders }) {
|
||||||
|
|
@ -205,11 +209,11 @@ export async function submitScanForReview({ imageData, name, candidateCardIds, a
|
||||||
return { ok: true, message: result.message };
|
return { ok: true, message: result.message };
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function fetchIdentifyByText({ ocrText, ocrConfidence, authHeaders }) {
|
export async function fetchIdentifyByText({ ocrText, ocrConfidence, cardNumber, authHeaders }) {
|
||||||
const response = await fetch('/api/cards/identify-by-text', {
|
const response = await fetch('/api/cards/identify-by-text', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: authHeaders,
|
headers: authHeaders,
|
||||||
body: JSON.stringify({ ocrText, ocrConfidence }),
|
body: JSON.stringify({ ocrText, ocrConfidence, cardNumber }),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
|
|
@ -245,16 +249,17 @@ export async function fetchVisionIdentify(imageData, authHeaders) {
|
||||||
* Returns { handled: true, outcome } when L1 resolves without escalation.
|
* Returns { handled: true, outcome } when L1 resolves without escalation.
|
||||||
*/
|
*/
|
||||||
export async function tryLayer1TextIdentify(imageData, authHeaders) {
|
export async function tryLayer1TextIdentify(imageData, authHeaders) {
|
||||||
const { recognizeCardNameStrip } = await import('./ocr-worker.js');
|
const { recognizeCardFields } = await import('./ocr-worker.js');
|
||||||
const ocr = await recognizeCardNameStrip(imageData);
|
const ocr = await recognizeCardFields(imageData);
|
||||||
|
|
||||||
if (ocr.text.length < 3) {
|
if (ocr.nameText.length < 3) {
|
||||||
return { handled: false };
|
return { handled: false };
|
||||||
}
|
}
|
||||||
|
|
||||||
const l1 = await fetchIdentifyByText({
|
const l1 = await fetchIdentifyByText({
|
||||||
ocrText: ocr.text,
|
ocrText: ocr.nameText,
|
||||||
ocrConfidence: ocr.confidence,
|
ocrConfidence: ocr.nameConfidence,
|
||||||
|
cardNumber: ocr.cardNumber || undefined,
|
||||||
authHeaders,
|
authHeaders,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -131,6 +131,7 @@ export function useScannerIdentification({
|
||||||
candidates: outcome.matches,
|
candidates: outcome.matches,
|
||||||
ocrMeta: outcome.ocrMeta,
|
ocrMeta: outcome.ocrMeta,
|
||||||
message: outcome.message,
|
message: outcome.message,
|
||||||
|
fromLayer1: Boolean(outcome.fromLayer1),
|
||||||
});
|
});
|
||||||
break;
|
break;
|
||||||
case 'notice':
|
case 'notice':
|
||||||
|
|
@ -179,6 +180,7 @@ export function useScannerIdentification({
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!disambiguation?.imageData) return;
|
if (!disambiguation?.imageData) return;
|
||||||
|
if (disambiguation.fromLayer1) return;
|
||||||
if (Date.now() < visionCooldownUntilRef.current) return;
|
if (Date.now() < visionCooldownUntilRef.current) return;
|
||||||
|
|
||||||
const refineKey = disambiguation.cardTracker?.id ?? 'modal';
|
const refineKey = disambiguation.cardTracker?.id ?? 'modal';
|
||||||
|
|
|
||||||
|
|
@ -32,19 +32,20 @@ export default async function handler(req, res) {
|
||||||
return res.status(401).json({ error: 'Authentication required' });
|
return res.status(401).json({ error: 'Authentication required' });
|
||||||
}
|
}
|
||||||
|
|
||||||
const { ocrText, ocrConfidence, game } = req.body || {};
|
const { ocrText, ocrConfidence, cardNumber, game } = req.body || {};
|
||||||
|
|
||||||
if (!ocrText || typeof ocrText !== 'string') {
|
if (!ocrText || typeof ocrText !== 'string') {
|
||||||
return res.status(400).json({ error: 'ocrText is required' });
|
return res.status(400).json({ error: 'ocrText is required' });
|
||||||
}
|
}
|
||||||
|
|
||||||
const matchResult = await matchTextInCatalog({ ocrText, game, ocrConfidence });
|
const matchResult = await matchTextInCatalog({ ocrText, cardNumber, game, ocrConfidence });
|
||||||
const latencyMs = Date.now() - startedAt;
|
const latencyMs = Date.now() - startedAt;
|
||||||
|
|
||||||
const ocrMeta = {
|
const ocrMeta = {
|
||||||
confidence: ocrConfidence ?? null,
|
confidence: ocrConfidence ?? null,
|
||||||
rawText: ocrText,
|
rawText: ocrText,
|
||||||
query: matchResult.query || ocrText,
|
query: matchResult.query || ocrText,
|
||||||
|
cardNumber: matchResult.cardNumber ?? cardNumber ?? null,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (matchResult.type === 'escalate') {
|
if (matchResult.type === 'escalate') {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { getUserFromRequest } from '../../../lib/permission-middleware';
|
import { getUserFromRequest } from '../../../lib/permission-middleware';
|
||||||
import { checkScanRateLimit } from '../../../lib/rate-limit.js';
|
import { checkScanRateLimit } from '../../../lib/rate-limit.js';
|
||||||
import { analyzeCardImage } from '../../../lib/scan-vision.js';
|
import { analyzeCardImage, VisionParseError } from '../../../lib/scan-vision.js';
|
||||||
import { matchCardInCatalog, logScanAttempt } from '../../../lib/card-catalog-match.js';
|
import { matchCardInCatalog, logScanAttempt } from '../../../lib/card-catalog-match.js';
|
||||||
|
|
||||||
function scanErrorResponse(error) {
|
function scanErrorResponse(error) {
|
||||||
|
|
@ -42,6 +42,18 @@ function scanErrorResponse(error) {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (error instanceof VisionParseError || msg.includes('Failed to parse structured vision')) {
|
||||||
|
return {
|
||||||
|
status: 200,
|
||||||
|
body: {
|
||||||
|
isCard: true,
|
||||||
|
card: null,
|
||||||
|
needsUserInput: true,
|
||||||
|
message: 'Could not read card details from scan — try again or pick from matches.',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
status: 500,
|
status: 500,
|
||||||
body: { error: 'Internal server error' },
|
body: { error: 'Internal server error' },
|
||||||
|
|
|
||||||
16
test/lib/card-text-match.test.js
Normal file
16
test/lib/card-text-match.test.js
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { extractCollectorNumberCandidate } from '../../lib/ocr-worker.js';
|
||||||
|
|
||||||
|
describe('extractCollectorNumberCandidate', () => {
|
||||||
|
it('extracts slash-style collector numbers', () => {
|
||||||
|
expect(extractCollectorNumberCandidate('015 / 208\nRare')).toBe('015/208');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('extracts compact numbers from the last numeric line', () => {
|
||||||
|
expect(extractCollectorNumberCandidate('Some set symbol\n123/456')).toBe('123/456');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns empty for non-numeric noise', () => {
|
||||||
|
expect(extractCollectorNumberCandidate('Creature — Human')).toBe('');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -97,16 +97,16 @@ describe('selectCardsReadyForVerification', () => {
|
||||||
const ready = {
|
const ready = {
|
||||||
id: 1,
|
id: 1,
|
||||||
status: 'detecting',
|
status: 'detecting',
|
||||||
stableCount: 6,
|
stableCount: 3,
|
||||||
scanAttempts: 0,
|
scanAttempts: 0,
|
||||||
firstSeen: now - 3000,
|
firstSeen: now - 900,
|
||||||
};
|
};
|
||||||
const tooFresh = {
|
const tooFresh = {
|
||||||
id: 2,
|
id: 2,
|
||||||
status: 'detecting',
|
status: 'detecting',
|
||||||
stableCount: 6,
|
stableCount: 3,
|
||||||
scanAttempts: 0,
|
scanAttempts: 0,
|
||||||
firstSeen: now - 1000,
|
firstSeen: now - 500,
|
||||||
};
|
};
|
||||||
|
|
||||||
expect(selectCardsReadyForVerification([ready, tooFresh], now)).toEqual([ready]);
|
expect(selectCardsReadyForVerification([ready, tooFresh], now)).toEqual([ready]);
|
||||||
|
|
|
||||||
|
|
@ -53,6 +53,7 @@ describe('resolveIdentifyOutcome', () => {
|
||||||
matches,
|
matches,
|
||||||
ocrMeta: expect.objectContaining({ abilities: [] }),
|
ocrMeta: expect.objectContaining({ abilities: [] }),
|
||||||
message: 'Pick one',
|
message: 'Pick one',
|
||||||
|
fromLayer1: false,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -63,6 +64,18 @@ describe('resolveIdentifyOutcome', () => {
|
||||||
message: 'Blurry',
|
message: 'Blurry',
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('marks L1 disambiguation so Gemini refine is skipped', () => {
|
||||||
|
const outcome = resolveIdentifyOutcome({
|
||||||
|
isCard: true,
|
||||||
|
layer: 1,
|
||||||
|
needsUserSelection: true,
|
||||||
|
matches: [{ id: 1, name: 'Bolt' }],
|
||||||
|
message: 'Pick one',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(outcome.fromLayer1).toBe(true);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('buildScannedCardPayload', () => {
|
describe('buildScannedCardPayload', () => {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue