Scanner identify upgrade — Phase 1 hot path (#156)
* docs(convoy): seed scanner identify upgrade epic and sub-convoys Baseline scan_attempts telemetry and three-phase plan for faster, more accurate card identification without touching scanner chrome. Co-authored-by: Cursor <cursoragent@cursor.com> * feat(scanner): tighten Layer-1 identify hot path (Phase 1) Cut verify hold-still gates, OCR collector numbers on Layer 1, request structured Gemini JSON, and skip automatic L2 refine when L1 opens the printing picker. Includes convoy UX/architecture briefs and unit tests. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
varutasu
GitHub
parent
73424aae59
commit
0b4f419f49
22 changed files with 1263 additions and 74 deletions
|
|
@ -79,6 +79,10 @@
|
|||
{"ts": "2026-08-15T00:14:27Z", "role": "role-ia-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 95, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:15:31Z", "role": "role-ui-designer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:16:36Z", "role": "role-ux-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 180, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "scanner-identify-upgrade", "repo": "scanner-identify-upgrade", "skip_flags": [], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "visual", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "visual", "a11y", "design", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:18:40Z", "role": "role-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 420, "model": "composer-2.5", "model_tier": "standard"}
|
||||
{"ts": "2026-08-15T00:19:56Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:20:23Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 2, "duration_s": 120, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
|
|
@ -88,4 +92,5 @@
|
|||
{"ts": "2026-08-15T00:25:54Z", "role": "role-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 210, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||
{"ts": "2026-08-15T00:26:05Z", "role": "role-a11y-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 78, "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||
{"ts": "2026-08-15T00:26:08Z", "role": "role-design-system-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 82, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||
{"ts": "2026-08-15T00:55:56Z", "role": "role-reviewer", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 0, "duration_s": 180, "multitask_group": "audit-tighten-scan-identify-hot-path-uncommitted", "model": "cursor-grok-4.5-high", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:56:39Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
|
|
|
|||
151
.convoys/improve-scan-card-detection.md
Normal file
151
.convoys/improve-scan-card-detection.md
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
---
|
||||
name: improve-scan-card-detection
|
||||
classification: feature
|
||||
success_metric: |
|
||||
After ship, L2 result_kind=not_a_card ≤15% of L2 (was 44.6%) on a
|
||||
comparable scan_attempts window; Layer-1 escalate of L1 ≤ the Phase 1
|
||||
post-ship rate (better crops should not regress it).
|
||||
skip:
|
||||
- ia
|
||||
- ui-design
|
||||
- flag
|
||||
status: open
|
||||
created: 2026-08-14
|
||||
depends_on:
|
||||
- tighten-scan-identify-hot-path
|
||||
umbrella: scanner-identify-upgrade
|
||||
model_policy:
|
||||
default_session: auto
|
||||
roles:
|
||||
role-conductor: composer-2.5-fast
|
||||
role-architect: composer-2.5
|
||||
role-ia-architect: composer-2.5-fast
|
||||
role-ux-reviewer: composer-2.5-fast
|
||||
role-ui-designer: composer-2.5-fast
|
||||
role-implementer: composer-2.5-fast
|
||||
role-reviewer: cursor-grok-4.5-high
|
||||
role-security-auditor: gpt-5.6-terra-medium
|
||||
role-design-system-auditor: cursor-grok-4.5-high
|
||||
role-a11y-auditor: cursor-grok-4.5-high
|
||||
role-doc-writer: auto
|
||||
escalate_to: claude-sonnet-5-thinking-medium
|
||||
escalate_to_premium: claude-4.6-opus-high-thinking
|
||||
never_premium:
|
||||
- role-reviewer
|
||||
- role-security-auditor
|
||||
- role-design-system-auditor
|
||||
- role-a11y-auditor
|
||||
- role-ui-designer
|
||||
- role-doc-writer
|
||||
---
|
||||
|
||||
# Convoy: improve-scan-card-detection
|
||||
|
||||
Phase 2 of `scanner-identify-upgrade`. Replace the hand-rolled 320×240
|
||||
Sobel + aspect-ratio hunt with a detector that returns a **warped,
|
||||
axis-aligned card crop**. Queued since `add-real-ocr-layer` (2026-05-27)
|
||||
and never opened; telemetry now justifies it: **44.6% of Gemini calls
|
||||
are `not_a_card`**.
|
||||
|
||||
`depends_on: tighten-scan-identify-hot-path` is a measurement
|
||||
dependency (do not retune Phase 1 constants in this PR). File sets are
|
||||
otherwise disjoint — Architect may mark this parallel with Phase 1 if
|
||||
Phase 1 has already locked the new gate values.
|
||||
|
||||
## Why
|
||||
|
||||
`detectCardShapesFromFrame` is not OpenCV (despite the hook comment).
|
||||
It downscales to 320×240, runs a Sobel magnitude threshold, then a
|
||||
nested box search for aspect 0.63–0.77. There is no four-corner
|
||||
homography. Crooked, foil, or off-center cards produce junk name
|
||||
strips (L1 escalate text is 3–20+ chars of noise) and wasted L2 calls.
|
||||
|
||||
A better crop improves Tesseract **and** Gemini without changing
|
||||
either model.
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope
|
||||
|
||||
- Replace or wrap `detectCardShapesFromFrame` so tracked bounds are
|
||||
card-shaped **and** a perspective-corrected JPEG can be produced for
|
||||
identify.
|
||||
- Architect picks **one** in-browser approach (do not land both):
|
||||
1. **OpenCV.js contours + approxPolyDP + warpPerspective** — no
|
||||
training, larger WASM.
|
||||
2. **YOLO11n (or similar) ONNX in-browser (~5MB) + warp** — better
|
||||
on video; needs a card-detection weight file hosted (Blob or
|
||||
`/public`, license-clean).
|
||||
- Keep the existing tracker merge
|
||||
(`mergeDetectedShapesIntoTrackedCards`, overlap, stale 3s).
|
||||
- Overlay brackets in `components/scanner/ScannerCamera.js` keep
|
||||
consuming `{x,y,width,height}` (or four corners mapped to a rect).
|
||||
Do not redesign the overlay.
|
||||
- Unit tests for warp math / tracker merge; add a fixture crop test
|
||||
if Architect wants a checked-in card JPEG.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Identify / Gemini / Tesseract / catalog match (Phase 1 + 3).
|
||||
- Python/CUDA microservice, PaddleOCR server, Roboflow-hosted detect.
|
||||
- Scanner chrome / cart (`scanner-mobile-checkout`).
|
||||
- Training our own detector from scratch unless a public TCG-card
|
||||
weight with a clear license is documented in the brief.
|
||||
|
||||
## Roles invoked
|
||||
|
||||
1. `role-ux-reviewer` — time-to-bracket, false-positive boxes, dual-card
|
||||
frames.
|
||||
2. `role-architect` — OpenCV.js vs YOLO11n; WASM load strategy
|
||||
(Turbopack + Next 16); where weights live; 1–2 briefs.
|
||||
3. `role-implementer`.
|
||||
4. Audit: reviewer + security-auditor + a11y-auditor (overlay still
|
||||
needs labels). Design-system only if overlay styling changes.
|
||||
|
||||
## Todos
|
||||
|
||||
- [ ] Architect: pick detector; document WASM / weight budget
|
||||
- [ ] Brief 1 — detect + warp library + tests
|
||||
- [ ] Brief 2 — wire `use-camera-scanner.js` + crop used by
|
||||
`identifyTrackedCardCapture`
|
||||
- [ ] Confirm `scanner-mobile-checkout` overlay still maps bounds
|
||||
- [ ] Re-measure L2 `not_a_card` share
|
||||
|
||||
## Likely file ownership
|
||||
|
||||
| Area | Files |
|
||||
| --- | --- |
|
||||
| Detect | `lib/scanner-card-detection.js`, `test/lib/scanner-card-detection.test.js` |
|
||||
| Warp helper | new `lib/scanner-card-warp.js` (if Architect splits) |
|
||||
| Camera loop | `lib/use-camera-scanner.js` |
|
||||
| Crop consumer | `lib/scanner-card-identify.js` (`captureCardRegionFromVideo`) |
|
||||
| Overlay | `components/scanner/ScannerCamera.js` only if bounds shape changes |
|
||||
|
||||
Do not edit `lib/ocr-worker.js` or `lib/scan-vision.js` here.
|
||||
|
||||
## Multitask dispatch
|
||||
|
||||
Serial unless Architect splits detect-lib vs camera-wire with disjoint
|
||||
files.
|
||||
|
||||
Audit group id: `audit-improve-scan-card-detection-<pr>`.
|
||||
|
||||
## CI impact
|
||||
|
||||
| Workflow / job | Behavior |
|
||||
| --- | --- |
|
||||
| `visual-diff.yml` | **May fire** if `ScannerCamera.js` changes |
|
||||
| `preview-smoke.yml` | Fires |
|
||||
| `ci.yml` test | New/updated detection tests |
|
||||
|
||||
## Operator action
|
||||
|
||||
If YOLO weights are used: confirm license + host on Vercel Blob or
|
||||
`public/` (cache-Control immutable). No new secrets.
|
||||
|
||||
## Conductor notes
|
||||
|
||||
Do not drop OpenCV.js in only to reimplement the current rectangle
|
||||
hunt. Success is a **rectified card image**, not a prettier box.
|
||||
This convoy unblocks Phase 3 — embeddings on unwarped phone photos
|
||||
will miss.
|
||||
161
.convoys/scan-visual-catalog-search.md
Normal file
161
.convoys/scan-visual-catalog-search.md
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
---
|
||||
name: scan-visual-catalog-search
|
||||
classification: feature
|
||||
success_metric: |
|
||||
After catalog embeddings exist and the new kNN path is on, ≥50% of
|
||||
legitimate card scans (scan_attempts excluding not_a_card) auto-match
|
||||
a printing with no picker — the original add-real-ocr-layer 70%
|
||||
target, measured honestly. Gemini L2 share of attempts falls vs the
|
||||
Phase 1 post-ship baseline.
|
||||
skip:
|
||||
- ia
|
||||
- ui-design
|
||||
- visual
|
||||
- a11y
|
||||
- design
|
||||
- flag
|
||||
status: open
|
||||
created: 2026-08-14
|
||||
depends_on:
|
||||
- improve-scan-card-detection
|
||||
umbrella: scanner-identify-upgrade
|
||||
model_policy:
|
||||
default_session: auto
|
||||
roles:
|
||||
role-conductor: composer-2.5-fast
|
||||
role-architect: composer-2.5
|
||||
role-ia-architect: composer-2.5-fast
|
||||
role-ux-reviewer: composer-2.5-fast
|
||||
role-ui-designer: composer-2.5-fast
|
||||
role-implementer: composer-2.5-fast
|
||||
role-reviewer: cursor-grok-4.5-high
|
||||
role-security-auditor: gpt-5.6-terra-medium
|
||||
role-design-system-auditor: cursor-grok-4.5-high
|
||||
role-a11y-auditor: cursor-grok-4.5-high
|
||||
role-doc-writer: auto
|
||||
escalate_to: claude-sonnet-5-thinking-medium
|
||||
escalate_to_premium: claude-4.6-opus-high-thinking
|
||||
never_premium:
|
||||
- role-reviewer
|
||||
- role-security-auditor
|
||||
- role-design-system-auditor
|
||||
- role-a11y-auditor
|
||||
- role-ui-designer
|
||||
- role-doc-writer
|
||||
---
|
||||
|
||||
# Convoy: scan-visual-catalog-search
|
||||
|
||||
Phase 3 of `scanner-identify-upgrade`. Identify cards by **what they
|
||||
look like**, not by reading the name. This is how Manabox / Delver-style
|
||||
scanners get printing-accurate matches on foil and alt-art.
|
||||
|
||||
Blocked on `improve-scan-card-detection`: embeddings of unwarped
|
||||
phone crops will not match catalog `image_url`s reliably.
|
||||
|
||||
## Why
|
||||
|
||||
Name-only `pg_trgm` cannot distinguish printings. Tesseract fails on
|
||||
foil and stylized type even when it emits 8–20 characters (76.7% L1
|
||||
escalate with non-empty text). Gemini can read a name but still
|
||||
returns a picker when set/number are missing, and costs 5/min.
|
||||
|
||||
The catalog already stores `cards.image_url` for every imported
|
||||
printing. Precompute an embedding per row; at scan time embed the
|
||||
warped crop and take top-k cosine. OCR / Gemini become hints and
|
||||
unknown-card fallback.
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope
|
||||
|
||||
- **Migration:** `pgvector` (confirm Neon availability) +
|
||||
`cards.embedding` (or a side table `card_embeddings`) + ANN index.
|
||||
Update `docs/SCHEMA_MAP.md`.
|
||||
- **Offline embed job:** new dated script or `npm run` task that
|
||||
reads `image_url`, writes vectors. Idempotent. Rate-limit the
|
||||
embedding provider. Do **not** edit historical `scripts/add-*.js`.
|
||||
- **Architect picks the embedder** (one):
|
||||
1. Gateway embedding model (same `AI_GATEWAY_API_KEY`, server-only).
|
||||
2. In-browser MobileCLIP-S2 / SigLIP ONNX for the *query* crop, with
|
||||
catalog vectors baked or fetched — only if weight license + size
|
||||
are acceptable.
|
||||
- **Identify path:** new Layer-0 (or replace L1) — kNN then
|
||||
auto-match / disambiguate / escalate to existing L2. Log
|
||||
`scan_attempts.layer = 0` (or Architect-ratified value).
|
||||
- **Similarity thresholds** analogous to 0.85 / 0.60, tuned on a
|
||||
held-out set of scan captures if any exist in Blob.
|
||||
- Auth + rate-limit on any new route. No client-side API keys
|
||||
(`forbidden-client-side-llm-keys` must stay green).
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Replacing Gemini entirely on day one — keep L2 for catalog misses
|
||||
and low similarity.
|
||||
- Training a custom card CNN.
|
||||
- Python GPU service.
|
||||
- Changing scanner chrome / cart.
|
||||
- Auto-approving the 27 pending `card_submissions`.
|
||||
|
||||
## Roles invoked
|
||||
|
||||
1. `role-architect` — embedder, schema, layer numbering, thresholds,
|
||||
brief split (migration / backfill / route). Security-sensitive:
|
||||
escalate to Sonnet if schema + new route land together.
|
||||
2. `role-implementer`.
|
||||
3. Audit: `role-reviewer` + `role-security-auditor` (required).
|
||||
|
||||
## Todos
|
||||
|
||||
- [ ] Architect: confirm `pgvector` on prod Neon tier
|
||||
- [ ] Brief 1 — migration + SCHEMA_MAP
|
||||
- [ ] Brief 2 — catalog backfill job (idempotent)
|
||||
- [ ] Brief 3 — identify kNN route + client escalate order
|
||||
- [ ] Threshold bake-off on real crops
|
||||
- [ ] Re-measure auto-match % excluding `not_a_card`
|
||||
|
||||
## Likely file ownership
|
||||
|
||||
| Area | Files |
|
||||
| --- | --- |
|
||||
| Schema | `migrations/*_card-embeddings.js`, `docs/SCHEMA_MAP.md` |
|
||||
| Backfill | new `scripts/` job (dated) or `lib/card-embed-backfill.js` |
|
||||
| Query | new `lib/card-visual-match.js`, `pages/api/scan/identify-by-image.js` **or** fold into existing identify |
|
||||
| Client | `lib/scanner-card-identify.js` (`tryLayer1TextIdentify` sibling) |
|
||||
|
||||
Do not rewrite `lib/scanner-card-detection.js` here.
|
||||
|
||||
## Multitask dispatch
|
||||
|
||||
Brief 1 first. Brief 2 after 1. Brief 3 after 1 (can overlap 2 if the
|
||||
route degrades to escalate-when-empty-index).
|
||||
|
||||
```
|
||||
/multitask role-reviewer + role-security-auditor
|
||||
```
|
||||
|
||||
Group id: `audit-scan-visual-catalog-search-<pr>`.
|
||||
|
||||
## CI impact
|
||||
|
||||
| Workflow / job | Behavior |
|
||||
| --- | --- |
|
||||
| `schema-map-fresh` | **Fires** — migration + SCHEMA_MAP |
|
||||
| `ci.yml` migrate | Must apply `pgvector` on CT 102 CI Postgres — Architect must verify the extension is available there or gate the migration |
|
||||
| `forbidden-client-side-llm-keys` | Blocking |
|
||||
|
||||
## Operator action
|
||||
|
||||
- Confirm Neon `pgvector` (or Neon’s equivalent) on the prod project.
|
||||
- Budget: one embedding per catalog image, plus one per live scan if
|
||||
the query embed is server-side. Architect publishes a cost note
|
||||
before Brief 2 runs against prod images.
|
||||
- No new browser secrets.
|
||||
|
||||
## Conductor notes
|
||||
|
||||
This is the accuracy leap. Do not start it to "try CLIP" before Phase 2
|
||||
crops are rectified — that wastes the backfill. If Architect finds
|
||||
`pgvector` unavailable on CI Postgres, stop and write a fallback
|
||||
(external index vs skip-CI-extension plan) rather than shipping an
|
||||
untestable migration.
|
||||
207
.convoys/scanner-identify-upgrade.md
Normal file
207
.convoys/scanner-identify-upgrade.md
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
---
|
||||
name: scanner-identify-upgrade
|
||||
classification: feature
|
||||
success_metric: |
|
||||
Of legitimate card scans (excluding not_a_card), ≥50% auto-match a
|
||||
catalog printing with no picker; Layer-1 escalate rate falls from 76.7%
|
||||
to ≤40%; median /api/scan/identify latency stays near today's 1.7s p50
|
||||
or improves. Measured on scan_attempts after Phase 1 ships.
|
||||
skip: []
|
||||
status: open
|
||||
created: 2026-08-14
|
||||
depends_on:
|
||||
- add-real-ocr-layer
|
||||
- server-side-scan-pipeline
|
||||
model_policy:
|
||||
default_session: auto
|
||||
roles:
|
||||
role-conductor: composer-2.5-fast
|
||||
role-architect: composer-2.5
|
||||
role-ia-architect: composer-2.5-fast
|
||||
role-ux-reviewer: composer-2.5-fast
|
||||
role-ui-designer: composer-2.5-fast
|
||||
role-implementer: composer-2.5-fast
|
||||
role-reviewer: cursor-grok-4.5-high
|
||||
role-security-auditor: gpt-5.6-terra-medium
|
||||
role-design-system-auditor: cursor-grok-4.5-high
|
||||
role-a11y-auditor: cursor-grok-4.5-high
|
||||
role-doc-writer: auto
|
||||
escalate_to: claude-sonnet-5-thinking-medium
|
||||
escalate_to_premium: claude-4.6-opus-high-thinking
|
||||
never_premium:
|
||||
- role-reviewer
|
||||
- role-security-auditor
|
||||
- role-design-system-auditor
|
||||
- role-a11y-auditor
|
||||
- role-ui-designer
|
||||
- role-doc-writer
|
||||
---
|
||||
|
||||
# Scanner identify upgrade — epic
|
||||
|
||||
Umbrella for making the camera scanner **faster and more accurate**
|
||||
without replacing the two-layer shape (cheap local path, then
|
||||
server-owned Gemini). Planning-only in this file. Each numbered
|
||||
sub-convoy is its own gated PR stream.
|
||||
|
||||
Worktree: `tcg-vault-worktrees/scanner-identify-upgrade` on
|
||||
`convoy/scanner-identify-upgrade` (branched from `main` @ `c6c1364`).
|
||||
Do **not** land these docs on `feat/scanner-mobile-checkout` — that
|
||||
convoy owns scanner chrome/cart and explicitly leaves identify out of
|
||||
scope.
|
||||
|
||||
## Why
|
||||
|
||||
`add-real-ocr-layer` (PR #38) targeted ≥70% of scans resolving at
|
||||
Layer-1 (Tesseract + `pg_trgm`) with zero Gemini calls. Live
|
||||
`scan_attempts` from the local Neon (2026-05-27 → 2026-08-12, n=224,
|
||||
1 user) shows that target was missed by a wide margin:
|
||||
|
||||
| Signal | Value | Target / note |
|
||||
| --- | --- | --- |
|
||||
| L1 auto-match (`result_kind=matched`) | **5.8% of L1** / 2.7% of all | ≥70% L1 resolve |
|
||||
| L1 escalate | **76.7% of L1** (79/103) | Should be the minority path |
|
||||
| L1 disambiguation | 17.5% of L1 (18/103) | Name-only matcher cannot pick printings |
|
||||
| End-to-end auto-match | **10.3%** (23/224) | User still picks or retries most cards |
|
||||
| L2 `not_a_card` | **44.6% of L2** (54/121) | Detector fires on non-cards |
|
||||
| L2 p50 / p90 latency | **1655 / 2349 ms** | Plus 3.5s hold-still *before* identify |
|
||||
| L1 p50 latency | **158 ms** | Fast, but almost never uniquely matches |
|
||||
| `card_submissions` | 27 pending, 0 reviewed | Catalog-gap queue is unread |
|
||||
|
||||
The six L1 "matches" include noisy Tesseract strips (`J ——`, `Rock
|
||||
Jockey ©`) at confidence 36–68. L1 is a cheap filter that rarely
|
||||
identifies a printing. Meanwhile every card waits
|
||||
`DETECTION_START_DELAY_MS` (1s) + `MIN_FIRST_SEEN_MS_FOR_VERIFY` (2.5s)
|
||||
before OCR starts, and L2 is capped at 5 Gemini calls / user / minute.
|
||||
|
||||
Users feel this as: hold still forever, then pick from a list, or get
|
||||
"not a card" / "saved for review."
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope (this epic)
|
||||
|
||||
1. **`tighten-scan-identify-hot-path`** — Phase 1. Cut the hold-still
|
||||
gate, OCR collector number on L1, structured Gemini JSON, optional
|
||||
`SCAN_VISION_MODEL` bump, stop the automatic second Gemini refine.
|
||||
2. **`improve-scan-card-detection`** — Phase 2. Replace the 320×240
|
||||
Sobel brute-force detector with a real card crop + perspective warp.
|
||||
This was queued in `add-real-ocr-layer` and never opened.
|
||||
3. **`scan-visual-catalog-search`** — Phase 3. Embed catalog
|
||||
`image_url`s; nearest-neighbor the warped crop. Gemini becomes
|
||||
fallback. This is the accuracy leap.
|
||||
|
||||
### Out of scope (this epic)
|
||||
|
||||
- Scanner chrome, cart, checkout (`scanner-mobile-checkout` /
|
||||
`scanner-rebuild`). Identify libs only.
|
||||
- Training a custom card CNN or standing up a Python/CUDA OCR service.
|
||||
- Swapping Tesseract for EasyOCR / PaddleOCR as the *primary* identifier.
|
||||
- Raising the L2 rate limit until Phase 1 hit-rate is re-measured.
|
||||
- Admin review of the 27 pending `card_submissions` (ops, not this epic).
|
||||
|
||||
## Baseline (do not re-query to "start" Phase 1)
|
||||
|
||||
Pulled 2026-08-14 from local `.env.local` → Neon `scan_attempts`.
|
||||
|
||||
```
|
||||
layer | result_kind | n | pct
|
||||
1 | escalate | 79 | 35.3
|
||||
1 | disambiguation | 18 | 8.0
|
||||
1 | matched | 6 | 2.7
|
||||
2 | not_a_card | 54 | 24.1
|
||||
2 | submitted | 26 | 11.6
|
||||
2 | disambiguation | 22 | 9.8
|
||||
2 | matched | 17 | 7.6
|
||||
2 | needs_input | 2 | 0.9
|
||||
```
|
||||
|
||||
L1 escalate text-length buckets (all had ≥3 chars — Tesseract is
|
||||
emitting text that `pg_trgm` cannot match): 3–7 chars n=28; 8–19 n=30;
|
||||
20+ n=21.
|
||||
|
||||
Re-measure with the same grouping after each sub-convoy ships.
|
||||
|
||||
## Dependency graph
|
||||
|
||||
```
|
||||
[baseline pulled 2026-08-14]
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────────┐
|
||||
│ 1. tighten-scan-identify-hot-path │
|
||||
│ gates, collector #, schema JSON│
|
||||
└───────────────┬───────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────────┐
|
||||
│ 2. improve-scan-card-detection │
|
||||
│ detect + homography crop │
|
||||
└───────────────┬───────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────────┐
|
||||
│ 3. scan-visual-catalog-search │
|
||||
│ embeddings + pgvector kNN │
|
||||
└───────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Strict-blockers:** #1 before #2 only if #2 would retune the same
|
||||
stability constants — otherwise #1 (identify files) and #2 (detection
|
||||
files) are file-disjoint and may run in parallel after Architect
|
||||
confirms. #3 needs a stable crop (#2) to be worth the embedding job;
|
||||
do not start #3 until #2 has a warped JPEG.
|
||||
|
||||
**Sibling:** `scanner-mobile-checkout` (other worktree / branch) must
|
||||
not edit `lib/ocr-worker.js`, `lib/scan-vision.js`,
|
||||
`lib/card-text-match.js`, `lib/scanner-card-identify.js`,
|
||||
`lib/scanner-card-detection.js`, or `/api/scan/identify`.
|
||||
|
||||
## Roles invoked
|
||||
|
||||
Umbrella is planning-only. Each sub-convoy lists its own roles.
|
||||
Default for #1: Architect → Implementer → audit fan-out (reviewer +
|
||||
security-auditor). UX reviewer on #1 and #2. Skip IA and UI Designer
|
||||
on all three (no new routes or visual language).
|
||||
|
||||
## Todos
|
||||
|
||||
- [x] Pull `scan_attempts` baseline (2026-08-14)
|
||||
- [x] Open worktree `scanner-identify-upgrade` from `main`
|
||||
- [x] Seed sub-convoys #1–#3
|
||||
- [ ] Architect: pick up `tighten-scan-identify-hot-path` first
|
||||
- [ ] Re-measure `scan_attempts` after #1 ships
|
||||
- [ ] Architect: `improve-scan-card-detection` (or parallel if files stay disjoint)
|
||||
- [ ] Architect: `scan-visual-catalog-search` after warped crops exist
|
||||
|
||||
## Worktree
|
||||
|
||||
| Checkout | Branch | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `tcg-vault-worktrees/scanner-identify-upgrade` | `convoy/scanner-identify-upgrade` | These convoy docs |
|
||||
| (later) Agents Window worktree per brief | `brief/scanner-identify-upgrade/<n>-<title>` | Implementer slices |
|
||||
|
||||
`scripts/wt.sh` is a deprecation stub. Create implementer worktrees from
|
||||
the Agents Window after Architect writes `slice_dependencies:`.
|
||||
|
||||
## Multitask dispatch
|
||||
|
||||
No implementer fan-out from this umbrella. After #1 Architect marks
|
||||
parallel-safe briefs (`depends_on: []` + disjoint `files:`):
|
||||
|
||||
```
|
||||
/multitask role-implementer briefs <ids>
|
||||
```
|
||||
|
||||
After each PR draft:
|
||||
|
||||
```
|
||||
/multitask role-reviewer + role-security-auditor
|
||||
```
|
||||
|
||||
Add design-system + a11y auditors only if the PR touches
|
||||
`components/` or `pages/scanner.js` (not expected in #1).
|
||||
|
||||
Group ids: `audit-tighten-scan-identify-hot-path-<pr>`,
|
||||
`audit-improve-scan-card-detection-<pr>`,
|
||||
`audit-scan-visual-catalog-search-<pr>`.
|
||||
|
|
@ -415,6 +415,7 @@ Total: ~14 convoys to get from current state to public-launch-ready. Estimate 4-
|
|||
|
||||
Follow-ups surfaced mid-convoy or mid-PR that didn't fit the original launch sequence but need to land before public traffic. Listed in priority order; not all will be P0/P1 — most are CI / DX / hygiene polish.
|
||||
|
||||
- **`scanner-identify-upgrade`** (priority: P1 scanner accuracy; opened 2026-08-14). Umbrella after measuring `scan_attempts` (n=224, 2026-05-27–2026-08-12): L1 auto-match 5.8% of L1 vs the `add-real-ocr-layer` ≥70% target; L1 escalate 76.7%; L2 `not_a_card` 44.6% of Gemini calls; end-to-end auto-match 10.3%. Sub-convoys: `tighten-scan-identify-hot-path` → `improve-scan-card-detection` (queued since PR #38, never opened) → `scan-visual-catalog-search`. Worktree `tcg-vault-worktrees/scanner-identify-upgrade` on `convoy/scanner-identify-upgrade`. File-disjoint from `scanner-mobile-checkout` (chrome/cart). See `.convoys/scanner-identify-upgrade.md`.
|
||||
- **`rotate-default-admin`** — **RESOLVED 2026-06-13** by PR #141 (`scripts/rotate-admin-password.js`). The convoy chose option B from the architect's three-option menu (close as no-op / build script / build forced-rotation flow): a parameterized one-shot rotation script that's safer than "manually change via app" (audit-trail-preserving via `updated_at`) and lighter than building a first-login forced-rotation flow in the app (that heavier option is the deferred `force-admin-password-reset-flow` convoy). Script reads `POSTGRES_URL` + `ADMIN_NEW_PASSWORD` env vars, validates target row exists + has `role='admin'`, refuses to rotate non-admin rows, verifies the new hash matches the supplied plaintext via `bcrypt.compare` post-update, never echoes the password. AGENTS.md Gotcha #4 documents the rotation workflow. Sibling test users (alice/bob in `scripts/create-test-users.js`) intentionally NOT rotated (dev fixtures, not real auth surfaces).
|
||||
- **`delete-dead-lorcana-import`** — **RESOLVED 2026-06-02** by PR #59 (`8262fec`). Deleted `pages/api/cards/import-lorcana.js` and `scripts/import-lorcana.js`; no dedicated convoy file (cleanup tracked here only). Entry kept for audit trail.
|
||||
- **`tighten-visual-diff-path-filter`** — **RESOLVED 2026-05-26** by `tighten-visual-diff-path-filter` convoy, squash commit `ba95462` (PR #26). Single-edit `paths:` filter change in `.github/workflows/visual-diff.yml`: inserted `'!pages/api/**'` immediately after `'pages/**'` (order-sensitive per GitHub Actions' minimatch path-filter semantics — exclusions only fire after a prior include matches). Verified the YAML deserialization order at gate time (`['pages/**', '!pages/api/**', 'components/**', 'styles/**', 'tailwind.config.js', 'postcss.config.js']`). `preview-smoke.yml` left untouched (no `paths:` filter; intentionally fires on every PR). Diff: 2 files, +279 / -0 (1 YAML entry + inline comment block + the planning convoy file). **Post-merge verification still pending** — the only true verification is that the next API-only PR after this merges does NOT trigger `Screenshot diff`. PR #30 (`single-sql-client`, squash `c403ea4`) was the **first API-only PR post-merge** and its CI Checks tab showed `Screenshot diff: not triggered` — empirical confirmation that the `!pages/api/**` exclusion fires correctly. The next-API-only-PR success line was originally specified in the convoy file's § Verification plan as the deferred-to-post-merge gate; this is that confirmation. Entry kept (not removed) to preserve the audit trail. See `.convoys/tighten-visual-diff-path-filter.md` § As-shipped.
|
||||
|
|
|
|||
328
.convoys/tighten-scan-identify-hot-path.md
Normal file
328
.convoys/tighten-scan-identify-hot-path.md
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
---
|
||||
name: tighten-scan-identify-hot-path
|
||||
classification: feature
|
||||
success_metric: |
|
||||
After ship, a new scan_attempts window shows L1 escalate ≤40% of L1
|
||||
(was 76.7%), L1 matched ≥25% of L1 (was 5.8%), and first identify
|
||||
attempt starts in ≤1.2s of tracking (was 3.5s). L2 p50 stays ≤1.8s.
|
||||
skip:
|
||||
- ia
|
||||
- ui-design
|
||||
- visual
|
||||
- flag
|
||||
status: in-progress
|
||||
created: 2026-08-14
|
||||
depends_on:
|
||||
- add-real-ocr-layer
|
||||
umbrella: scanner-identify-upgrade
|
||||
model_policy:
|
||||
default_session: auto
|
||||
roles:
|
||||
role-conductor: composer-2.5-fast
|
||||
role-architect: composer-2.5
|
||||
role-ia-architect: composer-2.5-fast
|
||||
role-ux-reviewer: composer-2.5-fast
|
||||
role-ui-designer: composer-2.5-fast
|
||||
role-implementer: composer-2.5-fast
|
||||
role-reviewer: cursor-grok-4.5-high
|
||||
role-security-auditor: gpt-5.6-terra-medium
|
||||
role-design-system-auditor: cursor-grok-4.5-high
|
||||
role-a11y-auditor: cursor-grok-4.5-high
|
||||
role-doc-writer: auto
|
||||
escalate_to: claude-sonnet-5-thinking-medium
|
||||
escalate_to_premium: claude-4.6-opus-high-thinking
|
||||
never_premium:
|
||||
- role-reviewer
|
||||
- role-security-auditor
|
||||
- role-design-system-auditor
|
||||
- role-a11y-auditor
|
||||
- role-ui-designer
|
||||
- role-doc-writer
|
||||
---
|
||||
|
||||
# Convoy: tighten-scan-identify-hot-path
|
||||
|
||||
Phase 1 of `scanner-identify-upgrade`. Make the **existing** two-layer
|
||||
pipeline faster and more unique-match capable. No new vendors, no
|
||||
detection rewrite, no embeddings.
|
||||
|
||||
## Why
|
||||
|
||||
Live telemetry (see umbrella) says Layer-1 is fast (p50 158ms) and
|
||||
almost useless as an identifier (5.8% auto-match, 76.7% escalate). The
|
||||
user still waits 3.5s of hold-still before that cheap path runs. L2
|
||||
Gemini is 1.7s p50 and spends a second token on every disambiguation
|
||||
refine. Collector number — the unique printing key L2 already uses —
|
||||
is never read on L1.
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope
|
||||
|
||||
- **Hold-still constants** in `lib/scanner-card-detection.js`:
|
||||
`MIN_FIRST_SEEN_MS_FOR_VERIFY` 2500 → ~800;
|
||||
`MIN_STABLE_COUNT_FOR_VERIFY` 6 → 3. Keep tests in
|
||||
`test/lib/scanner-card-detection.test.js` in lockstep.
|
||||
- **Name-strip OCR mode:** Tesseract PSM 7 (single line), JPEG quality
|
||||
0.92 on the crop used for OCR (`lib/ocr-worker.js`,
|
||||
`captureCardRegionFromVideo` quality if shared).
|
||||
- **Collector-number strip:** second crop of the bottom ~18% of the
|
||||
card; pass number (+ optional set hint) into `matchTextInCatalog`.
|
||||
When name + number uniquely match a `cards` row, auto-match even if
|
||||
many printings share the name.
|
||||
- **Structured Gemini output** in `lib/scan-vision.js`:
|
||||
`response_format` JSON schema (or Gateway equivalent). Delete the
|
||||
confidence-30 regex fallback as the success path; keep a hard parse
|
||||
error → `needs_input`.
|
||||
- **Model A/B:** default stays `google/gemini-2.5-flash-lite`. Document
|
||||
and allow `SCAN_VISION_MODEL=google/gemini-3.1-flash-lite` (GA on
|
||||
Vercel AI Gateway, ~2.5× input price). Do not flip prod default in
|
||||
this convoy unless Architect + a short identify bake-off say so.
|
||||
- **Stop automatic L2 refine** in `use-scanner-identification.js` when
|
||||
L1 already returned a printing list. User picks; Gemini refine is
|
||||
opt-in or only when L1 had no set/number hint.
|
||||
- **`docs/SCHEMA_MAP.md`:** `scan_attempts.layer` is 1 = Tesseract
|
||||
(shipped), not "Tesseract future".
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Detector / OpenCV / YOLO / perspective warp →
|
||||
`improve-scan-card-detection`.
|
||||
- pgvector / CLIP / catalog embeddings →
|
||||
`scan-visual-catalog-search`.
|
||||
- Changing `checkScanRateLimit` (5/min). Revisit after re-measure.
|
||||
- Scanner chrome, cart, checkout (`scanner-mobile-checkout`).
|
||||
- Replacing `tesseract.js`.
|
||||
|
||||
## Roles invoked
|
||||
|
||||
1. `role-ux-reviewer` — hold-still feel; when the picker still appears;
|
||||
no new screens.
|
||||
2. `role-architect` — ratify constant values, L1 match rules when a
|
||||
collector number is present, JSON schema shape, whether the model
|
||||
default flips. Write briefs + `slice_dependencies`.
|
||||
3. `role-implementer` — per brief.
|
||||
4. Audit: `role-reviewer` + `role-security-auditor`. Skip design-system
|
||||
+ a11y unless a brief unexpectedly touches `components/`.
|
||||
|
||||
## Todos
|
||||
|
||||
- [x] UX: confirm 800ms / 3-frame gate does not cause double-scans
|
||||
- [x] Architect: briefs + match-rule when number is present
|
||||
- [x] Brief A — detection constants + tests
|
||||
- [x] Brief B — OCR strips (name PSM 7 + collector number) +
|
||||
`card-text-match.js` / `identify-by-text`
|
||||
- [x] Brief C — `scan-vision.js` JSON schema + optional model id
|
||||
- [x] Brief D — skip automatic disambiguation Gemini refine
|
||||
- [x] SCHEMA_MAP layer-1 wording
|
||||
- [ ] Re-query `scan_attempts` after preview traffic
|
||||
|
||||
## Likely file ownership (Architect will lock)
|
||||
|
||||
| Area | Files |
|
||||
| --- | --- |
|
||||
| Gates | `lib/scanner-card-detection.js`, `test/lib/scanner-card-detection.test.js` |
|
||||
| OCR | `lib/ocr-worker.js` |
|
||||
| L1 match | `lib/card-text-match.js`, `pages/api/cards/identify-by-text.js` |
|
||||
| L2 vision | `lib/scan-vision.js`, `pages/api/scan/identify.js` |
|
||||
| Refine | `lib/use-scanner-identification.js`, `lib/scanner-card-identify.js` |
|
||||
| Docs | `docs/SCHEMA_MAP.md` |
|
||||
|
||||
Briefs A and C are file-disjoint and can run in parallel. B depends on
|
||||
nothing if it does not retouch detection constants. D depends on B
|
||||
(needs L1 payload shape).
|
||||
|
||||
## Multitask dispatch
|
||||
|
||||
Planning: UX → Architect (serial).
|
||||
|
||||
After Architect, if `slice_dependencies` marks A+C `depends_on: []`:
|
||||
|
||||
```
|
||||
/multitask role-implementer briefs A, C
|
||||
```
|
||||
|
||||
Then B, then D.
|
||||
|
||||
Audit group id: `audit-tighten-scan-identify-hot-path-<pr>`.
|
||||
|
||||
## CI impact
|
||||
|
||||
| Workflow / job | Behavior |
|
||||
| --- | --- |
|
||||
| `ci.yml` test | Fires — detection + identify unit tests change |
|
||||
| `visual-diff.yml` | Should **not** fire if `components/` / `pages/` / `styles/` untouched |
|
||||
| `preview-smoke.yml` | Fires |
|
||||
| `forbidden-client-side-llm-keys` | Must stay green — no client Gemini URLs |
|
||||
|
||||
## Operator action
|
||||
|
||||
Optional: set `SCAN_VISION_MODEL=google/gemini-3.1-flash-lite` on a
|
||||
preview to bake off against 2.5 Flash Lite. Leave prod on 2.5 until
|
||||
the bake-off.
|
||||
|
||||
## Conductor notes
|
||||
|
||||
Do not treat Tesseract replacement as in-scope if L1 escalate stays
|
||||
high after collector-number matching — that is a crop/detection
|
||||
problem (#2) or a visual-search problem (#3), not an OCR-engine
|
||||
problem. The six historical L1 matches already show Tesseract can
|
||||
read a name when the crop is clean; it cannot pick a printing.
|
||||
|
||||
## UX
|
||||
|
||||
No new routes or screens. Changes are timing + fewer automatic Gemini
|
||||
calls during disambiguation.
|
||||
|
||||
### Existing components to reuse
|
||||
|
||||
- `ScannerDisambiguation` (`components/scanner/ScannerDisambiguation.js`) —
|
||||
printing picker when L1/L2 cannot unique-match.
|
||||
- `ScannerToast` / status copy around tracked card (`components/scanner/ScannerCamera.js`)
|
||||
— continue showing detecting → verifying → found without new chrome.
|
||||
- `ScanDisambiguationDialog` pattern if still referenced — do not fork a
|
||||
second picker.
|
||||
|
||||
### Design direction alignment
|
||||
|
||||
Liquid Glass tokens unchanged. No new overlays. Faster identify should
|
||||
feel like the camera "wakes up" sooner (H1 visibility of system status).
|
||||
|
||||
### Patterns to follow
|
||||
|
||||
- Disambiguation stays a bottom sheet / modal pause (`verificationPausedRef`
|
||||
already wired in `use-scanner-identification.js`).
|
||||
- Error toasts debounced via `reportScannerError` (4s gap) — keep that
|
||||
when tightening gates so double-failures do not spam.
|
||||
|
||||
### A11y constraints
|
||||
|
||||
- No new interactive controls in this convoy.
|
||||
- Existing disambiguation list must remain keyboard-selectable (hand off
|
||||
unchanged to a11y auditor if Brief D touches the sheet).
|
||||
|
||||
### Interaction patterns
|
||||
|
||||
| Pattern | Requirement | Heuristic |
|
||||
| --- | --- | --- |
|
||||
| Hold-still gate | **required** — 800ms + 3 stable frames; user sees brackets sooner | H1 feedback |
|
||||
| Auto-match | **required** — collector number should skip picker for multi-printing names | H6 recognition vs recall |
|
||||
| Disambiguation refine | **required** — stop silent second Gemini call when L1 already opened picker; user tap only | H3 user control |
|
||||
| Rate-limit toast | **required** — unchanged copy when L2 throttled | H9 error recovery |
|
||||
| Loading during verify | **nice-to-have** — optional subtle "Reading…" on card status if trivial | H1 |
|
||||
|
||||
### Anti-patterns
|
||||
|
||||
- Adding a setup screen or settings toggle for gate timing (H4 consistency).
|
||||
- Auto-picking a printing without number evidence when multiple exact-name
|
||||
rows exist (H6).
|
||||
- Blocking the camera on L1 OCR worker load (H1 — worker stays async).
|
||||
|
||||
### Mobile / responsive
|
||||
|
||||
Gate timing applies equally on phone and desktop; no viewport-specific
|
||||
branches in this convoy.
|
||||
|
||||
## Architecture
|
||||
|
||||
### File plan
|
||||
|
||||
| File | Action | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `lib/scanner-card-detection.js` | modified | 800ms / 3-frame verify gates |
|
||||
| `test/lib/scanner-card-detection.test.js` | modified | Lock new constants |
|
||||
| `lib/ocr-worker.js` | modified | PSM 7 name strip + bottom collector strip |
|
||||
| `lib/card-text-match.js` | modified | Name + collector number unique match |
|
||||
| `pages/api/cards/identify-by-text.js` | modified | Accept `cardNumber` body field |
|
||||
| `test/lib/card-text-match.test.js` | new | Unit tests for number path (mock-free helpers) |
|
||||
| `lib/scanner-card-identify.js` | modified | Pass number to L1; JPEG 0.92; `fromLayer1` flag |
|
||||
| `lib/scan-vision.js` | modified | JSON schema response; stricter parse |
|
||||
| `pages/api/scan/identify.js` | modified | Handle vision parse failures → needs_input |
|
||||
| `lib/use-scanner-identification.js` | modified | Skip auto Gemini refine when `fromLayer1` |
|
||||
| `docs/SCHEMA_MAP.md` | modified | Layer 1 = Tesseract (shipped) |
|
||||
|
||||
### API surface
|
||||
|
||||
**POST `/api/cards/identify-by-text`** (modified)
|
||||
|
||||
- Auth: `getUserFromRequest` → 401
|
||||
- Body: `{ ocrText, ocrConfidence?, cardNumber?, game? }`
|
||||
- Response: unchanged envelope + optional `cardNumber` echo in `ocr` meta
|
||||
- Rate limit: none (unchanged)
|
||||
|
||||
**POST `/api/scan/identify`** (modified behavior only)
|
||||
|
||||
- On vision JSON parse failure: 200 with `needsUserInput: true` instead of
|
||||
silent regex fallback at confidence 30
|
||||
|
||||
### Schema diff
|
||||
|
||||
No migration. `scan_attempts.layer` semantics documented only.
|
||||
|
||||
### Test plan
|
||||
|
||||
- Update `scanner-card-detection.test.js` thresholds (800ms, stable 3).
|
||||
- New `card-text-match.test.js` for `extractCollectorNumberCandidate` and
|
||||
collector-aware disambiguation resolution (pure functions exported for test).
|
||||
- Extend `scanner-card-identify.test.js` for `fromLayer1` disambiguation
|
||||
payload if added to pure helpers.
|
||||
- Run `npm run test:run` + `npm run lint`.
|
||||
|
||||
### Risks
|
||||
|
||||
| Risk | Mitigation |
|
||||
| --- | --- |
|
||||
| Faster gate double-scans same card | `scanAttempts < 1` + tracker negative state unchanged |
|
||||
| OCR number strip reads set symbol garbage | Normalize with `extractCollectorNumberCandidate`; fall back to name-only |
|
||||
| Gateway rejects `response_format` | Catch 400, log once, fall back to prose prompt (Architect: implementer may feature-detect) |
|
||||
| L1 skip-refine hides catalog-gap auto-submit | Refine still runs for L2 disambiguation only |
|
||||
|
||||
### Decomposition
|
||||
|
||||
| Brief # | Title | Files | Depends on | Size |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | Faster verify gates | detection + test | — | S |
|
||||
| 2 | Collector number L1 | ocr-worker, card-text-match, identify-by-text, scanner-card-identify, test | — | M |
|
||||
| 3 | Structured vision JSON | scan-vision, identify API | — | S |
|
||||
| 4 | Skip L1 disambiguation refine | use-scanner-identification, SCHEMA_MAP | 2 | S |
|
||||
|
||||
### Slice dependencies
|
||||
|
||||
```yaml
|
||||
slice_dependencies:
|
||||
- brief: 1
|
||||
depends_on: []
|
||||
files:
|
||||
- lib/scanner-card-detection.js
|
||||
- test/lib/scanner-card-detection.test.js
|
||||
- brief: 2
|
||||
depends_on: []
|
||||
files:
|
||||
- lib/ocr-worker.js
|
||||
- lib/card-text-match.js
|
||||
- pages/api/cards/identify-by-text.js
|
||||
- lib/scanner-card-identify.js
|
||||
- test/lib/card-text-match.test.js
|
||||
- brief: 3
|
||||
depends_on: []
|
||||
files:
|
||||
- lib/scan-vision.js
|
||||
- pages/api/scan/identify.js
|
||||
- brief: 4
|
||||
depends_on: [2]
|
||||
files:
|
||||
- lib/use-scanner-identification.js
|
||||
- docs/SCHEMA_MAP.md
|
||||
```
|
||||
|
||||
### Decisions (post-UX)
|
||||
|
||||
| # | Decision |
|
||||
| --- | --- |
|
||||
| D1 | Gate values: `MIN_FIRST_SEEN_MS_FOR_VERIFY = 800`, `MIN_STABLE_COUNT_FOR_VERIFY = 3`. |
|
||||
| D2 | Collector strip: bottom 18% of crop; PSM 7 for both strips. |
|
||||
| D3 | When exact-name multiple printings AND parsed collector number matches exactly one row → auto-match. |
|
||||
| D4 | `SCAN_VISION_MODEL` default stays `google/gemini-2.5-flash-lite`; structured JSON either way. |
|
||||
| D5 | Auto Gemini refine skipped when disambiguation opened from L1 (`result.layer === 1`). |
|
||||
|
||||
Human gate 1: **approved** (operator requested full pipeline run 2026-08-14).
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
convoy: tighten-scan-identify-hot-path
|
||||
brief_number: 1
|
||||
depends_on: []
|
||||
recommended_model: composer-2.5-fast
|
||||
model_tier: fast
|
||||
files:
|
||||
- lib/scanner-card-detection.js
|
||||
- test/lib/scanner-card-detection.test.js
|
||||
---
|
||||
|
||||
# Brief 1: Faster verify gates
|
||||
|
||||
## Goal
|
||||
|
||||
Cut hold-still wait from 3.5s to ~1.4s by lowering verify gate constants.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [ ] `MIN_FIRST_SEEN_MS_FOR_VERIFY = 800`
|
||||
- [ ] `MIN_STABLE_COUNT_FOR_VERIFY = 3`
|
||||
- [ ] Tests updated and green
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
---
|
||||
convoy: tighten-scan-identify-hot-path
|
||||
brief_number: 2
|
||||
depends_on: []
|
||||
recommended_model: composer-2.5-fast
|
||||
model_tier: fast
|
||||
files:
|
||||
- lib/ocr-worker.js
|
||||
- lib/card-text-match.js
|
||||
- pages/api/cards/identify-by-text.js
|
||||
- lib/scanner-card-identify.js
|
||||
- test/lib/card-text-match.test.js
|
||||
---
|
||||
|
||||
# Brief 2: Collector number on Layer 1
|
||||
|
||||
## Goal
|
||||
|
||||
OCR name + collector number strips; unique-match printings when number resolves one row.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [ ] PSM 7 on name strip; bottom 18% number strip
|
||||
- [ ] `matchTextInCatalog` accepts `cardNumber`
|
||||
- [ ] identify-by-text passes `cardNumber`
|
||||
- [ ] JPEG capture quality 0.92 for OCR path
|
||||
- [ ] Unit tests for number extraction helper
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
convoy: tighten-scan-identify-hot-path
|
||||
brief_number: 3
|
||||
depends_on: []
|
||||
recommended_model: composer-2.5-fast
|
||||
model_tier: fast
|
||||
files:
|
||||
- lib/scan-vision.js
|
||||
- pages/api/scan/identify.js
|
||||
---
|
||||
|
||||
# Brief 3: Structured Gemini JSON
|
||||
|
||||
## Goal
|
||||
|
||||
Request JSON schema from AI Gateway; fail loud on parse errors.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [ ] `response_format` json_schema on gateway call
|
||||
- [ ] No confidence-30 regex success path
|
||||
- [ ] identify route returns needsUserInput on parse failure
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
convoy: tighten-scan-identify-hot-path
|
||||
brief_number: 4
|
||||
depends_on: [2]
|
||||
recommended_model: composer-2.5-fast
|
||||
model_tier: fast
|
||||
files:
|
||||
- lib/use-scanner-identification.js
|
||||
- docs/SCHEMA_MAP.md
|
||||
---
|
||||
|
||||
# Brief 4: Skip L1 disambiguation refine
|
||||
|
||||
## Goal
|
||||
|
||||
Do not auto-call Gemini when L1 already opened the printing picker.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [ ] `fromLayer1` on disambiguation state when `result.layer === 1`
|
||||
- [ ] useEffect refine skipped when `fromLayer1`
|
||||
- [ ] SCHEMA_MAP documents layer 1 = Tesseract
|
||||
|
|
@ -261,7 +261,7 @@ Index: `idx_card_submissions_status (status, created_at DESC)`.
|
|||
|
||||
### scan_attempts
|
||||
|
||||
Per-scan telemetry for the identify pipeline (layer 2 = Gemini today).
|
||||
Per-scan telemetry for the identify pipeline (layer 1 = Tesseract + pg_trgm, layer 2 = Gemini).
|
||||
|
||||
| Column | Type | Notes |
|
||||
| --- | --- | --- |
|
||||
|
|
@ -269,7 +269,7 @@ Per-scan telemetry for the identify pipeline (layer 2 = Gemini today).
|
|||
| `user_id` | FK → `users` SET NULL | |
|
||||
| `ocr_text` | `TEXT` | |
|
||||
| `ocr_confidence` | `INTEGER` | |
|
||||
| `layer` | `INTEGER` default `2` | OCR layer (1 = Tesseract future) |
|
||||
| `layer` | `INTEGER` default `2` | Identify layer (`1` = browser Tesseract + pg_trgm, `2` = Gemini vision) |
|
||||
| `matched_card_id` | FK → `cards` SET NULL | |
|
||||
| `result_kind` | `VARCHAR(32)` | e.g. `'matched'`, `'disambiguation'`, `'submitted'`, `'not_a_card'` |
|
||||
| `latency_ms` | `INTEGER` | End-to-end identify latency |
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import { sql } from '@vercel/postgres';
|
||||
|
||||
import { findPrintingByCollectorNumber } from './card-number-utils.js';
|
||||
|
||||
const MATCH_THRESHOLD = 0.85;
|
||||
const DISAMBIGUATION_THRESHOLD = 0.6;
|
||||
const MIN_QUERY_LENGTH = 3;
|
||||
|
|
@ -64,11 +66,53 @@ async function querySimilarCards(query, game) {
|
|||
`;
|
||||
}
|
||||
|
||||
function resolveExactNameMatches(exactNameMatches, cardNumber, query) {
|
||||
if (exactNameMatches.length === 1) {
|
||||
return {
|
||||
type: 'matched',
|
||||
card: exactNameMatches[0],
|
||||
query,
|
||||
similarity: exactNameMatches[0].sim,
|
||||
message: `Matched "${exactNameMatches[0].name}" via text search`,
|
||||
};
|
||||
}
|
||||
|
||||
if (exactNameMatches.length > 1 && cardNumber) {
|
||||
const uniqueByNumber = findPrintingByCollectorNumber(exactNameMatches, cardNumber);
|
||||
if (uniqueByNumber) {
|
||||
return {
|
||||
type: 'matched',
|
||||
card: uniqueByNumber,
|
||||
query,
|
||||
similarity: uniqueByNumber.sim,
|
||||
message: `Matched "${uniqueByNumber.name}" (${uniqueByNumber.card_number}) via collector number`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (exactNameMatches.length > 1) {
|
||||
return {
|
||||
type: 'disambiguation',
|
||||
matches: exactNameMatches.slice(0, 8).map(mapCardRow),
|
||||
query,
|
||||
message: `Found ${exactNameMatches.length} printings of "${query}". Select the correct one.`,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fuzzy match OCR text against cards.name using pg_trgm similarity.
|
||||
*/
|
||||
export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence = null }) {
|
||||
export async function matchTextInCatalog({
|
||||
ocrText,
|
||||
cardNumber = null,
|
||||
game = null,
|
||||
ocrConfidence = null,
|
||||
}) {
|
||||
const query = extractNameCandidate(ocrText);
|
||||
const trimmedNumber = cardNumber && String(cardNumber).trim() ? String(cardNumber).trim() : null;
|
||||
|
||||
if (!query || query.length < MIN_QUERY_LENGTH) {
|
||||
return {
|
||||
|
|
@ -76,6 +120,7 @@ export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence =
|
|||
reason: 'OCR text too short for catalog match',
|
||||
query,
|
||||
ocrConfidence,
|
||||
cardNumber: trimmedNumber,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -88,6 +133,7 @@ export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence =
|
|||
reason: `No catalog match above ${DISAMBIGUATION_THRESHOLD} similarity for "${query}"`,
|
||||
query,
|
||||
ocrConfidence,
|
||||
cardNumber: trimmedNumber,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -96,14 +142,9 @@ export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence =
|
|||
(row) => row.name?.toLowerCase() === normalizedQuery
|
||||
);
|
||||
|
||||
// Same card name, multiple printings — always ask the user.
|
||||
if (exactNameMatches.length > 1) {
|
||||
return {
|
||||
type: 'disambiguation',
|
||||
matches: exactNameMatches.slice(0, 8).map(mapCardRow),
|
||||
query,
|
||||
message: `Found ${exactNameMatches.length} printings of "${query}". Select the correct one.`,
|
||||
};
|
||||
const exactResolution = resolveExactNameMatches(exactNameMatches, trimmedNumber, query);
|
||||
if (exactResolution) {
|
||||
return exactResolution;
|
||||
}
|
||||
|
||||
const top = candidates[0];
|
||||
|
|
|
|||
|
|
@ -5,6 +5,12 @@
|
|||
|
||||
let workerPromise = null;
|
||||
|
||||
const NAME_STRIP_PSM = '7';
|
||||
const NUMBER_STRIP_PSM = '7';
|
||||
const NAME_STRIP_HEIGHT_RATIO = 0.35;
|
||||
const NUMBER_STRIP_HEIGHT_RATIO = 0.18;
|
||||
const OCR_JPEG_QUALITY = 0.92;
|
||||
|
||||
async function getWorker() {
|
||||
if (!workerPromise) {
|
||||
workerPromise = (async () => {
|
||||
|
|
@ -12,53 +18,116 @@ async function getWorker() {
|
|||
const worker = await createWorker('eng', 1, {
|
||||
logger: () => {},
|
||||
});
|
||||
await worker.setParameters({
|
||||
tessedit_pageseg_mode: '6',
|
||||
});
|
||||
return worker;
|
||||
})();
|
||||
}
|
||||
return workerPromise;
|
||||
}
|
||||
|
||||
function cropNameStrip(imageDataUrl) {
|
||||
function cropCardStrip(imageDataUrl, { fromTop = true, heightRatio }) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
img.onload = () => {
|
||||
const stripHeight = Math.max(1, Math.floor(img.height * 0.35));
|
||||
const stripHeight = Math.max(1, Math.floor(img.height * heightRatio));
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = img.width;
|
||||
canvas.height = stripHeight;
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(img, 0, 0, img.width, stripHeight, 0, 0, img.width, stripHeight);
|
||||
resolve(canvas.toDataURL('image/jpeg', 0.92));
|
||||
const sourceY = fromTop ? 0 : Math.max(0, img.height - stripHeight);
|
||||
ctx.drawImage(
|
||||
img,
|
||||
0,
|
||||
sourceY,
|
||||
img.width,
|
||||
stripHeight,
|
||||
0,
|
||||
0,
|
||||
canvas.width,
|
||||
canvas.height
|
||||
);
|
||||
resolve(canvas.toDataURL('image/jpeg', OCR_JPEG_QUALITY));
|
||||
};
|
||||
img.onerror = () => reject(new Error('Failed to load image for OCR'));
|
||||
img.src = imageDataUrl;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the most likely collector number token from OCR of the bottom strip.
|
||||
* @param {string} rawText
|
||||
* @returns {string}
|
||||
*/
|
||||
export function extractCollectorNumberCandidate(rawText) {
|
||||
if (!rawText || typeof rawText !== 'string') return '';
|
||||
|
||||
const lines = rawText
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.replace(/\s+/g, ' ').trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const joined = lines.join(' ');
|
||||
const slashMatch = joined.match(/\b(\d{1,4}\s*\/\s*\d{1,4}[A-Za-z]?)\b/);
|
||||
if (slashMatch) {
|
||||
return slashMatch[1].replace(/\s+/g, '');
|
||||
}
|
||||
|
||||
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
||||
const line = lines[i];
|
||||
if (/\d/.test(line) && line.length <= 12) {
|
||||
return line;
|
||||
}
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* OCR the top name strip of a card crop.
|
||||
* @param {string} imageDataUrl - full card crop data URL
|
||||
* @returns {Promise<{ text: string, confidence: number }>}
|
||||
*/
|
||||
export async function recognizeCardNameStrip(imageDataUrl) {
|
||||
const fields = await recognizeCardFields(imageDataUrl);
|
||||
return {
|
||||
text: fields.nameText,
|
||||
confidence: fields.nameConfidence,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* OCR name strip + bottom collector-number strip.
|
||||
* @param {string} imageDataUrl
|
||||
* @returns {Promise<{ nameText: string, nameConfidence: number, cardNumber: string, numberConfidence: number }>}
|
||||
*/
|
||||
export async function recognizeCardFields(imageDataUrl) {
|
||||
if (typeof window === 'undefined') {
|
||||
return { text: '', confidence: 0 };
|
||||
return { nameText: '', nameConfidence: 0, cardNumber: '', numberConfidence: 0 };
|
||||
}
|
||||
|
||||
try {
|
||||
const stripUrl = await cropNameStrip(imageDataUrl);
|
||||
const [nameStripUrl, numberStripUrl] = await Promise.all([
|
||||
cropCardStrip(imageDataUrl, { fromTop: true, heightRatio: NAME_STRIP_HEIGHT_RATIO }),
|
||||
cropCardStrip(imageDataUrl, { fromTop: false, heightRatio: NUMBER_STRIP_HEIGHT_RATIO }),
|
||||
]);
|
||||
|
||||
const worker = await getWorker();
|
||||
const { data } = await worker.recognize(stripUrl);
|
||||
await worker.setParameters({ tessedit_pageseg_mode: NAME_STRIP_PSM });
|
||||
const nameResult = await worker.recognize(nameStripUrl);
|
||||
|
||||
await worker.setParameters({ tessedit_pageseg_mode: NUMBER_STRIP_PSM });
|
||||
const numberResult = await worker.recognize(numberStripUrl);
|
||||
|
||||
const cardNumber = extractCollectorNumberCandidate(numberResult.data.text || '');
|
||||
|
||||
return {
|
||||
text: (data.text || '').trim(),
|
||||
confidence: Math.round(data.confidence || 0),
|
||||
nameText: (nameResult.data.text || '').trim(),
|
||||
nameConfidence: Math.round(nameResult.data.confidence || 0),
|
||||
cardNumber,
|
||||
numberConfidence: Math.round(numberResult.data.confidence || 0),
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn('[ocr-worker] recognizeCardNameStrip failed:', error);
|
||||
return { text: '', confidence: 0 };
|
||||
console.warn('[ocr-worker] recognizeCardFields failed:', error);
|
||||
return { nameText: '', nameConfidence: 0, cardNumber: '', numberConfidence: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,45 +8,105 @@ HOLOGRAPHIC / FOIL CARDS: Many cards have reflective foil surfaces with glare or
|
|||
|
||||
CRITICAL: Only respond with card data if you can clearly identify a TRADING CARD in the image. Ignore random objects, books, papers, phone screens, screenshots, and non-card gaming items. Blurry images with no readable card frame should be rejected.
|
||||
|
||||
If you detect a trading card, extract information in this JSON format:
|
||||
Respond with JSON only (no markdown fences) using this shape:
|
||||
{
|
||||
"isCard": true,
|
||||
"cardName": "exact card name as printed on the card",
|
||||
"setName": "set name if visible",
|
||||
"setCode": "set code/symbol if visible",
|
||||
"cardNumber": "collector number if visible",
|
||||
"game": "mtg, pokemon, or lorcana (lowercase)",
|
||||
"cardType": "creature, instant, sorcery, trainer, etc.",
|
||||
"cardName": "exact card name",
|
||||
"setName": "set name or null",
|
||||
"setCode": "set code or null",
|
||||
"cardNumber": "collector number or null",
|
||||
"game": "mtg, pokemon, or lorcana",
|
||||
"cardType": "creature, instant, etc.",
|
||||
"rarity": "common, uncommon, rare, mythic, etc.",
|
||||
"manaCost": "mana cost if visible",
|
||||
"hp": "HP or power if visible",
|
||||
"abilities": ["list of abilities or attacks if clearly readable"],
|
||||
"manaCost": "mana cost or null",
|
||||
"hp": "HP or power or null",
|
||||
"abilities": ["ability strings"],
|
||||
"confidence": 85,
|
||||
"rawText": "all text visible on the card"
|
||||
"rawText": "all visible text",
|
||||
"reason": null
|
||||
}
|
||||
|
||||
If NO trading card is clearly visible, respond with:
|
||||
{
|
||||
"isCard": false,
|
||||
"cardName": null,
|
||||
"setName": null,
|
||||
"setCode": null,
|
||||
"cardNumber": null,
|
||||
"game": null,
|
||||
"cardType": null,
|
||||
"rarity": null,
|
||||
"manaCost": null,
|
||||
"hp": null,
|
||||
"abilities": [],
|
||||
"confidence": 0,
|
||||
"rawText": null,
|
||||
"reason": "No trading card detected in image"
|
||||
}
|
||||
|
||||
Be conservative — only extract data you can clearly read. Quality over quantity.`;
|
||||
|
||||
const CARD_VISION_SCHEMA = {
|
||||
name: 'card_scan_result',
|
||||
strict: true,
|
||||
schema: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
isCard: { type: 'boolean' },
|
||||
cardName: { type: ['string', 'null'] },
|
||||
setName: { type: ['string', 'null'] },
|
||||
setCode: { type: ['string', 'null'] },
|
||||
cardNumber: { type: ['string', 'null'] },
|
||||
game: { type: ['string', 'null'] },
|
||||
cardType: { type: ['string', 'null'] },
|
||||
rarity: { type: ['string', 'null'] },
|
||||
manaCost: { type: ['string', 'null'] },
|
||||
hp: { type: ['string', 'null'] },
|
||||
abilities: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
},
|
||||
confidence: { type: 'number' },
|
||||
rawText: { type: ['string', 'null'] },
|
||||
reason: { type: ['string', 'null'] },
|
||||
},
|
||||
required: [
|
||||
'isCard',
|
||||
'cardName',
|
||||
'setName',
|
||||
'setCode',
|
||||
'cardNumber',
|
||||
'game',
|
||||
'cardType',
|
||||
'rarity',
|
||||
'manaCost',
|
||||
'hp',
|
||||
'abilities',
|
||||
'confidence',
|
||||
'rawText',
|
||||
'reason',
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
export class VisionParseError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = 'VisionParseError';
|
||||
}
|
||||
}
|
||||
|
||||
function parseVisionJson(content) {
|
||||
if (!content || typeof content !== 'string') {
|
||||
throw new VisionParseError('Empty vision model response');
|
||||
}
|
||||
|
||||
const cleanContent = content.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
|
||||
try {
|
||||
return JSON.parse(cleanContent);
|
||||
} catch {
|
||||
const cardNameMatch = content.match(/card.*?name.*?[:"]\s*([^"'\n,}]+)/i);
|
||||
return {
|
||||
isCard: !!cardNameMatch,
|
||||
cardName: cardNameMatch ? cardNameMatch[1].trim() : null,
|
||||
confidence: 30,
|
||||
rawText: content,
|
||||
reason: 'Failed to parse structured response',
|
||||
};
|
||||
throw new VisionParseError('Failed to parse structured vision response');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -59,6 +119,29 @@ function normalizeGame(game) {
|
|||
return value;
|
||||
}
|
||||
|
||||
function buildGatewayBody(imageDataUrl, { structured = true } = {}) {
|
||||
const body = {
|
||||
model: DEFAULT_VISION_MODEL,
|
||||
temperature: 0.1,
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: CARD_PROMPT },
|
||||
{ type: 'image_url', image_url: { url: imageDataUrl } },
|
||||
],
|
||||
}],
|
||||
};
|
||||
|
||||
if (structured) {
|
||||
body.response_format = {
|
||||
type: 'json_schema',
|
||||
json_schema: CARD_VISION_SCHEMA,
|
||||
};
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Server-side vision analysis via Vercel AI Gateway. Requires AI_GATEWAY_API_KEY.
|
||||
* @param {string} imageDataUrl - data:image/jpeg;base64,... capture from scanner
|
||||
|
|
@ -73,25 +156,26 @@ export async function analyzeCardImage(imageDataUrl) {
|
|||
throw new Error('Invalid image data format');
|
||||
}
|
||||
|
||||
const response = await fetch(GATEWAY_URL, {
|
||||
let response = await fetch(GATEWAY_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: DEFAULT_VISION_MODEL,
|
||||
temperature: 0.1,
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: CARD_PROMPT },
|
||||
{ type: 'image_url', image_url: { url: imageDataUrl } },
|
||||
],
|
||||
}],
|
||||
}),
|
||||
body: JSON.stringify(buildGatewayBody(imageDataUrl)),
|
||||
});
|
||||
|
||||
if (!response.ok && response.status === 400) {
|
||||
response = await fetch(GATEWAY_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(buildGatewayBody(imageDataUrl, { structured: false })),
|
||||
});
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}));
|
||||
const message = errorData.error?.message || errorData.message || 'Unknown error';
|
||||
|
|
@ -101,7 +185,7 @@ export async function analyzeCardImage(imageDataUrl) {
|
|||
const data = await response.json();
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('No response from vision model');
|
||||
throw new VisionParseError('No response from vision model');
|
||||
}
|
||||
|
||||
const result = parseVisionJson(content);
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ export const SHAPE_DETECTION_INTERVAL_MS = 200;
|
|||
export const VERIFICATION_INTERVAL_MS = 1000;
|
||||
|
||||
/** Minimum stable frames before a tracked card is sent for identification. */
|
||||
export const MIN_STABLE_COUNT_FOR_VERIFY = 6;
|
||||
export const MIN_STABLE_COUNT_FOR_VERIFY = 3;
|
||||
|
||||
/** Minimum time (ms) a card must be tracked before verification. */
|
||||
export const MIN_FIRST_SEEN_MS_FOR_VERIFY = 2500;
|
||||
export const MIN_FIRST_SEEN_MS_FOR_VERIFY = 800;
|
||||
|
||||
/** Delay after camera start before detection loops begin. */
|
||||
export const DETECTION_START_DELAY_MS = 1000;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
/** Default margin (px) around tracked bounds when cropping a card capture. */
|
||||
export const CAPTURE_MARGIN_PX = 20;
|
||||
|
||||
/** JPEG quality for card crops sent to OCR / vision. */
|
||||
export const OCR_CAPTURE_JPEG_QUALITY = 0.92;
|
||||
|
||||
/** Vision rate-limit backoff duration (ms). */
|
||||
export const VISION_RATE_LIMIT_MS = 60_000;
|
||||
|
||||
|
|
@ -79,6 +82,7 @@ export function resolveIdentifyOutcome(result) {
|
|||
matches: result.matches,
|
||||
ocrMeta,
|
||||
message: result.message,
|
||||
fromLayer1: result.layer === 1,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -182,7 +186,7 @@ export function captureCardRegionFromVideo(video, canvas, bounds, margin = CAPTU
|
|||
canvas.height
|
||||
);
|
||||
|
||||
return canvas.toDataURL('image/jpeg', 0.8);
|
||||
return canvas.toDataURL('image/jpeg', OCR_CAPTURE_JPEG_QUALITY);
|
||||
}
|
||||
|
||||
export async function submitScanForReview({ imageData, name, candidateCardIds, authHeaders }) {
|
||||
|
|
@ -205,11 +209,11 @@ export async function submitScanForReview({ imageData, name, candidateCardIds, a
|
|||
return { ok: true, message: result.message };
|
||||
}
|
||||
|
||||
export async function fetchIdentifyByText({ ocrText, ocrConfidence, authHeaders }) {
|
||||
export async function fetchIdentifyByText({ ocrText, ocrConfidence, cardNumber, authHeaders }) {
|
||||
const response = await fetch('/api/cards/identify-by-text', {
|
||||
method: 'POST',
|
||||
headers: authHeaders,
|
||||
body: JSON.stringify({ ocrText, ocrConfidence }),
|
||||
body: JSON.stringify({ ocrText, ocrConfidence, cardNumber }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
|
@ -245,16 +249,17 @@ export async function fetchVisionIdentify(imageData, authHeaders) {
|
|||
* Returns { handled: true, outcome } when L1 resolves without escalation.
|
||||
*/
|
||||
export async function tryLayer1TextIdentify(imageData, authHeaders) {
|
||||
const { recognizeCardNameStrip } = await import('./ocr-worker.js');
|
||||
const ocr = await recognizeCardNameStrip(imageData);
|
||||
const { recognizeCardFields } = await import('./ocr-worker.js');
|
||||
const ocr = await recognizeCardFields(imageData);
|
||||
|
||||
if (ocr.text.length < 3) {
|
||||
if (ocr.nameText.length < 3) {
|
||||
return { handled: false };
|
||||
}
|
||||
|
||||
const l1 = await fetchIdentifyByText({
|
||||
ocrText: ocr.text,
|
||||
ocrConfidence: ocr.confidence,
|
||||
ocrText: ocr.nameText,
|
||||
ocrConfidence: ocr.nameConfidence,
|
||||
cardNumber: ocr.cardNumber || undefined,
|
||||
authHeaders,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -131,6 +131,7 @@ export function useScannerIdentification({
|
|||
candidates: outcome.matches,
|
||||
ocrMeta: outcome.ocrMeta,
|
||||
message: outcome.message,
|
||||
fromLayer1: Boolean(outcome.fromLayer1),
|
||||
});
|
||||
break;
|
||||
case 'notice':
|
||||
|
|
@ -179,6 +180,7 @@ export function useScannerIdentification({
|
|||
|
||||
useEffect(() => {
|
||||
if (!disambiguation?.imageData) return;
|
||||
if (disambiguation.fromLayer1) return;
|
||||
if (Date.now() < visionCooldownUntilRef.current) return;
|
||||
|
||||
const refineKey = disambiguation.cardTracker?.id ?? 'modal';
|
||||
|
|
|
|||
|
|
@ -32,19 +32,20 @@ export default async function handler(req, res) {
|
|||
return res.status(401).json({ error: 'Authentication required' });
|
||||
}
|
||||
|
||||
const { ocrText, ocrConfidence, game } = req.body || {};
|
||||
const { ocrText, ocrConfidence, cardNumber, game } = req.body || {};
|
||||
|
||||
if (!ocrText || typeof ocrText !== 'string') {
|
||||
return res.status(400).json({ error: 'ocrText is required' });
|
||||
}
|
||||
|
||||
const matchResult = await matchTextInCatalog({ ocrText, game, ocrConfidence });
|
||||
const matchResult = await matchTextInCatalog({ ocrText, cardNumber, game, ocrConfidence });
|
||||
const latencyMs = Date.now() - startedAt;
|
||||
|
||||
const ocrMeta = {
|
||||
confidence: ocrConfidence ?? null,
|
||||
rawText: ocrText,
|
||||
query: matchResult.query || ocrText,
|
||||
cardNumber: matchResult.cardNumber ?? cardNumber ?? null,
|
||||
};
|
||||
|
||||
if (matchResult.type === 'escalate') {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { getUserFromRequest } from '../../../lib/permission-middleware';
|
||||
import { checkScanRateLimit } from '../../../lib/rate-limit.js';
|
||||
import { analyzeCardImage } from '../../../lib/scan-vision.js';
|
||||
import { analyzeCardImage, VisionParseError } from '../../../lib/scan-vision.js';
|
||||
import { matchCardInCatalog, logScanAttempt } from '../../../lib/card-catalog-match.js';
|
||||
|
||||
function scanErrorResponse(error) {
|
||||
|
|
@ -42,6 +42,18 @@ function scanErrorResponse(error) {
|
|||
};
|
||||
}
|
||||
|
||||
if (error instanceof VisionParseError || msg.includes('Failed to parse structured vision')) {
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
isCard: true,
|
||||
card: null,
|
||||
needsUserInput: true,
|
||||
message: 'Could not read card details from scan — try again or pick from matches.',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 500,
|
||||
body: { error: 'Internal server error' },
|
||||
|
|
|
|||
16
test/lib/card-text-match.test.js
Normal file
16
test/lib/card-text-match.test.js
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { extractCollectorNumberCandidate } from '../../lib/ocr-worker.js';
|
||||
|
||||
describe('extractCollectorNumberCandidate', () => {
|
||||
it('extracts slash-style collector numbers', () => {
|
||||
expect(extractCollectorNumberCandidate('015 / 208\nRare')).toBe('015/208');
|
||||
});
|
||||
|
||||
it('extracts compact numbers from the last numeric line', () => {
|
||||
expect(extractCollectorNumberCandidate('Some set symbol\n123/456')).toBe('123/456');
|
||||
});
|
||||
|
||||
it('returns empty for non-numeric noise', () => {
|
||||
expect(extractCollectorNumberCandidate('Creature — Human')).toBe('');
|
||||
});
|
||||
});
|
||||
|
|
@ -97,16 +97,16 @@ describe('selectCardsReadyForVerification', () => {
|
|||
const ready = {
|
||||
id: 1,
|
||||
status: 'detecting',
|
||||
stableCount: 6,
|
||||
stableCount: 3,
|
||||
scanAttempts: 0,
|
||||
firstSeen: now - 3000,
|
||||
firstSeen: now - 900,
|
||||
};
|
||||
const tooFresh = {
|
||||
id: 2,
|
||||
status: 'detecting',
|
||||
stableCount: 6,
|
||||
stableCount: 3,
|
||||
scanAttempts: 0,
|
||||
firstSeen: now - 1000,
|
||||
firstSeen: now - 500,
|
||||
};
|
||||
|
||||
expect(selectCardsReadyForVerification([ready, tooFresh], now)).toEqual([ready]);
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ describe('resolveIdentifyOutcome', () => {
|
|||
matches,
|
||||
ocrMeta: expect.objectContaining({ abilities: [] }),
|
||||
message: 'Pick one',
|
||||
fromLayer1: false,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -63,6 +64,18 @@ describe('resolveIdentifyOutcome', () => {
|
|||
message: 'Blurry',
|
||||
});
|
||||
});
|
||||
|
||||
it('marks L1 disambiguation so Gemini refine is skipped', () => {
|
||||
const outcome = resolveIdentifyOutcome({
|
||||
isCard: true,
|
||||
layer: 1,
|
||||
needsUserSelection: true,
|
||||
matches: [{ id: 1, name: 'Bolt' }],
|
||||
message: 'Pick one',
|
||||
});
|
||||
|
||||
expect(outcome.fromLayer1).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildScannedCardPayload', () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue