From 0b4f419f493484315134be423df031889b283163 Mon Sep 17 00:00:00 2001 From: varutasu <104105839+varutasu@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:21:20 -0500 Subject: [PATCH] =?UTF-8?q?Scanner=20identify=20upgrade=20=E2=80=94=20Phas?= =?UTF-8?q?e=201=20hot=20path=20(#156)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs(convoy): seed scanner identify upgrade epic and sub-convoys Baseline scan_attempts telemetry and three-phase plan for faster, more accurate card identification without touching scanner chrome. Co-authored-by: Cursor * feat(scanner): tighten Layer-1 identify hot path (Phase 1) Cut verify hold-still gates, OCR collector numbers on Layer 1, request structured Gemini JSON, and skip automatic L2 refine when L1 opens the printing picker. Includes convoy UX/architecture briefs and unit tests. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- .convoys/.metrics.jsonl | 5 + .convoys/improve-scan-card-detection.md | 151 ++++++++ .convoys/scan-visual-catalog-search.md | 161 +++++++++ .convoys/scanner-identify-upgrade.md | 207 +++++++++++ .convoys/ship-readiness.md | 1 + .convoys/tighten-scan-identify-hot-path.md | 328 ++++++++++++++++++ .../brief-1-faster-verify-gates.md | 22 ++ .../brief-2-collector-number-l1.md | 27 ++ .../brief-3-structured-vision-json.md | 22 ++ .../brief-4-skip-l1-refine.md | 22 ++ docs/SCHEMA_MAP.md | 4 +- lib/card-text-match.js | 59 +++- lib/ocr-worker.js | 97 +++++- lib/scan-vision.js | 148 ++++++-- lib/scanner-card-detection.js | 4 +- lib/scanner-card-identify.js | 21 +- lib/use-scanner-identification.js | 2 + pages/api/cards/identify-by-text.js | 5 +- pages/api/scan/identify.js | 14 +- test/lib/card-text-match.test.js | 16 + test/lib/scanner-card-detection.test.js | 8 +- test/lib/scanner-card-identify.test.js | 13 + 22 files changed, 1263 insertions(+), 74 deletions(-) create mode 100644 .convoys/improve-scan-card-detection.md create mode 100644 .convoys/scan-visual-catalog-search.md create mode 100644 .convoys/scanner-identify-upgrade.md create mode 100644 .convoys/tighten-scan-identify-hot-path.md create mode 100644 .convoys/tighten-scan-identify-hot-path/brief-1-faster-verify-gates.md create mode 100644 .convoys/tighten-scan-identify-hot-path/brief-2-collector-number-l1.md create mode 100644 .convoys/tighten-scan-identify-hot-path/brief-3-structured-vision-json.md create mode 100644 .convoys/tighten-scan-identify-hot-path/brief-4-skip-l1-refine.md create mode 100644 test/lib/card-text-match.test.js diff --git a/.convoys/.metrics.jsonl b/.convoys/.metrics.jsonl index d58efb8..3c48ae7 100644 --- a/.convoys/.metrics.jsonl +++ b/.convoys/.metrics.jsonl @@ -79,6 +79,10 @@ {"ts": "2026-08-15T00:14:27Z", "role": "role-ia-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 95, "model": "composer-2.5-fast", "model_tier": "fast"} {"ts": "2026-08-15T00:15:31Z", "role": "role-ui-designer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 90, "model": "composer-2.5-fast", "model_tier": "fast"} {"ts": "2026-08-15T00:16:36Z", "role": "role-ux-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 180, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "scanner-identify-upgrade", "repo": "scanner-identify-upgrade", "skip_flags": [], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "visual", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T00:18:01Z", "role": "role-conductor", "convoy": "scan-visual-catalog-search", "repo": "scanner-identify-upgrade", "skip_flags": ["ia", "ui-design", "visual", "a11y", "design", "flag"], "classification": "feature", "duration_s": 720, "model": "composer-2.5-fast", "model_tier": "fast"} {"ts": "2026-08-15T00:18:40Z", "role": "role-architect", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 420, "model": "composer-2.5", "model_tier": "standard"} {"ts": "2026-08-15T00:19:56Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} {"ts": "2026-08-15T00:20:23Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 2, "duration_s": 120, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} @@ -88,4 +92,5 @@ {"ts": "2026-08-15T00:25:54Z", "role": "role-reviewer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 210, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} {"ts": "2026-08-15T00:26:05Z", "role": "role-a11y-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 78, "model": "cursor-grok-4.5-high", "model_tier": "audit"} {"ts": "2026-08-15T00:26:08Z", "role": "role-design-system-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 82, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T00:55:56Z", "role": "role-reviewer", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 0, "duration_s": 180, "multitask_group": "audit-tighten-scan-identify-hot-path-uncommitted", "model": "cursor-grok-4.5-high", "model_tier": "fast"} {"ts": "2026-08-15T00:56:39Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} diff --git a/.convoys/improve-scan-card-detection.md b/.convoys/improve-scan-card-detection.md new file mode 100644 index 0000000..3840e95 --- /dev/null +++ b/.convoys/improve-scan-card-detection.md @@ -0,0 +1,151 @@ +--- +name: improve-scan-card-detection +classification: feature +success_metric: | + After ship, L2 result_kind=not_a_card ≤15% of L2 (was 44.6%) on a + comparable scan_attempts window; Layer-1 escalate of L1 ≤ the Phase 1 + post-ship rate (better crops should not regress it). +skip: + - ia + - ui-design + - flag +status: open +created: 2026-08-14 +depends_on: + - tighten-scan-identify-hot-path +umbrella: scanner-identify-upgrade +model_policy: + default_session: auto + roles: + role-conductor: composer-2.5-fast + role-architect: composer-2.5 + role-ia-architect: composer-2.5-fast + role-ux-reviewer: composer-2.5-fast + role-ui-designer: composer-2.5-fast + role-implementer: composer-2.5-fast + role-reviewer: cursor-grok-4.5-high + role-security-auditor: gpt-5.6-terra-medium + role-design-system-auditor: cursor-grok-4.5-high + role-a11y-auditor: cursor-grok-4.5-high + role-doc-writer: auto + escalate_to: claude-sonnet-5-thinking-medium + escalate_to_premium: claude-4.6-opus-high-thinking + never_premium: + - role-reviewer + - role-security-auditor + - role-design-system-auditor + - role-a11y-auditor + - role-ui-designer + - role-doc-writer +--- + +# Convoy: improve-scan-card-detection + +Phase 2 of `scanner-identify-upgrade`. Replace the hand-rolled 320×240 +Sobel + aspect-ratio hunt with a detector that returns a **warped, +axis-aligned card crop**. Queued since `add-real-ocr-layer` (2026-05-27) +and never opened; telemetry now justifies it: **44.6% of Gemini calls +are `not_a_card`**. + +`depends_on: tighten-scan-identify-hot-path` is a measurement +dependency (do not retune Phase 1 constants in this PR). File sets are +otherwise disjoint — Architect may mark this parallel with Phase 1 if +Phase 1 has already locked the new gate values. + +## Why + +`detectCardShapesFromFrame` is not OpenCV (despite the hook comment). +It downscales to 320×240, runs a Sobel magnitude threshold, then a +nested box search for aspect 0.63–0.77. There is no four-corner +homography. Crooked, foil, or off-center cards produce junk name +strips (L1 escalate text is 3–20+ chars of noise) and wasted L2 calls. + +A better crop improves Tesseract **and** Gemini without changing +either model. + +## Scope + +### In scope + +- Replace or wrap `detectCardShapesFromFrame` so tracked bounds are + card-shaped **and** a perspective-corrected JPEG can be produced for + identify. +- Architect picks **one** in-browser approach (do not land both): + 1. **OpenCV.js contours + approxPolyDP + warpPerspective** — no + training, larger WASM. + 2. **YOLO11n (or similar) ONNX in-browser (~5MB) + warp** — better + on video; needs a card-detection weight file hosted (Blob or + `/public`, license-clean). +- Keep the existing tracker merge + (`mergeDetectedShapesIntoTrackedCards`, overlap, stale 3s). +- Overlay brackets in `components/scanner/ScannerCamera.js` keep + consuming `{x,y,width,height}` (or four corners mapped to a rect). + Do not redesign the overlay. +- Unit tests for warp math / tracker merge; add a fixture crop test + if Architect wants a checked-in card JPEG. + +### Out of scope + +- Identify / Gemini / Tesseract / catalog match (Phase 1 + 3). +- Python/CUDA microservice, PaddleOCR server, Roboflow-hosted detect. +- Scanner chrome / cart (`scanner-mobile-checkout`). +- Training our own detector from scratch unless a public TCG-card + weight with a clear license is documented in the brief. + +## Roles invoked + +1. `role-ux-reviewer` — time-to-bracket, false-positive boxes, dual-card + frames. +2. `role-architect` — OpenCV.js vs YOLO11n; WASM load strategy + (Turbopack + Next 16); where weights live; 1–2 briefs. +3. `role-implementer`. +4. Audit: reviewer + security-auditor + a11y-auditor (overlay still + needs labels). Design-system only if overlay styling changes. + +## Todos + +- [ ] Architect: pick detector; document WASM / weight budget +- [ ] Brief 1 — detect + warp library + tests +- [ ] Brief 2 — wire `use-camera-scanner.js` + crop used by + `identifyTrackedCardCapture` +- [ ] Confirm `scanner-mobile-checkout` overlay still maps bounds +- [ ] Re-measure L2 `not_a_card` share + +## Likely file ownership + +| Area | Files | +| --- | --- | +| Detect | `lib/scanner-card-detection.js`, `test/lib/scanner-card-detection.test.js` | +| Warp helper | new `lib/scanner-card-warp.js` (if Architect splits) | +| Camera loop | `lib/use-camera-scanner.js` | +| Crop consumer | `lib/scanner-card-identify.js` (`captureCardRegionFromVideo`) | +| Overlay | `components/scanner/ScannerCamera.js` only if bounds shape changes | + +Do not edit `lib/ocr-worker.js` or `lib/scan-vision.js` here. + +## Multitask dispatch + +Serial unless Architect splits detect-lib vs camera-wire with disjoint +files. + +Audit group id: `audit-improve-scan-card-detection-`. + +## CI impact + +| Workflow / job | Behavior | +| --- | --- | +| `visual-diff.yml` | **May fire** if `ScannerCamera.js` changes | +| `preview-smoke.yml` | Fires | +| `ci.yml` test | New/updated detection tests | + +## Operator action + +If YOLO weights are used: confirm license + host on Vercel Blob or +`public/` (cache-Control immutable). No new secrets. + +## Conductor notes + +Do not drop OpenCV.js in only to reimplement the current rectangle +hunt. Success is a **rectified card image**, not a prettier box. +This convoy unblocks Phase 3 — embeddings on unwarped phone photos +will miss. diff --git a/.convoys/scan-visual-catalog-search.md b/.convoys/scan-visual-catalog-search.md new file mode 100644 index 0000000..f94b845 --- /dev/null +++ b/.convoys/scan-visual-catalog-search.md @@ -0,0 +1,161 @@ +--- +name: scan-visual-catalog-search +classification: feature +success_metric: | + After catalog embeddings exist and the new kNN path is on, ≥50% of + legitimate card scans (scan_attempts excluding not_a_card) auto-match + a printing with no picker — the original add-real-ocr-layer 70% + target, measured honestly. Gemini L2 share of attempts falls vs the + Phase 1 post-ship baseline. +skip: + - ia + - ui-design + - visual + - a11y + - design + - flag +status: open +created: 2026-08-14 +depends_on: + - improve-scan-card-detection +umbrella: scanner-identify-upgrade +model_policy: + default_session: auto + roles: + role-conductor: composer-2.5-fast + role-architect: composer-2.5 + role-ia-architect: composer-2.5-fast + role-ux-reviewer: composer-2.5-fast + role-ui-designer: composer-2.5-fast + role-implementer: composer-2.5-fast + role-reviewer: cursor-grok-4.5-high + role-security-auditor: gpt-5.6-terra-medium + role-design-system-auditor: cursor-grok-4.5-high + role-a11y-auditor: cursor-grok-4.5-high + role-doc-writer: auto + escalate_to: claude-sonnet-5-thinking-medium + escalate_to_premium: claude-4.6-opus-high-thinking + never_premium: + - role-reviewer + - role-security-auditor + - role-design-system-auditor + - role-a11y-auditor + - role-ui-designer + - role-doc-writer +--- + +# Convoy: scan-visual-catalog-search + +Phase 3 of `scanner-identify-upgrade`. Identify cards by **what they +look like**, not by reading the name. This is how Manabox / Delver-style +scanners get printing-accurate matches on foil and alt-art. + +Blocked on `improve-scan-card-detection`: embeddings of unwarped +phone crops will not match catalog `image_url`s reliably. + +## Why + +Name-only `pg_trgm` cannot distinguish printings. Tesseract fails on +foil and stylized type even when it emits 8–20 characters (76.7% L1 +escalate with non-empty text). Gemini can read a name but still +returns a picker when set/number are missing, and costs 5/min. + +The catalog already stores `cards.image_url` for every imported +printing. Precompute an embedding per row; at scan time embed the +warped crop and take top-k cosine. OCR / Gemini become hints and +unknown-card fallback. + +## Scope + +### In scope + +- **Migration:** `pgvector` (confirm Neon availability) + + `cards.embedding` (or a side table `card_embeddings`) + ANN index. + Update `docs/SCHEMA_MAP.md`. +- **Offline embed job:** new dated script or `npm run` task that + reads `image_url`, writes vectors. Idempotent. Rate-limit the + embedding provider. Do **not** edit historical `scripts/add-*.js`. +- **Architect picks the embedder** (one): + 1. Gateway embedding model (same `AI_GATEWAY_API_KEY`, server-only). + 2. In-browser MobileCLIP-S2 / SigLIP ONNX for the *query* crop, with + catalog vectors baked or fetched — only if weight license + size + are acceptable. +- **Identify path:** new Layer-0 (or replace L1) — kNN then + auto-match / disambiguate / escalate to existing L2. Log + `scan_attempts.layer = 0` (or Architect-ratified value). +- **Similarity thresholds** analogous to 0.85 / 0.60, tuned on a + held-out set of scan captures if any exist in Blob. +- Auth + rate-limit on any new route. No client-side API keys + (`forbidden-client-side-llm-keys` must stay green). + +### Out of scope + +- Replacing Gemini entirely on day one — keep L2 for catalog misses + and low similarity. +- Training a custom card CNN. +- Python GPU service. +- Changing scanner chrome / cart. +- Auto-approving the 27 pending `card_submissions`. + +## Roles invoked + +1. `role-architect` — embedder, schema, layer numbering, thresholds, + brief split (migration / backfill / route). Security-sensitive: + escalate to Sonnet if schema + new route land together. +2. `role-implementer`. +3. Audit: `role-reviewer` + `role-security-auditor` (required). + +## Todos + +- [ ] Architect: confirm `pgvector` on prod Neon tier +- [ ] Brief 1 — migration + SCHEMA_MAP +- [ ] Brief 2 — catalog backfill job (idempotent) +- [ ] Brief 3 — identify kNN route + client escalate order +- [ ] Threshold bake-off on real crops +- [ ] Re-measure auto-match % excluding `not_a_card` + +## Likely file ownership + +| Area | Files | +| --- | --- | +| Schema | `migrations/*_card-embeddings.js`, `docs/SCHEMA_MAP.md` | +| Backfill | new `scripts/` job (dated) or `lib/card-embed-backfill.js` | +| Query | new `lib/card-visual-match.js`, `pages/api/scan/identify-by-image.js` **or** fold into existing identify | +| Client | `lib/scanner-card-identify.js` (`tryLayer1TextIdentify` sibling) | + +Do not rewrite `lib/scanner-card-detection.js` here. + +## Multitask dispatch + +Brief 1 first. Brief 2 after 1. Brief 3 after 1 (can overlap 2 if the +route degrades to escalate-when-empty-index). + +``` +/multitask role-reviewer + role-security-auditor +``` + +Group id: `audit-scan-visual-catalog-search-`. + +## CI impact + +| Workflow / job | Behavior | +| --- | --- | +| `schema-map-fresh` | **Fires** — migration + SCHEMA_MAP | +| `ci.yml` migrate | Must apply `pgvector` on CT 102 CI Postgres — Architect must verify the extension is available there or gate the migration | +| `forbidden-client-side-llm-keys` | Blocking | + +## Operator action + +- Confirm Neon `pgvector` (or Neon’s equivalent) on the prod project. +- Budget: one embedding per catalog image, plus one per live scan if + the query embed is server-side. Architect publishes a cost note + before Brief 2 runs against prod images. +- No new browser secrets. + +## Conductor notes + +This is the accuracy leap. Do not start it to "try CLIP" before Phase 2 +crops are rectified — that wastes the backfill. If Architect finds +`pgvector` unavailable on CI Postgres, stop and write a fallback +(external index vs skip-CI-extension plan) rather than shipping an +untestable migration. diff --git a/.convoys/scanner-identify-upgrade.md b/.convoys/scanner-identify-upgrade.md new file mode 100644 index 0000000..cc3122e --- /dev/null +++ b/.convoys/scanner-identify-upgrade.md @@ -0,0 +1,207 @@ +--- +name: scanner-identify-upgrade +classification: feature +success_metric: | + Of legitimate card scans (excluding not_a_card), ≥50% auto-match a + catalog printing with no picker; Layer-1 escalate rate falls from 76.7% + to ≤40%; median /api/scan/identify latency stays near today's 1.7s p50 + or improves. Measured on scan_attempts after Phase 1 ships. +skip: [] +status: open +created: 2026-08-14 +depends_on: + - add-real-ocr-layer + - server-side-scan-pipeline +model_policy: + default_session: auto + roles: + role-conductor: composer-2.5-fast + role-architect: composer-2.5 + role-ia-architect: composer-2.5-fast + role-ux-reviewer: composer-2.5-fast + role-ui-designer: composer-2.5-fast + role-implementer: composer-2.5-fast + role-reviewer: cursor-grok-4.5-high + role-security-auditor: gpt-5.6-terra-medium + role-design-system-auditor: cursor-grok-4.5-high + role-a11y-auditor: cursor-grok-4.5-high + role-doc-writer: auto + escalate_to: claude-sonnet-5-thinking-medium + escalate_to_premium: claude-4.6-opus-high-thinking + never_premium: + - role-reviewer + - role-security-auditor + - role-design-system-auditor + - role-a11y-auditor + - role-ui-designer + - role-doc-writer +--- + +# Scanner identify upgrade — epic + +Umbrella for making the camera scanner **faster and more accurate** +without replacing the two-layer shape (cheap local path, then +server-owned Gemini). Planning-only in this file. Each numbered +sub-convoy is its own gated PR stream. + +Worktree: `tcg-vault-worktrees/scanner-identify-upgrade` on +`convoy/scanner-identify-upgrade` (branched from `main` @ `c6c1364`). +Do **not** land these docs on `feat/scanner-mobile-checkout` — that +convoy owns scanner chrome/cart and explicitly leaves identify out of +scope. + +## Why + +`add-real-ocr-layer` (PR #38) targeted ≥70% of scans resolving at +Layer-1 (Tesseract + `pg_trgm`) with zero Gemini calls. Live +`scan_attempts` from the local Neon (2026-05-27 → 2026-08-12, n=224, +1 user) shows that target was missed by a wide margin: + +| Signal | Value | Target / note | +| --- | --- | --- | +| L1 auto-match (`result_kind=matched`) | **5.8% of L1** / 2.7% of all | ≥70% L1 resolve | +| L1 escalate | **76.7% of L1** (79/103) | Should be the minority path | +| L1 disambiguation | 17.5% of L1 (18/103) | Name-only matcher cannot pick printings | +| End-to-end auto-match | **10.3%** (23/224) | User still picks or retries most cards | +| L2 `not_a_card` | **44.6% of L2** (54/121) | Detector fires on non-cards | +| L2 p50 / p90 latency | **1655 / 2349 ms** | Plus 3.5s hold-still *before* identify | +| L1 p50 latency | **158 ms** | Fast, but almost never uniquely matches | +| `card_submissions` | 27 pending, 0 reviewed | Catalog-gap queue is unread | + +The six L1 "matches" include noisy Tesseract strips (`J ——`, `Rock +Jockey ©`) at confidence 36–68. L1 is a cheap filter that rarely +identifies a printing. Meanwhile every card waits +`DETECTION_START_DELAY_MS` (1s) + `MIN_FIRST_SEEN_MS_FOR_VERIFY` (2.5s) +before OCR starts, and L2 is capped at 5 Gemini calls / user / minute. + +Users feel this as: hold still forever, then pick from a list, or get +"not a card" / "saved for review." + +## Scope + +### In scope (this epic) + +1. **`tighten-scan-identify-hot-path`** — Phase 1. Cut the hold-still + gate, OCR collector number on L1, structured Gemini JSON, optional + `SCAN_VISION_MODEL` bump, stop the automatic second Gemini refine. +2. **`improve-scan-card-detection`** — Phase 2. Replace the 320×240 + Sobel brute-force detector with a real card crop + perspective warp. + This was queued in `add-real-ocr-layer` and never opened. +3. **`scan-visual-catalog-search`** — Phase 3. Embed catalog + `image_url`s; nearest-neighbor the warped crop. Gemini becomes + fallback. This is the accuracy leap. + +### Out of scope (this epic) + +- Scanner chrome, cart, checkout (`scanner-mobile-checkout` / + `scanner-rebuild`). Identify libs only. +- Training a custom card CNN or standing up a Python/CUDA OCR service. +- Swapping Tesseract for EasyOCR / PaddleOCR as the *primary* identifier. +- Raising the L2 rate limit until Phase 1 hit-rate is re-measured. +- Admin review of the 27 pending `card_submissions` (ops, not this epic). + +## Baseline (do not re-query to "start" Phase 1) + +Pulled 2026-08-14 from local `.env.local` → Neon `scan_attempts`. + +``` +layer | result_kind | n | pct + 1 | escalate | 79 | 35.3 + 1 | disambiguation | 18 | 8.0 + 1 | matched | 6 | 2.7 + 2 | not_a_card | 54 | 24.1 + 2 | submitted | 26 | 11.6 + 2 | disambiguation | 22 | 9.8 + 2 | matched | 17 | 7.6 + 2 | needs_input | 2 | 0.9 +``` + +L1 escalate text-length buckets (all had ≥3 chars — Tesseract is +emitting text that `pg_trgm` cannot match): 3–7 chars n=28; 8–19 n=30; +20+ n=21. + +Re-measure with the same grouping after each sub-convoy ships. + +## Dependency graph + +``` + [baseline pulled 2026-08-14] + │ + ▼ + ┌───────────────────────────────────┐ + │ 1. tighten-scan-identify-hot-path │ + │ gates, collector #, schema JSON│ + └───────────────┬───────────────────┘ + │ + ▼ + ┌───────────────────────────────────┐ + │ 2. improve-scan-card-detection │ + │ detect + homography crop │ + └───────────────┬───────────────────┘ + │ + ▼ + ┌───────────────────────────────────┐ + │ 3. scan-visual-catalog-search │ + │ embeddings + pgvector kNN │ + └───────────────────────────────────┘ +``` + +**Strict-blockers:** #1 before #2 only if #2 would retune the same +stability constants — otherwise #1 (identify files) and #2 (detection +files) are file-disjoint and may run in parallel after Architect +confirms. #3 needs a stable crop (#2) to be worth the embedding job; +do not start #3 until #2 has a warped JPEG. + +**Sibling:** `scanner-mobile-checkout` (other worktree / branch) must +not edit `lib/ocr-worker.js`, `lib/scan-vision.js`, +`lib/card-text-match.js`, `lib/scanner-card-identify.js`, +`lib/scanner-card-detection.js`, or `/api/scan/identify`. + +## Roles invoked + +Umbrella is planning-only. Each sub-convoy lists its own roles. +Default for #1: Architect → Implementer → audit fan-out (reviewer + +security-auditor). UX reviewer on #1 and #2. Skip IA and UI Designer +on all three (no new routes or visual language). + +## Todos + +- [x] Pull `scan_attempts` baseline (2026-08-14) +- [x] Open worktree `scanner-identify-upgrade` from `main` +- [x] Seed sub-convoys #1–#3 +- [ ] Architect: pick up `tighten-scan-identify-hot-path` first +- [ ] Re-measure `scan_attempts` after #1 ships +- [ ] Architect: `improve-scan-card-detection` (or parallel if files stay disjoint) +- [ ] Architect: `scan-visual-catalog-search` after warped crops exist + +## Worktree + +| Checkout | Branch | Purpose | +| --- | --- | --- | +| `tcg-vault-worktrees/scanner-identify-upgrade` | `convoy/scanner-identify-upgrade` | These convoy docs | +| (later) Agents Window worktree per brief | `brief/scanner-identify-upgrade/-` | Implementer slices | + +`scripts/wt.sh` is a deprecation stub. Create implementer worktrees from +the Agents Window after Architect writes `slice_dependencies:`. + +## Multitask dispatch + +No implementer fan-out from this umbrella. After #1 Architect marks +parallel-safe briefs (`depends_on: []` + disjoint `files:`): + +``` +/multitask role-implementer briefs <ids> +``` + +After each PR draft: + +``` +/multitask role-reviewer + role-security-auditor +``` + +Add design-system + a11y auditors only if the PR touches +`components/` or `pages/scanner.js` (not expected in #1). + +Group ids: `audit-tighten-scan-identify-hot-path-<pr>`, +`audit-improve-scan-card-detection-<pr>`, +`audit-scan-visual-catalog-search-<pr>`. diff --git a/.convoys/ship-readiness.md b/.convoys/ship-readiness.md index 377042f..9381e2b 100644 --- a/.convoys/ship-readiness.md +++ b/.convoys/ship-readiness.md @@ -415,6 +415,7 @@ Total: ~14 convoys to get from current state to public-launch-ready. Estimate 4- Follow-ups surfaced mid-convoy or mid-PR that didn't fit the original launch sequence but need to land before public traffic. Listed in priority order; not all will be P0/P1 — most are CI / DX / hygiene polish. +- **`scanner-identify-upgrade`** (priority: P1 scanner accuracy; opened 2026-08-14). Umbrella after measuring `scan_attempts` (n=224, 2026-05-27–2026-08-12): L1 auto-match 5.8% of L1 vs the `add-real-ocr-layer` ≥70% target; L1 escalate 76.7%; L2 `not_a_card` 44.6% of Gemini calls; end-to-end auto-match 10.3%. Sub-convoys: `tighten-scan-identify-hot-path` → `improve-scan-card-detection` (queued since PR #38, never opened) → `scan-visual-catalog-search`. Worktree `tcg-vault-worktrees/scanner-identify-upgrade` on `convoy/scanner-identify-upgrade`. File-disjoint from `scanner-mobile-checkout` (chrome/cart). See `.convoys/scanner-identify-upgrade.md`. - **`rotate-default-admin`** — **RESOLVED 2026-06-13** by PR #141 (`scripts/rotate-admin-password.js`). The convoy chose option B from the architect's three-option menu (close as no-op / build script / build forced-rotation flow): a parameterized one-shot rotation script that's safer than "manually change via app" (audit-trail-preserving via `updated_at`) and lighter than building a first-login forced-rotation flow in the app (that heavier option is the deferred `force-admin-password-reset-flow` convoy). Script reads `POSTGRES_URL` + `ADMIN_NEW_PASSWORD` env vars, validates target row exists + has `role='admin'`, refuses to rotate non-admin rows, verifies the new hash matches the supplied plaintext via `bcrypt.compare` post-update, never echoes the password. AGENTS.md Gotcha #4 documents the rotation workflow. Sibling test users (alice/bob in `scripts/create-test-users.js`) intentionally NOT rotated (dev fixtures, not real auth surfaces). - **`delete-dead-lorcana-import`** — **RESOLVED 2026-06-02** by PR #59 (`8262fec`). Deleted `pages/api/cards/import-lorcana.js` and `scripts/import-lorcana.js`; no dedicated convoy file (cleanup tracked here only). Entry kept for audit trail. - **`tighten-visual-diff-path-filter`** — **RESOLVED 2026-05-26** by `tighten-visual-diff-path-filter` convoy, squash commit `ba95462` (PR #26). Single-edit `paths:` filter change in `.github/workflows/visual-diff.yml`: inserted `'!pages/api/**'` immediately after `'pages/**'` (order-sensitive per GitHub Actions' minimatch path-filter semantics — exclusions only fire after a prior include matches). Verified the YAML deserialization order at gate time (`['pages/**', '!pages/api/**', 'components/**', 'styles/**', 'tailwind.config.js', 'postcss.config.js']`). `preview-smoke.yml` left untouched (no `paths:` filter; intentionally fires on every PR). Diff: 2 files, +279 / -0 (1 YAML entry + inline comment block + the planning convoy file). **Post-merge verification still pending** — the only true verification is that the next API-only PR after this merges does NOT trigger `Screenshot diff`. PR #30 (`single-sql-client`, squash `c403ea4`) was the **first API-only PR post-merge** and its CI Checks tab showed `Screenshot diff: not triggered` — empirical confirmation that the `!pages/api/**` exclusion fires correctly. The next-API-only-PR success line was originally specified in the convoy file's § Verification plan as the deferred-to-post-merge gate; this is that confirmation. Entry kept (not removed) to preserve the audit trail. See `.convoys/tighten-visual-diff-path-filter.md` § As-shipped. diff --git a/.convoys/tighten-scan-identify-hot-path.md b/.convoys/tighten-scan-identify-hot-path.md new file mode 100644 index 0000000..bcaeea2 --- /dev/null +++ b/.convoys/tighten-scan-identify-hot-path.md @@ -0,0 +1,328 @@ +--- +name: tighten-scan-identify-hot-path +classification: feature +success_metric: | + After ship, a new scan_attempts window shows L1 escalate ≤40% of L1 + (was 76.7%), L1 matched ≥25% of L1 (was 5.8%), and first identify + attempt starts in ≤1.2s of tracking (was 3.5s). L2 p50 stays ≤1.8s. +skip: + - ia + - ui-design + - visual + - flag +status: in-progress +created: 2026-08-14 +depends_on: + - add-real-ocr-layer +umbrella: scanner-identify-upgrade +model_policy: + default_session: auto + roles: + role-conductor: composer-2.5-fast + role-architect: composer-2.5 + role-ia-architect: composer-2.5-fast + role-ux-reviewer: composer-2.5-fast + role-ui-designer: composer-2.5-fast + role-implementer: composer-2.5-fast + role-reviewer: cursor-grok-4.5-high + role-security-auditor: gpt-5.6-terra-medium + role-design-system-auditor: cursor-grok-4.5-high + role-a11y-auditor: cursor-grok-4.5-high + role-doc-writer: auto + escalate_to: claude-sonnet-5-thinking-medium + escalate_to_premium: claude-4.6-opus-high-thinking + never_premium: + - role-reviewer + - role-security-auditor + - role-design-system-auditor + - role-a11y-auditor + - role-ui-designer + - role-doc-writer +--- + +# Convoy: tighten-scan-identify-hot-path + +Phase 1 of `scanner-identify-upgrade`. Make the **existing** two-layer +pipeline faster and more unique-match capable. No new vendors, no +detection rewrite, no embeddings. + +## Why + +Live telemetry (see umbrella) says Layer-1 is fast (p50 158ms) and +almost useless as an identifier (5.8% auto-match, 76.7% escalate). The +user still waits 3.5s of hold-still before that cheap path runs. L2 +Gemini is 1.7s p50 and spends a second token on every disambiguation +refine. Collector number — the unique printing key L2 already uses — +is never read on L1. + +## Scope + +### In scope + +- **Hold-still constants** in `lib/scanner-card-detection.js`: + `MIN_FIRST_SEEN_MS_FOR_VERIFY` 2500 → ~800; + `MIN_STABLE_COUNT_FOR_VERIFY` 6 → 3. Keep tests in + `test/lib/scanner-card-detection.test.js` in lockstep. +- **Name-strip OCR mode:** Tesseract PSM 7 (single line), JPEG quality + 0.92 on the crop used for OCR (`lib/ocr-worker.js`, + `captureCardRegionFromVideo` quality if shared). +- **Collector-number strip:** second crop of the bottom ~18% of the + card; pass number (+ optional set hint) into `matchTextInCatalog`. + When name + number uniquely match a `cards` row, auto-match even if + many printings share the name. +- **Structured Gemini output** in `lib/scan-vision.js`: + `response_format` JSON schema (or Gateway equivalent). Delete the + confidence-30 regex fallback as the success path; keep a hard parse + error → `needs_input`. +- **Model A/B:** default stays `google/gemini-2.5-flash-lite`. Document + and allow `SCAN_VISION_MODEL=google/gemini-3.1-flash-lite` (GA on + Vercel AI Gateway, ~2.5× input price). Do not flip prod default in + this convoy unless Architect + a short identify bake-off say so. +- **Stop automatic L2 refine** in `use-scanner-identification.js` when + L1 already returned a printing list. User picks; Gemini refine is + opt-in or only when L1 had no set/number hint. +- **`docs/SCHEMA_MAP.md`:** `scan_attempts.layer` is 1 = Tesseract + (shipped), not "Tesseract future". + +### Out of scope + +- Detector / OpenCV / YOLO / perspective warp → + `improve-scan-card-detection`. +- pgvector / CLIP / catalog embeddings → + `scan-visual-catalog-search`. +- Changing `checkScanRateLimit` (5/min). Revisit after re-measure. +- Scanner chrome, cart, checkout (`scanner-mobile-checkout`). +- Replacing `tesseract.js`. + +## Roles invoked + +1. `role-ux-reviewer` — hold-still feel; when the picker still appears; + no new screens. +2. `role-architect` — ratify constant values, L1 match rules when a + collector number is present, JSON schema shape, whether the model + default flips. Write briefs + `slice_dependencies`. +3. `role-implementer` — per brief. +4. Audit: `role-reviewer` + `role-security-auditor`. Skip design-system + + a11y unless a brief unexpectedly touches `components/`. + +## Todos + +- [x] UX: confirm 800ms / 3-frame gate does not cause double-scans +- [x] Architect: briefs + match-rule when number is present +- [x] Brief A — detection constants + tests +- [x] Brief B — OCR strips (name PSM 7 + collector number) + + `card-text-match.js` / `identify-by-text` +- [x] Brief C — `scan-vision.js` JSON schema + optional model id +- [x] Brief D — skip automatic disambiguation Gemini refine +- [x] SCHEMA_MAP layer-1 wording +- [ ] Re-query `scan_attempts` after preview traffic + +## Likely file ownership (Architect will lock) + +| Area | Files | +| --- | --- | +| Gates | `lib/scanner-card-detection.js`, `test/lib/scanner-card-detection.test.js` | +| OCR | `lib/ocr-worker.js` | +| L1 match | `lib/card-text-match.js`, `pages/api/cards/identify-by-text.js` | +| L2 vision | `lib/scan-vision.js`, `pages/api/scan/identify.js` | +| Refine | `lib/use-scanner-identification.js`, `lib/scanner-card-identify.js` | +| Docs | `docs/SCHEMA_MAP.md` | + +Briefs A and C are file-disjoint and can run in parallel. B depends on +nothing if it does not retouch detection constants. D depends on B +(needs L1 payload shape). + +## Multitask dispatch + +Planning: UX → Architect (serial). + +After Architect, if `slice_dependencies` marks A+C `depends_on: []`: + +``` +/multitask role-implementer briefs A, C +``` + +Then B, then D. + +Audit group id: `audit-tighten-scan-identify-hot-path-<pr>`. + +## CI impact + +| Workflow / job | Behavior | +| --- | --- | +| `ci.yml` test | Fires — detection + identify unit tests change | +| `visual-diff.yml` | Should **not** fire if `components/` / `pages/` / `styles/` untouched | +| `preview-smoke.yml` | Fires | +| `forbidden-client-side-llm-keys` | Must stay green — no client Gemini URLs | + +## Operator action + +Optional: set `SCAN_VISION_MODEL=google/gemini-3.1-flash-lite` on a +preview to bake off against 2.5 Flash Lite. Leave prod on 2.5 until +the bake-off. + +## Conductor notes + +Do not treat Tesseract replacement as in-scope if L1 escalate stays +high after collector-number matching — that is a crop/detection +problem (#2) or a visual-search problem (#3), not an OCR-engine +problem. The six historical L1 matches already show Tesseract can +read a name when the crop is clean; it cannot pick a printing. + +## UX + +No new routes or screens. Changes are timing + fewer automatic Gemini +calls during disambiguation. + +### Existing components to reuse + +- `ScannerDisambiguation` (`components/scanner/ScannerDisambiguation.js`) — + printing picker when L1/L2 cannot unique-match. +- `ScannerToast` / status copy around tracked card (`components/scanner/ScannerCamera.js`) + — continue showing detecting → verifying → found without new chrome. +- `ScanDisambiguationDialog` pattern if still referenced — do not fork a + second picker. + +### Design direction alignment + +Liquid Glass tokens unchanged. No new overlays. Faster identify should +feel like the camera "wakes up" sooner (H1 visibility of system status). + +### Patterns to follow + +- Disambiguation stays a bottom sheet / modal pause (`verificationPausedRef` + already wired in `use-scanner-identification.js`). +- Error toasts debounced via `reportScannerError` (4s gap) — keep that + when tightening gates so double-failures do not spam. + +### A11y constraints + +- No new interactive controls in this convoy. +- Existing disambiguation list must remain keyboard-selectable (hand off + unchanged to a11y auditor if Brief D touches the sheet). + +### Interaction patterns + +| Pattern | Requirement | Heuristic | +| --- | --- | --- | +| Hold-still gate | **required** — 800ms + 3 stable frames; user sees brackets sooner | H1 feedback | +| Auto-match | **required** — collector number should skip picker for multi-printing names | H6 recognition vs recall | +| Disambiguation refine | **required** — stop silent second Gemini call when L1 already opened picker; user tap only | H3 user control | +| Rate-limit toast | **required** — unchanged copy when L2 throttled | H9 error recovery | +| Loading during verify | **nice-to-have** — optional subtle "Reading…" on card status if trivial | H1 | + +### Anti-patterns + +- Adding a setup screen or settings toggle for gate timing (H4 consistency). +- Auto-picking a printing without number evidence when multiple exact-name + rows exist (H6). +- Blocking the camera on L1 OCR worker load (H1 — worker stays async). + +### Mobile / responsive + +Gate timing applies equally on phone and desktop; no viewport-specific +branches in this convoy. + +## Architecture + +### File plan + +| File | Action | Purpose | +| --- | --- | --- | +| `lib/scanner-card-detection.js` | modified | 800ms / 3-frame verify gates | +| `test/lib/scanner-card-detection.test.js` | modified | Lock new constants | +| `lib/ocr-worker.js` | modified | PSM 7 name strip + bottom collector strip | +| `lib/card-text-match.js` | modified | Name + collector number unique match | +| `pages/api/cards/identify-by-text.js` | modified | Accept `cardNumber` body field | +| `test/lib/card-text-match.test.js` | new | Unit tests for number path (mock-free helpers) | +| `lib/scanner-card-identify.js` | modified | Pass number to L1; JPEG 0.92; `fromLayer1` flag | +| `lib/scan-vision.js` | modified | JSON schema response; stricter parse | +| `pages/api/scan/identify.js` | modified | Handle vision parse failures → needs_input | +| `lib/use-scanner-identification.js` | modified | Skip auto Gemini refine when `fromLayer1` | +| `docs/SCHEMA_MAP.md` | modified | Layer 1 = Tesseract (shipped) | + +### API surface + +**POST `/api/cards/identify-by-text`** (modified) + +- Auth: `getUserFromRequest` → 401 +- Body: `{ ocrText, ocrConfidence?, cardNumber?, game? }` +- Response: unchanged envelope + optional `cardNumber` echo in `ocr` meta +- Rate limit: none (unchanged) + +**POST `/api/scan/identify`** (modified behavior only) + +- On vision JSON parse failure: 200 with `needsUserInput: true` instead of + silent regex fallback at confidence 30 + +### Schema diff + +No migration. `scan_attempts.layer` semantics documented only. + +### Test plan + +- Update `scanner-card-detection.test.js` thresholds (800ms, stable 3). +- New `card-text-match.test.js` for `extractCollectorNumberCandidate` and + collector-aware disambiguation resolution (pure functions exported for test). +- Extend `scanner-card-identify.test.js` for `fromLayer1` disambiguation + payload if added to pure helpers. +- Run `npm run test:run` + `npm run lint`. + +### Risks + +| Risk | Mitigation | +| --- | --- | +| Faster gate double-scans same card | `scanAttempts < 1` + tracker negative state unchanged | +| OCR number strip reads set symbol garbage | Normalize with `extractCollectorNumberCandidate`; fall back to name-only | +| Gateway rejects `response_format` | Catch 400, log once, fall back to prose prompt (Architect: implementer may feature-detect) | +| L1 skip-refine hides catalog-gap auto-submit | Refine still runs for L2 disambiguation only | + +### Decomposition + +| Brief # | Title | Files | Depends on | Size | +| --- | --- | --- | --- | --- | +| 1 | Faster verify gates | detection + test | — | S | +| 2 | Collector number L1 | ocr-worker, card-text-match, identify-by-text, scanner-card-identify, test | — | M | +| 3 | Structured vision JSON | scan-vision, identify API | — | S | +| 4 | Skip L1 disambiguation refine | use-scanner-identification, SCHEMA_MAP | 2 | S | + +### Slice dependencies + +```yaml +slice_dependencies: + - brief: 1 + depends_on: [] + files: + - lib/scanner-card-detection.js + - test/lib/scanner-card-detection.test.js + - brief: 2 + depends_on: [] + files: + - lib/ocr-worker.js + - lib/card-text-match.js + - pages/api/cards/identify-by-text.js + - lib/scanner-card-identify.js + - test/lib/card-text-match.test.js + - brief: 3 + depends_on: [] + files: + - lib/scan-vision.js + - pages/api/scan/identify.js + - brief: 4 + depends_on: [2] + files: + - lib/use-scanner-identification.js + - docs/SCHEMA_MAP.md +``` + +### Decisions (post-UX) + +| # | Decision | +| --- | --- | +| D1 | Gate values: `MIN_FIRST_SEEN_MS_FOR_VERIFY = 800`, `MIN_STABLE_COUNT_FOR_VERIFY = 3`. | +| D2 | Collector strip: bottom 18% of crop; PSM 7 for both strips. | +| D3 | When exact-name multiple printings AND parsed collector number matches exactly one row → auto-match. | +| D4 | `SCAN_VISION_MODEL` default stays `google/gemini-2.5-flash-lite`; structured JSON either way. | +| D5 | Auto Gemini refine skipped when disambiguation opened from L1 (`result.layer === 1`). | + +Human gate 1: **approved** (operator requested full pipeline run 2026-08-14). diff --git a/.convoys/tighten-scan-identify-hot-path/brief-1-faster-verify-gates.md b/.convoys/tighten-scan-identify-hot-path/brief-1-faster-verify-gates.md new file mode 100644 index 0000000..9416e3f --- /dev/null +++ b/.convoys/tighten-scan-identify-hot-path/brief-1-faster-verify-gates.md @@ -0,0 +1,22 @@ +--- +convoy: tighten-scan-identify-hot-path +brief_number: 1 +depends_on: [] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/scanner-card-detection.js + - test/lib/scanner-card-detection.test.js +--- + +# Brief 1: Faster verify gates + +## Goal + +Cut hold-still wait from 3.5s to ~1.4s by lowering verify gate constants. + +## Acceptance criteria + +- [ ] `MIN_FIRST_SEEN_MS_FOR_VERIFY = 800` +- [ ] `MIN_STABLE_COUNT_FOR_VERIFY = 3` +- [ ] Tests updated and green diff --git a/.convoys/tighten-scan-identify-hot-path/brief-2-collector-number-l1.md b/.convoys/tighten-scan-identify-hot-path/brief-2-collector-number-l1.md new file mode 100644 index 0000000..3cb6d7f --- /dev/null +++ b/.convoys/tighten-scan-identify-hot-path/brief-2-collector-number-l1.md @@ -0,0 +1,27 @@ +--- +convoy: tighten-scan-identify-hot-path +brief_number: 2 +depends_on: [] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/ocr-worker.js + - lib/card-text-match.js + - pages/api/cards/identify-by-text.js + - lib/scanner-card-identify.js + - test/lib/card-text-match.test.js +--- + +# Brief 2: Collector number on Layer 1 + +## Goal + +OCR name + collector number strips; unique-match printings when number resolves one row. + +## Acceptance criteria + +- [ ] PSM 7 on name strip; bottom 18% number strip +- [ ] `matchTextInCatalog` accepts `cardNumber` +- [ ] identify-by-text passes `cardNumber` +- [ ] JPEG capture quality 0.92 for OCR path +- [ ] Unit tests for number extraction helper diff --git a/.convoys/tighten-scan-identify-hot-path/brief-3-structured-vision-json.md b/.convoys/tighten-scan-identify-hot-path/brief-3-structured-vision-json.md new file mode 100644 index 0000000..28f11ce --- /dev/null +++ b/.convoys/tighten-scan-identify-hot-path/brief-3-structured-vision-json.md @@ -0,0 +1,22 @@ +--- +convoy: tighten-scan-identify-hot-path +brief_number: 3 +depends_on: [] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/scan-vision.js + - pages/api/scan/identify.js +--- + +# Brief 3: Structured Gemini JSON + +## Goal + +Request JSON schema from AI Gateway; fail loud on parse errors. + +## Acceptance criteria + +- [ ] `response_format` json_schema on gateway call +- [ ] No confidence-30 regex success path +- [ ] identify route returns needsUserInput on parse failure diff --git a/.convoys/tighten-scan-identify-hot-path/brief-4-skip-l1-refine.md b/.convoys/tighten-scan-identify-hot-path/brief-4-skip-l1-refine.md new file mode 100644 index 0000000..eaf50cf --- /dev/null +++ b/.convoys/tighten-scan-identify-hot-path/brief-4-skip-l1-refine.md @@ -0,0 +1,22 @@ +--- +convoy: tighten-scan-identify-hot-path +brief_number: 4 +depends_on: [2] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/use-scanner-identification.js + - docs/SCHEMA_MAP.md +--- + +# Brief 4: Skip L1 disambiguation refine + +## Goal + +Do not auto-call Gemini when L1 already opened the printing picker. + +## Acceptance criteria + +- [ ] `fromLayer1` on disambiguation state when `result.layer === 1` +- [ ] useEffect refine skipped when `fromLayer1` +- [ ] SCHEMA_MAP documents layer 1 = Tesseract diff --git a/docs/SCHEMA_MAP.md b/docs/SCHEMA_MAP.md index bf6a9a1..2e430a8 100644 --- a/docs/SCHEMA_MAP.md +++ b/docs/SCHEMA_MAP.md @@ -261,7 +261,7 @@ Index: `idx_card_submissions_status (status, created_at DESC)`. ### scan_attempts -Per-scan telemetry for the identify pipeline (layer 2 = Gemini today). +Per-scan telemetry for the identify pipeline (layer 1 = Tesseract + pg_trgm, layer 2 = Gemini). | Column | Type | Notes | | --- | --- | --- | @@ -269,7 +269,7 @@ Per-scan telemetry for the identify pipeline (layer 2 = Gemini today). | `user_id` | FK → `users` SET NULL | | | `ocr_text` | `TEXT` | | | `ocr_confidence` | `INTEGER` | | -| `layer` | `INTEGER` default `2` | OCR layer (1 = Tesseract future) | +| `layer` | `INTEGER` default `2` | Identify layer (`1` = browser Tesseract + pg_trgm, `2` = Gemini vision) | | `matched_card_id` | FK → `cards` SET NULL | | | `result_kind` | `VARCHAR(32)` | e.g. `'matched'`, `'disambiguation'`, `'submitted'`, `'not_a_card'` | | `latency_ms` | `INTEGER` | End-to-end identify latency | diff --git a/lib/card-text-match.js b/lib/card-text-match.js index ea2fa7f..6483a86 100644 --- a/lib/card-text-match.js +++ b/lib/card-text-match.js @@ -1,5 +1,7 @@ import { sql } from '@vercel/postgres'; +import { findPrintingByCollectorNumber } from './card-number-utils.js'; + const MATCH_THRESHOLD = 0.85; const DISAMBIGUATION_THRESHOLD = 0.6; const MIN_QUERY_LENGTH = 3; @@ -64,11 +66,53 @@ async function querySimilarCards(query, game) { `; } +function resolveExactNameMatches(exactNameMatches, cardNumber, query) { + if (exactNameMatches.length === 1) { + return { + type: 'matched', + card: exactNameMatches[0], + query, + similarity: exactNameMatches[0].sim, + message: `Matched "${exactNameMatches[0].name}" via text search`, + }; + } + + if (exactNameMatches.length > 1 && cardNumber) { + const uniqueByNumber = findPrintingByCollectorNumber(exactNameMatches, cardNumber); + if (uniqueByNumber) { + return { + type: 'matched', + card: uniqueByNumber, + query, + similarity: uniqueByNumber.sim, + message: `Matched "${uniqueByNumber.name}" (${uniqueByNumber.card_number}) via collector number`, + }; + } + } + + if (exactNameMatches.length > 1) { + return { + type: 'disambiguation', + matches: exactNameMatches.slice(0, 8).map(mapCardRow), + query, + message: `Found ${exactNameMatches.length} printings of "${query}". Select the correct one.`, + }; + } + + return null; +} + /** * Fuzzy match OCR text against cards.name using pg_trgm similarity. */ -export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence = null }) { +export async function matchTextInCatalog({ + ocrText, + cardNumber = null, + game = null, + ocrConfidence = null, +}) { const query = extractNameCandidate(ocrText); + const trimmedNumber = cardNumber && String(cardNumber).trim() ? String(cardNumber).trim() : null; if (!query || query.length < MIN_QUERY_LENGTH) { return { @@ -76,6 +120,7 @@ export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence = reason: 'OCR text too short for catalog match', query, ocrConfidence, + cardNumber: trimmedNumber, }; } @@ -88,6 +133,7 @@ export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence = reason: `No catalog match above ${DISAMBIGUATION_THRESHOLD} similarity for "${query}"`, query, ocrConfidence, + cardNumber: trimmedNumber, }; } @@ -96,14 +142,9 @@ export async function matchTextInCatalog({ ocrText, game = null, ocrConfidence = (row) => row.name?.toLowerCase() === normalizedQuery ); - // Same card name, multiple printings — always ask the user. - if (exactNameMatches.length > 1) { - return { - type: 'disambiguation', - matches: exactNameMatches.slice(0, 8).map(mapCardRow), - query, - message: `Found ${exactNameMatches.length} printings of "${query}". Select the correct one.`, - }; + const exactResolution = resolveExactNameMatches(exactNameMatches, trimmedNumber, query); + if (exactResolution) { + return exactResolution; } const top = candidates[0]; diff --git a/lib/ocr-worker.js b/lib/ocr-worker.js index 6a98058..83af0a1 100644 --- a/lib/ocr-worker.js +++ b/lib/ocr-worker.js @@ -5,6 +5,12 @@ let workerPromise = null; +const NAME_STRIP_PSM = '7'; +const NUMBER_STRIP_PSM = '7'; +const NAME_STRIP_HEIGHT_RATIO = 0.35; +const NUMBER_STRIP_HEIGHT_RATIO = 0.18; +const OCR_JPEG_QUALITY = 0.92; + async function getWorker() { if (!workerPromise) { workerPromise = (async () => { @@ -12,53 +18,116 @@ async function getWorker() { const worker = await createWorker('eng', 1, { logger: () => {}, }); - await worker.setParameters({ - tessedit_pageseg_mode: '6', - }); return worker; })(); } return workerPromise; } -function cropNameStrip(imageDataUrl) { +function cropCardStrip(imageDataUrl, { fromTop = true, heightRatio }) { return new Promise((resolve, reject) => { const img = new Image(); img.onload = () => { - const stripHeight = Math.max(1, Math.floor(img.height * 0.35)); + const stripHeight = Math.max(1, Math.floor(img.height * heightRatio)); const canvas = document.createElement('canvas'); canvas.width = img.width; canvas.height = stripHeight; const ctx = canvas.getContext('2d'); - ctx.drawImage(img, 0, 0, img.width, stripHeight, 0, 0, img.width, stripHeight); - resolve(canvas.toDataURL('image/jpeg', 0.92)); + const sourceY = fromTop ? 0 : Math.max(0, img.height - stripHeight); + ctx.drawImage( + img, + 0, + sourceY, + img.width, + stripHeight, + 0, + 0, + canvas.width, + canvas.height + ); + resolve(canvas.toDataURL('image/jpeg', OCR_JPEG_QUALITY)); }; img.onerror = () => reject(new Error('Failed to load image for OCR')); img.src = imageDataUrl; }); } +/** + * Pick the most likely collector number token from OCR of the bottom strip. + * @param {string} rawText + * @returns {string} + */ +export function extractCollectorNumberCandidate(rawText) { + if (!rawText || typeof rawText !== 'string') return ''; + + const lines = rawText + .split(/\r?\n/) + .map((line) => line.replace(/\s+/g, ' ').trim()) + .filter(Boolean); + + const joined = lines.join(' '); + const slashMatch = joined.match(/\b(\d{1,4}\s*\/\s*\d{1,4}[A-Za-z]?)\b/); + if (slashMatch) { + return slashMatch[1].replace(/\s+/g, ''); + } + + for (let i = lines.length - 1; i >= 0; i -= 1) { + const line = lines[i]; + if (/\d/.test(line) && line.length <= 12) { + return line; + } + } + + return ''; +} + /** * OCR the top name strip of a card crop. * @param {string} imageDataUrl - full card crop data URL * @returns {Promise<{ text: string, confidence: number }>} */ export async function recognizeCardNameStrip(imageDataUrl) { + const fields = await recognizeCardFields(imageDataUrl); + return { + text: fields.nameText, + confidence: fields.nameConfidence, + }; +} + +/** + * OCR name strip + bottom collector-number strip. + * @param {string} imageDataUrl + * @returns {Promise<{ nameText: string, nameConfidence: number, cardNumber: string, numberConfidence: number }>} + */ +export async function recognizeCardFields(imageDataUrl) { if (typeof window === 'undefined') { - return { text: '', confidence: 0 }; + return { nameText: '', nameConfidence: 0, cardNumber: '', numberConfidence: 0 }; } try { - const stripUrl = await cropNameStrip(imageDataUrl); + const [nameStripUrl, numberStripUrl] = await Promise.all([ + cropCardStrip(imageDataUrl, { fromTop: true, heightRatio: NAME_STRIP_HEIGHT_RATIO }), + cropCardStrip(imageDataUrl, { fromTop: false, heightRatio: NUMBER_STRIP_HEIGHT_RATIO }), + ]); + const worker = await getWorker(); - const { data } = await worker.recognize(stripUrl); + await worker.setParameters({ tessedit_pageseg_mode: NAME_STRIP_PSM }); + const nameResult = await worker.recognize(nameStripUrl); + + await worker.setParameters({ tessedit_pageseg_mode: NUMBER_STRIP_PSM }); + const numberResult = await worker.recognize(numberStripUrl); + + const cardNumber = extractCollectorNumberCandidate(numberResult.data.text || ''); + return { - text: (data.text || '').trim(), - confidence: Math.round(data.confidence || 0), + nameText: (nameResult.data.text || '').trim(), + nameConfidence: Math.round(nameResult.data.confidence || 0), + cardNumber, + numberConfidence: Math.round(numberResult.data.confidence || 0), }; } catch (error) { - console.warn('[ocr-worker] recognizeCardNameStrip failed:', error); - return { text: '', confidence: 0 }; + console.warn('[ocr-worker] recognizeCardFields failed:', error); + return { nameText: '', nameConfidence: 0, cardNumber: '', numberConfidence: 0 }; } } diff --git a/lib/scan-vision.js b/lib/scan-vision.js index ed908bd..33eb2ff 100644 --- a/lib/scan-vision.js +++ b/lib/scan-vision.js @@ -8,45 +8,105 @@ HOLOGRAPHIC / FOIL CARDS: Many cards have reflective foil surfaces with glare or CRITICAL: Only respond with card data if you can clearly identify a TRADING CARD in the image. Ignore random objects, books, papers, phone screens, screenshots, and non-card gaming items. Blurry images with no readable card frame should be rejected. -If you detect a trading card, extract information in this JSON format: +Respond with JSON only (no markdown fences) using this shape: { "isCard": true, - "cardName": "exact card name as printed on the card", - "setName": "set name if visible", - "setCode": "set code/symbol if visible", - "cardNumber": "collector number if visible", - "game": "mtg, pokemon, or lorcana (lowercase)", - "cardType": "creature, instant, sorcery, trainer, etc.", + "cardName": "exact card name", + "setName": "set name or null", + "setCode": "set code or null", + "cardNumber": "collector number or null", + "game": "mtg, pokemon, or lorcana", + "cardType": "creature, instant, etc.", "rarity": "common, uncommon, rare, mythic, etc.", - "manaCost": "mana cost if visible", - "hp": "HP or power if visible", - "abilities": ["list of abilities or attacks if clearly readable"], + "manaCost": "mana cost or null", + "hp": "HP or power or null", + "abilities": ["ability strings"], "confidence": 85, - "rawText": "all text visible on the card" + "rawText": "all visible text", + "reason": null } If NO trading card is clearly visible, respond with: { "isCard": false, + "cardName": null, + "setName": null, + "setCode": null, + "cardNumber": null, + "game": null, + "cardType": null, + "rarity": null, + "manaCost": null, + "hp": null, + "abilities": [], "confidence": 0, + "rawText": null, "reason": "No trading card detected in image" } Be conservative — only extract data you can clearly read. Quality over quantity.`; +const CARD_VISION_SCHEMA = { + name: 'card_scan_result', + strict: true, + schema: { + type: 'object', + additionalProperties: false, + properties: { + isCard: { type: 'boolean' }, + cardName: { type: ['string', 'null'] }, + setName: { type: ['string', 'null'] }, + setCode: { type: ['string', 'null'] }, + cardNumber: { type: ['string', 'null'] }, + game: { type: ['string', 'null'] }, + cardType: { type: ['string', 'null'] }, + rarity: { type: ['string', 'null'] }, + manaCost: { type: ['string', 'null'] }, + hp: { type: ['string', 'null'] }, + abilities: { + type: 'array', + items: { type: 'string' }, + }, + confidence: { type: 'number' }, + rawText: { type: ['string', 'null'] }, + reason: { type: ['string', 'null'] }, + }, + required: [ + 'isCard', + 'cardName', + 'setName', + 'setCode', + 'cardNumber', + 'game', + 'cardType', + 'rarity', + 'manaCost', + 'hp', + 'abilities', + 'confidence', + 'rawText', + 'reason', + ], + }, +}; + +export class VisionParseError extends Error { + constructor(message) { + super(message); + this.name = 'VisionParseError'; + } +} + function parseVisionJson(content) { + if (!content || typeof content !== 'string') { + throw new VisionParseError('Empty vision model response'); + } + const cleanContent = content.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim(); try { return JSON.parse(cleanContent); } catch { - const cardNameMatch = content.match(/card.*?name.*?[:"]\s*([^"'\n,}]+)/i); - return { - isCard: !!cardNameMatch, - cardName: cardNameMatch ? cardNameMatch[1].trim() : null, - confidence: 30, - rawText: content, - reason: 'Failed to parse structured response', - }; + throw new VisionParseError('Failed to parse structured vision response'); } } @@ -59,6 +119,29 @@ function normalizeGame(game) { return value; } +function buildGatewayBody(imageDataUrl, { structured = true } = {}) { + const body = { + model: DEFAULT_VISION_MODEL, + temperature: 0.1, + messages: [{ + role: 'user', + content: [ + { type: 'text', text: CARD_PROMPT }, + { type: 'image_url', image_url: { url: imageDataUrl } }, + ], + }], + }; + + if (structured) { + body.response_format = { + type: 'json_schema', + json_schema: CARD_VISION_SCHEMA, + }; + } + + return body; +} + /** * Server-side vision analysis via Vercel AI Gateway. Requires AI_GATEWAY_API_KEY. * @param {string} imageDataUrl - data:image/jpeg;base64,... capture from scanner @@ -73,25 +156,26 @@ export async function analyzeCardImage(imageDataUrl) { throw new Error('Invalid image data format'); } - const response = await fetch(GATEWAY_URL, { + let response = await fetch(GATEWAY_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}`, }, - body: JSON.stringify({ - model: DEFAULT_VISION_MODEL, - temperature: 0.1, - messages: [{ - role: 'user', - content: [ - { type: 'text', text: CARD_PROMPT }, - { type: 'image_url', image_url: { url: imageDataUrl } }, - ], - }], - }), + body: JSON.stringify(buildGatewayBody(imageDataUrl)), }); + if (!response.ok && response.status === 400) { + response = await fetch(GATEWAY_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify(buildGatewayBody(imageDataUrl, { structured: false })), + }); + } + if (!response.ok) { const errorData = await response.json().catch(() => ({})); const message = errorData.error?.message || errorData.message || 'Unknown error'; @@ -101,7 +185,7 @@ export async function analyzeCardImage(imageDataUrl) { const data = await response.json(); const content = data.choices?.[0]?.message?.content; if (!content) { - throw new Error('No response from vision model'); + throw new VisionParseError('No response from vision model'); } const result = parseVisionJson(content); diff --git a/lib/scanner-card-detection.js b/lib/scanner-card-detection.js index bd5a1d7..c1b2734 100644 --- a/lib/scanner-card-detection.js +++ b/lib/scanner-card-detection.js @@ -12,10 +12,10 @@ export const SHAPE_DETECTION_INTERVAL_MS = 200; export const VERIFICATION_INTERVAL_MS = 1000; /** Minimum stable frames before a tracked card is sent for identification. */ -export const MIN_STABLE_COUNT_FOR_VERIFY = 6; +export const MIN_STABLE_COUNT_FOR_VERIFY = 3; /** Minimum time (ms) a card must be tracked before verification. */ -export const MIN_FIRST_SEEN_MS_FOR_VERIFY = 2500; +export const MIN_FIRST_SEEN_MS_FOR_VERIFY = 800; /** Delay after camera start before detection loops begin. */ export const DETECTION_START_DELAY_MS = 1000; diff --git a/lib/scanner-card-identify.js b/lib/scanner-card-identify.js index 6dc4286..7895e99 100644 --- a/lib/scanner-card-identify.js +++ b/lib/scanner-card-identify.js @@ -1,6 +1,9 @@ /** Default margin (px) around tracked bounds when cropping a card capture. */ export const CAPTURE_MARGIN_PX = 20; +/** JPEG quality for card crops sent to OCR / vision. */ +export const OCR_CAPTURE_JPEG_QUALITY = 0.92; + /** Vision rate-limit backoff duration (ms). */ export const VISION_RATE_LIMIT_MS = 60_000; @@ -79,6 +82,7 @@ export function resolveIdentifyOutcome(result) { matches: result.matches, ocrMeta, message: result.message, + fromLayer1: result.layer === 1, }; } @@ -182,7 +186,7 @@ export function captureCardRegionFromVideo(video, canvas, bounds, margin = CAPTU canvas.height ); - return canvas.toDataURL('image/jpeg', 0.8); + return canvas.toDataURL('image/jpeg', OCR_CAPTURE_JPEG_QUALITY); } export async function submitScanForReview({ imageData, name, candidateCardIds, authHeaders }) { @@ -205,11 +209,11 @@ export async function submitScanForReview({ imageData, name, candidateCardIds, a return { ok: true, message: result.message }; } -export async function fetchIdentifyByText({ ocrText, ocrConfidence, authHeaders }) { +export async function fetchIdentifyByText({ ocrText, ocrConfidence, cardNumber, authHeaders }) { const response = await fetch('/api/cards/identify-by-text', { method: 'POST', headers: authHeaders, - body: JSON.stringify({ ocrText, ocrConfidence }), + body: JSON.stringify({ ocrText, ocrConfidence, cardNumber }), }); if (!response.ok) { @@ -245,16 +249,17 @@ export async function fetchVisionIdentify(imageData, authHeaders) { * Returns { handled: true, outcome } when L1 resolves without escalation. */ export async function tryLayer1TextIdentify(imageData, authHeaders) { - const { recognizeCardNameStrip } = await import('./ocr-worker.js'); - const ocr = await recognizeCardNameStrip(imageData); + const { recognizeCardFields } = await import('./ocr-worker.js'); + const ocr = await recognizeCardFields(imageData); - if (ocr.text.length < 3) { + if (ocr.nameText.length < 3) { return { handled: false }; } const l1 = await fetchIdentifyByText({ - ocrText: ocr.text, - ocrConfidence: ocr.confidence, + ocrText: ocr.nameText, + ocrConfidence: ocr.nameConfidence, + cardNumber: ocr.cardNumber || undefined, authHeaders, }); diff --git a/lib/use-scanner-identification.js b/lib/use-scanner-identification.js index 2866461..f73e355 100644 --- a/lib/use-scanner-identification.js +++ b/lib/use-scanner-identification.js @@ -131,6 +131,7 @@ export function useScannerIdentification({ candidates: outcome.matches, ocrMeta: outcome.ocrMeta, message: outcome.message, + fromLayer1: Boolean(outcome.fromLayer1), }); break; case 'notice': @@ -179,6 +180,7 @@ export function useScannerIdentification({ useEffect(() => { if (!disambiguation?.imageData) return; + if (disambiguation.fromLayer1) return; if (Date.now() < visionCooldownUntilRef.current) return; const refineKey = disambiguation.cardTracker?.id ?? 'modal'; diff --git a/pages/api/cards/identify-by-text.js b/pages/api/cards/identify-by-text.js index 1dc5cd3..6be4bef 100644 --- a/pages/api/cards/identify-by-text.js +++ b/pages/api/cards/identify-by-text.js @@ -32,19 +32,20 @@ export default async function handler(req, res) { return res.status(401).json({ error: 'Authentication required' }); } - const { ocrText, ocrConfidence, game } = req.body || {}; + const { ocrText, ocrConfidence, cardNumber, game } = req.body || {}; if (!ocrText || typeof ocrText !== 'string') { return res.status(400).json({ error: 'ocrText is required' }); } - const matchResult = await matchTextInCatalog({ ocrText, game, ocrConfidence }); + const matchResult = await matchTextInCatalog({ ocrText, cardNumber, game, ocrConfidence }); const latencyMs = Date.now() - startedAt; const ocrMeta = { confidence: ocrConfidence ?? null, rawText: ocrText, query: matchResult.query || ocrText, + cardNumber: matchResult.cardNumber ?? cardNumber ?? null, }; if (matchResult.type === 'escalate') { diff --git a/pages/api/scan/identify.js b/pages/api/scan/identify.js index 5079823..a4b8008 100644 --- a/pages/api/scan/identify.js +++ b/pages/api/scan/identify.js @@ -1,6 +1,6 @@ import { getUserFromRequest } from '../../../lib/permission-middleware'; import { checkScanRateLimit } from '../../../lib/rate-limit.js'; -import { analyzeCardImage } from '../../../lib/scan-vision.js'; +import { analyzeCardImage, VisionParseError } from '../../../lib/scan-vision.js'; import { matchCardInCatalog, logScanAttempt } from '../../../lib/card-catalog-match.js'; function scanErrorResponse(error) { @@ -42,6 +42,18 @@ function scanErrorResponse(error) { }; } + if (error instanceof VisionParseError || msg.includes('Failed to parse structured vision')) { + return { + status: 200, + body: { + isCard: true, + card: null, + needsUserInput: true, + message: 'Could not read card details from scan — try again or pick from matches.', + }, + }; + } + return { status: 500, body: { error: 'Internal server error' }, diff --git a/test/lib/card-text-match.test.js b/test/lib/card-text-match.test.js new file mode 100644 index 0000000..a1563a2 --- /dev/null +++ b/test/lib/card-text-match.test.js @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest'; +import { extractCollectorNumberCandidate } from '../../lib/ocr-worker.js'; + +describe('extractCollectorNumberCandidate', () => { + it('extracts slash-style collector numbers', () => { + expect(extractCollectorNumberCandidate('015 / 208\nRare')).toBe('015/208'); + }); + + it('extracts compact numbers from the last numeric line', () => { + expect(extractCollectorNumberCandidate('Some set symbol\n123/456')).toBe('123/456'); + }); + + it('returns empty for non-numeric noise', () => { + expect(extractCollectorNumberCandidate('Creature — Human')).toBe(''); + }); +}); diff --git a/test/lib/scanner-card-detection.test.js b/test/lib/scanner-card-detection.test.js index 4e57c50..6728f58 100644 --- a/test/lib/scanner-card-detection.test.js +++ b/test/lib/scanner-card-detection.test.js @@ -97,16 +97,16 @@ describe('selectCardsReadyForVerification', () => { const ready = { id: 1, status: 'detecting', - stableCount: 6, + stableCount: 3, scanAttempts: 0, - firstSeen: now - 3000, + firstSeen: now - 900, }; const tooFresh = { id: 2, status: 'detecting', - stableCount: 6, + stableCount: 3, scanAttempts: 0, - firstSeen: now - 1000, + firstSeen: now - 500, }; expect(selectCardsReadyForVerification([ready, tooFresh], now)).toEqual([ready]); diff --git a/test/lib/scanner-card-identify.test.js b/test/lib/scanner-card-identify.test.js index bd7ff47..50c88cb 100644 --- a/test/lib/scanner-card-identify.test.js +++ b/test/lib/scanner-card-identify.test.js @@ -53,6 +53,7 @@ describe('resolveIdentifyOutcome', () => { matches, ocrMeta: expect.objectContaining({ abilities: [] }), message: 'Pick one', + fromLayer1: false, }); }); @@ -63,6 +64,18 @@ describe('resolveIdentifyOutcome', () => { message: 'Blurry', }); }); + + it('marks L1 disambiguation so Gemini refine is skipped', () => { + const outcome = resolveIdentifyOutcome({ + isCard: true, + layer: 1, + needsUserSelection: true, + matches: [{ id: 1, name: 'Bolt' }], + message: 'Pick one', + }); + + expect(outcome.fromLayer1).toBe(true); + }); }); describe('buildScannedCardPayload', () => {