diff --git a/.convoys/.metrics.jsonl b/.convoys/.metrics.jsonl index 3c48ae7..3b178d5 100644 --- a/.convoys/.metrics.jsonl +++ b/.convoys/.metrics.jsonl @@ -94,3 +94,10 @@ {"ts": "2026-08-15T00:26:08Z", "role": "role-design-system-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 82, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} {"ts": "2026-08-15T00:55:56Z", "role": "role-reviewer", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 0, "duration_s": 180, "multitask_group": "audit-tighten-scan-identify-hot-path-uncommitted", "model": "cursor-grok-4.5-high", "model_tier": "fast"} {"ts": "2026-08-15T00:56:39Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-ux-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-architect", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 300, "model": "composer-2.5", "model_tier": "standard"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-implementer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 1, "duration_s": 900, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-implementer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 2, "duration_s": 300, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} +{"ts": "2026-08-15T01:24:40Z", "role": "role-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} +{"ts": "2026-08-15T01:24:41Z", "role": "role-security-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"} +{"ts": "2026-08-15T01:24:41Z", "role": "role-a11y-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 60, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} diff --git a/.convoys/improve-scan-card-detection.md b/.convoys/improve-scan-card-detection.md index 3840e95..84690ca 100644 --- a/.convoys/improve-scan-card-detection.md +++ b/.convoys/improve-scan-card-detection.md @@ -9,7 +9,7 @@ skip: - ia - ui-design - flag -status: open +status: in-progress created: 2026-08-14 depends_on: - tighten-scan-identify-hot-path @@ -104,9 +104,9 @@ either model. ## Todos -- [ ] Architect: pick detector; document WASM / weight budget -- [ ] Brief 1 — detect + warp library + tests -- [ ] Brief 2 — wire `use-camera-scanner.js` + crop used by +- [x] Architect: pick detector; document WASM / weight budget +- [x] Brief 1 — detect + warp library + tests +- [x] Brief 2 — wire `use-camera-scanner.js` + crop used by `identifyTrackedCardCapture` - [ ] Confirm `scanner-mobile-checkout` overlay still maps bounds - [ ] Re-measure L2 `not_a_card` share @@ -149,3 +149,58 @@ Do not drop OpenCV.js in only to reimplement the current rectangle hunt. Success is a **rectified card image**, not a prettier box. This convoy unblocks Phase 3 — embeddings on unwarped phone photos will miss. + +## UX + +No new routes or screens. Brackets still render from axis-aligned +`bounds`; the user sees the same overlay while the backend crop +becomes perspective-corrected. + +### Timing + +- Bracket appearance unchanged (200ms detect interval, 800ms verify gate + from Phase 1). +- Warp adds ~20–40ms on capture only — not on the detect loop. + +### Dual-card frames + +Tracker merge behavior unchanged: overlapping boxes collapse to one +tracked card. Warp runs per tracked card at verify time. + +### Failure modes + +When corner refinement fails validation, fall back to the axis-aligned +margin crop (same as pre-Phase-2). No new error toast. + +## Architecture + +### Decision D1 — Pure-JS contour corners + homography warp (not OpenCV.js WASM) + +OpenCV.js adds ~8MB WASM and a Turbopack dynamic-import footgun. +Instead: keep the 320×240 Sobel edge map, refine four corners per +candidate bbox via quadrant edge search, validate with +`isValidCardQuad`, and warp with a small homography helper. + +Rejected: YOLO11n ONNX (weight hosting + license review), OpenCV.js +(full WASM budget). + +### Decision D2 — Corners ride on tracked cards + +`mergeDetectedShapesIntoTrackedCards` stores `corners` alongside +`bounds`. Overlay continues to use `bounds` only. + +### Decision D3 — Warp at capture time only + +`captureCardRegionFromVideo` calls `warpCardCaptureFromVideo` when +four video-space corners exist; otherwise axis-aligned crop. + +### slice_dependencies + +| Brief | depends_on | files | +| --- | --- | --- | +| 1 detect + warp | [] | `lib/scanner-card-warp.js`, `lib/scanner-card-detection.js`, tests | +| 2 wire capture | [1] | `lib/scanner-card-identify.js`, `lib/use-camera-scanner.js`, `ScannerCamera.js` | + +Serial implement: Brief 1 → Brief 2. + +Audit group id: `audit-improve-scan-card-detection-`. diff --git a/.convoys/improve-scan-card-detection/brief-1-detect-warp-lib.md b/.convoys/improve-scan-card-detection/brief-1-detect-warp-lib.md new file mode 100644 index 0000000..dd57282 --- /dev/null +++ b/.convoys/improve-scan-card-detection/brief-1-detect-warp-lib.md @@ -0,0 +1,27 @@ +--- +convoy: improve-scan-card-detection +brief_number: 1 +depends_on: [] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/scanner-card-warp.js + - lib/scanner-card-detection.js + - test/lib/scanner-card-warp.test.js + - test/lib/scanner-card-detection.test.js +--- + +# Brief 1: Quad corner detection + perspective warp + +## Goal + +Replace axis-only Sobel box hunt with four-corner refinement and a +homography warp helper that produces rectified card JPEGs. + +## Acceptance criteria + +- [ ] `lib/scanner-card-warp.js` exports `orderQuadCorners`, + `computeHomography`, `warpCardCaptureFromVideo` +- [ ] `detectCardShapesFromFrame` returns `corners` in video space +- [ ] Tracker merge preserves `corners` on tracked cards +- [ ] Unit tests for warp math + corner refinement diff --git a/.convoys/improve-scan-card-detection/brief-2-wire-capture.md b/.convoys/improve-scan-card-detection/brief-2-wire-capture.md new file mode 100644 index 0000000..1a6f554 --- /dev/null +++ b/.convoys/improve-scan-card-detection/brief-2-wire-capture.md @@ -0,0 +1,25 @@ +--- +convoy: improve-scan-card-detection +brief_number: 2 +depends_on: [1] +recommended_model: composer-2.5-fast +model_tier: fast +files: + - lib/scanner-card-identify.js + - lib/use-camera-scanner.js + - components/scanner/ScannerCamera.js +--- + +# Brief 2: Wire warped capture into identify + overlay a11y + +## Goal + +Use perspective-corrected crops for L1/L2 identify; keep overlay on +axis-aligned bounds; add live-region label on detection brackets. + +## Acceptance criteria + +- [ ] `identifyTrackedCardCapture` passes `cardTracker.corners` to capture +- [ ] `captureCardRegionFromVideo` warps when corners present, else fallback +- [ ] `use-camera-scanner.js` comment no longer claims OpenCV +- [ ] `DetectionFrame` exposes `role="status"` + `aria-label` diff --git a/components/scanner/ScannerCamera.js b/components/scanner/ScannerCamera.js index 11ed336..dd3ba48 100644 --- a/components/scanner/ScannerCamera.js +++ b/components/scanner/ScannerCamera.js @@ -10,9 +10,17 @@ import { useScannerFlash } from '../../lib/use-scanner-flash.js'; const TOAST_DURATION_MS = 2500; function DetectionFrame({ card, videoMetrics }) { + const label = + card.status === 'scanned' + ? `Card ${card.id} scanned` + : `Card ${card.id} identified`; + return (
threshold; } +function buildGrayscaleAndEdges(imageData, width, height) { + const data = imageData.data; + const grayscale = new Uint8Array(width * height); + const edges = new Uint8Array(width * height); + + for (let i = 0, px = 0; i < data.length; i += 4, px++) { + grayscale[px] = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]); + } + + for (let y = 1; y < height - 1; y++) { + for (let x = 1; x < width - 1; x++) { + const idx = y * width + x; + + const gx = + -grayscale[idx - 1] + + grayscale[idx + 1] + + -2 * grayscale[idx - 1 + width] + + 2 * grayscale[idx + 1 + width] + + -grayscale[idx - 1 + 2 * width] + + grayscale[idx + 1 + 2 * width]; + + const gy = + -grayscale[idx - width] - + 2 * grayscale[idx] - + grayscale[idx + width] + + grayscale[idx - width + 2 * width] + + 2 * grayscale[idx + 2 * width] + + grayscale[idx + width + 2 * width]; + + const magnitude = Math.sqrt(gx * gx + gy * gy); + edges[idx] = magnitude > 90 ? 255 : 0; + } + } + + return { grayscale, edges }; +} + +/** + * Refine a coarse card bbox into four corners using edge strength in each quadrant. + * Exported for unit tests. + */ +export function refineCardCornersFromEdges(edges, width, height, bbox) { + const { x, y, width: boxWidth, height: boxHeight } = bbox; + const centerX = x + boxWidth / 2; + const centerY = y + boxHeight / 2; + + const quadrants = [ + { minX: x, maxX: centerX, minY: y, maxY: centerY }, + { minX: centerX, maxX: x + boxWidth, minY: y, maxY: centerY }, + { minX: centerX, maxX: x + boxWidth, minY: centerY, maxY: y + boxHeight }, + { minX: x, maxX: centerX, minY: centerY, maxY: y + boxHeight }, + ]; + + const corners = quadrants.map((quad) => { + let best = null; + let bestScore = -1; + + for (let py = Math.floor(quad.minY); py < quad.maxY; py++) { + for (let px = Math.floor(quad.minX); px < quad.maxX; px++) { + if (px < 0 || py < 0 || px >= width || py >= height) continue; + const idx = py * width + px; + if (edges[idx] !== 255) continue; + + const dx = px - centerX; + const dy = py - centerY; + const score = Math.hypot(dx, dy); + if (score > bestScore) { + bestScore = score; + best = { x: px, y: py }; + } + } + } + + if (!best) { + return { + x: (quad.minX + quad.maxX) / 2, + y: (quad.minY + quad.maxY) / 2, + }; + } + + return best; + }); + + return orderQuadCorners(corners); +} + +function scoreCardCandidate(edges, width, height, bbox) { + const { x, y, width: w, height: h } = bbox; + let edgeCount = 0; + let totalPixels = 0; + let perimeterEdges = 0; + + for (let sy = y; sy < y + h; sy += 3) { + for (let sx = x; sx < x + w; sx += 3) { + if (sx < 0 || sy < 0 || sx >= width || sy >= height) continue; + const idx = sy * width + sx; + totalPixels++; + if (edges[idx] !== 255) continue; + edgeCount++; + const isPerimeter = + sx < x + w * 0.15 || sx > x + w * 0.85 || sy < y + h * 0.15 || sy > y + h * 0.85; + if (isPerimeter) perimeterEdges++; + } + } + + const edgeDensity = edgeCount / (totalPixels || 1); + const perimeterRatio = perimeterEdges / (edgeCount || 1); + if (edgeDensity <= 0.18 || edgeDensity >= 0.65 || perimeterRatio <= 0.35 || edgeCount <= 60) { + return null; + } + + return edgeDensity * 100 + perimeterRatio * 60 + edgeCount / 10; +} + +function findCardCandidates(edges, width, height) { + const minCardWidth = Math.floor(width * 0.2); + const maxCardWidth = Math.floor(width * 0.6); + const minCardHeight = Math.floor(height * 0.25); + const maxCardHeight = Math.floor(height * 0.7); + const candidates = []; + + for (let y = 0; y < height - minCardHeight; y += 12) { + for (let x = 0; x < width - minCardWidth; x += 12) { + for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 16) { + for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 16) { + const aspectRatio = w / h; + if (aspectRatio < 0.63 || aspectRatio > 0.77) continue; + + const score = scoreCardCandidate(edges, width, height, { x, y, width: w, height: h }); + if (score == null || score <= 50) continue; + + const corners = refineCardCornersFromEdges(edges, width, height, { + x, + y, + width: w, + height: h, + }); + const bounds = boundsFromCorners(corners); + if (!isValidCardQuad(corners)) continue; + + candidates.push({ + x: bounds.x, + y: bounds.y, + width: bounds.width, + height: bounds.height, + corners, + score: score + (isValidCardQuad(corners) ? 20 : 0), + aspectRatio, + }); + } + } + } + } + + return candidates.sort((a, b) => b.score - a.score).slice(0, 5); +} + /** * Draw the current video frame on `detectionCanvas` and return up to 5 card-like - * bounding boxes in video coordinates. + * detections in video coordinates, including perspective corners when found. */ export function detectCardShapesFromFrame(video, detectionCanvas) { if (!video || !detectionCanvas) return []; @@ -83,106 +253,27 @@ export function detectCardShapesFromFrame(video, detectionCanvas) { ctx.drawImage(video, 0, 0, detectionCanvas.width, detectionCanvas.height); const imageData = ctx.getImageData(0, 0, detectionCanvas.width, detectionCanvas.height); - const data = imageData.data; + const { edges } = buildGrayscaleAndEdges( + imageData, + detectionCanvas.width, + detectionCanvas.height + ); - const grayscale = []; - const edges = []; + const candidates = findCardCandidates(edges, detectionCanvas.width, detectionCanvas.height); - for (let i = 0; i < data.length; i += 4) { - const gray = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]); - grayscale.push(gray); - } + return candidates.map((candidate) => { + const videoBounds = convertToVideoCoordinates(candidate, detectionCanvas, video); + const videoCorners = candidate.corners.map((corner) => + convertPointToVideoCoordinates(corner, detectionCanvas, video) + ); - for (let y = 1; y < detectionCanvas.height - 1; y++) { - for (let x = 1; x < detectionCanvas.width - 1; x++) { - const idx = y * detectionCanvas.width + x; - - const gx = - -grayscale[idx - 1] + - grayscale[idx + 1] + - -2 * grayscale[idx - 1 + detectionCanvas.width] + - 2 * grayscale[idx + 1 + detectionCanvas.width] + - -grayscale[idx - 1 + 2 * detectionCanvas.width] + - grayscale[idx + 1 + 2 * detectionCanvas.width]; - - const gy = - -grayscale[idx - detectionCanvas.width] - - 2 * grayscale[idx] - - grayscale[idx + detectionCanvas.width] + - grayscale[idx - detectionCanvas.width + 2 * detectionCanvas.width] + - 2 * grayscale[idx + 2 * detectionCanvas.width] + - grayscale[idx + detectionCanvas.width + 2 * detectionCanvas.width]; - - const magnitude = Math.sqrt(gx * gx + gy * gy); - edges[idx] = magnitude > 100 ? 255 : 0; - } - } - - const cardShapes = []; - const { width, height } = detectionCanvas; - - const minCardWidth = Math.floor(width * 0.2); - const maxCardWidth = Math.floor(width * 0.6); - const minCardHeight = Math.floor(height * 0.25); - const maxCardHeight = Math.floor(height * 0.7); - - for (let y = 0; y < height - minCardHeight; y += 15) { - for (let x = 0; x < width - minCardWidth; x += 15) { - for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 20) { - for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 20) { - const aspectRatio = w / h; - if (aspectRatio < 0.63 || aspectRatio > 0.77) continue; - - let edgeCount = 0; - let totalPixels = 0; - let perimeterEdges = 0; - - for (let sy = y; sy < y + h; sy += 4) { - for (let sx = x; sx < x + w; sx += 4) { - const idx = sy * width + sx; - if (edges[idx] === 255) { - edgeCount++; - - const isPerimeter = - sx < x + w * 0.15 || - sx > x + w * 0.85 || - sy < y + h * 0.15 || - sy > y + h * 0.85; - if (isPerimeter) { - perimeterEdges++; - } - } - totalPixels++; - } - } - - const edgeDensity = edgeCount / totalPixels; - const perimeterRatio = perimeterEdges / (edgeCount || 1); - - if (edgeDensity > 0.2 && edgeDensity < 0.6 && perimeterRatio > 0.4 && edgeCount > 80) { - const score = edgeDensity * 100 + perimeterRatio * 60 + edgeCount / 10; - - if (score > 50) { - const videoCoords = convertToVideoCoordinates( - { x, y, width: w, height: h }, - detectionCanvas, - video - ); - - cardShapes.push({ - ...videoCoords, - score, - aspectRatio, - edgeDensity, - }); - } - } - } - } - } - } - - return cardShapes.sort((a, b) => b.score - a.score).slice(0, 5); + return { + ...videoBounds, + corners: videoCorners, + score: candidate.score, + aspectRatio: candidate.aspectRatio, + }; + }); } /** @@ -219,7 +310,15 @@ export function mergeDetectedShapesIntoTrackedCards( } if (matchedCard) { - matchedCard.bounds = { ...shape }; + matchedCard.bounds = { + x: shape.x, + y: shape.y, + width: shape.width, + height: shape.height, + }; + if (shape.corners?.length === 4) { + matchedCard.corners = shape.corners.map((corner) => ({ ...corner })); + } matchedCard.lastSeen = now; matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10); @@ -234,7 +333,13 @@ export function mergeDetectedShapesIntoTrackedCards( } else { const newCard = { id: idCounter++, - bounds: { ...shape }, + bounds: { + x: shape.x, + y: shape.y, + width: shape.width, + height: shape.height, + }, + corners: shape.corners?.length === 4 ? shape.corners.map((corner) => ({ ...corner })) : null, status: 'detecting', firstSeen: now, lastSeen: now, diff --git a/lib/scanner-card-identify.js b/lib/scanner-card-identify.js index 7895e99..5776cb9 100644 --- a/lib/scanner-card-identify.js +++ b/lib/scanner-card-identify.js @@ -1,3 +1,5 @@ +import { isValidCardQuad, warpCardCaptureFromVideo } from './scanner-card-warp.js'; + /** Default margin (px) around tracked bounds when cropping a card capture. */ export const CAPTURE_MARGIN_PX = 20; @@ -167,7 +169,23 @@ export function resolveDisambiguationRefineAction(result, { candidates }) { } /** Crop a tracked card region from the live video frame; returns a JPEG data URL. */ -export function captureCardRegionFromVideo(video, canvas, bounds, margin = CAPTURE_MARGIN_PX) { +export function captureCardRegionFromVideo( + video, + canvas, + bounds, + margin = CAPTURE_MARGIN_PX, + corners = null +) { + if (corners?.length === 4 && typeof document !== 'undefined' && isValidCardQuad(corners)) { + try { + return warpCardCaptureFromVideo(video, canvas, corners, { + jpegQuality: OCR_CAPTURE_JPEG_QUALITY, + }); + } catch (warpError) { + console.warn('Perspective warp failed, falling back to axis-aligned crop:', warpError); + } + } + const ctx = canvas.getContext('2d'); const { x, y, width, height } = bounds; @@ -283,7 +301,13 @@ export async function identifyTrackedCardCapture({ authHeaders, visionCooldownUntilMs = 0, }) { - const imageData = captureCardRegionFromVideo(video, canvas, cardTracker.bounds); + const imageData = captureCardRegionFromVideo( + video, + canvas, + cardTracker.bounds, + CAPTURE_MARGIN_PX, + cardTracker.corners + ); try { const l1 = await tryLayer1TextIdentify(imageData, authHeaders); diff --git a/lib/scanner-card-warp.js b/lib/scanner-card-warp.js new file mode 100644 index 0000000..4eed459 --- /dev/null +++ b/lib/scanner-card-warp.js @@ -0,0 +1,271 @@ +/** Standard trading-card aspect ratio (width / height). */ +export const CARD_ASPECT_RATIO = 5 / 7; + +/** Order four corners as top-left, top-right, bottom-right, bottom-left. */ +export function orderQuadCorners(points) { + if (!points || points.length !== 4) { + throw new Error('orderQuadCorners expects exactly four points'); + } + + const sortedByY = [...points].sort((a, b) => a.y - b.y); + const top = sortedByY.slice(0, 2).sort((a, b) => a.x - b.x); + const bottom = sortedByY.slice(2, 4).sort((a, b) => a.x - b.x); + + return [top[0], top[1], bottom[1], bottom[0]]; +} + +/** Axis-aligned bounds enclosing a quad. */ +export function boundsFromCorners(corners) { + const xs = corners.map((point) => point.x); + const ys = corners.map((point) => point.y); + const minX = Math.min(...xs); + const minY = Math.min(...ys); + const maxX = Math.max(...xs); + const maxY = Math.max(...ys); + + return { + x: minX, + y: minY, + width: maxX - minX, + height: maxY - minY, + }; +} + +/** True when corners form a convex quad with plausible card aspect ratio. */ +export function isValidCardQuad(corners, { minAspect = 0.6, maxAspect = 0.8 } = {}) { + if (!corners || corners.length !== 4) return false; + + const ordered = orderQuadCorners(corners); + const bounds = boundsFromCorners(ordered); + if (bounds.width < 20 || bounds.height < 28) return false; + + const aspect = bounds.width / bounds.height; + if (aspect < minAspect || aspect > maxAspect) return false; + + const area = polygonArea(ordered); + const boundsArea = bounds.width * bounds.height; + if (area / boundsArea < 0.55) return false; + + return isConvexQuad(ordered); +} + +function polygonArea(points) { + let sum = 0; + for (let i = 0; i < points.length; i++) { + const current = points[i]; + const next = points[(i + 1) % points.length]; + sum += current.x * next.y - next.x * current.y; + } + return Math.abs(sum) / 2; +} + +function isConvexQuad(points) { + let sign = 0; + for (let i = 0; i < 4; i++) { + const a = points[i]; + const b = points[(i + 1) % 4]; + const c = points[(i + 2) % 4]; + const cross = (b.x - a.x) * (c.y - b.y) - (b.y - a.y) * (c.x - b.x); + if (cross === 0) continue; + const currentSign = cross > 0 ? 1 : -1; + if (sign === 0) { + sign = currentSign; + } else if (sign !== currentSign) { + return false; + } + } + return sign !== 0; +} + +/** Solve 8×8 homography mapping src quad → axis-aligned dst rectangle. */ +export function computeHomography(srcCorners, dstWidth, dstHeight) { + const src = orderQuadCorners(srcCorners); + const dst = [ + { x: 0, y: 0 }, + { x: dstWidth, y: 0 }, + { x: dstWidth, y: dstHeight }, + { x: 0, y: dstHeight }, + ]; + + const rows = []; + for (let i = 0; i < 4; i++) { + const { x, y } = src[i]; + const { x: u, y: v } = dst[i]; + rows.push([x, y, 1, 0, 0, 0, -u * x, -u * y, u]); + rows.push([0, 0, 0, x, y, 1, -v * x, -v * y, v]); + } + + const h = solveLinearSystem(rows); + return [ + [h[0], h[1], h[2]], + [h[3], h[4], h[5]], + [h[6], h[7], 1], + ]; +} + +function solveLinearSystem(rows) { + const matrix = rows.map((row) => row.slice()); + const size = 8; + + for (let col = 0; col < size; col++) { + let pivotRow = col; + for (let row = col + 1; row < size; row++) { + if (Math.abs(matrix[row][col]) > Math.abs(matrix[pivotRow][col])) { + pivotRow = row; + } + } + + if (Math.abs(matrix[pivotRow][col]) < 1e-9) { + throw new Error('Homography system is singular'); + } + + [matrix[col], matrix[pivotRow]] = [matrix[pivotRow], matrix[col]]; + + const pivot = matrix[col][col]; + for (let j = col; j <= size; j++) { + matrix[col][j] /= pivot; + } + + for (let row = 0; row < size; row++) { + if (row === col) continue; + const factor = matrix[row][col]; + for (let j = col; j <= size; j++) { + matrix[row][j] -= factor * matrix[col][j]; + } + } + } + + return matrix.map((row) => row[size]); +} + +function applyHomographyInverse(matrix, x, y) { + const denom = matrix[2][0] * x + matrix[2][1] * y + matrix[2][2]; + const srcX = (matrix[0][0] * x + matrix[0][1] * y + matrix[0][2]) / denom; + const srcY = (matrix[1][0] * x + matrix[1][1] * y + matrix[1][2]) / denom; + return { x: srcX, y: srcY }; +} + +function invert3x3(matrix) { + const [ + [a, b, c], + [d, e, f], + [g, h, i], + ] = matrix; + + const A = e * i - f * h; + const B = -(d * i - f * g); + const C = d * h - e * g; + const D = -(b * i - c * h); + const E = a * i - c * g; + const F = -(a * h - b * g); + const G = b * f - c * e; + const H = -(a * f - c * d); + const I = a * e - b * d; + const det = a * A + b * B + c * C; + + if (Math.abs(det) < 1e-9) { + throw new Error('Homography matrix is not invertible'); + } + + const invDet = 1 / det; + return [ + [A * invDet, D * invDet, G * invDet], + [B * invDet, E * invDet, H * invDet], + [C * invDet, F * invDet, I * invDet], + ]; +} + +function sampleBilinear(data, width, height, x, y) { + const clampedX = Math.max(0, Math.min(width - 1, x)); + const clampedY = Math.max(0, Math.min(height - 1, y)); + const x0 = Math.floor(clampedX); + const y0 = Math.floor(clampedY); + const x1 = Math.min(x0 + 1, width - 1); + const y1 = Math.min(y0 + 1, height - 1); + const tx = clampedX - x0; + const ty = clampedY - y0; + + const idx = (row, col) => (row * width + col) * 4; + const sample = (row, col) => { + const base = idx(row, col); + return [data[base], data[base + 1], data[base + 2], data[base + 3]]; + }; + + const c00 = sample(y0, x0); + const c10 = sample(y0, x1); + const c01 = sample(y1, x0); + const c11 = sample(y1, x1); + + const out = [0, 0, 0, 255]; + for (let channel = 0; channel < 3; channel++) { + const top = c00[channel] * (1 - tx) + c10[channel] * tx; + const bottom = c01[channel] * (1 - tx) + c11[channel] * tx; + out[channel] = Math.round(top * (1 - ty) + bottom * ty); + } + return out; +} + +/** + * Perspective-correct a card region from the live video frame. + * Returns a JPEG data URL sized to the card aspect ratio. + */ +export function warpCardCaptureFromVideo( + video, + canvas, + corners, + { jpegQuality = 0.92, maxWidth = 480 } = {} +) { + if (!video || !canvas || !corners || corners.length !== 4) { + throw new Error('warpCardCaptureFromVideo requires video, canvas, and four corners'); + } + + const ordered = orderQuadCorners(corners); + const bounds = boundsFromCorners(ordered); + const outputHeight = Math.max(1, Math.round(maxWidth / CARD_ASPECT_RATIO)); + const outputWidth = maxWidth; + + const homography = computeHomography(ordered, outputWidth, outputHeight); + const inverse = invert3x3(homography); + + const sourceCanvas = document.createElement('canvas'); + sourceCanvas.width = video.videoWidth; + sourceCanvas.height = video.videoHeight; + const sourceCtx = sourceCanvas.getContext('2d'); + sourceCtx.drawImage(video, 0, 0); + const sourceData = sourceCtx.getImageData(0, 0, sourceCanvas.width, sourceCanvas.height).data; + + canvas.width = outputWidth; + canvas.height = outputHeight; + const ctx = canvas.getContext('2d'); + const output = ctx.createImageData(outputWidth, outputHeight); + + for (let y = 0; y < outputHeight; y++) { + for (let x = 0; x < outputWidth; x++) { + const mapped = applyHomographyInverse(inverse, x, y); + if ( + mapped.x < bounds.x - 5 || + mapped.y < bounds.y - 5 || + mapped.x > bounds.x + bounds.width + 5 || + mapped.y > bounds.y + bounds.height + 5 + ) { + continue; + } + + const rgba = sampleBilinear( + sourceData, + sourceCanvas.width, + sourceCanvas.height, + mapped.x, + mapped.y + ); + const outIdx = (y * outputWidth + x) * 4; + output.data[outIdx] = rgba[0]; + output.data[outIdx + 1] = rgba[1]; + output.data[outIdx + 2] = rgba[2]; + output.data[outIdx + 3] = 255; + } + } + + ctx.putImageData(output, 0, 0); + return canvas.toDataURL('image/jpeg', jpegQuality); +} diff --git a/lib/use-camera-scanner.js b/lib/use-camera-scanner.js index 71aaf44..496ec7f 100644 --- a/lib/use-camera-scanner.js +++ b/lib/use-camera-scanner.js @@ -9,7 +9,7 @@ import { } from './scanner-card-detection.js'; /** - * Camera stream + OpenCV shape detection loop for the card scanner. + * Camera stream + card shape detection loop for the card scanner. * Identification callbacks stay in the parent component. */ export function useCameraScanner({ diff --git a/test/lib/scanner-card-detection.test.js b/test/lib/scanner-card-detection.test.js index 6728f58..306dfd2 100644 --- a/test/lib/scanner-card-detection.test.js +++ b/test/lib/scanner-card-detection.test.js @@ -3,6 +3,7 @@ import { convertToVideoCoordinates, mergeDetectedShapesIntoTrackedCards, rectanglesOverlap, + refineCardCornersFromEdges, selectCardsReadyForVerification, TRACK_STALE_MS, } from '../../lib/scanner-card-detection.js'; @@ -37,7 +38,19 @@ describe('convertToVideoCoordinates', () => { describe('mergeDetectedShapesIntoTrackedCards', () => { it('creates a new tracked card for an unmatched shape', () => { const now = 1_000_000; - const shape = { x: 10, y: 20, width: 100, height: 140, score: 80 }; + const shape = { + x: 10, + y: 20, + width: 100, + height: 140, + score: 80, + corners: [ + { x: 10, y: 20 }, + { x: 110, y: 22 }, + { x: 108, y: 160 }, + { x: 12, y: 158 }, + ], + }; const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards([], [shape], 1, now); @@ -47,7 +60,8 @@ describe('mergeDetectedShapesIntoTrackedCards', () => { status: 'detecting', stableCount: 1, scanAttempts: 0, - bounds: shape, + bounds: { x: 10, y: 20, width: 100, height: 140 }, + corners: shape.corners, }); expect(nextCardId).toBe(2); }); @@ -125,3 +139,31 @@ describe('selectCardsReadyForVerification', () => { expect(selectCardsReadyForVerification([retried], now)).toHaveLength(0); }); }); + +describe('refineCardCornersFromEdges', () => { + it('picks strong edge pixels near each quadrant corner', () => { + const width = 120; + const height = 168; + const edges = new Uint8Array(width * height); + const mark = (x, y) => { + edges[y * width + x] = 255; + }; + + mark(8, 10); + mark(110, 12); + mark(108, 156); + mark(10, 154); + + const corners = refineCardCornersFromEdges(edges, width, height, { + x: 5, + y: 8, + width: 110, + height: 150, + }); + + expect(corners[0]).toEqual({ x: 8, y: 10 }); + expect(corners[1]).toEqual({ x: 110, y: 12 }); + expect(corners[2]).toEqual({ x: 108, y: 156 }); + expect(corners[3]).toEqual({ x: 10, y: 154 }); + }); +}); diff --git a/test/lib/scanner-card-warp.test.js b/test/lib/scanner-card-warp.test.js new file mode 100644 index 0000000..e48b67f --- /dev/null +++ b/test/lib/scanner-card-warp.test.js @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest'; +import { + boundsFromCorners, + computeHomography, + isValidCardQuad, + orderQuadCorners, +} from '../../lib/scanner-card-warp.js'; + +describe('orderQuadCorners', () => { + it('orders corners as top-left, top-right, bottom-right, bottom-left', () => { + const points = [ + { x: 10, y: 10 }, + { x: 90, y: 12 }, + { x: 88, y: 140 }, + { x: 12, y: 138 }, + ]; + + expect(orderQuadCorners(points)).toEqual([ + { x: 10, y: 10 }, + { x: 90, y: 12 }, + { x: 88, y: 140 }, + { x: 12, y: 138 }, + ]); + }); +}); + +describe('boundsFromCorners', () => { + it('returns the enclosing axis-aligned rectangle', () => { + const corners = [ + { x: 10, y: 20 }, + { x: 110, y: 25 }, + { x: 105, y: 165 }, + { x: 15, y: 160 }, + ]; + + expect(boundsFromCorners(corners)).toEqual({ + x: 10, + y: 20, + width: 100, + height: 145, + }); + }); +}); + +describe('isValidCardQuad', () => { + it('accepts a convex card-shaped quad', () => { + const corners = [ + { x: 0, y: 0 }, + { x: 50, y: 0 }, + { x: 50, y: 70 }, + { x: 0, y: 70 }, + ]; + expect(isValidCardQuad(corners)).toBe(true); + }); + + it('rejects a quad with implausible aspect ratio', () => { + const corners = [ + { x: 0, y: 0 }, + { x: 200, y: 0 }, + { x: 200, y: 70 }, + { x: 0, y: 70 }, + ]; + expect(isValidCardQuad(corners)).toBe(false); + }); +}); + +describe('computeHomography', () => { + it('maps source corners to the destination rectangle', () => { + const src = [ + { x: 0, y: 0 }, + { x: 100, y: 0 }, + { x: 100, y: 140 }, + { x: 0, y: 140 }, + ]; + const matrix = computeHomography(src, 200, 280); + + const mappedTopLeft = applyForward(matrix, 0, 0); + expect(mappedTopLeft.x).toBeCloseTo(0, 4); + expect(mappedTopLeft.y).toBeCloseTo(0, 4); + + const mappedBottomRight = applyForward(matrix, 100, 140); + expect(mappedBottomRight.x).toBeCloseTo(200, 4); + expect(mappedBottomRight.y).toBeCloseTo(280, 4); + }); +}); + +function applyForward(matrix, x, y) { + const denom = matrix[2][0] * x + matrix[2][1] * y + matrix[2][2]; + return { + x: (matrix[0][0] * x + matrix[0][1] * y + matrix[0][2]) / denom, + y: (matrix[1][0] * x + matrix[1][1] * y + matrix[1][2]) / denom, + }; +}