feat(scanner): add debug instrumentation for vision pipeline timing #155
11 changed files with 766 additions and 109 deletions
|
|
@ -94,3 +94,10 @@
|
|||
{"ts": "2026-08-15T00:26:08Z", "role": "role-design-system-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 82, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||
{"ts": "2026-08-15T00:55:56Z", "role": "role-reviewer", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 0, "duration_s": 180, "multitask_group": "audit-tighten-scan-identify-hot-path-uncommitted", "model": "cursor-grok-4.5-high", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T00:56:39Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T01:24:40Z", "role": "role-ux-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T01:24:40Z", "role": "role-architect", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 300, "model": "composer-2.5", "model_tier": "standard"}
|
||||
{"ts": "2026-08-15T01:24:40Z", "role": "role-implementer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 1, "duration_s": 900, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T01:24:40Z", "role": "role-implementer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 2, "duration_s": 300, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
|
||||
{"ts": "2026-08-15T01:24:40Z", "role": "role-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||
{"ts": "2026-08-15T01:24:41Z", "role": "role-security-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"}
|
||||
{"ts": "2026-08-15T01:24:41Z", "role": "role-a11y-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 60, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ skip:
|
|||
- ia
|
||||
- ui-design
|
||||
- flag
|
||||
status: open
|
||||
status: in-progress
|
||||
created: 2026-08-14
|
||||
depends_on:
|
||||
- tighten-scan-identify-hot-path
|
||||
|
|
@ -104,9 +104,9 @@ either model.
|
|||
|
||||
## Todos
|
||||
|
||||
- [ ] Architect: pick detector; document WASM / weight budget
|
||||
- [ ] Brief 1 — detect + warp library + tests
|
||||
- [ ] Brief 2 — wire `use-camera-scanner.js` + crop used by
|
||||
- [x] Architect: pick detector; document WASM / weight budget
|
||||
- [x] Brief 1 — detect + warp library + tests
|
||||
- [x] Brief 2 — wire `use-camera-scanner.js` + crop used by
|
||||
`identifyTrackedCardCapture`
|
||||
- [ ] Confirm `scanner-mobile-checkout` overlay still maps bounds
|
||||
- [ ] Re-measure L2 `not_a_card` share
|
||||
|
|
@ -149,3 +149,58 @@ Do not drop OpenCV.js in only to reimplement the current rectangle
|
|||
hunt. Success is a **rectified card image**, not a prettier box.
|
||||
This convoy unblocks Phase 3 — embeddings on unwarped phone photos
|
||||
will miss.
|
||||
|
||||
## UX
|
||||
|
||||
No new routes or screens. Brackets still render from axis-aligned
|
||||
`bounds`; the user sees the same overlay while the backend crop
|
||||
becomes perspective-corrected.
|
||||
|
||||
### Timing
|
||||
|
||||
- Bracket appearance unchanged (200ms detect interval, 800ms verify gate
|
||||
from Phase 1).
|
||||
- Warp adds ~20–40ms on capture only — not on the detect loop.
|
||||
|
||||
### Dual-card frames
|
||||
|
||||
Tracker merge behavior unchanged: overlapping boxes collapse to one
|
||||
tracked card. Warp runs per tracked card at verify time.
|
||||
|
||||
### Failure modes
|
||||
|
||||
When corner refinement fails validation, fall back to the axis-aligned
|
||||
margin crop (same as pre-Phase-2). No new error toast.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Decision D1 — Pure-JS contour corners + homography warp (not OpenCV.js WASM)
|
||||
|
||||
OpenCV.js adds ~8MB WASM and a Turbopack dynamic-import footgun.
|
||||
Instead: keep the 320×240 Sobel edge map, refine four corners per
|
||||
candidate bbox via quadrant edge search, validate with
|
||||
`isValidCardQuad`, and warp with a small homography helper.
|
||||
|
||||
Rejected: YOLO11n ONNX (weight hosting + license review), OpenCV.js
|
||||
(full WASM budget).
|
||||
|
||||
### Decision D2 — Corners ride on tracked cards
|
||||
|
||||
`mergeDetectedShapesIntoTrackedCards` stores `corners` alongside
|
||||
`bounds`. Overlay continues to use `bounds` only.
|
||||
|
||||
### Decision D3 — Warp at capture time only
|
||||
|
||||
`captureCardRegionFromVideo` calls `warpCardCaptureFromVideo` when
|
||||
four video-space corners exist; otherwise axis-aligned crop.
|
||||
|
||||
### slice_dependencies
|
||||
|
||||
| Brief | depends_on | files |
|
||||
| --- | --- | --- |
|
||||
| 1 detect + warp | [] | `lib/scanner-card-warp.js`, `lib/scanner-card-detection.js`, tests |
|
||||
| 2 wire capture | [1] | `lib/scanner-card-identify.js`, `lib/use-camera-scanner.js`, `ScannerCamera.js` |
|
||||
|
||||
Serial implement: Brief 1 → Brief 2.
|
||||
|
||||
Audit group id: `audit-improve-scan-card-detection-<pr>`.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
---
|
||||
convoy: improve-scan-card-detection
|
||||
brief_number: 1
|
||||
depends_on: []
|
||||
recommended_model: composer-2.5-fast
|
||||
model_tier: fast
|
||||
files:
|
||||
- lib/scanner-card-warp.js
|
||||
- lib/scanner-card-detection.js
|
||||
- test/lib/scanner-card-warp.test.js
|
||||
- test/lib/scanner-card-detection.test.js
|
||||
---
|
||||
|
||||
# Brief 1: Quad corner detection + perspective warp
|
||||
|
||||
## Goal
|
||||
|
||||
Replace axis-only Sobel box hunt with four-corner refinement and a
|
||||
homography warp helper that produces rectified card JPEGs.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [ ] `lib/scanner-card-warp.js` exports `orderQuadCorners`,
|
||||
`computeHomography`, `warpCardCaptureFromVideo`
|
||||
- [ ] `detectCardShapesFromFrame` returns `corners` in video space
|
||||
- [ ] Tracker merge preserves `corners` on tracked cards
|
||||
- [ ] Unit tests for warp math + corner refinement
|
||||
25
.convoys/improve-scan-card-detection/brief-2-wire-capture.md
Normal file
25
.convoys/improve-scan-card-detection/brief-2-wire-capture.md
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
---
|
||||
convoy: improve-scan-card-detection
|
||||
brief_number: 2
|
||||
depends_on: [1]
|
||||
recommended_model: composer-2.5-fast
|
||||
model_tier: fast
|
||||
files:
|
||||
- lib/scanner-card-identify.js
|
||||
- lib/use-camera-scanner.js
|
||||
- components/scanner/ScannerCamera.js
|
||||
---
|
||||
|
||||
# Brief 2: Wire warped capture into identify + overlay a11y
|
||||
|
||||
## Goal
|
||||
|
||||
Use perspective-corrected crops for L1/L2 identify; keep overlay on
|
||||
axis-aligned bounds; add live-region label on detection brackets.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [ ] `identifyTrackedCardCapture` passes `cardTracker.corners` to capture
|
||||
- [ ] `captureCardRegionFromVideo` warps when corners present, else fallback
|
||||
- [ ] `use-camera-scanner.js` comment no longer claims OpenCV
|
||||
- [ ] `DetectionFrame` exposes `role="status"` + `aria-label`
|
||||
|
|
@ -10,9 +10,17 @@ import { useScannerFlash } from '../../lib/use-scanner-flash.js';
|
|||
const TOAST_DURATION_MS = 2500;
|
||||
|
||||
function DetectionFrame({ card, videoMetrics }) {
|
||||
const label =
|
||||
card.status === 'scanned'
|
||||
? `Card ${card.id} scanned`
|
||||
: `Card ${card.id} identified`;
|
||||
|
||||
return (
|
||||
<div
|
||||
className="absolute pointer-events-none scan-frame"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
aria-label={label}
|
||||
style={{
|
||||
left: `${(card.bounds.x / videoMetrics.width) * 100}%`,
|
||||
top: `${(card.bounds.y / videoMetrics.height) * 100}%`,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
import {
|
||||
boundsFromCorners,
|
||||
isValidCardQuad,
|
||||
orderQuadCorners,
|
||||
} from './scanner-card-warp.js';
|
||||
|
||||
/** Detection canvas dimensions (smaller than video for performance). */
|
||||
export const DETECTION_CANVAS_WIDTH = 320;
|
||||
export const DETECTION_CANVAS_HEIGHT = 240;
|
||||
|
|
@ -52,6 +58,13 @@ export function convertToVideoCoordinates(detection, canvas, video) {
|
|||
};
|
||||
}
|
||||
|
||||
/** Convert a point from detection canvas space to video pixel space. */
|
||||
export function convertPointToVideoCoordinates(point, canvas, video) {
|
||||
const scaleX = video.videoWidth / canvas.width;
|
||||
const scaleY = video.videoHeight / canvas.height;
|
||||
return { x: point.x * scaleX, y: point.y * scaleY };
|
||||
}
|
||||
|
||||
/** True when overlap area exceeds `threshold` of the smaller rectangle. */
|
||||
export function rectanglesOverlap(rect1, rect2, threshold = 0.3) {
|
||||
const x1 = Math.max(rect1.x, rect2.x);
|
||||
|
|
@ -69,9 +82,166 @@ export function rectanglesOverlap(rect1, rect2, threshold = 0.3) {
|
|||
return overlapArea / smallerArea > threshold;
|
||||
}
|
||||
|
||||
function buildGrayscaleAndEdges(imageData, width, height) {
|
||||
const data = imageData.data;
|
||||
const grayscale = new Uint8Array(width * height);
|
||||
const edges = new Uint8Array(width * height);
|
||||
|
||||
for (let i = 0, px = 0; i < data.length; i += 4, px++) {
|
||||
grayscale[px] = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]);
|
||||
}
|
||||
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
const idx = y * width + x;
|
||||
|
||||
const gx =
|
||||
-grayscale[idx - 1] +
|
||||
grayscale[idx + 1] +
|
||||
-2 * grayscale[idx - 1 + width] +
|
||||
2 * grayscale[idx + 1 + width] +
|
||||
-grayscale[idx - 1 + 2 * width] +
|
||||
grayscale[idx + 1 + 2 * width];
|
||||
|
||||
const gy =
|
||||
-grayscale[idx - width] -
|
||||
2 * grayscale[idx] -
|
||||
grayscale[idx + width] +
|
||||
grayscale[idx - width + 2 * width] +
|
||||
2 * grayscale[idx + 2 * width] +
|
||||
grayscale[idx + width + 2 * width];
|
||||
|
||||
const magnitude = Math.sqrt(gx * gx + gy * gy);
|
||||
edges[idx] = magnitude > 90 ? 255 : 0;
|
||||
}
|
||||
}
|
||||
|
||||
return { grayscale, edges };
|
||||
}
|
||||
|
||||
/**
|
||||
* Refine a coarse card bbox into four corners using edge strength in each quadrant.
|
||||
* Exported for unit tests.
|
||||
*/
|
||||
export function refineCardCornersFromEdges(edges, width, height, bbox) {
|
||||
const { x, y, width: boxWidth, height: boxHeight } = bbox;
|
||||
const centerX = x + boxWidth / 2;
|
||||
const centerY = y + boxHeight / 2;
|
||||
|
||||
const quadrants = [
|
||||
{ minX: x, maxX: centerX, minY: y, maxY: centerY },
|
||||
{ minX: centerX, maxX: x + boxWidth, minY: y, maxY: centerY },
|
||||
{ minX: centerX, maxX: x + boxWidth, minY: centerY, maxY: y + boxHeight },
|
||||
{ minX: x, maxX: centerX, minY: centerY, maxY: y + boxHeight },
|
||||
];
|
||||
|
||||
const corners = quadrants.map((quad) => {
|
||||
let best = null;
|
||||
let bestScore = -1;
|
||||
|
||||
for (let py = Math.floor(quad.minY); py < quad.maxY; py++) {
|
||||
for (let px = Math.floor(quad.minX); px < quad.maxX; px++) {
|
||||
if (px < 0 || py < 0 || px >= width || py >= height) continue;
|
||||
const idx = py * width + px;
|
||||
if (edges[idx] !== 255) continue;
|
||||
|
||||
const dx = px - centerX;
|
||||
const dy = py - centerY;
|
||||
const score = Math.hypot(dx, dy);
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
best = { x: px, y: py };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!best) {
|
||||
return {
|
||||
x: (quad.minX + quad.maxX) / 2,
|
||||
y: (quad.minY + quad.maxY) / 2,
|
||||
};
|
||||
}
|
||||
|
||||
return best;
|
||||
});
|
||||
|
||||
return orderQuadCorners(corners);
|
||||
}
|
||||
|
||||
function scoreCardCandidate(edges, width, height, bbox) {
|
||||
const { x, y, width: w, height: h } = bbox;
|
||||
let edgeCount = 0;
|
||||
let totalPixels = 0;
|
||||
let perimeterEdges = 0;
|
||||
|
||||
for (let sy = y; sy < y + h; sy += 3) {
|
||||
for (let sx = x; sx < x + w; sx += 3) {
|
||||
if (sx < 0 || sy < 0 || sx >= width || sy >= height) continue;
|
||||
const idx = sy * width + sx;
|
||||
totalPixels++;
|
||||
if (edges[idx] !== 255) continue;
|
||||
edgeCount++;
|
||||
const isPerimeter =
|
||||
sx < x + w * 0.15 || sx > x + w * 0.85 || sy < y + h * 0.15 || sy > y + h * 0.85;
|
||||
if (isPerimeter) perimeterEdges++;
|
||||
}
|
||||
}
|
||||
|
||||
const edgeDensity = edgeCount / (totalPixels || 1);
|
||||
const perimeterRatio = perimeterEdges / (edgeCount || 1);
|
||||
if (edgeDensity <= 0.18 || edgeDensity >= 0.65 || perimeterRatio <= 0.35 || edgeCount <= 60) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return edgeDensity * 100 + perimeterRatio * 60 + edgeCount / 10;
|
||||
}
|
||||
|
||||
function findCardCandidates(edges, width, height) {
|
||||
const minCardWidth = Math.floor(width * 0.2);
|
||||
const maxCardWidth = Math.floor(width * 0.6);
|
||||
const minCardHeight = Math.floor(height * 0.25);
|
||||
const maxCardHeight = Math.floor(height * 0.7);
|
||||
const candidates = [];
|
||||
|
||||
for (let y = 0; y < height - minCardHeight; y += 12) {
|
||||
for (let x = 0; x < width - minCardWidth; x += 12) {
|
||||
for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 16) {
|
||||
for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 16) {
|
||||
const aspectRatio = w / h;
|
||||
if (aspectRatio < 0.63 || aspectRatio > 0.77) continue;
|
||||
|
||||
const score = scoreCardCandidate(edges, width, height, { x, y, width: w, height: h });
|
||||
if (score == null || score <= 50) continue;
|
||||
|
||||
const corners = refineCardCornersFromEdges(edges, width, height, {
|
||||
x,
|
||||
y,
|
||||
width: w,
|
||||
height: h,
|
||||
});
|
||||
const bounds = boundsFromCorners(corners);
|
||||
if (!isValidCardQuad(corners)) continue;
|
||||
|
||||
candidates.push({
|
||||
x: bounds.x,
|
||||
y: bounds.y,
|
||||
width: bounds.width,
|
||||
height: bounds.height,
|
||||
corners,
|
||||
score: score + (isValidCardQuad(corners) ? 20 : 0),
|
||||
aspectRatio,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return candidates.sort((a, b) => b.score - a.score).slice(0, 5);
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw the current video frame on `detectionCanvas` and return up to 5 card-like
|
||||
* bounding boxes in video coordinates.
|
||||
* detections in video coordinates, including perspective corners when found.
|
||||
*/
|
||||
export function detectCardShapesFromFrame(video, detectionCanvas) {
|
||||
if (!video || !detectionCanvas) return [];
|
||||
|
|
@ -83,106 +253,27 @@ export function detectCardShapesFromFrame(video, detectionCanvas) {
|
|||
ctx.drawImage(video, 0, 0, detectionCanvas.width, detectionCanvas.height);
|
||||
|
||||
const imageData = ctx.getImageData(0, 0, detectionCanvas.width, detectionCanvas.height);
|
||||
const data = imageData.data;
|
||||
|
||||
const grayscale = [];
|
||||
const edges = [];
|
||||
|
||||
for (let i = 0; i < data.length; i += 4) {
|
||||
const gray = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]);
|
||||
grayscale.push(gray);
|
||||
}
|
||||
|
||||
for (let y = 1; y < detectionCanvas.height - 1; y++) {
|
||||
for (let x = 1; x < detectionCanvas.width - 1; x++) {
|
||||
const idx = y * detectionCanvas.width + x;
|
||||
|
||||
const gx =
|
||||
-grayscale[idx - 1] +
|
||||
grayscale[idx + 1] +
|
||||
-2 * grayscale[idx - 1 + detectionCanvas.width] +
|
||||
2 * grayscale[idx + 1 + detectionCanvas.width] +
|
||||
-grayscale[idx - 1 + 2 * detectionCanvas.width] +
|
||||
grayscale[idx + 1 + 2 * detectionCanvas.width];
|
||||
|
||||
const gy =
|
||||
-grayscale[idx - detectionCanvas.width] -
|
||||
2 * grayscale[idx] -
|
||||
grayscale[idx + detectionCanvas.width] +
|
||||
grayscale[idx - detectionCanvas.width + 2 * detectionCanvas.width] +
|
||||
2 * grayscale[idx + 2 * detectionCanvas.width] +
|
||||
grayscale[idx + detectionCanvas.width + 2 * detectionCanvas.width];
|
||||
|
||||
const magnitude = Math.sqrt(gx * gx + gy * gy);
|
||||
edges[idx] = magnitude > 100 ? 255 : 0;
|
||||
}
|
||||
}
|
||||
|
||||
const cardShapes = [];
|
||||
const { width, height } = detectionCanvas;
|
||||
|
||||
const minCardWidth = Math.floor(width * 0.2);
|
||||
const maxCardWidth = Math.floor(width * 0.6);
|
||||
const minCardHeight = Math.floor(height * 0.25);
|
||||
const maxCardHeight = Math.floor(height * 0.7);
|
||||
|
||||
for (let y = 0; y < height - minCardHeight; y += 15) {
|
||||
for (let x = 0; x < width - minCardWidth; x += 15) {
|
||||
for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 20) {
|
||||
for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 20) {
|
||||
const aspectRatio = w / h;
|
||||
if (aspectRatio < 0.63 || aspectRatio > 0.77) continue;
|
||||
|
||||
let edgeCount = 0;
|
||||
let totalPixels = 0;
|
||||
let perimeterEdges = 0;
|
||||
|
||||
for (let sy = y; sy < y + h; sy += 4) {
|
||||
for (let sx = x; sx < x + w; sx += 4) {
|
||||
const idx = sy * width + sx;
|
||||
if (edges[idx] === 255) {
|
||||
edgeCount++;
|
||||
|
||||
const isPerimeter =
|
||||
sx < x + w * 0.15 ||
|
||||
sx > x + w * 0.85 ||
|
||||
sy < y + h * 0.15 ||
|
||||
sy > y + h * 0.85;
|
||||
if (isPerimeter) {
|
||||
perimeterEdges++;
|
||||
}
|
||||
}
|
||||
totalPixels++;
|
||||
}
|
||||
}
|
||||
|
||||
const edgeDensity = edgeCount / totalPixels;
|
||||
const perimeterRatio = perimeterEdges / (edgeCount || 1);
|
||||
|
||||
if (edgeDensity > 0.2 && edgeDensity < 0.6 && perimeterRatio > 0.4 && edgeCount > 80) {
|
||||
const score = edgeDensity * 100 + perimeterRatio * 60 + edgeCount / 10;
|
||||
|
||||
if (score > 50) {
|
||||
const videoCoords = convertToVideoCoordinates(
|
||||
{ x, y, width: w, height: h },
|
||||
detectionCanvas,
|
||||
video
|
||||
const { edges } = buildGrayscaleAndEdges(
|
||||
imageData,
|
||||
detectionCanvas.width,
|
||||
detectionCanvas.height
|
||||
);
|
||||
|
||||
cardShapes.push({
|
||||
...videoCoords,
|
||||
score,
|
||||
aspectRatio,
|
||||
edgeDensity,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const candidates = findCardCandidates(edges, detectionCanvas.width, detectionCanvas.height);
|
||||
|
||||
return cardShapes.sort((a, b) => b.score - a.score).slice(0, 5);
|
||||
return candidates.map((candidate) => {
|
||||
const videoBounds = convertToVideoCoordinates(candidate, detectionCanvas, video);
|
||||
const videoCorners = candidate.corners.map((corner) =>
|
||||
convertPointToVideoCoordinates(corner, detectionCanvas, video)
|
||||
);
|
||||
|
||||
return {
|
||||
...videoBounds,
|
||||
corners: videoCorners,
|
||||
score: candidate.score,
|
||||
aspectRatio: candidate.aspectRatio,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -219,7 +310,15 @@ export function mergeDetectedShapesIntoTrackedCards(
|
|||
}
|
||||
|
||||
if (matchedCard) {
|
||||
matchedCard.bounds = { ...shape };
|
||||
matchedCard.bounds = {
|
||||
x: shape.x,
|
||||
y: shape.y,
|
||||
width: shape.width,
|
||||
height: shape.height,
|
||||
};
|
||||
if (shape.corners?.length === 4) {
|
||||
matchedCard.corners = shape.corners.map((corner) => ({ ...corner }));
|
||||
}
|
||||
matchedCard.lastSeen = now;
|
||||
matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10);
|
||||
|
||||
|
|
@ -234,7 +333,13 @@ export function mergeDetectedShapesIntoTrackedCards(
|
|||
} else {
|
||||
const newCard = {
|
||||
id: idCounter++,
|
||||
bounds: { ...shape },
|
||||
bounds: {
|
||||
x: shape.x,
|
||||
y: shape.y,
|
||||
width: shape.width,
|
||||
height: shape.height,
|
||||
},
|
||||
corners: shape.corners?.length === 4 ? shape.corners.map((corner) => ({ ...corner })) : null,
|
||||
status: 'detecting',
|
||||
firstSeen: now,
|
||||
lastSeen: now,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import { isValidCardQuad, warpCardCaptureFromVideo } from './scanner-card-warp.js';
|
||||
|
||||
/** Default margin (px) around tracked bounds when cropping a card capture. */
|
||||
export const CAPTURE_MARGIN_PX = 20;
|
||||
|
||||
|
|
@ -167,7 +169,23 @@ export function resolveDisambiguationRefineAction(result, { candidates }) {
|
|||
}
|
||||
|
||||
/** Crop a tracked card region from the live video frame; returns a JPEG data URL. */
|
||||
export function captureCardRegionFromVideo(video, canvas, bounds, margin = CAPTURE_MARGIN_PX) {
|
||||
export function captureCardRegionFromVideo(
|
||||
video,
|
||||
canvas,
|
||||
bounds,
|
||||
margin = CAPTURE_MARGIN_PX,
|
||||
corners = null
|
||||
) {
|
||||
if (corners?.length === 4 && typeof document !== 'undefined' && isValidCardQuad(corners)) {
|
||||
try {
|
||||
return warpCardCaptureFromVideo(video, canvas, corners, {
|
||||
jpegQuality: OCR_CAPTURE_JPEG_QUALITY,
|
||||
});
|
||||
} catch (warpError) {
|
||||
console.warn('Perspective warp failed, falling back to axis-aligned crop:', warpError);
|
||||
}
|
||||
}
|
||||
|
||||
const ctx = canvas.getContext('2d');
|
||||
const { x, y, width, height } = bounds;
|
||||
|
||||
|
|
@ -283,7 +301,13 @@ export async function identifyTrackedCardCapture({
|
|||
authHeaders,
|
||||
visionCooldownUntilMs = 0,
|
||||
}) {
|
||||
const imageData = captureCardRegionFromVideo(video, canvas, cardTracker.bounds);
|
||||
const imageData = captureCardRegionFromVideo(
|
||||
video,
|
||||
canvas,
|
||||
cardTracker.bounds,
|
||||
CAPTURE_MARGIN_PX,
|
||||
cardTracker.corners
|
||||
);
|
||||
|
||||
try {
|
||||
const l1 = await tryLayer1TextIdentify(imageData, authHeaders);
|
||||
|
|
|
|||
271
lib/scanner-card-warp.js
Normal file
271
lib/scanner-card-warp.js
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
/** Standard trading-card aspect ratio (width / height). */
|
||||
export const CARD_ASPECT_RATIO = 5 / 7;
|
||||
|
||||
/** Order four corners as top-left, top-right, bottom-right, bottom-left. */
|
||||
export function orderQuadCorners(points) {
|
||||
if (!points || points.length !== 4) {
|
||||
throw new Error('orderQuadCorners expects exactly four points');
|
||||
}
|
||||
|
||||
const sortedByY = [...points].sort((a, b) => a.y - b.y);
|
||||
const top = sortedByY.slice(0, 2).sort((a, b) => a.x - b.x);
|
||||
const bottom = sortedByY.slice(2, 4).sort((a, b) => a.x - b.x);
|
||||
|
||||
return [top[0], top[1], bottom[1], bottom[0]];
|
||||
}
|
||||
|
||||
/** Axis-aligned bounds enclosing a quad. */
|
||||
export function boundsFromCorners(corners) {
|
||||
const xs = corners.map((point) => point.x);
|
||||
const ys = corners.map((point) => point.y);
|
||||
const minX = Math.min(...xs);
|
||||
const minY = Math.min(...ys);
|
||||
const maxX = Math.max(...xs);
|
||||
const maxY = Math.max(...ys);
|
||||
|
||||
return {
|
||||
x: minX,
|
||||
y: minY,
|
||||
width: maxX - minX,
|
||||
height: maxY - minY,
|
||||
};
|
||||
}
|
||||
|
||||
/** True when corners form a convex quad with plausible card aspect ratio. */
|
||||
export function isValidCardQuad(corners, { minAspect = 0.6, maxAspect = 0.8 } = {}) {
|
||||
if (!corners || corners.length !== 4) return false;
|
||||
|
||||
const ordered = orderQuadCorners(corners);
|
||||
const bounds = boundsFromCorners(ordered);
|
||||
if (bounds.width < 20 || bounds.height < 28) return false;
|
||||
|
||||
const aspect = bounds.width / bounds.height;
|
||||
if (aspect < minAspect || aspect > maxAspect) return false;
|
||||
|
||||
const area = polygonArea(ordered);
|
||||
const boundsArea = bounds.width * bounds.height;
|
||||
if (area / boundsArea < 0.55) return false;
|
||||
|
||||
return isConvexQuad(ordered);
|
||||
}
|
||||
|
||||
function polygonArea(points) {
|
||||
let sum = 0;
|
||||
for (let i = 0; i < points.length; i++) {
|
||||
const current = points[i];
|
||||
const next = points[(i + 1) % points.length];
|
||||
sum += current.x * next.y - next.x * current.y;
|
||||
}
|
||||
return Math.abs(sum) / 2;
|
||||
}
|
||||
|
||||
function isConvexQuad(points) {
|
||||
let sign = 0;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const a = points[i];
|
||||
const b = points[(i + 1) % 4];
|
||||
const c = points[(i + 2) % 4];
|
||||
const cross = (b.x - a.x) * (c.y - b.y) - (b.y - a.y) * (c.x - b.x);
|
||||
if (cross === 0) continue;
|
||||
const currentSign = cross > 0 ? 1 : -1;
|
||||
if (sign === 0) {
|
||||
sign = currentSign;
|
||||
} else if (sign !== currentSign) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return sign !== 0;
|
||||
}
|
||||
|
||||
/** Solve 8×8 homography mapping src quad → axis-aligned dst rectangle. */
|
||||
export function computeHomography(srcCorners, dstWidth, dstHeight) {
|
||||
const src = orderQuadCorners(srcCorners);
|
||||
const dst = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: dstWidth, y: 0 },
|
||||
{ x: dstWidth, y: dstHeight },
|
||||
{ x: 0, y: dstHeight },
|
||||
];
|
||||
|
||||
const rows = [];
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const { x, y } = src[i];
|
||||
const { x: u, y: v } = dst[i];
|
||||
rows.push([x, y, 1, 0, 0, 0, -u * x, -u * y, u]);
|
||||
rows.push([0, 0, 0, x, y, 1, -v * x, -v * y, v]);
|
||||
}
|
||||
|
||||
const h = solveLinearSystem(rows);
|
||||
return [
|
||||
[h[0], h[1], h[2]],
|
||||
[h[3], h[4], h[5]],
|
||||
[h[6], h[7], 1],
|
||||
];
|
||||
}
|
||||
|
||||
function solveLinearSystem(rows) {
|
||||
const matrix = rows.map((row) => row.slice());
|
||||
const size = 8;
|
||||
|
||||
for (let col = 0; col < size; col++) {
|
||||
let pivotRow = col;
|
||||
for (let row = col + 1; row < size; row++) {
|
||||
if (Math.abs(matrix[row][col]) > Math.abs(matrix[pivotRow][col])) {
|
||||
pivotRow = row;
|
||||
}
|
||||
}
|
||||
|
||||
if (Math.abs(matrix[pivotRow][col]) < 1e-9) {
|
||||
throw new Error('Homography system is singular');
|
||||
}
|
||||
|
||||
[matrix[col], matrix[pivotRow]] = [matrix[pivotRow], matrix[col]];
|
||||
|
||||
const pivot = matrix[col][col];
|
||||
for (let j = col; j <= size; j++) {
|
||||
matrix[col][j] /= pivot;
|
||||
}
|
||||
|
||||
for (let row = 0; row < size; row++) {
|
||||
if (row === col) continue;
|
||||
const factor = matrix[row][col];
|
||||
for (let j = col; j <= size; j++) {
|
||||
matrix[row][j] -= factor * matrix[col][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matrix.map((row) => row[size]);
|
||||
}
|
||||
|
||||
function applyHomographyInverse(matrix, x, y) {
|
||||
const denom = matrix[2][0] * x + matrix[2][1] * y + matrix[2][2];
|
||||
const srcX = (matrix[0][0] * x + matrix[0][1] * y + matrix[0][2]) / denom;
|
||||
const srcY = (matrix[1][0] * x + matrix[1][1] * y + matrix[1][2]) / denom;
|
||||
return { x: srcX, y: srcY };
|
||||
}
|
||||
|
||||
function invert3x3(matrix) {
|
||||
const [
|
||||
[a, b, c],
|
||||
[d, e, f],
|
||||
[g, h, i],
|
||||
] = matrix;
|
||||
|
||||
const A = e * i - f * h;
|
||||
const B = -(d * i - f * g);
|
||||
const C = d * h - e * g;
|
||||
const D = -(b * i - c * h);
|
||||
const E = a * i - c * g;
|
||||
const F = -(a * h - b * g);
|
||||
const G = b * f - c * e;
|
||||
const H = -(a * f - c * d);
|
||||
const I = a * e - b * d;
|
||||
const det = a * A + b * B + c * C;
|
||||
|
||||
if (Math.abs(det) < 1e-9) {
|
||||
throw new Error('Homography matrix is not invertible');
|
||||
}
|
||||
|
||||
const invDet = 1 / det;
|
||||
return [
|
||||
[A * invDet, D * invDet, G * invDet],
|
||||
[B * invDet, E * invDet, H * invDet],
|
||||
[C * invDet, F * invDet, I * invDet],
|
||||
];
|
||||
}
|
||||
|
||||
function sampleBilinear(data, width, height, x, y) {
|
||||
const clampedX = Math.max(0, Math.min(width - 1, x));
|
||||
const clampedY = Math.max(0, Math.min(height - 1, y));
|
||||
const x0 = Math.floor(clampedX);
|
||||
const y0 = Math.floor(clampedY);
|
||||
const x1 = Math.min(x0 + 1, width - 1);
|
||||
const y1 = Math.min(y0 + 1, height - 1);
|
||||
const tx = clampedX - x0;
|
||||
const ty = clampedY - y0;
|
||||
|
||||
const idx = (row, col) => (row * width + col) * 4;
|
||||
const sample = (row, col) => {
|
||||
const base = idx(row, col);
|
||||
return [data[base], data[base + 1], data[base + 2], data[base + 3]];
|
||||
};
|
||||
|
||||
const c00 = sample(y0, x0);
|
||||
const c10 = sample(y0, x1);
|
||||
const c01 = sample(y1, x0);
|
||||
const c11 = sample(y1, x1);
|
||||
|
||||
const out = [0, 0, 0, 255];
|
||||
for (let channel = 0; channel < 3; channel++) {
|
||||
const top = c00[channel] * (1 - tx) + c10[channel] * tx;
|
||||
const bottom = c01[channel] * (1 - tx) + c11[channel] * tx;
|
||||
out[channel] = Math.round(top * (1 - ty) + bottom * ty);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perspective-correct a card region from the live video frame.
|
||||
* Returns a JPEG data URL sized to the card aspect ratio.
|
||||
*/
|
||||
export function warpCardCaptureFromVideo(
|
||||
video,
|
||||
canvas,
|
||||
corners,
|
||||
{ jpegQuality = 0.92, maxWidth = 480 } = {}
|
||||
) {
|
||||
if (!video || !canvas || !corners || corners.length !== 4) {
|
||||
throw new Error('warpCardCaptureFromVideo requires video, canvas, and four corners');
|
||||
}
|
||||
|
||||
const ordered = orderQuadCorners(corners);
|
||||
const bounds = boundsFromCorners(ordered);
|
||||
const outputHeight = Math.max(1, Math.round(maxWidth / CARD_ASPECT_RATIO));
|
||||
const outputWidth = maxWidth;
|
||||
|
||||
const homography = computeHomography(ordered, outputWidth, outputHeight);
|
||||
const inverse = invert3x3(homography);
|
||||
|
||||
const sourceCanvas = document.createElement('canvas');
|
||||
sourceCanvas.width = video.videoWidth;
|
||||
sourceCanvas.height = video.videoHeight;
|
||||
const sourceCtx = sourceCanvas.getContext('2d');
|
||||
sourceCtx.drawImage(video, 0, 0);
|
||||
const sourceData = sourceCtx.getImageData(0, 0, sourceCanvas.width, sourceCanvas.height).data;
|
||||
|
||||
canvas.width = outputWidth;
|
||||
canvas.height = outputHeight;
|
||||
const ctx = canvas.getContext('2d');
|
||||
const output = ctx.createImageData(outputWidth, outputHeight);
|
||||
|
||||
for (let y = 0; y < outputHeight; y++) {
|
||||
for (let x = 0; x < outputWidth; x++) {
|
||||
const mapped = applyHomographyInverse(inverse, x, y);
|
||||
if (
|
||||
mapped.x < bounds.x - 5 ||
|
||||
mapped.y < bounds.y - 5 ||
|
||||
mapped.x > bounds.x + bounds.width + 5 ||
|
||||
mapped.y > bounds.y + bounds.height + 5
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const rgba = sampleBilinear(
|
||||
sourceData,
|
||||
sourceCanvas.width,
|
||||
sourceCanvas.height,
|
||||
mapped.x,
|
||||
mapped.y
|
||||
);
|
||||
const outIdx = (y * outputWidth + x) * 4;
|
||||
output.data[outIdx] = rgba[0];
|
||||
output.data[outIdx + 1] = rgba[1];
|
||||
output.data[outIdx + 2] = rgba[2];
|
||||
output.data[outIdx + 3] = 255;
|
||||
}
|
||||
}
|
||||
|
||||
ctx.putImageData(output, 0, 0);
|
||||
return canvas.toDataURL('image/jpeg', jpegQuality);
|
||||
}
|
||||
|
|
@ -9,7 +9,7 @@ import {
|
|||
} from './scanner-card-detection.js';
|
||||
|
||||
/**
|
||||
* Camera stream + OpenCV shape detection loop for the card scanner.
|
||||
* Camera stream + card shape detection loop for the card scanner.
|
||||
* Identification callbacks stay in the parent component.
|
||||
*/
|
||||
export function useCameraScanner({
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import {
|
|||
convertToVideoCoordinates,
|
||||
mergeDetectedShapesIntoTrackedCards,
|
||||
rectanglesOverlap,
|
||||
refineCardCornersFromEdges,
|
||||
selectCardsReadyForVerification,
|
||||
TRACK_STALE_MS,
|
||||
} from '../../lib/scanner-card-detection.js';
|
||||
|
|
@ -37,7 +38,19 @@ describe('convertToVideoCoordinates', () => {
|
|||
describe('mergeDetectedShapesIntoTrackedCards', () => {
|
||||
it('creates a new tracked card for an unmatched shape', () => {
|
||||
const now = 1_000_000;
|
||||
const shape = { x: 10, y: 20, width: 100, height: 140, score: 80 };
|
||||
const shape = {
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 100,
|
||||
height: 140,
|
||||
score: 80,
|
||||
corners: [
|
||||
{ x: 10, y: 20 },
|
||||
{ x: 110, y: 22 },
|
||||
{ x: 108, y: 160 },
|
||||
{ x: 12, y: 158 },
|
||||
],
|
||||
};
|
||||
|
||||
const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards([], [shape], 1, now);
|
||||
|
||||
|
|
@ -47,7 +60,8 @@ describe('mergeDetectedShapesIntoTrackedCards', () => {
|
|||
status: 'detecting',
|
||||
stableCount: 1,
|
||||
scanAttempts: 0,
|
||||
bounds: shape,
|
||||
bounds: { x: 10, y: 20, width: 100, height: 140 },
|
||||
corners: shape.corners,
|
||||
});
|
||||
expect(nextCardId).toBe(2);
|
||||
});
|
||||
|
|
@ -125,3 +139,31 @@ describe('selectCardsReadyForVerification', () => {
|
|||
expect(selectCardsReadyForVerification([retried], now)).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('refineCardCornersFromEdges', () => {
|
||||
it('picks strong edge pixels near each quadrant corner', () => {
|
||||
const width = 120;
|
||||
const height = 168;
|
||||
const edges = new Uint8Array(width * height);
|
||||
const mark = (x, y) => {
|
||||
edges[y * width + x] = 255;
|
||||
};
|
||||
|
||||
mark(8, 10);
|
||||
mark(110, 12);
|
||||
mark(108, 156);
|
||||
mark(10, 154);
|
||||
|
||||
const corners = refineCardCornersFromEdges(edges, width, height, {
|
||||
x: 5,
|
||||
y: 8,
|
||||
width: 110,
|
||||
height: 150,
|
||||
});
|
||||
|
||||
expect(corners[0]).toEqual({ x: 8, y: 10 });
|
||||
expect(corners[1]).toEqual({ x: 110, y: 12 });
|
||||
expect(corners[2]).toEqual({ x: 108, y: 156 });
|
||||
expect(corners[3]).toEqual({ x: 10, y: 154 });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
93
test/lib/scanner-card-warp.test.js
Normal file
93
test/lib/scanner-card-warp.test.js
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
boundsFromCorners,
|
||||
computeHomography,
|
||||
isValidCardQuad,
|
||||
orderQuadCorners,
|
||||
} from '../../lib/scanner-card-warp.js';
|
||||
|
||||
describe('orderQuadCorners', () => {
|
||||
it('orders corners as top-left, top-right, bottom-right, bottom-left', () => {
|
||||
const points = [
|
||||
{ x: 10, y: 10 },
|
||||
{ x: 90, y: 12 },
|
||||
{ x: 88, y: 140 },
|
||||
{ x: 12, y: 138 },
|
||||
];
|
||||
|
||||
expect(orderQuadCorners(points)).toEqual([
|
||||
{ x: 10, y: 10 },
|
||||
{ x: 90, y: 12 },
|
||||
{ x: 88, y: 140 },
|
||||
{ x: 12, y: 138 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('boundsFromCorners', () => {
|
||||
it('returns the enclosing axis-aligned rectangle', () => {
|
||||
const corners = [
|
||||
{ x: 10, y: 20 },
|
||||
{ x: 110, y: 25 },
|
||||
{ x: 105, y: 165 },
|
||||
{ x: 15, y: 160 },
|
||||
];
|
||||
|
||||
expect(boundsFromCorners(corners)).toEqual({
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 100,
|
||||
height: 145,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('isValidCardQuad', () => {
|
||||
it('accepts a convex card-shaped quad', () => {
|
||||
const corners = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 50, y: 0 },
|
||||
{ x: 50, y: 70 },
|
||||
{ x: 0, y: 70 },
|
||||
];
|
||||
expect(isValidCardQuad(corners)).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects a quad with implausible aspect ratio', () => {
|
||||
const corners = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 200, y: 0 },
|
||||
{ x: 200, y: 70 },
|
||||
{ x: 0, y: 70 },
|
||||
];
|
||||
expect(isValidCardQuad(corners)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeHomography', () => {
|
||||
it('maps source corners to the destination rectangle', () => {
|
||||
const src = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 100, y: 0 },
|
||||
{ x: 100, y: 140 },
|
||||
{ x: 0, y: 140 },
|
||||
];
|
||||
const matrix = computeHomography(src, 200, 280);
|
||||
|
||||
const mappedTopLeft = applyForward(matrix, 0, 0);
|
||||
expect(mappedTopLeft.x).toBeCloseTo(0, 4);
|
||||
expect(mappedTopLeft.y).toBeCloseTo(0, 4);
|
||||
|
||||
const mappedBottomRight = applyForward(matrix, 100, 140);
|
||||
expect(mappedBottomRight.x).toBeCloseTo(200, 4);
|
||||
expect(mappedBottomRight.y).toBeCloseTo(280, 4);
|
||||
});
|
||||
});
|
||||
|
||||
function applyForward(matrix, x, y) {
|
||||
const denom = matrix[2][0] * x + matrix[2][1] * y + matrix[2][2];
|
||||
return {
|
||||
x: (matrix[0][0] * x + matrix[0][1] * y + matrix[0][2]) / denom,
|
||||
y: (matrix[1][0] * x + matrix[1][1] * y + matrix[1][2]) / denom,
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue