feat(scanner): add debug instrumentation for vision pipeline timing #155

Merged
rstillwell merged 31 commits from feat/scanner-debug-mode into main 2026-09-01 18:11:03 -04:00
11 changed files with 766 additions and 109 deletions
Showing only changes of commit c52891a6b2 - Show all commits

View file

@ -94,3 +94,10 @@
{"ts": "2026-08-15T00:26:08Z", "role": "role-design-system-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 82, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"} {"ts": "2026-08-15T00:26:08Z", "role": "role-design-system-auditor", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "duration_s": 82, "multitask_group": "audit-scanner-mobile-checkout-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
{"ts": "2026-08-15T00:55:56Z", "role": "role-reviewer", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 0, "duration_s": 180, "multitask_group": "audit-tighten-scan-identify-hot-path-uncommitted", "model": "cursor-grok-4.5-high", "model_tier": "fast"} {"ts": "2026-08-15T00:55:56Z", "role": "role-reviewer", "convoy": "tighten-scan-identify-hot-path", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 0, "duration_s": 180, "multitask_group": "audit-tighten-scan-identify-hot-path-uncommitted", "model": "cursor-grok-4.5-high", "model_tier": "fast"}
{"ts": "2026-08-15T00:56:39Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"} {"ts": "2026-08-15T00:56:39Z", "role": "role-implementer", "convoy": "scanner-mobile-checkout", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 240, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
{"ts": "2026-08-15T01:24:40Z", "role": "role-ux-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "model": "composer-2.5-fast", "model_tier": "fast"}
{"ts": "2026-08-15T01:24:40Z", "role": "role-architect", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 300, "model": "composer-2.5", "model_tier": "standard"}
{"ts": "2026-08-15T01:24:40Z", "role": "role-implementer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 1, "duration_s": 900, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
{"ts": "2026-08-15T01:24:40Z", "role": "role-implementer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "brief": 2, "duration_s": 300, "outcome": "complete", "model": "composer-2.5-fast", "model_tier": "fast"}
{"ts": "2026-08-15T01:24:40Z", "role": "role-reviewer", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 180, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}
{"ts": "2026-08-15T01:24:41Z", "role": "role-security-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 120, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "gpt-5.6-terra-medium", "model_tier": "security"}
{"ts": "2026-08-15T01:24:41Z", "role": "role-a11y-auditor", "convoy": "improve-scan-card-detection", "repo": "scanner-identify-upgrade", "skip_flags": [], "duration_s": 60, "outcome": "approved", "multitask_group": "audit-improve-scan-card-detection-local", "model": "cursor-grok-4.5-high", "model_tier": "audit"}

View file

@ -9,7 +9,7 @@ skip:
- ia - ia
- ui-design - ui-design
- flag - flag
status: open status: in-progress
created: 2026-08-14 created: 2026-08-14
depends_on: depends_on:
- tighten-scan-identify-hot-path - tighten-scan-identify-hot-path
@ -104,9 +104,9 @@ either model.
## Todos ## Todos
- [ ] Architect: pick detector; document WASM / weight budget - [x] Architect: pick detector; document WASM / weight budget
- [ ] Brief 1 — detect + warp library + tests - [x] Brief 1 — detect + warp library + tests
- [ ] Brief 2 — wire `use-camera-scanner.js` + crop used by - [x] Brief 2 — wire `use-camera-scanner.js` + crop used by
`identifyTrackedCardCapture` `identifyTrackedCardCapture`
- [ ] Confirm `scanner-mobile-checkout` overlay still maps bounds - [ ] Confirm `scanner-mobile-checkout` overlay still maps bounds
- [ ] Re-measure L2 `not_a_card` share - [ ] Re-measure L2 `not_a_card` share
@ -149,3 +149,58 @@ Do not drop OpenCV.js in only to reimplement the current rectangle
hunt. Success is a **rectified card image**, not a prettier box. hunt. Success is a **rectified card image**, not a prettier box.
This convoy unblocks Phase 3 — embeddings on unwarped phone photos This convoy unblocks Phase 3 — embeddings on unwarped phone photos
will miss. will miss.
## UX
No new routes or screens. Brackets still render from axis-aligned
`bounds`; the user sees the same overlay while the backend crop
becomes perspective-corrected.
### Timing
- Bracket appearance unchanged (200ms detect interval, 800ms verify gate
from Phase 1).
- Warp adds ~2040ms on capture only — not on the detect loop.
### Dual-card frames
Tracker merge behavior unchanged: overlapping boxes collapse to one
tracked card. Warp runs per tracked card at verify time.
### Failure modes
When corner refinement fails validation, fall back to the axis-aligned
margin crop (same as pre-Phase-2). No new error toast.
## Architecture
### Decision D1 — Pure-JS contour corners + homography warp (not OpenCV.js WASM)
OpenCV.js adds ~8MB WASM and a Turbopack dynamic-import footgun.
Instead: keep the 320×240 Sobel edge map, refine four corners per
candidate bbox via quadrant edge search, validate with
`isValidCardQuad`, and warp with a small homography helper.
Rejected: YOLO11n ONNX (weight hosting + license review), OpenCV.js
(full WASM budget).
### Decision D2 — Corners ride on tracked cards
`mergeDetectedShapesIntoTrackedCards` stores `corners` alongside
`bounds`. Overlay continues to use `bounds` only.
### Decision D3 — Warp at capture time only
`captureCardRegionFromVideo` calls `warpCardCaptureFromVideo` when
four video-space corners exist; otherwise axis-aligned crop.
### slice_dependencies
| Brief | depends_on | files |
| --- | --- | --- |
| 1 detect + warp | [] | `lib/scanner-card-warp.js`, `lib/scanner-card-detection.js`, tests |
| 2 wire capture | [1] | `lib/scanner-card-identify.js`, `lib/use-camera-scanner.js`, `ScannerCamera.js` |
Serial implement: Brief 1 → Brief 2.
Audit group id: `audit-improve-scan-card-detection-<pr>`.

View file

@ -0,0 +1,27 @@
---
convoy: improve-scan-card-detection
brief_number: 1
depends_on: []
recommended_model: composer-2.5-fast
model_tier: fast
files:
- lib/scanner-card-warp.js
- lib/scanner-card-detection.js
- test/lib/scanner-card-warp.test.js
- test/lib/scanner-card-detection.test.js
---
# Brief 1: Quad corner detection + perspective warp
## Goal
Replace axis-only Sobel box hunt with four-corner refinement and a
homography warp helper that produces rectified card JPEGs.
## Acceptance criteria
- [ ] `lib/scanner-card-warp.js` exports `orderQuadCorners`,
`computeHomography`, `warpCardCaptureFromVideo`
- [ ] `detectCardShapesFromFrame` returns `corners` in video space
- [ ] Tracker merge preserves `corners` on tracked cards
- [ ] Unit tests for warp math + corner refinement

View file

@ -0,0 +1,25 @@
---
convoy: improve-scan-card-detection
brief_number: 2
depends_on: [1]
recommended_model: composer-2.5-fast
model_tier: fast
files:
- lib/scanner-card-identify.js
- lib/use-camera-scanner.js
- components/scanner/ScannerCamera.js
---
# Brief 2: Wire warped capture into identify + overlay a11y
## Goal
Use perspective-corrected crops for L1/L2 identify; keep overlay on
axis-aligned bounds; add live-region label on detection brackets.
## Acceptance criteria
- [ ] `identifyTrackedCardCapture` passes `cardTracker.corners` to capture
- [ ] `captureCardRegionFromVideo` warps when corners present, else fallback
- [ ] `use-camera-scanner.js` comment no longer claims OpenCV
- [ ] `DetectionFrame` exposes `role="status"` + `aria-label`

View file

@ -10,9 +10,17 @@ import { useScannerFlash } from '../../lib/use-scanner-flash.js';
const TOAST_DURATION_MS = 2500; const TOAST_DURATION_MS = 2500;
function DetectionFrame({ card, videoMetrics }) { function DetectionFrame({ card, videoMetrics }) {
const label =
card.status === 'scanned'
? `Card ${card.id} scanned`
: `Card ${card.id} identified`;
return ( return (
<div <div
className="absolute pointer-events-none scan-frame" className="absolute pointer-events-none scan-frame"
role="status"
aria-live="polite"
aria-label={label}
style={{ style={{
left: `${(card.bounds.x / videoMetrics.width) * 100}%`, left: `${(card.bounds.x / videoMetrics.width) * 100}%`,
top: `${(card.bounds.y / videoMetrics.height) * 100}%`, top: `${(card.bounds.y / videoMetrics.height) * 100}%`,

View file

@ -1,3 +1,9 @@
import {
boundsFromCorners,
isValidCardQuad,
orderQuadCorners,
} from './scanner-card-warp.js';
/** Detection canvas dimensions (smaller than video for performance). */ /** Detection canvas dimensions (smaller than video for performance). */
export const DETECTION_CANVAS_WIDTH = 320; export const DETECTION_CANVAS_WIDTH = 320;
export const DETECTION_CANVAS_HEIGHT = 240; export const DETECTION_CANVAS_HEIGHT = 240;
@ -52,6 +58,13 @@ export function convertToVideoCoordinates(detection, canvas, video) {
}; };
} }
/** Convert a point from detection canvas space to video pixel space. */
export function convertPointToVideoCoordinates(point, canvas, video) {
const scaleX = video.videoWidth / canvas.width;
const scaleY = video.videoHeight / canvas.height;
return { x: point.x * scaleX, y: point.y * scaleY };
}
/** True when overlap area exceeds `threshold` of the smaller rectangle. */ /** True when overlap area exceeds `threshold` of the smaller rectangle. */
export function rectanglesOverlap(rect1, rect2, threshold = 0.3) { export function rectanglesOverlap(rect1, rect2, threshold = 0.3) {
const x1 = Math.max(rect1.x, rect2.x); const x1 = Math.max(rect1.x, rect2.x);
@ -69,9 +82,166 @@ export function rectanglesOverlap(rect1, rect2, threshold = 0.3) {
return overlapArea / smallerArea > threshold; return overlapArea / smallerArea > threshold;
} }
function buildGrayscaleAndEdges(imageData, width, height) {
const data = imageData.data;
const grayscale = new Uint8Array(width * height);
const edges = new Uint8Array(width * height);
for (let i = 0, px = 0; i < data.length; i += 4, px++) {
grayscale[px] = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]);
}
for (let y = 1; y < height - 1; y++) {
for (let x = 1; x < width - 1; x++) {
const idx = y * width + x;
const gx =
-grayscale[idx - 1] +
grayscale[idx + 1] +
-2 * grayscale[idx - 1 + width] +
2 * grayscale[idx + 1 + width] +
-grayscale[idx - 1 + 2 * width] +
grayscale[idx + 1 + 2 * width];
const gy =
-grayscale[idx - width] -
2 * grayscale[idx] -
grayscale[idx + width] +
grayscale[idx - width + 2 * width] +
2 * grayscale[idx + 2 * width] +
grayscale[idx + width + 2 * width];
const magnitude = Math.sqrt(gx * gx + gy * gy);
edges[idx] = magnitude > 90 ? 255 : 0;
}
}
return { grayscale, edges };
}
/**
* Refine a coarse card bbox into four corners using edge strength in each quadrant.
* Exported for unit tests.
*/
export function refineCardCornersFromEdges(edges, width, height, bbox) {
const { x, y, width: boxWidth, height: boxHeight } = bbox;
const centerX = x + boxWidth / 2;
const centerY = y + boxHeight / 2;
const quadrants = [
{ minX: x, maxX: centerX, minY: y, maxY: centerY },
{ minX: centerX, maxX: x + boxWidth, minY: y, maxY: centerY },
{ minX: centerX, maxX: x + boxWidth, minY: centerY, maxY: y + boxHeight },
{ minX: x, maxX: centerX, minY: centerY, maxY: y + boxHeight },
];
const corners = quadrants.map((quad) => {
let best = null;
let bestScore = -1;
for (let py = Math.floor(quad.minY); py < quad.maxY; py++) {
for (let px = Math.floor(quad.minX); px < quad.maxX; px++) {
if (px < 0 || py < 0 || px >= width || py >= height) continue;
const idx = py * width + px;
if (edges[idx] !== 255) continue;
const dx = px - centerX;
const dy = py - centerY;
const score = Math.hypot(dx, dy);
if (score > bestScore) {
bestScore = score;
best = { x: px, y: py };
}
}
}
if (!best) {
return {
x: (quad.minX + quad.maxX) / 2,
y: (quad.minY + quad.maxY) / 2,
};
}
return best;
});
return orderQuadCorners(corners);
}
function scoreCardCandidate(edges, width, height, bbox) {
const { x, y, width: w, height: h } = bbox;
let edgeCount = 0;
let totalPixels = 0;
let perimeterEdges = 0;
for (let sy = y; sy < y + h; sy += 3) {
for (let sx = x; sx < x + w; sx += 3) {
if (sx < 0 || sy < 0 || sx >= width || sy >= height) continue;
const idx = sy * width + sx;
totalPixels++;
if (edges[idx] !== 255) continue;
edgeCount++;
const isPerimeter =
sx < x + w * 0.15 || sx > x + w * 0.85 || sy < y + h * 0.15 || sy > y + h * 0.85;
if (isPerimeter) perimeterEdges++;
}
}
const edgeDensity = edgeCount / (totalPixels || 1);
const perimeterRatio = perimeterEdges / (edgeCount || 1);
if (edgeDensity <= 0.18 || edgeDensity >= 0.65 || perimeterRatio <= 0.35 || edgeCount <= 60) {
return null;
}
return edgeDensity * 100 + perimeterRatio * 60 + edgeCount / 10;
}
function findCardCandidates(edges, width, height) {
const minCardWidth = Math.floor(width * 0.2);
const maxCardWidth = Math.floor(width * 0.6);
const minCardHeight = Math.floor(height * 0.25);
const maxCardHeight = Math.floor(height * 0.7);
const candidates = [];
for (let y = 0; y < height - minCardHeight; y += 12) {
for (let x = 0; x < width - minCardWidth; x += 12) {
for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 16) {
for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 16) {
const aspectRatio = w / h;
if (aspectRatio < 0.63 || aspectRatio > 0.77) continue;
const score = scoreCardCandidate(edges, width, height, { x, y, width: w, height: h });
if (score == null || score <= 50) continue;
const corners = refineCardCornersFromEdges(edges, width, height, {
x,
y,
width: w,
height: h,
});
const bounds = boundsFromCorners(corners);
if (!isValidCardQuad(corners)) continue;
candidates.push({
x: bounds.x,
y: bounds.y,
width: bounds.width,
height: bounds.height,
corners,
score: score + (isValidCardQuad(corners) ? 20 : 0),
aspectRatio,
});
}
}
}
}
return candidates.sort((a, b) => b.score - a.score).slice(0, 5);
}
/** /**
* Draw the current video frame on `detectionCanvas` and return up to 5 card-like * Draw the current video frame on `detectionCanvas` and return up to 5 card-like
* bounding boxes in video coordinates. * detections in video coordinates, including perspective corners when found.
*/ */
export function detectCardShapesFromFrame(video, detectionCanvas) { export function detectCardShapesFromFrame(video, detectionCanvas) {
if (!video || !detectionCanvas) return []; if (!video || !detectionCanvas) return [];
@ -83,106 +253,27 @@ export function detectCardShapesFromFrame(video, detectionCanvas) {
ctx.drawImage(video, 0, 0, detectionCanvas.width, detectionCanvas.height); ctx.drawImage(video, 0, 0, detectionCanvas.width, detectionCanvas.height);
const imageData = ctx.getImageData(0, 0, detectionCanvas.width, detectionCanvas.height); const imageData = ctx.getImageData(0, 0, detectionCanvas.width, detectionCanvas.height);
const data = imageData.data; const { edges } = buildGrayscaleAndEdges(
imageData,
detectionCanvas.width,
detectionCanvas.height
);
const grayscale = []; const candidates = findCardCandidates(edges, detectionCanvas.width, detectionCanvas.height);
const edges = [];
for (let i = 0; i < data.length; i += 4) { return candidates.map((candidate) => {
const gray = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]); const videoBounds = convertToVideoCoordinates(candidate, detectionCanvas, video);
grayscale.push(gray); const videoCorners = candidate.corners.map((corner) =>
} convertPointToVideoCoordinates(corner, detectionCanvas, video)
);
for (let y = 1; y < detectionCanvas.height - 1; y++) { return {
for (let x = 1; x < detectionCanvas.width - 1; x++) { ...videoBounds,
const idx = y * detectionCanvas.width + x; corners: videoCorners,
score: candidate.score,
const gx = aspectRatio: candidate.aspectRatio,
-grayscale[idx - 1] + };
grayscale[idx + 1] + });
-2 * grayscale[idx - 1 + detectionCanvas.width] +
2 * grayscale[idx + 1 + detectionCanvas.width] +
-grayscale[idx - 1 + 2 * detectionCanvas.width] +
grayscale[idx + 1 + 2 * detectionCanvas.width];
const gy =
-grayscale[idx - detectionCanvas.width] -
2 * grayscale[idx] -
grayscale[idx + detectionCanvas.width] +
grayscale[idx - detectionCanvas.width + 2 * detectionCanvas.width] +
2 * grayscale[idx + 2 * detectionCanvas.width] +
grayscale[idx + detectionCanvas.width + 2 * detectionCanvas.width];
const magnitude = Math.sqrt(gx * gx + gy * gy);
edges[idx] = magnitude > 100 ? 255 : 0;
}
}
const cardShapes = [];
const { width, height } = detectionCanvas;
const minCardWidth = Math.floor(width * 0.2);
const maxCardWidth = Math.floor(width * 0.6);
const minCardHeight = Math.floor(height * 0.25);
const maxCardHeight = Math.floor(height * 0.7);
for (let y = 0; y < height - minCardHeight; y += 15) {
for (let x = 0; x < width - minCardWidth; x += 15) {
for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 20) {
for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 20) {
const aspectRatio = w / h;
if (aspectRatio < 0.63 || aspectRatio > 0.77) continue;
let edgeCount = 0;
let totalPixels = 0;
let perimeterEdges = 0;
for (let sy = y; sy < y + h; sy += 4) {
for (let sx = x; sx < x + w; sx += 4) {
const idx = sy * width + sx;
if (edges[idx] === 255) {
edgeCount++;
const isPerimeter =
sx < x + w * 0.15 ||
sx > x + w * 0.85 ||
sy < y + h * 0.15 ||
sy > y + h * 0.85;
if (isPerimeter) {
perimeterEdges++;
}
}
totalPixels++;
}
}
const edgeDensity = edgeCount / totalPixels;
const perimeterRatio = perimeterEdges / (edgeCount || 1);
if (edgeDensity > 0.2 && edgeDensity < 0.6 && perimeterRatio > 0.4 && edgeCount > 80) {
const score = edgeDensity * 100 + perimeterRatio * 60 + edgeCount / 10;
if (score > 50) {
const videoCoords = convertToVideoCoordinates(
{ x, y, width: w, height: h },
detectionCanvas,
video
);
cardShapes.push({
...videoCoords,
score,
aspectRatio,
edgeDensity,
});
}
}
}
}
}
}
return cardShapes.sort((a, b) => b.score - a.score).slice(0, 5);
} }
/** /**
@ -219,7 +310,15 @@ export function mergeDetectedShapesIntoTrackedCards(
} }
if (matchedCard) { if (matchedCard) {
matchedCard.bounds = { ...shape }; matchedCard.bounds = {
x: shape.x,
y: shape.y,
width: shape.width,
height: shape.height,
};
if (shape.corners?.length === 4) {
matchedCard.corners = shape.corners.map((corner) => ({ ...corner }));
}
matchedCard.lastSeen = now; matchedCard.lastSeen = now;
matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10); matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10);
@ -234,7 +333,13 @@ export function mergeDetectedShapesIntoTrackedCards(
} else { } else {
const newCard = { const newCard = {
id: idCounter++, id: idCounter++,
bounds: { ...shape }, bounds: {
x: shape.x,
y: shape.y,
width: shape.width,
height: shape.height,
},
corners: shape.corners?.length === 4 ? shape.corners.map((corner) => ({ ...corner })) : null,
status: 'detecting', status: 'detecting',
firstSeen: now, firstSeen: now,
lastSeen: now, lastSeen: now,

View file

@ -1,3 +1,5 @@
import { isValidCardQuad, warpCardCaptureFromVideo } from './scanner-card-warp.js';
/** Default margin (px) around tracked bounds when cropping a card capture. */ /** Default margin (px) around tracked bounds when cropping a card capture. */
export const CAPTURE_MARGIN_PX = 20; export const CAPTURE_MARGIN_PX = 20;
@ -167,7 +169,23 @@ export function resolveDisambiguationRefineAction(result, { candidates }) {
} }
/** Crop a tracked card region from the live video frame; returns a JPEG data URL. */ /** Crop a tracked card region from the live video frame; returns a JPEG data URL. */
export function captureCardRegionFromVideo(video, canvas, bounds, margin = CAPTURE_MARGIN_PX) { export function captureCardRegionFromVideo(
video,
canvas,
bounds,
margin = CAPTURE_MARGIN_PX,
corners = null
) {
if (corners?.length === 4 && typeof document !== 'undefined' && isValidCardQuad(corners)) {
try {
return warpCardCaptureFromVideo(video, canvas, corners, {
jpegQuality: OCR_CAPTURE_JPEG_QUALITY,
});
} catch (warpError) {
console.warn('Perspective warp failed, falling back to axis-aligned crop:', warpError);
}
}
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
const { x, y, width, height } = bounds; const { x, y, width, height } = bounds;
@ -283,7 +301,13 @@ export async function identifyTrackedCardCapture({
authHeaders, authHeaders,
visionCooldownUntilMs = 0, visionCooldownUntilMs = 0,
}) { }) {
const imageData = captureCardRegionFromVideo(video, canvas, cardTracker.bounds); const imageData = captureCardRegionFromVideo(
video,
canvas,
cardTracker.bounds,
CAPTURE_MARGIN_PX,
cardTracker.corners
);
try { try {
const l1 = await tryLayer1TextIdentify(imageData, authHeaders); const l1 = await tryLayer1TextIdentify(imageData, authHeaders);

271
lib/scanner-card-warp.js Normal file
View file

@ -0,0 +1,271 @@
/** Standard trading-card aspect ratio (width / height). */
export const CARD_ASPECT_RATIO = 5 / 7;
/** Order four corners as top-left, top-right, bottom-right, bottom-left. */
export function orderQuadCorners(points) {
if (!points || points.length !== 4) {
throw new Error('orderQuadCorners expects exactly four points');
}
const sortedByY = [...points].sort((a, b) => a.y - b.y);
const top = sortedByY.slice(0, 2).sort((a, b) => a.x - b.x);
const bottom = sortedByY.slice(2, 4).sort((a, b) => a.x - b.x);
return [top[0], top[1], bottom[1], bottom[0]];
}
/** Axis-aligned bounds enclosing a quad. */
export function boundsFromCorners(corners) {
const xs = corners.map((point) => point.x);
const ys = corners.map((point) => point.y);
const minX = Math.min(...xs);
const minY = Math.min(...ys);
const maxX = Math.max(...xs);
const maxY = Math.max(...ys);
return {
x: minX,
y: minY,
width: maxX - minX,
height: maxY - minY,
};
}
/** True when corners form a convex quad with plausible card aspect ratio. */
export function isValidCardQuad(corners, { minAspect = 0.6, maxAspect = 0.8 } = {}) {
if (!corners || corners.length !== 4) return false;
const ordered = orderQuadCorners(corners);
const bounds = boundsFromCorners(ordered);
if (bounds.width < 20 || bounds.height < 28) return false;
const aspect = bounds.width / bounds.height;
if (aspect < minAspect || aspect > maxAspect) return false;
const area = polygonArea(ordered);
const boundsArea = bounds.width * bounds.height;
if (area / boundsArea < 0.55) return false;
return isConvexQuad(ordered);
}
function polygonArea(points) {
let sum = 0;
for (let i = 0; i < points.length; i++) {
const current = points[i];
const next = points[(i + 1) % points.length];
sum += current.x * next.y - next.x * current.y;
}
return Math.abs(sum) / 2;
}
function isConvexQuad(points) {
let sign = 0;
for (let i = 0; i < 4; i++) {
const a = points[i];
const b = points[(i + 1) % 4];
const c = points[(i + 2) % 4];
const cross = (b.x - a.x) * (c.y - b.y) - (b.y - a.y) * (c.x - b.x);
if (cross === 0) continue;
const currentSign = cross > 0 ? 1 : -1;
if (sign === 0) {
sign = currentSign;
} else if (sign !== currentSign) {
return false;
}
}
return sign !== 0;
}
/** Solve 8×8 homography mapping src quad → axis-aligned dst rectangle. */
export function computeHomography(srcCorners, dstWidth, dstHeight) {
const src = orderQuadCorners(srcCorners);
const dst = [
{ x: 0, y: 0 },
{ x: dstWidth, y: 0 },
{ x: dstWidth, y: dstHeight },
{ x: 0, y: dstHeight },
];
const rows = [];
for (let i = 0; i < 4; i++) {
const { x, y } = src[i];
const { x: u, y: v } = dst[i];
rows.push([x, y, 1, 0, 0, 0, -u * x, -u * y, u]);
rows.push([0, 0, 0, x, y, 1, -v * x, -v * y, v]);
}
const h = solveLinearSystem(rows);
return [
[h[0], h[1], h[2]],
[h[3], h[4], h[5]],
[h[6], h[7], 1],
];
}
function solveLinearSystem(rows) {
const matrix = rows.map((row) => row.slice());
const size = 8;
for (let col = 0; col < size; col++) {
let pivotRow = col;
for (let row = col + 1; row < size; row++) {
if (Math.abs(matrix[row][col]) > Math.abs(matrix[pivotRow][col])) {
pivotRow = row;
}
}
if (Math.abs(matrix[pivotRow][col]) < 1e-9) {
throw new Error('Homography system is singular');
}
[matrix[col], matrix[pivotRow]] = [matrix[pivotRow], matrix[col]];
const pivot = matrix[col][col];
for (let j = col; j <= size; j++) {
matrix[col][j] /= pivot;
}
for (let row = 0; row < size; row++) {
if (row === col) continue;
const factor = matrix[row][col];
for (let j = col; j <= size; j++) {
matrix[row][j] -= factor * matrix[col][j];
}
}
}
return matrix.map((row) => row[size]);
}
function applyHomographyInverse(matrix, x, y) {
const denom = matrix[2][0] * x + matrix[2][1] * y + matrix[2][2];
const srcX = (matrix[0][0] * x + matrix[0][1] * y + matrix[0][2]) / denom;
const srcY = (matrix[1][0] * x + matrix[1][1] * y + matrix[1][2]) / denom;
return { x: srcX, y: srcY };
}
function invert3x3(matrix) {
const [
[a, b, c],
[d, e, f],
[g, h, i],
] = matrix;
const A = e * i - f * h;
const B = -(d * i - f * g);
const C = d * h - e * g;
const D = -(b * i - c * h);
const E = a * i - c * g;
const F = -(a * h - b * g);
const G = b * f - c * e;
const H = -(a * f - c * d);
const I = a * e - b * d;
const det = a * A + b * B + c * C;
if (Math.abs(det) < 1e-9) {
throw new Error('Homography matrix is not invertible');
}
const invDet = 1 / det;
return [
[A * invDet, D * invDet, G * invDet],
[B * invDet, E * invDet, H * invDet],
[C * invDet, F * invDet, I * invDet],
];
}
function sampleBilinear(data, width, height, x, y) {
const clampedX = Math.max(0, Math.min(width - 1, x));
const clampedY = Math.max(0, Math.min(height - 1, y));
const x0 = Math.floor(clampedX);
const y0 = Math.floor(clampedY);
const x1 = Math.min(x0 + 1, width - 1);
const y1 = Math.min(y0 + 1, height - 1);
const tx = clampedX - x0;
const ty = clampedY - y0;
const idx = (row, col) => (row * width + col) * 4;
const sample = (row, col) => {
const base = idx(row, col);
return [data[base], data[base + 1], data[base + 2], data[base + 3]];
};
const c00 = sample(y0, x0);
const c10 = sample(y0, x1);
const c01 = sample(y1, x0);
const c11 = sample(y1, x1);
const out = [0, 0, 0, 255];
for (let channel = 0; channel < 3; channel++) {
const top = c00[channel] * (1 - tx) + c10[channel] * tx;
const bottom = c01[channel] * (1 - tx) + c11[channel] * tx;
out[channel] = Math.round(top * (1 - ty) + bottom * ty);
}
return out;
}
/**
* Perspective-correct a card region from the live video frame.
* Returns a JPEG data URL sized to the card aspect ratio.
*/
export function warpCardCaptureFromVideo(
video,
canvas,
corners,
{ jpegQuality = 0.92, maxWidth = 480 } = {}
) {
if (!video || !canvas || !corners || corners.length !== 4) {
throw new Error('warpCardCaptureFromVideo requires video, canvas, and four corners');
}
const ordered = orderQuadCorners(corners);
const bounds = boundsFromCorners(ordered);
const outputHeight = Math.max(1, Math.round(maxWidth / CARD_ASPECT_RATIO));
const outputWidth = maxWidth;
const homography = computeHomography(ordered, outputWidth, outputHeight);
const inverse = invert3x3(homography);
const sourceCanvas = document.createElement('canvas');
sourceCanvas.width = video.videoWidth;
sourceCanvas.height = video.videoHeight;
const sourceCtx = sourceCanvas.getContext('2d');
sourceCtx.drawImage(video, 0, 0);
const sourceData = sourceCtx.getImageData(0, 0, sourceCanvas.width, sourceCanvas.height).data;
canvas.width = outputWidth;
canvas.height = outputHeight;
const ctx = canvas.getContext('2d');
const output = ctx.createImageData(outputWidth, outputHeight);
for (let y = 0; y < outputHeight; y++) {
for (let x = 0; x < outputWidth; x++) {
const mapped = applyHomographyInverse(inverse, x, y);
if (
mapped.x < bounds.x - 5 ||
mapped.y < bounds.y - 5 ||
mapped.x > bounds.x + bounds.width + 5 ||
mapped.y > bounds.y + bounds.height + 5
) {
continue;
}
const rgba = sampleBilinear(
sourceData,
sourceCanvas.width,
sourceCanvas.height,
mapped.x,
mapped.y
);
const outIdx = (y * outputWidth + x) * 4;
output.data[outIdx] = rgba[0];
output.data[outIdx + 1] = rgba[1];
output.data[outIdx + 2] = rgba[2];
output.data[outIdx + 3] = 255;
}
}
ctx.putImageData(output, 0, 0);
return canvas.toDataURL('image/jpeg', jpegQuality);
}

View file

@ -9,7 +9,7 @@ import {
} from './scanner-card-detection.js'; } from './scanner-card-detection.js';
/** /**
* Camera stream + OpenCV shape detection loop for the card scanner. * Camera stream + card shape detection loop for the card scanner.
* Identification callbacks stay in the parent component. * Identification callbacks stay in the parent component.
*/ */
export function useCameraScanner({ export function useCameraScanner({

View file

@ -3,6 +3,7 @@ import {
convertToVideoCoordinates, convertToVideoCoordinates,
mergeDetectedShapesIntoTrackedCards, mergeDetectedShapesIntoTrackedCards,
rectanglesOverlap, rectanglesOverlap,
refineCardCornersFromEdges,
selectCardsReadyForVerification, selectCardsReadyForVerification,
TRACK_STALE_MS, TRACK_STALE_MS,
} from '../../lib/scanner-card-detection.js'; } from '../../lib/scanner-card-detection.js';
@ -37,7 +38,19 @@ describe('convertToVideoCoordinates', () => {
describe('mergeDetectedShapesIntoTrackedCards', () => { describe('mergeDetectedShapesIntoTrackedCards', () => {
it('creates a new tracked card for an unmatched shape', () => { it('creates a new tracked card for an unmatched shape', () => {
const now = 1_000_000; const now = 1_000_000;
const shape = { x: 10, y: 20, width: 100, height: 140, score: 80 }; const shape = {
x: 10,
y: 20,
width: 100,
height: 140,
score: 80,
corners: [
{ x: 10, y: 20 },
{ x: 110, y: 22 },
{ x: 108, y: 160 },
{ x: 12, y: 158 },
],
};
const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards([], [shape], 1, now); const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards([], [shape], 1, now);
@ -47,7 +60,8 @@ describe('mergeDetectedShapesIntoTrackedCards', () => {
status: 'detecting', status: 'detecting',
stableCount: 1, stableCount: 1,
scanAttempts: 0, scanAttempts: 0,
bounds: shape, bounds: { x: 10, y: 20, width: 100, height: 140 },
corners: shape.corners,
}); });
expect(nextCardId).toBe(2); expect(nextCardId).toBe(2);
}); });
@ -125,3 +139,31 @@ describe('selectCardsReadyForVerification', () => {
expect(selectCardsReadyForVerification([retried], now)).toHaveLength(0); expect(selectCardsReadyForVerification([retried], now)).toHaveLength(0);
}); });
}); });
describe('refineCardCornersFromEdges', () => {
it('picks strong edge pixels near each quadrant corner', () => {
const width = 120;
const height = 168;
const edges = new Uint8Array(width * height);
const mark = (x, y) => {
edges[y * width + x] = 255;
};
mark(8, 10);
mark(110, 12);
mark(108, 156);
mark(10, 154);
const corners = refineCardCornersFromEdges(edges, width, height, {
x: 5,
y: 8,
width: 110,
height: 150,
});
expect(corners[0]).toEqual({ x: 8, y: 10 });
expect(corners[1]).toEqual({ x: 110, y: 12 });
expect(corners[2]).toEqual({ x: 108, y: 156 });
expect(corners[3]).toEqual({ x: 10, y: 154 });
});
});

View file

@ -0,0 +1,93 @@
import { describe, expect, it } from 'vitest';
import {
boundsFromCorners,
computeHomography,
isValidCardQuad,
orderQuadCorners,
} from '../../lib/scanner-card-warp.js';
describe('orderQuadCorners', () => {
it('orders corners as top-left, top-right, bottom-right, bottom-left', () => {
const points = [
{ x: 10, y: 10 },
{ x: 90, y: 12 },
{ x: 88, y: 140 },
{ x: 12, y: 138 },
];
expect(orderQuadCorners(points)).toEqual([
{ x: 10, y: 10 },
{ x: 90, y: 12 },
{ x: 88, y: 140 },
{ x: 12, y: 138 },
]);
});
});
describe('boundsFromCorners', () => {
it('returns the enclosing axis-aligned rectangle', () => {
const corners = [
{ x: 10, y: 20 },
{ x: 110, y: 25 },
{ x: 105, y: 165 },
{ x: 15, y: 160 },
];
expect(boundsFromCorners(corners)).toEqual({
x: 10,
y: 20,
width: 100,
height: 145,
});
});
});
describe('isValidCardQuad', () => {
it('accepts a convex card-shaped quad', () => {
const corners = [
{ x: 0, y: 0 },
{ x: 50, y: 0 },
{ x: 50, y: 70 },
{ x: 0, y: 70 },
];
expect(isValidCardQuad(corners)).toBe(true);
});
it('rejects a quad with implausible aspect ratio', () => {
const corners = [
{ x: 0, y: 0 },
{ x: 200, y: 0 },
{ x: 200, y: 70 },
{ x: 0, y: 70 },
];
expect(isValidCardQuad(corners)).toBe(false);
});
});
describe('computeHomography', () => {
it('maps source corners to the destination rectangle', () => {
const src = [
{ x: 0, y: 0 },
{ x: 100, y: 0 },
{ x: 100, y: 140 },
{ x: 0, y: 140 },
];
const matrix = computeHomography(src, 200, 280);
const mappedTopLeft = applyForward(matrix, 0, 0);
expect(mappedTopLeft.x).toBeCloseTo(0, 4);
expect(mappedTopLeft.y).toBeCloseTo(0, 4);
const mappedBottomRight = applyForward(matrix, 100, 140);
expect(mappedBottomRight.x).toBeCloseTo(200, 4);
expect(mappedBottomRight.y).toBeCloseTo(280, 4);
});
});
function applyForward(matrix, x, y) {
const denom = matrix[2][0] * x + matrix[2][1] * y + matrix[2][2];
return {
x: (matrix[0][0] * x + matrix[0][1] * y + matrix[0][2]) / denom,
y: (matrix[1][0] * x + matrix[1][1] * y + matrix[1][2]) / denom,
};
}