rstillwell/deckhearth
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

refactor(scanner): extract card detection and tracking lib (Brief 2) #69

Merged
varutasu merged 1 commit from refactor/camera-scanner-split-brief-2 into main 2026-06-02 13:52:40 -04:00
Conversation 3 Commits 1 Files changed 3 +321 -200
3 changed files with 321 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

213
components/CameraScanner.js
View file

@ -1,5 +1,9 @@
import { useState, useEffect, useRef } from 'react';
import { rateLimitCooldownUntil, uploadScanCapture } from '../lib/scan-capture-upload.js';
import {
detectCardShapesFromFrame,
mergeDetectedShapesIntoTrackedCards,
} from '../lib/scanner-card-detection.js';
import ScanDisambiguationDialog from './ScanDisambiguationDialog.js';
export default function CameraScanner({ onCardScanned, onError }) {
@ -42,209 +46,18 @@ export default function CameraScanner({ onCardScanned, onError }) {
// Continuous shape detection for card-like rectangles
const detectCardShapes = () => {
if (!videoRef.current || !detectionCanvasRef.current || !isStreaming) return [];
const video = videoRef.current;
const canvas = detectionCanvasRef.current;
const ctx = canvas.getContext('2d');
// Set canvas size for detection (smaller for performance)
canvas.width = 320;
canvas.height = 240;
// Draw current video frame
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
// Get image data for analysis
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
const data = imageData.data;
// Convert to grayscale and detect edges
const grayscale = [];
const edges = [];
for (let i = 0; i < data.length; i += 4) {
const gray = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]);
grayscale.push(gray);
}
// Simple edge detection (Sobel-like)
for (let y = 1; y < canvas.height - 1; y++) {
for (let x = 1; x < canvas.width - 1; x++) {
const idx = y * canvas.width + x;
const gx = -grayscale[idx - 1] + grayscale[idx + 1] +
-2 * grayscale[idx - 1 + canvas.width] + 2 * grayscale[idx + 1 + canvas.width] +
-grayscale[idx - 1 + 2 * canvas.width] + grayscale[idx + 1 + 2 * canvas.width];
const gy = -grayscale[idx - canvas.width] - 2 * grayscale[idx] - grayscale[idx + canvas.width] +
grayscale[idx - canvas.width + 2 * canvas.width] + 2 * grayscale[idx + 2 * canvas.width] + grayscale[idx + canvas.width + 2 * canvas.width];
const magnitude = Math.sqrt(gx * gx + gy * gy);
edges[idx] = magnitude > 100 ? 255 : 0; // Threshold for edge detection
}
}
// Find rectangular regions that could be cards
const cardShapes = [];
const { width, height } = canvas;
// Card aspect ratio constraints (typical trading cards are ~2.5:3.5 ratio)
const minCardWidth = Math.floor(width * 0.2); // Increased from 0.15
const maxCardWidth = Math.floor(width * 0.6); // Decreased from 0.8
const minCardHeight = Math.floor(height * 0.25); // Increased from 0.2
const maxCardHeight = Math.floor(height * 0.7); // Decreased from 0.9
// Scan for edge-dense rectangular regions
for (let y = 0; y < height - minCardHeight; y += 15) { // Increased step
for (let x = 0; x < width - minCardWidth; x += 15) { // Increased step
for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 20) { // Increased step
for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 20) { // Increased step
// Check aspect ratio (cards are typically 0.65-0.75)
const aspectRatio = w / h;
if (aspectRatio < 0.63 || aspectRatio > 0.77) continue; // Stricter range
// Count edges in this region
let edgeCount = 0;
let totalPixels = 0;
let perimeterEdges = 0;
// Sample the region
for (let sy = y; sy < y + h; sy += 4) { // Increased step for performance
for (let sx = x; sx < x + w; sx += 4) { // Increased step for performance
const idx = sy * width + sx;
if (edges[idx] === 255) {
edgeCount++;
// Check if this edge is near the perimeter (cards have strong borders)
const isPerimeter = (sx < x + w * 0.15 || sx > x + w * 0.85 ||
sy < y + h * 0.15 || sy > y + h * 0.85);
if (isPerimeter) {
perimeterEdges++;
}
}
totalPixels++;
}
}
const edgeDensity = edgeCount / totalPixels;
const perimeterRatio = perimeterEdges / (edgeCount || 1);
// Much stricter criteria for card-like objects
if (edgeDensity > 0.2 && edgeDensity < 0.6 && perimeterRatio > 0.4 && edgeCount > 80) {
const score = edgeDensity * 100 + perimeterRatio * 60 + (edgeCount / 10);
// Higher threshold for accepting shapes
if (score > 50) {
// Convert back to video coordinates
const videoCoords = convertToVideoCoordinates({ x, y, width: w, height: h }, canvas, video);
cardShapes.push({
...videoCoords,
score,
aspectRatio,
edgeDensity,
});
}
}
}
}
}
}
// Sort by score and return top candidates to allow multiple cards
return cardShapes.sort((a, b) => b.score - a.score).slice(0, 5); // Allow up to 5 cards simultaneously
return detectCardShapesFromFrame(videoRef.current, detectionCanvasRef.current);
};
// Convert detection coordinates to video coordinates
const convertToVideoCoordinates = (detection, canvas, video) => {
const scaleX = video.videoWidth / canvas.width;
const scaleY = video.videoHeight / canvas.height;
return {
x: detection.x * scaleX,
y: detection.y * scaleY,
width: detection.width * scaleX,
height: detection.height * scaleY
};
};
// Check if two rectangles overlap significantly
const rectanglesOverlap = (rect1, rect2, threshold = 0.3) => { // Reduced default threshold
const x1 = Math.max(rect1.x, rect2.x);
const y1 = Math.max(rect1.y, rect2.y);
const x2 = Math.min(rect1.x + rect1.width, rect2.x + rect2.width);
const y2 = Math.min(rect1.y + rect1.height, rect2.y + rect2.height);
if (x2 <= x1 || y2 <= y1) return false;
const overlapArea = (x2 - x1) * (y2 - y1);
const rect1Area = rect1.width * rect1.height;
const rect2Area = rect2.width * rect2.height;
const smallerArea = Math.min(rect1Area, rect2Area);
// Use smaller area as denominator for better tracking of moving cards
return (overlapArea / smallerArea) > threshold;
};
// Update tracked cards with new detections
const updateTrackedCards = (detectedShapes) => {
const currentTime = Date.now();
const updatedCards = [...trackedCardsRef.current];
// Remove cards that haven't been seen recently (3 seconds for better responsiveness)
for (let i = updatedCards.length - 1; i >= 0; i--) {
if (currentTime - updatedCards[i].lastSeen > 3000) {
console.log(`🗑️ Removing stale tracked card ${updatedCards[i].id} (last seen ${Math.round((currentTime - updatedCards[i].lastSeen)/1000)}s ago)`);
updatedCards.splice(i, 1);
}
}
// Match detected shapes with existing tracked cards
detectedShapes.forEach(shape => {
let matchedCard = null;
// Find existing card that overlaps with this shape
for (const card of updatedCards) {
if (rectanglesOverlap(shape, card.bounds, 0.4)) { // Slightly more lenient overlap
matchedCard = card;
break;
}
}
if (matchedCard) {
// Update existing card position and timestamp
matchedCard.bounds = { ...shape };
matchedCard.lastSeen = currentTime;
matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10);
// Reset scan attempts if card moved significantly (allows re-scanning)
const positionChange = Math.abs(matchedCard.bounds.x - shape.x) + Math.abs(matchedCard.bounds.y - shape.y);
if (positionChange > 50 && matchedCard.status === 'negative') {
console.log(`🔄 Card ${matchedCard.id} moved significantly, allowing re-scan`);
matchedCard.status = 'detecting';
matchedCard.scanAttempts = 0;
matchedCard.stableCount = 1;
}
} else {
// Create new tracked card
const newCard = {
id: nextCardIdRef.current++,
bounds: { ...shape },
status: 'detecting', // 'detecting', 'confirmed', 'negative', 'scanned'
firstSeen: currentTime,
lastSeen: currentTime,
stableCount: 1,
scanAttempts: 0
};
updatedCards.push(newCard);
console.log(`🎯 New card shape detected: ${newCard.id}`);
}
});
trackedCardsRef.current = updatedCards;
setTrackedCards(updatedCards);
const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards(
trackedCardsRef.current,
detectedShapes,
nextCardIdRef.current
);
nextCardIdRef.current = nextCardId;
trackedCardsRef.current = cards;
setTrackedCards(cards);
};
const emitScannedCard = async (cardTracker, imageData, finalCard, ocrMeta = {}) => {

217
lib/scanner-card-detection.js Normal file
View file

@ -0,0 +1,217 @@
/** Detection canvas dimensions (smaller than video for performance). */
export const DETECTION_CANVAS_WIDTH = 320;
export const DETECTION_CANVAS_HEIGHT = 240;
/** Milliseconds before a tracked card is dropped when not re-detected. */
export const TRACK_STALE_MS = 3000;
/** Convert detection-canvas coordinates to video pixel coordinates. */
export function convertToVideoCoordinates(detection, canvas, video) {
const scaleX = video.videoWidth / canvas.width;
const scaleY = video.videoHeight / canvas.height;
return {
x: detection.x * scaleX,
y: detection.y * scaleY,
width: detection.width * scaleX,
height: detection.height * scaleY,
};
}
/** True when overlap area exceeds `threshold` of the smaller rectangle. */
export function rectanglesOverlap(rect1, rect2, threshold = 0.3) {
const x1 = Math.max(rect1.x, rect2.x);
const y1 = Math.max(rect1.y, rect2.y);
const x2 = Math.min(rect1.x + rect1.width, rect2.x + rect2.width);
const y2 = Math.min(rect1.y + rect1.height, rect2.y + rect2.height);
if (x2 <= x1 || y2 <= y1) return false;
const overlapArea = (x2 - x1) * (y2 - y1);
const rect1Area = rect1.width * rect1.height;
const rect2Area = rect2.width * rect2.height;
const smallerArea = Math.min(rect1Area, rect2Area);
return overlapArea / smallerArea > threshold;
}
/**
* Draw the current video frame on `detectionCanvas` and return up to 5 card-like
* bounding boxes in video coordinates.
*/
export function detectCardShapesFromFrame(video, detectionCanvas) {
if (!video || !detectionCanvas) return [];
const ctx = detectionCanvas.getContext('2d');
detectionCanvas.width = DETECTION_CANVAS_WIDTH;
detectionCanvas.height = DETECTION_CANVAS_HEIGHT;
ctx.drawImage(video, 0, 0, detectionCanvas.width, detectionCanvas.height);
const imageData = ctx.getImageData(0, 0, detectionCanvas.width, detectionCanvas.height);
const data = imageData.data;
const grayscale = [];
const edges = [];
for (let i = 0; i < data.length; i += 4) {
const gray = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]);
grayscale.push(gray);
}
for (let y = 1; y < detectionCanvas.height - 1; y++) {
for (let x = 1; x < detectionCanvas.width - 1; x++) {
const idx = y * detectionCanvas.width + x;
const gx =
-grayscale[idx - 1] +
grayscale[idx + 1] +
-2 * grayscale[idx - 1 + detectionCanvas.width] +
2 * grayscale[idx + 1 + detectionCanvas.width] +
-grayscale[idx - 1 + 2 * detectionCanvas.width] +
grayscale[idx + 1 + 2 * detectionCanvas.width];
const gy =
-grayscale[idx - detectionCanvas.width] -
2 * grayscale[idx] -
grayscale[idx + detectionCanvas.width] +
grayscale[idx - detectionCanvas.width + 2 * detectionCanvas.width] +
2 * grayscale[idx + 2 * detectionCanvas.width] +
grayscale[idx + detectionCanvas.width + 2 * detectionCanvas.width];
const magnitude = Math.sqrt(gx * gx + gy * gy);
edges[idx] = magnitude > 100 ? 255 : 0;
}
}
const cardShapes = [];
const { width, height } = detectionCanvas;
const minCardWidth = Math.floor(width * 0.2);
const maxCardWidth = Math.floor(width * 0.6);
const minCardHeight = Math.floor(height * 0.25);
const maxCardHeight = Math.floor(height * 0.7);
for (let y = 0; y < height - minCardHeight; y += 15) {
for (let x = 0; x < width - minCardWidth; x += 15) {
for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 20) {
for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 20) {
const aspectRatio = w / h;
if (aspectRatio < 0.63 || aspectRatio > 0.77) continue;
let edgeCount = 0;
let totalPixels = 0;
let perimeterEdges = 0;
for (let sy = y; sy < y + h; sy += 4) {
for (let sx = x; sx < x + w; sx += 4) {
const idx = sy * width + sx;
if (edges[idx] === 255) {
edgeCount++;
const isPerimeter =
sx < x + w * 0.15 ||
sx > x + w * 0.85 ||
sy < y + h * 0.15 ||
sy > y + h * 0.85;
if (isPerimeter) {
perimeterEdges++;
}
}
totalPixels++;
}
}
const edgeDensity = edgeCount / totalPixels;
const perimeterRatio = perimeterEdges / (edgeCount || 1);
if (edgeDensity > 0.2 && edgeDensity < 0.6 && perimeterRatio > 0.4 && edgeCount > 80) {
const score = edgeDensity * 100 + perimeterRatio * 60 + edgeCount / 10;
if (score > 50) {
const videoCoords = convertToVideoCoordinates(
{ x, y, width: w, height: h },
detectionCanvas,
video
);
cardShapes.push({
...videoCoords,
score,
aspectRatio,
edgeDensity,
});
}
}
}
}
}
}
return cardShapes.sort((a, b) => b.score - a.score).slice(0, 5);
}
/**
* Merge fresh shape detections into the tracked-card list.
* Returns updated cards and the next card id counter.
*/
export function mergeDetectedShapesIntoTrackedCards(
trackedCards,
detectedShapes,
nextCardId,
now = Date.now()
) {
const updatedCards = [...trackedCards];
for (let i = updatedCards.length - 1; i >= 0; i--) {
if (now - updatedCards[i].lastSeen > TRACK_STALE_MS) {
console.log(
`🗑️ Removing stale tracked card ${updatedCards[i].id} (last seen ${Math.round((now - updatedCards[i].lastSeen) / 1000)}s ago)`
);
updatedCards.splice(i, 1);
}
}
let idCounter = nextCardId;
detectedShapes.forEach((shape) => {
let matchedCard = null;
for (const card of updatedCards) {
if (rectanglesOverlap(shape, card.bounds, 0.4)) {
matchedCard = card;
break;
}
}
if (matchedCard) {
matchedCard.bounds = { ...shape };
matchedCard.lastSeen = now;
matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10);
const positionChange =
Math.abs(matchedCard.bounds.x - shape.x) + Math.abs(matchedCard.bounds.y - shape.y);
if (positionChange > 50 && matchedCard.status === 'negative') {
console.log(`🔄 Card ${matchedCard.id} moved significantly, allowing re-scan`);
matchedCard.status = 'detecting';
matchedCard.scanAttempts = 0;
matchedCard.stableCount = 1;
}
} else {
const newCard = {
id: idCounter++,
bounds: { ...shape },
status: 'detecting',
firstSeen: now,
lastSeen: now,
stableCount: 1,
scanAttempts: 0,
};
updatedCards.push(newCard);
console.log(`🎯 New card shape detected: ${newCard.id}`);
}
});
return { cards: updatedCards, nextCardId: idCounter };
}

91
test/lib/scanner-card-detection.test.js Normal file
View file

@ -0,0 +1,91 @@
import { describe, expect, it } from 'vitest';
import {
convertToVideoCoordinates,
mergeDetectedShapesIntoTrackedCards,
rectanglesOverlap,
TRACK_STALE_MS,
} from '../../lib/scanner-card-detection.js';
describe('rectanglesOverlap', () => {
it('returns true when rectangles mostly overlap', () => {
const a = { x: 0, y: 0, width: 100, height: 140 };
const b = { x: 10, y: 10, width: 100, height: 140 };
expect(rectanglesOverlap(a, b, 0.3)).toBe(true);
});
it('returns false when rectangles do not touch', () => {
const a = { x: 0, y: 0, width: 50, height: 70 };
const b = { x: 200, y: 200, width: 50, height: 70 };
expect(rectanglesOverlap(a, b)).toBe(false);
});
});
describe('convertToVideoCoordinates', () => {
it('scales detection canvas coords to video pixel space', () => {
const canvas = { width: 320, height: 240 };
const video = { videoWidth: 1280, videoHeight: 960 };
expect(convertToVideoCoordinates({ x: 10, y: 20, width: 100, height: 140 }, canvas, video)).toEqual({
x: 40,
y: 80,
width: 400,
height: 560,
});
});
});
describe('mergeDetectedShapesIntoTrackedCards', () => {
it('creates a new tracked card for an unmatched shape', () => {
const now = 1_000_000;
const shape = { x: 10, y: 20, width: 100, height: 140, score: 80 };
const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards([], [shape], 1, now);
expect(cards).toHaveLength(1);
expect(cards[0]).toMatchObject({
id: 1,
status: 'detecting',
stableCount: 1,
scanAttempts: 0,
bounds: shape,
});
expect(nextCardId).toBe(2);
});
it('updates an overlapping tracked card instead of creating a duplicate', () => {
const now = 1_000_000;
const existing = {
id: 5,
bounds: { x: 10, y: 20, width: 100, height: 140 },
status: 'detecting',
firstSeen: now - 500,
lastSeen: now - 100,
stableCount: 3,
scanAttempts: 0,
};
const moved = { x: 12, y: 22, width: 100, height: 140, score: 82 };
const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards([existing], [moved], 6, now);
expect(cards).toHaveLength(1);
expect(cards[0].id).toBe(5);
expect(cards[0].stableCount).toBe(4);
expect(cards[0].lastSeen).toBe(now);
expect(nextCardId).toBe(6);
});
it('drops cards not seen within TRACK_STALE_MS', () => {
const now = 10_000;
const stale = {
id: 1,
bounds: { x: 0, y: 0, width: 50, height: 70 },
status: 'detecting',
firstSeen: now - TRACK_STALE_MS - 1,
lastSeen: now - TRACK_STALE_MS - 1,
stableCount: 2,
scanAttempts: 0,
};
const { cards } = mergeDetectedShapesIntoTrackedCards([stale], [], 2, now);
expect(cards).toHaveLength(0);
});
});