diff --git a/components/CameraScanner.js b/components/CameraScanner.js index 1498de6..7245896 100644 --- a/components/CameraScanner.js +++ b/components/CameraScanner.js @@ -1,5 +1,9 @@ import { useState, useEffect, useRef } from 'react'; import { rateLimitCooldownUntil, uploadScanCapture } from '../lib/scan-capture-upload.js'; +import { + detectCardShapesFromFrame, + mergeDetectedShapesIntoTrackedCards, +} from '../lib/scanner-card-detection.js'; import ScanDisambiguationDialog from './ScanDisambiguationDialog.js'; export default function CameraScanner({ onCardScanned, onError }) { @@ -42,209 +46,18 @@ export default function CameraScanner({ onCardScanned, onError }) { // Continuous shape detection for card-like rectangles const detectCardShapes = () => { if (!videoRef.current || !detectionCanvasRef.current || !isStreaming) return []; - - const video = videoRef.current; - const canvas = detectionCanvasRef.current; - const ctx = canvas.getContext('2d'); - - // Set canvas size for detection (smaller for performance) - canvas.width = 320; - canvas.height = 240; - - // Draw current video frame - ctx.drawImage(video, 0, 0, canvas.width, canvas.height); - - // Get image data for analysis - const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); - const data = imageData.data; - - // Convert to grayscale and detect edges - const grayscale = []; - const edges = []; - - for (let i = 0; i < data.length; i += 4) { - const gray = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]); - grayscale.push(gray); - } - - // Simple edge detection (Sobel-like) - for (let y = 1; y < canvas.height - 1; y++) { - for (let x = 1; x < canvas.width - 1; x++) { - const idx = y * canvas.width + x; - - const gx = -grayscale[idx - 1] + grayscale[idx + 1] + - -2 * grayscale[idx - 1 + canvas.width] + 2 * grayscale[idx + 1 + canvas.width] + - -grayscale[idx - 1 + 2 * canvas.width] + grayscale[idx + 1 + 2 * canvas.width]; - - const gy = -grayscale[idx - canvas.width] - 2 * grayscale[idx] - grayscale[idx + canvas.width] + - grayscale[idx - canvas.width + 2 * canvas.width] + 2 * grayscale[idx + 2 * canvas.width] + grayscale[idx + canvas.width + 2 * canvas.width]; - - const magnitude = Math.sqrt(gx * gx + gy * gy); - edges[idx] = magnitude > 100 ? 255 : 0; // Threshold for edge detection - } - } - - // Find rectangular regions that could be cards - const cardShapes = []; - const { width, height } = canvas; - - // Card aspect ratio constraints (typical trading cards are ~2.5:3.5 ratio) - const minCardWidth = Math.floor(width * 0.2); // Increased from 0.15 - const maxCardWidth = Math.floor(width * 0.6); // Decreased from 0.8 - const minCardHeight = Math.floor(height * 0.25); // Increased from 0.2 - const maxCardHeight = Math.floor(height * 0.7); // Decreased from 0.9 - - // Scan for edge-dense rectangular regions - for (let y = 0; y < height - minCardHeight; y += 15) { // Increased step - for (let x = 0; x < width - minCardWidth; x += 15) { // Increased step - for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 20) { // Increased step - for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 20) { // Increased step - - // Check aspect ratio (cards are typically 0.65-0.75) - const aspectRatio = w / h; - if (aspectRatio < 0.63 || aspectRatio > 0.77) continue; // Stricter range - - // Count edges in this region - let edgeCount = 0; - let totalPixels = 0; - let perimeterEdges = 0; - - // Sample the region - for (let sy = y; sy < y + h; sy += 4) { // Increased step for performance - for (let sx = x; sx < x + w; sx += 4) { // Increased step for performance - const idx = sy * width + sx; - if (edges[idx] === 255) { - edgeCount++; - - // Check if this edge is near the perimeter (cards have strong borders) - const isPerimeter = (sx < x + w * 0.15 || sx > x + w * 0.85 || - sy < y + h * 0.15 || sy > y + h * 0.85); - if (isPerimeter) { - perimeterEdges++; - } - } - totalPixels++; - } - } - - const edgeDensity = edgeCount / totalPixels; - const perimeterRatio = perimeterEdges / (edgeCount || 1); - - // Much stricter criteria for card-like objects - if (edgeDensity > 0.2 && edgeDensity < 0.6 && perimeterRatio > 0.4 && edgeCount > 80) { - const score = edgeDensity * 100 + perimeterRatio * 60 + (edgeCount / 10); - - // Higher threshold for accepting shapes - if (score > 50) { - // Convert back to video coordinates - const videoCoords = convertToVideoCoordinates({ x, y, width: w, height: h }, canvas, video); - - cardShapes.push({ - ...videoCoords, - score, - aspectRatio, - edgeDensity, - }); - } - } - } - } - } - } - - // Sort by score and return top candidates to allow multiple cards - return cardShapes.sort((a, b) => b.score - a.score).slice(0, 5); // Allow up to 5 cards simultaneously + return detectCardShapesFromFrame(videoRef.current, detectionCanvasRef.current); }; - // Convert detection coordinates to video coordinates - const convertToVideoCoordinates = (detection, canvas, video) => { - const scaleX = video.videoWidth / canvas.width; - const scaleY = video.videoHeight / canvas.height; - - return { - x: detection.x * scaleX, - y: detection.y * scaleY, - width: detection.width * scaleX, - height: detection.height * scaleY - }; - }; - - // Check if two rectangles overlap significantly - const rectanglesOverlap = (rect1, rect2, threshold = 0.3) => { // Reduced default threshold - const x1 = Math.max(rect1.x, rect2.x); - const y1 = Math.max(rect1.y, rect2.y); - const x2 = Math.min(rect1.x + rect1.width, rect2.x + rect2.width); - const y2 = Math.min(rect1.y + rect1.height, rect2.y + rect2.height); - - if (x2 <= x1 || y2 <= y1) return false; - - const overlapArea = (x2 - x1) * (y2 - y1); - const rect1Area = rect1.width * rect1.height; - const rect2Area = rect2.width * rect2.height; - const smallerArea = Math.min(rect1Area, rect2Area); - - // Use smaller area as denominator for better tracking of moving cards - return (overlapArea / smallerArea) > threshold; - }; - - // Update tracked cards with new detections const updateTrackedCards = (detectedShapes) => { - const currentTime = Date.now(); - const updatedCards = [...trackedCardsRef.current]; - - // Remove cards that haven't been seen recently (3 seconds for better responsiveness) - for (let i = updatedCards.length - 1; i >= 0; i--) { - if (currentTime - updatedCards[i].lastSeen > 3000) { - console.log(`🗑️ Removing stale tracked card ${updatedCards[i].id} (last seen ${Math.round((currentTime - updatedCards[i].lastSeen)/1000)}s ago)`); - updatedCards.splice(i, 1); - } - } - - // Match detected shapes with existing tracked cards - detectedShapes.forEach(shape => { - let matchedCard = null; - - // Find existing card that overlaps with this shape - for (const card of updatedCards) { - if (rectanglesOverlap(shape, card.bounds, 0.4)) { // Slightly more lenient overlap - matchedCard = card; - break; - } - } - - if (matchedCard) { - // Update existing card position and timestamp - matchedCard.bounds = { ...shape }; - matchedCard.lastSeen = currentTime; - matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10); - - // Reset scan attempts if card moved significantly (allows re-scanning) - const positionChange = Math.abs(matchedCard.bounds.x - shape.x) + Math.abs(matchedCard.bounds.y - shape.y); - if (positionChange > 50 && matchedCard.status === 'negative') { - console.log(`🔄 Card ${matchedCard.id} moved significantly, allowing re-scan`); - matchedCard.status = 'detecting'; - matchedCard.scanAttempts = 0; - matchedCard.stableCount = 1; - } - } else { - // Create new tracked card - const newCard = { - id: nextCardIdRef.current++, - bounds: { ...shape }, - status: 'detecting', // 'detecting', 'confirmed', 'negative', 'scanned' - firstSeen: currentTime, - lastSeen: currentTime, - stableCount: 1, - scanAttempts: 0 - }; - - updatedCards.push(newCard); - console.log(`🎯 New card shape detected: ${newCard.id}`); - } - }); - - trackedCardsRef.current = updatedCards; - setTrackedCards(updatedCards); + const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards( + trackedCardsRef.current, + detectedShapes, + nextCardIdRef.current + ); + nextCardIdRef.current = nextCardId; + trackedCardsRef.current = cards; + setTrackedCards(cards); }; const emitScannedCard = async (cardTracker, imageData, finalCard, ocrMeta = {}) => { diff --git a/lib/scanner-card-detection.js b/lib/scanner-card-detection.js new file mode 100644 index 0000000..de954ec --- /dev/null +++ b/lib/scanner-card-detection.js @@ -0,0 +1,217 @@ +/** Detection canvas dimensions (smaller than video for performance). */ +export const DETECTION_CANVAS_WIDTH = 320; +export const DETECTION_CANVAS_HEIGHT = 240; + +/** Milliseconds before a tracked card is dropped when not re-detected. */ +export const TRACK_STALE_MS = 3000; + +/** Convert detection-canvas coordinates to video pixel coordinates. */ +export function convertToVideoCoordinates(detection, canvas, video) { + const scaleX = video.videoWidth / canvas.width; + const scaleY = video.videoHeight / canvas.height; + + return { + x: detection.x * scaleX, + y: detection.y * scaleY, + width: detection.width * scaleX, + height: detection.height * scaleY, + }; +} + +/** True when overlap area exceeds `threshold` of the smaller rectangle. */ +export function rectanglesOverlap(rect1, rect2, threshold = 0.3) { + const x1 = Math.max(rect1.x, rect2.x); + const y1 = Math.max(rect1.y, rect2.y); + const x2 = Math.min(rect1.x + rect1.width, rect2.x + rect2.width); + const y2 = Math.min(rect1.y + rect1.height, rect2.y + rect2.height); + + if (x2 <= x1 || y2 <= y1) return false; + + const overlapArea = (x2 - x1) * (y2 - y1); + const rect1Area = rect1.width * rect1.height; + const rect2Area = rect2.width * rect2.height; + const smallerArea = Math.min(rect1Area, rect2Area); + + return overlapArea / smallerArea > threshold; +} + +/** + * Draw the current video frame on `detectionCanvas` and return up to 5 card-like + * bounding boxes in video coordinates. + */ +export function detectCardShapesFromFrame(video, detectionCanvas) { + if (!video || !detectionCanvas) return []; + + const ctx = detectionCanvas.getContext('2d'); + detectionCanvas.width = DETECTION_CANVAS_WIDTH; + detectionCanvas.height = DETECTION_CANVAS_HEIGHT; + + ctx.drawImage(video, 0, 0, detectionCanvas.width, detectionCanvas.height); + + const imageData = ctx.getImageData(0, 0, detectionCanvas.width, detectionCanvas.height); + const data = imageData.data; + + const grayscale = []; + const edges = []; + + for (let i = 0; i < data.length; i += 4) { + const gray = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]); + grayscale.push(gray); + } + + for (let y = 1; y < detectionCanvas.height - 1; y++) { + for (let x = 1; x < detectionCanvas.width - 1; x++) { + const idx = y * detectionCanvas.width + x; + + const gx = + -grayscale[idx - 1] + + grayscale[idx + 1] + + -2 * grayscale[idx - 1 + detectionCanvas.width] + + 2 * grayscale[idx + 1 + detectionCanvas.width] + + -grayscale[idx - 1 + 2 * detectionCanvas.width] + + grayscale[idx + 1 + 2 * detectionCanvas.width]; + + const gy = + -grayscale[idx - detectionCanvas.width] - + 2 * grayscale[idx] - + grayscale[idx + detectionCanvas.width] + + grayscale[idx - detectionCanvas.width + 2 * detectionCanvas.width] + + 2 * grayscale[idx + 2 * detectionCanvas.width] + + grayscale[idx + detectionCanvas.width + 2 * detectionCanvas.width]; + + const magnitude = Math.sqrt(gx * gx + gy * gy); + edges[idx] = magnitude > 100 ? 255 : 0; + } + } + + const cardShapes = []; + const { width, height } = detectionCanvas; + + const minCardWidth = Math.floor(width * 0.2); + const maxCardWidth = Math.floor(width * 0.6); + const minCardHeight = Math.floor(height * 0.25); + const maxCardHeight = Math.floor(height * 0.7); + + for (let y = 0; y < height - minCardHeight; y += 15) { + for (let x = 0; x < width - minCardWidth; x += 15) { + for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 20) { + for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 20) { + const aspectRatio = w / h; + if (aspectRatio < 0.63 || aspectRatio > 0.77) continue; + + let edgeCount = 0; + let totalPixels = 0; + let perimeterEdges = 0; + + for (let sy = y; sy < y + h; sy += 4) { + for (let sx = x; sx < x + w; sx += 4) { + const idx = sy * width + sx; + if (edges[idx] === 255) { + edgeCount++; + + const isPerimeter = + sx < x + w * 0.15 || + sx > x + w * 0.85 || + sy < y + h * 0.15 || + sy > y + h * 0.85; + if (isPerimeter) { + perimeterEdges++; + } + } + totalPixels++; + } + } + + const edgeDensity = edgeCount / totalPixels; + const perimeterRatio = perimeterEdges / (edgeCount || 1); + + if (edgeDensity > 0.2 && edgeDensity < 0.6 && perimeterRatio > 0.4 && edgeCount > 80) { + const score = edgeDensity * 100 + perimeterRatio * 60 + edgeCount / 10; + + if (score > 50) { + const videoCoords = convertToVideoCoordinates( + { x, y, width: w, height: h }, + detectionCanvas, + video + ); + + cardShapes.push({ + ...videoCoords, + score, + aspectRatio, + edgeDensity, + }); + } + } + } + } + } + } + + return cardShapes.sort((a, b) => b.score - a.score).slice(0, 5); +} + +/** + * Merge fresh shape detections into the tracked-card list. + * Returns updated cards and the next card id counter. + */ +export function mergeDetectedShapesIntoTrackedCards( + trackedCards, + detectedShapes, + nextCardId, + now = Date.now() +) { + const updatedCards = [...trackedCards]; + + for (let i = updatedCards.length - 1; i >= 0; i--) { + if (now - updatedCards[i].lastSeen > TRACK_STALE_MS) { + console.log( + `🗑️ Removing stale tracked card ${updatedCards[i].id} (last seen ${Math.round((now - updatedCards[i].lastSeen) / 1000)}s ago)` + ); + updatedCards.splice(i, 1); + } + } + + let idCounter = nextCardId; + + detectedShapes.forEach((shape) => { + let matchedCard = null; + + for (const card of updatedCards) { + if (rectanglesOverlap(shape, card.bounds, 0.4)) { + matchedCard = card; + break; + } + } + + if (matchedCard) { + matchedCard.bounds = { ...shape }; + matchedCard.lastSeen = now; + matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10); + + const positionChange = + Math.abs(matchedCard.bounds.x - shape.x) + Math.abs(matchedCard.bounds.y - shape.y); + if (positionChange > 50 && matchedCard.status === 'negative') { + console.log(`🔄 Card ${matchedCard.id} moved significantly, allowing re-scan`); + matchedCard.status = 'detecting'; + matchedCard.scanAttempts = 0; + matchedCard.stableCount = 1; + } + } else { + const newCard = { + id: idCounter++, + bounds: { ...shape }, + status: 'detecting', + firstSeen: now, + lastSeen: now, + stableCount: 1, + scanAttempts: 0, + }; + + updatedCards.push(newCard); + console.log(`🎯 New card shape detected: ${newCard.id}`); + } + }); + + return { cards: updatedCards, nextCardId: idCounter }; +} diff --git a/test/lib/scanner-card-detection.test.js b/test/lib/scanner-card-detection.test.js new file mode 100644 index 0000000..229e2b0 --- /dev/null +++ b/test/lib/scanner-card-detection.test.js @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; +import { + convertToVideoCoordinates, + mergeDetectedShapesIntoTrackedCards, + rectanglesOverlap, + TRACK_STALE_MS, +} from '../../lib/scanner-card-detection.js'; + +describe('rectanglesOverlap', () => { + it('returns true when rectangles mostly overlap', () => { + const a = { x: 0, y: 0, width: 100, height: 140 }; + const b = { x: 10, y: 10, width: 100, height: 140 }; + expect(rectanglesOverlap(a, b, 0.3)).toBe(true); + }); + + it('returns false when rectangles do not touch', () => { + const a = { x: 0, y: 0, width: 50, height: 70 }; + const b = { x: 200, y: 200, width: 50, height: 70 }; + expect(rectanglesOverlap(a, b)).toBe(false); + }); +}); + +describe('convertToVideoCoordinates', () => { + it('scales detection canvas coords to video pixel space', () => { + const canvas = { width: 320, height: 240 }; + const video = { videoWidth: 1280, videoHeight: 960 }; + expect(convertToVideoCoordinates({ x: 10, y: 20, width: 100, height: 140 }, canvas, video)).toEqual({ + x: 40, + y: 80, + width: 400, + height: 560, + }); + }); +}); + +describe('mergeDetectedShapesIntoTrackedCards', () => { + it('creates a new tracked card for an unmatched shape', () => { + const now = 1_000_000; + const shape = { x: 10, y: 20, width: 100, height: 140, score: 80 }; + + const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards([], [shape], 1, now); + + expect(cards).toHaveLength(1); + expect(cards[0]).toMatchObject({ + id: 1, + status: 'detecting', + stableCount: 1, + scanAttempts: 0, + bounds: shape, + }); + expect(nextCardId).toBe(2); + }); + + it('updates an overlapping tracked card instead of creating a duplicate', () => { + const now = 1_000_000; + const existing = { + id: 5, + bounds: { x: 10, y: 20, width: 100, height: 140 }, + status: 'detecting', + firstSeen: now - 500, + lastSeen: now - 100, + stableCount: 3, + scanAttempts: 0, + }; + const moved = { x: 12, y: 22, width: 100, height: 140, score: 82 }; + + const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards([existing], [moved], 6, now); + + expect(cards).toHaveLength(1); + expect(cards[0].id).toBe(5); + expect(cards[0].stableCount).toBe(4); + expect(cards[0].lastSeen).toBe(now); + expect(nextCardId).toBe(6); + }); + + it('drops cards not seen within TRACK_STALE_MS', () => { + const now = 10_000; + const stale = { + id: 1, + bounds: { x: 0, y: 0, width: 50, height: 70 }, + status: 'detecting', + firstSeen: now - TRACK_STALE_MS - 1, + lastSeen: now - TRACK_STALE_MS - 1, + stableCount: 2, + scanAttempts: 0, + }; + + const { cards } = mergeDetectedShapesIntoTrackedCards([stale], [], 2, now); + expect(cards).toHaveLength(0); + }); +});