import { useState, useEffect, useRef } from 'react'; export default function CameraScanner({ onCardScanned, onError }) { const [isStreaming, setIsStreaming] = useState(false); const [isDetecting, setIsDetecting] = useState(false); const [disambiguation, setDisambiguation] = useState(null); const [scanNotice, setScanNotice] = useState(null); const [submittingReview, setSubmittingReview] = useState(false); const videoRef = useRef(null); const canvasRef = useRef(null); const detectionCanvasRef = useRef(null); const streamRef = useRef(null); const detectionIntervalRef = useRef(null); const trackingIntervalRef = useRef(null); // Card tracking state const [trackedCards, setTrackedCards] = useState([]); // Array of tracked card objects const trackedCardsRef = useRef([]); const nextCardIdRef = useRef(1); const visionCooldownUntilRef = useRef(0); const activeVerificationRef = useRef(0); const lastErrorAtRef = useRef(0); const disambiguationRefineRef = useRef(null); // Mana symbol settings const [manaSymbolSettings, setManaSymbolSettings] = useState({ useSVG: false }); // Configure canvas contexts for optimal performance useEffect(() => { if (canvasRef.current) { const ctx = canvasRef.current.getContext('2d', { willReadFrequently: true }); } if (detectionCanvasRef.current) { const ctx = detectionCanvasRef.current.getContext('2d', { willReadFrequently: true }); } }, []); // Continuous shape detection for card-like rectangles const detectCardShapes = () => { if (!videoRef.current || !detectionCanvasRef.current || !isStreaming) return []; const video = videoRef.current; const canvas = detectionCanvasRef.current; const ctx = canvas.getContext('2d'); // Set canvas size for detection (smaller for performance) canvas.width = 320; canvas.height = 240; // Draw current video frame ctx.drawImage(video, 0, 0, canvas.width, canvas.height); // Get image data for analysis const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); const data = imageData.data; // Convert to grayscale and detect edges const grayscale = []; const edges = []; for (let i = 0; i < data.length; i += 4) { const gray = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]); grayscale.push(gray); } // Simple edge detection (Sobel-like) for (let y = 1; y < canvas.height - 1; y++) { for (let x = 1; x < canvas.width - 1; x++) { const idx = y * canvas.width + x; const gx = -grayscale[idx - 1] + grayscale[idx + 1] + -2 * grayscale[idx - 1 + canvas.width] + 2 * grayscale[idx + 1 + canvas.width] + -grayscale[idx - 1 + 2 * canvas.width] + grayscale[idx + 1 + 2 * canvas.width]; const gy = -grayscale[idx - canvas.width] - 2 * grayscale[idx] - grayscale[idx + canvas.width] + grayscale[idx - canvas.width + 2 * canvas.width] + 2 * grayscale[idx + 2 * canvas.width] + grayscale[idx + canvas.width + 2 * canvas.width]; const magnitude = Math.sqrt(gx * gx + gy * gy); edges[idx] = magnitude > 100 ? 255 : 0; // Threshold for edge detection } } // Find rectangular regions that could be cards const cardShapes = []; const { width, height } = canvas; // Card aspect ratio constraints (typical trading cards are ~2.5:3.5 ratio) const minCardWidth = Math.floor(width * 0.2); // Increased from 0.15 const maxCardWidth = Math.floor(width * 0.6); // Decreased from 0.8 const minCardHeight = Math.floor(height * 0.25); // Increased from 0.2 const maxCardHeight = Math.floor(height * 0.7); // Decreased from 0.9 // Scan for edge-dense rectangular regions for (let y = 0; y < height - minCardHeight; y += 15) { // Increased step for (let x = 0; x < width - minCardWidth; x += 15) { // Increased step for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 20) { // Increased step for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 20) { // Increased step // Check aspect ratio (cards are typically 0.65-0.75) const aspectRatio = w / h; if (aspectRatio < 0.63 || aspectRatio > 0.77) continue; // Stricter range // Count edges in this region let edgeCount = 0; let totalPixels = 0; let perimeterEdges = 0; // Sample the region for (let sy = y; sy < y + h; sy += 4) { // Increased step for performance for (let sx = x; sx < x + w; sx += 4) { // Increased step for performance const idx = sy * width + sx; if (edges[idx] === 255) { edgeCount++; // Check if this edge is near the perimeter (cards have strong borders) const isPerimeter = (sx < x + w * 0.15 || sx > x + w * 0.85 || sy < y + h * 0.15 || sy > y + h * 0.85); if (isPerimeter) { perimeterEdges++; } } totalPixels++; } } const edgeDensity = edgeCount / totalPixels; const perimeterRatio = perimeterEdges / (edgeCount || 1); // Much stricter criteria for card-like objects if (edgeDensity > 0.2 && edgeDensity < 0.6 && perimeterRatio > 0.4 && edgeCount > 80) { const score = edgeDensity * 100 + perimeterRatio * 60 + (edgeCount / 10); // Higher threshold for accepting shapes if (score > 50) { // Convert back to video coordinates const videoCoords = convertToVideoCoordinates({ x, y, width: w, height: h }, canvas, video); cardShapes.push({ ...videoCoords, score, aspectRatio, edgeDensity, timestamp: Date.now() }); } } } } } } // Sort by score and return top candidates to allow multiple cards return cardShapes.sort((a, b) => b.score - a.score).slice(0, 5); // Allow up to 5 cards simultaneously }; // Convert detection coordinates to video coordinates const convertToVideoCoordinates = (detection, canvas, video) => { const scaleX = video.videoWidth / canvas.width; const scaleY = video.videoHeight / canvas.height; return { x: detection.x * scaleX, y: detection.y * scaleY, width: detection.width * scaleX, height: detection.height * scaleY }; }; // Check if two rectangles overlap significantly const rectanglesOverlap = (rect1, rect2, threshold = 0.3) => { // Reduced default threshold const x1 = Math.max(rect1.x, rect2.x); const y1 = Math.max(rect1.y, rect2.y); const x2 = Math.min(rect1.x + rect1.width, rect2.x + rect2.width); const y2 = Math.min(rect1.y + rect1.height, rect2.y + rect2.height); if (x2 <= x1 || y2 <= y1) return false; const overlapArea = (x2 - x1) * (y2 - y1); const rect1Area = rect1.width * rect1.height; const rect2Area = rect2.width * rect2.height; const smallerArea = Math.min(rect1Area, rect2Area); // Use smaller area as denominator for better tracking of moving cards return (overlapArea / smallerArea) > threshold; }; // Update tracked cards with new detections const updateTrackedCards = (detectedShapes) => { const currentTime = Date.now(); const updatedCards = [...trackedCardsRef.current]; // Remove cards that haven't been seen recently (3 seconds for better responsiveness) for (let i = updatedCards.length - 1; i >= 0; i--) { if (currentTime - updatedCards[i].lastSeen > 3000) { console.log(`πŸ—‘οΈ Removing stale tracked card ${updatedCards[i].id} (last seen ${Math.round((currentTime - updatedCards[i].lastSeen)/1000)}s ago)`); updatedCards.splice(i, 1); } } // Match detected shapes with existing tracked cards detectedShapes.forEach(shape => { let matchedCard = null; // Find existing card that overlaps with this shape for (const card of updatedCards) { if (rectanglesOverlap(shape, card.bounds, 0.4)) { // Slightly more lenient overlap matchedCard = card; break; } } if (matchedCard) { // Update existing card position and timestamp matchedCard.bounds = { ...shape }; matchedCard.lastSeen = currentTime; matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10); // Reset scan attempts if card moved significantly (allows re-scanning) const positionChange = Math.abs(matchedCard.bounds.x - shape.x) + Math.abs(matchedCard.bounds.y - shape.y); if (positionChange > 50 && matchedCard.status === 'negative') { console.log(`πŸ”„ Card ${matchedCard.id} moved significantly, allowing re-scan`); matchedCard.status = 'detecting'; matchedCard.scanAttempts = 0; matchedCard.stableCount = 1; } } else { // Create new tracked card const newCard = { id: nextCardIdRef.current++, bounds: { ...shape }, status: 'detecting', // 'detecting', 'confirmed', 'negative', 'scanned' firstSeen: currentTime, lastSeen: currentTime, stableCount: 1, scanAttempts: 0 }; updatedCards.push(newCard); console.log(`🎯 New card shape detected: ${newCard.id}`); } }); trackedCardsRef.current = updatedCards; setTrackedCards(updatedCards); }; const emitScannedCard = (cardTracker, imageData, finalCard, ocrMeta = {}) => { cardTracker.status = 'scanned'; const originalTitle = document.title; document.title = `πŸ“Έ ${finalCard.name} - Card Scanner`; setTimeout(() => { document.title = originalTitle; }, 3000); onCardScanned({ name: finalCard.name, set: finalCard.set_name, setCode: finalCard.set_code, cardNumber: finalCard.card_number, game: finalCard.game, cardType: finalCard.card_type, rarity: finalCard.rarity, hp: finalCard.hp || ocrMeta.hp, manaCost: finalCard.mana_cost || ocrMeta.manaCost, abilities: ocrMeta.abilities || [], ocrText: ocrMeta.rawText, confidence: ocrMeta.confidence, capturedImage: imageData, image_url: finalCard.image_url, databaseId: finalCard.id, isExisting: true, }); }; const handleDisambiguationPick = (candidate) => { if (!disambiguation) return; const { cardTracker, imageData, ocrMeta } = disambiguation; emitScannedCard(cardTracker, imageData, candidate, ocrMeta); setDisambiguation(null); disambiguationRefineRef.current = null; }; const showScanNotice = (message) => { setScanNotice(message); setTimeout(() => setScanNotice(null), 8000); }; const handleReviewSubmitted = (cardTracker, message) => { if (cardTracker) cardTracker.status = 'confirmed'; setDisambiguation(null); disambiguationRefineRef.current = null; showScanNotice(message); }; const handleNotInCatalog = async () => { if (!disambiguation || submittingReview) return; setSubmittingReview(true); const { cardTracker, imageData, candidates, ocrMeta } = disambiguation; const guessedName = ocrMeta?.cardName || ocrMeta?.query || candidates?.[0]?.name || null; try { const response = await fetch('/api/scan/submit-for-review', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('auth_token')}`, }, body: JSON.stringify({ imageData, name: guessedName, candidateCardIds: (candidates || []).map((c) => c.id), }), }); if (response.status === 429) { visionCooldownUntilRef.current = Date.now() + 60_000; reportScannerError('Too many scan attempts. Please wait a moment and try again.'); return; } if (!response.ok) { const errBody = await response.json().catch(() => ({})); throw new Error(errBody.error || 'Failed to submit scan for review'); } const result = await response.json(); handleReviewSubmitted(cardTracker, result.message); } catch (error) { reportScannerError(error.message || 'Failed to submit scan for review'); } finally { setSubmittingReview(false); } }; const candidateMatchesSetHint = (candidate, setName, setCode) => { if (!setName && !setCode) return true; const hint = (setName || setCode || '').toLowerCase(); const setNameLower = (candidate.set_name || '').toLowerCase(); const setCodeLower = (candidate.set_code || '').toLowerCase(); return setNameLower.includes(hint) || hint.includes(setNameLower) || setCodeLower === hint; }; const processIdentifyResponse = async (cardTracker, imageData, result) => { if (!result.isCard) { cardTracker.status = 'negative'; reportScannerError(result.reason || 'No trading card detected'); return; } const ocrMeta = { confidence: result.ocr?.confidence ?? result.card?.ocr?.confidence, rawText: result.ocr?.rawText ?? result.card?.ocr?.rawText, abilities: result.card?.ocr?.abilities || [], hp: result.card?.hp, manaCost: result.card?.mana_cost, }; if (result.card) { cardTracker.status = 'confirmed'; emitScannedCard(cardTracker, imageData, result.card, ocrMeta); return; } if (result.needsUserSelection && result.matches?.length) { cardTracker.status = 'confirmed'; setDisambiguation({ cardTracker, imageData, candidates: result.matches, ocrMeta, message: result.message, }); return; } if (result.needsReview) { cardTracker.status = 'confirmed'; showScanNotice(result.message || 'Scan saved for admin review.'); return; } if (result.needsUserInput) { cardTracker.status = 'negative'; reportScannerError(result.message || 'Could not identify card β€” try again or submit for review.'); return; } cardTracker.status = 'negative'; reportScannerError('Could not identify card from scan.'); }; const reportScannerError = (message) => { if (disambiguation) return; const now = Date.now(); if (now - lastErrorAtRef.current < 4000) return; lastErrorAtRef.current = now; onError?.(message); }; useEffect(() => { if (!disambiguation?.imageData) return; if (Date.now() < visionCooldownUntilRef.current) return; const refineKey = disambiguation.cardTracker?.id ?? 'modal'; if (disambiguationRefineRef.current === refineKey) return; disambiguationRefineRef.current = refineKey; let cancelled = false; (async () => { try { const response = await fetch('/api/scan/identify', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('auth_token')}`, }, body: JSON.stringify({ imageData: disambiguation.imageData }), }); if (cancelled) return; if (response.status === 429) { visionCooldownUntilRef.current = Date.now() + 60_000; return; } if (!response.ok) return; const result = await response.json(); if (cancelled) return; if (result.needsReview) { handleReviewSubmitted(disambiguation.cardTracker, result.message); return; } if (result.card) { emitScannedCard( disambiguation.cardTracker, disambiguation.imageData, result.card, { confidence: result.ocr?.confidence, rawText: result.ocr?.rawText, abilities: result.card?.ocr?.abilities || [], } ); setDisambiguation(null); disambiguationRefineRef.current = null; return; } const setHint = result.ocr?.setName || result.ocr?.setCode; if (result.matches?.length && setHint) { const filtered = disambiguation.candidates.filter((candidate) => candidateMatchesSetHint(candidate, result.ocr.setName, result.ocr.setCode) ); if (filtered.length === 0 && setHint) { try { const submitRes = await fetch('/api/scan/submit-for-review', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('auth_token')}`, }, body: JSON.stringify({ imageData: disambiguation.imageData, name: result.ocr?.cardName || disambiguation.candidates[0]?.name, candidateCardIds: disambiguation.candidates.map((c) => c.id), }), }); if (submitRes.ok) { const submitResult = await submitRes.json(); handleReviewSubmitted(disambiguation.cardTracker, submitResult.message); } } catch { setDisambiguation((current) => current ? { ...current, visionHint: setHint, message: `No "${setHint}" printing in our catalog. Tap "My card isn't listed" to save for admin review.`, } : current ); } return; } if (filtered.length === 1) { handleDisambiguationPick(filtered[0]); return; } if (filtered.length > 1 && filtered.length < disambiguation.candidates.length) { setDisambiguation((current) => current ? { ...current, candidates: filtered, message: `Narrowed to ${filtered.length} printings matching "${setHint}".`, visionHint: setHint, } : current ); } } } catch (error) { console.warn('Disambiguation vision refine failed:', error); } })(); return () => { cancelled = true; }; }, [disambiguation?.cardTracker?.id, disambiguation?.imageData]); // Server-side card identification const verifyCardShape = async (cardTracker) => { if (!videoRef.current || !canvasRef.current || cardTracker.status !== 'detecting') return; if (disambiguation) return; if (activeVerificationRef.current >= 1) return; activeVerificationRef.current += 1; cardTracker.status = 'verifying'; try { cardTracker.scanAttempts++; const video = videoRef.current; const canvas = canvasRef.current; const ctx = canvas.getContext('2d'); const { x, y, width, height } = cardTracker.bounds; const margin = 20; canvas.width = width + margin * 2; canvas.height = height + margin * 2; ctx.drawImage( video, Math.max(0, x - margin), Math.max(0, y - margin), width + margin * 2, height + margin * 2, 0, 0, canvas.width, canvas.height ); const imageData = canvas.toDataURL('image/jpeg', 0.8); const authHeaders = { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('auth_token')}`, }; // Layer 1: local OCR + pg_trgm catalog match (no vision LLM) try { const { recognizeCardNameStrip } = await import('../lib/ocr-worker.js'); const ocr = await recognizeCardNameStrip(imageData); if (ocr.text.length >= 3) { const l1Response = await fetch('/api/cards/identify-by-text', { method: 'POST', headers: authHeaders, body: JSON.stringify({ ocrText: ocr.text, ocrConfidence: ocr.confidence, }), }); if (l1Response.ok) { const l1Result = await l1Response.json(); if (!l1Result.escalate) { cardTracker.status = 'confirmed'; await processIdentifyResponse(cardTracker, imageData, l1Result); return; } } } } catch (l1Error) { console.warn('Layer-1 OCR path failed, escalating to vision:', l1Error); } // Layer 2: vision via AI Gateway (skip while rate-limited) if (Date.now() < visionCooldownUntilRef.current) { cardTracker.status = 'detecting'; return; } const response = await fetch('/api/scan/identify', { method: 'POST', headers: authHeaders, body: JSON.stringify({ imageData }), }); if (response.status === 429) { visionCooldownUntilRef.current = Date.now() + 60_000; reportScannerError('Too many scan attempts. Please wait a moment and try again.'); cardTracker.status = 'negative'; return; } if (!response.ok) { const errBody = await response.json().catch(() => ({})); throw new Error(errBody.error || `Scan identify failed: ${response.status}`); } const result = await response.json(); cardTracker.status = 'confirmed'; await processIdentifyResponse(cardTracker, imageData, result); } catch (error) { console.error(`Error verifying card ${cardTracker.id}:`, error); cardTracker.status = 'negative'; reportScannerError(error.message || 'Scan failed'); } finally { activeVerificationRef.current = Math.max(0, activeVerificationRef.current - 1); } }; // Start continuous detection const startDetection = () => { if (detectionIntervalRef.current || !isStreaming) return; console.log('🎯 Starting continuous card detection...'); setIsDetecting(true); // Shape detection every 200ms detectionIntervalRef.current = setInterval(() => { const shapes = detectCardShapes(); updateTrackedCards(shapes); }, 200); // Card verification every 1 second trackingIntervalRef.current = setInterval(() => { const cardsToVerify = trackedCardsRef.current.filter(card => card.status === 'detecting' && card.stableCount >= 6 && card.scanAttempts < 1 && Date.now() - card.firstSeen > 2500 ); const cardsToProcess = cardsToVerify.slice(0, 1); cardsToProcess.forEach(card => { verifyCardShape(card); }); }, 1000); // Back to 1 second intervals }; // Stop detection const stopDetection = () => { console.log('πŸ›‘ Stopping card detection...'); setIsDetecting(false); if (detectionIntervalRef.current) { clearInterval(detectionIntervalRef.current); detectionIntervalRef.current = null; } if (trackingIntervalRef.current) { clearInterval(trackingIntervalRef.current); trackingIntervalRef.current = null; } // Clear tracked cards trackedCardsRef.current = []; setTrackedCards([]); }; // Start camera stream const startCamera = async () => { try { console.log('πŸŽ₯ Starting camera...'); const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment', width: { ideal: 1280 }, height: { ideal: 720 }, aspectRatio: { ideal: 16/9 } } }); console.log('πŸ“Ή Camera stream obtained:', stream); if (videoRef.current) { videoRef.current.srcObject = stream; streamRef.current = stream; videoRef.current.onloadedmetadata = () => { console.log('πŸ“Ί Video metadata loaded, attempting to play...'); videoRef.current?.play().then(() => { console.log('▢️ Video playback started successfully'); setIsStreaming(true); }).catch((err) => { console.error('❌ Video playback failed:', err); onError(`Video playback failed: ${err.message}`); }); }; videoRef.current.onerror = (err) => { console.error('❌ Video element error:', err); onError('Video element error occurred'); }; // Add a fallback timeout setTimeout(() => { if (!isStreaming && videoRef.current && videoRef.current.readyState >= 2) { console.log('πŸ”„ Fallback: Attempting to play video directly...'); videoRef.current.play().then(() => { console.log('▢️ Fallback video playback started'); setIsStreaming(true); }).catch(console.error); } }, 2000); } else { console.error('❌ Video element not available'); onError('Video element not available'); } } catch (err) { console.error('❌ Camera access error:', err); onError(`Unable to access camera: ${err.message}`); } }; // Stop camera stream const stopCamera = () => { setIsStreaming(false); stopDetection(); if (streamRef.current) { streamRef.current.getTracks().forEach(track => track.stop()); streamRef.current = null; } if (videoRef.current) { videoRef.current.srcObject = null; } }; // Auto-start detection when camera starts useEffect(() => { if (isStreaming && !isDetecting) { // Small delay to let camera stabilize setTimeout(() => { startDetection(); }, 1000); } }, [isStreaming]); // Cleanup on unmount useEffect(() => { return () => { stopCamera(); }; }, []); return (
{scanNotice && (
{scanNotice}
)} {/* Camera Feed Container */}
{/* Video Element - Always rendered but visibility controlled */}
{/* Hidden canvases for image processing */} {/* Detection Info Panel - Only show when streaming */} {isStreaming && (
🎯

Smart Detection Active

● Shape recognition
● Server identification
● Position tracking
● Database lookup
)} {disambiguation && (

Which card is this?

{disambiguation.message || 'Multiple matches found. Select the correct printing.'}

{disambiguation.visionHint && (

Vision detected set: {disambiguation.visionHint}

)}
{disambiguation.candidates.map((candidate) => ( ))}
)}
); }