import { useState, useEffect, useRef } from 'react'; import { rateLimitCooldownUntil, uploadScanCapture } from '../lib/scan-capture-upload.js'; import { detectCardShapesFromFrame, mergeDetectedShapesIntoTrackedCards, } from '../lib/scanner-card-detection.js'; import ScanDisambiguationDialog from './ScanDisambiguationDialog.js'; export default function CameraScanner({ onCardScanned, onError }) { const [isStreaming, setIsStreaming] = useState(false); const [isDetecting, setIsDetecting] = useState(false); const [disambiguation, setDisambiguation] = useState(null); const [scanNotice, setScanNotice] = useState(null); const [submittingReview, setSubmittingReview] = useState(false); const videoRef = useRef(null); const canvasRef = useRef(null); const detectionCanvasRef = useRef(null); const streamRef = useRef(null); const detectionIntervalRef = useRef(null); const trackingIntervalRef = useRef(null); // Card tracking state const [trackedCards, setTrackedCards] = useState([]); // Array of tracked card objects const trackedCardsRef = useRef([]); const nextCardIdRef = useRef(1); const visionCooldownUntilRef = useRef(0); const activeVerificationRef = useRef(0); const lastErrorAtRef = useRef(0); const disambiguationRefineRef = useRef(null); // Mana symbol settings const [manaSymbolSettings, setManaSymbolSettings] = useState({ useSVG: false }); const [videoMetrics, setVideoMetrics] = useState({ width: 0, height: 0 }); // Configure canvas contexts for optimal performance useEffect(() => { if (canvasRef.current) { const ctx = canvasRef.current.getContext('2d', { willReadFrequently: true }); } if (detectionCanvasRef.current) { const ctx = detectionCanvasRef.current.getContext('2d', { willReadFrequently: true }); } }, []); // Continuous shape detection for card-like rectangles const detectCardShapes = () => { if (!videoRef.current || !detectionCanvasRef.current || !isStreaming) return []; return detectCardShapesFromFrame(videoRef.current, detectionCanvasRef.current); }; const updateTrackedCards = (detectedShapes) => { const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards( trackedCardsRef.current, detectedShapes, nextCardIdRef.current ); nextCardIdRef.current = nextCardId; trackedCardsRef.current = cards; setTrackedCards(cards); }; const emitScannedCard = async (cardTracker, imageData, finalCard, ocrMeta = {}) => { cardTracker.status = 'scanned'; const originalTitle = document.title; document.title = `📸 ${finalCard.name} - Card Scanner`; setTimeout(() => { document.title = originalTitle; }, 3000); let scanImageUrl = null; if (imageData) { try { scanImageUrl = await uploadScanCapture(imageData); } catch (uploadError) { console.warn('Scan image upload failed:', uploadError); } } onCardScanned({ name: finalCard.name, set: finalCard.set_name, setCode: finalCard.set_code, cardNumber: finalCard.card_number, game: finalCard.game, cardType: finalCard.card_type, rarity: finalCard.rarity, hp: finalCard.hp || ocrMeta.hp, manaCost: finalCard.mana_cost || ocrMeta.manaCost, abilities: ocrMeta.abilities || [], ocrText: ocrMeta.rawText, confidence: ocrMeta.confidence, capturedImage: imageData, scanImageUrl, image_url: finalCard.image_url, databaseId: finalCard.id, isExisting: true, }); }; const handleDisambiguationPick = async (candidate) => { if (!disambiguation) return; const { cardTracker, imageData, ocrMeta } = disambiguation; await emitScannedCard(cardTracker, imageData, candidate, ocrMeta); setDisambiguation(null); disambiguationRefineRef.current = null; }; const showScanNotice = (message) => { setScanNotice(message); setTimeout(() => setScanNotice(null), 8000); }; const handleReviewSubmitted = (cardTracker, message) => { if (cardTracker) cardTracker.status = 'confirmed'; setDisambiguation(null); disambiguationRefineRef.current = null; showScanNotice(message); }; const handleNotInCatalog = async () => { if (!disambiguation || submittingReview) return; setSubmittingReview(true); const { cardTracker, imageData, candidates, ocrMeta } = disambiguation; const guessedName = ocrMeta?.cardName || ocrMeta?.query || candidates?.[0]?.name || null; try { const response = await fetch('/api/scan/submit-for-review', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('auth_token')}`, }, body: JSON.stringify({ imageData, name: guessedName, candidateCardIds: (candidates || []).map((c) => c.id), }), }); if (response.status === 429) { visionCooldownUntilRef.current = rateLimitCooldownUntil(60_000); reportScannerError('Too many scan attempts. Please wait a moment and try again.'); return; } if (!response.ok) { const errBody = await response.json().catch(() => ({})); throw new Error(errBody.error || 'Failed to submit scan for review'); } const result = await response.json(); handleReviewSubmitted(cardTracker, result.message); } catch (error) { reportScannerError(error.message || 'Failed to submit scan for review'); } finally { setSubmittingReview(false); } }; const candidateMatchesSetHint = (candidate, setName, setCode) => { if (!setName && !setCode) return true; const hint = (setName || setCode || '').toLowerCase(); const setNameLower = (candidate.set_name || '').toLowerCase(); const setCodeLower = (candidate.set_code || '').toLowerCase(); return setNameLower.includes(hint) || hint.includes(setNameLower) || setCodeLower === hint; }; const processIdentifyResponse = async (cardTracker, imageData, result) => { if (!result.isCard) { cardTracker.status = 'negative'; reportScannerError(result.reason || 'No trading card detected'); return; } const ocrMeta = { confidence: result.ocr?.confidence ?? result.card?.ocr?.confidence, rawText: result.ocr?.rawText ?? result.card?.ocr?.rawText, abilities: result.card?.ocr?.abilities || [], hp: result.card?.hp, manaCost: result.card?.mana_cost, }; if (result.card) { cardTracker.status = 'confirmed'; await emitScannedCard(cardTracker, imageData, result.card, ocrMeta); return; } if (result.needsUserSelection && result.matches?.length) { cardTracker.status = 'confirmed'; setDisambiguation({ cardTracker, imageData, candidates: result.matches, ocrMeta, message: result.message, }); return; } if (result.needsReview) { cardTracker.status = 'confirmed'; showScanNotice(result.message || 'Scan saved for admin review.'); return; } if (result.needsUserInput) { cardTracker.status = 'negative'; reportScannerError(result.message || 'Could not identify card — try again or submit for review.'); return; } cardTracker.status = 'negative'; reportScannerError('Could not identify card from scan.'); }; const reportScannerError = (message) => { if (disambiguation) return; const now = Date.now(); if (now - lastErrorAtRef.current < 4000) return; lastErrorAtRef.current = now; onError?.(message); }; useEffect(() => { if (!disambiguation?.imageData) return; if (Date.now() < visionCooldownUntilRef.current) return; const refineKey = disambiguation.cardTracker?.id ?? 'modal'; if (disambiguationRefineRef.current === refineKey) return; disambiguationRefineRef.current = refineKey; let cancelled = false; (async () => { try { const response = await fetch('/api/scan/identify', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('auth_token')}`, }, body: JSON.stringify({ imageData: disambiguation.imageData }), }); if (cancelled) return; if (response.status === 429) { visionCooldownUntilRef.current = Date.now() + 60_000; return; } if (!response.ok) return; const result = await response.json(); if (cancelled) return; if (result.needsReview) { handleReviewSubmitted(disambiguation.cardTracker, result.message); return; } if (result.card) { emitScannedCard( disambiguation.cardTracker, disambiguation.imageData, result.card, { confidence: result.ocr?.confidence, rawText: result.ocr?.rawText, abilities: result.card?.ocr?.abilities || [], } ); setDisambiguation(null); disambiguationRefineRef.current = null; return; } const setHint = result.ocr?.setName || result.ocr?.setCode; if (result.matches?.length && setHint) { const filtered = disambiguation.candidates.filter((candidate) => candidateMatchesSetHint(candidate, result.ocr.setName, result.ocr.setCode) ); if (filtered.length === 0 && setHint) { try { const submitRes = await fetch('/api/scan/submit-for-review', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('auth_token')}`, }, body: JSON.stringify({ imageData: disambiguation.imageData, name: result.ocr?.cardName || disambiguation.candidates[0]?.name, candidateCardIds: disambiguation.candidates.map((c) => c.id), }), }); if (submitRes.ok) { const submitResult = await submitRes.json(); handleReviewSubmitted(disambiguation.cardTracker, submitResult.message); } } catch { setDisambiguation((current) => current ? { ...current, visionHint: setHint, message: `No "${setHint}" printing in our catalog. Tap "My card isn't listed" to save for admin review.`, } : current ); } return; } if (filtered.length === 1) { handleDisambiguationPick(filtered[0]); return; } if (filtered.length > 1 && filtered.length < disambiguation.candidates.length) { setDisambiguation((current) => current ? { ...current, candidates: filtered, message: `Narrowed to ${filtered.length} printings matching "${setHint}".`, visionHint: setHint, } : current ); } } } catch (error) { console.warn('Disambiguation vision refine failed:', error); } })(); return () => { cancelled = true; }; // eslint-disable-next-line react-hooks/exhaustive-deps -- refine runs per disambiguation session, not per handler identity }, [disambiguation?.cardTracker?.id, disambiguation?.imageData]); // Server-side card identification const verifyCardShape = async (cardTracker) => { if (!videoRef.current || !canvasRef.current || cardTracker.status !== 'detecting') return; if (disambiguation) return; if (activeVerificationRef.current >= 1) return; activeVerificationRef.current += 1; cardTracker.status = 'verifying'; try { cardTracker.scanAttempts++; const video = videoRef.current; const canvas = canvasRef.current; const ctx = canvas.getContext('2d'); const { x, y, width, height } = cardTracker.bounds; const margin = 20; canvas.width = width + margin * 2; canvas.height = height + margin * 2; ctx.drawImage( video, Math.max(0, x - margin), Math.max(0, y - margin), width + margin * 2, height + margin * 2, 0, 0, canvas.width, canvas.height ); const imageData = canvas.toDataURL('image/jpeg', 0.8); const authHeaders = { 'Content-Type': 'application/json', Authorization: `Bearer ${localStorage.getItem('auth_token')}`, }; // Layer 1: local OCR + pg_trgm catalog match (no vision LLM) try { const { recognizeCardNameStrip } = await import('../lib/ocr-worker.js'); const ocr = await recognizeCardNameStrip(imageData); if (ocr.text.length >= 3) { const l1Response = await fetch('/api/cards/identify-by-text', { method: 'POST', headers: authHeaders, body: JSON.stringify({ ocrText: ocr.text, ocrConfidence: ocr.confidence, }), }); if (l1Response.ok) { const l1Result = await l1Response.json(); if (!l1Result.escalate) { cardTracker.status = 'confirmed'; await processIdentifyResponse(cardTracker, imageData, l1Result); return; } } } } catch (l1Error) { console.warn('Layer-1 OCR path failed, escalating to vision:', l1Error); } // Layer 2: vision via AI Gateway (skip while rate-limited) if (Date.now() < visionCooldownUntilRef.current) { cardTracker.status = 'detecting'; return; } const response = await fetch('/api/scan/identify', { method: 'POST', headers: authHeaders, body: JSON.stringify({ imageData }), }); if (response.status === 429) { visionCooldownUntilRef.current = Date.now() + 60_000; reportScannerError('Too many scan attempts. Please wait a moment and try again.'); cardTracker.status = 'negative'; return; } if (!response.ok) { const errBody = await response.json().catch(() => ({})); throw new Error(errBody.error || `Scan identify failed: ${response.status}`); } const result = await response.json(); cardTracker.status = 'confirmed'; await processIdentifyResponse(cardTracker, imageData, result); } catch (error) { console.error(`Error verifying card ${cardTracker.id}:`, error); cardTracker.status = 'negative'; reportScannerError(error.message || 'Scan failed'); } finally { activeVerificationRef.current = Math.max(0, activeVerificationRef.current - 1); } }; // Start continuous detection const startDetection = () => { if (detectionIntervalRef.current || !isStreaming) return; console.log('🎯 Starting continuous card detection...'); setIsDetecting(true); // Shape detection every 200ms detectionIntervalRef.current = setInterval(() => { const shapes = detectCardShapes(); updateTrackedCards(shapes); }, 200); // Card verification every 1 second trackingIntervalRef.current = setInterval(() => { const cardsToVerify = trackedCardsRef.current.filter(card => card.status === 'detecting' && card.stableCount >= 6 && card.scanAttempts < 1 && Date.now() - card.firstSeen > 2500 ); const cardsToProcess = cardsToVerify.slice(0, 1); cardsToProcess.forEach(card => { verifyCardShape(card); }); }, 1000); // Back to 1 second intervals }; // Stop detection const stopDetection = () => { console.log('🛑 Stopping card detection...'); setIsDetecting(false); if (detectionIntervalRef.current) { clearInterval(detectionIntervalRef.current); detectionIntervalRef.current = null; } if (trackingIntervalRef.current) { clearInterval(trackingIntervalRef.current); trackingIntervalRef.current = null; } // Clear tracked cards trackedCardsRef.current = []; setTrackedCards([]); }; // Start camera stream const startCamera = async () => { try { console.log('🎥 Starting camera...'); const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment', width: { ideal: 1280 }, height: { ideal: 720 }, aspectRatio: { ideal: 16/9 } } }); console.log('📹 Camera stream obtained:', stream); if (videoRef.current) { videoRef.current.srcObject = stream; streamRef.current = stream; videoRef.current.onloadedmetadata = () => { console.log('📺 Video metadata loaded, attempting to play...'); videoRef.current?.play().then(() => { console.log('▶️ Video playback started successfully'); setIsStreaming(true); }).catch((err) => { console.error('❌ Video playback failed:', err); onError(`Video playback failed: ${err.message}`); }); }; videoRef.current.onerror = (err) => { console.error('❌ Video element error:', err); onError('Video element error occurred'); }; // Add a fallback timeout setTimeout(() => { if (!isStreaming && videoRef.current && videoRef.current.readyState >= 2) { console.log('🔄 Fallback: Attempting to play video directly...'); videoRef.current.play().then(() => { console.log('▶️ Fallback video playback started'); setIsStreaming(true); }).catch(console.error); } }, 2000); } else { console.error('❌ Video element not available'); onError('Video element not available'); } } catch (err) { console.error('❌ Camera access error:', err); onError(`Unable to access camera: ${err.message}`); } }; // Stop camera stream const stopCamera = () => { setIsStreaming(false); stopDetection(); if (streamRef.current) { streamRef.current.getTracks().forEach(track => track.stop()); streamRef.current = null; } if (videoRef.current) { videoRef.current.srcObject = null; } }; useEffect(() => { const video = videoRef.current; if (!video || !isStreaming) return undefined; const syncVideoMetrics = () => { setVideoMetrics({ width: video.videoWidth || 0, height: video.videoHeight || 0, }); }; video.addEventListener('loadedmetadata', syncVideoMetrics); video.addEventListener('resize', syncVideoMetrics); syncVideoMetrics(); return () => { video.removeEventListener('loadedmetadata', syncVideoMetrics); video.removeEventListener('resize', syncVideoMetrics); }; }, [isStreaming]); // Auto-start detection when camera starts useEffect(() => { if (isStreaming && !isDetecting) { // Small delay to let camera stabilize const timerId = setTimeout(() => { startDetection(); }, 1000); return () => clearTimeout(timerId); } return undefined; // eslint-disable-next-line react-hooks/exhaustive-deps -- intentional: start detection once per stream session }, [isStreaming]); // Cleanup on unmount useEffect(() => { return () => { stopCamera(); }; // eslint-disable-next-line react-hooks/exhaustive-deps -- run stopCamera only on unmount }, []); return (
{scanNotice && (
{scanNotice}
)} {/* Camera Feed Container */}
{/* Video Element - Always rendered but visibility controlled */}
{/* Hidden canvases for image processing */} {/* Detection Info Panel - Only show when streaming */} {isStreaming && (
🎯

Smart Detection Active

Shape recognition
Server identification
Position tracking
Database lookup
)} { setDisambiguation(null); disambiguationRefineRef.current = null; }} />
); }