import { useState, useEffect, useRef } from 'react'; import { aiCardOCR, ollamaCardOCR, puterCardOCR, geminiCardOCR } from '../lib/ai-ocr'; export default function CameraScanner({ onCardScanned, onError }) { const [isStreaming, setIsStreaming] = useState(false); const [isDetecting, setIsDetecting] = useState(false); const [scanningAnimation, setScanningAnimation] = useState(false); const [scanResult, setScanResult] = useState(null); const [isProcessing, setIsProcessing] = useState(false); const [ocrSettings, setOcrSettings] = useState({ service: 'gemini', openaiApiKey: '', geminiApiKey: '', ollamaUrl: 'http://localhost:11434' }); const videoRef = useRef(null); const canvasRef = useRef(null); const detectionCanvasRef = useRef(null); const streamRef = useRef(null); const detectionIntervalRef = useRef(null); const trackingIntervalRef = useRef(null); // Card tracking state const [trackedCards, setTrackedCards] = useState([]); // Array of tracked card objects const trackedCardsRef = useRef([]); const nextCardIdRef = useRef(1); // Mana symbol settings const [manaSymbolSettings, setManaSymbolSettings] = useState({ useSVG: false }); // Load OCR settings from localStorage only (never fetch server-side API keys) useEffect(() => { let settings = { service: 'gemini', openaiApiKey: '', geminiApiKey: '', ollamaUrl: 'http://localhost:11434' }; const savedSettings = localStorage.getItem('ocrSettings'); if (savedSettings) { try { const parsed = JSON.parse(savedSettings); settings = { ...settings, ...parsed }; } catch (error) { console.error('Failed to load OCR settings:', error); } } setOcrSettings(settings); if (settings.openaiApiKey) { aiCardOCR.setApiKey(settings.openaiApiKey); } if (settings.geminiApiKey) { geminiCardOCR.setApiKey(settings.geminiApiKey); } if (settings.ollamaUrl) { ollamaCardOCR.setBaseUrl(settings.ollamaUrl); } }, []); // Configure canvas contexts for optimal performance useEffect(() => { if (canvasRef.current) { const ctx = canvasRef.current.getContext('2d', { willReadFrequently: true }); } if (detectionCanvasRef.current) { const ctx = detectionCanvasRef.current.getContext('2d', { willReadFrequently: true }); } }, []); // Continuous shape detection for card-like rectangles const detectCardShapes = () => { if (!videoRef.current || !detectionCanvasRef.current || !isStreaming) return []; const video = videoRef.current; const canvas = detectionCanvasRef.current; const ctx = canvas.getContext('2d'); // Set canvas size for detection (smaller for performance) canvas.width = 320; canvas.height = 240; // Draw current video frame ctx.drawImage(video, 0, 0, canvas.width, canvas.height); // Get image data for analysis const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); const data = imageData.data; // Convert to grayscale and detect edges const grayscale = []; const edges = []; for (let i = 0; i < data.length; i += 4) { const gray = Math.round(0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]); grayscale.push(gray); } // Simple edge detection (Sobel-like) for (let y = 1; y < canvas.height - 1; y++) { for (let x = 1; x < canvas.width - 1; x++) { const idx = y * canvas.width + x; const gx = -grayscale[idx - 1] + grayscale[idx + 1] + -2 * grayscale[idx - 1 + canvas.width] + 2 * grayscale[idx + 1 + canvas.width] + -grayscale[idx - 1 + 2 * canvas.width] + grayscale[idx + 1 + 2 * canvas.width]; const gy = -grayscale[idx - canvas.width] - 2 * grayscale[idx] - grayscale[idx + canvas.width] + grayscale[idx - canvas.width + 2 * canvas.width] + 2 * grayscale[idx + 2 * canvas.width] + grayscale[idx + canvas.width + 2 * canvas.width]; const magnitude = Math.sqrt(gx * gx + gy * gy); edges[idx] = magnitude > 100 ? 255 : 0; // Threshold for edge detection } } // Find rectangular regions that could be cards const cardShapes = []; const { width, height } = canvas; // Card aspect ratio constraints (typical trading cards are ~2.5:3.5 ratio) const minCardWidth = Math.floor(width * 0.2); // Increased from 0.15 const maxCardWidth = Math.floor(width * 0.6); // Decreased from 0.8 const minCardHeight = Math.floor(height * 0.25); // Increased from 0.2 const maxCardHeight = Math.floor(height * 0.7); // Decreased from 0.9 // Scan for edge-dense rectangular regions for (let y = 0; y < height - minCardHeight; y += 15) { // Increased step for (let x = 0; x < width - minCardWidth; x += 15) { // Increased step for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 20) { // Increased step for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 20) { // Increased step // Check aspect ratio (cards are typically 0.65-0.75) const aspectRatio = w / h; if (aspectRatio < 0.63 || aspectRatio > 0.77) continue; // Stricter range // Count edges in this region let edgeCount = 0; let totalPixels = 0; let perimeterEdges = 0; // Sample the region for (let sy = y; sy < y + h; sy += 4) { // Increased step for performance for (let sx = x; sx < x + w; sx += 4) { // Increased step for performance const idx = sy * width + sx; if (edges[idx] === 255) { edgeCount++; // Check if this edge is near the perimeter (cards have strong borders) const isPerimeter = (sx < x + w * 0.15 || sx > x + w * 0.85 || sy < y + h * 0.15 || sy > y + h * 0.85); if (isPerimeter) { perimeterEdges++; } } totalPixels++; } } const edgeDensity = edgeCount / totalPixels; const perimeterRatio = perimeterEdges / (edgeCount || 1); // Much stricter criteria for card-like objects if (edgeDensity > 0.2 && edgeDensity < 0.6 && perimeterRatio > 0.4 && edgeCount > 80) { const score = edgeDensity * 100 + perimeterRatio * 60 + (edgeCount / 10); // Higher threshold for accepting shapes if (score > 50) { // Convert back to video coordinates const videoCoords = convertToVideoCoordinates({ x, y, width: w, height: h }, canvas, video); cardShapes.push({ ...videoCoords, score, aspectRatio, edgeDensity, timestamp: Date.now() }); } } } } } } // Sort by score and return top candidates to allow multiple cards return cardShapes.sort((a, b) => b.score - a.score).slice(0, 5); // Allow up to 5 cards simultaneously }; // Convert detection coordinates to video coordinates const convertToVideoCoordinates = (detection, canvas, video) => { const scaleX = video.videoWidth / canvas.width; const scaleY = video.videoHeight / canvas.height; return { x: detection.x * scaleX, y: detection.y * scaleY, width: detection.width * scaleX, height: detection.height * scaleY }; }; // Check if two rectangles overlap significantly const rectanglesOverlap = (rect1, rect2, threshold = 0.3) => { // Reduced default threshold const x1 = Math.max(rect1.x, rect2.x); const y1 = Math.max(rect1.y, rect2.y); const x2 = Math.min(rect1.x + rect1.width, rect2.x + rect2.width); const y2 = Math.min(rect1.y + rect1.height, rect2.y + rect2.height); if (x2 <= x1 || y2 <= y1) return false; const overlapArea = (x2 - x1) * (y2 - y1); const rect1Area = rect1.width * rect1.height; const rect2Area = rect2.width * rect2.height; const smallerArea = Math.min(rect1Area, rect2Area); // Use smaller area as denominator for better tracking of moving cards return (overlapArea / smallerArea) > threshold; }; // Update tracked cards with new detections const updateTrackedCards = (detectedShapes) => { const currentTime = Date.now(); const updatedCards = [...trackedCardsRef.current]; // Remove cards that haven't been seen recently (3 seconds for better responsiveness) for (let i = updatedCards.length - 1; i >= 0; i--) { if (currentTime - updatedCards[i].lastSeen > 3000) { console.log(`🗑️ Removing stale tracked card ${updatedCards[i].id} (last seen ${Math.round((currentTime - updatedCards[i].lastSeen)/1000)}s ago)`); updatedCards.splice(i, 1); } } // Match detected shapes with existing tracked cards detectedShapes.forEach(shape => { let matchedCard = null; // Find existing card that overlaps with this shape for (const card of updatedCards) { if (rectanglesOverlap(shape, card.bounds, 0.4)) { // Slightly more lenient overlap matchedCard = card; break; } } if (matchedCard) { // Update existing card position and timestamp matchedCard.bounds = { ...shape }; matchedCard.lastSeen = currentTime; matchedCard.stableCount = Math.min(matchedCard.stableCount + 1, 10); // Reset scan attempts if card moved significantly (allows re-scanning) const positionChange = Math.abs(matchedCard.bounds.x - shape.x) + Math.abs(matchedCard.bounds.y - shape.y); if (positionChange > 50 && matchedCard.status === 'negative') { console.log(`🔄 Card ${matchedCard.id} moved significantly, allowing re-scan`); matchedCard.status = 'detecting'; matchedCard.scanAttempts = 0; matchedCard.stableCount = 1; } } else { // Create new tracked card const newCard = { id: nextCardIdRef.current++, bounds: { ...shape }, status: 'detecting', // 'detecting', 'confirmed', 'negative', 'scanned' firstSeen: currentTime, lastSeen: currentTime, stableCount: 1, scanAttempts: 0 }; updatedCards.push(newCard); console.log(`🎯 New card shape detected: ${newCard.id}`); } }); trackedCardsRef.current = updatedCards; setTrackedCards(updatedCards); }; // Quick card verification using AI const verifyCardShape = async (cardTracker) => { if (!videoRef.current || !canvasRef.current || cardTracker.status !== 'detecting') return; try { console.log(`🔍 Verifying card ${cardTracker.id}...`); cardTracker.scanAttempts++; const video = videoRef.current; const canvas = canvasRef.current; const ctx = canvas.getContext('2d'); // Capture the specific region const { x, y, width, height } = cardTracker.bounds; const margin = 20; // Add some margin around the detected area canvas.width = width + margin * 2; canvas.height = height + margin * 2; // Draw the card region with margin ctx.drawImage( video, Math.max(0, x - margin), Math.max(0, y - margin), width + margin * 2, height + margin * 2, 0, 0, canvas.width, canvas.height ); const imageData = canvas.toDataURL('image/jpeg', 0.8); // Quick AI verification let ocrResult; console.log('🔍 Current OCR settings:', ocrSettings); if (ocrSettings.service === 'puter') { console.log('🎯 Using Puter.js (Free AI Vision)...'); ocrResult = await puterCardOCR.analyzeCard(imageData); console.log('✅ Puter.js Vision result:', ocrResult); } else if (ocrSettings.service === 'openai') { console.log('🤖 Using OpenAI Vision API...'); ocrResult = await aiCardOCR.analyzeCard(imageData); console.log('✅ OpenAI Vision result:', ocrResult); } else if (ocrSettings.service === 'ollama') { console.log('🦙 Using Ollama Vision...'); ocrResult = await ollamaCardOCR.analyzeCard(imageData); console.log('✅ Ollama Vision result:', ocrResult); } else if (ocrSettings.service === 'gemini') { console.log('🤖 Using Gemini Vision API...'); ocrResult = await geminiCardOCR.analyzeCard(imageData); console.log('✅ Gemini Vision result:', ocrResult); } else { throw new Error('No OCR service configured'); } if (ocrResult.isCard && ocrResult.confidence > 60) { // Confirmed as a card! cardTracker.status = 'confirmed'; cardTracker.cardData = ocrResult; console.log(`✅ Card ${cardTracker.id} confirmed: ${ocrResult.cardName}`); // Process the card through database lookup await processConfirmedCard(cardTracker, imageData); } else { // Not a card or low confidence cardTracker.status = 'negative'; console.log(`❌ Card ${cardTracker.id} rejected: ${ocrResult.reason || 'Low confidence'}`); } } catch (error) { console.error(`Error verifying card ${cardTracker.id}:`, error); cardTracker.status = 'negative'; } }; // Process confirmed card through database lookup const processConfirmedCard = async (cardTracker, imageData) => { try { const ocrResult = cardTracker.cardData; // Database lookup console.log('🔍 Cross-referencing with database...'); const dbResponse = await fetch('/api/cards/find-or-create', { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${localStorage.getItem('auth_token')}` }, body: JSON.stringify({ name: ocrResult.cardName.trim(), set: ocrResult.setName, setCode: ocrResult.setCode, cardNumber: ocrResult.cardNumber, game: ocrResult.game, cardType: ocrResult.cardType, rarity: ocrResult.rarity, hp: ocrResult.hp, manaCost: ocrResult.manaCost, ocrData: { confidence: ocrResult.confidence, rawText: ocrResult.rawText, abilities: ocrResult.abilities, flavorText: ocrResult.flavorText, artist: ocrResult.artist } }) }); if (!dbResponse.ok) { throw new Error(`Database lookup failed: ${dbResponse.status}`); } const dbResult = await dbResponse.json(); // Handle successful card identification if (dbResult.card) { const finalCard = dbResult.card; // Mark as scanned and send to parent cardTracker.status = 'scanned'; // Update browser tab title with card name const originalTitle = document.title; document.title = `📸 ${finalCard.name} - Card Scanner`; // Reset title after 3 seconds setTimeout(() => { document.title = originalTitle; }, 3000); onCardScanned({ name: finalCard.name, set: finalCard.set_name, setCode: finalCard.set_code, cardNumber: finalCard.card_number, game: finalCard.game, cardType: finalCard.card_type, rarity: finalCard.rarity, hp: finalCard.hp || ocrResult.hp, manaCost: finalCard.mana_cost || ocrResult.manaCost, abilities: ocrResult.abilities, ocrText: ocrResult.rawText, confidence: ocrResult.confidence, capturedImage: imageData, image_url: finalCard.image_url, databaseId: finalCard.id, isExisting: dbResult.isExisting }); console.log(`🎉 Card ${cardTracker.id} successfully scanned: ${finalCard.name}`); } } catch (error) { console.error(`Error processing card ${cardTracker.id}:`, error); cardTracker.status = 'negative'; } }; // Start continuous detection const startDetection = () => { if (detectionIntervalRef.current || !isStreaming) return; console.log('🎯 Starting continuous card detection...'); setIsDetecting(true); // Shape detection every 200ms detectionIntervalRef.current = setInterval(() => { const shapes = detectCardShapes(); updateTrackedCards(shapes); }, 200); // Card verification every 1 second trackingIntervalRef.current = setInterval(() => { const cardsToVerify = trackedCardsRef.current.filter(card => card.status === 'detecting' && card.stableCount >= 4 && // Reduced back to 4 for better responsiveness card.scanAttempts < 2 && // Allow 2 attempts again Date.now() - card.firstSeen > 1500 // Reduced to 1.5 seconds ); // Verify up to 2 cards simultaneously to allow multi-card scanning const cardsToProcess = cardsToVerify.slice(0, 2); cardsToProcess.forEach(card => { verifyCardShape(card); }); }, 1000); // Back to 1 second intervals }; // Stop detection const stopDetection = () => { console.log('🛑 Stopping card detection...'); setIsDetecting(false); if (detectionIntervalRef.current) { clearInterval(detectionIntervalRef.current); detectionIntervalRef.current = null; } if (trackingIntervalRef.current) { clearInterval(trackingIntervalRef.current); trackingIntervalRef.current = null; } // Clear tracked cards trackedCardsRef.current = []; setTrackedCards([]); }; // Start camera stream const startCamera = async () => { try { console.log('🎥 Starting camera...'); const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment', width: { ideal: 1280 }, height: { ideal: 720 }, aspectRatio: { ideal: 16/9 } } }); console.log('📹 Camera stream obtained:', stream); if (videoRef.current) { videoRef.current.srcObject = stream; streamRef.current = stream; videoRef.current.onloadedmetadata = () => { console.log('📺 Video metadata loaded, attempting to play...'); videoRef.current?.play().then(() => { console.log('▶️ Video playback started successfully'); setIsStreaming(true); }).catch((err) => { console.error('❌ Video playback failed:', err); onError(`Video playback failed: ${err.message}`); }); }; videoRef.current.onerror = (err) => { console.error('❌ Video element error:', err); onError('Video element error occurred'); }; // Add a fallback timeout setTimeout(() => { if (!isStreaming && videoRef.current && videoRef.current.readyState >= 2) { console.log('🔄 Fallback: Attempting to play video directly...'); videoRef.current.play().then(() => { console.log('▶️ Fallback video playback started'); setIsStreaming(true); }).catch(console.error); } }, 2000); } else { console.error('❌ Video element not available'); onError('Video element not available'); } } catch (err) { console.error('❌ Camera access error:', err); onError(`Unable to access camera: ${err.message}`); } }; // Stop camera stream const stopCamera = () => { setIsStreaming(false); stopDetection(); if (streamRef.current) { streamRef.current.getTracks().forEach(track => track.stop()); streamRef.current = null; } if (videoRef.current) { videoRef.current.srcObject = null; } }; // Auto-start detection when camera starts useEffect(() => { if (isStreaming && !isDetecting) { // Small delay to let camera stabilize setTimeout(() => { startDetection(); }, 1000); } }, [isStreaming]); // Cleanup on unmount useEffect(() => { return () => { stopCamera(); }; }, []); return (