import { useState, useEffect, useRef } from 'react'; import { aiCardOCR, ollamaCardOCR, puterCardOCR, geminiCardOCR } from '../lib/ai-ocr'; export default function CameraScanner({ onCardScanned, onError }) { const [isStreaming, setIsStreaming] = useState(false); const [isProcessing, setIsProcessing] = useState(false); const [scanResult, setScanResult] = useState(null); const [capturedImage, setCapturedImage] = useState(null); const [isAutoScanning, setIsAutoScanning] = useState(true); const [detectedCard, setDetectedCard] = useState(null); const [scanningAnimation, setScanningAnimation] = useState(false); const [toast, setToast] = useState(null); const [isDetecting, setIsDetecting] = useState(false); const [ocrSettings, setOcrSettings] = useState({ service: 'gemini', // Default to Gemini openaiApiKey: '', geminiApiKey: '', ollamaUrl: 'http://localhost:11434' }); const videoRef = useRef(null); const canvasRef = useRef(null); const detectionCanvasRef = useRef(null); // Separate canvas for computer vision const streamRef = useRef(null); const autoScanIntervalRef = useRef(null); const detectionIntervalRef = useRef(null); const lastScanTimeRef = useRef(0); const lastDetectionTimeRef = useRef(0); const detectionHistoryRef = useRef([]); // Track detection stability const stableDetectionRef = useRef(null); // Current stable detection // Load OCR settings from localStorage useEffect(() => { const loadOcrSettings = async () => { let settings = { service: 'gemini', // Default to Gemini openaiApiKey: '', geminiApiKey: '', ollamaUrl: 'http://localhost:11434' }; // Load saved settings const savedSettings = localStorage.getItem('ocrSettings'); if (savedSettings) { try { const parsed = JSON.parse(savedSettings); settings = { ...settings, ...parsed }; } catch (error) { console.error('Failed to load OCR settings:', error); } } // Try to auto-load Gemini API key from environment if not already set if (!settings.geminiApiKey) { try { const response = await fetch('/api/config/gemini'); if (response.ok) { const data = await response.json(); if (data.hasKey && data.apiKey) { settings.geminiApiKey = data.apiKey; settings.service = 'gemini'; // Ensure Gemini is selected console.log('βœ… Auto-configured Gemini API key from environment'); } } } catch (error) { console.log('Could not auto-load Gemini API key:', error); } } setOcrSettings(settings); // Configure AI services if (settings.openaiApiKey) { aiCardOCR.setApiKey(settings.openaiApiKey); } if (settings.geminiApiKey) { geminiCardOCR.setApiKey(settings.geminiApiKey); } if (settings.ollamaUrl) { ollamaCardOCR.setBaseUrl(settings.ollamaUrl); } }; loadOcrSettings(); }, []); // Configure canvas contexts for optimal performance useEffect(() => { if (canvasRef.current) { const ctx = canvasRef.current.getContext('2d', { willReadFrequently: true }); } if (detectionCanvasRef.current) { const ctx = detectionCanvasRef.current.getContext('2d', { willReadFrequently: true }); } }, []); // Computer vision-based card detection (fast, no API calls) const detectCardWithComputerVision = () => { if (!videoRef.current || !detectionCanvasRef.current) return null; const video = videoRef.current; const canvas = detectionCanvasRef.current; const ctx = canvas.getContext('2d'); // Set canvas size for detection (smaller for performance) canvas.width = 320; canvas.height = 240; // Draw current video frame ctx.drawImage(video, 0, 0, canvas.width, canvas.height); // Get image data for analysis const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); const data = imageData.data; // Card detection algorithm using computer vision const cardDetection = analyzeImageForCard(data, canvas.width, canvas.height); return cardDetection; }; // Computer vision algorithm to detect rectangular card-like objects const analyzeImageForCard = (imageData, width, height) => { // Convert to grayscale and detect edges const grayscale = []; const edges = []; // Convert to grayscale for (let i = 0; i < imageData.length; i += 4) { const gray = Math.round(0.299 * imageData[i] + 0.587 * imageData[i + 1] + 0.114 * imageData[i + 2]); grayscale.push(gray); } // Enhanced edge detection with stronger thresholds for (let y = 1; y < height - 1; y++) { for (let x = 1; x < width - 1; x++) { const idx = y * width + x; // Horizontal gradient const gx = -grayscale[idx - width - 1] - 2 * grayscale[idx - 1] - grayscale[idx + width - 1] + grayscale[idx - width + 1] + 2 * grayscale[idx + 1] + grayscale[idx + width + 1]; // Vertical gradient const gy = -grayscale[idx - width - 1] - 2 * grayscale[idx - width] - grayscale[idx - width + 1] + grayscale[idx + width - 1] + 2 * grayscale[idx + width] + grayscale[idx + width + 1]; // Edge magnitude const magnitude = Math.sqrt(gx * gx + gy * gy); edges[idx] = magnitude > 80 ? 255 : 0; // Increased threshold from 50 to 80 } } // Look for rectangular regions with stricter criteria const cardCandidates = findRectangularRegions(edges, width, height); // Score candidates based on card-like properties with higher standards const bestCandidate = scoreCardCandidates(cardCandidates, width, height); return bestCandidate; }; // Find rectangular regions that could be cards with stricter criteria const findRectangularRegions = (edges, width, height) => { const candidates = []; const minCardWidth = Math.floor(width * 0.2); // Increased from 15% to 20% const maxCardWidth = Math.floor(width * 0.7); // Decreased from 80% to 70% const minCardHeight = Math.floor(height * 0.25); // Increased from 20% to 25% const maxCardHeight = Math.floor(height * 0.8); // Decreased from 90% to 80% // Scan for edge-dense rectangular regions with larger steps for performance for (let y = 0; y < height - minCardHeight; y += 15) { // Increased step from 10 to 15 for (let x = 0; x < width - minCardWidth; x += 15) { // Increased step from 10 to 15 for (let w = minCardWidth; w <= maxCardWidth && x + w < width; w += 25) { // Increased step from 20 to 25 for (let h = minCardHeight; h <= maxCardHeight && y + h < height; h += 25) { // Increased step from 20 to 25 // Stricter card-like aspect ratio check const aspectRatio = w / h; if (aspectRatio < 0.65 || aspectRatio > 0.77) continue; // Narrowed from 0.6-0.8 to 0.65-0.77 // Count edges in this region with more selective sampling let edgeCount = 0; let totalPixels = 0; let cornerEdges = 0; // Count edges near corners (cards have defined corners) // Sample the region (not every pixel for performance) for (let sy = y; sy < y + h; sy += 4) { // Increased step from 3 to 4 for (let sx = x; sx < x + w; sx += 4) { // Increased step from 3 to 4 const idx = sy * width + sx; if (edges[idx] === 255) { edgeCount++; // Check if this edge is near a corner (cards have distinct corners) const isNearCorner = ( (sx < x + w * 0.2 && sy < y + h * 0.2) || // Top-left (sx > x + w * 0.8 && sy < y + h * 0.2) || // Top-right (sx < x + w * 0.2 && sy > y + h * 0.8) || // Bottom-left (sx > x + w * 0.8 && sy > y + h * 0.8) // Bottom-right ); if (isNearCorner) cornerEdges++; } totalPixels++; } } const edgeDensity = edgeCount / totalPixels; const cornerEdgeDensity = cornerEdges / (edgeCount || 1); // Stricter criteria: cards should have moderate edge density AND corner definition if (edgeDensity > 0.08 && edgeDensity < 0.25 && cornerEdgeDensity > 0.1) { // Increased min from 0.05 to 0.08, added corner requirement // Additional check: look for rectangular perimeter (cards have clear borders) const perimeterStrength = checkRectangularPerimeter(edges, x, y, w, h, width); if (perimeterStrength > 0.3) { // Only accept if perimeter is well-defined candidates.push({ x: x, y: y, width: w, height: h, edgeDensity: edgeDensity, aspectRatio: aspectRatio, cornerEdgeDensity: cornerEdgeDensity, perimeterStrength: perimeterStrength }); } } } } } } return candidates; }; // Check if the region has a well-defined rectangular perimeter (like a card border) const checkRectangularPerimeter = (edges, x, y, w, h, frameWidth) => { let perimeterEdges = 0; let perimeterPixels = 0; // Check top and bottom edges for (let sx = x; sx < x + w; sx += 2) { // Top edge const topIdx = y * frameWidth + sx; if (topIdx >= 0 && topIdx < edges.length) { if (edges[topIdx] === 255) perimeterEdges++; perimeterPixels++; } // Bottom edge const bottomIdx = (y + h - 1) * frameWidth + sx; if (bottomIdx >= 0 && bottomIdx < edges.length) { if (edges[bottomIdx] === 255) perimeterEdges++; perimeterPixels++; } } // Check left and right edges for (let sy = y; sy < y + h; sy += 2) { // Left edge const leftIdx = sy * frameWidth + x; if (leftIdx >= 0 && leftIdx < edges.length) { if (edges[leftIdx] === 255) perimeterEdges++; perimeterPixels++; } // Right edge const rightIdx = sy * frameWidth + (x + w - 1); if (rightIdx >= 0 && rightIdx < edges.length) { if (edges[rightIdx] === 255) perimeterEdges++; perimeterPixels++; } } return perimeterPixels > 0 ? perimeterEdges / perimeterPixels : 0; }; // Score and select the best card candidate with higher standards const scoreCardCandidates = (candidates, frameWidth, frameHeight) => { if (candidates.length === 0) return null; let bestCandidate = null; let bestScore = 0; for (const candidate of candidates) { let score = 0; // Prefer candidates with good aspect ratio (closer to 0.71 - typical card ratio) const aspectRatioScore = 1 - Math.abs(candidate.aspectRatio - 0.71); score += aspectRatioScore * 35; // Increased weight from 30 to 35 // Prefer moderate edge density (not too sparse, not too dense) const edgeDensityScore = Math.min(candidate.edgeDensity * 100, 25); score += edgeDensityScore; // Reward corner definition (cards have clear corners) const cornerScore = candidate.cornerEdgeDensity * 25; score += cornerScore; // Reward strong perimeter (cards have borders) const perimeterScore = candidate.perimeterStrength * 20; score += perimeterScore; // Prefer cards that are reasonably sized const cardArea = (candidate.width / frameWidth) * (candidate.height / frameHeight); const sizeScore = cardArea > 0.05 && cardArea < 0.4 ? 15 : 0; // Reward reasonable size score += sizeScore; // Prefer cards closer to center (people usually center cards when scanning) const centerX = frameWidth / 2; const centerY = frameHeight / 2; const cardCenterX = candidate.x + candidate.width / 2; const cardCenterY = candidate.y + candidate.height / 2; const distanceFromCenter = Math.sqrt( Math.pow(cardCenterX - centerX, 2) + Math.pow(cardCenterY - centerY, 2) ); const maxDistance = Math.sqrt(Math.pow(centerX, 2) + Math.pow(centerY, 2)); const centerScore = (1 - distanceFromCenter / maxDistance) * 10; // Reduced weight from 15 to 10 score += centerScore; if (score > bestScore) { bestScore = score; bestCandidate = candidate; } } // Much higher threshold - only return candidate if it's very likely to be a card return bestScore > 70 ? bestCandidate : null; // Increased from 40 to 70 }; // Convert canvas coordinates to video coordinates (percentage) const convertToVideoCoordinates = (candidate) => { if (!candidate || !detectionCanvasRef.current || !videoRef.current) return null; const canvas = detectionCanvasRef.current; const video = videoRef.current; // Convert from detection canvas coordinates to video percentage return { x: (candidate.x / canvas.width) * 100, y: (candidate.y / canvas.height) * 100, width: (candidate.width / canvas.width) * 100, height: (candidate.height / canvas.height) * 100 }; }; // Fast computer vision detection effect useEffect(() => { if (isStreaming && isAutoScanning && !isProcessing) { // Run computer vision detection every 500ms (fast, no API calls) detectionIntervalRef.current = setInterval(() => { const candidate = detectCardWithComputerVision(); // Add to detection history for stability tracking const now = Date.now(); detectionHistoryRef.current.push({ timestamp: now, detected: !!candidate, candidate: candidate }); // Keep only last 6 detections (3 seconds of history at 500ms intervals) detectionHistoryRef.current = detectionHistoryRef.current.filter( detection => now - detection.timestamp < 3000 ); // Check for stable detection (at least 4 out of last 6 detections must be positive) const recentDetections = detectionHistoryRef.current.slice(-6); const positiveDetections = recentDetections.filter(d => d.detected).length; const isStableDetection = recentDetections.length >= 4 && positiveDetections >= 4; if (isStableDetection && candidate) { // Only update if this is a new stable detection or significantly different position const shouldUpdate = !stableDetectionRef.current || Math.abs(candidate.x - stableDetectionRef.current.x) > 10 || Math.abs(candidate.y - stableDetectionRef.current.y) > 10; if (shouldUpdate) { const videoCoords = convertToVideoCoordinates(candidate); if (videoCoords) { setDetectedCard(videoCoords); stableDetectionRef.current = candidate; } } } else if (!isStableDetection) { // Clear detection if not stable setDetectedCard(null); stableDetectionRef.current = null; } }, 500); // Run AI analysis only when card is detected and stable for 3 seconds autoScanIntervalRef.current = setInterval(() => { const now = Date.now(); if (detectedCard && now - lastScanTimeRef.current > 3000) { processDetectedCard(); } }, 1000); } else { if (detectionIntervalRef.current) { clearInterval(detectionIntervalRef.current); detectionIntervalRef.current = null; } if (autoScanIntervalRef.current) { clearInterval(autoScanIntervalRef.current); autoScanIntervalRef.current = null; } // Clear detection history when not scanning detectionHistoryRef.current = []; stableDetectionRef.current = null; } return () => { if (detectionIntervalRef.current) { clearInterval(detectionIntervalRef.current); } if (autoScanIntervalRef.current) { clearInterval(autoScanIntervalRef.current); } }; }, [isStreaming, isAutoScanning, isProcessing, detectedCard]); // Show toast notification const showToast = (message, type = 'success') => { setToast({ message, type }); setTimeout(() => setToast(null), 3000); }; // Start camera stream with card aspect ratio const startCamera = async () => { try { const stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'environment', // Use back camera on mobile width: { ideal: 1280 }, height: { ideal: 720 }, // 16:9 aspect ratio, good for cards aspectRatio: { ideal: 16/9 } } }); if (videoRef.current) { videoRef.current.srcObject = stream; streamRef.current = stream; // Wait for video to be ready videoRef.current.onloadedmetadata = () => { videoRef.current?.play().then(() => { setIsStreaming(true); }).catch((err) => { onError(`Video playback failed: ${err.message}`); }); }; videoRef.current.onerror = (err) => { onError('Video element error occurred'); }; } else { onError('Video element not available'); } } catch (err) { console.error('Camera access error:', err); onError(`Unable to access camera: ${err.message}`); } }; // Stop camera stream const stopCamera = () => { if (streamRef.current) { streamRef.current.getTracks().forEach(track => track.stop()); streamRef.current = null; } if (autoScanIntervalRef.current) { clearInterval(autoScanIntervalRef.current); autoScanIntervalRef.current = null; } if (detectionIntervalRef.current) { clearInterval(detectionIntervalRef.current); detectionIntervalRef.current = null; } setIsStreaming(false); setCapturedImage(null); setScanResult(null); setDetectedCard(null); setIsDetecting(false); // Clear detection state detectionHistoryRef.current = []; stableDetectionRef.current = null; }; // Process the detected card with full AI analysis const processDetectedCard = async () => { if (!detectedCard || isProcessing) return; lastScanTimeRef.current = Date.now(); setScanningAnimation(true); // Capture image const imageDataUrl = captureImageData(); if (imageDataUrl) { await processImage(imageDataUrl, true); // true for auto-scan } setTimeout(() => setScanningAnimation(false), 2000); }; // Capture image from video stream const captureImageData = () => { if (!videoRef.current || !canvasRef.current) return null; const video = videoRef.current; const canvas = canvasRef.current; const ctx = canvas.getContext('2d'); // Set canvas dimensions to match video canvas.width = video.videoWidth; canvas.height = video.videoHeight; // Draw current video frame to canvas ctx?.drawImage(video, 0, 0, canvas.width, canvas.height); // Get image data URL return canvas.toDataURL('image/jpeg', 0.8); }; // Manual capture for testing const captureImage = () => { const imageDataUrl = captureImageData(); if (imageDataUrl) { setCapturedImage(imageDataUrl); processImage(imageDataUrl, false); } }; // Process image with AI OCR (only called when computer vision detects a card) const processImage = async (imageData, isAutoScan = false) => { setIsProcessing(true); setScanResult(null); try { let ocrResult; console.log('πŸ” Current OCR settings:', ocrSettings); if (ocrSettings.service === 'puter') { console.log('🎯 Using Puter.js (Free AI Vision)...'); ocrResult = await puterCardOCR.analyzeCard(imageData); console.log('βœ… Puter.js Vision result:', ocrResult); } else if (ocrSettings.service === 'openai') { console.log('πŸ€– Using OpenAI Vision API...'); ocrResult = await aiCardOCR.analyzeCard(imageData); console.log('βœ… OpenAI Vision result:', ocrResult); } else if (ocrSettings.service === 'ollama') { console.log('πŸ¦™ Using Ollama Vision...'); ocrResult = await ollamaCardOCR.analyzeCard(imageData); console.log('βœ… Ollama Vision result:', ocrResult); } else if (ocrSettings.service === 'gemini') { console.log('πŸ€– Using Gemini Vision API...'); ocrResult = await geminiCardOCR.analyzeCard(imageData); console.log('βœ… Gemini Vision result:', ocrResult); } else { throw new Error('No OCR service configured'); } // First check: Is this actually a trading card? if (!ocrResult.isCard) { if (isAutoScan) { console.log('❌ No trading card detected in auto-scan, continuing...'); setDetectedCard(null); // Remove highlight return; } else { // Show error for manual scans onError(ocrResult.reason || 'No trading card detected in image'); return; } } // Second check: Do we have a card name? if (!ocrResult.cardName || ocrResult.cardName.trim().length < 2) { if (isAutoScan) { console.log('❌ Card name not clear in auto-scan, continuing...'); setDetectedCard(null); return; } else { onError('Could not read card name clearly. Please try again with better lighting.'); return; } } // Third check: Cross-reference with database console.log('πŸ” Cross-referencing with database...'); const dbResponse = await fetch('/api/cards/find-or-create', { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${localStorage.getItem('auth_token')}` }, body: JSON.stringify({ name: ocrResult.cardName.trim(), set: ocrResult.setName, setCode: ocrResult.setCode, cardNumber: ocrResult.cardNumber, game: ocrResult.game, cardType: ocrResult.cardType, rarity: ocrResult.rarity, hp: ocrResult.hp, manaCost: ocrResult.manaCost, ocrData: { confidence: ocrResult.confidence, rawText: ocrResult.rawText, abilities: ocrResult.abilities, flavorText: ocrResult.flavorText, artist: ocrResult.artist } }) }); if (!dbResponse.ok) { throw new Error(`Database lookup failed: ${dbResponse.status}`); } const dbResult = await dbResponse.json(); // Handle different database response scenarios if (dbResult.needsUserSelection) { // Multiple matches found - let user choose if (isAutoScan) { showToast(`⚠️ Multiple matches for "${ocrResult.cardName}" - use manual mode to select`, 'warning'); setDetectedCard(null); return; } else { // Show selection modal for manual scans setScanResult({ type: 'multipleMatches', cardName: ocrResult.cardName, matches: dbResult.matches, confidence: ocrResult.confidence }); return; } } if (dbResult.needsUserInput) { // Low confidence or unclear card if (isAutoScan) { console.log('❌ Low confidence match, skipping auto-scan'); setDetectedCard(null); return; } else { onError(dbResult.message || 'Card not found in database and confidence is too low'); return; } } // Success! We have a confirmed card match const finalCard = dbResult.card; const cleanCardName = finalCard.name; setScanResult({ type: 'success', cardName: cleanCardName, setName: finalCard.set_name, game: finalCard.game, rarity: finalCard.rarity, confidence: ocrResult.confidence, isExisting: dbResult.isExisting, message: dbResult.message }); // Show success toast for auto-scan if (isAutoScan) { showToast(`πŸ“Έ Captured: ${cleanCardName}`, 'success'); setDetectedCard(null); // Remove highlight after successful scan } // Send the confirmed card data to parent onCardScanned({ name: cleanCardName, set: finalCard.set_name, setCode: finalCard.set_code, cardNumber: finalCard.card_number, game: finalCard.game, cardType: finalCard.card_type, rarity: finalCard.rarity, hp: finalCard.hp || ocrResult.hp, manaCost: finalCard.mana_cost || ocrResult.manaCost, abilities: ocrResult.abilities, ocrText: ocrResult.rawText, confidence: ocrResult.confidence, capturedImage: imageData, image_url: finalCard.image_url, // Add the database card image databaseId: finalCard.id, isExisting: dbResult.isExisting }); } catch (error) { console.error('Card processing error:', error); if (!isAutoScan) { onError(`Card processing failed: ${error.message}`); } else { // For auto-scans, just continue silently setDetectedCard(null); } } finally { setIsProcessing(false); } }; // Cleanup on unmount useEffect(() => { return () => { stopCamera(); }; }, []); return (
{/* Camera Controls */}
{!isStreaming ? ( ) : ( <> )}
{/* Camera Preview with Card-like Aspect Ratio */}
); }