deckhearth/components/CameraScanner.js
varutasu 7f3cf62344
refactor(scanner): extract card identification pipeline (Brief 3) (#70)
Move Layer-1/Layer-2 identify flow, outcome resolution, disambiguation
refine helpers, and scan-for-review API calls into lib/scanner-card-identify.js.
Remove unused manaSymbolSettings state from CameraScanner.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-02 15:34:13 -05:00

718 lines
25 KiB
JavaScript

import { useState, useEffect, useRef } from 'react';
import { rateLimitCooldownUntil, uploadScanCapture } from '../lib/scan-capture-upload.js';
import {
detectCardShapesFromFrame,
mergeDetectedShapesIntoTrackedCards,
} from '../lib/scanner-card-detection.js';
import {
buildScannedCardPayload,
getScanAuthHeaders,
identifyTrackedCardCapture,
resolveDisambiguationRefineAction,
submitScanForReview,
VISION_RATE_LIMIT_MS,
fetchVisionIdentify,
} from '../lib/scanner-card-identify.js';
import ScanDisambiguationDialog from './ScanDisambiguationDialog.js';
export default function CameraScanner({ onCardScanned, onError }) {
const [isStreaming, setIsStreaming] = useState(false);
const [isDetecting, setIsDetecting] = useState(false);
const [disambiguation, setDisambiguation] = useState(null);
const [scanNotice, setScanNotice] = useState(null);
const [submittingReview, setSubmittingReview] = useState(false);
const videoRef = useRef(null);
const canvasRef = useRef(null);
const detectionCanvasRef = useRef(null);
const streamRef = useRef(null);
const detectionIntervalRef = useRef(null);
const trackingIntervalRef = useRef(null);
// Card tracking state
const [trackedCards, setTrackedCards] = useState([]); // Array of tracked card objects
const trackedCardsRef = useRef([]);
const nextCardIdRef = useRef(1);
const visionCooldownUntilRef = useRef(0);
const activeVerificationRef = useRef(0);
const lastErrorAtRef = useRef(0);
const disambiguationRefineRef = useRef(null);
const [videoMetrics, setVideoMetrics] = useState({ width: 0, height: 0 });
// Configure canvas contexts for optimal performance
useEffect(() => {
if (canvasRef.current) {
const ctx = canvasRef.current.getContext('2d', { willReadFrequently: true });
}
if (detectionCanvasRef.current) {
const ctx = detectionCanvasRef.current.getContext('2d', { willReadFrequently: true });
}
}, []);
// Continuous shape detection for card-like rectangles
const detectCardShapes = () => {
if (!videoRef.current || !detectionCanvasRef.current || !isStreaming) return [];
return detectCardShapesFromFrame(videoRef.current, detectionCanvasRef.current);
};
const updateTrackedCards = (detectedShapes) => {
const { cards, nextCardId } = mergeDetectedShapesIntoTrackedCards(
trackedCardsRef.current,
detectedShapes,
nextCardIdRef.current
);
nextCardIdRef.current = nextCardId;
trackedCardsRef.current = cards;
setTrackedCards(cards);
};
const emitScannedCard = async (cardTracker, imageData, finalCard, ocrMeta = {}) => {
cardTracker.status = 'scanned';
const originalTitle = document.title;
document.title = `📸 ${finalCard.name} - Card Scanner`;
setTimeout(() => {
document.title = originalTitle;
}, 3000);
let scanImageUrl = null;
if (imageData) {
try {
scanImageUrl = await uploadScanCapture(imageData);
} catch (uploadError) {
console.warn('Scan image upload failed:', uploadError);
}
}
onCardScanned(buildScannedCardPayload(finalCard, { imageData, scanImageUrl, ocrMeta }));
};
const handleDisambiguationPick = async (candidate) => {
if (!disambiguation) return;
const { cardTracker, imageData, ocrMeta } = disambiguation;
await emitScannedCard(cardTracker, imageData, candidate, ocrMeta);
setDisambiguation(null);
disambiguationRefineRef.current = null;
};
const showScanNotice = (message) => {
setScanNotice(message);
setTimeout(() => setScanNotice(null), 8000);
};
const handleReviewSubmitted = (cardTracker, message) => {
if (cardTracker) cardTracker.status = 'confirmed';
setDisambiguation(null);
disambiguationRefineRef.current = null;
showScanNotice(message);
};
const handleNotInCatalog = async () => {
if (!disambiguation || submittingReview) return;
setSubmittingReview(true);
const { cardTracker, imageData, candidates, ocrMeta } = disambiguation;
const guessedName =
ocrMeta?.cardName ||
ocrMeta?.query ||
candidates?.[0]?.name ||
null;
try {
const result = await submitScanForReview({
imageData,
name: guessedName,
candidateCardIds: (candidates || []).map((c) => c.id),
authHeaders: getScanAuthHeaders(),
});
if (result.rateLimited) {
visionCooldownUntilRef.current = rateLimitCooldownUntil(VISION_RATE_LIMIT_MS);
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
return;
}
handleReviewSubmitted(cardTracker, result.message);
} catch (error) {
reportScannerError(error.message || 'Failed to submit scan for review');
} finally {
setSubmittingReview(false);
}
};
const applyIdentifyOutcome = async (cardTracker, imageData, outcome) => {
cardTracker.status = outcome.cardStatus;
switch (outcome.type) {
case 'emit':
await emitScannedCard(cardTracker, imageData, outcome.card, outcome.ocrMeta);
break;
case 'disambiguation':
setDisambiguation({
cardTracker,
imageData,
candidates: outcome.matches,
ocrMeta: outcome.ocrMeta,
message: outcome.message,
});
break;
case 'notice':
showScanNotice(outcome.message);
break;
case 'error':
reportScannerError(outcome.message);
break;
default:
break;
}
};
const reportScannerError = (message) => {
if (disambiguation) return;
const now = Date.now();
if (now - lastErrorAtRef.current < 4000) return;
lastErrorAtRef.current = now;
onError?.(message);
};
useEffect(() => {
if (!disambiguation?.imageData) return;
if (Date.now() < visionCooldownUntilRef.current) return;
const refineKey = disambiguation.cardTracker?.id ?? 'modal';
if (disambiguationRefineRef.current === refineKey) return;
disambiguationRefineRef.current = refineKey;
let cancelled = false;
(async () => {
try {
const authHeaders = getScanAuthHeaders();
const vision = await fetchVisionIdentify(disambiguation.imageData, authHeaders);
if (cancelled) return;
if (vision.rateLimited) {
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
return;
}
const action = resolveDisambiguationRefineAction(vision.result, {
candidates: disambiguation.candidates,
});
if (cancelled) return;
switch (action.type) {
case 'review_submitted':
handleReviewSubmitted(disambiguation.cardTracker, action.message);
break;
case 'emit':
await emitScannedCard(
disambiguation.cardTracker,
disambiguation.imageData,
action.card,
action.ocrMeta
);
setDisambiguation(null);
disambiguationRefineRef.current = null;
break;
case 'pick':
await handleDisambiguationPick(action.candidate);
break;
case 'catalog_gap':
try {
const submitResult = await submitScanForReview({
imageData: disambiguation.imageData,
...action.submitPayload,
authHeaders,
});
if (submitResult.ok) {
handleReviewSubmitted(disambiguation.cardTracker, submitResult.message);
}
} catch {
setDisambiguation((current) =>
current
? {
...current,
visionHint: action.setHint,
message: action.hintMessage,
}
: current
);
}
break;
case 'narrow':
setDisambiguation((current) =>
current
? {
...current,
candidates: action.filtered,
message: action.message,
visionHint: action.visionHint,
}
: current
);
break;
default:
break;
}
} catch (error) {
console.warn('Disambiguation vision refine failed:', error);
}
})();
return () => {
cancelled = true;
};
// eslint-disable-next-line react-hooks/exhaustive-deps -- refine runs per disambiguation session, not per handler identity
}, [disambiguation?.cardTracker?.id, disambiguation?.imageData]);
const verifyCardShape = async (cardTracker) => {
if (!videoRef.current || !canvasRef.current || cardTracker.status !== 'detecting') return;
if (disambiguation) return;
if (activeVerificationRef.current >= 1) return;
activeVerificationRef.current += 1;
cardTracker.status = 'verifying';
try {
cardTracker.scanAttempts++;
const identification = await identifyTrackedCardCapture({
video: videoRef.current,
canvas: canvasRef.current,
cardTracker,
authHeaders: getScanAuthHeaders(),
visionCooldownUntilMs: visionCooldownUntilRef.current,
});
if (identification.retry) {
cardTracker.status = 'detecting';
return;
}
if (identification.rateLimited) {
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
cardTracker.status = 'negative';
return;
}
if (identification.handled && identification.outcome) {
cardTracker.status = 'confirmed';
await applyIdentifyOutcome(cardTracker, identification.imageData, identification.outcome);
}
} catch (error) {
console.error(`Error verifying card ${cardTracker.id}:`, error);
cardTracker.status = 'negative';
reportScannerError(error.message || 'Scan failed');
} finally {
activeVerificationRef.current = Math.max(0, activeVerificationRef.current - 1);
}
};
// Start continuous detection
const startDetection = () => {
if (detectionIntervalRef.current || !isStreaming) return;
console.log('🎯 Starting continuous card detection...');
setIsDetecting(true);
// Shape detection every 200ms
detectionIntervalRef.current = setInterval(() => {
const shapes = detectCardShapes();
updateTrackedCards(shapes);
}, 200);
// Card verification every 1 second
trackingIntervalRef.current = setInterval(() => {
const cardsToVerify = trackedCardsRef.current.filter(card =>
card.status === 'detecting' &&
card.stableCount >= 6 &&
card.scanAttempts < 1 &&
Date.now() - card.firstSeen > 2500
);
const cardsToProcess = cardsToVerify.slice(0, 1);
cardsToProcess.forEach(card => {
verifyCardShape(card);
});
}, 1000); // Back to 1 second intervals
};
// Stop detection
const stopDetection = () => {
console.log('🛑 Stopping card detection...');
setIsDetecting(false);
if (detectionIntervalRef.current) {
clearInterval(detectionIntervalRef.current);
detectionIntervalRef.current = null;
}
if (trackingIntervalRef.current) {
clearInterval(trackingIntervalRef.current);
trackingIntervalRef.current = null;
}
// Clear tracked cards
trackedCardsRef.current = [];
setTrackedCards([]);
};
// Start camera stream
const startCamera = async () => {
try {
console.log('🎥 Starting camera...');
const stream = await navigator.mediaDevices.getUserMedia({
video: {
facingMode: 'environment',
width: { ideal: 1280 },
height: { ideal: 720 },
aspectRatio: { ideal: 16/9 }
}
});
console.log('📹 Camera stream obtained:', stream);
if (videoRef.current) {
videoRef.current.srcObject = stream;
streamRef.current = stream;
videoRef.current.onloadedmetadata = () => {
console.log('📺 Video metadata loaded, attempting to play...');
videoRef.current?.play().then(() => {
console.log('▶️ Video playback started successfully');
setIsStreaming(true);
}).catch((err) => {
console.error('❌ Video playback failed:', err);
onError(`Video playback failed: ${err.message}`);
});
};
videoRef.current.onerror = (err) => {
console.error('❌ Video element error:', err);
onError('Video element error occurred');
};
// Add a fallback timeout
setTimeout(() => {
if (!isStreaming && videoRef.current && videoRef.current.readyState >= 2) {
console.log('🔄 Fallback: Attempting to play video directly...');
videoRef.current.play().then(() => {
console.log('▶️ Fallback video playback started');
setIsStreaming(true);
}).catch(console.error);
}
}, 2000);
} else {
console.error('❌ Video element not available');
onError('Video element not available');
}
} catch (err) {
console.error('❌ Camera access error:', err);
onError(`Unable to access camera: ${err.message}`);
}
};
// Stop camera stream
const stopCamera = () => {
setIsStreaming(false);
stopDetection();
if (streamRef.current) {
streamRef.current.getTracks().forEach(track => track.stop());
streamRef.current = null;
}
if (videoRef.current) {
videoRef.current.srcObject = null;
}
};
useEffect(() => {
const video = videoRef.current;
if (!video || !isStreaming) return undefined;
const syncVideoMetrics = () => {
setVideoMetrics({
width: video.videoWidth || 0,
height: video.videoHeight || 0,
});
};
video.addEventListener('loadedmetadata', syncVideoMetrics);
video.addEventListener('resize', syncVideoMetrics);
syncVideoMetrics();
return () => {
video.removeEventListener('loadedmetadata', syncVideoMetrics);
video.removeEventListener('resize', syncVideoMetrics);
};
}, [isStreaming]);
// Auto-start detection when camera starts
useEffect(() => {
if (isStreaming && !isDetecting) {
// Small delay to let camera stabilize
const timerId = setTimeout(() => {
startDetection();
}, 1000);
return () => clearTimeout(timerId);
}
return undefined;
// eslint-disable-next-line react-hooks/exhaustive-deps -- intentional: start detection once per stream session
}, [isStreaming]);
// Cleanup on unmount
useEffect(() => {
return () => {
stopCamera();
};
// eslint-disable-next-line react-hooks/exhaustive-deps -- run stopCamera only on unmount
}, []);
return (
<div className="w-full h-full flex flex-col">
{scanNotice && (
<div
className="mx-4 mt-3 px-4 py-3 rounded-lg text-sm border"
style={{
backgroundColor: 'var(--bg-secondary)',
borderColor: 'var(--accent-ember)',
color: 'var(--text-primary)',
}}
role="status"
>
{scanNotice}
</div>
)}
{/* Camera Feed Container */}
<div
className="flex-1 relative rounded-2xl overflow-hidden mb-4"
style={{
backgroundColor: 'var(--bg-tertiary)',
border: '2px solid var(--border)'
}}
>
{/* Video Element - Always rendered but visibility controlled */}
<video
ref={videoRef}
className={`absolute inset-0 w-full h-full object-cover ${isStreaming ? 'block' : 'hidden'}`}
autoPlay
playsInline
muted
aria-label="Card scanner camera feed"
/>
{/* Card Detection Overlays - Only when streaming */}
{isStreaming && videoMetrics.width > 0 && videoMetrics.height > 0 && trackedCards
.filter(card => card.status === 'confirmed' || card.status === 'scanned')
.map(card => (
<div
key={card.id}
className="absolute border-2 rounded-lg transition-all duration-200"
style={{
left: `${(card.bounds.x / videoMetrics.width) * 100}%`,
top: `${(card.bounds.y / videoMetrics.height) * 100}%`,
width: `${(card.bounds.width / videoMetrics.width) * 100}%`,
height: `${(card.bounds.height / videoMetrics.height) * 100}%`,
borderColor:
card.status === 'confirmed' ? '#10B981' : // Green for confirmed
card.status === 'scanned' ? '#3B82F6' : // Blue for scanned
'#10B981', // Default to green
borderWidth: '3px',
boxShadow: `0 0 15px ${
card.status === 'confirmed' ? '#10B98150' :
card.status === 'scanned' ? '#3B82F650' :
'#10B98150'
}`
}}
>
{/* Status Label */}
<div
className="absolute -top-8 left-0 px-3 py-1 rounded-full text-xs font-bold text-white shadow-lg"
style={{
backgroundColor:
card.status === 'confirmed' ? '#10B981' :
card.status === 'scanned' ? '#3B82F6' :
'#10B981'
}}
>
{card.status === 'confirmed' ? '✅ Card Found' :
card.status === 'scanned' ? '📸 Scanned' :
'✅ Card Found'}
</div>
</div>
))}
{/* Top Status Bar - Only when streaming */}
{isStreaming && (
<div className="absolute top-4 left-4 right-4 flex justify-between items-center">
{/* Detection Status */}
{isDetecting && (
<div className="bg-black bg-opacity-80 text-white px-4 py-2 rounded-full text-sm font-medium shadow-lg backdrop-blur-sm">
<div className="flex items-center gap-2">
<div className="w-2 h-2 bg-green-400 rounded-full animate-pulse"></div>
🎯 Scanning {trackedCards.filter(card => card.status === 'confirmed' || card.status === 'scanned').length > 0 &&
`(${trackedCards.filter(card => card.status === 'confirmed' || card.status === 'scanned').length} found)`}
</div>
</div>
)}
<div className="flex-1"></div>
{/* Recording Indicator */}
<div className="bg-black bg-opacity-80 text-white px-4 py-2 rounded-full text-sm font-medium shadow-lg backdrop-blur-sm">
<div className="flex items-center gap-2">
<div className="w-2 h-2 bg-red-500 rounded-full animate-pulse"></div>
LIVE
</div>
</div>
</div>
)}
{/* Bottom Controls Overlay - Only when streaming */}
{isStreaming && (
<div className="absolute bottom-6 left-1/2 transform -translate-x-1/2">
<div className="flex items-center gap-4">
{/* Stop Button */}
<button
onClick={stopCamera}
className="w-16 h-16 rounded-full flex items-center justify-center text-white shadow-2xl hover:scale-105 transition-all duration-200 backdrop-blur-sm"
style={{
backgroundColor: 'rgba(239, 68, 68, 0.9)',
border: '3px solid rgba(255, 255, 255, 0.3)'
}}
aria-label="Stop camera"
>
<div className="w-6 h-6 bg-white rounded-sm" aria-hidden="true"></div>
</button>
</div>
</div>
)}
{/* Camera Placeholder with Centered Start Button - Only when not streaming */}
{!isStreaming && (
<div className="absolute inset-0 flex flex-col items-center justify-center">
{/* Background Pattern */}
<div className="absolute inset-0 opacity-5">
<div className="w-full h-full" style={{
backgroundImage: `radial-gradient(circle at 25% 25%, var(--text-primary) 2px, transparent 2px),
radial-gradient(circle at 75% 75%, var(--text-primary) 2px, transparent 2px)`,
backgroundSize: '50px 50px'
}}></div>
</div>
{/* Camera Icon and Content */}
<div className="text-center z-10 mb-8">
<div className="font-semibold text-2xl mb-3" style={{ color: 'var(--text-primary)' }}>
Camera Ready
</div>
<div className="text-lg opacity-75 mb-6" style={{ color: 'var(--text-secondary)' }}>
Position trading cards in view for smart detection
</div>
</div>
{/* Centered Start Camera Button */}
<button
onClick={startCamera}
className="w-20 h-20 rounded-full flex items-center justify-center text-white shadow-2xl hover:scale-110 transition-all duration-300 relative overflow-hidden group"
style={{
backgroundColor: 'var(--accent-ember)',
border: '4px solid rgba(255, 255, 255, 0.2)'
}}
aria-label="Start camera"
>
{/* Button Glow Effect */}
<div className="absolute inset-0 rounded-full opacity-0 group-hover:opacity-100 transition-opacity duration-300"
style={{
background: `radial-gradient(circle, rgba(255,255,255,0.3) 0%, transparent 70%)`
}}></div>
{/* Play Icon */}
<div className="relative z-10" aria-hidden="true">
<svg width="28" height="28" viewBox="0 0 24 24" fill="currentColor">
<path d="M8 5v14l11-7z"/>
</svg>
</div>
{/* Pulse Ring */}
<div className="absolute inset-0 rounded-full border-2 border-white opacity-60 animate-ping"></div>
</button>
{/* Feature Hints */}
<div className="mt-8 text-center max-w-md">
<div className="grid grid-cols-2 gap-4 text-sm" style={{ color: 'var(--text-secondary)' }}>
<div className="flex items-center gap-2">
<div className="w-2 h-2 rounded-full" style={{ backgroundColor: 'var(--accent-gold)' }}></div>
<span>AI Detection</span>
</div>
<div className="flex items-center gap-2">
<div className="w-2 h-2 rounded-full" style={{ backgroundColor: 'var(--accent-ember)' }}></div>
<span>Real-time Tracking</span>
</div>
<div className="flex items-center gap-2">
<div className="w-2 h-2 rounded-full" style={{ backgroundColor: 'var(--accent-flame)' }}></div>
<span>Smart Recognition</span>
</div>
<div className="flex items-center gap-2">
<div className="w-2 h-2 rounded-full bg-green-500"></div>
<span>Auto Scanning</span>
</div>
</div>
</div>
</div>
)}
</div>
{/* Hidden canvases for image processing */}
<canvas ref={canvasRef} className="hidden" />
<canvas ref={detectionCanvasRef} className="hidden" />
{/* Detection Info Panel - Only show when streaming */}
{isStreaming && (
<div className="rounded-xl p-3 border" style={{ backgroundColor: 'var(--bg-secondary)', borderColor: 'var(--border)' }}>
<div className="flex items-center gap-3 mb-2">
<div className="w-6 h-6 rounded-full flex items-center justify-center" style={{ backgroundColor: 'var(--accent-ember)' }}>
<span className="text-white text-xs">🎯</span>
</div>
<h3 className="font-medium text-sm" style={{ color: 'var(--text-primary)' }}>
Smart Detection Active
</h3>
</div>
<div className="grid grid-cols-2 gap-2 text-xs" style={{ color: 'var(--text-secondary)' }}>
<div className="flex items-center gap-2">
<span className="text-green-500"></span>
<span>Shape recognition</span>
</div>
<div className="flex items-center gap-2">
<span className="text-blue-500"></span>
<span>Server identification</span>
</div>
<div className="flex items-center gap-2">
<span className="text-yellow-500"></span>
<span>Position tracking</span>
</div>
<div className="flex items-center gap-2">
<span className="text-purple-500"></span>
<span>Database lookup</span>
</div>
</div>
</div>
)}
<ScanDisambiguationDialog
disambiguation={disambiguation}
submittingReview={submittingReview}
onPick={handleDisambiguationPick}
onNotInCatalog={handleNotInCatalog}
onCancel={() => {
setDisambiguation(null);
disambiguationRefineRef.current = null;
}}
/>
</div>
);
}