feat(scanner): add debug instrumentation for vision pipeline timing #155

Merged
rstillwell merged 31 commits from feat/scanner-debug-mode into main 2026-09-01 18:11:03 -04:00
5 changed files with 274 additions and 27 deletions
Showing only changes of commit 6fab22d315 - Show all commits

View file

@ -9,27 +9,44 @@ import { useScannerFlash } from '../../lib/use-scanner-flash.js';
import { import {
cardGuideToContainerStyle, cardGuideToContainerStyle,
computeObjectCoverLayout, computeObjectCoverLayout,
layoutGuideToVideoBounds,
videoBoundsToContainerStyle, videoBoundsToContainerStyle,
} from '../../lib/scanner-video-layout.js'; } from '../../lib/scanner-video-layout.js';
const TOAST_DURATION_MS = 2500; const TOAST_DURATION_MS = 2500;
function CardGuideFrame({ layout }) { function CardGuideFrame({ layout, onTap, isManualMode, isIdentifying, tapDisabled }) {
if (!layout) return null; if (!layout) return null;
const style = cardGuideToContainerStyle(layout); const style = cardGuideToContainerStyle(layout);
const className = [
'absolute scan-guide',
isManualMode ? 'scan-guide-tappable' : 'pointer-events-none',
isIdentifying ? 'scan-guide-active' : '',
]
.filter(Boolean)
.join(' ');
return ( const frame = (
<div <>
className="absolute pointer-events-none scan-guide"
aria-hidden="true"
style={style}
>
<span className="scan-bracket scan-bracket-tl" /> <span className="scan-bracket scan-bracket-tl" />
<span className="scan-bracket scan-bracket-tr" /> <span className="scan-bracket scan-bracket-tr" />
<span className="scan-bracket scan-bracket-bl" /> <span className="scan-bracket scan-bracket-bl" />
<span className="scan-bracket scan-bracket-br" /> <span className="scan-bracket scan-bracket-br" />
<style jsx>{` <style jsx>{`
.scan-guide {
transition: box-shadow 0.2s ease, transform 0.15s ease;
}
.scan-guide-tappable {
cursor: pointer;
touch-action: manipulation;
}
.scan-guide-tappable:active:not(:disabled) {
transform: scale(0.985);
}
.scan-guide-active {
box-shadow: inset 0 0 0 2px color-mix(in srgb, var(--accent-flame) 55%, transparent);
}
.scan-bracket { .scan-bracket {
position: absolute; position: absolute;
width: 28px; width: 28px;
@ -37,15 +54,40 @@ function CardGuideFrame({ layout }) {
border-color: color-mix(in srgb, var(--text-primary) 70%, transparent); border-color: color-mix(in srgb, var(--text-primary) 70%, transparent);
border-style: solid; border-style: solid;
} }
.scan-guide-active .scan-bracket {
border-color: var(--accent-ember);
}
.scan-bracket-tl { top: 0; left: 0; border-width: 2px 0 0 2px; } .scan-bracket-tl { top: 0; left: 0; border-width: 2px 0 0 2px; }
.scan-bracket-tr { top: 0; right: 0; border-width: 2px 2px 0 0; } .scan-bracket-tr { top: 0; right: 0; border-width: 2px 2px 0 0; }
.scan-bracket-bl { bottom: 0; left: 0; border-width: 0 0 2px 2px; } .scan-bracket-bl { bottom: 0; left: 0; border-width: 0 0 2px 2px; }
.scan-bracket-br { bottom: 0; right: 0; border-width: 0 2px 2px 0; } .scan-bracket-br { bottom: 0; right: 0; border-width: 0 2px 2px 0; }
`}</style> `}</style>
</>
);
if (!isManualMode) {
return (
<div className={className} aria-hidden="true" style={style}>
{frame}
</div> </div>
); );
} }
return (
<button
type="button"
className={className}
style={{ ...style, background: 'transparent', padding: 0, border: 'none' }}
onClick={onTap}
disabled={tapDisabled}
aria-label="Tap to scan card in frame"
aria-busy={isIdentifying || undefined}
>
{frame}
</button>
);
}
function DetectionFrame({ card, layout }) { function DetectionFrame({ card, layout }) {
const label = const label =
card.status === 'scanned' card.status === 'scanned'
@ -116,6 +158,41 @@ function DetectionFrame({ card, layout }) {
); );
} }
function ShutterButton({ onClick, busy, label = 'Scan card' }) {
return (
<button
type="button"
onClick={onClick}
disabled={busy}
className="relative w-14 h-14 rounded-full flex items-center justify-center flex-shrink-0 transition-transform active:scale-95 focus:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60"
style={{
minWidth: 56,
minHeight: 56,
backgroundColor: 'color-mix(in srgb, var(--text-primary) 12%, transparent)',
border: '3px solid var(--text-primary)',
'--tw-ring-color': 'var(--accent-ember)',
'--tw-ring-offset-color': 'transparent',
}}
aria-label={label}
aria-busy={busy || undefined}
>
{busy ? (
<div
className="w-5 h-5 rounded-full border-2 border-t-transparent animate-spin"
style={{ borderColor: 'var(--accent-ember)', borderTopColor: 'transparent' }}
aria-hidden="true"
/>
) : (
<span
className="w-10 h-10 rounded-full"
style={{ backgroundColor: 'var(--text-primary)' }}
aria-hidden="true"
/>
)}
</button>
);
}
function ChromeIconButton({ function ChromeIconButton({
onClick, onClick,
label, label,
@ -160,6 +237,10 @@ function CameraViewport({
isStreaming, isStreaming,
displayLayout, displayLayout,
foundCards, foundCards,
isManualMode,
isIdentifying,
onGuideTap,
manualScanDisabled,
}) { }) {
return ( return (
<div <div
@ -177,7 +258,15 @@ function CameraViewport({
aria-label="Card scanner camera feed" aria-label="Card scanner camera feed"
/> />
{isStreaming && displayLayout && <CardGuideFrame layout={displayLayout} />} {isStreaming && displayLayout && (
<CardGuideFrame
layout={displayLayout}
isManualMode={isManualMode}
isIdentifying={isIdentifying}
onTap={onGuideTap}
tapDisabled={manualScanDisabled}
/>
)}
{isStreaming && {isStreaming &&
displayLayout && displayLayout &&
@ -240,8 +329,13 @@ export default function ScannerCamera({
streamRef, streamRef,
facingMode, facingMode,
switchFacingMode, switchFacingMode,
triggerManualScan,
} = camera; } = camera;
const isIdentifying = Boolean(identification.isIdentifying);
const isManualMode = !autoDetectOn;
const manualScanDisabled = isIdentifying || isCheckoutOpen || Boolean(identification.disambiguation);
const sound = useScannerSound(); const sound = useScannerSound();
const flash = useScannerFlash(streamRef); const flash = useScannerFlash(streamRef);
@ -339,12 +433,35 @@ export default function ScannerCamera({
viewportSize.height viewportSize.height
) )
: null; : null;
const handleManualScan = useCallback(() => {
if (manualScanDisabled || !triggerManualScan) return;
const guideBounds = displayLayout ? layoutGuideToVideoBounds(displayLayout) : null;
const result = triggerManualScan(guideBounds);
if (result?.ok) {
navigator.vibrate?.(30);
return;
}
if (result?.reason === 'no_card') {
clearTimeout(toastTimerRef.current);
setToast({ message: 'Hold the card in the frame', visible: true, type: 'info' });
toastTimerRef.current = setTimeout(() => {
setToast((current) => ({ ...current, visible: false }));
}, TOAST_DURATION_MS);
}
}, [displayLayout, manualScanDisabled, triggerManualScan]);
const showFlash = flash.flashSupported && facingMode === 'environment'; const showFlash = flash.flashSupported && facingMode === 'environment';
const scanStatus = isCheckoutOpen const scanStatus = (() => {
? 'Checkout open' if (isCheckoutOpen) return 'Checkout open';
: isStreaming if (!isStreaming) return 'Starting camera…';
? 'Scanning…' if (isIdentifying) return 'Identifying card…';
: 'Starting camera…'; if (isManualMode) return 'Tap frame or scan button';
return 'Scanning…';
})();
const switchCameraLabel = const switchCameraLabel =
facingMode === 'environment' ? 'Switch to front camera' : 'Switch to rear camera'; facingMode === 'environment' ? 'Switch to front camera' : 'Switch to rear camera';
@ -371,6 +488,10 @@ export default function ScannerCamera({
isStreaming={isStreaming} isStreaming={isStreaming}
displayLayout={displayLayout} displayLayout={displayLayout}
foundCards={foundCards} foundCards={foundCards}
isManualMode={isManualMode}
isIdentifying={isIdentifying}
onGuideTap={handleManualScan}
manualScanDisabled={manualScanDisabled}
/> />
</GlassSurface> </GlassSurface>
) : ( ) : (
@ -380,6 +501,10 @@ export default function ScannerCamera({
isStreaming={isStreaming} isStreaming={isStreaming}
displayLayout={displayLayout} displayLayout={displayLayout}
foundCards={foundCards} foundCards={foundCards}
isManualMode={isManualMode}
isIdentifying={isIdentifying}
onGuideTap={handleManualScan}
manualScanDisabled={manualScanDisabled}
/> />
)} )}
@ -584,6 +709,22 @@ export default function ScannerCamera({
)} )}
</div> </div>
{isManualMode && !isWorkstation ? (
<div className="flex flex-col items-center gap-1 flex-shrink-0 px-1">
<ShutterButton
onClick={handleManualScan}
busy={isIdentifying}
label={isIdentifying ? 'Identifying card' : 'Scan card'}
/>
<span
className="text-[10px] font-medium text-center max-w-[88px] leading-tight"
style={{ color: 'var(--text-secondary)' }}
aria-live="polite"
>
{scanStatus}
</span>
</div>
) : (
<p <p
className="text-xs font-medium truncate text-center flex-1 px-1" className="text-xs font-medium truncate text-center flex-1 px-1"
style={{ color: 'var(--text-secondary)' }} style={{ color: 'var(--text-secondary)' }}
@ -591,11 +732,14 @@ export default function ScannerCamera({
> >
{scanStatus} {scanStatus}
</p> </p>
)}
<div className="flex flex-1 min-w-0 justify-end">
<ScannerCountPill <ScannerCountPill
count={resolvedCartCount} count={resolvedCartCount}
onOpenCheckout={onOpenCheckout} onOpenCheckout={onOpenCheckout}
/> />
</div>
</GlassSurface> </GlassSurface>
</div> </div>
)} )}

View file

@ -96,3 +96,29 @@ export function cardGuideToContainerStyle(layout) {
height: `${(guide.height / layout.containerHeight) * 100}%`, height: `${(guide.height / layout.containerHeight) * 100}%`,
}; };
} }
/** Map a container-space rectangle to video pixel bounds (inverse of object-cover). */
export function containerRectToVideoBounds(rect, layout) {
const { videoWidth, videoHeight, renderedWidth, renderedHeight, offsetX, offsetY } = layout;
const toVideoX = (containerX) => ((containerX - offsetX) / renderedWidth) * videoWidth;
const toVideoY = (containerY) => ((containerY - offsetY) / renderedHeight) * videoHeight;
const x1 = toVideoX(rect.left);
const y1 = toVideoY(rect.top);
const x2 = toVideoX(rect.left + rect.width);
const y2 = toVideoY(rect.top + rect.height);
const x = Math.max(0, Math.min(x1, x2));
const y = Math.max(0, Math.min(y1, y2));
const width = Math.min(videoWidth - x, Math.abs(x2 - x1));
const height = Math.min(videoHeight - y, Math.abs(y2 - y1));
return { x, y, width, height };
}
/** Centered card guide converted to video pixel bounds. */
export function layoutGuideToVideoBounds(layout) {
const guide = computeCardGuideBounds(layout.containerWidth, layout.containerHeight);
return containerRectToVideoBounds(guide, layout);
}

View file

@ -3,6 +3,7 @@ import {
detectCardShapesFromFrame, detectCardShapesFromFrame,
DETECTION_START_DELAY_MS, DETECTION_START_DELAY_MS,
mergeDetectedShapesIntoTrackedCards, mergeDetectedShapesIntoTrackedCards,
MIN_STABLE_COUNT_FOR_VERIFY,
selectCardsReadyForVerification, selectCardsReadyForVerification,
SHAPE_DETECTION_INTERVAL_MS, SHAPE_DETECTION_INTERVAL_MS,
VERIFICATION_INTERVAL_MS, VERIFICATION_INTERVAL_MS,
@ -26,6 +27,7 @@ export function useCameraScanner({
onError, onError,
onVerifyCard, onVerifyCard,
verificationPausedRef, verificationPausedRef,
autoDetectPausedRef,
facingMode = 'environment', facingMode = 'environment',
}) { }) {
const [isStreaming, setIsStreaming] = useState(false); const [isStreaming, setIsStreaming] = useState(false);
@ -174,6 +176,60 @@ export function useCameraScanner({
setTrackedCards(next); setTrackedCards(next);
}, []); }, []);
const resetScanTrackers = useCallback(() => {
trackedCardsRef.current = [];
setTrackedCards([]);
nextCardIdRef.current = 1;
}, []);
/**
* Manual scan: fresh detection pass, then verify the best candidate (or guide bounds).
* @returns {{ ok: boolean, reason?: string }}
*/
const triggerManualScan = useCallback(
(guideVideoBounds = null) => {
if (!isStreamingRef.current) {
return { ok: false, reason: 'not_streaming' };
}
if (verificationPausedRef?.current) {
return { ok: false, reason: 'paused' };
}
resetScanTrackers();
const shapes = detectCardShapes();
updateTrackedCards(shapes);
let target = trackedCardsRef.current
.filter((card) => card.status === 'detecting')
.sort((a, b) => b.stableCount - a.stableCount)[0];
if (!target && guideVideoBounds?.width > 0 && guideVideoBounds?.height > 0) {
target = {
id: nextCardIdRef.current++,
bounds: guideVideoBounds,
corners: null,
status: 'detecting',
firstSeen: Date.now(),
lastSeen: Date.now(),
stableCount: MIN_STABLE_COUNT_FOR_VERIFY,
scanAttempts: 0,
};
trackedCardsRef.current = [target];
setTrackedCards([target]);
}
if (!target) {
return { ok: false, reason: 'no_card' };
}
onVerifyCard?.(target);
return { ok: true };
},
[onVerifyCard, resetScanTrackers, verificationPausedRef]
);
const startDetection = () => { const startDetection = () => {
if (detectionIntervalRef.current || !isStreamingRef.current) return; if (detectionIntervalRef.current || !isStreamingRef.current) return;
@ -188,6 +244,7 @@ export function useCameraScanner({
trackingIntervalRef.current = setInterval(() => { trackingIntervalRef.current = setInterval(() => {
if (document.hidden) return; if (document.hidden) return;
if (verificationPausedRef?.current) return; if (verificationPausedRef?.current) return;
if (autoDetectPausedRef?.current) return;
const cardsToVerify = selectCardsReadyForVerification(trackedCardsRef.current); const cardsToVerify = selectCardsReadyForVerification(trackedCardsRef.current);
cardsToVerify.slice(0, 1).forEach((card) => { cardsToVerify.slice(0, 1).forEach((card) => {
@ -372,6 +429,8 @@ export function useCameraScanner({
startCamera, startCamera,
stopCamera, stopCamera,
removeTrackedCard, removeTrackedCard,
resetScanTrackers,
triggerManualScan,
streamRef, streamRef,
facingMode: activeFacingMode, facingMode: activeFacingMode,
switchFacingMode, switchFacingMode,

View file

@ -34,13 +34,17 @@ export default function Scanner() {
const [inspectorCommitError, setInspectorCommitError] = useState(null); const [inspectorCommitError, setInspectorCommitError] = useState(null);
const [focusedCardId, setFocusedCardId] = useState(null); const [focusedCardId, setFocusedCardId] = useState(null);
const [stripActiveTab, setStripActiveTab] = useState(TAB_RECENT); const [stripActiveTab, setStripActiveTab] = useState(TAB_RECENT);
const [isAutoDetectPaused, setIsAutoDetectPaused] = useState(false); const [isAutoDetectPaused, setIsAutoDetectPaused] = useState(() => {
if (typeof window === 'undefined') return false;
return window.matchMedia('(max-width: 767px)').matches;
});
const [pageToast, setPageToast] = useState({ message: '', visible: false, type: 'success' }); const [pageToast, setPageToast] = useState({ message: '', visible: false, type: 'success' });
const [galleryBusy, setGalleryBusy] = useState(false); const [galleryBusy, setGalleryBusy] = useState(false);
const [batchBusy, setBatchBusy] = useState(false); const [batchBusy, setBatchBusy] = useState(false);
const [batchProgress, setBatchProgress] = useState(null); const [batchProgress, setBatchProgress] = useState(null);
const verificationPausedRef = useRef(false); const verificationPausedRef = useRef(false);
const autoDetectPausedRef = useRef(false);
const prevCardCountRef = useRef(0); const prevCardCountRef = useRef(0);
const scannedCardsRef = useRef([]); const scannedCardsRef = useRef([]);
const toastTimerRef = useRef(null); const toastTimerRef = useRef(null);
@ -58,6 +62,7 @@ export default function Scanner() {
onError: (msg) => console.error('Camera error:', msg), onError: (msg) => console.error('Camera error:', msg),
onVerifyCard: (cardTracker) => onVerifyCardRef.current?.(cardTracker), onVerifyCard: (cardTracker) => onVerifyCardRef.current?.(cardTracker),
verificationPausedRef, verificationPausedRef,
autoDetectPausedRef,
}); });
const identification = useScannerIdentification({ const identification = useScannerIdentification({
@ -105,8 +110,9 @@ export default function Scanner() {
verificationPausedRef.current = verificationPausedRef.current =
mobileCheckoutPauses || mobileCheckoutPauses ||
isListPickerOpen || isListPickerOpen ||
isAutoDetectPaused ||
Boolean(identification.disambiguation); Boolean(identification.disambiguation);
autoDetectPausedRef.current = isAutoDetectPaused;
}, [ }, [
isCheckoutOpen, isCheckoutOpen,
isListPickerOpen, isListPickerOpen,

View file

@ -3,6 +3,7 @@ import {
cardGuideToContainerStyle, cardGuideToContainerStyle,
computeCardGuideBounds, computeCardGuideBounds,
computeObjectCoverLayout, computeObjectCoverLayout,
layoutGuideToVideoBounds,
videoBoundsToContainerStyle, videoBoundsToContainerStyle,
} from '../../lib/scanner-video-layout.js'; } from '../../lib/scanner-video-layout.js';
@ -47,3 +48,14 @@ describe('card guide layout', () => {
expect(style.width).toMatch(/%$/); expect(style.width).toMatch(/%$/);
}); });
}); });
describe('layoutGuideToVideoBounds', () => {
it('maps the centered guide back to video pixel space', () => {
const layout = computeObjectCoverLayout(1280, 720, 390, 844);
const bounds = layoutGuideToVideoBounds(layout);
expect(bounds.width).toBeGreaterThan(0);
expect(bounds.height).toBeGreaterThan(0);
expect(bounds.x).toBeGreaterThanOrEqual(0);
expect(bounds.y).toBeGreaterThanOrEqual(0);
});
});