Add pgvector embeddings on cards, server-side cohere/embed-v4.0 via AI Gateway, kNN identify route, and L0→L1→L2 client orchestration with empty-index fast escalate and id-cursor backfill job. Co-authored-by: Cursor <cursoragent@cursor.com>
383 lines
12 KiB
JavaScript
383 lines
12 KiB
JavaScript
import { useEffect, useRef, useState } from 'react';
|
|
import { rateLimitCooldownUntil, uploadScanCapture } from './scan-capture-upload.js';
|
|
import {
|
|
buildScannedCardPayload,
|
|
getScanAuthHeaders,
|
|
identifyTrackedCardCapture,
|
|
resolveDisambiguationRefineAction,
|
|
resolveIdentifyOutcome,
|
|
submitScanForReview,
|
|
tryLayer0VisualIdentify,
|
|
tryLayer1TextIdentify,
|
|
VISION_RATE_LIMIT_MS,
|
|
fetchVisionIdentify,
|
|
} from './scanner-card-identify.js';
|
|
|
|
/**
|
|
* Card identification, disambiguation, and review-submission flow for the scanner.
|
|
* Camera refs and verify-card wiring are supplied by the parent + useCameraScanner.
|
|
*/
|
|
function readFileToImageData(file) {
|
|
return new Promise((resolve, reject) => {
|
|
const img = new Image();
|
|
const url = URL.createObjectURL(file);
|
|
|
|
img.onload = () => {
|
|
const canvas = document.createElement('canvas');
|
|
canvas.width = img.naturalWidth;
|
|
canvas.height = img.naturalHeight;
|
|
const ctx = canvas.getContext('2d', { willReadFrequently: true });
|
|
ctx.drawImage(img, 0, 0);
|
|
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
|
URL.revokeObjectURL(url);
|
|
resolve(imageData);
|
|
};
|
|
|
|
img.onerror = () => {
|
|
URL.revokeObjectURL(url);
|
|
reject(new Error('Failed to load image from gallery'));
|
|
};
|
|
|
|
img.src = url;
|
|
});
|
|
}
|
|
|
|
export function useScannerIdentification({
|
|
onCardScanned,
|
|
onError,
|
|
videoRef,
|
|
canvasRef,
|
|
onVerifyCardRef,
|
|
verificationPausedRef,
|
|
}) {
|
|
const [disambiguation, setDisambiguation] = useState(null);
|
|
const [scanNotice, setScanNotice] = useState(null);
|
|
const [submittingReview, setSubmittingReview] = useState(false);
|
|
|
|
const visionCooldownUntilRef = useRef(0);
|
|
const activeVerificationRef = useRef(0);
|
|
const lastErrorAtRef = useRef(0);
|
|
const disambiguationRefineRef = useRef(null);
|
|
|
|
useEffect(() => {
|
|
if (verificationPausedRef) {
|
|
verificationPausedRef.current = Boolean(disambiguation);
|
|
}
|
|
}, [disambiguation, verificationPausedRef]);
|
|
|
|
const emitScannedCard = async (cardTracker, imageData, finalCard, ocrMeta = {}) => {
|
|
cardTracker.status = 'scanned';
|
|
|
|
const originalTitle = document.title;
|
|
document.title = `📸 ${finalCard.name} - Card Scanner`;
|
|
setTimeout(() => {
|
|
document.title = originalTitle;
|
|
}, 3000);
|
|
|
|
let scanImageUrl = null;
|
|
if (imageData) {
|
|
try {
|
|
scanImageUrl = await uploadScanCapture(imageData);
|
|
} catch (uploadError) {
|
|
console.warn('Scan image upload failed:', uploadError);
|
|
}
|
|
}
|
|
|
|
onCardScanned(buildScannedCardPayload(finalCard, { imageData, scanImageUrl, ocrMeta }));
|
|
};
|
|
|
|
const showScanNotice = (message) => {
|
|
setScanNotice(message);
|
|
setTimeout(() => setScanNotice(null), 8000);
|
|
};
|
|
|
|
const handleReviewSubmitted = (cardTracker, message) => {
|
|
if (cardTracker) cardTracker.status = 'confirmed';
|
|
setDisambiguation(null);
|
|
disambiguationRefineRef.current = null;
|
|
showScanNotice(message);
|
|
};
|
|
|
|
const handleDisambiguationPick = async (candidate) => {
|
|
if (!disambiguation) return;
|
|
const { cardTracker, imageData, ocrMeta } = disambiguation;
|
|
await emitScannedCard(cardTracker, imageData, candidate, ocrMeta);
|
|
setDisambiguation(null);
|
|
disambiguationRefineRef.current = null;
|
|
};
|
|
|
|
const cancelDisambiguation = () => {
|
|
setDisambiguation(null);
|
|
disambiguationRefineRef.current = null;
|
|
};
|
|
|
|
const reportScannerError = (message) => {
|
|
if (disambiguation) return;
|
|
const now = Date.now();
|
|
if (now - lastErrorAtRef.current < 4000) return;
|
|
lastErrorAtRef.current = now;
|
|
onError?.(message);
|
|
};
|
|
|
|
const applyIdentifyOutcome = async (cardTracker, imageData, outcome) => {
|
|
cardTracker.status = outcome.cardStatus;
|
|
|
|
switch (outcome.type) {
|
|
case 'emit':
|
|
await emitScannedCard(cardTracker, imageData, outcome.card, outcome.ocrMeta);
|
|
break;
|
|
case 'disambiguation':
|
|
setDisambiguation({
|
|
cardTracker,
|
|
imageData,
|
|
candidates: outcome.matches,
|
|
ocrMeta: outcome.ocrMeta,
|
|
message: outcome.message,
|
|
fromLayer1: Boolean(outcome.fromLayer1),
|
|
fromLayer0: Boolean(outcome.fromLayer0),
|
|
});
|
|
break;
|
|
case 'notice':
|
|
showScanNotice(outcome.message);
|
|
break;
|
|
case 'error':
|
|
reportScannerError(outcome.message);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
};
|
|
|
|
const handleNotInCatalog = async () => {
|
|
if (!disambiguation || submittingReview) return;
|
|
setSubmittingReview(true);
|
|
|
|
const { cardTracker, imageData, candidates, ocrMeta } = disambiguation;
|
|
const guessedName =
|
|
ocrMeta?.cardName ||
|
|
ocrMeta?.query ||
|
|
candidates?.[0]?.name ||
|
|
null;
|
|
|
|
try {
|
|
const result = await submitScanForReview({
|
|
imageData,
|
|
name: guessedName,
|
|
candidateCardIds: (candidates || []).map((c) => c.id),
|
|
authHeaders: getScanAuthHeaders(),
|
|
});
|
|
|
|
if (result.rateLimited) {
|
|
visionCooldownUntilRef.current = rateLimitCooldownUntil(VISION_RATE_LIMIT_MS);
|
|
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
|
return;
|
|
}
|
|
|
|
handleReviewSubmitted(cardTracker, result.message);
|
|
} catch (error) {
|
|
reportScannerError(error.message || 'Failed to submit scan for review');
|
|
} finally {
|
|
setSubmittingReview(false);
|
|
}
|
|
};
|
|
|
|
useEffect(() => {
|
|
if (!disambiguation?.imageData) return;
|
|
if (disambiguation.fromLayer1 || disambiguation.fromLayer0) return;
|
|
if (Date.now() < visionCooldownUntilRef.current) return;
|
|
|
|
const refineKey = disambiguation.cardTracker?.id ?? 'modal';
|
|
if (disambiguationRefineRef.current === refineKey) return;
|
|
disambiguationRefineRef.current = refineKey;
|
|
|
|
let cancelled = false;
|
|
|
|
(async () => {
|
|
try {
|
|
const authHeaders = getScanAuthHeaders();
|
|
const vision = await fetchVisionIdentify(disambiguation.imageData, authHeaders);
|
|
if (cancelled) return;
|
|
|
|
if (vision.rateLimited) {
|
|
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
|
|
return;
|
|
}
|
|
|
|
const action = resolveDisambiguationRefineAction(vision.result, {
|
|
candidates: disambiguation.candidates,
|
|
});
|
|
if (cancelled) return;
|
|
|
|
switch (action.type) {
|
|
case 'review_submitted':
|
|
handleReviewSubmitted(disambiguation.cardTracker, action.message);
|
|
break;
|
|
case 'emit':
|
|
await emitScannedCard(
|
|
disambiguation.cardTracker,
|
|
disambiguation.imageData,
|
|
action.card,
|
|
action.ocrMeta
|
|
);
|
|
setDisambiguation(null);
|
|
disambiguationRefineRef.current = null;
|
|
break;
|
|
case 'pick':
|
|
await handleDisambiguationPick(action.candidate);
|
|
break;
|
|
case 'catalog_gap':
|
|
try {
|
|
const submitResult = await submitScanForReview({
|
|
imageData: disambiguation.imageData,
|
|
...action.submitPayload,
|
|
authHeaders,
|
|
});
|
|
if (submitResult.ok) {
|
|
handleReviewSubmitted(disambiguation.cardTracker, submitResult.message);
|
|
}
|
|
} catch {
|
|
setDisambiguation((current) =>
|
|
current
|
|
? {
|
|
...current,
|
|
visionHint: action.setHint,
|
|
message: action.hintMessage,
|
|
}
|
|
: current
|
|
);
|
|
}
|
|
break;
|
|
case 'narrow':
|
|
setDisambiguation((current) =>
|
|
current
|
|
? {
|
|
...current,
|
|
candidates: action.filtered,
|
|
message: action.message,
|
|
visionHint: action.visionHint,
|
|
}
|
|
: current
|
|
);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
} catch (error) {
|
|
console.warn('Disambiguation vision refine failed:', error);
|
|
}
|
|
})();
|
|
|
|
return () => {
|
|
cancelled = true;
|
|
};
|
|
// eslint-disable-next-line react-hooks/exhaustive-deps -- refine runs per disambiguation session, not per handler identity
|
|
}, [disambiguation?.cardTracker?.id, disambiguation?.imageData]);
|
|
|
|
const verifyCardShape = async (cardTracker) => {
|
|
if (!videoRef.current || !canvasRef.current || cardTracker.status !== 'detecting') return;
|
|
if (disambiguation) return;
|
|
if (activeVerificationRef.current >= 1) return;
|
|
|
|
activeVerificationRef.current += 1;
|
|
cardTracker.status = 'verifying';
|
|
|
|
try {
|
|
cardTracker.scanAttempts++;
|
|
|
|
const identification = await identifyTrackedCardCapture({
|
|
video: videoRef.current,
|
|
canvas: canvasRef.current,
|
|
cardTracker,
|
|
authHeaders: getScanAuthHeaders(),
|
|
visionCooldownUntilMs: visionCooldownUntilRef.current,
|
|
});
|
|
|
|
if (identification.retry) {
|
|
cardTracker.status = 'detecting';
|
|
return;
|
|
}
|
|
|
|
if (identification.rateLimited) {
|
|
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
|
|
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
|
cardTracker.status = 'negative';
|
|
return;
|
|
}
|
|
|
|
if (identification.handled && identification.outcome) {
|
|
cardTracker.status = 'confirmed';
|
|
await applyIdentifyOutcome(cardTracker, identification.imageData, identification.outcome);
|
|
}
|
|
} catch (error) {
|
|
console.error(`Error verifying card ${cardTracker.id}:`, error);
|
|
cardTracker.status = 'negative';
|
|
reportScannerError(error.message || 'Scan failed');
|
|
} finally {
|
|
activeVerificationRef.current = Math.max(0, activeVerificationRef.current - 1);
|
|
}
|
|
};
|
|
|
|
useEffect(() => {
|
|
if (onVerifyCardRef) {
|
|
onVerifyCardRef.current = verifyCardShape;
|
|
}
|
|
});
|
|
|
|
const identifyFromGalleryFile = async (file) => {
|
|
if (!file || verificationPausedRef?.current) return;
|
|
|
|
try {
|
|
const imageData = await readFileToImageData(file);
|
|
const authHeaders = getScanAuthHeaders();
|
|
const syntheticTracker = { id: `gallery-${Date.now()}`, status: 'verifying' };
|
|
|
|
const l0 = await tryLayer0VisualIdentify(imageData, authHeaders);
|
|
if (l0.handled && l0.outcome) {
|
|
await applyIdentifyOutcome(syntheticTracker, imageData, l0.outcome);
|
|
return;
|
|
}
|
|
|
|
const result = await tryLayer1TextIdentify(imageData, authHeaders);
|
|
|
|
if (result.handled && result.outcome) {
|
|
await applyIdentifyOutcome(syntheticTracker, imageData, result.outcome);
|
|
return;
|
|
}
|
|
|
|
if (Date.now() < visionCooldownUntilRef.current) {
|
|
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
|
return;
|
|
}
|
|
|
|
const vision = await fetchVisionIdentify(imageData, authHeaders);
|
|
if (vision.rateLimited) {
|
|
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
|
|
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
|
return;
|
|
}
|
|
|
|
if (vision.ok && vision.result) {
|
|
await applyIdentifyOutcome(
|
|
syntheticTracker,
|
|
imageData,
|
|
resolveIdentifyOutcome(vision.result)
|
|
);
|
|
return;
|
|
}
|
|
|
|
reportScannerError('Could not identify card from gallery image');
|
|
} catch (error) {
|
|
reportScannerError(error.message || 'Gallery identify failed');
|
|
}
|
|
};
|
|
|
|
return {
|
|
disambiguation,
|
|
scanNotice,
|
|
submittingReview,
|
|
handleDisambiguationPick,
|
|
handleNotInCatalog,
|
|
cancelDisambiguation,
|
|
identifyFromGalleryFile,
|
|
};
|
|
}
|