refactor(scanner): extract card identification pipeline (Brief 3) #70
3 changed files with 551 additions and 238 deletions
|
|
@ -4,6 +4,15 @@ import {
|
|||
detectCardShapesFromFrame,
|
||||
mergeDetectedShapesIntoTrackedCards,
|
||||
} from '../lib/scanner-card-detection.js';
|
||||
import {
|
||||
buildScannedCardPayload,
|
||||
getScanAuthHeaders,
|
||||
identifyTrackedCardCapture,
|
||||
resolveDisambiguationRefineAction,
|
||||
submitScanForReview,
|
||||
VISION_RATE_LIMIT_MS,
|
||||
fetchVisionIdentify,
|
||||
} from '../lib/scanner-card-identify.js';
|
||||
import ScanDisambiguationDialog from './ScanDisambiguationDialog.js';
|
||||
|
||||
export default function CameraScanner({ onCardScanned, onError }) {
|
||||
|
|
@ -29,8 +38,6 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
|||
const lastErrorAtRef = useRef(0);
|
||||
const disambiguationRefineRef = useRef(null);
|
||||
|
||||
// Mana symbol settings
|
||||
const [manaSymbolSettings, setManaSymbolSettings] = useState({ useSVG: false });
|
||||
const [videoMetrics, setVideoMetrics] = useState({ width: 0, height: 0 });
|
||||
|
||||
// Configure canvas contexts for optimal performance
|
||||
|
|
@ -78,25 +85,7 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
|||
}
|
||||
}
|
||||
|
||||
onCardScanned({
|
||||
name: finalCard.name,
|
||||
set: finalCard.set_name,
|
||||
setCode: finalCard.set_code,
|
||||
cardNumber: finalCard.card_number,
|
||||
game: finalCard.game,
|
||||
cardType: finalCard.card_type,
|
||||
rarity: finalCard.rarity,
|
||||
hp: finalCard.hp || ocrMeta.hp,
|
||||
manaCost: finalCard.mana_cost || ocrMeta.manaCost,
|
||||
abilities: ocrMeta.abilities || [],
|
||||
ocrText: ocrMeta.rawText,
|
||||
confidence: ocrMeta.confidence,
|
||||
capturedImage: imageData,
|
||||
scanImageUrl,
|
||||
image_url: finalCard.image_url,
|
||||
databaseId: finalCard.id,
|
||||
isExisting: true,
|
||||
});
|
||||
onCardScanned(buildScannedCardPayload(finalCard, { imageData, scanImageUrl, ocrMeta }));
|
||||
};
|
||||
|
||||
const handleDisambiguationPick = async (candidate) => {
|
||||
|
|
@ -131,31 +120,19 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
|||
null;
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/scan/submit-for-review', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
imageData,
|
||||
name: guessedName,
|
||||
candidateCardIds: (candidates || []).map((c) => c.id),
|
||||
}),
|
||||
const result = await submitScanForReview({
|
||||
imageData,
|
||||
name: guessedName,
|
||||
candidateCardIds: (candidates || []).map((c) => c.id),
|
||||
authHeaders: getScanAuthHeaders(),
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
visionCooldownUntilRef.current = rateLimitCooldownUntil(60_000);
|
||||
if (result.rateLimited) {
|
||||
visionCooldownUntilRef.current = rateLimitCooldownUntil(VISION_RATE_LIMIT_MS);
|
||||
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const errBody = await response.json().catch(() => ({}));
|
||||
throw new Error(errBody.error || 'Failed to submit scan for review');
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
handleReviewSubmitted(cardTracker, result.message);
|
||||
} catch (error) {
|
||||
reportScannerError(error.message || 'Failed to submit scan for review');
|
||||
|
|
@ -164,61 +141,31 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
|||
}
|
||||
};
|
||||
|
||||
const candidateMatchesSetHint = (candidate, setName, setCode) => {
|
||||
if (!setName && !setCode) return true;
|
||||
const hint = (setName || setCode || '').toLowerCase();
|
||||
const setNameLower = (candidate.set_name || '').toLowerCase();
|
||||
const setCodeLower = (candidate.set_code || '').toLowerCase();
|
||||
return setNameLower.includes(hint) || hint.includes(setNameLower) || setCodeLower === hint;
|
||||
};
|
||||
const applyIdentifyOutcome = async (cardTracker, imageData, outcome) => {
|
||||
cardTracker.status = outcome.cardStatus;
|
||||
|
||||
const processIdentifyResponse = async (cardTracker, imageData, result) => {
|
||||
if (!result.isCard) {
|
||||
cardTracker.status = 'negative';
|
||||
reportScannerError(result.reason || 'No trading card detected');
|
||||
return;
|
||||
switch (outcome.type) {
|
||||
case 'emit':
|
||||
await emitScannedCard(cardTracker, imageData, outcome.card, outcome.ocrMeta);
|
||||
break;
|
||||
case 'disambiguation':
|
||||
setDisambiguation({
|
||||
cardTracker,
|
||||
imageData,
|
||||
candidates: outcome.matches,
|
||||
ocrMeta: outcome.ocrMeta,
|
||||
message: outcome.message,
|
||||
});
|
||||
break;
|
||||
case 'notice':
|
||||
showScanNotice(outcome.message);
|
||||
break;
|
||||
case 'error':
|
||||
reportScannerError(outcome.message);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
const ocrMeta = {
|
||||
confidence: result.ocr?.confidence ?? result.card?.ocr?.confidence,
|
||||
rawText: result.ocr?.rawText ?? result.card?.ocr?.rawText,
|
||||
abilities: result.card?.ocr?.abilities || [],
|
||||
hp: result.card?.hp,
|
||||
manaCost: result.card?.mana_cost,
|
||||
};
|
||||
|
||||
if (result.card) {
|
||||
cardTracker.status = 'confirmed';
|
||||
await emitScannedCard(cardTracker, imageData, result.card, ocrMeta);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.needsUserSelection && result.matches?.length) {
|
||||
cardTracker.status = 'confirmed';
|
||||
setDisambiguation({
|
||||
cardTracker,
|
||||
imageData,
|
||||
candidates: result.matches,
|
||||
ocrMeta,
|
||||
message: result.message,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.needsReview) {
|
||||
cardTracker.status = 'confirmed';
|
||||
showScanNotice(result.message || 'Scan saved for admin review.');
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.needsUserInput) {
|
||||
cardTracker.status = 'negative';
|
||||
reportScannerError(result.message || 'Could not identify card — try again or submit for review.');
|
||||
return;
|
||||
}
|
||||
|
||||
cardTracker.status = 'negative';
|
||||
reportScannerError('Could not identify card from scan.');
|
||||
};
|
||||
|
||||
const reportScannerError = (message) => {
|
||||
|
|
@ -241,70 +188,45 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
|||
|
||||
(async () => {
|
||||
try {
|
||||
const response = await fetch('/api/scan/identify', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
||||
},
|
||||
body: JSON.stringify({ imageData: disambiguation.imageData }),
|
||||
const authHeaders = getScanAuthHeaders();
|
||||
const vision = await fetchVisionIdentify(disambiguation.imageData, authHeaders);
|
||||
if (cancelled) return;
|
||||
|
||||
if (vision.rateLimited) {
|
||||
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
|
||||
return;
|
||||
}
|
||||
|
||||
const action = resolveDisambiguationRefineAction(vision.result, {
|
||||
candidates: disambiguation.candidates,
|
||||
});
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
if (response.status === 429) {
|
||||
visionCooldownUntilRef.current = Date.now() + 60_000;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!response.ok) return;
|
||||
|
||||
const result = await response.json();
|
||||
if (cancelled) return;
|
||||
|
||||
if (result.needsReview) {
|
||||
handleReviewSubmitted(disambiguation.cardTracker, result.message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.card) {
|
||||
emitScannedCard(
|
||||
disambiguation.cardTracker,
|
||||
disambiguation.imageData,
|
||||
result.card,
|
||||
{
|
||||
confidence: result.ocr?.confidence,
|
||||
rawText: result.ocr?.rawText,
|
||||
abilities: result.card?.ocr?.abilities || [],
|
||||
}
|
||||
);
|
||||
setDisambiguation(null);
|
||||
disambiguationRefineRef.current = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const setHint = result.ocr?.setName || result.ocr?.setCode;
|
||||
if (result.matches?.length && setHint) {
|
||||
const filtered = disambiguation.candidates.filter((candidate) =>
|
||||
candidateMatchesSetHint(candidate, result.ocr.setName, result.ocr.setCode)
|
||||
);
|
||||
|
||||
if (filtered.length === 0 && setHint) {
|
||||
switch (action.type) {
|
||||
case 'review_submitted':
|
||||
handleReviewSubmitted(disambiguation.cardTracker, action.message);
|
||||
break;
|
||||
case 'emit':
|
||||
await emitScannedCard(
|
||||
disambiguation.cardTracker,
|
||||
disambiguation.imageData,
|
||||
action.card,
|
||||
action.ocrMeta
|
||||
);
|
||||
setDisambiguation(null);
|
||||
disambiguationRefineRef.current = null;
|
||||
break;
|
||||
case 'pick':
|
||||
await handleDisambiguationPick(action.candidate);
|
||||
break;
|
||||
case 'catalog_gap':
|
||||
try {
|
||||
const submitRes = await fetch('/api/scan/submit-for-review', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
imageData: disambiguation.imageData,
|
||||
name: result.ocr?.cardName || disambiguation.candidates[0]?.name,
|
||||
candidateCardIds: disambiguation.candidates.map((c) => c.id),
|
||||
}),
|
||||
const submitResult = await submitScanForReview({
|
||||
imageData: disambiguation.imageData,
|
||||
...action.submitPayload,
|
||||
authHeaders,
|
||||
});
|
||||
if (submitRes.ok) {
|
||||
const submitResult = await submitRes.json();
|
||||
if (submitResult.ok) {
|
||||
handleReviewSubmitted(disambiguation.cardTracker, submitResult.message);
|
||||
}
|
||||
} catch {
|
||||
|
|
@ -312,32 +234,27 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
|||
current
|
||||
? {
|
||||
...current,
|
||||
visionHint: setHint,
|
||||
message: `No "${setHint}" printing in our catalog. Tap "My card isn't listed" to save for admin review.`,
|
||||
visionHint: action.setHint,
|
||||
message: action.hintMessage,
|
||||
}
|
||||
: current
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (filtered.length === 1) {
|
||||
handleDisambiguationPick(filtered[0]);
|
||||
return;
|
||||
}
|
||||
|
||||
if (filtered.length > 1 && filtered.length < disambiguation.candidates.length) {
|
||||
break;
|
||||
case 'narrow':
|
||||
setDisambiguation((current) =>
|
||||
current
|
||||
? {
|
||||
...current,
|
||||
candidates: filtered,
|
||||
message: `Narrowed to ${filtered.length} printings matching "${setHint}".`,
|
||||
visionHint: setHint,
|
||||
candidates: action.filtered,
|
||||
message: action.message,
|
||||
visionHint: action.visionHint,
|
||||
}
|
||||
: current
|
||||
);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Disambiguation vision refine failed:', error);
|
||||
|
|
@ -350,7 +267,6 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
|||
// eslint-disable-next-line react-hooks/exhaustive-deps -- refine runs per disambiguation session, not per handler identity
|
||||
}, [disambiguation?.cardTracker?.id, disambiguation?.imageData]);
|
||||
|
||||
// Server-side card identification
|
||||
const verifyCardShape = async (cardTracker) => {
|
||||
if (!videoRef.current || !canvasRef.current || cardTracker.status !== 'detecting') return;
|
||||
if (disambiguation) return;
|
||||
|
|
@ -362,86 +278,30 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
|||
try {
|
||||
cardTracker.scanAttempts++;
|
||||
|
||||
const video = videoRef.current;
|
||||
const canvas = canvasRef.current;
|
||||
const ctx = canvas.getContext('2d');
|
||||
const identification = await identifyTrackedCardCapture({
|
||||
video: videoRef.current,
|
||||
canvas: canvasRef.current,
|
||||
cardTracker,
|
||||
authHeaders: getScanAuthHeaders(),
|
||||
visionCooldownUntilMs: visionCooldownUntilRef.current,
|
||||
});
|
||||
|
||||
const { x, y, width, height } = cardTracker.bounds;
|
||||
const margin = 20;
|
||||
|
||||
canvas.width = width + margin * 2;
|
||||
canvas.height = height + margin * 2;
|
||||
|
||||
ctx.drawImage(
|
||||
video,
|
||||
Math.max(0, x - margin), Math.max(0, y - margin),
|
||||
width + margin * 2, height + margin * 2,
|
||||
0, 0,
|
||||
canvas.width, canvas.height
|
||||
);
|
||||
|
||||
const imageData = canvas.toDataURL('image/jpeg', 0.8);
|
||||
|
||||
const authHeaders = {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
||||
};
|
||||
|
||||
// Layer 1: local OCR + pg_trgm catalog match (no vision LLM)
|
||||
try {
|
||||
const { recognizeCardNameStrip } = await import('../lib/ocr-worker.js');
|
||||
const ocr = await recognizeCardNameStrip(imageData);
|
||||
|
||||
if (ocr.text.length >= 3) {
|
||||
const l1Response = await fetch('/api/cards/identify-by-text', {
|
||||
method: 'POST',
|
||||
headers: authHeaders,
|
||||
body: JSON.stringify({
|
||||
ocrText: ocr.text,
|
||||
ocrConfidence: ocr.confidence,
|
||||
}),
|
||||
});
|
||||
|
||||
if (l1Response.ok) {
|
||||
const l1Result = await l1Response.json();
|
||||
if (!l1Result.escalate) {
|
||||
cardTracker.status = 'confirmed';
|
||||
await processIdentifyResponse(cardTracker, imageData, l1Result);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (l1Error) {
|
||||
console.warn('Layer-1 OCR path failed, escalating to vision:', l1Error);
|
||||
}
|
||||
|
||||
// Layer 2: vision via AI Gateway (skip while rate-limited)
|
||||
if (Date.now() < visionCooldownUntilRef.current) {
|
||||
if (identification.retry) {
|
||||
cardTracker.status = 'detecting';
|
||||
return;
|
||||
}
|
||||
|
||||
const response = await fetch('/api/scan/identify', {
|
||||
method: 'POST',
|
||||
headers: authHeaders,
|
||||
body: JSON.stringify({ imageData }),
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
visionCooldownUntilRef.current = Date.now() + 60_000;
|
||||
if (identification.rateLimited) {
|
||||
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
|
||||
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
||||
cardTracker.status = 'negative';
|
||||
return;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const errBody = await response.json().catch(() => ({}));
|
||||
throw new Error(errBody.error || `Scan identify failed: ${response.status}`);
|
||||
if (identification.handled && identification.outcome) {
|
||||
cardTracker.status = 'confirmed';
|
||||
await applyIdentifyOutcome(cardTracker, identification.imageData, identification.outcome);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
cardTracker.status = 'confirmed';
|
||||
await processIdentifyResponse(cardTracker, imageData, result);
|
||||
} catch (error) {
|
||||
console.error(`Error verifying card ${cardTracker.id}:`, error);
|
||||
cardTracker.status = 'negative';
|
||||
|
|
@ -855,4 +715,4 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
|||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
306
lib/scanner-card-identify.js
Normal file
306
lib/scanner-card-identify.js
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
/** Default margin (px) around tracked bounds when cropping a card capture. */
|
||||
export const CAPTURE_MARGIN_PX = 20;
|
||||
|
||||
/** Vision rate-limit backoff duration (ms). */
|
||||
export const VISION_RATE_LIMIT_MS = 60_000;
|
||||
|
||||
export function getScanAuthHeaders() {
|
||||
return {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
||||
};
|
||||
}
|
||||
|
||||
export function candidateMatchesSetHint(candidate, setName, setCode) {
|
||||
if (!setName && !setCode) return true;
|
||||
const hint = (setName || setCode || '').toLowerCase();
|
||||
const setNameLower = (candidate.set_name || '').toLowerCase();
|
||||
const setCodeLower = (candidate.set_code || '').toLowerCase();
|
||||
return setNameLower.includes(hint) || hint.includes(setNameLower) || setCodeLower === hint;
|
||||
}
|
||||
|
||||
export function buildOcrMetaFromIdentifyResult(result) {
|
||||
return {
|
||||
confidence: result.ocr?.confidence ?? result.card?.ocr?.confidence,
|
||||
rawText: result.ocr?.rawText ?? result.card?.ocr?.rawText,
|
||||
abilities: result.card?.ocr?.abilities || [],
|
||||
hp: result.card?.hp,
|
||||
manaCost: result.card?.mana_cost,
|
||||
cardName: result.ocr?.cardName,
|
||||
query: result.ocr?.query,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildScannedCardPayload(finalCard, { imageData, scanImageUrl, ocrMeta = {} }) {
|
||||
return {
|
||||
name: finalCard.name,
|
||||
set: finalCard.set_name,
|
||||
setCode: finalCard.set_code,
|
||||
cardNumber: finalCard.card_number,
|
||||
game: finalCard.game,
|
||||
cardType: finalCard.card_type,
|
||||
rarity: finalCard.rarity,
|
||||
hp: finalCard.hp || ocrMeta.hp,
|
||||
manaCost: finalCard.mana_cost || ocrMeta.manaCost,
|
||||
abilities: ocrMeta.abilities || [],
|
||||
ocrText: ocrMeta.rawText,
|
||||
confidence: ocrMeta.confidence,
|
||||
capturedImage: imageData,
|
||||
scanImageUrl,
|
||||
image_url: finalCard.image_url,
|
||||
databaseId: finalCard.id,
|
||||
isExisting: true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a server identify response to a UI action the scanner component applies.
|
||||
* @returns {{ type: string, cardStatus: string, [key: string]: unknown }}
|
||||
*/
|
||||
export function resolveIdentifyOutcome(result) {
|
||||
if (!result.isCard) {
|
||||
return {
|
||||
type: 'error',
|
||||
cardStatus: 'negative',
|
||||
message: result.reason || 'No trading card detected',
|
||||
};
|
||||
}
|
||||
|
||||
const ocrMeta = buildOcrMetaFromIdentifyResult(result);
|
||||
|
||||
if (result.card) {
|
||||
return { type: 'emit', cardStatus: 'confirmed', card: result.card, ocrMeta };
|
||||
}
|
||||
|
||||
if (result.needsUserSelection && result.matches?.length) {
|
||||
return {
|
||||
type: 'disambiguation',
|
||||
cardStatus: 'confirmed',
|
||||
matches: result.matches,
|
||||
ocrMeta,
|
||||
message: result.message,
|
||||
};
|
||||
}
|
||||
|
||||
if (result.needsReview) {
|
||||
return {
|
||||
type: 'notice',
|
||||
cardStatus: 'confirmed',
|
||||
message: result.message || 'Scan saved for admin review.',
|
||||
};
|
||||
}
|
||||
|
||||
if (result.needsUserInput) {
|
||||
return {
|
||||
type: 'error',
|
||||
cardStatus: 'negative',
|
||||
message: result.message || 'Could not identify card — try again or submit for review.',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'error',
|
||||
cardStatus: 'negative',
|
||||
message: 'Could not identify card from scan.',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* After a vision refine call during disambiguation, decide the next UI step.
|
||||
*/
|
||||
export function resolveDisambiguationRefineAction(result, { candidates }) {
|
||||
if (result.needsReview) {
|
||||
return { type: 'review_submitted', message: result.message };
|
||||
}
|
||||
|
||||
if (result.card) {
|
||||
return {
|
||||
type: 'emit',
|
||||
card: result.card,
|
||||
ocrMeta: {
|
||||
confidence: result.ocr?.confidence,
|
||||
rawText: result.ocr?.rawText,
|
||||
abilities: result.card?.ocr?.abilities || [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const setHint = result.ocr?.setName || result.ocr?.setCode;
|
||||
if (!result.matches?.length || !setHint) {
|
||||
return { type: 'noop' };
|
||||
}
|
||||
|
||||
const filtered = candidates.filter((candidate) =>
|
||||
candidateMatchesSetHint(candidate, result.ocr.setName, result.ocr.setCode)
|
||||
);
|
||||
|
||||
if (filtered.length === 0) {
|
||||
return {
|
||||
type: 'catalog_gap',
|
||||
setHint,
|
||||
submitPayload: {
|
||||
name: result.ocr?.cardName || candidates[0]?.name,
|
||||
candidateCardIds: candidates.map((c) => c.id),
|
||||
},
|
||||
hintMessage: `No "${setHint}" printing in our catalog. Tap "My card isn't listed" to save for admin review.`,
|
||||
};
|
||||
}
|
||||
|
||||
if (filtered.length === 1) {
|
||||
return { type: 'pick', candidate: filtered[0] };
|
||||
}
|
||||
|
||||
if (filtered.length > 1 && filtered.length < candidates.length) {
|
||||
return {
|
||||
type: 'narrow',
|
||||
filtered,
|
||||
visionHint: setHint,
|
||||
message: `Narrowed to ${filtered.length} printings matching "${setHint}".`,
|
||||
};
|
||||
}
|
||||
|
||||
return { type: 'noop' };
|
||||
}
|
||||
|
||||
/** Crop a tracked card region from the live video frame; returns a JPEG data URL. */
|
||||
export function captureCardRegionFromVideo(video, canvas, bounds, margin = CAPTURE_MARGIN_PX) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
const { x, y, width, height } = bounds;
|
||||
|
||||
canvas.width = width + margin * 2;
|
||||
canvas.height = height + margin * 2;
|
||||
|
||||
ctx.drawImage(
|
||||
video,
|
||||
Math.max(0, x - margin),
|
||||
Math.max(0, y - margin),
|
||||
width + margin * 2,
|
||||
height + margin * 2,
|
||||
0,
|
||||
0,
|
||||
canvas.width,
|
||||
canvas.height
|
||||
);
|
||||
|
||||
return canvas.toDataURL('image/jpeg', 0.8);
|
||||
}
|
||||
|
||||
export async function submitScanForReview({ imageData, name, candidateCardIds, authHeaders }) {
|
||||
const response = await fetch('/api/scan/submit-for-review', {
|
||||
method: 'POST',
|
||||
headers: authHeaders,
|
||||
body: JSON.stringify({ imageData, name, candidateCardIds }),
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
return { ok: false, rateLimited: true };
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const errBody = await response.json().catch(() => ({}));
|
||||
throw new Error(errBody.error || 'Failed to submit scan for review');
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
return { ok: true, message: result.message };
|
||||
}
|
||||
|
||||
export async function fetchIdentifyByText({ ocrText, ocrConfidence, authHeaders }) {
|
||||
const response = await fetch('/api/cards/identify-by-text', {
|
||||
method: 'POST',
|
||||
headers: authHeaders,
|
||||
body: JSON.stringify({ ocrText, ocrConfidence }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
return { ok: true, result };
|
||||
}
|
||||
|
||||
export async function fetchVisionIdentify(imageData, authHeaders) {
|
||||
const response = await fetch('/api/scan/identify', {
|
||||
method: 'POST',
|
||||
headers: authHeaders,
|
||||
body: JSON.stringify({ imageData }),
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
return { ok: false, rateLimited: true };
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const errBody = await response.json().catch(() => ({}));
|
||||
throw new Error(errBody.error || `Scan identify failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
return { ok: true, result };
|
||||
}
|
||||
|
||||
/**
|
||||
* Layer 1: local OCR name strip + pg_trgm catalog match.
|
||||
* Returns { handled: true, outcome } when L1 resolves without escalation.
|
||||
*/
|
||||
export async function tryLayer1TextIdentify(imageData, authHeaders) {
|
||||
const { recognizeCardNameStrip } = await import('./ocr-worker.js');
|
||||
const ocr = await recognizeCardNameStrip(imageData);
|
||||
|
||||
if (ocr.text.length < 3) {
|
||||
return { handled: false };
|
||||
}
|
||||
|
||||
const l1 = await fetchIdentifyByText({
|
||||
ocrText: ocr.text,
|
||||
ocrConfidence: ocr.confidence,
|
||||
authHeaders,
|
||||
});
|
||||
|
||||
if (!l1.ok || l1.result.escalate) {
|
||||
return { handled: false };
|
||||
}
|
||||
|
||||
return {
|
||||
handled: true,
|
||||
outcome: resolveIdentifyOutcome(l1.result),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run Layer 1 then Layer 2 identification for a tracked card capture.
|
||||
*/
|
||||
export async function identifyTrackedCardCapture({
|
||||
video,
|
||||
canvas,
|
||||
cardTracker,
|
||||
authHeaders,
|
||||
visionCooldownUntilMs = 0,
|
||||
}) {
|
||||
const imageData = captureCardRegionFromVideo(video, canvas, cardTracker.bounds);
|
||||
|
||||
try {
|
||||
const l1 = await tryLayer1TextIdentify(imageData, authHeaders);
|
||||
if (l1.handled) {
|
||||
return { imageData, ...l1 };
|
||||
}
|
||||
} catch (l1Error) {
|
||||
console.warn('Layer-1 OCR path failed, escalating to vision:', l1Error);
|
||||
}
|
||||
|
||||
if (Date.now() < visionCooldownUntilMs) {
|
||||
return { imageData, retry: true };
|
||||
}
|
||||
|
||||
const vision = await fetchVisionIdentify(imageData, authHeaders);
|
||||
if (vision.rateLimited) {
|
||||
return { imageData, rateLimited: true };
|
||||
}
|
||||
|
||||
return {
|
||||
imageData,
|
||||
handled: true,
|
||||
outcome: resolveIdentifyOutcome(vision.result),
|
||||
};
|
||||
}
|
||||
147
test/lib/scanner-card-identify.test.js
Normal file
147
test/lib/scanner-card-identify.test.js
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildOcrMetaFromIdentifyResult,
|
||||
buildScannedCardPayload,
|
||||
candidateMatchesSetHint,
|
||||
resolveDisambiguationRefineAction,
|
||||
resolveIdentifyOutcome,
|
||||
} from '../../lib/scanner-card-identify.js';
|
||||
|
||||
describe('candidateMatchesSetHint', () => {
|
||||
it('matches when set code equals hint', () => {
|
||||
expect(candidateMatchesSetHint({ set_name: 'Base Set', set_code: 'BS' }, null, 'BS')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when no hint is provided', () => {
|
||||
expect(candidateMatchesSetHint({ set_name: 'Any Set' }, null, null)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false for unrelated sets', () => {
|
||||
expect(candidateMatchesSetHint({ set_name: 'Jungle', set_code: 'JU' }, 'Base Set', null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveIdentifyOutcome', () => {
|
||||
it('returns emit when a single card is matched', () => {
|
||||
const card = { id: 1, name: 'Pikachu', mana_cost: '1R' };
|
||||
const outcome = resolveIdentifyOutcome({
|
||||
isCard: true,
|
||||
card,
|
||||
ocr: { confidence: 90, rawText: 'Pikachu' },
|
||||
});
|
||||
|
||||
expect(outcome).toMatchObject({
|
||||
type: 'emit',
|
||||
cardStatus: 'confirmed',
|
||||
card,
|
||||
});
|
||||
expect(outcome.ocrMeta.confidence).toBe(90);
|
||||
});
|
||||
|
||||
it('returns disambiguation when multiple printings match', () => {
|
||||
const matches = [{ id: 1, name: 'Lightning Bolt' }, { id: 2, name: 'Lightning Bolt' }];
|
||||
const outcome = resolveIdentifyOutcome({
|
||||
isCard: true,
|
||||
needsUserSelection: true,
|
||||
matches,
|
||||
message: 'Pick one',
|
||||
});
|
||||
|
||||
expect(outcome).toEqual({
|
||||
type: 'disambiguation',
|
||||
cardStatus: 'confirmed',
|
||||
matches,
|
||||
ocrMeta: expect.objectContaining({ abilities: [] }),
|
||||
message: 'Pick one',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns error when the frame is not a card', () => {
|
||||
expect(resolveIdentifyOutcome({ isCard: false, reason: 'Blurry' })).toEqual({
|
||||
type: 'error',
|
||||
cardStatus: 'negative',
|
||||
message: 'Blurry',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildScannedCardPayload', () => {
|
||||
it('maps catalog fields and OCR metadata into the scanner queue shape', () => {
|
||||
const payload = buildScannedCardPayload(
|
||||
{
|
||||
id: 42,
|
||||
name: 'Seel',
|
||||
set_name: 'Perfect Order',
|
||||
set_code: 'PO',
|
||||
card_number: '015/208',
|
||||
game: 'pokemon',
|
||||
card_type: 'Pokemon',
|
||||
rarity: 'Common',
|
||||
image_url: 'https://example.com/seel.jpg',
|
||||
},
|
||||
{
|
||||
imageData: 'data:image/jpeg;base64,abc',
|
||||
scanImageUrl: 'https://blob.example/seel.jpg',
|
||||
ocrMeta: { rawText: 'Seel', confidence: 88, abilities: ['Freeze-Dry'] },
|
||||
}
|
||||
);
|
||||
|
||||
expect(payload).toMatchObject({
|
||||
name: 'Seel',
|
||||
set: 'Perfect Order',
|
||||
databaseId: 42,
|
||||
scanImageUrl: 'https://blob.example/seel.jpg',
|
||||
ocrText: 'Seel',
|
||||
confidence: 88,
|
||||
abilities: ['Freeze-Dry'],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveDisambiguationRefineAction', () => {
|
||||
const candidates = [
|
||||
{ id: 1, name: 'Bolt', set_name: 'Alpha', set_code: 'LEA' },
|
||||
{ id: 2, name: 'Bolt', set_name: 'Beta', set_code: 'LEB' },
|
||||
];
|
||||
|
||||
it('auto-picks when vision narrows to one printing', () => {
|
||||
const action = resolveDisambiguationRefineAction(
|
||||
{
|
||||
matches: [{ id: 1 }],
|
||||
ocr: { setCode: 'LEA' },
|
||||
},
|
||||
{ candidates }
|
||||
);
|
||||
|
||||
expect(action).toEqual({ type: 'pick', candidate: candidates[0] });
|
||||
});
|
||||
|
||||
it('requests admin review path when set hint misses the catalog', () => {
|
||||
const action = resolveDisambiguationRefineAction(
|
||||
{
|
||||
matches: [{}],
|
||||
ocr: { setName: 'Unknown Set', cardName: 'Bolt' },
|
||||
},
|
||||
{ candidates }
|
||||
);
|
||||
|
||||
expect(action.type).toBe('catalog_gap');
|
||||
expect(action.submitPayload.candidateCardIds).toEqual([1, 2]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildOcrMetaFromIdentifyResult', () => {
|
||||
it('prefers top-level OCR fields over nested card OCR', () => {
|
||||
expect(
|
||||
buildOcrMetaFromIdentifyResult({
|
||||
ocr: { confidence: 95, rawText: 'From top' },
|
||||
card: { ocr: { confidence: 50, rawText: 'From card', abilities: ['Flying'] }, hp: 4 },
|
||||
})
|
||||
).toMatchObject({
|
||||
confidence: 95,
|
||||
rawText: 'From top',
|
||||
abilities: ['Flying'],
|
||||
hp: 4,
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue