refactor(scanner): extract card identification pipeline (Brief 3)
Move Layer-1/Layer-2 identify flow, outcome resolution, disambiguation refine helpers, and scan-for-review API calls into lib/scanner-card-identify.js. Remove unused manaSymbolSettings state from CameraScanner. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
a6813ef764
commit
b1ea422b15
3 changed files with 551 additions and 238 deletions
|
|
@ -4,6 +4,15 @@ import {
|
||||||
detectCardShapesFromFrame,
|
detectCardShapesFromFrame,
|
||||||
mergeDetectedShapesIntoTrackedCards,
|
mergeDetectedShapesIntoTrackedCards,
|
||||||
} from '../lib/scanner-card-detection.js';
|
} from '../lib/scanner-card-detection.js';
|
||||||
|
import {
|
||||||
|
buildScannedCardPayload,
|
||||||
|
getScanAuthHeaders,
|
||||||
|
identifyTrackedCardCapture,
|
||||||
|
resolveDisambiguationRefineAction,
|
||||||
|
submitScanForReview,
|
||||||
|
VISION_RATE_LIMIT_MS,
|
||||||
|
fetchVisionIdentify,
|
||||||
|
} from '../lib/scanner-card-identify.js';
|
||||||
import ScanDisambiguationDialog from './ScanDisambiguationDialog.js';
|
import ScanDisambiguationDialog from './ScanDisambiguationDialog.js';
|
||||||
|
|
||||||
export default function CameraScanner({ onCardScanned, onError }) {
|
export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
|
|
@ -29,8 +38,6 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
const lastErrorAtRef = useRef(0);
|
const lastErrorAtRef = useRef(0);
|
||||||
const disambiguationRefineRef = useRef(null);
|
const disambiguationRefineRef = useRef(null);
|
||||||
|
|
||||||
// Mana symbol settings
|
|
||||||
const [manaSymbolSettings, setManaSymbolSettings] = useState({ useSVG: false });
|
|
||||||
const [videoMetrics, setVideoMetrics] = useState({ width: 0, height: 0 });
|
const [videoMetrics, setVideoMetrics] = useState({ width: 0, height: 0 });
|
||||||
|
|
||||||
// Configure canvas contexts for optimal performance
|
// Configure canvas contexts for optimal performance
|
||||||
|
|
@ -78,25 +85,7 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
onCardScanned({
|
onCardScanned(buildScannedCardPayload(finalCard, { imageData, scanImageUrl, ocrMeta }));
|
||||||
name: finalCard.name,
|
|
||||||
set: finalCard.set_name,
|
|
||||||
setCode: finalCard.set_code,
|
|
||||||
cardNumber: finalCard.card_number,
|
|
||||||
game: finalCard.game,
|
|
||||||
cardType: finalCard.card_type,
|
|
||||||
rarity: finalCard.rarity,
|
|
||||||
hp: finalCard.hp || ocrMeta.hp,
|
|
||||||
manaCost: finalCard.mana_cost || ocrMeta.manaCost,
|
|
||||||
abilities: ocrMeta.abilities || [],
|
|
||||||
ocrText: ocrMeta.rawText,
|
|
||||||
confidence: ocrMeta.confidence,
|
|
||||||
capturedImage: imageData,
|
|
||||||
scanImageUrl,
|
|
||||||
image_url: finalCard.image_url,
|
|
||||||
databaseId: finalCard.id,
|
|
||||||
isExisting: true,
|
|
||||||
});
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleDisambiguationPick = async (candidate) => {
|
const handleDisambiguationPick = async (candidate) => {
|
||||||
|
|
@ -131,31 +120,19 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
null;
|
null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch('/api/scan/submit-for-review', {
|
const result = await submitScanForReview({
|
||||||
method: 'POST',
|
imageData,
|
||||||
headers: {
|
name: guessedName,
|
||||||
'Content-Type': 'application/json',
|
candidateCardIds: (candidates || []).map((c) => c.id),
|
||||||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
authHeaders: getScanAuthHeaders(),
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
imageData,
|
|
||||||
name: guessedName,
|
|
||||||
candidateCardIds: (candidates || []).map((c) => c.id),
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (response.status === 429) {
|
if (result.rateLimited) {
|
||||||
visionCooldownUntilRef.current = rateLimitCooldownUntil(60_000);
|
visionCooldownUntilRef.current = rateLimitCooldownUntil(VISION_RATE_LIMIT_MS);
|
||||||
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const errBody = await response.json().catch(() => ({}));
|
|
||||||
throw new Error(errBody.error || 'Failed to submit scan for review');
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await response.json();
|
|
||||||
handleReviewSubmitted(cardTracker, result.message);
|
handleReviewSubmitted(cardTracker, result.message);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
reportScannerError(error.message || 'Failed to submit scan for review');
|
reportScannerError(error.message || 'Failed to submit scan for review');
|
||||||
|
|
@ -164,61 +141,31 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const candidateMatchesSetHint = (candidate, setName, setCode) => {
|
const applyIdentifyOutcome = async (cardTracker, imageData, outcome) => {
|
||||||
if (!setName && !setCode) return true;
|
cardTracker.status = outcome.cardStatus;
|
||||||
const hint = (setName || setCode || '').toLowerCase();
|
|
||||||
const setNameLower = (candidate.set_name || '').toLowerCase();
|
|
||||||
const setCodeLower = (candidate.set_code || '').toLowerCase();
|
|
||||||
return setNameLower.includes(hint) || hint.includes(setNameLower) || setCodeLower === hint;
|
|
||||||
};
|
|
||||||
|
|
||||||
const processIdentifyResponse = async (cardTracker, imageData, result) => {
|
switch (outcome.type) {
|
||||||
if (!result.isCard) {
|
case 'emit':
|
||||||
cardTracker.status = 'negative';
|
await emitScannedCard(cardTracker, imageData, outcome.card, outcome.ocrMeta);
|
||||||
reportScannerError(result.reason || 'No trading card detected');
|
break;
|
||||||
return;
|
case 'disambiguation':
|
||||||
|
setDisambiguation({
|
||||||
|
cardTracker,
|
||||||
|
imageData,
|
||||||
|
candidates: outcome.matches,
|
||||||
|
ocrMeta: outcome.ocrMeta,
|
||||||
|
message: outcome.message,
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
case 'notice':
|
||||||
|
showScanNotice(outcome.message);
|
||||||
|
break;
|
||||||
|
case 'error':
|
||||||
|
reportScannerError(outcome.message);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
const ocrMeta = {
|
|
||||||
confidence: result.ocr?.confidence ?? result.card?.ocr?.confidence,
|
|
||||||
rawText: result.ocr?.rawText ?? result.card?.ocr?.rawText,
|
|
||||||
abilities: result.card?.ocr?.abilities || [],
|
|
||||||
hp: result.card?.hp,
|
|
||||||
manaCost: result.card?.mana_cost,
|
|
||||||
};
|
|
||||||
|
|
||||||
if (result.card) {
|
|
||||||
cardTracker.status = 'confirmed';
|
|
||||||
await emitScannedCard(cardTracker, imageData, result.card, ocrMeta);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (result.needsUserSelection && result.matches?.length) {
|
|
||||||
cardTracker.status = 'confirmed';
|
|
||||||
setDisambiguation({
|
|
||||||
cardTracker,
|
|
||||||
imageData,
|
|
||||||
candidates: result.matches,
|
|
||||||
ocrMeta,
|
|
||||||
message: result.message,
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (result.needsReview) {
|
|
||||||
cardTracker.status = 'confirmed';
|
|
||||||
showScanNotice(result.message || 'Scan saved for admin review.');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (result.needsUserInput) {
|
|
||||||
cardTracker.status = 'negative';
|
|
||||||
reportScannerError(result.message || 'Could not identify card — try again or submit for review.');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
cardTracker.status = 'negative';
|
|
||||||
reportScannerError('Could not identify card from scan.');
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const reportScannerError = (message) => {
|
const reportScannerError = (message) => {
|
||||||
|
|
@ -241,70 +188,45 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
|
|
||||||
(async () => {
|
(async () => {
|
||||||
try {
|
try {
|
||||||
const response = await fetch('/api/scan/identify', {
|
const authHeaders = getScanAuthHeaders();
|
||||||
method: 'POST',
|
const vision = await fetchVisionIdentify(disambiguation.imageData, authHeaders);
|
||||||
headers: {
|
if (cancelled) return;
|
||||||
'Content-Type': 'application/json',
|
|
||||||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
if (vision.rateLimited) {
|
||||||
},
|
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
|
||||||
body: JSON.stringify({ imageData: disambiguation.imageData }),
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const action = resolveDisambiguationRefineAction(vision.result, {
|
||||||
|
candidates: disambiguation.candidates,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (cancelled) return;
|
if (cancelled) return;
|
||||||
|
|
||||||
if (response.status === 429) {
|
switch (action.type) {
|
||||||
visionCooldownUntilRef.current = Date.now() + 60_000;
|
case 'review_submitted':
|
||||||
return;
|
handleReviewSubmitted(disambiguation.cardTracker, action.message);
|
||||||
}
|
break;
|
||||||
|
case 'emit':
|
||||||
if (!response.ok) return;
|
await emitScannedCard(
|
||||||
|
disambiguation.cardTracker,
|
||||||
const result = await response.json();
|
disambiguation.imageData,
|
||||||
if (cancelled) return;
|
action.card,
|
||||||
|
action.ocrMeta
|
||||||
if (result.needsReview) {
|
);
|
||||||
handleReviewSubmitted(disambiguation.cardTracker, result.message);
|
setDisambiguation(null);
|
||||||
return;
|
disambiguationRefineRef.current = null;
|
||||||
}
|
break;
|
||||||
|
case 'pick':
|
||||||
if (result.card) {
|
await handleDisambiguationPick(action.candidate);
|
||||||
emitScannedCard(
|
break;
|
||||||
disambiguation.cardTracker,
|
case 'catalog_gap':
|
||||||
disambiguation.imageData,
|
|
||||||
result.card,
|
|
||||||
{
|
|
||||||
confidence: result.ocr?.confidence,
|
|
||||||
rawText: result.ocr?.rawText,
|
|
||||||
abilities: result.card?.ocr?.abilities || [],
|
|
||||||
}
|
|
||||||
);
|
|
||||||
setDisambiguation(null);
|
|
||||||
disambiguationRefineRef.current = null;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const setHint = result.ocr?.setName || result.ocr?.setCode;
|
|
||||||
if (result.matches?.length && setHint) {
|
|
||||||
const filtered = disambiguation.candidates.filter((candidate) =>
|
|
||||||
candidateMatchesSetHint(candidate, result.ocr.setName, result.ocr.setCode)
|
|
||||||
);
|
|
||||||
|
|
||||||
if (filtered.length === 0 && setHint) {
|
|
||||||
try {
|
try {
|
||||||
const submitRes = await fetch('/api/scan/submit-for-review', {
|
const submitResult = await submitScanForReview({
|
||||||
method: 'POST',
|
imageData: disambiguation.imageData,
|
||||||
headers: {
|
...action.submitPayload,
|
||||||
'Content-Type': 'application/json',
|
authHeaders,
|
||||||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
imageData: disambiguation.imageData,
|
|
||||||
name: result.ocr?.cardName || disambiguation.candidates[0]?.name,
|
|
||||||
candidateCardIds: disambiguation.candidates.map((c) => c.id),
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
if (submitRes.ok) {
|
if (submitResult.ok) {
|
||||||
const submitResult = await submitRes.json();
|
|
||||||
handleReviewSubmitted(disambiguation.cardTracker, submitResult.message);
|
handleReviewSubmitted(disambiguation.cardTracker, submitResult.message);
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
|
|
@ -312,32 +234,27 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
current
|
current
|
||||||
? {
|
? {
|
||||||
...current,
|
...current,
|
||||||
visionHint: setHint,
|
visionHint: action.setHint,
|
||||||
message: `No "${setHint}" printing in our catalog. Tap "My card isn't listed" to save for admin review.`,
|
message: action.hintMessage,
|
||||||
}
|
}
|
||||||
: current
|
: current
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return;
|
break;
|
||||||
}
|
case 'narrow':
|
||||||
|
|
||||||
if (filtered.length === 1) {
|
|
||||||
handleDisambiguationPick(filtered[0]);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (filtered.length > 1 && filtered.length < disambiguation.candidates.length) {
|
|
||||||
setDisambiguation((current) =>
|
setDisambiguation((current) =>
|
||||||
current
|
current
|
||||||
? {
|
? {
|
||||||
...current,
|
...current,
|
||||||
candidates: filtered,
|
candidates: action.filtered,
|
||||||
message: `Narrowed to ${filtered.length} printings matching "${setHint}".`,
|
message: action.message,
|
||||||
visionHint: setHint,
|
visionHint: action.visionHint,
|
||||||
}
|
}
|
||||||
: current
|
: current
|
||||||
);
|
);
|
||||||
}
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn('Disambiguation vision refine failed:', error);
|
console.warn('Disambiguation vision refine failed:', error);
|
||||||
|
|
@ -350,7 +267,6 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps -- refine runs per disambiguation session, not per handler identity
|
// eslint-disable-next-line react-hooks/exhaustive-deps -- refine runs per disambiguation session, not per handler identity
|
||||||
}, [disambiguation?.cardTracker?.id, disambiguation?.imageData]);
|
}, [disambiguation?.cardTracker?.id, disambiguation?.imageData]);
|
||||||
|
|
||||||
// Server-side card identification
|
|
||||||
const verifyCardShape = async (cardTracker) => {
|
const verifyCardShape = async (cardTracker) => {
|
||||||
if (!videoRef.current || !canvasRef.current || cardTracker.status !== 'detecting') return;
|
if (!videoRef.current || !canvasRef.current || cardTracker.status !== 'detecting') return;
|
||||||
if (disambiguation) return;
|
if (disambiguation) return;
|
||||||
|
|
@ -362,86 +278,30 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
try {
|
try {
|
||||||
cardTracker.scanAttempts++;
|
cardTracker.scanAttempts++;
|
||||||
|
|
||||||
const video = videoRef.current;
|
const identification = await identifyTrackedCardCapture({
|
||||||
const canvas = canvasRef.current;
|
video: videoRef.current,
|
||||||
const ctx = canvas.getContext('2d');
|
canvas: canvasRef.current,
|
||||||
|
cardTracker,
|
||||||
|
authHeaders: getScanAuthHeaders(),
|
||||||
|
visionCooldownUntilMs: visionCooldownUntilRef.current,
|
||||||
|
});
|
||||||
|
|
||||||
const { x, y, width, height } = cardTracker.bounds;
|
if (identification.retry) {
|
||||||
const margin = 20;
|
|
||||||
|
|
||||||
canvas.width = width + margin * 2;
|
|
||||||
canvas.height = height + margin * 2;
|
|
||||||
|
|
||||||
ctx.drawImage(
|
|
||||||
video,
|
|
||||||
Math.max(0, x - margin), Math.max(0, y - margin),
|
|
||||||
width + margin * 2, height + margin * 2,
|
|
||||||
0, 0,
|
|
||||||
canvas.width, canvas.height
|
|
||||||
);
|
|
||||||
|
|
||||||
const imageData = canvas.toDataURL('image/jpeg', 0.8);
|
|
||||||
|
|
||||||
const authHeaders = {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Layer 1: local OCR + pg_trgm catalog match (no vision LLM)
|
|
||||||
try {
|
|
||||||
const { recognizeCardNameStrip } = await import('../lib/ocr-worker.js');
|
|
||||||
const ocr = await recognizeCardNameStrip(imageData);
|
|
||||||
|
|
||||||
if (ocr.text.length >= 3) {
|
|
||||||
const l1Response = await fetch('/api/cards/identify-by-text', {
|
|
||||||
method: 'POST',
|
|
||||||
headers: authHeaders,
|
|
||||||
body: JSON.stringify({
|
|
||||||
ocrText: ocr.text,
|
|
||||||
ocrConfidence: ocr.confidence,
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (l1Response.ok) {
|
|
||||||
const l1Result = await l1Response.json();
|
|
||||||
if (!l1Result.escalate) {
|
|
||||||
cardTracker.status = 'confirmed';
|
|
||||||
await processIdentifyResponse(cardTracker, imageData, l1Result);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (l1Error) {
|
|
||||||
console.warn('Layer-1 OCR path failed, escalating to vision:', l1Error);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Layer 2: vision via AI Gateway (skip while rate-limited)
|
|
||||||
if (Date.now() < visionCooldownUntilRef.current) {
|
|
||||||
cardTracker.status = 'detecting';
|
cardTracker.status = 'detecting';
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = await fetch('/api/scan/identify', {
|
if (identification.rateLimited) {
|
||||||
method: 'POST',
|
visionCooldownUntilRef.current = Date.now() + VISION_RATE_LIMIT_MS;
|
||||||
headers: authHeaders,
|
|
||||||
body: JSON.stringify({ imageData }),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (response.status === 429) {
|
|
||||||
visionCooldownUntilRef.current = Date.now() + 60_000;
|
|
||||||
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
reportScannerError('Too many scan attempts. Please wait a moment and try again.');
|
||||||
cardTracker.status = 'negative';
|
cardTracker.status = 'negative';
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (identification.handled && identification.outcome) {
|
||||||
const errBody = await response.json().catch(() => ({}));
|
cardTracker.status = 'confirmed';
|
||||||
throw new Error(errBody.error || `Scan identify failed: ${response.status}`);
|
await applyIdentifyOutcome(cardTracker, identification.imageData, identification.outcome);
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await response.json();
|
|
||||||
cardTracker.status = 'confirmed';
|
|
||||||
await processIdentifyResponse(cardTracker, imageData, result);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error verifying card ${cardTracker.id}:`, error);
|
console.error(`Error verifying card ${cardTracker.id}:`, error);
|
||||||
cardTracker.status = 'negative';
|
cardTracker.status = 'negative';
|
||||||
|
|
@ -855,4 +715,4 @@ export default function CameraScanner({ onCardScanned, onError }) {
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
306
lib/scanner-card-identify.js
Normal file
306
lib/scanner-card-identify.js
Normal file
|
|
@ -0,0 +1,306 @@
|
||||||
|
/** Default margin (px) around tracked bounds when cropping a card capture. */
|
||||||
|
export const CAPTURE_MARGIN_PX = 20;
|
||||||
|
|
||||||
|
/** Vision rate-limit backoff duration (ms). */
|
||||||
|
export const VISION_RATE_LIMIT_MS = 60_000;
|
||||||
|
|
||||||
|
export function getScanAuthHeaders() {
|
||||||
|
return {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
Authorization: `Bearer ${localStorage.getItem('auth_token')}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function candidateMatchesSetHint(candidate, setName, setCode) {
|
||||||
|
if (!setName && !setCode) return true;
|
||||||
|
const hint = (setName || setCode || '').toLowerCase();
|
||||||
|
const setNameLower = (candidate.set_name || '').toLowerCase();
|
||||||
|
const setCodeLower = (candidate.set_code || '').toLowerCase();
|
||||||
|
return setNameLower.includes(hint) || hint.includes(setNameLower) || setCodeLower === hint;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildOcrMetaFromIdentifyResult(result) {
|
||||||
|
return {
|
||||||
|
confidence: result.ocr?.confidence ?? result.card?.ocr?.confidence,
|
||||||
|
rawText: result.ocr?.rawText ?? result.card?.ocr?.rawText,
|
||||||
|
abilities: result.card?.ocr?.abilities || [],
|
||||||
|
hp: result.card?.hp,
|
||||||
|
manaCost: result.card?.mana_cost,
|
||||||
|
cardName: result.ocr?.cardName,
|
||||||
|
query: result.ocr?.query,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildScannedCardPayload(finalCard, { imageData, scanImageUrl, ocrMeta = {} }) {
|
||||||
|
return {
|
||||||
|
name: finalCard.name,
|
||||||
|
set: finalCard.set_name,
|
||||||
|
setCode: finalCard.set_code,
|
||||||
|
cardNumber: finalCard.card_number,
|
||||||
|
game: finalCard.game,
|
||||||
|
cardType: finalCard.card_type,
|
||||||
|
rarity: finalCard.rarity,
|
||||||
|
hp: finalCard.hp || ocrMeta.hp,
|
||||||
|
manaCost: finalCard.mana_cost || ocrMeta.manaCost,
|
||||||
|
abilities: ocrMeta.abilities || [],
|
||||||
|
ocrText: ocrMeta.rawText,
|
||||||
|
confidence: ocrMeta.confidence,
|
||||||
|
capturedImage: imageData,
|
||||||
|
scanImageUrl,
|
||||||
|
image_url: finalCard.image_url,
|
||||||
|
databaseId: finalCard.id,
|
||||||
|
isExisting: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Map a server identify response to a UI action the scanner component applies.
|
||||||
|
* @returns {{ type: string, cardStatus: string, [key: string]: unknown }}
|
||||||
|
*/
|
||||||
|
export function resolveIdentifyOutcome(result) {
|
||||||
|
if (!result.isCard) {
|
||||||
|
return {
|
||||||
|
type: 'error',
|
||||||
|
cardStatus: 'negative',
|
||||||
|
message: result.reason || 'No trading card detected',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const ocrMeta = buildOcrMetaFromIdentifyResult(result);
|
||||||
|
|
||||||
|
if (result.card) {
|
||||||
|
return { type: 'emit', cardStatus: 'confirmed', card: result.card, ocrMeta };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.needsUserSelection && result.matches?.length) {
|
||||||
|
return {
|
||||||
|
type: 'disambiguation',
|
||||||
|
cardStatus: 'confirmed',
|
||||||
|
matches: result.matches,
|
||||||
|
ocrMeta,
|
||||||
|
message: result.message,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.needsReview) {
|
||||||
|
return {
|
||||||
|
type: 'notice',
|
||||||
|
cardStatus: 'confirmed',
|
||||||
|
message: result.message || 'Scan saved for admin review.',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.needsUserInput) {
|
||||||
|
return {
|
||||||
|
type: 'error',
|
||||||
|
cardStatus: 'negative',
|
||||||
|
message: result.message || 'Could not identify card — try again or submit for review.',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
type: 'error',
|
||||||
|
cardStatus: 'negative',
|
||||||
|
message: 'Could not identify card from scan.',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* After a vision refine call during disambiguation, decide the next UI step.
|
||||||
|
*/
|
||||||
|
export function resolveDisambiguationRefineAction(result, { candidates }) {
|
||||||
|
if (result.needsReview) {
|
||||||
|
return { type: 'review_submitted', message: result.message };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.card) {
|
||||||
|
return {
|
||||||
|
type: 'emit',
|
||||||
|
card: result.card,
|
||||||
|
ocrMeta: {
|
||||||
|
confidence: result.ocr?.confidence,
|
||||||
|
rawText: result.ocr?.rawText,
|
||||||
|
abilities: result.card?.ocr?.abilities || [],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const setHint = result.ocr?.setName || result.ocr?.setCode;
|
||||||
|
if (!result.matches?.length || !setHint) {
|
||||||
|
return { type: 'noop' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const filtered = candidates.filter((candidate) =>
|
||||||
|
candidateMatchesSetHint(candidate, result.ocr.setName, result.ocr.setCode)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (filtered.length === 0) {
|
||||||
|
return {
|
||||||
|
type: 'catalog_gap',
|
||||||
|
setHint,
|
||||||
|
submitPayload: {
|
||||||
|
name: result.ocr?.cardName || candidates[0]?.name,
|
||||||
|
candidateCardIds: candidates.map((c) => c.id),
|
||||||
|
},
|
||||||
|
hintMessage: `No "${setHint}" printing in our catalog. Tap "My card isn't listed" to save for admin review.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (filtered.length === 1) {
|
||||||
|
return { type: 'pick', candidate: filtered[0] };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (filtered.length > 1 && filtered.length < candidates.length) {
|
||||||
|
return {
|
||||||
|
type: 'narrow',
|
||||||
|
filtered,
|
||||||
|
visionHint: setHint,
|
||||||
|
message: `Narrowed to ${filtered.length} printings matching "${setHint}".`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { type: 'noop' };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Crop a tracked card region from the live video frame; returns a JPEG data URL. */
|
||||||
|
export function captureCardRegionFromVideo(video, canvas, bounds, margin = CAPTURE_MARGIN_PX) {
|
||||||
|
const ctx = canvas.getContext('2d');
|
||||||
|
const { x, y, width, height } = bounds;
|
||||||
|
|
||||||
|
canvas.width = width + margin * 2;
|
||||||
|
canvas.height = height + margin * 2;
|
||||||
|
|
||||||
|
ctx.drawImage(
|
||||||
|
video,
|
||||||
|
Math.max(0, x - margin),
|
||||||
|
Math.max(0, y - margin),
|
||||||
|
width + margin * 2,
|
||||||
|
height + margin * 2,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
canvas.width,
|
||||||
|
canvas.height
|
||||||
|
);
|
||||||
|
|
||||||
|
return canvas.toDataURL('image/jpeg', 0.8);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function submitScanForReview({ imageData, name, candidateCardIds, authHeaders }) {
|
||||||
|
const response = await fetch('/api/scan/submit-for-review', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: authHeaders,
|
||||||
|
body: JSON.stringify({ imageData, name, candidateCardIds }),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.status === 429) {
|
||||||
|
return { ok: false, rateLimited: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errBody = await response.json().catch(() => ({}));
|
||||||
|
throw new Error(errBody.error || 'Failed to submit scan for review');
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
return { ok: true, message: result.message };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchIdentifyByText({ ocrText, ocrConfidence, authHeaders }) {
|
||||||
|
const response = await fetch('/api/cards/identify-by-text', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: authHeaders,
|
||||||
|
body: JSON.stringify({ ocrText, ocrConfidence }),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
return { ok: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
return { ok: true, result };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchVisionIdentify(imageData, authHeaders) {
|
||||||
|
const response = await fetch('/api/scan/identify', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: authHeaders,
|
||||||
|
body: JSON.stringify({ imageData }),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.status === 429) {
|
||||||
|
return { ok: false, rateLimited: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errBody = await response.json().catch(() => ({}));
|
||||||
|
throw new Error(errBody.error || `Scan identify failed: ${response.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
return { ok: true, result };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Layer 1: local OCR name strip + pg_trgm catalog match.
|
||||||
|
* Returns { handled: true, outcome } when L1 resolves without escalation.
|
||||||
|
*/
|
||||||
|
export async function tryLayer1TextIdentify(imageData, authHeaders) {
|
||||||
|
const { recognizeCardNameStrip } = await import('./ocr-worker.js');
|
||||||
|
const ocr = await recognizeCardNameStrip(imageData);
|
||||||
|
|
||||||
|
if (ocr.text.length < 3) {
|
||||||
|
return { handled: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
const l1 = await fetchIdentifyByText({
|
||||||
|
ocrText: ocr.text,
|
||||||
|
ocrConfidence: ocr.confidence,
|
||||||
|
authHeaders,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!l1.ok || l1.result.escalate) {
|
||||||
|
return { handled: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
handled: true,
|
||||||
|
outcome: resolveIdentifyOutcome(l1.result),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run Layer 1 then Layer 2 identification for a tracked card capture.
|
||||||
|
*/
|
||||||
|
export async function identifyTrackedCardCapture({
|
||||||
|
video,
|
||||||
|
canvas,
|
||||||
|
cardTracker,
|
||||||
|
authHeaders,
|
||||||
|
visionCooldownUntilMs = 0,
|
||||||
|
}) {
|
||||||
|
const imageData = captureCardRegionFromVideo(video, canvas, cardTracker.bounds);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const l1 = await tryLayer1TextIdentify(imageData, authHeaders);
|
||||||
|
if (l1.handled) {
|
||||||
|
return { imageData, ...l1 };
|
||||||
|
}
|
||||||
|
} catch (l1Error) {
|
||||||
|
console.warn('Layer-1 OCR path failed, escalating to vision:', l1Error);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Date.now() < visionCooldownUntilMs) {
|
||||||
|
return { imageData, retry: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
const vision = await fetchVisionIdentify(imageData, authHeaders);
|
||||||
|
if (vision.rateLimited) {
|
||||||
|
return { imageData, rateLimited: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
imageData,
|
||||||
|
handled: true,
|
||||||
|
outcome: resolveIdentifyOutcome(vision.result),
|
||||||
|
};
|
||||||
|
}
|
||||||
147
test/lib/scanner-card-identify.test.js
Normal file
147
test/lib/scanner-card-identify.test.js
Normal file
|
|
@ -0,0 +1,147 @@
|
||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import {
|
||||||
|
buildOcrMetaFromIdentifyResult,
|
||||||
|
buildScannedCardPayload,
|
||||||
|
candidateMatchesSetHint,
|
||||||
|
resolveDisambiguationRefineAction,
|
||||||
|
resolveIdentifyOutcome,
|
||||||
|
} from '../../lib/scanner-card-identify.js';
|
||||||
|
|
||||||
|
describe('candidateMatchesSetHint', () => {
|
||||||
|
it('matches when set code equals hint', () => {
|
||||||
|
expect(candidateMatchesSetHint({ set_name: 'Base Set', set_code: 'BS' }, null, 'BS')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns true when no hint is provided', () => {
|
||||||
|
expect(candidateMatchesSetHint({ set_name: 'Any Set' }, null, null)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for unrelated sets', () => {
|
||||||
|
expect(candidateMatchesSetHint({ set_name: 'Jungle', set_code: 'JU' }, 'Base Set', null)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('resolveIdentifyOutcome', () => {
|
||||||
|
it('returns emit when a single card is matched', () => {
|
||||||
|
const card = { id: 1, name: 'Pikachu', mana_cost: '1R' };
|
||||||
|
const outcome = resolveIdentifyOutcome({
|
||||||
|
isCard: true,
|
||||||
|
card,
|
||||||
|
ocr: { confidence: 90, rawText: 'Pikachu' },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(outcome).toMatchObject({
|
||||||
|
type: 'emit',
|
||||||
|
cardStatus: 'confirmed',
|
||||||
|
card,
|
||||||
|
});
|
||||||
|
expect(outcome.ocrMeta.confidence).toBe(90);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns disambiguation when multiple printings match', () => {
|
||||||
|
const matches = [{ id: 1, name: 'Lightning Bolt' }, { id: 2, name: 'Lightning Bolt' }];
|
||||||
|
const outcome = resolveIdentifyOutcome({
|
||||||
|
isCard: true,
|
||||||
|
needsUserSelection: true,
|
||||||
|
matches,
|
||||||
|
message: 'Pick one',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(outcome).toEqual({
|
||||||
|
type: 'disambiguation',
|
||||||
|
cardStatus: 'confirmed',
|
||||||
|
matches,
|
||||||
|
ocrMeta: expect.objectContaining({ abilities: [] }),
|
||||||
|
message: 'Pick one',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns error when the frame is not a card', () => {
|
||||||
|
expect(resolveIdentifyOutcome({ isCard: false, reason: 'Blurry' })).toEqual({
|
||||||
|
type: 'error',
|
||||||
|
cardStatus: 'negative',
|
||||||
|
message: 'Blurry',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('buildScannedCardPayload', () => {
|
||||||
|
it('maps catalog fields and OCR metadata into the scanner queue shape', () => {
|
||||||
|
const payload = buildScannedCardPayload(
|
||||||
|
{
|
||||||
|
id: 42,
|
||||||
|
name: 'Seel',
|
||||||
|
set_name: 'Perfect Order',
|
||||||
|
set_code: 'PO',
|
||||||
|
card_number: '015/208',
|
||||||
|
game: 'pokemon',
|
||||||
|
card_type: 'Pokemon',
|
||||||
|
rarity: 'Common',
|
||||||
|
image_url: 'https://example.com/seel.jpg',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
imageData: 'data:image/jpeg;base64,abc',
|
||||||
|
scanImageUrl: 'https://blob.example/seel.jpg',
|
||||||
|
ocrMeta: { rawText: 'Seel', confidence: 88, abilities: ['Freeze-Dry'] },
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(payload).toMatchObject({
|
||||||
|
name: 'Seel',
|
||||||
|
set: 'Perfect Order',
|
||||||
|
databaseId: 42,
|
||||||
|
scanImageUrl: 'https://blob.example/seel.jpg',
|
||||||
|
ocrText: 'Seel',
|
||||||
|
confidence: 88,
|
||||||
|
abilities: ['Freeze-Dry'],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('resolveDisambiguationRefineAction', () => {
|
||||||
|
const candidates = [
|
||||||
|
{ id: 1, name: 'Bolt', set_name: 'Alpha', set_code: 'LEA' },
|
||||||
|
{ id: 2, name: 'Bolt', set_name: 'Beta', set_code: 'LEB' },
|
||||||
|
];
|
||||||
|
|
||||||
|
it('auto-picks when vision narrows to one printing', () => {
|
||||||
|
const action = resolveDisambiguationRefineAction(
|
||||||
|
{
|
||||||
|
matches: [{ id: 1 }],
|
||||||
|
ocr: { setCode: 'LEA' },
|
||||||
|
},
|
||||||
|
{ candidates }
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(action).toEqual({ type: 'pick', candidate: candidates[0] });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('requests admin review path when set hint misses the catalog', () => {
|
||||||
|
const action = resolveDisambiguationRefineAction(
|
||||||
|
{
|
||||||
|
matches: [{}],
|
||||||
|
ocr: { setName: 'Unknown Set', cardName: 'Bolt' },
|
||||||
|
},
|
||||||
|
{ candidates }
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(action.type).toBe('catalog_gap');
|
||||||
|
expect(action.submitPayload.candidateCardIds).toEqual([1, 2]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('buildOcrMetaFromIdentifyResult', () => {
|
||||||
|
it('prefers top-level OCR fields over nested card OCR', () => {
|
||||||
|
expect(
|
||||||
|
buildOcrMetaFromIdentifyResult({
|
||||||
|
ocr: { confidence: 95, rawText: 'From top' },
|
||||||
|
card: { ocr: { confidence: 50, rawText: 'From card', abilities: ['Flying'] }, hp: 4 },
|
||||||
|
})
|
||||||
|
).toMatchObject({
|
||||||
|
confidence: 95,
|
||||||
|
rawText: 'From top',
|
||||||
|
abilities: ['Flying'],
|
||||||
|
hp: 4,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
Reference in a new issue