From 27bdf9d4151e2a02da4ae1dcf7022de13089f8bc Mon Sep 17 00:00:00 2001 From: Randall Stillwell Date: Tue, 7 Apr 2026 23:10:59 -0500 Subject: [PATCH] Add survey side recovery from source PDFs Card-level reprocessing only re-OCRs existing images. When the old page count bug caused survey pages to never be extracted, those images don't exist to reprocess. New /api/cards/recover-survey endpoint: - Finds all cards missing frontImagePath (survey side) - Locates their source PDFs in MinIO via the ProcessingJob - Re-extracts the correct survey page for each card - Stores the image and runs OCR to fill in survey data - Processes sequentially with rate-limit delays Settings page shows amber banner when missing backs are detected with a one-click "Recover Missing Backs" button. Made-with: Cursor --- src/app/api/cards/recover-survey/route.ts | 145 ++++++++++++++++++++++ src/app/settings/page.tsx | 49 ++++++++ 2 files changed, 194 insertions(+) create mode 100644 src/app/api/cards/recover-survey/route.ts diff --git a/src/app/api/cards/recover-survey/route.ts b/src/app/api/cards/recover-survey/route.ts new file mode 100644 index 0000000..9c44b29 --- /dev/null +++ b/src/app/api/cards/recover-survey/route.ts @@ -0,0 +1,145 @@ +import { NextResponse } from "next/server"; +import { prisma } from "@/lib/db"; +import { getBuffer, uploadBuffer } from "@/lib/minio"; +import { pdfToImages, imageToBase64 } from "@/lib/pdf"; +import { ocrImage } from "@/lib/ai-ocr"; + +const DELAY_BETWEEN_OCR_MS = 3_000; + +export async function GET() { + const missing = await prisma.responseCard.count({ + where: { frontImagePath: null, backImagePath: { not: null } }, + }); + return NextResponse.json({ missingCount: missing }); +} + +export async function POST() { + try { + const cards = await prisma.responseCard.findMany({ + where: { frontImagePath: null, backImagePath: { not: null } }, + select: { id: true, sourceFile: true, name: true }, + }); + + if (cards.length === 0) { + return NextResponse.json({ ok: true, recovered: 0, message: "No cards with missing survey sides" }); + } + + const sourceFiles = [...new Set(cards.map((c) => c.sourceFile).filter(Boolean))] as string[]; + console.log(`[recover-survey] ${cards.length} card(s) missing survey, from ${sourceFiles.length} source file(s)`); + + (async () => { + let recovered = 0; + let failed = 0; + + for (const sourceFile of sourceFiles) { + try { + const jobIdMatch = sourceFile.match(/^sources\/([^/]+)\//); + if (!jobIdMatch) { + console.warn(`[recover-survey] Cannot parse job ID from sourceFile: ${sourceFile}`); + continue; + } + const jobId = jobIdMatch[1]; + + const job = await prisma.processingJob.findUnique({ where: { id: jobId } }); + if (!job) { + console.warn(`[recover-survey] Job ${jobId} not found`); + continue; + } + + const orderedCardIds = (job.cardIds as string[] | null) ?? []; + const affectedCards = cards.filter((c) => c.sourceFile === sourceFile); + const isPdf = job.fileName.toLowerCase().endsWith(".pdf"); + + if (!isPdf) { + console.log(`[recover-survey] Skipping non-PDF source: ${sourceFile}`); + continue; + } + + let pdfBuffer: Buffer; + try { + pdfBuffer = await getBuffer(sourceFile); + } catch { + console.warn(`[recover-survey] Source PDF not found in storage: ${sourceFile}`); + continue; + } + + console.log(`[recover-survey] Re-extracting pages from ${sourceFile}`); + const pageImages = await pdfToImages(pdfBuffer); + console.log(`[recover-survey] Got ${pageImages.length} pages from ${job.fileName}`); + + for (const card of affectedCards) { + const cardIndex = orderedCardIds.indexOf(card.id); + if (cardIndex === -1) { + console.warn(`[recover-survey] Card ${card.id} not found in job ${jobId} cardIds`); + continue; + } + + const surveyPageIndex = cardIndex * 2 + 1; + if (surveyPageIndex >= pageImages.length) { + console.log(`[recover-survey] Card "${card.name}" (${card.id}): no survey page at index ${surveyPageIndex} (only ${pageImages.length} pages)`); + continue; + } + + const surveyImage = pageImages[surveyPageIndex]; + const imgKey = `images/${card.id}/survey.jpg`; + + try { + await uploadBuffer(imgKey, surveyImage.buffer, "image/jpeg"); + await prisma.responseCard.update({ + where: { id: card.id }, + data: { frontImagePath: imgKey }, + }); + + const base64 = await imageToBase64(surveyImage.buffer); + const result = await ocrImage(base64, "survey"); + + await prisma.responseCard.update({ + where: { id: card.id }, + data: { + messageTopics: result.data.messageTopics ?? [], + messageTopicsOther: asString(result.data.messageTopicsOther), + nextStep: result.data.nextStep ?? [], + attendanceDuration: asString(result.data.attendanceDuration), + campusPreference: result.data.campusPreference ?? [], + campusPreferenceOther: asString(result.data.campusPreferenceOther), + howHeard: result.data.howHeard ?? [], + howHeardOther: asString(result.data.howHeardOther), + serviceAttended: asString(result.data.serviceAttended), + }, + }); + + recovered++; + console.log(`[recover-survey] Recovered survey for "${card.name}" (${card.id})`); + } catch (err) { + failed++; + console.error(`[recover-survey] Failed OCR for card ${card.id}:`, err); + } + + await new Promise((r) => setTimeout(r, DELAY_BETWEEN_OCR_MS)); + } + } catch (err) { + console.error(`[recover-survey] Error processing source ${sourceFile}:`, err); + } + } + + console.log(`[recover-survey] Done: ${recovered} recovered, ${failed} failed`); + })().catch((err) => { + console.error("[recover-survey] Background task failed:", err); + }); + + return NextResponse.json({ + ok: true, + queued: cards.length, + sourceFiles: sourceFiles.length, + message: `Recovery started for ${cards.length} card(s) from ${sourceFiles.length} PDF(s)`, + }); + } catch (error) { + console.error("[recover-survey POST]", error); + return NextResponse.json({ error: "Recovery failed" }, { status: 500 }); + } +} + +function asString(v: unknown): string | null { + if (v === null || v === undefined) return null; + return String(v); +} diff --git a/src/app/settings/page.tsx b/src/app/settings/page.tsx index 7417783..3b02f2a 100644 --- a/src/app/settings/page.tsx +++ b/src/app/settings/page.tsx @@ -21,6 +21,7 @@ import { LayoutGrid, Globe, Check, + RefreshCw, } from "lucide-react"; import { Header } from "@/components/layout/header"; @@ -191,6 +192,8 @@ export default function SettingsPage() { const [subscribing, setSubscribing] = React.useState(false); const [pushingAll, setPushingAll] = React.useState(false); const [webhookTestStatus, setWebhookTestStatus] = React.useState<"idle" | "testing" | "success" | "error">("idle"); + const [recoverCount, setRecoverCount] = React.useState(null); + const [recovering, setRecovering] = React.useState(false); const [notifPrefs, setNotifPrefs] = React.useState(DEFAULT_NOTIFICATION_PREFS); React.useEffect(() => { @@ -226,6 +229,10 @@ export default function SettingsPage() { setLoading(false); }); fetchCleanupStatus(); + fetch("/api/cards/recover-survey") + .then((r) => r.json()) + .then((data) => { if (data.missingCount !== undefined) setRecoverCount(data.missingCount); }) + .catch(() => {}); }, [fetchCleanupStatus]); const handleSave = async () => { @@ -467,6 +474,24 @@ export default function SettingsPage() { }); }; + const recoverSurveys = async () => { + setRecovering(true); + try { + const res = await fetch("/api/cards/recover-survey", { method: "POST" }); + const data = await res.json(); + if (res.ok) { + toast.success(data.message || `Recovery started for ${data.queued} card(s)`); + setRecoverCount(0); + } else { + toast.error(data.error || "Recovery failed"); + } + } catch { + toast.error("Failed to start recovery"); + } finally { + setRecovering(false); + } + }; + const runCleanup = async () => { setCleaning(true); try { @@ -722,6 +747,30 @@ export default function SettingsPage() { + + {recoverCount !== null && recoverCount > 0 && ( +
+
+
+ {recoverCount}{" "} + card(s) are missing their survey (back) side. + + Re-extracts missing pages from the original source PDFs and runs OCR. + +
+ +
+
+ )}