From 1ec50d9033cb462f1d316ecd12f3eb155039ecf5 Mon Sep 17 00:00:00 2001 From: Randall Stillwell Date: Wed, 11 Mar 2026 09:24:15 -0500 Subject: [PATCH] Integrate Vercel AI SDK for multi-provider OCR Replace the raw Ollama fetch-based OCR with the Vercel AI SDK, adding support for OpenAI, Google Gemini, Anthropic, and Ollama as selectable providers from the Settings page. Uses generateText with Output.object() and Zod schemas for type-safe structured data extraction. Made-with: Cursor --- .env.example | 7 +- package-lock.json | 157 ++++++++++++++++++++++++++++++ package.json | 5 + prisma/schema.prisma | 3 + src/app/api/ai-test/route.ts | 61 ++++++++++++ src/app/api/settings/route.ts | 2 + src/app/settings/page.tsx | 149 ++++++++++++++++++++++------- src/lib/ai-ocr.ts | 175 ++++++++++++++++++++++++++++++++++ src/lib/ocr.ts | 2 +- 9 files changed, 527 insertions(+), 34 deletions(-) create mode 100644 src/app/api/ai-test/route.ts create mode 100644 src/lib/ai-ocr.ts diff --git a/.env.example b/.env.example index ac41e22..34e9e7b 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,12 @@ # Database (shared PostgreSQL on CT 102) DATABASE_URL="postgresql://echos_ocr:YOUR_PASSWORD@192.168.68.102:5432/echos_ocr" -# Ollama Vision LLM (CT 108) +# AI Provider API Keys (set the ones you need) +OPENAI_API_KEY="" +GOOGLE_GENERATIVE_AI_API_KEY="" +ANTHROPIC_API_KEY="" + +# Ollama (only needed if using Ollama as the AI provider) OLLAMA_BASE_URL="http://192.168.68.108:11434" OLLAMA_MODEL="llava:7b" diff --git a/package-lock.json b/package-lock.json index 87c32d0..7b54f6c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,12 +9,17 @@ "version": "0.1.0", "hasInstallScript": true, "dependencies": { + "@ai-sdk/anthropic": "^3.0.58", + "@ai-sdk/google": "^3.0.43", + "@ai-sdk/openai": "^3.0.41", + "@ai-sdk/openai-compatible": "^2.0.35", "@aws-sdk/client-s3": "^3.1005.0", "@aws-sdk/s3-request-presigner": "^3.1005.0", "@base-ui/react": "^1.2.0", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", "@tanstack/react-table": "^8.21.3", + "ai": "^6.0.116", "chokidar": "^5.0.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -48,6 +53,116 @@ "typescript": "^5" } }, + "node_modules/@ai-sdk/anthropic": { + "version": "3.0.58", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.58.tgz", + "integrity": "sha512-/53SACgmVukO4bkms4dpxpRlYhW8Ct6QZRe6sj1Pi5H00hYhxIrqfiLbZBGxkdRvjsBQeP/4TVGsXgH5rQeb8Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "3.0.66", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.66.tgz", + "integrity": "sha512-SIQ0YY0iMuv+07HLsZ+bB990zUJ6S4ujORAh+Jv1V2KGNn73qQKnGO0JBk+w+Res8YqOFSycwDoWcFlQrVxS4A==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19", + "@vercel/oidc": "3.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/google": { + "version": "3.0.43", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-3.0.43.tgz", + "integrity": "sha512-NGCgP5g8HBxrNdxvF8Dhww+UKfqAkZAmyYBvbu9YLoBkzAmGKDBGhVptN/oXPB5Vm0jggMdoLycZ8JReQM8Zqg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai": { + "version": "3.0.41", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.41.tgz", + "integrity": "sha512-IZ42A+FO+vuEQCVNqlnAPYQnnUpUfdJIwn1BEDOBywiEHa23fw7PahxVtlX9zm3/zMvTW4JKPzWyvAgDu+SQ2A==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai-compatible": { + "version": "2.0.35", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai-compatible/-/openai-compatible-2.0.35.tgz", + "integrity": "sha512-g3wA57IAQFb+3j4YuFndgkUdXyRETZVvbfAWM+UX7bZSxA3xjes0v3XKgIdKdekPtDGsh4ZX2byHD0gJIMPfiA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.8.tgz", + "integrity": "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "4.0.19", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.19.tgz", + "integrity": "sha512-3eG55CrSWCu2SXlqq2QCsFjo3+E7+Gmg7i/oRVoSZzIodTuDSfLb3MRje67xE9RFea73Zao7Lm4mADIfUETKGg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", @@ -2917,6 +3032,15 @@ "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", "license": "MIT" }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@prisma/adapter-pg": { "version": "7.4.2", "resolved": "https://registry.npmjs.org/@prisma/adapter-pg/-/adapter-pg-7.4.2.tgz", @@ -5352,6 +5476,15 @@ "win32" ] }, + "node_modules/@vercel/oidc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", + "integrity": "sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -5397,6 +5530,24 @@ "node": ">= 14" } }, + "node_modules/ai": { + "version": "6.0.116", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.116.tgz", + "integrity": "sha512-7yM+cTmyRLeNIXwt4Vj+mrrJgVQ9RMIW5WO0ydoLoYkewIvsMcvUmqS4j2RJTUXaF1HphwmSKUMQ/HypNRGOmA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "3.0.66", + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.19", + "@opentelemetry/api": "1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/ajv": { "version": "6.14.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", @@ -9181,6 +9332,12 @@ "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "license": "MIT" }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", diff --git a/package.json b/package.json index 40b5ffc..65d28c9 100644 --- a/package.json +++ b/package.json @@ -13,12 +13,17 @@ "postinstall": "npx prisma generate" }, "dependencies": { + "@ai-sdk/anthropic": "^3.0.58", + "@ai-sdk/google": "^3.0.43", + "@ai-sdk/openai": "^3.0.41", + "@ai-sdk/openai-compatible": "^2.0.35", "@aws-sdk/client-s3": "^3.1005.0", "@aws-sdk/s3-request-presigner": "^3.1005.0", "@base-ui/react": "^1.2.0", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", "@tanstack/react-table": "^8.21.3", + "ai": "^6.0.116", "chokidar": "^5.0.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6c558ec..1fa8c0b 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -82,4 +82,7 @@ model AppSettings { sourceRetentionDays Int @default(30) imageRetentionDays Int @default(180) + + aiProvider String @default("ollama") + aiModel String @default("") } diff --git a/src/app/api/ai-test/route.ts b/src/app/api/ai-test/route.ts new file mode 100644 index 0000000..fadf538 --- /dev/null +++ b/src/app/api/ai-test/route.ts @@ -0,0 +1,61 @@ +import { NextRequest, NextResponse } from "next/server"; +import { generateText } from "ai"; +import { openai } from "@ai-sdk/openai"; +import { google } from "@ai-sdk/google"; +import { anthropic } from "@ai-sdk/anthropic"; +import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; +import type { LanguageModel } from "ai"; + +const DEFAULT_MODELS: Record = { + openai: "gpt-4o-mini", + google: "gemini-2.5-flash", + anthropic: "claude-sonnet-4-20250514", + ollama: "llava:7b", +}; + +function getTestModel( + provider: string, + modelId: string, + ollamaUrl: string +): LanguageModel { + const resolvedModel = modelId || DEFAULT_MODELS[provider] || DEFAULT_MODELS.ollama; + + switch (provider) { + case "openai": + return openai(resolvedModel); + case "google": + return google(resolvedModel); + case "anthropic": + return anthropic(resolvedModel); + case "ollama": { + const baseURL = (ollamaUrl || process.env.OLLAMA_BASE_URL || "http://192.168.68.108:11434") + "/v1"; + const ollama = createOpenAICompatible({ name: "ollama", baseURL }); + return ollama.chatModel(resolvedModel); + } + default: + throw new Error(`Unknown provider: ${provider}`); + } +} + +export async function POST(request: NextRequest) { + try { + const body = await request.json().catch(() => ({})); + const provider = body.provider || "ollama"; + const modelId = body.model || ""; + const ollamaUrl = body.ollamaUrl || ""; + + const model = getTestModel(provider, modelId, ollamaUrl); + + const { text } = await generateText({ + model, + prompt: "Reply with exactly: OK", + maxOutputTokens: 10, + }); + + return NextResponse.json({ ok: true, response: text.trim() }); + } catch (err) { + const message = err instanceof Error ? err.message : "Unknown error"; + console.error("[ai-test]", message); + return NextResponse.json({ error: message }, { status: 500 }); + } +} diff --git a/src/app/api/settings/route.ts b/src/app/api/settings/route.ts index b458f52..80922db 100644 --- a/src/app/api/settings/route.ts +++ b/src/app/api/settings/route.ts @@ -35,6 +35,8 @@ export async function PUT(request: NextRequest) { if (body.watching != null) data.watching = Boolean(body.watching); if (body.sourceRetentionDays != null) data.sourceRetentionDays = Math.max(1, parseInt(String(body.sourceRetentionDays)) || 30); if (body.imageRetentionDays != null) data.imageRetentionDays = Math.max(1, parseInt(String(body.imageRetentionDays)) || 180); + if (body.aiProvider != null) data.aiProvider = String(body.aiProvider); + if (body.aiModel != null) data.aiModel = String(body.aiModel); const settings = await prisma.appSettings.upsert({ where: { id: "singleton" }, diff --git a/src/app/settings/page.tsx b/src/app/settings/page.tsx index 14eb06e..ee3d567 100644 --- a/src/app/settings/page.tsx +++ b/src/app/settings/page.tsx @@ -2,7 +2,7 @@ import * as React from "react"; import { toast } from "sonner"; -import { Loader2, Save, Wifi, WifiOff, Trash2 } from "lucide-react"; +import { Loader2, Save, Wifi, WifiOff, Trash2, Brain } from "lucide-react"; import { Header } from "@/components/layout/header"; import { Button } from "@/components/ui/button"; @@ -10,6 +10,20 @@ import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/com import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Badge } from "@/components/ui/badge"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; + +const AI_PROVIDERS = [ + { value: "openai", label: "OpenAI", defaultModel: "gpt-4o-mini", hint: "GPT-4o, GPT-4o-mini" }, + { value: "google", label: "Google Gemini", defaultModel: "gemini-2.5-flash", hint: "Gemini 2.5 Flash, Gemini 2.5 Pro" }, + { value: "anthropic", label: "Anthropic", defaultModel: "claude-sonnet-4-20250514", hint: "Claude Sonnet, Claude Haiku" }, + { value: "ollama", label: "Ollama (Local)", defaultModel: "llava:7b", hint: "llava:7b, moondream, llama3.2-vision" }, +] as const; type Settings = { ollamaUrl: string; @@ -18,6 +32,8 @@ type Settings = { watching: boolean; sourceRetentionDays: number; imageRetentionDays: number; + aiProvider: string; + aiModel: string; }; export default function SettingsPage() { @@ -28,10 +44,12 @@ export default function SettingsPage() { watching: false, sourceRetentionDays: 30, imageRetentionDays: 180, + aiProvider: "ollama", + aiModel: "", }); const [loading, setLoading] = React.useState(true); const [saving, setSaving] = React.useState(false); - const [ollamaStatus, setOllamaStatus] = React.useState<"unknown" | "connected" | "error">("unknown"); + const [aiTestStatus, setAiTestStatus] = React.useState<"idle" | "testing" | "success" | "error">("idle"); const [cleanupStatus, setCleanupStatus] = React.useState<{ sourcesEligible: number; imagesEligible: number } | null>(null); const [cleaning, setCleaning] = React.useState(false); @@ -74,26 +92,44 @@ export default function SettingsPage() { } }; - const testOllamaConnection = async () => { - setOllamaStatus("unknown"); + const testAiProvider = async () => { + setAiTestStatus("testing"); try { - const url = settings.ollamaUrl || "http://192.168.68.108:11434"; - const res = await fetch(`${url}/api/tags`, { - signal: AbortSignal.timeout(5000), + const res = await fetch("/api/ai-test", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + provider: settings.aiProvider, + model: settings.aiModel, + ollamaUrl: settings.ollamaUrl, + }), + signal: AbortSignal.timeout(15000), }); if (res.ok) { - setOllamaStatus("connected"); - toast.success("Connected to Ollama"); + setAiTestStatus("success"); + toast.success("AI provider connected successfully"); } else { - setOllamaStatus("error"); - toast.error("Ollama responded with an error"); + const data = await res.json().catch(() => ({})); + setAiTestStatus("error"); + toast.error(data.error || "AI provider test failed"); } } catch { - setOllamaStatus("error"); - toast.error("Cannot reach Ollama. Check the URL and ensure it's running."); + setAiTestStatus("error"); + toast.error("Cannot reach AI provider. Check your configuration and API keys."); } }; + const handleProviderChange = (value: string | null) => { + if (!value) return; + const provider = AI_PROVIDERS.find((p) => p.value === value); + setSettings((s) => ({ + ...s, + aiProvider: value, + aiModel: provider?.defaultModel || "", + })); + setAiTestStatus("idle"); + }; + const toggleWatch = async () => { try { const action = settings.watching ? "stop" : "start"; @@ -149,51 +185,100 @@ export default function SettingsPage() {
- {/* Ollama Configuration */} + {/* AI Provider */}
- Ollama Configuration - Connect to your Ollama instance for OCR processing + + + AI Provider + + Choose the AI model for OCR processing
- {ollamaStatus === "connected" && } - {ollamaStatus === "error" && } - {ollamaStatus === "connected" ? "Connected" : ollamaStatus === "error" ? "Error" : "Not tested"} + {aiTestStatus === "success" && } + {aiTestStatus === "error" && } + {aiTestStatus === "testing" && } + {aiTestStatus === "success" + ? "Connected" + : aiTestStatus === "error" + ? "Error" + : aiTestStatus === "testing" + ? "Testing..." + : "Not tested"}
- - setSettings((s) => ({ ...s, ollamaUrl: e.target.value }))} - placeholder="http://192.168.68.108:11434" - /> + +
setSettings((s) => ({ ...s, model: e.target.value }))} - placeholder="llava:7b" + value={settings.aiModel} + onChange={(e) => setSettings((s) => ({ ...s, aiModel: e.target.value }))} + placeholder={AI_PROVIDERS.find((p) => p.value === settings.aiProvider)?.defaultModel || ""} />

- Recommended: llava:7b, moondream, or llama3.2-vision + {AI_PROVIDERS.find((p) => p.value === settings.aiProvider)?.hint || ""}

- diff --git a/src/lib/ai-ocr.ts b/src/lib/ai-ocr.ts new file mode 100644 index 0000000..1bc429f --- /dev/null +++ b/src/lib/ai-ocr.ts @@ -0,0 +1,175 @@ +import { generateText, Output } from "ai"; +import { openai } from "@ai-sdk/openai"; +import { google } from "@ai-sdk/google"; +import { anthropic } from "@ai-sdk/anthropic"; +import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; +import { z } from "zod"; +import { prisma } from "./db"; +import type { LanguageModel } from "ai"; + +const DEFAULT_MODELS: Record = { + openai: "gpt-4o-mini", + google: "gemini-2.5-flash", + anthropic: "claude-sonnet-4-20250514", + ollama: "llava:7b", +}; + +async function getSettings() { + let settings = await prisma.appSettings.findUnique({ + where: { id: "singleton" }, + }); + if (!settings) { + settings = await prisma.appSettings.create({ + data: { id: "singleton" }, + }); + } + return settings; +} + +function getModelInstance( + provider: string, + modelId: string, + ollamaUrl: string +): LanguageModel { + const resolvedModel = modelId || DEFAULT_MODELS[provider] || DEFAULT_MODELS.ollama; + + switch (provider) { + case "openai": + return openai(resolvedModel); + + case "google": + return google(resolvedModel); + + case "anthropic": + return anthropic(resolvedModel); + + case "ollama": { + const baseURL = (ollamaUrl || process.env.OLLAMA_BASE_URL || "http://192.168.68.108:11434") + "/v1"; + const ollama = createOpenAICompatible({ + name: "ollama", + baseURL, + }); + return ollama.chatModel(resolvedModel); + } + + default: + throw new Error(`Unknown AI provider: ${provider}`); + } +} + +const responseCardSchema = z.object({ + name: z.string().nullable().describe("Full name as written on the card"), + gender: z.string().nullable().describe("Male or Female"), + dateOfBirth: z.string().nullable().describe("Date of birth as written"), + maritalStatus: z.string().nullable().describe("Married, Single, or Other"), + maritalStatusOther: z.string().nullable().describe("Value if Other is checked"), + visitType: z.string().nullable().describe("First/Second Time Guest or Update My Information"), + cellPhone: z.string().nullable().describe("Cell phone number"), + homePhone: z.string().nullable().describe("Home phone number"), + email: z.string().nullable().describe("Email address"), + address: z.string().nullable().describe("Street address"), + aptNumber: z.string().nullable().describe("Apartment number"), + city: z.string().nullable().describe("City"), + state: z.string().nullable().describe("State"), + zip: z.string().nullable().describe("ZIP code"), + prayerRequests: z.string().nullable().describe("Written prayer requests"), + prayerForTeam: z.boolean().describe("Whether prayer team checkbox is checked"), + prayerConfidential: z.boolean().describe("Whether confidential checkbox is checked"), + confidence: z.number().min(0).max(100).describe("Your confidence in the OCR accuracy, 0-100"), +}); + +const surveySchema = z.object({ + messageTopics: z.array(z.string()).describe( + "Checked topics from: Stress, Marriage, Revival, Addiction, Parenting, Miracles, Forgiveness, Finances, My Identity, Conflict Resolution, The Holy Spirit, Understanding The Bible, Spiritual Warfare, Sharing My Faith, Anxiety, Heaven, Spiritual Gifts" + ), + messageTopicsOther: z.string().nullable().describe("Value if Other is filled in"), + nextStep: z.array(z.string()).describe("Checked items from: Baptism, Next Steps"), + attendanceDuration: z.string().nullable().describe( + "Less than 6 months, 6 Months - 1 Year, 1-3 Years, 4-6 Years, or 7+ Years" + ), + campusPreference: z.array(z.string()).describe( + "Checked locations from: Beulah, Pace/Milton, Gulf Breeze, Warrington" + ), + campusPreferenceOther: z.string().nullable().describe("Value if Other is filled in"), + howHeard: z.array(z.string()).describe( + "Checked items from: This is my church home, Regular Attender, Drove by, Social Media, Google, Personal Invite" + ), + howHeardOther: z.string().nullable().describe("Value if Other is filled in"), + serviceAttended: z.string().nullable().describe("Service letter: A, B, C, or D"), + confidence: z.number().min(0).max(100).describe("Your confidence in the OCR accuracy, 0-100"), +}); + +const RESPONSE_SYSTEM_PROMPT = `You are analyzing a scanned church response card. This is the PERSONAL INFORMATION side. + +Extract ALL of the following fields from the image. For checkboxes, determine if they are checked or unchecked. +For handwritten text, read it as accurately as possible. + +Be precise: return null for fields you cannot read. Set prayerForTeam and prayerConfidential to false if the checkboxes are not clearly marked.`; + +const SURVEY_SYSTEM_PROMPT = `You are analyzing a scanned church Easter survey form. This is the SURVEY side. + +Extract ALL of the following fields. For checkboxes, determine if they are checked (filled/marked) or unchecked (empty). +Return empty arrays for checkbox groups where nothing is checked. + +Be precise: return null for fields you cannot read.`; + +export interface OcrResult { + data: Record; + confidence: number; + raw: string; + side: "response" | "survey"; +} + +async function extractStructured>( + model: LanguageModel, + schema: z.ZodType, + systemPrompt: string, + imageBase64: string +): Promise { + const { output, text } = await generateText({ + model, + output: Output.object({ schema }), + messages: [ + { role: "system", content: systemPrompt }, + { + role: "user", + content: [ + { type: "text", text: "Extract all data from this scanned card image." }, + { type: "image", image: Buffer.from(imageBase64, "base64") }, + ], + }, + ], + }); + + if (!output) { + throw new Error(`AI model did not return structured output. Raw text: ${(text || "").slice(0, 500)}`); + } + + return output; +} + +export async function ocrImage( + imageBase64: string, + side: "response" | "survey" +): Promise { + const settings = await getSettings(); + const provider = settings.aiProvider || "ollama"; + const modelId = settings.aiModel || ""; + const model = getModelInstance(provider, modelId, settings.ollamaUrl); + + let output: Record; + + if (side === "response") { + output = await extractStructured(model, responseCardSchema, RESPONSE_SYSTEM_PROMPT, imageBase64); + } else { + output = await extractStructured(model, surveySchema, SURVEY_SYSTEM_PROMPT, imageBase64); + } + + const confidence = typeof output.confidence === "number" ? output.confidence : 50; + const data = { ...output }; + delete data.confidence; + + return { data, confidence, raw: JSON.stringify(output), side }; +} + +export { DEFAULT_MODELS }; diff --git a/src/lib/ocr.ts b/src/lib/ocr.ts index 1e22255..0d783ac 100644 --- a/src/lib/ocr.ts +++ b/src/lib/ocr.ts @@ -1,6 +1,6 @@ import { prisma } from "./db"; import { uploadBuffer } from "./minio"; -import { ocrImage } from "./ollama"; +import { ocrImage } from "./ai-ocr"; import { pdfToImages, imageToBase64, processUploadedImage } from "./pdf"; export async function processFile(