diff --git a/Dockerfile b/Dockerfile index 4c06273..2323d68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,10 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static COPY --from=builder /app/prisma ./prisma COPY --from=builder /app/prisma.config.ts ./prisma.config.ts +COPY --from=builder /app/node_modules/prisma ./node_modules/prisma +COPY --from=builder /app/node_modules/@prisma ./node_modules/@prisma +COPY --from=builder /app/node_modules/dotenv ./node_modules/dotenv +COPY --from=builder /app/start.sh ./start.sh RUN mkdir -p /data/watch && chown -R nextjs:nodejs /data @@ -41,4 +45,4 @@ EXPOSE 3000 ENV PORT=3000 ENV HOSTNAME="0.0.0.0" -CMD ["node", "server.js"] +CMD ["sh", "start.sh"] diff --git a/package-lock.json b/package-lock.json index 22a1ce1..05470e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "echos-ocr", "version": "0.1.0", + "hasInstallScript": true, "dependencies": { "@aws-sdk/client-s3": "^3.1005.0", "@aws-sdk/s3-request-presigner": "^3.1005.0", @@ -19,6 +20,7 @@ "clsx": "^2.1.1", "cmdk": "^1.1.1", "date-fns": "^4.1.0", + "dotenv": "^17.3.1", "lucide-react": "^0.577.0", "minio": "^8.0.7", "next": "16.1.6", diff --git a/package.json b/package.json index 8dfcaab..f4745a0 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "clsx": "^2.1.1", "cmdk": "^1.1.1", "date-fns": "^4.1.0", + "dotenv": "^17.3.1", "lucide-react": "^0.577.0", "minio": "^8.0.7", "next": "16.1.6", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 3ba9ca5..6c558ec 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -79,4 +79,7 @@ model AppSettings { model String @default("llava:7b") watchDir String @default("") watching Boolean @default(false) + + sourceRetentionDays Int @default(30) + imageRetentionDays Int @default(180) } diff --git a/src/app/api/cleanup/route.ts b/src/app/api/cleanup/route.ts new file mode 100644 index 0000000..d3faa0a --- /dev/null +++ b/src/app/api/cleanup/route.ts @@ -0,0 +1,134 @@ +import { NextRequest, NextResponse } from "next/server"; +import { prisma } from "@/lib/db"; +import { deleteObject, listObjects } from "@/lib/minio"; + +export async function POST(request: NextRequest) { + try { + const body = await request.json().catch(() => ({})); + const dryRun = body.dryRun === true; + + let settings = await prisma.appSettings.findUnique({ + where: { id: "singleton" }, + }); + if (!settings) { + settings = await prisma.appSettings.create({ + data: { id: "singleton" }, + }); + } + + const now = new Date(); + const sourcesCutoff = new Date(now.getTime() - settings.sourceRetentionDays * 86400000); + const imagesCutoff = new Date(now.getTime() - settings.imageRetentionDays * 86400000); + + const results = { + sourcesDeleted: 0, + imagesDeleted: 0, + cardsUpdated: 0, + jobsPurged: 0, + dryRun, + }; + + // 1. Purge source PDFs older than sourceRetentionDays + const sourceKeys = await listObjects("sources/"); + const jobsWithSources = await prisma.processingJob.findMany({ + where: { createdAt: { lt: sourcesCutoff }, status: "complete" }, + select: { id: true }, + }); + const oldJobIds = new Set(jobsWithSources.map((j) => j.id)); + + for (const key of sourceKeys) { + const jobId = key.split("/")[1]; + if (jobId && oldJobIds.has(jobId)) { + if (!dryRun) await deleteObject(key); + results.sourcesDeleted++; + } + } + + // 2. Purge card images older than imageRetentionDays + const oldCards = await prisma.responseCard.findMany({ + where: { createdAt: { lt: imagesCutoff } }, + select: { id: true, frontImagePath: true, backImagePath: true }, + }); + + for (const card of oldCards) { + if (card.frontImagePath) { + if (!dryRun) await deleteObject(card.frontImagePath); + results.imagesDeleted++; + } + if (card.backImagePath) { + if (!dryRun) await deleteObject(card.backImagePath); + results.imagesDeleted++; + } + if (!dryRun) { + await prisma.responseCard.update({ + where: { id: card.id }, + data: { frontImagePath: null, backImagePath: null }, + }); + } + results.cardsUpdated++; + } + + // 3. Purge completed processing jobs older than sourceRetentionDays + if (!dryRun) { + const deleted = await prisma.processingJob.deleteMany({ + where: { createdAt: { lt: sourcesCutoff }, status: { in: ["complete", "error"] } }, + }); + results.jobsPurged = deleted.count; + } else { + results.jobsPurged = await prisma.processingJob.count({ + where: { createdAt: { lt: sourcesCutoff }, status: { in: ["complete", "error"] } }, + }); + } + + return NextResponse.json(results); + } catch (error) { + console.error("[cleanup POST]", error); + return NextResponse.json( + { error: "Cleanup failed" }, + { status: 500 } + ); + } +} + +export async function GET() { + try { + let settings = await prisma.appSettings.findUnique({ + where: { id: "singleton" }, + }); + if (!settings) { + settings = await prisma.appSettings.create({ + data: { id: "singleton" }, + }); + } + + const now = new Date(); + const sourcesCutoff = new Date(now.getTime() - settings.sourceRetentionDays * 86400000); + const imagesCutoff = new Date(now.getTime() - settings.imageRetentionDays * 86400000); + + const sourcesEligible = await prisma.processingJob.count({ + where: { createdAt: { lt: sourcesCutoff }, status: "complete" }, + }); + const imagesEligible = await prisma.responseCard.count({ + where: { + createdAt: { lt: imagesCutoff }, + OR: [ + { frontImagePath: { not: null } }, + { backImagePath: { not: null } }, + ], + }, + }); + + return NextResponse.json({ + sourceRetentionDays: settings.sourceRetentionDays, + imageRetentionDays: settings.imageRetentionDays, + sourcesEligible, + imagesEligible, + }); + } catch (error) { + console.error("[cleanup GET]", error); + return NextResponse.json( + { error: "Failed to check cleanup status" }, + { status: 500 } + ); + } +} diff --git a/src/app/api/settings/route.ts b/src/app/api/settings/route.ts index 60d5e73..b458f52 100644 --- a/src/app/api/settings/route.ts +++ b/src/app/api/settings/route.ts @@ -33,6 +33,8 @@ export async function PUT(request: NextRequest) { if (body.model != null) data.model = String(body.model); if (body.watchDir != null) data.watchDir = String(body.watchDir); if (body.watching != null) data.watching = Boolean(body.watching); + if (body.sourceRetentionDays != null) data.sourceRetentionDays = Math.max(1, parseInt(String(body.sourceRetentionDays)) || 30); + if (body.imageRetentionDays != null) data.imageRetentionDays = Math.max(1, parseInt(String(body.imageRetentionDays)) || 180); const settings = await prisma.appSettings.upsert({ where: { id: "singleton" }, diff --git a/src/app/settings/page.tsx b/src/app/settings/page.tsx index a85b438..14eb06e 100644 --- a/src/app/settings/page.tsx +++ b/src/app/settings/page.tsx @@ -2,7 +2,7 @@ import * as React from "react"; import { toast } from "sonner"; -import { Loader2, Save, Wifi, WifiOff } from "lucide-react"; +import { Loader2, Save, Wifi, WifiOff, Trash2 } from "lucide-react"; import { Header } from "@/components/layout/header"; import { Button } from "@/components/ui/button"; @@ -16,6 +16,8 @@ type Settings = { model: string; watchDir: string; watching: boolean; + sourceRetentionDays: number; + imageRetentionDays: number; }; export default function SettingsPage() { @@ -24,10 +26,23 @@ export default function SettingsPage() { model: "", watchDir: "", watching: false, + sourceRetentionDays: 30, + imageRetentionDays: 180, }); const [loading, setLoading] = React.useState(true); const [saving, setSaving] = React.useState(false); const [ollamaStatus, setOllamaStatus] = React.useState<"unknown" | "connected" | "error">("unknown"); + const [cleanupStatus, setCleanupStatus] = React.useState<{ sourcesEligible: number; imagesEligible: number } | null>(null); + const [cleaning, setCleaning] = React.useState(false); + + const fetchCleanupStatus = React.useCallback(() => { + fetch("/api/cleanup") + .then((r) => r.json()) + .then((data) => { + if (data.sourcesEligible !== undefined) setCleanupStatus(data); + }) + .catch(() => {}); + }, []); React.useEffect(() => { fetch("/api/settings") @@ -39,7 +54,8 @@ export default function SettingsPage() { .catch(() => { setLoading(false); }); - }, []); + fetchCleanupStatus(); + }, [fetchCleanupStatus]); const handleSave = async () => { setSaving(true); @@ -94,6 +110,27 @@ export default function SettingsPage() { } }; + const runCleanup = async () => { + setCleaning(true); + try { + const res = await fetch("/api/cleanup", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ dryRun: false }), + }); + if (!res.ok) throw new Error(); + const data = await res.json(); + toast.success( + `Cleanup complete: ${data.sourcesDeleted} source files, ${data.imagesDeleted} images, ${data.jobsPurged} jobs removed` + ); + fetchCleanupStatus(); + } catch { + toast.error("Cleanup failed"); + } finally { + setCleaning(false); + } + }; + if (loading) { return (
@@ -198,8 +235,80 @@ export default function SettingsPage() { + {/* Storage & Cleanup */} + + + Storage & Cleanup + Auto-purge uploaded files and images to save storage + + +
+ + + setSettings((s) => ({ + ...s, + sourceRetentionDays: parseInt(e.target.value) || 30, + })) + } + /> +

+ Original uploaded PDFs are deleted after this many days. Card data is kept. +

+
+
+ + + setSettings((s) => ({ + ...s, + imageRetentionDays: parseInt(e.target.value) || 180, + })) + } + /> +

+ Scanned card images are removed after this many days. Card data is kept. +

+
+
+
+
+ {cleanupStatus ? ( + <> + {cleanupStatus.sourcesEligible} source files + {" and "} + {cleanupStatus.imagesEligible} card images eligible + + ) : ( + "Checking..." + )} +
+ +
+
+ + + {/* Monday.com Integration (placeholder) */} - + Monday.com Integration Push scanned card data to Monday.com boards (coming soon) diff --git a/src/lib/minio.ts b/src/lib/minio.ts index add1a10..92f906f 100644 --- a/src/lib/minio.ts +++ b/src/lib/minio.ts @@ -1,4 +1,4 @@ -import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from "@aws-sdk/client-s3"; +import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT || "192.168.68.105"; @@ -44,3 +44,24 @@ export async function deleteObject(key: string): Promise { new DeleteObjectCommand({ Bucket: BUCKET, Key: key }) ); } + +export async function listObjects(prefix: string): Promise { + const keys: string[] = []; + let continuationToken: string | undefined; + + do { + const res = await s3.send( + new ListObjectsV2Command({ + Bucket: BUCKET, + Prefix: prefix, + ContinuationToken: continuationToken, + }) + ); + for (const obj of res.Contents ?? []) { + if (obj.Key) keys.push(obj.Key); + } + continuationToken = res.NextContinuationToken; + } while (continuationToken); + + return keys; +} diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..e39b135 --- /dev/null +++ b/start.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +echo "Running database migrations..." +npx prisma db push --skip-generate 2>&1 || echo "Warning: prisma db push failed, tables may already exist" + +echo "Starting server..." +exec node server.js