From 21c863da0144d2cf2dd6ef9c82310d1dda3f4293 Mon Sep 17 00:00:00 2001 From: Randall Stillwell Date: Wed, 15 Apr 2026 23:51:44 -0500 Subject: [PATCH] Migrate to Vercel + Supabase + Upstash production stack - Replace pdf2pic/GraphicsMagick with pdfjs-dist + @napi-rs/canvas for Vercel-compatible PDF rasterization - Replace MinIO with Supabase Storage (S3-compatible); rename minio.ts to storage.ts and update all imports - Replace in-memory job queue with Upstash QStash; upload route now persists files to storage before enqueuing, /api/jobs/process handles the QStash callback - Convert email watcher from persistent IMAP connection to stateless scanInbox() polled by Vercel Cron every 2 minutes - Add FTP watcher (basic-ftp) with cron polling for scanner integration via Dreamhost FTP drop directory - Add FTP config fields to AppSettings schema - Remove folder watcher (chokidar), standalone output, Docker-only code - Update next.config.ts, middleware, instrumentation for serverless - Add vercel.json with cron schedules for email and FTP polling - Add migration scripts for database (pg_dump/restore) and storage (S3-to-S3 copy) with verification Made-with: Cursor --- .env.example | 54 ++- .gitignore | 3 + next.config.ts | 9 +- package-lock.json | 392 +++++++++++++++++----- package.json | 6 +- prisma/schema.prisma | 9 + scripts/migrate-database.sh | 76 +++++ scripts/migrate-storage.ts | 173 ++++++++++ scripts/verify-migration.ts | 131 ++++++++ src/app/api/cards/[id]/route.ts | 2 +- src/app/api/cards/recover-survey/route.ts | 2 +- src/app/api/cleanup/route.ts | 2 +- src/app/api/email-watch/poll/route.ts | 12 + src/app/api/email-watch/route.ts | 51 ++- src/app/api/ftp-watch/poll/route.ts | 12 + src/app/api/ftp-watch/route.ts | 63 ++++ src/app/api/ftp-watch/test/route.ts | 23 ++ src/app/api/health/route.ts | 31 +- src/app/api/images/[...path]/route.ts | 2 +- src/app/api/jobs/process/route.ts | 50 +++ src/app/api/settings/route.ts | 9 + src/app/api/upload/route.ts | 10 +- src/app/api/watch/route.ts | 42 +-- src/instrumentation.ts | 55 +-- src/lib/db.ts | 1 + src/lib/email-watcher.ts | 157 +-------- src/lib/ftp-watcher.ts | 147 ++++++++ src/lib/integrations.ts | 2 +- src/lib/ocr.ts | 2 +- src/lib/pdf.ts | 128 +++---- src/lib/processing-queue.ts | 52 +-- src/lib/{minio.ts => storage.ts} | 20 +- src/lib/watcher.ts | 99 ------ src/middleware.ts | 3 + vercel.json | 12 + 35 files changed, 1216 insertions(+), 626 deletions(-) create mode 100755 scripts/migrate-database.sh create mode 100644 scripts/migrate-storage.ts create mode 100644 scripts/verify-migration.ts create mode 100644 src/app/api/email-watch/poll/route.ts create mode 100644 src/app/api/ftp-watch/poll/route.ts create mode 100644 src/app/api/ftp-watch/route.ts create mode 100644 src/app/api/ftp-watch/test/route.ts create mode 100644 src/app/api/jobs/process/route.ts create mode 100644 src/lib/ftp-watcher.ts rename src/lib/{minio.ts => storage.ts} (76%) delete mode 100644 src/lib/watcher.ts create mode 100644 vercel.json diff --git a/.env.example b/.env.example index b3cc8fb..1eb0b54 100644 --- a/.env.example +++ b/.env.example @@ -1,44 +1,40 @@ -# Database (shared PostgreSQL on CT 102) -DATABASE_URL="postgresql://echos_ocr:YOUR_PASSWORD@192.168.68.102:5432/echos_ocr" +# ─── Database (Supabase pooled connection) ──────────────────── +DATABASE_URL="postgresql://postgres.:@aws-0-us-east-1.pooler.supabase.com:6543/postgres" -# Vercel AI Gateway (routes to OpenAI, Google, Anthropic, etc.) +# ─── Object Storage (Supabase Storage, S3-compatible) ───────── +STORAGE_ENDPOINT="https://.supabase.co/storage/v1/s3" +STORAGE_REGION="us-east-1" +STORAGE_ACCESS_KEY="" +STORAGE_SECRET_KEY="" +STORAGE_BUCKET="echos-ocr" + +# ─── Upstash QStash (job queue) ─────────────────────────────── +QSTASH_TOKEN="" +QSTASH_CURRENT_SIGNING_KEY="" +QSTASH_NEXT_SIGNING_KEY="" + +# ─── Vercel AI Gateway ──────────────────────────────────────── AI_GATEWAY_API_KEY="" -# Ollama (only needed if using Ollama as the AI provider) -OLLAMA_BASE_URL="http://192.168.68.108:11434" - -# MinIO S3 Storage (CT 105) -MINIO_ENDPOINT="192.168.68.105" -MINIO_PORT="9000" -MINIO_ACCESS_KEY="minioadmin" -MINIO_SECRET_KEY="YOUR_MINIO_SECRET" -MINIO_BUCKET="echos-ocr" - -# Folder Watch (optional, mount a host path into the container) -WATCH_DIR="" - -# Auth.js (required — generate with: npx auth secret) +# ─── Auth.js (required — generate with: npx auth secret) ───── AUTH_SECRET="" -# Set AUTH_URL to your external URL when behind a reverse proxy -AUTH_URL="https://staging.echoocr.stillwell.cloud" +AUTH_URL="https://echoocr.yourdomain.com" -# Authentik OIDC SSO (optional — enables "Sign in with SSO" button) -# Create an OAuth2/OIDC provider in Authentik and set these values. +# ─── OIDC SSO (optional) ────────────────────────────────────── AUTHENTIK_ISSUER="" AUTHENTIK_CLIENT_ID="" AUTHENTIK_CLIENT_SECRET="" -# Legacy Authentik forward-auth (deprecated — will be removed) -AUTHENTIK_URL="https://auth.stillwell.cloud" -AUTHENTIK_API_TOKEN="" - -# SMTP for outbound email (verification, invitations) -# Falls back to EMAIL_IMAP_* values if not set -SMTP_HOST="" +# ─── SMTP for outbound email ────────────────────────────────── +SMTP_HOST="smtp.dreamhost.com" SMTP_PORT="587" SMTP_USER="" SMTP_PASS="" SMTP_FROM="" -# Environment indicator (set to "staging" for staging deployments) +# ─── Vercel Cron Secret ──────────────────────────────────────── +# Vercel auto-sets this on Pro. Used to authenticate cron job requests. +CRON_SECRET="" + +# ─── Environment indicator ──────────────────────────────────── NEXT_PUBLIC_ENV="" diff --git a/.gitignore b/.gitignore index e10be6e..1fce156 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,9 @@ yarn-error.log* .env.local .env.*.local +# database dumps +*.dump + # vercel .vercel diff --git a/next.config.ts b/next.config.ts index aa8f345..4202ada 100644 --- a/next.config.ts +++ b/next.config.ts @@ -1,20 +1,15 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { - output: "standalone", images: { remotePatterns: [ - { - protocol: "http", - hostname: "**", - }, { protocol: "https", - hostname: "**", + hostname: "**.supabase.co", }, ], }, - serverExternalPackages: ["sharp", "pdf2pic", "chokidar", "pg", "imapflow", "mailparser"], + serverExternalPackages: ["sharp", "@napi-rs/canvas", "pg", "basic-ftp", "imapflow", "mailparser"], }; export default nextConfig; diff --git a/package-lock.json b/package-lock.json index 3566be7..0918d26 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,12 +15,14 @@ "@aws-sdk/client-s3": "^3.1005.0", "@aws-sdk/s3-request-presigner": "^3.1005.0", "@base-ui/react": "^1.2.0", + "@napi-rs/canvas": "^0.1.98", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", "@tanstack/react-table": "^8.21.3", + "@upstash/qstash": "^2.10.1", "ai": "^6.0.116", + "basic-ftp": "^5.3.0", "bcryptjs": "^3.0.3", - "chokidar": "^5.0.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -34,7 +36,7 @@ "next-themes": "^0.4.6", "nodemailer": "^7.0.13", "pdf-lib": "^1.17.1", - "pdf2pic": "^3.2.0", + "pdfjs-dist": "^5.6.205", "pg": "^8.20.0", "prisma": "^7.4.2", "react": "^19.2.4", @@ -2771,6 +2773,255 @@ "node": ">=18" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.98.tgz", + "integrity": "sha512-WDg3lxYMqlrg49sDVUlrHVfIEPsd5AjYDRuGD6Fu82K5agJx0UnWA+l5qd53GNLRiMN2WhOw7FLR+Er5QB/0SA==", + "license": "MIT", + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.98", + "@napi-rs/canvas-darwin-arm64": "0.1.98", + "@napi-rs/canvas-darwin-x64": "0.1.98", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.98", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.98", + "@napi-rs/canvas-linux-arm64-musl": "0.1.98", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.98", + "@napi-rs/canvas-linux-x64-gnu": "0.1.98", + "@napi-rs/canvas-linux-x64-musl": "0.1.98", + "@napi-rs/canvas-win32-arm64-msvc": "0.1.98", + "@napi-rs/canvas-win32-x64-msvc": "0.1.98" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.98.tgz", + "integrity": "sha512-O45Ifr0WZJUrSyg0QgB+67TiC0zYBRkBK+d43ZV4JtlwH3XttiVxLvlxEeULiH5y1MSELruspF0bjF6xXwJNPQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.98.tgz", + "integrity": "sha512-1b/nQhw6Isdv14JokUqat+i5wrAYD+ce3egiotedBGRUjVxYSj4s2uQCh2bFsyX5/9A5iTKVGsWoQhFft+j7Lg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.98.tgz", + "integrity": "sha512-oefzfBM8mwnyYp6S+yNXwjCoLdkOalFG24mssHgvrJDS0FulOryyI35Q7GdJGmrzuL4oo1XW3ZTOcTBLdJ8Zkg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.98.tgz", + "integrity": "sha512-NDH5QXGmf8wlo5yhijCNGVFiJk7an5GvHwb2LHyfLQWY/6/S48i5+YtY6FPqPVVCUckNGudYOfXEJnb3/FiJGQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.98.tgz", + "integrity": "sha512-KBLLM6tu1xs80LSAqdSLBKkgct0S23MCEf/aq8yxzg5imAceqp1ulKeELgWaYm27MgpUhm3Q7jmegX12FfphwA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.98.tgz", + "integrity": "sha512-mfMNhjN5zDcJafqQ6sHj4Tc3YMTRxP5UA3MHtp/ssytBR/k6XO0x+1IIPtscnUKwha+ql1++WjDCGEgqu8OfWQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.98.tgz", + "integrity": "sha512-nfW8esrcaeuhrO3qGA5cwuyk4Ak6cn2eB0LtEYtqROIl+fz06CNGNCU0M95+Tspw5ZgfSbc98SaigT5r5B3LVQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.98.tgz", + "integrity": "sha512-318UT8j6Gro2bTjtutjQXHWp9SLTNw+WRS4wQ6XIRPAyzBGnGHg7x2ndD+oqkPrrSRIbYLA5WoBcCasaF7lSTQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.98.tgz", + "integrity": "sha512-0vZhI74UxnA4VqlW4UvM0dFRrjE1RLEe/OXSBjzytGIxV+yOG4exlrhGoIpAQaIpQQQXMCdb1EmbvPC1k9vEqQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.98.tgz", + "integrity": "sha512-oiC/IxgFEEVcZ7VH7JXXlmgsqRvmFb57PIQ4gQck35IKFZCNUvdNCcN3OeoLP7Hpf5160MWJf9jj/+E5V0bSvw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.98", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.98.tgz", + "integrity": "sha512-ZqstKAJBSyZetU8udUvBQWPlGN9buawFvjuo9mgCAxzbOoJAgXX39ihec/nn42T5Vb6/qyn45eTimx5ND9kMEw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", @@ -5565,6 +5816,26 @@ "win32" ] }, + "node_modules/@upstash/qstash": { + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/@upstash/qstash/-/qstash-2.10.1.tgz", + "integrity": "sha512-LsTAPxPk0dFvhlEnRqwObRS94b8mmDHYaBlF3IaONUL+Scq6i9cZraA6936KSUVe+Vq3eNQle3CjOZkStPUFTw==", + "license": "MIT", + "dependencies": { + "crypto-js": ">=4.2.0", + "jose": "^5.2.3", + "neverthrow": "^7.0.1" + } + }, + "node_modules/@upstash/qstash/node_modules/jose": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/jose/-/jose-5.10.0.tgz", + "integrity": "sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/@vercel/oidc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", @@ -5808,18 +6079,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/array-parallel": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/array-parallel/-/array-parallel-0.1.3.tgz", - "integrity": "sha512-TDPTwSWW5E4oiFiKmz6RGJ/a80Y91GuLgUYuLd49+XBS75tYo8PNgaT2K/OxuQYqkoI852MDGBorg9OcUSTQ8w==", - "license": "MIT" - }, - "node_modules/array-series": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/array-series/-/array-series-0.1.5.tgz", - "integrity": "sha512-L0XlBwfx9QetHOsbLDrE/vh2t018w9462HM3iaFfxRiK83aJjAt/Ja3NMkOW7FICwWTlQBa3ZbL5FKhuQWkDrg==", - "license": "MIT" - }, "node_modules/array.prototype.findlast": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/array.prototype.findlast/-/array.prototype.findlast-1.2.5.tgz", @@ -6042,6 +6301,15 @@ "node": ">=6.0.0" } }, + "node_modules/basic-ftp": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.0.tgz", + "integrity": "sha512-5K9eNNn7ywHPsYnFwjKgYH8Hf8B5emh7JKcPaVjjrMJFQQwGpwowEnZNEtHs7DfR7hCZsmaK3VA4HUK0YarT+w==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bcryptjs": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-3.0.3.tgz", @@ -6337,21 +6605,6 @@ "regexp-to-ast": "0.5.0" } }, - "node_modules/chokidar": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", - "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", - "license": "MIT", - "dependencies": { - "readdirp": "^5.0.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/citty": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/citty/-/citty-0.1.6.tgz", @@ -6670,6 +6923,12 @@ "node": ">= 8" } }, + "node_modules/crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==", + "license": "MIT" + }, "node_modules/cssesc": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", @@ -8531,31 +8790,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/gm": { - "version": "1.25.1", - "resolved": "https://registry.npmjs.org/gm/-/gm-1.25.1.tgz", - "integrity": "sha512-jgcs2vKir9hFogGhXIfs0ODhJTfIrbECCehg38tqFgHm8zqXx7kAJyCYAFK4jTjx71AxrkFtkJBawbAxYUPX9A==", - "deprecated": "The gm module has been sunset. Please migrate to an alternative. https://github.com/aheckmann/gm?tab=readme-ov-file#2025-02-24-this-project-is-not-maintained", - "license": "MIT", - "dependencies": { - "array-parallel": "~0.1.3", - "array-series": "~0.1.5", - "cross-spawn": "^7.0.5", - "debug": "^3.1.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gm/node_modules/debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" - } - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -10483,6 +10717,15 @@ "node": ">= 0.6" } }, + "node_modules/neverthrow": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/neverthrow/-/neverthrow-7.2.0.tgz", + "integrity": "sha512-iGBUfFB7yPczHHtA8dksKTJ9E8TESNTAx1UQWW6TzMF280vo9jdPYpLUXrMN1BCkPdHFdNG3fxOt2CUad8KhAw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/next": { "version": "16.1.6", "resolved": "https://registry.npmjs.org/next/-/next-16.1.6.tgz", @@ -10664,6 +10907,13 @@ "integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==", "license": "MIT" }, + "node_modules/node-readable-to-web-readable-stream": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/node-readable-to-web-readable-stream/-/node-readable-to-web-readable-stream-0.4.2.tgz", + "integrity": "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ==", + "license": "MIT", + "optional": true + }, "node_modules/node-releases": { "version": "2.0.36", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz", @@ -11187,20 +11437,17 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "license": "0BSD" }, - "node_modules/pdf2pic": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/pdf2pic/-/pdf2pic-3.2.0.tgz", - "integrity": "sha512-p0bp+Mp4iJy2hqSCLvJ521rDaZkzBvDFT9O9Y0BUID3I04/eDaebAFM5t8hoWeo2BCf42cDijLCGJWTOtkJVpA==", - "license": "MIT", - "dependencies": { - "gm": "^1.25.1" - }, + "node_modules/pdfjs-dist": { + "version": "5.6.205", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.6.205.tgz", + "integrity": "sha512-tlUj+2IDa7G1SbvBNN74UHRLJybZDWYom+k6p5KIZl7huBvsA4APi6mKL+zCxd3tLjN5hOOEE9Tv7VdzO88pfg==", + "license": "Apache-2.0", "engines": { - "node": ">=14" + "node": ">=20.19.0 || >=22.13.0 || >=24" }, - "funding": { - "type": "paypal", - "url": "https://www.paypal.me/yakovmeister" + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.96", + "node-readable-to-web-readable-stream": "^0.4.2" } }, "node_modules/peberminta": { @@ -11870,19 +12117,6 @@ } } }, - "node_modules/readdirp": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", - "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", diff --git a/package.json b/package.json index c3195d3..424f979 100644 --- a/package.json +++ b/package.json @@ -19,12 +19,14 @@ "@aws-sdk/client-s3": "^3.1005.0", "@aws-sdk/s3-request-presigner": "^3.1005.0", "@base-ui/react": "^1.2.0", + "@napi-rs/canvas": "^0.1.98", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", "@tanstack/react-table": "^8.21.3", + "@upstash/qstash": "^2.10.1", "ai": "^6.0.116", + "basic-ftp": "^5.3.0", "bcryptjs": "^3.0.3", - "chokidar": "^5.0.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -38,7 +40,7 @@ "next-themes": "^0.4.6", "nodemailer": "^7.0.13", "pdf-lib": "^1.17.1", - "pdf2pic": "^3.2.0", + "pdfjs-dist": "^5.6.205", "pg": "^8.20.0", "prisma": "^7.4.2", "react": "^19.2.4", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 5fb6c48..c8b6536 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -321,6 +321,15 @@ model AppSettings { emailWatching Boolean @default(false) emailProcessed String @default("mark_read") emailProcessedFolder String @default("Processed") + + ftpEnabled Boolean @default(false) + ftpHost String @default("") + ftpPort Int @default(21) + ftpUser String @default("") + ftpPass String @default("") + ftpTls Boolean @default(true) + ftpIncomingDir String @default("/incoming") + ftpProcessedDir String @default("/processed") } model ActivityLog { diff --git a/scripts/migrate-database.sh b/scripts/migrate-database.sh new file mode 100755 index 0000000..b6dbd12 --- /dev/null +++ b/scripts/migrate-database.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ─── Database Migration: Homelab PostgreSQL → Supabase ──────── +# +# Prerequisites: +# - pg_dump and pg_restore installed locally +# - Network access to both the homelab PG and Supabase PG +# - Supabase project created with direct connection string +# +# Usage: +# HOMELAB_DB_URL="postgresql://echos_ocr:PASSWORD@192.168.68.102:5432/echos_ocr" \ +# SUPABASE_DB_URL="postgresql://postgres.REF:PASSWORD@db.REF.supabase.co:5432/postgres" \ +# bash scripts/migrate-database.sh + +if [ -z "${HOMELAB_DB_URL:-}" ]; then + echo "Error: HOMELAB_DB_URL is not set" + echo "Example: postgresql://echos_ocr:PASSWORD@192.168.68.102:5432/echos_ocr" + exit 1 +fi + +if [ -z "${SUPABASE_DB_URL:-}" ]; then + echo "Error: SUPABASE_DB_URL is not set (use the DIRECT connection, port 5432, not pooled)" + echo "Example: postgresql://postgres.REF:PASSWORD@db.REF.supabase.co:5432/postgres" + exit 1 +fi + +DUMP_FILE="echos_ocr_backup_$(date +%Y%m%d_%H%M%S).dump" + +echo "=== Step 1: Dumping homelab database ===" +pg_dump "${HOMELAB_DB_URL}" -Fc -f "${DUMP_FILE}" +echo "Dump created: ${DUMP_FILE} ($(du -h "${DUMP_FILE}" | cut -f1))" + +echo "" +echo "=== Step 2: Restoring to Supabase ===" +pg_restore "${SUPABASE_DB_URL}" --no-owner --no-privileges --clean --if-exists "${DUMP_FILE}" 2>&1 || true +echo "Restore complete (some warnings about existing objects are normal)" + +echo "" +echo "=== Step 3: Pushing schema with Prisma ===" +echo "Running 'npx prisma db push' to ensure schema alignment..." +DATABASE_URL="${SUPABASE_DB_URL}" npx prisma db push --skip-generate 2>&1 +echo "Schema push complete" + +echo "" +echo "=== Step 4: Verifying row counts ===" +psql "${SUPABASE_DB_URL}" -c " +SELECT 'User' as table_name, count(*) FROM \"User\" +UNION ALL SELECT 'Organization', count(*) FROM \"Organization\" +UNION ALL SELECT 'OrgMember', count(*) FROM \"OrgMember\" +UNION ALL SELECT 'ResponseCard', count(*) FROM \"ResponseCard\" +UNION ALL SELECT 'ProcessingJob', count(*) FROM \"ProcessingJob\" +UNION ALL SELECT 'ActivityLog', count(*) FROM \"ActivityLog\" +UNION ALL SELECT 'Notification', count(*) FROM \"Notification\" +UNION ALL SELECT 'Integration', count(*) FROM \"Integration\" +ORDER BY table_name; +" + +echo "" +echo "=== Compare with homelab counts ===" +psql "${HOMELAB_DB_URL}" -c " +SELECT 'User' as table_name, count(*) FROM \"User\" +UNION ALL SELECT 'Organization', count(*) FROM \"Organization\" +UNION ALL SELECT 'OrgMember', count(*) FROM \"OrgMember\" +UNION ALL SELECT 'ResponseCard', count(*) FROM \"ResponseCard\" +UNION ALL SELECT 'ProcessingJob', count(*) FROM \"ProcessingJob\" +UNION ALL SELECT 'ActivityLog', count(*) FROM \"ActivityLog\" +UNION ALL SELECT 'Notification', count(*) FROM \"Notification\" +UNION ALL SELECT 'Integration', count(*) FROM \"Integration\" +ORDER BY table_name; +" + +echo "" +echo "=== Migration complete ===" +echo "Backup file retained at: ${DUMP_FILE}" +echo "Verify the counts above match between homelab and Supabase." diff --git a/scripts/migrate-storage.ts b/scripts/migrate-storage.ts new file mode 100644 index 0000000..3836acd --- /dev/null +++ b/scripts/migrate-storage.ts @@ -0,0 +1,173 @@ +/** + * Storage Migration: MinIO → Supabase Storage + * + * Copies all objects from a MinIO bucket to a Supabase Storage bucket, + * preserving the same key paths so database references remain valid. + * + * Usage: + * npx tsx scripts/migrate-storage.ts + * + * Required env vars: + * MINIO_ENDPOINT - e.g. 192.168.68.105 + * MINIO_PORT - e.g. 9000 + * MINIO_ACCESS_KEY - e.g. minioadmin + * MINIO_SECRET_KEY - your secret + * MINIO_BUCKET - e.g. echos-ocr + * + * STORAGE_ENDPOINT - e.g. https://xxx.supabase.co/storage/v1/s3 + * STORAGE_REGION - e.g. us-east-1 + * STORAGE_ACCESS_KEY - Supabase S3 access key + * STORAGE_SECRET_KEY - Supabase S3 secret key + * STORAGE_BUCKET - e.g. echos-ocr + */ + +import "dotenv/config"; +import { + S3Client, + ListObjectsV2Command, + GetObjectCommand, + PutObjectCommand, + HeadObjectCommand, +} from "@aws-sdk/client-s3"; + +const sourceClient = new S3Client({ + endpoint: `http://${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT || "9000"}`, + region: "us-east-1", + credentials: { + accessKeyId: process.env.MINIO_ACCESS_KEY || "", + secretAccessKey: process.env.MINIO_SECRET_KEY || "", + }, + forcePathStyle: true, +}); + +const destClient = new S3Client({ + endpoint: process.env.STORAGE_ENDPOINT || "", + region: process.env.STORAGE_REGION || "us-east-1", + credentials: { + accessKeyId: process.env.STORAGE_ACCESS_KEY || "", + secretAccessKey: process.env.STORAGE_SECRET_KEY || "", + }, + forcePathStyle: true, +}); + +const SOURCE_BUCKET = process.env.MINIO_BUCKET || "echos-ocr"; +const DEST_BUCKET = process.env.STORAGE_BUCKET || "echos-ocr"; + +async function listAllKeys(): Promise { + const keys: string[] = []; + let continuationToken: string | undefined; + + do { + const res = await sourceClient.send( + new ListObjectsV2Command({ + Bucket: SOURCE_BUCKET, + ContinuationToken: continuationToken, + }) + ); + for (const obj of res.Contents ?? []) { + if (obj.Key) keys.push(obj.Key); + } + continuationToken = res.NextContinuationToken; + } while (continuationToken); + + return keys; +} + +async function objectExists(key: string): Promise { + try { + await destClient.send( + new HeadObjectCommand({ Bucket: DEST_BUCKET, Key: key }) + ); + return true; + } catch { + return false; + } +} + +async function copyObject(key: string): Promise { + const res = await sourceClient.send( + new GetObjectCommand({ Bucket: SOURCE_BUCKET, Key: key }) + ); + const stream = res.Body as ReadableStream; + const buffer = Buffer.from(await new Response(stream).arrayBuffer()); + + await destClient.send( + new PutObjectCommand({ + Bucket: DEST_BUCKET, + Key: key, + Body: buffer, + ContentType: res.ContentType || "application/octet-stream", + }) + ); +} + +async function main() { + console.log("=== Storage Migration: MinIO → Supabase Storage ==="); + console.log(`Source: ${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT} / ${SOURCE_BUCKET}`); + console.log(`Dest: ${process.env.STORAGE_ENDPOINT} / ${DEST_BUCKET}`); + console.log(""); + + console.log("Listing all objects in source bucket..."); + const keys = await listAllKeys(); + console.log(`Found ${keys.length} object(s) to migrate`); + + if (keys.length === 0) { + console.log("Nothing to migrate."); + return; + } + + const sourcesCount = keys.filter((k) => k.startsWith("sources/")).length; + const imagesCount = keys.filter((k) => k.startsWith("images/")).length; + const otherCount = keys.length - sourcesCount - imagesCount; + console.log(` sources/: ${sourcesCount}, images/: ${imagesCount}, other: ${otherCount}`); + console.log(""); + + let copied = 0; + let skipped = 0; + let errors = 0; + + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const progress = `[${i + 1}/${keys.length}]`; + + try { + const exists = await objectExists(key); + if (exists) { + skipped++; + if ((i + 1) % 50 === 0) { + console.log(`${progress} Skipped (already exists): ${key}`); + } + continue; + } + + await copyObject(key); + copied++; + + if (copied % 10 === 0 || (i + 1) === keys.length) { + console.log(`${progress} Copied: ${key} (${copied} total)`); + } + } catch (err) { + errors++; + console.error(`${progress} ERROR copying ${key}:`, err instanceof Error ? err.message : err); + } + } + + console.log(""); + console.log("=== Migration Summary ==="); + console.log(`Total objects: ${keys.length}`); + console.log(`Copied: ${copied}`); + console.log(`Skipped: ${skipped} (already existed in destination)`); + console.log(`Errors: ${errors}`); + + if (errors > 0) { + console.log("\nSome objects failed to copy. Re-run this script to retry (it skips already-copied objects)."); + process.exit(1); + } + + console.log("\nAll objects migrated successfully."); +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/scripts/verify-migration.ts b/scripts/verify-migration.ts new file mode 100644 index 0000000..e683756 --- /dev/null +++ b/scripts/verify-migration.ts @@ -0,0 +1,131 @@ +/** + * Post-Migration Verification + * + * Checks that the database and storage are intact after migration. + * + * Usage: + * DATABASE_URL="postgresql://..." \ + * STORAGE_ENDPOINT="https://..." \ + * STORAGE_ACCESS_KEY="..." \ + * STORAGE_SECRET_KEY="..." \ + * STORAGE_BUCKET="echos-ocr" \ + * npx tsx scripts/verify-migration.ts + */ + +import "dotenv/config"; +import pg from "pg"; +import { S3Client, ListObjectsV2Command, HeadObjectCommand } from "@aws-sdk/client-s3"; + +const pool = new pg.Pool({ + connectionString: process.env.DATABASE_URL, + max: 3, +}); + +const s3 = new S3Client({ + endpoint: process.env.STORAGE_ENDPOINT || "", + region: process.env.STORAGE_REGION || "us-east-1", + credentials: { + accessKeyId: process.env.STORAGE_ACCESS_KEY || "", + secretAccessKey: process.env.STORAGE_SECRET_KEY || "", + }, + forcePathStyle: true, +}); + +const BUCKET = process.env.STORAGE_BUCKET || "echos-ocr"; + +async function countTable(table: string): Promise { + const res = await pool.query(`SELECT count(*)::int as cnt FROM "${table}"`); + return res.rows[0].cnt; +} + +async function countStorageObjects(prefix: string): Promise { + let count = 0; + let continuationToken: string | undefined; + + do { + const res = await s3.send( + new ListObjectsV2Command({ + Bucket: BUCKET, + Prefix: prefix, + ContinuationToken: continuationToken, + }) + ); + count += res.KeyCount || 0; + continuationToken = res.NextContinuationToken; + } while (continuationToken); + + return count; +} + +async function spotCheck(): Promise<{ checked: number; ok: number; missing: number }> { + const res = await pool.query(` + SELECT id, "frontImagePath", "backImagePath" + FROM "ResponseCard" + WHERE "frontImagePath" IS NOT NULL OR "backImagePath" IS NOT NULL + ORDER BY "createdAt" DESC + LIMIT 10 + `); + + let checked = 0; + let ok = 0; + let missing = 0; + + for (const row of res.rows) { + for (const path of [row.frontImagePath, row.backImagePath]) { + if (!path) continue; + checked++; + try { + await s3.send(new HeadObjectCommand({ Bucket: BUCKET, Key: path })); + ok++; + } catch { + console.log(` MISSING: ${path} (card ${row.id})`); + missing++; + } + } + } + + return { checked, ok, missing }; +} + +async function main() { + console.log("=== Post-Migration Verification ===\n"); + + console.log("--- Database Row Counts ---"); + const tables = [ + "User", "Organization", "OrgMember", "Location", + "ResponseCard", "ProcessingJob", "ActivityLog", + "Notification", "Integration", "AppSettings", + ]; + for (const table of tables) { + try { + const count = await countTable(table); + console.log(` ${table.padEnd(20)} ${count}`); + } catch (err) { + console.log(` ${table.padEnd(20)} ERROR: ${(err as Error).message}`); + } + } + + console.log("\n--- Storage Object Counts ---"); + const imageCount = await countStorageObjects("images/"); + const sourceCount = await countStorageObjects("sources/"); + console.log(` images/ ${imageCount}`); + console.log(` sources/ ${sourceCount}`); + + console.log("\n--- Spot Check: 10 newest cards ---"); + const spotResult = await spotCheck(); + console.log(` Checked: ${spotResult.checked}, OK: ${spotResult.ok}, Missing: ${spotResult.missing}`); + + if (spotResult.missing > 0) { + console.log("\n WARNING: Some storage objects are missing. Storage migration may be incomplete."); + } else { + console.log("\n All spot-checked objects found in storage."); + } + + console.log("\n=== Verification Complete ==="); + await pool.end(); +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/src/app/api/cards/[id]/route.ts b/src/app/api/cards/[id]/route.ts index 926a622..817fa19 100644 --- a/src/app/api/cards/[id]/route.ts +++ b/src/app/api/cards/[id]/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/db"; -import { deleteObject } from "@/lib/minio"; +import { deleteObject } from "@/lib/storage"; import { fireIntegrationEvent } from "@/lib/integrations"; import { logActivity, diffCardFields } from "@/lib/activity-log"; import { getOrCreateUser, RoleError } from "@/lib/auth"; diff --git a/src/app/api/cards/recover-survey/route.ts b/src/app/api/cards/recover-survey/route.ts index 9c44b29..396946f 100644 --- a/src/app/api/cards/recover-survey/route.ts +++ b/src/app/api/cards/recover-survey/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from "next/server"; import { prisma } from "@/lib/db"; -import { getBuffer, uploadBuffer } from "@/lib/minio"; +import { getBuffer, uploadBuffer } from "@/lib/storage"; import { pdfToImages, imageToBase64 } from "@/lib/pdf"; import { ocrImage } from "@/lib/ai-ocr"; diff --git a/src/app/api/cleanup/route.ts b/src/app/api/cleanup/route.ts index d3faa0a..7ae7b59 100644 --- a/src/app/api/cleanup/route.ts +++ b/src/app/api/cleanup/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/db"; -import { deleteObject, listObjects } from "@/lib/minio"; +import { deleteObject, listObjects } from "@/lib/storage"; export async function POST(request: NextRequest) { try { diff --git a/src/app/api/email-watch/poll/route.ts b/src/app/api/email-watch/poll/route.ts new file mode 100644 index 0000000..8fd2423 --- /dev/null +++ b/src/app/api/email-watch/poll/route.ts @@ -0,0 +1,12 @@ +import { NextRequest, NextResponse } from "next/server"; +import { scanInbox } from "@/lib/email-watcher"; + +export async function GET(request: NextRequest) { + const authHeader = request.headers.get("authorization"); + if (authHeader !== `Bearer ${process.env.CRON_SECRET}`) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const result = await scanInbox(); + return NextResponse.json(result); +} diff --git a/src/app/api/email-watch/route.ts b/src/app/api/email-watch/route.ts index dc3f9e6..a81580a 100644 --- a/src/app/api/email-watch/route.ts +++ b/src/app/api/email-watch/route.ts @@ -1,20 +1,15 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/db"; -import { - startEmailWatching, - stopEmailWatching, - isEmailWatching, - scanInbox, -} from "@/lib/email-watcher"; +import { scanInbox } from "@/lib/email-watcher"; export async function POST(request: NextRequest) { try { const body = await request.json().catch(() => ({})); const action = body.action as string | undefined; - if (!action || !["start", "stop", "scan"].includes(action)) { + if (!action || !["enable", "disable", "scan"].includes(action)) { return NextResponse.json( - { error: "Invalid action. Use 'start', 'stop', or 'scan'" }, + { error: "Invalid action. Use 'enable', 'disable', or 'scan'" }, { status: 400 } ); } @@ -27,18 +22,18 @@ export async function POST(request: NextRequest) { return NextResponse.json({ ok: true, ...result }); } - if (action === "start") { - const settings = await prisma.appSettings.findUnique({ - where: { id: "singleton" }, - }); + const settings = await prisma.appSettings.findUnique({ + where: { id: "singleton" }, + }); - if (!settings) { - return NextResponse.json( - { error: "Settings not configured" }, - { status: 400 } - ); - } + if (!settings) { + return NextResponse.json( + { error: "Settings not configured" }, + { status: 400 } + ); + } + if (action === "enable") { if (!settings.emailImapHost || !settings.emailImapUser || !settings.emailImapPass) { return NextResponse.json( { error: "IMAP host, username, and password are required" }, @@ -46,21 +41,19 @@ export async function POST(request: NextRequest) { ); } - await startEmailWatching({ - host: settings.emailImapHost, - port: settings.emailImapPort, - user: settings.emailImapUser, - pass: settings.emailImapPass, - tls: settings.emailImapTls, - folder: settings.emailFolder, - processedAction: settings.emailProcessed, - processedFolder: settings.emailProcessedFolder, + await prisma.appSettings.update({ + where: { id: "singleton" }, + data: { emailWatching: true }, }); } else { - await stopEmailWatching(); + await prisma.appSettings.update({ + where: { id: "singleton" }, + data: { emailWatching: false }, + }); } - return NextResponse.json({ emailWatching: isEmailWatching() }); + const updated = await prisma.appSettings.findUnique({ where: { id: "singleton" } }); + return NextResponse.json({ emailWatching: updated?.emailWatching ?? false }); } catch (error) { console.error("[email-watch POST]", error); const message = diff --git a/src/app/api/ftp-watch/poll/route.ts b/src/app/api/ftp-watch/poll/route.ts new file mode 100644 index 0000000..c7a0c94 --- /dev/null +++ b/src/app/api/ftp-watch/poll/route.ts @@ -0,0 +1,12 @@ +import { NextRequest, NextResponse } from "next/server"; +import { pollFtp } from "@/lib/ftp-watcher"; + +export async function GET(request: NextRequest) { + const authHeader = request.headers.get("authorization"); + if (authHeader !== `Bearer ${process.env.CRON_SECRET}`) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const result = await pollFtp(); + return NextResponse.json(result); +} diff --git a/src/app/api/ftp-watch/route.ts b/src/app/api/ftp-watch/route.ts new file mode 100644 index 0000000..8da864d --- /dev/null +++ b/src/app/api/ftp-watch/route.ts @@ -0,0 +1,63 @@ +import { NextRequest, NextResponse } from "next/server"; +import { prisma } from "@/lib/db"; +import { pollFtp } from "@/lib/ftp-watcher"; + +export async function POST(request: NextRequest) { + try { + const body = await request.json().catch(() => ({})); + const action = body.action as string | undefined; + + if (!action || !["enable", "disable", "scan"].includes(action)) { + return NextResponse.json( + { error: "Invalid action. Use 'enable', 'disable', or 'scan'" }, + { status: 400 } + ); + } + + if (action === "scan") { + const result = await pollFtp(); + if (result.error) { + return NextResponse.json({ error: result.error, ...result }, { status: 500 }); + } + return NextResponse.json({ ok: true, ...result }); + } + + const settings = await prisma.appSettings.findUnique({ + where: { id: "singleton" }, + }); + + if (!settings) { + return NextResponse.json( + { error: "Settings not configured" }, + { status: 400 } + ); + } + + if (action === "enable") { + if (!settings.ftpHost || !settings.ftpUser || !settings.ftpPass) { + return NextResponse.json( + { error: "FTP host, username, and password are required" }, + { status: 400 } + ); + } + + await prisma.appSettings.update({ + where: { id: "singleton" }, + data: { ftpEnabled: true }, + }); + } else { + await prisma.appSettings.update({ + where: { id: "singleton" }, + data: { ftpEnabled: false }, + }); + } + + const updated = await prisma.appSettings.findUnique({ where: { id: "singleton" } }); + return NextResponse.json({ ftpEnabled: updated?.ftpEnabled ?? false }); + } catch (error) { + console.error("[ftp-watch POST]", error); + const message = + error instanceof Error ? error.message : "Failed to update FTP watch"; + return NextResponse.json({ error: message }, { status: 500 }); + } +} diff --git a/src/app/api/ftp-watch/test/route.ts b/src/app/api/ftp-watch/test/route.ts new file mode 100644 index 0000000..979c3e2 --- /dev/null +++ b/src/app/api/ftp-watch/test/route.ts @@ -0,0 +1,23 @@ +import { NextRequest, NextResponse } from "next/server"; +import { testFtpConnection } from "@/lib/ftp-watcher"; + +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const result = await testFtpConnection({ + host: body.host, + port: body.port || 21, + user: body.user, + pass: body.pass, + tls: body.tls ?? true, + incomingDir: body.incomingDir || "/incoming", + }); + return NextResponse.json(result); + } catch (error) { + console.error("[ftp-watch/test POST]", error); + return NextResponse.json( + { ok: false, error: "Test failed" }, + { status: 500 } + ); + } +} diff --git a/src/app/api/health/route.ts b/src/app/api/health/route.ts index 7d990a0..e04a11a 100644 --- a/src/app/api/health/route.ts +++ b/src/app/api/health/route.ts @@ -1,44 +1,27 @@ import { NextResponse } from "next/server"; import { HeadBucketCommand } from "@aws-sdk/client-s3"; -import { s3, BUCKET } from "@/lib/minio"; +import { s3, BUCKET } from "@/lib/storage"; export async function GET() { const results: Record = { - minio: { status: "unknown" }, - minioConfig: { - endpoint: process.env.MINIO_ENDPOINT || "192.168.68.105", - port: process.env.MINIO_PORT || "9000", + storage: { status: "unknown" }, + storageConfig: { + endpoint: process.env.STORAGE_ENDPOINT || "(not set)", bucket: BUCKET, }, }; try { await s3.send(new HeadBucketCommand({ Bucket: BUCKET })); - results.minio = { status: "ok", bucket: BUCKET }; + results.storage = { status: "ok", bucket: BUCKET }; } catch (error: unknown) { - const err = error as Error & { $response?: { body?: unknown }; $metadata?: unknown; Code?: string }; - results.minio = { + const err = error as Error & { $metadata?: unknown; Code?: string }; + results.storage = { status: "error", message: err.message, code: err.Code, metadata: err.$metadata, }; - - // Try a raw fetch to see what the endpoint actually returns - try { - const endpoint = `http://${process.env.MINIO_ENDPOINT || "192.168.68.105"}:${process.env.MINIO_PORT || "9000"}`; - const rawRes = await fetch(endpoint, { signal: AbortSignal.timeout(5000) }); - const rawBody = await rawRes.text(); - results.rawEndpointResponse = { - status: rawRes.status, - contentType: rawRes.headers.get("content-type"), - bodyPreview: rawBody.slice(0, 500), - }; - } catch (fetchErr: unknown) { - results.rawEndpointResponse = { - error: (fetchErr as Error).message, - }; - } } return NextResponse.json(results, { status: 200 }); diff --git a/src/app/api/images/[...path]/route.ts b/src/app/api/images/[...path]/route.ts index b908dad..5bc3802 100644 --- a/src/app/api/images/[...path]/route.ts +++ b/src/app/api/images/[...path]/route.ts @@ -1,5 +1,5 @@ import { NextRequest, NextResponse } from "next/server"; -import { getBuffer } from "@/lib/minio"; +import { getBuffer } from "@/lib/storage"; const EXT_TO_CONTENT_TYPE: Record = { jpg: "image/jpeg", diff --git a/src/app/api/jobs/process/route.ts b/src/app/api/jobs/process/route.ts new file mode 100644 index 0000000..f98797c --- /dev/null +++ b/src/app/api/jobs/process/route.ts @@ -0,0 +1,50 @@ +import { NextRequest, NextResponse } from "next/server"; +import { Receiver } from "@upstash/qstash"; +import { prisma } from "@/lib/db"; +import { getBuffer } from "@/lib/storage"; +import { processFile } from "@/lib/ocr"; + +const receiver = new Receiver({ + currentSigningKey: process.env.QSTASH_CURRENT_SIGNING_KEY!, + nextSigningKey: process.env.QSTASH_NEXT_SIGNING_KEY!, +}); + +export async function POST(request: NextRequest) { + try { + const body = await request.text(); + + const isValid = await receiver.verify({ + signature: request.headers.get("upstash-signature") || "", + body, + }); + + if (!isValid) { + return NextResponse.json({ error: "Invalid signature" }, { status: 401 }); + } + + const { jobId } = JSON.parse(body) as { jobId: string }; + + const job = await prisma.processingJob.findUnique({ where: { id: jobId } }); + if (!job) { + return NextResponse.json({ error: "Job not found" }, { status: 404 }); + } + + if (job.status !== "queued") { + return NextResponse.json({ ok: true, skipped: true, reason: `Job status is ${job.status}` }); + } + + const sourceKey = `sources/${job.id}/${job.fileName}`; + const fileBuffer = await getBuffer(sourceKey); + const isPdf = job.fileName.toLowerCase().endsWith(".pdf"); + + const cardIds = await processFile(job.id, job.fileName, fileBuffer, isPdf); + + return NextResponse.json({ ok: true, cardIds }); + } catch (error) { + console.error("[jobs/process POST]", error); + return NextResponse.json( + { error: error instanceof Error ? error.message : "Processing failed" }, + { status: 500 } + ); + } +} diff --git a/src/app/api/settings/route.ts b/src/app/api/settings/route.ts index 93b9515..18849f5 100644 --- a/src/app/api/settings/route.ts +++ b/src/app/api/settings/route.ts @@ -58,6 +58,15 @@ export async function PUT(request: NextRequest) { if (body.emailProcessed != null) data.emailProcessed = String(body.emailProcessed); if (body.emailProcessedFolder != null) data.emailProcessedFolder = String(body.emailProcessedFolder); + if (body.ftpEnabled != null) data.ftpEnabled = Boolean(body.ftpEnabled); + if (body.ftpHost != null) data.ftpHost = String(body.ftpHost); + if (body.ftpPort != null) data.ftpPort = Math.max(1, parseInt(String(body.ftpPort)) || 21); + if (body.ftpUser != null) data.ftpUser = String(body.ftpUser); + if (body.ftpPass != null) data.ftpPass = String(body.ftpPass); + if (body.ftpTls != null) data.ftpTls = Boolean(body.ftpTls); + if (body.ftpIncomingDir != null) data.ftpIncomingDir = String(body.ftpIncomingDir); + if (body.ftpProcessedDir != null) data.ftpProcessedDir = String(body.ftpProcessedDir); + const settings = await prisma.appSettings.upsert({ where: { id: "singleton" }, create: { diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts index eb5da0e..59a8b1d 100644 --- a/src/app/api/upload/route.ts +++ b/src/app/api/upload/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { prisma } from "@/lib/db"; +import { uploadBuffer } from "@/lib/storage"; import { enqueueProcessing } from "@/lib/processing-queue"; const MAX_UPLOAD_SIZE = 50 * 1024 * 1024; // 50 MB @@ -49,7 +50,6 @@ export async function POST(request: NextRequest) { const buffer = Buffer.from(await file.arrayBuffer()); const fileName = file.name || `upload-${Date.now()}`; - const isPdf = contentType === "application/pdf"; const job = await prisma.processingJob.create({ data: { @@ -59,11 +59,11 @@ export async function POST(request: NextRequest) { }, }); - jobIds.push(job.id); + const sourceKey = `sources/${job.id}/${fileName}`; + await uploadBuffer(sourceKey, buffer, contentType); - enqueueProcessing(job.id, fileName, buffer, isPdf).catch((err) => { - console.error(`[upload] Background processing failed for job ${job.id}:`, err); - }); + await enqueueProcessing(job.id); + jobIds.push(job.id); } return NextResponse.json({ jobIds }); diff --git a/src/app/api/watch/route.ts b/src/app/api/watch/route.ts index b394741..8b89a5f 100644 --- a/src/app/api/watch/route.ts +++ b/src/app/api/watch/route.ts @@ -1,38 +1,8 @@ -import { NextRequest, NextResponse } from "next/server"; -import { startWatching, stopWatching, isWatching } from "@/lib/watcher"; +import { NextResponse } from "next/server"; -export async function POST(request: NextRequest) { - try { - const body = await request.json().catch(() => ({})); - const action = body.action as string | undefined; - const watchDir = body.watchDir as string | undefined; - - if (!action || !["start", "stop"].includes(action)) { - return NextResponse.json( - { error: "Invalid action. Use 'start' or 'stop'" }, - { status: 400 } - ); - } - - if (action === "start") { - if (!watchDir) { - return NextResponse.json( - { error: "watchDir is required to start watching" }, - { status: 400 } - ); - } - await startWatching(watchDir); - } else { - await stopWatching(); - } - - return NextResponse.json({ - watching: isWatching(), - watchDir: watchDir || "", - }); - } catch (error) { - console.error("[watch POST]", error); - const message = error instanceof Error ? error.message : "Failed to update watch settings"; - return NextResponse.json({ error: message }, { status: 500 }); - } +export async function POST() { + return NextResponse.json( + { error: "Folder watching has been replaced by FTP watching. Use /api/ftp-watch instead." }, + { status: 410 } + ); } diff --git a/src/instrumentation.ts b/src/instrumentation.ts index f6b7f4c..8192dde 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -1,54 +1,5 @@ export async function register() { - // Only run on the Node.js server runtime, not during build or in the edge runtime - if (process.env.NEXT_RUNTIME !== "nodejs") return; - - const { prisma } = await import("@/lib/db"); - - let settings; - try { - settings = await prisma.appSettings.findUnique({ - where: { id: "singleton" }, - }); - } catch (err) { - console.error("[instrumentation] Could not read settings:", err); - return; - } - - if (!settings) return; - - // Resume email monitoring - if ( - settings.emailWatching && - settings.emailImapHost && - settings.emailImapUser && - settings.emailImapPass - ) { - try { - const { startEmailWatching } = await import("@/lib/email-watcher"); - await startEmailWatching({ - host: settings.emailImapHost, - port: settings.emailImapPort, - user: settings.emailImapUser, - pass: settings.emailImapPass, - tls: settings.emailImapTls, - folder: settings.emailFolder, - processedAction: settings.emailProcessed, - processedFolder: settings.emailProcessedFolder, - }); - console.log("[instrumentation] Email monitoring resumed"); - } catch (err) { - console.error("[instrumentation] Failed to resume email monitoring:", err); - } - } - - // Resume folder watching - if (settings.watching && settings.watchDir) { - try { - const { startWatching } = await import("@/lib/watcher"); - await startWatching(settings.watchDir); - console.log("[instrumentation] Folder watching resumed"); - } catch (err) { - console.error("[instrumentation] Failed to resume folder watching:", err); - } - } + // Background watchers (email + FTP) are now handled by Vercel Cron Jobs + // that call /api/email-watch/poll and /api/ftp-watch/poll every 2 minutes. + // No persistent processes to start on boot. } diff --git a/src/lib/db.ts b/src/lib/db.ts index 8c48fd5..7da7dfc 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -9,6 +9,7 @@ const globalForPrisma = globalThis as unknown as { function createPrismaClient() { const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL, + max: 5, }); const adapter = new PrismaPg(pool); return new PrismaClient({ adapter }); diff --git a/src/lib/email-watcher.ts b/src/lib/email-watcher.ts index 694ab74..ee186da 100644 --- a/src/lib/email-watcher.ts +++ b/src/lib/email-watcher.ts @@ -1,6 +1,7 @@ import { ImapFlow } from "imapflow"; import { simpleParser } from "mailparser"; import { prisma } from "./db"; +import { uploadBuffer } from "./storage"; import { enqueueProcessing } from "./processing-queue"; const ALLOWED_CONTENT_TYPES = [ @@ -24,11 +25,6 @@ type EmailConfig = { processedFolder: string; }; -let client: ImapFlow | null = null; -let watching = false; -let reconnectTimer: ReturnType | null = null; -let currentConfig: EmailConfig | null = null; - function log(msg: string, ...args: unknown[]) { console.log(`[email-watcher] ${msg}`, ...args); } @@ -79,10 +75,11 @@ async function handleMessage(client: ImapFlow, uid: number, config: EmailConfig) }, }); - enqueueProcessing(job.id, fileName, buffer, isPdf).catch((err) => { - logError(`Processing failed for ${fileName}:`, err); - }); + const sourceKey = `sources/${job.id}/${fileName}`; + const contentType = isPdf ? "application/pdf" : "image/jpeg"; + await uploadBuffer(sourceKey, buffer, contentType); + await enqueueProcessing(job.id); log(`Queued: ${fileName} (job ${job.id})`); } catch (err) { logError(`Failed to create job for ${fileName}:`, err); @@ -103,150 +100,16 @@ async function handleMessage(client: ImapFlow, uid: number, config: EmailConfig) } } -async function pollLoop(config: EmailConfig) { - if (!client || !watching) return; - - try { - const lock = await client.getMailboxLock(config.folder); - try { - const searchResult = await client.search({ seen: false }, { uid: true }); - const unseen = Array.isArray(searchResult) ? searchResult : []; - if (unseen.length > 0) { - log(`Found ${unseen.length} unseen message(s)`); - for (const uid of unseen) { - if (!watching) break; - try { - await handleMessage(client, uid, config); - } catch (err) { - logError(`Error handling message UID ${uid}:`, err); - } - } - } - } finally { - lock.release(); - } - } catch (err) { - logError("Error during poll:", err); - } - - if (!watching || !client) return; - - try { - await client.idle(); - } catch { - // IDLE interrupted or connection lost — reconnect handles it - } - - if (watching) { - setImmediate(() => pollLoop(config)); - } -} - -async function connect(config: EmailConfig) { - if (client) { - try { await client.logout(); } catch {} - client = null; - } - - client = new ImapFlow({ - host: config.host, - port: config.port, - secure: config.tls, - auth: { user: config.user, pass: config.pass }, - logger: false, - emitLogs: false, - }); - - client.on("error", (err: Error) => { - logError("Connection error:", err.message); - if (watching) scheduleReconnect(config); - }); - - client.on("close", () => { - log("Connection closed"); - if (watching) scheduleReconnect(config); - }); - - await client.connect(); - log(`Connected to ${config.host}:${config.port} as ${config.user}`); - - pollLoop(config); -} - -function scheduleReconnect(config: EmailConfig) { - if (reconnectTimer) return; - const delay = 10_000; - log(`Reconnecting in ${delay / 1000}s...`); - reconnectTimer = setTimeout(async () => { - reconnectTimer = null; - if (!watching) return; - try { - await connect(config); - } catch (err) { - logError("Reconnect failed:", err); - scheduleReconnect(config); - } - }, delay); -} - -export async function startEmailWatching(config: EmailConfig): Promise { - if (watching) await stopEmailWatching(); - - watching = true; - currentConfig = config; - - await prisma.appSettings.upsert({ - where: { id: "singleton" }, - update: { emailWatching: true }, - create: { id: "singleton", emailWatching: true }, - }); - - await connect(config); -} - -export async function stopEmailWatching(): Promise { - watching = false; - currentConfig = null; - - if (reconnectTimer) { - clearTimeout(reconnectTimer); - reconnectTimer = null; - } - - if (client) { - try { await client.logout(); } catch {} - client = null; - } - - await prisma.appSettings.upsert({ - where: { id: "singleton" }, - update: { emailWatching: false }, - create: { id: "singleton", emailWatching: false }, - }); - - log("Stopped"); -} - -export function isEmailWatching(): boolean { - return watching; -} - -export function getEmailWatchStatus() { - return { - watching, - connected: client !== null && client.usable !== false, - config: currentConfig - ? { host: currentConfig.host, user: currentConfig.user, folder: currentConfig.folder } - : null, - }; -} - export async function scanInbox(): Promise<{ processed: number; skipped: number; error?: string }> { const settings = await prisma.appSettings.findUnique({ where: { id: "singleton" } }); if (!settings?.emailImapHost || !settings?.emailImapUser || !settings?.emailImapPass) { return { processed: 0, skipped: 0, error: "IMAP not configured" }; } + if (!settings.emailWatching) { + return { processed: 0, skipped: 0, error: "Email watching is disabled" }; + } + const config: EmailConfig = { host: settings.emailImapHost, port: settings.emailImapPort, @@ -272,7 +135,7 @@ export async function scanInbox(): Promise<{ processed: number; skipped: number; try { await scanClient.connect(); - log(`[scan] Connected for manual inbox scan`); + log(`[scan] Connected for inbox scan`); const lock = await scanClient.getMailboxLock(config.folder); try { diff --git a/src/lib/ftp-watcher.ts b/src/lib/ftp-watcher.ts new file mode 100644 index 0000000..effab2e --- /dev/null +++ b/src/lib/ftp-watcher.ts @@ -0,0 +1,147 @@ +import { Client } from "basic-ftp"; +import { Writable } from "stream"; +import { prisma } from "./db"; +import { uploadBuffer } from "./storage"; +import { enqueueProcessing } from "./processing-queue"; + +const ALLOWED_EXTENSIONS = [".pdf", ".jpg", ".jpeg", ".png", ".webp"]; +const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB + +function log(msg: string, ...args: unknown[]) { + console.log(`[ftp-watcher] ${msg}`, ...args); +} + +function logError(msg: string, ...args: unknown[]) { + console.error(`[ftp-watcher] ${msg}`, ...args); +} + +async function downloadToBuffer(client: Client, remotePath: string): Promise { + const chunks: Buffer[] = []; + const writable = new Writable({ + write(chunk, _encoding, callback) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + callback(); + }, + }); + await client.downloadTo(writable, remotePath); + return Buffer.concat(chunks); +} + +export async function pollFtp(): Promise<{ processed: number; skipped: number; error?: string }> { + const settings = await prisma.appSettings.findUnique({ where: { id: "singleton" } }); + + if (!settings?.ftpEnabled) { + return { processed: 0, skipped: 0, error: "FTP watching is disabled" }; + } + + if (!settings.ftpHost || !settings.ftpUser || !settings.ftpPass) { + return { processed: 0, skipped: 0, error: "FTP not configured" }; + } + + const client = new Client(); + let processed = 0; + let skipped = 0; + + try { + await client.access({ + host: settings.ftpHost, + port: settings.ftpPort, + user: settings.ftpUser, + password: settings.ftpPass, + secure: settings.ftpTls, + }); + + log(`Connected to ${settings.ftpHost}:${settings.ftpPort}`); + + const files = await client.list(settings.ftpIncomingDir); + const validFiles = files.filter((f) => { + if (f.isDirectory) return false; + const ext = f.name.toLowerCase().match(/\.[^.]+$/)?.[0] || ""; + return ALLOWED_EXTENSIONS.includes(ext); + }); + + log(`Found ${validFiles.length} file(s) in ${settings.ftpIncomingDir}`); + + for (const file of validFiles) { + const remotePath = `${settings.ftpIncomingDir}/${file.name}`; + const ext = file.name.toLowerCase().match(/\.[^.]+$/)?.[0] || ""; + + if (file.size > MAX_FILE_SIZE) { + log(`Skipping "${file.name}" — exceeds 50 MB (${(file.size / 1024 / 1024).toFixed(1)} MB)`); + skipped++; + continue; + } + + try { + const buffer = await downloadToBuffer(client, remotePath); + const isPdf = ext === ".pdf"; + const contentType = isPdf ? "application/pdf" : "image/jpeg"; + + const job = await prisma.processingJob.create({ + data: { + fileName: file.name, + filePath: `ftp/${file.name}`, + status: "queued", + }, + }); + + const sourceKey = `sources/${job.id}/${file.name}`; + await uploadBuffer(sourceKey, buffer, contentType); + await enqueueProcessing(job.id); + + const destPath = `${settings.ftpProcessedDir}/${file.name}`; + try { + await client.ensureDir(settings.ftpProcessedDir); + await client.rename(remotePath, destPath); + } catch { + log(`Could not move "${file.name}" to processed dir, removing instead`); + try { await client.remove(remotePath); } catch {} + } + + processed++; + log(`Queued: ${file.name} (job ${job.id})`); + } catch (err) { + logError(`Failed to process "${file.name}":`, err); + skipped++; + } + } + + client.close(); + } catch (err) { + const message = err instanceof Error ? err.message : "FTP poll failed"; + logError("Error:", message); + client.close(); + return { processed, skipped, error: message }; + } + + log(`Complete: ${processed} processed, ${skipped} skipped`); + return { processed, skipped }; +} + +export async function testFtpConnection(config: { + host: string; + port: number; + user: string; + pass: string; + tls: boolean; + incomingDir: string; +}): Promise<{ ok: boolean; files?: number; error?: string }> { + const client = new Client(); + try { + await client.access({ + host: config.host, + port: config.port, + user: config.user, + password: config.pass, + secure: config.tls, + }); + + const files = await client.list(config.incomingDir); + client.close(); + return { ok: true, files: files.length }; + } catch (err) { + const message = err instanceof Error ? err.message : "Connection failed"; + client.close(); + return { ok: false, error: message }; + } +} diff --git a/src/lib/integrations.ts b/src/lib/integrations.ts index dedd0c2..4d760fd 100644 --- a/src/lib/integrations.ts +++ b/src/lib/integrations.ts @@ -1,5 +1,5 @@ import { prisma } from "./db"; -import { getBuffer } from "./minio"; +import { getBuffer } from "./storage"; import { logActivity, diffCardFields } from "./activity-log"; import { createNotification } from "./notifications"; import { diff --git a/src/lib/ocr.ts b/src/lib/ocr.ts index 7fde243..4512c4c 100644 --- a/src/lib/ocr.ts +++ b/src/lib/ocr.ts @@ -1,6 +1,6 @@ import { prisma } from "./db"; import { Prisma } from "@/generated/prisma/client"; -import { uploadBuffer, getBuffer, deleteObject } from "./minio"; +import { uploadBuffer, getBuffer, deleteObject } from "./storage"; import { ocrImage } from "./ai-ocr"; import { pdfToImages, imageToBase64, processUploadedImage } from "./pdf"; import { fireIntegrationEvent } from "./integrations"; diff --git a/src/lib/pdf.ts b/src/lib/pdf.ts index d874fd0..f58a90a 100644 --- a/src/lib/pdf.ts +++ b/src/lib/pdf.ts @@ -1,7 +1,10 @@ -import { fromBuffer } from "pdf2pic"; -import { PDFDocument } from "pdf-lib"; +import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs"; import sharp from "sharp"; +const TARGET_DPI = 200; +const PDF_DEFAULT_DPI = 72; +const SCALE = TARGET_DPI / PDF_DEFAULT_DPI; + export interface PageImage { page: number; buffer: Buffer; @@ -9,68 +12,71 @@ export interface PageImage { height: number; } -export async function pdfToImages(pdfBuffer: Buffer): Promise { - const converter = fromBuffer(pdfBuffer, { - density: 200, - format: "jpeg", - width: 2200, - height: 2200, - quality: 90, - preserveAspectRatio: true, - }); - - const pageCount = await getPdfPageCount(pdfBuffer); - const images: PageImage[] = []; - - for (let i = 1; i <= pageCount; i++) { - const result = await converter(i, { responseType: "buffer" }); - if (result.buffer) { - let buf = result.buffer as Buffer; - let meta = await sharp(buf).metadata(); - const w = meta.width || 0; - const h = meta.height || 0; - - // Duplex scanners flip the sheet between sides, so odd pages - // (back/response card) are 180° rotated relative to even pages - // (front/survey). Landscape pages need rotation to portrait; - // the direction depends on which side of the sheet was scanned. - const isBackSide = i % 2 === 1; - - if (w > h) { - const angle = isBackSide ? -90 : 90; - buf = await sharp(buf).rotate(angle).jpeg({ quality: 90 }).toBuffer(); - meta = await sharp(buf).metadata(); - } else if (isBackSide) { - buf = await sharp(buf).rotate(180).jpeg({ quality: 90 }).toBuffer(); - meta = await sharp(buf).metadata(); - } - - images.push({ - page: i, - buffer: buf, - width: meta.width || 1600, - height: meta.height || 2200, - }); - } - } - - return images; +interface CanvasAndContext { + canvas: { toBuffer(mime: string): Buffer }; + context: unknown; } -async function getPdfPageCount(pdfBuffer: Buffer): Promise { - try { - const doc = await PDFDocument.load(pdfBuffer, { ignoreEncryption: true }); - const count = doc.getPageCount(); - console.log(`[pdf] Page count (pdf-lib): ${count}`); - return count; - } catch (err) { - console.warn("[pdf] pdf-lib page count failed, falling back to regex:", err); - const text = pdfBuffer.toString("latin1"); - const matches = text.match(/\/Type\s*\/Page(?!s)/g); - const count = matches ? matches.length : 2; - console.log(`[pdf] Page count (regex fallback): ${count}`); - return count; +interface CanvasFactory { + create(width: number, height: number): CanvasAndContext; +} + +export async function pdfToImages(pdfBuffer: Buffer): Promise { + const data = new Uint8Array(pdfBuffer); + const loadingTask = getDocument({ data, useSystemFonts: true }); + const pdfDocument = await loadingTask.promise; + const pageCount = pdfDocument.numPages; + const images: PageImage[] = []; + + const canvasFactory = pdfDocument.canvasFactory as unknown as CanvasFactory; + + console.log(`[pdf] Rendering ${pageCount} page(s) at ${TARGET_DPI} DPI (scale ${SCALE.toFixed(2)})`); + + for (let i = 1; i <= pageCount; i++) { + const page = await pdfDocument.getPage(i); + const viewport = page.getViewport({ scale: SCALE }); + + const { canvas, context } = canvasFactory.create( + Math.floor(viewport.width), + Math.floor(viewport.height) + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await page.render({ canvas: canvas as any, viewport } as any).promise; + + const pngBuffer = canvas.toBuffer("image/png"); + page.cleanup(); + + let buf = await sharp(pngBuffer).jpeg({ quality: 90 }).toBuffer(); + let meta = await sharp(buf).metadata(); + const w = meta.width || 0; + const h = meta.height || 0; + + // Duplex scanners flip the sheet between sides, so odd pages + // (back/response card) are 180-deg rotated relative to even pages + // (front/survey). Landscape pages need rotation to portrait; + // the direction depends on which side of the sheet was scanned. + const isBackSide = i % 2 === 1; + + if (w > h) { + const angle = isBackSide ? -90 : 90; + buf = await sharp(buf).rotate(angle).jpeg({ quality: 90 }).toBuffer(); + meta = await sharp(buf).metadata(); + } else if (isBackSide) { + buf = await sharp(buf).rotate(180).jpeg({ quality: 90 }).toBuffer(); + meta = await sharp(buf).metadata(); + } + + images.push({ + page: i, + buffer: buf, + width: meta.width || Math.floor(viewport.width), + height: meta.height || Math.floor(viewport.height), + }); } + + await pdfDocument.destroy(); + return images; } export async function imageToBase64(buffer: Buffer): Promise { diff --git a/src/lib/processing-queue.ts b/src/lib/processing-queue.ts index a5aa31e..e5d3bf6 100644 --- a/src/lib/processing-queue.ts +++ b/src/lib/processing-queue.ts @@ -1,47 +1,19 @@ -import { processFile } from "./ocr"; +import { Client } from "@upstash/qstash"; -const MAX_CONCURRENT = 1; -const DELAY_BETWEEN_JOBS_MS = 3_000; -const queue: QueueItem[] = []; -let running = 0; +const qstash = new Client({ token: process.env.QSTASH_TOKEN! }); -type QueueItem = { - jobId: string; - fileName: string; - fileBuffer: Buffer; - isPdf: boolean; - resolve: (ids: string[]) => void; - reject: (err: unknown) => void; -}; - -function drain() { - while (running < MAX_CONCURRENT && queue.length > 0) { - const item = queue.shift()!; - running++; - processFile(item.jobId, item.fileName, item.fileBuffer, item.isPdf) - .then(item.resolve) - .catch(item.reject) - .finally(() => { - running--; - if (queue.length > 0) { - setTimeout(drain, DELAY_BETWEEN_JOBS_MS); - } - }); +function getBaseUrl(): string { + if (process.env.VERCEL_URL) { + return `https://${process.env.VERCEL_URL}`; } + return process.env.AUTH_URL || "http://localhost:3000"; } -export function enqueueProcessing( - jobId: string, - fileName: string, - fileBuffer: Buffer, - isPdf: boolean -): Promise { - return new Promise((resolve, reject) => { - queue.push({ jobId, fileName, fileBuffer, isPdf, resolve, reject }); - if (running < MAX_CONCURRENT) drain(); +export async function enqueueProcessing(jobId: string): Promise { + const baseUrl = getBaseUrl(); + await qstash.publishJSON({ + url: `${baseUrl}/api/jobs/process`, + body: { jobId }, + retries: 2, }); } - -export function getQueueStatus() { - return { running, queued: queue.length, maxConcurrent: MAX_CONCURRENT }; -} diff --git a/src/lib/minio.ts b/src/lib/storage.ts similarity index 76% rename from src/lib/minio.ts rename to src/lib/storage.ts index ca1eec8..1d18218 100644 --- a/src/lib/minio.ts +++ b/src/lib/storage.ts @@ -1,23 +1,23 @@ import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; -const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT || "192.168.68.105"; -const MINIO_PORT = parseInt(process.env.MINIO_PORT || "9000"); -const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY || "minioadmin"; -const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY || ""; -const MINIO_BUCKET = process.env.MINIO_BUCKET || "echos-ocr"; +const S3_ENDPOINT = process.env.STORAGE_ENDPOINT || ""; +const S3_REGION = process.env.STORAGE_REGION || "us-east-1"; +const S3_ACCESS_KEY = process.env.STORAGE_ACCESS_KEY || ""; +const S3_SECRET_KEY = process.env.STORAGE_SECRET_KEY || ""; +const S3_BUCKET = process.env.STORAGE_BUCKET || "echos-ocr"; export const s3 = new S3Client({ - endpoint: `http://${MINIO_ENDPOINT}:${MINIO_PORT}`, - region: "us-east-1", + endpoint: S3_ENDPOINT, + region: S3_REGION, credentials: { - accessKeyId: MINIO_ACCESS_KEY, - secretAccessKey: MINIO_SECRET_KEY, + accessKeyId: S3_ACCESS_KEY, + secretAccessKey: S3_SECRET_KEY, }, forcePathStyle: true, }); -export const BUCKET = MINIO_BUCKET; +export const BUCKET = S3_BUCKET; export async function uploadBuffer(key: string, buffer: Buffer, contentType: string): Promise { await s3.send( diff --git a/src/lib/watcher.ts b/src/lib/watcher.ts deleted file mode 100644 index eb20bc6..0000000 --- a/src/lib/watcher.ts +++ /dev/null @@ -1,99 +0,0 @@ -import chokidar, { type FSWatcher } from "chokidar"; -import fs from "fs/promises"; -import path from "path"; -import { prisma } from "./db"; -import { enqueueProcessing } from "./processing-queue"; - -let watcher: FSWatcher | null = null; - -const processedFiles = new Set(); -const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB - -export async function startWatching(watchDir: string): Promise { - if (watcher) { - await stopWatching(); - } - - try { - await fs.access(watchDir); - } catch { - throw new Error(`Watch directory does not exist: ${watchDir}`); - } - - watcher = chokidar.watch(watchDir, { - ignored: /(^|[/\\])\../, - persistent: true, - ignoreInitial: false, - awaitWriteFinish: { - stabilityThreshold: 2000, - pollInterval: 500, - }, - }); - - watcher.on("add", async (filePath: string) => { - const ext = path.extname(filePath).toLowerCase(); - const allowed = [".pdf", ".jpg", ".jpeg", ".png", ".webp"]; - if (!allowed.includes(ext)) return; - if (processedFiles.has(filePath)) return; - processedFiles.add(filePath); - - const fileName = path.basename(filePath); - const isPdf = ext === ".pdf"; - - try { - const stat = await fs.stat(filePath); - if (stat.size > MAX_FILE_SIZE) { - console.log(`[watcher] Skipping ${fileName} — exceeds 50 MB size limit (${(stat.size / 1024 / 1024).toFixed(1)} MB)`); - return; - } - - const buffer = await fs.readFile(filePath); - const job = await prisma.processingJob.create({ - data: { - fileName, - filePath, - status: "queued", - }, - }); - - enqueueProcessing(job.id, fileName, buffer, isPdf).catch((err) => { - console.error(`[watcher] Processing failed for ${fileName}:`, err); - }); - - console.log(`[watcher] Queued: ${fileName}`); - } catch (err) { - console.error(`[watcher] Failed to read file ${filePath}:`, err); - } - }); - - watcher.on("error", (error: unknown) => { - console.error("[watcher] Error:", error); - }); - - await prisma.appSettings.upsert({ - where: { id: "singleton" }, - update: { watching: true, watchDir }, - create: { id: "singleton", watching: true, watchDir }, - }); - - console.log(`[watcher] Started watching: ${watchDir}`); -} - -export async function stopWatching(): Promise { - if (watcher) { - await watcher.close(); - watcher = null; - } - - await prisma.appSettings.upsert({ - where: { id: "singleton" }, - update: { watching: false }, - create: { id: "singleton", watching: false }, - }); - - console.log("[watcher] Stopped"); -} - -export function isWatching(): boolean { - return watcher !== null; -} diff --git a/src/middleware.ts b/src/middleware.ts index b7f2007..3966e13 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -9,6 +9,9 @@ const publicPaths = [ "/api/health", "/api/setup", "/api/onboarding", + "/api/jobs/process", + "/api/email-watch/poll", + "/api/ftp-watch/poll", ]; const onboardingExemptPaths = [ diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..86e780c --- /dev/null +++ b/vercel.json @@ -0,0 +1,12 @@ +{ + "crons": [ + { + "path": "/api/email-watch/poll", + "schedule": "*/2 * * * *" + }, + { + "path": "/api/ftp-watch/poll", + "schedule": "*/2 * * * *" + } + ] +}