Migrate to Vercel + Supabase + Upstash production stack
- Replace pdf2pic/GraphicsMagick with pdfjs-dist + @napi-rs/canvas for Vercel-compatible PDF rasterization - Replace MinIO with Supabase Storage (S3-compatible); rename minio.ts to storage.ts and update all imports - Replace in-memory job queue with Upstash QStash; upload route now persists files to storage before enqueuing, /api/jobs/process handles the QStash callback - Convert email watcher from persistent IMAP connection to stateless scanInbox() polled by Vercel Cron every 2 minutes - Add FTP watcher (basic-ftp) with cron polling for scanner integration via Dreamhost FTP drop directory - Add FTP config fields to AppSettings schema - Remove folder watcher (chokidar), standalone output, Docker-only code - Update next.config.ts, middleware, instrumentation for serverless - Add vercel.json with cron schedules for email and FTP polling - Add migration scripts for database (pg_dump/restore) and storage (S3-to-S3 copy) with verification Made-with: Cursor
This commit is contained in:
Randall Stillwell
parent
20bc4cbb03
commit
21c863da01
35 changed files with 1216 additions and 626 deletions
54
.env.example
54
.env.example
|
|
@ -1,44 +1,40 @@
|
|||
# Database (shared PostgreSQL on CT 102)
|
||||
DATABASE_URL="postgresql://echos_ocr:YOUR_PASSWORD@192.168.68.102:5432/echos_ocr"
|
||||
# ─── Database (Supabase pooled connection) ────────────────────
|
||||
DATABASE_URL="postgresql://postgres.<ref>:<password>@aws-0-us-east-1.pooler.supabase.com:6543/postgres"
|
||||
|
||||
# Vercel AI Gateway (routes to OpenAI, Google, Anthropic, etc.)
|
||||
# ─── Object Storage (Supabase Storage, S3-compatible) ─────────
|
||||
STORAGE_ENDPOINT="https://<ref>.supabase.co/storage/v1/s3"
|
||||
STORAGE_REGION="us-east-1"
|
||||
STORAGE_ACCESS_KEY=""
|
||||
STORAGE_SECRET_KEY=""
|
||||
STORAGE_BUCKET="echos-ocr"
|
||||
|
||||
# ─── Upstash QStash (job queue) ───────────────────────────────
|
||||
QSTASH_TOKEN=""
|
||||
QSTASH_CURRENT_SIGNING_KEY=""
|
||||
QSTASH_NEXT_SIGNING_KEY=""
|
||||
|
||||
# ─── Vercel AI Gateway ────────────────────────────────────────
|
||||
AI_GATEWAY_API_KEY=""
|
||||
|
||||
# Ollama (only needed if using Ollama as the AI provider)
|
||||
OLLAMA_BASE_URL="http://192.168.68.108:11434"
|
||||
|
||||
# MinIO S3 Storage (CT 105)
|
||||
MINIO_ENDPOINT="192.168.68.105"
|
||||
MINIO_PORT="9000"
|
||||
MINIO_ACCESS_KEY="minioadmin"
|
||||
MINIO_SECRET_KEY="YOUR_MINIO_SECRET"
|
||||
MINIO_BUCKET="echos-ocr"
|
||||
|
||||
# Folder Watch (optional, mount a host path into the container)
|
||||
WATCH_DIR=""
|
||||
|
||||
# Auth.js (required — generate with: npx auth secret)
|
||||
# ─── Auth.js (required — generate with: npx auth secret) ─────
|
||||
AUTH_SECRET=""
|
||||
# Set AUTH_URL to your external URL when behind a reverse proxy
|
||||
AUTH_URL="https://staging.echoocr.stillwell.cloud"
|
||||
AUTH_URL="https://echoocr.yourdomain.com"
|
||||
|
||||
# Authentik OIDC SSO (optional — enables "Sign in with SSO" button)
|
||||
# Create an OAuth2/OIDC provider in Authentik and set these values.
|
||||
# ─── OIDC SSO (optional) ──────────────────────────────────────
|
||||
AUTHENTIK_ISSUER=""
|
||||
AUTHENTIK_CLIENT_ID=""
|
||||
AUTHENTIK_CLIENT_SECRET=""
|
||||
|
||||
# Legacy Authentik forward-auth (deprecated — will be removed)
|
||||
AUTHENTIK_URL="https://auth.stillwell.cloud"
|
||||
AUTHENTIK_API_TOKEN=""
|
||||
|
||||
# SMTP for outbound email (verification, invitations)
|
||||
# Falls back to EMAIL_IMAP_* values if not set
|
||||
SMTP_HOST=""
|
||||
# ─── SMTP for outbound email ──────────────────────────────────
|
||||
SMTP_HOST="smtp.dreamhost.com"
|
||||
SMTP_PORT="587"
|
||||
SMTP_USER=""
|
||||
SMTP_PASS=""
|
||||
SMTP_FROM=""
|
||||
|
||||
# Environment indicator (set to "staging" for staging deployments)
|
||||
# ─── Vercel Cron Secret ────────────────────────────────────────
|
||||
# Vercel auto-sets this on Pro. Used to authenticate cron job requests.
|
||||
CRON_SECRET=""
|
||||
|
||||
# ─── Environment indicator ────────────────────────────────────
|
||||
NEXT_PUBLIC_ENV=""
|
||||
|
|
|
|||
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -35,6 +35,9 @@ yarn-error.log*
|
|||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# database dumps
|
||||
*.dump
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +1,15 @@
|
|||
import type { NextConfig } from "next";
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
output: "standalone",
|
||||
images: {
|
||||
remotePatterns: [
|
||||
{
|
||||
protocol: "http",
|
||||
hostname: "**",
|
||||
},
|
||||
{
|
||||
protocol: "https",
|
||||
hostname: "**",
|
||||
hostname: "**.supabase.co",
|
||||
},
|
||||
],
|
||||
},
|
||||
serverExternalPackages: ["sharp", "pdf2pic", "chokidar", "pg", "imapflow", "mailparser"],
|
||||
serverExternalPackages: ["sharp", "@napi-rs/canvas", "pg", "basic-ftp", "imapflow", "mailparser"],
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
|
|
|
|||
392
package-lock.json
generated
392
package-lock.json
generated
|
|
@ -15,12 +15,14 @@
|
|||
"@aws-sdk/client-s3": "^3.1005.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.1005.0",
|
||||
"@base-ui/react": "^1.2.0",
|
||||
"@napi-rs/canvas": "^0.1.98",
|
||||
"@prisma/adapter-pg": "^7.4.2",
|
||||
"@prisma/client": "^7.4.2",
|
||||
"@tanstack/react-table": "^8.21.3",
|
||||
"@upstash/qstash": "^2.10.1",
|
||||
"ai": "^6.0.116",
|
||||
"basic-ftp": "^5.3.0",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"chokidar": "^5.0.0",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"cmdk": "^1.1.1",
|
||||
|
|
@ -34,7 +36,7 @@
|
|||
"next-themes": "^0.4.6",
|
||||
"nodemailer": "^7.0.13",
|
||||
"pdf-lib": "^1.17.1",
|
||||
"pdf2pic": "^3.2.0",
|
||||
"pdfjs-dist": "^5.6.205",
|
||||
"pg": "^8.20.0",
|
||||
"prisma": "^7.4.2",
|
||||
"react": "^19.2.4",
|
||||
|
|
@ -2771,6 +2773,255 @@
|
|||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.98.tgz",
|
||||
"integrity": "sha512-WDg3lxYMqlrg49sDVUlrHVfIEPsd5AjYDRuGD6Fu82K5agJx0UnWA+l5qd53GNLRiMN2WhOw7FLR+Er5QB/0SA==",
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
"e2e/*"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas-android-arm64": "0.1.98",
|
||||
"@napi-rs/canvas-darwin-arm64": "0.1.98",
|
||||
"@napi-rs/canvas-darwin-x64": "0.1.98",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.98",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "0.1.98",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "0.1.98",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.98",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "0.1.98",
|
||||
"@napi-rs/canvas-linux-x64-musl": "0.1.98",
|
||||
"@napi-rs/canvas-win32-arm64-msvc": "0.1.98",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "0.1.98"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.98.tgz",
|
||||
"integrity": "sha512-O45Ifr0WZJUrSyg0QgB+67TiC0zYBRkBK+d43ZV4JtlwH3XttiVxLvlxEeULiH5y1MSELruspF0bjF6xXwJNPQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.98.tgz",
|
||||
"integrity": "sha512-1b/nQhw6Isdv14JokUqat+i5wrAYD+ce3egiotedBGRUjVxYSj4s2uQCh2bFsyX5/9A5iTKVGsWoQhFft+j7Lg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.98.tgz",
|
||||
"integrity": "sha512-oefzfBM8mwnyYp6S+yNXwjCoLdkOalFG24mssHgvrJDS0FulOryyI35Q7GdJGmrzuL4oo1XW3ZTOcTBLdJ8Zkg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.98.tgz",
|
||||
"integrity": "sha512-NDH5QXGmf8wlo5yhijCNGVFiJk7an5GvHwb2LHyfLQWY/6/S48i5+YtY6FPqPVVCUckNGudYOfXEJnb3/FiJGQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.98.tgz",
|
||||
"integrity": "sha512-KBLLM6tu1xs80LSAqdSLBKkgct0S23MCEf/aq8yxzg5imAceqp1ulKeELgWaYm27MgpUhm3Q7jmegX12FfphwA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.98.tgz",
|
||||
"integrity": "sha512-mfMNhjN5zDcJafqQ6sHj4Tc3YMTRxP5UA3MHtp/ssytBR/k6XO0x+1IIPtscnUKwha+ql1++WjDCGEgqu8OfWQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.98.tgz",
|
||||
"integrity": "sha512-nfW8esrcaeuhrO3qGA5cwuyk4Ak6cn2eB0LtEYtqROIl+fz06CNGNCU0M95+Tspw5ZgfSbc98SaigT5r5B3LVQ==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.98.tgz",
|
||||
"integrity": "sha512-318UT8j6Gro2bTjtutjQXHWp9SLTNw+WRS4wQ6XIRPAyzBGnGHg7x2ndD+oqkPrrSRIbYLA5WoBcCasaF7lSTQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.98.tgz",
|
||||
"integrity": "sha512-0vZhI74UxnA4VqlW4UvM0dFRrjE1RLEe/OXSBjzytGIxV+yOG4exlrhGoIpAQaIpQQQXMCdb1EmbvPC1k9vEqQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.98.tgz",
|
||||
"integrity": "sha512-oiC/IxgFEEVcZ7VH7JXXlmgsqRvmFb57PIQ4gQck35IKFZCNUvdNCcN3OeoLP7Hpf5160MWJf9jj/+E5V0bSvw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||
"version": "0.1.98",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.98.tgz",
|
||||
"integrity": "sha512-ZqstKAJBSyZetU8udUvBQWPlGN9buawFvjuo9mgCAxzbOoJAgXX39ihec/nn42T5Vb6/qyn45eTimx5ND9kMEw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/wasm-runtime": {
|
||||
"version": "0.2.12",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz",
|
||||
|
|
@ -5565,6 +5816,26 @@
|
|||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@upstash/qstash": {
|
||||
"version": "2.10.1",
|
||||
"resolved": "https://registry.npmjs.org/@upstash/qstash/-/qstash-2.10.1.tgz",
|
||||
"integrity": "sha512-LsTAPxPk0dFvhlEnRqwObRS94b8mmDHYaBlF3IaONUL+Scq6i9cZraA6936KSUVe+Vq3eNQle3CjOZkStPUFTw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"crypto-js": ">=4.2.0",
|
||||
"jose": "^5.2.3",
|
||||
"neverthrow": "^7.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@upstash/qstash/node_modules/jose": {
|
||||
"version": "5.10.0",
|
||||
"resolved": "https://registry.npmjs.org/jose/-/jose-5.10.0.tgz",
|
||||
"integrity": "sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/panva"
|
||||
}
|
||||
},
|
||||
"node_modules/@vercel/oidc": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz",
|
||||
|
|
@ -5808,18 +6079,6 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/array-parallel": {
|
||||
"version": "0.1.3",
|
||||
"resolved": "https://registry.npmjs.org/array-parallel/-/array-parallel-0.1.3.tgz",
|
||||
"integrity": "sha512-TDPTwSWW5E4oiFiKmz6RGJ/a80Y91GuLgUYuLd49+XBS75tYo8PNgaT2K/OxuQYqkoI852MDGBorg9OcUSTQ8w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/array-series": {
|
||||
"version": "0.1.5",
|
||||
"resolved": "https://registry.npmjs.org/array-series/-/array-series-0.1.5.tgz",
|
||||
"integrity": "sha512-L0XlBwfx9QetHOsbLDrE/vh2t018w9462HM3iaFfxRiK83aJjAt/Ja3NMkOW7FICwWTlQBa3ZbL5FKhuQWkDrg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/array.prototype.findlast": {
|
||||
"version": "1.2.5",
|
||||
"resolved": "https://registry.npmjs.org/array.prototype.findlast/-/array.prototype.findlast-1.2.5.tgz",
|
||||
|
|
@ -6042,6 +6301,15 @@
|
|||
"node": ">=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/basic-ftp": {
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.0.tgz",
|
||||
"integrity": "sha512-5K9eNNn7ywHPsYnFwjKgYH8Hf8B5emh7JKcPaVjjrMJFQQwGpwowEnZNEtHs7DfR7hCZsmaK3VA4HUK0YarT+w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bcryptjs": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-3.0.3.tgz",
|
||||
|
|
@ -6337,21 +6605,6 @@
|
|||
"regexp-to-ast": "0.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/chokidar": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz",
|
||||
"integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"readdirp": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 20.19.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://paulmillr.com/funding/"
|
||||
}
|
||||
},
|
||||
"node_modules/citty": {
|
||||
"version": "0.1.6",
|
||||
"resolved": "https://registry.npmjs.org/citty/-/citty-0.1.6.tgz",
|
||||
|
|
@ -6670,6 +6923,12 @@
|
|||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/crypto-js": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz",
|
||||
"integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/cssesc": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
|
||||
|
|
@ -8531,31 +8790,6 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/gm": {
|
||||
"version": "1.25.1",
|
||||
"resolved": "https://registry.npmjs.org/gm/-/gm-1.25.1.tgz",
|
||||
"integrity": "sha512-jgcs2vKir9hFogGhXIfs0ODhJTfIrbECCehg38tqFgHm8zqXx7kAJyCYAFK4jTjx71AxrkFtkJBawbAxYUPX9A==",
|
||||
"deprecated": "The gm module has been sunset. Please migrate to an alternative. https://github.com/aheckmann/gm?tab=readme-ov-file#2025-02-24-this-project-is-not-maintained",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"array-parallel": "~0.1.3",
|
||||
"array-series": "~0.1.5",
|
||||
"cross-spawn": "^7.0.5",
|
||||
"debug": "^3.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/gm/node_modules/debug": {
|
||||
"version": "3.2.7",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
|
||||
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/gopd": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
|
||||
|
|
@ -10483,6 +10717,15 @@
|
|||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/neverthrow": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/neverthrow/-/neverthrow-7.2.0.tgz",
|
||||
"integrity": "sha512-iGBUfFB7yPczHHtA8dksKTJ9E8TESNTAx1UQWW6TzMF280vo9jdPYpLUXrMN1BCkPdHFdNG3fxOt2CUad8KhAw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/next": {
|
||||
"version": "16.1.6",
|
||||
"resolved": "https://registry.npmjs.org/next/-/next-16.1.6.tgz",
|
||||
|
|
@ -10664,6 +10907,13 @@
|
|||
"integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/node-readable-to-web-readable-stream": {
|
||||
"version": "0.4.2",
|
||||
"resolved": "https://registry.npmjs.org/node-readable-to-web-readable-stream/-/node-readable-to-web-readable-stream-0.4.2.tgz",
|
||||
"integrity": "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ==",
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/node-releases": {
|
||||
"version": "2.0.36",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz",
|
||||
|
|
@ -11187,20 +11437,17 @@
|
|||
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
|
||||
"license": "0BSD"
|
||||
},
|
||||
"node_modules/pdf2pic": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pdf2pic/-/pdf2pic-3.2.0.tgz",
|
||||
"integrity": "sha512-p0bp+Mp4iJy2hqSCLvJ521rDaZkzBvDFT9O9Y0BUID3I04/eDaebAFM5t8hoWeo2BCf42cDijLCGJWTOtkJVpA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"gm": "^1.25.1"
|
||||
},
|
||||
"node_modules/pdfjs-dist": {
|
||||
"version": "5.6.205",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.6.205.tgz",
|
||||
"integrity": "sha512-tlUj+2IDa7G1SbvBNN74UHRLJybZDWYom+k6p5KIZl7huBvsA4APi6mKL+zCxd3tLjN5hOOEE9Tv7VdzO88pfg==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
"node": ">=20.19.0 || >=22.13.0 || >=24"
|
||||
},
|
||||
"funding": {
|
||||
"type": "paypal",
|
||||
"url": "https://www.paypal.me/yakovmeister"
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas": "^0.1.96",
|
||||
"node-readable-to-web-readable-stream": "^0.4.2"
|
||||
}
|
||||
},
|
||||
"node_modules/peberminta": {
|
||||
|
|
@ -11870,19 +12117,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/readdirp": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz",
|
||||
"integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 20.19.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "individual",
|
||||
"url": "https://paulmillr.com/funding/"
|
||||
}
|
||||
},
|
||||
"node_modules/real-require": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz",
|
||||
|
|
|
|||
|
|
@ -19,12 +19,14 @@
|
|||
"@aws-sdk/client-s3": "^3.1005.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.1005.0",
|
||||
"@base-ui/react": "^1.2.0",
|
||||
"@napi-rs/canvas": "^0.1.98",
|
||||
"@prisma/adapter-pg": "^7.4.2",
|
||||
"@prisma/client": "^7.4.2",
|
||||
"@tanstack/react-table": "^8.21.3",
|
||||
"@upstash/qstash": "^2.10.1",
|
||||
"ai": "^6.0.116",
|
||||
"basic-ftp": "^5.3.0",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"chokidar": "^5.0.0",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"cmdk": "^1.1.1",
|
||||
|
|
@ -38,7 +40,7 @@
|
|||
"next-themes": "^0.4.6",
|
||||
"nodemailer": "^7.0.13",
|
||||
"pdf-lib": "^1.17.1",
|
||||
"pdf2pic": "^3.2.0",
|
||||
"pdfjs-dist": "^5.6.205",
|
||||
"pg": "^8.20.0",
|
||||
"prisma": "^7.4.2",
|
||||
"react": "^19.2.4",
|
||||
|
|
|
|||
|
|
@ -321,6 +321,15 @@ model AppSettings {
|
|||
emailWatching Boolean @default(false)
|
||||
emailProcessed String @default("mark_read")
|
||||
emailProcessedFolder String @default("Processed")
|
||||
|
||||
ftpEnabled Boolean @default(false)
|
||||
ftpHost String @default("")
|
||||
ftpPort Int @default(21)
|
||||
ftpUser String @default("")
|
||||
ftpPass String @default("")
|
||||
ftpTls Boolean @default(true)
|
||||
ftpIncomingDir String @default("/incoming")
|
||||
ftpProcessedDir String @default("/processed")
|
||||
}
|
||||
|
||||
model ActivityLog {
|
||||
|
|
|
|||
76
scripts/migrate-database.sh
Executable file
76
scripts/migrate-database.sh
Executable file
|
|
@ -0,0 +1,76 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# ─── Database Migration: Homelab PostgreSQL → Supabase ────────
|
||||
#
|
||||
# Prerequisites:
|
||||
# - pg_dump and pg_restore installed locally
|
||||
# - Network access to both the homelab PG and Supabase PG
|
||||
# - Supabase project created with direct connection string
|
||||
#
|
||||
# Usage:
|
||||
# HOMELAB_DB_URL="postgresql://echos_ocr:PASSWORD@192.168.68.102:5432/echos_ocr" \
|
||||
# SUPABASE_DB_URL="postgresql://postgres.REF:PASSWORD@db.REF.supabase.co:5432/postgres" \
|
||||
# bash scripts/migrate-database.sh
|
||||
|
||||
if [ -z "${HOMELAB_DB_URL:-}" ]; then
|
||||
echo "Error: HOMELAB_DB_URL is not set"
|
||||
echo "Example: postgresql://echos_ocr:PASSWORD@192.168.68.102:5432/echos_ocr"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${SUPABASE_DB_URL:-}" ]; then
|
||||
echo "Error: SUPABASE_DB_URL is not set (use the DIRECT connection, port 5432, not pooled)"
|
||||
echo "Example: postgresql://postgres.REF:PASSWORD@db.REF.supabase.co:5432/postgres"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DUMP_FILE="echos_ocr_backup_$(date +%Y%m%d_%H%M%S).dump"
|
||||
|
||||
echo "=== Step 1: Dumping homelab database ==="
|
||||
pg_dump "${HOMELAB_DB_URL}" -Fc -f "${DUMP_FILE}"
|
||||
echo "Dump created: ${DUMP_FILE} ($(du -h "${DUMP_FILE}" | cut -f1))"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 2: Restoring to Supabase ==="
|
||||
pg_restore "${SUPABASE_DB_URL}" --no-owner --no-privileges --clean --if-exists "${DUMP_FILE}" 2>&1 || true
|
||||
echo "Restore complete (some warnings about existing objects are normal)"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 3: Pushing schema with Prisma ==="
|
||||
echo "Running 'npx prisma db push' to ensure schema alignment..."
|
||||
DATABASE_URL="${SUPABASE_DB_URL}" npx prisma db push --skip-generate 2>&1
|
||||
echo "Schema push complete"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 4: Verifying row counts ==="
|
||||
psql "${SUPABASE_DB_URL}" -c "
|
||||
SELECT 'User' as table_name, count(*) FROM \"User\"
|
||||
UNION ALL SELECT 'Organization', count(*) FROM \"Organization\"
|
||||
UNION ALL SELECT 'OrgMember', count(*) FROM \"OrgMember\"
|
||||
UNION ALL SELECT 'ResponseCard', count(*) FROM \"ResponseCard\"
|
||||
UNION ALL SELECT 'ProcessingJob', count(*) FROM \"ProcessingJob\"
|
||||
UNION ALL SELECT 'ActivityLog', count(*) FROM \"ActivityLog\"
|
||||
UNION ALL SELECT 'Notification', count(*) FROM \"Notification\"
|
||||
UNION ALL SELECT 'Integration', count(*) FROM \"Integration\"
|
||||
ORDER BY table_name;
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== Compare with homelab counts ==="
|
||||
psql "${HOMELAB_DB_URL}" -c "
|
||||
SELECT 'User' as table_name, count(*) FROM \"User\"
|
||||
UNION ALL SELECT 'Organization', count(*) FROM \"Organization\"
|
||||
UNION ALL SELECT 'OrgMember', count(*) FROM \"OrgMember\"
|
||||
UNION ALL SELECT 'ResponseCard', count(*) FROM \"ResponseCard\"
|
||||
UNION ALL SELECT 'ProcessingJob', count(*) FROM \"ProcessingJob\"
|
||||
UNION ALL SELECT 'ActivityLog', count(*) FROM \"ActivityLog\"
|
||||
UNION ALL SELECT 'Notification', count(*) FROM \"Notification\"
|
||||
UNION ALL SELECT 'Integration', count(*) FROM \"Integration\"
|
||||
ORDER BY table_name;
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "=== Migration complete ==="
|
||||
echo "Backup file retained at: ${DUMP_FILE}"
|
||||
echo "Verify the counts above match between homelab and Supabase."
|
||||
173
scripts/migrate-storage.ts
Normal file
173
scripts/migrate-storage.ts
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
/**
|
||||
* Storage Migration: MinIO → Supabase Storage
|
||||
*
|
||||
* Copies all objects from a MinIO bucket to a Supabase Storage bucket,
|
||||
* preserving the same key paths so database references remain valid.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/migrate-storage.ts
|
||||
*
|
||||
* Required env vars:
|
||||
* MINIO_ENDPOINT - e.g. 192.168.68.105
|
||||
* MINIO_PORT - e.g. 9000
|
||||
* MINIO_ACCESS_KEY - e.g. minioadmin
|
||||
* MINIO_SECRET_KEY - your secret
|
||||
* MINIO_BUCKET - e.g. echos-ocr
|
||||
*
|
||||
* STORAGE_ENDPOINT - e.g. https://xxx.supabase.co/storage/v1/s3
|
||||
* STORAGE_REGION - e.g. us-east-1
|
||||
* STORAGE_ACCESS_KEY - Supabase S3 access key
|
||||
* STORAGE_SECRET_KEY - Supabase S3 secret key
|
||||
* STORAGE_BUCKET - e.g. echos-ocr
|
||||
*/
|
||||
|
||||
import "dotenv/config";
|
||||
import {
|
||||
S3Client,
|
||||
ListObjectsV2Command,
|
||||
GetObjectCommand,
|
||||
PutObjectCommand,
|
||||
HeadObjectCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
|
||||
const sourceClient = new S3Client({
|
||||
endpoint: `http://${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT || "9000"}`,
|
||||
region: "us-east-1",
|
||||
credentials: {
|
||||
accessKeyId: process.env.MINIO_ACCESS_KEY || "",
|
||||
secretAccessKey: process.env.MINIO_SECRET_KEY || "",
|
||||
},
|
||||
forcePathStyle: true,
|
||||
});
|
||||
|
||||
const destClient = new S3Client({
|
||||
endpoint: process.env.STORAGE_ENDPOINT || "",
|
||||
region: process.env.STORAGE_REGION || "us-east-1",
|
||||
credentials: {
|
||||
accessKeyId: process.env.STORAGE_ACCESS_KEY || "",
|
||||
secretAccessKey: process.env.STORAGE_SECRET_KEY || "",
|
||||
},
|
||||
forcePathStyle: true,
|
||||
});
|
||||
|
||||
const SOURCE_BUCKET = process.env.MINIO_BUCKET || "echos-ocr";
|
||||
const DEST_BUCKET = process.env.STORAGE_BUCKET || "echos-ocr";
|
||||
|
||||
async function listAllKeys(): Promise<string[]> {
|
||||
const keys: string[] = [];
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
do {
|
||||
const res = await sourceClient.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: SOURCE_BUCKET,
|
||||
ContinuationToken: continuationToken,
|
||||
})
|
||||
);
|
||||
for (const obj of res.Contents ?? []) {
|
||||
if (obj.Key) keys.push(obj.Key);
|
||||
}
|
||||
continuationToken = res.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
async function objectExists(key: string): Promise<boolean> {
|
||||
try {
|
||||
await destClient.send(
|
||||
new HeadObjectCommand({ Bucket: DEST_BUCKET, Key: key })
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function copyObject(key: string): Promise<void> {
|
||||
const res = await sourceClient.send(
|
||||
new GetObjectCommand({ Bucket: SOURCE_BUCKET, Key: key })
|
||||
);
|
||||
const stream = res.Body as ReadableStream;
|
||||
const buffer = Buffer.from(await new Response(stream).arrayBuffer());
|
||||
|
||||
await destClient.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: DEST_BUCKET,
|
||||
Key: key,
|
||||
Body: buffer,
|
||||
ContentType: res.ContentType || "application/octet-stream",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("=== Storage Migration: MinIO → Supabase Storage ===");
|
||||
console.log(`Source: ${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT} / ${SOURCE_BUCKET}`);
|
||||
console.log(`Dest: ${process.env.STORAGE_ENDPOINT} / ${DEST_BUCKET}`);
|
||||
console.log("");
|
||||
|
||||
console.log("Listing all objects in source bucket...");
|
||||
const keys = await listAllKeys();
|
||||
console.log(`Found ${keys.length} object(s) to migrate`);
|
||||
|
||||
if (keys.length === 0) {
|
||||
console.log("Nothing to migrate.");
|
||||
return;
|
||||
}
|
||||
|
||||
const sourcesCount = keys.filter((k) => k.startsWith("sources/")).length;
|
||||
const imagesCount = keys.filter((k) => k.startsWith("images/")).length;
|
||||
const otherCount = keys.length - sourcesCount - imagesCount;
|
||||
console.log(` sources/: ${sourcesCount}, images/: ${imagesCount}, other: ${otherCount}`);
|
||||
console.log("");
|
||||
|
||||
let copied = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
const key = keys[i];
|
||||
const progress = `[${i + 1}/${keys.length}]`;
|
||||
|
||||
try {
|
||||
const exists = await objectExists(key);
|
||||
if (exists) {
|
||||
skipped++;
|
||||
if ((i + 1) % 50 === 0) {
|
||||
console.log(`${progress} Skipped (already exists): ${key}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
await copyObject(key);
|
||||
copied++;
|
||||
|
||||
if (copied % 10 === 0 || (i + 1) === keys.length) {
|
||||
console.log(`${progress} Copied: ${key} (${copied} total)`);
|
||||
}
|
||||
} catch (err) {
|
||||
errors++;
|
||||
console.error(`${progress} ERROR copying ${key}:`, err instanceof Error ? err.message : err);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("");
|
||||
console.log("=== Migration Summary ===");
|
||||
console.log(`Total objects: ${keys.length}`);
|
||||
console.log(`Copied: ${copied}`);
|
||||
console.log(`Skipped: ${skipped} (already existed in destination)`);
|
||||
console.log(`Errors: ${errors}`);
|
||||
|
||||
if (errors > 0) {
|
||||
console.log("\nSome objects failed to copy. Re-run this script to retry (it skips already-copied objects).");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("\nAll objects migrated successfully.");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("Fatal error:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
131
scripts/verify-migration.ts
Normal file
131
scripts/verify-migration.ts
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
/**
|
||||
* Post-Migration Verification
|
||||
*
|
||||
* Checks that the database and storage are intact after migration.
|
||||
*
|
||||
* Usage:
|
||||
* DATABASE_URL="postgresql://..." \
|
||||
* STORAGE_ENDPOINT="https://..." \
|
||||
* STORAGE_ACCESS_KEY="..." \
|
||||
* STORAGE_SECRET_KEY="..." \
|
||||
* STORAGE_BUCKET="echos-ocr" \
|
||||
* npx tsx scripts/verify-migration.ts
|
||||
*/
|
||||
|
||||
import "dotenv/config";
|
||||
import pg from "pg";
|
||||
import { S3Client, ListObjectsV2Command, HeadObjectCommand } from "@aws-sdk/client-s3";
|
||||
|
||||
const pool = new pg.Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
max: 3,
|
||||
});
|
||||
|
||||
const s3 = new S3Client({
|
||||
endpoint: process.env.STORAGE_ENDPOINT || "",
|
||||
region: process.env.STORAGE_REGION || "us-east-1",
|
||||
credentials: {
|
||||
accessKeyId: process.env.STORAGE_ACCESS_KEY || "",
|
||||
secretAccessKey: process.env.STORAGE_SECRET_KEY || "",
|
||||
},
|
||||
forcePathStyle: true,
|
||||
});
|
||||
|
||||
const BUCKET = process.env.STORAGE_BUCKET || "echos-ocr";
|
||||
|
||||
async function countTable(table: string): Promise<number> {
|
||||
const res = await pool.query(`SELECT count(*)::int as cnt FROM "${table}"`);
|
||||
return res.rows[0].cnt;
|
||||
}
|
||||
|
||||
async function countStorageObjects(prefix: string): Promise<number> {
|
||||
let count = 0;
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
do {
|
||||
const res = await s3.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: BUCKET,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: continuationToken,
|
||||
})
|
||||
);
|
||||
count += res.KeyCount || 0;
|
||||
continuationToken = res.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
async function spotCheck(): Promise<{ checked: number; ok: number; missing: number }> {
|
||||
const res = await pool.query(`
|
||||
SELECT id, "frontImagePath", "backImagePath"
|
||||
FROM "ResponseCard"
|
||||
WHERE "frontImagePath" IS NOT NULL OR "backImagePath" IS NOT NULL
|
||||
ORDER BY "createdAt" DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
let checked = 0;
|
||||
let ok = 0;
|
||||
let missing = 0;
|
||||
|
||||
for (const row of res.rows) {
|
||||
for (const path of [row.frontImagePath, row.backImagePath]) {
|
||||
if (!path) continue;
|
||||
checked++;
|
||||
try {
|
||||
await s3.send(new HeadObjectCommand({ Bucket: BUCKET, Key: path }));
|
||||
ok++;
|
||||
} catch {
|
||||
console.log(` MISSING: ${path} (card ${row.id})`);
|
||||
missing++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { checked, ok, missing };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("=== Post-Migration Verification ===\n");
|
||||
|
||||
console.log("--- Database Row Counts ---");
|
||||
const tables = [
|
||||
"User", "Organization", "OrgMember", "Location",
|
||||
"ResponseCard", "ProcessingJob", "ActivityLog",
|
||||
"Notification", "Integration", "AppSettings",
|
||||
];
|
||||
for (const table of tables) {
|
||||
try {
|
||||
const count = await countTable(table);
|
||||
console.log(` ${table.padEnd(20)} ${count}`);
|
||||
} catch (err) {
|
||||
console.log(` ${table.padEnd(20)} ERROR: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\n--- Storage Object Counts ---");
|
||||
const imageCount = await countStorageObjects("images/");
|
||||
const sourceCount = await countStorageObjects("sources/");
|
||||
console.log(` images/ ${imageCount}`);
|
||||
console.log(` sources/ ${sourceCount}`);
|
||||
|
||||
console.log("\n--- Spot Check: 10 newest cards ---");
|
||||
const spotResult = await spotCheck();
|
||||
console.log(` Checked: ${spotResult.checked}, OK: ${spotResult.ok}, Missing: ${spotResult.missing}`);
|
||||
|
||||
if (spotResult.missing > 0) {
|
||||
console.log("\n WARNING: Some storage objects are missing. Storage migration may be incomplete.");
|
||||
} else {
|
||||
console.log("\n All spot-checked objects found in storage.");
|
||||
}
|
||||
|
||||
console.log("\n=== Verification Complete ===");
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("Fatal error:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { deleteObject } from "@/lib/minio";
|
||||
import { deleteObject } from "@/lib/storage";
|
||||
import { fireIntegrationEvent } from "@/lib/integrations";
|
||||
import { logActivity, diffCardFields } from "@/lib/activity-log";
|
||||
import { getOrCreateUser, RoleError } from "@/lib/auth";
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getBuffer, uploadBuffer } from "@/lib/minio";
|
||||
import { getBuffer, uploadBuffer } from "@/lib/storage";
|
||||
import { pdfToImages, imageToBase64 } from "@/lib/pdf";
|
||||
import { ocrImage } from "@/lib/ai-ocr";
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { deleteObject, listObjects } from "@/lib/minio";
|
||||
import { deleteObject, listObjects } from "@/lib/storage";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
|
|
|
|||
12
src/app/api/email-watch/poll/route.ts
Normal file
12
src/app/api/email-watch/poll/route.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { scanInbox } from "@/lib/email-watcher";
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const authHeader = request.headers.get("authorization");
|
||||
if (authHeader !== `Bearer ${process.env.CRON_SECRET}`) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
|
||||
const result = await scanInbox();
|
||||
return NextResponse.json(result);
|
||||
}
|
||||
|
|
@ -1,20 +1,15 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import {
|
||||
startEmailWatching,
|
||||
stopEmailWatching,
|
||||
isEmailWatching,
|
||||
scanInbox,
|
||||
} from "@/lib/email-watcher";
|
||||
import { scanInbox } from "@/lib/email-watcher";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const body = await request.json().catch(() => ({}));
|
||||
const action = body.action as string | undefined;
|
||||
|
||||
if (!action || !["start", "stop", "scan"].includes(action)) {
|
||||
if (!action || !["enable", "disable", "scan"].includes(action)) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid action. Use 'start', 'stop', or 'scan'" },
|
||||
{ error: "Invalid action. Use 'enable', 'disable', or 'scan'" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
|
@ -27,18 +22,18 @@ export async function POST(request: NextRequest) {
|
|||
return NextResponse.json({ ok: true, ...result });
|
||||
}
|
||||
|
||||
if (action === "start") {
|
||||
const settings = await prisma.appSettings.findUnique({
|
||||
where: { id: "singleton" },
|
||||
});
|
||||
const settings = await prisma.appSettings.findUnique({
|
||||
where: { id: "singleton" },
|
||||
});
|
||||
|
||||
if (!settings) {
|
||||
return NextResponse.json(
|
||||
{ error: "Settings not configured" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
if (!settings) {
|
||||
return NextResponse.json(
|
||||
{ error: "Settings not configured" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
if (action === "enable") {
|
||||
if (!settings.emailImapHost || !settings.emailImapUser || !settings.emailImapPass) {
|
||||
return NextResponse.json(
|
||||
{ error: "IMAP host, username, and password are required" },
|
||||
|
|
@ -46,21 +41,19 @@ export async function POST(request: NextRequest) {
|
|||
);
|
||||
}
|
||||
|
||||
await startEmailWatching({
|
||||
host: settings.emailImapHost,
|
||||
port: settings.emailImapPort,
|
||||
user: settings.emailImapUser,
|
||||
pass: settings.emailImapPass,
|
||||
tls: settings.emailImapTls,
|
||||
folder: settings.emailFolder,
|
||||
processedAction: settings.emailProcessed,
|
||||
processedFolder: settings.emailProcessedFolder,
|
||||
await prisma.appSettings.update({
|
||||
where: { id: "singleton" },
|
||||
data: { emailWatching: true },
|
||||
});
|
||||
} else {
|
||||
await stopEmailWatching();
|
||||
await prisma.appSettings.update({
|
||||
where: { id: "singleton" },
|
||||
data: { emailWatching: false },
|
||||
});
|
||||
}
|
||||
|
||||
return NextResponse.json({ emailWatching: isEmailWatching() });
|
||||
const updated = await prisma.appSettings.findUnique({ where: { id: "singleton" } });
|
||||
return NextResponse.json({ emailWatching: updated?.emailWatching ?? false });
|
||||
} catch (error) {
|
||||
console.error("[email-watch POST]", error);
|
||||
const message =
|
||||
|
|
|
|||
12
src/app/api/ftp-watch/poll/route.ts
Normal file
12
src/app/api/ftp-watch/poll/route.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { pollFtp } from "@/lib/ftp-watcher";
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const authHeader = request.headers.get("authorization");
|
||||
if (authHeader !== `Bearer ${process.env.CRON_SECRET}`) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
|
||||
const result = await pollFtp();
|
||||
return NextResponse.json(result);
|
||||
}
|
||||
63
src/app/api/ftp-watch/route.ts
Normal file
63
src/app/api/ftp-watch/route.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { pollFtp } from "@/lib/ftp-watcher";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const body = await request.json().catch(() => ({}));
|
||||
const action = body.action as string | undefined;
|
||||
|
||||
if (!action || !["enable", "disable", "scan"].includes(action)) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid action. Use 'enable', 'disable', or 'scan'" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
if (action === "scan") {
|
||||
const result = await pollFtp();
|
||||
if (result.error) {
|
||||
return NextResponse.json({ error: result.error, ...result }, { status: 500 });
|
||||
}
|
||||
return NextResponse.json({ ok: true, ...result });
|
||||
}
|
||||
|
||||
const settings = await prisma.appSettings.findUnique({
|
||||
where: { id: "singleton" },
|
||||
});
|
||||
|
||||
if (!settings) {
|
||||
return NextResponse.json(
|
||||
{ error: "Settings not configured" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
if (action === "enable") {
|
||||
if (!settings.ftpHost || !settings.ftpUser || !settings.ftpPass) {
|
||||
return NextResponse.json(
|
||||
{ error: "FTP host, username, and password are required" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
await prisma.appSettings.update({
|
||||
where: { id: "singleton" },
|
||||
data: { ftpEnabled: true },
|
||||
});
|
||||
} else {
|
||||
await prisma.appSettings.update({
|
||||
where: { id: "singleton" },
|
||||
data: { ftpEnabled: false },
|
||||
});
|
||||
}
|
||||
|
||||
const updated = await prisma.appSettings.findUnique({ where: { id: "singleton" } });
|
||||
return NextResponse.json({ ftpEnabled: updated?.ftpEnabled ?? false });
|
||||
} catch (error) {
|
||||
console.error("[ftp-watch POST]", error);
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Failed to update FTP watch";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
23
src/app/api/ftp-watch/test/route.ts
Normal file
23
src/app/api/ftp-watch/test/route.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { testFtpConnection } from "@/lib/ftp-watcher";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const body = await request.json();
|
||||
const result = await testFtpConnection({
|
||||
host: body.host,
|
||||
port: body.port || 21,
|
||||
user: body.user,
|
||||
pass: body.pass,
|
||||
tls: body.tls ?? true,
|
||||
incomingDir: body.incomingDir || "/incoming",
|
||||
});
|
||||
return NextResponse.json(result);
|
||||
} catch (error) {
|
||||
console.error("[ftp-watch/test POST]", error);
|
||||
return NextResponse.json(
|
||||
{ ok: false, error: "Test failed" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,44 +1,27 @@
|
|||
import { NextResponse } from "next/server";
|
||||
import { HeadBucketCommand } from "@aws-sdk/client-s3";
|
||||
import { s3, BUCKET } from "@/lib/minio";
|
||||
import { s3, BUCKET } from "@/lib/storage";
|
||||
|
||||
export async function GET() {
|
||||
const results: Record<string, unknown> = {
|
||||
minio: { status: "unknown" },
|
||||
minioConfig: {
|
||||
endpoint: process.env.MINIO_ENDPOINT || "192.168.68.105",
|
||||
port: process.env.MINIO_PORT || "9000",
|
||||
storage: { status: "unknown" },
|
||||
storageConfig: {
|
||||
endpoint: process.env.STORAGE_ENDPOINT || "(not set)",
|
||||
bucket: BUCKET,
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await s3.send(new HeadBucketCommand({ Bucket: BUCKET }));
|
||||
results.minio = { status: "ok", bucket: BUCKET };
|
||||
results.storage = { status: "ok", bucket: BUCKET };
|
||||
} catch (error: unknown) {
|
||||
const err = error as Error & { $response?: { body?: unknown }; $metadata?: unknown; Code?: string };
|
||||
results.minio = {
|
||||
const err = error as Error & { $metadata?: unknown; Code?: string };
|
||||
results.storage = {
|
||||
status: "error",
|
||||
message: err.message,
|
||||
code: err.Code,
|
||||
metadata: err.$metadata,
|
||||
};
|
||||
|
||||
// Try a raw fetch to see what the endpoint actually returns
|
||||
try {
|
||||
const endpoint = `http://${process.env.MINIO_ENDPOINT || "192.168.68.105"}:${process.env.MINIO_PORT || "9000"}`;
|
||||
const rawRes = await fetch(endpoint, { signal: AbortSignal.timeout(5000) });
|
||||
const rawBody = await rawRes.text();
|
||||
results.rawEndpointResponse = {
|
||||
status: rawRes.status,
|
||||
contentType: rawRes.headers.get("content-type"),
|
||||
bodyPreview: rawBody.slice(0, 500),
|
||||
};
|
||||
} catch (fetchErr: unknown) {
|
||||
results.rawEndpointResponse = {
|
||||
error: (fetchErr as Error).message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json(results, { status: 200 });
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { getBuffer } from "@/lib/minio";
|
||||
import { getBuffer } from "@/lib/storage";
|
||||
|
||||
const EXT_TO_CONTENT_TYPE: Record<string, string> = {
|
||||
jpg: "image/jpeg",
|
||||
|
|
|
|||
50
src/app/api/jobs/process/route.ts
Normal file
50
src/app/api/jobs/process/route.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { Receiver } from "@upstash/qstash";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getBuffer } from "@/lib/storage";
|
||||
import { processFile } from "@/lib/ocr";
|
||||
|
||||
const receiver = new Receiver({
|
||||
currentSigningKey: process.env.QSTASH_CURRENT_SIGNING_KEY!,
|
||||
nextSigningKey: process.env.QSTASH_NEXT_SIGNING_KEY!,
|
||||
});
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const body = await request.text();
|
||||
|
||||
const isValid = await receiver.verify({
|
||||
signature: request.headers.get("upstash-signature") || "",
|
||||
body,
|
||||
});
|
||||
|
||||
if (!isValid) {
|
||||
return NextResponse.json({ error: "Invalid signature" }, { status: 401 });
|
||||
}
|
||||
|
||||
const { jobId } = JSON.parse(body) as { jobId: string };
|
||||
|
||||
const job = await prisma.processingJob.findUnique({ where: { id: jobId } });
|
||||
if (!job) {
|
||||
return NextResponse.json({ error: "Job not found" }, { status: 404 });
|
||||
}
|
||||
|
||||
if (job.status !== "queued") {
|
||||
return NextResponse.json({ ok: true, skipped: true, reason: `Job status is ${job.status}` });
|
||||
}
|
||||
|
||||
const sourceKey = `sources/${job.id}/${job.fileName}`;
|
||||
const fileBuffer = await getBuffer(sourceKey);
|
||||
const isPdf = job.fileName.toLowerCase().endsWith(".pdf");
|
||||
|
||||
const cardIds = await processFile(job.id, job.fileName, fileBuffer, isPdf);
|
||||
|
||||
return NextResponse.json({ ok: true, cardIds });
|
||||
} catch (error) {
|
||||
console.error("[jobs/process POST]", error);
|
||||
return NextResponse.json(
|
||||
{ error: error instanceof Error ? error.message : "Processing failed" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -58,6 +58,15 @@ export async function PUT(request: NextRequest) {
|
|||
if (body.emailProcessed != null) data.emailProcessed = String(body.emailProcessed);
|
||||
if (body.emailProcessedFolder != null) data.emailProcessedFolder = String(body.emailProcessedFolder);
|
||||
|
||||
if (body.ftpEnabled != null) data.ftpEnabled = Boolean(body.ftpEnabled);
|
||||
if (body.ftpHost != null) data.ftpHost = String(body.ftpHost);
|
||||
if (body.ftpPort != null) data.ftpPort = Math.max(1, parseInt(String(body.ftpPort)) || 21);
|
||||
if (body.ftpUser != null) data.ftpUser = String(body.ftpUser);
|
||||
if (body.ftpPass != null) data.ftpPass = String(body.ftpPass);
|
||||
if (body.ftpTls != null) data.ftpTls = Boolean(body.ftpTls);
|
||||
if (body.ftpIncomingDir != null) data.ftpIncomingDir = String(body.ftpIncomingDir);
|
||||
if (body.ftpProcessedDir != null) data.ftpProcessedDir = String(body.ftpProcessedDir);
|
||||
|
||||
const settings = await prisma.appSettings.upsert({
|
||||
where: { id: "singleton" },
|
||||
create: {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { uploadBuffer } from "@/lib/storage";
|
||||
import { enqueueProcessing } from "@/lib/processing-queue";
|
||||
|
||||
const MAX_UPLOAD_SIZE = 50 * 1024 * 1024; // 50 MB
|
||||
|
|
@ -49,7 +50,6 @@ export async function POST(request: NextRequest) {
|
|||
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
const fileName = file.name || `upload-${Date.now()}`;
|
||||
const isPdf = contentType === "application/pdf";
|
||||
|
||||
const job = await prisma.processingJob.create({
|
||||
data: {
|
||||
|
|
@ -59,11 +59,11 @@ export async function POST(request: NextRequest) {
|
|||
},
|
||||
});
|
||||
|
||||
jobIds.push(job.id);
|
||||
const sourceKey = `sources/${job.id}/${fileName}`;
|
||||
await uploadBuffer(sourceKey, buffer, contentType);
|
||||
|
||||
enqueueProcessing(job.id, fileName, buffer, isPdf).catch((err) => {
|
||||
console.error(`[upload] Background processing failed for job ${job.id}:`, err);
|
||||
});
|
||||
await enqueueProcessing(job.id);
|
||||
jobIds.push(job.id);
|
||||
}
|
||||
|
||||
return NextResponse.json({ jobIds });
|
||||
|
|
|
|||
|
|
@ -1,38 +1,8 @@
|
|||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { startWatching, stopWatching, isWatching } from "@/lib/watcher";
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const body = await request.json().catch(() => ({}));
|
||||
const action = body.action as string | undefined;
|
||||
const watchDir = body.watchDir as string | undefined;
|
||||
|
||||
if (!action || !["start", "stop"].includes(action)) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid action. Use 'start' or 'stop'" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
if (action === "start") {
|
||||
if (!watchDir) {
|
||||
return NextResponse.json(
|
||||
{ error: "watchDir is required to start watching" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
await startWatching(watchDir);
|
||||
} else {
|
||||
await stopWatching();
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
watching: isWatching(),
|
||||
watchDir: watchDir || "",
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[watch POST]", error);
|
||||
const message = error instanceof Error ? error.message : "Failed to update watch settings";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
export async function POST() {
|
||||
return NextResponse.json(
|
||||
{ error: "Folder watching has been replaced by FTP watching. Use /api/ftp-watch instead." },
|
||||
{ status: 410 }
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,54 +1,5 @@
|
|||
export async function register() {
|
||||
// Only run on the Node.js server runtime, not during build or in the edge runtime
|
||||
if (process.env.NEXT_RUNTIME !== "nodejs") return;
|
||||
|
||||
const { prisma } = await import("@/lib/db");
|
||||
|
||||
let settings;
|
||||
try {
|
||||
settings = await prisma.appSettings.findUnique({
|
||||
where: { id: "singleton" },
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("[instrumentation] Could not read settings:", err);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!settings) return;
|
||||
|
||||
// Resume email monitoring
|
||||
if (
|
||||
settings.emailWatching &&
|
||||
settings.emailImapHost &&
|
||||
settings.emailImapUser &&
|
||||
settings.emailImapPass
|
||||
) {
|
||||
try {
|
||||
const { startEmailWatching } = await import("@/lib/email-watcher");
|
||||
await startEmailWatching({
|
||||
host: settings.emailImapHost,
|
||||
port: settings.emailImapPort,
|
||||
user: settings.emailImapUser,
|
||||
pass: settings.emailImapPass,
|
||||
tls: settings.emailImapTls,
|
||||
folder: settings.emailFolder,
|
||||
processedAction: settings.emailProcessed,
|
||||
processedFolder: settings.emailProcessedFolder,
|
||||
});
|
||||
console.log("[instrumentation] Email monitoring resumed");
|
||||
} catch (err) {
|
||||
console.error("[instrumentation] Failed to resume email monitoring:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// Resume folder watching
|
||||
if (settings.watching && settings.watchDir) {
|
||||
try {
|
||||
const { startWatching } = await import("@/lib/watcher");
|
||||
await startWatching(settings.watchDir);
|
||||
console.log("[instrumentation] Folder watching resumed");
|
||||
} catch (err) {
|
||||
console.error("[instrumentation] Failed to resume folder watching:", err);
|
||||
}
|
||||
}
|
||||
// Background watchers (email + FTP) are now handled by Vercel Cron Jobs
|
||||
// that call /api/email-watch/poll and /api/ftp-watch/poll every 2 minutes.
|
||||
// No persistent processes to start on boot.
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ const globalForPrisma = globalThis as unknown as {
|
|||
function createPrismaClient() {
|
||||
const pool = new pg.Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
max: 5,
|
||||
});
|
||||
const adapter = new PrismaPg(pool);
|
||||
return new PrismaClient({ adapter });
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { ImapFlow } from "imapflow";
|
||||
import { simpleParser } from "mailparser";
|
||||
import { prisma } from "./db";
|
||||
import { uploadBuffer } from "./storage";
|
||||
import { enqueueProcessing } from "./processing-queue";
|
||||
|
||||
const ALLOWED_CONTENT_TYPES = [
|
||||
|
|
@ -24,11 +25,6 @@ type EmailConfig = {
|
|||
processedFolder: string;
|
||||
};
|
||||
|
||||
let client: ImapFlow | null = null;
|
||||
let watching = false;
|
||||
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let currentConfig: EmailConfig | null = null;
|
||||
|
||||
function log(msg: string, ...args: unknown[]) {
|
||||
console.log(`[email-watcher] ${msg}`, ...args);
|
||||
}
|
||||
|
|
@ -79,10 +75,11 @@ async function handleMessage(client: ImapFlow, uid: number, config: EmailConfig)
|
|||
},
|
||||
});
|
||||
|
||||
enqueueProcessing(job.id, fileName, buffer, isPdf).catch((err) => {
|
||||
logError(`Processing failed for ${fileName}:`, err);
|
||||
});
|
||||
const sourceKey = `sources/${job.id}/${fileName}`;
|
||||
const contentType = isPdf ? "application/pdf" : "image/jpeg";
|
||||
await uploadBuffer(sourceKey, buffer, contentType);
|
||||
|
||||
await enqueueProcessing(job.id);
|
||||
log(`Queued: ${fileName} (job ${job.id})`);
|
||||
} catch (err) {
|
||||
logError(`Failed to create job for ${fileName}:`, err);
|
||||
|
|
@ -103,150 +100,16 @@ async function handleMessage(client: ImapFlow, uid: number, config: EmailConfig)
|
|||
}
|
||||
}
|
||||
|
||||
async function pollLoop(config: EmailConfig) {
|
||||
if (!client || !watching) return;
|
||||
|
||||
try {
|
||||
const lock = await client.getMailboxLock(config.folder);
|
||||
try {
|
||||
const searchResult = await client.search({ seen: false }, { uid: true });
|
||||
const unseen = Array.isArray(searchResult) ? searchResult : [];
|
||||
if (unseen.length > 0) {
|
||||
log(`Found ${unseen.length} unseen message(s)`);
|
||||
for (const uid of unseen) {
|
||||
if (!watching) break;
|
||||
try {
|
||||
await handleMessage(client, uid, config);
|
||||
} catch (err) {
|
||||
logError(`Error handling message UID ${uid}:`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
} catch (err) {
|
||||
logError("Error during poll:", err);
|
||||
}
|
||||
|
||||
if (!watching || !client) return;
|
||||
|
||||
try {
|
||||
await client.idle();
|
||||
} catch {
|
||||
// IDLE interrupted or connection lost — reconnect handles it
|
||||
}
|
||||
|
||||
if (watching) {
|
||||
setImmediate(() => pollLoop(config));
|
||||
}
|
||||
}
|
||||
|
||||
async function connect(config: EmailConfig) {
|
||||
if (client) {
|
||||
try { await client.logout(); } catch {}
|
||||
client = null;
|
||||
}
|
||||
|
||||
client = new ImapFlow({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
secure: config.tls,
|
||||
auth: { user: config.user, pass: config.pass },
|
||||
logger: false,
|
||||
emitLogs: false,
|
||||
});
|
||||
|
||||
client.on("error", (err: Error) => {
|
||||
logError("Connection error:", err.message);
|
||||
if (watching) scheduleReconnect(config);
|
||||
});
|
||||
|
||||
client.on("close", () => {
|
||||
log("Connection closed");
|
||||
if (watching) scheduleReconnect(config);
|
||||
});
|
||||
|
||||
await client.connect();
|
||||
log(`Connected to ${config.host}:${config.port} as ${config.user}`);
|
||||
|
||||
pollLoop(config);
|
||||
}
|
||||
|
||||
function scheduleReconnect(config: EmailConfig) {
|
||||
if (reconnectTimer) return;
|
||||
const delay = 10_000;
|
||||
log(`Reconnecting in ${delay / 1000}s...`);
|
||||
reconnectTimer = setTimeout(async () => {
|
||||
reconnectTimer = null;
|
||||
if (!watching) return;
|
||||
try {
|
||||
await connect(config);
|
||||
} catch (err) {
|
||||
logError("Reconnect failed:", err);
|
||||
scheduleReconnect(config);
|
||||
}
|
||||
}, delay);
|
||||
}
|
||||
|
||||
export async function startEmailWatching(config: EmailConfig): Promise<void> {
|
||||
if (watching) await stopEmailWatching();
|
||||
|
||||
watching = true;
|
||||
currentConfig = config;
|
||||
|
||||
await prisma.appSettings.upsert({
|
||||
where: { id: "singleton" },
|
||||
update: { emailWatching: true },
|
||||
create: { id: "singleton", emailWatching: true },
|
||||
});
|
||||
|
||||
await connect(config);
|
||||
}
|
||||
|
||||
export async function stopEmailWatching(): Promise<void> {
|
||||
watching = false;
|
||||
currentConfig = null;
|
||||
|
||||
if (reconnectTimer) {
|
||||
clearTimeout(reconnectTimer);
|
||||
reconnectTimer = null;
|
||||
}
|
||||
|
||||
if (client) {
|
||||
try { await client.logout(); } catch {}
|
||||
client = null;
|
||||
}
|
||||
|
||||
await prisma.appSettings.upsert({
|
||||
where: { id: "singleton" },
|
||||
update: { emailWatching: false },
|
||||
create: { id: "singleton", emailWatching: false },
|
||||
});
|
||||
|
||||
log("Stopped");
|
||||
}
|
||||
|
||||
export function isEmailWatching(): boolean {
|
||||
return watching;
|
||||
}
|
||||
|
||||
export function getEmailWatchStatus() {
|
||||
return {
|
||||
watching,
|
||||
connected: client !== null && client.usable !== false,
|
||||
config: currentConfig
|
||||
? { host: currentConfig.host, user: currentConfig.user, folder: currentConfig.folder }
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function scanInbox(): Promise<{ processed: number; skipped: number; error?: string }> {
|
||||
const settings = await prisma.appSettings.findUnique({ where: { id: "singleton" } });
|
||||
if (!settings?.emailImapHost || !settings?.emailImapUser || !settings?.emailImapPass) {
|
||||
return { processed: 0, skipped: 0, error: "IMAP not configured" };
|
||||
}
|
||||
|
||||
if (!settings.emailWatching) {
|
||||
return { processed: 0, skipped: 0, error: "Email watching is disabled" };
|
||||
}
|
||||
|
||||
const config: EmailConfig = {
|
||||
host: settings.emailImapHost,
|
||||
port: settings.emailImapPort,
|
||||
|
|
@ -272,7 +135,7 @@ export async function scanInbox(): Promise<{ processed: number; skipped: number;
|
|||
|
||||
try {
|
||||
await scanClient.connect();
|
||||
log(`[scan] Connected for manual inbox scan`);
|
||||
log(`[scan] Connected for inbox scan`);
|
||||
|
||||
const lock = await scanClient.getMailboxLock(config.folder);
|
||||
try {
|
||||
|
|
|
|||
147
src/lib/ftp-watcher.ts
Normal file
147
src/lib/ftp-watcher.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
import { Client } from "basic-ftp";
|
||||
import { Writable } from "stream";
|
||||
import { prisma } from "./db";
|
||||
import { uploadBuffer } from "./storage";
|
||||
import { enqueueProcessing } from "./processing-queue";
|
||||
|
||||
const ALLOWED_EXTENSIONS = [".pdf", ".jpg", ".jpeg", ".png", ".webp"];
|
||||
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB
|
||||
|
||||
function log(msg: string, ...args: unknown[]) {
|
||||
console.log(`[ftp-watcher] ${msg}`, ...args);
|
||||
}
|
||||
|
||||
function logError(msg: string, ...args: unknown[]) {
|
||||
console.error(`[ftp-watcher] ${msg}`, ...args);
|
||||
}
|
||||
|
||||
async function downloadToBuffer(client: Client, remotePath: string): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
const writable = new Writable({
|
||||
write(chunk, _encoding, callback) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
callback();
|
||||
},
|
||||
});
|
||||
await client.downloadTo(writable, remotePath);
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
export async function pollFtp(): Promise<{ processed: number; skipped: number; error?: string }> {
|
||||
const settings = await prisma.appSettings.findUnique({ where: { id: "singleton" } });
|
||||
|
||||
if (!settings?.ftpEnabled) {
|
||||
return { processed: 0, skipped: 0, error: "FTP watching is disabled" };
|
||||
}
|
||||
|
||||
if (!settings.ftpHost || !settings.ftpUser || !settings.ftpPass) {
|
||||
return { processed: 0, skipped: 0, error: "FTP not configured" };
|
||||
}
|
||||
|
||||
const client = new Client();
|
||||
let processed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
try {
|
||||
await client.access({
|
||||
host: settings.ftpHost,
|
||||
port: settings.ftpPort,
|
||||
user: settings.ftpUser,
|
||||
password: settings.ftpPass,
|
||||
secure: settings.ftpTls,
|
||||
});
|
||||
|
||||
log(`Connected to ${settings.ftpHost}:${settings.ftpPort}`);
|
||||
|
||||
const files = await client.list(settings.ftpIncomingDir);
|
||||
const validFiles = files.filter((f) => {
|
||||
if (f.isDirectory) return false;
|
||||
const ext = f.name.toLowerCase().match(/\.[^.]+$/)?.[0] || "";
|
||||
return ALLOWED_EXTENSIONS.includes(ext);
|
||||
});
|
||||
|
||||
log(`Found ${validFiles.length} file(s) in ${settings.ftpIncomingDir}`);
|
||||
|
||||
for (const file of validFiles) {
|
||||
const remotePath = `${settings.ftpIncomingDir}/${file.name}`;
|
||||
const ext = file.name.toLowerCase().match(/\.[^.]+$/)?.[0] || "";
|
||||
|
||||
if (file.size > MAX_FILE_SIZE) {
|
||||
log(`Skipping "${file.name}" — exceeds 50 MB (${(file.size / 1024 / 1024).toFixed(1)} MB)`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const buffer = await downloadToBuffer(client, remotePath);
|
||||
const isPdf = ext === ".pdf";
|
||||
const contentType = isPdf ? "application/pdf" : "image/jpeg";
|
||||
|
||||
const job = await prisma.processingJob.create({
|
||||
data: {
|
||||
fileName: file.name,
|
||||
filePath: `ftp/${file.name}`,
|
||||
status: "queued",
|
||||
},
|
||||
});
|
||||
|
||||
const sourceKey = `sources/${job.id}/${file.name}`;
|
||||
await uploadBuffer(sourceKey, buffer, contentType);
|
||||
await enqueueProcessing(job.id);
|
||||
|
||||
const destPath = `${settings.ftpProcessedDir}/${file.name}`;
|
||||
try {
|
||||
await client.ensureDir(settings.ftpProcessedDir);
|
||||
await client.rename(remotePath, destPath);
|
||||
} catch {
|
||||
log(`Could not move "${file.name}" to processed dir, removing instead`);
|
||||
try { await client.remove(remotePath); } catch {}
|
||||
}
|
||||
|
||||
processed++;
|
||||
log(`Queued: ${file.name} (job ${job.id})`);
|
||||
} catch (err) {
|
||||
logError(`Failed to process "${file.name}":`, err);
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
client.close();
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "FTP poll failed";
|
||||
logError("Error:", message);
|
||||
client.close();
|
||||
return { processed, skipped, error: message };
|
||||
}
|
||||
|
||||
log(`Complete: ${processed} processed, ${skipped} skipped`);
|
||||
return { processed, skipped };
|
||||
}
|
||||
|
||||
export async function testFtpConnection(config: {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
pass: string;
|
||||
tls: boolean;
|
||||
incomingDir: string;
|
||||
}): Promise<{ ok: boolean; files?: number; error?: string }> {
|
||||
const client = new Client();
|
||||
try {
|
||||
await client.access({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
user: config.user,
|
||||
password: config.pass,
|
||||
secure: config.tls,
|
||||
});
|
||||
|
||||
const files = await client.list(config.incomingDir);
|
||||
client.close();
|
||||
return { ok: true, files: files.length };
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Connection failed";
|
||||
client.close();
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { prisma } from "./db";
|
||||
import { getBuffer } from "./minio";
|
||||
import { getBuffer } from "./storage";
|
||||
import { logActivity, diffCardFields } from "./activity-log";
|
||||
import { createNotification } from "./notifications";
|
||||
import {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { prisma } from "./db";
|
||||
import { Prisma } from "@/generated/prisma/client";
|
||||
import { uploadBuffer, getBuffer, deleteObject } from "./minio";
|
||||
import { uploadBuffer, getBuffer, deleteObject } from "./storage";
|
||||
import { ocrImage } from "./ai-ocr";
|
||||
import { pdfToImages, imageToBase64, processUploadedImage } from "./pdf";
|
||||
import { fireIntegrationEvent } from "./integrations";
|
||||
|
|
|
|||
128
src/lib/pdf.ts
128
src/lib/pdf.ts
|
|
@ -1,7 +1,10 @@
|
|||
import { fromBuffer } from "pdf2pic";
|
||||
import { PDFDocument } from "pdf-lib";
|
||||
import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs";
|
||||
import sharp from "sharp";
|
||||
|
||||
const TARGET_DPI = 200;
|
||||
const PDF_DEFAULT_DPI = 72;
|
||||
const SCALE = TARGET_DPI / PDF_DEFAULT_DPI;
|
||||
|
||||
export interface PageImage {
|
||||
page: number;
|
||||
buffer: Buffer;
|
||||
|
|
@ -9,68 +12,71 @@ export interface PageImage {
|
|||
height: number;
|
||||
}
|
||||
|
||||
export async function pdfToImages(pdfBuffer: Buffer): Promise<PageImage[]> {
|
||||
const converter = fromBuffer(pdfBuffer, {
|
||||
density: 200,
|
||||
format: "jpeg",
|
||||
width: 2200,
|
||||
height: 2200,
|
||||
quality: 90,
|
||||
preserveAspectRatio: true,
|
||||
});
|
||||
|
||||
const pageCount = await getPdfPageCount(pdfBuffer);
|
||||
const images: PageImage[] = [];
|
||||
|
||||
for (let i = 1; i <= pageCount; i++) {
|
||||
const result = await converter(i, { responseType: "buffer" });
|
||||
if (result.buffer) {
|
||||
let buf = result.buffer as Buffer;
|
||||
let meta = await sharp(buf).metadata();
|
||||
const w = meta.width || 0;
|
||||
const h = meta.height || 0;
|
||||
|
||||
// Duplex scanners flip the sheet between sides, so odd pages
|
||||
// (back/response card) are 180° rotated relative to even pages
|
||||
// (front/survey). Landscape pages need rotation to portrait;
|
||||
// the direction depends on which side of the sheet was scanned.
|
||||
const isBackSide = i % 2 === 1;
|
||||
|
||||
if (w > h) {
|
||||
const angle = isBackSide ? -90 : 90;
|
||||
buf = await sharp(buf).rotate(angle).jpeg({ quality: 90 }).toBuffer();
|
||||
meta = await sharp(buf).metadata();
|
||||
} else if (isBackSide) {
|
||||
buf = await sharp(buf).rotate(180).jpeg({ quality: 90 }).toBuffer();
|
||||
meta = await sharp(buf).metadata();
|
||||
}
|
||||
|
||||
images.push({
|
||||
page: i,
|
||||
buffer: buf,
|
||||
width: meta.width || 1600,
|
||||
height: meta.height || 2200,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return images;
|
||||
interface CanvasAndContext {
|
||||
canvas: { toBuffer(mime: string): Buffer };
|
||||
context: unknown;
|
||||
}
|
||||
|
||||
async function getPdfPageCount(pdfBuffer: Buffer): Promise<number> {
|
||||
try {
|
||||
const doc = await PDFDocument.load(pdfBuffer, { ignoreEncryption: true });
|
||||
const count = doc.getPageCount();
|
||||
console.log(`[pdf] Page count (pdf-lib): ${count}`);
|
||||
return count;
|
||||
} catch (err) {
|
||||
console.warn("[pdf] pdf-lib page count failed, falling back to regex:", err);
|
||||
const text = pdfBuffer.toString("latin1");
|
||||
const matches = text.match(/\/Type\s*\/Page(?!s)/g);
|
||||
const count = matches ? matches.length : 2;
|
||||
console.log(`[pdf] Page count (regex fallback): ${count}`);
|
||||
return count;
|
||||
interface CanvasFactory {
|
||||
create(width: number, height: number): CanvasAndContext;
|
||||
}
|
||||
|
||||
export async function pdfToImages(pdfBuffer: Buffer): Promise<PageImage[]> {
|
||||
const data = new Uint8Array(pdfBuffer);
|
||||
const loadingTask = getDocument({ data, useSystemFonts: true });
|
||||
const pdfDocument = await loadingTask.promise;
|
||||
const pageCount = pdfDocument.numPages;
|
||||
const images: PageImage[] = [];
|
||||
|
||||
const canvasFactory = pdfDocument.canvasFactory as unknown as CanvasFactory;
|
||||
|
||||
console.log(`[pdf] Rendering ${pageCount} page(s) at ${TARGET_DPI} DPI (scale ${SCALE.toFixed(2)})`);
|
||||
|
||||
for (let i = 1; i <= pageCount; i++) {
|
||||
const page = await pdfDocument.getPage(i);
|
||||
const viewport = page.getViewport({ scale: SCALE });
|
||||
|
||||
const { canvas, context } = canvasFactory.create(
|
||||
Math.floor(viewport.width),
|
||||
Math.floor(viewport.height)
|
||||
);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await page.render({ canvas: canvas as any, viewport } as any).promise;
|
||||
|
||||
const pngBuffer = canvas.toBuffer("image/png");
|
||||
page.cleanup();
|
||||
|
||||
let buf = await sharp(pngBuffer).jpeg({ quality: 90 }).toBuffer();
|
||||
let meta = await sharp(buf).metadata();
|
||||
const w = meta.width || 0;
|
||||
const h = meta.height || 0;
|
||||
|
||||
// Duplex scanners flip the sheet between sides, so odd pages
|
||||
// (back/response card) are 180-deg rotated relative to even pages
|
||||
// (front/survey). Landscape pages need rotation to portrait;
|
||||
// the direction depends on which side of the sheet was scanned.
|
||||
const isBackSide = i % 2 === 1;
|
||||
|
||||
if (w > h) {
|
||||
const angle = isBackSide ? -90 : 90;
|
||||
buf = await sharp(buf).rotate(angle).jpeg({ quality: 90 }).toBuffer();
|
||||
meta = await sharp(buf).metadata();
|
||||
} else if (isBackSide) {
|
||||
buf = await sharp(buf).rotate(180).jpeg({ quality: 90 }).toBuffer();
|
||||
meta = await sharp(buf).metadata();
|
||||
}
|
||||
|
||||
images.push({
|
||||
page: i,
|
||||
buffer: buf,
|
||||
width: meta.width || Math.floor(viewport.width),
|
||||
height: meta.height || Math.floor(viewport.height),
|
||||
});
|
||||
}
|
||||
|
||||
await pdfDocument.destroy();
|
||||
return images;
|
||||
}
|
||||
|
||||
export async function imageToBase64(buffer: Buffer): Promise<string> {
|
||||
|
|
|
|||
|
|
@ -1,47 +1,19 @@
|
|||
import { processFile } from "./ocr";
|
||||
import { Client } from "@upstash/qstash";
|
||||
|
||||
const MAX_CONCURRENT = 1;
|
||||
const DELAY_BETWEEN_JOBS_MS = 3_000;
|
||||
const queue: QueueItem[] = [];
|
||||
let running = 0;
|
||||
const qstash = new Client({ token: process.env.QSTASH_TOKEN! });
|
||||
|
||||
type QueueItem = {
|
||||
jobId: string;
|
||||
fileName: string;
|
||||
fileBuffer: Buffer;
|
||||
isPdf: boolean;
|
||||
resolve: (ids: string[]) => void;
|
||||
reject: (err: unknown) => void;
|
||||
};
|
||||
|
||||
function drain() {
|
||||
while (running < MAX_CONCURRENT && queue.length > 0) {
|
||||
const item = queue.shift()!;
|
||||
running++;
|
||||
processFile(item.jobId, item.fileName, item.fileBuffer, item.isPdf)
|
||||
.then(item.resolve)
|
||||
.catch(item.reject)
|
||||
.finally(() => {
|
||||
running--;
|
||||
if (queue.length > 0) {
|
||||
setTimeout(drain, DELAY_BETWEEN_JOBS_MS);
|
||||
}
|
||||
});
|
||||
function getBaseUrl(): string {
|
||||
if (process.env.VERCEL_URL) {
|
||||
return `https://${process.env.VERCEL_URL}`;
|
||||
}
|
||||
return process.env.AUTH_URL || "http://localhost:3000";
|
||||
}
|
||||
|
||||
export function enqueueProcessing(
|
||||
jobId: string,
|
||||
fileName: string,
|
||||
fileBuffer: Buffer,
|
||||
isPdf: boolean
|
||||
): Promise<string[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
queue.push({ jobId, fileName, fileBuffer, isPdf, resolve, reject });
|
||||
if (running < MAX_CONCURRENT) drain();
|
||||
export async function enqueueProcessing(jobId: string): Promise<void> {
|
||||
const baseUrl = getBaseUrl();
|
||||
await qstash.publishJSON({
|
||||
url: `${baseUrl}/api/jobs/process`,
|
||||
body: { jobId },
|
||||
retries: 2,
|
||||
});
|
||||
}
|
||||
|
||||
export function getQueueStatus() {
|
||||
return { running, queued: queue.length, maxConcurrent: MAX_CONCURRENT };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,23 +1,23 @@
|
|||
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3";
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
|
||||
const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT || "192.168.68.105";
|
||||
const MINIO_PORT = parseInt(process.env.MINIO_PORT || "9000");
|
||||
const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY || "minioadmin";
|
||||
const MINIO_SECRET_KEY = process.env.MINIO_SECRET_KEY || "";
|
||||
const MINIO_BUCKET = process.env.MINIO_BUCKET || "echos-ocr";
|
||||
const S3_ENDPOINT = process.env.STORAGE_ENDPOINT || "";
|
||||
const S3_REGION = process.env.STORAGE_REGION || "us-east-1";
|
||||
const S3_ACCESS_KEY = process.env.STORAGE_ACCESS_KEY || "";
|
||||
const S3_SECRET_KEY = process.env.STORAGE_SECRET_KEY || "";
|
||||
const S3_BUCKET = process.env.STORAGE_BUCKET || "echos-ocr";
|
||||
|
||||
export const s3 = new S3Client({
|
||||
endpoint: `http://${MINIO_ENDPOINT}:${MINIO_PORT}`,
|
||||
region: "us-east-1",
|
||||
endpoint: S3_ENDPOINT,
|
||||
region: S3_REGION,
|
||||
credentials: {
|
||||
accessKeyId: MINIO_ACCESS_KEY,
|
||||
secretAccessKey: MINIO_SECRET_KEY,
|
||||
accessKeyId: S3_ACCESS_KEY,
|
||||
secretAccessKey: S3_SECRET_KEY,
|
||||
},
|
||||
forcePathStyle: true,
|
||||
});
|
||||
|
||||
export const BUCKET = MINIO_BUCKET;
|
||||
export const BUCKET = S3_BUCKET;
|
||||
|
||||
export async function uploadBuffer(key: string, buffer: Buffer, contentType: string): Promise<string> {
|
||||
await s3.send(
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
import chokidar, { type FSWatcher } from "chokidar";
|
||||
import fs from "fs/promises";
|
||||
import path from "path";
|
||||
import { prisma } from "./db";
|
||||
import { enqueueProcessing } from "./processing-queue";
|
||||
|
||||
let watcher: FSWatcher | null = null;
|
||||
|
||||
const processedFiles = new Set<string>();
|
||||
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB
|
||||
|
||||
export async function startWatching(watchDir: string): Promise<void> {
|
||||
if (watcher) {
|
||||
await stopWatching();
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.access(watchDir);
|
||||
} catch {
|
||||
throw new Error(`Watch directory does not exist: ${watchDir}`);
|
||||
}
|
||||
|
||||
watcher = chokidar.watch(watchDir, {
|
||||
ignored: /(^|[/\\])\../,
|
||||
persistent: true,
|
||||
ignoreInitial: false,
|
||||
awaitWriteFinish: {
|
||||
stabilityThreshold: 2000,
|
||||
pollInterval: 500,
|
||||
},
|
||||
});
|
||||
|
||||
watcher.on("add", async (filePath: string) => {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const allowed = [".pdf", ".jpg", ".jpeg", ".png", ".webp"];
|
||||
if (!allowed.includes(ext)) return;
|
||||
if (processedFiles.has(filePath)) return;
|
||||
processedFiles.add(filePath);
|
||||
|
||||
const fileName = path.basename(filePath);
|
||||
const isPdf = ext === ".pdf";
|
||||
|
||||
try {
|
||||
const stat = await fs.stat(filePath);
|
||||
if (stat.size > MAX_FILE_SIZE) {
|
||||
console.log(`[watcher] Skipping ${fileName} — exceeds 50 MB size limit (${(stat.size / 1024 / 1024).toFixed(1)} MB)`);
|
||||
return;
|
||||
}
|
||||
|
||||
const buffer = await fs.readFile(filePath);
|
||||
const job = await prisma.processingJob.create({
|
||||
data: {
|
||||
fileName,
|
||||
filePath,
|
||||
status: "queued",
|
||||
},
|
||||
});
|
||||
|
||||
enqueueProcessing(job.id, fileName, buffer, isPdf).catch((err) => {
|
||||
console.error(`[watcher] Processing failed for ${fileName}:`, err);
|
||||
});
|
||||
|
||||
console.log(`[watcher] Queued: ${fileName}`);
|
||||
} catch (err) {
|
||||
console.error(`[watcher] Failed to read file ${filePath}:`, err);
|
||||
}
|
||||
});
|
||||
|
||||
watcher.on("error", (error: unknown) => {
|
||||
console.error("[watcher] Error:", error);
|
||||
});
|
||||
|
||||
await prisma.appSettings.upsert({
|
||||
where: { id: "singleton" },
|
||||
update: { watching: true, watchDir },
|
||||
create: { id: "singleton", watching: true, watchDir },
|
||||
});
|
||||
|
||||
console.log(`[watcher] Started watching: ${watchDir}`);
|
||||
}
|
||||
|
||||
export async function stopWatching(): Promise<void> {
|
||||
if (watcher) {
|
||||
await watcher.close();
|
||||
watcher = null;
|
||||
}
|
||||
|
||||
await prisma.appSettings.upsert({
|
||||
where: { id: "singleton" },
|
||||
update: { watching: false },
|
||||
create: { id: "singleton", watching: false },
|
||||
});
|
||||
|
||||
console.log("[watcher] Stopped");
|
||||
}
|
||||
|
||||
export function isWatching(): boolean {
|
||||
return watcher !== null;
|
||||
}
|
||||
|
|
@ -9,6 +9,9 @@ const publicPaths = [
|
|||
"/api/health",
|
||||
"/api/setup",
|
||||
"/api/onboarding",
|
||||
"/api/jobs/process",
|
||||
"/api/email-watch/poll",
|
||||
"/api/ftp-watch/poll",
|
||||
];
|
||||
|
||||
const onboardingExemptPaths = [
|
||||
|
|
|
|||
12
vercel.json
Normal file
12
vercel.json
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"crons": [
|
||||
{
|
||||
"path": "/api/email-watch/poll",
|
||||
"schedule": "*/2 * * * *"
|
||||
},
|
||||
{
|
||||
"path": "/api/ftp-watch/poll",
|
||||
"schedule": "*/2 * * * *"
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Reference in a new issue