From 028840a88797eb6caa9f868718229111596a875a Mon Sep 17 00:00:00 2001 From: Randall Stillwell Date: Fri, 17 Apr 2026 15:12:54 -0500 Subject: [PATCH] Add SFTP support: replace FTP-only TLS toggle with protocol selector Dreamhost (and most modern hosts) use SFTP (port 22), not FTP (port 21). The previous implementation only supported FTP/FTPS via basic-ftp, causing timeouts when connecting to SFTP servers. Changes: - Add ssh2-sftp-client for SFTP connections - Replace ftpTls boolean with ftpProtocol ("sftp" | "ftp" | "ftps") in schema - Rewrite ftp-watcher.ts to support both SFTP and FTP/FTPS protocols - Update UI with protocol dropdown that auto-switches the default port - Add ssh2/ssh2-sftp-client to serverExternalPackages in next.config - Default to SFTP on port 22 Made-with: Cursor --- next.config.ts | 2 +- package-lock.json | 197 ++++++++++ package.json | 2 + prisma/schema.prisma | 4 +- .../settings/upload-sources/page.tsx | 64 ++-- src/app/api/ftp-watch/test/route.ts | 7 +- src/app/api/settings/route.ts | 3 +- src/lib/ftp-watcher.ts | 338 +++++++++++++----- 8 files changed, 502 insertions(+), 115 deletions(-) diff --git a/next.config.ts b/next.config.ts index 4202ada..2940b6a 100644 --- a/next.config.ts +++ b/next.config.ts @@ -9,7 +9,7 @@ const nextConfig: NextConfig = { }, ], }, - serverExternalPackages: ["sharp", "@napi-rs/canvas", "pg", "basic-ftp", "imapflow", "mailparser"], + serverExternalPackages: ["sharp", "@napi-rs/canvas", "pg", "basic-ftp", "ssh2", "ssh2-sftp-client", "imapflow", "mailparser"], }; export default nextConfig; diff --git a/package-lock.json b/package-lock.json index 2c181b0..edbc585 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,6 +48,7 @@ "shadcn": "^4.0.2", "sharp": "^0.34.5", "sonner": "^2.0.7", + "ssh2-sftp-client": "^12.1.1", "tailwind-merge": "^3.5.0", "tw-animate-css": "^1.4.0", "zod": "^4.3.6" @@ -60,6 +61,7 @@ "@types/pg": "^8.18.0", "@types/react": "^19", "@types/react-dom": "^19", + "@types/ssh2-sftp-client": "^9.0.6", "eslint": "^9", "eslint-config-next": "16.1.6", "tailwindcss": "^4", @@ -5353,6 +5355,43 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/ssh2": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2-sftp-client": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@types/ssh2-sftp-client/-/ssh2-sftp-client-9.0.6.tgz", + "integrity": "sha512-4+KvXO/V77y9VjI2op2T8+RCGI/GXQAwR0q5Qkj/EJ5YSeyKszqZP6F8i3H3txYoBqjc7sgorqyvBP3+w1EHyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ssh2": "^1.0.0" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/ssh2/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/statuses": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.6.tgz", @@ -6318,6 +6357,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, "node_modules/ast-types": { "version": "0.16.1", "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.16.1.tgz", @@ -6429,6 +6477,15 @@ "node": ">=10.0.0" } }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "license": "BSD-3-Clause", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, "node_modules/bcryptjs": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-3.0.3.tgz", @@ -6524,6 +6581,21 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/buildcheck": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -6933,6 +7005,21 @@ "dev": true, "license": "MIT" }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/confbox": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.4.tgz", @@ -7037,6 +7124,20 @@ } } }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -10972,6 +11073,13 @@ "node": ">=8.0.0" } }, + "node_modules/nan": { + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", + "license": "MIT", + "optional": true + }, "node_modules/nanoid": { "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", @@ -12614,6 +12722,20 @@ } } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", @@ -12944,6 +13066,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safe-push-apply": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-push-apply/-/safe-push-apply-1.0.0.tgz", @@ -13475,6 +13617,40 @@ "node": ">= 0.6" } }, + "node_modules/ssh2": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.23.0" + } + }, + "node_modules/ssh2-sftp-client": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-12.1.1.tgz", + "integrity": "sha512-wYVDgwkpcKG2iPGQQ+QR33xkWqLFIaVrYvA+uON4pmxTPaPuB81f1aooUEPN75e/9DCK6rrKYXb6zR6zP3+EtA==", + "license": "Apache-2.0", + "dependencies": { + "concat-stream": "^2.0.0", + "ssh2": "^1.16.0" + }, + "engines": { + "node": ">=18.20.4" + }, + "funding": { + "type": "individual", + "url": "https://square.link/u/4g7sPflL" + } + }, "node_modules/stable-hash": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stable-hash/-/stable-hash-0.0.5.tgz", @@ -13529,6 +13705,15 @@ "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==", "license": "MIT" }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", @@ -14040,6 +14225,12 @@ "url": "https://github.com/sponsors/Wombosvideo" } }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "license": "Unlicense" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -14160,6 +14351,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "license": "MIT" + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/package.json b/package.json index 9ec1497..aad6ae6 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "shadcn": "^4.0.2", "sharp": "^0.34.5", "sonner": "^2.0.7", + "ssh2-sftp-client": "^12.1.1", "tailwind-merge": "^3.5.0", "tw-animate-css": "^1.4.0", "zod": "^4.3.6" @@ -64,6 +65,7 @@ "@types/pg": "^8.18.0", "@types/react": "^19", "@types/react-dom": "^19", + "@types/ssh2-sftp-client": "^9.0.6", "eslint": "^9", "eslint-config-next": "16.1.6", "tailwindcss": "^4", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index accdc75..06aa79c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -427,10 +427,10 @@ model AppSettings { ftpEnabled Boolean @default(false) ftpHost String @default("") - ftpPort Int @default(21) + ftpPort Int @default(22) ftpUser String @default("") ftpPass String @default("") - ftpTls Boolean @default(true) + ftpProtocol String @default("sftp") // "sftp" | "ftp" | "ftps" ftpIncomingDir String @default("/incoming") ftpProcessedDir String @default("/processed") } diff --git a/src/app/(dashboard)/settings/upload-sources/page.tsx b/src/app/(dashboard)/settings/upload-sources/page.tsx index 54d6ba6..2a9b3dd 100644 --- a/src/app/(dashboard)/settings/upload-sources/page.tsx +++ b/src/app/(dashboard)/settings/upload-sources/page.tsx @@ -48,7 +48,7 @@ type SourceSettings = { ftpPort: number; ftpUser: string; ftpPass: string; - ftpTls: boolean; + ftpProtocol: "sftp" | "ftp" | "ftps"; ftpIncomingDir: string; ftpProcessedDir: string; watchDir: string; @@ -68,10 +68,10 @@ export default function UploadSourcesPage() { emailProcessedFolder: "Processed", ftpEnabled: false, ftpHost: "", - ftpPort: 21, + ftpPort: 22, ftpUser: "", ftpPass: "", - ftpTls: true, + ftpProtocol: "sftp", ftpIncomingDir: "/incoming", ftpProcessedDir: "/processed", watchDir: "", @@ -103,10 +103,10 @@ export default function UploadSourcesPage() { emailProcessedFolder: data.emailProcessedFolder || "Processed", ftpEnabled: data.ftpEnabled || false, ftpHost: data.ftpHost || "", - ftpPort: data.ftpPort || 21, + ftpPort: data.ftpPort || 22, ftpUser: data.ftpUser || "", ftpPass: data.ftpPass || "", - ftpTls: data.ftpTls ?? true, + ftpProtocol: data.ftpProtocol || "sftp", ftpIncomingDir: data.ftpIncomingDir || "/incoming", ftpProcessedDir: data.ftpProcessedDir || "/processed", watchDir: data.watchDir || "", @@ -195,7 +195,7 @@ export default function UploadSourcesPage() { port: settings.ftpPort, user: settings.ftpUser, pass: settings.ftpPass, - tls: settings.ftpTls, + protocol: settings.ftpProtocol, incomingDir: settings.ftpIncomingDir, }), }); @@ -373,11 +373,12 @@ export default function UploadSourcesPage() { - FTP Server + SFTP / FTP Server - Poll an FTP server for new scanned files. Files are downloaded, - processed, then moved to a “processed” directory. + Poll a remote server for new scanned files via SFTP, FTP, or FTPS. + Files are downloaded, processed, then moved to a + “processed” directory. @@ -393,9 +394,33 @@ export default function UploadSourcesPage() { -
+
- + + +
+
+ @@ -412,11 +437,14 @@ export default function UploadSourcesPage() { onChange={(e) => setSettings((s) => ({ ...s, - ftpPort: parseInt(e.target.value) || 21, + ftpPort: parseInt(e.target.value) || 22, })) } />
+
+ +
-
-
- - setSettings((s) => ({ ...s, ftpTls: !!v })) - } - /> - -
-
-
diff --git a/src/app/api/ftp-watch/test/route.ts b/src/app/api/ftp-watch/test/route.ts index 00bfef8..e79178c 100644 --- a/src/app/api/ftp-watch/test/route.ts +++ b/src/app/api/ftp-watch/test/route.ts @@ -16,12 +16,15 @@ export async function POST(request: NextRequest) { pass = settings?.ftpPass || ""; } + const protocol = body.protocol || "sftp"; + const defaultPort = protocol === "sftp" ? 22 : 21; + const result = await testFtpConnection({ host: body.host, - port: body.port || 21, + port: body.port || defaultPort, user: body.user, pass, - tls: body.tls ?? true, + protocol, incomingDir: body.incomingDir || "/incoming", }); return NextResponse.json(result); diff --git a/src/app/api/settings/route.ts b/src/app/api/settings/route.ts index 6faa2c5..9e40021 100644 --- a/src/app/api/settings/route.ts +++ b/src/app/api/settings/route.ts @@ -67,7 +67,8 @@ export async function PUT(request: NextRequest) { if (body.ftpPort != null) data.ftpPort = Math.max(1, parseInt(String(body.ftpPort)) || 21); if (body.ftpUser != null) data.ftpUser = String(body.ftpUser); if (body.ftpPass != null) data.ftpPass = String(body.ftpPass); - if (body.ftpTls != null) data.ftpTls = Boolean(body.ftpTls); + if (body.ftpProtocol != null && ["sftp", "ftp", "ftps"].includes(body.ftpProtocol)) + data.ftpProtocol = String(body.ftpProtocol); if (body.ftpIncomingDir != null) data.ftpIncomingDir = String(body.ftpIncomingDir); if (body.ftpProcessedDir != null) data.ftpProcessedDir = String(body.ftpProcessedDir); diff --git a/src/lib/ftp-watcher.ts b/src/lib/ftp-watcher.ts index a8bd70d..8403611 100644 --- a/src/lib/ftp-watcher.ts +++ b/src/lib/ftp-watcher.ts @@ -1,4 +1,5 @@ -import { Client } from "basic-ftp"; +import { Client as FtpClient } from "basic-ftp"; +import SftpClient from "ssh2-sftp-client"; import { Writable } from "stream"; import { prisma } from "./db"; import { uploadBuffer } from "./storage"; @@ -6,6 +7,25 @@ import { enqueueProcessing } from "./processing-queue"; const ALLOWED_EXTENSIONS = [".pdf", ".jpg", ".jpeg", ".png", ".webp"]; const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB +const CONNECT_TIMEOUT = 15_000; + +type Protocol = "sftp" | "ftp" | "ftps"; + +interface ConnectionConfig { + host: string; + port: number; + user: string; + pass: string; + protocol: Protocol; + incomingDir: string; + processedDir?: string; +} + +interface FileEntry { + name: string; + size: number; + isDirectory: boolean; +} function log(msg: string, ...args: unknown[]) { console.log(`[ftp-watcher] ${msg}`, ...args); @@ -15,7 +35,71 @@ function logError(msg: string, ...args: unknown[]) { console.error(`[ftp-watcher] ${msg}`, ...args); } -async function downloadToBuffer(client: Client, remotePath: string): Promise { +// ── SFTP helpers ────────────────────────────────────────────────────────── + +async function sftpConnect(config: ConnectionConfig): Promise { + const sftp = new SftpClient(); + await sftp.connect({ + host: config.host, + port: config.port, + username: config.user, + password: config.pass, + readyTimeout: CONNECT_TIMEOUT, + retries: 0, + }); + return sftp; +} + +async function sftpList(sftp: SftpClient, dir: string): Promise { + const items = await sftp.list(dir); + return items.map((f) => ({ + name: f.name, + size: f.size, + isDirectory: f.type === "d", + })); +} + +async function sftpDownload(sftp: SftpClient, remotePath: string): Promise { + const result = await sftp.get(remotePath); + if (Buffer.isBuffer(result)) return result; + if (typeof result === "string") return Buffer.from(result); + return Buffer.from(result as unknown as ArrayBuffer); +} + +async function sftpMove( + sftp: SftpClient, + src: string, + dest: string, + destDir: string +): Promise { + try { + const exists = await sftp.exists(destDir); + if (!exists) await sftp.mkdir(destDir, true); + await sftp.rename(src, dest); + } catch { + log(`Could not move file to processed dir, removing instead`); + try { + await sftp.delete(src); + } catch {} + } +} + +// ── FTP / FTPS helpers ──────────────────────────────────────────────────── + +async function ftpConnect(config: ConnectionConfig): Promise { + const client = new FtpClient(CONNECT_TIMEOUT); + await client.access({ + host: config.host, + port: config.port, + user: config.user, + password: config.pass, + secure: config.protocol === "ftps", + secureOptions: { rejectUnauthorized: false }, + }); + return client; +} + +async function ftpDownloadToBuffer(client: FtpClient, remotePath: string): Promise { const chunks: Buffer[] = []; const writable = new Writable({ write(chunk, _encoding, callback) { @@ -27,6 +111,8 @@ async function downloadToBuffer(client: Client, remotePath: string): Promise { const settings = await prisma.appSettings.findUnique({ where: { id: "singleton" } }); @@ -38,112 +124,194 @@ export async function pollFtp(): Promise<{ processed: number; skipped: number; e return { processed: 0, skipped: 0, error: "FTP not configured" }; } - const client = new Client(15_000); + const protocol = (settings.ftpProtocol || "sftp") as Protocol; + const config: ConnectionConfig = { + host: settings.ftpHost, + port: settings.ftpPort, + user: settings.ftpUser, + pass: settings.ftpPass, + protocol, + incomingDir: settings.ftpIncomingDir, + processedDir: settings.ftpProcessedDir, + }; + let processed = 0; let skipped = 0; - try { - await client.access({ - host: settings.ftpHost, - port: settings.ftpPort, - user: settings.ftpUser, - password: settings.ftpPass, - secure: settings.ftpTls, - secureOptions: { rejectUnauthorized: false }, + if (protocol === "sftp") { + const sftp = await sftpConnect(config).catch((err) => { + throw new Error(`SFTP connect failed: ${err.message}`); }); - log(`Connected to ${settings.ftpHost}:${settings.ftpPort}`); + try { + log(`Connected via SFTP to ${config.host}:${config.port}`); + const allFiles = await sftpList(sftp, config.incomingDir); + const validFiles = allFiles.filter((f) => { + if (f.isDirectory) return false; + const ext = f.name.toLowerCase().match(/\.[^.]+$/)?.[0] || ""; + return ALLOWED_EXTENSIONS.includes(ext); + }); - const files = await client.list(settings.ftpIncomingDir); - const validFiles = files.filter((f) => { - if (f.isDirectory) return false; - const ext = f.name.toLowerCase().match(/\.[^.]+$/)?.[0] || ""; - return ALLOWED_EXTENSIONS.includes(ext); - }); + log(`Found ${validFiles.length} file(s) in ${config.incomingDir}`); - log(`Found ${validFiles.length} file(s) in ${settings.ftpIncomingDir}`); + for (const file of validFiles) { + const remotePath = `${config.incomingDir}/${file.name}`; + const ext = file.name.toLowerCase().match(/\.[^.]+$/)?.[0] || ""; - for (const file of validFiles) { - const remotePath = `${settings.ftpIncomingDir}/${file.name}`; - const ext = file.name.toLowerCase().match(/\.[^.]+$/)?.[0] || ""; - - if (file.size > MAX_FILE_SIZE) { - log(`Skipping "${file.name}" — exceeds 50 MB (${(file.size / 1024 / 1024).toFixed(1)} MB)`); - skipped++; - continue; - } - - try { - const buffer = await downloadToBuffer(client, remotePath); - const isPdf = ext === ".pdf"; - const contentType = isPdf ? "application/pdf" : "image/jpeg"; - - const job = await prisma.processingJob.create({ - data: { - fileName: file.name, - filePath: `ftp/${file.name}`, - status: "queued", - }, - }); - - const sourceKey = `sources/${job.id}/${file.name}`; - await uploadBuffer(sourceKey, buffer, contentType); - await enqueueProcessing(job.id); - - const destPath = `${settings.ftpProcessedDir}/${file.name}`; - try { - await client.ensureDir(settings.ftpProcessedDir); - await client.rename(remotePath, destPath); - } catch { - log(`Could not move "${file.name}" to processed dir, removing instead`); - try { await client.remove(remotePath); } catch {} + if (file.size > MAX_FILE_SIZE) { + log(`Skipping "${file.name}" — exceeds 50 MB`); + skipped++; + continue; } - processed++; - log(`Queued: ${file.name} (job ${job.id})`); - } catch (err) { - logError(`Failed to process "${file.name}":`, err); - skipped++; - } - } + try { + const buffer = await sftpDownload(sftp, remotePath); + const isPdf = ext === ".pdf"; + const contentType = isPdf ? "application/pdf" : "image/jpeg"; - client.close(); - } catch (err) { - const message = err instanceof Error ? err.message : "FTP poll failed"; - logError("Error:", message); - client.close(); - return { processed, skipped, error: message }; + const job = await prisma.processingJob.create({ + data: { fileName: file.name, filePath: `ftp/${file.name}`, status: "queued" }, + }); + + await uploadBuffer(`sources/${job.id}/${file.name}`, buffer, contentType); + await enqueueProcessing(job.id); + + await sftpMove( + sftp, + remotePath, + `${config.processedDir}/${file.name}`, + config.processedDir! + ); + + processed++; + log(`Queued: ${file.name} (job ${job.id})`); + } catch (err) { + logError(`Failed to process "${file.name}":`, err); + skipped++; + } + } + + await sftp.end(); + } catch (err) { + await sftp.end().catch(() => {}); + const message = err instanceof Error ? err.message : "SFTP poll failed"; + logError("Error:", message); + return { processed, skipped, error: message }; + } + } else { + let client: FtpClient | null = null; + try { + client = await ftpConnect(config); + log(`Connected via ${protocol.toUpperCase()} to ${config.host}:${config.port}`); + + const files = await client.list(config.incomingDir); + const validFiles = files.filter((f) => { + if (f.isDirectory) return false; + const ext = f.name.toLowerCase().match(/\.[^.]+$/)?.[0] || ""; + return ALLOWED_EXTENSIONS.includes(ext); + }); + + log(`Found ${validFiles.length} file(s) in ${config.incomingDir}`); + + for (const file of validFiles) { + const remotePath = `${config.incomingDir}/${file.name}`; + const ext = file.name.toLowerCase().match(/\.[^.]+$/)?.[0] || ""; + + if (file.size > MAX_FILE_SIZE) { + log(`Skipping "${file.name}" — exceeds 50 MB`); + skipped++; + continue; + } + + try { + const buffer = await ftpDownloadToBuffer(client, remotePath); + const isPdf = ext === ".pdf"; + const contentType = isPdf ? "application/pdf" : "image/jpeg"; + + const job = await prisma.processingJob.create({ + data: { fileName: file.name, filePath: `ftp/${file.name}`, status: "queued" }, + }); + + await uploadBuffer(`sources/${job.id}/${file.name}`, buffer, contentType); + await enqueueProcessing(job.id); + + const destPath = `${config.processedDir}/${file.name}`; + try { + await client.ensureDir(config.processedDir!); + await client.rename(remotePath, destPath); + } catch { + log(`Could not move "${file.name}" to processed dir, removing instead`); + try { await client.remove(remotePath); } catch {} + } + + processed++; + log(`Queued: ${file.name} (job ${job.id})`); + } catch (err) { + logError(`Failed to process "${file.name}":`, err); + skipped++; + } + } + + client.close(); + } catch (err) { + client?.close(); + const message = err instanceof Error ? err.message : "FTP poll failed"; + logError("Error:", message); + return { processed, skipped, error: message }; + } } log(`Complete: ${processed} processed, ${skipped} skipped`); return { processed, skipped }; } +// ── Test connection ────────────────────────────────────────────────────── + export async function testFtpConnection(config: { host: string; port: number; user: string; pass: string; - tls: boolean; + protocol: Protocol; incomingDir: string; }): Promise<{ ok: boolean; files?: number; error?: string }> { - const client = new Client(15_000); - try { - await client.access({ - host: config.host, - port: config.port, - user: config.user, - password: config.pass, - secure: config.tls, - secureOptions: { rejectUnauthorized: false }, - }); - - const files = await client.list(config.incomingDir); - client.close(); - return { ok: true, files: files.length }; - } catch (err) { - const message = err instanceof Error ? err.message : "Connection failed"; - client.close(); - return { ok: false, error: message }; + if (config.protocol === "sftp") { + const sftp = new SftpClient(); + try { + await sftp.connect({ + host: config.host, + port: config.port, + username: config.user, + password: config.pass, + readyTimeout: CONNECT_TIMEOUT, + retries: 0, + }); + const files = await sftp.list(config.incomingDir); + await sftp.end(); + return { ok: true, files: files.length }; + } catch (err) { + await sftp.end().catch(() => {}); + const message = err instanceof Error ? err.message : "SFTP connection failed"; + return { ok: false, error: message }; + } + } else { + const client = new FtpClient(CONNECT_TIMEOUT); + try { + await client.access({ + host: config.host, + port: config.port, + user: config.user, + password: config.pass, + secure: config.protocol === "ftps", + secureOptions: { rejectUnauthorized: false }, + }); + const files = await client.list(config.incomingDir); + client.close(); + return { ok: true, files: files.length }; + } catch (err) { + client.close(); + const message = err instanceof Error ? err.message : "FTP connection failed"; + return { ok: false, error: message }; + } } }