Add SFTP support: replace FTP-only TLS toggle with protocol selector
Dreamhost (and most modern hosts) use SFTP (port 22), not FTP (port 21).
The previous implementation only supported FTP/FTPS via basic-ftp, causing
timeouts when connecting to SFTP servers.
Changes:
- Add ssh2-sftp-client for SFTP connections
- Replace ftpTls boolean with ftpProtocol ("sftp" | "ftp" | "ftps") in schema
- Rewrite ftp-watcher.ts to support both SFTP and FTP/FTPS protocols
- Update UI with protocol dropdown that auto-switches the default port
- Add ssh2/ssh2-sftp-client to serverExternalPackages in next.config
- Default to SFTP on port 22
Made-with: Cursor
This commit is contained in:
Randall Stillwell
parent
bc5ad6dbdb
commit
028840a887
8 changed files with 502 additions and 115 deletions
|
|
@ -9,7 +9,7 @@ const nextConfig: NextConfig = {
|
|||
},
|
||||
],
|
||||
},
|
||||
serverExternalPackages: ["sharp", "@napi-rs/canvas", "pg", "basic-ftp", "imapflow", "mailparser"],
|
||||
serverExternalPackages: ["sharp", "@napi-rs/canvas", "pg", "basic-ftp", "ssh2", "ssh2-sftp-client", "imapflow", "mailparser"],
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
|
|
|
|||
197
package-lock.json
generated
197
package-lock.json
generated
|
|
@ -48,6 +48,7 @@
|
|||
"shadcn": "^4.0.2",
|
||||
"sharp": "^0.34.5",
|
||||
"sonner": "^2.0.7",
|
||||
"ssh2-sftp-client": "^12.1.1",
|
||||
"tailwind-merge": "^3.5.0",
|
||||
"tw-animate-css": "^1.4.0",
|
||||
"zod": "^4.3.6"
|
||||
|
|
@ -60,6 +61,7 @@
|
|||
"@types/pg": "^8.18.0",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"@types/ssh2-sftp-client": "^9.0.6",
|
||||
"eslint": "^9",
|
||||
"eslint-config-next": "16.1.6",
|
||||
"tailwindcss": "^4",
|
||||
|
|
@ -5353,6 +5355,43 @@
|
|||
"@types/react": "^19.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/ssh2": {
|
||||
"version": "1.15.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz",
|
||||
"integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "^18.11.18"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/ssh2-sftp-client": {
|
||||
"version": "9.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@types/ssh2-sftp-client/-/ssh2-sftp-client-9.0.6.tgz",
|
||||
"integrity": "sha512-4+KvXO/V77y9VjI2op2T8+RCGI/GXQAwR0q5Qkj/EJ5YSeyKszqZP6F8i3H3txYoBqjc7sgorqyvBP3+w1EHyg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/ssh2": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/ssh2/node_modules/@types/node": {
|
||||
"version": "18.19.130",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz",
|
||||
"integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~5.26.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/ssh2/node_modules/undici-types": {
|
||||
"version": "5.26.5",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/statuses": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.6.tgz",
|
||||
|
|
@ -6318,6 +6357,15 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/asn1": {
|
||||
"version": "0.2.6",
|
||||
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
|
||||
"integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safer-buffer": "~2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ast-types": {
|
||||
"version": "0.16.1",
|
||||
"resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.16.1.tgz",
|
||||
|
|
@ -6429,6 +6477,15 @@
|
|||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bcrypt-pbkdf": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
|
||||
"integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"tweetnacl": "^0.14.3"
|
||||
}
|
||||
},
|
||||
"node_modules/bcryptjs": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-3.0.3.tgz",
|
||||
|
|
@ -6524,6 +6581,21 @@
|
|||
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
|
||||
}
|
||||
},
|
||||
"node_modules/buffer-from": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
|
||||
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/buildcheck": {
|
||||
"version": "0.0.7",
|
||||
"resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz",
|
||||
"integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==",
|
||||
"optional": true,
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bundle-name": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz",
|
||||
|
|
@ -6933,6 +7005,21 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/concat-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
|
||||
"integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
|
||||
"engines": [
|
||||
"node >= 6.0"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^3.0.2",
|
||||
"typedarray": "^0.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/confbox": {
|
||||
"version": "0.2.4",
|
||||
"resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.4.tgz",
|
||||
|
|
@ -7037,6 +7124,20 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/cpu-features": {
|
||||
"version": "0.0.10",
|
||||
"resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz",
|
||||
"integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==",
|
||||
"hasInstallScript": true,
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"buildcheck": "~0.0.6",
|
||||
"nan": "^2.19.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/cross-spawn": {
|
||||
"version": "7.0.6",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
|
||||
|
|
@ -10972,6 +11073,13 @@
|
|||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/nan": {
|
||||
"version": "2.26.2",
|
||||
"resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz",
|
||||
"integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==",
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.11",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz",
|
||||
|
|
@ -12614,6 +12722,20 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/readable-stream": {
|
||||
"version": "3.6.2",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
|
||||
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"inherits": "^2.0.3",
|
||||
"string_decoder": "^1.1.1",
|
||||
"util-deprecate": "^1.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/real-require": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz",
|
||||
|
|
@ -12944,6 +13066,26 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/feross"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://www.patreon.com/feross"
|
||||
},
|
||||
{
|
||||
"type": "consulting",
|
||||
"url": "https://feross.org/support"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/safe-push-apply": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/safe-push-apply/-/safe-push-apply-1.0.0.tgz",
|
||||
|
|
@ -13475,6 +13617,40 @@
|
|||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/ssh2": {
|
||||
"version": "1.17.0",
|
||||
"resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz",
|
||||
"integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==",
|
||||
"hasInstallScript": true,
|
||||
"dependencies": {
|
||||
"asn1": "^0.2.6",
|
||||
"bcrypt-pbkdf": "^1.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.16.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"cpu-features": "~0.0.10",
|
||||
"nan": "^2.23.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ssh2-sftp-client": {
|
||||
"version": "12.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-12.1.1.tgz",
|
||||
"integrity": "sha512-wYVDgwkpcKG2iPGQQ+QR33xkWqLFIaVrYvA+uON4pmxTPaPuB81f1aooUEPN75e/9DCK6rrKYXb6zR6zP3+EtA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"concat-stream": "^2.0.0",
|
||||
"ssh2": "^1.16.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.20.4"
|
||||
},
|
||||
"funding": {
|
||||
"type": "individual",
|
||||
"url": "https://square.link/u/4g7sPflL"
|
||||
}
|
||||
},
|
||||
"node_modules/stable-hash": {
|
||||
"version": "0.0.5",
|
||||
"resolved": "https://registry.npmjs.org/stable-hash/-/stable-hash-0.0.5.tgz",
|
||||
|
|
@ -13529,6 +13705,15 @@
|
|||
"integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safe-buffer": "~5.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/string-width": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz",
|
||||
|
|
@ -14040,6 +14225,12 @@
|
|||
"url": "https://github.com/sponsors/Wombosvideo"
|
||||
}
|
||||
},
|
||||
"node_modules/tweetnacl": {
|
||||
"version": "0.14.5",
|
||||
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
|
||||
"integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==",
|
||||
"license": "Unlicense"
|
||||
},
|
||||
"node_modules/type-check": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
|
||||
|
|
@ -14160,6 +14351,12 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/typedarray": {
|
||||
"version": "0.0.6",
|
||||
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
|
||||
"integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@
|
|||
"shadcn": "^4.0.2",
|
||||
"sharp": "^0.34.5",
|
||||
"sonner": "^2.0.7",
|
||||
"ssh2-sftp-client": "^12.1.1",
|
||||
"tailwind-merge": "^3.5.0",
|
||||
"tw-animate-css": "^1.4.0",
|
||||
"zod": "^4.3.6"
|
||||
|
|
@ -64,6 +65,7 @@
|
|||
"@types/pg": "^8.18.0",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"@types/ssh2-sftp-client": "^9.0.6",
|
||||
"eslint": "^9",
|
||||
"eslint-config-next": "16.1.6",
|
||||
"tailwindcss": "^4",
|
||||
|
|
|
|||
|
|
@ -427,10 +427,10 @@ model AppSettings {
|
|||
|
||||
ftpEnabled Boolean @default(false)
|
||||
ftpHost String @default("")
|
||||
ftpPort Int @default(21)
|
||||
ftpPort Int @default(22)
|
||||
ftpUser String @default("")
|
||||
ftpPass String @default("")
|
||||
ftpTls Boolean @default(true)
|
||||
ftpProtocol String @default("sftp") // "sftp" | "ftp" | "ftps"
|
||||
ftpIncomingDir String @default("/incoming")
|
||||
ftpProcessedDir String @default("/processed")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ type SourceSettings = {
|
|||
ftpPort: number;
|
||||
ftpUser: string;
|
||||
ftpPass: string;
|
||||
ftpTls: boolean;
|
||||
ftpProtocol: "sftp" | "ftp" | "ftps";
|
||||
ftpIncomingDir: string;
|
||||
ftpProcessedDir: string;
|
||||
watchDir: string;
|
||||
|
|
@ -68,10 +68,10 @@ export default function UploadSourcesPage() {
|
|||
emailProcessedFolder: "Processed",
|
||||
ftpEnabled: false,
|
||||
ftpHost: "",
|
||||
ftpPort: 21,
|
||||
ftpPort: 22,
|
||||
ftpUser: "",
|
||||
ftpPass: "",
|
||||
ftpTls: true,
|
||||
ftpProtocol: "sftp",
|
||||
ftpIncomingDir: "/incoming",
|
||||
ftpProcessedDir: "/processed",
|
||||
watchDir: "",
|
||||
|
|
@ -103,10 +103,10 @@ export default function UploadSourcesPage() {
|
|||
emailProcessedFolder: data.emailProcessedFolder || "Processed",
|
||||
ftpEnabled: data.ftpEnabled || false,
|
||||
ftpHost: data.ftpHost || "",
|
||||
ftpPort: data.ftpPort || 21,
|
||||
ftpPort: data.ftpPort || 22,
|
||||
ftpUser: data.ftpUser || "",
|
||||
ftpPass: data.ftpPass || "",
|
||||
ftpTls: data.ftpTls ?? true,
|
||||
ftpProtocol: data.ftpProtocol || "sftp",
|
||||
ftpIncomingDir: data.ftpIncomingDir || "/incoming",
|
||||
ftpProcessedDir: data.ftpProcessedDir || "/processed",
|
||||
watchDir: data.watchDir || "",
|
||||
|
|
@ -195,7 +195,7 @@ export default function UploadSourcesPage() {
|
|||
port: settings.ftpPort,
|
||||
user: settings.ftpUser,
|
||||
pass: settings.ftpPass,
|
||||
tls: settings.ftpTls,
|
||||
protocol: settings.ftpProtocol,
|
||||
incomingDir: settings.ftpIncomingDir,
|
||||
}),
|
||||
});
|
||||
|
|
@ -373,11 +373,12 @@ export default function UploadSourcesPage() {
|
|||
<CardHeader>
|
||||
<CardTitle className="flex items-center gap-2 text-base">
|
||||
<Server className="size-4" />
|
||||
FTP Server
|
||||
SFTP / FTP Server
|
||||
</CardTitle>
|
||||
<CardDescription>
|
||||
Poll an FTP server for new scanned files. Files are downloaded,
|
||||
processed, then moved to a “processed” directory.
|
||||
Poll a remote server for new scanned files via SFTP, FTP, or FTPS.
|
||||
Files are downloaded, processed, then moved to a
|
||||
“processed” directory.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
|
|
@ -393,9 +394,33 @@ export default function UploadSourcesPage() {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-4 sm:grid-cols-2">
|
||||
<div className="grid gap-4 sm:grid-cols-3">
|
||||
<div className="space-y-2">
|
||||
<Label>FTP Host</Label>
|
||||
<Label>Protocol</Label>
|
||||
<Select
|
||||
value={settings.ftpProtocol}
|
||||
onValueChange={(v) => {
|
||||
const proto = v as "sftp" | "ftp" | "ftps";
|
||||
const defaultPort = proto === "sftp" ? 22 : 21;
|
||||
setSettings((s) => ({
|
||||
...s,
|
||||
ftpProtocol: proto,
|
||||
ftpPort: defaultPort,
|
||||
}));
|
||||
}}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="sftp">SFTP (SSH)</SelectItem>
|
||||
<SelectItem value="ftp">FTP</SelectItem>
|
||||
<SelectItem value="ftps">FTPS (TLS)</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label>Host</Label>
|
||||
<Input
|
||||
value={settings.ftpHost}
|
||||
onChange={(e) =>
|
||||
|
|
@ -412,11 +437,14 @@ export default function UploadSourcesPage() {
|
|||
onChange={(e) =>
|
||||
setSettings((s) => ({
|
||||
...s,
|
||||
ftpPort: parseInt(e.target.value) || 21,
|
||||
ftpPort: parseInt(e.target.value) || 22,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-4 sm:grid-cols-2">
|
||||
<div className="space-y-2">
|
||||
<Label>Username</Label>
|
||||
<Input
|
||||
|
|
@ -438,18 +466,6 @@ export default function UploadSourcesPage() {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-4">
|
||||
<div className="flex items-center gap-2">
|
||||
<Switch
|
||||
checked={settings.ftpTls}
|
||||
onCheckedChange={(v) =>
|
||||
setSettings((s) => ({ ...s, ftpTls: !!v }))
|
||||
}
|
||||
/>
|
||||
<Label className="text-sm">Use TLS</Label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-4 sm:grid-cols-2">
|
||||
<div className="space-y-2">
|
||||
<Label>Incoming Directory</Label>
|
||||
|
|
|
|||
|
|
@ -16,12 +16,15 @@ export async function POST(request: NextRequest) {
|
|||
pass = settings?.ftpPass || "";
|
||||
}
|
||||
|
||||
const protocol = body.protocol || "sftp";
|
||||
const defaultPort = protocol === "sftp" ? 22 : 21;
|
||||
|
||||
const result = await testFtpConnection({
|
||||
host: body.host,
|
||||
port: body.port || 21,
|
||||
port: body.port || defaultPort,
|
||||
user: body.user,
|
||||
pass,
|
||||
tls: body.tls ?? true,
|
||||
protocol,
|
||||
incomingDir: body.incomingDir || "/incoming",
|
||||
});
|
||||
return NextResponse.json(result);
|
||||
|
|
|
|||
|
|
@ -67,7 +67,8 @@ export async function PUT(request: NextRequest) {
|
|||
if (body.ftpPort != null) data.ftpPort = Math.max(1, parseInt(String(body.ftpPort)) || 21);
|
||||
if (body.ftpUser != null) data.ftpUser = String(body.ftpUser);
|
||||
if (body.ftpPass != null) data.ftpPass = String(body.ftpPass);
|
||||
if (body.ftpTls != null) data.ftpTls = Boolean(body.ftpTls);
|
||||
if (body.ftpProtocol != null && ["sftp", "ftp", "ftps"].includes(body.ftpProtocol))
|
||||
data.ftpProtocol = String(body.ftpProtocol);
|
||||
if (body.ftpIncomingDir != null) data.ftpIncomingDir = String(body.ftpIncomingDir);
|
||||
if (body.ftpProcessedDir != null) data.ftpProcessedDir = String(body.ftpProcessedDir);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { Client } from "basic-ftp";
|
||||
import { Client as FtpClient } from "basic-ftp";
|
||||
import SftpClient from "ssh2-sftp-client";
|
||||
import { Writable } from "stream";
|
||||
import { prisma } from "./db";
|
||||
import { uploadBuffer } from "./storage";
|
||||
|
|
@ -6,6 +7,25 @@ import { enqueueProcessing } from "./processing-queue";
|
|||
|
||||
const ALLOWED_EXTENSIONS = [".pdf", ".jpg", ".jpeg", ".png", ".webp"];
|
||||
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50 MB
|
||||
const CONNECT_TIMEOUT = 15_000;
|
||||
|
||||
type Protocol = "sftp" | "ftp" | "ftps";
|
||||
|
||||
interface ConnectionConfig {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
pass: string;
|
||||
protocol: Protocol;
|
||||
incomingDir: string;
|
||||
processedDir?: string;
|
||||
}
|
||||
|
||||
interface FileEntry {
|
||||
name: string;
|
||||
size: number;
|
||||
isDirectory: boolean;
|
||||
}
|
||||
|
||||
function log(msg: string, ...args: unknown[]) {
|
||||
console.log(`[ftp-watcher] ${msg}`, ...args);
|
||||
|
|
@ -15,7 +35,71 @@ function logError(msg: string, ...args: unknown[]) {
|
|||
console.error(`[ftp-watcher] ${msg}`, ...args);
|
||||
}
|
||||
|
||||
async function downloadToBuffer(client: Client, remotePath: string): Promise<Buffer> {
|
||||
// ── SFTP helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
async function sftpConnect(config: ConnectionConfig): Promise<SftpClient> {
|
||||
const sftp = new SftpClient();
|
||||
await sftp.connect({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
username: config.user,
|
||||
password: config.pass,
|
||||
readyTimeout: CONNECT_TIMEOUT,
|
||||
retries: 0,
|
||||
});
|
||||
return sftp;
|
||||
}
|
||||
|
||||
async function sftpList(sftp: SftpClient, dir: string): Promise<FileEntry[]> {
|
||||
const items = await sftp.list(dir);
|
||||
return items.map((f) => ({
|
||||
name: f.name,
|
||||
size: f.size,
|
||||
isDirectory: f.type === "d",
|
||||
}));
|
||||
}
|
||||
|
||||
async function sftpDownload(sftp: SftpClient, remotePath: string): Promise<Buffer> {
|
||||
const result = await sftp.get(remotePath);
|
||||
if (Buffer.isBuffer(result)) return result;
|
||||
if (typeof result === "string") return Buffer.from(result);
|
||||
return Buffer.from(result as unknown as ArrayBuffer);
|
||||
}
|
||||
|
||||
async function sftpMove(
|
||||
sftp: SftpClient,
|
||||
src: string,
|
||||
dest: string,
|
||||
destDir: string
|
||||
): Promise<void> {
|
||||
try {
|
||||
const exists = await sftp.exists(destDir);
|
||||
if (!exists) await sftp.mkdir(destDir, true);
|
||||
await sftp.rename(src, dest);
|
||||
} catch {
|
||||
log(`Could not move file to processed dir, removing instead`);
|
||||
try {
|
||||
await sftp.delete(src);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
// ── FTP / FTPS helpers ────────────────────────────────────────────────────
|
||||
|
||||
async function ftpConnect(config: ConnectionConfig): Promise<FtpClient> {
|
||||
const client = new FtpClient(CONNECT_TIMEOUT);
|
||||
await client.access({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
user: config.user,
|
||||
password: config.pass,
|
||||
secure: config.protocol === "ftps",
|
||||
secureOptions: { rejectUnauthorized: false },
|
||||
});
|
||||
return client;
|
||||
}
|
||||
|
||||
async function ftpDownloadToBuffer(client: FtpClient, remotePath: string): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
const writable = new Writable({
|
||||
write(chunk, _encoding, callback) {
|
||||
|
|
@ -27,6 +111,8 @@ async function downloadToBuffer(client: Client, remotePath: string): Promise<Buf
|
|||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
// ── Poll (main entry point) ──────────────────────────────────────────────
|
||||
|
||||
export async function pollFtp(): Promise<{ processed: number; skipped: number; error?: string }> {
|
||||
const settings = await prisma.appSettings.findUnique({ where: { id: "singleton" } });
|
||||
|
||||
|
|
@ -38,112 +124,194 @@ export async function pollFtp(): Promise<{ processed: number; skipped: number; e
|
|||
return { processed: 0, skipped: 0, error: "FTP not configured" };
|
||||
}
|
||||
|
||||
const client = new Client(15_000);
|
||||
const protocol = (settings.ftpProtocol || "sftp") as Protocol;
|
||||
const config: ConnectionConfig = {
|
||||
host: settings.ftpHost,
|
||||
port: settings.ftpPort,
|
||||
user: settings.ftpUser,
|
||||
pass: settings.ftpPass,
|
||||
protocol,
|
||||
incomingDir: settings.ftpIncomingDir,
|
||||
processedDir: settings.ftpProcessedDir,
|
||||
};
|
||||
|
||||
let processed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
try {
|
||||
await client.access({
|
||||
host: settings.ftpHost,
|
||||
port: settings.ftpPort,
|
||||
user: settings.ftpUser,
|
||||
password: settings.ftpPass,
|
||||
secure: settings.ftpTls,
|
||||
secureOptions: { rejectUnauthorized: false },
|
||||
if (protocol === "sftp") {
|
||||
const sftp = await sftpConnect(config).catch((err) => {
|
||||
throw new Error(`SFTP connect failed: ${err.message}`);
|
||||
});
|
||||
|
||||
log(`Connected to ${settings.ftpHost}:${settings.ftpPort}`);
|
||||
try {
|
||||
log(`Connected via SFTP to ${config.host}:${config.port}`);
|
||||
const allFiles = await sftpList(sftp, config.incomingDir);
|
||||
const validFiles = allFiles.filter((f) => {
|
||||
if (f.isDirectory) return false;
|
||||
const ext = f.name.toLowerCase().match(/\.[^.]+$/)?.[0] || "";
|
||||
return ALLOWED_EXTENSIONS.includes(ext);
|
||||
});
|
||||
|
||||
const files = await client.list(settings.ftpIncomingDir);
|
||||
const validFiles = files.filter((f) => {
|
||||
if (f.isDirectory) return false;
|
||||
const ext = f.name.toLowerCase().match(/\.[^.]+$/)?.[0] || "";
|
||||
return ALLOWED_EXTENSIONS.includes(ext);
|
||||
});
|
||||
log(`Found ${validFiles.length} file(s) in ${config.incomingDir}`);
|
||||
|
||||
log(`Found ${validFiles.length} file(s) in ${settings.ftpIncomingDir}`);
|
||||
for (const file of validFiles) {
|
||||
const remotePath = `${config.incomingDir}/${file.name}`;
|
||||
const ext = file.name.toLowerCase().match(/\.[^.]+$/)?.[0] || "";
|
||||
|
||||
for (const file of validFiles) {
|
||||
const remotePath = `${settings.ftpIncomingDir}/${file.name}`;
|
||||
const ext = file.name.toLowerCase().match(/\.[^.]+$/)?.[0] || "";
|
||||
|
||||
if (file.size > MAX_FILE_SIZE) {
|
||||
log(`Skipping "${file.name}" — exceeds 50 MB (${(file.size / 1024 / 1024).toFixed(1)} MB)`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const buffer = await downloadToBuffer(client, remotePath);
|
||||
const isPdf = ext === ".pdf";
|
||||
const contentType = isPdf ? "application/pdf" : "image/jpeg";
|
||||
|
||||
const job = await prisma.processingJob.create({
|
||||
data: {
|
||||
fileName: file.name,
|
||||
filePath: `ftp/${file.name}`,
|
||||
status: "queued",
|
||||
},
|
||||
});
|
||||
|
||||
const sourceKey = `sources/${job.id}/${file.name}`;
|
||||
await uploadBuffer(sourceKey, buffer, contentType);
|
||||
await enqueueProcessing(job.id);
|
||||
|
||||
const destPath = `${settings.ftpProcessedDir}/${file.name}`;
|
||||
try {
|
||||
await client.ensureDir(settings.ftpProcessedDir);
|
||||
await client.rename(remotePath, destPath);
|
||||
} catch {
|
||||
log(`Could not move "${file.name}" to processed dir, removing instead`);
|
||||
try { await client.remove(remotePath); } catch {}
|
||||
if (file.size > MAX_FILE_SIZE) {
|
||||
log(`Skipping "${file.name}" — exceeds 50 MB`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
processed++;
|
||||
log(`Queued: ${file.name} (job ${job.id})`);
|
||||
} catch (err) {
|
||||
logError(`Failed to process "${file.name}":`, err);
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
try {
|
||||
const buffer = await sftpDownload(sftp, remotePath);
|
||||
const isPdf = ext === ".pdf";
|
||||
const contentType = isPdf ? "application/pdf" : "image/jpeg";
|
||||
|
||||
client.close();
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "FTP poll failed";
|
||||
logError("Error:", message);
|
||||
client.close();
|
||||
return { processed, skipped, error: message };
|
||||
const job = await prisma.processingJob.create({
|
||||
data: { fileName: file.name, filePath: `ftp/${file.name}`, status: "queued" },
|
||||
});
|
||||
|
||||
await uploadBuffer(`sources/${job.id}/${file.name}`, buffer, contentType);
|
||||
await enqueueProcessing(job.id);
|
||||
|
||||
await sftpMove(
|
||||
sftp,
|
||||
remotePath,
|
||||
`${config.processedDir}/${file.name}`,
|
||||
config.processedDir!
|
||||
);
|
||||
|
||||
processed++;
|
||||
log(`Queued: ${file.name} (job ${job.id})`);
|
||||
} catch (err) {
|
||||
logError(`Failed to process "${file.name}":`, err);
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
await sftp.end();
|
||||
} catch (err) {
|
||||
await sftp.end().catch(() => {});
|
||||
const message = err instanceof Error ? err.message : "SFTP poll failed";
|
||||
logError("Error:", message);
|
||||
return { processed, skipped, error: message };
|
||||
}
|
||||
} else {
|
||||
let client: FtpClient | null = null;
|
||||
try {
|
||||
client = await ftpConnect(config);
|
||||
log(`Connected via ${protocol.toUpperCase()} to ${config.host}:${config.port}`);
|
||||
|
||||
const files = await client.list(config.incomingDir);
|
||||
const validFiles = files.filter((f) => {
|
||||
if (f.isDirectory) return false;
|
||||
const ext = f.name.toLowerCase().match(/\.[^.]+$/)?.[0] || "";
|
||||
return ALLOWED_EXTENSIONS.includes(ext);
|
||||
});
|
||||
|
||||
log(`Found ${validFiles.length} file(s) in ${config.incomingDir}`);
|
||||
|
||||
for (const file of validFiles) {
|
||||
const remotePath = `${config.incomingDir}/${file.name}`;
|
||||
const ext = file.name.toLowerCase().match(/\.[^.]+$/)?.[0] || "";
|
||||
|
||||
if (file.size > MAX_FILE_SIZE) {
|
||||
log(`Skipping "${file.name}" — exceeds 50 MB`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const buffer = await ftpDownloadToBuffer(client, remotePath);
|
||||
const isPdf = ext === ".pdf";
|
||||
const contentType = isPdf ? "application/pdf" : "image/jpeg";
|
||||
|
||||
const job = await prisma.processingJob.create({
|
||||
data: { fileName: file.name, filePath: `ftp/${file.name}`, status: "queued" },
|
||||
});
|
||||
|
||||
await uploadBuffer(`sources/${job.id}/${file.name}`, buffer, contentType);
|
||||
await enqueueProcessing(job.id);
|
||||
|
||||
const destPath = `${config.processedDir}/${file.name}`;
|
||||
try {
|
||||
await client.ensureDir(config.processedDir!);
|
||||
await client.rename(remotePath, destPath);
|
||||
} catch {
|
||||
log(`Could not move "${file.name}" to processed dir, removing instead`);
|
||||
try { await client.remove(remotePath); } catch {}
|
||||
}
|
||||
|
||||
processed++;
|
||||
log(`Queued: ${file.name} (job ${job.id})`);
|
||||
} catch (err) {
|
||||
logError(`Failed to process "${file.name}":`, err);
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
client.close();
|
||||
} catch (err) {
|
||||
client?.close();
|
||||
const message = err instanceof Error ? err.message : "FTP poll failed";
|
||||
logError("Error:", message);
|
||||
return { processed, skipped, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
log(`Complete: ${processed} processed, ${skipped} skipped`);
|
||||
return { processed, skipped };
|
||||
}
|
||||
|
||||
// ── Test connection ──────────────────────────────────────────────────────
|
||||
|
||||
export async function testFtpConnection(config: {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
pass: string;
|
||||
tls: boolean;
|
||||
protocol: Protocol;
|
||||
incomingDir: string;
|
||||
}): Promise<{ ok: boolean; files?: number; error?: string }> {
|
||||
const client = new Client(15_000);
|
||||
try {
|
||||
await client.access({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
user: config.user,
|
||||
password: config.pass,
|
||||
secure: config.tls,
|
||||
secureOptions: { rejectUnauthorized: false },
|
||||
});
|
||||
|
||||
const files = await client.list(config.incomingDir);
|
||||
client.close();
|
||||
return { ok: true, files: files.length };
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Connection failed";
|
||||
client.close();
|
||||
return { ok: false, error: message };
|
||||
if (config.protocol === "sftp") {
|
||||
const sftp = new SftpClient();
|
||||
try {
|
||||
await sftp.connect({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
username: config.user,
|
||||
password: config.pass,
|
||||
readyTimeout: CONNECT_TIMEOUT,
|
||||
retries: 0,
|
||||
});
|
||||
const files = await sftp.list(config.incomingDir);
|
||||
await sftp.end();
|
||||
return { ok: true, files: files.length };
|
||||
} catch (err) {
|
||||
await sftp.end().catch(() => {});
|
||||
const message = err instanceof Error ? err.message : "SFTP connection failed";
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
} else {
|
||||
const client = new FtpClient(CONNECT_TIMEOUT);
|
||||
try {
|
||||
await client.access({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
user: config.user,
|
||||
password: config.pass,
|
||||
secure: config.protocol === "ftps",
|
||||
secureOptions: { rejectUnauthorized: false },
|
||||
});
|
||||
const files = await client.list(config.incomingDir);
|
||||
client.close();
|
||||
return { ok: true, files: files.length };
|
||||
} catch (err) {
|
||||
client.close();
|
||||
const message = err instanceof Error ? err.message : "FTP connection failed";
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue