2026-04-07 12:41:52 -04:00
|
|
|
import { ImapFlow } from "imapflow";
|
|
|
|
|
import { simpleParser } from "mailparser";
|
|
|
|
|
import { prisma } from "./db";
|
2026-04-16 00:51:44 -04:00
|
|
|
import { uploadBuffer } from "./storage";
|
2026-04-07 14:44:04 -04:00
|
|
|
import { enqueueProcessing } from "./processing-queue";
|
2026-04-07 12:41:52 -04:00
|
|
|
|
|
|
|
|
const ALLOWED_CONTENT_TYPES = [
|
|
|
|
|
"application/pdf",
|
|
|
|
|
"image/jpeg",
|
|
|
|
|
"image/jpg",
|
|
|
|
|
"image/png",
|
|
|
|
|
"image/webp",
|
|
|
|
|
];
|
|
|
|
|
|
2026-04-07 14:44:04 -04:00
|
|
|
const MAX_ATTACHMENT_SIZE = 50 * 1024 * 1024; // 50 MB
|
|
|
|
|
|
2026-04-07 12:41:52 -04:00
|
|
|
type EmailConfig = {
|
|
|
|
|
host: string;
|
|
|
|
|
port: number;
|
|
|
|
|
user: string;
|
|
|
|
|
pass: string;
|
|
|
|
|
tls: boolean;
|
|
|
|
|
folder: string;
|
|
|
|
|
processedAction: string;
|
|
|
|
|
processedFolder: string;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
function log(msg: string, ...args: unknown[]) {
|
|
|
|
|
console.log(`[email-watcher] ${msg}`, ...args);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function logError(msg: string, ...args: unknown[]) {
|
|
|
|
|
console.error(`[email-watcher] ${msg}`, ...args);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function handleMessage(client: ImapFlow, uid: number, config: EmailConfig) {
|
|
|
|
|
const result = await client.fetchOne(String(uid), { source: true }, { uid: true });
|
|
|
|
|
if (!result) return;
|
|
|
|
|
const rawSource = (result as unknown as { source?: Buffer }).source;
|
|
|
|
|
if (!rawSource) return;
|
|
|
|
|
|
|
|
|
|
const parsed = await simpleParser(rawSource);
|
|
|
|
|
const subject = parsed.subject || "(no subject)";
|
|
|
|
|
const from = parsed.from?.text || "unknown";
|
|
|
|
|
|
|
|
|
|
const attachments = (parsed.attachments || []).filter((att) => {
|
|
|
|
|
const ct = att.contentType?.toLowerCase() || "";
|
|
|
|
|
return ALLOWED_CONTENT_TYPES.includes(ct);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (attachments.length === 0) {
|
|
|
|
|
log(`Skipping email from ${from} "${subject}" — no valid attachments`);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
log(`Processing email from ${from} "${subject}" — ${attachments.length} attachment(s)`);
|
|
|
|
|
|
|
|
|
|
for (const att of attachments) {
|
|
|
|
|
const fileName = att.filename || `email-attachment-${Date.now()}`;
|
|
|
|
|
const buffer = att.content;
|
2026-04-07 14:44:04 -04:00
|
|
|
|
|
|
|
|
if (buffer.length > MAX_ATTACHMENT_SIZE) {
|
|
|
|
|
log(`Skipping attachment "${fileName}" — exceeds 50 MB size limit (${(buffer.length / 1024 / 1024).toFixed(1)} MB)`);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-07 12:41:52 -04:00
|
|
|
try {
|
|
|
|
|
const job = await prisma.processingJob.create({
|
|
|
|
|
data: {
|
|
|
|
|
fileName,
|
|
|
|
|
filePath: `email/${fileName}`,
|
|
|
|
|
status: "queued",
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
|
2026-04-16 00:51:44 -04:00
|
|
|
const sourceKey = `sources/${job.id}/${fileName}`;
|
Add dynamic fields, people directory, analytics, security hardening, and UX polish
Phase 1 - Security & Bug Fixes:
- Add requireApiAuth helper and protect all 25 unprotected API routes
- Add org-tenant scoping to all card, job, stats, and notification queries
- Fix SSRF in ai-test, mask secrets in settings API, fix middleware bypass
- Fix cards pagination routing, stat filter sync, drag-drop file passing
- Add PUT /api/auth/me for profile persistence, stuck job recovery
- Fix email watcher MIME type detection
Phase 2 - Dynamic Fields & Digital Survey:
- Add FormTemplate, FormField, Person, PasswordResetToken models to schema
- Add fieldData, formTemplateId, firstName, lastName, personId to ResponseCard
- Build FormTemplate CRUD API with field management and org scoping
- Build Form Builder UI with field ordering, type config, and section management
- Refactor card detail page to render fields dynamically from templates
- Add dynamic OCR prompt/schema generation from template fields
- Build public survey page at /s/[orgSlug]/[formSlug] with branding
- Add QR code generation API and share section component
Phase 3 - People & Analytics:
- Build People CRUD API with merge and batch auto-link endpoints
- Build People list and detail pages with search, merge dialog
- Add auto-link logic in OCR completion to match/create Person records
- Add /api/stats/trends endpoint with time series and team activity
- Build Reports page with Recharts (area charts, bar charts, pipeline)
- Upgrade dashboard with sparklines and People stat card
Phase 4 - UX Polish:
- Replace silent error handling with toast notifications across all pages
- Add loading skeletons, differentiated empty states
- Add ARIA labels, skip-to-content link, accessible column toggle
- Add forgot password flow, Cmd+K command palette, Collection Days pages
- Unify Echo branding and theme toggle consistency
Made-with: Cursor
2026-04-17 00:29:26 -04:00
|
|
|
await uploadBuffer(sourceKey, buffer, att.contentType || "application/octet-stream");
|
2026-04-07 12:41:52 -04:00
|
|
|
|
2026-04-16 00:51:44 -04:00
|
|
|
await enqueueProcessing(job.id);
|
2026-04-07 12:41:52 -04:00
|
|
|
log(`Queued: ${fileName} (job ${job.id})`);
|
|
|
|
|
} catch (err) {
|
|
|
|
|
logError(`Failed to create job for ${fileName}:`, err);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (config.processedAction === "mark_read") {
|
|
|
|
|
await client.messageFlagsAdd(String(uid), ["\\Seen"], { uid: true });
|
|
|
|
|
} else if (config.processedAction === "move") {
|
|
|
|
|
try {
|
|
|
|
|
await client.messageMove(String(uid), config.processedFolder, { uid: true });
|
|
|
|
|
} catch {
|
|
|
|
|
log(`Could not move to "${config.processedFolder}", marking as read instead`);
|
|
|
|
|
await client.messageFlagsAdd(String(uid), ["\\Seen"], { uid: true });
|
|
|
|
|
}
|
|
|
|
|
} else if (config.processedAction === "delete") {
|
|
|
|
|
await client.messageDelete(String(uid), { uid: true });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-10 14:04:45 -04:00
|
|
|
export async function scanInbox(): Promise<{ processed: number; skipped: number; error?: string }> {
|
|
|
|
|
const settings = await prisma.appSettings.findUnique({ where: { id: "singleton" } });
|
|
|
|
|
if (!settings?.emailImapHost || !settings?.emailImapUser || !settings?.emailImapPass) {
|
|
|
|
|
return { processed: 0, skipped: 0, error: "IMAP not configured" };
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-16 00:51:44 -04:00
|
|
|
if (!settings.emailWatching) {
|
|
|
|
|
return { processed: 0, skipped: 0, error: "Email watching is disabled" };
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-10 14:04:45 -04:00
|
|
|
const config: EmailConfig = {
|
|
|
|
|
host: settings.emailImapHost,
|
|
|
|
|
port: settings.emailImapPort,
|
|
|
|
|
user: settings.emailImapUser,
|
|
|
|
|
pass: settings.emailImapPass,
|
|
|
|
|
tls: settings.emailImapTls,
|
|
|
|
|
folder: settings.emailFolder,
|
|
|
|
|
processedAction: settings.emailProcessed,
|
|
|
|
|
processedFolder: settings.emailProcessedFolder,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const scanClient = new ImapFlow({
|
|
|
|
|
host: config.host,
|
|
|
|
|
port: config.port,
|
|
|
|
|
secure: config.tls,
|
|
|
|
|
auth: { user: config.user, pass: config.pass },
|
|
|
|
|
logger: false,
|
|
|
|
|
emitLogs: false,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
let processed = 0;
|
|
|
|
|
let skipped = 0;
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
await scanClient.connect();
|
2026-04-16 00:51:44 -04:00
|
|
|
log(`[scan] Connected for inbox scan`);
|
2026-04-10 14:04:45 -04:00
|
|
|
|
|
|
|
|
const lock = await scanClient.getMailboxLock(config.folder);
|
|
|
|
|
try {
|
|
|
|
|
const searchResult = await scanClient.search({ seen: false }, { uid: true });
|
|
|
|
|
const unseen = Array.isArray(searchResult) ? searchResult : [];
|
|
|
|
|
log(`[scan] Found ${unseen.length} unseen message(s)`);
|
|
|
|
|
|
|
|
|
|
for (const uid of unseen) {
|
|
|
|
|
try {
|
|
|
|
|
await handleMessage(scanClient, uid, config);
|
|
|
|
|
processed++;
|
|
|
|
|
} catch (err) {
|
|
|
|
|
logError(`[scan] Error handling message UID ${uid}:`, err);
|
|
|
|
|
skipped++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
lock.release();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await scanClient.logout();
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const message = err instanceof Error ? err.message : "Scan failed";
|
|
|
|
|
logError("[scan] Error:", message);
|
|
|
|
|
try { await scanClient.logout(); } catch {}
|
|
|
|
|
return { processed, skipped, error: message };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
log(`[scan] Complete: ${processed} processed, ${skipped} skipped`);
|
|
|
|
|
return { processed, skipped };
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-07 12:41:52 -04:00
|
|
|
export async function testEmailConnection(config: Omit<EmailConfig, "processedAction" | "processedFolder">): Promise<{ ok: boolean; folders?: string[]; error?: string }> {
|
|
|
|
|
let testClient: ImapFlow | null = null;
|
|
|
|
|
try {
|
|
|
|
|
testClient = new ImapFlow({
|
|
|
|
|
host: config.host,
|
|
|
|
|
port: config.port,
|
|
|
|
|
secure: config.tls,
|
|
|
|
|
auth: { user: config.user, pass: config.pass },
|
|
|
|
|
logger: false,
|
|
|
|
|
emitLogs: false,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await testClient.connect();
|
|
|
|
|
const folders: string[] = [];
|
|
|
|
|
const list = await testClient.list();
|
|
|
|
|
for (const folder of list) {
|
|
|
|
|
folders.push(folder.path);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await testClient.logout();
|
|
|
|
|
return { ok: true, folders };
|
|
|
|
|
} catch (err) {
|
|
|
|
|
const message = err instanceof Error ? err.message : "Connection failed";
|
|
|
|
|
return { ok: false, error: message };
|
|
|
|
|
} finally {
|
|
|
|
|
if (testClient) {
|
|
|
|
|
try { await testClient.logout(); } catch {}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|