Submatter/apps/orchestrator/src/index.ts

import { db } from "@tasks/database";
import IoRedis from "ioredis";

import { detectStalledRunsForResume } from "./db/convoy.js";
import { loadEnv } from "./env.js";
import { enqueueJob } from "./queue/redis-queue.js";
import { runOrchestratorTick } from "./tick.js";

async function main(): Promise<void> {
  const env = loadEnv();
  const redis = env.REDIS_URL ? new IoRedis(env.REDIS_URL) : null;

  if (env.ORCHESTRATOR_DRY_RUN) {
    console.info("[orchestrator] DRY-RUN mode — jobs will be logged, not enqueued");
  } else if (!redis) {
    console.warn("[orchestrator] REDIS_URL not set — running in DRY-RUN mode");
  }

  console.info(`[orchestrator] polling every ${env.ORCHESTRATOR_POLL_MS}ms`);

  const loop = async () => {
    try {
      const results = await runOrchestratorTick(db, redis, {
        ...env,
        ORCHESTRATOR_DRY_RUN: env.ORCHESTRATOR_DRY_RUN || !redis,
      });

      const stalled = await detectStalledRunsForResume(db, env.ORCHESTRATOR_STALL_MINUTES);
      if (stalled.length > 0) {
        console.warn(
          `[orchestrator] stall detector: ${stalled.length} open agent_runs older than ${env.ORCHESTRATOR_STALL_MINUTES}m`,
        );

        for (const run of stalled) {
          if (env.ORCHESTRATOR_DRY_RUN || !redis) {
            console.info(
              `[orchestrator] DRY-RUN resume_from_checkpoint convoy=${run.convoyId} role=${run.roleId} run=${run.agentRunId}`,
            );
            continue;
          }

          const job = await enqueueJob(redis, {
            type: "role_dispatch",
            workspaceId: run.workspaceId,
            convoyId: run.convoyId,
            roleId: run.roleId,
            briefId: run.briefId,
            resumeFromCheckpoint: true,
          });
          console.info(
            `[orchestrator] re-dispatched job=${job.id} resume_from_checkpoint run=${run.agentRunId}`,
          );
        }
      }

      if (results.length === 0) {
        console.info("[orchestrator] no active convoys");
      }
    } catch (error) {
      console.error("[orchestrator] tick failed:", error);
    }
  };

  await loop();
  setInterval(loop, env.ORCHESTRATOR_POLL_MS);
}

main().catch((error) => {
  console.error("[orchestrator] fatal:", error);
  process.exit(1);
});