aigen/server/utils/comfyLifecycle.ts

import { execFile, spawn } from 'node:child_process'
import { promisify } from 'node:util'
import { getComfyHost, setComfyHostOverride, comfyConfigured } from '~/server/utils/comfy'

const execFileAsync = promisify(execFile)

export type ComfyLifecycleState = 'online' | 'booting' | 'busy' | 'starting' | 'offline'

export interface ComfyLifecycleStatus {
  state: ComfyLifecycleState
  message: string
  host: string
  processRunning?: boolean
  queueRunning?: number
  queuePending?: number
}

type StatusFn = (status: ComfyLifecycleStatus) => void

let gate: Promise<unknown> = Promise.resolve()
let lastStartAt = 0

function sleep(ms: number) {
  return new Promise(resolve => setTimeout(resolve, ms))
}

function log(event: string, detail: Record<string, unknown> = {}) {
  console.log(JSON.stringify({
    src: 'comfy-lifecycle',
    event,
    at: new Date().toISOString(),
    ...detail
  }))
}

function settings() {
  const config = useRuntimeConfig()
  return {
    host: getComfyHost(),
    launchCmd: String(config.comfyLaunchCmd || process.env.COMFY_LAUNCH_CMD || '').trim(),
    controlUrl: String(config.comfyControlUrl || process.env.COMFY_CONTROL_URL || '').replace(/\/$/, ''),
    controlToken: String(config.comfyControlToken || process.env.COMFY_CONTROL_TOKEN || ''),
    healthTimeoutMs: 2500,
    bootPollAttempts: Number(process.env.COMFY_BOOT_POLL_ATTEMPTS || 20),
    bootPollDelayMs: 3000,
    startTimeoutMs: Number(process.env.COMFY_START_TIMEOUT_MS || 180_000),
    busyWaitMs: Number(process.env.COMFY_BUSY_WAIT_MS || 180_000)
  }
}

export async function checkComfyHttp(timeoutMs = 2500, host?: string) {
  let resolved = host || ''
  try {
    if (!resolved) {
      if (!comfyConfigured()) return { ok: false as const, host: '', status: 0, stats: null }
      resolved = getComfyHost()
    }
    const res = await fetch(`${resolved}/system_stats`, { signal: AbortSignal.timeout(timeoutMs) })
    if (!res.ok) return { ok: false as const, host: resolved, status: res.status, stats: null }
    const stats = await res.json().catch(() => null)
    return { ok: true as const, host: resolved, status: res.status, stats }
  } catch {
    return { ok: false as const, host: resolved, status: 0, stats: null }
  }
}

export async function fetchQueue() {
  if (!comfyConfigured()) return { running: 0, pending: 0 }
  try {
    const host = getComfyHost()
    const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(2500) })
    if (!res.ok) return { running: 0, pending: 0 }
    const payload = await res.json() as { queue_running?: unknown[]; queue_pending?: unknown[] }
    return {
      running: Array.isArray(payload.queue_running) ? payload.queue_running.length : 0,
      pending: Array.isArray(payload.queue_pending) ? payload.queue_pending.length : 0
    }
  } catch {
    return { running: 0, pending: 0 }
  }
}

async function controlRequest(path: string, method = 'GET') {
  const { controlUrl, controlToken } = settings()
  if (!controlUrl) return null
  try {
    const res = await fetch(`${controlUrl}${path}`, {
      method,
      headers: {
        Accept: 'application/json',
        ...(controlToken ? { Authorization: `Bearer ${controlToken}` } : {})
      },
      signal: AbortSignal.timeout(5000)
    })
    if (!res.ok) return { ok: false, status: res.status }
    return await res.json() as {
      ok?: boolean
      http?: boolean
      process?: boolean
      pid?: number
      started?: boolean
      already?: boolean
      port?: number | null
      proxyPort?: number | null
    }
  } catch {
    return null
  }
}

async function findLocalComfyProcess() {
  const platform = process.platform
  try {
    if (platform === 'win32') {
      const { stdout } = await execFileAsync('tasklist', ['/FO', 'CSV', '/NH'], { timeout: 4000, windowsHide: true })
      const hit = stdout.split(/\r?\n/).some(line => /comfy|python/i.test(line) && /comfy/i.test(line))
      if (hit) return true
      const { stdout: verbose } = await execFileAsync('wmic', ['process', 'where', "name='python.exe'", 'get', 'commandline'], { timeout: 4000, windowsHide: true }).catch(() => ({ stdout: '' }))
      return /comfyui|main\.py/i.test(verbose)
    }
    const { stdout } = await execFileAsync('ps', ['-ax', '-o', 'args='], { timeout: 4000 })
    return /comfyui|ComfyUI|main\.py.*8188/i.test(stdout)
  } catch {
    return false
  }
}

async function processRunning() {
  const remote = await controlRequest('/status')
  if (remote?.process) return true
  return findLocalComfyProcess()
}

function spawnLaunchCommand(command: string) {
  log('spawn', { command: command.slice(0, 180) })
  const child = spawn(command, {
    shell: true,
    detached: true,
    stdio: 'ignore',
    windowsHide: true
  })
  child.unref()
}

async function startComfy() {
  const now = Date.now()
  if (now - lastStartAt < 15_000) {
    log('start-skipped', { reason: 'recent-start' })
    return { started: false, reason: 'recent-start' as const }
  }
  lastStartAt = now

  const remote = await controlRequest('/start', 'POST')
  if (remote?.ok || remote?.started) {
    log('start-remote', { pid: remote.pid || null })
    return { started: true, reason: 'remote' as const }
  }

  const { launchCmd } = settings()
  if (launchCmd) {
    spawnLaunchCommand(launchCmd)
    return { started: true, reason: 'launch-cmd' as const }
  }

  if (process.platform === 'win32') {
    const fallback = process.env.LOCALAPPDATA
      ? `"${process.env.LOCALAPPDATA}\\Programs\\Comfy Desktop\\Comfy Desktop.exe"`
      : ''
    if (fallback) {
      spawnLaunchCommand(`start "" ${fallback}`)
      return { started: true, reason: 'comfy-desktop' as const }
    }
  }

  return { started: false, reason: 'no-launcher' as const }
}

async function pollUntilHealthy(attempts: number, delayMs: number, onStatus: StatusFn, state: ComfyLifecycleState, message: string) {
  const host = getComfyHost()
  for (let i = 1; i <= attempts; i++) {
    onStatus({ state, message: `${message} (${i}/${attempts})`, host, processRunning: true })
    const health = await checkComfyHttp()
    if (health.ok) return true
    if (i < attempts) await sleep(delayMs)
  }
  return false
}

async function waitWhileBusy(onStatus: StatusFn) {
  const { busyWaitMs } = settings()
  const host = getComfyHost()
  const started = Date.now()
  while (Date.now() - started < busyWaitMs) {
    const queue = await fetchQueue()
    if (queue.running === 0) return
    onStatus({
      state: 'busy',
      message: `ComfyUI is busy with another prompt (${queue.running} running, ${queue.pending} queued). Waiting up to 3 minutes, then this job will be saved.`,
      host,
      queueRunning: queue.running,
      queuePending: queue.pending
    })
    log('busy', queue)
    await sleep(2000)
  }
  throw createError({
    statusCode: 409,
    statusMessage: 'ComfyUI is still busy after 3 minutes. Your prompt, still, and settings were saved.',
    data: { code: 'COMFY_BUSY' }
  })
}

function hostWithPort(host: string, port: number) {
  try {
    const url = new URL(host)
    url.port = String(port)
    return url.origin
  } catch {
    return host
  }
}

async function ensureUnlocked(onStatus: StatusFn) {
  const cfg = settings()
  const host = cfg.host
  onStatus({ state: 'offline', message: 'Checking ComfyUI...', host })

  const health = await checkComfyHttp(cfg.healthTimeoutMs)
  if (health.ok) {
    log('http-ok', { host })
    await waitWhileBusy(onStatus)
    onStatus({ state: 'online', message: 'ComfyUI online', host })
    return
  }

  log('http-fail', { host })
  const remote = await controlRequest('/status')
  if (remote?.http && remote.proxyPort) {
    const proxyUrl = hostWithPort(host, remote.proxyPort)
    onStatus({
      state: 'booting',
      message: `Configured Comfy port is blocked. Trying the desktop LAN proxy on ${remote.proxyPort}...`,
      host: proxyUrl,
      processRunning: true
    })
    const viaProxy = await checkComfyHttp(cfg.healthTimeoutMs, proxyUrl)
    if (viaProxy.ok) {
      setComfyHostOverride(proxyUrl)
      log('http-ok-proxy', { host: proxyUrl, localPort: remote.port || null })
      await waitWhileBusy(onStatus)
      onStatus({ state: 'online', message: 'ComfyUI online', host: proxyUrl, processRunning: true })
      return
    }
  }
  if (remote?.http) {
    const localPort = remote.port ? ` Port ${remote.port} is healthy on the desktop.` : ''
    throw createError({
      statusCode: 503,
      statusMessage: `Comfy Desktop is open and the local API is up, but this server cannot reach ${host}.${localPort} Open the MiniMax instance “ComfyUI (1)” and wait until the graph editor loads — the instance picker is not enough.`
    })
  }
  const running = await processRunning()
  if (running) {
    onStatus({
      state: 'booting',
      message: 'ComfyUI process is up but the API is not ready yet. Waiting...',
      host,
      processRunning: true
    })
    const ready = await pollUntilHealthy(cfg.bootPollAttempts, cfg.bootPollDelayMs, onStatus, 'booting', 'Waiting for ComfyUI to finish booting')
    if (ready) {
      await waitWhileBusy(onStatus)
      onStatus({ state: 'online', message: 'ComfyUI online', host, processRunning: true })
      return
    }
    onStatus({
      state: 'starting',
      message: 'ComfyUI is running but the API never came up. Attempting a relaunch...',
      host,
      processRunning: true
    })
  }

  if (!running) {
    onStatus({
      state: 'starting',
      message: 'ComfyUI is offline. Starting the desktop instance...',
      host,
      processRunning: false
    })
  }
  const started = await startComfy()
  if (!started.started) {
    throw createError({
      statusCode: 503,
      statusMessage: 'ComfyUI is offline and no launch command is configured. Start Comfy Desktop, or set COMFY_LAUNCH_CMD / COMFY_CONTROL_URL.'
    })
  }

  const deadline = Date.now() + cfg.startTimeoutMs
  let attempt = 0
  while (Date.now() < deadline) {
    attempt += 1
    onStatus({
      state: 'starting',
      message: `Starting ComfyUI... waiting for /system_stats (${attempt})`,
      host
    })
    const next = await checkComfyHttp()
    if (next.ok) {
      log('started', { attempt, reason: started.reason })
      await waitWhileBusy(onStatus)
      onStatus({ state: 'online', message: 'ComfyUI online', host, processRunning: true })
      return
    }
    await sleep(2000)
  }

  throw createError({
    statusCode: 503,
    statusMessage: 'Started Comfy Desktop but the API never answered. If the instance picker is showing, click “ComfyUI (1)” (MiniMax) and wait until the graph editor is loaded. Cold start can take a couple of minutes.'
  })
}

export function ensureComfyReady(onStatus: StatusFn = () => undefined) {
  const run = gate.then(() => ensureUnlocked(onStatus))
  gate = run.then(() => undefined, () => undefined)
  return run
}