aigen/server/utils/comfyLifecycle.ts

import { execFile, spawn } from 'node:child_process'
import { promisify } from 'node:util'

const execFileAsync = promisify(execFile)

export type ComfyLifecycleState = 'online' | 'booting' | 'busy' | 'starting' | 'offline'

export interface ComfyLifecycleStatus {
  state: ComfyLifecycleState
  message: string
  host: string
  processRunning?: boolean
  queueRunning?: number
  queuePending?: number
}

type StatusFn = (status: ComfyLifecycleStatus) => void

let gate: Promise<unknown> = Promise.resolve()
let lastStartAt = 0

function sleep(ms: number) {
  return new Promise(resolve => setTimeout(resolve, ms))
}

function log(event: string, detail: Record<string, unknown> = {}) {
  console.log(JSON.stringify({
    src: 'comfy-lifecycle',
    event,
    at: new Date().toISOString(),
    ...detail
  }))
}

function settings() {
  const config = useRuntimeConfig()
  return {
    host: getComfyHost(),
    launchCmd: String(config.comfyLaunchCmd || process.env.COMFY_LAUNCH_CMD || '').trim(),
    controlUrl: String(config.comfyControlUrl || process.env.COMFY_CONTROL_URL || '').replace(/\/$/, ''),
    controlToken: String(config.comfyControlToken || process.env.COMFY_CONTROL_TOKEN || ''),
    healthTimeoutMs: 2500,
    bootPollAttempts: 5,
    bootPollDelayMs: 2000,
    startTimeoutMs: 30_000,
    busyWaitMs: Number(process.env.COMFY_BUSY_WAIT_MS || 180_000)
  }
}

export async function checkComfyHttp(timeoutMs = 2500) {
  const host = getComfyHost()
  try {
    const res = await fetch(`${host}/system_stats`, { signal: AbortSignal.timeout(timeoutMs) })
    if (!res.ok) return { ok: false as const, host, status: res.status, stats: null }
    const stats = await res.json().catch(() => null)
    return { ok: true as const, host, status: res.status, stats }
  } catch {
    return { ok: false as const, host, status: 0, stats: null }
  }
}

async function fetchQueue() {
  const host = getComfyHost()
  try {
    const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(2500) })
    if (!res.ok) return { running: 0, pending: 0 }
    const payload = await res.json() as { queue_running?: unknown[]; queue_pending?: unknown[] }
    return {
      running: Array.isArray(payload.queue_running) ? payload.queue_running.length : 0,
      pending: Array.isArray(payload.queue_pending) ? payload.queue_pending.length : 0
    }
  } catch {
    return { running: 0, pending: 0 }
  }
}

async function controlRequest(path: string, method = 'GET') {
  const { controlUrl, controlToken } = settings()
  if (!controlUrl) return null
  try {
    const res = await fetch(`${controlUrl}${path}`, {
      method,
      headers: {
        Accept: 'application/json',
        ...(controlToken ? { Authorization: `Bearer ${controlToken}` } : {})
      },
      signal: AbortSignal.timeout(5000)
    })
    if (!res.ok) return { ok: false, status: res.status }
    return await res.json() as {
      ok?: boolean
      http?: boolean
      process?: boolean
      pid?: number
      started?: boolean
    }
  } catch {
    return null
  }
}

async function findLocalComfyProcess() {
  const platform = process.platform
  try {
    if (platform === 'win32') {
      const { stdout } = await execFileAsync('tasklist', ['/FO', 'CSV', '/NH'], { timeout: 4000, windowsHide: true })
      const hit = stdout.split(/\r?\n/).some(line => /comfy|python/i.test(line) && /comfy/i.test(line))
      if (hit) return true
      const { stdout: verbose } = await execFileAsync('wmic', ['process', 'where', "name='python.exe'", 'get', 'commandline'], { timeout: 4000, windowsHide: true }).catch(() => ({ stdout: '' }))
      return /comfyui|main\.py/i.test(verbose)
    }
    const { stdout } = await execFileAsync('ps', ['-ax', '-o', 'args='], { timeout: 4000 })
    return /comfyui|ComfyUI|main\.py.*8188/i.test(stdout)
  } catch {
    return false
  }
}

async function processRunning() {
  const remote = await controlRequest('/status')
  if (remote?.process) return true
  return findLocalComfyProcess()
}

function spawnLaunchCommand(command: string) {
  log('spawn', { command: command.slice(0, 180) })
  const child = spawn(command, {
    shell: true,
    detached: true,
    stdio: 'ignore',
    windowsHide: true
  })
  child.unref()
}

async function startComfy() {
  const now = Date.now()
  if (now - lastStartAt < 15_000) {
    log('start-skipped', { reason: 'recent-start' })
    return { started: false, reason: 'recent-start' as const }
  }
  lastStartAt = now

  const remote = await controlRequest('/start', 'POST')
  if (remote?.ok || remote?.started) {
    log('start-remote', { pid: remote.pid || null })
    return { started: true, reason: 'remote' as const }
  }

  const { launchCmd } = settings()
  if (launchCmd) {
    spawnLaunchCommand(launchCmd)
    return { started: true, reason: 'launch-cmd' as const }
  }

  if (process.platform === 'win32') {
    const fallback = process.env.LOCALAPPDATA
      ? `"${process.env.LOCALAPPDATA}\\Programs\\Comfy Desktop\\Comfy Desktop.exe"`
      : ''
    if (fallback) {
      spawnLaunchCommand(`start "" ${fallback}`)
      return { started: true, reason: 'comfy-desktop' as const }
    }
  }

  return { started: false, reason: 'no-launcher' as const }
}

async function pollUntilHealthy(attempts: number, delayMs: number, onStatus: StatusFn, state: ComfyLifecycleState, message: string) {
  const host = getComfyHost()
  for (let i = 1; i <= attempts; i++) {
    onStatus({ state, message: `${message} (${i}/${attempts})`, host, processRunning: true })
    const health = await checkComfyHttp()
    if (health.ok) return true
    if (i < attempts) await sleep(delayMs)
  }
  return false
}

async function waitWhileBusy(onStatus: StatusFn) {
  const { busyWaitMs } = settings()
  const host = getComfyHost()
  const started = Date.now()
  while (Date.now() - started < busyWaitMs) {
    const queue = await fetchQueue()
    if (queue.running === 0) return
    onStatus({
      state: 'busy',
      message: `ComfyUI is busy with another prompt (${queue.running} running, ${queue.pending} queued). Waiting...`,
      host,
      queueRunning: queue.running,
      queuePending: queue.pending
    })
    log('busy', queue)
    await sleep(2000)
  }
  throw createError({
    statusCode: 409,
    statusMessage: 'ComfyUI is busy executing another prompt. Try again in a moment.'
  })
}

async function ensureUnlocked(onStatus: StatusFn) {
  const cfg = settings()
  const host = cfg.host
  onStatus({ state: 'offline', message: 'Checking ComfyUI...', host })

  const health = await checkComfyHttp(cfg.healthTimeoutMs)
  if (health.ok) {
    log('http-ok', { host })
    await waitWhileBusy(onStatus)
    onStatus({ state: 'online', message: 'ComfyUI online', host })
    return
  }

  log('http-fail', { host })
  const running = await processRunning()
  if (running) {
    onStatus({
      state: 'booting',
      message: 'ComfyUI process is up but the API is not ready yet. Waiting...',
      host,
      processRunning: true
    })
    const ready = await pollUntilHealthy(cfg.bootPollAttempts, cfg.bootPollDelayMs, onStatus, 'booting', 'Waiting for ComfyUI to finish booting')
    if (ready) {
      await waitWhileBusy(onStatus)
      onStatus({ state: 'online', message: 'ComfyUI online', host, processRunning: true })
      return
    }
    onStatus({
      state: 'starting',
      message: 'ComfyUI is running but the API never came up. Attempting a relaunch...',
      host,
      processRunning: true
    })
  }

  if (!running) {
    onStatus({
      state: 'starting',
      message: 'ComfyUI is offline. Starting the desktop instance...',
      host,
      processRunning: false
    })
  }
  const started = await startComfy()
  if (!started.started) {
    throw createError({
      statusCode: 503,
      statusMessage: 'ComfyUI is offline and no launch command is configured. Start Comfy Desktop, or set COMFY_LAUNCH_CMD / COMFY_CONTROL_URL.'
    })
  }

  const deadline = Date.now() + cfg.startTimeoutMs
  let attempt = 0
  while (Date.now() < deadline) {
    attempt += 1
    onStatus({
      state: 'starting',
      message: `Starting ComfyUI... waiting for /system_stats (${attempt})`,
      host
    })
    const next = await checkComfyHttp()
    if (next.ok) {
      log('started', { attempt, reason: started.reason })
      await waitWhileBusy(onStatus)
      onStatus({ state: 'online', message: 'ComfyUI online', host, processRunning: true })
      return
    }
    await sleep(2000)
  }

  throw createError({
    statusCode: 503,
    statusMessage: 'Started ComfyUI but it did not become healthy within 30 seconds.'
  })
}

export function ensureComfyReady(onStatus: StatusFn = () => undefined) {
  const run = gate.then(() => ensureUnlocked(onStatus))
  gate = run.then(() => undefined, () => undefined)
  return run
}