318 lines
10 KiB
TypeScript
318 lines
10 KiB
TypeScript
import { assertSharedGpu, sharedGpuHeaders, withSharedGpuStart } from '~/server/utils/sharedGpu'
|
|
import { getImageComfyHost, setImageComfyHostOverride, imageComfyConfigured, getBeastImageHost, getSidecarImageHost, withImageComfyHost, type ImageEditBox } from '~/server/utils/imageComfy'
|
|
|
|
export type ImageComfyState = 'online' | 'booting' | 'busy' | 'starting' | 'offline'
|
|
|
|
export interface ImageComfyStatus {
|
|
state: ImageComfyState
|
|
message: string
|
|
host: string
|
|
box?: ImageEditBox
|
|
processRunning?: boolean
|
|
queueRunning?: number
|
|
queuePending?: number
|
|
}
|
|
|
|
export interface ImageEditTarget {
|
|
host: string
|
|
box: ImageEditBox
|
|
label: string
|
|
}
|
|
|
|
type StatusFn = (status: ImageComfyStatus) => void
|
|
|
|
let gate: Promise<unknown> = Promise.resolve()
|
|
let lastStartAt = 0
|
|
|
|
function sleep(ms: number) {
|
|
return new Promise(resolve => setTimeout(resolve, ms))
|
|
}
|
|
|
|
function log(event: string, detail: Record<string, unknown> = {}) {
|
|
console.log(JSON.stringify({
|
|
src: 'image-comfy-lifecycle',
|
|
event,
|
|
at: new Date().toISOString(),
|
|
...detail
|
|
}))
|
|
}
|
|
|
|
function settings() {
|
|
const config = useRuntimeConfig()
|
|
return {
|
|
host: getImageComfyHost(),
|
|
launchCmd: String(config.imageComfyLaunchCmd || process.env.IMAGE_COMFY_LAUNCH_CMD || '').trim(),
|
|
controlUrl: String(config.imageComfyControlUrl || process.env.IMAGE_COMFY_CONTROL_URL || '').replace(/\/$/, ''),
|
|
controlToken: String(config.imageComfyControlToken || process.env.IMAGE_COMFY_CONTROL_TOKEN || ''),
|
|
healthTimeoutMs: 2500,
|
|
bootPollAttempts: Number(process.env.IMAGE_COMFY_BOOT_POLL_ATTEMPTS || 20),
|
|
bootPollDelayMs: 3000,
|
|
startTimeoutMs: Number(process.env.IMAGE_COMFY_START_TIMEOUT_MS || 180_000),
|
|
busyWaitMs: Number(process.env.IMAGE_COMFY_BUSY_WAIT_MS || 180_000)
|
|
}
|
|
}
|
|
|
|
export async function checkImageComfyHttp(timeoutMs = 5000, host?: string) {
|
|
let resolved = host || ''
|
|
try {
|
|
if (!resolved) {
|
|
if (!imageComfyConfigured()) return { ok: false as const, host: '', status: 0, stats: null, timedOut: false }
|
|
resolved = getImageComfyHost()
|
|
}
|
|
if (!resolved) return { ok: false as const, host: '', status: 0, stats: null, timedOut: false }
|
|
const res = await fetch(`${resolved}/system_stats`, { signal: AbortSignal.timeout(timeoutMs) })
|
|
if (!res.ok) return { ok: false as const, host: resolved, status: res.status, stats: null, timedOut: false }
|
|
const stats = await res.json().catch(() => null)
|
|
return { ok: true as const, host: resolved, status: res.status, stats, timedOut: false }
|
|
} catch (error) {
|
|
const timedOut = error instanceof Error && (error.name === 'TimeoutError' || error.name === 'AbortError' || /timeout/i.test(error.message))
|
|
return { ok: false as const, host: resolved, status: 0, stats: null, timedOut }
|
|
}
|
|
}
|
|
|
|
export async function fetchQueueAt(host: string) {
|
|
if (!host) return { running: 0, pending: 0, ok: false }
|
|
try {
|
|
const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(5000) })
|
|
if (!res.ok) return { running: 0, pending: 0, ok: false }
|
|
const payload = await res.json() as { queue_running?: unknown[]; queue_pending?: unknown[] }
|
|
return {
|
|
running: Array.isArray(payload.queue_running) ? payload.queue_running.length : 0,
|
|
pending: Array.isArray(payload.queue_pending) ? payload.queue_pending.length : 0,
|
|
ok: true
|
|
}
|
|
} catch {
|
|
return { running: 0, pending: 0, ok: false }
|
|
}
|
|
}
|
|
|
|
export async function fetchImageQueue() {
|
|
return fetchQueueAt(getImageComfyHost())
|
|
}
|
|
|
|
async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
|
|
const { controlUrl, controlToken } = settings()
|
|
if (!controlUrl) return null
|
|
if (method !== 'GET') await assertSharedGpu()
|
|
try {
|
|
const res = await fetch(`${controlUrl}${path}`, {
|
|
method,
|
|
headers: {
|
|
Accept: 'application/json',
|
|
...(controlToken ? { Authorization: `Bearer ${controlToken}` } : {}),
|
|
...(method !== 'GET' ? sharedGpuHeaders() : {})
|
|
},
|
|
signal: AbortSignal.timeout(timeoutMs)
|
|
})
|
|
if (!res.ok) return { ok: false, status: res.status }
|
|
return await res.json() as {
|
|
ok?: boolean
|
|
http?: boolean
|
|
process?: boolean
|
|
started?: boolean
|
|
already?: boolean
|
|
port?: number | null
|
|
proxyPort?: number | null
|
|
}
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
async function waitWhileBusy(onStatus: StatusFn) {
|
|
const { busyWaitMs, host } = settings()
|
|
const started = Date.now()
|
|
while (Date.now() - started < busyWaitMs) {
|
|
const queue = await fetchImageQueue()
|
|
if (queue.running === 0) return
|
|
onStatus({
|
|
state: 'busy',
|
|
message: `Image ComfyUI is busy with another prompt (${queue.running} running, ${queue.pending} queued). Waiting up to 3 minutes.`,
|
|
host,
|
|
queueRunning: queue.running,
|
|
queuePending: queue.pending
|
|
})
|
|
await sleep(2000)
|
|
}
|
|
throw createError({
|
|
statusCode: 409,
|
|
statusMessage: 'Image ComfyUI is still busy after 3 minutes.',
|
|
data: { code: 'IMAGE_COMFY_BUSY' }
|
|
})
|
|
}
|
|
|
|
function hostWithPort(host: string, port: number) {
|
|
try {
|
|
const url = new URL(host)
|
|
url.port = String(port)
|
|
return url.origin
|
|
} catch {
|
|
return host
|
|
}
|
|
}
|
|
|
|
async function startImageComfy() {
|
|
const now = Date.now()
|
|
if (now - lastStartAt < 15_000) {
|
|
log('start-skipped', { reason: 'recent-start' })
|
|
return { started: false, reason: 'recent-start' as const }
|
|
}
|
|
lastStartAt = now
|
|
const remote = await controlRequest('/start', 'POST', 25_000)
|
|
if (remote?.ok || remote?.started) {
|
|
log('start-remote', { port: remote.port || null })
|
|
return { started: true, reason: 'remote' as const }
|
|
}
|
|
return { started: false, reason: 'no-launcher' as const }
|
|
}
|
|
|
|
async function ensureUnlocked(onStatus: StatusFn) {
|
|
const cfg = settings()
|
|
const host = cfg.host
|
|
onStatus({ state: 'offline', message: 'Checking image ComfyUI...', host })
|
|
|
|
const health = await checkImageComfyHttp(cfg.healthTimeoutMs)
|
|
if (health.ok) {
|
|
await waitWhileBusy(onStatus)
|
|
onStatus({ state: 'online', message: 'Image ComfyUI online', host })
|
|
return
|
|
}
|
|
if (health.timedOut) {
|
|
log('http-busy', { host, timeoutMs: cfg.healthTimeoutMs })
|
|
onStatus({
|
|
state: 'busy',
|
|
message: 'Image ComfyUI is loaded but slow to answer. Not starting another instance.',
|
|
host,
|
|
processRunning: true
|
|
})
|
|
await waitWhileBusy(onStatus)
|
|
onStatus({ state: 'online', message: 'Image ComfyUI online', host, processRunning: true })
|
|
return
|
|
}
|
|
|
|
const remote = await controlRequest('/status')
|
|
if (remote?.http && remote.proxyPort) {
|
|
const proxyUrl = hostWithPort(host, remote.proxyPort)
|
|
const viaProxy = await checkImageComfyHttp(cfg.healthTimeoutMs, proxyUrl)
|
|
if (viaProxy.ok) {
|
|
setImageComfyHostOverride(proxyUrl)
|
|
await waitWhileBusy(onStatus)
|
|
onStatus({ state: 'online', message: 'Image ComfyUI online', host: proxyUrl, processRunning: true })
|
|
return
|
|
}
|
|
}
|
|
|
|
onStatus({
|
|
state: 'starting',
|
|
message: 'Image ComfyUI is offline. Starting the Beast instance...',
|
|
host,
|
|
processRunning: Boolean(remote?.process)
|
|
})
|
|
const started = await startImageComfy()
|
|
if (!started.started && started.reason === 'no-launcher') {
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: 'Image ComfyUI is offline and the Beast host agent did not start it. Confirm COMFY_CONTROL_URL is running.'
|
|
})
|
|
}
|
|
|
|
const deadline = Date.now() + cfg.startTimeoutMs
|
|
let attempt = 0
|
|
while (Date.now() < deadline) {
|
|
attempt += 1
|
|
onStatus({
|
|
state: 'starting',
|
|
message: `Starting image ComfyUI... waiting for /system_stats (${attempt})`,
|
|
host
|
|
})
|
|
const next = await checkImageComfyHttp()
|
|
if (next.ok) {
|
|
await waitWhileBusy(onStatus)
|
|
onStatus({ state: 'online', message: 'Image ComfyUI online', host, processRunning: true })
|
|
return
|
|
}
|
|
await sleep(2000)
|
|
}
|
|
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: 'Started image ComfyUI but the API never answered. Headless Qwen Edit can take a couple of minutes to load.'
|
|
})
|
|
}
|
|
|
|
export function ensureImageComfyReady(onStatus: StatusFn = () => undefined) {
|
|
const run = gate.then(() => withSharedGpuStart(() => ensureUnlocked(onStatus), async () => { await assertSharedGpu() }))
|
|
gate = run.then(() => undefined, () => undefined)
|
|
return run
|
|
}
|
|
|
|
let sidecarGate: Promise<unknown> = Promise.resolve()
|
|
|
|
async function ensureSidecarUnlocked(onStatus: StatusFn) {
|
|
const host = getSidecarImageHost()
|
|
if (!host) {
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: 'Qwen VL is not configured. Set COMFY_HOST or IMAGE_COMFY_HOST to Beast.'
|
|
})
|
|
}
|
|
|
|
onStatus({ state: 'offline', message: 'Checking Beast Qwen VL…', host, box: 'beast' })
|
|
const health = await checkImageComfyHttp(2500, host)
|
|
if (health.ok) {
|
|
onStatus({ state: 'online', message: 'Beast Qwen VL online', host, box: 'beast' })
|
|
return
|
|
}
|
|
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: `Beast ComfyUI is offline (${host}). Recommend needs Qwen VL on that instance and will not start a sidecar.`
|
|
})
|
|
}
|
|
|
|
export function ensureSidecarReady(onStatus: StatusFn = () => undefined) {
|
|
const run = sidecarGate.then(() => ensureSidecarUnlocked(onStatus))
|
|
sidecarGate = run.then(() => undefined, () => undefined)
|
|
return run
|
|
}
|
|
|
|
export async function pickImageEditHost(onStatus: StatusFn = () => undefined): Promise<ImageEditTarget> {
|
|
const beast = getBeastImageHost()
|
|
if (!beast) {
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: 'Beast ComfyUI is not configured. Set COMFY_HOST.'
|
|
})
|
|
}
|
|
|
|
const health = await checkImageComfyHttp(2500, beast)
|
|
if (!health.ok) {
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: 'Beast is offline. Image edit runs on the 5080 only.'
|
|
})
|
|
}
|
|
const queue = await fetchQueueAt(beast)
|
|
if (queue.running === 0) {
|
|
onStatus({
|
|
state: 'online',
|
|
message: 'Beast is free. Editing on this machine.',
|
|
host: beast,
|
|
box: 'beast',
|
|
queueRunning: 0,
|
|
queuePending: queue.pending
|
|
})
|
|
return { host: beast, box: 'beast', label: 'Beast' }
|
|
}
|
|
onStatus({
|
|
state: 'busy',
|
|
message: `Beast is busy (${queue.running} running). Waiting for a free slot...`,
|
|
host: beast,
|
|
box: 'beast',
|
|
queueRunning: queue.running,
|
|
queuePending: queue.pending
|
|
})
|
|
await withImageComfyHost(beast, () => waitWhileBusy(onStatus))
|
|
return { host: beast, box: 'beast', label: 'Beast' }
|
|
}
|