From 55f8cb1def8c2d624480e7e5548b4092830c6ae9 Mon Sep 17 00:00:00 2001 From: Towsty Date: Fri, 28 Aug 2026 20:01:05 -0500 Subject: [PATCH] Do not mark Beast offline when /system_stats times out during a render. Co-authored-by: Cursor --- pages/index.vue | 27 ++++++++++++++++++--------- server/api/health.get.ts | 26 ++++++++++++++------------ server/api/studio-queue.get.ts | 3 +-- server/utils/comfyLifecycle.ts | 13 +++++++------ server/utils/imageComfyLifecycle.ts | 13 +++++++------ 5 files changed, 47 insertions(+), 35 deletions(-) diff --git a/pages/index.vue b/pages/index.vue index 0994a18..556639f 100644 --- a/pages/index.vue +++ b/pages/index.vue @@ -2742,6 +2742,7 @@ function scheduleLibraryRefresh() { }, 1500) } +let healthFailStreak = 0 function applyComfyHealth(health: { comfy?: { ok?: boolean; busy?: boolean } imageComfy?: { @@ -2753,19 +2754,27 @@ function applyComfyHealth(health: { sidecar?: { ok?: boolean; busy?: boolean; host?: string; configured?: boolean } } } | null | undefined) { - comfyOk.value = Boolean(health?.comfy?.ok) - comfyBusy.value = Boolean(health?.comfy?.ok && health?.comfy?.busy) + const nextComfyOk = Boolean(health?.comfy?.ok) + const nextImageOk = Boolean(health?.imageComfy?.ok) + const reachable = nextComfyOk || nextImageOk + if (reachable) healthFailStreak = 0 + else healthFailStreak += 1 + const keepLast = !reachable && (healthFailStreak < 3 || videoBusy.value || editBusy.value) + if (!keepLast) { + comfyOk.value = nextComfyOk + imageComfyOk.value = nextImageOk + imageBeastOk.value = Boolean(health?.imageComfy?.beast?.ok) + imageSidecarOk.value = Boolean(health?.imageComfy?.sidecar?.ok) + } + comfyBusy.value = Boolean((keepLast ? comfyOk.value : nextComfyOk) && health?.comfy?.busy) imageComfyConfigured.value = Boolean(health?.imageComfy?.configured) - imageComfyOk.value = Boolean(health?.imageComfy?.ok) - imageComfyBusy.value = Boolean(health?.imageComfy?.ok && health?.imageComfy?.busy) - imageComfyHost.value = String(health?.imageComfy?.host || '') - imageBeastOk.value = Boolean(health?.imageComfy?.beast?.ok) + imageComfyBusy.value = Boolean((keepLast ? imageComfyOk.value : nextImageOk) && health?.imageComfy?.busy) + imageComfyHost.value = String(health?.imageComfy?.host || imageComfyHost.value || '') imageBeastBusy.value = Boolean(health?.imageComfy?.beast?.busy) - imageBeastHost.value = String(health?.imageComfy?.beast?.host || '') - imageSidecarOk.value = Boolean(health?.imageComfy?.sidecar?.ok) + imageBeastHost.value = String(health?.imageComfy?.beast?.host || imageBeastHost.value || '') imageSidecarBusy.value = Boolean(health?.imageComfy?.sidecar?.busy) imageSidecarConfigured.value = Boolean(health?.imageComfy?.sidecar?.configured) - imageSidecarHost.value = String(health?.imageComfy?.sidecar?.host || '') + imageSidecarHost.value = String(health?.imageComfy?.sidecar?.host || imageSidecarHost.value || '') } async function pollLoras() { diff --git a/server/api/health.get.ts b/server/api/health.get.ts index 11bc9f5..c69fe66 100644 --- a/server/api/health.get.ts +++ b/server/api/health.get.ts @@ -13,21 +13,23 @@ export default defineEventHandler(async () => { videoConfigured ? checkComfyHttp().catch(() => ({ ok: false, host: '', stats: null })) : Promise.resolve({ ok: false, host: '', stats: null }), - videoConfigured ? fetchQueue() : Promise.resolve({ running: 0, pending: 0 }), + videoConfigured ? fetchQueue() : Promise.resolve({ running: 0, pending: 0, ok: false }), beastHost - ? checkImageComfyHttp(2500, beastHost).catch(() => ({ ok: false, host: beastHost, stats: null })) + ? checkImageComfyHttp(5000, beastHost).catch(() => ({ ok: false, host: beastHost, stats: null })) : Promise.resolve({ ok: false, host: beastHost, stats: null }), - beastHost ? fetchQueueAt(beastHost) : Promise.resolve({ running: 0, pending: 0 }), + beastHost ? fetchQueueAt(beastHost) : Promise.resolve({ running: 0, pending: 0, ok: false }), promptHost && !promptIsBeast - ? checkImageComfyHttp(2500, promptHost).catch(() => ({ ok: false, host: promptHost, stats: null })) + ? checkImageComfyHttp(5000, promptHost).catch(() => ({ ok: false, host: promptHost, stats: null })) : Promise.resolve({ ok: false, host: promptHost, stats: null }), - promptHost && !promptIsBeast ? fetchQueueAt(promptHost) : Promise.resolve({ running: 0, pending: 0 }) + promptHost && !promptIsBeast ? fetchQueueAt(promptHost) : Promise.resolve({ running: 0, pending: 0, ok: false }) ]) - const busy = Boolean(comfy.ok && queue.running > 0) - const beastBusy = Boolean(beast.ok && beastQueue.running > 0) - const promptOk = promptIsBeast ? Boolean(beast.ok || comfy.ok) : Boolean(prompt.ok) - const promptBusy = promptIsBeast ? (beastBusy || busy) : Boolean(prompt.ok && promptQueue.running > 0) - const imageOk = Boolean(beast.ok || comfy.ok) + const comfyReachable = Boolean(comfy.ok || queue.ok) + const beastReachable = Boolean(beast.ok || beastQueue.ok || comfyReachable) + const busy = Boolean(comfyReachable && queue.running > 0) + const beastBusy = Boolean(beastReachable && beastQueue.running > 0) + const promptOk = promptIsBeast ? beastReachable : Boolean(prompt.ok || promptQueue.ok) + const promptBusy = promptIsBeast ? (beastBusy || busy) : Boolean(promptOk && promptQueue.running > 0) + const imageOk = Boolean(beastReachable || comfyReachable) const imageBusy = Boolean(imageConfigured && (beastBusy || busy)) const preferredHost = beastHost || comfy.host || '' const promptQueueRunning = promptIsBeast ? ((beastQueue.running || queue.running) || 0) : (promptQueue.running || 0) @@ -36,7 +38,7 @@ export default defineEventHandler(async () => { ok: true, service: useRuntimeConfig().public.instanceName || 'aigen', comfy: { - ok: comfy.ok, + ok: comfyReachable, busy, host: comfy.host || '', queueRunning: queue.running, @@ -54,7 +56,7 @@ export default defineEventHandler(async () => { beast: { label: 'Beast', configured: Boolean(beastHost), - ok: Boolean(beast.ok || comfy.ok), + ok: beastReachable, busy: beastBusy || busy, host: preferredHost }, diff --git a/server/api/studio-queue.get.ts b/server/api/studio-queue.get.ts index c2f5ba8..1b4b0c3 100644 --- a/server/api/studio-queue.get.ts +++ b/server/api/studio-queue.get.ts @@ -1,5 +1,5 @@ import { listShotQueues, summarizeQueue } from '~/server/utils/shotQueue' -import { kickStudioQueue, listStudioJobs, summarizeStudioJob } from '~/server/utils/studioQueue' +import { listStudioJobs, summarizeStudioJob } from '~/server/utils/studioQueue' export default defineEventHandler((event) => { const { owner } = assertLibraryOwner(event) @@ -51,7 +51,6 @@ export default defineEventHandler((event) => { .filter(queue => !claimed.has(queue.id) && queue.status !== 'complete') .map(summarizeQueue) : [] - if (jobs.some(job => job.status === 'waiting')) void kickStudioQueue() return { jobs, orphans, diff --git a/server/utils/comfyLifecycle.ts b/server/utils/comfyLifecycle.ts index 3ad1ebd..f79e1da 100644 --- a/server/utils/comfyLifecycle.ts +++ b/server/utils/comfyLifecycle.ts @@ -48,7 +48,7 @@ function settings() { } } -export async function checkComfyHttp(timeoutMs = 2500, host?: string) { +export async function checkComfyHttp(timeoutMs = 5000, host?: string) { let resolved = host || '' try { if (!resolved) { @@ -65,18 +65,19 @@ export async function checkComfyHttp(timeoutMs = 2500, host?: string) { } export async function fetchQueue() { - if (!comfyConfigured()) return { running: 0, pending: 0 } + if (!comfyConfigured()) return { running: 0, pending: 0, ok: false } try { const host = getComfyHost() - const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(2500) }) - if (!res.ok) return { running: 0, pending: 0 } + const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(5000) }) + if (!res.ok) return { running: 0, pending: 0, ok: false } const payload = await res.json() as { queue_running?: unknown[]; queue_pending?: unknown[] } return { running: Array.isArray(payload.queue_running) ? payload.queue_running.length : 0, - pending: Array.isArray(payload.queue_pending) ? payload.queue_pending.length : 0 + pending: Array.isArray(payload.queue_pending) ? payload.queue_pending.length : 0, + ok: true } } catch { - return { running: 0, pending: 0 } + return { running: 0, pending: 0, ok: false } } } diff --git a/server/utils/imageComfyLifecycle.ts b/server/utils/imageComfyLifecycle.ts index 10de690..83b7412 100644 --- a/server/utils/imageComfyLifecycle.ts +++ b/server/utils/imageComfyLifecycle.ts @@ -51,7 +51,7 @@ function settings() { } } -export async function checkImageComfyHttp(timeoutMs = 2500, host?: string) { +export async function checkImageComfyHttp(timeoutMs = 5000, host?: string) { let resolved = host || '' try { if (!resolved) { @@ -69,17 +69,18 @@ export async function checkImageComfyHttp(timeoutMs = 2500, host?: string) { } export async function fetchQueueAt(host: string) { - if (!host) return { running: 0, pending: 0 } + if (!host) return { running: 0, pending: 0, ok: false } try { - const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(2500) }) - if (!res.ok) return { running: 0, pending: 0 } + const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(5000) }) + if (!res.ok) return { running: 0, pending: 0, ok: false } const payload = await res.json() as { queue_running?: unknown[]; queue_pending?: unknown[] } return { running: Array.isArray(payload.queue_running) ? payload.queue_running.length : 0, - pending: Array.isArray(payload.queue_pending) ? payload.queue_pending.length : 0 + pending: Array.isArray(payload.queue_pending) ? payload.queue_pending.length : 0, + ok: true } } catch { - return { running: 0, pending: 0 } + return { running: 0, pending: 0, ok: false } } }