Fix generation progress getting stuck at 1% after Comfy finishes.

Poll Comfy history and the job snapshot so the browser still gets the video when the live stream is buffered or the websocket misses completion.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-08-25 18:23:23 -05:00
co-authored by Cursor
parent ca51fddea3
commit c4e6af9f77
8 changed files with 326 additions and 182 deletions
+9 -1
View File
@@ -38,6 +38,14 @@ export default defineNuxtConfig({
} }
}, },
nitro: { nitro: {
preset: 'node-server' preset: 'node-server',
routeRules: {
'/api/generate/**': {
headers: {
'Cache-Control': 'no-cache, no-store, no-transform',
'X-Accel-Buffering': 'no'
}
}
}
} }
}) })
+61 -25
View File
@@ -365,6 +365,8 @@ const currentPassword = ref('')
const nextPassword = ref('') const nextPassword = ref('')
let timer: ReturnType<typeof setInterval> | null = null let timer: ReturnType<typeof setInterval> | null = null
let source: EventSource | null = null let source: EventSource | null = null
let pollTimer: ReturnType<typeof setInterval> | null = null
let settledUi = false
const elapsedLabel = computed(() => { const elapsedLabel = computed(() => {
const seconds = Math.floor(elapsedMs.value / 1000) const seconds = Math.floor(elapsedMs.value / 1000)
@@ -593,6 +595,7 @@ async function generate() {
return return
} }
busy.value = true busy.value = true
settledUi = false
progress.value = 1 progress.value = 1
videoUrl.value = '' videoUrl.value = ''
awaitingReveal.value = false awaitingReveal.value = false
@@ -629,37 +632,70 @@ async function generate() {
} }
} }
function listen(id: string, hidden: boolean) { function stopListen() {
source?.close() source?.close()
source = null
if (pollTimer) {
clearInterval(pollTimer)
pollTimer = null
}
}
function applyJobEvent(payload: Record<string, any>, hidden: boolean) {
if (payload.message) statusMessage.value = payload.message
if (typeof payload.progress === 'number') progress.value = payload.progress
if (typeof payload.step === 'number') step.value = payload.step
if (typeof payload.maxStep === 'number') maxStep.value = payload.maxStep
const failed = payload.type === 'error' || payload.status === 'error' || payload.status === 'cancelled'
const complete = payload.type === 'complete' || payload.status === 'complete'
if (settledUi) return
if (complete && (payload.clipId || payload.filename)) {
settledUi = true
const clipUrl = payload.clipId
? `/api/library/clips/${payload.clipId}/video`
: `/api/media?filename=${encodeURIComponent(payload.filename)}&subfolder=${encodeURIComponent(payload.subfolder || 'video')}&type=${encodeURIComponent(payload.mediaType || 'output')}`
videoUrl.value = clipUrl
awaitingReveal.value = Boolean(payload.hideThumbnail || hidden)
busy.value = false
stopTimer()
stopListen()
loadLibrary().catch(() => null)
return
}
if (failed) {
settledUi = true
busy.value = false
stopTimer()
stopListen()
toast(payload.error || payload.message || 'Generation failed')
}
}
function listen(id: string, hidden: boolean) {
stopListen()
source = new EventSource(`/api/generate/${id}/stream`) source = new EventSource(`/api/generate/${id}/stream`)
source.onmessage = (event) => { source.onmessage = (event) => {
const payload = JSON.parse(event.data) applyJobEvent(JSON.parse(event.data), hidden)
if (payload.message) statusMessage.value = payload.message }
if (typeof payload.progress === 'number') progress.value = payload.progress const tick = async () => {
if (typeof payload.step === 'number') step.value = payload.step if (settledUi) {
if (typeof payload.maxStep === 'number') maxStep.value = payload.maxStep stopListen()
if (payload.type === 'complete') { return
const clipUrl = payload.clipId
? `/api/library/clips/${payload.clipId}/video`
: `/api/media?filename=${encodeURIComponent(payload.filename)}&subfolder=${encodeURIComponent(payload.subfolder || 'video')}&type=${encodeURIComponent(payload.mediaType || 'output')}`
videoUrl.value = clipUrl
awaitingReveal.value = Boolean(payload.hideThumbnail || hidden)
busy.value = false
stopTimer()
source?.close()
loadLibrary().catch(() => null)
} }
if (payload.type === 'error') { try {
busy.value = false const snap = await $fetch<Record<string, any>>(`/api/generate/${id}`)
stopTimer() applyJobEvent(snap, hidden)
source?.close() } catch {
toast(payload.error || payload.message || 'Generation failed') // Job snapshot can 404 briefly after a restart; keep polling while busy.
} }
} }
source.onerror = () => { void tick()
if (!busy.value) return pollTimer = setInterval(() => {
toast('Progress stream disconnected. The ComfyUI job may still be running.') void tick()
} }, 1000)
} }
async function cancelJob() { async function cancelJob() {
+3 -1
View File
@@ -95,9 +95,11 @@ async function runGeneration(
length: number length: number
} }
) { ) {
const done = watchComfyJob(job)
job.status = 'uploading' job.status = 'uploading'
const uploaded = await uploadImage(params.image) const uploaded = await uploadImage(params.image)
emitJob(job, { type: 'status', message: 'Queueing MiniMax H3 job...', progress: 6 }) emitJob(job, { type: 'status', message: 'Queueing MiniMax H3 job...', progress: 6 })
await waitForComfySocket(job, 4000)
const graph = buildWorkflow({ const graph = buildWorkflow({
prompt: params.prompt, prompt: params.prompt,
@@ -114,5 +116,5 @@ async function runGeneration(
job.promptId = queued.prompt_id job.promptId = queued.prompt_id
job.status = 'running' job.status = 'running'
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 }) emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
await watchComfyJob(job) await done
} }
+8
View File
@@ -0,0 +1,8 @@
export default defineEventHandler((event) => {
const id = getRouterParam(event, 'id')
const job = id ? getJob(id) : undefined
if (!job) {
throw createError({ statusCode: 404, statusMessage: 'Job not found' })
}
return jobSnapshot(job)
})
+12 -11
View File
@@ -5,22 +5,18 @@ export default defineEventHandler(async (event) => {
throw createError({ statusCode: 404, statusMessage: 'Job not found' }) throw createError({ statusCode: 404, statusMessage: 'Job not found' })
} }
setResponseHeaders(event, {
'Cache-Control': 'no-cache, no-store, no-transform',
'X-Accel-Buffering': 'no',
Connection: 'keep-alive'
})
const stream = createEventStream(event) const stream = createEventStream(event)
const send = async (payload: unknown) => { const send = async (payload: unknown) => {
await stream.push(JSON.stringify(payload)) await stream.push(JSON.stringify(payload))
} }
await send({ await send(jobSnapshot(job))
type: 'hello',
status: job.status,
message: job.message,
progress: job.progress,
step: job.step,
maxStep: job.maxStep,
promptId: job.promptId,
elapsedMs: Date.now() - job.startedAt
})
for (const past of job.events) { for (const past of job.events) {
await send(past) await send(past)
} }
@@ -33,7 +29,12 @@ export default defineEventHandler(async (event) => {
}) })
}) })
const ping = setInterval(() => {
void send(jobSnapshot(job))
}, 1000)
stream.onClosed(() => { stream.onClosed(() => {
clearInterval(ping)
unsubscribe() unsubscribe()
}) })
+52 -17
View File
@@ -68,28 +68,63 @@ export async function fetchHistory(promptId: string) {
return (await res.json()) as Record<string, unknown> return (await res.json()) as Record<string, unknown>
} }
export function extractVideo(history: Record<string, unknown> | null, promptId: string) { function isVideoFile(item: { filename?: string; format?: string } | null | undefined) {
if (!history) return null if (!item) return false
const entry = (history[promptId] || Object.values(history)[0]) as { outputs?: Record<string, Record<string, unknown>> } | undefined const name = String(item.filename || '').toLowerCase()
const outputs = entry?.outputs || {} const format = String(item.format || '').toLowerCase()
for (const node of Object.values(outputs)) { return name.endsWith('.mp4') || name.endsWith('.webm') || format.includes('video') || format.includes('mp4')
for (const key of ['videos', 'gifs', 'images', 'files']) { }
const items = node[key]
if (!Array.isArray(items)) continue function normalizeVideo(item: { filename?: string; subfolder?: string; type?: string }) {
const match = items.find((item: { filename?: string }) => String(item.filename || '').toLowerCase().endsWith('.mp4')) return {
|| items[0] filename: String(item.filename),
if (match?.filename) { subfolder: String(item.subfolder || ''),
return { type: String(item.type || 'output')
filename: String(match.filename), }
subfolder: String(match.subfolder || 'video'), }
type: String(match.type || 'output')
} function findVideo(value: unknown, depth = 0): { filename: string; subfolder: string; type: string } | null {
} if (!value || typeof value !== 'object' || depth > 8) return null
if (Array.isArray(value)) {
const files = value.filter((item): item is { filename?: string; format?: string; subfolder?: string; type?: string } => Boolean(item && typeof item === 'object'))
const match = files.find(item => String(item.filename || '').toLowerCase().endsWith('.mp4'))
|| files.find(isVideoFile)
if (match?.filename) return normalizeVideo(match)
for (const item of value) {
const nested = findVideo(item, depth + 1)
if (nested) return nested
} }
return null
}
const record = value as { filename?: string; format?: string; subfolder?: string; type?: string }
if (record.filename && isVideoFile(record)) return normalizeVideo(record)
for (const nested of Object.values(value as Record<string, unknown>)) {
const found = findVideo(nested, depth + 1)
if (found) return found
} }
return null return null
} }
export function extractVideo(history: Record<string, unknown> | null, promptId: string) {
if (!history) return null
const entry = (history[promptId] || Object.values(history)[0]) as { outputs?: Record<string, unknown> } | undefined
return findVideo(entry?.outputs || {})
}
export function inspectHistory(history: Record<string, unknown> | null, promptId: string) {
const video = extractVideo(history, promptId)
if (!history) return { video: null, completed: false, error: null as string | null }
const entry = (history[promptId] || Object.values(history)[0]) as {
status?: { status_str?: string; completed?: boolean; messages?: unknown[] }
} | undefined
const status = entry?.status
const error = status?.status_str === 'error'
? 'ComfyUI reported an execution error'
: null
const completed = Boolean(video) || status?.completed === true || status?.status_str === 'success'
return { video, completed, error }
}
export async function probeComfy() { export async function probeComfy() {
try { try {
const res = await fetch(`${comfyBase()}/system_stats`, { signal: AbortSignal.timeout(2500) }) const res = await fetch(`${comfyBase()}/system_stats`, { signal: AbortSignal.timeout(2500) })
+20
View File
@@ -42,6 +42,7 @@ export interface Job {
thumb?: Buffer thumb?: Buffer
} }
error?: string error?: string
socketReady?: boolean
events: JobEvent[] events: JobEvent[]
listeners: Set<(event: JobEvent) => void> listeners: Set<(event: JobEvent) => void>
} }
@@ -74,6 +75,25 @@ export function getJob(id: string) {
return jobs.get(id) return jobs.get(id)
} }
export function jobSnapshot(job: Job) {
return {
type: 'snapshot' as const,
status: job.status,
message: job.message,
progress: job.progress,
step: job.step,
maxStep: job.maxStep,
promptId: job.promptId,
elapsedMs: Date.now() - job.startedAt,
filename: job.video?.filename,
subfolder: job.video?.subfolder,
mediaType: job.video?.type,
clipId: job.clipId,
hideThumbnail: job.hideThumbnail,
error: job.error
}
}
export function emitJob(job: Job, event: JobEvent) { export function emitJob(job: Job, event: JobEvent) {
if (event.message) job.message = event.message if (event.message) job.message = event.message
if (typeof event.progress === 'number') job.progress = event.progress if (typeof event.progress === 'number') job.progress = event.progress
+161 -127
View File
@@ -3,7 +3,7 @@ import type { Job } from '~/server/utils/jobs'
function classifyError(message: string) { function classifyError(message: string) {
const lower = message.toLowerCase() const lower = message.toLowerCase()
if (lower.includes('out of memory') || lower.includes('cuda') && lower.includes('alloc') || lower.includes('vram')) { if (lower.includes('out of memory') || (lower.includes('cuda') && lower.includes('alloc')) || lower.includes('vram')) {
return 'ComfyUI VRAM allocation failed. Try Turbo (8 steps) or a smaller aspect ratio.' return 'ComfyUI VRAM allocation failed. Try Turbo (8 steps) or a smaller aspect ratio.'
} }
if (lower.includes('econnrefused') || lower.includes('unreachable') || lower.includes('fetch failed')) { if (lower.includes('econnrefused') || lower.includes('unreachable') || lower.includes('fetch failed')) {
@@ -15,154 +15,188 @@ function classifyError(message: string) {
return message return message
} }
export async function watchComfyJob(job: Job) { function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms))
}
export function watchComfyJob(job: Job): Promise<void> {
const ws = new WebSocket(comfyWsUrl(job.clientId)) const ws = new WebSocket(comfyWsUrl(job.clientId))
let settled = false let settled = false
let pollTimer: ReturnType<typeof setInterval> | null = null
let timeout: ReturnType<typeof setTimeout> | null = null
const finish = async (error?: string) => { return new Promise((resolve) => {
if (settled) return const cleanup = () => {
settled = true if (timeout) clearTimeout(timeout)
try { ws.close() } catch { /* ignore */ } if (pollTimer) clearInterval(pollTimer)
if (error) { timeout = null
job.status = job.status === 'cancelled' ? 'cancelled' : 'error' pollTimer = null
job.error = classifyError(error) try { ws.close() } catch { /* ignore */ }
emitJob(job, { type: 'error', error: job.error, message: job.error })
return
} }
emitJob(job, { type: 'status', message: 'Fetching output...', progress: 96 })
const history = await fetchHistory(job.promptId || '') const finish = async (error?: string) => {
const video = extractVideo(history, job.promptId || '') if (settled) return
if (!video) { settled = true
job.status = 'error' cleanup()
job.error = 'Workflow finished but no MP4 was found in ComfyUI history.'
emitJob(job, { type: 'error', error: job.error, message: job.error })
return
}
job.video = video
if (job.library) {
emitJob(job, { type: 'status', message: 'Saving to library...', progress: 98 })
try { try {
const buffer = await downloadComfyVideo(video) if (error) {
const clip = await saveClip({ job.status = job.status === 'cancelled' ? 'cancelled' : 'error'
folderId: job.library.folderId, job.error = classifyError(error)
prompt: job.library.prompt, emitJob(job, { type: 'error', error: job.error, message: job.error })
aspect: job.library.aspect, return
width: job.library.width, }
height: job.library.height, emitJob(job, { type: 'status', message: 'Fetching output...', progress: 96 })
steps: job.library.steps, const history = await fetchHistory(job.promptId || '')
turbo: job.library.turbo, const video = extractVideo(history, job.promptId || '')
seed: job.library.seed, if (!video) {
hideThumbnail: job.library.hideThumbnail, job.status = 'error'
video: buffer, job.error = 'Workflow finished but no MP4 was found in ComfyUI history.'
thumb: job.library.hideThumbnail ? null : job.library.thumb emitJob(job, { type: 'error', error: job.error, message: job.error })
return
}
job.video = video
if (job.library) {
emitJob(job, { type: 'status', message: 'Saving to library...', progress: 98 })
try {
const buffer = await downloadComfyVideo(video)
const clip = await saveClip({
folderId: job.library.folderId,
prompt: job.library.prompt,
aspect: job.library.aspect,
width: job.library.width,
height: job.library.height,
steps: job.library.steps,
turbo: job.library.turbo,
seed: job.library.seed,
hideThumbnail: job.library.hideThumbnail,
video: buffer,
thumb: job.library.hideThumbnail ? null : job.library.thumb
})
job.clipId = clip.id
job.hideThumbnail = clip.hideThumbnail
job.library.thumb = undefined
} catch (saveError) {
const message = saveError instanceof Error ? saveError.message : String(saveError)
job.status = 'error'
job.error = `Video generated but library save failed: ${message}`
emitJob(job, { type: 'error', error: job.error, message: job.error })
return
}
}
job.status = 'complete'
emitJob(job, {
type: 'complete',
message: 'Video ready',
progress: 100,
filename: video.filename,
subfolder: video.subfolder,
mediaType: video.type,
clipId: job.clipId,
hideThumbnail: job.hideThumbnail
}) })
job.clipId = clip.id } finally {
job.hideThumbnail = clip.hideThumbnail resolve()
job.library.thumb = undefined
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
job.status = 'error'
job.error = `Video generated but library save failed: ${message}`
emitJob(job, { type: 'error', error: job.error, message: job.error })
return
} }
} }
job.status = 'complete'
emitJob(job, {
type: 'complete',
message: 'Video ready',
progress: 100,
filename: video.filename,
subfolder: video.subfolder,
mediaType: video.type,
clipId: job.clipId,
hideThumbnail: job.hideThumbnail
})
}
const timeout = setTimeout(() => { const pollHistory = async () => {
finish('Timed out waiting for ComfyUI (15 minutes).') if (settled || !job.promptId) return
}, 15 * 60 * 1000) try {
const inspected = inspectHistory(await fetchHistory(job.promptId), job.promptId)
ws.addEventListener('open', () => { if (inspected.error) {
emitJob(job, { type: 'status', message: 'Connected to ComfyUI', progress: 8 }) await finish(inspected.error)
}) return
}
ws.addEventListener('error', () => { if (inspected.video || inspected.completed) {
if (!settled) finish('ComfyUI WebSocket connection dropped.')
})
ws.addEventListener('close', () => {
if (!settled) {
// Fall back to history polling in case completion arrived as HTTP only.
setTimeout(async () => {
if (settled) return
const history = await fetchHistory(job.promptId || '')
if (extractVideo(history, job.promptId || '')) {
clearTimeout(timeout)
await finish() await finish()
} }
}, 1500) } catch {
// Transient ComfyUI history misses are expected while the graph is still running.
}
} }
})
ws.addEventListener('message', async (event) => { timeout = setTimeout(() => {
let payload: { type?: string; data?: Record<string, unknown> } void finish('Timed out waiting for ComfyUI (15 minutes).')
try { }, 15 * 60 * 1000)
payload = JSON.parse(String(event.data))
} catch {
return
}
const data = payload.data || {}
if (job.promptId && data.prompt_id && data.prompt_id !== job.promptId) return
if (payload.type === 'progress') { pollTimer = setInterval(() => {
const value = Number(data.value || 0) void pollHistory()
const max = Number(data.max || 1) }, 2000)
const pct = 12 + Math.round((value / Math.max(max, 1)) * 70)
job.status = 'running' ws.addEventListener('open', () => {
job.socketReady = true
emitJob(job, { type: 'status', message: 'Connected to ComfyUI', progress: Math.max(job.progress, 4) })
})
ws.addEventListener('error', () => {
if (settled) return
emitJob(job, { emitJob(job, {
type: 'progress', type: 'status',
step: value, message: job.promptId
maxStep: max, ? 'Live socket unavailable, polling ComfyUI history...'
progress: pct, : 'Waiting for ComfyUI socket...',
node: data.node ? String(data.node) : null, progress: Math.max(job.progress, 4)
message: `Sampling step ${value}/${max} (${Math.round((value / Math.max(max, 1)) * 100)}%)`
}) })
} })
if (payload.type === 'executing') { ws.addEventListener('message', async (event) => {
const node = data.node === null || data.node === undefined ? null : String(data.node) let payload: { type?: string; data?: Record<string, unknown> }
if (node === null) { try {
clearTimeout(timeout) payload = JSON.parse(String(event.data))
await finish() } catch {
return return
} }
const label = NODE_LABELS[node] || `Running node ${node}` const data = payload.data || {}
const encoding = isEncodingNode(node) if (job.promptId && data.prompt_id && data.prompt_id !== job.promptId) return
emitJob(job, {
type: 'executing',
node,
progress: encoding ? 90 : Math.max(job.progress, 10),
message: encoding ? 'Encoding video...' : `${label}...`
})
}
if (payload.type === 'execution_error') { if (payload.type === 'progress') {
clearTimeout(timeout) const value = Number(data.value || 0)
const message = String(data.exception_message || data.message || 'ComfyUI node execution failed') const max = Number(data.max || 1)
await finish(message) const pct = 12 + Math.round((value / Math.max(max, 1)) * 70)
} job.status = 'running'
emitJob(job, {
type: 'progress',
step: value,
maxStep: max,
progress: pct,
node: data.node ? String(data.node) : null,
message: `Sampling step ${value}/${max} (${Math.round((value / Math.max(max, 1)) * 100)}%)`
})
}
if (payload.type === 'execution_interrupted') { if (payload.type === 'executing') {
clearTimeout(timeout) const node = data.node === null || data.node === undefined ? null : String(data.node)
job.status = 'cancelled' if (node === null) {
await finish('Job interrupted.') await finish()
} return
}
const label = NODE_LABELS[node] || `Running node ${node}`
const encoding = isEncodingNode(node)
emitJob(job, {
type: 'executing',
node,
progress: encoding ? 90 : Math.max(job.progress, 10),
message: encoding ? 'Encoding video...' : `${label}...`
})
}
if (payload.type === 'execution_error') {
const message = String(data.exception_message || data.message || 'ComfyUI node execution failed')
await finish(message)
}
if (payload.type === 'execution_interrupted') {
job.status = 'cancelled'
await finish('Job interrupted.')
}
})
}) })
}
return () => { export async function waitForComfySocket(job: Job, ms = 4000) {
clearTimeout(timeout) const started = Date.now()
try { ws.close() } catch { /* ignore */ } while (Date.now() - started < ms) {
if (job.socketReady) return
await sleep(100)
} }
} }