towsty/aigen
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Recover finished Comfy videos after relay restarts.

Browse Source 
Persist queued jobs to disk, import leftover MP4s from Comfy history, and do not mark a run complete until the saved video exists.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-08-25 18:43:15 -05:00
co-authored by Cursor
parent f0f80b5873
commit b7196ecde3
10 changed files with 342 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/utils/watch.ts
+53 -52
View File
@@ -34,55 +34,47 @@ export function watchComfyJob(job: Job): Promise<void> {
try { ws.close() } catch { /* ignore */ }
}
const finish = async (error?: string) => {
const fail = async (error: string) => {
if (settled) return
settled = true
cleanup()
job.status = job.status === 'cancelled' ? 'cancelled' : 'error'
job.error = classifyError(error)
emitJob(job, { type: 'error', error: job.error, message: job.error })
if (job.id) deletePendingJob(job.id)
resolve()
}
const succeed = async () => {
if (settled || !job.promptId) return false
const history = await fetchHistory(job.promptId)
const video = extractVideo(history, job.promptId)
if (!video) return false
settled = true
cleanup()
try {
if (error) {
job.status = job.status === 'cancelled' ? 'cancelled' : 'error'
job.error = classifyError(error)
emitJob(job, { type: 'error', error: job.error, message: job.error })
return
}
emitJob(job, { type: 'status', message: 'Fetching output...', progress: 96 })
const history = await fetchHistory(job.promptId || '')
const video = extractVideo(history, job.promptId || '')
if (!video) {
job.status = 'error'
job.error = 'Workflow finished but no MP4 was found in ComfyUI history.'
emitJob(job, { type: 'error', error: job.error, message: job.error })
return
}
job.video = video
emitJob(job, { type: 'status', message: 'Saving to library...', progress: 98 })
if (job.library) {
emitJob(job, { type: 'status', message: 'Saving to library...', progress: 98 })
try {
const buffer = await downloadComfyVideo(video)
const clip = await saveClip({
ownerKey: job.library.ownerKey,
folderId: job.library.folderId,
prompt: job.library.prompt,
aspect: job.library.aspect,
width: job.library.width,
height: job.library.height,
steps: job.library.steps,
turbo: job.library.turbo,
seed: job.library.seed,
hideThumbnail: job.library.hideThumbnail,
video: buffer,
thumb: job.library.hideThumbnail ? null : job.library.thumb
})
job.clipId = clip.id
job.hideThumbnail = clip.hideThumbnail
job.library.thumb = undefined
} catch (saveError) {
const message = saveError instanceof Error ? saveError.message : String(saveError)
job.status = 'error'
job.error = `Video generated but library save failed: ${message}`
emitJob(job, { type: 'error', error: job.error, message: job.error })
return
}
const buffer = await downloadComfyVideo(video)
const clip = await saveClip({
ownerKey: job.library.ownerKey,
folderId: job.library.folderId,
prompt: job.library.prompt,
aspect: job.library.aspect,
width: job.library.width,
height: job.library.height,
steps: job.library.steps,
turbo: job.library.turbo,
seed: job.library.seed,
hideThumbnail: job.library.hideThumbnail,
video: buffer,
thumb: job.library.hideThumbnail ? null : job.library.thumb,
comfyFilename: video.filename
})
job.clipId = clip.id
job.hideThumbnail = clip.hideThumbnail
job.library.thumb = undefined
}
job.status = 'complete'
emitJob(job, {
@@ -95,9 +87,15 @@ export function watchComfyJob(job: Job): Promise<void> {
clipId: job.clipId,
hideThumbnail: job.hideThumbnail
})
} finally {
resolve()
deletePendingJob(job.id)
} catch (saveError) {
const message = saveError instanceof Error ? saveError.message : String(saveError)
job.status = 'error'
job.error = `Video generated but library save failed: ${message}`
emitJob(job, { type: 'error', error: job.error, message: job.error })
}
resolve()
return true
}
const pollHistory = async () => {
@@ -105,11 +103,11 @@ export function watchComfyJob(job: Job): Promise<void> {
try {
const inspected = inspectHistory(await fetchHistory(job.promptId), job.promptId)
if (inspected.error) {
await finish(inspected.error)
await fail(inspected.error)
return
}
if (inspected.video || inspected.completed) {
await finish()
if (inspected.video) {
await succeed()
}
} catch {
// Transient ComfyUI history misses are expected while the graph is still running.
@@ -117,7 +115,7 @@ export function watchComfyJob(job: Job): Promise<void> {
}
timeout = setTimeout(() => {
void finish('Timed out waiting for ComfyUI (15 minutes).')
void fail('Timed out waiting for ComfyUI (15 minutes).')
}, 15 * 60 * 1000)
pollTimer = setInterval(() => {
@@ -168,7 +166,10 @@ export function watchComfyJob(job: Job): Promise<void> {
if (payload.type === 'executing') {
const node = data.node === null || data.node === undefined ? null : String(data.node)
if (node === null) {
await finish()
const saved = await succeed()
if (!saved) {
emitJob(job, { type: 'status', message: 'Waiting for ComfyUI to save the MP4...', progress: 96 })
}
return
}
const label = NODE_LABELS[node] || `Running node ${node}`
@@ -183,12 +184,12 @@ export function watchComfyJob(job: Job): Promise<void> {
if (payload.type === 'execution_error') {
const message = String(data.exception_message || data.message || 'ComfyUI node execution failed')
await finish(message)
await fail(message)
}
if (payload.type === 'execution_interrupted') {
job.status = 'cancelled'
await finish('Job interrupted.')
await fail('Job interrupted.')
}
})
})