Stop killing live videos mid-save: never fake-complete or age-out jobs just because Comfy is briefly idle.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-09-05 09:49:00 -05:00
co-authored by Cursor
parent 34c39bf85d
commit 279fabfb7b
+35 -45
View File
@@ -297,11 +297,9 @@ async function reapZombieLiveJobs() {
if (queue.running > 0 || queue.pending > 0) return if (queue.running > 0 || queue.pending > 0) return
const { fetchHistory } = await import('~/server/utils/comfy') const { fetchHistory } = await import('~/server/utils/comfy')
for (const job of listJobs()) { for (const job of listJobs()) {
// Never interrupt download/stitch/library save — Comfy is idle then by design.
if (job.saving) continue if (job.saving) continue
// Stuck extension loops claimed the GPU forever after a crash/sleep. if (job.library?.chainContinuing) continue
if (job.library?.chainContinuing && jobAgeMs(job) >= ZOMBIE_EMPTY_COMFY_MS) {
job.library.chainContinuing = false
}
if (job.status !== 'running' && job.status !== 'uploading' && job.status !== 'queued') continue if (job.status !== 'running' && job.status !== 'uploading' && job.status !== 'queued') continue
if (jobIsLocallySubmitting(job)) continue if (jobIsLocallySubmitting(job)) continue
const zombieMs = (job.kind === 'edit' || job.kind === 'music') ? 45_000 : ZOMBIE_EMPTY_COMFY_MS const zombieMs = (job.kind === 'edit' || job.kind === 'music') ? 45_000 : ZOMBIE_EMPTY_COMFY_MS
@@ -321,29 +319,24 @@ async function reapZombieLiveJobs() {
failZombieLiveJob(job, job.error || `ComfyUI ${status}`) failZombieLiveJob(job, job.error || `ComfyUI ${status}`)
continue continue
} }
// History still present + Comfy idle: waiter never settled (common for image jobs). // History success + Comfy idle: the socket waiter usually finishes the save.
// Mark complete and advance the studio queue so waiting items can run. // Never fake-complete video here — that skipped library save and left ghosts in the queue.
job.status = 'complete' // Image/music: only settle if media was already persisted and the waiter died before settle.
job.error = undefined if (job.kind === 'edit' && job.stillId) {
if (job.library) job.library.chainContinuing = false job.status = 'complete'
await onLiveVideoSettled(job).catch(() => null) job.error = undefined
await onLiveVideoSettled(job).catch(() => null)
} else if (job.kind === 'music' && (job.clipId || job.audio)) {
job.status = 'complete'
job.error = undefined
await onLiveVideoSettled(job).catch(() => null)
}
} }
} }
function liveJobOwnsGpu(job: Job) { function liveJobOwnsGpu(job: Job) {
if (job.library?.stopAfterCurrent) return false if (job.library?.stopAfterCurrent) return false
if (job.library?.chainContinuing) { if (job.library?.chainContinuing) return true
// Never let a stuck chainContinuing flag block the queue after Comfy went idle.
if (job.status === 'complete' || job.status === 'error' || job.status === 'cancelled') {
job.library.chainContinuing = false
return false
}
if (jobAgeMs(job) >= ZOMBIE_EMPTY_COMFY_MS * 2) {
job.library.chainContinuing = false
return false
}
return true
}
if (job.saving) return true if (job.saving) return true
if (job.status === 'uploading') return true if (job.status === 'uploading') return true
if (job.status === 'running') { if (job.status === 'running') {
@@ -355,19 +348,24 @@ function liveJobOwnsGpu(job: Job) {
return false return false
} }
/** Drop soft in-memory GPU claims when Comfy itself is idle so Start now can fire. */ /**
function clearSoftGpuClaimsWhenComfyIdle(opts?: { aggressive?: boolean }) { * Force-start only: drop dead in-memory claims when Comfy is idle.
const ageCap = opts?.aggressive ? SUBMIT_WINDOW_MS : ZOMBIE_EMPTY_COMFY_MS * Must never use job startedAt alone — real videos run far longer than ZOMBIE_EMPTY_COMFY_MS
* and sit with an empty Comfy queue while downloading/stitching.
*/
function clearDeadGpuClaimsForForceStart() {
for (const live of listJobs()) { for (const live of listJobs()) {
if (live.library?.chainContinuing && jobAgeMs(live) >= ageCap) { if (live.saving) continue
live.library.chainContinuing = false
}
if (live.saving && jobAgeMs(live) >= ZOMBIE_EMPTY_COMFY_MS) {
live.saving = false
}
if (live.status !== 'running' && live.status !== 'queued' && live.status !== 'uploading') continue
if (jobIsLocallySubmitting(live)) continue if (jobIsLocallySubmitting(live)) continue
if (jobAgeMs(live) < ageCap) continue if (live.status !== 'running' && live.status !== 'queued' && live.status !== 'uploading') {
if (live.library?.chainContinuing) live.library.chainContinuing = false
continue
}
// Active prompt still owned by a living waiter — leave it alone.
if (live.promptId && jobAgeMs(live) < QUEUED_GRACE_MS) continue
if (live.promptId) continue
if (live.library?.chainContinuing) live.library.chainContinuing = false
if (jobAgeMs(live) < SUBMIT_WINDOW_MS) continue
failZombieLiveJob(live, 'Cleared stuck live job so the studio queue can start') failZombieLiveJob(live, 'Cleared stuck live job so the studio queue can start')
} }
} }
@@ -382,9 +380,7 @@ async function forceStartWaitingIfGpuIdle(owner: string, id: string) {
const queue = await fetchLiveQueue().catch(() => null) const queue = await fetchLiveQueue().catch(() => null)
const comfyRendering = Boolean(queue && queue.running > 0) const comfyRendering = Boolean(queue && queue.running > 0)
if (comfyRendering) return false if (comfyRendering) return false
// Soft ownership (chainContinuing / orphan running) used to block Start now forever clearDeadGpuClaimsForForceStart()
// while Comfy's queue was already empty.
clearSoftGpuClaimsWhenComfyIdle({ aggressive: true })
if (listJobs().some(liveJobOwnsGpu)) return false if (listJobs().some(liveJobOwnsGpu)) return false
try { try {
await startStudioJob(after) await startStudioJob(after)
@@ -987,16 +983,10 @@ async function dispatchStudioQueue() {
}) })
} }
if (await videoJobsBusy()) { if (await videoJobsBusy()) {
const queue = await fetchLiveQueue().catch(() => null) if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting'))) {
if (queue && queue.running === 0 && queue.pending === 0) { scheduleKickRetry()
clearSoftGpuClaimsWhenComfyIdle()
}
if (await videoJobsBusy()) {
if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting'))) {
scheduleKickRetry()
}
return
} }
return
} }
// Re-read after videoJobsBusy — reap/settle may have rewritten disk rows. // Re-read after videoJobsBusy — reap/settle may have rewritten disk rows.
for (const owner of owners) { for (const owner of owners) {