Stop killing live videos mid-save: never fake-complete or age-out jobs just because Comfy is briefly idle.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+35
-45
@@ -297,11 +297,9 @@ async function reapZombieLiveJobs() {
|
||||
if (queue.running > 0 || queue.pending > 0) return
|
||||
const { fetchHistory } = await import('~/server/utils/comfy')
|
||||
for (const job of listJobs()) {
|
||||
// Never interrupt download/stitch/library save — Comfy is idle then by design.
|
||||
if (job.saving) continue
|
||||
// Stuck extension loops claimed the GPU forever after a crash/sleep.
|
||||
if (job.library?.chainContinuing && jobAgeMs(job) >= ZOMBIE_EMPTY_COMFY_MS) {
|
||||
job.library.chainContinuing = false
|
||||
}
|
||||
if (job.library?.chainContinuing) continue
|
||||
if (job.status !== 'running' && job.status !== 'uploading' && job.status !== 'queued') continue
|
||||
if (jobIsLocallySubmitting(job)) continue
|
||||
const zombieMs = (job.kind === 'edit' || job.kind === 'music') ? 45_000 : ZOMBIE_EMPTY_COMFY_MS
|
||||
@@ -321,29 +319,24 @@ async function reapZombieLiveJobs() {
|
||||
failZombieLiveJob(job, job.error || `ComfyUI ${status}`)
|
||||
continue
|
||||
}
|
||||
// History still present + Comfy idle: waiter never settled (common for image jobs).
|
||||
// Mark complete and advance the studio queue so waiting items can run.
|
||||
job.status = 'complete'
|
||||
job.error = undefined
|
||||
if (job.library) job.library.chainContinuing = false
|
||||
await onLiveVideoSettled(job).catch(() => null)
|
||||
// History success + Comfy idle: the socket waiter usually finishes the save.
|
||||
// Never fake-complete video here — that skipped library save and left ghosts in the queue.
|
||||
// Image/music: only settle if media was already persisted and the waiter died before settle.
|
||||
if (job.kind === 'edit' && job.stillId) {
|
||||
job.status = 'complete'
|
||||
job.error = undefined
|
||||
await onLiveVideoSettled(job).catch(() => null)
|
||||
} else if (job.kind === 'music' && (job.clipId || job.audio)) {
|
||||
job.status = 'complete'
|
||||
job.error = undefined
|
||||
await onLiveVideoSettled(job).catch(() => null)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function liveJobOwnsGpu(job: Job) {
|
||||
if (job.library?.stopAfterCurrent) return false
|
||||
if (job.library?.chainContinuing) {
|
||||
// Never let a stuck chainContinuing flag block the queue after Comfy went idle.
|
||||
if (job.status === 'complete' || job.status === 'error' || job.status === 'cancelled') {
|
||||
job.library.chainContinuing = false
|
||||
return false
|
||||
}
|
||||
if (jobAgeMs(job) >= ZOMBIE_EMPTY_COMFY_MS * 2) {
|
||||
job.library.chainContinuing = false
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
if (job.library?.chainContinuing) return true
|
||||
if (job.saving) return true
|
||||
if (job.status === 'uploading') return true
|
||||
if (job.status === 'running') {
|
||||
@@ -355,19 +348,24 @@ function liveJobOwnsGpu(job: Job) {
|
||||
return false
|
||||
}
|
||||
|
||||
/** Drop soft in-memory GPU claims when Comfy itself is idle so Start now can fire. */
|
||||
function clearSoftGpuClaimsWhenComfyIdle(opts?: { aggressive?: boolean }) {
|
||||
const ageCap = opts?.aggressive ? SUBMIT_WINDOW_MS : ZOMBIE_EMPTY_COMFY_MS
|
||||
/**
|
||||
* Force-start only: drop dead in-memory claims when Comfy is idle.
|
||||
* Must never use job startedAt alone — real videos run far longer than ZOMBIE_EMPTY_COMFY_MS
|
||||
* and sit with an empty Comfy queue while downloading/stitching.
|
||||
*/
|
||||
function clearDeadGpuClaimsForForceStart() {
|
||||
for (const live of listJobs()) {
|
||||
if (live.library?.chainContinuing && jobAgeMs(live) >= ageCap) {
|
||||
live.library.chainContinuing = false
|
||||
}
|
||||
if (live.saving && jobAgeMs(live) >= ZOMBIE_EMPTY_COMFY_MS) {
|
||||
live.saving = false
|
||||
}
|
||||
if (live.status !== 'running' && live.status !== 'queued' && live.status !== 'uploading') continue
|
||||
if (live.saving) continue
|
||||
if (jobIsLocallySubmitting(live)) continue
|
||||
if (jobAgeMs(live) < ageCap) continue
|
||||
if (live.status !== 'running' && live.status !== 'queued' && live.status !== 'uploading') {
|
||||
if (live.library?.chainContinuing) live.library.chainContinuing = false
|
||||
continue
|
||||
}
|
||||
// Active prompt still owned by a living waiter — leave it alone.
|
||||
if (live.promptId && jobAgeMs(live) < QUEUED_GRACE_MS) continue
|
||||
if (live.promptId) continue
|
||||
if (live.library?.chainContinuing) live.library.chainContinuing = false
|
||||
if (jobAgeMs(live) < SUBMIT_WINDOW_MS) continue
|
||||
failZombieLiveJob(live, 'Cleared stuck live job so the studio queue can start')
|
||||
}
|
||||
}
|
||||
@@ -382,9 +380,7 @@ async function forceStartWaitingIfGpuIdle(owner: string, id: string) {
|
||||
const queue = await fetchLiveQueue().catch(() => null)
|
||||
const comfyRendering = Boolean(queue && queue.running > 0)
|
||||
if (comfyRendering) return false
|
||||
// Soft ownership (chainContinuing / orphan running) used to block Start now forever
|
||||
// while Comfy's queue was already empty.
|
||||
clearSoftGpuClaimsWhenComfyIdle({ aggressive: true })
|
||||
clearDeadGpuClaimsForForceStart()
|
||||
if (listJobs().some(liveJobOwnsGpu)) return false
|
||||
try {
|
||||
await startStudioJob(after)
|
||||
@@ -987,16 +983,10 @@ async function dispatchStudioQueue() {
|
||||
})
|
||||
}
|
||||
if (await videoJobsBusy()) {
|
||||
const queue = await fetchLiveQueue().catch(() => null)
|
||||
if (queue && queue.running === 0 && queue.pending === 0) {
|
||||
clearSoftGpuClaimsWhenComfyIdle()
|
||||
}
|
||||
if (await videoJobsBusy()) {
|
||||
if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting'))) {
|
||||
scheduleKickRetry()
|
||||
}
|
||||
return
|
||||
if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting'))) {
|
||||
scheduleKickRetry()
|
||||
}
|
||||
return
|
||||
}
|
||||
// Re-read after videoJobsBusy — reap/settle may have rewritten disk rows.
|
||||
for (const owner of owners) {
|
||||
|
||||
Reference in New Issue
Block a user