Stop killing live videos mid-save: never fake-complete or age-out jobs just because Comfy is briefly idle.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+35
-45
@@ -297,11 +297,9 @@ async function reapZombieLiveJobs() {
|
|||||||
if (queue.running > 0 || queue.pending > 0) return
|
if (queue.running > 0 || queue.pending > 0) return
|
||||||
const { fetchHistory } = await import('~/server/utils/comfy')
|
const { fetchHistory } = await import('~/server/utils/comfy')
|
||||||
for (const job of listJobs()) {
|
for (const job of listJobs()) {
|
||||||
|
// Never interrupt download/stitch/library save — Comfy is idle then by design.
|
||||||
if (job.saving) continue
|
if (job.saving) continue
|
||||||
// Stuck extension loops claimed the GPU forever after a crash/sleep.
|
if (job.library?.chainContinuing) continue
|
||||||
if (job.library?.chainContinuing && jobAgeMs(job) >= ZOMBIE_EMPTY_COMFY_MS) {
|
|
||||||
job.library.chainContinuing = false
|
|
||||||
}
|
|
||||||
if (job.status !== 'running' && job.status !== 'uploading' && job.status !== 'queued') continue
|
if (job.status !== 'running' && job.status !== 'uploading' && job.status !== 'queued') continue
|
||||||
if (jobIsLocallySubmitting(job)) continue
|
if (jobIsLocallySubmitting(job)) continue
|
||||||
const zombieMs = (job.kind === 'edit' || job.kind === 'music') ? 45_000 : ZOMBIE_EMPTY_COMFY_MS
|
const zombieMs = (job.kind === 'edit' || job.kind === 'music') ? 45_000 : ZOMBIE_EMPTY_COMFY_MS
|
||||||
@@ -321,29 +319,24 @@ async function reapZombieLiveJobs() {
|
|||||||
failZombieLiveJob(job, job.error || `ComfyUI ${status}`)
|
failZombieLiveJob(job, job.error || `ComfyUI ${status}`)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// History still present + Comfy idle: waiter never settled (common for image jobs).
|
// History success + Comfy idle: the socket waiter usually finishes the save.
|
||||||
// Mark complete and advance the studio queue so waiting items can run.
|
// Never fake-complete video here — that skipped library save and left ghosts in the queue.
|
||||||
job.status = 'complete'
|
// Image/music: only settle if media was already persisted and the waiter died before settle.
|
||||||
job.error = undefined
|
if (job.kind === 'edit' && job.stillId) {
|
||||||
if (job.library) job.library.chainContinuing = false
|
job.status = 'complete'
|
||||||
await onLiveVideoSettled(job).catch(() => null)
|
job.error = undefined
|
||||||
|
await onLiveVideoSettled(job).catch(() => null)
|
||||||
|
} else if (job.kind === 'music' && (job.clipId || job.audio)) {
|
||||||
|
job.status = 'complete'
|
||||||
|
job.error = undefined
|
||||||
|
await onLiveVideoSettled(job).catch(() => null)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function liveJobOwnsGpu(job: Job) {
|
function liveJobOwnsGpu(job: Job) {
|
||||||
if (job.library?.stopAfterCurrent) return false
|
if (job.library?.stopAfterCurrent) return false
|
||||||
if (job.library?.chainContinuing) {
|
if (job.library?.chainContinuing) return true
|
||||||
// Never let a stuck chainContinuing flag block the queue after Comfy went idle.
|
|
||||||
if (job.status === 'complete' || job.status === 'error' || job.status === 'cancelled') {
|
|
||||||
job.library.chainContinuing = false
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if (jobAgeMs(job) >= ZOMBIE_EMPTY_COMFY_MS * 2) {
|
|
||||||
job.library.chainContinuing = false
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if (job.saving) return true
|
if (job.saving) return true
|
||||||
if (job.status === 'uploading') return true
|
if (job.status === 'uploading') return true
|
||||||
if (job.status === 'running') {
|
if (job.status === 'running') {
|
||||||
@@ -355,19 +348,24 @@ function liveJobOwnsGpu(job: Job) {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Drop soft in-memory GPU claims when Comfy itself is idle so Start now can fire. */
|
/**
|
||||||
function clearSoftGpuClaimsWhenComfyIdle(opts?: { aggressive?: boolean }) {
|
* Force-start only: drop dead in-memory claims when Comfy is idle.
|
||||||
const ageCap = opts?.aggressive ? SUBMIT_WINDOW_MS : ZOMBIE_EMPTY_COMFY_MS
|
* Must never use job startedAt alone — real videos run far longer than ZOMBIE_EMPTY_COMFY_MS
|
||||||
|
* and sit with an empty Comfy queue while downloading/stitching.
|
||||||
|
*/
|
||||||
|
function clearDeadGpuClaimsForForceStart() {
|
||||||
for (const live of listJobs()) {
|
for (const live of listJobs()) {
|
||||||
if (live.library?.chainContinuing && jobAgeMs(live) >= ageCap) {
|
if (live.saving) continue
|
||||||
live.library.chainContinuing = false
|
|
||||||
}
|
|
||||||
if (live.saving && jobAgeMs(live) >= ZOMBIE_EMPTY_COMFY_MS) {
|
|
||||||
live.saving = false
|
|
||||||
}
|
|
||||||
if (live.status !== 'running' && live.status !== 'queued' && live.status !== 'uploading') continue
|
|
||||||
if (jobIsLocallySubmitting(live)) continue
|
if (jobIsLocallySubmitting(live)) continue
|
||||||
if (jobAgeMs(live) < ageCap) continue
|
if (live.status !== 'running' && live.status !== 'queued' && live.status !== 'uploading') {
|
||||||
|
if (live.library?.chainContinuing) live.library.chainContinuing = false
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Active prompt still owned by a living waiter — leave it alone.
|
||||||
|
if (live.promptId && jobAgeMs(live) < QUEUED_GRACE_MS) continue
|
||||||
|
if (live.promptId) continue
|
||||||
|
if (live.library?.chainContinuing) live.library.chainContinuing = false
|
||||||
|
if (jobAgeMs(live) < SUBMIT_WINDOW_MS) continue
|
||||||
failZombieLiveJob(live, 'Cleared stuck live job so the studio queue can start')
|
failZombieLiveJob(live, 'Cleared stuck live job so the studio queue can start')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -382,9 +380,7 @@ async function forceStartWaitingIfGpuIdle(owner: string, id: string) {
|
|||||||
const queue = await fetchLiveQueue().catch(() => null)
|
const queue = await fetchLiveQueue().catch(() => null)
|
||||||
const comfyRendering = Boolean(queue && queue.running > 0)
|
const comfyRendering = Boolean(queue && queue.running > 0)
|
||||||
if (comfyRendering) return false
|
if (comfyRendering) return false
|
||||||
// Soft ownership (chainContinuing / orphan running) used to block Start now forever
|
clearDeadGpuClaimsForForceStart()
|
||||||
// while Comfy's queue was already empty.
|
|
||||||
clearSoftGpuClaimsWhenComfyIdle({ aggressive: true })
|
|
||||||
if (listJobs().some(liveJobOwnsGpu)) return false
|
if (listJobs().some(liveJobOwnsGpu)) return false
|
||||||
try {
|
try {
|
||||||
await startStudioJob(after)
|
await startStudioJob(after)
|
||||||
@@ -987,16 +983,10 @@ async function dispatchStudioQueue() {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (await videoJobsBusy()) {
|
if (await videoJobsBusy()) {
|
||||||
const queue = await fetchLiveQueue().catch(() => null)
|
if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting'))) {
|
||||||
if (queue && queue.running === 0 && queue.pending === 0) {
|
scheduleKickRetry()
|
||||||
clearSoftGpuClaimsWhenComfyIdle()
|
|
||||||
}
|
|
||||||
if (await videoJobsBusy()) {
|
|
||||||
if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting'))) {
|
|
||||||
scheduleKickRetry()
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
// Re-read after videoJobsBusy — reap/settle may have rewritten disk rows.
|
// Re-read after videoJobsBusy — reap/settle may have rewritten disk rows.
|
||||||
for (const owner of owners) {
|
for (const owner of owners) {
|
||||||
|
|||||||
Reference in New Issue
Block a user