diff --git a/server/utils/studioQueue.ts b/server/utils/studioQueue.ts index bf4b138..3de1c2c 100644 --- a/server/utils/studioQueue.ts +++ b/server/utils/studioQueue.ts @@ -297,11 +297,9 @@ async function reapZombieLiveJobs() { if (queue.running > 0 || queue.pending > 0) return const { fetchHistory } = await import('~/server/utils/comfy') for (const job of listJobs()) { + // Never interrupt download/stitch/library save — Comfy is idle then by design. if (job.saving) continue - // Stuck extension loops claimed the GPU forever after a crash/sleep. - if (job.library?.chainContinuing && jobAgeMs(job) >= ZOMBIE_EMPTY_COMFY_MS) { - job.library.chainContinuing = false - } + if (job.library?.chainContinuing) continue if (job.status !== 'running' && job.status !== 'uploading' && job.status !== 'queued') continue if (jobIsLocallySubmitting(job)) continue const zombieMs = (job.kind === 'edit' || job.kind === 'music') ? 45_000 : ZOMBIE_EMPTY_COMFY_MS @@ -321,29 +319,24 @@ async function reapZombieLiveJobs() { failZombieLiveJob(job, job.error || `ComfyUI ${status}`) continue } - // History still present + Comfy idle: waiter never settled (common for image jobs). - // Mark complete and advance the studio queue so waiting items can run. - job.status = 'complete' - job.error = undefined - if (job.library) job.library.chainContinuing = false - await onLiveVideoSettled(job).catch(() => null) + // History success + Comfy idle: the socket waiter usually finishes the save. + // Never fake-complete video here — that skipped library save and left ghosts in the queue. + // Image/music: only settle if media was already persisted and the waiter died before settle. + if (job.kind === 'edit' && job.stillId) { + job.status = 'complete' + job.error = undefined + await onLiveVideoSettled(job).catch(() => null) + } else if (job.kind === 'music' && (job.clipId || job.audio)) { + job.status = 'complete' + job.error = undefined + await onLiveVideoSettled(job).catch(() => null) + } } } function liveJobOwnsGpu(job: Job) { if (job.library?.stopAfterCurrent) return false - if (job.library?.chainContinuing) { - // Never let a stuck chainContinuing flag block the queue after Comfy went idle. - if (job.status === 'complete' || job.status === 'error' || job.status === 'cancelled') { - job.library.chainContinuing = false - return false - } - if (jobAgeMs(job) >= ZOMBIE_EMPTY_COMFY_MS * 2) { - job.library.chainContinuing = false - return false - } - return true - } + if (job.library?.chainContinuing) return true if (job.saving) return true if (job.status === 'uploading') return true if (job.status === 'running') { @@ -355,19 +348,24 @@ function liveJobOwnsGpu(job: Job) { return false } -/** Drop soft in-memory GPU claims when Comfy itself is idle so Start now can fire. */ -function clearSoftGpuClaimsWhenComfyIdle(opts?: { aggressive?: boolean }) { - const ageCap = opts?.aggressive ? SUBMIT_WINDOW_MS : ZOMBIE_EMPTY_COMFY_MS +/** + * Force-start only: drop dead in-memory claims when Comfy is idle. + * Must never use job startedAt alone — real videos run far longer than ZOMBIE_EMPTY_COMFY_MS + * and sit with an empty Comfy queue while downloading/stitching. + */ +function clearDeadGpuClaimsForForceStart() { for (const live of listJobs()) { - if (live.library?.chainContinuing && jobAgeMs(live) >= ageCap) { - live.library.chainContinuing = false - } - if (live.saving && jobAgeMs(live) >= ZOMBIE_EMPTY_COMFY_MS) { - live.saving = false - } - if (live.status !== 'running' && live.status !== 'queued' && live.status !== 'uploading') continue + if (live.saving) continue if (jobIsLocallySubmitting(live)) continue - if (jobAgeMs(live) < ageCap) continue + if (live.status !== 'running' && live.status !== 'queued' && live.status !== 'uploading') { + if (live.library?.chainContinuing) live.library.chainContinuing = false + continue + } + // Active prompt still owned by a living waiter — leave it alone. + if (live.promptId && jobAgeMs(live) < QUEUED_GRACE_MS) continue + if (live.promptId) continue + if (live.library?.chainContinuing) live.library.chainContinuing = false + if (jobAgeMs(live) < SUBMIT_WINDOW_MS) continue failZombieLiveJob(live, 'Cleared stuck live job so the studio queue can start') } } @@ -382,9 +380,7 @@ async function forceStartWaitingIfGpuIdle(owner: string, id: string) { const queue = await fetchLiveQueue().catch(() => null) const comfyRendering = Boolean(queue && queue.running > 0) if (comfyRendering) return false - // Soft ownership (chainContinuing / orphan running) used to block Start now forever - // while Comfy's queue was already empty. - clearSoftGpuClaimsWhenComfyIdle({ aggressive: true }) + clearDeadGpuClaimsForForceStart() if (listJobs().some(liveJobOwnsGpu)) return false try { await startStudioJob(after) @@ -987,16 +983,10 @@ async function dispatchStudioQueue() { }) } if (await videoJobsBusy()) { - const queue = await fetchLiveQueue().catch(() => null) - if (queue && queue.running === 0 && queue.pending === 0) { - clearSoftGpuClaimsWhenComfyIdle() - } - if (await videoJobsBusy()) { - if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting'))) { - scheduleKickRetry() - } - return + if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting'))) { + scheduleKickRetry() } + return } // Re-read after videoJobsBusy — reap/settle may have rewritten disk rows. for (const owner of owners) {