Fix queue dispatch deadlocks and stale completion recovery

This commit is contained in:
Towsty
2026-09-06 16:08:50 -05:00
parent 069108ae44
commit 911ddeab55
4 changed files with 189 additions and 10 deletions
+17 -9
View File
@@ -309,7 +309,14 @@ async function reapZombieLiveJobs() {
if (jobAgeMs(job) >= QUEUED_GRACE_MS) failZombieLiveJob(job, 'Job never started on ComfyUI')
continue
}
const history = await fetchHistory(job.promptId).catch(() => null)
let history: Awaited<ReturnType<typeof fetchHistory>>
try {
history = await fetchHistory(job.promptId)
} catch {
// A failed history request is not evidence that Comfy lost the prompt.
continue
}
if (!history) continue
const entry = history?.[job.promptId] as { status?: { status_str?: string; completed?: boolean } } | undefined
if (!entry) {
failZombieLiveJob(job, 'ComfyUI lost this job (empty queue after a restart or clear). Generate again.')
@@ -755,10 +762,9 @@ export async function retryStudioJob(owner: string, id: string) {
function pruneDone(jobs: StudioJob[]) {
const cutoff = Date.now() - 1000 * 60 * 60 * 24
return jobs.filter(job => {
if (job.status === 'waiting' || job.status === 'running' || job.status === 'held') return true
return job.updatedAt > cutoff
}).slice(-40)
const active = (job: StudioJob) => job.status === 'waiting' || job.status === 'running' || job.status === 'held'
const recentDone = new Set(jobs.filter(job => !active(job) && job.updatedAt > cutoff).slice(-40))
return jobs.filter(job => active(job) || recentDone.has(job))
}
export async function markStudioLive(owner: string, id: string, liveJobId: string, shotQueueId?: string) {
@@ -1599,12 +1605,12 @@ function findStudioRowForLive(jobs: StudioJob[], job: Job) {
const byQueue = jobs.find(item => (
item.shotQueueId === job.library?.queueId
&& item.status === 'running'
&& !item.liveJobId
))
if (byQueue) return byQueue
}
// Last resort: sole running row for this owner (liveJobId lost after a restart).
const running = jobs.filter(item => item.status === 'running')
if (running.length === 1) return running[0]
// A delayed callback from an older job must never settle a newer running job.
// Missing associations are handled by repairStaleJobs, not guessed here.
return undefined
}
@@ -1704,7 +1710,9 @@ export async function onLiveVideoSettled(job: Job) {
syncPausedFlag(store)
})
await kickStudioQueue()
// Settlement is also called from inside dispatchStudioQueue's recovery checks.
// Awaiting a dispatch queued behind that same dispatch deadlocks the entire queue.
void kickStudioQueue()
}
function interruptedRunning(job: Job, remaining: number, userPause: boolean, cutInHold: boolean) {