Fix queue dispatch deadlocks and stale completion recovery
This commit is contained in:
@@ -309,7 +309,14 @@ async function reapZombieLiveJobs() {
|
||||
if (jobAgeMs(job) >= QUEUED_GRACE_MS) failZombieLiveJob(job, 'Job never started on ComfyUI')
|
||||
continue
|
||||
}
|
||||
const history = await fetchHistory(job.promptId).catch(() => null)
|
||||
let history: Awaited<ReturnType<typeof fetchHistory>>
|
||||
try {
|
||||
history = await fetchHistory(job.promptId)
|
||||
} catch {
|
||||
// A failed history request is not evidence that Comfy lost the prompt.
|
||||
continue
|
||||
}
|
||||
if (!history) continue
|
||||
const entry = history?.[job.promptId] as { status?: { status_str?: string; completed?: boolean } } | undefined
|
||||
if (!entry) {
|
||||
failZombieLiveJob(job, 'ComfyUI lost this job (empty queue after a restart or clear). Generate again.')
|
||||
@@ -755,10 +762,9 @@ export async function retryStudioJob(owner: string, id: string) {
|
||||
|
||||
function pruneDone(jobs: StudioJob[]) {
|
||||
const cutoff = Date.now() - 1000 * 60 * 60 * 24
|
||||
return jobs.filter(job => {
|
||||
if (job.status === 'waiting' || job.status === 'running' || job.status === 'held') return true
|
||||
return job.updatedAt > cutoff
|
||||
}).slice(-40)
|
||||
const active = (job: StudioJob) => job.status === 'waiting' || job.status === 'running' || job.status === 'held'
|
||||
const recentDone = new Set(jobs.filter(job => !active(job) && job.updatedAt > cutoff).slice(-40))
|
||||
return jobs.filter(job => active(job) || recentDone.has(job))
|
||||
}
|
||||
|
||||
export async function markStudioLive(owner: string, id: string, liveJobId: string, shotQueueId?: string) {
|
||||
@@ -1599,12 +1605,12 @@ function findStudioRowForLive(jobs: StudioJob[], job: Job) {
|
||||
const byQueue = jobs.find(item => (
|
||||
item.shotQueueId === job.library?.queueId
|
||||
&& item.status === 'running'
|
||||
&& !item.liveJobId
|
||||
))
|
||||
if (byQueue) return byQueue
|
||||
}
|
||||
// Last resort: sole running row for this owner (liveJobId lost after a restart).
|
||||
const running = jobs.filter(item => item.status === 'running')
|
||||
if (running.length === 1) return running[0]
|
||||
// A delayed callback from an older job must never settle a newer running job.
|
||||
// Missing associations are handled by repairStaleJobs, not guessed here.
|
||||
return undefined
|
||||
}
|
||||
|
||||
@@ -1704,7 +1710,9 @@ export async function onLiveVideoSettled(job: Job) {
|
||||
syncPausedFlag(store)
|
||||
})
|
||||
|
||||
await kickStudioQueue()
|
||||
// Settlement is also called from inside dispatchStudioQueue's recovery checks.
|
||||
// Awaiting a dispatch queued behind that same dispatch deadlocks the entire queue.
|
||||
void kickStudioQueue()
|
||||
}
|
||||
|
||||
function interruptedRunning(job: Job, remaining: number, userPause: boolean, cutInHold: boolean) {
|
||||
|
||||
Reference in New Issue
Block a user