Fix queue race: keep the running studio slot until settle finishes, then start the next waiting job.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+18
-21
@@ -259,12 +259,13 @@ function jobAgeMs(job: { startedAt: number }) {
|
|||||||
return Date.now() - job.startedAt
|
return Date.now() - job.startedAt
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Uploading, or created just now and not yet on Comfy. Paused leftovers do not count. */
|
/** Uploading, or created and not yet on Comfy (includes wake / model load). */
|
||||||
export function jobIsLocallySubmitting(job: Job) {
|
export function jobIsLocallySubmitting(job: Job) {
|
||||||
if (job.library?.stopAfterCurrent) return false
|
if (job.library?.stopAfterCurrent) return false
|
||||||
if (job.status === 'uploading') return true
|
if (job.status === 'uploading') return true
|
||||||
if (job.status === 'queued' && !job.promptId && jobAgeMs(job) < SUBMIT_WINDOW_MS) return true
|
// Must cover ensureComfyReady's busy wait (up to ~3 min). The old 45s window
|
||||||
if (job.status === 'running' && !job.promptId && jobAgeMs(job) < SUBMIT_WINDOW_MS) return true
|
// let the dispatcher start the next queue item while the current one was still waking Comfy.
|
||||||
|
if ((job.status === 'queued' || job.status === 'running') && !job.promptId && jobAgeMs(job) < QUEUED_GRACE_MS) return true
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -339,12 +340,11 @@ function liveJobOwnsGpu(job: Job) {
|
|||||||
if (job.library?.chainContinuing) return true
|
if (job.library?.chainContinuing) return true
|
||||||
if (job.saving) return true
|
if (job.saving) return true
|
||||||
if (job.status === 'uploading') return true
|
if (job.status === 'uploading') return true
|
||||||
if (job.status === 'running') {
|
if (job.status === 'running' || job.status === 'queued') {
|
||||||
// Without a prompt id past the submit window, this is a zombie — do not block the queue.
|
// Without a prompt id past the wake/submit grace, this is a zombie — do not block forever.
|
||||||
if (!job.promptId && jobAgeMs(job) >= SUBMIT_WINDOW_MS) return false
|
if (!job.promptId && jobAgeMs(job) >= QUEUED_GRACE_MS) return false
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
if (job.status === 'queued') return Boolean(job.promptId) || jobAgeMs(job) < SUBMIT_WINDOW_MS
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -397,16 +397,11 @@ export async function videoJobsBusy() {
|
|||||||
// An empty Comfy queue is not idle: the 3s buffer + last-frame extract between
|
// An empty Comfy queue is not idle: the 3s buffer + last-frame extract between
|
||||||
// shots leaves Comfy empty while the current video still owns the GPU.
|
// shots leaves Comfy empty while the current video still owns the GPU.
|
||||||
if (listJobs().some(liveJobOwnsGpu)) return true
|
if (listJobs().some(liveJobOwnsGpu)) return true
|
||||||
// Studio "running" only blocks when a live job still owns the GPU, or a fresh
|
// Studio row still "running" owns the queue slot until onLiveVideoSettled clears it.
|
||||||
// pending file exists. Orphan disk rows used to block the whole queue forever.
|
// Do not peek under the live job here — that raced settle and started the next item early.
|
||||||
if (listOwnersWithStudioQueues().some(owner => listStudioJobs(owner).some((job) => {
|
if (listOwnersWithStudioQueues().some(owner => listStudioJobs(owner).some(job => job.status === 'running'))) {
|
||||||
if (job.status !== 'running') return false
|
return true
|
||||||
if (job.liveJobId) {
|
|
||||||
const live = getJob(job.liveJobId)
|
|
||||||
if (live && liveJobOwnsGpu(live)) return true
|
|
||||||
}
|
}
|
||||||
return pendingAlive(job)
|
|
||||||
}))) return true
|
|
||||||
const queue = await fetchLiveQueue()
|
const queue = await fetchLiveQueue()
|
||||||
if (queue) {
|
if (queue) {
|
||||||
if (queue.running > 0) return true
|
if (queue.running > 0) return true
|
||||||
@@ -825,11 +820,9 @@ function pickHeld(jobs: StudioJob[]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function pickWaiting(jobs: StudioJob[]) {
|
function pickWaiting(jobs: StudioJob[]) {
|
||||||
const waiting = jobs.filter(job => job.status === 'waiting')
|
const waiting = jobs
|
||||||
if (!waiting.length) return null
|
.filter(job => job.status === 'waiting')
|
||||||
if (jobs.some(job => job.pausedByUser === true)) {
|
.sort((a, b) => a.createdAt - b.createdAt)
|
||||||
return waiting.reduce((latest, job) => job.createdAt >= latest.createdAt ? job : latest)
|
|
||||||
}
|
|
||||||
return waiting[0] || null
|
return waiting[0] || null
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -927,6 +920,9 @@ function repairStaleJobs(jobs: StudioJob[]) {
|
|||||||
}
|
}
|
||||||
if (job.status !== 'running') continue
|
if (job.status !== 'running') continue
|
||||||
const live = job.liveJobId ? getJob(job.liveJobId) : undefined
|
const live = job.liveJobId ? getJob(job.liveJobId) : undefined
|
||||||
|
// Live already complete: onLiveVideoSettled owns finalization. Stealing the row here
|
||||||
|
// started the next queue item while purge/settle was still running → next jobs failed.
|
||||||
|
if (live && live.status === 'complete') continue
|
||||||
const liveBusy = live && (live.status === 'queued' || live.status === 'uploading' || live.status === 'running')
|
const liveBusy = live && (live.status === 'queued' || live.status === 'uploading' || live.status === 'running')
|
||||||
if (liveBusy || pendingAlive(job)) continue
|
if (liveBusy || pendingAlive(job)) continue
|
||||||
const liveFailed = live && (live.status === 'error' || live.status === 'cancelled')
|
const liveFailed = live && (live.status === 'error' || live.status === 'cancelled')
|
||||||
@@ -940,6 +936,7 @@ function repairStaleJobs(jobs: StudioJob[]) {
|
|||||||
job.updatedAt = Date.now()
|
job.updatedAt = Date.now()
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// No live job and no fresh pending — orphaned running row.
|
||||||
if (job.shotQueueId) {
|
if (job.shotQueueId) {
|
||||||
job.status = 'held'
|
job.status = 'held'
|
||||||
job.liveJobId = undefined
|
job.liveJobId = undefined
|
||||||
|
|||||||
Reference in New Issue
Block a user