Stop cancelled video jobs from coming back after delete.

A leftover process-remaining wake was flipping the row back to running or error, so Remove looked like it did nothing.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-08-30 12:47:59 -05:00
co-authored by Cursor
parent 201eb1107f
commit c88a8494d5
5 changed files with 80 additions and 29 deletions
+38 -18
View File
@@ -346,20 +346,28 @@ export async function patchStudioJob(owner: string, id: string, patch: (job: Stu
})
}
function canCancelStudioStatus(status: StudioJobStatus) {
return status === 'waiting' || status === 'held' || status === 'running' || status === 'error'
}
export async function cancelStudioJob(owner: string, id: string) {
const before = readJobs(owner).find(item => item.id === id)
if (!before) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' })
if (before.status !== 'waiting' && before.status !== 'held' && before.status !== 'running') {
if (!canCancelStudioStatus(before.status)) {
throw createError({ statusCode: 409, statusMessage: 'That job is no longer in the queue' })
}
const stopLive = before.status === 'running'
const liveJobId = before.liveJobId
const shotQueueId = before.shotQueueId
if (shotQueueId) {
const { markShotQueueCancelled, setQueueJob } = await import('~/server/utils/shotQueue')
markShotQueueCancelled(shotQueueId)
setQueueJob(shotQueueId, null)
}
const result = await mutateStore(owner, (store) => {
const job = store.jobs.find(item => item.id === id)
if (!job) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' })
if (job.status !== 'waiting' && job.status !== 'held' && job.status !== 'running') {
if (!canCancelStudioStatus(job.status)) {
throw createError({ statusCode: 409, statusMessage: 'That job is no longer in the queue' })
}
job.status = 'cancelled'
@@ -368,6 +376,8 @@ export async function cancelStudioJob(owner: string, id: string) {
job.pauseAfterCurrent = false
job.pausedByUser = false
job.liveJobId = undefined
job.shotQueueId = undefined
job.lastError = undefined
job.updatedAt = Date.now()
const stillCutIn = store.jobs.some(item => item.status === 'waiting' && item.cutIn)
if (!stillCutIn) {
@@ -380,13 +390,10 @@ export async function cancelStudioJob(owner: string, id: string) {
syncPausedFlag(store)
return structuredClone(job)
})
if (stopLive) {
await stopLiveGeneration(liveJobId, shotQueueId)
}
if (result.shotQueueId) {
const { deleteShotQueue, setQueueJob } = await import('~/server/utils/shotQueue')
setQueueJob(result.shotQueueId, null)
await deleteShotQueue(owner, result.shotQueueId, { force: true }).catch(() => null)
await stopLiveGeneration(liveJobId, shotQueueId)
if (shotQueueId) {
const { deleteShotQueue } = await import('~/server/utils/shotQueue')
await deleteShotQueue(owner, shotQueueId, { force: true }).catch(() => null)
}
const jobs = readJobs(owner)
if (!jobs.some(item => item.status === 'waiting' && item.cutIn)) {
@@ -425,12 +432,13 @@ async function stopLiveGeneration(liveJobId?: string, shotQueueId?: string) {
}
export async function cancelOrphanQueue(owner: string, queueId: string) {
const { getShotQueue, getQueueJobId, queueIsProcessing, deleteShotQueue } = await import('~/server/utils/shotQueue')
const { getShotQueue, getQueueJobId, queueIsProcessing, deleteShotQueue, markShotQueueCancelled } = await import('~/server/utils/shotQueue')
const queue = getShotQueue(owner, queueId)
if (!queue) throw createError({ statusCode: 404, statusMessage: 'Queue not found' })
markShotQueueCancelled(queueId)
const generating = queue.status === 'running' || queueIsProcessing(queueId)
const liveJobId = getQueueJobId(queueId) || queue.currentJobId
if (generating) {
if (generating || liveJobId) {
await stopLiveGeneration(liveJobId, queueId)
}
await deleteShotQueue(owner, queueId, { force: true }).catch(() => null)
@@ -650,33 +658,43 @@ async function dispatchStudioQueue() {
}
async function resumeHeldStudioJob(item: StudioJob) {
if (!item.shotQueueId) {
const latest = readJobs(item.ownerKey).find(job => job.id === item.id)
if (!latest || latest.status === 'cancelled') return false
if (!latest.shotQueueId) {
await patchStudioJob(item.ownerKey, item.id, (job) => {
if (job.status === 'cancelled') return
job.status = 'complete'
job.holdForCutIn = false
})
return false
}
const { isShotQueueCancelled } = await import('~/server/utils/shotQueue')
if (isShotQueueCancelled(latest.shotQueueId)) return false
await patchStudioJob(item.ownerKey, item.id, (job) => {
if (job.status === 'cancelled') return
job.status = 'running'
job.lastError = undefined
})
if (readJobs(item.ownerKey).find(job => job.id === item.id)?.status === 'cancelled') return false
const { startQueueBurst } = await import('~/server/utils/videoChain')
const count = item.resumeAutoRun ? 'all' as const : 1
const count = latest.resumeAutoRun ? 'all' as const : 1
try {
const started = await startQueueBurst(item.ownerKey, item.shotQueueId, count)
const started = await startQueueBurst(item.ownerKey, latest.shotQueueId, count)
await patchStudioJob(item.ownerKey, item.id, (job) => {
if (job.status === 'cancelled') return
job.status = 'running'
job.liveJobId = started.jobId
job.holdForCutIn = false
job.cutIn = false
})
return true
return readJobs(item.ownerKey).find(job => job.id === item.id)?.status === 'running'
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
if (/already processing/i.test(message)) return true
if (/removed|not found/i.test(message)) return false
const transient = /offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker/i.test(message)
await patchStudioJob(item.ownerKey, item.id, (job) => {
if (job.status === 'cancelled') return
job.status = transient ? 'held' : 'error'
job.lastError = message
job.holdForCutIn = transient
@@ -1116,7 +1134,7 @@ export async function onLiveVideoSettled(job: Job) {
await mutateStore(owner, (store) => {
const row = store.jobs.find(item => item.liveJobId === job.id)
|| (job.library?.queueId ? store.jobs.find(item => item.shotQueueId === job.library?.queueId) : undefined)
if (!row) {
if (!row || row.status === 'cancelled') {
syncPausedFlag(store)
return
}
@@ -1321,9 +1339,11 @@ export async function toggleStudioQueuePause(owner: string, options: {
}
export async function onStudioQueueBurstStarted(owner: string, queueId: string, liveJobId: string) {
const { isShotQueueCancelled } = await import('~/server/utils/shotQueue')
if (isShotQueueCancelled(queueId)) return
await mutateStore(owner, (store) => {
const row = store.jobs.find(job => job.shotQueueId === queueId)
if (!row) {
if (!row || row.status === 'cancelled') {
syncPausedFlag(store)
return
}