Let failed studio jobs retry instead of sitting in the queue with only Remove.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-09-02 08:19:21 -05:00
co-authored by Cursor
parent bdec8872b6
commit 1314e8e89f
4 changed files with 88 additions and 4 deletions
@@ -0,0 +1,7 @@
import { retryStudioJob } from '~/server/utils/studioQueue'
export default defineEventHandler(async (event) => {
const { owner } = assertLibraryOwner(event)
const id = String(getRouterParam(event, 'id') || '')
return await retryStudioJob(owner, id)
})
+41 -4
View File
@@ -492,6 +492,36 @@ export async function requestCutIn(owner: string, id: string) {
return updated
}
export async function retryStudioJob(owner: string, id: string) {
const jobsNow = readJobs(owner)
const target = jobsNow.find(job => job.id === id)
if (!target) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' })
if (target.status !== 'error' && target.status !== 'held' && target.status !== 'waiting') {
throw createError({ statusCode: 409, statusMessage: 'Only a failed or paused job can be retried' })
}
const updated = await mutate(owner, (jobs) => {
const job = jobs.find(item => item.id === id)
if (!job) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' })
if (job.status !== 'error' && job.status !== 'held' && job.status !== 'waiting') {
throw createError({ statusCode: 409, statusMessage: 'Only a failed or paused job can be retried' })
}
job.status = 'waiting'
job.cutIn = true
job.lastError = undefined
job.liveJobId = undefined
job.pausedByUser = false
job.pauseAfterCurrent = false
job.holdForCutIn = false
job.updatedAt = Date.now()
for (const item of jobs) {
if (item.id !== id && item.status === 'waiting') item.cutIn = false
}
return structuredClone(job)
})
await kickStudioQueue()
return updated
}
function pruneDone(jobs: StudioJob[]) {
const cutoff = Date.now() - 1000 * 60 * 60 * 24
return jobs.filter(job => {
@@ -585,6 +615,10 @@ function pendingAlive(job: StudioJob) {
))
}
function isTransientComfyError(error?: string) {
return /still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker/i.test(String(error || ''))
}
function repairStaleJobs(jobs: StudioJob[]) {
for (const job of jobs) {
if (job.status === 'error' && /allowIdentityRefs is not defined/i.test(String(job.lastError || ''))) {
@@ -594,6 +628,13 @@ function repairStaleJobs(jobs: StudioJob[]) {
job.updatedAt = Date.now()
continue
}
if (job.status === 'error' && isTransientComfyError(job.lastError)) {
job.status = 'waiting'
job.liveJobId = undefined
job.lastError = undefined
job.updatedAt = Date.now()
continue
}
if (job.status !== 'running') continue
const live = job.liveJobId ? getJob(job.liveJobId) : undefined
const liveBusy = live && (live.status === 'queued' || live.status === 'uploading' || live.status === 'running')
@@ -1123,10 +1164,6 @@ function remainingStudioShots(job: Job) {
return Math.max(0, (library.chainTotal || 1) - (library.chainStep || 1))
}
function isTransientComfyError(error?: string) {
return /still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker/i.test(String(error || ''))
}
export async function onLiveVideoSettled(job: Job) {
const owner = job.library?.ownerKey
if (!owner) {