Let failed studio jobs retry instead of sitting in the queue with only Remove.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-09-02 08:19:21 -05:00
co-authored by Cursor
parent bdec8872b6
commit 1314e8e89f
4 changed files with 88 additions and 4 deletions
+18
View File
@@ -1167,6 +1167,14 @@
>
{{ job.cutIn ? 'Next' : (canCutInStudioJob ? 'Run next' : 'Start now') }}
</button>
<button
v-if="job.status === 'error' || (job.status === 'held' && !job.holdForCutIn)"
type="button"
class="shrink-0 rounded-full border border-amber-300/40 px-2.5 py-1 text-xs text-amber-100 hover:border-amber-200 hover:text-white"
@click="retryStudioJob(job.id)"
>
Retry
</button>
<button
v-if="job.status === 'waiting' || job.status === 'held' || job.status === 'running' || job.status === 'error'"
type="button"
@@ -5085,6 +5093,16 @@ async function cutInStudioJob(id: string) {
}
}
async function retryStudioJob(id: string) {
try {
await $fetch(`/api/studio-queue/${id}/retry`, { method: 'POST' })
toast('Queued to run as soon as the GPU is free.')
await refreshStudioQueue()
} catch (error: any) {
toast(error?.data?.statusMessage || error?.statusMessage || 'Could not retry that job')
}
}
async function dismissStudioJob(id: string) {
try {
await $fetch(`/api/studio-queue/${id}`, { method: 'DELETE' })
+22
View File
@@ -84,6 +84,15 @@
>
{{ job.cutIn ? 'Next after current shot' : (canCutIn ? 'Run next' : 'Start now') }}
</button>
<button
v-if="job.status === 'error'"
type="button"
class="rounded-full border border-amber-300/40 px-3 py-1.5 text-xs text-amber-100 hover:border-amber-200 disabled:cursor-not-allowed disabled:opacity-50"
:disabled="Boolean(busyId)"
@click="retryJob(job)"
>
Retry
</button>
<template v-if="job.shots">
<template v-if="canProcessShots(job)">
<button
@@ -756,6 +765,19 @@ async function cutIn(job: StudioJobRow) {
}
}
async function retryJob(job: StudioJobRow) {
busyId.value = job.id
error.value = ''
try {
await $fetch(`/api/studio-queue/${job.id}/retry`, { method: 'POST' })
await loadQueues()
} catch (err: any) {
error.value = err?.data?.statusMessage || err?.statusMessage || err?.message || 'Could not retry that job'
} finally {
busyId.value = ''
}
}
function listenJob(job: StudioJobRow) {
if (!job.liveJobId || sources.has(job.id)) return
const source = new EventSource(`/api/generate/${job.liveJobId}/stream`)
@@ -0,0 +1,7 @@
import { retryStudioJob } from '~/server/utils/studioQueue'
export default defineEventHandler(async (event) => {
const { owner } = assertLibraryOwner(event)
const id = String(getRouterParam(event, 'id') || '')
return await retryStudioJob(owner, id)
})
+41 -4
View File
@@ -492,6 +492,36 @@ export async function requestCutIn(owner: string, id: string) {
return updated
}
export async function retryStudioJob(owner: string, id: string) {
const jobsNow = readJobs(owner)
const target = jobsNow.find(job => job.id === id)
if (!target) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' })
if (target.status !== 'error' && target.status !== 'held' && target.status !== 'waiting') {
throw createError({ statusCode: 409, statusMessage: 'Only a failed or paused job can be retried' })
}
const updated = await mutate(owner, (jobs) => {
const job = jobs.find(item => item.id === id)
if (!job) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' })
if (job.status !== 'error' && job.status !== 'held' && job.status !== 'waiting') {
throw createError({ statusCode: 409, statusMessage: 'Only a failed or paused job can be retried' })
}
job.status = 'waiting'
job.cutIn = true
job.lastError = undefined
job.liveJobId = undefined
job.pausedByUser = false
job.pauseAfterCurrent = false
job.holdForCutIn = false
job.updatedAt = Date.now()
for (const item of jobs) {
if (item.id !== id && item.status === 'waiting') item.cutIn = false
}
return structuredClone(job)
})
await kickStudioQueue()
return updated
}
function pruneDone(jobs: StudioJob[]) {
const cutoff = Date.now() - 1000 * 60 * 60 * 24
return jobs.filter(job => {
@@ -585,6 +615,10 @@ function pendingAlive(job: StudioJob) {
))
}
function isTransientComfyError(error?: string) {
return /still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker/i.test(String(error || ''))
}
function repairStaleJobs(jobs: StudioJob[]) {
for (const job of jobs) {
if (job.status === 'error' && /allowIdentityRefs is not defined/i.test(String(job.lastError || ''))) {
@@ -594,6 +628,13 @@ function repairStaleJobs(jobs: StudioJob[]) {
job.updatedAt = Date.now()
continue
}
if (job.status === 'error' && isTransientComfyError(job.lastError)) {
job.status = 'waiting'
job.liveJobId = undefined
job.lastError = undefined
job.updatedAt = Date.now()
continue
}
if (job.status !== 'running') continue
const live = job.liveJobId ? getJob(job.liveJobId) : undefined
const liveBusy = live && (live.status === 'queued' || live.status === 'uploading' || live.status === 'running')
@@ -1123,10 +1164,6 @@ function remainingStudioShots(job: Job) {
return Math.max(0, (library.chainTotal || 1) - (library.chainStep || 1))
}
function isTransientComfyError(error?: string) {
return /still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker/i.test(String(error || ''))
}
export async function onLiveVideoSettled(job: Job) {
const owner = job.library?.ownerKey
if (!owner) {