Let failed studio jobs retry instead of sitting in the queue with only Remove.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1167,6 +1167,14 @@
|
||||
>
|
||||
{{ job.cutIn ? 'Next' : (canCutInStudioJob ? 'Run next' : 'Start now') }}
|
||||
</button>
|
||||
<button
|
||||
v-if="job.status === 'error' || (job.status === 'held' && !job.holdForCutIn)"
|
||||
type="button"
|
||||
class="shrink-0 rounded-full border border-amber-300/40 px-2.5 py-1 text-xs text-amber-100 hover:border-amber-200 hover:text-white"
|
||||
@click="retryStudioJob(job.id)"
|
||||
>
|
||||
Retry
|
||||
</button>
|
||||
<button
|
||||
v-if="job.status === 'waiting' || job.status === 'held' || job.status === 'running' || job.status === 'error'"
|
||||
type="button"
|
||||
@@ -5085,6 +5093,16 @@ async function cutInStudioJob(id: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function retryStudioJob(id: string) {
|
||||
try {
|
||||
await $fetch(`/api/studio-queue/${id}/retry`, { method: 'POST' })
|
||||
toast('Queued to run as soon as the GPU is free.')
|
||||
await refreshStudioQueue()
|
||||
} catch (error: any) {
|
||||
toast(error?.data?.statusMessage || error?.statusMessage || 'Could not retry that job')
|
||||
}
|
||||
}
|
||||
|
||||
async function dismissStudioJob(id: string) {
|
||||
try {
|
||||
await $fetch(`/api/studio-queue/${id}`, { method: 'DELETE' })
|
||||
|
||||
@@ -84,6 +84,15 @@
|
||||
>
|
||||
{{ job.cutIn ? 'Next after current shot' : (canCutIn ? 'Run next' : 'Start now') }}
|
||||
</button>
|
||||
<button
|
||||
v-if="job.status === 'error'"
|
||||
type="button"
|
||||
class="rounded-full border border-amber-300/40 px-3 py-1.5 text-xs text-amber-100 hover:border-amber-200 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
:disabled="Boolean(busyId)"
|
||||
@click="retryJob(job)"
|
||||
>
|
||||
Retry
|
||||
</button>
|
||||
<template v-if="job.shots">
|
||||
<template v-if="canProcessShots(job)">
|
||||
<button
|
||||
@@ -756,6 +765,19 @@ async function cutIn(job: StudioJobRow) {
|
||||
}
|
||||
}
|
||||
|
||||
async function retryJob(job: StudioJobRow) {
|
||||
busyId.value = job.id
|
||||
error.value = ''
|
||||
try {
|
||||
await $fetch(`/api/studio-queue/${job.id}/retry`, { method: 'POST' })
|
||||
await loadQueues()
|
||||
} catch (err: any) {
|
||||
error.value = err?.data?.statusMessage || err?.statusMessage || err?.message || 'Could not retry that job'
|
||||
} finally {
|
||||
busyId.value = ''
|
||||
}
|
||||
}
|
||||
|
||||
function listenJob(job: StudioJobRow) {
|
||||
if (!job.liveJobId || sources.has(job.id)) return
|
||||
const source = new EventSource(`/api/generate/${job.liveJobId}/stream`)
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { retryStudioJob } from '~/server/utils/studioQueue'
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
const { owner } = assertLibraryOwner(event)
|
||||
const id = String(getRouterParam(event, 'id') || '')
|
||||
return await retryStudioJob(owner, id)
|
||||
})
|
||||
@@ -492,6 +492,36 @@ export async function requestCutIn(owner: string, id: string) {
|
||||
return updated
|
||||
}
|
||||
|
||||
export async function retryStudioJob(owner: string, id: string) {
|
||||
const jobsNow = readJobs(owner)
|
||||
const target = jobsNow.find(job => job.id === id)
|
||||
if (!target) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' })
|
||||
if (target.status !== 'error' && target.status !== 'held' && target.status !== 'waiting') {
|
||||
throw createError({ statusCode: 409, statusMessage: 'Only a failed or paused job can be retried' })
|
||||
}
|
||||
const updated = await mutate(owner, (jobs) => {
|
||||
const job = jobs.find(item => item.id === id)
|
||||
if (!job) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' })
|
||||
if (job.status !== 'error' && job.status !== 'held' && job.status !== 'waiting') {
|
||||
throw createError({ statusCode: 409, statusMessage: 'Only a failed or paused job can be retried' })
|
||||
}
|
||||
job.status = 'waiting'
|
||||
job.cutIn = true
|
||||
job.lastError = undefined
|
||||
job.liveJobId = undefined
|
||||
job.pausedByUser = false
|
||||
job.pauseAfterCurrent = false
|
||||
job.holdForCutIn = false
|
||||
job.updatedAt = Date.now()
|
||||
for (const item of jobs) {
|
||||
if (item.id !== id && item.status === 'waiting') item.cutIn = false
|
||||
}
|
||||
return structuredClone(job)
|
||||
})
|
||||
await kickStudioQueue()
|
||||
return updated
|
||||
}
|
||||
|
||||
function pruneDone(jobs: StudioJob[]) {
|
||||
const cutoff = Date.now() - 1000 * 60 * 60 * 24
|
||||
return jobs.filter(job => {
|
||||
@@ -585,6 +615,10 @@ function pendingAlive(job: StudioJob) {
|
||||
))
|
||||
}
|
||||
|
||||
function isTransientComfyError(error?: string) {
|
||||
return /still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker/i.test(String(error || ''))
|
||||
}
|
||||
|
||||
function repairStaleJobs(jobs: StudioJob[]) {
|
||||
for (const job of jobs) {
|
||||
if (job.status === 'error' && /allowIdentityRefs is not defined/i.test(String(job.lastError || ''))) {
|
||||
@@ -594,6 +628,13 @@ function repairStaleJobs(jobs: StudioJob[]) {
|
||||
job.updatedAt = Date.now()
|
||||
continue
|
||||
}
|
||||
if (job.status === 'error' && isTransientComfyError(job.lastError)) {
|
||||
job.status = 'waiting'
|
||||
job.liveJobId = undefined
|
||||
job.lastError = undefined
|
||||
job.updatedAt = Date.now()
|
||||
continue
|
||||
}
|
||||
if (job.status !== 'running') continue
|
||||
const live = job.liveJobId ? getJob(job.liveJobId) : undefined
|
||||
const liveBusy = live && (live.status === 'queued' || live.status === 'uploading' || live.status === 'running')
|
||||
@@ -1123,10 +1164,6 @@ function remainingStudioShots(job: Job) {
|
||||
return Math.max(0, (library.chainTotal || 1) - (library.chainStep || 1))
|
||||
}
|
||||
|
||||
function isTransientComfyError(error?: string) {
|
||||
return /still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker/i.test(String(error || ''))
|
||||
}
|
||||
|
||||
export async function onLiveVideoSettled(job: Job) {
|
||||
const owner = job.library?.ownerKey
|
||||
if (!owner) {
|
||||
|
||||
Reference in New Issue
Block a user