diff --git a/pages/index.vue b/pages/index.vue index 1537b5d..eb7c173 100644 --- a/pages/index.vue +++ b/pages/index.vue @@ -1167,6 +1167,14 @@ > {{ job.cutIn ? 'Next' : (canCutInStudioJob ? 'Run next' : 'Start now') }} + + Retry + {{ job.cutIn ? 'Next after current shot' : (canCutIn ? 'Run next' : 'Start now') }} + + Retry + { + const { owner } = assertLibraryOwner(event) + const id = String(getRouterParam(event, 'id') || '') + return await retryStudioJob(owner, id) +}) diff --git a/server/utils/studioQueue.ts b/server/utils/studioQueue.ts index c057fff..e7d98de 100644 --- a/server/utils/studioQueue.ts +++ b/server/utils/studioQueue.ts @@ -492,6 +492,36 @@ export async function requestCutIn(owner: string, id: string) { return updated } +export async function retryStudioJob(owner: string, id: string) { + const jobsNow = readJobs(owner) + const target = jobsNow.find(job => job.id === id) + if (!target) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' }) + if (target.status !== 'error' && target.status !== 'held' && target.status !== 'waiting') { + throw createError({ statusCode: 409, statusMessage: 'Only a failed or paused job can be retried' }) + } + const updated = await mutate(owner, (jobs) => { + const job = jobs.find(item => item.id === id) + if (!job) throw createError({ statusCode: 404, statusMessage: 'Queued job not found' }) + if (job.status !== 'error' && job.status !== 'held' && job.status !== 'waiting') { + throw createError({ statusCode: 409, statusMessage: 'Only a failed or paused job can be retried' }) + } + job.status = 'waiting' + job.cutIn = true + job.lastError = undefined + job.liveJobId = undefined + job.pausedByUser = false + job.pauseAfterCurrent = false + job.holdForCutIn = false + job.updatedAt = Date.now() + for (const item of jobs) { + if (item.id !== id && item.status === 'waiting') item.cutIn = false + } + return structuredClone(job) + }) + await kickStudioQueue() + return updated +} + function pruneDone(jobs: StudioJob[]) { const cutoff = Date.now() - 1000 * 60 * 60 * 24 return jobs.filter(job => { @@ -585,6 +615,10 @@ function pendingAlive(job: StudioJob) { )) } +function isTransientComfyError(error?: string) { + return /still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker/i.test(String(error || '')) +} + function repairStaleJobs(jobs: StudioJob[]) { for (const job of jobs) { if (job.status === 'error' && /allowIdentityRefs is not defined/i.test(String(job.lastError || ''))) { @@ -594,6 +628,13 @@ function repairStaleJobs(jobs: StudioJob[]) { job.updatedAt = Date.now() continue } + if (job.status === 'error' && isTransientComfyError(job.lastError)) { + job.status = 'waiting' + job.liveJobId = undefined + job.lastError = undefined + job.updatedAt = Date.now() + continue + } if (job.status !== 'running') continue const live = job.liveJobId ? getJob(job.liveJobId) : undefined const liveBusy = live && (live.status === 'queued' || live.status === 'uploading' || live.status === 'running') @@ -1123,10 +1164,6 @@ function remainingStudioShots(job: Job) { return Math.max(0, (library.chainTotal || 1) - (library.chainStep || 1)) } -function isTransientComfyError(error?: string) { - return /still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker/i.test(String(error || '')) -} - export async function onLiveVideoSettled(job: Job) { const owner = job.library?.ownerKey if (!owner) {