Add Force clear for stuck queues including running YuE jobs.

Queue page and Music get a nuclear clear that cancels studio/live work, wipes shot batches, and force-stops Comfy so images/video/other music can run again.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-09-03 23:03:05 -05:00
co-authored by Cursor
parent 7da9e66f43
commit 780b407b3c
7 changed files with 105 additions and 10 deletions
+2 -2
View File
@@ -84,7 +84,7 @@
type="button"
class="block w-full rounded-xl px-3 py-2 text-left text-sm text-red-200 hover:bg-white/5 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
:disabled="gpuForceResetting || gpuSleeping || gpuWaking"
title="Clear stuck jobs and kill Comfy even if a generation is wedged"
title="Clear stuck Studio/music/shot jobs and kill Comfy even if a generation is wedged"
role="menuitem"
@click="forceResetGpu"
>{{ gpuForceResetting ? 'Force resetting…' : 'Force reset' }}</button>
@@ -275,7 +275,7 @@ async function sleepGpu() {
async function forceResetGpu() {
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
const ok = window.confirm(
'Force reset clears stuck Studio/queue jobs and kills Comfy even if a generation is wedged. Continue?'
'Force reset cancels every running/waiting Studio and music job, wipes shot batches, and kills Comfy even if wedged. Continue?'
)
if (!ok) return
menuOpen.value = false
+1 -1
View File
@@ -735,7 +735,7 @@ async function wakeGpu() {
async function forceResetGpu() {
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
const ok = window.confirm(
'Force reset clears stuck Studio/queue jobs and kills Comfy even if a generation is wedged. Continue?'
'Force reset cancels every running/waiting Studio and music job, wipes shot batches, and kills Comfy even if wedged. Continue?'
)
if (!ok) return
gpuForceResetting.value = true
+39
View File
@@ -173,6 +173,16 @@
/>
{{ busy ? (status || 'Generating…') : 'Generate' }}
</button>
<button
v-if="busy || queued"
type="button"
class="inline-flex w-full items-center justify-center rounded-2xl border border-red-400/40 px-4 py-2.5 text-sm text-red-200 hover:border-red-300 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
:disabled="forceClearing"
title="Cancel stuck music/studio jobs and kill Comfy if wedged"
@click="forceClearQueue"
>
{{ forceClearing ? 'Force clearing…' : 'Force clear stuck queue' }}
</button>
<p v-if="blockReason" class="text-xs text-zinc-400">{{ blockReason }}</p>
<p v-else-if="queued" class="text-xs text-zinc-400">
Waiting in the job queue.
@@ -270,6 +280,7 @@ const lyricsEl = ref<HTMLTextAreaElement | null>(null)
const ace15 = ref(false)
const engineFamily = ref<'ace' | 'yue'>('ace')
const busy = ref(false)
const forceClearing = ref(false)
const error = ref('')
const status = ref('')
const progress = ref(0)
@@ -568,6 +579,34 @@ function listen(id: string) {
}
}
async function forceClearQueue() {
if (forceClearing.value) return
if (!confirm(
'Force clear cancels every running and waiting job (including this track), wipes shot batches, and kills Comfy if wedged. Continue?'
)) return
forceClearing.value = true
error.value = ''
try {
await $fetch('/api/comfy/force-reset', { method: 'POST' })
settled = true
busy.value = false
queued.value = false
progress.value = 0
status.value = 'Queue force-cleared. Poke Comfy when you want the GPU again.'
studioJobId.value = ''
liveMusicId.value = ''
jobId.value = ''
if (source) {
source.close()
source = null
}
} catch (err: any) {
error.value = err?.data?.statusMessage || err?.statusMessage || err?.message || 'Could not force-clear the queue'
} finally {
forceClearing.value = false
}
}
async function generate() {
if (blocked.value) {
error.value = blockReason.value
+40 -4
View File
@@ -4,6 +4,7 @@
subtitle="Queue"
current="queue"
:queue-count="waitingCount || pendingCount"
@gpu="loadQueues"
/>
<main class="mx-auto min-w-0 max-w-7xl space-y-6 px-4 py-6 sm:px-6">
@@ -31,15 +32,25 @@
<button
type="button"
class="min-h-10 rounded-full border border-red-400/40 px-3 py-1.5 text-xs text-red-200 hover:border-red-300 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
:disabled="!clearableCount || Boolean(busyId)"
:disabled="!clearableCount || Boolean(busyId) || forceClearing"
@click="clearAll"
>
Clear queue
Clear waiting
</button>
<button
type="button"
class="min-h-10 rounded-full bg-red-500/90 px-3 py-1.5 text-xs font-semibold text-white hover:bg-red-400 disabled:cursor-not-allowed disabled:opacity-50"
:disabled="forceClearing || Boolean(busyId)"
title="Cancel running and waiting jobs, wipe shot batches, and kill Comfy if it is wedged"
@click="forceClearAll"
>
{{ forceClearing ? 'Force clearing…' : 'Force clear' }}
</button>
</div>
</div>
<p v-if="error" class="rounded-2xl border border-red-400/30 bg-red-400/10 px-4 py-3 text-sm text-red-200">{{ error }}</p>
<p v-else-if="forceClearNote" class="rounded-2xl border border-emerald-400/30 bg-emerald-400/10 px-4 py-3 text-sm text-emerald-100">{{ forceClearNote }}</p>
<p v-else-if="willPauseLatch" class="rounded-2xl border border-amber-300/30 bg-amber-400/10 px-4 py-3 text-sm text-amber-100">
Will pause after this video. The current clip keeps generating; this batch will stop after it finishes.
</p>
@@ -557,6 +568,8 @@ const pendingCount = ref(0)
const queuePaused = ref(false)
const error = ref('')
const busyId = ref('')
const forceClearing = ref(false)
const forceClearNote = ref('')
const liveMessage = ref<Record<string, string>>({})
const dirty = new Set<string>()
const sources = new Map<string, EventSource>()
@@ -1194,11 +1207,11 @@ async function remove(queue: Queue) {
async function clearAll() {
const n = clearableCount.value
if (!n) {
error.value = 'Pause the job that’s generating before clearing the queue.'
error.value = 'Nothing waiting to clear. Use Force clear if a running job is stuck.'
return
}
const running = jobs.value.some(job => job.status === 'running') || orphans.value.some(queue => queue.status === 'running')
const extra = running ? ' The job that’s generating will be left until you stop it.' : ''
const extra = running ? ' The job that’s generating will be left until you Force clear or cancel it.' : ''
if (!confirm(`Clear waiting and paused jobs from the queue? Saved clips stay in the library.${extra}`)) return
error.value = ''
try {
@@ -1212,6 +1225,29 @@ async function clearAll() {
}
}
async function forceClearAll() {
if (forceClearing.value) return
if (!confirm(
'Force clear cancels every running and waiting job (including stuck YuE/music), wipes shot batches, and kills Comfy if it is wedged. Saved library clips stay. Continue?'
)) return
forceClearing.value = true
error.value = ''
forceClearNote.value = ''
try {
const result = await $fetch<{
message?: string
cleared?: { cancelledStudio?: number; clearedPending?: number; cancelledLive?: number; clearedShotQueues?: number }
}>('/api/comfy/force-reset', { method: 'POST' })
const c = result.cleared || {}
await loadQueues()
forceClearNote.value = `Force cleared · studio ${c.cancelledStudio || 0}, live ${c.cancelledLive || 0}, shots ${c.clearedShotQueues || 0}. Comfy is stopped — Poke when ready.`
} catch (err: any) {
error.value = err?.data?.statusMessage || err?.statusMessage || err?.message || 'Could not force-clear the queue'
} finally {
forceClearing.value = false
}
}
let saveTimer: ReturnType<typeof setTimeout> | null = null
function saveSegment(queue: Queue, segment: Segment) {
markDirty(queue.id, segment.index)
+1 -1
View File
@@ -8,7 +8,7 @@ export default defineEventHandler(async (event) => {
const comfy = await requestComfyForceReset()
return {
ok: true,
message: 'Cleared stuck jobs and force-stopped Comfy. Use Wake / Poke when you want the GPU again.',
message: 'Cleared every stuck Studio/music/shot job and force-stopped Comfy. Use Wake / Poke when you want the GPU again.',
cleared,
comfy
}
+14
View File
@@ -484,6 +484,20 @@ export async function clearShotQueues(owner: string) {
})
}
/** Nuclear clear: wipe every shot batch for this owner, including ones marked running. */
export async function forceClearAllShotQueues(owner: string) {
const before = listShotQueues(owner)
for (const queue of before) {
markShotQueueCancelled(queue.id)
activeJobs.delete(queue.id)
}
return mutate(owner, (queues) => {
const removed = queues.length
queues.splice(0, queues.length)
return { removed }
})
}
export function listOwnersWithQueues() {
const root = join(libraryRoot(), 'users')
if (!existsSync(root)) return []
+8 -2
View File
@@ -483,7 +483,7 @@ export async function cancelStudioJob(owner: string, id: string) {
return result
}
/** Wipe waiting/running/held studio rows + live/pending jobs for this owner (stuck recovery). */
/** Wipe waiting/running/held studio rows + live/pending/shot queues for this owner (stuck recovery). */
export async function clearStuckStudioWork(owner: string) {
const liveIds = new Set<string>()
const cancelledStudio: string[] = []
@@ -530,10 +530,16 @@ export async function clearStuckStudioWork(owner: string) {
for (const id of liveIds) deletePendingJob(id)
const { forceClearAllShotQueues } = await import('~/server/utils/shotQueue')
const shots = await forceClearAllShotQueues(owner).catch(() => ({ removed: 0 }))
kickStudioQueue()
return {
cancelledStudio: cancelledStudio.length,
clearedPending: clearedPending.length,
cancelledLive: liveIds.size
cancelledLive: liveIds.size,
clearedShotQueues: Number(shots?.removed || 0)
}
}