Add Force clear for stuck queues including running YuE jobs.
Queue page and Music get a nuclear clear that cancels studio/live work, wipes shot batches, and force-stops Comfy so images/video/other music can run again. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -84,7 +84,7 @@
|
||||
type="button"
|
||||
class="block w-full rounded-xl px-3 py-2 text-left text-sm text-red-200 hover:bg-white/5 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
:disabled="gpuForceResetting || gpuSleeping || gpuWaking"
|
||||
title="Clear stuck jobs and kill Comfy even if a generation is wedged"
|
||||
title="Clear stuck Studio/music/shot jobs and kill Comfy even if a generation is wedged"
|
||||
role="menuitem"
|
||||
@click="forceResetGpu"
|
||||
>{{ gpuForceResetting ? 'Force resetting…' : 'Force reset' }}</button>
|
||||
@@ -275,7 +275,7 @@ async function sleepGpu() {
|
||||
async function forceResetGpu() {
|
||||
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
|
||||
const ok = window.confirm(
|
||||
'Force reset clears stuck Studio/queue jobs and kills Comfy even if a generation is wedged. Continue?'
|
||||
'Force reset cancels every running/waiting Studio and music job, wipes shot batches, and kills Comfy even if wedged. Continue?'
|
||||
)
|
||||
if (!ok) return
|
||||
menuOpen.value = false
|
||||
|
||||
@@ -735,7 +735,7 @@ async function wakeGpu() {
|
||||
async function forceResetGpu() {
|
||||
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
|
||||
const ok = window.confirm(
|
||||
'Force reset clears stuck Studio/queue jobs and kills Comfy even if a generation is wedged. Continue?'
|
||||
'Force reset cancels every running/waiting Studio and music job, wipes shot batches, and kills Comfy even if wedged. Continue?'
|
||||
)
|
||||
if (!ok) return
|
||||
gpuForceResetting.value = true
|
||||
|
||||
@@ -173,6 +173,16 @@
|
||||
/>
|
||||
{{ busy ? (status || 'Generating…') : 'Generate' }}
|
||||
</button>
|
||||
<button
|
||||
v-if="busy || queued"
|
||||
type="button"
|
||||
class="inline-flex w-full items-center justify-center rounded-2xl border border-red-400/40 px-4 py-2.5 text-sm text-red-200 hover:border-red-300 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
:disabled="forceClearing"
|
||||
title="Cancel stuck music/studio jobs and kill Comfy if wedged"
|
||||
@click="forceClearQueue"
|
||||
>
|
||||
{{ forceClearing ? 'Force clearing…' : 'Force clear stuck queue' }}
|
||||
</button>
|
||||
<p v-if="blockReason" class="text-xs text-zinc-400">{{ blockReason }}</p>
|
||||
<p v-else-if="queued" class="text-xs text-zinc-400">
|
||||
Waiting in the job queue.
|
||||
@@ -270,6 +280,7 @@ const lyricsEl = ref<HTMLTextAreaElement | null>(null)
|
||||
const ace15 = ref(false)
|
||||
const engineFamily = ref<'ace' | 'yue'>('ace')
|
||||
const busy = ref(false)
|
||||
const forceClearing = ref(false)
|
||||
const error = ref('')
|
||||
const status = ref('')
|
||||
const progress = ref(0)
|
||||
@@ -568,6 +579,34 @@ function listen(id: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function forceClearQueue() {
|
||||
if (forceClearing.value) return
|
||||
if (!confirm(
|
||||
'Force clear cancels every running and waiting job (including this track), wipes shot batches, and kills Comfy if wedged. Continue?'
|
||||
)) return
|
||||
forceClearing.value = true
|
||||
error.value = ''
|
||||
try {
|
||||
await $fetch('/api/comfy/force-reset', { method: 'POST' })
|
||||
settled = true
|
||||
busy.value = false
|
||||
queued.value = false
|
||||
progress.value = 0
|
||||
status.value = 'Queue force-cleared. Poke Comfy when you want the GPU again.'
|
||||
studioJobId.value = ''
|
||||
liveMusicId.value = ''
|
||||
jobId.value = ''
|
||||
if (source) {
|
||||
source.close()
|
||||
source = null
|
||||
}
|
||||
} catch (err: any) {
|
||||
error.value = err?.data?.statusMessage || err?.statusMessage || err?.message || 'Could not force-clear the queue'
|
||||
} finally {
|
||||
forceClearing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function generate() {
|
||||
if (blocked.value) {
|
||||
error.value = blockReason.value
|
||||
|
||||
+40
-4
@@ -4,6 +4,7 @@
|
||||
subtitle="Queue"
|
||||
current="queue"
|
||||
:queue-count="waitingCount || pendingCount"
|
||||
@gpu="loadQueues"
|
||||
/>
|
||||
|
||||
<main class="mx-auto min-w-0 max-w-7xl space-y-6 px-4 py-6 sm:px-6">
|
||||
@@ -31,15 +32,25 @@
|
||||
<button
|
||||
type="button"
|
||||
class="min-h-10 rounded-full border border-red-400/40 px-3 py-1.5 text-xs text-red-200 hover:border-red-300 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
:disabled="!clearableCount || Boolean(busyId)"
|
||||
:disabled="!clearableCount || Boolean(busyId) || forceClearing"
|
||||
@click="clearAll"
|
||||
>
|
||||
Clear queue
|
||||
Clear waiting
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
class="min-h-10 rounded-full bg-red-500/90 px-3 py-1.5 text-xs font-semibold text-white hover:bg-red-400 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
:disabled="forceClearing || Boolean(busyId)"
|
||||
title="Cancel running and waiting jobs, wipe shot batches, and kill Comfy if it is wedged"
|
||||
@click="forceClearAll"
|
||||
>
|
||||
{{ forceClearing ? 'Force clearing…' : 'Force clear' }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p v-if="error" class="rounded-2xl border border-red-400/30 bg-red-400/10 px-4 py-3 text-sm text-red-200">{{ error }}</p>
|
||||
<p v-else-if="forceClearNote" class="rounded-2xl border border-emerald-400/30 bg-emerald-400/10 px-4 py-3 text-sm text-emerald-100">{{ forceClearNote }}</p>
|
||||
<p v-else-if="willPauseLatch" class="rounded-2xl border border-amber-300/30 bg-amber-400/10 px-4 py-3 text-sm text-amber-100">
|
||||
Will pause after this video. The current clip keeps generating; this batch will stop after it finishes.
|
||||
</p>
|
||||
@@ -557,6 +568,8 @@ const pendingCount = ref(0)
|
||||
const queuePaused = ref(false)
|
||||
const error = ref('')
|
||||
const busyId = ref('')
|
||||
const forceClearing = ref(false)
|
||||
const forceClearNote = ref('')
|
||||
const liveMessage = ref<Record<string, string>>({})
|
||||
const dirty = new Set<string>()
|
||||
const sources = new Map<string, EventSource>()
|
||||
@@ -1194,11 +1207,11 @@ async function remove(queue: Queue) {
|
||||
async function clearAll() {
|
||||
const n = clearableCount.value
|
||||
if (!n) {
|
||||
error.value = 'Pause the job that’s generating before clearing the queue.'
|
||||
error.value = 'Nothing waiting to clear. Use Force clear if a running job is stuck.'
|
||||
return
|
||||
}
|
||||
const running = jobs.value.some(job => job.status === 'running') || orphans.value.some(queue => queue.status === 'running')
|
||||
const extra = running ? ' The job that’s generating will be left until you stop it.' : ''
|
||||
const extra = running ? ' The job that’s generating will be left until you Force clear or cancel it.' : ''
|
||||
if (!confirm(`Clear waiting and paused jobs from the queue? Saved clips stay in the library.${extra}`)) return
|
||||
error.value = ''
|
||||
try {
|
||||
@@ -1212,6 +1225,29 @@ async function clearAll() {
|
||||
}
|
||||
}
|
||||
|
||||
async function forceClearAll() {
|
||||
if (forceClearing.value) return
|
||||
if (!confirm(
|
||||
'Force clear cancels every running and waiting job (including stuck YuE/music), wipes shot batches, and kills Comfy if it is wedged. Saved library clips stay. Continue?'
|
||||
)) return
|
||||
forceClearing.value = true
|
||||
error.value = ''
|
||||
forceClearNote.value = ''
|
||||
try {
|
||||
const result = await $fetch<{
|
||||
message?: string
|
||||
cleared?: { cancelledStudio?: number; clearedPending?: number; cancelledLive?: number; clearedShotQueues?: number }
|
||||
}>('/api/comfy/force-reset', { method: 'POST' })
|
||||
const c = result.cleared || {}
|
||||
await loadQueues()
|
||||
forceClearNote.value = `Force cleared · studio ${c.cancelledStudio || 0}, live ${c.cancelledLive || 0}, shots ${c.clearedShotQueues || 0}. Comfy is stopped — Poke when ready.`
|
||||
} catch (err: any) {
|
||||
error.value = err?.data?.statusMessage || err?.statusMessage || err?.message || 'Could not force-clear the queue'
|
||||
} finally {
|
||||
forceClearing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
let saveTimer: ReturnType<typeof setTimeout> | null = null
|
||||
function saveSegment(queue: Queue, segment: Segment) {
|
||||
markDirty(queue.id, segment.index)
|
||||
|
||||
@@ -8,7 +8,7 @@ export default defineEventHandler(async (event) => {
|
||||
const comfy = await requestComfyForceReset()
|
||||
return {
|
||||
ok: true,
|
||||
message: 'Cleared stuck jobs and force-stopped Comfy. Use Wake / Poke when you want the GPU again.',
|
||||
message: 'Cleared every stuck Studio/music/shot job and force-stopped Comfy. Use Wake / Poke when you want the GPU again.',
|
||||
cleared,
|
||||
comfy
|
||||
}
|
||||
|
||||
@@ -484,6 +484,20 @@ export async function clearShotQueues(owner: string) {
|
||||
})
|
||||
}
|
||||
|
||||
/** Nuclear clear: wipe every shot batch for this owner, including ones marked running. */
|
||||
export async function forceClearAllShotQueues(owner: string) {
|
||||
const before = listShotQueues(owner)
|
||||
for (const queue of before) {
|
||||
markShotQueueCancelled(queue.id)
|
||||
activeJobs.delete(queue.id)
|
||||
}
|
||||
return mutate(owner, (queues) => {
|
||||
const removed = queues.length
|
||||
queues.splice(0, queues.length)
|
||||
return { removed }
|
||||
})
|
||||
}
|
||||
|
||||
export function listOwnersWithQueues() {
|
||||
const root = join(libraryRoot(), 'users')
|
||||
if (!existsSync(root)) return []
|
||||
|
||||
@@ -483,7 +483,7 @@ export async function cancelStudioJob(owner: string, id: string) {
|
||||
return result
|
||||
}
|
||||
|
||||
/** Wipe waiting/running/held studio rows + live/pending jobs for this owner (stuck recovery). */
|
||||
/** Wipe waiting/running/held studio rows + live/pending/shot queues for this owner (stuck recovery). */
|
||||
export async function clearStuckStudioWork(owner: string) {
|
||||
const liveIds = new Set<string>()
|
||||
const cancelledStudio: string[] = []
|
||||
@@ -530,10 +530,16 @@ export async function clearStuckStudioWork(owner: string) {
|
||||
|
||||
for (const id of liveIds) deletePendingJob(id)
|
||||
|
||||
const { forceClearAllShotQueues } = await import('~/server/utils/shotQueue')
|
||||
const shots = await forceClearAllShotQueues(owner).catch(() => ({ removed: 0 }))
|
||||
|
||||
kickStudioQueue()
|
||||
|
||||
return {
|
||||
cancelledStudio: cancelledStudio.length,
|
||||
clearedPending: clearedPending.length,
|
||||
cancelledLive: liveIds.size
|
||||
cancelledLive: liveIds.size,
|
||||
clearedShotQueues: Number(shots?.removed || 0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user