Add Force clear for stuck queues including running YuE jobs.
Queue page and Music get a nuclear clear that cancels studio/live work, wipes shot batches, and force-stops Comfy so images/video/other music can run again. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -84,7 +84,7 @@
|
|||||||
type="button"
|
type="button"
|
||||||
class="block w-full rounded-xl px-3 py-2 text-left text-sm text-red-200 hover:bg-white/5 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
|
class="block w-full rounded-xl px-3 py-2 text-left text-sm text-red-200 hover:bg-white/5 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
:disabled="gpuForceResetting || gpuSleeping || gpuWaking"
|
:disabled="gpuForceResetting || gpuSleeping || gpuWaking"
|
||||||
title="Clear stuck jobs and kill Comfy even if a generation is wedged"
|
title="Clear stuck Studio/music/shot jobs and kill Comfy even if a generation is wedged"
|
||||||
role="menuitem"
|
role="menuitem"
|
||||||
@click="forceResetGpu"
|
@click="forceResetGpu"
|
||||||
>{{ gpuForceResetting ? 'Force resetting…' : 'Force reset' }}</button>
|
>{{ gpuForceResetting ? 'Force resetting…' : 'Force reset' }}</button>
|
||||||
@@ -275,7 +275,7 @@ async function sleepGpu() {
|
|||||||
async function forceResetGpu() {
|
async function forceResetGpu() {
|
||||||
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
|
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
|
||||||
const ok = window.confirm(
|
const ok = window.confirm(
|
||||||
'Force reset clears stuck Studio/queue jobs and kills Comfy even if a generation is wedged. Continue?'
|
'Force reset cancels every running/waiting Studio and music job, wipes shot batches, and kills Comfy even if wedged. Continue?'
|
||||||
)
|
)
|
||||||
if (!ok) return
|
if (!ok) return
|
||||||
menuOpen.value = false
|
menuOpen.value = false
|
||||||
|
|||||||
@@ -735,7 +735,7 @@ async function wakeGpu() {
|
|||||||
async function forceResetGpu() {
|
async function forceResetGpu() {
|
||||||
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
|
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
|
||||||
const ok = window.confirm(
|
const ok = window.confirm(
|
||||||
'Force reset clears stuck Studio/queue jobs and kills Comfy even if a generation is wedged. Continue?'
|
'Force reset cancels every running/waiting Studio and music job, wipes shot batches, and kills Comfy even if wedged. Continue?'
|
||||||
)
|
)
|
||||||
if (!ok) return
|
if (!ok) return
|
||||||
gpuForceResetting.value = true
|
gpuForceResetting.value = true
|
||||||
|
|||||||
@@ -173,6 +173,16 @@
|
|||||||
/>
|
/>
|
||||||
{{ busy ? (status || 'Generating…') : 'Generate' }}
|
{{ busy ? (status || 'Generating…') : 'Generate' }}
|
||||||
</button>
|
</button>
|
||||||
|
<button
|
||||||
|
v-if="busy || queued"
|
||||||
|
type="button"
|
||||||
|
class="inline-flex w-full items-center justify-center rounded-2xl border border-red-400/40 px-4 py-2.5 text-sm text-red-200 hover:border-red-300 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
|
:disabled="forceClearing"
|
||||||
|
title="Cancel stuck music/studio jobs and kill Comfy if wedged"
|
||||||
|
@click="forceClearQueue"
|
||||||
|
>
|
||||||
|
{{ forceClearing ? 'Force clearing…' : 'Force clear stuck queue' }}
|
||||||
|
</button>
|
||||||
<p v-if="blockReason" class="text-xs text-zinc-400">{{ blockReason }}</p>
|
<p v-if="blockReason" class="text-xs text-zinc-400">{{ blockReason }}</p>
|
||||||
<p v-else-if="queued" class="text-xs text-zinc-400">
|
<p v-else-if="queued" class="text-xs text-zinc-400">
|
||||||
Waiting in the job queue.
|
Waiting in the job queue.
|
||||||
@@ -270,6 +280,7 @@ const lyricsEl = ref<HTMLTextAreaElement | null>(null)
|
|||||||
const ace15 = ref(false)
|
const ace15 = ref(false)
|
||||||
const engineFamily = ref<'ace' | 'yue'>('ace')
|
const engineFamily = ref<'ace' | 'yue'>('ace')
|
||||||
const busy = ref(false)
|
const busy = ref(false)
|
||||||
|
const forceClearing = ref(false)
|
||||||
const error = ref('')
|
const error = ref('')
|
||||||
const status = ref('')
|
const status = ref('')
|
||||||
const progress = ref(0)
|
const progress = ref(0)
|
||||||
@@ -568,6 +579,34 @@ function listen(id: string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function forceClearQueue() {
|
||||||
|
if (forceClearing.value) return
|
||||||
|
if (!confirm(
|
||||||
|
'Force clear cancels every running and waiting job (including this track), wipes shot batches, and kills Comfy if wedged. Continue?'
|
||||||
|
)) return
|
||||||
|
forceClearing.value = true
|
||||||
|
error.value = ''
|
||||||
|
try {
|
||||||
|
await $fetch('/api/comfy/force-reset', { method: 'POST' })
|
||||||
|
settled = true
|
||||||
|
busy.value = false
|
||||||
|
queued.value = false
|
||||||
|
progress.value = 0
|
||||||
|
status.value = 'Queue force-cleared. Poke Comfy when you want the GPU again.'
|
||||||
|
studioJobId.value = ''
|
||||||
|
liveMusicId.value = ''
|
||||||
|
jobId.value = ''
|
||||||
|
if (source) {
|
||||||
|
source.close()
|
||||||
|
source = null
|
||||||
|
}
|
||||||
|
} catch (err: any) {
|
||||||
|
error.value = err?.data?.statusMessage || err?.statusMessage || err?.message || 'Could not force-clear the queue'
|
||||||
|
} finally {
|
||||||
|
forceClearing.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function generate() {
|
async function generate() {
|
||||||
if (blocked.value) {
|
if (blocked.value) {
|
||||||
error.value = blockReason.value
|
error.value = blockReason.value
|
||||||
|
|||||||
+40
-4
@@ -4,6 +4,7 @@
|
|||||||
subtitle="Queue"
|
subtitle="Queue"
|
||||||
current="queue"
|
current="queue"
|
||||||
:queue-count="waitingCount || pendingCount"
|
:queue-count="waitingCount || pendingCount"
|
||||||
|
@gpu="loadQueues"
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<main class="mx-auto min-w-0 max-w-7xl space-y-6 px-4 py-6 sm:px-6">
|
<main class="mx-auto min-w-0 max-w-7xl space-y-6 px-4 py-6 sm:px-6">
|
||||||
@@ -31,15 +32,25 @@
|
|||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="min-h-10 rounded-full border border-red-400/40 px-3 py-1.5 text-xs text-red-200 hover:border-red-300 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
|
class="min-h-10 rounded-full border border-red-400/40 px-3 py-1.5 text-xs text-red-200 hover:border-red-300 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
:disabled="!clearableCount || Boolean(busyId)"
|
:disabled="!clearableCount || Boolean(busyId) || forceClearing"
|
||||||
@click="clearAll"
|
@click="clearAll"
|
||||||
>
|
>
|
||||||
Clear queue
|
Clear waiting
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="min-h-10 rounded-full bg-red-500/90 px-3 py-1.5 text-xs font-semibold text-white hover:bg-red-400 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
|
:disabled="forceClearing || Boolean(busyId)"
|
||||||
|
title="Cancel running and waiting jobs, wipe shot batches, and kill Comfy if it is wedged"
|
||||||
|
@click="forceClearAll"
|
||||||
|
>
|
||||||
|
{{ forceClearing ? 'Force clearing…' : 'Force clear' }}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<p v-if="error" class="rounded-2xl border border-red-400/30 bg-red-400/10 px-4 py-3 text-sm text-red-200">{{ error }}</p>
|
<p v-if="error" class="rounded-2xl border border-red-400/30 bg-red-400/10 px-4 py-3 text-sm text-red-200">{{ error }}</p>
|
||||||
|
<p v-else-if="forceClearNote" class="rounded-2xl border border-emerald-400/30 bg-emerald-400/10 px-4 py-3 text-sm text-emerald-100">{{ forceClearNote }}</p>
|
||||||
<p v-else-if="willPauseLatch" class="rounded-2xl border border-amber-300/30 bg-amber-400/10 px-4 py-3 text-sm text-amber-100">
|
<p v-else-if="willPauseLatch" class="rounded-2xl border border-amber-300/30 bg-amber-400/10 px-4 py-3 text-sm text-amber-100">
|
||||||
Will pause after this video. The current clip keeps generating; this batch will stop after it finishes.
|
Will pause after this video. The current clip keeps generating; this batch will stop after it finishes.
|
||||||
</p>
|
</p>
|
||||||
@@ -557,6 +568,8 @@ const pendingCount = ref(0)
|
|||||||
const queuePaused = ref(false)
|
const queuePaused = ref(false)
|
||||||
const error = ref('')
|
const error = ref('')
|
||||||
const busyId = ref('')
|
const busyId = ref('')
|
||||||
|
const forceClearing = ref(false)
|
||||||
|
const forceClearNote = ref('')
|
||||||
const liveMessage = ref<Record<string, string>>({})
|
const liveMessage = ref<Record<string, string>>({})
|
||||||
const dirty = new Set<string>()
|
const dirty = new Set<string>()
|
||||||
const sources = new Map<string, EventSource>()
|
const sources = new Map<string, EventSource>()
|
||||||
@@ -1194,11 +1207,11 @@ async function remove(queue: Queue) {
|
|||||||
async function clearAll() {
|
async function clearAll() {
|
||||||
const n = clearableCount.value
|
const n = clearableCount.value
|
||||||
if (!n) {
|
if (!n) {
|
||||||
error.value = 'Pause the job that’s generating before clearing the queue.'
|
error.value = 'Nothing waiting to clear. Use Force clear if a running job is stuck.'
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
const running = jobs.value.some(job => job.status === 'running') || orphans.value.some(queue => queue.status === 'running')
|
const running = jobs.value.some(job => job.status === 'running') || orphans.value.some(queue => queue.status === 'running')
|
||||||
const extra = running ? ' The job that’s generating will be left until you stop it.' : ''
|
const extra = running ? ' The job that’s generating will be left until you Force clear or cancel it.' : ''
|
||||||
if (!confirm(`Clear waiting and paused jobs from the queue? Saved clips stay in the library.${extra}`)) return
|
if (!confirm(`Clear waiting and paused jobs from the queue? Saved clips stay in the library.${extra}`)) return
|
||||||
error.value = ''
|
error.value = ''
|
||||||
try {
|
try {
|
||||||
@@ -1212,6 +1225,29 @@ async function clearAll() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function forceClearAll() {
|
||||||
|
if (forceClearing.value) return
|
||||||
|
if (!confirm(
|
||||||
|
'Force clear cancels every running and waiting job (including stuck YuE/music), wipes shot batches, and kills Comfy if it is wedged. Saved library clips stay. Continue?'
|
||||||
|
)) return
|
||||||
|
forceClearing.value = true
|
||||||
|
error.value = ''
|
||||||
|
forceClearNote.value = ''
|
||||||
|
try {
|
||||||
|
const result = await $fetch<{
|
||||||
|
message?: string
|
||||||
|
cleared?: { cancelledStudio?: number; clearedPending?: number; cancelledLive?: number; clearedShotQueues?: number }
|
||||||
|
}>('/api/comfy/force-reset', { method: 'POST' })
|
||||||
|
const c = result.cleared || {}
|
||||||
|
await loadQueues()
|
||||||
|
forceClearNote.value = `Force cleared · studio ${c.cancelledStudio || 0}, live ${c.cancelledLive || 0}, shots ${c.clearedShotQueues || 0}. Comfy is stopped — Poke when ready.`
|
||||||
|
} catch (err: any) {
|
||||||
|
error.value = err?.data?.statusMessage || err?.statusMessage || err?.message || 'Could not force-clear the queue'
|
||||||
|
} finally {
|
||||||
|
forceClearing.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let saveTimer: ReturnType<typeof setTimeout> | null = null
|
let saveTimer: ReturnType<typeof setTimeout> | null = null
|
||||||
function saveSegment(queue: Queue, segment: Segment) {
|
function saveSegment(queue: Queue, segment: Segment) {
|
||||||
markDirty(queue.id, segment.index)
|
markDirty(queue.id, segment.index)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ export default defineEventHandler(async (event) => {
|
|||||||
const comfy = await requestComfyForceReset()
|
const comfy = await requestComfyForceReset()
|
||||||
return {
|
return {
|
||||||
ok: true,
|
ok: true,
|
||||||
message: 'Cleared stuck jobs and force-stopped Comfy. Use Wake / Poke when you want the GPU again.',
|
message: 'Cleared every stuck Studio/music/shot job and force-stopped Comfy. Use Wake / Poke when you want the GPU again.',
|
||||||
cleared,
|
cleared,
|
||||||
comfy
|
comfy
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -484,6 +484,20 @@ export async function clearShotQueues(owner: string) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Nuclear clear: wipe every shot batch for this owner, including ones marked running. */
|
||||||
|
export async function forceClearAllShotQueues(owner: string) {
|
||||||
|
const before = listShotQueues(owner)
|
||||||
|
for (const queue of before) {
|
||||||
|
markShotQueueCancelled(queue.id)
|
||||||
|
activeJobs.delete(queue.id)
|
||||||
|
}
|
||||||
|
return mutate(owner, (queues) => {
|
||||||
|
const removed = queues.length
|
||||||
|
queues.splice(0, queues.length)
|
||||||
|
return { removed }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
export function listOwnersWithQueues() {
|
export function listOwnersWithQueues() {
|
||||||
const root = join(libraryRoot(), 'users')
|
const root = join(libraryRoot(), 'users')
|
||||||
if (!existsSync(root)) return []
|
if (!existsSync(root)) return []
|
||||||
|
|||||||
@@ -483,7 +483,7 @@ export async function cancelStudioJob(owner: string, id: string) {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Wipe waiting/running/held studio rows + live/pending jobs for this owner (stuck recovery). */
|
/** Wipe waiting/running/held studio rows + live/pending/shot queues for this owner (stuck recovery). */
|
||||||
export async function clearStuckStudioWork(owner: string) {
|
export async function clearStuckStudioWork(owner: string) {
|
||||||
const liveIds = new Set<string>()
|
const liveIds = new Set<string>()
|
||||||
const cancelledStudio: string[] = []
|
const cancelledStudio: string[] = []
|
||||||
@@ -530,10 +530,16 @@ export async function clearStuckStudioWork(owner: string) {
|
|||||||
|
|
||||||
for (const id of liveIds) deletePendingJob(id)
|
for (const id of liveIds) deletePendingJob(id)
|
||||||
|
|
||||||
|
const { forceClearAllShotQueues } = await import('~/server/utils/shotQueue')
|
||||||
|
const shots = await forceClearAllShotQueues(owner).catch(() => ({ removed: 0 }))
|
||||||
|
|
||||||
|
kickStudioQueue()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
cancelledStudio: cancelledStudio.length,
|
cancelledStudio: cancelledStudio.length,
|
||||||
clearedPending: clearedPending.length,
|
clearedPending: clearedPending.length,
|
||||||
cancelledLive: liveIds.size
|
cancelledLive: liveIds.size,
|
||||||
|
clearedShotQueues: Number(shots?.removed || 0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user