Add Force reset to clear stuck jobs and kill a wedged Comfy.
Settings and the header menu can cancel invisible queue/pending work and force-stop Beast when Sleep is blocked. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -75,11 +75,19 @@
|
||||
type="button"
|
||||
class="block w-full rounded-xl px-3 py-2 text-left text-sm hover:bg-white/5 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
:class="gpuSleeping ? 'text-amber-100' : 'text-zinc-200 hover:text-white'"
|
||||
:disabled="gpuSleeping || gpuWaking || gpuBusy"
|
||||
:disabled="gpuSleeping || gpuWaking || gpuForceResetting"
|
||||
:title="gpuError || (gpuBusy ? 'Cancel the running job first' : 'Stop Comfy on the GPU to free VRAM. Does not shut the PC down.')"
|
||||
role="menuitem"
|
||||
@click="sleepGpu"
|
||||
>{{ gpuSleeping ? 'Sleeping…' : 'Sleep' }}</button>
|
||||
<button
|
||||
type="button"
|
||||
class="block w-full rounded-xl px-3 py-2 text-left text-sm text-red-200 hover:bg-white/5 hover:text-red-100 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
:disabled="gpuForceResetting || gpuSleeping || gpuWaking"
|
||||
title="Clear stuck jobs and kill Comfy even if a generation is wedged"
|
||||
role="menuitem"
|
||||
@click="forceResetGpu"
|
||||
>{{ gpuForceResetting ? 'Force resetting…' : 'Force reset' }}</button>
|
||||
<div class="my-1 border-t border-white/10" />
|
||||
<NuxtLink
|
||||
to="/settings"
|
||||
@@ -142,6 +150,7 @@ const authEnabled = ref(false)
|
||||
const menuOpen = ref(false)
|
||||
const gpuWaking = ref(false)
|
||||
const gpuSleeping = ref(false)
|
||||
const gpuForceResetting = ref(false)
|
||||
const gpuError = ref('')
|
||||
const comfyOk = ref(false)
|
||||
const imageOk = ref(false)
|
||||
@@ -224,7 +233,7 @@ async function pollHealth() {
|
||||
}
|
||||
|
||||
async function pokeGpu() {
|
||||
if (gpuWaking.value || gpuSleeping.value) return
|
||||
if (gpuWaking.value || gpuSleeping.value || gpuForceResetting.value) return
|
||||
gpuWaking.value = true
|
||||
gpuError.value = ''
|
||||
try {
|
||||
@@ -246,7 +255,7 @@ async function pokeGpu() {
|
||||
}
|
||||
|
||||
async function sleepGpu() {
|
||||
if (gpuSleeping.value || gpuWaking.value || gpuBusy.value) return
|
||||
if (gpuSleeping.value || gpuWaking.value || gpuForceResetting.value || gpuBusy.value) return
|
||||
gpuSleeping.value = true
|
||||
gpuError.value = ''
|
||||
try {
|
||||
@@ -263,6 +272,31 @@ async function sleepGpu() {
|
||||
}
|
||||
}
|
||||
|
||||
async function forceResetGpu() {
|
||||
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
|
||||
const ok = window.confirm(
|
||||
'Force reset clears stuck Studio/queue jobs and kills Comfy even if a generation is wedged. Continue?'
|
||||
)
|
||||
if (!ok) return
|
||||
menuOpen.value = false
|
||||
gpuForceResetting.value = true
|
||||
gpuError.value = ''
|
||||
try {
|
||||
await $fetch('/api/comfy/force-reset', { method: 'POST' })
|
||||
comfyOk.value = false
|
||||
imageOk.value = false
|
||||
comfyBusy.value = false
|
||||
imageBusy.value = false
|
||||
comfyAsleep.value = true
|
||||
await pollHealth()
|
||||
} catch (error: any) {
|
||||
gpuError.value = error?.data?.statusMessage || error?.statusMessage || error?.message || 'Could not force-reset Comfy.'
|
||||
} finally {
|
||||
gpuForceResetting.value = false
|
||||
emit('gpu')
|
||||
}
|
||||
}
|
||||
|
||||
function onWindowClick() {
|
||||
closeMenu()
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
type="button"
|
||||
class="rounded-xl border px-3 py-2 text-sm disabled:cursor-not-allowed disabled:opacity-60"
|
||||
:class="gpuSleeping ? 'border-amber-300/40 text-amber-100' : 'border-white/15 text-zinc-200 hover:border-amber-300/50 hover:text-white'"
|
||||
:disabled="gpuSleeping || gpuWaking"
|
||||
:disabled="gpuSleeping || gpuWaking || gpuForceResetting"
|
||||
@click="sleepGpu"
|
||||
>
|
||||
{{ gpuSleeping ? 'Stopping GPU…' : 'Sleep GPU' }}
|
||||
@@ -40,12 +40,24 @@
|
||||
type="button"
|
||||
class="rounded-xl border px-3 py-2 text-sm disabled:cursor-not-allowed disabled:opacity-60"
|
||||
:class="gpuWaking ? 'border-amber-300/40 text-amber-100' : 'border-amber-400/40 text-amber-200 hover:border-amber-300 hover:text-amber-50'"
|
||||
:disabled="gpuWaking || gpuSleeping"
|
||||
:disabled="gpuWaking || gpuSleeping || gpuForceResetting"
|
||||
@click="wakeGpu"
|
||||
>
|
||||
{{ gpuWaking ? 'Starting GPU…' : 'Wake GPU' }}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
class="rounded-xl border border-red-400/40 px-3 py-2 text-sm text-red-200 hover:border-red-300 hover:text-red-50 disabled:cursor-not-allowed disabled:opacity-60"
|
||||
:disabled="gpuForceResetting || gpuSleeping || gpuWaking"
|
||||
title="Cancel stuck Studio jobs, clear Comfy’s queue, and kill Comfy even if a job is wedged"
|
||||
@click="forceResetGpu"
|
||||
>
|
||||
{{ gpuForceResetting ? 'Force resetting…' : 'Force reset GPU' }}
|
||||
</button>
|
||||
</div>
|
||||
<p class="text-[11px] text-zinc-500">
|
||||
Force reset: clears stuck/invisible queue jobs, interrupts Comfy, and kills it if Sleep is blocked. Does not shut the PC down.
|
||||
</p>
|
||||
<p v-if="gpuMessage" class="text-xs" :class="gpuError ? 'text-red-300' : 'text-zinc-500'">{{ gpuMessage }}</p>
|
||||
</div>
|
||||
<label class="mt-4 flex cursor-pointer items-center justify-between gap-3 rounded-2xl border border-white/10 bg-zinc-950/50 px-4 py-3 text-sm">
|
||||
@@ -497,6 +509,7 @@ const storageLoading = ref(true)
|
||||
const storageError = ref('')
|
||||
const gpuSleeping = ref(false)
|
||||
const gpuWaking = ref(false)
|
||||
const gpuForceResetting = ref(false)
|
||||
const gpuError = ref(false)
|
||||
const gpuMessage = ref('')
|
||||
const notifyComplete = ref(true)
|
||||
@@ -688,7 +701,7 @@ function onDefaultVideoDuration(value: number) {
|
||||
}
|
||||
|
||||
async function sleepGpu() {
|
||||
if (gpuSleeping.value || gpuWaking.value) return
|
||||
if (gpuSleeping.value || gpuWaking.value || gpuForceResetting.value) return
|
||||
gpuSleeping.value = true
|
||||
gpuError.value = false
|
||||
gpuMessage.value = 'Stopping Comfy on the GPU…'
|
||||
@@ -704,7 +717,7 @@ async function sleepGpu() {
|
||||
}
|
||||
|
||||
async function wakeGpu() {
|
||||
if (gpuWaking.value || gpuSleeping.value) return
|
||||
if (gpuWaking.value || gpuSleeping.value || gpuForceResetting.value) return
|
||||
gpuWaking.value = true
|
||||
gpuError.value = false
|
||||
gpuMessage.value = 'Starting Comfy on the GPU…'
|
||||
@@ -719,6 +732,31 @@ async function wakeGpu() {
|
||||
}
|
||||
}
|
||||
|
||||
async function forceResetGpu() {
|
||||
if (gpuForceResetting.value || gpuSleeping.value || gpuWaking.value) return
|
||||
const ok = window.confirm(
|
||||
'Force reset clears stuck Studio/queue jobs and kills Comfy even if a generation is wedged. Continue?'
|
||||
)
|
||||
if (!ok) return
|
||||
gpuForceResetting.value = true
|
||||
gpuError.value = false
|
||||
gpuMessage.value = 'Force-resetting Comfy and clearing stuck jobs…'
|
||||
try {
|
||||
const result = await $fetch<{
|
||||
message?: string
|
||||
cleared?: { cancelledStudio?: number; clearedPending?: number; cancelledLive?: number }
|
||||
}>('/api/comfy/force-reset', { method: 'POST' })
|
||||
const c = result.cleared || {}
|
||||
gpuMessage.value = result.message
|
||||
|| `Cleared ${c.cancelledStudio || 0} queue job(s), ${c.clearedPending || 0} pending, ${c.cancelledLive || 0} live. Comfy is stopped — Wake when ready.`
|
||||
} catch (error: any) {
|
||||
gpuError.value = true
|
||||
gpuMessage.value = error?.data?.statusMessage || error?.statusMessage || 'Could not force-reset Comfy.'
|
||||
} finally {
|
||||
gpuForceResetting.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadLoraKeys() {
|
||||
loraRowsLoading.value = true
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { libraryOwnerKey } from '~/server/utils/library'
|
||||
import { requestComfyForceReset } from '~/server/utils/comfyLifecycle'
|
||||
import { clearStuckStudioWork } from '~/server/utils/studioQueue'
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
const owner = libraryOwnerKey(event)
|
||||
const cleared = await clearStuckStudioWork(owner)
|
||||
const comfy = await requestComfyForceReset()
|
||||
return {
|
||||
ok: true,
|
||||
message: 'Cleared stuck jobs and force-stopped Comfy. Use Wake / Poke when you want the GPU again.',
|
||||
cleared,
|
||||
comfy
|
||||
}
|
||||
})
|
||||
@@ -82,7 +82,7 @@ export async function fetchQueue() {
|
||||
}
|
||||
}
|
||||
|
||||
async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
|
||||
async function controlRequest(path: string, method = 'GET', timeoutMs = 5000, body?: Record<string, unknown>) {
|
||||
const { controlUrl, controlToken } = settings()
|
||||
if (!controlUrl) return null
|
||||
try {
|
||||
@@ -90,11 +90,13 @@ async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
|
||||
method,
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
...(body ? { 'Content-Type': 'application/json' } : {}),
|
||||
...(controlToken ? { Authorization: `Bearer ${controlToken}` } : {})
|
||||
},
|
||||
...(body ? { body: JSON.stringify(body) } : {}),
|
||||
signal: AbortSignal.timeout(timeoutMs)
|
||||
})
|
||||
const body = await res.json().catch(() => null) as {
|
||||
const payload = await res.json().catch(() => null) as {
|
||||
ok?: boolean
|
||||
http?: boolean
|
||||
process?: boolean
|
||||
@@ -109,6 +111,7 @@ async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
|
||||
proxyPort?: number | null
|
||||
stopped?: boolean
|
||||
killed?: unknown
|
||||
forced?: boolean
|
||||
error?: string
|
||||
message?: string
|
||||
training?: { busy?: boolean; jobId?: string | null; status?: string | null; name?: string | null; message?: string }
|
||||
@@ -120,15 +123,16 @@ async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
|
||||
return {
|
||||
ok: false as const,
|
||||
status: res.status,
|
||||
error: body?.error,
|
||||
message: body?.message,
|
||||
training: body?.training,
|
||||
running: body?.running,
|
||||
pending: body?.pending,
|
||||
killed: body?.killed
|
||||
error: payload?.error,
|
||||
message: payload?.message,
|
||||
training: payload?.training,
|
||||
running: payload?.running,
|
||||
pending: payload?.pending,
|
||||
killed: payload?.killed,
|
||||
forced: payload?.forced
|
||||
}
|
||||
}
|
||||
return body || { ok: true }
|
||||
return payload || { ok: true }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
@@ -216,6 +220,39 @@ export async function requestComfySleep() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Interrupt + clear Comfy queue, then kill the process even if a job is wedged. */
|
||||
export async function requestComfyForceReset() {
|
||||
const { controlUrl } = settings()
|
||||
if (!controlUrl) {
|
||||
throw createError({
|
||||
statusCode: 503,
|
||||
statusMessage: 'COMFY_CONTROL_URL is not set. The studio cannot poke the Beast host agent.'
|
||||
})
|
||||
}
|
||||
const { interruptComfy } = await import('~/server/utils/comfy')
|
||||
await interruptComfy().catch(() => false)
|
||||
const remote = await controlRequest('/stop', 'POST', 30_000, { force: true })
|
||||
if (!remote) {
|
||||
throw createError({
|
||||
statusCode: 503,
|
||||
statusMessage: 'Beast host agent is unreachable. Confirm it is running on the desktop.'
|
||||
})
|
||||
}
|
||||
if (remote.ok === false) {
|
||||
throw createError({
|
||||
statusCode: Number(remote.status) || 502,
|
||||
statusMessage: String(remote.message || remote.error || 'Beast host agent refused to force-stop Comfy.')
|
||||
})
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
stopped: true,
|
||||
asleep: true,
|
||||
forced: true,
|
||||
killed: remote.killed ?? null
|
||||
}
|
||||
}
|
||||
|
||||
async function findLocalComfyProcess() {
|
||||
const platform = process.platform
|
||||
try {
|
||||
|
||||
@@ -483,6 +483,60 @@ export async function cancelStudioJob(owner: string, id: string) {
|
||||
return result
|
||||
}
|
||||
|
||||
/** Wipe waiting/running/held studio rows + live/pending jobs for this owner (stuck recovery). */
|
||||
export async function clearStuckStudioWork(owner: string) {
|
||||
const liveIds = new Set<string>()
|
||||
const cancelledStudio: string[] = []
|
||||
const clearedPending: string[] = []
|
||||
|
||||
for (const job of listJobs()) {
|
||||
if (job.library?.ownerKey !== owner) continue
|
||||
if (job.status === 'complete' || job.status === 'error' || job.status === 'cancelled') continue
|
||||
liveIds.add(job.id)
|
||||
job.status = 'cancelled'
|
||||
job.error = 'Cleared by force reset'
|
||||
if (job.library) {
|
||||
job.library.stopAfterCurrent = true
|
||||
job.library.queueAutoRun = false
|
||||
}
|
||||
emitJob(job, { type: 'error', error: 'Cleared by force reset', message: 'Cleared by force reset' })
|
||||
}
|
||||
|
||||
for (const pending of listPendingJobs()) {
|
||||
if (pending.ownerKey !== owner) continue
|
||||
deletePendingJob(pending.jobId)
|
||||
clearedPending.push(pending.jobId)
|
||||
}
|
||||
|
||||
const before = readJobs(owner)
|
||||
for (const job of before) {
|
||||
if (!canCancelStudioStatus(job.status)) continue
|
||||
try {
|
||||
await cancelStudioJob(owner, job.id)
|
||||
cancelledStudio.push(job.id)
|
||||
} catch {
|
||||
await mutateStore(owner, (store) => {
|
||||
const row = store.jobs.find(item => item.id === job.id)
|
||||
if (!row) return
|
||||
row.status = 'cancelled'
|
||||
row.liveJobId = undefined
|
||||
row.lastError = 'Cleared by force reset'
|
||||
row.updatedAt = Date.now()
|
||||
syncPausedFlag(store)
|
||||
}).catch(() => null)
|
||||
cancelledStudio.push(job.id)
|
||||
}
|
||||
}
|
||||
|
||||
for (const id of liveIds) deletePendingJob(id)
|
||||
|
||||
return {
|
||||
cancelledStudio: cancelledStudio.length,
|
||||
clearedPending: clearedPending.length,
|
||||
cancelledLive: liveIds.size
|
||||
}
|
||||
}
|
||||
|
||||
async function stopLiveGeneration(liveJobId?: string, shotQueueId?: string) {
|
||||
if (shotQueueId) {
|
||||
const { setQueueJob, pauseShotQueue } = await import('~/server/utils/shotQueue')
|
||||
|
||||
Reference in New Issue
Block a user