Add Force reset to clear stuck jobs and kill a wedged Comfy.

Settings and the header menu can cancel invisible queue/pending work and force-stop Beast when Sleep is blocked.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-09-03 21:24:10 -05:00
co-authored by Cursor
parent ea66ed8e3d
commit 5df9e2d8ac
5 changed files with 194 additions and 16 deletions
+46 -9
View File
@@ -82,7 +82,7 @@ export async function fetchQueue() {
}
}
async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
async function controlRequest(path: string, method = 'GET', timeoutMs = 5000, body?: Record<string, unknown>) {
const { controlUrl, controlToken } = settings()
if (!controlUrl) return null
try {
@@ -90,11 +90,13 @@ async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
method,
headers: {
Accept: 'application/json',
...(body ? { 'Content-Type': 'application/json' } : {}),
...(controlToken ? { Authorization: `Bearer ${controlToken}` } : {})
},
...(body ? { body: JSON.stringify(body) } : {}),
signal: AbortSignal.timeout(timeoutMs)
})
const body = await res.json().catch(() => null) as {
const payload = await res.json().catch(() => null) as {
ok?: boolean
http?: boolean
process?: boolean
@@ -109,6 +111,7 @@ async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
proxyPort?: number | null
stopped?: boolean
killed?: unknown
forced?: boolean
error?: string
message?: string
training?: { busy?: boolean; jobId?: string | null; status?: string | null; name?: string | null; message?: string }
@@ -120,15 +123,16 @@ async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
return {
ok: false as const,
status: res.status,
error: body?.error,
message: body?.message,
training: body?.training,
running: body?.running,
pending: body?.pending,
killed: body?.killed
error: payload?.error,
message: payload?.message,
training: payload?.training,
running: payload?.running,
pending: payload?.pending,
killed: payload?.killed,
forced: payload?.forced
}
}
return body || { ok: true }
return payload || { ok: true }
} catch {
return null
}
@@ -216,6 +220,39 @@ export async function requestComfySleep() {
}
}
/** Interrupt + clear Comfy queue, then kill the process even if a job is wedged. */
export async function requestComfyForceReset() {
const { controlUrl } = settings()
if (!controlUrl) {
throw createError({
statusCode: 503,
statusMessage: 'COMFY_CONTROL_URL is not set. The studio cannot poke the Beast host agent.'
})
}
const { interruptComfy } = await import('~/server/utils/comfy')
await interruptComfy().catch(() => false)
const remote = await controlRequest('/stop', 'POST', 30_000, { force: true })
if (!remote) {
throw createError({
statusCode: 503,
statusMessage: 'Beast host agent is unreachable. Confirm it is running on the desktop.'
})
}
if (remote.ok === false) {
throw createError({
statusCode: Number(remote.status) || 502,
statusMessage: String(remote.message || remote.error || 'Beast host agent refused to force-stop Comfy.')
})
}
return {
ok: true,
stopped: true,
asleep: true,
forced: true,
killed: remote.killed ?? null
}
}
async function findLocalComfyProcess() {
const platform = process.platform
try {
+54
View File
@@ -483,6 +483,60 @@ export async function cancelStudioJob(owner: string, id: string) {
return result
}
/** Wipe waiting/running/held studio rows + live/pending jobs for this owner (stuck recovery). */
export async function clearStuckStudioWork(owner: string) {
const liveIds = new Set<string>()
const cancelledStudio: string[] = []
const clearedPending: string[] = []
for (const job of listJobs()) {
if (job.library?.ownerKey !== owner) continue
if (job.status === 'complete' || job.status === 'error' || job.status === 'cancelled') continue
liveIds.add(job.id)
job.status = 'cancelled'
job.error = 'Cleared by force reset'
if (job.library) {
job.library.stopAfterCurrent = true
job.library.queueAutoRun = false
}
emitJob(job, { type: 'error', error: 'Cleared by force reset', message: 'Cleared by force reset' })
}
for (const pending of listPendingJobs()) {
if (pending.ownerKey !== owner) continue
deletePendingJob(pending.jobId)
clearedPending.push(pending.jobId)
}
const before = readJobs(owner)
for (const job of before) {
if (!canCancelStudioStatus(job.status)) continue
try {
await cancelStudioJob(owner, job.id)
cancelledStudio.push(job.id)
} catch {
await mutateStore(owner, (store) => {
const row = store.jobs.find(item => item.id === job.id)
if (!row) return
row.status = 'cancelled'
row.liveJobId = undefined
row.lastError = 'Cleared by force reset'
row.updatedAt = Date.now()
syncPausedFlag(store)
}).catch(() => null)
cancelledStudio.push(job.id)
}
}
for (const id of liveIds) deletePendingJob(id)
return {
cancelledStudio: cancelledStudio.length,
clearedPending: clearedPending.length,
cancelledLive: liveIds.size
}
}
async function stopLiveGeneration(liveJobId?: string, shotQueueId?: string) {
if (shotQueueId) {
const { setQueueJob, pauseShotQueue } = await import('~/server/utils/shotQueue')