Add Force reset to clear stuck jobs and kill a wedged Comfy.
Settings and the header menu can cancel invisible queue/pending work and force-stop Beast when Sleep is blocked. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -82,7 +82,7 @@ export async function fetchQueue() {
|
||||
}
|
||||
}
|
||||
|
||||
async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
|
||||
async function controlRequest(path: string, method = 'GET', timeoutMs = 5000, body?: Record<string, unknown>) {
|
||||
const { controlUrl, controlToken } = settings()
|
||||
if (!controlUrl) return null
|
||||
try {
|
||||
@@ -90,11 +90,13 @@ async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
|
||||
method,
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
...(body ? { 'Content-Type': 'application/json' } : {}),
|
||||
...(controlToken ? { Authorization: `Bearer ${controlToken}` } : {})
|
||||
},
|
||||
...(body ? { body: JSON.stringify(body) } : {}),
|
||||
signal: AbortSignal.timeout(timeoutMs)
|
||||
})
|
||||
const body = await res.json().catch(() => null) as {
|
||||
const payload = await res.json().catch(() => null) as {
|
||||
ok?: boolean
|
||||
http?: boolean
|
||||
process?: boolean
|
||||
@@ -109,6 +111,7 @@ async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
|
||||
proxyPort?: number | null
|
||||
stopped?: boolean
|
||||
killed?: unknown
|
||||
forced?: boolean
|
||||
error?: string
|
||||
message?: string
|
||||
training?: { busy?: boolean; jobId?: string | null; status?: string | null; name?: string | null; message?: string }
|
||||
@@ -120,15 +123,16 @@ async function controlRequest(path: string, method = 'GET', timeoutMs = 5000) {
|
||||
return {
|
||||
ok: false as const,
|
||||
status: res.status,
|
||||
error: body?.error,
|
||||
message: body?.message,
|
||||
training: body?.training,
|
||||
running: body?.running,
|
||||
pending: body?.pending,
|
||||
killed: body?.killed
|
||||
error: payload?.error,
|
||||
message: payload?.message,
|
||||
training: payload?.training,
|
||||
running: payload?.running,
|
||||
pending: payload?.pending,
|
||||
killed: payload?.killed,
|
||||
forced: payload?.forced
|
||||
}
|
||||
}
|
||||
return body || { ok: true }
|
||||
return payload || { ok: true }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
@@ -216,6 +220,39 @@ export async function requestComfySleep() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Interrupt + clear Comfy queue, then kill the process even if a job is wedged. */
|
||||
export async function requestComfyForceReset() {
|
||||
const { controlUrl } = settings()
|
||||
if (!controlUrl) {
|
||||
throw createError({
|
||||
statusCode: 503,
|
||||
statusMessage: 'COMFY_CONTROL_URL is not set. The studio cannot poke the Beast host agent.'
|
||||
})
|
||||
}
|
||||
const { interruptComfy } = await import('~/server/utils/comfy')
|
||||
await interruptComfy().catch(() => false)
|
||||
const remote = await controlRequest('/stop', 'POST', 30_000, { force: true })
|
||||
if (!remote) {
|
||||
throw createError({
|
||||
statusCode: 503,
|
||||
statusMessage: 'Beast host agent is unreachable. Confirm it is running on the desktop.'
|
||||
})
|
||||
}
|
||||
if (remote.ok === false) {
|
||||
throw createError({
|
||||
statusCode: Number(remote.status) || 502,
|
||||
statusMessage: String(remote.message || remote.error || 'Beast host agent refused to force-stop Comfy.')
|
||||
})
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
stopped: true,
|
||||
asleep: true,
|
||||
forced: true,
|
||||
killed: remote.killed ?? null
|
||||
}
|
||||
}
|
||||
|
||||
async function findLocalComfyProcess() {
|
||||
const platform = process.platform
|
||||
try {
|
||||
|
||||
@@ -483,6 +483,60 @@ export async function cancelStudioJob(owner: string, id: string) {
|
||||
return result
|
||||
}
|
||||
|
||||
/** Wipe waiting/running/held studio rows + live/pending jobs for this owner (stuck recovery). */
|
||||
export async function clearStuckStudioWork(owner: string) {
|
||||
const liveIds = new Set<string>()
|
||||
const cancelledStudio: string[] = []
|
||||
const clearedPending: string[] = []
|
||||
|
||||
for (const job of listJobs()) {
|
||||
if (job.library?.ownerKey !== owner) continue
|
||||
if (job.status === 'complete' || job.status === 'error' || job.status === 'cancelled') continue
|
||||
liveIds.add(job.id)
|
||||
job.status = 'cancelled'
|
||||
job.error = 'Cleared by force reset'
|
||||
if (job.library) {
|
||||
job.library.stopAfterCurrent = true
|
||||
job.library.queueAutoRun = false
|
||||
}
|
||||
emitJob(job, { type: 'error', error: 'Cleared by force reset', message: 'Cleared by force reset' })
|
||||
}
|
||||
|
||||
for (const pending of listPendingJobs()) {
|
||||
if (pending.ownerKey !== owner) continue
|
||||
deletePendingJob(pending.jobId)
|
||||
clearedPending.push(pending.jobId)
|
||||
}
|
||||
|
||||
const before = readJobs(owner)
|
||||
for (const job of before) {
|
||||
if (!canCancelStudioStatus(job.status)) continue
|
||||
try {
|
||||
await cancelStudioJob(owner, job.id)
|
||||
cancelledStudio.push(job.id)
|
||||
} catch {
|
||||
await mutateStore(owner, (store) => {
|
||||
const row = store.jobs.find(item => item.id === job.id)
|
||||
if (!row) return
|
||||
row.status = 'cancelled'
|
||||
row.liveJobId = undefined
|
||||
row.lastError = 'Cleared by force reset'
|
||||
row.updatedAt = Date.now()
|
||||
syncPausedFlag(store)
|
||||
}).catch(() => null)
|
||||
cancelledStudio.push(job.id)
|
||||
}
|
||||
}
|
||||
|
||||
for (const id of liveIds) deletePendingJob(id)
|
||||
|
||||
return {
|
||||
cancelledStudio: cancelledStudio.length,
|
||||
clearedPending: clearedPending.length,
|
||||
cancelledLive: liveIds.size
|
||||
}
|
||||
}
|
||||
|
||||
async function stopLiveGeneration(liveJobId?: string, shotQueueId?: string) {
|
||||
if (shotQueueId) {
|
||||
const { setQueueJob, pauseShotQueue } = await import('~/server/utils/shotQueue')
|
||||
|
||||
Reference in New Issue
Block a user