Keep extension chains alive and save each finished segment.
The overall bar made a long queue look stuck at ~9% during the first clip, and nothing was written until the last step. Save checkpoints, use per-step progress, and stop treating a short Comfy history lag as a dropped job.
This commit is contained in:
+48
-23
@@ -57,8 +57,8 @@ export function emitChainJob(job: Job, event: JobEvent, samplePct?: number) {
|
||||
if (next.message) {
|
||||
next.message = formatChainMessage(job, next.message, samplePct)
|
||||
}
|
||||
if (chained && typeof next.progress === 'number' && next.type !== 'complete') {
|
||||
next.progress = mapChainProgress(job, next.progress)
|
||||
if (chained && typeof next.progress === 'number' && next.type !== 'complete' && next.type !== 'checkpoint') {
|
||||
next.overallProgress = mapChainProgress(job, next.progress)
|
||||
}
|
||||
if (chained) {
|
||||
next.chainStep = step
|
||||
@@ -78,8 +78,17 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let localProgress = 0
|
||||
const startedAt = Date.now()
|
||||
let lastActivity = Date.now()
|
||||
const IDLE_MS = 30 * 60 * 1000
|
||||
const ABSOLUTE_MS = 3 * 60 * 60 * 1000
|
||||
|
||||
const markActivity = () => {
|
||||
lastActivity = Date.now()
|
||||
}
|
||||
|
||||
const emitLocal = (event: JobEvent, samplePct?: number) => {
|
||||
markActivity()
|
||||
if (typeof event.progress === 'number') localProgress = event.progress
|
||||
emitChainJob(job, event, samplePct)
|
||||
}
|
||||
@@ -92,6 +101,25 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
|
||||
try { ws.close() } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
const armIdle = () => {
|
||||
if (timeout) clearTimeout(timeout)
|
||||
if (settled) return
|
||||
const absLeft = ABSOLUTE_MS - (Date.now() - startedAt)
|
||||
if (absLeft <= 0) {
|
||||
void fail('Timed out waiting for ComfyUI (3 hours).')
|
||||
return
|
||||
}
|
||||
const idleLeft = IDLE_MS - (Date.now() - lastActivity)
|
||||
timeout = setTimeout(() => {
|
||||
if (settled) return
|
||||
if (Date.now() - lastActivity >= IDLE_MS) {
|
||||
void fail('Timed out waiting for ComfyUI progress (30 minutes with no updates). The desktop job may still be running.')
|
||||
return
|
||||
}
|
||||
armIdle()
|
||||
}, Math.max(1000, Math.min(idleLeft, absLeft)))
|
||||
}
|
||||
|
||||
const fail = async (error: string) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
@@ -131,23 +159,6 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
|
||||
tmpDir: job.library.extendTmpDir!
|
||||
})
|
||||
}
|
||||
if (!persist) {
|
||||
job.segmentBuffer = buffer
|
||||
await purgeComfyArtifacts({
|
||||
video,
|
||||
imageName: job.library.imageName,
|
||||
imageSubfolder: job.library.imageSubfolder,
|
||||
promptId: job.promptId
|
||||
})
|
||||
deletePendingJob(job.id)
|
||||
emitLocal({
|
||||
type: 'status',
|
||||
message: stitching ? 'Extension stitched' : 'Initial segment ready',
|
||||
progress: 99
|
||||
})
|
||||
resolve()
|
||||
return true
|
||||
}
|
||||
const clip = await saveClip({
|
||||
ownerKey: job.library.ownerKey,
|
||||
folderId: job.library.folderId,
|
||||
@@ -172,12 +183,27 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
|
||||
job.clipId = clip.id
|
||||
job.hideThumbnail = clip.hideThumbnail
|
||||
job.library.thumb = undefined
|
||||
job.library.name = nextClipPartName(clip.name)
|
||||
await purgeComfyArtifacts({
|
||||
video,
|
||||
imageName: job.library.imageName,
|
||||
imageSubfolder: job.library.imageSubfolder,
|
||||
promptId: job.promptId
|
||||
})
|
||||
if (!persist) {
|
||||
job.segmentBuffer = buffer
|
||||
deletePendingJob(job.id)
|
||||
emitLocal({
|
||||
type: 'checkpoint',
|
||||
message: stitching ? 'Extension checkpoint saved' : 'Initial segment saved',
|
||||
progress: 99,
|
||||
clipId: clip.id,
|
||||
hideThumbnail: job.hideThumbnail,
|
||||
folderLocked: job.library.folderLocked
|
||||
})
|
||||
resolve()
|
||||
return true
|
||||
}
|
||||
removeExtendTemp(job.library.extendTmpDir)
|
||||
job.library.extendTmpDir = undefined
|
||||
}
|
||||
@@ -226,9 +252,10 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
|
||||
await succeed()
|
||||
return
|
||||
}
|
||||
markActivity()
|
||||
if (await isComfyPromptDropped(job.promptId)) {
|
||||
dropMisses += 1
|
||||
if (dropMisses >= 3) {
|
||||
if (dropMisses >= 20) {
|
||||
await fail('ComfyUI dropped this job. Reloading the Comfy interface clears the queue. Generate again.')
|
||||
}
|
||||
} else {
|
||||
@@ -239,9 +266,7 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
|
||||
}
|
||||
}
|
||||
|
||||
timeout = setTimeout(() => {
|
||||
void fail('Timed out waiting for ComfyUI (15 minutes).')
|
||||
}, 15 * 60 * 1000)
|
||||
armIdle()
|
||||
|
||||
pollTimer = setInterval(() => {
|
||||
void pollHistory()
|
||||
|
||||
Reference in New Issue
Block a user