Keep extension chains alive and save each finished segment.

The overall bar made a long queue look stuck at ~9% during the first clip, and nothing was written until the last step. Save checkpoints, use per-step progress, and stop treating a short Comfy history lag as a dropped job.
This commit is contained in:
2026-08-26 02:21:55 -05:00
parent f6feb41d62
commit d78c3fee7d
6 changed files with 78 additions and 30 deletions
+48 -23
View File
@@ -57,8 +57,8 @@ export function emitChainJob(job: Job, event: JobEvent, samplePct?: number) {
if (next.message) {
next.message = formatChainMessage(job, next.message, samplePct)
}
if (chained && typeof next.progress === 'number' && next.type !== 'complete') {
next.progress = mapChainProgress(job, next.progress)
if (chained && typeof next.progress === 'number' && next.type !== 'complete' && next.type !== 'checkpoint') {
next.overallProgress = mapChainProgress(job, next.progress)
}
if (chained) {
next.chainStep = step
@@ -78,8 +78,17 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
return new Promise((resolve) => {
let localProgress = 0
const startedAt = Date.now()
let lastActivity = Date.now()
const IDLE_MS = 30 * 60 * 1000
const ABSOLUTE_MS = 3 * 60 * 60 * 1000
const markActivity = () => {
lastActivity = Date.now()
}
const emitLocal = (event: JobEvent, samplePct?: number) => {
markActivity()
if (typeof event.progress === 'number') localProgress = event.progress
emitChainJob(job, event, samplePct)
}
@@ -92,6 +101,25 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
try { ws.close() } catch { /* ignore */ }
}
const armIdle = () => {
if (timeout) clearTimeout(timeout)
if (settled) return
const absLeft = ABSOLUTE_MS - (Date.now() - startedAt)
if (absLeft <= 0) {
void fail('Timed out waiting for ComfyUI (3 hours).')
return
}
const idleLeft = IDLE_MS - (Date.now() - lastActivity)
timeout = setTimeout(() => {
if (settled) return
if (Date.now() - lastActivity >= IDLE_MS) {
void fail('Timed out waiting for ComfyUI progress (30 minutes with no updates). The desktop job may still be running.')
return
}
armIdle()
}, Math.max(1000, Math.min(idleLeft, absLeft)))
}
const fail = async (error: string) => {
if (settled) return
settled = true
@@ -131,23 +159,6 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
tmpDir: job.library.extendTmpDir!
})
}
if (!persist) {
job.segmentBuffer = buffer
await purgeComfyArtifacts({
video,
imageName: job.library.imageName,
imageSubfolder: job.library.imageSubfolder,
promptId: job.promptId
})
deletePendingJob(job.id)
emitLocal({
type: 'status',
message: stitching ? 'Extension stitched' : 'Initial segment ready',
progress: 99
})
resolve()
return true
}
const clip = await saveClip({
ownerKey: job.library.ownerKey,
folderId: job.library.folderId,
@@ -172,12 +183,27 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
job.clipId = clip.id
job.hideThumbnail = clip.hideThumbnail
job.library.thumb = undefined
job.library.name = nextClipPartName(clip.name)
await purgeComfyArtifacts({
video,
imageName: job.library.imageName,
imageSubfolder: job.library.imageSubfolder,
promptId: job.promptId
})
if (!persist) {
job.segmentBuffer = buffer
deletePendingJob(job.id)
emitLocal({
type: 'checkpoint',
message: stitching ? 'Extension checkpoint saved' : 'Initial segment saved',
progress: 99,
clipId: clip.id,
hideThumbnail: job.hideThumbnail,
folderLocked: job.library.folderLocked
})
resolve()
return true
}
removeExtendTemp(job.library.extendTmpDir)
job.library.extendTmpDir = undefined
}
@@ -226,9 +252,10 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
await succeed()
return
}
markActivity()
if (await isComfyPromptDropped(job.promptId)) {
dropMisses += 1
if (dropMisses >= 3) {
if (dropMisses >= 20) {
await fail('ComfyUI dropped this job. Reloading the Comfy interface clears the queue. Generate again.')
}
} else {
@@ -239,9 +266,7 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
}
}
timeout = setTimeout(() => {
void fail('Timed out waiting for ComfyUI (15 minutes).')
}, 15 * 60 * 1000)
armIdle()
pollTimer = setInterval(() => {
void pollHistory()