From d78c3fee7dce65f0a1bfc6eb9b917c3976a34ff1 Mon Sep 17 00:00:00 2001 From: Ian Date: Wed, 26 Aug 2026 02:21:55 -0500 Subject: [PATCH] Keep extension chains alive and save each finished segment. The overall bar made a long queue look stuck at ~9% during the first clip, and nothing was written until the last step. Save checkpoints, use per-step progress, and stop treating a short Comfy history lag as a dropped job. --- pages/index.vue | 16 +++++++++ server/api/generate.post.ts | 1 + server/utils/comfy.ts | 14 ++++---- server/utils/ffmpeg.ts | 5 ++- server/utils/jobs.ts | 1 + server/utils/watch.ts | 71 +++++++++++++++++++++++++------------ 6 files changed, 78 insertions(+), 30 deletions(-) diff --git a/pages/index.vue b/pages/index.vue index 26af379..d1413f7 100644 --- a/pages/index.vue +++ b/pages/index.vue @@ -320,6 +320,7 @@

Step {{ chainStep }}/{{ chainTotal }}{{ chainLabel ? `: ${chainLabel}` : '' }} +

@@ -696,6 +697,7 @@ const maxStep = ref(8) const chainStep = ref(1) const chainTotal = ref(1) const chainLabel = ref('') +const overallProgress = ref(0) const elapsedMs = ref(0) const videoUrl = ref('') const jobId = ref('') @@ -1297,6 +1299,7 @@ async function generateExtension() { chainStep.value = 1 chainTotal.value = 1 chainLabel.value = '' + overallProgress.value = 0 statusMessage.value = 'Extracting last frame & stitching extension...' startTimer() try { @@ -1473,6 +1476,7 @@ async function generate() { chainStep.value = 1 chainTotal.value = 1 + extensionQueue.value.length chainLabel.value = extensionQueue.value.length ? 'Initial' : '' + overallProgress.value = 0 startTimer() try { const body = new FormData() @@ -1545,12 +1549,24 @@ function applyJobEvent(payload: Record, hidden: boolean, folderLock if (typeof payload.chainStep === 'number') chainStep.value = payload.chainStep if (typeof payload.chainTotal === 'number') chainTotal.value = payload.chainTotal if (payload.chainLabel) chainLabel.value = String(payload.chainLabel) + if (typeof payload.overallProgress === 'number') overallProgress.value = payload.overallProgress const failed = payload.type === 'error' || payload.status === 'error' || payload.status === 'cancelled' const complete = payload.type === 'complete' || payload.status === 'complete' const deferred = payload.type === 'deferred' || payload.status === 'deferred' if (settledUi) return + if (payload.type === 'checkpoint' && payload.clipId) { + loadLibrary().catch(() => null) + const locked = folderLocked || payload.folderLocked === true + if (!watchingLibrary.value && !locked) { + currentClipId.value = payload.clipId + videoUrl.value = clipVideoUrl(payload.clipId) + concealOutput.value = hidden || payload.hideThumbnail === true + } + return + } + if (deferred) { settledUi = true busy.value = false diff --git a/server/api/generate.post.ts b/server/api/generate.post.ts index 25a5da5..f05ddf2 100644 --- a/server/api/generate.post.ts +++ b/server/api/generate.post.ts @@ -402,6 +402,7 @@ async function runGeneration( await extractLastFrame(part1Path, framePath) const frame = readFileSync(framePath) job.library.extendPart1Path = part1Path + job.library.prompt = ext.prompt const sound = await probeHasAudio(currentPath) const seed = Math.floor(Math.random() * 2_147_483_647) diff --git a/server/utils/comfy.ts b/server/utils/comfy.ts index 8c3f7cd..3956750 100644 --- a/server/utils/comfy.ts +++ b/server/utils/comfy.ts @@ -209,25 +209,27 @@ export async function isComfyPromptDropped(promptId: string) { if (queue.running > 0 || queue.pending > 0) return false const history = await fetchHistory(promptId) const entry = history?.[promptId] as { status?: { status_str?: string; completed?: boolean } } | undefined - if (!entry) return true + if (!entry) return false const inspected = inspectHistory(history, promptId) if (inspected.video) return false const status = entry.status?.status_str if (status === 'error' || status === 'interrupted') return true if (entry.status?.completed) return true - return true + return false } export function inspectHistory(history: Record | null, promptId: string) { const video = extractVideo(history, promptId) + if (video) return { video, completed: true, error: null as string | null } if (!history) return { video: null, completed: false, error: null as string | null } const entry = (history[promptId] || history) as { status?: { status_str?: string; completed?: boolean } } - const error = entry?.status?.status_str === 'error' || entry?.status?.status_str === 'interrupted' - ? (entry.status.status_str === 'interrupted' ? 'Job interrupted.' : 'ComfyUI reported an execution error') - : null - return { video, completed: Boolean(video), error } + const status = entry?.status?.status_str + if (status === 'interrupted') return { video: null, completed: false, error: 'Job interrupted.' } + if (status === 'error') return { video: null, completed: false, error: 'ComfyUI reported an execution error' } + if (entry?.status?.completed) return { video: null, completed: true, error: 'ComfyUI finished without an MP4' } + return { video: null, completed: false, error: null as string | null } } export async function probeComfy() { diff --git a/server/utils/ffmpeg.ts b/server/utils/ffmpeg.ts index 32479ad..91bb0a4 100644 --- a/server/utils/ffmpeg.ts +++ b/server/utils/ffmpeg.ts @@ -249,9 +249,12 @@ export async function concatMp4(part1Path: string, part2Path: string, outputPath } export async function stitchExtension(params: { part1Path: string; part2: Buffer; tmpDir: string }) { - const part2Path = join(params.tmpDir, 'part2.mp4') + const part2Path = join(params.tmpDir, `part2_${Date.now()}.mp4`) const finalPath = join(params.tmpDir, 'final_stitched_video.mp4') writeFileSync(part2Path, params.part2) + if (!existsSync(part2Path) || !params.part2.length) { + throw new Error('Extension segment was missing before stitch') + } try { await xfadeStitch(params.part1Path, part2Path, finalPath) } catch (error) { diff --git a/server/utils/jobs.ts b/server/utils/jobs.ts index 67b34a8..085eb71 100644 --- a/server/utils/jobs.ts +++ b/server/utils/jobs.ts @@ -23,6 +23,7 @@ export interface JobEvent { chainStep?: number chainTotal?: number chainLabel?: string + overallProgress?: number } export interface Job { diff --git a/server/utils/watch.ts b/server/utils/watch.ts index 73d72de..d81b995 100644 --- a/server/utils/watch.ts +++ b/server/utils/watch.ts @@ -57,8 +57,8 @@ export function emitChainJob(job: Job, event: JobEvent, samplePct?: number) { if (next.message) { next.message = formatChainMessage(job, next.message, samplePct) } - if (chained && typeof next.progress === 'number' && next.type !== 'complete') { - next.progress = mapChainProgress(job, next.progress) + if (chained && typeof next.progress === 'number' && next.type !== 'complete' && next.type !== 'checkpoint') { + next.overallProgress = mapChainProgress(job, next.progress) } if (chained) { next.chainStep = step @@ -78,8 +78,17 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr return new Promise((resolve) => { let localProgress = 0 + const startedAt = Date.now() + let lastActivity = Date.now() + const IDLE_MS = 30 * 60 * 1000 + const ABSOLUTE_MS = 3 * 60 * 60 * 1000 + + const markActivity = () => { + lastActivity = Date.now() + } const emitLocal = (event: JobEvent, samplePct?: number) => { + markActivity() if (typeof event.progress === 'number') localProgress = event.progress emitChainJob(job, event, samplePct) } @@ -92,6 +101,25 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr try { ws.close() } catch { /* ignore */ } } + const armIdle = () => { + if (timeout) clearTimeout(timeout) + if (settled) return + const absLeft = ABSOLUTE_MS - (Date.now() - startedAt) + if (absLeft <= 0) { + void fail('Timed out waiting for ComfyUI (3 hours).') + return + } + const idleLeft = IDLE_MS - (Date.now() - lastActivity) + timeout = setTimeout(() => { + if (settled) return + if (Date.now() - lastActivity >= IDLE_MS) { + void fail('Timed out waiting for ComfyUI progress (30 minutes with no updates). The desktop job may still be running.') + return + } + armIdle() + }, Math.max(1000, Math.min(idleLeft, absLeft))) + } + const fail = async (error: string) => { if (settled) return settled = true @@ -131,23 +159,6 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr tmpDir: job.library.extendTmpDir! }) } - if (!persist) { - job.segmentBuffer = buffer - await purgeComfyArtifacts({ - video, - imageName: job.library.imageName, - imageSubfolder: job.library.imageSubfolder, - promptId: job.promptId - }) - deletePendingJob(job.id) - emitLocal({ - type: 'status', - message: stitching ? 'Extension stitched' : 'Initial segment ready', - progress: 99 - }) - resolve() - return true - } const clip = await saveClip({ ownerKey: job.library.ownerKey, folderId: job.library.folderId, @@ -172,12 +183,27 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr job.clipId = clip.id job.hideThumbnail = clip.hideThumbnail job.library.thumb = undefined + job.library.name = nextClipPartName(clip.name) await purgeComfyArtifacts({ video, imageName: job.library.imageName, imageSubfolder: job.library.imageSubfolder, promptId: job.promptId }) + if (!persist) { + job.segmentBuffer = buffer + deletePendingJob(job.id) + emitLocal({ + type: 'checkpoint', + message: stitching ? 'Extension checkpoint saved' : 'Initial segment saved', + progress: 99, + clipId: clip.id, + hideThumbnail: job.hideThumbnail, + folderLocked: job.library.folderLocked + }) + resolve() + return true + } removeExtendTemp(job.library.extendTmpDir) job.library.extendTmpDir = undefined } @@ -226,9 +252,10 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr await succeed() return } + markActivity() if (await isComfyPromptDropped(job.promptId)) { dropMisses += 1 - if (dropMisses >= 3) { + if (dropMisses >= 20) { await fail('ComfyUI dropped this job. Reloading the Comfy interface clears the queue. Generate again.') } } else { @@ -239,9 +266,7 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr } } - timeout = setTimeout(() => { - void fail('Timed out waiting for ComfyUI (15 minutes).') - }, 15 * 60 * 1000) + armIdle() pollTimer = setInterval(() => { void pollHistory()