towsty/aigen
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Keep extension chains alive and save each finished segment.

Browse Source 
The overall bar made a long queue look stuck at ~9% during the first clip, and nothing was written until the last step. Save checkpoints, use per-step progress, and stop treating a short Comfy history lag as a dropped job.
This commit is contained in:
towsty
2026-08-26 02:21:55 -05:00
parent f6feb41d62
commit d78c3fee7d
6 changed files with 78 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pages/index.vue
+16
View File
@@ -320,6 +320,7 @@
</div>
<p v-if="chainTotal > 1" class="mb-2 text-xs text-amber-200/80">
Step {{ chainStep }}/{{ chainTotal }}{{ chainLabel ? `: ${chainLabel}` : '' }}
<template v-if="overallProgress"> · {{ overallProgress }}% overall</template>
</p>
<div class="h-2 overflow-hidden rounded-full bg-zinc-800">
<div class="h-full rounded-full bg-amber-400 progress-glow transition-all duration-300" :style="{ width: progress + '%' }" />
@@ -696,6 +697,7 @@ const maxStep = ref(8)
const chainStep = ref(1)
const chainTotal = ref(1)
const chainLabel = ref('')
const overallProgress = ref(0)
const elapsedMs = ref(0)
const videoUrl = ref('')
const jobId = ref('')
@@ -1297,6 +1299,7 @@ async function generateExtension() {
chainStep.value = 1
chainTotal.value = 1
chainLabel.value = ''
overallProgress.value = 0
statusMessage.value = 'Extracting last frame & stitching extension...'
startTimer()
try {
@@ -1473,6 +1476,7 @@ async function generate() {
chainStep.value = 1
chainTotal.value = 1 + extensionQueue.value.length
chainLabel.value = extensionQueue.value.length ? 'Initial' : ''
overallProgress.value = 0
startTimer()
try {
const body = new FormData()
@@ -1545,12 +1549,24 @@ function applyJobEvent(payload: Record<string, any>, hidden: boolean, folderLock
if (typeof payload.chainStep === 'number') chainStep.value = payload.chainStep
if (typeof payload.chainTotal === 'number') chainTotal.value = payload.chainTotal
if (payload.chainLabel) chainLabel.value = String(payload.chainLabel)
if (typeof payload.overallProgress === 'number') overallProgress.value = payload.overallProgress
const failed = payload.type === 'error' || payload.status === 'error' || payload.status === 'cancelled'
const complete = payload.type === 'complete' || payload.status === 'complete'
const deferred = payload.type === 'deferred' || payload.status === 'deferred'
if (settledUi) return
if (payload.type === 'checkpoint' && payload.clipId) {
loadLibrary().catch(() => null)
const locked = folderLocked || payload.folderLocked === true
if (!watchingLibrary.value && !locked) {
currentClipId.value = payload.clipId
videoUrl.value = clipVideoUrl(payload.clipId)
concealOutput.value = hidden || payload.hideThumbnail === true
}
return
}
if (deferred) {
settledUi = true
busy.value = false
server/api/generate.post.ts
+1
View File
@@ -402,6 +402,7 @@ async function runGeneration(
await extractLastFrame(part1Path, framePath)
const frame = readFileSync(framePath)
job.library.extendPart1Path = part1Path
job.library.prompt = ext.prompt
const sound = await probeHasAudio(currentPath)
const seed = Math.floor(Math.random() * 2_147_483_647)
server/utils/comfy.ts
+8 -6
View File
@@ -209,25 +209,27 @@ export async function isComfyPromptDropped(promptId: string) {
if (queue.running > 0 || queue.pending > 0) return false
const history = await fetchHistory(promptId)
const entry = history?.[promptId] as { status?: { status_str?: string; completed?: boolean } } | undefined
if (!entry) return true
if (!entry) return false
const inspected = inspectHistory(history, promptId)
if (inspected.video) return false
const status = entry.status?.status_str
if (status === 'error' || status === 'interrupted') return true
if (entry.status?.completed) return true
return true
return false
}
export function inspectHistory(history: Record<string, unknown> | null, promptId: string) {
const video = extractVideo(history, promptId)
if (video) return { video, completed: true, error: null as string | null }
if (!history) return { video: null, completed: false, error: null as string | null }
const entry = (history[promptId] || history) as {
status?: { status_str?: string; completed?: boolean }
}
const error = entry?.status?.status_str === 'error' || entry?.status?.status_str === 'interrupted'
? (entry.status.status_str === 'interrupted' ? 'Job interrupted.' : 'ComfyUI reported an execution error')
: null
return { video, completed: Boolean(video), error }
const status = entry?.status?.status_str
if (status === 'interrupted') return { video: null, completed: false, error: 'Job interrupted.' }
if (status === 'error') return { video: null, completed: false, error: 'ComfyUI reported an execution error' }
if (entry?.status?.completed) return { video: null, completed: true, error: 'ComfyUI finished without an MP4' }
return { video: null, completed: false, error: null as string | null }
}
export async function probeComfy() {
server/utils/ffmpeg.ts
+4 -1
View File
@@ -249,9 +249,12 @@ export async function concatMp4(part1Path: string, part2Path: string, outputPath
}
export async function stitchExtension(params: { part1Path: string; part2: Buffer; tmpDir: string }) {
const part2Path = join(params.tmpDir, 'part2.mp4')
const part2Path = join(params.tmpDir, `part2_${Date.now()}.mp4`)
const finalPath = join(params.tmpDir, 'final_stitched_video.mp4')
writeFileSync(part2Path, params.part2)
if (!existsSync(part2Path) || !params.part2.length) {
throw new Error('Extension segment was missing before stitch')
}
try {
await xfadeStitch(params.part1Path, part2Path, finalPath)
} catch (error) {
server/utils/jobs.ts
+1
View File
@@ -23,6 +23,7 @@ export interface JobEvent {
chainStep?: number
chainTotal?: number
chainLabel?: string
overallProgress?: number
}
export interface Job {
server/utils/watch.ts
+48 -23
View File
@@ -57,8 +57,8 @@ export function emitChainJob(job: Job, event: JobEvent, samplePct?: number) {
if (next.message) {
next.message = formatChainMessage(job, next.message, samplePct)
}
if (chained && typeof next.progress === 'number' && next.type !== 'complete') {
next.progress = mapChainProgress(job, next.progress)
if (chained && typeof next.progress === 'number' && next.type !== 'complete' && next.type !== 'checkpoint') {
next.overallProgress = mapChainProgress(job, next.progress)
}
if (chained) {
next.chainStep = step
@@ -78,8 +78,17 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
return new Promise((resolve) => {
let localProgress = 0
const startedAt = Date.now()
let lastActivity = Date.now()
const IDLE_MS = 30 * 60 * 1000
const ABSOLUTE_MS = 3 * 60 * 60 * 1000
const markActivity = () => {
lastActivity = Date.now()
}
const emitLocal = (event: JobEvent, samplePct?: number) => {
markActivity()
if (typeof event.progress === 'number') localProgress = event.progress
emitChainJob(job, event, samplePct)
}
@@ -92,6 +101,25 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
try { ws.close() } catch { /* ignore */ }
}
const armIdle = () => {
if (timeout) clearTimeout(timeout)
if (settled) return
const absLeft = ABSOLUTE_MS - (Date.now() - startedAt)
if (absLeft <= 0) {
void fail('Timed out waiting for ComfyUI (3 hours).')
return
}
const idleLeft = IDLE_MS - (Date.now() - lastActivity)
timeout = setTimeout(() => {
if (settled) return
if (Date.now() - lastActivity >= IDLE_MS) {
void fail('Timed out waiting for ComfyUI progress (30 minutes with no updates). The desktop job may still be running.')
return
}
armIdle()
}, Math.max(1000, Math.min(idleLeft, absLeft)))
}
const fail = async (error: string) => {
if (settled) return
settled = true
@@ -131,23 +159,6 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
tmpDir: job.library.extendTmpDir!
})
}
if (!persist) {
job.segmentBuffer = buffer
await purgeComfyArtifacts({
video,
imageName: job.library.imageName,
imageSubfolder: job.library.imageSubfolder,
promptId: job.promptId
})
deletePendingJob(job.id)
emitLocal({
type: 'status',
message: stitching ? 'Extension stitched' : 'Initial segment ready',
progress: 99
})
resolve()
return true
}
const clip = await saveClip({
ownerKey: job.library.ownerKey,
folderId: job.library.folderId,
@@ -172,12 +183,27 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
job.clipId = clip.id
job.hideThumbnail = clip.hideThumbnail
job.library.thumb = undefined
job.library.name = nextClipPartName(clip.name)
await purgeComfyArtifacts({
video,
imageName: job.library.imageName,
imageSubfolder: job.library.imageSubfolder,
promptId: job.promptId
})
if (!persist) {
job.segmentBuffer = buffer
deletePendingJob(job.id)
emitLocal({
type: 'checkpoint',
message: stitching ? 'Extension checkpoint saved' : 'Initial segment saved',
progress: 99,
clipId: clip.id,
hideThumbnail: job.hideThumbnail,
folderLocked: job.library.folderLocked
})
resolve()
return true
}
removeExtendTemp(job.library.extendTmpDir)
job.library.extendTmpDir = undefined
}
@@ -226,9 +252,10 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
await succeed()
return
}
markActivity()
if (await isComfyPromptDropped(job.promptId)) {
dropMisses += 1
if (dropMisses >= 3) {
if (dropMisses >= 20) {
await fail('ComfyUI dropped this job. Reloading the Comfy interface clears the queue. Generate again.')
}
} else {
@@ -239,9 +266,7 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
}
}
timeout = setTimeout(() => {
void fail('Timed out waiting for ComfyUI (15 minutes).')
}, 15 * 60 * 1000)
armIdle()
pollTimer = setInterval(() => {
void pollHistory()