Ship extension queues, library extend, and smoother seam stitching.

Users can pre-queue extensions, extend from the library with p2 titles, blend seams with xfade, and a unreadable catalog is no longer replaced with an empty one. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-26 00:01:25 -05:00
parent 22d99f18db
commit f5c709924b
7 changed files with 653 additions and 79 deletions
@@ -92,6 +92,132 @@ export async function probeHasAudio(filePath: string) {
  }
 }

+const XFADE_DURATION = 0.12
+
+async function probeDuration(filePath: string) {
+  const out = await runFfprobe([
+    '-v', 'error',
+    '-show_entries', 'format=duration',
+    '-of', 'default=nokey=1:noprint_wrappers=1',
+    filePath
+  ])
+  const duration = Number(out)
+  if (!Number.isFinite(duration) || duration <= 0) {
+    throw new Error('Could not read video duration')
+  }
+  return duration
+}
+
+async function probeFps(filePath: string) {
+  const out = await runFfprobe([
+    '-v', 'error',
+    '-select_streams', 'v:0',
+    '-show_entries', 'stream=avg_frame_rate,r_frame_rate',
+    '-of', 'default=nokey=1:noprint_wrappers=1',
+    filePath
+  ])
+  const line = out.split(/\r?\n/).map(item => item.trim()).find(Boolean) || ''
+  const [num, den] = line.split('/').map(Number)
+  if (Number.isFinite(num) && Number.isFinite(den) && den > 0) return num / den
+  const fps = Number(line)
+  return Number.isFinite(fps) && fps > 0 ? fps : 24
+}
+
+async function probeSize(filePath: string) {
+  const out = await runFfprobe([
+    '-v', 'error',
+    '-select_streams', 'v:0',
+    '-show_entries', 'stream=width,height',
+    '-of', 'csv=p=0',
+    filePath
+  ])
+  const [width, height] = out.split(',').map(Number)
+  return {
+    width: Number.isFinite(width) && width > 0 ? width : 1344,
+    height: Number.isFinite(height) && height > 0 ? height : 768
+  }
+}
+
+async function trimLeadingFrame(inputPath: string, outputPath: string) {
+  const fps = await probeFps(inputPath)
+  const hasAudio = await probeHasAudio(inputPath)
+  const args = [
+    '-y',
+    '-i', inputPath,
+    '-vf', 'trim=start_frame=1,setpts=PTS-STARTPTS'
+  ]
+  if (hasAudio) {
+    args.push('-af', `atrim=start=${(1 / fps).toFixed(6)},asetpts=PTS-STARTPTS`, '-c:a', 'aac')
+  } else {
+    args.push('-an')
+  }
+  args.push(
+    '-c:v', 'libx264',
+    '-preset', 'fast',
+    '-crf', '17',
+    '-pix_fmt', 'yuv420p',
+    '-movflags', '+faststart',
+    outputPath
+  )
+  await runFfmpeg(args, 180000)
+  if (!existsSync(outputPath)) {
+    throw new Error('FFmpeg did not write the trimmed extension segment')
+  }
+}
+
+async function xfadeStitch(part1Path: string, part2Path: string, outputPath: string) {
+  const [duration1, duration2, fps, size, audio1, audio2] = await Promise.all([
+    probeDuration(part1Path),
+    probeDuration(part2Path),
+    probeFps(part1Path),
+    probeSize(part1Path),
+    probeHasAudio(part1Path),
+    probeHasAudio(part2Path)
+  ])
+  const fade = Math.min(
+    XFADE_DURATION,
+    Math.max(0.04, duration1 - 0.05),
+    Math.max(0.04, duration2 - 0.05)
+  )
+  const offset = Math.max(0, Number((duration1 - fade).toFixed(3)))
+  const fpsLabel = Number.isInteger(fps) ? String(fps) : fps.toFixed(3)
+  const prepare = `fps=${fpsLabel},scale=${size.width}:${size.height}:flags=bicubic,format=yuv420p,setpts=PTS-STARTPTS`
+  const videoFilter = `[0:v]${prepare}[v0];[1:v]${prepare}[v1];[v0][v1]xfade=transition=fade:duration=${fade.toFixed(3)}:offset=${offset},setpts=PTS-STARTPTS[v]`
+  const encode = [
+    '-c:v', 'libx264',
+    '-preset', 'fast',
+    '-crf', '17',
+    '-pix_fmt', 'yuv420p',
+    '-movflags', '+faststart'
+  ]
+  if (audio1 && audio2) {
+    await runFfmpeg([
+      '-y',
+      '-i', part1Path,
+      '-i', part2Path,
+      '-filter_complex', `${videoFilter};[0:a]asetpts=PTS-STARTPTS[a0];[1:a]asetpts=PTS-STARTPTS[a1];[a0][a1]acrossfade=d=${fade.toFixed(3)}:c1=tri:c2=tri[a]`,
+      '-map', '[v]',
+      '-map', '[a]',
+      ...encode,
+      '-c:a', 'aac',
+      outputPath
+    ], 300000)
+  } else {
+    await runFfmpeg([
+      '-y',
+      '-i', part1Path,
+      '-i', part2Path,
+      '-filter_complex', videoFilter,
+      '-map', '[v]',
+      ...encode,
+      outputPath
+    ], 300000)
+  }
+  if (!existsSync(outputPath)) {
+    throw new Error('FFmpeg did not write the blended video')
+  }
+}
+
 export async function concatMp4(part1Path: string, part2Path: string, outputPath: string) {
  const listPath = join(dirname(outputPath), 'concat_list.txt')
  writeFileSync(listPath, `file '${concatPath(part1Path)}'\nfile '${concatPath(part2Path)}'\n`)
@@ -111,7 +237,7 @@ export async function concatMp4(part1Path: string, part2Path: string, outputPath
          '-map', '[a]',
          '-c:v', 'libx264',
          '-preset', 'fast',
-          '-crf', '18',
+          '-crf', '17',
          '-pix_fmt', 'yuv420p',
          '-c:a', 'aac',
          '-movflags', '+faststart',
@@ -126,7 +252,7 @@ export async function concatMp4(part1Path: string, part2Path: string, outputPath
          '-map', '[v]',
          '-c:v', 'libx264',
          '-preset', 'fast',
-          '-crf', '18',
+          '-crf', '17',
          '-pix_fmt', 'yuv420p',
          '-movflags', '+faststart',
          outputPath
@@ -145,11 +271,24 @@ export async function concatMp4(part1Path: string, part2Path: string, outputPath

 export async function stitchExtension(params: { part1Path: string; part2: Buffer; tmpDir: string }) {
  const part2Path = join(params.tmpDir, 'part2.mp4')
+  const trimmedPath = join(params.tmpDir, 'part2_trim.mp4')
  const finalPath = join(params.tmpDir, 'final_stitched_video.mp4')
  writeFileSync(part2Path, params.part2)
-  await concatMp4(params.part1Path, part2Path, finalPath)
+  try {
+    await trimLeadingFrame(part2Path, trimmedPath)
+    await xfadeStitch(params.part1Path, trimmedPath, finalPath)
+  } catch (blendError) {
+    try {
+      const trimmedReady = existsSync(trimmedPath)
+      await concatMp4(params.part1Path, trimmedReady ? trimmedPath : part2Path, finalPath)
+    } catch (concatError) {
+      const blendMessage = blendError instanceof Error ? blendError.message : String(blendError)
+      const concatMessage = concatError instanceof Error ? concatError.message : String(concatError)
+      throw new Error(`Seam blend failed (${blendMessage}). Concat fallback also failed (${concatMessage}).`)
+    }
+  }
  const stitched = readFileSync(finalPath)
-  for (const extra of [part2Path, join(params.tmpDir, 'concat_list.txt'), join(params.tmpDir, 'last_frame.png')]) {
+  for (const extra of [part2Path, trimmedPath, join(params.tmpDir, 'concat_list.txt'), join(params.tmpDir, 'last_frame.png')]) {
    try { unlinkSync(extra) } catch { /* ignore */ }
  }
  return stitched
@@ -20,6 +20,9 @@ export interface JobEvent {
  folderLocked?: boolean
  jobId?: string
  draftId?: string
+  chainStep?: number
+  chainTotal?: number
+  chainLabel?: string
 }

 export interface Job {
@@ -61,12 +64,18 @@ export interface Job {
    duration?: number
    sound?: boolean
    draftId?: string
+    extensions?: { prompt: string; duration: number }[]
+    chainIndex?: number
+    chainStep?: number
+    chainTotal?: number
+    chainLabel?: string
    extendTmpDir?: string
    extendPart1Path?: string
    extendSourceClipId?: string
  }
  error?: string
  socketReady?: boolean
+  segmentBuffer?: Buffer
  events: JobEvent[]
  listeners: Set<(event: JobEvent) => void>
 }
@@ -121,7 +130,10 @@ export function jobSnapshot(job: Job) {
    hideThumbnail: job.hideThumbnail,
    error: job.error,
    folderLocked: job.library?.folderLocked,
-    draftId: job.library?.draftId
+    draftId: job.library?.draftId,
+    chainStep: job.library?.chainStep,
+    chainTotal: job.library?.chainTotal,
+    chainLabel: job.library?.chainLabel
  }
 }

@@ -131,6 +143,13 @@ export function emitJob(job: Job, event: JobEvent) {
  if (typeof event.progress === 'number') job.progress = event.progress
  if (typeof event.step === 'number') job.step = event.step
  if (typeof event.maxStep === 'number') job.maxStep = event.maxStep
+  if (typeof event.chainStep === 'number') {
+    if (job.library) job.library.chainStep = event.chainStep
+  }
+  if (typeof event.chainTotal === 'number') {
+    if (job.library) job.library.chainTotal = event.chainTotal
+  }
+  if (event.chainLabel && job.library) job.library.chainLabel = event.chainLabel
  event.elapsedMs = Date.now() - job.startedAt
  job.events.push(event)
  if (job.events.length > 200) job.events.shift()
@@ -57,6 +57,11 @@ export interface PublicFolder {
  clipCount: number
 }

+export interface QueuedExtension {
+  prompt: string
+  duration: number
+}
+
 export interface RetryDraft {
  id: string
  createdAt: number
@@ -79,6 +84,7 @@ export interface RetryDraft {
  fps?: number
  samplerName?: string
  scheduler?: string
+  extensions?: QueuedExtension[]
 }

 interface Catalog {
@@ -198,11 +204,15 @@ function ensureOwner(owner: string) {

 function readCatalog(owner: string): Catalog {
  ensureOwner(owner)
+  const path = catalogPath(owner)
  try {
-    return normalizeCatalog(JSON.parse(readFileSync(catalogPath(owner), 'utf8')))
+    return normalizeCatalog(JSON.parse(readFileSync(path, 'utf8')))
  } catch {
+    if (existsSync(path)) {
+      throw createError({ statusCode: 500, statusMessage: 'Library catalog could not be read' })
+    }
    const created = emptyCatalog()
-    writeFileSync(catalogPath(owner), JSON.stringify(created, null, 2))
+    writeFileSync(path, JSON.stringify(created, null, 2))
    return created
  }
 }
@@ -233,6 +243,16 @@ export function clipTitle(clip: Pick<LibraryClip, 'name' | 'prompt'>) {
  return fromPrompt || 'Untitled clip'
 }

+export function nextClipPartName(name: string) {
+  const trimmed = (name || '').trim()
+  if (!trimmed) return ''
+  const match = trimmed.match(/^(.*)\s+p(\d+)$/i)
+  const base = (match ? match[1] : trimmed).trim() || trimmed
+  const part = match ? Number(match[2]) + 1 : 2
+  const suffix = ` p${part}`
+  return `${base.slice(0, Math.max(1, 80 - suffix.length))}${suffix}`
+}
+
 export function safeDownloadName(name: string) {
  const cleaned = name.replace(/[<>:"/\\|?*'`\u0000-\u001f]/g, '-').replace(/\s+/g, ' ').trim()
  return cleaned.slice(0, 80) || 'aigen'
@@ -689,6 +709,7 @@ export async function saveRetryDraft(params: {
  fps?: number
  samplerName?: string
  scheduler?: string
+  extensions?: QueuedExtension[]
 }) {
  return mutate(params.ownerKey, (catalog) => {
    const existing = params.id ? catalog.drafts.find(item => item.id === params.id) : undefined
@@ -713,7 +734,8 @@ export async function saveRetryDraft(params: {
      cfg: params.cfg,
      fps: params.fps,
      samplerName: params.samplerName,
-      scheduler: params.scheduler
+      scheduler: params.scheduler,
+      extensions: params.extensions?.length ? params.extensions : undefined
    }
    catalog.drafts = [draft, ...catalog.drafts.filter(item => item.id !== draft.id)].slice(0, 20)
    return draft
@@ -1,5 +1,5 @@
 import { existsSync } from 'node:fs'
-import type { Job } from '~/server/utils/jobs'
+import type { Job, JobEvent } from '~/server/utils/jobs'

 function classifyError(message: string) {
  const lower = message.toLowerCase()
@@ -19,13 +19,71 @@ function sleep(ms: number) {
  return new Promise(resolve => setTimeout(resolve, ms))
 }

-export function watchComfyJob(job: Job): Promise<void> {
+function chainMeta(job: Job) {
+  const total = Math.max(1, job.library?.chainTotal || 1)
+  const index = job.library?.chainIndex ?? 0
+  return {
+    total,
+    index,
+    step: index + 1,
+    label: job.library?.chainLabel || (index === 0 ? 'Initial' : `Extension ${index}`),
+    chained: total > 1
+  }
+}
+
+export function mapChainProgress(job: Job, localPct: number) {
+  const { total, index, chained } = chainMeta(job)
+  const local = Math.max(0, Math.min(100, localPct))
+  if (!chained) return Math.round(local)
+  const share = 100 / total
+  return Math.min(99, Math.round(index * share + (Math.min(local, 99) / 100) * share))
+}
+
+export function formatChainMessage(job: Job, message: string, samplePct?: number) {
+  const { chained, step, total, label } = chainMeta(job)
+  if (!chained) return message
+  if (typeof samplePct === 'number') {
+    return `Step ${step}/${total}: Generating ${label} (${samplePct}%)`
+  }
+  if (/waiting 3s buffer/i.test(message)) {
+    return `Step ${step}/${total}: Waiting 3s buffer...`
+  }
+  return `Step ${step}/${total}: ${message}`
+}
+
+export function emitChainJob(job: Job, event: JobEvent, samplePct?: number) {
+  const { chained, step, total, label } = chainMeta(job)
+  const next: JobEvent = { ...event }
+  if (next.message) {
+    next.message = formatChainMessage(job, next.message, samplePct)
+  }
+  if (chained && typeof next.progress === 'number' && next.type !== 'complete') {
+    next.progress = mapChainProgress(job, next.progress)
+  }
+  if (chained) {
+    next.chainStep = step
+    next.chainTotal = total
+    next.chainLabel = label
+  }
+  emitJob(job, next)
+}
+
+export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Promise<void> {
+  const persist = options.persist !== false
+  job.socketReady = false
  const ws = new WebSocket(comfyWsUrl(job.clientId))
  let settled = false
  let pollTimer: ReturnType<typeof setInterval> | null = null
  let timeout: ReturnType<typeof setTimeout> | null = null

  return new Promise((resolve) => {
+    let localProgress = 0
+
+    const emitLocal = (event: JobEvent, samplePct?: number) => {
+      if (typeof event.progress === 'number') localProgress = event.progress
+      emitChainJob(job, event, samplePct)
+    }
+
    const cleanup = () => {
      if (timeout) clearTimeout(timeout)
      if (pollTimer) clearInterval(pollTimer)
@@ -40,7 +98,7 @@ export function watchComfyJob(job: Job): Promise<void> {
      cleanup()
      job.status = job.status === 'cancelled' ? 'cancelled' : 'error'
      job.error = classifyError(error)
-      emitJob(job, { type: 'error', error: job.error, message: job.error })
+      emitChainJob(job, { type: 'error', error: job.error, message: job.error })
      if (job.id) deletePendingJob(job.id)
      removeExtendTemp(job.library?.extendTmpDir)
      resolve()
@@ -56,9 +114,9 @@ export function watchComfyJob(job: Job): Promise<void> {
      try {
        job.video = video
        const stitching = Boolean(job.library?.extendPart1Path && job.library.extendTmpDir)
-        emitJob(job, {
+        emitLocal({
          type: 'status',
-          message: stitching ? 'Extracting last frame & stitching extension...' : 'Saving to library...',
+          message: stitching ? 'Extracting last frame & stitching extension...' : (persist ? 'Saving to library...' : 'Downloading segment...'),
          progress: stitching ? 97 : 98
        })
        if (job.library) {
@@ -73,6 +131,23 @@ export function watchComfyJob(job: Job): Promise<void> {
              tmpDir: job.library.extendTmpDir!
            })
          }
+          if (!persist) {
+            job.segmentBuffer = buffer
+            await purgeComfyArtifacts({
+              video,
+              imageName: job.library.imageName,
+              imageSubfolder: job.library.imageSubfolder,
+              promptId: job.promptId
+            })
+            deletePendingJob(job.id)
+            emitLocal({
+              type: 'status',
+              message: stitching ? 'Extension stitched' : 'Initial segment ready',
+              progress: 99
+            })
+            resolve()
+            return true
+          }
          const clip = await saveClip({
            ownerKey: job.library.ownerKey,
            folderId: job.library.folderId,
@@ -111,22 +186,27 @@ export function watchComfyJob(job: Job): Promise<void> {
          type: 'complete',
          message: job.library?.folderLocked
            ? 'Saved to the locked folder. Unlock it to view.'
-            : (job.library?.extendSourceClipId ? 'Extended video ready' : 'Video ready'),
+            : (job.library?.extendSourceClipId || (job.library?.chainTotal || 1) > 1 ? 'Extended video ready' : 'Video ready'),
          progress: 100,
          filename: job.library?.folderLocked ? undefined : video.filename,
          subfolder: job.library?.folderLocked ? undefined : video.subfolder,
          mediaType: job.library?.folderLocked ? undefined : video.type,
          clipId: job.clipId,
          hideThumbnail: job.hideThumbnail,
-          folderLocked: job.library?.folderLocked
+          folderLocked: job.library?.folderLocked,
+          chainStep: job.library?.chainStep,
+          chainTotal: job.library?.chainTotal,
+          chainLabel: job.library?.chainLabel
        })
        deletePendingJob(job.id)
      } catch (saveError) {
        removeExtendTemp(job.library?.extendTmpDir)
        const message = saveError instanceof Error ? saveError.message : String(saveError)
        job.status = 'error'
-        job.error = `Video generated but library save failed: ${message}`
-        emitJob(job, { type: 'error', error: job.error, message: job.error })
+        job.error = persist
+          ? `Video generated but library save failed: ${message}`
+          : `Segment finished but could not be prepared: ${message}`
+        emitChainJob(job, { type: 'error', error: job.error, message: job.error })
      }
      resolve()
      return true
@@ -169,7 +249,7 @@ export function watchComfyJob(job: Job): Promise<void> {

    ws.addEventListener('open', () => {
      job.socketReady = true
-      emitJob(job, { type: 'status', message: 'Connected to ComfyUI', progress: Math.max(job.progress, 4) })
+      emitLocal({ type: 'status', message: 'Connected to ComfyUI', progress: Math.max(localProgress, 4) })
    })

    ws.addEventListener('close', () => {
@@ -191,15 +271,16 @@ export function watchComfyJob(job: Job): Promise<void> {
        const value = Number(data.value || 0)
        const max = Number(data.max || 1)
        const pct = 12 + Math.round((value / Math.max(max, 1)) * 70)
+        const samplePct = Math.round((value / Math.max(max, 1)) * 100)
        job.status = 'running'
-        emitJob(job, {
+        emitLocal({
          type: 'progress',
          step: value,
          maxStep: max,
          progress: pct,
          node: data.node ? String(data.node) : null,
-          message: `Sampling step ${value}/${max} (${Math.round((value / Math.max(max, 1)) * 100)}%)`
-        })
+          message: `Sampling step ${value}/${max} (${samplePct}%)`
+        }, samplePct)
      }

      if (payload.type === 'executing') {
@@ -207,16 +288,16 @@ export function watchComfyJob(job: Job): Promise<void> {
        if (node === null) {
          const saved = await succeed()
          if (!saved) {
-            emitJob(job, { type: 'status', message: 'Waiting for ComfyUI to save the MP4...', progress: 96 })
+            emitLocal({ type: 'status', message: 'Waiting for ComfyUI to save the MP4...', progress: 96 })
          }
          return
        }
        const label = NODE_LABELS[node] || `Running node ${node}`
        const encoding = isEncodingNode(node)
-        emitJob(job, {
+        emitLocal({
          type: 'executing',
          node,
-          progress: encoding ? 90 : Math.max(job.progress, 10),
+          progress: encoding ? 90 : Math.max(localProgress, 10),
          message: encoding ? 'Encoding video...' : `${label}...`
        })
      }