Add clip extension from the last frame with FFmpeg stitching.

Let a finished player continue into a new MiniMax segment, then losslessly concat the parts and discard the intermediate files so the library keeps one extended MP4.

Co-authored-by: Cursor <cursoragent@cursor.com>

server/utils/ffmpeg.ts

@@ -0,0 +1,156 @@
+import { existsSync, writeFileSync, readFileSync, unlinkSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { spawn } from 'node:child_process'
+
+function collectError(stderr: string, fallback: string) {
+  const text = stderr.replace(/\r/g, '').trim()
+  const lines = text.split('\n').map(line => line.trim()).filter(Boolean)
+  return lines.slice(-6).join(' ') || fallback
+}
+
+export function runFfmpeg(args: string[], timeoutMs = 120000) {
+  return new Promise<void>((resolve, reject) => {
+    const proc = spawn('ffmpeg', args, { stdio: ['ignore', 'ignore', 'pipe'] })
+    let stderr = ''
+    proc.stderr?.on('data', (chunk: Buffer | string) => { stderr += String(chunk) })
+    const timer = setTimeout(() => {
+      try { proc.kill('SIGKILL') } catch { /* ignore */ }
+      reject(new Error(`FFmpeg timed out after ${Math.round(timeoutMs / 1000)}s`))
+    }, timeoutMs)
+    proc.on('error', (error) => {
+      clearTimeout(timer)
+      const message = (error as NodeJS.ErrnoException).code === 'ENOENT'
+        ? 'FFmpeg is not installed on this server'
+        : error.message
+      reject(new Error(message))
+    })
+    proc.on('close', (code) => {
+      clearTimeout(timer)
+      if (code === 0) resolve()
+      else reject(new Error(collectError(stderr, `FFmpeg exited with code ${code}`)))
+    })
+  })
+}
+
+function runFfprobe(args: string[], timeoutMs = 15000) {
+  return new Promise<string>((resolve, reject) => {
+    const proc = spawn('ffprobe', args, { stdio: ['ignore', 'pipe', 'pipe'] })
+    let out = ''
+    let err = ''
+    proc.stdout?.on('data', (chunk: Buffer | string) => { out += String(chunk) })
+    proc.stderr?.on('data', (chunk: Buffer | string) => { err += String(chunk) })
+    const timer = setTimeout(() => {
+      try { proc.kill('SIGKILL') } catch { /* ignore */ }
+      reject(new Error('ffprobe timed out'))
+    }, timeoutMs)
+    proc.on('error', (error) => {
+      clearTimeout(timer)
+      reject(error)
+    })
+    proc.on('close', (code) => {
+      clearTimeout(timer)
+      if (code === 0) resolve(out.trim())
+      else reject(new Error(collectError(err, `ffprobe exited with code ${code}`)))
+    })
+  })
+}
+
+function concatPath(filePath: string) {
+  return filePath.replace(/\\/g, '/').replace(/'/g, "'\\''")
+}
+
+export async function extractLastFrame(inputPath: string, outputPath: string) {
+  if (!existsSync(inputPath)) {
+    throw new Error('Source video is missing')
+  }
+  await runFfmpeg([
+    '-y',
+    '-sseof', '-1',
+    '-i', inputPath,
+    '-update', '1',
+    '-frames:v', '1',
+    '-q:v', '1',
+    outputPath
+  ], 30000)
+  if (!existsSync(outputPath)) {
+    throw new Error('FFmpeg did not write the last frame')
+  }
+}
+
+export async function probeHasAudio(filePath: string) {
+  try {
+    const out = await runFfprobe([
+      '-v', 'error',
+      '-select_streams', 'a:0',
+      '-show_entries', 'stream=codec_type',
+      '-of', 'csv=p=0',
+      filePath
+    ])
+    return out.includes('audio')
+  } catch {
+    return false
+  }
+}
+
+export async function concatMp4(part1Path: string, part2Path: string, outputPath: string) {
+  const listPath = join(dirname(outputPath), 'concat_list.txt')
+  writeFileSync(listPath, `file '${concatPath(part1Path)}'\nfile '${concatPath(part2Path)}'\n`)
+  try {
+    await runFfmpeg(['-y', '-f', 'concat', '-safe', '0', '-i', listPath, '-c', 'copy', outputPath], 180000)
+  } catch (copyError) {
+    const audio1 = await probeHasAudio(part1Path)
+    const audio2 = await probeHasAudio(part2Path)
+    try {
+      if (audio1 && audio2) {
+        await runFfmpeg([
+          '-y',
+          '-i', part1Path,
+          '-i', part2Path,
+          '-filter_complex', '[0:v][0:a][1:v][1:a]concat=n=2:v=1:a=1[v][a]',
+          '-map', '[v]',
+          '-map', '[a]',
+          '-c:v', 'libx264',
+          '-preset', 'fast',
+          '-crf', '18',
+          '-pix_fmt', 'yuv420p',
+          '-c:a', 'aac',
+          '-movflags', '+faststart',
+          outputPath
+        ], 300000)
+      } else {
+        await runFfmpeg([
+          '-y',
+          '-i', part1Path,
+          '-i', part2Path,
+          '-filter_complex', '[0:v][1:v]concat=n=2:v=1:a=0[v]',
+          '-map', '[v]',
+          '-c:v', 'libx264',
+          '-preset', 'fast',
+          '-crf', '18',
+          '-pix_fmt', 'yuv420p',
+          '-movflags', '+faststart',
+          outputPath
+        ], 300000)
+      }
+    } catch (encodeError) {
+      const copyMessage = copyError instanceof Error ? copyError.message : String(copyError)
+      const encodeMessage = encodeError instanceof Error ? encodeError.message : String(encodeError)
+      throw new Error(`FFmpeg concat failed (${copyMessage}). Re-encode also failed (${encodeMessage}).`)
+    }
+  }
+  if (!existsSync(outputPath)) {
+    throw new Error('FFmpeg did not write the stitched video')
+  }
+}
+
+export async function stitchExtension(params: { part1Path: string; part2: Buffer; tmpDir: string }) {
+  const part2Path = join(params.tmpDir, 'part2.mp4')
+  const finalPath = join(params.tmpDir, 'final_stitched_video.mp4')
+  writeFileSync(part2Path, params.part2)
+  await concatMp4(params.part1Path, part2Path, finalPath)
+  const stitched = readFileSync(finalPath)
+  for (const extra of [part2Path, join(params.tmpDir, 'concat_list.txt'), join(params.tmpDir, 'last_frame.png')]) {
+    try { unlinkSync(extra) } catch { /* ignore */ }
+  }
+  return stitched
+}

server/utils/jobs.ts

@@ -61,6 +61,9 @@ export interface Job {
     duration?: number
     sound?: boolean
     draftId?: string
+    extendTmpDir?: string
+    extendPart1Path?: string
+    extendSourceClipId?: string
   }
   error?: string
   socketReady?: boolean

server/utils/library.ts

@@ -462,6 +462,21 @@ export function clipVideoPath(owner: string, id: string) {
   return join(clipDir(owner, id), 'video.mp4')
 }
 
+export function clipLastFramePath(owner: string, id: string) {
+  return join(clipDir(owner, id), 'last-frame.png')
+}
+
+export function extendTempDir(owner: string, jobId: string) {
+  const dir = join(ownerRoot(owner), 'tmp', 'extend', jobId)
+  mkdirSync(dir, { recursive: true })
+  return dir
+}
+
+export function removeExtendTemp(dir?: string) {
+  if (!dir) return
+  rmSync(dir, { recursive: true, force: true })
+}
+
 export function clipThumbPath(owner: string, id: string) {
   const dir = clipDir(owner, id)
   for (const name of ['thumb.jpg', 'thumb.png', 'thumb']) {

server/utils/pending.ts

@@ -20,6 +20,8 @@ export interface PendingJob {
   startedAt: number
   imageName?: string
   imageSubfolder?: string
+  extendTmpDir?: string
+  extendPart1Path?: string
 }
 
 function pendingRoot() {
@@ -68,7 +70,24 @@ export async function completePendingIfReady(pending: PendingJob) {
   const history = await fetchHistory(pending.promptId)
   const video = extractVideo(history, pending.promptId)
   if (!video) return null
-  const buffer = await downloadComfyVideo(video)
+  let buffer = await downloadComfyVideo(video)
+  if (pending.extendPart1Path && pending.extendTmpDir) {
+    if (!existsSync(pending.extendPart1Path)) {
+      removeExtendTemp(pending.extendTmpDir)
+      throw createError({ statusCode: 500, statusMessage: 'Extension source clip was missing during stitch' })
+    }
+    try {
+      buffer = await stitchExtension({
+        part1Path: pending.extendPart1Path,
+        part2: buffer,
+        tmpDir: pending.extendTmpDir
+      })
+    } catch (error) {
+      removeExtendTemp(pending.extendTmpDir)
+      throw error
+    }
+    removeExtendTemp(pending.extendTmpDir)
+  }
   const clip = await saveClip({
     ownerKey: pending.ownerKey,
     folderId: pending.folderId,
@@ -90,7 +109,9 @@ export async function completePendingIfReady(pending: PendingJob) {
   return {
     type: 'complete' as const,
     status: 'complete' as const,
-    message: pending.folderLocked ? 'Saved to the locked folder. Unlock it to view.' : 'Video ready',
+    message: pending.folderLocked
+      ? 'Saved to the locked folder. Unlock it to view.'
+      : (pending.extendPart1Path ? 'Extended video ready' : 'Video ready'),
     progress: 100,
     jobId: pending.jobId,
     clipId: clip.id,

server/utils/watch.ts

@@ -1,4 +1,4 @@
-import { NODE_LABELS, isEncodingNode } from '~/server/utils/workflow'
+import { existsSync } from 'node:fs'
 import type { Job } from '~/server/utils/jobs'
 
 function classifyError(message: string) {
@@ -42,6 +42,7 @@ export function watchComfyJob(job: Job): Promise<void> {
       job.error = classifyError(error)
       emitJob(job, { type: 'error', error: job.error, message: job.error })
       if (job.id) deletePendingJob(job.id)
+      removeExtendTemp(job.library?.extendTmpDir)
       resolve()
     }
 
@@ -54,9 +55,24 @@ export function watchComfyJob(job: Job): Promise<void> {
       cleanup()
       try {
         job.video = video
-        emitJob(job, { type: 'status', message: 'Saving to library...', progress: 98 })
+        const stitching = Boolean(job.library?.extendPart1Path && job.library.extendTmpDir)
+        emitJob(job, {
+          type: 'status',
+          message: stitching ? 'Extracting last frame & stitching extension...' : 'Saving to library...',
+          progress: stitching ? 97 : 98
+        })
         if (job.library) {
-          const buffer = await downloadComfyVideo(video)
+          let buffer = await downloadComfyVideo(video)
+          if (stitching) {
+            if (!existsSync(job.library.extendPart1Path!)) {
+              throw new Error('Extension source clip was missing during stitch')
+            }
+            buffer = await stitchExtension({
+              part1Path: job.library.extendPart1Path!,
+              part2: buffer,
+              tmpDir: job.library.extendTmpDir!
+            })
+          }
           const clip = await saveClip({
             ownerKey: job.library.ownerKey,
             folderId: job.library.folderId,
@@ -87,11 +103,15 @@ export function watchComfyJob(job: Job): Promise<void> {
             imageSubfolder: job.library.imageSubfolder,
             promptId: job.promptId
           })
+          removeExtendTemp(job.library.extendTmpDir)
+          job.library.extendTmpDir = undefined
         }
         job.status = 'complete'
         emitJob(job, {
           type: 'complete',
-          message: job.library?.folderLocked ? 'Saved to the locked folder. Unlock it to view.' : 'Video ready',
+          message: job.library?.folderLocked
+            ? 'Saved to the locked folder. Unlock it to view.'
+            : (job.library?.extendSourceClipId ? 'Extended video ready' : 'Video ready'),
           progress: 100,
           filename: job.library?.folderLocked ? undefined : video.filename,
           subfolder: job.library?.folderLocked ? undefined : video.subfolder,
@@ -102,6 +122,7 @@ export function watchComfyJob(job: Job): Promise<void> {
         })
         deletePendingJob(job.id)
       } catch (saveError) {
+        removeExtendTemp(job.library?.extendTmpDir)
         const message = saveError instanceof Error ? saveError.message : String(saveError)
         job.status = 'error'
         job.error = `Video generated but library save failed: ${message}`