Add clip extension from the last frame with FFmpeg stitching.

Let a finished player continue into a new MiniMax segment, then losslessly concat the parts and discard the intermediate files so the library keeps one extended MP4.

Co-authored-by: Cursor <cursoragent@cursor.com>

server/api/extend.post.ts

@@ -0,0 +1,233 @@
+import { copyFileSync, existsSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
+const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
+
+function parseExtendDuration(raw: unknown) {
+  const seconds = Number(raw)
+  return seconds === 3 || seconds === 5 ? seconds : 5
+}
+
+function frameLength(seconds: number, fps: number) {
+  return Math.max(5, Math.floor(seconds * fps))
+}
+
+function parseFps(raw: unknown) {
+  const fps = Number(raw)
+  return fps === 12 || fps === 30 || fps === 24 ? fps : 24
+}
+
+function parseCfg(raw: unknown, turbo: boolean) {
+  const fallback = turbo ? 1.5 : 4
+  const value = Number(raw)
+  if (!Number.isFinite(value)) return fallback
+  const snapped = Math.round(value * 2) / 2
+  return Math.min(10, Math.max(1, snapped))
+}
+
+function parseSampler(raw: unknown) {
+  const value = String(raw || '')
+  return SAMPLERS.has(value) ? value : 'res_multistep'
+}
+
+function parseScheduler(raw: unknown) {
+  const value = String(raw || '')
+  return SCHEDULERS.has(value) ? value : 'simple'
+}
+
+export default defineEventHandler(async (event) => {
+  const body = await readBody<{ clipId?: string; prompt?: string; duration?: number }>(event).catch(() => ({}))
+  const clipId = String(body?.clipId || '').trim()
+  const prompt = String(body?.prompt || '').trim()
+  if (!clipId) {
+    throw createError({ statusCode: 400, statusMessage: 'A clip is required to extend' })
+  }
+  if (!prompt) {
+    throw createError({ statusCode: 400, statusMessage: 'An extension prompt is required' })
+  }
+
+  const { owner } = assertLibraryOwner(event)
+  const source = getClip(owner, clipId)
+  assertFolderAccess(event, source.folderId)
+  const sourcePath = clipVideoPath(owner, source.id)
+  if (!existsSync(sourcePath)) {
+    throw createError({ statusCode: 404, statusMessage: 'Source video is missing' })
+  }
+
+  const destFolder = publicLibrary(event).folders.find(folder => folder.id === source.folderId)
+  const folderLocked = Boolean(destFolder?.protected && !destFolder.unlocked)
+  const durationSeconds = parseExtendDuration(body?.duration)
+  const turbo = source.turbo !== false
+  const fps = parseFps(source.fps)
+  const cfg = parseCfg(source.cfg, turbo)
+  const samplerName = parseSampler(source.samplerName)
+  const scheduler = parseScheduler(source.scheduler)
+  const steps = Number(source.steps || (turbo ? 8 : 20))
+  const seed = Math.floor(Math.random() * 2_147_483_647)
+  const length = frameLength(durationSeconds, fps)
+  const width = source.width || 1344
+  const height = source.height || 768
+  const clipName = `${clipTitle(source)} extended`.slice(0, 80)
+
+  const job = createJob()
+  job.maxStep = steps
+  job.hideThumbnail = source.hideThumbnail
+  const tmpDir = extendTempDir(owner, job.id)
+  const part1Path = join(tmpDir, 'part1.mp4')
+  const framePath = join(tmpDir, 'last_frame.png')
+  job.library = {
+    ownerKey: owner,
+    folderId: source.folderId,
+    hideThumbnail: source.hideThumbnail,
+    folderLocked,
+    name: clipName,
+    prompt,
+    aspect: source.aspect || '16:9',
+    width,
+    height,
+    steps,
+    turbo,
+    seed,
+    cfg,
+    fps,
+    samplerName,
+    scheduler,
+    duration: durationSeconds,
+    extendTmpDir: tmpDir,
+    extendPart1Path: part1Path,
+    extendSourceClipId: source.id
+  }
+
+  emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 })
+
+  void runExtend(job, {
+    sourcePath,
+    part1Path,
+    framePath,
+    prompt,
+    width,
+    height,
+    steps,
+    seed,
+    turbo,
+    length,
+    cfg,
+    fps,
+    samplerName,
+    scheduler
+  }).catch((error) => {
+    removeExtendTemp(job.library?.extendTmpDir)
+    const message = error instanceof Error ? error.message : String(error)
+    job.status = 'error'
+    job.error = message
+    emitJob(job, { type: 'error', error: message, message })
+  })
+
+  return {
+    jobId: job.id,
+    clientId: job.clientId,
+    seed,
+    width,
+    height,
+    steps,
+    duration: durationSeconds,
+    hideThumbnail: source.hideThumbnail,
+    folderLocked
+  }
+})
+
+async function runExtend(
+  job: ReturnType<typeof createJob>,
+  params: {
+    sourcePath: string
+    part1Path: string
+    framePath: string
+    prompt: string
+    width: number
+    height: number
+    steps: number
+    seed: number
+    turbo: boolean
+    length: number
+    cfg: number
+    fps: number
+    samplerName: string
+    scheduler: string
+  }
+) {
+  copyFileSync(params.sourcePath, params.part1Path)
+  await extractLastFrame(params.part1Path, params.framePath)
+  const frame = readFileSync(params.framePath)
+  job.library!.thumb = frame
+  const sound = await probeHasAudio(params.part1Path)
+
+  await ensureComfyReady((status) => {
+    emitJob(job, {
+      type: status.state === 'busy' ? 'busy' : 'status',
+      message: status.message,
+      progress: status.state === 'online' ? Math.max(job.progress, 3) : Math.max(job.progress, 2),
+      busy: status.state === 'busy',
+      queueRunning: status.queueRunning,
+      queuePending: status.queuePending
+    })
+  })
+
+  const done = watchComfyJob(job)
+  job.status = 'uploading'
+  emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 4 })
+  const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
+  if (job.library) {
+    job.library.imageName = uploaded.name
+    job.library.imageSubfolder = uploaded.subfolder
+  }
+  emitJob(job, { type: 'status', message: 'Queueing extension on MiniMax H3...', progress: 6 })
+  await waitForComfySocket(job, 4000)
+
+  const graph = buildWorkflow({
+    prompt: params.prompt,
+    imageName: uploaded.name,
+    width: params.width,
+    height: params.height,
+    steps: params.steps,
+    seed: params.seed,
+    turbo: params.turbo,
+    length: params.length,
+    cfg: params.cfg,
+    fps: params.fps,
+    samplerName: params.samplerName,
+    scheduler: params.scheduler,
+    filenamePrefix: comfyFilenamePrefix(),
+    sound
+  })
+
+  const queued = await queuePrompt(graph, job.clientId)
+  job.promptId = queued.prompt_id
+  job.status = 'running'
+  if (job.library && job.promptId) {
+    writePendingJob({
+      jobId: job.id,
+      promptId: job.promptId,
+      clientId: job.clientId,
+      ownerKey: job.library.ownerKey,
+      folderId: job.library.folderId,
+      hideThumbnail: job.library.hideThumbnail,
+      folderLocked: job.library.folderLocked,
+      name: job.library.name,
+      prompt: job.library.prompt,
+      aspect: job.library.aspect,
+      width: job.library.width,
+      height: job.library.height,
+      steps: job.library.steps,
+      turbo: job.library.turbo,
+      seed: job.library.seed,
+      startedAt: job.startedAt,
+      imageName: job.library.imageName,
+      imageSubfolder: job.library.imageSubfolder,
+      extendTmpDir: job.library.extendTmpDir,
+      extendPart1Path: job.library.extendPart1Path
+    })
+  }
+  emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
+  await done
+}

server/api/generate.post.ts

@@ -273,7 +273,9 @@ async function runGeneration(
       seed: job.library.seed,
       startedAt: job.startedAt,
       imageName: job.library.imageName,
-      imageSubfolder: job.library.imageSubfolder
+      imageSubfolder: job.library.imageSubfolder,
+      extendTmpDir: job.library.extendTmpDir,
+      extendPart1Path: job.library.extendPart1Path
     })
   }
   emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })

server/api/interrupt.post.ts

@@ -5,6 +5,7 @@ export default defineEventHandler(async (event) => {
   if (job) {
     job.status = 'cancelled'
     emitJob(job, { type: 'status', message: 'Cancelling job...' })
+    removeExtendTemp(job.library?.extendTmpDir)
   }
   if (jobId) deletePendingJob(jobId)
   const ok = await interruptComfy()

server/api/library/clips/[id]/last-frame.get.ts

@@ -0,0 +1,25 @@
+import { existsSync, readFileSync } from 'node:fs'
+
+export default defineEventHandler(async (event) => {
+  const { owner } = assertLibraryOwner(event)
+  const id = String(getRouterParam(event, 'id') || '')
+  const clip = getClip(owner, id)
+  assertFolderAccess(event, clip.folderId)
+  const videoPath = clipVideoPath(owner, clip.id)
+  if (!existsSync(videoPath)) {
+    throw createError({ statusCode: 404, statusMessage: 'Video file is missing' })
+  }
+  const framePath = clipLastFramePath(owner, clip.id)
+  if (!existsSync(framePath)) {
+    try {
+      await extractLastFrame(videoPath, framePath)
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error)
+      throw createError({ statusCode: 500, statusMessage: `Could not extract the last frame: ${message}` })
+    }
+  }
+  const buf = readFileSync(framePath)
+  setHeader(event, 'Content-Type', 'image/png')
+  setHeader(event, 'Cache-Control', 'private, no-store')
+  return buf
+})

server/utils/ffmpeg.ts

@@ -0,0 +1,156 @@
+import { existsSync, writeFileSync, readFileSync, unlinkSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { spawn } from 'node:child_process'
+
+function collectError(stderr: string, fallback: string) {
+  const text = stderr.replace(/\r/g, '').trim()
+  const lines = text.split('\n').map(line => line.trim()).filter(Boolean)
+  return lines.slice(-6).join(' ') || fallback
+}
+
+export function runFfmpeg(args: string[], timeoutMs = 120000) {
+  return new Promise<void>((resolve, reject) => {
+    const proc = spawn('ffmpeg', args, { stdio: ['ignore', 'ignore', 'pipe'] })
+    let stderr = ''
+    proc.stderr?.on('data', (chunk: Buffer | string) => { stderr += String(chunk) })
+    const timer = setTimeout(() => {
+      try { proc.kill('SIGKILL') } catch { /* ignore */ }
+      reject(new Error(`FFmpeg timed out after ${Math.round(timeoutMs / 1000)}s`))
+    }, timeoutMs)
+    proc.on('error', (error) => {
+      clearTimeout(timer)
+      const message = (error as NodeJS.ErrnoException).code === 'ENOENT'
+        ? 'FFmpeg is not installed on this server'
+        : error.message
+      reject(new Error(message))
+    })
+    proc.on('close', (code) => {
+      clearTimeout(timer)
+      if (code === 0) resolve()
+      else reject(new Error(collectError(stderr, `FFmpeg exited with code ${code}`)))
+    })
+  })
+}
+
+function runFfprobe(args: string[], timeoutMs = 15000) {
+  return new Promise<string>((resolve, reject) => {
+    const proc = spawn('ffprobe', args, { stdio: ['ignore', 'pipe', 'pipe'] })
+    let out = ''
+    let err = ''
+    proc.stdout?.on('data', (chunk: Buffer | string) => { out += String(chunk) })
+    proc.stderr?.on('data', (chunk: Buffer | string) => { err += String(chunk) })
+    const timer = setTimeout(() => {
+      try { proc.kill('SIGKILL') } catch { /* ignore */ }
+      reject(new Error('ffprobe timed out'))
+    }, timeoutMs)
+    proc.on('error', (error) => {
+      clearTimeout(timer)
+      reject(error)
+    })
+    proc.on('close', (code) => {
+      clearTimeout(timer)
+      if (code === 0) resolve(out.trim())
+      else reject(new Error(collectError(err, `ffprobe exited with code ${code}`)))
+    })
+  })
+}
+
+function concatPath(filePath: string) {
+  return filePath.replace(/\\/g, '/').replace(/'/g, "'\\''")
+}
+
+export async function extractLastFrame(inputPath: string, outputPath: string) {
+  if (!existsSync(inputPath)) {
+    throw new Error('Source video is missing')
+  }
+  await runFfmpeg([
+    '-y',
+    '-sseof', '-1',
+    '-i', inputPath,
+    '-update', '1',
+    '-frames:v', '1',
+    '-q:v', '1',
+    outputPath
+  ], 30000)
+  if (!existsSync(outputPath)) {
+    throw new Error('FFmpeg did not write the last frame')
+  }
+}
+
+export async function probeHasAudio(filePath: string) {
+  try {
+    const out = await runFfprobe([
+      '-v', 'error',
+      '-select_streams', 'a:0',
+      '-show_entries', 'stream=codec_type',
+      '-of', 'csv=p=0',
+      filePath
+    ])
+    return out.includes('audio')
+  } catch {
+    return false
+  }
+}
+
+export async function concatMp4(part1Path: string, part2Path: string, outputPath: string) {
+  const listPath = join(dirname(outputPath), 'concat_list.txt')
+  writeFileSync(listPath, `file '${concatPath(part1Path)}'\nfile '${concatPath(part2Path)}'\n`)
+  try {
+    await runFfmpeg(['-y', '-f', 'concat', '-safe', '0', '-i', listPath, '-c', 'copy', outputPath], 180000)
+  } catch (copyError) {
+    const audio1 = await probeHasAudio(part1Path)
+    const audio2 = await probeHasAudio(part2Path)
+    try {
+      if (audio1 && audio2) {
+        await runFfmpeg([
+          '-y',
+          '-i', part1Path,
+          '-i', part2Path,
+          '-filter_complex', '[0:v][0:a][1:v][1:a]concat=n=2:v=1:a=1[v][a]',
+          '-map', '[v]',
+          '-map', '[a]',
+          '-c:v', 'libx264',
+          '-preset', 'fast',
+          '-crf', '18',
+          '-pix_fmt', 'yuv420p',
+          '-c:a', 'aac',
+          '-movflags', '+faststart',
+          outputPath
+        ], 300000)
+      } else {
+        await runFfmpeg([
+          '-y',
+          '-i', part1Path,
+          '-i', part2Path,
+          '-filter_complex', '[0:v][1:v]concat=n=2:v=1:a=0[v]',
+          '-map', '[v]',
+          '-c:v', 'libx264',
+          '-preset', 'fast',
+          '-crf', '18',
+          '-pix_fmt', 'yuv420p',
+          '-movflags', '+faststart',
+          outputPath
+        ], 300000)
+      }
+    } catch (encodeError) {
+      const copyMessage = copyError instanceof Error ? copyError.message : String(copyError)
+      const encodeMessage = encodeError instanceof Error ? encodeError.message : String(encodeError)
+      throw new Error(`FFmpeg concat failed (${copyMessage}). Re-encode also failed (${encodeMessage}).`)
+    }
+  }
+  if (!existsSync(outputPath)) {
+    throw new Error('FFmpeg did not write the stitched video')
+  }
+}
+
+export async function stitchExtension(params: { part1Path: string; part2: Buffer; tmpDir: string }) {
+  const part2Path = join(params.tmpDir, 'part2.mp4')
+  const finalPath = join(params.tmpDir, 'final_stitched_video.mp4')
+  writeFileSync(part2Path, params.part2)
+  await concatMp4(params.part1Path, part2Path, finalPath)
+  const stitched = readFileSync(finalPath)
+  for (const extra of [part2Path, join(params.tmpDir, 'concat_list.txt'), join(params.tmpDir, 'last_frame.png')]) {
+    try { unlinkSync(extra) } catch { /* ignore */ }
+  }
+  return stitched
+}

server/utils/jobs.ts

@@ -61,6 +61,9 @@ export interface Job {
     duration?: number
     sound?: boolean
     draftId?: string
+    extendTmpDir?: string
+    extendPart1Path?: string
+    extendSourceClipId?: string
   }
   error?: string
   socketReady?: boolean

server/utils/library.ts

@@ -462,6 +462,21 @@ export function clipVideoPath(owner: string, id: string) {
   return join(clipDir(owner, id), 'video.mp4')
 }
 
+export function clipLastFramePath(owner: string, id: string) {
+  return join(clipDir(owner, id), 'last-frame.png')
+}
+
+export function extendTempDir(owner: string, jobId: string) {
+  const dir = join(ownerRoot(owner), 'tmp', 'extend', jobId)
+  mkdirSync(dir, { recursive: true })
+  return dir
+}
+
+export function removeExtendTemp(dir?: string) {
+  if (!dir) return
+  rmSync(dir, { recursive: true, force: true })
+}
+
 export function clipThumbPath(owner: string, id: string) {
   const dir = clipDir(owner, id)
   for (const name of ['thumb.jpg', 'thumb.png', 'thumb']) {

server/utils/pending.ts

@@ -20,6 +20,8 @@ export interface PendingJob {
   startedAt: number
   imageName?: string
   imageSubfolder?: string
+  extendTmpDir?: string
+  extendPart1Path?: string
 }
 
 function pendingRoot() {
@@ -68,7 +70,24 @@ export async function completePendingIfReady(pending: PendingJob) {
   const history = await fetchHistory(pending.promptId)
   const video = extractVideo(history, pending.promptId)
   if (!video) return null
-  const buffer = await downloadComfyVideo(video)
+  let buffer = await downloadComfyVideo(video)
+  if (pending.extendPart1Path && pending.extendTmpDir) {
+    if (!existsSync(pending.extendPart1Path)) {
+      removeExtendTemp(pending.extendTmpDir)
+      throw createError({ statusCode: 500, statusMessage: 'Extension source clip was missing during stitch' })
+    }
+    try {
+      buffer = await stitchExtension({
+        part1Path: pending.extendPart1Path,
+        part2: buffer,
+        tmpDir: pending.extendTmpDir
+      })
+    } catch (error) {
+      removeExtendTemp(pending.extendTmpDir)
+      throw error
+    }
+    removeExtendTemp(pending.extendTmpDir)
+  }
   const clip = await saveClip({
     ownerKey: pending.ownerKey,
     folderId: pending.folderId,
@@ -90,7 +109,9 @@ export async function completePendingIfReady(pending: PendingJob) {
   return {
     type: 'complete' as const,
     status: 'complete' as const,
-    message: pending.folderLocked ? 'Saved to the locked folder. Unlock it to view.' : 'Video ready',
+    message: pending.folderLocked
+      ? 'Saved to the locked folder. Unlock it to view.'
+      : (pending.extendPart1Path ? 'Extended video ready' : 'Video ready'),
     progress: 100,
     jobId: pending.jobId,
     clipId: clip.id,

server/utils/watch.ts

@@ -1,4 +1,4 @@
-import { NODE_LABELS, isEncodingNode } from '~/server/utils/workflow'
+import { existsSync } from 'node:fs'
 import type { Job } from '~/server/utils/jobs'
 
 function classifyError(message: string) {
@@ -42,6 +42,7 @@ export function watchComfyJob(job: Job): Promise<void> {
       job.error = classifyError(error)
       emitJob(job, { type: 'error', error: job.error, message: job.error })
       if (job.id) deletePendingJob(job.id)
+      removeExtendTemp(job.library?.extendTmpDir)
       resolve()
     }
 
@@ -54,9 +55,24 @@ export function watchComfyJob(job: Job): Promise<void> {
       cleanup()
       try {
         job.video = video
-        emitJob(job, { type: 'status', message: 'Saving to library...', progress: 98 })
+        const stitching = Boolean(job.library?.extendPart1Path && job.library.extendTmpDir)
+        emitJob(job, {
+          type: 'status',
+          message: stitching ? 'Extracting last frame & stitching extension...' : 'Saving to library...',
+          progress: stitching ? 97 : 98
+        })
         if (job.library) {
-          const buffer = await downloadComfyVideo(video)
+          let buffer = await downloadComfyVideo(video)
+          if (stitching) {
+            if (!existsSync(job.library.extendPart1Path!)) {
+              throw new Error('Extension source clip was missing during stitch')
+            }
+            buffer = await stitchExtension({
+              part1Path: job.library.extendPart1Path!,
+              part2: buffer,
+              tmpDir: job.library.extendTmpDir!
+            })
+          }
           const clip = await saveClip({
             ownerKey: job.library.ownerKey,
             folderId: job.library.folderId,
@@ -87,11 +103,15 @@ export function watchComfyJob(job: Job): Promise<void> {
             imageSubfolder: job.library.imageSubfolder,
             promptId: job.promptId
           })
+          removeExtendTemp(job.library.extendTmpDir)
+          job.library.extendTmpDir = undefined
         }
         job.status = 'complete'
         emitJob(job, {
           type: 'complete',
-          message: job.library?.folderLocked ? 'Saved to the locked folder. Unlock it to view.' : 'Video ready',
+          message: job.library?.folderLocked
+            ? 'Saved to the locked folder. Unlock it to view.'
+            : (job.library?.extendSourceClipId ? 'Extended video ready' : 'Video ready'),
           progress: 100,
           filename: job.library?.folderLocked ? undefined : video.filename,
           subfolder: job.library?.folderLocked ? undefined : video.subfolder,
@@ -102,6 +122,7 @@ export function watchComfyJob(job: Job): Promise<void> {
         })
         deletePendingJob(job.id)
       } catch (saveError) {
+        removeExtendTemp(job.library?.extendTmpDir)
         const message = saveError instanceof Error ? saveError.message : String(saveError)
         job.status = 'error'
         job.error = `Video generated but library save failed: ${message}`