Add clip extension from the last frame with FFmpeg stitching.
Let a finished player continue into a new MiniMax segment, then losslessly concat the parts and discard the intermediate files so the library keeps one extended MP4. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+182
-11
@@ -331,18 +331,84 @@
|
||||
<span class="font-display text-lg font-bold">Saved to a locked folder</span>
|
||||
<span class="text-sm text-amber-100/80">Unlock that folder in the library to view this clip</span>
|
||||
</div>
|
||||
<div v-else-if="!videoUrl" class="flex h-full min-h-[12rem] items-center justify-center text-sm text-zinc-500">
|
||||
Generated MP4 appears here
|
||||
</div>
|
||||
</div>
|
||||
<a
|
||||
v-if="videoUrl && !awaitingReveal && !lockedSave"
|
||||
:href="videoUrl"
|
||||
:download="`${outputDownloadName}.mp4`"
|
||||
class="mt-4 inline-flex rounded-2xl border border-white/15 px-4 py-2 text-sm hover:border-amber-300/50"
|
||||
<div
|
||||
v-else-if="!videoUrl"
|
||||
class="flex h-full min-h-[12rem] items-center justify-center text-sm text-zinc-500"
|
||||
>
|
||||
Download MP4
|
||||
</a>
|
||||
Generated MP4 appears here
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="videoUrl && !awaitingReveal && !lockedSave" class="mt-4 flex flex-wrap gap-3">
|
||||
<a
|
||||
:href="videoUrl"
|
||||
:download="`${outputDownloadName}.mp4`"
|
||||
class="inline-flex rounded-2xl border border-white/15 px-4 py-2 text-sm hover:border-amber-300/50"
|
||||
>
|
||||
Download MP4
|
||||
</a>
|
||||
<button
|
||||
v-if="canExtend"
|
||||
type="button"
|
||||
class="inline-flex rounded-2xl border border-amber-300/40 px-4 py-2 text-sm text-amber-100 hover:border-amber-300/70"
|
||||
:disabled="busy"
|
||||
@click="openExtend"
|
||||
>
|
||||
Extend Video
|
||||
</button>
|
||||
</div>
|
||||
<div v-if="extendOpen && canExtend" class="mt-4 space-y-4 rounded-2xl border border-white/10 bg-zinc-950/50 p-4">
|
||||
<div class="flex items-start justify-between gap-3">
|
||||
<div>
|
||||
<h3 class="text-sm font-semibold text-amber-100">Extend this clip</h3>
|
||||
<p class="mt-1 text-xs text-zinc-500">Continue from the last frame with a new motion prompt, then stitch the segments together.</p>
|
||||
</div>
|
||||
<button type="button" class="text-xs text-zinc-400 hover:text-white" @click="closeExtend">Close</button>
|
||||
</div>
|
||||
<div class="overflow-hidden rounded-xl bg-black/40">
|
||||
<img v-if="lastFrameUrl" :src="lastFrameUrl" alt="Last frame" class="max-h-48 w-full object-contain">
|
||||
<p v-else class="px-3 py-8 text-center text-xs text-zinc-500">
|
||||
{{ lastFrameLoading ? 'Reading last frame…' : 'Last frame preview unavailable' }}
|
||||
</p>
|
||||
</div>
|
||||
<label class="block text-sm">
|
||||
<span class="mb-1 block text-zinc-400">Extension prompt</span>
|
||||
<textarea
|
||||
v-model="extendPrompt"
|
||||
rows="4"
|
||||
class="w-full rounded-xl border border-white/10 bg-zinc-950 px-3 py-2 text-sm outline-none ring-amber-300/40 placeholder:text-zinc-600 focus:ring-2"
|
||||
placeholder="Continue the motion from this frame…"
|
||||
/>
|
||||
</label>
|
||||
<div>
|
||||
<p class="mb-2 text-xs uppercase tracking-wider text-zinc-500">Extension duration</p>
|
||||
<div class="grid grid-cols-2 gap-2">
|
||||
<button
|
||||
type="button"
|
||||
class="rounded-xl border px-3 py-2 text-sm"
|
||||
:class="extendDuration === 3 ? 'border-amber-300 bg-amber-400/10' : 'border-white/10'"
|
||||
@click="extendDuration = 3"
|
||||
>
|
||||
3s
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
class="rounded-xl border px-3 py-2 text-sm"
|
||||
:class="extendDuration === 5 ? 'border-amber-300 bg-amber-400/10' : 'border-white/10'"
|
||||
@click="extendDuration = 5"
|
||||
>
|
||||
5s
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
class="w-full rounded-2xl bg-amber-400 px-4 py-2.5 text-sm font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
:disabled="busy || !extendPrompt.trim() || lastFrameLoading"
|
||||
@click="generateExtension"
|
||||
>
|
||||
Generate Extension
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="glass rounded-3xl p-5 sm:p-6">
|
||||
@@ -614,6 +680,13 @@ const maxStep = ref(8)
|
||||
const elapsedMs = ref(0)
|
||||
const videoUrl = ref('')
|
||||
const jobId = ref('')
|
||||
const currentClipId = ref('')
|
||||
const extendOpen = ref(false)
|
||||
const extendPrompt = ref('')
|
||||
const extendDuration = ref(5)
|
||||
const lastFrameUrl = ref('')
|
||||
const lastFrameLoading = ref(false)
|
||||
let lastFrameObjectUrl = ''
|
||||
const awaitingReveal = ref(false)
|
||||
const concealOutput = ref(false)
|
||||
const lockedSave = ref(false)
|
||||
@@ -686,6 +759,13 @@ function normalizeAspect(raw?: string, width?: number, height?: number): AspectI
|
||||
const outputSize = computed(() => PRESETS[aspect.value])
|
||||
const frameCount = computed(() => Math.max(5, Math.floor(duration.value * fps.value)))
|
||||
const cfgPct = computed(() => ((cfg.value - CFG_MIN) / (CFG_MAX - CFG_MIN)) * 100)
|
||||
const currentClip = computed(() => clips.value.find(clip => clip.id === currentClipId.value))
|
||||
const canExtend = computed(() => Boolean(
|
||||
videoUrl.value
|
||||
&& !lockedSave.value
|
||||
&& !awaitingReveal.value
|
||||
&& (currentClipId.value || clipIdFromPlayer())
|
||||
))
|
||||
|
||||
const visibleClips = computed(() => clips.value.filter(clip => clip.folderId === browseFolderId.value))
|
||||
const currentFolder = computed(() => folders.value.find(folder => folder.id === folderId.value))
|
||||
@@ -756,6 +836,7 @@ onBeforeUnmount(() => {
|
||||
healthTimer = null
|
||||
stopTimer()
|
||||
stopListen()
|
||||
closeExtend()
|
||||
})
|
||||
|
||||
watch(folderId, () => {
|
||||
@@ -1092,11 +1173,95 @@ async function lockFolder(id: string) {
|
||||
}
|
||||
}
|
||||
|
||||
function clipIdFromPlayer() {
|
||||
const match = videoUrl.value.match(/\/api\/library\/clips\/([^/?#]+)\/video/)
|
||||
return match?.[1] || currentClipId.value
|
||||
}
|
||||
|
||||
function clipVideoUrl(clipId: string) {
|
||||
return `/api/library/clips/${clipId}/video?t=${Date.now()}`
|
||||
}
|
||||
|
||||
function closeExtend() {
|
||||
extendOpen.value = false
|
||||
lastFrameLoading.value = false
|
||||
if (lastFrameObjectUrl) {
|
||||
URL.revokeObjectURL(lastFrameObjectUrl)
|
||||
lastFrameObjectUrl = ''
|
||||
}
|
||||
lastFrameUrl.value = ''
|
||||
}
|
||||
|
||||
async function openExtend() {
|
||||
const id = clipIdFromPlayer()
|
||||
if (!id) {
|
||||
toast('Load a generated clip before extending it.')
|
||||
return
|
||||
}
|
||||
currentClipId.value = id
|
||||
const clip = clips.value.find(item => item.id === id)
|
||||
extendPrompt.value = clip?.prompt || prompt.value
|
||||
extendDuration.value = 5
|
||||
extendOpen.value = true
|
||||
lastFrameLoading.value = true
|
||||
lastFrameUrl.value = ''
|
||||
player.value?.pause()
|
||||
try {
|
||||
const blob = await $fetch<Blob>(`/api/library/clips/${id}/last-frame`, { responseType: 'blob' })
|
||||
if (lastFrameObjectUrl) URL.revokeObjectURL(lastFrameObjectUrl)
|
||||
lastFrameObjectUrl = URL.createObjectURL(blob)
|
||||
lastFrameUrl.value = lastFrameObjectUrl
|
||||
} catch (error: any) {
|
||||
toast(error?.data?.statusMessage || 'Could not extract the last frame')
|
||||
} finally {
|
||||
lastFrameLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function generateExtension() {
|
||||
const id = clipIdFromPlayer()
|
||||
if (!id || !extendPrompt.value.trim()) return
|
||||
statusBusy.value = false
|
||||
busy.value = true
|
||||
listenGen += 1
|
||||
stopListen()
|
||||
settledUi = false
|
||||
progress.value = 1
|
||||
awaitingReveal.value = false
|
||||
concealOutput.value = hideThumbnail.value || currentClip.value?.hideThumbnail === true
|
||||
lockedSave.value = false
|
||||
statusMessage.value = 'Extracting last frame & stitching extension...'
|
||||
startTimer()
|
||||
try {
|
||||
const started = await $fetch<{ jobId: string; steps: number; hideThumbnail?: boolean; folderLocked?: boolean }>('/api/extend', {
|
||||
method: 'POST',
|
||||
body: {
|
||||
clipId: id,
|
||||
prompt: extendPrompt.value.trim(),
|
||||
duration: extendDuration.value
|
||||
}
|
||||
})
|
||||
closeExtend()
|
||||
jobId.value = started.jobId
|
||||
maxStep.value = started.steps
|
||||
settledUi = false
|
||||
persistActiveJob(started.jobId, started.hideThumbnail === true || concealOutput.value, started.folderLocked === true)
|
||||
listen(started.jobId, started.hideThumbnail === true || concealOutput.value, started.folderLocked === true)
|
||||
void pollComfyHealth()
|
||||
} catch (error: any) {
|
||||
busy.value = false
|
||||
statusBusy.value = false
|
||||
stopTimer()
|
||||
clearActiveJob()
|
||||
const message = error?.data?.statusMessage || error?.statusMessage || error?.message || 'Failed to start extension'
|
||||
statusMessage.value = message
|
||||
toast(message)
|
||||
}
|
||||
}
|
||||
|
||||
function openClip(item: LibraryClip) {
|
||||
closeExtend()
|
||||
currentClipId.value = item.id
|
||||
lockedSave.value = false
|
||||
concealOutput.value = item.hideThumbnail
|
||||
videoUrl.value = clipVideoUrl(item.id)
|
||||
@@ -1116,6 +1281,8 @@ async function removeClip(item: LibraryClip) {
|
||||
awaitingReveal.value = false
|
||||
concealOutput.value = false
|
||||
lockedSave.value = false
|
||||
currentClipId.value = ''
|
||||
closeExtend()
|
||||
}
|
||||
await loadLibrary()
|
||||
} catch (error: any) {
|
||||
@@ -1208,6 +1375,8 @@ async function generate() {
|
||||
settledUi = false
|
||||
progress.value = 1
|
||||
videoUrl.value = ''
|
||||
currentClipId.value = ''
|
||||
closeExtend()
|
||||
awaitingReveal.value = false
|
||||
concealOutput.value = hideThumbnail.value
|
||||
lockedSave.value = false
|
||||
@@ -1308,7 +1477,9 @@ function applyJobEvent(payload: Record<string, any>, hidden: boolean, folderLock
|
||||
lockedSave.value = true
|
||||
browseFolderId.value = folderId.value
|
||||
statusMessage.value = 'Saved to the locked folder. Unlock it to view.'
|
||||
currentClipId.value = payload.clipId || currentClipId.value
|
||||
} else if (payload.clipId) {
|
||||
currentClipId.value = payload.clipId
|
||||
videoUrl.value = clipVideoUrl(payload.clipId)
|
||||
awaitingReveal.value = Boolean(payload.hideThumbnail || hidden)
|
||||
concealOutput.value = awaitingReveal.value
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
import { copyFileSync, existsSync, readFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
|
||||
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
|
||||
|
||||
function parseExtendDuration(raw: unknown) {
|
||||
const seconds = Number(raw)
|
||||
return seconds === 3 || seconds === 5 ? seconds : 5
|
||||
}
|
||||
|
||||
function frameLength(seconds: number, fps: number) {
|
||||
return Math.max(5, Math.floor(seconds * fps))
|
||||
}
|
||||
|
||||
function parseFps(raw: unknown) {
|
||||
const fps = Number(raw)
|
||||
return fps === 12 || fps === 30 || fps === 24 ? fps : 24
|
||||
}
|
||||
|
||||
function parseCfg(raw: unknown, turbo: boolean) {
|
||||
const fallback = turbo ? 1.5 : 4
|
||||
const value = Number(raw)
|
||||
if (!Number.isFinite(value)) return fallback
|
||||
const snapped = Math.round(value * 2) / 2
|
||||
return Math.min(10, Math.max(1, snapped))
|
||||
}
|
||||
|
||||
function parseSampler(raw: unknown) {
|
||||
const value = String(raw || '')
|
||||
return SAMPLERS.has(value) ? value : 'res_multistep'
|
||||
}
|
||||
|
||||
function parseScheduler(raw: unknown) {
|
||||
const value = String(raw || '')
|
||||
return SCHEDULERS.has(value) ? value : 'simple'
|
||||
}
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
const body = await readBody<{ clipId?: string; prompt?: string; duration?: number }>(event).catch(() => ({}))
|
||||
const clipId = String(body?.clipId || '').trim()
|
||||
const prompt = String(body?.prompt || '').trim()
|
||||
if (!clipId) {
|
||||
throw createError({ statusCode: 400, statusMessage: 'A clip is required to extend' })
|
||||
}
|
||||
if (!prompt) {
|
||||
throw createError({ statusCode: 400, statusMessage: 'An extension prompt is required' })
|
||||
}
|
||||
|
||||
const { owner } = assertLibraryOwner(event)
|
||||
const source = getClip(owner, clipId)
|
||||
assertFolderAccess(event, source.folderId)
|
||||
const sourcePath = clipVideoPath(owner, source.id)
|
||||
if (!existsSync(sourcePath)) {
|
||||
throw createError({ statusCode: 404, statusMessage: 'Source video is missing' })
|
||||
}
|
||||
|
||||
const destFolder = publicLibrary(event).folders.find(folder => folder.id === source.folderId)
|
||||
const folderLocked = Boolean(destFolder?.protected && !destFolder.unlocked)
|
||||
const durationSeconds = parseExtendDuration(body?.duration)
|
||||
const turbo = source.turbo !== false
|
||||
const fps = parseFps(source.fps)
|
||||
const cfg = parseCfg(source.cfg, turbo)
|
||||
const samplerName = parseSampler(source.samplerName)
|
||||
const scheduler = parseScheduler(source.scheduler)
|
||||
const steps = Number(source.steps || (turbo ? 8 : 20))
|
||||
const seed = Math.floor(Math.random() * 2_147_483_647)
|
||||
const length = frameLength(durationSeconds, fps)
|
||||
const width = source.width || 1344
|
||||
const height = source.height || 768
|
||||
const clipName = `${clipTitle(source)} extended`.slice(0, 80)
|
||||
|
||||
const job = createJob()
|
||||
job.maxStep = steps
|
||||
job.hideThumbnail = source.hideThumbnail
|
||||
const tmpDir = extendTempDir(owner, job.id)
|
||||
const part1Path = join(tmpDir, 'part1.mp4')
|
||||
const framePath = join(tmpDir, 'last_frame.png')
|
||||
job.library = {
|
||||
ownerKey: owner,
|
||||
folderId: source.folderId,
|
||||
hideThumbnail: source.hideThumbnail,
|
||||
folderLocked,
|
||||
name: clipName,
|
||||
prompt,
|
||||
aspect: source.aspect || '16:9',
|
||||
width,
|
||||
height,
|
||||
steps,
|
||||
turbo,
|
||||
seed,
|
||||
cfg,
|
||||
fps,
|
||||
samplerName,
|
||||
scheduler,
|
||||
duration: durationSeconds,
|
||||
extendTmpDir: tmpDir,
|
||||
extendPart1Path: part1Path,
|
||||
extendSourceClipId: source.id
|
||||
}
|
||||
|
||||
emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 })
|
||||
|
||||
void runExtend(job, {
|
||||
sourcePath,
|
||||
part1Path,
|
||||
framePath,
|
||||
prompt,
|
||||
width,
|
||||
height,
|
||||
steps,
|
||||
seed,
|
||||
turbo,
|
||||
length,
|
||||
cfg,
|
||||
fps,
|
||||
samplerName,
|
||||
scheduler
|
||||
}).catch((error) => {
|
||||
removeExtendTemp(job.library?.extendTmpDir)
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
job.status = 'error'
|
||||
job.error = message
|
||||
emitJob(job, { type: 'error', error: message, message })
|
||||
})
|
||||
|
||||
return {
|
||||
jobId: job.id,
|
||||
clientId: job.clientId,
|
||||
seed,
|
||||
width,
|
||||
height,
|
||||
steps,
|
||||
duration: durationSeconds,
|
||||
hideThumbnail: source.hideThumbnail,
|
||||
folderLocked
|
||||
}
|
||||
})
|
||||
|
||||
async function runExtend(
|
||||
job: ReturnType<typeof createJob>,
|
||||
params: {
|
||||
sourcePath: string
|
||||
part1Path: string
|
||||
framePath: string
|
||||
prompt: string
|
||||
width: number
|
||||
height: number
|
||||
steps: number
|
||||
seed: number
|
||||
turbo: boolean
|
||||
length: number
|
||||
cfg: number
|
||||
fps: number
|
||||
samplerName: string
|
||||
scheduler: string
|
||||
}
|
||||
) {
|
||||
copyFileSync(params.sourcePath, params.part1Path)
|
||||
await extractLastFrame(params.part1Path, params.framePath)
|
||||
const frame = readFileSync(params.framePath)
|
||||
job.library!.thumb = frame
|
||||
const sound = await probeHasAudio(params.part1Path)
|
||||
|
||||
await ensureComfyReady((status) => {
|
||||
emitJob(job, {
|
||||
type: status.state === 'busy' ? 'busy' : 'status',
|
||||
message: status.message,
|
||||
progress: status.state === 'online' ? Math.max(job.progress, 3) : Math.max(job.progress, 2),
|
||||
busy: status.state === 'busy',
|
||||
queueRunning: status.queueRunning,
|
||||
queuePending: status.queuePending
|
||||
})
|
||||
})
|
||||
|
||||
const done = watchComfyJob(job)
|
||||
job.status = 'uploading'
|
||||
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 4 })
|
||||
const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
|
||||
if (job.library) {
|
||||
job.library.imageName = uploaded.name
|
||||
job.library.imageSubfolder = uploaded.subfolder
|
||||
}
|
||||
emitJob(job, { type: 'status', message: 'Queueing extension on MiniMax H3...', progress: 6 })
|
||||
await waitForComfySocket(job, 4000)
|
||||
|
||||
const graph = buildWorkflow({
|
||||
prompt: params.prompt,
|
||||
imageName: uploaded.name,
|
||||
width: params.width,
|
||||
height: params.height,
|
||||
steps: params.steps,
|
||||
seed: params.seed,
|
||||
turbo: params.turbo,
|
||||
length: params.length,
|
||||
cfg: params.cfg,
|
||||
fps: params.fps,
|
||||
samplerName: params.samplerName,
|
||||
scheduler: params.scheduler,
|
||||
filenamePrefix: comfyFilenamePrefix(),
|
||||
sound
|
||||
})
|
||||
|
||||
const queued = await queuePrompt(graph, job.clientId)
|
||||
job.promptId = queued.prompt_id
|
||||
job.status = 'running'
|
||||
if (job.library && job.promptId) {
|
||||
writePendingJob({
|
||||
jobId: job.id,
|
||||
promptId: job.promptId,
|
||||
clientId: job.clientId,
|
||||
ownerKey: job.library.ownerKey,
|
||||
folderId: job.library.folderId,
|
||||
hideThumbnail: job.library.hideThumbnail,
|
||||
folderLocked: job.library.folderLocked,
|
||||
name: job.library.name,
|
||||
prompt: job.library.prompt,
|
||||
aspect: job.library.aspect,
|
||||
width: job.library.width,
|
||||
height: job.library.height,
|
||||
steps: job.library.steps,
|
||||
turbo: job.library.turbo,
|
||||
seed: job.library.seed,
|
||||
startedAt: job.startedAt,
|
||||
imageName: job.library.imageName,
|
||||
imageSubfolder: job.library.imageSubfolder,
|
||||
extendTmpDir: job.library.extendTmpDir,
|
||||
extendPart1Path: job.library.extendPart1Path
|
||||
})
|
||||
}
|
||||
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
|
||||
await done
|
||||
}
|
||||
@@ -273,7 +273,9 @@ async function runGeneration(
|
||||
seed: job.library.seed,
|
||||
startedAt: job.startedAt,
|
||||
imageName: job.library.imageName,
|
||||
imageSubfolder: job.library.imageSubfolder
|
||||
imageSubfolder: job.library.imageSubfolder,
|
||||
extendTmpDir: job.library.extendTmpDir,
|
||||
extendPart1Path: job.library.extendPart1Path
|
||||
})
|
||||
}
|
||||
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
|
||||
|
||||
@@ -5,6 +5,7 @@ export default defineEventHandler(async (event) => {
|
||||
if (job) {
|
||||
job.status = 'cancelled'
|
||||
emitJob(job, { type: 'status', message: 'Cancelling job...' })
|
||||
removeExtendTemp(job.library?.extendTmpDir)
|
||||
}
|
||||
if (jobId) deletePendingJob(jobId)
|
||||
const ok = await interruptComfy()
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import { existsSync, readFileSync } from 'node:fs'
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
const { owner } = assertLibraryOwner(event)
|
||||
const id = String(getRouterParam(event, 'id') || '')
|
||||
const clip = getClip(owner, id)
|
||||
assertFolderAccess(event, clip.folderId)
|
||||
const videoPath = clipVideoPath(owner, clip.id)
|
||||
if (!existsSync(videoPath)) {
|
||||
throw createError({ statusCode: 404, statusMessage: 'Video file is missing' })
|
||||
}
|
||||
const framePath = clipLastFramePath(owner, clip.id)
|
||||
if (!existsSync(framePath)) {
|
||||
try {
|
||||
await extractLastFrame(videoPath, framePath)
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
throw createError({ statusCode: 500, statusMessage: `Could not extract the last frame: ${message}` })
|
||||
}
|
||||
}
|
||||
const buf = readFileSync(framePath)
|
||||
setHeader(event, 'Content-Type', 'image/png')
|
||||
setHeader(event, 'Cache-Control', 'private, no-store')
|
||||
return buf
|
||||
})
|
||||
@@ -0,0 +1,156 @@
|
||||
import { existsSync, writeFileSync, readFileSync, unlinkSync } from 'node:fs'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { spawn } from 'node:child_process'
|
||||
|
||||
function collectError(stderr: string, fallback: string) {
|
||||
const text = stderr.replace(/\r/g, '').trim()
|
||||
const lines = text.split('\n').map(line => line.trim()).filter(Boolean)
|
||||
return lines.slice(-6).join(' ') || fallback
|
||||
}
|
||||
|
||||
export function runFfmpeg(args: string[], timeoutMs = 120000) {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const proc = spawn('ffmpeg', args, { stdio: ['ignore', 'ignore', 'pipe'] })
|
||||
let stderr = ''
|
||||
proc.stderr?.on('data', (chunk: Buffer | string) => { stderr += String(chunk) })
|
||||
const timer = setTimeout(() => {
|
||||
try { proc.kill('SIGKILL') } catch { /* ignore */ }
|
||||
reject(new Error(`FFmpeg timed out after ${Math.round(timeoutMs / 1000)}s`))
|
||||
}, timeoutMs)
|
||||
proc.on('error', (error) => {
|
||||
clearTimeout(timer)
|
||||
const message = (error as NodeJS.ErrnoException).code === 'ENOENT'
|
||||
? 'FFmpeg is not installed on this server'
|
||||
: error.message
|
||||
reject(new Error(message))
|
||||
})
|
||||
proc.on('close', (code) => {
|
||||
clearTimeout(timer)
|
||||
if (code === 0) resolve()
|
||||
else reject(new Error(collectError(stderr, `FFmpeg exited with code ${code}`)))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function runFfprobe(args: string[], timeoutMs = 15000) {
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const proc = spawn('ffprobe', args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
let out = ''
|
||||
let err = ''
|
||||
proc.stdout?.on('data', (chunk: Buffer | string) => { out += String(chunk) })
|
||||
proc.stderr?.on('data', (chunk: Buffer | string) => { err += String(chunk) })
|
||||
const timer = setTimeout(() => {
|
||||
try { proc.kill('SIGKILL') } catch { /* ignore */ }
|
||||
reject(new Error('ffprobe timed out'))
|
||||
}, timeoutMs)
|
||||
proc.on('error', (error) => {
|
||||
clearTimeout(timer)
|
||||
reject(error)
|
||||
})
|
||||
proc.on('close', (code) => {
|
||||
clearTimeout(timer)
|
||||
if (code === 0) resolve(out.trim())
|
||||
else reject(new Error(collectError(err, `ffprobe exited with code ${code}`)))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function concatPath(filePath: string) {
|
||||
return filePath.replace(/\\/g, '/').replace(/'/g, "'\\''")
|
||||
}
|
||||
|
||||
export async function extractLastFrame(inputPath: string, outputPath: string) {
|
||||
if (!existsSync(inputPath)) {
|
||||
throw new Error('Source video is missing')
|
||||
}
|
||||
await runFfmpeg([
|
||||
'-y',
|
||||
'-sseof', '-1',
|
||||
'-i', inputPath,
|
||||
'-update', '1',
|
||||
'-frames:v', '1',
|
||||
'-q:v', '1',
|
||||
outputPath
|
||||
], 30000)
|
||||
if (!existsSync(outputPath)) {
|
||||
throw new Error('FFmpeg did not write the last frame')
|
||||
}
|
||||
}
|
||||
|
||||
export async function probeHasAudio(filePath: string) {
|
||||
try {
|
||||
const out = await runFfprobe([
|
||||
'-v', 'error',
|
||||
'-select_streams', 'a:0',
|
||||
'-show_entries', 'stream=codec_type',
|
||||
'-of', 'csv=p=0',
|
||||
filePath
|
||||
])
|
||||
return out.includes('audio')
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export async function concatMp4(part1Path: string, part2Path: string, outputPath: string) {
|
||||
const listPath = join(dirname(outputPath), 'concat_list.txt')
|
||||
writeFileSync(listPath, `file '${concatPath(part1Path)}'\nfile '${concatPath(part2Path)}'\n`)
|
||||
try {
|
||||
await runFfmpeg(['-y', '-f', 'concat', '-safe', '0', '-i', listPath, '-c', 'copy', outputPath], 180000)
|
||||
} catch (copyError) {
|
||||
const audio1 = await probeHasAudio(part1Path)
|
||||
const audio2 = await probeHasAudio(part2Path)
|
||||
try {
|
||||
if (audio1 && audio2) {
|
||||
await runFfmpeg([
|
||||
'-y',
|
||||
'-i', part1Path,
|
||||
'-i', part2Path,
|
||||
'-filter_complex', '[0:v][0:a][1:v][1:a]concat=n=2:v=1:a=1[v][a]',
|
||||
'-map', '[v]',
|
||||
'-map', '[a]',
|
||||
'-c:v', 'libx264',
|
||||
'-preset', 'fast',
|
||||
'-crf', '18',
|
||||
'-pix_fmt', 'yuv420p',
|
||||
'-c:a', 'aac',
|
||||
'-movflags', '+faststart',
|
||||
outputPath
|
||||
], 300000)
|
||||
} else {
|
||||
await runFfmpeg([
|
||||
'-y',
|
||||
'-i', part1Path,
|
||||
'-i', part2Path,
|
||||
'-filter_complex', '[0:v][1:v]concat=n=2:v=1:a=0[v]',
|
||||
'-map', '[v]',
|
||||
'-c:v', 'libx264',
|
||||
'-preset', 'fast',
|
||||
'-crf', '18',
|
||||
'-pix_fmt', 'yuv420p',
|
||||
'-movflags', '+faststart',
|
||||
outputPath
|
||||
], 300000)
|
||||
}
|
||||
} catch (encodeError) {
|
||||
const copyMessage = copyError instanceof Error ? copyError.message : String(copyError)
|
||||
const encodeMessage = encodeError instanceof Error ? encodeError.message : String(encodeError)
|
||||
throw new Error(`FFmpeg concat failed (${copyMessage}). Re-encode also failed (${encodeMessage}).`)
|
||||
}
|
||||
}
|
||||
if (!existsSync(outputPath)) {
|
||||
throw new Error('FFmpeg did not write the stitched video')
|
||||
}
|
||||
}
|
||||
|
||||
export async function stitchExtension(params: { part1Path: string; part2: Buffer; tmpDir: string }) {
|
||||
const part2Path = join(params.tmpDir, 'part2.mp4')
|
||||
const finalPath = join(params.tmpDir, 'final_stitched_video.mp4')
|
||||
writeFileSync(part2Path, params.part2)
|
||||
await concatMp4(params.part1Path, part2Path, finalPath)
|
||||
const stitched = readFileSync(finalPath)
|
||||
for (const extra of [part2Path, join(params.tmpDir, 'concat_list.txt'), join(params.tmpDir, 'last_frame.png')]) {
|
||||
try { unlinkSync(extra) } catch { /* ignore */ }
|
||||
}
|
||||
return stitched
|
||||
}
|
||||
@@ -61,6 +61,9 @@ export interface Job {
|
||||
duration?: number
|
||||
sound?: boolean
|
||||
draftId?: string
|
||||
extendTmpDir?: string
|
||||
extendPart1Path?: string
|
||||
extendSourceClipId?: string
|
||||
}
|
||||
error?: string
|
||||
socketReady?: boolean
|
||||
|
||||
@@ -462,6 +462,21 @@ export function clipVideoPath(owner: string, id: string) {
|
||||
return join(clipDir(owner, id), 'video.mp4')
|
||||
}
|
||||
|
||||
export function clipLastFramePath(owner: string, id: string) {
|
||||
return join(clipDir(owner, id), 'last-frame.png')
|
||||
}
|
||||
|
||||
export function extendTempDir(owner: string, jobId: string) {
|
||||
const dir = join(ownerRoot(owner), 'tmp', 'extend', jobId)
|
||||
mkdirSync(dir, { recursive: true })
|
||||
return dir
|
||||
}
|
||||
|
||||
export function removeExtendTemp(dir?: string) {
|
||||
if (!dir) return
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
export function clipThumbPath(owner: string, id: string) {
|
||||
const dir = clipDir(owner, id)
|
||||
for (const name of ['thumb.jpg', 'thumb.png', 'thumb']) {
|
||||
|
||||
+23
-2
@@ -20,6 +20,8 @@ export interface PendingJob {
|
||||
startedAt: number
|
||||
imageName?: string
|
||||
imageSubfolder?: string
|
||||
extendTmpDir?: string
|
||||
extendPart1Path?: string
|
||||
}
|
||||
|
||||
function pendingRoot() {
|
||||
@@ -68,7 +70,24 @@ export async function completePendingIfReady(pending: PendingJob) {
|
||||
const history = await fetchHistory(pending.promptId)
|
||||
const video = extractVideo(history, pending.promptId)
|
||||
if (!video) return null
|
||||
const buffer = await downloadComfyVideo(video)
|
||||
let buffer = await downloadComfyVideo(video)
|
||||
if (pending.extendPart1Path && pending.extendTmpDir) {
|
||||
if (!existsSync(pending.extendPart1Path)) {
|
||||
removeExtendTemp(pending.extendTmpDir)
|
||||
throw createError({ statusCode: 500, statusMessage: 'Extension source clip was missing during stitch' })
|
||||
}
|
||||
try {
|
||||
buffer = await stitchExtension({
|
||||
part1Path: pending.extendPart1Path,
|
||||
part2: buffer,
|
||||
tmpDir: pending.extendTmpDir
|
||||
})
|
||||
} catch (error) {
|
||||
removeExtendTemp(pending.extendTmpDir)
|
||||
throw error
|
||||
}
|
||||
removeExtendTemp(pending.extendTmpDir)
|
||||
}
|
||||
const clip = await saveClip({
|
||||
ownerKey: pending.ownerKey,
|
||||
folderId: pending.folderId,
|
||||
@@ -90,7 +109,9 @@ export async function completePendingIfReady(pending: PendingJob) {
|
||||
return {
|
||||
type: 'complete' as const,
|
||||
status: 'complete' as const,
|
||||
message: pending.folderLocked ? 'Saved to the locked folder. Unlock it to view.' : 'Video ready',
|
||||
message: pending.folderLocked
|
||||
? 'Saved to the locked folder. Unlock it to view.'
|
||||
: (pending.extendPart1Path ? 'Extended video ready' : 'Video ready'),
|
||||
progress: 100,
|
||||
jobId: pending.jobId,
|
||||
clipId: clip.id,
|
||||
|
||||
+25
-4
@@ -1,4 +1,4 @@
|
||||
import { NODE_LABELS, isEncodingNode } from '~/server/utils/workflow'
|
||||
import { existsSync } from 'node:fs'
|
||||
import type { Job } from '~/server/utils/jobs'
|
||||
|
||||
function classifyError(message: string) {
|
||||
@@ -42,6 +42,7 @@ export function watchComfyJob(job: Job): Promise<void> {
|
||||
job.error = classifyError(error)
|
||||
emitJob(job, { type: 'error', error: job.error, message: job.error })
|
||||
if (job.id) deletePendingJob(job.id)
|
||||
removeExtendTemp(job.library?.extendTmpDir)
|
||||
resolve()
|
||||
}
|
||||
|
||||
@@ -54,9 +55,24 @@ export function watchComfyJob(job: Job): Promise<void> {
|
||||
cleanup()
|
||||
try {
|
||||
job.video = video
|
||||
emitJob(job, { type: 'status', message: 'Saving to library...', progress: 98 })
|
||||
const stitching = Boolean(job.library?.extendPart1Path && job.library.extendTmpDir)
|
||||
emitJob(job, {
|
||||
type: 'status',
|
||||
message: stitching ? 'Extracting last frame & stitching extension...' : 'Saving to library...',
|
||||
progress: stitching ? 97 : 98
|
||||
})
|
||||
if (job.library) {
|
||||
const buffer = await downloadComfyVideo(video)
|
||||
let buffer = await downloadComfyVideo(video)
|
||||
if (stitching) {
|
||||
if (!existsSync(job.library.extendPart1Path!)) {
|
||||
throw new Error('Extension source clip was missing during stitch')
|
||||
}
|
||||
buffer = await stitchExtension({
|
||||
part1Path: job.library.extendPart1Path!,
|
||||
part2: buffer,
|
||||
tmpDir: job.library.extendTmpDir!
|
||||
})
|
||||
}
|
||||
const clip = await saveClip({
|
||||
ownerKey: job.library.ownerKey,
|
||||
folderId: job.library.folderId,
|
||||
@@ -87,11 +103,15 @@ export function watchComfyJob(job: Job): Promise<void> {
|
||||
imageSubfolder: job.library.imageSubfolder,
|
||||
promptId: job.promptId
|
||||
})
|
||||
removeExtendTemp(job.library.extendTmpDir)
|
||||
job.library.extendTmpDir = undefined
|
||||
}
|
||||
job.status = 'complete'
|
||||
emitJob(job, {
|
||||
type: 'complete',
|
||||
message: job.library?.folderLocked ? 'Saved to the locked folder. Unlock it to view.' : 'Video ready',
|
||||
message: job.library?.folderLocked
|
||||
? 'Saved to the locked folder. Unlock it to view.'
|
||||
: (job.library?.extendSourceClipId ? 'Extended video ready' : 'Video ready'),
|
||||
progress: 100,
|
||||
filename: job.library?.folderLocked ? undefined : video.filename,
|
||||
subfolder: job.library?.folderLocked ? undefined : video.subfolder,
|
||||
@@ -102,6 +122,7 @@ export function watchComfyJob(job: Job): Promise<void> {
|
||||
})
|
||||
deletePendingJob(job.id)
|
||||
} catch (saveError) {
|
||||
removeExtendTemp(job.library?.extendTmpDir)
|
||||
const message = saveError instanceof Error ? saveError.message : String(saveError)
|
||||
job.status = 'error'
|
||||
job.error = `Video generated but library save failed: ${message}`
|
||||
|
||||
Reference in New Issue
Block a user