Add clip extension from the last frame with FFmpeg stitching.
Let a finished player continue into a new MiniMax segment, then losslessly concat the parts and discard the intermediate files so the library keeps one extended MP4. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+174
-3
@@ -331,18 +331,84 @@
|
|||||||
<span class="font-display text-lg font-bold">Saved to a locked folder</span>
|
<span class="font-display text-lg font-bold">Saved to a locked folder</span>
|
||||||
<span class="text-sm text-amber-100/80">Unlock that folder in the library to view this clip</span>
|
<span class="text-sm text-amber-100/80">Unlock that folder in the library to view this clip</span>
|
||||||
</div>
|
</div>
|
||||||
<div v-else-if="!videoUrl" class="flex h-full min-h-[12rem] items-center justify-center text-sm text-zinc-500">
|
<div
|
||||||
|
v-else-if="!videoUrl"
|
||||||
|
class="flex h-full min-h-[12rem] items-center justify-center text-sm text-zinc-500"
|
||||||
|
>
|
||||||
Generated MP4 appears here
|
Generated MP4 appears here
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div v-if="videoUrl && !awaitingReveal && !lockedSave" class="mt-4 flex flex-wrap gap-3">
|
||||||
<a
|
<a
|
||||||
v-if="videoUrl && !awaitingReveal && !lockedSave"
|
|
||||||
:href="videoUrl"
|
:href="videoUrl"
|
||||||
:download="`${outputDownloadName}.mp4`"
|
:download="`${outputDownloadName}.mp4`"
|
||||||
class="mt-4 inline-flex rounded-2xl border border-white/15 px-4 py-2 text-sm hover:border-amber-300/50"
|
class="inline-flex rounded-2xl border border-white/15 px-4 py-2 text-sm hover:border-amber-300/50"
|
||||||
>
|
>
|
||||||
Download MP4
|
Download MP4
|
||||||
</a>
|
</a>
|
||||||
|
<button
|
||||||
|
v-if="canExtend"
|
||||||
|
type="button"
|
||||||
|
class="inline-flex rounded-2xl border border-amber-300/40 px-4 py-2 text-sm text-amber-100 hover:border-amber-300/70"
|
||||||
|
:disabled="busy"
|
||||||
|
@click="openExtend"
|
||||||
|
>
|
||||||
|
Extend Video
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div v-if="extendOpen && canExtend" class="mt-4 space-y-4 rounded-2xl border border-white/10 bg-zinc-950/50 p-4">
|
||||||
|
<div class="flex items-start justify-between gap-3">
|
||||||
|
<div>
|
||||||
|
<h3 class="text-sm font-semibold text-amber-100">Extend this clip</h3>
|
||||||
|
<p class="mt-1 text-xs text-zinc-500">Continue from the last frame with a new motion prompt, then stitch the segments together.</p>
|
||||||
|
</div>
|
||||||
|
<button type="button" class="text-xs text-zinc-400 hover:text-white" @click="closeExtend">Close</button>
|
||||||
|
</div>
|
||||||
|
<div class="overflow-hidden rounded-xl bg-black/40">
|
||||||
|
<img v-if="lastFrameUrl" :src="lastFrameUrl" alt="Last frame" class="max-h-48 w-full object-contain">
|
||||||
|
<p v-else class="px-3 py-8 text-center text-xs text-zinc-500">
|
||||||
|
{{ lastFrameLoading ? 'Reading last frame…' : 'Last frame preview unavailable' }}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<label class="block text-sm">
|
||||||
|
<span class="mb-1 block text-zinc-400">Extension prompt</span>
|
||||||
|
<textarea
|
||||||
|
v-model="extendPrompt"
|
||||||
|
rows="4"
|
||||||
|
class="w-full rounded-xl border border-white/10 bg-zinc-950 px-3 py-2 text-sm outline-none ring-amber-300/40 placeholder:text-zinc-600 focus:ring-2"
|
||||||
|
placeholder="Continue the motion from this frame…"
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<div>
|
||||||
|
<p class="mb-2 text-xs uppercase tracking-wider text-zinc-500">Extension duration</p>
|
||||||
|
<div class="grid grid-cols-2 gap-2">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="rounded-xl border px-3 py-2 text-sm"
|
||||||
|
:class="extendDuration === 3 ? 'border-amber-300 bg-amber-400/10' : 'border-white/10'"
|
||||||
|
@click="extendDuration = 3"
|
||||||
|
>
|
||||||
|
3s
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="rounded-xl border px-3 py-2 text-sm"
|
||||||
|
:class="extendDuration === 5 ? 'border-amber-300 bg-amber-400/10' : 'border-white/10'"
|
||||||
|
@click="extendDuration = 5"
|
||||||
|
>
|
||||||
|
5s
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="w-full rounded-2xl bg-amber-400 px-4 py-2.5 text-sm font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
|
:disabled="busy || !extendPrompt.trim() || lastFrameLoading"
|
||||||
|
@click="generateExtension"
|
||||||
|
>
|
||||||
|
Generate Extension
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="glass rounded-3xl p-5 sm:p-6">
|
<div class="glass rounded-3xl p-5 sm:p-6">
|
||||||
@@ -614,6 +680,13 @@ const maxStep = ref(8)
|
|||||||
const elapsedMs = ref(0)
|
const elapsedMs = ref(0)
|
||||||
const videoUrl = ref('')
|
const videoUrl = ref('')
|
||||||
const jobId = ref('')
|
const jobId = ref('')
|
||||||
|
const currentClipId = ref('')
|
||||||
|
const extendOpen = ref(false)
|
||||||
|
const extendPrompt = ref('')
|
||||||
|
const extendDuration = ref(5)
|
||||||
|
const lastFrameUrl = ref('')
|
||||||
|
const lastFrameLoading = ref(false)
|
||||||
|
let lastFrameObjectUrl = ''
|
||||||
const awaitingReveal = ref(false)
|
const awaitingReveal = ref(false)
|
||||||
const concealOutput = ref(false)
|
const concealOutput = ref(false)
|
||||||
const lockedSave = ref(false)
|
const lockedSave = ref(false)
|
||||||
@@ -686,6 +759,13 @@ function normalizeAspect(raw?: string, width?: number, height?: number): AspectI
|
|||||||
const outputSize = computed(() => PRESETS[aspect.value])
|
const outputSize = computed(() => PRESETS[aspect.value])
|
||||||
const frameCount = computed(() => Math.max(5, Math.floor(duration.value * fps.value)))
|
const frameCount = computed(() => Math.max(5, Math.floor(duration.value * fps.value)))
|
||||||
const cfgPct = computed(() => ((cfg.value - CFG_MIN) / (CFG_MAX - CFG_MIN)) * 100)
|
const cfgPct = computed(() => ((cfg.value - CFG_MIN) / (CFG_MAX - CFG_MIN)) * 100)
|
||||||
|
const currentClip = computed(() => clips.value.find(clip => clip.id === currentClipId.value))
|
||||||
|
const canExtend = computed(() => Boolean(
|
||||||
|
videoUrl.value
|
||||||
|
&& !lockedSave.value
|
||||||
|
&& !awaitingReveal.value
|
||||||
|
&& (currentClipId.value || clipIdFromPlayer())
|
||||||
|
))
|
||||||
|
|
||||||
const visibleClips = computed(() => clips.value.filter(clip => clip.folderId === browseFolderId.value))
|
const visibleClips = computed(() => clips.value.filter(clip => clip.folderId === browseFolderId.value))
|
||||||
const currentFolder = computed(() => folders.value.find(folder => folder.id === folderId.value))
|
const currentFolder = computed(() => folders.value.find(folder => folder.id === folderId.value))
|
||||||
@@ -756,6 +836,7 @@ onBeforeUnmount(() => {
|
|||||||
healthTimer = null
|
healthTimer = null
|
||||||
stopTimer()
|
stopTimer()
|
||||||
stopListen()
|
stopListen()
|
||||||
|
closeExtend()
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(folderId, () => {
|
watch(folderId, () => {
|
||||||
@@ -1092,11 +1173,95 @@ async function lockFolder(id: string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function clipIdFromPlayer() {
|
||||||
|
const match = videoUrl.value.match(/\/api\/library\/clips\/([^/?#]+)\/video/)
|
||||||
|
return match?.[1] || currentClipId.value
|
||||||
|
}
|
||||||
|
|
||||||
function clipVideoUrl(clipId: string) {
|
function clipVideoUrl(clipId: string) {
|
||||||
return `/api/library/clips/${clipId}/video?t=${Date.now()}`
|
return `/api/library/clips/${clipId}/video?t=${Date.now()}`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function closeExtend() {
|
||||||
|
extendOpen.value = false
|
||||||
|
lastFrameLoading.value = false
|
||||||
|
if (lastFrameObjectUrl) {
|
||||||
|
URL.revokeObjectURL(lastFrameObjectUrl)
|
||||||
|
lastFrameObjectUrl = ''
|
||||||
|
}
|
||||||
|
lastFrameUrl.value = ''
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openExtend() {
|
||||||
|
const id = clipIdFromPlayer()
|
||||||
|
if (!id) {
|
||||||
|
toast('Load a generated clip before extending it.')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
currentClipId.value = id
|
||||||
|
const clip = clips.value.find(item => item.id === id)
|
||||||
|
extendPrompt.value = clip?.prompt || prompt.value
|
||||||
|
extendDuration.value = 5
|
||||||
|
extendOpen.value = true
|
||||||
|
lastFrameLoading.value = true
|
||||||
|
lastFrameUrl.value = ''
|
||||||
|
player.value?.pause()
|
||||||
|
try {
|
||||||
|
const blob = await $fetch<Blob>(`/api/library/clips/${id}/last-frame`, { responseType: 'blob' })
|
||||||
|
if (lastFrameObjectUrl) URL.revokeObjectURL(lastFrameObjectUrl)
|
||||||
|
lastFrameObjectUrl = URL.createObjectURL(blob)
|
||||||
|
lastFrameUrl.value = lastFrameObjectUrl
|
||||||
|
} catch (error: any) {
|
||||||
|
toast(error?.data?.statusMessage || 'Could not extract the last frame')
|
||||||
|
} finally {
|
||||||
|
lastFrameLoading.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function generateExtension() {
|
||||||
|
const id = clipIdFromPlayer()
|
||||||
|
if (!id || !extendPrompt.value.trim()) return
|
||||||
|
statusBusy.value = false
|
||||||
|
busy.value = true
|
||||||
|
listenGen += 1
|
||||||
|
stopListen()
|
||||||
|
settledUi = false
|
||||||
|
progress.value = 1
|
||||||
|
awaitingReveal.value = false
|
||||||
|
concealOutput.value = hideThumbnail.value || currentClip.value?.hideThumbnail === true
|
||||||
|
lockedSave.value = false
|
||||||
|
statusMessage.value = 'Extracting last frame & stitching extension...'
|
||||||
|
startTimer()
|
||||||
|
try {
|
||||||
|
const started = await $fetch<{ jobId: string; steps: number; hideThumbnail?: boolean; folderLocked?: boolean }>('/api/extend', {
|
||||||
|
method: 'POST',
|
||||||
|
body: {
|
||||||
|
clipId: id,
|
||||||
|
prompt: extendPrompt.value.trim(),
|
||||||
|
duration: extendDuration.value
|
||||||
|
}
|
||||||
|
})
|
||||||
|
closeExtend()
|
||||||
|
jobId.value = started.jobId
|
||||||
|
maxStep.value = started.steps
|
||||||
|
settledUi = false
|
||||||
|
persistActiveJob(started.jobId, started.hideThumbnail === true || concealOutput.value, started.folderLocked === true)
|
||||||
|
listen(started.jobId, started.hideThumbnail === true || concealOutput.value, started.folderLocked === true)
|
||||||
|
void pollComfyHealth()
|
||||||
|
} catch (error: any) {
|
||||||
|
busy.value = false
|
||||||
|
statusBusy.value = false
|
||||||
|
stopTimer()
|
||||||
|
clearActiveJob()
|
||||||
|
const message = error?.data?.statusMessage || error?.statusMessage || error?.message || 'Failed to start extension'
|
||||||
|
statusMessage.value = message
|
||||||
|
toast(message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function openClip(item: LibraryClip) {
|
function openClip(item: LibraryClip) {
|
||||||
|
closeExtend()
|
||||||
|
currentClipId.value = item.id
|
||||||
lockedSave.value = false
|
lockedSave.value = false
|
||||||
concealOutput.value = item.hideThumbnail
|
concealOutput.value = item.hideThumbnail
|
||||||
videoUrl.value = clipVideoUrl(item.id)
|
videoUrl.value = clipVideoUrl(item.id)
|
||||||
@@ -1116,6 +1281,8 @@ async function removeClip(item: LibraryClip) {
|
|||||||
awaitingReveal.value = false
|
awaitingReveal.value = false
|
||||||
concealOutput.value = false
|
concealOutput.value = false
|
||||||
lockedSave.value = false
|
lockedSave.value = false
|
||||||
|
currentClipId.value = ''
|
||||||
|
closeExtend()
|
||||||
}
|
}
|
||||||
await loadLibrary()
|
await loadLibrary()
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
@@ -1208,6 +1375,8 @@ async function generate() {
|
|||||||
settledUi = false
|
settledUi = false
|
||||||
progress.value = 1
|
progress.value = 1
|
||||||
videoUrl.value = ''
|
videoUrl.value = ''
|
||||||
|
currentClipId.value = ''
|
||||||
|
closeExtend()
|
||||||
awaitingReveal.value = false
|
awaitingReveal.value = false
|
||||||
concealOutput.value = hideThumbnail.value
|
concealOutput.value = hideThumbnail.value
|
||||||
lockedSave.value = false
|
lockedSave.value = false
|
||||||
@@ -1308,7 +1477,9 @@ function applyJobEvent(payload: Record<string, any>, hidden: boolean, folderLock
|
|||||||
lockedSave.value = true
|
lockedSave.value = true
|
||||||
browseFolderId.value = folderId.value
|
browseFolderId.value = folderId.value
|
||||||
statusMessage.value = 'Saved to the locked folder. Unlock it to view.'
|
statusMessage.value = 'Saved to the locked folder. Unlock it to view.'
|
||||||
|
currentClipId.value = payload.clipId || currentClipId.value
|
||||||
} else if (payload.clipId) {
|
} else if (payload.clipId) {
|
||||||
|
currentClipId.value = payload.clipId
|
||||||
videoUrl.value = clipVideoUrl(payload.clipId)
|
videoUrl.value = clipVideoUrl(payload.clipId)
|
||||||
awaitingReveal.value = Boolean(payload.hideThumbnail || hidden)
|
awaitingReveal.value = Boolean(payload.hideThumbnail || hidden)
|
||||||
concealOutput.value = awaitingReveal.value
|
concealOutput.value = awaitingReveal.value
|
||||||
|
|||||||
@@ -0,0 +1,233 @@
|
|||||||
|
import { copyFileSync, existsSync, readFileSync } from 'node:fs'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
|
||||||
|
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
|
||||||
|
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
|
||||||
|
|
||||||
|
function parseExtendDuration(raw: unknown) {
|
||||||
|
const seconds = Number(raw)
|
||||||
|
return seconds === 3 || seconds === 5 ? seconds : 5
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameLength(seconds: number, fps: number) {
|
||||||
|
return Math.max(5, Math.floor(seconds * fps))
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseFps(raw: unknown) {
|
||||||
|
const fps = Number(raw)
|
||||||
|
return fps === 12 || fps === 30 || fps === 24 ? fps : 24
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseCfg(raw: unknown, turbo: boolean) {
|
||||||
|
const fallback = turbo ? 1.5 : 4
|
||||||
|
const value = Number(raw)
|
||||||
|
if (!Number.isFinite(value)) return fallback
|
||||||
|
const snapped = Math.round(value * 2) / 2
|
||||||
|
return Math.min(10, Math.max(1, snapped))
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseSampler(raw: unknown) {
|
||||||
|
const value = String(raw || '')
|
||||||
|
return SAMPLERS.has(value) ? value : 'res_multistep'
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseScheduler(raw: unknown) {
|
||||||
|
const value = String(raw || '')
|
||||||
|
return SCHEDULERS.has(value) ? value : 'simple'
|
||||||
|
}
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event) => {
|
||||||
|
const body = await readBody<{ clipId?: string; prompt?: string; duration?: number }>(event).catch(() => ({}))
|
||||||
|
const clipId = String(body?.clipId || '').trim()
|
||||||
|
const prompt = String(body?.prompt || '').trim()
|
||||||
|
if (!clipId) {
|
||||||
|
throw createError({ statusCode: 400, statusMessage: 'A clip is required to extend' })
|
||||||
|
}
|
||||||
|
if (!prompt) {
|
||||||
|
throw createError({ statusCode: 400, statusMessage: 'An extension prompt is required' })
|
||||||
|
}
|
||||||
|
|
||||||
|
const { owner } = assertLibraryOwner(event)
|
||||||
|
const source = getClip(owner, clipId)
|
||||||
|
assertFolderAccess(event, source.folderId)
|
||||||
|
const sourcePath = clipVideoPath(owner, source.id)
|
||||||
|
if (!existsSync(sourcePath)) {
|
||||||
|
throw createError({ statusCode: 404, statusMessage: 'Source video is missing' })
|
||||||
|
}
|
||||||
|
|
||||||
|
const destFolder = publicLibrary(event).folders.find(folder => folder.id === source.folderId)
|
||||||
|
const folderLocked = Boolean(destFolder?.protected && !destFolder.unlocked)
|
||||||
|
const durationSeconds = parseExtendDuration(body?.duration)
|
||||||
|
const turbo = source.turbo !== false
|
||||||
|
const fps = parseFps(source.fps)
|
||||||
|
const cfg = parseCfg(source.cfg, turbo)
|
||||||
|
const samplerName = parseSampler(source.samplerName)
|
||||||
|
const scheduler = parseScheduler(source.scheduler)
|
||||||
|
const steps = Number(source.steps || (turbo ? 8 : 20))
|
||||||
|
const seed = Math.floor(Math.random() * 2_147_483_647)
|
||||||
|
const length = frameLength(durationSeconds, fps)
|
||||||
|
const width = source.width || 1344
|
||||||
|
const height = source.height || 768
|
||||||
|
const clipName = `${clipTitle(source)} extended`.slice(0, 80)
|
||||||
|
|
||||||
|
const job = createJob()
|
||||||
|
job.maxStep = steps
|
||||||
|
job.hideThumbnail = source.hideThumbnail
|
||||||
|
const tmpDir = extendTempDir(owner, job.id)
|
||||||
|
const part1Path = join(tmpDir, 'part1.mp4')
|
||||||
|
const framePath = join(tmpDir, 'last_frame.png')
|
||||||
|
job.library = {
|
||||||
|
ownerKey: owner,
|
||||||
|
folderId: source.folderId,
|
||||||
|
hideThumbnail: source.hideThumbnail,
|
||||||
|
folderLocked,
|
||||||
|
name: clipName,
|
||||||
|
prompt,
|
||||||
|
aspect: source.aspect || '16:9',
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
steps,
|
||||||
|
turbo,
|
||||||
|
seed,
|
||||||
|
cfg,
|
||||||
|
fps,
|
||||||
|
samplerName,
|
||||||
|
scheduler,
|
||||||
|
duration: durationSeconds,
|
||||||
|
extendTmpDir: tmpDir,
|
||||||
|
extendPart1Path: part1Path,
|
||||||
|
extendSourceClipId: source.id
|
||||||
|
}
|
||||||
|
|
||||||
|
emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 })
|
||||||
|
|
||||||
|
void runExtend(job, {
|
||||||
|
sourcePath,
|
||||||
|
part1Path,
|
||||||
|
framePath,
|
||||||
|
prompt,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
steps,
|
||||||
|
seed,
|
||||||
|
turbo,
|
||||||
|
length,
|
||||||
|
cfg,
|
||||||
|
fps,
|
||||||
|
samplerName,
|
||||||
|
scheduler
|
||||||
|
}).catch((error) => {
|
||||||
|
removeExtendTemp(job.library?.extendTmpDir)
|
||||||
|
const message = error instanceof Error ? error.message : String(error)
|
||||||
|
job.status = 'error'
|
||||||
|
job.error = message
|
||||||
|
emitJob(job, { type: 'error', error: message, message })
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
jobId: job.id,
|
||||||
|
clientId: job.clientId,
|
||||||
|
seed,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
steps,
|
||||||
|
duration: durationSeconds,
|
||||||
|
hideThumbnail: source.hideThumbnail,
|
||||||
|
folderLocked
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
async function runExtend(
|
||||||
|
job: ReturnType<typeof createJob>,
|
||||||
|
params: {
|
||||||
|
sourcePath: string
|
||||||
|
part1Path: string
|
||||||
|
framePath: string
|
||||||
|
prompt: string
|
||||||
|
width: number
|
||||||
|
height: number
|
||||||
|
steps: number
|
||||||
|
seed: number
|
||||||
|
turbo: boolean
|
||||||
|
length: number
|
||||||
|
cfg: number
|
||||||
|
fps: number
|
||||||
|
samplerName: string
|
||||||
|
scheduler: string
|
||||||
|
}
|
||||||
|
) {
|
||||||
|
copyFileSync(params.sourcePath, params.part1Path)
|
||||||
|
await extractLastFrame(params.part1Path, params.framePath)
|
||||||
|
const frame = readFileSync(params.framePath)
|
||||||
|
job.library!.thumb = frame
|
||||||
|
const sound = await probeHasAudio(params.part1Path)
|
||||||
|
|
||||||
|
await ensureComfyReady((status) => {
|
||||||
|
emitJob(job, {
|
||||||
|
type: status.state === 'busy' ? 'busy' : 'status',
|
||||||
|
message: status.message,
|
||||||
|
progress: status.state === 'online' ? Math.max(job.progress, 3) : Math.max(job.progress, 2),
|
||||||
|
busy: status.state === 'busy',
|
||||||
|
queueRunning: status.queueRunning,
|
||||||
|
queuePending: status.queuePending
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
const done = watchComfyJob(job)
|
||||||
|
job.status = 'uploading'
|
||||||
|
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 4 })
|
||||||
|
const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
|
||||||
|
if (job.library) {
|
||||||
|
job.library.imageName = uploaded.name
|
||||||
|
job.library.imageSubfolder = uploaded.subfolder
|
||||||
|
}
|
||||||
|
emitJob(job, { type: 'status', message: 'Queueing extension on MiniMax H3...', progress: 6 })
|
||||||
|
await waitForComfySocket(job, 4000)
|
||||||
|
|
||||||
|
const graph = buildWorkflow({
|
||||||
|
prompt: params.prompt,
|
||||||
|
imageName: uploaded.name,
|
||||||
|
width: params.width,
|
||||||
|
height: params.height,
|
||||||
|
steps: params.steps,
|
||||||
|
seed: params.seed,
|
||||||
|
turbo: params.turbo,
|
||||||
|
length: params.length,
|
||||||
|
cfg: params.cfg,
|
||||||
|
fps: params.fps,
|
||||||
|
samplerName: params.samplerName,
|
||||||
|
scheduler: params.scheduler,
|
||||||
|
filenamePrefix: comfyFilenamePrefix(),
|
||||||
|
sound
|
||||||
|
})
|
||||||
|
|
||||||
|
const queued = await queuePrompt(graph, job.clientId)
|
||||||
|
job.promptId = queued.prompt_id
|
||||||
|
job.status = 'running'
|
||||||
|
if (job.library && job.promptId) {
|
||||||
|
writePendingJob({
|
||||||
|
jobId: job.id,
|
||||||
|
promptId: job.promptId,
|
||||||
|
clientId: job.clientId,
|
||||||
|
ownerKey: job.library.ownerKey,
|
||||||
|
folderId: job.library.folderId,
|
||||||
|
hideThumbnail: job.library.hideThumbnail,
|
||||||
|
folderLocked: job.library.folderLocked,
|
||||||
|
name: job.library.name,
|
||||||
|
prompt: job.library.prompt,
|
||||||
|
aspect: job.library.aspect,
|
||||||
|
width: job.library.width,
|
||||||
|
height: job.library.height,
|
||||||
|
steps: job.library.steps,
|
||||||
|
turbo: job.library.turbo,
|
||||||
|
seed: job.library.seed,
|
||||||
|
startedAt: job.startedAt,
|
||||||
|
imageName: job.library.imageName,
|
||||||
|
imageSubfolder: job.library.imageSubfolder,
|
||||||
|
extendTmpDir: job.library.extendTmpDir,
|
||||||
|
extendPart1Path: job.library.extendPart1Path
|
||||||
|
})
|
||||||
|
}
|
||||||
|
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
|
||||||
|
await done
|
||||||
|
}
|
||||||
@@ -273,7 +273,9 @@ async function runGeneration(
|
|||||||
seed: job.library.seed,
|
seed: job.library.seed,
|
||||||
startedAt: job.startedAt,
|
startedAt: job.startedAt,
|
||||||
imageName: job.library.imageName,
|
imageName: job.library.imageName,
|
||||||
imageSubfolder: job.library.imageSubfolder
|
imageSubfolder: job.library.imageSubfolder,
|
||||||
|
extendTmpDir: job.library.extendTmpDir,
|
||||||
|
extendPart1Path: job.library.extendPart1Path
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
|
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ export default defineEventHandler(async (event) => {
|
|||||||
if (job) {
|
if (job) {
|
||||||
job.status = 'cancelled'
|
job.status = 'cancelled'
|
||||||
emitJob(job, { type: 'status', message: 'Cancelling job...' })
|
emitJob(job, { type: 'status', message: 'Cancelling job...' })
|
||||||
|
removeExtendTemp(job.library?.extendTmpDir)
|
||||||
}
|
}
|
||||||
if (jobId) deletePendingJob(jobId)
|
if (jobId) deletePendingJob(jobId)
|
||||||
const ok = await interruptComfy()
|
const ok = await interruptComfy()
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import { existsSync, readFileSync } from 'node:fs'
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event) => {
|
||||||
|
const { owner } = assertLibraryOwner(event)
|
||||||
|
const id = String(getRouterParam(event, 'id') || '')
|
||||||
|
const clip = getClip(owner, id)
|
||||||
|
assertFolderAccess(event, clip.folderId)
|
||||||
|
const videoPath = clipVideoPath(owner, clip.id)
|
||||||
|
if (!existsSync(videoPath)) {
|
||||||
|
throw createError({ statusCode: 404, statusMessage: 'Video file is missing' })
|
||||||
|
}
|
||||||
|
const framePath = clipLastFramePath(owner, clip.id)
|
||||||
|
if (!existsSync(framePath)) {
|
||||||
|
try {
|
||||||
|
await extractLastFrame(videoPath, framePath)
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error)
|
||||||
|
throw createError({ statusCode: 500, statusMessage: `Could not extract the last frame: ${message}` })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const buf = readFileSync(framePath)
|
||||||
|
setHeader(event, 'Content-Type', 'image/png')
|
||||||
|
setHeader(event, 'Cache-Control', 'private, no-store')
|
||||||
|
return buf
|
||||||
|
})
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
import { existsSync, writeFileSync, readFileSync, unlinkSync } from 'node:fs'
|
||||||
|
import { dirname, join } from 'node:path'
|
||||||
|
import { spawn } from 'node:child_process'
|
||||||
|
|
||||||
|
function collectError(stderr: string, fallback: string) {
|
||||||
|
const text = stderr.replace(/\r/g, '').trim()
|
||||||
|
const lines = text.split('\n').map(line => line.trim()).filter(Boolean)
|
||||||
|
return lines.slice(-6).join(' ') || fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runFfmpeg(args: string[], timeoutMs = 120000) {
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
const proc = spawn('ffmpeg', args, { stdio: ['ignore', 'ignore', 'pipe'] })
|
||||||
|
let stderr = ''
|
||||||
|
proc.stderr?.on('data', (chunk: Buffer | string) => { stderr += String(chunk) })
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
try { proc.kill('SIGKILL') } catch { /* ignore */ }
|
||||||
|
reject(new Error(`FFmpeg timed out after ${Math.round(timeoutMs / 1000)}s`))
|
||||||
|
}, timeoutMs)
|
||||||
|
proc.on('error', (error) => {
|
||||||
|
clearTimeout(timer)
|
||||||
|
const message = (error as NodeJS.ErrnoException).code === 'ENOENT'
|
||||||
|
? 'FFmpeg is not installed on this server'
|
||||||
|
: error.message
|
||||||
|
reject(new Error(message))
|
||||||
|
})
|
||||||
|
proc.on('close', (code) => {
|
||||||
|
clearTimeout(timer)
|
||||||
|
if (code === 0) resolve()
|
||||||
|
else reject(new Error(collectError(stderr, `FFmpeg exited with code ${code}`)))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function runFfprobe(args: string[], timeoutMs = 15000) {
|
||||||
|
return new Promise<string>((resolve, reject) => {
|
||||||
|
const proc = spawn('ffprobe', args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||||
|
let out = ''
|
||||||
|
let err = ''
|
||||||
|
proc.stdout?.on('data', (chunk: Buffer | string) => { out += String(chunk) })
|
||||||
|
proc.stderr?.on('data', (chunk: Buffer | string) => { err += String(chunk) })
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
try { proc.kill('SIGKILL') } catch { /* ignore */ }
|
||||||
|
reject(new Error('ffprobe timed out'))
|
||||||
|
}, timeoutMs)
|
||||||
|
proc.on('error', (error) => {
|
||||||
|
clearTimeout(timer)
|
||||||
|
reject(error)
|
||||||
|
})
|
||||||
|
proc.on('close', (code) => {
|
||||||
|
clearTimeout(timer)
|
||||||
|
if (code === 0) resolve(out.trim())
|
||||||
|
else reject(new Error(collectError(err, `ffprobe exited with code ${code}`)))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function concatPath(filePath: string) {
|
||||||
|
return filePath.replace(/\\/g, '/').replace(/'/g, "'\\''")
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function extractLastFrame(inputPath: string, outputPath: string) {
|
||||||
|
if (!existsSync(inputPath)) {
|
||||||
|
throw new Error('Source video is missing')
|
||||||
|
}
|
||||||
|
await runFfmpeg([
|
||||||
|
'-y',
|
||||||
|
'-sseof', '-1',
|
||||||
|
'-i', inputPath,
|
||||||
|
'-update', '1',
|
||||||
|
'-frames:v', '1',
|
||||||
|
'-q:v', '1',
|
||||||
|
outputPath
|
||||||
|
], 30000)
|
||||||
|
if (!existsSync(outputPath)) {
|
||||||
|
throw new Error('FFmpeg did not write the last frame')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function probeHasAudio(filePath: string) {
|
||||||
|
try {
|
||||||
|
const out = await runFfprobe([
|
||||||
|
'-v', 'error',
|
||||||
|
'-select_streams', 'a:0',
|
||||||
|
'-show_entries', 'stream=codec_type',
|
||||||
|
'-of', 'csv=p=0',
|
||||||
|
filePath
|
||||||
|
])
|
||||||
|
return out.includes('audio')
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function concatMp4(part1Path: string, part2Path: string, outputPath: string) {
|
||||||
|
const listPath = join(dirname(outputPath), 'concat_list.txt')
|
||||||
|
writeFileSync(listPath, `file '${concatPath(part1Path)}'\nfile '${concatPath(part2Path)}'\n`)
|
||||||
|
try {
|
||||||
|
await runFfmpeg(['-y', '-f', 'concat', '-safe', '0', '-i', listPath, '-c', 'copy', outputPath], 180000)
|
||||||
|
} catch (copyError) {
|
||||||
|
const audio1 = await probeHasAudio(part1Path)
|
||||||
|
const audio2 = await probeHasAudio(part2Path)
|
||||||
|
try {
|
||||||
|
if (audio1 && audio2) {
|
||||||
|
await runFfmpeg([
|
||||||
|
'-y',
|
||||||
|
'-i', part1Path,
|
||||||
|
'-i', part2Path,
|
||||||
|
'-filter_complex', '[0:v][0:a][1:v][1:a]concat=n=2:v=1:a=1[v][a]',
|
||||||
|
'-map', '[v]',
|
||||||
|
'-map', '[a]',
|
||||||
|
'-c:v', 'libx264',
|
||||||
|
'-preset', 'fast',
|
||||||
|
'-crf', '18',
|
||||||
|
'-pix_fmt', 'yuv420p',
|
||||||
|
'-c:a', 'aac',
|
||||||
|
'-movflags', '+faststart',
|
||||||
|
outputPath
|
||||||
|
], 300000)
|
||||||
|
} else {
|
||||||
|
await runFfmpeg([
|
||||||
|
'-y',
|
||||||
|
'-i', part1Path,
|
||||||
|
'-i', part2Path,
|
||||||
|
'-filter_complex', '[0:v][1:v]concat=n=2:v=1:a=0[v]',
|
||||||
|
'-map', '[v]',
|
||||||
|
'-c:v', 'libx264',
|
||||||
|
'-preset', 'fast',
|
||||||
|
'-crf', '18',
|
||||||
|
'-pix_fmt', 'yuv420p',
|
||||||
|
'-movflags', '+faststart',
|
||||||
|
outputPath
|
||||||
|
], 300000)
|
||||||
|
}
|
||||||
|
} catch (encodeError) {
|
||||||
|
const copyMessage = copyError instanceof Error ? copyError.message : String(copyError)
|
||||||
|
const encodeMessage = encodeError instanceof Error ? encodeError.message : String(encodeError)
|
||||||
|
throw new Error(`FFmpeg concat failed (${copyMessage}). Re-encode also failed (${encodeMessage}).`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!existsSync(outputPath)) {
|
||||||
|
throw new Error('FFmpeg did not write the stitched video')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function stitchExtension(params: { part1Path: string; part2: Buffer; tmpDir: string }) {
|
||||||
|
const part2Path = join(params.tmpDir, 'part2.mp4')
|
||||||
|
const finalPath = join(params.tmpDir, 'final_stitched_video.mp4')
|
||||||
|
writeFileSync(part2Path, params.part2)
|
||||||
|
await concatMp4(params.part1Path, part2Path, finalPath)
|
||||||
|
const stitched = readFileSync(finalPath)
|
||||||
|
for (const extra of [part2Path, join(params.tmpDir, 'concat_list.txt'), join(params.tmpDir, 'last_frame.png')]) {
|
||||||
|
try { unlinkSync(extra) } catch { /* ignore */ }
|
||||||
|
}
|
||||||
|
return stitched
|
||||||
|
}
|
||||||
@@ -61,6 +61,9 @@ export interface Job {
|
|||||||
duration?: number
|
duration?: number
|
||||||
sound?: boolean
|
sound?: boolean
|
||||||
draftId?: string
|
draftId?: string
|
||||||
|
extendTmpDir?: string
|
||||||
|
extendPart1Path?: string
|
||||||
|
extendSourceClipId?: string
|
||||||
}
|
}
|
||||||
error?: string
|
error?: string
|
||||||
socketReady?: boolean
|
socketReady?: boolean
|
||||||
|
|||||||
@@ -462,6 +462,21 @@ export function clipVideoPath(owner: string, id: string) {
|
|||||||
return join(clipDir(owner, id), 'video.mp4')
|
return join(clipDir(owner, id), 'video.mp4')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function clipLastFramePath(owner: string, id: string) {
|
||||||
|
return join(clipDir(owner, id), 'last-frame.png')
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extendTempDir(owner: string, jobId: string) {
|
||||||
|
const dir = join(ownerRoot(owner), 'tmp', 'extend', jobId)
|
||||||
|
mkdirSync(dir, { recursive: true })
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
export function removeExtendTemp(dir?: string) {
|
||||||
|
if (!dir) return
|
||||||
|
rmSync(dir, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
|
||||||
export function clipThumbPath(owner: string, id: string) {
|
export function clipThumbPath(owner: string, id: string) {
|
||||||
const dir = clipDir(owner, id)
|
const dir = clipDir(owner, id)
|
||||||
for (const name of ['thumb.jpg', 'thumb.png', 'thumb']) {
|
for (const name of ['thumb.jpg', 'thumb.png', 'thumb']) {
|
||||||
|
|||||||
+23
-2
@@ -20,6 +20,8 @@ export interface PendingJob {
|
|||||||
startedAt: number
|
startedAt: number
|
||||||
imageName?: string
|
imageName?: string
|
||||||
imageSubfolder?: string
|
imageSubfolder?: string
|
||||||
|
extendTmpDir?: string
|
||||||
|
extendPart1Path?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
function pendingRoot() {
|
function pendingRoot() {
|
||||||
@@ -68,7 +70,24 @@ export async function completePendingIfReady(pending: PendingJob) {
|
|||||||
const history = await fetchHistory(pending.promptId)
|
const history = await fetchHistory(pending.promptId)
|
||||||
const video = extractVideo(history, pending.promptId)
|
const video = extractVideo(history, pending.promptId)
|
||||||
if (!video) return null
|
if (!video) return null
|
||||||
const buffer = await downloadComfyVideo(video)
|
let buffer = await downloadComfyVideo(video)
|
||||||
|
if (pending.extendPart1Path && pending.extendTmpDir) {
|
||||||
|
if (!existsSync(pending.extendPart1Path)) {
|
||||||
|
removeExtendTemp(pending.extendTmpDir)
|
||||||
|
throw createError({ statusCode: 500, statusMessage: 'Extension source clip was missing during stitch' })
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
buffer = await stitchExtension({
|
||||||
|
part1Path: pending.extendPart1Path,
|
||||||
|
part2: buffer,
|
||||||
|
tmpDir: pending.extendTmpDir
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
removeExtendTemp(pending.extendTmpDir)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
removeExtendTemp(pending.extendTmpDir)
|
||||||
|
}
|
||||||
const clip = await saveClip({
|
const clip = await saveClip({
|
||||||
ownerKey: pending.ownerKey,
|
ownerKey: pending.ownerKey,
|
||||||
folderId: pending.folderId,
|
folderId: pending.folderId,
|
||||||
@@ -90,7 +109,9 @@ export async function completePendingIfReady(pending: PendingJob) {
|
|||||||
return {
|
return {
|
||||||
type: 'complete' as const,
|
type: 'complete' as const,
|
||||||
status: 'complete' as const,
|
status: 'complete' as const,
|
||||||
message: pending.folderLocked ? 'Saved to the locked folder. Unlock it to view.' : 'Video ready',
|
message: pending.folderLocked
|
||||||
|
? 'Saved to the locked folder. Unlock it to view.'
|
||||||
|
: (pending.extendPart1Path ? 'Extended video ready' : 'Video ready'),
|
||||||
progress: 100,
|
progress: 100,
|
||||||
jobId: pending.jobId,
|
jobId: pending.jobId,
|
||||||
clipId: clip.id,
|
clipId: clip.id,
|
||||||
|
|||||||
+25
-4
@@ -1,4 +1,4 @@
|
|||||||
import { NODE_LABELS, isEncodingNode } from '~/server/utils/workflow'
|
import { existsSync } from 'node:fs'
|
||||||
import type { Job } from '~/server/utils/jobs'
|
import type { Job } from '~/server/utils/jobs'
|
||||||
|
|
||||||
function classifyError(message: string) {
|
function classifyError(message: string) {
|
||||||
@@ -42,6 +42,7 @@ export function watchComfyJob(job: Job): Promise<void> {
|
|||||||
job.error = classifyError(error)
|
job.error = classifyError(error)
|
||||||
emitJob(job, { type: 'error', error: job.error, message: job.error })
|
emitJob(job, { type: 'error', error: job.error, message: job.error })
|
||||||
if (job.id) deletePendingJob(job.id)
|
if (job.id) deletePendingJob(job.id)
|
||||||
|
removeExtendTemp(job.library?.extendTmpDir)
|
||||||
resolve()
|
resolve()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,9 +55,24 @@ export function watchComfyJob(job: Job): Promise<void> {
|
|||||||
cleanup()
|
cleanup()
|
||||||
try {
|
try {
|
||||||
job.video = video
|
job.video = video
|
||||||
emitJob(job, { type: 'status', message: 'Saving to library...', progress: 98 })
|
const stitching = Boolean(job.library?.extendPart1Path && job.library.extendTmpDir)
|
||||||
|
emitJob(job, {
|
||||||
|
type: 'status',
|
||||||
|
message: stitching ? 'Extracting last frame & stitching extension...' : 'Saving to library...',
|
||||||
|
progress: stitching ? 97 : 98
|
||||||
|
})
|
||||||
if (job.library) {
|
if (job.library) {
|
||||||
const buffer = await downloadComfyVideo(video)
|
let buffer = await downloadComfyVideo(video)
|
||||||
|
if (stitching) {
|
||||||
|
if (!existsSync(job.library.extendPart1Path!)) {
|
||||||
|
throw new Error('Extension source clip was missing during stitch')
|
||||||
|
}
|
||||||
|
buffer = await stitchExtension({
|
||||||
|
part1Path: job.library.extendPart1Path!,
|
||||||
|
part2: buffer,
|
||||||
|
tmpDir: job.library.extendTmpDir!
|
||||||
|
})
|
||||||
|
}
|
||||||
const clip = await saveClip({
|
const clip = await saveClip({
|
||||||
ownerKey: job.library.ownerKey,
|
ownerKey: job.library.ownerKey,
|
||||||
folderId: job.library.folderId,
|
folderId: job.library.folderId,
|
||||||
@@ -87,11 +103,15 @@ export function watchComfyJob(job: Job): Promise<void> {
|
|||||||
imageSubfolder: job.library.imageSubfolder,
|
imageSubfolder: job.library.imageSubfolder,
|
||||||
promptId: job.promptId
|
promptId: job.promptId
|
||||||
})
|
})
|
||||||
|
removeExtendTemp(job.library.extendTmpDir)
|
||||||
|
job.library.extendTmpDir = undefined
|
||||||
}
|
}
|
||||||
job.status = 'complete'
|
job.status = 'complete'
|
||||||
emitJob(job, {
|
emitJob(job, {
|
||||||
type: 'complete',
|
type: 'complete',
|
||||||
message: job.library?.folderLocked ? 'Saved to the locked folder. Unlock it to view.' : 'Video ready',
|
message: job.library?.folderLocked
|
||||||
|
? 'Saved to the locked folder. Unlock it to view.'
|
||||||
|
: (job.library?.extendSourceClipId ? 'Extended video ready' : 'Video ready'),
|
||||||
progress: 100,
|
progress: 100,
|
||||||
filename: job.library?.folderLocked ? undefined : video.filename,
|
filename: job.library?.folderLocked ? undefined : video.filename,
|
||||||
subfolder: job.library?.folderLocked ? undefined : video.subfolder,
|
subfolder: job.library?.folderLocked ? undefined : video.subfolder,
|
||||||
@@ -102,6 +122,7 @@ export function watchComfyJob(job: Job): Promise<void> {
|
|||||||
})
|
})
|
||||||
deletePendingJob(job.id)
|
deletePendingJob(job.id)
|
||||||
} catch (saveError) {
|
} catch (saveError) {
|
||||||
|
removeExtendTemp(job.library?.extendTmpDir)
|
||||||
const message = saveError instanceof Error ? saveError.message : String(saveError)
|
const message = saveError instanceof Error ? saveError.message : String(saveError)
|
||||||
job.status = 'error'
|
job.status = 'error'
|
||||||
job.error = `Video generated but library save failed: ${message}`
|
job.error = `Video generated but library save failed: ${message}`
|
||||||
|
|||||||
Reference in New Issue
Block a user