Keep queueing video extensions while a clip is already generating.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+44
-30
@@ -261,7 +261,6 @@
|
|||||||
:scripts="directorScripts"
|
:scripts="directorScripts"
|
||||||
v-model:selected-id="directorScriptId"
|
v-model:selected-id="directorScriptId"
|
||||||
v-model:name="directorScriptName"
|
v-model:name="directorScriptName"
|
||||||
:disabled="videoBusy"
|
|
||||||
@load="loadDirectorScript"
|
@load="loadDirectorScript"
|
||||||
@save="saveDirectorScript"
|
@save="saveDirectorScript"
|
||||||
@remove="deleteDirectorScript"
|
@remove="deleteDirectorScript"
|
||||||
@@ -402,7 +401,6 @@
|
|||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="text-xs text-red-300 hover:text-red-200"
|
class="text-xs text-red-300 hover:text-red-200"
|
||||||
:disabled="editBusy"
|
|
||||||
@click="removeEditPass(item.id)"
|
@click="removeEditPass(item.id)"
|
||||||
>
|
>
|
||||||
Remove
|
Remove
|
||||||
@@ -415,13 +413,11 @@
|
|||||||
:placeholder="extraPassPlaceholder"
|
:placeholder="extraPassPlaceholder"
|
||||||
:keep-items="activeKeepPrompts"
|
:keep-items="activeKeepPrompts"
|
||||||
:keep-heading="keepPromptHeading"
|
:keep-heading="keepPromptHeading"
|
||||||
:disabled="editBusy"
|
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="w-full rounded-2xl border border-dashed border-white/15 px-4 py-2.5 text-sm text-zinc-300 transition hover:border-amber-300/40 hover:text-amber-100 disabled:cursor-not-allowed disabled:opacity-50"
|
class="w-full rounded-2xl border border-dashed border-white/15 px-4 py-2.5 text-sm text-zinc-300 transition hover:border-amber-300/40 hover:text-amber-100 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
:disabled="editBusy"
|
|
||||||
@click="queueEditPass"
|
@click="queueEditPass"
|
||||||
>
|
>
|
||||||
{{ extraPassButtonLabel }}
|
{{ extraPassButtonLabel }}
|
||||||
@@ -521,7 +517,6 @@
|
|||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="text-xs text-red-300 hover:text-red-200"
|
class="text-xs text-red-300 hover:text-red-200"
|
||||||
:disabled="videoBusy"
|
|
||||||
@click="removeQueuedExtension(item.id)"
|
@click="removeQueuedExtension(item.id)"
|
||||||
>
|
>
|
||||||
Remove
|
Remove
|
||||||
@@ -569,7 +564,6 @@
|
|||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="w-full rounded-2xl border border-dashed border-white/15 px-4 py-2.5 text-sm text-zinc-300 transition hover:border-amber-300/40 hover:text-amber-100 disabled:cursor-not-allowed disabled:opacity-50"
|
class="w-full rounded-2xl border border-dashed border-white/15 px-4 py-2.5 text-sm text-zinc-300 transition hover:border-amber-300/40 hover:text-amber-100 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
:disabled="videoBusy"
|
|
||||||
@click="queueExtension"
|
@click="queueExtension"
|
||||||
>
|
>
|
||||||
+ Queue Extension
|
+ Queue Extension
|
||||||
@@ -1443,11 +1437,12 @@
|
|||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="w-full rounded-2xl bg-amber-400 px-4 py-2.5 text-sm font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50"
|
class="w-full rounded-2xl bg-amber-400 px-4 py-2.5 text-sm font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
:disabled="!extendPrompt.trim() || lastFrameLoading || videoBusy"
|
:disabled="!extendPrompt.trim()"
|
||||||
@click="generateExtension"
|
@click="generateExtension"
|
||||||
>
|
>
|
||||||
Generate Extension
|
{{ videoBusy ? 'Queue extension' : 'Generate Extension' }}
|
||||||
</button>
|
</button>
|
||||||
|
<p v-if="videoBusy" class="text-xs text-zinc-500">The current job keeps running in Output. This extension waits in the job queue.</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -5744,27 +5739,19 @@ async function openExtend(item?: LibraryClip) {
|
|||||||
async function generateExtension() {
|
async function generateExtension() {
|
||||||
const id = clipIdFromPlayer()
|
const id = clipIdFromPlayer()
|
||||||
if (!id || !extendPrompt.value.trim()) return
|
if (!id || !extendPrompt.value.trim()) return
|
||||||
const switching = videoBusy.value
|
const hideOut = hideThumbnail.value
|
||||||
statusBusy.value = false
|
|
||||||
videoBusy.value = true
|
|
||||||
watchingLibrary.value = false
|
|
||||||
stopListen('video')
|
|
||||||
videoSettledUi = false
|
|
||||||
progress.value = 1
|
|
||||||
awaitingReveal.value = false
|
|
||||||
concealOutput.value = hideThumbnail.value || currentClip.value?.hideThumbnail === true
|
|
||||||
lockedSave.value = false
|
|
||||||
chainStep.value = 1
|
|
||||||
chainTotal.value = 1
|
|
||||||
chainLabel.value = ''
|
|
||||||
overallProgress.value = 0
|
|
||||||
completedChainStep.value = 0
|
|
||||||
activeChainPlan.value = []
|
|
||||||
statusMessage.value = 'Extracting last frame & stitching extension...'
|
|
||||||
startTimer('video')
|
|
||||||
try {
|
try {
|
||||||
await ensureGenerationNotifyPermission()
|
await ensureGenerationNotifyPermission()
|
||||||
const started = await $fetch<{ jobId: string; steps: number; hideThumbnail?: boolean; folderLocked?: boolean }>('/api/extend', {
|
const started = await $fetch<{
|
||||||
|
jobId: string
|
||||||
|
studioJobId?: string
|
||||||
|
queued?: boolean
|
||||||
|
steps: number
|
||||||
|
hideThumbnail?: boolean
|
||||||
|
folderLocked?: boolean
|
||||||
|
width?: number
|
||||||
|
height?: number
|
||||||
|
}>('/api/extend', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
body: {
|
body: {
|
||||||
clipId: id,
|
clipId: id,
|
||||||
@@ -5776,13 +5763,40 @@ async function generateExtension() {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
closeExtend()
|
closeExtend()
|
||||||
if (switching) toast('Switched to this extension. The other job will keep saving to the library.')
|
if (started.queued) {
|
||||||
videoJobId.value = started.jobId
|
outputStudioJobId.value = started.studioJobId || ''
|
||||||
maxStep.value = started.steps
|
outputFocus.value = 'video'
|
||||||
|
videoAwaitingBeast.value = true
|
||||||
|
statusMessage.value = 'Queued. Waiting for Beast to be free.'
|
||||||
|
toast('Queued behind the current job.')
|
||||||
|
await refreshStudioQueue()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
statusBusy.value = false
|
||||||
|
videoBusy.value = true
|
||||||
|
watchingLibrary.value = false
|
||||||
|
stopListen('video')
|
||||||
videoSettledUi = false
|
videoSettledUi = false
|
||||||
|
progress.value = 1
|
||||||
|
awaitingReveal.value = false
|
||||||
|
concealOutput.value = started.hideThumbnail === true || hideOut
|
||||||
|
lockedSave.value = false
|
||||||
|
chainStep.value = 1
|
||||||
|
chainTotal.value = 1
|
||||||
|
chainLabel.value = ''
|
||||||
|
overallProgress.value = 0
|
||||||
|
completedChainStep.value = 0
|
||||||
|
activeChainPlan.value = []
|
||||||
|
statusMessage.value = 'Extracting last frame & stitching extension...'
|
||||||
|
startTimer('video')
|
||||||
|
if (started.width && started.height) playerSize.value = { width: started.width, height: started.height }
|
||||||
|
videoJobId.value = started.jobId
|
||||||
|
outputStudioJobId.value = started.studioJobId || ''
|
||||||
|
maxStep.value = started.steps
|
||||||
persistActiveJob('video', started.jobId, started.hideThumbnail === true || concealOutput.value, started.folderLocked === true)
|
persistActiveJob('video', started.jobId, started.hideThumbnail === true || concealOutput.value, started.folderLocked === true)
|
||||||
listen('video', started.jobId, started.hideThumbnail === true || concealOutput.value, started.folderLocked === true)
|
listen('video', started.jobId, started.hideThumbnail === true || concealOutput.value, started.folderLocked === true)
|
||||||
void pollComfyHealth()
|
void pollComfyHealth()
|
||||||
|
void refreshStudioQueue()
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
videoBusy.value = false
|
videoBusy.value = false
|
||||||
statusBusy.value = false
|
statusBusy.value = false
|
||||||
|
|||||||
+54
-238
@@ -1,44 +1,10 @@
|
|||||||
import { copyFileSync, existsSync, readFileSync } from 'node:fs'
|
import { existsSync } from 'node:fs'
|
||||||
import { join } from 'node:path'
|
import { addStudioJob, kickStudioQueue, listStudioJobs, videoJobsBusy } from '~/server/utils/studioQueue'
|
||||||
import { composeShotPrompt } from '~/utils/globalLocks'
|
import { persistLoraFields, parsePostedLoraStack, listStudioLoras } from '~/server/utils/loras'
|
||||||
import { composePromptParts } from '~/utils/promptParts'
|
|
||||||
import { parsePostedLoraStack, listStudioLoras, persistLoraFields, ensureComfyLoraNames } from '~/server/utils/loras'
|
|
||||||
import { readLoraStack } from '~/utils/loras'
|
import { readLoraStack } from '~/utils/loras'
|
||||||
import { clampVideoCfg } from '~/utils/generationPresets'
|
import { parseExtendDuration } from '~/server/utils/extendChain'
|
||||||
import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
|
import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
|
||||||
|
|
||||||
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
|
|
||||||
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
|
|
||||||
|
|
||||||
function parseExtendDuration(raw: unknown) {
|
|
||||||
const seconds = Number(raw)
|
|
||||||
if (!Number.isFinite(seconds)) return 5
|
|
||||||
return Math.min(120, Math.max(0.5, Math.round(seconds * 10) / 10))
|
|
||||||
}
|
|
||||||
|
|
||||||
function frameLength(seconds: number, fps: number) {
|
|
||||||
return Math.max(5, Math.floor(seconds * fps))
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseFps(raw: unknown) {
|
|
||||||
const fps = Number(raw)
|
|
||||||
return fps === 12 || fps === 30 || fps === 24 ? fps : 24
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseCfg(raw: unknown, turbo: boolean) {
|
|
||||||
return clampVideoCfg(raw, turbo ? 1.5 : 4)
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseSampler(raw: unknown) {
|
|
||||||
const value = String(raw || '')
|
|
||||||
return SAMPLERS.has(value) ? value : 'res_multistep'
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseScheduler(raw: unknown) {
|
|
||||||
const value = String(raw || '')
|
|
||||||
return SCHEDULERS.has(value) ? value : 'simple'
|
|
||||||
}
|
|
||||||
|
|
||||||
export default defineEventHandler(async (event) => {
|
export default defineEventHandler(async (event) => {
|
||||||
const body = await readBody<{ clipId?: string; prompt?: string; promptPre?: string; promptPost?: string; duration?: number; lora?: unknown; loraStack?: unknown }>(event).catch(() => ({}))
|
const body = await readBody<{ clipId?: string; prompt?: string; promptPre?: string; promptPost?: string; duration?: number; lora?: unknown; loraStack?: unknown }>(event).catch(() => ({}))
|
||||||
const clipId = String(body?.clipId || '').trim()
|
const clipId = String(body?.clipId || '').trim()
|
||||||
@@ -63,222 +29,72 @@ export default defineEventHandler(async (event) => {
|
|||||||
const durationSeconds = parseExtendDuration(body?.duration)
|
const durationSeconds = parseExtendDuration(body?.duration)
|
||||||
const promptPre = String(body?.promptPre ?? source.promptPre ?? '').trim()
|
const promptPre = String(body?.promptPre ?? source.promptPre ?? '').trim()
|
||||||
const promptPost = String(body?.promptPost ?? source.promptPost ?? '').trim()
|
const promptPost = String(body?.promptPost ?? source.promptPost ?? '').trim()
|
||||||
const composedPrompt = composeShotPrompt({
|
|
||||||
globalLocks: source.globalLocks,
|
|
||||||
prompt: composePromptParts(promptPre, prompt, promptPost),
|
|
||||||
shotIndex: (source.chainIndex || 0) + 1,
|
|
||||||
familyRefs: source.permanenceRefs
|
|
||||||
})
|
|
||||||
const workflow = parseVideoWorkflow(source.workflow)
|
const workflow = parseVideoWorkflow(source.workflow)
|
||||||
if (isLtxWorkflow(workflow) && !ltxWorkflowEnabled()) {
|
if (isLtxWorkflow(workflow) && !ltxWorkflowEnabled()) {
|
||||||
throw createError({ statusCode: 400, statusMessage: LTX_DISABLED_MESSAGE })
|
throw createError({ statusCode: 400, statusMessage: LTX_DISABLED_MESSAGE })
|
||||||
}
|
}
|
||||||
const turbo = source.turbo !== false
|
|
||||||
const fps = parseFps(source.fps)
|
|
||||||
const cfg = parseCfg(source.cfg, turbo)
|
|
||||||
const samplerName = parseSampler(source.samplerName)
|
|
||||||
const scheduler = parseScheduler(source.scheduler)
|
|
||||||
const steps = Number(source.steps || defaultVideoSteps(turbo, workflow))
|
|
||||||
const seed = Math.floor(Math.random() * 2_147_483_647)
|
|
||||||
await listStudioLoras()
|
await listStudioLoras()
|
||||||
const postedStack = parsePostedLoraStack(body?.loraStack ?? body?.lora, 'video')
|
const postedStack = parsePostedLoraStack(body?.loraStack ?? body?.lora, 'video')
|
||||||
const loraStack = postedStack.length ? postedStack : parsePostedLoraStack(readLoraStack(source), 'video')
|
const loraStack = postedStack.length ? postedStack : parsePostedLoraStack(readLoraStack(source), 'video')
|
||||||
const loraFields = persistLoraFields(loraStack)
|
const familyId = source.familyId || crypto.randomUUID()
|
||||||
const length = frameLength(durationSeconds, fps)
|
const turbo = source.turbo !== false
|
||||||
const width = source.width || 1344
|
const steps = Number(source.steps || defaultVideoSteps(turbo, workflow))
|
||||||
const height = source.height || 768
|
const seed = Math.floor(Math.random() * 2_147_483_647)
|
||||||
const clipName = nextClipPartName(clipTitle(source))
|
|
||||||
|
|
||||||
const job = createJob()
|
const studio = await addStudioJob({
|
||||||
job.kind = 'video'
|
|
||||||
job.maxStep = steps
|
|
||||||
job.hideThumbnail = source.hideThumbnail
|
|
||||||
const tmpDir = extendTempDir(owner, job.id)
|
|
||||||
const part1Path = join(tmpDir, 'part1.mp4')
|
|
||||||
const framePath = join(tmpDir, 'last_frame.png')
|
|
||||||
job.library = {
|
|
||||||
ownerKey: owner,
|
ownerKey: owner,
|
||||||
folderId: source.folderId,
|
familyId,
|
||||||
hideThumbnail: source.hideThumbnail,
|
payload: {
|
||||||
folderLocked,
|
prompt,
|
||||||
name: clipName,
|
promptMid: prompt,
|
||||||
prompt,
|
promptPre: promptPre || undefined,
|
||||||
promptMid: prompt,
|
promptPost: promptPost || undefined,
|
||||||
promptPre: promptPre || undefined,
|
name: nextClipPartName(clipTitle(source)),
|
||||||
promptPost: promptPost || undefined,
|
folderId: source.folderId,
|
||||||
aspect: source.aspect || '16:9',
|
aspect: source.aspect || '16:9',
|
||||||
width,
|
width: source.width || 1344,
|
||||||
height,
|
height: source.height || 768,
|
||||||
steps,
|
steps,
|
||||||
turbo,
|
turbo,
|
||||||
seed,
|
seed,
|
||||||
cfg,
|
cfg: Number(source.cfg || 0),
|
||||||
fps,
|
fps: Number(source.fps || 24),
|
||||||
samplerName,
|
samplerName: source.samplerName || 'res_multistep',
|
||||||
scheduler,
|
scheduler: source.scheduler || 'simple',
|
||||||
duration: durationSeconds,
|
duration: durationSeconds,
|
||||||
extendTmpDir: tmpDir,
|
sound: source.sound !== false,
|
||||||
extendPart1Path: part1Path,
|
workflow,
|
||||||
extendSourceClipId: source.id,
|
useIdentityRefs: false,
|
||||||
familyId: source.familyId || crypto.randomUUID(),
|
hideThumbnail: source.hideThumbnail,
|
||||||
parentClipId: source.id,
|
folderLocked,
|
||||||
chainIndex: (source.chainIndex || 0) + 1,
|
referenceStillIds: [null, null, null, null],
|
||||||
workflow,
|
extensions: [],
|
||||||
globalLocks: source.globalLocks,
|
queueAutoRun: false,
|
||||||
permanenceRefs: source.permanenceRefs,
|
extendFromClipId: clipId,
|
||||||
...loraFields
|
globalLocks: source.globalLocks,
|
||||||
}
|
permanenceRefs: source.permanenceRefs,
|
||||||
|
...persistLoraFields(loraStack)
|
||||||
emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 })
|
}
|
||||||
|
|
||||||
void runExtend(job, {
|
|
||||||
sourcePath,
|
|
||||||
part1Path,
|
|
||||||
framePath,
|
|
||||||
prompt: composedPrompt,
|
|
||||||
width,
|
|
||||||
height,
|
|
||||||
steps,
|
|
||||||
seed,
|
|
||||||
turbo,
|
|
||||||
length,
|
|
||||||
cfg,
|
|
||||||
fps,
|
|
||||||
samplerName,
|
|
||||||
scheduler,
|
|
||||||
...loraFields
|
|
||||||
}).catch((error) => {
|
|
||||||
removeExtendTemp(job.library?.extendTmpDir)
|
|
||||||
const message = error instanceof Error ? error.message : String(error)
|
|
||||||
job.status = 'error'
|
|
||||||
job.error = message
|
|
||||||
emitJob(job, { type: 'error', error: message, message })
|
|
||||||
})
|
})
|
||||||
|
await kickStudioQueue()
|
||||||
|
let latest = listStudioJobs(owner).find(item => item.id === studio.id)
|
||||||
|
if (!latest?.liveJobId && !(await videoJobsBusy())) {
|
||||||
|
await kickStudioQueue()
|
||||||
|
latest = listStudioJobs(owner).find(item => item.id === studio.id)
|
||||||
|
}
|
||||||
|
const liveJobId = latest?.liveJobId || ''
|
||||||
|
|
||||||
return {
|
return {
|
||||||
jobId: job.id,
|
jobId: liveJobId || studio.id,
|
||||||
clientId: job.clientId,
|
studioJobId: studio.id,
|
||||||
|
queued: !liveJobId,
|
||||||
|
clientId: '',
|
||||||
seed,
|
seed,
|
||||||
width,
|
width: source.width || 1344,
|
||||||
height,
|
height: source.height || 768,
|
||||||
steps,
|
steps,
|
||||||
duration: durationSeconds,
|
duration: durationSeconds,
|
||||||
hideThumbnail: source.hideThumbnail,
|
hideThumbnail: source.hideThumbnail,
|
||||||
folderLocked
|
folderLocked
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
async function runExtend(
|
|
||||||
job: ReturnType<typeof createJob>,
|
|
||||||
params: {
|
|
||||||
sourcePath: string
|
|
||||||
part1Path: string
|
|
||||||
framePath: string
|
|
||||||
prompt: string
|
|
||||||
width: number
|
|
||||||
height: number
|
|
||||||
steps: number
|
|
||||||
seed: number
|
|
||||||
turbo: boolean
|
|
||||||
length: number
|
|
||||||
cfg: number
|
|
||||||
fps: number
|
|
||||||
samplerName: string
|
|
||||||
scheduler: string
|
|
||||||
loraName?: string
|
|
||||||
loraStack?: import('~/utils/loras').LoraStackItem[]
|
|
||||||
}
|
|
||||||
) {
|
|
||||||
copyFileSync(params.sourcePath, params.part1Path)
|
|
||||||
await extractLastFrame(params.sourcePath, params.framePath)
|
|
||||||
const frame = readFileSync(params.framePath)
|
|
||||||
if (!frame.length || frame.length < 64) {
|
|
||||||
throw new Error('Could not extract the last frame of the clip being extended')
|
|
||||||
}
|
|
||||||
if (job.library) job.library.thumb = undefined
|
|
||||||
const sound = await probeHasAudio(params.part1Path)
|
|
||||||
|
|
||||||
await ensureComfyReady((status) => {
|
|
||||||
emitJob(job, {
|
|
||||||
type: status.state === 'busy' ? 'busy' : 'status',
|
|
||||||
message: status.message,
|
|
||||||
progress: status.state === 'online' ? Math.max(job.progress, 3) : Math.max(job.progress, 2),
|
|
||||||
busy: status.state === 'busy',
|
|
||||||
queueRunning: status.queueRunning,
|
|
||||||
queuePending: status.queuePending
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
const done = watchComfyJob(job)
|
|
||||||
job.status = 'uploading'
|
|
||||||
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 4 })
|
|
||||||
const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
|
|
||||||
if (job.library) {
|
|
||||||
job.library.imageName = uploaded.name
|
|
||||||
job.library.imageSubfolder = uploaded.subfolder
|
|
||||||
}
|
|
||||||
emitJob(job, { type: 'status', message: 'Queueing extension on MiniMax H3...', progress: 6 })
|
|
||||||
await waitForComfySocket(job, 4000)
|
|
||||||
await ensureComfyLoraNames('video')
|
|
||||||
|
|
||||||
const graph = buildWorkflow({
|
|
||||||
prompt: params.prompt,
|
|
||||||
imageName: uploaded.name,
|
|
||||||
width: params.width,
|
|
||||||
height: params.height,
|
|
||||||
steps: params.steps,
|
|
||||||
seed: params.seed,
|
|
||||||
turbo: params.turbo,
|
|
||||||
length: params.length,
|
|
||||||
cfg: params.cfg,
|
|
||||||
fps: params.fps,
|
|
||||||
samplerName: params.samplerName,
|
|
||||||
scheduler: params.scheduler,
|
|
||||||
filenamePrefix: comfyFilenamePrefix(),
|
|
||||||
sound,
|
|
||||||
workflow: job.library?.workflow,
|
|
||||||
duration: job.library?.duration,
|
|
||||||
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName
|
|
||||||
})
|
|
||||||
|
|
||||||
const queued = await queuePrompt(graph, job.clientId)
|
|
||||||
job.promptId = queued.prompt_id
|
|
||||||
job.status = 'running'
|
|
||||||
if (job.library && job.promptId) {
|
|
||||||
writePendingJob({
|
|
||||||
jobId: job.id,
|
|
||||||
promptId: job.promptId,
|
|
||||||
clientId: job.clientId,
|
|
||||||
ownerKey: job.library.ownerKey,
|
|
||||||
folderId: job.library.folderId,
|
|
||||||
hideThumbnail: job.library.hideThumbnail,
|
|
||||||
folderLocked: job.library.folderLocked,
|
|
||||||
name: job.library.name,
|
|
||||||
prompt: job.library.prompt,
|
|
||||||
aspect: job.library.aspect,
|
|
||||||
width: job.library.width,
|
|
||||||
height: job.library.height,
|
|
||||||
steps: job.library.steps,
|
|
||||||
turbo: job.library.turbo,
|
|
||||||
seed: job.library.seed,
|
|
||||||
startedAt: job.startedAt,
|
|
||||||
imageName: job.library.imageName,
|
|
||||||
imageSubfolder: job.library.imageSubfolder,
|
|
||||||
extendTmpDir: job.library.extendTmpDir,
|
|
||||||
extendPart1Path: job.library.extendPart1Path,
|
|
||||||
familyId: job.library.familyId,
|
|
||||||
parentClipId: job.library.parentClipId,
|
|
||||||
chainIndex: job.library.chainIndex,
|
|
||||||
stillId: job.library.stillId,
|
|
||||||
sound: job.library.sound,
|
|
||||||
workflow: job.library.workflow,
|
|
||||||
duration: job.library.duration,
|
|
||||||
globalLocks: job.library.globalLocks,
|
|
||||||
permanenceRefs: job.library.permanenceRefs,
|
|
||||||
loraName: job.library.loraName,
|
|
||||||
loraStack: job.library.loraStack
|
|
||||||
})
|
|
||||||
}
|
|
||||||
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
|
|
||||||
await done
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,287 @@
|
|||||||
|
import { copyFileSync, existsSync, readFileSync } from 'node:fs'
|
||||||
|
import { join } from 'node:path'
|
||||||
|
import { composeShotPrompt } from '~/utils/globalLocks'
|
||||||
|
import { composePromptParts } from '~/utils/promptParts'
|
||||||
|
import { parsePostedLoraStack, listStudioLoras, persistLoraFields, ensureComfyLoraNames } from '~/server/utils/loras'
|
||||||
|
import { readLoraStack } from '~/utils/loras'
|
||||||
|
import { clampVideoCfg } from '~/utils/generationPresets'
|
||||||
|
import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
|
||||||
|
import { createJob, emitJob, type Job } from '~/server/utils/jobs'
|
||||||
|
import {
|
||||||
|
clipTitle,
|
||||||
|
clipVideoPath,
|
||||||
|
extendTempDir,
|
||||||
|
getClip,
|
||||||
|
nextClipPartName,
|
||||||
|
removeExtendTemp
|
||||||
|
} from '~/server/utils/library'
|
||||||
|
import { extractLastFrame, probeHasAudio } from '~/server/utils/ffmpeg'
|
||||||
|
import { ensureComfyReady, waitForComfySocket, watchComfyJob } from '~/server/utils/comfyLifecycle'
|
||||||
|
import { comfyFilenamePrefix, queuePrompt, uploadImage } from '~/server/utils/comfy'
|
||||||
|
import { buildWorkflow } from '~/server/utils/workflow'
|
||||||
|
import { writePendingJob } from '~/server/utils/pending'
|
||||||
|
|
||||||
|
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
|
||||||
|
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
|
||||||
|
|
||||||
|
export function parseExtendDuration(raw: unknown) {
|
||||||
|
const seconds = Number(raw)
|
||||||
|
if (!Number.isFinite(seconds)) return 5
|
||||||
|
return Math.min(120, Math.max(0.5, Math.round(seconds * 10) / 10))
|
||||||
|
}
|
||||||
|
|
||||||
|
function frameLength(seconds: number, fps: number) {
|
||||||
|
return Math.max(5, Math.floor(seconds * fps))
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseFps(raw: unknown) {
|
||||||
|
const fps = Number(raw)
|
||||||
|
return fps === 12 || fps === 30 || fps === 24 ? fps : 24
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseCfg(raw: unknown, turbo: boolean) {
|
||||||
|
return clampVideoCfg(raw, turbo ? 1.5 : 4)
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseSampler(raw: unknown) {
|
||||||
|
const value = String(raw || '')
|
||||||
|
return SAMPLERS.has(value) ? value : 'res_multistep'
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseScheduler(raw: unknown) {
|
||||||
|
const value = String(raw || '')
|
||||||
|
return SCHEDULERS.has(value) ? value : 'simple'
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function beginExtendFromClip(params: {
|
||||||
|
ownerKey: string
|
||||||
|
clipId: string
|
||||||
|
prompt: string
|
||||||
|
promptPre?: string
|
||||||
|
promptPost?: string
|
||||||
|
duration?: number
|
||||||
|
loraStack?: unknown
|
||||||
|
folderLocked?: boolean
|
||||||
|
}) {
|
||||||
|
const source = getClip(params.ownerKey, params.clipId)
|
||||||
|
const sourcePath = clipVideoPath(params.ownerKey, source.id)
|
||||||
|
if (!existsSync(sourcePath)) {
|
||||||
|
throw createError({ statusCode: 404, statusMessage: 'Source video is missing' })
|
||||||
|
}
|
||||||
|
const prompt = String(params.prompt || '').trim()
|
||||||
|
if (!prompt) {
|
||||||
|
throw createError({ statusCode: 400, statusMessage: 'An extension prompt is required' })
|
||||||
|
}
|
||||||
|
|
||||||
|
const durationSeconds = parseExtendDuration(params.duration)
|
||||||
|
const promptPre = String(params.promptPre ?? source.promptPre ?? '').trim()
|
||||||
|
const promptPost = String(params.promptPost ?? source.promptPost ?? '').trim()
|
||||||
|
const composedPrompt = composeShotPrompt({
|
||||||
|
globalLocks: source.globalLocks,
|
||||||
|
prompt: composePromptParts(promptPre, prompt, promptPost),
|
||||||
|
shotIndex: (source.chainIndex || 0) + 1,
|
||||||
|
familyRefs: source.permanenceRefs
|
||||||
|
})
|
||||||
|
const workflow = parseVideoWorkflow(source.workflow)
|
||||||
|
if (isLtxWorkflow(workflow) && !ltxWorkflowEnabled()) {
|
||||||
|
throw createError({ statusCode: 400, statusMessage: LTX_DISABLED_MESSAGE })
|
||||||
|
}
|
||||||
|
const turbo = source.turbo !== false
|
||||||
|
const fps = parseFps(source.fps)
|
||||||
|
const cfg = parseCfg(source.cfg, turbo)
|
||||||
|
const samplerName = parseSampler(source.samplerName)
|
||||||
|
const scheduler = parseScheduler(source.scheduler)
|
||||||
|
const steps = Number(source.steps || defaultVideoSteps(turbo, workflow))
|
||||||
|
const seed = Math.floor(Math.random() * 2_147_483_647)
|
||||||
|
await listStudioLoras()
|
||||||
|
const postedStack = parsePostedLoraStack(params.loraStack, 'video')
|
||||||
|
const loraStack = postedStack.length ? postedStack : parsePostedLoraStack(readLoraStack(source), 'video')
|
||||||
|
const loraFields = persistLoraFields(loraStack)
|
||||||
|
const length = frameLength(durationSeconds, fps)
|
||||||
|
const width = source.width || 1344
|
||||||
|
const height = source.height || 768
|
||||||
|
const clipName = nextClipPartName(clipTitle(source))
|
||||||
|
|
||||||
|
const job = createJob()
|
||||||
|
job.kind = 'video'
|
||||||
|
job.maxStep = steps
|
||||||
|
job.hideThumbnail = source.hideThumbnail
|
||||||
|
const tmpDir = extendTempDir(params.ownerKey, job.id)
|
||||||
|
const part1Path = join(tmpDir, 'part1.mp4')
|
||||||
|
const framePath = join(tmpDir, 'last_frame.png')
|
||||||
|
job.library = {
|
||||||
|
ownerKey: params.ownerKey,
|
||||||
|
folderId: source.folderId,
|
||||||
|
hideThumbnail: source.hideThumbnail,
|
||||||
|
folderLocked: params.folderLocked,
|
||||||
|
name: clipName,
|
||||||
|
prompt,
|
||||||
|
promptMid: prompt,
|
||||||
|
promptPre: promptPre || undefined,
|
||||||
|
promptPost: promptPost || undefined,
|
||||||
|
aspect: source.aspect || '16:9',
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
steps,
|
||||||
|
turbo,
|
||||||
|
seed,
|
||||||
|
cfg,
|
||||||
|
fps,
|
||||||
|
samplerName,
|
||||||
|
scheduler,
|
||||||
|
duration: durationSeconds,
|
||||||
|
extendTmpDir: tmpDir,
|
||||||
|
extendPart1Path: part1Path,
|
||||||
|
extendSourceClipId: source.id,
|
||||||
|
familyId: source.familyId || crypto.randomUUID(),
|
||||||
|
parentClipId: source.id,
|
||||||
|
chainIndex: (source.chainIndex || 0) + 1,
|
||||||
|
workflow,
|
||||||
|
globalLocks: source.globalLocks,
|
||||||
|
permanenceRefs: source.permanenceRefs,
|
||||||
|
...loraFields
|
||||||
|
}
|
||||||
|
|
||||||
|
emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 })
|
||||||
|
|
||||||
|
void runExtend(job, {
|
||||||
|
sourcePath,
|
||||||
|
part1Path,
|
||||||
|
framePath,
|
||||||
|
prompt: composedPrompt,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
steps,
|
||||||
|
seed,
|
||||||
|
turbo,
|
||||||
|
length,
|
||||||
|
cfg,
|
||||||
|
fps,
|
||||||
|
samplerName,
|
||||||
|
scheduler,
|
||||||
|
...loraFields
|
||||||
|
}).catch((error) => {
|
||||||
|
removeExtendTemp(job.library?.extendTmpDir)
|
||||||
|
const message = error instanceof Error ? error.message : String(error)
|
||||||
|
job.status = 'error'
|
||||||
|
job.error = message
|
||||||
|
emitJob(job, { type: 'error', error: message, message })
|
||||||
|
})
|
||||||
|
|
||||||
|
return job
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runExtend(
|
||||||
|
job: Job,
|
||||||
|
params: {
|
||||||
|
sourcePath: string
|
||||||
|
part1Path: string
|
||||||
|
framePath: string
|
||||||
|
prompt: string
|
||||||
|
width: number
|
||||||
|
height: number
|
||||||
|
steps: number
|
||||||
|
seed: number
|
||||||
|
turbo: boolean
|
||||||
|
length: number
|
||||||
|
cfg: number
|
||||||
|
fps: number
|
||||||
|
samplerName: string
|
||||||
|
scheduler: string
|
||||||
|
loraName?: string
|
||||||
|
loraStack?: import('~/utils/loras').LoraStackItem[]
|
||||||
|
}
|
||||||
|
) {
|
||||||
|
copyFileSync(params.sourcePath, params.part1Path)
|
||||||
|
await extractLastFrame(params.sourcePath, params.framePath)
|
||||||
|
const frame = readFileSync(params.framePath)
|
||||||
|
if (!frame.length || frame.length < 64) {
|
||||||
|
throw new Error('Could not extract the last frame of the clip being extended')
|
||||||
|
}
|
||||||
|
if (job.library) job.library.thumb = undefined
|
||||||
|
const sound = await probeHasAudio(params.part1Path)
|
||||||
|
|
||||||
|
await ensureComfyReady((status) => {
|
||||||
|
emitJob(job, {
|
||||||
|
type: status.state === 'busy' ? 'busy' : 'status',
|
||||||
|
message: status.message,
|
||||||
|
progress: status.state === 'online' ? Math.max(job.progress, 3) : Math.max(job.progress, 2),
|
||||||
|
busy: status.state === 'busy',
|
||||||
|
queueRunning: status.queueRunning,
|
||||||
|
queuePending: status.queuePending
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
const done = watchComfyJob(job)
|
||||||
|
job.status = 'uploading'
|
||||||
|
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 4 })
|
||||||
|
const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
|
||||||
|
if (job.library) {
|
||||||
|
job.library.imageName = uploaded.name
|
||||||
|
job.library.imageSubfolder = uploaded.subfolder
|
||||||
|
}
|
||||||
|
emitJob(job, { type: 'status', message: 'Queueing extension on MiniMax H3...', progress: 6 })
|
||||||
|
await waitForComfySocket(job, 4000)
|
||||||
|
await ensureComfyLoraNames('video')
|
||||||
|
|
||||||
|
const graph = buildWorkflow({
|
||||||
|
prompt: params.prompt,
|
||||||
|
imageName: uploaded.name,
|
||||||
|
width: params.width,
|
||||||
|
height: params.height,
|
||||||
|
steps: params.steps,
|
||||||
|
seed: params.seed,
|
||||||
|
turbo: params.turbo,
|
||||||
|
length: params.length,
|
||||||
|
cfg: params.cfg,
|
||||||
|
fps: params.fps,
|
||||||
|
samplerName: params.samplerName,
|
||||||
|
scheduler: params.scheduler,
|
||||||
|
filenamePrefix: comfyFilenamePrefix(),
|
||||||
|
sound,
|
||||||
|
workflow: job.library?.workflow,
|
||||||
|
duration: job.library?.duration,
|
||||||
|
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName
|
||||||
|
})
|
||||||
|
|
||||||
|
const queued = await queuePrompt(graph, job.clientId)
|
||||||
|
job.promptId = queued.prompt_id
|
||||||
|
job.status = 'running'
|
||||||
|
if (job.library && job.promptId) {
|
||||||
|
writePendingJob({
|
||||||
|
jobId: job.id,
|
||||||
|
promptId: job.promptId,
|
||||||
|
clientId: job.clientId,
|
||||||
|
ownerKey: job.library.ownerKey,
|
||||||
|
folderId: job.library.folderId,
|
||||||
|
hideThumbnail: job.library.hideThumbnail,
|
||||||
|
folderLocked: job.library.folderLocked,
|
||||||
|
name: job.library.name,
|
||||||
|
prompt: job.library.prompt,
|
||||||
|
aspect: job.library.aspect,
|
||||||
|
width: job.library.width,
|
||||||
|
height: job.library.height,
|
||||||
|
steps: job.library.steps,
|
||||||
|
turbo: job.library.turbo,
|
||||||
|
seed: job.library.seed,
|
||||||
|
startedAt: job.startedAt,
|
||||||
|
imageName: job.library.imageName,
|
||||||
|
imageSubfolder: job.library.imageSubfolder,
|
||||||
|
extendTmpDir: job.library.extendTmpDir,
|
||||||
|
extendPart1Path: job.library.extendPart1Path,
|
||||||
|
familyId: job.library.familyId,
|
||||||
|
parentClipId: job.library.parentClipId,
|
||||||
|
chainIndex: job.library.chainIndex,
|
||||||
|
stillId: job.library.stillId,
|
||||||
|
sound: job.library.sound,
|
||||||
|
workflow: job.library.workflow,
|
||||||
|
duration: job.library.duration,
|
||||||
|
globalLocks: job.library.globalLocks,
|
||||||
|
permanenceRefs: job.library.permanenceRefs,
|
||||||
|
loraName: job.library.loraName,
|
||||||
|
loraStack: job.library.loraStack
|
||||||
|
})
|
||||||
|
}
|
||||||
|
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
|
||||||
|
await done
|
||||||
|
}
|
||||||
@@ -66,6 +66,7 @@ export interface StudioJobPayload {
|
|||||||
maskStillId?: string
|
maskStillId?: string
|
||||||
maskStillFilename?: string
|
maskStillFilename?: string
|
||||||
refineStrength?: number
|
refineStrength?: number
|
||||||
|
extendFromClipId?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StudioJob {
|
export interface StudioJob {
|
||||||
@@ -897,11 +898,44 @@ async function startStudioEditJob(item: StudioJob) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function startStudioExtendJob(item: StudioJob) {
|
||||||
|
let live: Job | undefined
|
||||||
|
try {
|
||||||
|
const { beginExtendFromClip } = await import('~/server/utils/extendChain')
|
||||||
|
live = await beginExtendFromClip({
|
||||||
|
ownerKey: item.ownerKey,
|
||||||
|
clipId: String(item.payload.extendFromClipId || ''),
|
||||||
|
prompt: item.payload.prompt,
|
||||||
|
promptPre: item.payload.promptPre,
|
||||||
|
promptPost: item.payload.promptPost,
|
||||||
|
duration: item.payload.duration,
|
||||||
|
loraStack: item.payload.loraStack || item.payload.loraName,
|
||||||
|
folderLocked: item.payload.folderLocked
|
||||||
|
})
|
||||||
|
await markStudioLive(item.ownerKey, item.id, live.id)
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error)
|
||||||
|
if (live && (live.status === 'queued' || live.status === 'uploading' || live.status === 'running')) {
|
||||||
|
live.status = 'error'
|
||||||
|
live.error = message
|
||||||
|
}
|
||||||
|
await patchStudioJob(item.ownerKey, item.id, (job) => {
|
||||||
|
job.status = 'error'
|
||||||
|
job.lastError = message
|
||||||
|
}).catch(() => null)
|
||||||
|
kickStudioQueue()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function startStudioJob(item: StudioJob) {
|
export async function startStudioJob(item: StudioJob) {
|
||||||
if (studioJobKind(item) === 'edit') {
|
if (studioJobKind(item) === 'edit') {
|
||||||
await startStudioEditJob(item)
|
await startStudioEditJob(item)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if (item.payload.extendFromClipId) {
|
||||||
|
await startStudioExtendJob(item)
|
||||||
|
return
|
||||||
|
}
|
||||||
let live: Job | undefined
|
let live: Job | undefined
|
||||||
try {
|
try {
|
||||||
const { createJob } = await import('~/server/utils/jobs')
|
const { createJob } = await import('~/server/utils/jobs')
|
||||||
|
|||||||
Reference in New Issue
Block a user