Add LoRA stacks with per-item strengths and named video/image generation presets so setups survive reload without mixing Klein onto MiniMax. Keep LTX Pink Cherry and xAIGen LoRAs on xAIGen, and run Recommend on Beast.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-08-28 15:24:44 -05:00
co-authored by Cursor
parent 0af3cddb14
commit f8db0d626a
36 changed files with 1767 additions and 246 deletions
+5 -1
View File
@@ -1,4 +1,5 @@
import { addStudioJob, kickStudioQueue, listStudioJobs } from '~/server/utils/studioQueue'
import { parsePostedLoraStack, listStudioLoras, persistLoraFields } from '~/server/utils/loras'
import { comfyConfigured } from '~/server/utils/comfy'
import { imageDimensions } from '~/server/utils/resolution'
@@ -81,6 +82,8 @@ export default defineEventHandler(async (event) => {
: Math.floor(Math.random() * 2_147_483_647)
const size = imageDimensions(image.data)
const clipName = (fields.name || '').trim().slice(0, 80)
await listStudioLoras()
const loraStack = parsePostedLoraStack(fields.loraStack || fields.lora, 'image')
const still = await rememberInputStill({
ownerKey,
@@ -137,7 +140,8 @@ export default defineEventHandler(async (event) => {
negative: (fields.negative || '').trim(),
passes,
referenceStillId: savedRef?.id,
referenceStillFilename: savedRef?.filename
referenceStillFilename: savedRef?.filename,
...persistLoraFields(loraStack)
}
})
await kickStudioQueue()
+25 -7
View File
@@ -1,6 +1,9 @@
import { copyFileSync, existsSync, readFileSync } from 'node:fs'
import { join } from 'node:path'
import { composeShotPrompt } from '~/utils/globalLocks'
import { parsePostedLoraStack, listStudioLoras, persistLoraFields } from '~/server/utils/loras'
import { readLoraStack } from '~/utils/loras'
import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
@@ -39,7 +42,7 @@ function parseScheduler(raw: unknown) {
}
export default defineEventHandler(async (event) => {
const body = await readBody<{ clipId?: string; prompt?: string; duration?: number }>(event).catch(() => ({}))
const body = await readBody<{ clipId?: string; prompt?: string; duration?: number; lora?: unknown; loraStack?: unknown }>(event).catch(() => ({}))
const clipId = String(body?.clipId || '').trim()
const prompt = String(body?.prompt || '').trim()
if (!clipId) {
@@ -66,13 +69,21 @@ export default defineEventHandler(async (event) => {
shotIndex: (source.chainIndex || 0) + 1,
familyRefs: source.permanenceRefs
})
const workflow = parseVideoWorkflow(source.workflow)
if (isLtxWorkflow(workflow) && !ltxWorkflowEnabled()) {
throw createError({ statusCode: 400, statusMessage: LTX_DISABLED_MESSAGE })
}
const turbo = source.turbo !== false
const fps = parseFps(source.fps)
const cfg = parseCfg(source.cfg, turbo)
const samplerName = parseSampler(source.samplerName)
const scheduler = parseScheduler(source.scheduler)
const steps = Number(source.steps || defaultVideoSteps(turbo, parseVideoWorkflow(source.workflow)))
const steps = Number(source.steps || defaultVideoSteps(turbo, workflow))
const seed = Math.floor(Math.random() * 2_147_483_647)
await listStudioLoras()
const postedStack = parsePostedLoraStack(body?.loraStack ?? body?.lora, 'video')
const loraStack = postedStack.length ? postedStack : parsePostedLoraStack(readLoraStack(source), 'video')
const loraFields = persistLoraFields(loraStack)
const length = frameLength(durationSeconds, fps)
const width = source.width || 1344
const height = source.height || 768
@@ -109,9 +120,10 @@ export default defineEventHandler(async (event) => {
familyId: source.familyId || crypto.randomUUID(),
parentClipId: source.id,
chainIndex: (source.chainIndex || 0) + 1,
workflow: parseVideoWorkflow(source.workflow),
workflow,
globalLocks: source.globalLocks,
permanenceRefs: source.permanenceRefs
permanenceRefs: source.permanenceRefs,
...loraFields
}
emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 })
@@ -130,7 +142,8 @@ export default defineEventHandler(async (event) => {
cfg,
fps,
samplerName,
scheduler
scheduler,
...loraFields
}).catch((error) => {
removeExtendTemp(job.library?.extendTmpDir)
const message = error instanceof Error ? error.message : String(error)
@@ -169,6 +182,8 @@ async function runExtend(
fps: number
samplerName: string
scheduler: string
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
}
) {
copyFileSync(params.sourcePath, params.part1Path)
@@ -218,7 +233,8 @@ async function runExtend(
filenamePrefix: comfyFilenamePrefix(),
sound,
workflow: job.library?.workflow,
duration: job.library?.duration
duration: job.library?.duration,
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName
})
const queued = await queuePrompt(graph, job.clientId)
@@ -254,7 +270,9 @@ async function runExtend(
workflow: job.library.workflow,
duration: job.library.duration,
globalLocks: job.library.globalLocks,
permanenceRefs: job.library.permanenceRefs
permanenceRefs: job.library.permanenceRefs,
loraName: job.library.loraName,
loraStack: job.library.loraStack
})
}
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
+29 -9
View File
@@ -1,5 +1,7 @@
import { addStudioJob, kickStudioQueue, listStudioJobs } from '~/server/utils/studioQueue'
import { defaultVideoSteps, isLtxWorkflow, isTextToVideo, parseVideoWorkflow } from '~/utils/videoModels'
import { listStudioLoras, parsePostedLoraStack, parseShotLoraStacks, persistLoraFields } from '~/server/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
import { defaultVideoSteps, isLtxWorkflow, isTextToVideo, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
import { allowIdentityRefs, normalizePermanenceRefs, resolveGlobalLocks, type PermanenceRef } from '~/utils/globalLocks'
function parseDuration(raw: unknown) {
@@ -13,18 +15,23 @@ function parseExtendDuration(raw: unknown) {
}
function parseExtensions(raw: string | undefined) {
if (!raw) return [] as { prompt: string; duration: number; permanenceRefs?: PermanenceRef[] }[]
if (!raw) return [] as { prompt: string; duration: number; permanenceRefs?: PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[]
try {
const parsed = JSON.parse(raw)
if (!Array.isArray(parsed)) return []
return parsed
.map((item: { prompt?: unknown; duration?: unknown; permanenceRefs?: unknown }) => ({
prompt: String(item?.prompt || '').trim(),
duration: parseExtendDuration(item?.duration),
permanenceRefs: normalizePermanenceRefs(item?.permanenceRefs)
}))
.map((item: { prompt?: unknown; duration?: unknown; permanenceRefs?: unknown; loraName?: unknown; loraStack?: unknown }) => {
const loraStack = parsePostedLoraStack(item?.loraStack ?? item?.loraName, 'video')
return {
prompt: String(item?.prompt || '').trim(),
duration: parseExtendDuration(item?.duration),
permanenceRefs: normalizePermanenceRefs(item?.permanenceRefs),
...persistLoraFields(loraStack)
}
})
.filter(item => item.prompt)
} catch {
} catch (error) {
if (error && typeof error === 'object' && 'statusCode' in error) throw error
return []
}
}
@@ -101,7 +108,11 @@ export default defineEventHandler(async (event) => {
}
}
await listStudioLoras()
const workflow = parseVideoWorkflow(fields.workflow)
if (isLtxWorkflow(workflow) && !ltxWorkflowEnabled()) {
throw createError({ statusCode: 400, statusMessage: LTX_DISABLED_MESSAGE })
}
const textToVideo = isTextToVideo(workflow)
if (!image && !textToVideo) {
throw createError({ statusCode: 400, statusMessage: 'An initial image is required' })
@@ -116,10 +127,16 @@ export default defineEventHandler(async (event) => {
const extensions = parseExtensions(fields.extensions)
const permanenceRefs = parseFamilyPermanenceRefs(fields.permanenceRefs)
const shotPermanenceRefs = parseShotPermanenceRefs(fields.shotPermanenceRefs, 1 + extensions.length)
const defaultStack = parsePostedLoraStack(fields.loraStack || fields.lora, 'video')
const shotLoraStacks = parseShotLoraStacks(fields.shotLoraStacks || fields.shotLoras, 1 + extensions.length)
if (defaultStack.length && !shotLoraStacks[0].length) shotLoraStacks[0] = defaultStack
for (const [index, item] of extensions.entries()) {
if (!shotPermanenceRefs[index + 1]?.length && item.permanenceRefs?.length) {
shotPermanenceRefs[index + 1] = item.permanenceRefs
}
const shotStack = shotLoraStacks[index + 1]
if (!shotStack.length && item.loraStack?.length) shotLoraStacks[index + 1] = item.loraStack
else if (shotStack.length) Object.assign(item, persistLoraFields(shotStack))
}
const requestedIdentity = workflow === 'v2' && !textToVideo && !isLtxWorkflow(workflow) && fields.useIdentityRefs === 'true'
const useIdentityRefs = allowIdentityRefs(requestedIdentity, permanenceRefs, globalLocks, shotPermanenceRefs)
@@ -216,7 +233,10 @@ export default defineEventHandler(async (event) => {
queueAutoRun,
globalLocks: globalLocks || undefined,
permanenceRefs: permanenceRefs.length ? permanenceRefs : undefined,
shotPermanenceRefs: shotPermanenceRefs.some(item => item.length) ? shotPermanenceRefs : undefined
shotPermanenceRefs: shotPermanenceRefs.some(item => item.length) ? shotPermanenceRefs : undefined,
...persistLoraFields(defaultStack),
shotLoras: shotLoraStacks.some(stack => stack.length) ? shotLoraStacks.map(stack => stack[0]?.name || '') : undefined,
shotLoraStacks: shotLoraStacks.some(stack => stack.length) ? shotLoraStacks : undefined
}
})
await kickStudioQueue()
+10
View File
@@ -0,0 +1,10 @@
import { listGenerationPresets } from '~/server/utils/generationPresets'
import { parseGenerationPresetKind } from '~/utils/generationPresets'
export default defineEventHandler((event) => {
const { owner } = assertLibraryOwner(event)
const kind = parseGenerationPresetKind(getQuery(event).kind)
return {
presets: listGenerationPresets(owner, kind)
}
})
+25
View File
@@ -0,0 +1,25 @@
import { listGenerationPresets, saveGenerationPreset } from '~/server/utils/generationPresets'
import { parseGenerationPresetKind } from '~/utils/generationPresets'
export default defineEventHandler(async (event) => {
const { owner } = assertLibraryOwner(event)
const body = await readBody<{
name?: string
kind?: string
loraStack?: unknown
settings?: unknown
overwrite?: boolean
}>(event).catch(() => ({}))
const kind = parseGenerationPresetKind(body?.kind)
const preset = await saveGenerationPreset(owner, {
name: body?.name || '',
kind,
loraStack: body?.loraStack,
settings: body?.settings,
overwrite: body?.overwrite === true
})
return {
preset,
presets: listGenerationPresets(owner, kind)
}
})
@@ -0,0 +1,13 @@
import { deleteGenerationPreset, listGenerationPresets } from '~/server/utils/generationPresets'
import { parseGenerationPresetKind } from '~/utils/generationPresets'
export default defineEventHandler(async (event) => {
const { owner } = assertLibraryOwner(event)
const id = String(getRouterParam(event, 'id') || '')
const kind = parseGenerationPresetKind(getQuery(event).kind)
await deleteGenerationPreset(owner, id)
return {
ok: true,
presets: listGenerationPresets(owner, kind)
}
})
+19 -15
View File
@@ -1,14 +1,15 @@
import { checkComfyHttp, fetchQueue } from '~/server/utils/comfyLifecycle'
import { comfyConfigured } from '~/server/utils/comfy'
import { imageComfyConfigured, getBeastImageHost, getSidecarImageHost } from '~/server/utils/imageComfy'
import { imageComfyConfigured, getBeastImageHost, getSidecarImageHost, sameImageHost } from '~/server/utils/imageComfy'
import { checkImageComfyHttp, fetchQueueAt } from '~/server/utils/imageComfyLifecycle'
export default defineEventHandler(async () => {
const videoConfigured = comfyConfigured()
const imageConfigured = imageComfyConfigured()
const beastHost = getBeastImageHost()
const sidecarHost = getSidecarImageHost()
const [comfy, queue, beast, beastQueue, sidecar, sidecarQueue] = await Promise.all([
const promptHost = getSidecarImageHost()
const promptIsBeast = Boolean(promptHost && beastHost && sameImageHost(promptHost, beastHost))
const [comfy, queue, beast, beastQueue, prompt, promptQueue] = await Promise.all([
videoConfigured
? checkComfyHttp().catch(() => ({ ok: false, host: '', stats: null }))
: Promise.resolve({ ok: false, host: '', stats: null }),
@@ -17,17 +18,20 @@ export default defineEventHandler(async () => {
? checkImageComfyHttp(2500, beastHost).catch(() => ({ ok: false, host: beastHost, stats: null }))
: Promise.resolve({ ok: false, host: beastHost, stats: null }),
beastHost ? fetchQueueAt(beastHost) : Promise.resolve({ running: 0, pending: 0 }),
sidecarHost
? checkImageComfyHttp(2500, sidecarHost).catch(() => ({ ok: false, host: sidecarHost, stats: null }))
: Promise.resolve({ ok: false, host: sidecarHost, stats: null }),
sidecarHost ? fetchQueueAt(sidecarHost) : Promise.resolve({ running: 0, pending: 0 })
promptHost && !promptIsBeast
? checkImageComfyHttp(2500, promptHost).catch(() => ({ ok: false, host: promptHost, stats: null }))
: Promise.resolve({ ok: false, host: promptHost, stats: null }),
promptHost && !promptIsBeast ? fetchQueueAt(promptHost) : Promise.resolve({ running: 0, pending: 0 })
])
const busy = Boolean(comfy.ok && queue.running > 0)
const beastBusy = Boolean(beast.ok && beastQueue.running > 0)
const sidecarBusy = Boolean(sidecar.ok && sidecarQueue.running > 0)
const promptOk = promptIsBeast ? Boolean(beast.ok || comfy.ok) : Boolean(prompt.ok)
const promptBusy = promptIsBeast ? (beastBusy || busy) : Boolean(prompt.ok && promptQueue.running > 0)
const imageOk = Boolean(beast.ok || comfy.ok)
const imageBusy = Boolean(imageConfigured && (beastBusy || busy))
const preferredHost = beastHost || comfy.host || ''
const promptQueueRunning = promptIsBeast ? ((beastQueue.running || queue.running) || 0) : (promptQueue.running || 0)
const promptQueuePending = promptIsBeast ? ((beastQueue.pending || queue.pending) || 0) : (promptQueue.pending || 0)
return {
ok: true,
service: useRuntimeConfig().public.instanceName || 'aigen',
@@ -55,13 +59,13 @@ export default defineEventHandler(async () => {
host: preferredHost
},
sidecar: {
label: 'Sidecar',
configured: Boolean(sidecarHost),
ok: Boolean(sidecar.ok),
busy: sidecarBusy,
host: sidecarHost,
queueRunning: sidecarQueue.running || 0,
queuePending: sidecarQueue.pending || 0
label: 'Qwen VL',
configured: Boolean(promptHost),
ok: promptOk,
busy: promptBusy,
host: promptHost,
queueRunning: promptQueueRunning,
queuePending: promptQueuePending
}
}
}
+9
View File
@@ -0,0 +1,9 @@
import { listStudioLoras } from '~/server/utils/loras'
export default defineEventHandler(async () => {
const listed = await listStudioLoras()
return {
image: listed.image,
video: listed.video
}
})
+1 -1
View File
@@ -5,7 +5,7 @@ export default defineEventHandler(async (event) => {
if (!promptComfyConfigured()) {
throw createError({
statusCode: 503,
statusMessage: 'Sidecar Qwen VL is not configured. Set IMAGE_COMFY_HOST.'
statusMessage: 'Qwen VL is not configured. Set COMFY_HOST or IMAGE_COMFY_HOST to Beast.'
})
}
+3 -1
View File
@@ -1,4 +1,5 @@
import { getShotQueue, patchShotQueueSegments, updateShotQueue, summarizeQueue } from '~/server/utils/shotQueue'
import { listStudioLoras } from '~/server/utils/loras'
export default defineEventHandler(async (event) => {
const { owner } = assertLibraryOwner(event)
@@ -6,10 +7,11 @@ export default defineEventHandler(async (event) => {
const body = await readBody<{
name?: string
autoRun?: boolean
segments?: Array<{ index: number; prompt?: string; duration?: number }>
segments?: Array<{ index: number; prompt?: string; duration?: number; loraName?: string; loraStack?: unknown }>
}>().catch(() => ({}))
if (Array.isArray(body?.segments) && body.segments.length) {
await listStudioLoras()
await patchShotQueueSegments(owner, id, body.segments)
}
if (typeof body?.name === 'string' || typeof body?.autoRun === 'boolean') {
+7 -2
View File
@@ -23,8 +23,13 @@ export default defineEventHandler((event) => {
]
: undefined)
: [
{ prompt: job.payload.prompt, duration: job.payload.duration },
...(job.payload.extensions || [])
{ prompt: job.payload.prompt, duration: job.payload.duration, loraName: job.payload.shotLoras?.[0] || job.payload.loraName, loraStack: job.payload.shotLoraStacks?.[0] || job.payload.loraStack },
...(job.payload.extensions || []).map((item, index) => ({
prompt: item.prompt,
duration: item.duration,
loraName: item.loraName || job.payload.shotLoras?.[index + 1],
loraStack: item.loraStack || job.payload.shotLoraStacks?.[index + 1]
}))
]
return {
...summarizeStudioJob(job),
+184
View File
@@ -0,0 +1,184 @@
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { ownerRoot } from '~/server/utils/library'
import { filterLoraStackForStudio, normalizeLoraStack } from '~/utils/loras'
import {
normalizePresetName,
parseGenerationPresetKind,
type GenerationPreset,
type GenerationPresetKind,
type ImagePresetSettings,
type VideoPresetSettings
} from '~/utils/generationPresets'
import { coerceVideoWorkflow, isXaigenStudio, parseVideoWorkflow } from '~/utils/videoModels'
const writeChains = new Map<string, Promise<unknown>>()
function presetsPath(owner: string) {
return join(ownerRoot(owner), 'generation-presets.json')
}
function ensureOwner(owner: string) {
mkdirSync(ownerRoot(owner), { recursive: true })
}
function readPresets(owner: string): GenerationPreset[] {
ensureOwner(owner)
const path = presetsPath(owner)
if (!existsSync(path)) return []
try {
const parsed = JSON.parse(readFileSync(path, 'utf8'))
return Array.isArray(parsed) ? parsed.map(normalizeStoredPreset).filter((item): item is GenerationPreset => Boolean(item)) : []
} catch {
return []
}
}
function writePresets(owner: string, presets: GenerationPreset[]) {
ensureOwner(owner)
const path = presetsPath(owner)
const tmp = `${path}.tmp`
writeFileSync(tmp, JSON.stringify(presets, null, 2))
renameSync(tmp, path)
}
function mutate<T>(owner: string, fn: (presets: GenerationPreset[]) => T): Promise<T> {
const prev = writeChains.get(owner) || Promise.resolve()
const run = prev.then(() => {
const presets = readPresets(owner)
const result = fn(presets)
writePresets(owner, presets)
return result
})
writeChains.set(owner, run.then(() => undefined, () => undefined))
return run
}
function clampInt(raw: unknown, fallback: number, min: number, max: number) {
const value = Number(raw)
if (!Number.isFinite(value)) return fallback
return Math.min(max, Math.max(min, Math.round(value)))
}
function clampNum(raw: unknown, fallback: number, min: number, max: number, step = 0.1) {
const value = Number(raw)
if (!Number.isFinite(value)) return fallback
const snapped = Math.round(value / step) * step
return Math.min(max, Math.max(min, Math.round(snapped * 100) / 100))
}
function sanitizeVideoSettings(raw: unknown): VideoPresetSettings {
const rec = raw && typeof raw === 'object' ? raw as Record<string, unknown> : {}
const workflow = coerceVideoWorkflow(parseVideoWorkflow(rec.workflow))
const turbo = rec.turbo !== false
const sampler = String(rec.samplerName || '')
const scheduler = String(rec.scheduler || '')
const aspect = String(rec.aspect || 'auto')
return {
workflow,
turbo,
cfg: clampNum(rec.cfg, turbo ? 1.5 : 4, 1, 10, 0.5),
fps: rec.fps === 12 || rec.fps === 30 || rec.fps === 24 ? Number(rec.fps) : 24,
samplerName: sampler === 'euler' || sampler === 'dpmpp_2m' || sampler === 'res_multistep' ? sampler : 'res_multistep',
scheduler: scheduler === 'ddim_uniform' || scheduler === 'sgm_uniform' || scheduler === 'simple' ? scheduler : 'simple',
duration: clampNum(rec.duration, 5, 0.5, 120, 0.1),
aspect: aspect === '16:9' || aspect === '9:16' || aspect === '1:1' || aspect === 'auto' ? aspect : 'auto',
sound: rec.sound !== false
}
}
function sanitizeImageSettings(raw: unknown): ImagePresetSettings {
const rec = raw && typeof raw === 'object' ? raw as Record<string, unknown> : {}
return {
steps: clampInt(rec.steps, 20, 1, 40),
cfg: clampNum(rec.cfg, 1, 0.5, 4, 0.1),
negative: String(rec.negative || '').slice(0, 2000)
}
}
function normalizeStoredPreset(raw: unknown): GenerationPreset | null {
if (!raw || typeof raw !== 'object') return null
const rec = raw as Record<string, unknown>
const name = normalizePresetName(rec.name)
if (!name) return null
const kind = parseGenerationPresetKind(rec.kind)
const id = String(rec.id || '').trim() || crypto.randomUUID()
const createdAt = Number(rec.createdAt) || Date.now()
return {
id,
name,
kind,
createdAt,
updatedAt: Number(rec.updatedAt) || createdAt,
loraStack: normalizeLoraStack(rec.loraStack),
settings: kind === 'image' ? sanitizeImageSettings(rec.settings) : sanitizeVideoSettings(rec.settings)
}
}
function publicPreset(preset: GenerationPreset): GenerationPreset {
const xaigen = isXaigenStudio()
return {
...preset,
loraStack: filterLoraStackForStudio(preset.loraStack, xaigen),
settings: preset.kind === 'video'
? { ...preset.settings, workflow: coerceVideoWorkflow((preset.settings as VideoPresetSettings).workflow) }
: preset.settings
}
}
export function listGenerationPresets(owner: string, kind?: GenerationPresetKind) {
return readPresets(owner)
.filter(item => !kind || item.kind === kind)
.map(publicPreset)
.sort((a, b) => a.name.localeCompare(b.name) || b.updatedAt - a.updatedAt)
}
export async function saveGenerationPreset(owner: string, input: {
name: string
kind: GenerationPresetKind
loraStack?: unknown
settings?: unknown
overwrite?: boolean
}) {
const name = normalizePresetName(input.name)
if (!name) {
throw createError({ statusCode: 400, statusMessage: 'A preset name is required' })
}
const kind = parseGenerationPresetKind(input.kind)
const xaigen = isXaigenStudio()
const loraStack = filterLoraStackForStudio(input.loraStack, xaigen)
const settings = kind === 'image' ? sanitizeImageSettings(input.settings) : sanitizeVideoSettings(input.settings)
const now = Date.now()
return mutate(owner, (presets) => {
const existing = presets.find(item => item.kind === kind && item.name.toLowerCase() === name.toLowerCase())
if (existing && !input.overwrite) {
throw createError({ statusCode: 409, statusMessage: `Overwrite preset “${existing.name}”?` })
}
if (existing) {
existing.updatedAt = now
existing.loraStack = loraStack
existing.settings = settings
return publicPreset(existing)
}
const created: GenerationPreset = {
id: crypto.randomUUID(),
name,
kind,
createdAt: now,
updatedAt: now,
loraStack,
settings
}
presets.unshift(created)
return publicPreset(created)
})
}
export async function deleteGenerationPreset(owner: string, id: string) {
return mutate(owner, (presets) => {
const index = presets.findIndex(item => item.id === id)
if (index < 0) throw createError({ statusCode: 404, statusMessage: 'Preset not found' })
const [removed] = presets.splice(index, 1)
return removed
})
}
+4 -1
View File
@@ -18,6 +18,8 @@ export type EditRunParams = {
steps: number
seed: number
cfg: number
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
}
export async function runEdit(job: Job, params: EditRunParams) {
@@ -97,7 +99,8 @@ export async function runEdit(job: Job, params: EditRunParams) {
steps: params.steps,
seed,
cfg: params.cfg,
filenamePrefix: `aigen_edit_${job.id.slice(0, 8)}_p${index + 1}`
filenamePrefix: `aigen_edit_${job.id.slice(0, 8)}_p${index + 1}`,
loraStack: params.loraStack || params.loraName
})
const queued = await queuePrompt(graph, job.clientId)
job.promptId = queued.prompt_id
+18 -18
View File
@@ -38,13 +38,24 @@ export function getBeastImageHost() {
}
}
function isRetiredSidecarHost(raw: string) {
return /192\.168\.77\.101(?!\d)/.test(raw)
}
export function getSidecarImageHost() {
const config = useRuntimeConfig()
const fallback = String(config.imageComfyFallbackHost || process.env.IMAGE_COMFY_FALLBACK_HOST || '').trim()
if (fallback) return normalizeHost(fallback)
const legacy = String(config.imageComfyHost || process.env.IMAGE_COMFY_HOST || '').trim()
const port = String(config.imageComfyPort || process.env.IMAGE_COMFY_PORT || '').trim()
return normalizeHost(legacy, port)
const candidates: Array<[string, string]> = [
[String(config.promptComfyHost || process.env.PROMPT_COMFY_HOST || '').trim(), ''],
[String(config.imageComfyHost || process.env.IMAGE_COMFY_HOST || '').trim(), port],
[String(config.imageComfyFallbackHost || process.env.IMAGE_COMFY_FALLBACK_HOST || '').trim(), '']
]
for (const [raw, extraPort] of candidates) {
if (!raw || isRetiredSidecarHost(raw)) continue
const host = normalizeHost(raw, extraPort)
if (host && !isRetiredSidecarHost(host)) return host
}
return getBeastImageHost()
}
export function getImageComfyHost() {
@@ -352,19 +363,8 @@ async function purgeImageOnDesktop(opts: {
output?: { filename: string; subfolder: string; type: string }
}) {
const config = useRuntimeConfig()
const host = getImageComfyHost()
const sidecar = getSidecarImageHost()
const useSidecarAgent = sidecar && sameImageHost(host, sidecar)
const controlUrl = String(
useSidecarAgent
? (config.imageComfyControlUrl || process.env.IMAGE_COMFY_CONTROL_URL || '')
: (config.comfyControlUrl || process.env.COMFY_CONTROL_URL || '')
).replace(/\/$/, '')
const token = String(
useSidecarAgent
? (config.imageComfyControlToken || process.env.IMAGE_COMFY_CONTROL_TOKEN || '')
: (config.comfyControlToken || process.env.COMFY_CONTROL_TOKEN || '')
)
const controlUrl = String(config.comfyControlUrl || process.env.COMFY_CONTROL_URL || '').replace(/\/$/, '')
const token = String(config.comfyControlToken || process.env.COMFY_CONTROL_TOKEN || '')
if (!controlUrl) return
try {
await fetch(`${controlUrl}/purge`, {
@@ -382,7 +382,7 @@ async function purgeImageOnDesktop(opts: {
signal: AbortSignal.timeout(4000)
})
} catch {
/* Sidecar host agent is optional; Comfy HTTP delete is the primary path. */
/* Host agent purge is optional; Comfy HTTP delete is the primary path. */
}
}
+8 -93
View File
@@ -187,7 +187,7 @@ async function ensureUnlocked(onStatus: StatusFn) {
onStatus({
state: 'starting',
message: 'Image ComfyUI is offline. Starting the sidecar instance...',
message: 'Image ComfyUI is offline. Starting the Beast instance...',
host,
processRunning: Boolean(remote?.process)
})
@@ -195,7 +195,7 @@ async function ensureUnlocked(onStatus: StatusFn) {
if (!started.started && started.reason === 'no-launcher') {
throw createError({
statusCode: 503,
statusMessage: 'Image ComfyUI is offline and the sidecar host agent did not start it. Confirm the agent on 192.168.77.101:8199 is running.'
statusMessage: 'Image ComfyUI is offline and the Beast host agent did not start it. Confirm COMFY_CONTROL_URL is running.'
})
}
@@ -230,111 +230,26 @@ export function ensureImageComfyReady(onStatus: StatusFn = () => undefined) {
}
let sidecarGate: Promise<unknown> = Promise.resolve()
let sidecarLastStartAt = 0
function sidecarControlSettings() {
const config = useRuntimeConfig()
return {
host: getSidecarImageHost(),
controlUrl: String(config.imageComfyControlUrl || process.env.IMAGE_COMFY_CONTROL_URL || '').replace(/\/$/, ''),
controlToken: String(config.imageComfyControlToken || process.env.IMAGE_COMFY_CONTROL_TOKEN || ''),
healthTimeoutMs: 2500,
startTimeoutMs: Number(process.env.IMAGE_COMFY_START_TIMEOUT_MS || 180_000)
}
}
async function sidecarControlRequest(path: string, method = 'GET') {
const { controlUrl, controlToken } = sidecarControlSettings()
if (!controlUrl) return null
try {
const res = await fetch(`${controlUrl}${path}`, {
method,
headers: {
Accept: 'application/json',
...(controlToken ? { Authorization: `Bearer ${controlToken}` } : {})
},
signal: AbortSignal.timeout(5000)
})
if (!res.ok) return { ok: false, status: res.status }
return await res.json() as {
ok?: boolean
http?: boolean
process?: boolean
started?: boolean
already?: boolean
}
} catch {
return null
}
}
async function startSidecarComfy() {
const now = Date.now()
if (now - sidecarLastStartAt < 15_000) {
log('sidecar-start-skipped', { reason: 'recent-start' })
return { started: false, reason: 'recent-start' as const }
}
sidecarLastStartAt = now
const remote = await sidecarControlRequest('/start', 'POST')
if (remote?.ok || remote?.started) {
log('sidecar-start-remote', {})
return { started: true, reason: 'remote' as const }
}
return { started: false, reason: 'no-launcher' as const }
}
async function ensureSidecarUnlocked(onStatus: StatusFn) {
const cfg = sidecarControlSettings()
const host = cfg.host
const host = getSidecarImageHost()
if (!host) {
throw createError({
statusCode: 503,
statusMessage: 'Sidecar Qwen VL is not configured. Set IMAGE_COMFY_HOST or IMAGE_COMFY_FALLBACK_HOST.'
statusMessage: 'Qwen VL is not configured. Set COMFY_HOST or IMAGE_COMFY_HOST to Beast.'
})
}
onStatus({ state: 'offline', message: 'Checking Sidecar Qwen VL…', host, box: 'sidecar' })
const health = await checkImageComfyHttp(cfg.healthTimeoutMs, host)
onStatus({ state: 'offline', message: 'Checking Beast Qwen VL…', host, box: 'beast' })
const health = await checkImageComfyHttp(2500, host)
if (health.ok) {
onStatus({ state: 'online', message: 'Sidecar Qwen VL online', host, box: 'sidecar' })
onStatus({ state: 'online', message: 'Beast Qwen VL online', host, box: 'beast' })
return
}
onStatus({
state: 'starting',
message: 'Sidecar is offline. Starting Qwen VL on 192.168.77.101…',
host,
box: 'sidecar'
})
const started = await startSidecarComfy()
if (!started.started && started.reason === 'no-launcher') {
throw createError({
statusCode: 503,
statusMessage: 'Sidecar is offline and the host agent did not start it. Confirm the agent on 192.168.77.101:8199 is running.'
})
}
const deadline = Date.now() + cfg.startTimeoutMs
let attempt = 0
while (Date.now() < deadline) {
attempt += 1
onStatus({
state: 'starting',
message: `Starting Sidecar Qwen VL… waiting for Comfy (${attempt})`,
host,
box: 'sidecar'
})
const next = await checkImageComfyHttp(cfg.healthTimeoutMs, host)
if (next.ok) {
onStatus({ state: 'online', message: 'Sidecar Qwen VL online', host, box: 'sidecar', processRunning: true })
return
}
await sleep(2000)
}
throw createError({
statusCode: 503,
statusMessage: 'Started Sidecar Qwen VL but it never answered. The 101 box can take a couple of minutes to load after a cold boot.'
statusMessage: `Beast ComfyUI is offline (${host}). Recommend needs Qwen VL on that instance and will not start a sidecar.`
})
}
+5
View File
@@ -1,4 +1,6 @@
import template from '../assets/workflow_flux2_klein_edit.json'
import { applyUserLoraToGraph } from '~/server/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
const IMAGE1 = '76'
const IMAGE2 = '81'
@@ -27,6 +29,8 @@ export interface EditParams {
seed: number
cfg: number
filenamePrefix?: string
loraName?: string
loraStack?: LoraStackItem[]
}
export function buildEditWorkflow(params: EditParams) {
@@ -61,6 +65,7 @@ export function buildEditWorkflow(params: EditParams) {
if (dual ? singleNode : dualNode) delete graph[id]
}
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
return graph
}
+5 -1
View File
@@ -77,7 +77,7 @@ export interface Job {
duration?: number
sound?: boolean
draftId?: string
extensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[]
extensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: import('~/utils/loras').LoraStackItem[] }[]
chainIndex?: number
chainStep?: number
chainTotal?: number
@@ -95,6 +95,10 @@ export interface Job {
queueAutoRun?: boolean
queueBudget?: number
stopAfterCurrent?: boolean
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
}
error?: string
socketReady?: boolean
+23 -3
View File
@@ -46,6 +46,8 @@ export interface LibraryClip {
hideInput?: boolean
globalLocks?: string
permanenceRefs?: PermanenceRef[]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
}
export type StillRole = 'input' | 'output'
@@ -82,6 +84,8 @@ export interface QueuedExtension {
prompt: string
duration: number
permanenceRefs?: PermanenceRef[]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
}
export interface RetryDraft {
@@ -111,6 +115,10 @@ export interface RetryDraft {
globalLocks?: string
permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
}
interface Catalog {
@@ -639,7 +647,7 @@ function writeFrameMeta(dest: string, video: string, mode: FrameSourceMeta['mode
const meta: FrameSourceMeta = { videoMtime: info.mtimeMs, videoSize: info.size, mode }
if (typeof seek === 'number') meta.seek = seek
writeFileSync(frameMetaPath(dest), JSON.stringify(meta))
} catch { /* thumbs still work without the sidecar */ }
} catch { /* thumbs still work without Comfy */ }
}
function frameCacheFresh(video: string, dest: string, mode: FrameSourceMeta['mode']) {
@@ -1209,6 +1217,10 @@ export async function saveRetryDraft(params: {
globalLocks?: string
permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
}) {
return mutate(params.ownerKey, (catalog) => {
const existing = params.id ? catalog.drafts.find(item => item.id === params.id) : undefined
@@ -1238,7 +1250,11 @@ export async function saveRetryDraft(params: {
workflow: params.workflow,
globalLocks: params.globalLocks,
permanenceRefs: params.permanenceRefs,
shotPermanenceRefs: params.shotPermanenceRefs
shotPermanenceRefs: params.shotPermanenceRefs,
loraName: params.loraName,
loraStack: params.loraStack,
shotLoras: params.shotLoras,
shotLoraStacks: params.shotLoraStacks
}
catalog.drafts = [draft, ...catalog.drafts.filter(item => item.id !== draft.id)].slice(0, 20)
return draft
@@ -1392,6 +1408,8 @@ export async function saveClip(params: {
sound?: boolean
globalLocks?: string
permanenceRefs?: PermanenceRef[]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
segmentFirstFrame?: Buffer
}) {
const catalog = readCatalog(params.ownerKey)
@@ -1423,7 +1441,9 @@ export async function saveClip(params: {
sound: params.sound,
hideInput: params.hideInput === true,
globalLocks: params.globalLocks,
permanenceRefs: params.permanenceRefs?.length ? normalizePermanenceRefs(params.permanenceRefs) : undefined
permanenceRefs: params.permanenceRefs?.length ? normalizePermanenceRefs(params.permanenceRefs) : undefined,
loraName: params.loraName || undefined,
loraStack: params.loraStack?.length ? params.loraStack : undefined
}
mkdirSync(clipDir(params.ownerKey, clip.id), { recursive: true })
const videoPath = clipVideoPath(params.ownerKey, clip.id)
+305
View File
@@ -0,0 +1,305 @@
import { comfyConfigured, comfyFetch, getComfyHost } from '~/server/utils/comfy'
import { getBeastImageHost, imageComfyFetch, sameImageHost } from '~/server/utils/imageComfy'
import {
XAIGEN_LORA_MESSAGE,
filterLorasForStudio,
isXaigenOnlyLora,
normalizeLoraName,
normalizeLoraStack,
persistLoraFields,
type LoraKind,
type LoraStackItem
} from '~/utils/loras'
import { isXaigenStudio } from '~/utils/videoModels'
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
type WorkflowGraph = Record<string, WorkflowNode>
const LORA_LOADER = 'LoraLoader'
const LORA_MODEL_ONLY = 'LoraLoaderModelOnly'
const OBJECT_INFO_NODES = [
LORA_LOADER,
LORA_MODEL_ONLY,
'Power Lora Loader (rgthree)',
'Lora Loader Stack (rgthree)',
'LoraLoaderModelOnly [10]',
'WanVideoLoraSelect',
'LoraLoaderStacked'
]
const MODEL_FEED_CLASSES = new Set(['BasicGuider', 'CFGGuider', 'KSampler', 'KSamplerAdvanced'])
const CACHE_MS = 20_000
type LoraCache = {
at: number
image: string[]
video: string[]
}
let cache: LoraCache | null = null
let inflight: Promise<LoraCache> | null = null
function uniqueNames(values: unknown[]) {
const seen = new Set<string>()
const out: string[] = []
for (const value of values) {
const name = String(value || '').trim()
if (!name || seen.has(name)) continue
seen.add(name)
out.push(name)
}
return out
}
function comboFromSpec(spec: unknown): string[] {
if (!Array.isArray(spec)) return []
const first = spec[0]
if (Array.isArray(first)) return first.filter((item): item is string => typeof item === 'string' && item.trim().length > 0)
if (typeof first === 'string' && first.trim()) return spec.filter((item): item is string => typeof item === 'string' && item.trim().length > 0)
return []
}
function namesFromObjectInfoNode(info: unknown) {
if (!info || typeof info !== 'object') return [] as string[]
const input = (info as { input?: { required?: Record<string, unknown>; optional?: Record<string, unknown> } }).input
const buckets = [input?.required, input?.optional]
const names: string[] = []
for (const bucket of buckets) {
if (!bucket) continue
for (const [key, spec] of Object.entries(bucket)) {
if (!/lora/i.test(key)) continue
names.push(...comboFromSpec(spec))
}
}
return uniqueNames(names)
}
async function fetchJson(path: string, via: 'video' | 'image') {
const res = via === 'image'
? await imageComfyFetch(path, { signal: AbortSignal.timeout(8000) })
: await comfyFetch(path, { signal: AbortSignal.timeout(8000) })
if (!res.ok) return null
return res.json().catch(() => null)
}
async function fetchModelsLoras(via: 'video' | 'image') {
const payload = await fetchJson('/models/loras', via)
if (Array.isArray(payload)) return uniqueNames(payload)
if (payload && typeof payload === 'object' && Array.isArray((payload as { loras?: unknown[] }).loras)) {
return uniqueNames((payload as { loras: unknown[] }).loras)
}
return [] as string[]
}
async function fetchObjectInfoLoras(via: 'video' | 'image') {
const image: string[] = []
const video: string[] = []
let any = false
for (const node of OBJECT_INFO_NODES) {
const info = await fetchJson(`/object_info/${encodeURIComponent(node)}`, via)
if (!info || typeof info !== 'object') continue
const record = info as Record<string, unknown>
const body = record[node] || (Object.keys(record).length === 1 ? record[Object.keys(record)[0] as string] : record)
const names = namesFromObjectInfoNode(body)
if (!names.length) continue
any = true
if (node === LORA_LOADER) image.push(...names)
else video.push(...names)
if (node !== LORA_LOADER && node !== LORA_MODEL_ONLY) {
image.push(...names)
video.push(...names)
}
}
if (any) {
return { image: uniqueNames(image), video: uniqueNames(video) }
}
const all = await fetchJson('/object_info', via)
if (!all || typeof all !== 'object') return { image: [] as string[], video: [] as string[] }
const imageAll: string[] = []
const videoAll: string[] = []
for (const [classType, info] of Object.entries(all as Record<string, unknown>)) {
if (!/lora/i.test(classType)) continue
const names = namesFromObjectInfoNode(info)
if (classType === LORA_LOADER) imageAll.push(...names)
else videoAll.push(...names)
}
return { image: uniqueNames(imageAll), video: uniqueNames(videoAll) }
}
async function discoverFromHost(via: 'video' | 'image') {
const models = await fetchModelsLoras(via).catch(() => [] as string[])
const fromInfo = await fetchObjectInfoLoras(via).catch(() => ({ image: [] as string[], video: [] as string[] }))
const image = uniqueNames([...fromInfo.image, ...models])
const video = uniqueNames([...fromInfo.video, ...models])
if (!image.length && video.length) return { image: video, video }
if (!video.length && image.length) return { image, video: image }
return { image, video }
}
async function loadLoraCache(): Promise<LoraCache> {
const now = Date.now()
if (cache && now - cache.at < CACHE_MS) return cache
if (inflight) return inflight
inflight = (async () => {
const videoHost = comfyConfigured() ? getComfyHost() : ''
const imageHost = getBeastImageHost()
const same = Boolean(videoHost && imageHost && sameImageHost(videoHost, imageHost))
const video = videoHost
? await discoverFromHost('video').catch(() => ({ image: [] as string[], video: [] as string[] }))
: { image: [] as string[], video: [] as string[] }
const image = imageHost && !same
? await discoverFromHost('image').catch(() => ({ image: [] as string[], video: [] as string[] }))
: video
const xaigen = isXaigenStudio()
const next: LoraCache = {
at: Date.now(),
image: filterLorasForStudio(uniqueNames([...image.image, ...video.image]), xaigen),
video: filterLorasForStudio(uniqueNames([...video.video, ...image.video]), xaigen)
}
cache = next
return next
})().finally(() => {
inflight = null
})
return inflight
}
export async function listStudioLoras() {
try {
return await loadLoraCache()
} catch {
return { at: 0, image: [] as string[], video: [] as string[] }
}
}
export function assertLoraAllowed(raw: unknown, kind: LoraKind) {
const name = normalizeLoraName(raw)
if (!name) return ''
if (isXaigenOnlyLora(name) && !isXaigenStudio()) {
throw createError({ statusCode: 400, statusMessage: XAIGEN_LORA_MESSAGE })
}
const known = cache?.[kind]
if (known?.length && !known.includes(name) && !known.some(item => item.replace(/\\/g, '/') === name)) {
throw createError({ statusCode: 400, statusMessage: `Unknown ${kind} LoRA` })
}
return name
}
export function parsePostedLora(raw: unknown, kind: LoraKind) {
return parsePostedLoraStack(raw, kind)[0]?.name || ''
}
export function parsePostedLoraStack(raw: unknown, kind: LoraKind): LoraStackItem[] {
const out: LoraStackItem[] = []
for (const item of normalizeLoraStack(raw)) {
const name = assertLoraAllowed(item.name, kind)
if (!name) continue
out.push({ ...item, name })
}
return out
}
export function parseShotLoras(raw: unknown, shotCount: number, kind: LoraKind = 'video') {
return parseShotLoraStacks(raw, shotCount, kind).map(stack => stack[0]?.name || '')
}
export function parseShotLoraStacks(raw: unknown, shotCount: number, kind: LoraKind = 'video'): LoraStackItem[][] {
const empty = Array.from({ length: shotCount }, () => [] as LoraStackItem[])
if (!raw) return empty
let parsed: unknown = raw
if (typeof raw === 'string') {
try {
parsed = JSON.parse(raw)
} catch {
return empty
}
}
if (!Array.isArray(parsed)) return empty
return Array.from({ length: shotCount }, (_, index) => parsePostedLoraStack(parsed[index], kind))
}
export { persistLoraFields }
function linkSource(value: unknown): string | null {
return Array.isArray(value) && typeof value[0] === 'string' ? value[0] : null
}
function alreadyHasLora(graph: WorkflowGraph, name: string) {
const wanted = name.replace(/\\/g, '/').toLowerCase()
return Object.values(graph).some((node) => {
if (node.class_type !== LORA_LOADER && node.class_type !== LORA_MODEL_ONLY) return false
return String(node.inputs.lora_name || '').replace(/\\/g, '/').toLowerCase() === wanted
})
}
function injectAfter(
graph: WorkflowGraph,
sourceId: string,
nodeId: string,
node: WorkflowNode
) {
if (!graph[sourceId] || graph[nodeId]) return
graph[nodeId] = node
for (const [id, other] of Object.entries(graph)) {
if (id === nodeId) continue
for (const [key, value] of Object.entries(other.inputs)) {
const src = linkSource(value)
if (src === sourceId && Array.isArray(value)) {
other.inputs[key] = [nodeId, value[1]]
}
}
}
}
function findModelFeed(graph: WorkflowGraph) {
for (const node of Object.values(graph)) {
if (!MODEL_FEED_CLASSES.has(node.class_type)) continue
const source = linkSource(node.inputs.model)
if (source && graph[source]) return source
}
return ''
}
export function applyUserLoraToGraph(graph: WorkflowGraph, stack?: unknown) {
const items = normalizeLoraStack(stack).filter(item => !alreadyHasLora(graph, item.name))
if (!items.length) return
const clipLoaders = Object.entries(graph).filter(([, node]) => node.class_type === LORA_LOADER)
if (clipLoaders.length) {
for (const [id] of clipLoaders) {
let sourceId = id
for (const [index, item] of items.entries()) {
const nodeId = `user:lora:${id}:${index}`
injectAfter(graph, sourceId, nodeId, {
class_type: LORA_LOADER,
inputs: {
lora_name: item.name,
strength_model: item.strengthModel,
strength_clip: item.strengthClip,
model: [sourceId, 0],
clip: [sourceId, 1]
},
_meta: { title: items.length === 1 ? 'User LoRA' : `User LoRA ${index + 1}` }
})
sourceId = nodeId
}
}
return
}
let source = findModelFeed(graph)
if (!source) return
for (const [index, item] of items.entries()) {
const nodeId = index === 0 ? 'user:lora' : `user:lora:${index}`
injectAfter(graph, source, nodeId, {
class_type: LORA_MODEL_ONLY,
inputs: {
lora_name: item.name,
strength_model: item.strengthModel,
model: [source, 0]
},
_meta: { title: items.length === 1 ? 'User LoRA' : `User LoRA ${index + 1}` }
})
source = nodeId
}
}
+17 -3
View File
@@ -47,13 +47,17 @@ export interface PendingJob {
samplerName?: string
scheduler?: string
hideInput?: boolean
extensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[]
remainingExtensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[]
extensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: import('~/utils/loras').LoraStackItem[] }[]
remainingExtensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: import('~/utils/loras').LoraStackItem[] }[]
currentClipId?: string
queueId?: string
queueAutoRun?: boolean
queueBudget?: number
stopAfterCurrent?: boolean
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
}
function pendingRoot() {
@@ -139,6 +143,10 @@ export function pendingFromJob(job: Job, overrides: Partial<PendingJob> = {}): P
globalLocks: library.globalLocks,
permanenceRefs: library.permanenceRefs,
shotPermanenceRefs: library.shotPermanenceRefs,
loraName: library.loraName,
loraStack: library.loraStack,
shotLoras: library.shotLoras,
shotLoraStacks: library.shotLoraStacks,
queueId: library.queueId,
queueAutoRun: library.queueAutoRun,
queueBudget: library.queueBudget,
@@ -191,7 +199,11 @@ export function libraryFromPending(pending: PendingJob): NonNullable<Job['librar
stopAfterCurrent: pending.stopAfterCurrent,
globalLocks: pending.globalLocks,
permanenceRefs: pending.permanenceRefs,
shotPermanenceRefs: pending.shotPermanenceRefs
shotPermanenceRefs: pending.shotPermanenceRefs,
loraName: pending.loraName,
loraStack: pending.loraStack,
shotLoras: pending.shotLoras,
shotLoraStacks: pending.shotLoraStacks
}
}
@@ -318,6 +330,8 @@ export async function completePendingIfReady(pending: PendingJob) {
sound: pending.sound,
globalLocks: pending.globalLocks,
permanenceRefs: mergePermanenceRefs(pending.permanenceRefs, pending.shotPermanenceRefs?.[pending.chainIndex || 0]),
loraName: pending.loraName,
loraStack: pending.loraStack,
segmentFirstFrame
})
await purgeComfyArtifacts({
+45 -25
View File
@@ -17,23 +17,7 @@ export interface PromptJob {
const jobs = new Map<string, PromptJob>()
const MAX_JOBS = 20
function normalizeHost(raw: string, port = '') {
let host = String(raw || '').trim().replace(/\/$/, '')
if (!host) return ''
if (!/^https?:\/\//i.test(host)) host = `http://${host}`
try {
const url = new URL(host)
if (port && !url.port) url.port = port
return url.origin
} catch {
return port ? `${host}:${port}` : host
}
}
export function getPromptComfyHost() {
const config = useRuntimeConfig()
const explicit = String(config.promptComfyHost || process.env.PROMPT_COMFY_HOST || '').trim()
if (explicit) return normalizeHost(explicit)
return getSidecarImageHost()
}
@@ -44,14 +28,14 @@ export function promptComfyConfigured() {
async function promptComfyFetch(path: string, init?: RequestInit) {
const host = getPromptComfyHost()
if (!host) {
throw createError({ statusCode: 503, statusMessage: 'Sidecar Qwen VL is not configured. Set IMAGE_COMFY_HOST.' })
throw createError({ statusCode: 503, statusMessage: 'Qwen VL is not configured. Set COMFY_HOST or IMAGE_COMFY_HOST to Beast.' })
}
try {
return await fetch(`${host}${path}`, init)
} catch (error) {
throw createError({
statusCode: 502,
statusMessage: `Sidecar Qwen VL is unreachable (${host})`,
statusMessage: `Beast Qwen VL is unreachable (${host})`,
data: { cause: error instanceof Error ? error.message : String(error) }
})
}
@@ -75,7 +59,7 @@ async function uploadPromptImage(file: { filename: string; data: Buffer; type?:
body.append('type', 'input')
const res = await promptComfyFetch('/upload/image', { method: 'POST', body })
if (!res.ok) {
throw createError({ statusCode: 502, statusMessage: `Sidecar image upload failed (${res.status})` })
throw createError({ statusCode: 502, statusMessage: `Beast image upload failed (${res.status})` })
}
const uploaded = (await res.json()) as { name: string; subfolder?: string; type?: string }
return uploaded.name || filename
@@ -89,14 +73,48 @@ async function queuePromptGraph(graph: unknown, clientId: string) {
})
const payload = await res.json().catch(() => ({}))
if (!res.ok) {
const raw = JSON.stringify(payload)
if (/does not exist|class_type|not found/i.test(raw)) {
throw createError({
statusCode: 503,
statusMessage: 'Beast ComfyUI rejected the Qwen VL graph. The GGUF VLM nodes are missing on this instance — Recommend will not use the sidecar.',
data: payload
})
}
const message = (payload as { error?: { message?: string } }).error?.message
|| (payload as { node_errors?: unknown }).node_errors
|| `Sidecar queue failed (${res.status})`
|| `Beast queue failed (${res.status})`
throw createError({ statusCode: 502, statusMessage: String(message), data: payload })
}
return payload as { prompt_id: string }
}
const QWEN_VL_NODE_TYPES = ['GGUF_VLM_ModelLoader', 'GGUF_VLM_ImageToText', 'AigenPreviewText']
async function assertQwenVlWorkflow() {
const res = await promptComfyFetch('/object_info')
if (!res.ok) {
throw createError({
statusCode: 502,
statusMessage: `Beast ComfyUI did not return node info (${res.status})`
})
}
const info = await res.json().catch(() => null)
if (!info || typeof info !== 'object') {
throw createError({
statusCode: 502,
statusMessage: 'Beast ComfyUI returned unreadable node info'
})
}
const missing = QWEN_VL_NODE_TYPES.filter(name => !(name in (info as Record<string, unknown>)))
if (missing.length) {
throw createError({
statusCode: 503,
statusMessage: `Beast ComfyUI is missing Qwen VL nodes (${missing.join(', ')}). Recommend is wired to Beast only and will not fall back to the sidecar.`
})
}
}
async function fetchPromptHistory(promptId: string) {
const res = await promptComfyFetch(`/history/${encodeURIComponent(promptId)}`)
if (!res.ok) return null
@@ -149,9 +167,9 @@ function historyError(history: Record<string, unknown> | null, promptId: string)
const err = entry?.status?.messages?.find(([type]) => type === 'execution_error')?.[1]
const message = String(err?.exception_message || err?.message || '').trim()
if (status === 'interrupted') return 'Prompt recommend was interrupted.'
if (status === 'error') return message || 'Sidecar Qwen VL reported an execution error'
if (status === 'error') return message || 'Beast Qwen VL reported an execution error'
if (entry?.status?.completed && !extractPromptText(history, promptId)) {
return 'Sidecar Qwen VL finished without prompt text'
return 'Beast Qwen VL finished without prompt text'
}
return null
}
@@ -168,7 +186,7 @@ async function waitForPromptText(promptId: string, timeoutMs = 240_000) {
}
await new Promise(resolve => setTimeout(resolve, 1200))
}
throw createError({ statusCode: 504, statusMessage: 'Sidecar Qwen VL timed out while writing the prompt' })
throw createError({ statusCode: 504, statusMessage: 'Beast Qwen VL timed out while writing the prompt' })
}
export function createPromptJob(): PromptJob {
@@ -221,11 +239,13 @@ export async function runPromptRecommend(job: PromptJob, params: {
mode: string
}) {
job.status = 'running'
job.message = 'Checking Sidecar Qwen VL…'
job.message = 'Checking Beast Qwen VL…'
await ensureSidecarReady((status) => {
job.message = status.message
})
job.message = 'Uploading still to Sidecar Qwen VL…'
job.message = 'Confirming Qwen VL nodes on Beast…'
await assertQwenVlWorkflow()
job.message = 'Uploading still to Beast Qwen VL…'
const imageName = await uploadPromptImage(params.image, job.id)
job.message = 'Qwen VL is reading the still…'
const graph = buildVisionPromptWorkflow({
+31 -8
View File
@@ -1,6 +1,8 @@
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { getJob, listJobs, type Job } from '~/server/utils/jobs'
import { parsePostedLoraStack, persistLoraFields } from '~/server/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
export type ShotQueueStatus = 'idle' | 'running' | 'paused' | 'complete' | 'error'
export type ShotSegmentStatus = 'pending' | 'running' | 'complete' | 'error'
@@ -13,6 +15,8 @@ export interface ShotQueueSegment {
clipId?: string
error?: string
permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]
loraName?: string
loraStack?: LoraStackItem[]
}
export interface ShotQueue {
@@ -46,6 +50,8 @@ export interface ShotQueue {
referenceStillIds?: Array<string | null>
globalLocks?: string
permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]
loraName?: string
loraStack?: LoraStackItem[]
segments: ShotQueueSegment[]
lastError?: string
}
@@ -163,8 +169,10 @@ export async function createShotQueue(params: {
referenceStillIds?: Array<string | null>
globalLocks?: string
permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]
initial: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }
extensions: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[]
loraName?: string
loraStack?: LoraStackItem[]
initial: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }
extensions: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[]
jobId?: string
}): Promise<ShotQueue> {
const now = Date.now()
@@ -198,14 +206,17 @@ export async function createShotQueue(params: {
referenceStillIds: params.referenceStillIds,
globalLocks: params.globalLocks,
permanenceRefs: params.permanenceRefs,
loraName: params.loraName,
loraStack: params.loraStack,
segments: [
{ index: 0, prompt: params.initial.prompt, duration: params.initial.duration, status: 'running', permanenceRefs: params.initial.permanenceRefs },
{ index: 0, prompt: params.initial.prompt, duration: params.initial.duration, status: 'running', permanenceRefs: params.initial.permanenceRefs, ...persistLoraFields(params.initial.loraStack || params.initial.loraName) },
...params.extensions.map((item, index) => ({
index: index + 1,
prompt: item.prompt,
duration: item.duration,
status: 'pending' as const,
permanenceRefs: item.permanenceRefs
permanenceRefs: item.permanenceRefs,
...persistLoraFields(item.loraStack || item.loraName)
}))
]
}
@@ -230,7 +241,7 @@ export async function updateShotQueue(owner: string, id: string, patch: (queue:
export async function patchShotQueueSegments(
owner: string,
id: string,
segments: Array<{ index: number; prompt?: string; duration?: number }>
segments: Array<{ index: number; prompt?: string; duration?: number; loraName?: string; loraStack?: unknown }>
) {
return updateShotQueue(owner, id, (queue) => {
if (queue.status === 'complete') {
@@ -248,6 +259,17 @@ export async function patchShotQueueSegments(
if (typeof next.duration === 'number' && Number.isFinite(next.duration)) {
segment.duration = Math.min(120, Math.max(0.5, Math.round(next.duration * 10) / 10))
}
if ('loraStack' in next || 'loraName' in next) {
const stack = parsePostedLoraStack(next.loraStack ?? next.loraName, 'video')
const fields = persistLoraFields(stack)
if (fields.loraStack) {
segment.loraStack = fields.loraStack
segment.loraName = fields.loraName
} else {
delete segment.loraStack
delete segment.loraName
}
}
}
})
}
@@ -426,10 +448,10 @@ export function listOwnersWithQueues() {
.filter(owner => existsSync(queuesPath(owner)))
}
export function remainingFromQueue(queue: ShotQueue): { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[] {
export function remainingFromQueue(queue: ShotQueue): { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[] {
return queue.segments
.filter(segment => segment.index > 0)
.map(segment => ({ prompt: segment.prompt, duration: segment.duration, permanenceRefs: segment.permanenceRefs }))
.map(segment => ({ prompt: segment.prompt, duration: segment.duration, permanenceRefs: segment.permanenceRefs, ...persistLoraFields(segment.loraStack || segment.loraName) }))
}
export function lastCompletedIndex(queue: ShotQueue) {
@@ -443,6 +465,7 @@ export function liveSegmentPrompt(queue: ShotQueue, extensionIndex: number) {
return {
prompt: segment?.prompt || '',
duration: segment?.duration || 5,
permanenceRefs: segment?.permanenceRefs
permanenceRefs: segment?.permanenceRefs,
...persistLoraFields(segment?.loraStack || segment?.loraName)
}
}
+33 -12
View File
@@ -3,8 +3,9 @@ import { join } from 'node:path'
import { getJob, listJobs, emitJob, type Job } from '~/server/utils/jobs'
import { listPendingJobs, patchPendingJob, readPendingJob, deletePendingJob } from '~/server/utils/pending'
import { getShotQueue } from '~/server/utils/shotQueue'
import { parseVideoWorkflow, type VideoWorkflowId } from '~/utils/videoModels'
import { allowIdentityRefs, type PermanenceRef } from '~/utils/globalLocks'
import { isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow, type VideoWorkflowId } from '~/utils/videoModels'
import { persistLoraFields } from '~/server/utils/loras'
import { resolveLoraStack } from '~/utils/loras'
export type StudioJobStatus = 'waiting' | 'running' | 'held' | 'complete' | 'error' | 'cancelled'
export type StudioJobKind = 'video' | 'edit'
@@ -33,11 +34,15 @@ export interface StudioJobPayload {
hideInput?: boolean
folderLocked?: boolean
referenceStillIds: Array<string | null>
extensions: { prompt: string; duration: number; permanenceRefs?: PermanenceRef[] }[]
extensions: { prompt: string; duration: number; permanenceRefs?: PermanenceRef[]; loraName?: string; loraStack?: import('~/utils/loras').LoraStackItem[] }[]
queueAutoRun: boolean
globalLocks?: string
permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
negative?: string
passes?: { prompt: string }[]
referenceStillId?: string
@@ -636,7 +641,8 @@ async function startStudioEditJob(item: StudioJob) {
chainStep: 1,
chainTotal: 1 + passes.length,
chainLabel: passes.length ? 'Pass 1' : undefined,
passes
passes,
...persistLoraFields(payload.loraStack || payload.loraName)
}
})
@@ -649,7 +655,8 @@ async function startStudioEditJob(item: StudioJob) {
negative: payload.negative || '',
steps: payload.steps,
seed: job.library?.seed || payload.seed,
cfg: payload.cfg
cfg: payload.cfg,
...persistLoraFields(payload.loraStack || payload.loraName)
}).catch((error) => {
const message = error instanceof Error ? error.message : String(error)
if (job.status !== 'error' && job.status !== 'cancelled' && job.status !== 'deferred') {
@@ -680,6 +687,10 @@ export async function startStudioJob(item: StudioJob) {
const { existsSync, readFileSync } = await import('node:fs')
const payload = item.payload
const workflow = parseVideoWorkflow(payload.workflow)
if (isLtxWorkflow(workflow) && !ltxWorkflowEnabled()) {
throw new Error(LTX_DISABLED_MESSAGE)
}
const identity = allowIdentityRefs(
payload.useIdentityRefs,
payload.permanenceRefs,
@@ -720,13 +731,16 @@ export async function startStudioJob(item: StudioJob) {
chainTotal: 1 + extensions.length,
chainLabel: extensions.length ? 'Initial' : undefined,
familyId: item.familyId,
workflow: parseVideoWorkflow(payload.workflow),
workflow: workflow,
useIdentityRefs: identity,
queueAutoRun: extensions.length ? payload.queueAutoRun : false,
queueBudget: extensions.length && payload.queueAutoRun ? extensions.length : 0,
globalLocks: payload.globalLocks,
permanenceRefs: payload.permanenceRefs,
shotPermanenceRefs: payload.shotPermanenceRefs
shotPermanenceRefs: payload.shotPermanenceRefs,
...persistLoraFields(resolveLoraStack(payload.loraStack || payload.loraName, payload.shotLoraStacks?.[0] || payload.shotLoras?.[0])),
shotLoras: payload.shotLoras,
shotLoraStacks: payload.shotLoraStacks
}
if (extensions.length) {
@@ -749,14 +763,18 @@ export async function startStudioJob(item: StudioJob) {
fps: payload.fps,
samplerName: payload.samplerName,
scheduler: payload.scheduler,
workflow: parseVideoWorkflow(payload.workflow),
workflow: workflow,
sound: payload.sound,
useIdentityRefs: identity,
referenceStillIds: payload.referenceStillIds,
globalLocks: payload.globalLocks,
permanenceRefs: payload.permanenceRefs,
initial: { prompt: payload.prompt, duration: payload.duration, permanenceRefs: payload.shotPermanenceRefs?.[0] },
extensions,
...persistLoraFields(payload.loraStack || payload.loraName),
initial: { prompt: payload.prompt, duration: payload.duration, permanenceRefs: payload.shotPermanenceRefs?.[0], ...persistLoraFields(resolveLoraStack(payload.loraStack || payload.loraName, payload.shotLoraStacks?.[0] || payload.shotLoras?.[0])) },
extensions: extensions.map((item, index) => ({
...item,
...persistLoraFields(item.loraStack || item.loraName || payload.shotLoraStacks?.[index + 1] || payload.shotLoras?.[index + 1])
})),
jobId: job.id
})
job.library.queueId = queue.id
@@ -802,10 +820,13 @@ export async function startStudioJob(item: StudioJob) {
samplerName: payload.samplerName,
scheduler: payload.scheduler,
extensions,
workflow: parseVideoWorkflow(payload.workflow),
workflow: workflow,
duration: payload.duration,
useIdentityRefs: identity,
referenceImages
referenceImages,
...persistLoraFields(resolveLoraStack(payload.loraStack || payload.loraName, payload.shotLoraStacks?.[0] || payload.shotLoras?.[0])),
shotLoras: payload.shotLoras,
shotLoraStacks: payload.shotLoraStacks
}).catch((error) => {
const message = error instanceof Error ? error.message : String(error)
if (job.status !== 'error' && job.status !== 'cancelled' && job.status !== 'deferred') {
+24 -5
View File
@@ -21,6 +21,9 @@ import {
updateShotQueue
} from '~/server/utils/shotQueue'
import { composeShotPrompt, allowIdentityRefs, type PermanenceRef } from '~/utils/globalLocks'
import { persistLoraFields } from '~/server/utils/loras'
import { readLoraStack, resolveLoraStack } from '~/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
export type ChainImage = { filename: string; data: Buffer; type?: string }
@@ -38,7 +41,7 @@ type VideoChainParams = {
fps: number
samplerName: string
scheduler: string
extensions: { prompt: string; duration: number }[]
extensions: { prompt: string; duration: number; loraName?: string; loraStack?: LoraStackItem[]; permanenceRefs?: PermanenceRef[] }[]
workflow: VideoWorkflowId
duration: number
useIdentityRefs: boolean
@@ -46,6 +49,10 @@ type VideoChainParams = {
globalLocks?: string
permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: LoraStackItem[][]
}
function sleep(ms: number) {
@@ -119,7 +126,10 @@ function paramsFromJob(job: Job): VideoChainParams {
referenceImages,
globalLocks: library.globalLocks,
permanenceRefs: library.permanenceRefs,
shotPermanenceRefs: library.shotPermanenceRefs
shotPermanenceRefs: library.shotPermanenceRefs,
...persistLoraFields(readLoraStack(library)),
shotLoras: library.shotLoras,
shotLoraStacks: library.shotLoraStacks
}
}
@@ -137,6 +147,7 @@ export async function queueMiniMax(
job.status = 'uploading'
const chainIndex = job.library?.chainIndex || 0
const graphId = chainIndex > 0 ? workflowForExtension(params.workflow) : params.workflow
if (job.library) Object.assign(job.library, persistLoraFields(params.loraStack || params.loraName || job.library.loraStack || job.library.loraName))
const engineName = isLtxWorkflow(graphId) ? 'LTX-2.3' : 'MiniMax H3'
const hasImage = Boolean(params.image?.data?.length)
const uploading = !hasImage
@@ -197,7 +208,8 @@ export async function queueMiniMax(
workflow: graphId,
duration: params.duration,
useIdentityRefs: params.useIdentityRefs,
referenceImageNames: params.useIdentityRefs ? referenceNames : []
referenceImageNames: params.useIdentityRefs ? referenceNames : [],
...persistLoraFields(params.loraStack || params.loraName)
})
const queued = await queuePrompt(graph, job.clientId)
@@ -305,6 +317,10 @@ export async function continueQueuedExtensions(
prompt: live.prompt || extensions[i].prompt,
duration: live.duration || extensions[i].duration
}
const shotStack = resolveLoraStack(
readLoraStack(liveQueue).length ? readLoraStack(liveQueue) : (params.loraStack || params.loraName),
live.loraStack || live.loraName || extensions[i]?.loraStack || extensions[i]?.loraName
)
if (!ext.prompt.trim()) {
throw new Error(`Shot ${i + 2} needs a prompt`)
}
@@ -328,6 +344,7 @@ export async function continueQueuedExtensions(
job.library.extendPart1Path = undefined
job.library.thumb = undefined
job.library.duration = ext.duration
Object.assign(job.library, persistLoraFields(shotStack))
if (liveQueue) {
await updateShotQueue(job.library.ownerKey, liveQueue.id, (queue) => {
const segment = queue.segments.find(item => item.index === i + 1)
@@ -393,7 +410,8 @@ export async function continueQueuedExtensions(
workflow: workflowForExtension(params.workflow),
duration: ext.duration,
useIdentityRefs: identity,
referenceImages: identity ? params.referenceImages : []
referenceImages: identity ? params.referenceImages : [],
...persistLoraFields(shotStack)
})
shotsLeft -= 1
@@ -548,7 +566,8 @@ export async function startQueueBurst(owner: string, queueId: string, count: num
queueBudget: n,
globalLocks: queue.globalLocks,
permanenceRefs: queue.permanenceRefs,
shotPermanenceRefs: queue.segments.map(segment => segment.permanenceRefs || [])
shotPermanenceRefs: queue.segments.map(segment => segment.permanenceRefs || []),
...persistLoraFields(readLoraStack(queue))
}
setQueueJob(queue.id, job.id)
await updateShotQueue(owner, queue.id, (next) => {
+11 -2
View File
@@ -83,10 +83,17 @@ export function emitChainJob(job: Job, event: JobEvent, samplePct?: number) {
emitJob(job, next)
}
function nodeLabel(node: string) {
if (!node) return ''
if (NODE_LABELS[node]) return NODE_LABELS[node]
if (node.startsWith('user:lora')) return 'Applying LoRA'
return ''
}
function isNonSamplerProgress(node: string) {
if (!node) return false
if (isEncodingNode(node)) return true
const label = NODE_LABELS[node] || ''
const label = nodeLabel(node)
return /save|checkpoint|video combine|vhs/i.test(label) && !/sampler/i.test(label)
}
@@ -235,6 +242,8 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
sound: job.library.sound,
globalLocks: job.library.globalLocks,
permanenceRefs: clipRefsFromJob(job.library),
loraName: job.library.loraName,
loraStack: job.library.loraStack,
segmentFirstFrame
})
job.clipId = clip.id
@@ -400,7 +409,7 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
}
return
}
const label = NODE_LABELS[node] || `Running node ${node}`
const label = nodeLabel(node) || `Running node ${node}`
const encoding = isEncodingNode(node)
emitLocal({
type: 'executing',
+12
View File
@@ -3,10 +3,14 @@ import workflowV1 from '../assets/workflow_minimax_video.json'
import workflowV2 from '../assets/workflow_minimax_video_v2.json'
import workflowLtx from '../assets/workflow_ltx_video.json'
import { buildIdentityPrompt } from '~/utils/identityPrompt'
import { applyUserLoraToGraph } from '~/server/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
import {
isLtxWorkflow,
isTextToVideo,
LTX_DISABLED_MESSAGE,
LTX_NEGATIVE,
ltxWorkflowEnabled,
parseVideoWorkflow,
type VideoWorkflowId
} from '~/utils/videoModels'
@@ -33,6 +37,8 @@ export interface GenerateParams {
duration?: number
useIdentityRefs?: boolean
referenceImageNames?: string[]
loraName?: string
loraStack?: LoraStackItem[]
}
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
@@ -74,6 +80,9 @@ function templateFor(id: VideoWorkflowId) {
export function buildWorkflow(params: GenerateParams) {
const version = parseVideoWorkflow(params.workflow)
if (isLtxWorkflow(version) && !ltxWorkflowEnabled()) {
throw new Error(LTX_DISABLED_MESSAGE)
}
if (isLtxWorkflow(version)) return buildLtxWorkflow(params, version)
const graph = structuredClone(templateFor(version === 't2v' ? 'v1' : version === 'v2' ? 'v2' : 'v1'))
@@ -216,6 +225,7 @@ export function buildWorkflow(params: GenerateParams) {
if (graph['105:104']?.inputs) delete graph['105:104'].inputs.audio_vae
}
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
return graph
}
@@ -288,6 +298,7 @@ function buildLtxWorkflow(params: GenerateParams, version: VideoWorkflowId) {
if (graph['12']?.inputs) graph['12'].inputs.latent = ['8', 0]
if (graph['14']?.inputs) graph['14'].inputs.latent_image = ['8', 0]
}
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
return graph
}
@@ -366,6 +377,7 @@ const LABEL_OVERRIDES: Record<string, string> = {
'105:104': 'Encoding prompt & latent',
'105:107': 'Computing duration',
'105:111': 'Setting duration',
'user:lora': 'Applying LoRA',
'105:121': 'Applying turbo LoRA',
'105:122': 'Selecting model path',
'105:123': 'Selecting steps',