Add LoRA stacks with per-item strengths and named video/image generation presets so setups survive reload without mixing Klein onto MiniMax. Keep LTX Pink Cherry and xAIGen LoRAs on xAIGen, and run Recommend on Beast.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-08-28 15:24:44 -05:00
co-authored by Cursor
parent 0af3cddb14
commit f8db0d626a
36 changed files with 1767 additions and 246 deletions
+8 -13
View File
@@ -22,24 +22,19 @@ LIBRARY_DIR=/data/library
# Private clone (xaigen): password login, isolated library, Comfy prefix # Private clone (xaigen): password login, isolated library, Comfy prefix
# AUTH_MODE=password # AUTH_MODE=password
# NUXT_PUBLIC_INSTANCE_NAME=xAIGen # NUXT_PUBLIC_INSTANCE_NAME=xAIGen
# NUXT_PUBLIC_STUDIO=xaigen
# AUTH_USERNAME= # AUTH_USERNAME=
# AUTH_PASSWORD= # AUTH_PASSWORD=
# COMFY_FILENAME_PREFIX=video/xAIGen # COMFY_FILENAME_PREFIX=video/xAIGen
# Desktop Comfy files are deleted after a successful library save (set false to keep them). # Desktop Comfy files are deleted after a successful library save (set false to keep them).
# Also removes Qwen Edit input/output files from the image sidecar after saveStill. # Also removes Qwen Edit input/output files from Beast after saveStill.
# PURGE_COMFY_OUTPUTS=true # PURGE_COMFY_OUTPUTS=true
# Second Comfy box (Sidecar / Qwen VL prompt helper). Image edits stay on Beast (COMFY_HOST). # Recommend / Qwen VL uses Beast (same Comfy as video). Do not point this at 192.168.77.101.
# Recommend sends the still + draft prompt here only — never to Beast. IMAGE_COMFY_HOST=http://192.168.77.7:8198
IMAGE_COMFY_HOST=http://192.168.77.101:8188 # Optional override if prompt-recommend Comfy is not IMAGE_COMFY_HOST / COMFY_HOST
# Optional override if prompt-recommend Comfy is not IMAGE_COMFY_HOST # PROMPT_COMFY_HOST=http://192.168.77.7:8198
# PROMPT_COMFY_HOST=http://192.168.77.101:8188 IMAGE_COMFY_PORT=8198
IMAGE_COMFY_PORT=8188
# Optional override if image Comfy on Beast is not the video instance # Optional override if image Comfy on Beast is not the video instance
# IMAGE_COMFY_PRIMARY_HOST=http://192.168.77.7:8198 # IMAGE_COMFY_PRIMARY_HOST=http://192.168.77.7:8198
# IMAGE_COMFY_FALLBACK_HOST=http://192.168.77.101:8188 IMAGE_COMFY_LAUNCH_CMD=
# Spawned on the sidecar PC (host agent /start and start_comfyui.bat). Listen is 0.0.0.0 so Coolify can reach :8188.
IMAGE_COMFY_LAUNCH_CMD=C:\ComfyUI-QwenEdit\python\python.exe C:\ComfyUI-QwenEdit\ComfyUI\main.py --listen 0.0.0.0 --port 8188 --disable-auto-launch --lowvram --fp8_e4m3fn-unet --disable-cuda-malloc
# Optional host agent on the sidecar (scripts/comfy-host-agent.mjs)
# IMAGE_COMFY_CONTROL_URL=http://192.168.77.101:8199
# IMAGE_COMFY_CONTROL_TOKEN=
+74
View File
@@ -0,0 +1,74 @@
<template>
<div class="flex min-w-0 flex-wrap items-end gap-2">
<label class="min-w-[9rem] flex-1 text-xs">
<span class="mb-1 block text-zinc-500">Preset</span>
<select
:value="selectedId"
class="w-full rounded-xl border border-white/10 bg-zinc-950 px-2 py-1 text-xs outline-none ring-amber-300/40 focus:ring-2"
:disabled="disabled"
@change="onPick"
>
<option value="">Load…</option>
<option
v-for="preset in presets"
:key="preset.id"
:value="preset.id"
>{{ preset.name }}</option>
</select>
</label>
<label class="w-32 text-xs">
<span class="mb-1 block text-zinc-500">Name</span>
<input
:value="name"
class="w-full rounded-xl border border-white/10 bg-zinc-950 px-2 py-1 text-xs outline-none ring-amber-300/40 placeholder:text-zinc-600 focus:ring-2"
placeholder="Save as…"
maxlength="80"
:disabled="disabled"
@input="emit('update:name', ($event.target as HTMLInputElement).value)"
@keydown.enter.prevent="emit('save')"
>
</label>
<button
type="button"
class="rounded-xl border border-white/10 px-2 py-1 text-xs text-zinc-300 hover:border-amber-300/50 hover:text-white disabled:opacity-50"
:disabled="disabled || !name.trim()"
@click="emit('save')"
>
Save
</button>
<button
v-if="selectedId"
type="button"
class="rounded-xl border border-white/10 px-2 py-1 text-xs text-zinc-500 hover:border-red-400/40 hover:text-red-300 disabled:opacity-50"
:disabled="disabled"
@click="emit('remove')"
>
Delete
</button>
</div>
</template>
<script setup lang="ts">
import type { GenerationPreset } from '~/utils/generationPresets'
defineProps<{
presets: GenerationPreset[]
selectedId: string
name: string
disabled?: boolean
}>()
const emit = defineEmits<{
load: [id: string]
save: []
remove: []
'update:name': [value: string]
'update:selectedId': [value: string]
}>()
function onPick(event: Event) {
const id = (event.target as HTMLSelectElement).value
emit('update:selectedId', id)
if (id) emit('load', id)
}
</script>
+140
View File
@@ -0,0 +1,140 @@
<template>
<div class="block min-w-0" :class="compact ? 'text-xs' : 'text-sm'">
<span v-if="label" class="mb-1 block text-zinc-400">{{ label }}</span>
<ul v-if="model.length" class="space-y-2">
<li
v-for="(item, index) in model"
:key="item.name"
class="rounded-xl border border-white/10 bg-zinc-950/80"
:class="compact ? 'px-2 py-1.5' : 'px-3 py-2'"
>
<div class="flex items-center justify-between gap-2">
<span class="min-w-0 truncate text-zinc-200">{{ loraDisplayName(item.name) }}</span>
<button
type="button"
class="shrink-0 text-[11px] text-zinc-500 hover:text-red-300 disabled:opacity-50"
:disabled="disabled"
@click="removeAt(index)"
>
Remove
</button>
</div>
<div class="mt-1.5 grid grid-cols-2 gap-2">
<label class="block min-w-0">
<span class="mb-0.5 flex justify-between text-[10px] uppercase tracking-wider text-zinc-500">
<span>Model</span>
<span class="tabular-nums normal-case text-zinc-400">{{ formatLoraStrength(item.strengthModel) }}</span>
</span>
<input
type="range"
class="aigen-slider w-full"
:min="LORA_STRENGTH_MIN"
:max="LORA_STRENGTH_MAX"
:step="LORA_STRENGTH_STEP"
:value="item.strengthModel"
:disabled="disabled"
@input="setStrength(index, 'strengthModel', $event)"
>
</label>
<label class="block min-w-0" :class="{ 'opacity-50': !clipStrength }">
<span class="mb-0.5 flex justify-between text-[10px] uppercase tracking-wider text-zinc-500">
<span>CLIP</span>
<span class="tabular-nums normal-case text-zinc-400">{{ formatLoraStrength(item.strengthClip) }}</span>
</span>
<input
type="range"
class="aigen-slider w-full"
:min="LORA_STRENGTH_MIN"
:max="LORA_STRENGTH_MAX"
:step="LORA_STRENGTH_STEP"
:value="item.strengthClip"
:disabled="disabled || !clipStrength"
@input="setStrength(index, 'strengthClip', $event)"
>
</label>
</div>
</li>
</ul>
<p v-if="!clipStrength && model.length" class="mt-1 text-[11px] text-zinc-500">CLIP unused on this graph — model strength only.</p>
<select
v-if="availableNames.length && model.length < LORA_STACK_MAX"
class="w-full rounded-xl border border-white/10 bg-zinc-950 outline-none ring-amber-300/40 focus:ring-2 disabled:opacity-50"
:class="[
compact ? 'px-2 py-1 text-xs' : 'px-3 py-2 text-sm',
model.length ? 'mt-2' : ''
]"
:disabled="disabled"
:value="''"
@change="onAdd"
>
<option value="">{{ model.length ? addLabel : emptyLabel }}</option>
<option
v-for="name in availableNames"
:key="name"
:value="name"
>{{ loraDisplayName(name) }}</option>
</select>
<p v-else-if="!model.length" class="text-[11px] text-zinc-500">{{ emptyLabel }}</p>
</div>
</template>
<script setup lang="ts">
import {
LORA_STACK_MAX,
LORA_STRENGTH_MAX,
LORA_STRENGTH_MIN,
LORA_STRENGTH_STEP,
clampLoraStrength,
formatLoraStrength,
loraDisplayName,
makeLoraStackItem,
type LoraStackItem
} from '~/utils/loras'
const props = withDefaults(defineProps<{
names: string[]
label?: string
emptyLabel?: string
addLabel?: string
compact?: boolean
disabled?: boolean
clipStrength?: boolean
}>(), {
label: 'LoRA',
emptyLabel: 'None',
addLabel: 'Add LoRA',
compact: false,
disabled: false,
clipStrength: true
})
const model = defineModel<LoraStackItem[]>({ default: () => [] })
const availableNames = computed(() => {
const taken = new Set(model.value.map(item => item.name.replace(/\\/g, '/').toLowerCase()))
return props.names.filter(name => !taken.has(name.replace(/\\/g, '/').toLowerCase()))
})
function onAdd(event: Event) {
const name = (event.target as HTMLSelectElement).value
;(event.target as HTMLSelectElement).value = ''
addLora(name)
}
function addLora(name: string) {
const item = makeLoraStackItem(name)
if (!item || model.value.length >= LORA_STACK_MAX) return
const taken = new Set(model.value.map(entry => entry.name.replace(/\\/g, '/').toLowerCase()))
if (taken.has(item.name.replace(/\\/g, '/').toLowerCase())) return
model.value = [...model.value, item]
}
function removeAt(index: number) {
model.value = model.value.filter((_, i) => i !== index)
}
function setStrength(index: number, key: 'strengthModel' | 'strengthClip', event: Event) {
const value = clampLoraStrength((event.target as HTMLInputElement).value)
model.value = model.value.map((item, i) => i === index ? { ...item, [key]: value } : item)
}
</script>
+1 -1
View File
@@ -127,7 +127,7 @@
<div> <div>
<h3 class="text-sm font-semibold uppercase tracking-wider text-zinc-500">Prompt helper</h3> <h3 class="text-sm font-semibold uppercase tracking-wider text-zinc-500">Prompt helper</h3>
<p class="mt-1 text-sm text-zinc-400"> <p class="mt-1 text-sm text-zinc-400">
Instruction sets sent to Sidecar Qwen VL with your still and draft. Use these later for multi-shot video scripts. Instruction sets sent to Beast Qwen VL with your still and draft. Use these later for multi-shot video scripts.
</p> </p>
</div> </div>
+3 -1
View File
@@ -53,7 +53,9 @@ export default defineNuxtConfig({
public: { public: {
authEnabled: process.env.NUXT_PUBLIC_AUTH_ENABLED === 'true', authEnabled: process.env.NUXT_PUBLIC_AUTH_ENABLED === 'true',
authMode: process.env.NUXT_PUBLIC_AUTH_MODE || process.env.AUTH_MODE || '', authMode: process.env.NUXT_PUBLIC_AUTH_MODE || process.env.AUTH_MODE || '',
instanceName: process.env.NUXT_PUBLIC_INSTANCE_NAME || 'AIGen' instanceName: process.env.NUXT_PUBLIC_INSTANCE_NAME || 'AIGen',
// NUXT_PUBLIC_STUDIO=xaigen on the xAIGen Coolify app (LTX Pink Cherry + xAIGen-prefixed LoRAs)
studio: process.env.NUXT_PUBLIC_STUDIO || ''
} }
}, },
nitro: { nitro: {
+360 -22
View File
@@ -6,7 +6,7 @@
<div class="flex h-10 w-10 items-center justify-center rounded-xl bg-amber-400 text-zinc-950 font-display font-extrabold">A</div> <div class="flex h-10 w-10 items-center justify-center rounded-xl bg-amber-400 text-zinc-950 font-display font-extrabold">A</div>
<div> <div>
<h1 class="font-display text-xl font-bold leading-none">{{ instanceName }}</h1> <h1 class="font-display text-xl font-bold leading-none">{{ instanceName }}</h1>
<p class="hidden text-[11px] uppercase tracking-[0.22em] text-zinc-500 sm:block">MiniMax H3 · LTX-2.3 · Flux.2 Klein</p> <p class="hidden text-[11px] uppercase tracking-[0.22em] text-zinc-500 sm:block">{{ ltxEnabled ? 'MiniMax H3 · LTX-2.3 · Flux.2 Klein' : 'MiniMax H3 · Flux.2 Klein' }}</p>
</div> </div>
</div> </div>
<div class="flex items-center gap-3 text-sm"> <div class="flex items-center gap-3 text-sm">
@@ -193,7 +193,7 @@
</p> </p>
<div v-if="recommendBusy || recommendText || recommendError" class="mt-3 max-h-64 overflow-y-auto rounded-2xl border border-amber-300/20 bg-amber-400/5 p-4"> <div v-if="recommendBusy || recommendText || recommendError" class="mt-3 max-h-64 overflow-y-auto rounded-2xl border border-amber-300/20 bg-amber-400/5 p-4">
<p class="text-xs font-medium uppercase tracking-wide text-amber-200/80">Recommended prompt</p> <p class="text-xs font-medium uppercase tracking-wide text-amber-200/80">Recommended prompt</p>
<p v-if="recommendBusy" class="mt-2 text-sm text-zinc-400">{{ recommendMessage || 'Sidecar Qwen VL is reading the still…' }}</p> <p v-if="recommendBusy" class="mt-2 text-sm text-zinc-400">{{ recommendMessage || 'Beast Qwen VL is reading the still…' }}</p>
<p v-else-if="recommendError" class="mt-2 text-sm text-red-300">{{ recommendError }}</p> <p v-else-if="recommendError" class="mt-2 text-sm text-red-300">{{ recommendError }}</p>
<div v-else-if="recommendedShots.length > 1" class="mt-2 space-y-2"> <div v-else-if="recommendedShots.length > 1" class="mt-2 space-y-2">
<div <div
@@ -308,6 +308,17 @@
<span class="font-normal text-zinc-500"> · shot {{ shot.n }} · {{ formatClipDuration(duration) }}</span> <span class="font-normal text-zinc-500"> · shot {{ shot.n }} · {{ formatClipDuration(duration) }}</span>
</p> </p>
<p class="mt-1 whitespace-pre-wrap text-xs text-zinc-300 [overflow-wrap:anywhere]">{{ composedShotPreview(shot.prompt, index, shot.n) }}</p> <p class="mt-1 whitespace-pre-wrap text-xs text-zinc-300 [overflow-wrap:anywhere]">{{ composedShotPreview(shot.prompt, index, shot.n) }}</p>
<LoraSelect
class="mt-2"
:model-value="shotLoraStacks[shot.n] || []"
:names="videoLoraOptions"
label="LoRA"
empty-label="Default"
add-label="Add LoRA"
compact
:clip-strength="false"
@update:model-value="setShotLoraStack(shot.n, $event)"
/>
<PermanenceRefStrip <PermanenceRefStrip
class="mt-2" class="mt-2"
:refs="shotPermanenceRefs[shot.n] || []" :refs="shotPermanenceRefs[shot.n] || []"
@@ -350,6 +361,17 @@
placeholder="Motion prompt for this extension…" placeholder="Motion prompt for this extension…"
/> />
<DurationField v-model="item.duration" class="mt-3" label="Duration" /> <DurationField v-model="item.duration" class="mt-3" label="Duration" />
<LoraSelect
class="mt-3"
:model-value="item.loraStack || []"
:names="videoLoraOptions"
label="LoRA"
empty-label="Default"
add-label="Add LoRA"
compact
:clip-strength="false"
@update:model-value="item.loraStack = $event"
/>
<PermanenceRefStrip <PermanenceRefStrip
class="mt-3" class="mt-3"
:refs="shotPermanenceRefs[index + 2] || []" :refs="shotPermanenceRefs[index + 2] || []"
@@ -442,7 +464,7 @@
</div> </div>
<div v-if="studioMode === 'video'" class="space-y-3"> <div v-if="studioMode === 'video'" class="space-y-3">
<div> <div v-if="ltxEnabled">
<p class="mb-2 text-xs uppercase tracking-wider text-zinc-500">Video model</p> <p class="mb-2 text-xs uppercase tracking-wider text-zinc-500">Video model</p>
<div class="grid grid-cols-2 gap-2"> <div class="grid grid-cols-2 gap-2">
<button <button
@@ -568,6 +590,29 @@
</div> </div>
</div> </div>
<div v-if="studioMode === 'video'" class="max-w-md space-y-2">
<LoraSelect v-model="videoLoraStack" :names="videoLoraOptions" label="LoRA" empty-label="None" compact :clip-strength="false" />
<GenerationPresetBar
:presets="videoGenerationPresets"
v-model:selected-id="videoPresetId"
v-model:name="videoPresetName"
@load="loadGenerationPreset"
@save="saveGenerationPreset"
@remove="deleteGenerationPreset"
/>
</div>
<div v-else class="max-w-md space-y-2">
<LoraSelect v-model="imageLoraStack" :names="imageLoraOptions" label="LoRA" empty-label="None" compact />
<GenerationPresetBar
:presets="imageGenerationPresets"
v-model:selected-id="imagePresetId"
v-model:name="imagePresetName"
@load="loadGenerationPreset"
@save="saveGenerationPreset"
@remove="deleteGenerationPreset"
/>
</div>
<details v-if="studioMode === 'video'" class="rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3"> <details v-if="studioMode === 'video'" class="rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3">
<summary class="cursor-pointer text-sm font-medium">Generation settings</summary> <summary class="cursor-pointer text-sm font-medium">Generation settings</summary>
<div class="mt-4 space-y-4"> <div class="mt-4 space-y-4">
@@ -643,6 +688,7 @@
<span class="mb-1 block text-zinc-400">Seed</span> <span class="mb-1 block text-zinc-400">Seed</span>
<input v-model="seedInput" class="w-full rounded-xl border border-white/10 bg-zinc-950 px-3 py-2" placeholder="random"> <input v-model="seedInput" class="w-full rounded-xl border border-white/10 bg-zinc-950 px-3 py-2" placeholder="random">
</label> </label>
<LoraSelect v-model="videoLoraStack" :names="videoLoraOptions" label="LoRA" empty-label="None" :clip-strength="false" />
<div> <div>
<p class="mb-2 text-xs uppercase tracking-wider text-zinc-500">Sampler & scheduler</p> <p class="mb-2 text-xs uppercase tracking-wider text-zinc-500">Sampler & scheduler</p>
<div class="grid grid-cols-2 gap-3"> <div class="grid grid-cols-2 gap-3">
@@ -691,6 +737,7 @@
<input v-model="seedInput" class="w-full rounded-xl border border-white/10 bg-zinc-950 px-3 py-2 text-sm" placeholder="random"> <input v-model="seedInput" class="w-full rounded-xl border border-white/10 bg-zinc-950 px-3 py-2 text-sm" placeholder="random">
</label> </label>
</div> </div>
<LoraSelect v-model="imageLoraStack" :names="imageLoraOptions" label="LoRA" empty-label="None" />
<p class="text-xs text-zinc-500">Graph default is 20 steps, CFG 1, euler. A second still switches the two-image branch on.</p> <p class="text-xs text-zinc-500">Graph default is 20 steps, CFG 1, euler. A second still switches the two-image branch on.</p>
</div> </div>
</details> </details>
@@ -975,6 +1022,7 @@
label="Extension duration" label="Extension duration"
hint="Type any length, use the slider, or drag up/down in the box." hint="Type any length, use the slider, or drag up/down in the box."
/> />
<LoraSelect v-model="extendLoraStack" :names="videoLoraOptions" label="LoRA" empty-label="Same as clip / none" compact :clip-strength="false" />
<button <button
type="button" type="button"
class="w-full rounded-2xl bg-amber-400 px-4 py-2.5 text-sm font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50" class="w-full rounded-2xl bg-amber-400 px-4 py-2.5 text-sm font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50"
@@ -1550,9 +1598,20 @@ import {
type PermanenceRef type PermanenceRef
} from '~/utils/globalLocks' } from '~/utils/globalLocks'
import { import {
filterLoraStackForStudio,
loraStacksEqual,
normalizeLoraStack,
persistLoraFields,
readLoraStack,
type LoraStackItem
} from '~/utils/loras'
import type { GenerationPreset } from '~/utils/generationPresets'
import {
coerceVideoWorkflow,
composeVideoWorkflow, composeVideoWorkflow,
isLtxWorkflow, isLtxWorkflow,
isTextToVideo, isTextToVideo,
ltxWorkflowEnabled,
minimaxGraphOf, minimaxGraphOf,
parseVideoWorkflow, parseVideoWorkflow,
videoEngineOf, videoEngineOf,
@@ -1622,6 +1681,8 @@ interface LibraryClip {
hideInput?: boolean hideInput?: boolean
globalLocks?: string globalLocks?: string
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
loraName?: string
loraStack?: LoraStackItem[]
} }
interface RetryDraft { interface RetryDraft {
@@ -1646,17 +1707,23 @@ interface RetryDraft {
fps?: number fps?: number
samplerName?: string samplerName?: string
scheduler?: string scheduler?: string
extensions?: { prompt: string; duration: number; permanenceRefs?: PermanenceRef[] }[] extensions?: { prompt: string; duration: number; permanenceRefs?: PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[]
workflow?: string workflow?: string
globalLocks?: string globalLocks?: string
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][] shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: LoraStackItem[][]
} }
interface QueuedExtension { interface QueuedExtension {
id: string id: string
prompt: string prompt: string
duration: number duration: number
loraName?: string
loraStack?: LoraStackItem[]
} }
interface QueuedEditPass { interface QueuedEditPass {
@@ -1721,6 +1788,18 @@ const aspect = ref<AspectId>('auto')
const imageWidth = ref(0) const imageWidth = ref(0)
const imageHeight = ref(0) const imageHeight = ref(0)
const turbo = ref(true) const turbo = ref(true)
const videoLoraStack = ref<LoraStackItem[]>([])
const imageLoraStack = ref<LoraStackItem[]>([])
const videoLoras = ref<string[]>([])
const imageLoras = ref<string[]>([])
const shotLoraStacks = ref<Record<number, LoraStackItem[]>>({})
const extendLoraStack = ref<LoraStackItem[]>([])
const videoGenerationPresets = ref<GenerationPreset[]>([])
const imageGenerationPresets = ref<GenerationPreset[]>([])
const videoPresetId = ref('')
const imagePresetId = ref('')
const videoPresetName = ref('')
const imagePresetName = ref('')
const CFG_MIN = 1 const CFG_MIN = 1
const CFG_MAX = 10 const CFG_MAX = 10
const CFG_STEP = 0.5 const CFG_STEP = 0.5
@@ -1814,6 +1893,10 @@ const userName = ref('')
const authEnabled = ref(false) const authEnabled = ref(false)
const authMode = ref('') const authMode = ref('')
const instanceName = ref(String(runtimeConfig.public.instanceName || 'AIGen')) const instanceName = ref(String(runtimeConfig.public.instanceName || 'AIGen'))
const ltxEnabled = computed(() => ltxWorkflowEnabled({
studio: runtimeConfig.public.studio,
instanceName: instanceName.value
}))
const showPrivacyToggles = computed(() => /xaigen/i.test(String(instanceName.value))) const showPrivacyToggles = computed(() => /xaigen/i.test(String(instanceName.value)))
const folders = ref<LibraryFolder[]>([]) const folders = ref<LibraryFolder[]>([])
const clips = ref<LibraryClip[]>([]) const clips = ref<LibraryClip[]>([])
@@ -1871,6 +1954,13 @@ const minimaxGraph = ref<MiniMaxGraphId>('v1')
const videoWorkflow = computed(() => composeVideoWorkflow(videoEngine.value, videoStart.value, minimaxGraph.value)) const videoWorkflow = computed(() => composeVideoWorkflow(videoEngine.value, videoStart.value, minimaxGraph.value))
const textToVideo = computed(() => isTextToVideo(videoWorkflow.value)) const textToVideo = computed(() => isTextToVideo(videoWorkflow.value))
const ltxVideo = computed(() => isLtxWorkflow(videoWorkflow.value)) const ltxVideo = computed(() => isLtxWorkflow(videoWorkflow.value))
const videoLoraOptions = computed(() => mergeLoraOptions(videoLoras.value, [
...videoLoraStack.value.map(item => item.name),
...extendLoraStack.value.map(item => item.name),
...Object.values(shotLoraStacks.value).flatMap(stack => stack.map(item => item.name)),
...extensionQueue.value.flatMap(item => (item.loraStack || []).map(entry => entry.name))
]))
const imageLoraOptions = computed(() => mergeLoraOptions(imageLoras.value, imageLoraStack.value.map(item => item.name)))
const useIdentityRefs = ref(false) const useIdentityRefs = ref(false)
const identityRefs = ref<(File | null)[]>([null, null, null, null]) const identityRefs = ref<(File | null)[]>([null, null, null, null])
const identityPreviews = ref(['', '', '', '']) const identityPreviews = ref(['', '', '', ''])
@@ -2074,12 +2164,12 @@ const recommendDisabled = computed(() => {
return false return false
}) })
const recommendHint = computed(() => { const recommendHint = computed(() => {
if (!imageSidecarConfigured.value) return 'Sidecar Qwen VL is not configured. Set IMAGE_COMFY_HOST.' if (!imageSidecarConfigured.value) return 'Qwen VL is not configured. Set COMFY_HOST or IMAGE_COMFY_HOST to Beast.'
if (!file.value) return 'Load a still first.' if (!file.value) return 'Load a still first.'
if (!prompt.value.trim()) return 'Type a draft prompt to enhance.' if (!prompt.value.trim()) return 'Type a draft prompt to enhance.'
if (!imageSidecarOk.value) return 'Sidecar is offline. Recommend will start Qwen VL on 192.168.77.101.' if (!imageSidecarOk.value) return 'Beast Comfy is offline. Recommend needs Qwen VL on that instance.'
if (imageSidecarBusy.value) return 'Sidecar is busy — this may wait in its queue.' if (imageSidecarBusy.value) return 'Beast Comfy is busy — this may wait in its queue.'
return 'Send this still and draft to Sidecar Qwen VL.' return 'Send this still and draft to Beast Qwen VL.'
}) })
const editBlocked = computed(() => { const editBlocked = computed(() => {
if (!file.value || !prompt.value.trim() || !folderId.value) return true if (!file.value || !prompt.value.trim() || !folderId.value) return true
@@ -2548,6 +2638,12 @@ function applyComfyHealth(health: {
imageSidecarHost.value = String(health?.imageComfy?.sidecar?.host || '') imageSidecarHost.value = String(health?.imageComfy?.sidecar?.host || '')
} }
async function pollLoras() {
const listed = await $fetch<{ image?: string[]; video?: string[] }>('/api/loras').catch(() => ({ image: [], video: [] }))
videoLoras.value = listed.video || []
imageLoras.value = listed.image || []
}
async function pollComfyHealth() { async function pollComfyHealth() {
const health = await $fetch<{ comfy?: { ok?: boolean; busy?: boolean } }>('/api/health').catch(() => ({ comfy: { ok: false, busy: false } })) const health = await $fetch<{ comfy?: { ok?: boolean; busy?: boolean } }>('/api/health').catch(() => ({ comfy: { ok: false, busy: false } }))
applyComfyHealth(health) applyComfyHealth(health)
@@ -2570,9 +2666,16 @@ onMounted(async () => {
const start = localStorage.getItem('aigen-video-start') const start = localStorage.getItem('aigen-video-start')
if (engine === 'ltx' || engine === 'minimax') videoEngine.value = engine if (engine === 'ltx' || engine === 'minimax') videoEngine.value = engine
if (start === 'text' || start === 'still') videoStart.value = start if (start === 'text' || start === 'still') videoStart.value = start
if (videoEngine.value === 'ltx') applyEngineDefaults('ltx') if (!ltxEnabled.value && videoEngine.value === 'ltx') {
videoEngine.value = 'minimax'
applyEngineDefaults('minimax')
} else if (videoEngine.value === 'ltx') {
applyEngineDefaults('ltx')
}
} catch { /* ignore */ } } catch { /* ignore */ }
useIdentityRefs.value = localStorage.getItem('aigen-use-identity-refs') === 'true' useIdentityRefs.value = localStorage.getItem('aigen-use-identity-refs') === 'true'
videoLoraStack.value = readStoredLoraStack('aigen-video-lora')
imageLoraStack.value = readStoredLoraStack('aigen-image-lora')
globalLocks.value = localStorage.getItem(LOCKS_STORE) || '' globalLocks.value = localStorage.getItem(LOCKS_STORE) || ''
try { try {
const savedRefs = JSON.parse(localStorage.getItem(REFS_STORE) || '') const savedRefs = JSON.parse(localStorage.getItem(REFS_STORE) || '')
@@ -2591,6 +2694,8 @@ onMounted(async () => {
loadLibrary().catch(() => applyLibrary({ folders: [], clips: [], stills: [], drafts: [] })) loadLibrary().catch(() => applyLibrary({ folders: [], clips: [], stills: [], drafts: [] }))
]) ])
applyComfyHealth(health) applyComfyHealth(health)
void pollLoras()
void loadGenerationPresets()
userName.value = me.user?.name || me.user?.email || '' userName.value = me.user?.name || me.user?.email || ''
authEnabled.value = Boolean(me.authEnabled) authEnabled.value = Boolean(me.authEnabled)
authMode.value = me.authMode || '' authMode.value = me.authMode || ''
@@ -2600,6 +2705,7 @@ onMounted(async () => {
healthTimer = setInterval(() => { healthTimer = setInterval(() => {
if (typeof document !== 'undefined' && document.hidden) return if (typeof document !== 'undefined' && document.hidden) return
void pollComfyHealth() void pollComfyHealth()
void pollLoras()
void refreshStudioQueue() void refreshStudioQueue()
if (activeQueueId.value) void refreshLiveQueue() if (activeQueueId.value) void refreshLiveQueue()
}, 5000) }, 5000)
@@ -2680,6 +2786,13 @@ watch(autoplayEnabled, (value) => {
} }
}) })
watch(ltxEnabled, (on) => {
if (!on && videoEngine.value === 'ltx') {
videoEngine.value = 'minimax'
applyEngineDefaults('minimax')
}
})
watch(videoWorkflow, (value) => { watch(videoWorkflow, (value) => {
try { try {
localStorage.setItem('aigen-video-workflow', value) localStorage.setItem('aigen-video-workflow', value)
@@ -2720,6 +2833,14 @@ watch(turbo, (on) => {
cfg.value = ltxVideo.value ? 3.5 : (on ? CFG_TURBO : CFG_QUALITY) cfg.value = ltxVideo.value ? 3.5 : (on ? CFG_TURBO : CFG_QUALITY)
}) })
watch(videoLoraStack, (value) => {
try { localStorage.setItem('aigen-video-lora', JSON.stringify(value)) } catch { /* ignore */ }
}, { deep: true })
watch(imageLoraStack, (value) => {
try { localStorage.setItem('aigen-image-lora', JSON.stringify(value)) } catch { /* ignore */ }
}, { deep: true })
function clampDuration(value: number) { function clampDuration(value: number) {
if (!Number.isFinite(value)) return 5 if (!Number.isFinite(value)) return 5
return Math.min(120, Math.max(0.5, Math.round(value * 10) / 10)) return Math.min(120, Math.max(0.5, Math.round(value * 10) / 10))
@@ -2752,6 +2873,189 @@ function restoreScheduler(raw?: string) {
return raw === 'ddim_uniform' || raw === 'sgm_uniform' || raw === 'simple' ? raw : 'simple' return raw === 'ddim_uniform' || raw === 'sgm_uniform' || raw === 'simple' ? raw : 'simple'
} }
function readStoredLoraStack(key: string) {
try {
return readLoraStack(localStorage.getItem(key) || '')
} catch {
return [] as LoraStackItem[]
}
}
function mergeLoraOptions(listed: string[], extra: string[]) {
const names = [...listed]
for (const name of extra) {
const value = String(name || '').trim()
if (value && !names.includes(value)) names.push(value)
}
return names
}
function setShotLoraStack(shotN: number, stack: LoraStackItem[]) {
const next = { ...shotLoraStacks.value }
const items = normalizeLoraStack(stack)
if (items.length) next[shotN] = items
else delete next[shotN]
shotLoraStacks.value = next
}
function applyLoraSelection(stack?: unknown, shotStacks?: Array<unknown>) {
videoLoraStack.value = readLoraStack(stack as { loraStack?: unknown; loraName?: unknown } | string | null)
const next: Record<number, LoraStackItem[]> = {}
;(shotStacks || []).forEach((item, index) => {
const value = readLoraStack(item as { loraStack?: unknown; loraName?: unknown } | string | null)
if (value.length && !loraStacksEqual(value, videoLoraStack.value)) next[index + 1] = value
})
shotLoraStacks.value = next
}
const currentPresetKind = computed(() => studioMode.value === 'edit' ? 'image' : 'video')
async function loadGenerationPresets() {
const [video, image] = await Promise.all([
$fetch<{ presets?: GenerationPreset[] }>('/api/generation-presets', { query: { kind: 'video' } }).catch(() => ({ presets: [] })),
$fetch<{ presets?: GenerationPreset[] }>('/api/generation-presets', { query: { kind: 'image' } }).catch(() => ({ presets: [] }))
])
videoGenerationPresets.value = video.presets || []
imageGenerationPresets.value = image.presets || []
}
function currentPresetSnapshot() {
if (studioMode.value === 'edit') {
return {
kind: 'image' as const,
loraStack: filterLoraStackForStudio(imageLoraStack.value, ltxEnabled.value),
settings: {
steps: editSteps.value,
cfg: editCfg.value,
negative: editNegative.value
}
}
}
return {
kind: 'video' as const,
loraStack: filterLoraStackForStudio(videoLoraStack.value, ltxEnabled.value),
settings: {
workflow: videoWorkflow.value,
turbo: turbo.value,
cfg: cfg.value,
fps: fps.value,
samplerName: samplerName.value,
scheduler: scheduler.value,
duration: duration.value,
aspect: aspect.value,
sound: withSound.value
}
}
}
function applyGenerationPreset(preset: GenerationPreset) {
const stack = filterLoraStackForStudio(preset.loraStack, ltxEnabled.value)
const skipped = preset.loraStack.length - stack.length
if (preset.kind === 'image') {
imageLoraStack.value = stack
const settings = preset.settings as { steps?: number; cfg?: number; negative?: string }
if (typeof settings.steps === 'number') editSteps.value = settings.steps
if (typeof settings.cfg === 'number') editCfg.value = settings.cfg
if (typeof settings.negative === 'string') editNegative.value = settings.negative
imagePresetId.value = preset.id
imagePresetName.value = preset.name
} else {
videoLoraStack.value = stack
const settings = preset.settings as {
workflow?: string
turbo?: boolean
cfg?: number
fps?: number
samplerName?: string
scheduler?: string
duration?: number
aspect?: string
sound?: boolean
}
const wantedWorkflow = String(settings.workflow || '')
applyVideoWorkflow(wantedWorkflow)
if (!ltxEnabled.value && isLtxWorkflow(parseVideoWorkflow(wantedWorkflow))) {
toast('LTX Pink Cherry is not available on this instance')
}
restoreAdvanced({
turbo: settings.turbo !== false,
aspect: settings.aspect,
cfg: settings.cfg,
fps: settings.fps,
samplerName: settings.samplerName,
scheduler: settings.scheduler
})
if (typeof settings.duration === 'number') duration.value = clampDuration(settings.duration)
if (typeof settings.sound === 'boolean') withSound.value = settings.sound
videoPresetId.value = preset.id
videoPresetName.value = preset.name
}
toast(skipped
? `Loaded “${preset.name}”. Skipped ${skipped} xAIGen-only LoRA${skipped === 1 ? '' : 's'}.`
: `Loaded “${preset.name}”.`)
}
function loadGenerationPreset(id: string) {
const list = currentPresetKind.value === 'image' ? imageGenerationPresets.value : videoGenerationPresets.value
const preset = list.find(item => item.id === id)
if (!preset) return
applyGenerationPreset(preset)
}
async function saveGenerationPreset() {
const name = (currentPresetKind.value === 'image' ? imagePresetName.value : videoPresetName.value).trim()
if (!name) {
toast('Name the preset before saving.')
return
}
const list = currentPresetKind.value === 'image' ? imageGenerationPresets.value : videoGenerationPresets.value
const existing = list.find(item => item.name.toLowerCase() === name.toLowerCase())
if (existing && !window.confirm(`Overwrite preset “${existing.name}”?`)) return
try {
const body = { name, overwrite: Boolean(existing), ...currentPresetSnapshot() }
const saved = await $fetch<{ preset: GenerationPreset; presets: GenerationPreset[] }>('/api/generation-presets', {
method: 'POST',
body
})
if (saved.preset.kind === 'image') {
imageGenerationPresets.value = saved.presets
imagePresetId.value = saved.preset.id
imagePresetName.value = saved.preset.name
} else {
videoGenerationPresets.value = saved.presets
videoPresetId.value = saved.preset.id
videoPresetName.value = saved.preset.name
}
toast(existing ? `Overwrote “${saved.preset.name}”.` : `Saved “${saved.preset.name}”.`)
} catch (error: any) {
toast(error?.data?.statusMessage || 'Could not save that preset')
}
}
async function deleteGenerationPreset() {
const id = currentPresetKind.value === 'image' ? imagePresetId.value : videoPresetId.value
const list = currentPresetKind.value === 'image' ? imageGenerationPresets.value : videoGenerationPresets.value
const preset = list.find(item => item.id === id)
if (!id || !preset) return
if (!window.confirm(`Delete preset “${preset.name}”?`)) return
try {
const saved = await $fetch<{ presets: GenerationPreset[] }>(`/api/generation-presets/${id}`, {
method: 'DELETE',
query: { kind: currentPresetKind.value }
})
if (currentPresetKind.value === 'image') {
imageGenerationPresets.value = saved.presets
imagePresetId.value = ''
} else {
videoGenerationPresets.value = saved.presets
videoPresetId.value = ''
}
toast(`Deleted “${preset.name}”.`)
} catch (error: any) {
toast(error?.data?.statusMessage || 'Could not delete that preset')
}
}
function restoreAdvanced(item: { turbo: boolean; aspect?: string; width?: number; height?: number; cfg?: number; fps?: number; samplerName?: string; scheduler?: string }) { function restoreAdvanced(item: { turbo: boolean; aspect?: string; width?: number; height?: number; cfg?: number; fps?: number; samplerName?: string; scheduler?: string }) {
cfgTouched.value = true cfgTouched.value = true
turbo.value = item.turbo turbo.value = item.turbo
@@ -2803,8 +3107,8 @@ async function recommendPrompt() {
recommendText.value = '' recommendText.value = ''
recommendApplied.value = false recommendApplied.value = false
recommendMessage.value = imageSidecarOk.value recommendMessage.value = imageSidecarOk.value
? 'Uploading still to Sidecar Qwen VL…' ? 'Uploading still to Beast Qwen VL…'
: 'Sidecar is offline. Starting Qwen VL…' : 'Checking Beast Comfy…'
const body = new FormData() const body = new FormData()
body.append('image', file.value) body.append('image', file.value)
body.append('prompt', prompt.value) body.append('prompt', prompt.value)
@@ -2823,7 +3127,7 @@ async function recommendPrompt() {
recommendMessage.value = snap.message || recommendMessage.value recommendMessage.value = snap.message || recommendMessage.value
if (snap.status === 'complete') { if (snap.status === 'complete') {
recommendText.value = String(snap.text || '').trim() recommendText.value = String(snap.text || '').trim()
if (!recommendText.value) recommendError.value = 'Sidecar returned an empty prompt' if (!recommendText.value) recommendError.value = 'Beast Qwen VL returned an empty prompt'
break break
} }
if (snap.status === 'error') { if (snap.status === 'error') {
@@ -2919,6 +3223,7 @@ function clearVideoForm() {
globalLocks.value = '' globalLocks.value = ''
familyPermanenceRefs.value = [] familyPermanenceRefs.value = []
shotPermanenceRefs.value = {} shotPermanenceRefs.value = {}
shotLoraStacks.value = {}
identityRefs.value.forEach((_, index) => clearIdentityRef(index)) identityRefs.value.forEach((_, index) => clearIdentityRef(index))
recommendText.value = '' recommendText.value = ''
recommendError.value = '' recommendError.value = ''
@@ -3276,11 +3581,15 @@ async function restoreStudioJob(id: string) {
hideThumbnail?: boolean hideThumbnail?: boolean
hideInput?: boolean hideInput?: boolean
referenceStillIds?: Array<string | null> referenceStillIds?: Array<string | null>
extensions?: { prompt: string; duration: number; permanenceRefs?: PermanenceRef[] }[] extensions?: { prompt: string; duration: number; permanenceRefs?: PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[]
queueAutoRun?: boolean queueAutoRun?: boolean
globalLocks?: string globalLocks?: string
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][] shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: LoraStackItem[][]
} }
}>(`/api/studio-queue/${id}`) }>(`/api/studio-queue/${id}`)
const payload = job.payload const payload = job.payload
@@ -3319,7 +3628,8 @@ async function restoreStudioJob(id: string) {
extensionQueue.value = extensions.map(item => ({ extensionQueue.value = extensions.map(item => ({
id: crypto.randomUUID(), id: crypto.randomUUID(),
prompt: item.prompt, prompt: item.prompt,
duration: clampDuration(Number(item.duration)) duration: clampDuration(Number(item.duration)),
loraStack: readLoraStack(item)
})) }))
queueAutoRun.value = payload.queueAutoRun === true queueAutoRun.value = payload.queueAutoRun === true
globalLocks.value = payload.globalLocks || '' globalLocks.value = payload.globalLocks || ''
@@ -3329,6 +3639,10 @@ async function restoreStudioJob(id: string) {
if (refs?.length) shotRefs[index + 1] = refs if (refs?.length) shotRefs[index + 1] = refs
}) })
shotPermanenceRefs.value = shotRefs shotPermanenceRefs.value = shotRefs
applyLoraSelection(
payload,
payload.shotLoraStacks || payload.shotLoras || [payload, ...extensions]
)
const stillId = payload.stillId const stillId = payload.stillId
if (stillId) { if (stillId) {
const blob = await $fetch<Blob>(`/api/library/stills/${stillId}`, { responseType: 'blob' }) const blob = await $fetch<Blob>(`/api/library/stills/${stillId}`, { responseType: 'blob' })
@@ -3577,11 +3891,16 @@ async function rerun(item: LibraryClip, collection = false) {
if (extras.length) nextShots[index + 1] = extras if (extras.length) nextShots[index + 1] = extras
}) })
shotPermanenceRefs.value = nextShots shotPermanenceRefs.value = nextShots
applyLoraSelection(
restoreAll ? (ordered[0] || target) : target,
restoreAll ? ordered : undefined
)
extensionQueue.value = restoreAll extensionQueue.value = restoreAll
? parts.slice(1).map(part => ({ ? parts.slice(1).map(part => ({
id: crypto.randomUUID(), id: crypto.randomUUID(),
prompt: part.prompt, prompt: part.prompt,
duration: clipGenerateDuration(part) duration: clipGenerateDuration(part),
loraStack: loraStacksEqual(readLoraStack(part), readLoraStack(ordered[0] || target)) ? [] : readLoraStack(part)
})) }))
: [] : []
try { try {
@@ -3622,10 +3941,12 @@ async function loadDraft(draft: RetryDraft) {
if (refs?.length) draftShots[index + 1] = refs if (refs?.length) draftShots[index + 1] = refs
}) })
shotPermanenceRefs.value = draftShots shotPermanenceRefs.value = draftShots
applyLoraSelection(draft, draft.shotLoraStacks || draft.shotLoras)
extensionQueue.value = (draft.extensions || []).map(item => ({ extensionQueue.value = (draft.extensions || []).map(item => ({
id: crypto.randomUUID(), id: crypto.randomUUID(),
prompt: String(item.prompt || ''), prompt: String(item.prompt || ''),
duration: clampDuration(Number(item.duration)) duration: clampDuration(Number(item.duration)),
loraStack: readLoraStack(item)
})) }))
try { try {
const blob = await $fetch<Blob>(`/api/library/drafts/${draft.id}/still`, { responseType: 'blob' }) const blob = await $fetch<Blob>(`/api/library/drafts/${draft.id}/still`, { responseType: 'blob' })
@@ -3718,6 +4039,7 @@ async function openExtend(item?: LibraryClip) {
const clip = clips.value.find(entry => entry.id === id) const clip = clips.value.find(entry => entry.id === id)
extendPrompt.value = clip?.prompt || prompt.value extendPrompt.value = clip?.prompt || prompt.value
extendDuration.value = clampDuration(duration.value) extendDuration.value = clampDuration(duration.value)
extendLoraStack.value = readLoraStack(clip)
extendOpen.value = true extendOpen.value = true
lastFrameLoading.value = true lastFrameLoading.value = true
lastFrameUrl.value = '' lastFrameUrl.value = ''
@@ -3764,7 +4086,8 @@ async function generateExtension() {
body: { body: {
clipId: id, clipId: id,
prompt: extendPrompt.value.trim(), prompt: extendPrompt.value.trim(),
duration: extendDuration.value duration: extendDuration.value,
loraStack: extendLoraStack.value
} }
}) })
closeExtend() closeExtend()
@@ -4242,7 +4565,8 @@ function queueExtension() {
extensionQueue.value.push({ extensionQueue.value.push({
id: crypto.randomUUID(), id: crypto.randomUUID(),
prompt: prompt.value.trim(), prompt: prompt.value.trim(),
duration: duration.value duration: duration.value,
loraStack: []
}) })
} }
@@ -4287,6 +4611,7 @@ async function editImage() {
body.append('hideThumbnail', String(showPrivacyToggles.value && hideThumbnail.value)) body.append('hideThumbnail', String(showPrivacyToggles.value && hideThumbnail.value))
body.append('name', clipName.value.trim()) body.append('name', clipName.value.trim())
if (extraPasses.length) body.append('passes', JSON.stringify(extraPasses)) if (extraPasses.length) body.append('passes', JSON.stringify(extraPasses))
if (imageLoraStack.value.length) body.append('loraStack', JSON.stringify(imageLoraStack.value))
const downloadName = clipFileName(clipName.value.trim() || prompt.value.split(/[.!?\n]/)[0] || 'aigen-edit') const downloadName = clipFileName(clipName.value.trim() || prompt.value.split(/[.!?\n]/)[0] || 'aigen-edit')
const started = await $fetch<{ const started = await $fetch<{
jobId: string jobId: string
@@ -4369,6 +4694,7 @@ function applyEngineDefaults(engine: VideoEngineId) {
} }
function selectVideoEngine(engine: VideoEngineId) { function selectVideoEngine(engine: VideoEngineId) {
if (engine === 'ltx' && !ltxEnabled.value) return
if (videoEngine.value === engine) return if (videoEngine.value === engine) return
cfgTouched.value = false cfgTouched.value = false
videoEngine.value = engine videoEngine.value = engine
@@ -4376,13 +4702,19 @@ function selectVideoEngine(engine: VideoEngineId) {
} }
function applyVideoWorkflow(raw?: string) { function applyVideoWorkflow(raw?: string) {
const parsed = parseVideoWorkflow(raw) const parsed = coerceVideoWorkflow(raw, ltxEnabled.value)
videoEngine.value = videoEngineOf(parsed) videoEngine.value = videoEngineOf(parsed)
videoStart.value = videoStartOf(parsed) videoStart.value = videoStartOf(parsed)
minimaxGraph.value = minimaxGraphOf(parsed) minimaxGraph.value = minimaxGraphOf(parsed)
} }
async function generate() { async function generate() {
if (!ltxEnabled.value && isLtxWorkflow(videoWorkflow.value)) {
videoEngine.value = 'minimax'
applyEngineDefaults('minimax')
toast('LTX Pink Cherry is not available on this instance')
return
}
if ((!file.value && !textToVideo.value) || !prompt.value.trim()) return if ((!file.value && !textToVideo.value) || !prompt.value.trim()) return
if (!queueReady.value) { if (!queueReady.value) {
toast(shotScriptMode.value ? 'Each shot in the script needs a prompt.' : 'Each queued extension needs a prompt.') toast(shotScriptMode.value ? 'Each shot in the script needs a prompt.' : 'Each queued extension needs a prompt.')
@@ -4395,8 +4727,8 @@ async function generate() {
const shots = shotScriptMode.value ? parsedShots.value : null const shots = shotScriptMode.value ? parsedShots.value : null
const initialPrompt = (shots?.[0]?.prompt || prompt.value).trim() const initialPrompt = (shots?.[0]?.prompt || prompt.value).trim()
const queued = shots const queued = shots
? shots.slice(1).map(shot => ({ prompt: shot.prompt.trim(), duration: duration.value })) ? shots.slice(1).map(shot => ({ prompt: shot.prompt.trim(), duration: duration.value, ...persistLoraFields(shotLoraStacks.value[shot.n] || []) }))
: extensionQueue.value.map(item => ({ prompt: item.prompt.trim(), duration: item.duration })) : extensionQueue.value.map(item => ({ prompt: item.prompt.trim(), duration: item.duration, ...persistLoraFields(item.loraStack || []) }))
const hideOut = hideThumbnail.value const hideOut = hideThumbnail.value
const folderLockedPref = false const folderLockedPref = false
try { try {
@@ -4432,11 +4764,17 @@ async function generate() {
? shots.map(shot => shotPermanenceRefs.value[shot.n] || []) ? shots.map(shot => shotPermanenceRefs.value[shot.n] || [])
: [shotPermanenceRefs.value[1] || [], ...queued.map((_, index) => shotPermanenceRefs.value[index + 2] || [])] : [shotPermanenceRefs.value[1] || [], ...queued.map((_, index) => shotPermanenceRefs.value[index + 2] || [])]
body.append('shotPermanenceRefs', JSON.stringify(shotRefList)) body.append('shotPermanenceRefs', JSON.stringify(shotRefList))
const shotLoraStacksList = shots
? shots.map(shot => shotLoraStacks.value[shot.n] || [])
: [shotLoraStacks.value[1] || [], ...queued.map(item => item.loraStack || [])]
if (videoLoraStack.value.length) body.append('loraStack', JSON.stringify(videoLoraStack.value))
body.append('shotLoraStacks', JSON.stringify(shotLoraStacksList))
if (activeDraftId) body.append('draftId', activeDraftId) if (activeDraftId) body.append('draftId', activeDraftId)
if (queued.length) { if (queued.length) {
body.append('extensions', JSON.stringify(queued.map((item, index) => ({ body.append('extensions', JSON.stringify(queued.map((item, index) => ({
...item, ...item,
permanenceRefs: shotRefList[index + 1] || [] permanenceRefs: shotRefList[index + 1] || [],
...persistLoraFields(shotLoraStacksList[index + 1] || [])
})))) }))))
body.append('queueAutoRun', String(queueAutoRun.value)) body.append('queueAutoRun', String(queueAutoRun.value))
} }
+68 -2
View File
@@ -200,6 +200,19 @@
label="Duration" label="Duration"
@update:modelValue="saveSegment(job.shots, segment)" @update:modelValue="saveSegment(job.shots, segment)"
/> />
<LoraSelect
v-if="segment.status !== 'complete' && segment.status !== 'running'"
class="mt-3 max-w-md"
:model-value="readLoraStack(segment)"
:names="videoLoraOptions"
label="LoRA"
empty-label="Default"
add-label="Add LoRA"
compact
:clip-strength="false"
@update:model-value="(value) => { setSegmentStack(segment, value); saveSegment(job.shots, segment) }"
/>
<p v-else-if="segmentLoraLabel(job.shots, segment)" class="mt-1 text-[11px] text-zinc-500">{{ segmentLoraLabel(job.shots, segment) }}</p>
<div v-if="segment.clipId" class="mt-2"> <div v-if="segment.clipId" class="mt-2">
<a <a
class="text-xs text-amber-200 hover:text-amber-100" class="text-xs text-amber-200 hover:text-amber-100"
@@ -221,6 +234,7 @@
> >
<p class="text-sm font-medium text-zinc-100">{{ job.kind === 'edit' ? `Pass ${index + 1}` : `Shot ${index + 1}` }} <span class="font-normal text-zinc-500">· queued with this job</span></p> <p class="text-sm font-medium text-zinc-100">{{ job.kind === 'edit' ? `Pass ${index + 1}` : `Shot ${index + 1}` }} <span class="font-normal text-zinc-500">· queued with this job</span></p>
<p class="mt-1 max-h-24 overflow-y-auto whitespace-pre-wrap text-xs text-zinc-400 [overflow-wrap:anywhere]">{{ shot.prompt }}</p> <p class="mt-1 max-h-24 overflow-y-auto whitespace-pre-wrap text-xs text-zinc-400 [overflow-wrap:anywhere]">{{ shot.prompt }}</p>
<p v-if="loraStackLabel(readLoraStack(shot))" class="mt-1 text-[11px] text-zinc-500">{{ loraStackLabel(readLoraStack(shot)) }}</p>
</li> </li>
</ol> </ol>
</section> </section>
@@ -328,6 +342,19 @@
label="Duration" label="Duration"
@update:modelValue="saveSegment(queue, segment)" @update:modelValue="saveSegment(queue, segment)"
/> />
<LoraSelect
v-if="segment.status !== 'complete' && segment.status !== 'running'"
class="mt-3 max-w-md"
:model-value="readLoraStack(segment)"
:names="videoLoraOptions"
label="LoRA"
empty-label="Default"
add-label="Add LoRA"
compact
:clip-strength="false"
@update:model-value="(value) => { setSegmentStack(segment, value); saveSegment(queue, segment) }"
/>
<p v-else-if="segmentLoraLabel(queue, segment)" class="mt-1 text-[11px] text-zinc-500">{{ segmentLoraLabel(queue, segment) }}</p>
<div v-if="segment.clipId" class="mt-2"> <div v-if="segment.clipId" class="mt-2">
<a <a
class="text-xs text-amber-200 hover:text-amber-100" class="text-xs text-amber-200 hover:text-amber-100"
@@ -382,6 +409,8 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import { loraStackLabel, persistLoraFields, readLoraStack, type LoraStackItem } from '~/utils/loras'
type Segment = { type Segment = {
index: number index: number
prompt: string prompt: string
@@ -389,6 +418,8 @@ type Segment = {
status: 'pending' | 'running' | 'complete' | 'error' status: 'pending' | 'running' | 'complete' | 'error'
clipId?: string clipId?: string
error?: string error?: string
loraName?: string
loraStack?: LoraStackItem[]
} }
type Queue = { type Queue = {
@@ -401,6 +432,8 @@ type Queue = {
totalCount: number totalCount: number
currentJobId?: string currentJobId?: string
stopAfterCurrent?: boolean stopAfterCurrent?: boolean
loraName?: string
loraStack?: LoraStackItem[]
segments: Segment[] segments: Segment[]
} }
@@ -422,7 +455,7 @@ type StudioJobRow = {
pausedByUser?: boolean pausedByUser?: boolean
lastError?: string lastError?: string
shots?: Queue | null shots?: Queue | null
plannedShots?: { prompt: string; duration: number }[] plannedShots?: { prompt: string; duration: number; loraName?: string; loraStack?: LoraStackItem[] }[]
completedCount?: number completedCount?: number
} }
@@ -436,6 +469,7 @@ type PendingRemove = {
} }
const instanceName = ref(String(useRuntimeConfig().public.instanceName || 'AIGen')) const instanceName = ref(String(useRuntimeConfig().public.instanceName || 'AIGen'))
const videoLoras = ref<string[]>([])
const authEnabled = ref(false) const authEnabled = ref(false)
const jobs = ref<StudioJobRow[]>([]) const jobs = ref<StudioJobRow[]>([])
const orphans = ref<Queue[]>([]) const orphans = ref<Queue[]>([])
@@ -450,6 +484,24 @@ const sources = new Map<string, EventSource>()
const pendingRemove = ref<PendingRemove | null>(null) const pendingRemove = ref<PendingRemove | null>(null)
let pollTimer: ReturnType<typeof setInterval> | null = null let pollTimer: ReturnType<typeof setInterval> | null = null
const videoLoraOptions = computed(() => {
const names = [...videoLoras.value]
for (const queue of [...jobs.value.map(job => job.shots), ...orphans.value]) {
if (!queue) continue
if (queue.loraName && !names.includes(queue.loraName)) names.push(queue.loraName)
for (const item of queue.loraStack || []) {
if (item.name && !names.includes(item.name)) names.push(item.name)
}
for (const segment of queue.segments) {
if (segment.loraName && !names.includes(segment.loraName)) names.push(segment.loraName)
for (const item of segment.loraStack || []) {
if (item.name && !names.includes(item.name)) names.push(item.name)
}
}
}
return names
})
const canCutIn = computed(() => jobs.value.some(job => job.status === 'running' || job.status === 'held')) const canCutIn = computed(() => jobs.value.some(job => job.status === 'running' || job.status === 'held'))
const pauseBusy = computed(() => busyId.value === 'pause') const pauseBusy = computed(() => busyId.value === 'pause')
const anythingGenerating = computed(() => ( const anythingGenerating = computed(() => (
@@ -569,6 +621,17 @@ function segmentCaption(queue: Queue, segment: Segment) {
return '· waiting' return '· waiting'
} }
function setSegmentStack(segment: Segment, stack: LoraStackItem[]) {
const fields = persistLoraFields(stack)
segment.loraStack = fields.loraStack
segment.loraName = fields.loraName
}
function segmentLoraLabel(queue: Queue, segment: Segment) {
const stack = readLoraStack(segment).length ? readLoraStack(segment) : readLoraStack(queue)
return loraStackLabel(stack)
}
function segmentClass(queue: Queue, segment: Segment) { function segmentClass(queue: Queue, segment: Segment) {
if (segment.status === 'complete') return 'border-emerald-400/25 bg-emerald-400/5' if (segment.status === 'complete') return 'border-emerald-400/25 bg-emerald-400/5'
if (segment.status === 'running') return 'border-amber-300/50 bg-amber-400/10' if (segment.status === 'running') return 'border-amber-300/50 bg-amber-400/10'
@@ -949,7 +1012,7 @@ function saveSegment(queue: Queue, segment: Segment) {
saveTimer = setTimeout(() => { saveTimer = setTimeout(() => {
void $fetch(`/api/queues/${queue.id}`, { void $fetch(`/api/queues/${queue.id}`, {
method: 'PATCH', method: 'PATCH',
body: { segments: [{ index: segment.index, prompt: segment.prompt, duration: segment.duration }] } body: { segments: [{ index: segment.index, prompt: segment.prompt, duration: segment.duration, loraName: segment.loraName || '', loraStack: segment.loraStack || [] }] }
}).then(() => { }).then(() => {
dirty.delete(dirtyKey(queue.id, segment.index)) dirty.delete(dirtyKey(queue.id, segment.index))
}).catch(() => null) }).catch(() => null)
@@ -960,6 +1023,9 @@ onMounted(async () => {
const me = await $fetch<{ user?: { name?: string }; authEnabled?: boolean; instanceName?: string }>('/api/auth/me').catch(() => ({ user: null })) const me = await $fetch<{ user?: { name?: string }; authEnabled?: boolean; instanceName?: string }>('/api/auth/me').catch(() => ({ user: null }))
authEnabled.value = Boolean(me.authEnabled) authEnabled.value = Boolean(me.authEnabled)
instanceName.value = me.instanceName || instanceName.value instanceName.value = me.instanceName || instanceName.value
void $fetch<{ video?: string[] }>('/api/loras').then((listed) => {
videoLoras.value = listed.video || []
}).catch(() => null)
window.addEventListener('keydown', onRemoveKey) window.addEventListener('keydown', onRemoveKey)
await loadQueues().catch((err: any) => { await loadQueues().catch((err: any) => {
error.value = err?.data?.statusMessage || err?.message || 'Could not load the queue' error.value = err?.data?.statusMessage || err?.message || 'Could not load the queue'
+5 -1
View File
@@ -1,4 +1,5 @@
import { addStudioJob, kickStudioQueue, listStudioJobs } from '~/server/utils/studioQueue' import { addStudioJob, kickStudioQueue, listStudioJobs } from '~/server/utils/studioQueue'
import { parsePostedLoraStack, listStudioLoras, persistLoraFields } from '~/server/utils/loras'
import { comfyConfigured } from '~/server/utils/comfy' import { comfyConfigured } from '~/server/utils/comfy'
import { imageDimensions } from '~/server/utils/resolution' import { imageDimensions } from '~/server/utils/resolution'
@@ -81,6 +82,8 @@ export default defineEventHandler(async (event) => {
: Math.floor(Math.random() * 2_147_483_647) : Math.floor(Math.random() * 2_147_483_647)
const size = imageDimensions(image.data) const size = imageDimensions(image.data)
const clipName = (fields.name || '').trim().slice(0, 80) const clipName = (fields.name || '').trim().slice(0, 80)
await listStudioLoras()
const loraStack = parsePostedLoraStack(fields.loraStack || fields.lora, 'image')
const still = await rememberInputStill({ const still = await rememberInputStill({
ownerKey, ownerKey,
@@ -137,7 +140,8 @@ export default defineEventHandler(async (event) => {
negative: (fields.negative || '').trim(), negative: (fields.negative || '').trim(),
passes, passes,
referenceStillId: savedRef?.id, referenceStillId: savedRef?.id,
referenceStillFilename: savedRef?.filename referenceStillFilename: savedRef?.filename,
...persistLoraFields(loraStack)
} }
}) })
await kickStudioQueue() await kickStudioQueue()
+25 -7
View File
@@ -1,6 +1,9 @@
import { copyFileSync, existsSync, readFileSync } from 'node:fs' import { copyFileSync, existsSync, readFileSync } from 'node:fs'
import { join } from 'node:path' import { join } from 'node:path'
import { composeShotPrompt } from '~/utils/globalLocks' import { composeShotPrompt } from '~/utils/globalLocks'
import { parsePostedLoraStack, listStudioLoras, persistLoraFields } from '~/server/utils/loras'
import { readLoraStack } from '~/utils/loras'
import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m']) const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform']) const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
@@ -39,7 +42,7 @@ function parseScheduler(raw: unknown) {
} }
export default defineEventHandler(async (event) => { export default defineEventHandler(async (event) => {
const body = await readBody<{ clipId?: string; prompt?: string; duration?: number }>(event).catch(() => ({})) const body = await readBody<{ clipId?: string; prompt?: string; duration?: number; lora?: unknown; loraStack?: unknown }>(event).catch(() => ({}))
const clipId = String(body?.clipId || '').trim() const clipId = String(body?.clipId || '').trim()
const prompt = String(body?.prompt || '').trim() const prompt = String(body?.prompt || '').trim()
if (!clipId) { if (!clipId) {
@@ -66,13 +69,21 @@ export default defineEventHandler(async (event) => {
shotIndex: (source.chainIndex || 0) + 1, shotIndex: (source.chainIndex || 0) + 1,
familyRefs: source.permanenceRefs familyRefs: source.permanenceRefs
}) })
const workflow = parseVideoWorkflow(source.workflow)
if (isLtxWorkflow(workflow) && !ltxWorkflowEnabled()) {
throw createError({ statusCode: 400, statusMessage: LTX_DISABLED_MESSAGE })
}
const turbo = source.turbo !== false const turbo = source.turbo !== false
const fps = parseFps(source.fps) const fps = parseFps(source.fps)
const cfg = parseCfg(source.cfg, turbo) const cfg = parseCfg(source.cfg, turbo)
const samplerName = parseSampler(source.samplerName) const samplerName = parseSampler(source.samplerName)
const scheduler = parseScheduler(source.scheduler) const scheduler = parseScheduler(source.scheduler)
const steps = Number(source.steps || defaultVideoSteps(turbo, parseVideoWorkflow(source.workflow))) const steps = Number(source.steps || defaultVideoSteps(turbo, workflow))
const seed = Math.floor(Math.random() * 2_147_483_647) const seed = Math.floor(Math.random() * 2_147_483_647)
await listStudioLoras()
const postedStack = parsePostedLoraStack(body?.loraStack ?? body?.lora, 'video')
const loraStack = postedStack.length ? postedStack : parsePostedLoraStack(readLoraStack(source), 'video')
const loraFields = persistLoraFields(loraStack)
const length = frameLength(durationSeconds, fps) const length = frameLength(durationSeconds, fps)
const width = source.width || 1344 const width = source.width || 1344
const height = source.height || 768 const height = source.height || 768
@@ -109,9 +120,10 @@ export default defineEventHandler(async (event) => {
familyId: source.familyId || crypto.randomUUID(), familyId: source.familyId || crypto.randomUUID(),
parentClipId: source.id, parentClipId: source.id,
chainIndex: (source.chainIndex || 0) + 1, chainIndex: (source.chainIndex || 0) + 1,
workflow: parseVideoWorkflow(source.workflow), workflow,
globalLocks: source.globalLocks, globalLocks: source.globalLocks,
permanenceRefs: source.permanenceRefs permanenceRefs: source.permanenceRefs,
...loraFields
} }
emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 }) emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 })
@@ -130,7 +142,8 @@ export default defineEventHandler(async (event) => {
cfg, cfg,
fps, fps,
samplerName, samplerName,
scheduler scheduler,
...loraFields
}).catch((error) => { }).catch((error) => {
removeExtendTemp(job.library?.extendTmpDir) removeExtendTemp(job.library?.extendTmpDir)
const message = error instanceof Error ? error.message : String(error) const message = error instanceof Error ? error.message : String(error)
@@ -169,6 +182,8 @@ async function runExtend(
fps: number fps: number
samplerName: string samplerName: string
scheduler: string scheduler: string
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
} }
) { ) {
copyFileSync(params.sourcePath, params.part1Path) copyFileSync(params.sourcePath, params.part1Path)
@@ -218,7 +233,8 @@ async function runExtend(
filenamePrefix: comfyFilenamePrefix(), filenamePrefix: comfyFilenamePrefix(),
sound, sound,
workflow: job.library?.workflow, workflow: job.library?.workflow,
duration: job.library?.duration duration: job.library?.duration,
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName
}) })
const queued = await queuePrompt(graph, job.clientId) const queued = await queuePrompt(graph, job.clientId)
@@ -254,7 +270,9 @@ async function runExtend(
workflow: job.library.workflow, workflow: job.library.workflow,
duration: job.library.duration, duration: job.library.duration,
globalLocks: job.library.globalLocks, globalLocks: job.library.globalLocks,
permanenceRefs: job.library.permanenceRefs permanenceRefs: job.library.permanenceRefs,
loraName: job.library.loraName,
loraStack: job.library.loraStack
}) })
} }
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 }) emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
+27 -7
View File
@@ -1,5 +1,7 @@
import { addStudioJob, kickStudioQueue, listStudioJobs } from '~/server/utils/studioQueue' import { addStudioJob, kickStudioQueue, listStudioJobs } from '~/server/utils/studioQueue'
import { defaultVideoSteps, isLtxWorkflow, isTextToVideo, parseVideoWorkflow } from '~/utils/videoModels' import { listStudioLoras, parsePostedLoraStack, parseShotLoraStacks, persistLoraFields } from '~/server/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
import { defaultVideoSteps, isLtxWorkflow, isTextToVideo, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
import { allowIdentityRefs, normalizePermanenceRefs, resolveGlobalLocks, type PermanenceRef } from '~/utils/globalLocks' import { allowIdentityRefs, normalizePermanenceRefs, resolveGlobalLocks, type PermanenceRef } from '~/utils/globalLocks'
function parseDuration(raw: unknown) { function parseDuration(raw: unknown) {
@@ -13,18 +15,23 @@ function parseExtendDuration(raw: unknown) {
} }
function parseExtensions(raw: string | undefined) { function parseExtensions(raw: string | undefined) {
if (!raw) return [] as { prompt: string; duration: number; permanenceRefs?: PermanenceRef[] }[] if (!raw) return [] as { prompt: string; duration: number; permanenceRefs?: PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[]
try { try {
const parsed = JSON.parse(raw) const parsed = JSON.parse(raw)
if (!Array.isArray(parsed)) return [] if (!Array.isArray(parsed)) return []
return parsed return parsed
.map((item: { prompt?: unknown; duration?: unknown; permanenceRefs?: unknown }) => ({ .map((item: { prompt?: unknown; duration?: unknown; permanenceRefs?: unknown; loraName?: unknown; loraStack?: unknown }) => {
const loraStack = parsePostedLoraStack(item?.loraStack ?? item?.loraName, 'video')
return {
prompt: String(item?.prompt || '').trim(), prompt: String(item?.prompt || '').trim(),
duration: parseExtendDuration(item?.duration), duration: parseExtendDuration(item?.duration),
permanenceRefs: normalizePermanenceRefs(item?.permanenceRefs) permanenceRefs: normalizePermanenceRefs(item?.permanenceRefs),
})) ...persistLoraFields(loraStack)
}
})
.filter(item => item.prompt) .filter(item => item.prompt)
} catch { } catch (error) {
if (error && typeof error === 'object' && 'statusCode' in error) throw error
return [] return []
} }
} }
@@ -101,7 +108,11 @@ export default defineEventHandler(async (event) => {
} }
} }
await listStudioLoras()
const workflow = parseVideoWorkflow(fields.workflow) const workflow = parseVideoWorkflow(fields.workflow)
if (isLtxWorkflow(workflow) && !ltxWorkflowEnabled()) {
throw createError({ statusCode: 400, statusMessage: LTX_DISABLED_MESSAGE })
}
const textToVideo = isTextToVideo(workflow) const textToVideo = isTextToVideo(workflow)
if (!image && !textToVideo) { if (!image && !textToVideo) {
throw createError({ statusCode: 400, statusMessage: 'An initial image is required' }) throw createError({ statusCode: 400, statusMessage: 'An initial image is required' })
@@ -116,10 +127,16 @@ export default defineEventHandler(async (event) => {
const extensions = parseExtensions(fields.extensions) const extensions = parseExtensions(fields.extensions)
const permanenceRefs = parseFamilyPermanenceRefs(fields.permanenceRefs) const permanenceRefs = parseFamilyPermanenceRefs(fields.permanenceRefs)
const shotPermanenceRefs = parseShotPermanenceRefs(fields.shotPermanenceRefs, 1 + extensions.length) const shotPermanenceRefs = parseShotPermanenceRefs(fields.shotPermanenceRefs, 1 + extensions.length)
const defaultStack = parsePostedLoraStack(fields.loraStack || fields.lora, 'video')
const shotLoraStacks = parseShotLoraStacks(fields.shotLoraStacks || fields.shotLoras, 1 + extensions.length)
if (defaultStack.length && !shotLoraStacks[0].length) shotLoraStacks[0] = defaultStack
for (const [index, item] of extensions.entries()) { for (const [index, item] of extensions.entries()) {
if (!shotPermanenceRefs[index + 1]?.length && item.permanenceRefs?.length) { if (!shotPermanenceRefs[index + 1]?.length && item.permanenceRefs?.length) {
shotPermanenceRefs[index + 1] = item.permanenceRefs shotPermanenceRefs[index + 1] = item.permanenceRefs
} }
const shotStack = shotLoraStacks[index + 1]
if (!shotStack.length && item.loraStack?.length) shotLoraStacks[index + 1] = item.loraStack
else if (shotStack.length) Object.assign(item, persistLoraFields(shotStack))
} }
const requestedIdentity = workflow === 'v2' && !textToVideo && !isLtxWorkflow(workflow) && fields.useIdentityRefs === 'true' const requestedIdentity = workflow === 'v2' && !textToVideo && !isLtxWorkflow(workflow) && fields.useIdentityRefs === 'true'
const useIdentityRefs = allowIdentityRefs(requestedIdentity, permanenceRefs, globalLocks, shotPermanenceRefs) const useIdentityRefs = allowIdentityRefs(requestedIdentity, permanenceRefs, globalLocks, shotPermanenceRefs)
@@ -216,7 +233,10 @@ export default defineEventHandler(async (event) => {
queueAutoRun, queueAutoRun,
globalLocks: globalLocks || undefined, globalLocks: globalLocks || undefined,
permanenceRefs: permanenceRefs.length ? permanenceRefs : undefined, permanenceRefs: permanenceRefs.length ? permanenceRefs : undefined,
shotPermanenceRefs: shotPermanenceRefs.some(item => item.length) ? shotPermanenceRefs : undefined shotPermanenceRefs: shotPermanenceRefs.some(item => item.length) ? shotPermanenceRefs : undefined,
...persistLoraFields(defaultStack),
shotLoras: shotLoraStacks.some(stack => stack.length) ? shotLoraStacks.map(stack => stack[0]?.name || '') : undefined,
shotLoraStacks: shotLoraStacks.some(stack => stack.length) ? shotLoraStacks : undefined
} }
}) })
await kickStudioQueue() await kickStudioQueue()
+10
View File
@@ -0,0 +1,10 @@
import { listGenerationPresets } from '~/server/utils/generationPresets'
import { parseGenerationPresetKind } from '~/utils/generationPresets'
export default defineEventHandler((event) => {
const { owner } = assertLibraryOwner(event)
const kind = parseGenerationPresetKind(getQuery(event).kind)
return {
presets: listGenerationPresets(owner, kind)
}
})
+25
View File
@@ -0,0 +1,25 @@
import { listGenerationPresets, saveGenerationPreset } from '~/server/utils/generationPresets'
import { parseGenerationPresetKind } from '~/utils/generationPresets'
export default defineEventHandler(async (event) => {
const { owner } = assertLibraryOwner(event)
const body = await readBody<{
name?: string
kind?: string
loraStack?: unknown
settings?: unknown
overwrite?: boolean
}>(event).catch(() => ({}))
const kind = parseGenerationPresetKind(body?.kind)
const preset = await saveGenerationPreset(owner, {
name: body?.name || '',
kind,
loraStack: body?.loraStack,
settings: body?.settings,
overwrite: body?.overwrite === true
})
return {
preset,
presets: listGenerationPresets(owner, kind)
}
})
@@ -0,0 +1,13 @@
import { deleteGenerationPreset, listGenerationPresets } from '~/server/utils/generationPresets'
import { parseGenerationPresetKind } from '~/utils/generationPresets'
export default defineEventHandler(async (event) => {
const { owner } = assertLibraryOwner(event)
const id = String(getRouterParam(event, 'id') || '')
const kind = parseGenerationPresetKind(getQuery(event).kind)
await deleteGenerationPreset(owner, id)
return {
ok: true,
presets: listGenerationPresets(owner, kind)
}
})
+19 -15
View File
@@ -1,14 +1,15 @@
import { checkComfyHttp, fetchQueue } from '~/server/utils/comfyLifecycle' import { checkComfyHttp, fetchQueue } from '~/server/utils/comfyLifecycle'
import { comfyConfigured } from '~/server/utils/comfy' import { comfyConfigured } from '~/server/utils/comfy'
import { imageComfyConfigured, getBeastImageHost, getSidecarImageHost } from '~/server/utils/imageComfy' import { imageComfyConfigured, getBeastImageHost, getSidecarImageHost, sameImageHost } from '~/server/utils/imageComfy'
import { checkImageComfyHttp, fetchQueueAt } from '~/server/utils/imageComfyLifecycle' import { checkImageComfyHttp, fetchQueueAt } from '~/server/utils/imageComfyLifecycle'
export default defineEventHandler(async () => { export default defineEventHandler(async () => {
const videoConfigured = comfyConfigured() const videoConfigured = comfyConfigured()
const imageConfigured = imageComfyConfigured() const imageConfigured = imageComfyConfigured()
const beastHost = getBeastImageHost() const beastHost = getBeastImageHost()
const sidecarHost = getSidecarImageHost() const promptHost = getSidecarImageHost()
const [comfy, queue, beast, beastQueue, sidecar, sidecarQueue] = await Promise.all([ const promptIsBeast = Boolean(promptHost && beastHost && sameImageHost(promptHost, beastHost))
const [comfy, queue, beast, beastQueue, prompt, promptQueue] = await Promise.all([
videoConfigured videoConfigured
? checkComfyHttp().catch(() => ({ ok: false, host: '', stats: null })) ? checkComfyHttp().catch(() => ({ ok: false, host: '', stats: null }))
: Promise.resolve({ ok: false, host: '', stats: null }), : Promise.resolve({ ok: false, host: '', stats: null }),
@@ -17,17 +18,20 @@ export default defineEventHandler(async () => {
? checkImageComfyHttp(2500, beastHost).catch(() => ({ ok: false, host: beastHost, stats: null })) ? checkImageComfyHttp(2500, beastHost).catch(() => ({ ok: false, host: beastHost, stats: null }))
: Promise.resolve({ ok: false, host: beastHost, stats: null }), : Promise.resolve({ ok: false, host: beastHost, stats: null }),
beastHost ? fetchQueueAt(beastHost) : Promise.resolve({ running: 0, pending: 0 }), beastHost ? fetchQueueAt(beastHost) : Promise.resolve({ running: 0, pending: 0 }),
sidecarHost promptHost && !promptIsBeast
? checkImageComfyHttp(2500, sidecarHost).catch(() => ({ ok: false, host: sidecarHost, stats: null })) ? checkImageComfyHttp(2500, promptHost).catch(() => ({ ok: false, host: promptHost, stats: null }))
: Promise.resolve({ ok: false, host: sidecarHost, stats: null }), : Promise.resolve({ ok: false, host: promptHost, stats: null }),
sidecarHost ? fetchQueueAt(sidecarHost) : Promise.resolve({ running: 0, pending: 0 }) promptHost && !promptIsBeast ? fetchQueueAt(promptHost) : Promise.resolve({ running: 0, pending: 0 })
]) ])
const busy = Boolean(comfy.ok && queue.running > 0) const busy = Boolean(comfy.ok && queue.running > 0)
const beastBusy = Boolean(beast.ok && beastQueue.running > 0) const beastBusy = Boolean(beast.ok && beastQueue.running > 0)
const sidecarBusy = Boolean(sidecar.ok && sidecarQueue.running > 0) const promptOk = promptIsBeast ? Boolean(beast.ok || comfy.ok) : Boolean(prompt.ok)
const promptBusy = promptIsBeast ? (beastBusy || busy) : Boolean(prompt.ok && promptQueue.running > 0)
const imageOk = Boolean(beast.ok || comfy.ok) const imageOk = Boolean(beast.ok || comfy.ok)
const imageBusy = Boolean(imageConfigured && (beastBusy || busy)) const imageBusy = Boolean(imageConfigured && (beastBusy || busy))
const preferredHost = beastHost || comfy.host || '' const preferredHost = beastHost || comfy.host || ''
const promptQueueRunning = promptIsBeast ? ((beastQueue.running || queue.running) || 0) : (promptQueue.running || 0)
const promptQueuePending = promptIsBeast ? ((beastQueue.pending || queue.pending) || 0) : (promptQueue.pending || 0)
return { return {
ok: true, ok: true,
service: useRuntimeConfig().public.instanceName || 'aigen', service: useRuntimeConfig().public.instanceName || 'aigen',
@@ -55,13 +59,13 @@ export default defineEventHandler(async () => {
host: preferredHost host: preferredHost
}, },
sidecar: { sidecar: {
label: 'Sidecar', label: 'Qwen VL',
configured: Boolean(sidecarHost), configured: Boolean(promptHost),
ok: Boolean(sidecar.ok), ok: promptOk,
busy: sidecarBusy, busy: promptBusy,
host: sidecarHost, host: promptHost,
queueRunning: sidecarQueue.running || 0, queueRunning: promptQueueRunning,
queuePending: sidecarQueue.pending || 0 queuePending: promptQueuePending
} }
} }
} }
+9
View File
@@ -0,0 +1,9 @@
import { listStudioLoras } from '~/server/utils/loras'
export default defineEventHandler(async () => {
const listed = await listStudioLoras()
return {
image: listed.image,
video: listed.video
}
})
+1 -1
View File
@@ -5,7 +5,7 @@ export default defineEventHandler(async (event) => {
if (!promptComfyConfigured()) { if (!promptComfyConfigured()) {
throw createError({ throw createError({
statusCode: 503, statusCode: 503,
statusMessage: 'Sidecar Qwen VL is not configured. Set IMAGE_COMFY_HOST.' statusMessage: 'Qwen VL is not configured. Set COMFY_HOST or IMAGE_COMFY_HOST to Beast.'
}) })
} }
+3 -1
View File
@@ -1,4 +1,5 @@
import { getShotQueue, patchShotQueueSegments, updateShotQueue, summarizeQueue } from '~/server/utils/shotQueue' import { getShotQueue, patchShotQueueSegments, updateShotQueue, summarizeQueue } from '~/server/utils/shotQueue'
import { listStudioLoras } from '~/server/utils/loras'
export default defineEventHandler(async (event) => { export default defineEventHandler(async (event) => {
const { owner } = assertLibraryOwner(event) const { owner } = assertLibraryOwner(event)
@@ -6,10 +7,11 @@ export default defineEventHandler(async (event) => {
const body = await readBody<{ const body = await readBody<{
name?: string name?: string
autoRun?: boolean autoRun?: boolean
segments?: Array<{ index: number; prompt?: string; duration?: number }> segments?: Array<{ index: number; prompt?: string; duration?: number; loraName?: string; loraStack?: unknown }>
}>().catch(() => ({})) }>().catch(() => ({}))
if (Array.isArray(body?.segments) && body.segments.length) { if (Array.isArray(body?.segments) && body.segments.length) {
await listStudioLoras()
await patchShotQueueSegments(owner, id, body.segments) await patchShotQueueSegments(owner, id, body.segments)
} }
if (typeof body?.name === 'string' || typeof body?.autoRun === 'boolean') { if (typeof body?.name === 'string' || typeof body?.autoRun === 'boolean') {
+7 -2
View File
@@ -23,8 +23,13 @@ export default defineEventHandler((event) => {
] ]
: undefined) : undefined)
: [ : [
{ prompt: job.payload.prompt, duration: job.payload.duration }, { prompt: job.payload.prompt, duration: job.payload.duration, loraName: job.payload.shotLoras?.[0] || job.payload.loraName, loraStack: job.payload.shotLoraStacks?.[0] || job.payload.loraStack },
...(job.payload.extensions || []) ...(job.payload.extensions || []).map((item, index) => ({
prompt: item.prompt,
duration: item.duration,
loraName: item.loraName || job.payload.shotLoras?.[index + 1],
loraStack: item.loraStack || job.payload.shotLoraStacks?.[index + 1]
}))
] ]
return { return {
...summarizeStudioJob(job), ...summarizeStudioJob(job),
+184
View File
@@ -0,0 +1,184 @@
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { ownerRoot } from '~/server/utils/library'
import { filterLoraStackForStudio, normalizeLoraStack } from '~/utils/loras'
import {
normalizePresetName,
parseGenerationPresetKind,
type GenerationPreset,
type GenerationPresetKind,
type ImagePresetSettings,
type VideoPresetSettings
} from '~/utils/generationPresets'
import { coerceVideoWorkflow, isXaigenStudio, parseVideoWorkflow } from '~/utils/videoModels'
const writeChains = new Map<string, Promise<unknown>>()
function presetsPath(owner: string) {
return join(ownerRoot(owner), 'generation-presets.json')
}
function ensureOwner(owner: string) {
mkdirSync(ownerRoot(owner), { recursive: true })
}
function readPresets(owner: string): GenerationPreset[] {
ensureOwner(owner)
const path = presetsPath(owner)
if (!existsSync(path)) return []
try {
const parsed = JSON.parse(readFileSync(path, 'utf8'))
return Array.isArray(parsed) ? parsed.map(normalizeStoredPreset).filter((item): item is GenerationPreset => Boolean(item)) : []
} catch {
return []
}
}
function writePresets(owner: string, presets: GenerationPreset[]) {
ensureOwner(owner)
const path = presetsPath(owner)
const tmp = `${path}.tmp`
writeFileSync(tmp, JSON.stringify(presets, null, 2))
renameSync(tmp, path)
}
function mutate<T>(owner: string, fn: (presets: GenerationPreset[]) => T): Promise<T> {
const prev = writeChains.get(owner) || Promise.resolve()
const run = prev.then(() => {
const presets = readPresets(owner)
const result = fn(presets)
writePresets(owner, presets)
return result
})
writeChains.set(owner, run.then(() => undefined, () => undefined))
return run
}
function clampInt(raw: unknown, fallback: number, min: number, max: number) {
const value = Number(raw)
if (!Number.isFinite(value)) return fallback
return Math.min(max, Math.max(min, Math.round(value)))
}
function clampNum(raw: unknown, fallback: number, min: number, max: number, step = 0.1) {
const value = Number(raw)
if (!Number.isFinite(value)) return fallback
const snapped = Math.round(value / step) * step
return Math.min(max, Math.max(min, Math.round(snapped * 100) / 100))
}
function sanitizeVideoSettings(raw: unknown): VideoPresetSettings {
const rec = raw && typeof raw === 'object' ? raw as Record<string, unknown> : {}
const workflow = coerceVideoWorkflow(parseVideoWorkflow(rec.workflow))
const turbo = rec.turbo !== false
const sampler = String(rec.samplerName || '')
const scheduler = String(rec.scheduler || '')
const aspect = String(rec.aspect || 'auto')
return {
workflow,
turbo,
cfg: clampNum(rec.cfg, turbo ? 1.5 : 4, 1, 10, 0.5),
fps: rec.fps === 12 || rec.fps === 30 || rec.fps === 24 ? Number(rec.fps) : 24,
samplerName: sampler === 'euler' || sampler === 'dpmpp_2m' || sampler === 'res_multistep' ? sampler : 'res_multistep',
scheduler: scheduler === 'ddim_uniform' || scheduler === 'sgm_uniform' || scheduler === 'simple' ? scheduler : 'simple',
duration: clampNum(rec.duration, 5, 0.5, 120, 0.1),
aspect: aspect === '16:9' || aspect === '9:16' || aspect === '1:1' || aspect === 'auto' ? aspect : 'auto',
sound: rec.sound !== false
}
}
function sanitizeImageSettings(raw: unknown): ImagePresetSettings {
const rec = raw && typeof raw === 'object' ? raw as Record<string, unknown> : {}
return {
steps: clampInt(rec.steps, 20, 1, 40),
cfg: clampNum(rec.cfg, 1, 0.5, 4, 0.1),
negative: String(rec.negative || '').slice(0, 2000)
}
}
function normalizeStoredPreset(raw: unknown): GenerationPreset | null {
if (!raw || typeof raw !== 'object') return null
const rec = raw as Record<string, unknown>
const name = normalizePresetName(rec.name)
if (!name) return null
const kind = parseGenerationPresetKind(rec.kind)
const id = String(rec.id || '').trim() || crypto.randomUUID()
const createdAt = Number(rec.createdAt) || Date.now()
return {
id,
name,
kind,
createdAt,
updatedAt: Number(rec.updatedAt) || createdAt,
loraStack: normalizeLoraStack(rec.loraStack),
settings: kind === 'image' ? sanitizeImageSettings(rec.settings) : sanitizeVideoSettings(rec.settings)
}
}
function publicPreset(preset: GenerationPreset): GenerationPreset {
const xaigen = isXaigenStudio()
return {
...preset,
loraStack: filterLoraStackForStudio(preset.loraStack, xaigen),
settings: preset.kind === 'video'
? { ...preset.settings, workflow: coerceVideoWorkflow((preset.settings as VideoPresetSettings).workflow) }
: preset.settings
}
}
export function listGenerationPresets(owner: string, kind?: GenerationPresetKind) {
return readPresets(owner)
.filter(item => !kind || item.kind === kind)
.map(publicPreset)
.sort((a, b) => a.name.localeCompare(b.name) || b.updatedAt - a.updatedAt)
}
export async function saveGenerationPreset(owner: string, input: {
name: string
kind: GenerationPresetKind
loraStack?: unknown
settings?: unknown
overwrite?: boolean
}) {
const name = normalizePresetName(input.name)
if (!name) {
throw createError({ statusCode: 400, statusMessage: 'A preset name is required' })
}
const kind = parseGenerationPresetKind(input.kind)
const xaigen = isXaigenStudio()
const loraStack = filterLoraStackForStudio(input.loraStack, xaigen)
const settings = kind === 'image' ? sanitizeImageSettings(input.settings) : sanitizeVideoSettings(input.settings)
const now = Date.now()
return mutate(owner, (presets) => {
const existing = presets.find(item => item.kind === kind && item.name.toLowerCase() === name.toLowerCase())
if (existing && !input.overwrite) {
throw createError({ statusCode: 409, statusMessage: `Overwrite preset “${existing.name}”?` })
}
if (existing) {
existing.updatedAt = now
existing.loraStack = loraStack
existing.settings = settings
return publicPreset(existing)
}
const created: GenerationPreset = {
id: crypto.randomUUID(),
name,
kind,
createdAt: now,
updatedAt: now,
loraStack,
settings
}
presets.unshift(created)
return publicPreset(created)
})
}
export async function deleteGenerationPreset(owner: string, id: string) {
return mutate(owner, (presets) => {
const index = presets.findIndex(item => item.id === id)
if (index < 0) throw createError({ statusCode: 404, statusMessage: 'Preset not found' })
const [removed] = presets.splice(index, 1)
return removed
})
}
+4 -1
View File
@@ -18,6 +18,8 @@ export type EditRunParams = {
steps: number steps: number
seed: number seed: number
cfg: number cfg: number
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
} }
export async function runEdit(job: Job, params: EditRunParams) { export async function runEdit(job: Job, params: EditRunParams) {
@@ -97,7 +99,8 @@ export async function runEdit(job: Job, params: EditRunParams) {
steps: params.steps, steps: params.steps,
seed, seed,
cfg: params.cfg, cfg: params.cfg,
filenamePrefix: `aigen_edit_${job.id.slice(0, 8)}_p${index + 1}` filenamePrefix: `aigen_edit_${job.id.slice(0, 8)}_p${index + 1}`,
loraStack: params.loraStack || params.loraName
}) })
const queued = await queuePrompt(graph, job.clientId) const queued = await queuePrompt(graph, job.clientId)
job.promptId = queued.prompt_id job.promptId = queued.prompt_id
+18 -18
View File
@@ -38,13 +38,24 @@ export function getBeastImageHost() {
} }
} }
function isRetiredSidecarHost(raw: string) {
return /192\.168\.77\.101(?!\d)/.test(raw)
}
export function getSidecarImageHost() { export function getSidecarImageHost() {
const config = useRuntimeConfig() const config = useRuntimeConfig()
const fallback = String(config.imageComfyFallbackHost || process.env.IMAGE_COMFY_FALLBACK_HOST || '').trim()
if (fallback) return normalizeHost(fallback)
const legacy = String(config.imageComfyHost || process.env.IMAGE_COMFY_HOST || '').trim()
const port = String(config.imageComfyPort || process.env.IMAGE_COMFY_PORT || '').trim() const port = String(config.imageComfyPort || process.env.IMAGE_COMFY_PORT || '').trim()
return normalizeHost(legacy, port) const candidates: Array<[string, string]> = [
[String(config.promptComfyHost || process.env.PROMPT_COMFY_HOST || '').trim(), ''],
[String(config.imageComfyHost || process.env.IMAGE_COMFY_HOST || '').trim(), port],
[String(config.imageComfyFallbackHost || process.env.IMAGE_COMFY_FALLBACK_HOST || '').trim(), '']
]
for (const [raw, extraPort] of candidates) {
if (!raw || isRetiredSidecarHost(raw)) continue
const host = normalizeHost(raw, extraPort)
if (host && !isRetiredSidecarHost(host)) return host
}
return getBeastImageHost()
} }
export function getImageComfyHost() { export function getImageComfyHost() {
@@ -352,19 +363,8 @@ async function purgeImageOnDesktop(opts: {
output?: { filename: string; subfolder: string; type: string } output?: { filename: string; subfolder: string; type: string }
}) { }) {
const config = useRuntimeConfig() const config = useRuntimeConfig()
const host = getImageComfyHost() const controlUrl = String(config.comfyControlUrl || process.env.COMFY_CONTROL_URL || '').replace(/\/$/, '')
const sidecar = getSidecarImageHost() const token = String(config.comfyControlToken || process.env.COMFY_CONTROL_TOKEN || '')
const useSidecarAgent = sidecar && sameImageHost(host, sidecar)
const controlUrl = String(
useSidecarAgent
? (config.imageComfyControlUrl || process.env.IMAGE_COMFY_CONTROL_URL || '')
: (config.comfyControlUrl || process.env.COMFY_CONTROL_URL || '')
).replace(/\/$/, '')
const token = String(
useSidecarAgent
? (config.imageComfyControlToken || process.env.IMAGE_COMFY_CONTROL_TOKEN || '')
: (config.comfyControlToken || process.env.COMFY_CONTROL_TOKEN || '')
)
if (!controlUrl) return if (!controlUrl) return
try { try {
await fetch(`${controlUrl}/purge`, { await fetch(`${controlUrl}/purge`, {
@@ -382,7 +382,7 @@ async function purgeImageOnDesktop(opts: {
signal: AbortSignal.timeout(4000) signal: AbortSignal.timeout(4000)
}) })
} catch { } catch {
/* Sidecar host agent is optional; Comfy HTTP delete is the primary path. */ /* Host agent purge is optional; Comfy HTTP delete is the primary path. */
} }
} }
+8 -93
View File
@@ -187,7 +187,7 @@ async function ensureUnlocked(onStatus: StatusFn) {
onStatus({ onStatus({
state: 'starting', state: 'starting',
message: 'Image ComfyUI is offline. Starting the sidecar instance...', message: 'Image ComfyUI is offline. Starting the Beast instance...',
host, host,
processRunning: Boolean(remote?.process) processRunning: Boolean(remote?.process)
}) })
@@ -195,7 +195,7 @@ async function ensureUnlocked(onStatus: StatusFn) {
if (!started.started && started.reason === 'no-launcher') { if (!started.started && started.reason === 'no-launcher') {
throw createError({ throw createError({
statusCode: 503, statusCode: 503,
statusMessage: 'Image ComfyUI is offline and the sidecar host agent did not start it. Confirm the agent on 192.168.77.101:8199 is running.' statusMessage: 'Image ComfyUI is offline and the Beast host agent did not start it. Confirm COMFY_CONTROL_URL is running.'
}) })
} }
@@ -230,111 +230,26 @@ export function ensureImageComfyReady(onStatus: StatusFn = () => undefined) {
} }
let sidecarGate: Promise<unknown> = Promise.resolve() let sidecarGate: Promise<unknown> = Promise.resolve()
let sidecarLastStartAt = 0
function sidecarControlSettings() {
const config = useRuntimeConfig()
return {
host: getSidecarImageHost(),
controlUrl: String(config.imageComfyControlUrl || process.env.IMAGE_COMFY_CONTROL_URL || '').replace(/\/$/, ''),
controlToken: String(config.imageComfyControlToken || process.env.IMAGE_COMFY_CONTROL_TOKEN || ''),
healthTimeoutMs: 2500,
startTimeoutMs: Number(process.env.IMAGE_COMFY_START_TIMEOUT_MS || 180_000)
}
}
async function sidecarControlRequest(path: string, method = 'GET') {
const { controlUrl, controlToken } = sidecarControlSettings()
if (!controlUrl) return null
try {
const res = await fetch(`${controlUrl}${path}`, {
method,
headers: {
Accept: 'application/json',
...(controlToken ? { Authorization: `Bearer ${controlToken}` } : {})
},
signal: AbortSignal.timeout(5000)
})
if (!res.ok) return { ok: false, status: res.status }
return await res.json() as {
ok?: boolean
http?: boolean
process?: boolean
started?: boolean
already?: boolean
}
} catch {
return null
}
}
async function startSidecarComfy() {
const now = Date.now()
if (now - sidecarLastStartAt < 15_000) {
log('sidecar-start-skipped', { reason: 'recent-start' })
return { started: false, reason: 'recent-start' as const }
}
sidecarLastStartAt = now
const remote = await sidecarControlRequest('/start', 'POST')
if (remote?.ok || remote?.started) {
log('sidecar-start-remote', {})
return { started: true, reason: 'remote' as const }
}
return { started: false, reason: 'no-launcher' as const }
}
async function ensureSidecarUnlocked(onStatus: StatusFn) { async function ensureSidecarUnlocked(onStatus: StatusFn) {
const cfg = sidecarControlSettings() const host = getSidecarImageHost()
const host = cfg.host
if (!host) { if (!host) {
throw createError({ throw createError({
statusCode: 503, statusCode: 503,
statusMessage: 'Sidecar Qwen VL is not configured. Set IMAGE_COMFY_HOST or IMAGE_COMFY_FALLBACK_HOST.' statusMessage: 'Qwen VL is not configured. Set COMFY_HOST or IMAGE_COMFY_HOST to Beast.'
}) })
} }
onStatus({ state: 'offline', message: 'Checking Sidecar Qwen VL…', host, box: 'sidecar' }) onStatus({ state: 'offline', message: 'Checking Beast Qwen VL…', host, box: 'beast' })
const health = await checkImageComfyHttp(cfg.healthTimeoutMs, host) const health = await checkImageComfyHttp(2500, host)
if (health.ok) { if (health.ok) {
onStatus({ state: 'online', message: 'Sidecar Qwen VL online', host, box: 'sidecar' }) onStatus({ state: 'online', message: 'Beast Qwen VL online', host, box: 'beast' })
return return
} }
onStatus({
state: 'starting',
message: 'Sidecar is offline. Starting Qwen VL on 192.168.77.101…',
host,
box: 'sidecar'
})
const started = await startSidecarComfy()
if (!started.started && started.reason === 'no-launcher') {
throw createError({
statusCode: 503,
statusMessage: 'Sidecar is offline and the host agent did not start it. Confirm the agent on 192.168.77.101:8199 is running.'
})
}
const deadline = Date.now() + cfg.startTimeoutMs
let attempt = 0
while (Date.now() < deadline) {
attempt += 1
onStatus({
state: 'starting',
message: `Starting Sidecar Qwen VL… waiting for Comfy (${attempt})`,
host,
box: 'sidecar'
})
const next = await checkImageComfyHttp(cfg.healthTimeoutMs, host)
if (next.ok) {
onStatus({ state: 'online', message: 'Sidecar Qwen VL online', host, box: 'sidecar', processRunning: true })
return
}
await sleep(2000)
}
throw createError({ throw createError({
statusCode: 503, statusCode: 503,
statusMessage: 'Started Sidecar Qwen VL but it never answered. The 101 box can take a couple of minutes to load after a cold boot.' statusMessage: `Beast ComfyUI is offline (${host}). Recommend needs Qwen VL on that instance and will not start a sidecar.`
}) })
} }
+5
View File
@@ -1,4 +1,6 @@
import template from '../assets/workflow_flux2_klein_edit.json' import template from '../assets/workflow_flux2_klein_edit.json'
import { applyUserLoraToGraph } from '~/server/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
const IMAGE1 = '76' const IMAGE1 = '76'
const IMAGE2 = '81' const IMAGE2 = '81'
@@ -27,6 +29,8 @@ export interface EditParams {
seed: number seed: number
cfg: number cfg: number
filenamePrefix?: string filenamePrefix?: string
loraName?: string
loraStack?: LoraStackItem[]
} }
export function buildEditWorkflow(params: EditParams) { export function buildEditWorkflow(params: EditParams) {
@@ -61,6 +65,7 @@ export function buildEditWorkflow(params: EditParams) {
if (dual ? singleNode : dualNode) delete graph[id] if (dual ? singleNode : dualNode) delete graph[id]
} }
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
return graph return graph
} }
+5 -1
View File
@@ -77,7 +77,7 @@ export interface Job {
duration?: number duration?: number
sound?: boolean sound?: boolean
draftId?: string draftId?: string
extensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[] extensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: import('~/utils/loras').LoraStackItem[] }[]
chainIndex?: number chainIndex?: number
chainStep?: number chainStep?: number
chainTotal?: number chainTotal?: number
@@ -95,6 +95,10 @@ export interface Job {
queueAutoRun?: boolean queueAutoRun?: boolean
queueBudget?: number queueBudget?: number
stopAfterCurrent?: boolean stopAfterCurrent?: boolean
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
} }
error?: string error?: string
socketReady?: boolean socketReady?: boolean
+23 -3
View File
@@ -46,6 +46,8 @@ export interface LibraryClip {
hideInput?: boolean hideInput?: boolean
globalLocks?: string globalLocks?: string
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
} }
export type StillRole = 'input' | 'output' export type StillRole = 'input' | 'output'
@@ -82,6 +84,8 @@ export interface QueuedExtension {
prompt: string prompt: string
duration: number duration: number
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
} }
export interface RetryDraft { export interface RetryDraft {
@@ -111,6 +115,10 @@ export interface RetryDraft {
globalLocks?: string globalLocks?: string
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][] shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
} }
interface Catalog { interface Catalog {
@@ -639,7 +647,7 @@ function writeFrameMeta(dest: string, video: string, mode: FrameSourceMeta['mode
const meta: FrameSourceMeta = { videoMtime: info.mtimeMs, videoSize: info.size, mode } const meta: FrameSourceMeta = { videoMtime: info.mtimeMs, videoSize: info.size, mode }
if (typeof seek === 'number') meta.seek = seek if (typeof seek === 'number') meta.seek = seek
writeFileSync(frameMetaPath(dest), JSON.stringify(meta)) writeFileSync(frameMetaPath(dest), JSON.stringify(meta))
} catch { /* thumbs still work without the sidecar */ } } catch { /* thumbs still work without Comfy */ }
} }
function frameCacheFresh(video: string, dest: string, mode: FrameSourceMeta['mode']) { function frameCacheFresh(video: string, dest: string, mode: FrameSourceMeta['mode']) {
@@ -1209,6 +1217,10 @@ export async function saveRetryDraft(params: {
globalLocks?: string globalLocks?: string
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][] shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
}) { }) {
return mutate(params.ownerKey, (catalog) => { return mutate(params.ownerKey, (catalog) => {
const existing = params.id ? catalog.drafts.find(item => item.id === params.id) : undefined const existing = params.id ? catalog.drafts.find(item => item.id === params.id) : undefined
@@ -1238,7 +1250,11 @@ export async function saveRetryDraft(params: {
workflow: params.workflow, workflow: params.workflow,
globalLocks: params.globalLocks, globalLocks: params.globalLocks,
permanenceRefs: params.permanenceRefs, permanenceRefs: params.permanenceRefs,
shotPermanenceRefs: params.shotPermanenceRefs shotPermanenceRefs: params.shotPermanenceRefs,
loraName: params.loraName,
loraStack: params.loraStack,
shotLoras: params.shotLoras,
shotLoraStacks: params.shotLoraStacks
} }
catalog.drafts = [draft, ...catalog.drafts.filter(item => item.id !== draft.id)].slice(0, 20) catalog.drafts = [draft, ...catalog.drafts.filter(item => item.id !== draft.id)].slice(0, 20)
return draft return draft
@@ -1392,6 +1408,8 @@ export async function saveClip(params: {
sound?: boolean sound?: boolean
globalLocks?: string globalLocks?: string
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
segmentFirstFrame?: Buffer segmentFirstFrame?: Buffer
}) { }) {
const catalog = readCatalog(params.ownerKey) const catalog = readCatalog(params.ownerKey)
@@ -1423,7 +1441,9 @@ export async function saveClip(params: {
sound: params.sound, sound: params.sound,
hideInput: params.hideInput === true, hideInput: params.hideInput === true,
globalLocks: params.globalLocks, globalLocks: params.globalLocks,
permanenceRefs: params.permanenceRefs?.length ? normalizePermanenceRefs(params.permanenceRefs) : undefined permanenceRefs: params.permanenceRefs?.length ? normalizePermanenceRefs(params.permanenceRefs) : undefined,
loraName: params.loraName || undefined,
loraStack: params.loraStack?.length ? params.loraStack : undefined
} }
mkdirSync(clipDir(params.ownerKey, clip.id), { recursive: true }) mkdirSync(clipDir(params.ownerKey, clip.id), { recursive: true })
const videoPath = clipVideoPath(params.ownerKey, clip.id) const videoPath = clipVideoPath(params.ownerKey, clip.id)
+305
View File
@@ -0,0 +1,305 @@
import { comfyConfigured, comfyFetch, getComfyHost } from '~/server/utils/comfy'
import { getBeastImageHost, imageComfyFetch, sameImageHost } from '~/server/utils/imageComfy'
import {
XAIGEN_LORA_MESSAGE,
filterLorasForStudio,
isXaigenOnlyLora,
normalizeLoraName,
normalizeLoraStack,
persistLoraFields,
type LoraKind,
type LoraStackItem
} from '~/utils/loras'
import { isXaigenStudio } from '~/utils/videoModels'
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
type WorkflowGraph = Record<string, WorkflowNode>
const LORA_LOADER = 'LoraLoader'
const LORA_MODEL_ONLY = 'LoraLoaderModelOnly'
const OBJECT_INFO_NODES = [
LORA_LOADER,
LORA_MODEL_ONLY,
'Power Lora Loader (rgthree)',
'Lora Loader Stack (rgthree)',
'LoraLoaderModelOnly [10]',
'WanVideoLoraSelect',
'LoraLoaderStacked'
]
const MODEL_FEED_CLASSES = new Set(['BasicGuider', 'CFGGuider', 'KSampler', 'KSamplerAdvanced'])
const CACHE_MS = 20_000
type LoraCache = {
at: number
image: string[]
video: string[]
}
let cache: LoraCache | null = null
let inflight: Promise<LoraCache> | null = null
function uniqueNames(values: unknown[]) {
const seen = new Set<string>()
const out: string[] = []
for (const value of values) {
const name = String(value || '').trim()
if (!name || seen.has(name)) continue
seen.add(name)
out.push(name)
}
return out
}
function comboFromSpec(spec: unknown): string[] {
if (!Array.isArray(spec)) return []
const first = spec[0]
if (Array.isArray(first)) return first.filter((item): item is string => typeof item === 'string' && item.trim().length > 0)
if (typeof first === 'string' && first.trim()) return spec.filter((item): item is string => typeof item === 'string' && item.trim().length > 0)
return []
}
function namesFromObjectInfoNode(info: unknown) {
if (!info || typeof info !== 'object') return [] as string[]
const input = (info as { input?: { required?: Record<string, unknown>; optional?: Record<string, unknown> } }).input
const buckets = [input?.required, input?.optional]
const names: string[] = []
for (const bucket of buckets) {
if (!bucket) continue
for (const [key, spec] of Object.entries(bucket)) {
if (!/lora/i.test(key)) continue
names.push(...comboFromSpec(spec))
}
}
return uniqueNames(names)
}
async function fetchJson(path: string, via: 'video' | 'image') {
const res = via === 'image'
? await imageComfyFetch(path, { signal: AbortSignal.timeout(8000) })
: await comfyFetch(path, { signal: AbortSignal.timeout(8000) })
if (!res.ok) return null
return res.json().catch(() => null)
}
async function fetchModelsLoras(via: 'video' | 'image') {
const payload = await fetchJson('/models/loras', via)
if (Array.isArray(payload)) return uniqueNames(payload)
if (payload && typeof payload === 'object' && Array.isArray((payload as { loras?: unknown[] }).loras)) {
return uniqueNames((payload as { loras: unknown[] }).loras)
}
return [] as string[]
}
async function fetchObjectInfoLoras(via: 'video' | 'image') {
const image: string[] = []
const video: string[] = []
let any = false
for (const node of OBJECT_INFO_NODES) {
const info = await fetchJson(`/object_info/${encodeURIComponent(node)}`, via)
if (!info || typeof info !== 'object') continue
const record = info as Record<string, unknown>
const body = record[node] || (Object.keys(record).length === 1 ? record[Object.keys(record)[0] as string] : record)
const names = namesFromObjectInfoNode(body)
if (!names.length) continue
any = true
if (node === LORA_LOADER) image.push(...names)
else video.push(...names)
if (node !== LORA_LOADER && node !== LORA_MODEL_ONLY) {
image.push(...names)
video.push(...names)
}
}
if (any) {
return { image: uniqueNames(image), video: uniqueNames(video) }
}
const all = await fetchJson('/object_info', via)
if (!all || typeof all !== 'object') return { image: [] as string[], video: [] as string[] }
const imageAll: string[] = []
const videoAll: string[] = []
for (const [classType, info] of Object.entries(all as Record<string, unknown>)) {
if (!/lora/i.test(classType)) continue
const names = namesFromObjectInfoNode(info)
if (classType === LORA_LOADER) imageAll.push(...names)
else videoAll.push(...names)
}
return { image: uniqueNames(imageAll), video: uniqueNames(videoAll) }
}
async function discoverFromHost(via: 'video' | 'image') {
const models = await fetchModelsLoras(via).catch(() => [] as string[])
const fromInfo = await fetchObjectInfoLoras(via).catch(() => ({ image: [] as string[], video: [] as string[] }))
const image = uniqueNames([...fromInfo.image, ...models])
const video = uniqueNames([...fromInfo.video, ...models])
if (!image.length && video.length) return { image: video, video }
if (!video.length && image.length) return { image, video: image }
return { image, video }
}
async function loadLoraCache(): Promise<LoraCache> {
const now = Date.now()
if (cache && now - cache.at < CACHE_MS) return cache
if (inflight) return inflight
inflight = (async () => {
const videoHost = comfyConfigured() ? getComfyHost() : ''
const imageHost = getBeastImageHost()
const same = Boolean(videoHost && imageHost && sameImageHost(videoHost, imageHost))
const video = videoHost
? await discoverFromHost('video').catch(() => ({ image: [] as string[], video: [] as string[] }))
: { image: [] as string[], video: [] as string[] }
const image = imageHost && !same
? await discoverFromHost('image').catch(() => ({ image: [] as string[], video: [] as string[] }))
: video
const xaigen = isXaigenStudio()
const next: LoraCache = {
at: Date.now(),
image: filterLorasForStudio(uniqueNames([...image.image, ...video.image]), xaigen),
video: filterLorasForStudio(uniqueNames([...video.video, ...image.video]), xaigen)
}
cache = next
return next
})().finally(() => {
inflight = null
})
return inflight
}
export async function listStudioLoras() {
try {
return await loadLoraCache()
} catch {
return { at: 0, image: [] as string[], video: [] as string[] }
}
}
export function assertLoraAllowed(raw: unknown, kind: LoraKind) {
const name = normalizeLoraName(raw)
if (!name) return ''
if (isXaigenOnlyLora(name) && !isXaigenStudio()) {
throw createError({ statusCode: 400, statusMessage: XAIGEN_LORA_MESSAGE })
}
const known = cache?.[kind]
if (known?.length && !known.includes(name) && !known.some(item => item.replace(/\\/g, '/') === name)) {
throw createError({ statusCode: 400, statusMessage: `Unknown ${kind} LoRA` })
}
return name
}
export function parsePostedLora(raw: unknown, kind: LoraKind) {
return parsePostedLoraStack(raw, kind)[0]?.name || ''
}
export function parsePostedLoraStack(raw: unknown, kind: LoraKind): LoraStackItem[] {
const out: LoraStackItem[] = []
for (const item of normalizeLoraStack(raw)) {
const name = assertLoraAllowed(item.name, kind)
if (!name) continue
out.push({ ...item, name })
}
return out
}
export function parseShotLoras(raw: unknown, shotCount: number, kind: LoraKind = 'video') {
return parseShotLoraStacks(raw, shotCount, kind).map(stack => stack[0]?.name || '')
}
export function parseShotLoraStacks(raw: unknown, shotCount: number, kind: LoraKind = 'video'): LoraStackItem[][] {
const empty = Array.from({ length: shotCount }, () => [] as LoraStackItem[])
if (!raw) return empty
let parsed: unknown = raw
if (typeof raw === 'string') {
try {
parsed = JSON.parse(raw)
} catch {
return empty
}
}
if (!Array.isArray(parsed)) return empty
return Array.from({ length: shotCount }, (_, index) => parsePostedLoraStack(parsed[index], kind))
}
export { persistLoraFields }
function linkSource(value: unknown): string | null {
return Array.isArray(value) && typeof value[0] === 'string' ? value[0] : null
}
function alreadyHasLora(graph: WorkflowGraph, name: string) {
const wanted = name.replace(/\\/g, '/').toLowerCase()
return Object.values(graph).some((node) => {
if (node.class_type !== LORA_LOADER && node.class_type !== LORA_MODEL_ONLY) return false
return String(node.inputs.lora_name || '').replace(/\\/g, '/').toLowerCase() === wanted
})
}
function injectAfter(
graph: WorkflowGraph,
sourceId: string,
nodeId: string,
node: WorkflowNode
) {
if (!graph[sourceId] || graph[nodeId]) return
graph[nodeId] = node
for (const [id, other] of Object.entries(graph)) {
if (id === nodeId) continue
for (const [key, value] of Object.entries(other.inputs)) {
const src = linkSource(value)
if (src === sourceId && Array.isArray(value)) {
other.inputs[key] = [nodeId, value[1]]
}
}
}
}
function findModelFeed(graph: WorkflowGraph) {
for (const node of Object.values(graph)) {
if (!MODEL_FEED_CLASSES.has(node.class_type)) continue
const source = linkSource(node.inputs.model)
if (source && graph[source]) return source
}
return ''
}
export function applyUserLoraToGraph(graph: WorkflowGraph, stack?: unknown) {
const items = normalizeLoraStack(stack).filter(item => !alreadyHasLora(graph, item.name))
if (!items.length) return
const clipLoaders = Object.entries(graph).filter(([, node]) => node.class_type === LORA_LOADER)
if (clipLoaders.length) {
for (const [id] of clipLoaders) {
let sourceId = id
for (const [index, item] of items.entries()) {
const nodeId = `user:lora:${id}:${index}`
injectAfter(graph, sourceId, nodeId, {
class_type: LORA_LOADER,
inputs: {
lora_name: item.name,
strength_model: item.strengthModel,
strength_clip: item.strengthClip,
model: [sourceId, 0],
clip: [sourceId, 1]
},
_meta: { title: items.length === 1 ? 'User LoRA' : `User LoRA ${index + 1}` }
})
sourceId = nodeId
}
}
return
}
let source = findModelFeed(graph)
if (!source) return
for (const [index, item] of items.entries()) {
const nodeId = index === 0 ? 'user:lora' : `user:lora:${index}`
injectAfter(graph, source, nodeId, {
class_type: LORA_MODEL_ONLY,
inputs: {
lora_name: item.name,
strength_model: item.strengthModel,
model: [source, 0]
},
_meta: { title: items.length === 1 ? 'User LoRA' : `User LoRA ${index + 1}` }
})
source = nodeId
}
}
+17 -3
View File
@@ -47,13 +47,17 @@ export interface PendingJob {
samplerName?: string samplerName?: string
scheduler?: string scheduler?: string
hideInput?: boolean hideInput?: boolean
extensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[] extensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: import('~/utils/loras').LoraStackItem[] }[]
remainingExtensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[] remainingExtensions?: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: import('~/utils/loras').LoraStackItem[] }[]
currentClipId?: string currentClipId?: string
queueId?: string queueId?: string
queueAutoRun?: boolean queueAutoRun?: boolean
queueBudget?: number queueBudget?: number
stopAfterCurrent?: boolean stopAfterCurrent?: boolean
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
} }
function pendingRoot() { function pendingRoot() {
@@ -139,6 +143,10 @@ export function pendingFromJob(job: Job, overrides: Partial<PendingJob> = {}): P
globalLocks: library.globalLocks, globalLocks: library.globalLocks,
permanenceRefs: library.permanenceRefs, permanenceRefs: library.permanenceRefs,
shotPermanenceRefs: library.shotPermanenceRefs, shotPermanenceRefs: library.shotPermanenceRefs,
loraName: library.loraName,
loraStack: library.loraStack,
shotLoras: library.shotLoras,
shotLoraStacks: library.shotLoraStacks,
queueId: library.queueId, queueId: library.queueId,
queueAutoRun: library.queueAutoRun, queueAutoRun: library.queueAutoRun,
queueBudget: library.queueBudget, queueBudget: library.queueBudget,
@@ -191,7 +199,11 @@ export function libraryFromPending(pending: PendingJob): NonNullable<Job['librar
stopAfterCurrent: pending.stopAfterCurrent, stopAfterCurrent: pending.stopAfterCurrent,
globalLocks: pending.globalLocks, globalLocks: pending.globalLocks,
permanenceRefs: pending.permanenceRefs, permanenceRefs: pending.permanenceRefs,
shotPermanenceRefs: pending.shotPermanenceRefs shotPermanenceRefs: pending.shotPermanenceRefs,
loraName: pending.loraName,
loraStack: pending.loraStack,
shotLoras: pending.shotLoras,
shotLoraStacks: pending.shotLoraStacks
} }
} }
@@ -318,6 +330,8 @@ export async function completePendingIfReady(pending: PendingJob) {
sound: pending.sound, sound: pending.sound,
globalLocks: pending.globalLocks, globalLocks: pending.globalLocks,
permanenceRefs: mergePermanenceRefs(pending.permanenceRefs, pending.shotPermanenceRefs?.[pending.chainIndex || 0]), permanenceRefs: mergePermanenceRefs(pending.permanenceRefs, pending.shotPermanenceRefs?.[pending.chainIndex || 0]),
loraName: pending.loraName,
loraStack: pending.loraStack,
segmentFirstFrame segmentFirstFrame
}) })
await purgeComfyArtifacts({ await purgeComfyArtifacts({
+45 -25
View File
@@ -17,23 +17,7 @@ export interface PromptJob {
const jobs = new Map<string, PromptJob>() const jobs = new Map<string, PromptJob>()
const MAX_JOBS = 20 const MAX_JOBS = 20
function normalizeHost(raw: string, port = '') {
let host = String(raw || '').trim().replace(/\/$/, '')
if (!host) return ''
if (!/^https?:\/\//i.test(host)) host = `http://${host}`
try {
const url = new URL(host)
if (port && !url.port) url.port = port
return url.origin
} catch {
return port ? `${host}:${port}` : host
}
}
export function getPromptComfyHost() { export function getPromptComfyHost() {
const config = useRuntimeConfig()
const explicit = String(config.promptComfyHost || process.env.PROMPT_COMFY_HOST || '').trim()
if (explicit) return normalizeHost(explicit)
return getSidecarImageHost() return getSidecarImageHost()
} }
@@ -44,14 +28,14 @@ export function promptComfyConfigured() {
async function promptComfyFetch(path: string, init?: RequestInit) { async function promptComfyFetch(path: string, init?: RequestInit) {
const host = getPromptComfyHost() const host = getPromptComfyHost()
if (!host) { if (!host) {
throw createError({ statusCode: 503, statusMessage: 'Sidecar Qwen VL is not configured. Set IMAGE_COMFY_HOST.' }) throw createError({ statusCode: 503, statusMessage: 'Qwen VL is not configured. Set COMFY_HOST or IMAGE_COMFY_HOST to Beast.' })
} }
try { try {
return await fetch(`${host}${path}`, init) return await fetch(`${host}${path}`, init)
} catch (error) { } catch (error) {
throw createError({ throw createError({
statusCode: 502, statusCode: 502,
statusMessage: `Sidecar Qwen VL is unreachable (${host})`, statusMessage: `Beast Qwen VL is unreachable (${host})`,
data: { cause: error instanceof Error ? error.message : String(error) } data: { cause: error instanceof Error ? error.message : String(error) }
}) })
} }
@@ -75,7 +59,7 @@ async function uploadPromptImage(file: { filename: string; data: Buffer; type?:
body.append('type', 'input') body.append('type', 'input')
const res = await promptComfyFetch('/upload/image', { method: 'POST', body }) const res = await promptComfyFetch('/upload/image', { method: 'POST', body })
if (!res.ok) { if (!res.ok) {
throw createError({ statusCode: 502, statusMessage: `Sidecar image upload failed (${res.status})` }) throw createError({ statusCode: 502, statusMessage: `Beast image upload failed (${res.status})` })
} }
const uploaded = (await res.json()) as { name: string; subfolder?: string; type?: string } const uploaded = (await res.json()) as { name: string; subfolder?: string; type?: string }
return uploaded.name || filename return uploaded.name || filename
@@ -89,14 +73,48 @@ async function queuePromptGraph(graph: unknown, clientId: string) {
}) })
const payload = await res.json().catch(() => ({})) const payload = await res.json().catch(() => ({}))
if (!res.ok) { if (!res.ok) {
const raw = JSON.stringify(payload)
if (/does not exist|class_type|not found/i.test(raw)) {
throw createError({
statusCode: 503,
statusMessage: 'Beast ComfyUI rejected the Qwen VL graph. The GGUF VLM nodes are missing on this instance — Recommend will not use the sidecar.',
data: payload
})
}
const message = (payload as { error?: { message?: string } }).error?.message const message = (payload as { error?: { message?: string } }).error?.message
|| (payload as { node_errors?: unknown }).node_errors || (payload as { node_errors?: unknown }).node_errors
|| `Sidecar queue failed (${res.status})` || `Beast queue failed (${res.status})`
throw createError({ statusCode: 502, statusMessage: String(message), data: payload }) throw createError({ statusCode: 502, statusMessage: String(message), data: payload })
} }
return payload as { prompt_id: string } return payload as { prompt_id: string }
} }
const QWEN_VL_NODE_TYPES = ['GGUF_VLM_ModelLoader', 'GGUF_VLM_ImageToText', 'AigenPreviewText']
async function assertQwenVlWorkflow() {
const res = await promptComfyFetch('/object_info')
if (!res.ok) {
throw createError({
statusCode: 502,
statusMessage: `Beast ComfyUI did not return node info (${res.status})`
})
}
const info = await res.json().catch(() => null)
if (!info || typeof info !== 'object') {
throw createError({
statusCode: 502,
statusMessage: 'Beast ComfyUI returned unreadable node info'
})
}
const missing = QWEN_VL_NODE_TYPES.filter(name => !(name in (info as Record<string, unknown>)))
if (missing.length) {
throw createError({
statusCode: 503,
statusMessage: `Beast ComfyUI is missing Qwen VL nodes (${missing.join(', ')}). Recommend is wired to Beast only and will not fall back to the sidecar.`
})
}
}
async function fetchPromptHistory(promptId: string) { async function fetchPromptHistory(promptId: string) {
const res = await promptComfyFetch(`/history/${encodeURIComponent(promptId)}`) const res = await promptComfyFetch(`/history/${encodeURIComponent(promptId)}`)
if (!res.ok) return null if (!res.ok) return null
@@ -149,9 +167,9 @@ function historyError(history: Record<string, unknown> | null, promptId: string)
const err = entry?.status?.messages?.find(([type]) => type === 'execution_error')?.[1] const err = entry?.status?.messages?.find(([type]) => type === 'execution_error')?.[1]
const message = String(err?.exception_message || err?.message || '').trim() const message = String(err?.exception_message || err?.message || '').trim()
if (status === 'interrupted') return 'Prompt recommend was interrupted.' if (status === 'interrupted') return 'Prompt recommend was interrupted.'
if (status === 'error') return message || 'Sidecar Qwen VL reported an execution error' if (status === 'error') return message || 'Beast Qwen VL reported an execution error'
if (entry?.status?.completed && !extractPromptText(history, promptId)) { if (entry?.status?.completed && !extractPromptText(history, promptId)) {
return 'Sidecar Qwen VL finished without prompt text' return 'Beast Qwen VL finished without prompt text'
} }
return null return null
} }
@@ -168,7 +186,7 @@ async function waitForPromptText(promptId: string, timeoutMs = 240_000) {
} }
await new Promise(resolve => setTimeout(resolve, 1200)) await new Promise(resolve => setTimeout(resolve, 1200))
} }
throw createError({ statusCode: 504, statusMessage: 'Sidecar Qwen VL timed out while writing the prompt' }) throw createError({ statusCode: 504, statusMessage: 'Beast Qwen VL timed out while writing the prompt' })
} }
export function createPromptJob(): PromptJob { export function createPromptJob(): PromptJob {
@@ -221,11 +239,13 @@ export async function runPromptRecommend(job: PromptJob, params: {
mode: string mode: string
}) { }) {
job.status = 'running' job.status = 'running'
job.message = 'Checking Sidecar Qwen VL…' job.message = 'Checking Beast Qwen VL…'
await ensureSidecarReady((status) => { await ensureSidecarReady((status) => {
job.message = status.message job.message = status.message
}) })
job.message = 'Uploading still to Sidecar Qwen VL…' job.message = 'Confirming Qwen VL nodes on Beast…'
await assertQwenVlWorkflow()
job.message = 'Uploading still to Beast Qwen VL…'
const imageName = await uploadPromptImage(params.image, job.id) const imageName = await uploadPromptImage(params.image, job.id)
job.message = 'Qwen VL is reading the still…' job.message = 'Qwen VL is reading the still…'
const graph = buildVisionPromptWorkflow({ const graph = buildVisionPromptWorkflow({
+31 -8
View File
@@ -1,6 +1,8 @@
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
import { join } from 'node:path' import { join } from 'node:path'
import { getJob, listJobs, type Job } from '~/server/utils/jobs' import { getJob, listJobs, type Job } from '~/server/utils/jobs'
import { parsePostedLoraStack, persistLoraFields } from '~/server/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
export type ShotQueueStatus = 'idle' | 'running' | 'paused' | 'complete' | 'error' export type ShotQueueStatus = 'idle' | 'running' | 'paused' | 'complete' | 'error'
export type ShotSegmentStatus = 'pending' | 'running' | 'complete' | 'error' export type ShotSegmentStatus = 'pending' | 'running' | 'complete' | 'error'
@@ -13,6 +15,8 @@ export interface ShotQueueSegment {
clipId?: string clipId?: string
error?: string error?: string
permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]
loraName?: string
loraStack?: LoraStackItem[]
} }
export interface ShotQueue { export interface ShotQueue {
@@ -46,6 +50,8 @@ export interface ShotQueue {
referenceStillIds?: Array<string | null> referenceStillIds?: Array<string | null>
globalLocks?: string globalLocks?: string
permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]
loraName?: string
loraStack?: LoraStackItem[]
segments: ShotQueueSegment[] segments: ShotQueueSegment[]
lastError?: string lastError?: string
} }
@@ -163,8 +169,10 @@ export async function createShotQueue(params: {
referenceStillIds?: Array<string | null> referenceStillIds?: Array<string | null>
globalLocks?: string globalLocks?: string
permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]
initial: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] } loraName?: string
extensions: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[] loraStack?: LoraStackItem[]
initial: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }
extensions: { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[]
jobId?: string jobId?: string
}): Promise<ShotQueue> { }): Promise<ShotQueue> {
const now = Date.now() const now = Date.now()
@@ -198,14 +206,17 @@ export async function createShotQueue(params: {
referenceStillIds: params.referenceStillIds, referenceStillIds: params.referenceStillIds,
globalLocks: params.globalLocks, globalLocks: params.globalLocks,
permanenceRefs: params.permanenceRefs, permanenceRefs: params.permanenceRefs,
loraName: params.loraName,
loraStack: params.loraStack,
segments: [ segments: [
{ index: 0, prompt: params.initial.prompt, duration: params.initial.duration, status: 'running', permanenceRefs: params.initial.permanenceRefs }, { index: 0, prompt: params.initial.prompt, duration: params.initial.duration, status: 'running', permanenceRefs: params.initial.permanenceRefs, ...persistLoraFields(params.initial.loraStack || params.initial.loraName) },
...params.extensions.map((item, index) => ({ ...params.extensions.map((item, index) => ({
index: index + 1, index: index + 1,
prompt: item.prompt, prompt: item.prompt,
duration: item.duration, duration: item.duration,
status: 'pending' as const, status: 'pending' as const,
permanenceRefs: item.permanenceRefs permanenceRefs: item.permanenceRefs,
...persistLoraFields(item.loraStack || item.loraName)
})) }))
] ]
} }
@@ -230,7 +241,7 @@ export async function updateShotQueue(owner: string, id: string, patch: (queue:
export async function patchShotQueueSegments( export async function patchShotQueueSegments(
owner: string, owner: string,
id: string, id: string,
segments: Array<{ index: number; prompt?: string; duration?: number }> segments: Array<{ index: number; prompt?: string; duration?: number; loraName?: string; loraStack?: unknown }>
) { ) {
return updateShotQueue(owner, id, (queue) => { return updateShotQueue(owner, id, (queue) => {
if (queue.status === 'complete') { if (queue.status === 'complete') {
@@ -248,6 +259,17 @@ export async function patchShotQueueSegments(
if (typeof next.duration === 'number' && Number.isFinite(next.duration)) { if (typeof next.duration === 'number' && Number.isFinite(next.duration)) {
segment.duration = Math.min(120, Math.max(0.5, Math.round(next.duration * 10) / 10)) segment.duration = Math.min(120, Math.max(0.5, Math.round(next.duration * 10) / 10))
} }
if ('loraStack' in next || 'loraName' in next) {
const stack = parsePostedLoraStack(next.loraStack ?? next.loraName, 'video')
const fields = persistLoraFields(stack)
if (fields.loraStack) {
segment.loraStack = fields.loraStack
segment.loraName = fields.loraName
} else {
delete segment.loraStack
delete segment.loraName
}
}
} }
}) })
} }
@@ -426,10 +448,10 @@ export function listOwnersWithQueues() {
.filter(owner => existsSync(queuesPath(owner))) .filter(owner => existsSync(queuesPath(owner)))
} }
export function remainingFromQueue(queue: ShotQueue): { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[] }[] { export function remainingFromQueue(queue: ShotQueue): { prompt: string; duration: number; permanenceRefs?: import('~/utils/globalLocks').PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[] {
return queue.segments return queue.segments
.filter(segment => segment.index > 0) .filter(segment => segment.index > 0)
.map(segment => ({ prompt: segment.prompt, duration: segment.duration, permanenceRefs: segment.permanenceRefs })) .map(segment => ({ prompt: segment.prompt, duration: segment.duration, permanenceRefs: segment.permanenceRefs, ...persistLoraFields(segment.loraStack || segment.loraName) }))
} }
export function lastCompletedIndex(queue: ShotQueue) { export function lastCompletedIndex(queue: ShotQueue) {
@@ -443,6 +465,7 @@ export function liveSegmentPrompt(queue: ShotQueue, extensionIndex: number) {
return { return {
prompt: segment?.prompt || '', prompt: segment?.prompt || '',
duration: segment?.duration || 5, duration: segment?.duration || 5,
permanenceRefs: segment?.permanenceRefs permanenceRefs: segment?.permanenceRefs,
...persistLoraFields(segment?.loraStack || segment?.loraName)
} }
} }
+33 -12
View File
@@ -3,8 +3,9 @@ import { join } from 'node:path'
import { getJob, listJobs, emitJob, type Job } from '~/server/utils/jobs' import { getJob, listJobs, emitJob, type Job } from '~/server/utils/jobs'
import { listPendingJobs, patchPendingJob, readPendingJob, deletePendingJob } from '~/server/utils/pending' import { listPendingJobs, patchPendingJob, readPendingJob, deletePendingJob } from '~/server/utils/pending'
import { getShotQueue } from '~/server/utils/shotQueue' import { getShotQueue } from '~/server/utils/shotQueue'
import { parseVideoWorkflow, type VideoWorkflowId } from '~/utils/videoModels' import { isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow, type VideoWorkflowId } from '~/utils/videoModels'
import { allowIdentityRefs, type PermanenceRef } from '~/utils/globalLocks' import { persistLoraFields } from '~/server/utils/loras'
import { resolveLoraStack } from '~/utils/loras'
export type StudioJobStatus = 'waiting' | 'running' | 'held' | 'complete' | 'error' | 'cancelled' export type StudioJobStatus = 'waiting' | 'running' | 'held' | 'complete' | 'error' | 'cancelled'
export type StudioJobKind = 'video' | 'edit' export type StudioJobKind = 'video' | 'edit'
@@ -33,11 +34,15 @@ export interface StudioJobPayload {
hideInput?: boolean hideInput?: boolean
folderLocked?: boolean folderLocked?: boolean
referenceStillIds: Array<string | null> referenceStillIds: Array<string | null>
extensions: { prompt: string; duration: number; permanenceRefs?: PermanenceRef[] }[] extensions: { prompt: string; duration: number; permanenceRefs?: PermanenceRef[]; loraName?: string; loraStack?: import('~/utils/loras').LoraStackItem[] }[]
queueAutoRun: boolean queueAutoRun: boolean
globalLocks?: string globalLocks?: string
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][] shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: import('~/utils/loras').LoraStackItem[][]
negative?: string negative?: string
passes?: { prompt: string }[] passes?: { prompt: string }[]
referenceStillId?: string referenceStillId?: string
@@ -636,7 +641,8 @@ async function startStudioEditJob(item: StudioJob) {
chainStep: 1, chainStep: 1,
chainTotal: 1 + passes.length, chainTotal: 1 + passes.length,
chainLabel: passes.length ? 'Pass 1' : undefined, chainLabel: passes.length ? 'Pass 1' : undefined,
passes passes,
...persistLoraFields(payload.loraStack || payload.loraName)
} }
}) })
@@ -649,7 +655,8 @@ async function startStudioEditJob(item: StudioJob) {
negative: payload.negative || '', negative: payload.negative || '',
steps: payload.steps, steps: payload.steps,
seed: job.library?.seed || payload.seed, seed: job.library?.seed || payload.seed,
cfg: payload.cfg cfg: payload.cfg,
...persistLoraFields(payload.loraStack || payload.loraName)
}).catch((error) => { }).catch((error) => {
const message = error instanceof Error ? error.message : String(error) const message = error instanceof Error ? error.message : String(error)
if (job.status !== 'error' && job.status !== 'cancelled' && job.status !== 'deferred') { if (job.status !== 'error' && job.status !== 'cancelled' && job.status !== 'deferred') {
@@ -680,6 +687,10 @@ export async function startStudioJob(item: StudioJob) {
const { existsSync, readFileSync } = await import('node:fs') const { existsSync, readFileSync } = await import('node:fs')
const payload = item.payload const payload = item.payload
const workflow = parseVideoWorkflow(payload.workflow)
if (isLtxWorkflow(workflow) && !ltxWorkflowEnabled()) {
throw new Error(LTX_DISABLED_MESSAGE)
}
const identity = allowIdentityRefs( const identity = allowIdentityRefs(
payload.useIdentityRefs, payload.useIdentityRefs,
payload.permanenceRefs, payload.permanenceRefs,
@@ -720,13 +731,16 @@ export async function startStudioJob(item: StudioJob) {
chainTotal: 1 + extensions.length, chainTotal: 1 + extensions.length,
chainLabel: extensions.length ? 'Initial' : undefined, chainLabel: extensions.length ? 'Initial' : undefined,
familyId: item.familyId, familyId: item.familyId,
workflow: parseVideoWorkflow(payload.workflow), workflow: workflow,
useIdentityRefs: identity, useIdentityRefs: identity,
queueAutoRun: extensions.length ? payload.queueAutoRun : false, queueAutoRun: extensions.length ? payload.queueAutoRun : false,
queueBudget: extensions.length && payload.queueAutoRun ? extensions.length : 0, queueBudget: extensions.length && payload.queueAutoRun ? extensions.length : 0,
globalLocks: payload.globalLocks, globalLocks: payload.globalLocks,
permanenceRefs: payload.permanenceRefs, permanenceRefs: payload.permanenceRefs,
shotPermanenceRefs: payload.shotPermanenceRefs shotPermanenceRefs: payload.shotPermanenceRefs,
...persistLoraFields(resolveLoraStack(payload.loraStack || payload.loraName, payload.shotLoraStacks?.[0] || payload.shotLoras?.[0])),
shotLoras: payload.shotLoras,
shotLoraStacks: payload.shotLoraStacks
} }
if (extensions.length) { if (extensions.length) {
@@ -749,14 +763,18 @@ export async function startStudioJob(item: StudioJob) {
fps: payload.fps, fps: payload.fps,
samplerName: payload.samplerName, samplerName: payload.samplerName,
scheduler: payload.scheduler, scheduler: payload.scheduler,
workflow: parseVideoWorkflow(payload.workflow), workflow: workflow,
sound: payload.sound, sound: payload.sound,
useIdentityRefs: identity, useIdentityRefs: identity,
referenceStillIds: payload.referenceStillIds, referenceStillIds: payload.referenceStillIds,
globalLocks: payload.globalLocks, globalLocks: payload.globalLocks,
permanenceRefs: payload.permanenceRefs, permanenceRefs: payload.permanenceRefs,
initial: { prompt: payload.prompt, duration: payload.duration, permanenceRefs: payload.shotPermanenceRefs?.[0] }, ...persistLoraFields(payload.loraStack || payload.loraName),
extensions, initial: { prompt: payload.prompt, duration: payload.duration, permanenceRefs: payload.shotPermanenceRefs?.[0], ...persistLoraFields(resolveLoraStack(payload.loraStack || payload.loraName, payload.shotLoraStacks?.[0] || payload.shotLoras?.[0])) },
extensions: extensions.map((item, index) => ({
...item,
...persistLoraFields(item.loraStack || item.loraName || payload.shotLoraStacks?.[index + 1] || payload.shotLoras?.[index + 1])
})),
jobId: job.id jobId: job.id
}) })
job.library.queueId = queue.id job.library.queueId = queue.id
@@ -802,10 +820,13 @@ export async function startStudioJob(item: StudioJob) {
samplerName: payload.samplerName, samplerName: payload.samplerName,
scheduler: payload.scheduler, scheduler: payload.scheduler,
extensions, extensions,
workflow: parseVideoWorkflow(payload.workflow), workflow: workflow,
duration: payload.duration, duration: payload.duration,
useIdentityRefs: identity, useIdentityRefs: identity,
referenceImages referenceImages,
...persistLoraFields(resolveLoraStack(payload.loraStack || payload.loraName, payload.shotLoraStacks?.[0] || payload.shotLoras?.[0])),
shotLoras: payload.shotLoras,
shotLoraStacks: payload.shotLoraStacks
}).catch((error) => { }).catch((error) => {
const message = error instanceof Error ? error.message : String(error) const message = error instanceof Error ? error.message : String(error)
if (job.status !== 'error' && job.status !== 'cancelled' && job.status !== 'deferred') { if (job.status !== 'error' && job.status !== 'cancelled' && job.status !== 'deferred') {
+24 -5
View File
@@ -21,6 +21,9 @@ import {
updateShotQueue updateShotQueue
} from '~/server/utils/shotQueue' } from '~/server/utils/shotQueue'
import { composeShotPrompt, allowIdentityRefs, type PermanenceRef } from '~/utils/globalLocks' import { composeShotPrompt, allowIdentityRefs, type PermanenceRef } from '~/utils/globalLocks'
import { persistLoraFields } from '~/server/utils/loras'
import { readLoraStack, resolveLoraStack } from '~/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
export type ChainImage = { filename: string; data: Buffer; type?: string } export type ChainImage = { filename: string; data: Buffer; type?: string }
@@ -38,7 +41,7 @@ type VideoChainParams = {
fps: number fps: number
samplerName: string samplerName: string
scheduler: string scheduler: string
extensions: { prompt: string; duration: number }[] extensions: { prompt: string; duration: number; loraName?: string; loraStack?: LoraStackItem[]; permanenceRefs?: PermanenceRef[] }[]
workflow: VideoWorkflowId workflow: VideoWorkflowId
duration: number duration: number
useIdentityRefs: boolean useIdentityRefs: boolean
@@ -46,6 +49,10 @@ type VideoChainParams = {
globalLocks?: string globalLocks?: string
permanenceRefs?: PermanenceRef[] permanenceRefs?: PermanenceRef[]
shotPermanenceRefs?: PermanenceRef[][] shotPermanenceRefs?: PermanenceRef[][]
loraName?: string
loraStack?: LoraStackItem[]
shotLoras?: string[]
shotLoraStacks?: LoraStackItem[][]
} }
function sleep(ms: number) { function sleep(ms: number) {
@@ -119,7 +126,10 @@ function paramsFromJob(job: Job): VideoChainParams {
referenceImages, referenceImages,
globalLocks: library.globalLocks, globalLocks: library.globalLocks,
permanenceRefs: library.permanenceRefs, permanenceRefs: library.permanenceRefs,
shotPermanenceRefs: library.shotPermanenceRefs shotPermanenceRefs: library.shotPermanenceRefs,
...persistLoraFields(readLoraStack(library)),
shotLoras: library.shotLoras,
shotLoraStacks: library.shotLoraStacks
} }
} }
@@ -137,6 +147,7 @@ export async function queueMiniMax(
job.status = 'uploading' job.status = 'uploading'
const chainIndex = job.library?.chainIndex || 0 const chainIndex = job.library?.chainIndex || 0
const graphId = chainIndex > 0 ? workflowForExtension(params.workflow) : params.workflow const graphId = chainIndex > 0 ? workflowForExtension(params.workflow) : params.workflow
if (job.library) Object.assign(job.library, persistLoraFields(params.loraStack || params.loraName || job.library.loraStack || job.library.loraName))
const engineName = isLtxWorkflow(graphId) ? 'LTX-2.3' : 'MiniMax H3' const engineName = isLtxWorkflow(graphId) ? 'LTX-2.3' : 'MiniMax H3'
const hasImage = Boolean(params.image?.data?.length) const hasImage = Boolean(params.image?.data?.length)
const uploading = !hasImage const uploading = !hasImage
@@ -197,7 +208,8 @@ export async function queueMiniMax(
workflow: graphId, workflow: graphId,
duration: params.duration, duration: params.duration,
useIdentityRefs: params.useIdentityRefs, useIdentityRefs: params.useIdentityRefs,
referenceImageNames: params.useIdentityRefs ? referenceNames : [] referenceImageNames: params.useIdentityRefs ? referenceNames : [],
...persistLoraFields(params.loraStack || params.loraName)
}) })
const queued = await queuePrompt(graph, job.clientId) const queued = await queuePrompt(graph, job.clientId)
@@ -305,6 +317,10 @@ export async function continueQueuedExtensions(
prompt: live.prompt || extensions[i].prompt, prompt: live.prompt || extensions[i].prompt,
duration: live.duration || extensions[i].duration duration: live.duration || extensions[i].duration
} }
const shotStack = resolveLoraStack(
readLoraStack(liveQueue).length ? readLoraStack(liveQueue) : (params.loraStack || params.loraName),
live.loraStack || live.loraName || extensions[i]?.loraStack || extensions[i]?.loraName
)
if (!ext.prompt.trim()) { if (!ext.prompt.trim()) {
throw new Error(`Shot ${i + 2} needs a prompt`) throw new Error(`Shot ${i + 2} needs a prompt`)
} }
@@ -328,6 +344,7 @@ export async function continueQueuedExtensions(
job.library.extendPart1Path = undefined job.library.extendPart1Path = undefined
job.library.thumb = undefined job.library.thumb = undefined
job.library.duration = ext.duration job.library.duration = ext.duration
Object.assign(job.library, persistLoraFields(shotStack))
if (liveQueue) { if (liveQueue) {
await updateShotQueue(job.library.ownerKey, liveQueue.id, (queue) => { await updateShotQueue(job.library.ownerKey, liveQueue.id, (queue) => {
const segment = queue.segments.find(item => item.index === i + 1) const segment = queue.segments.find(item => item.index === i + 1)
@@ -393,7 +410,8 @@ export async function continueQueuedExtensions(
workflow: workflowForExtension(params.workflow), workflow: workflowForExtension(params.workflow),
duration: ext.duration, duration: ext.duration,
useIdentityRefs: identity, useIdentityRefs: identity,
referenceImages: identity ? params.referenceImages : [] referenceImages: identity ? params.referenceImages : [],
...persistLoraFields(shotStack)
}) })
shotsLeft -= 1 shotsLeft -= 1
@@ -548,7 +566,8 @@ export async function startQueueBurst(owner: string, queueId: string, count: num
queueBudget: n, queueBudget: n,
globalLocks: queue.globalLocks, globalLocks: queue.globalLocks,
permanenceRefs: queue.permanenceRefs, permanenceRefs: queue.permanenceRefs,
shotPermanenceRefs: queue.segments.map(segment => segment.permanenceRefs || []) shotPermanenceRefs: queue.segments.map(segment => segment.permanenceRefs || []),
...persistLoraFields(readLoraStack(queue))
} }
setQueueJob(queue.id, job.id) setQueueJob(queue.id, job.id)
await updateShotQueue(owner, queue.id, (next) => { await updateShotQueue(owner, queue.id, (next) => {
+11 -2
View File
@@ -83,10 +83,17 @@ export function emitChainJob(job: Job, event: JobEvent, samplePct?: number) {
emitJob(job, next) emitJob(job, next)
} }
function nodeLabel(node: string) {
if (!node) return ''
if (NODE_LABELS[node]) return NODE_LABELS[node]
if (node.startsWith('user:lora')) return 'Applying LoRA'
return ''
}
function isNonSamplerProgress(node: string) { function isNonSamplerProgress(node: string) {
if (!node) return false if (!node) return false
if (isEncodingNode(node)) return true if (isEncodingNode(node)) return true
const label = NODE_LABELS[node] || '' const label = nodeLabel(node)
return /save|checkpoint|video combine|vhs/i.test(label) && !/sampler/i.test(label) return /save|checkpoint|video combine|vhs/i.test(label) && !/sampler/i.test(label)
} }
@@ -235,6 +242,8 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
sound: job.library.sound, sound: job.library.sound,
globalLocks: job.library.globalLocks, globalLocks: job.library.globalLocks,
permanenceRefs: clipRefsFromJob(job.library), permanenceRefs: clipRefsFromJob(job.library),
loraName: job.library.loraName,
loraStack: job.library.loraStack,
segmentFirstFrame segmentFirstFrame
}) })
job.clipId = clip.id job.clipId = clip.id
@@ -400,7 +409,7 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
} }
return return
} }
const label = NODE_LABELS[node] || `Running node ${node}` const label = nodeLabel(node) || `Running node ${node}`
const encoding = isEncodingNode(node) const encoding = isEncodingNode(node)
emitLocal({ emitLocal({
type: 'executing', type: 'executing',
+12
View File
@@ -3,10 +3,14 @@ import workflowV1 from '../assets/workflow_minimax_video.json'
import workflowV2 from '../assets/workflow_minimax_video_v2.json' import workflowV2 from '../assets/workflow_minimax_video_v2.json'
import workflowLtx from '../assets/workflow_ltx_video.json' import workflowLtx from '../assets/workflow_ltx_video.json'
import { buildIdentityPrompt } from '~/utils/identityPrompt' import { buildIdentityPrompt } from '~/utils/identityPrompt'
import { applyUserLoraToGraph } from '~/server/utils/loras'
import type { LoraStackItem } from '~/utils/loras'
import { import {
isLtxWorkflow, isLtxWorkflow,
isTextToVideo, isTextToVideo,
LTX_DISABLED_MESSAGE,
LTX_NEGATIVE, LTX_NEGATIVE,
ltxWorkflowEnabled,
parseVideoWorkflow, parseVideoWorkflow,
type VideoWorkflowId type VideoWorkflowId
} from '~/utils/videoModels' } from '~/utils/videoModels'
@@ -33,6 +37,8 @@ export interface GenerateParams {
duration?: number duration?: number
useIdentityRefs?: boolean useIdentityRefs?: boolean
referenceImageNames?: string[] referenceImageNames?: string[]
loraName?: string
loraStack?: LoraStackItem[]
} }
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } } type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
@@ -74,6 +80,9 @@ function templateFor(id: VideoWorkflowId) {
export function buildWorkflow(params: GenerateParams) { export function buildWorkflow(params: GenerateParams) {
const version = parseVideoWorkflow(params.workflow) const version = parseVideoWorkflow(params.workflow)
if (isLtxWorkflow(version) && !ltxWorkflowEnabled()) {
throw new Error(LTX_DISABLED_MESSAGE)
}
if (isLtxWorkflow(version)) return buildLtxWorkflow(params, version) if (isLtxWorkflow(version)) return buildLtxWorkflow(params, version)
const graph = structuredClone(templateFor(version === 't2v' ? 'v1' : version === 'v2' ? 'v2' : 'v1')) const graph = structuredClone(templateFor(version === 't2v' ? 'v1' : version === 'v2' ? 'v2' : 'v1'))
@@ -216,6 +225,7 @@ export function buildWorkflow(params: GenerateParams) {
if (graph['105:104']?.inputs) delete graph['105:104'].inputs.audio_vae if (graph['105:104']?.inputs) delete graph['105:104'].inputs.audio_vae
} }
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
return graph return graph
} }
@@ -288,6 +298,7 @@ function buildLtxWorkflow(params: GenerateParams, version: VideoWorkflowId) {
if (graph['12']?.inputs) graph['12'].inputs.latent = ['8', 0] if (graph['12']?.inputs) graph['12'].inputs.latent = ['8', 0]
if (graph['14']?.inputs) graph['14'].inputs.latent_image = ['8', 0] if (graph['14']?.inputs) graph['14'].inputs.latent_image = ['8', 0]
} }
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
return graph return graph
} }
@@ -366,6 +377,7 @@ const LABEL_OVERRIDES: Record<string, string> = {
'105:104': 'Encoding prompt & latent', '105:104': 'Encoding prompt & latent',
'105:107': 'Computing duration', '105:107': 'Computing duration',
'105:111': 'Setting duration', '105:111': 'Setting duration',
'user:lora': 'Applying LoRA',
'105:121': 'Applying turbo LoRA', '105:121': 'Applying turbo LoRA',
'105:122': 'Selecting model path', '105:122': 'Selecting model path',
'105:123': 'Selecting steps', '105:123': 'Selecting steps',
+40
View File
@@ -0,0 +1,40 @@
import type { LoraStackItem } from '~/utils/loras'
import type { VideoWorkflowId } from '~/utils/videoModels'
export type GenerationPresetKind = 'video' | 'image'
export type VideoPresetSettings = {
workflow?: VideoWorkflowId
turbo?: boolean
cfg?: number
fps?: number
samplerName?: string
scheduler?: string
duration?: number
aspect?: string
sound?: boolean
}
export type ImagePresetSettings = {
steps?: number
cfg?: number
negative?: string
}
export type GenerationPreset = {
id: string
name: string
kind: GenerationPresetKind
createdAt: number
updatedAt: number
loraStack: LoraStackItem[]
settings: VideoPresetSettings | ImagePresetSettings
}
export function parseGenerationPresetKind(raw: unknown): GenerationPresetKind {
return raw === 'image' ? 'image' : 'video'
}
export function normalizePresetName(raw: unknown) {
return String(raw || '').trim().replace(/\s+/g, ' ').slice(0, 80)
}
+177
View File
@@ -0,0 +1,177 @@
export const XAIGEN_LORA_MESSAGE = 'This LoRA is only available on xAIGen'
export type LoraKind = 'image' | 'video'
export type LoraStackItem = {
name: string
strengthModel: number
strengthClip: number
}
export type LoraStack = LoraStackItem[]
export const LORA_STRENGTH_MIN = -2
export const LORA_STRENGTH_MAX = 2
export const LORA_STRENGTH_DEFAULT = 1
export const LORA_STRENGTH_STEP = 0.05
export const LORA_STACK_MAX = 8
export function loraBasename(name: string) {
const trimmed = String(name || '').trim().replace(/\\/g, '/')
const parts = trimmed.split('/').filter(Boolean)
return parts[parts.length - 1] || trimmed
}
export function isXaigenOnlyLora(name: string) {
return /^xaigen/i.test(loraBasename(name))
}
export function loraDisplayName(name: string) {
return loraBasename(name).replace(/\.(safetensors|sft|ckpt|pt|bin)$/i, '')
}
export function normalizeLoraName(raw: unknown) {
const value = String(raw || '').trim().replace(/\\/g, '/')
if (!value) return ''
if (value.length > 240) return ''
if (value.includes('..') || value.startsWith('/') || /[\0\n\r]/.test(value)) return ''
if (!/\.(safetensors|sft|ckpt|pt|bin)$/i.test(value)) return ''
return value
}
export function clampLoraStrength(raw: unknown) {
const value = Number(raw)
if (!Number.isFinite(value)) return LORA_STRENGTH_DEFAULT
const snapped = Math.round(value / LORA_STRENGTH_STEP) * LORA_STRENGTH_STEP
return Math.min(LORA_STRENGTH_MAX, Math.max(LORA_STRENGTH_MIN, Math.round(snapped * 100) / 100))
}
export function formatLoraStrength(value: number) {
const n = clampLoraStrength(value)
return Number.isInteger(n) ? String(n) : n.toFixed(2).replace(/0$/, '')
}
function loraKey(name: string) {
return name.replace(/\\/g, '/').toLowerCase()
}
export function makeLoraStackItem(
name: string,
strengthModel: unknown = LORA_STRENGTH_DEFAULT,
strengthClip: unknown = LORA_STRENGTH_DEFAULT
): LoraStackItem | null {
const normalized = normalizeLoraName(name)
if (!normalized) return null
return {
name: normalized,
strengthModel: clampLoraStrength(strengthModel),
strengthClip: clampLoraStrength(strengthClip)
}
}
function parseStackEntry(raw: unknown): LoraStackItem | null {
if (typeof raw === 'string') return makeLoraStackItem(raw)
if (!raw || typeof raw !== 'object') return null
const rec = raw as Record<string, unknown>
const name = rec.name ?? rec.loraName ?? rec.lora_name
return makeLoraStackItem(
name,
rec.strengthModel ?? rec.strength_model ?? LORA_STRENGTH_DEFAULT,
rec.strengthClip ?? rec.strength_clip ?? LORA_STRENGTH_DEFAULT
)
}
export function normalizeLoraStack(raw: unknown): LoraStackItem[] {
if (raw == null || raw === '') return []
let parsed: unknown = raw
if (typeof raw === 'string') {
const trimmed = raw.trim()
if (!trimmed) return []
if (trimmed.startsWith('[') || trimmed.startsWith('{')) {
try {
parsed = JSON.parse(trimmed)
} catch {
const item = makeLoraStackItem(trimmed)
return item ? [item] : []
}
} else {
const item = makeLoraStackItem(trimmed)
return item ? [item] : []
}
}
const entries = Array.isArray(parsed) ? parsed : [parsed]
const seen = new Set<string>()
const out: LoraStackItem[] = []
for (const entry of entries) {
if (out.length >= LORA_STACK_MAX) break
const item = parseStackEntry(entry)
if (!item) continue
const key = loraKey(item.name)
if (seen.has(key)) continue
seen.add(key)
out.push(item)
}
return out
}
export function readLoraStack(value?: { loraStack?: unknown; loraName?: unknown } | string | unknown[] | null) {
if (value == null || value === '') return [] as LoraStackItem[]
if (typeof value === 'string' || Array.isArray(value)) return normalizeLoraStack(value)
const stack = normalizeLoraStack(value.loraStack)
if (stack.length) return stack
return normalizeLoraStack(value.loraName)
}
export function resolveLoraName(fallback?: string | null, override?: string | null) {
const over = normalizeLoraName(override)
if (over) return over
return normalizeLoraName(fallback)
}
export function resolveLoraStack(fallback?: unknown, override?: unknown) {
const over = normalizeLoraStack(override)
if (over.length) return over
return normalizeLoraStack(fallback)
}
export function primaryLoraName(stack?: LoraStackItem[] | null) {
return stack?.[0]?.name || ''
}
export function loraStackLabel(stack?: LoraStackItem[] | null) {
return (stack || []).map(item => loraDisplayName(item.name)).join(' · ')
}
export function persistLoraFields(stack?: unknown) {
const items = normalizeLoraStack(stack)
if (!items.length) return { loraName: undefined as string | undefined, loraStack: undefined as LoraStackItem[] | undefined }
return { loraName: items[0].name, loraStack: items }
}
export function loraStacksEqual(a?: unknown, b?: unknown) {
const left = normalizeLoraStack(a)
const right = normalizeLoraStack(b)
if (left.length !== right.length) return false
return left.every((item, index) => (
loraKey(item.name) === loraKey(right[index].name)
&& item.strengthModel === right[index].strengthModel
&& item.strengthClip === right[index].strengthClip
))
}
export function filterLorasForStudio(names: string[], xaigenStudio: boolean) {
const seen = new Set<string>()
const out: string[] = []
for (const raw of names) {
const name = String(raw || '').trim()
if (!name || seen.has(name)) continue
if (!xaigenStudio && isXaigenOnlyLora(name)) continue
seen.add(name)
out.push(name)
}
return out
}
export function filterLoraStackForStudio(stack: unknown, xaigenStudio: boolean) {
return normalizeLoraStack(stack).filter(item => xaigenStudio || !isXaigenOnlyLora(item.name))
}
+25
View File
@@ -12,6 +12,25 @@ export function parseVideoWorkflow(raw: unknown): VideoWorkflowId {
return 'v1' return 'v1'
} }
export const LTX_DISABLED_MESSAGE = 'LTX Pink Cherry is not available on this instance'
export function isXaigenStudio(publicConfig?: { studio?: unknown; instanceName?: unknown }) {
let pub = publicConfig
if (!pub) {
try {
pub = useRuntimeConfig().public as { studio?: unknown; instanceName?: unknown }
} catch {
pub = {}
}
}
return /^xaigen$/i.test(String(pub.studio || '').trim())
|| /xaigen/i.test(String(pub.instanceName || '').trim())
}
export function ltxWorkflowEnabled(publicConfig?: { studio?: unknown; instanceName?: unknown }) {
return isXaigenStudio(publicConfig)
}
export function videoEngineOf(id: VideoWorkflowId): VideoEngineId { export function videoEngineOf(id: VideoWorkflowId): VideoEngineId {
return id.startsWith('ltx') ? 'ltx' : 'minimax' return id.startsWith('ltx') ? 'ltx' : 'minimax'
} }
@@ -44,6 +63,12 @@ export function composeVideoWorkflow(engine: VideoEngineId, start: VideoStartId,
return graph === 'v2' ? 'v2' : 'v1' return graph === 'v2' ? 'v2' : 'v1'
} }
export function coerceVideoWorkflow(raw: unknown, allowLtx?: boolean): VideoWorkflowId {
const id = parseVideoWorkflow(raw)
if ((allowLtx ?? ltxWorkflowEnabled()) || !isLtxWorkflow(id)) return id
return composeVideoWorkflow('minimax', videoStartOf(id), 'v1')
}
export function defaultVideoSteps(turbo: boolean, workflow: string = 'v1') { export function defaultVideoSteps(turbo: boolean, workflow: string = 'v1') {
const id = parseVideoWorkflow(workflow) const id = parseVideoWorkflow(workflow)
if (!turbo) return isLtxWorkflow(id) ? 12 : 20 if (!turbo) return isLtxWorkflow(id) ? 12 : 20