Ship MiniMax v2 identity refs and Qwen sidecar edits.

Keep v1 as the default video graph, relay image edits to the Qwen Lightning box, and keep /api/health up if one Comfy host is unset.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-08-26 14:29:15 -05:00
co-authored by Cursor
parent 551c29a62a
commit c55a48a57e
22 changed files with 2061 additions and 75 deletions
+7
View File
@@ -27,3 +27,10 @@ LIBRARY_DIR=/data/library
# COMFY_FILENAME_PREFIX=video/xAIGen # COMFY_FILENAME_PREFIX=video/xAIGen
# Desktop Comfy files are deleted after a successful library save (set false to keep them) # Desktop Comfy files are deleted after a successful library save (set false to keep them)
# PURGE_COMFY_OUTPUTS=true # PURGE_COMFY_OUTPUTS=true
# Second Comfy box (sidecar / Qwen image edit)
IMAGE_COMFY_HOST=http://192.168.77.101:8188
IMAGE_COMFY_PORT=8188
# Optional host agent on the sidecar (scripts/comfy-host-agent.mjs)
# IMAGE_COMFY_CONTROL_URL=http://192.168.77.101:8199
# IMAGE_COMFY_CONTROL_TOKEN=
+2 -1
View File
@@ -4,7 +4,7 @@
<div class="glass w-full max-w-3xl rounded-3xl p-5 sm:p-6"> <div class="glass w-full max-w-3xl rounded-3xl p-5 sm:p-6">
<div class="flex items-start justify-between gap-3"> <div class="flex items-start justify-between gap-3">
<div> <div>
<h2 class="font-display text-2xl font-bold">Choose an image</h2> <h2 class="font-display text-2xl font-bold">{{ heading || 'Choose an image' }}</h2>
<p class="mt-1 text-sm text-zinc-400">Pick a still from this folder or upload a new one.</p> <p class="mt-1 text-sm text-zinc-400">Pick a still from this folder or upload a new one.</p>
</div> </div>
<div class="flex items-center gap-2"> <div class="flex items-center gap-2">
@@ -147,6 +147,7 @@ const props = defineProps<{
stills: PickerStill[] stills: PickerStill[]
folderId: string folderId: string
hideInputPreview?: boolean hideInputPreview?: boolean
heading?: string
}>() }>()
const emit = defineEmits<{ const emit = defineEmits<{
+5
View File
@@ -29,6 +29,11 @@ export default defineNuxtConfig({
comfyLaunchCmd: process.env.COMFY_LAUNCH_CMD || '', comfyLaunchCmd: process.env.COMFY_LAUNCH_CMD || '',
comfyControlUrl: process.env.COMFY_CONTROL_URL || '', comfyControlUrl: process.env.COMFY_CONTROL_URL || '',
comfyControlToken: process.env.COMFY_CONTROL_TOKEN || '', comfyControlToken: process.env.COMFY_CONTROL_TOKEN || '',
imageComfyHost: process.env.IMAGE_COMFY_HOST || '',
imageComfyPort: process.env.IMAGE_COMFY_PORT || '',
imageComfyLaunchCmd: process.env.IMAGE_COMFY_LAUNCH_CMD || '',
imageComfyControlUrl: process.env.IMAGE_COMFY_CONTROL_URL || '',
imageComfyControlToken: process.env.IMAGE_COMFY_CONTROL_TOKEN || '',
sessionSecret: process.env.NUXT_SESSION_SECRET || '', sessionSecret: process.env.NUXT_SESSION_SECRET || '',
oidcClientId: process.env.NUXT_OIDC_CLIENT_ID || '', oidcClientId: process.env.NUXT_OIDC_CLIENT_ID || '',
oidcClientSecret: process.env.NUXT_OIDC_CLIENT_SECRET || '', oidcClientSecret: process.env.NUXT_OIDC_CLIENT_SECRET || '',
+387 -22
View File
@@ -6,7 +6,7 @@
<div class="flex h-10 w-10 items-center justify-center rounded-xl bg-amber-400 text-zinc-950 font-display font-extrabold">A</div> <div class="flex h-10 w-10 items-center justify-center rounded-xl bg-amber-400 text-zinc-950 font-display font-extrabold">A</div>
<div> <div>
<h1 class="font-display text-xl font-bold leading-none">{{ instanceName }}</h1> <h1 class="font-display text-xl font-bold leading-none">{{ instanceName }}</h1>
<p class="text-[11px] uppercase tracking-[0.22em] text-zinc-500">MiniMax H3 · ComfyUI relay</p> <p class="text-[11px] uppercase tracking-[0.22em] text-zinc-500">MiniMax H3 · Qwen Edit · ComfyUI</p>
</div> </div>
</div> </div>
<div class="flex items-center gap-3 text-sm"> <div class="flex items-center gap-3 text-sm">
@@ -17,6 +17,14 @@
<span class="h-2 w-2 rounded-full" :class="comfyBadge.dot" /> <span class="h-2 w-2 rounded-full" :class="comfyBadge.dot" />
{{ comfyBadge.label }} {{ comfyBadge.label }}
</span> </span>
<span
class="hidden lg:inline-flex items-center gap-2 rounded-full border px-3 py-1"
:class="imageComfyBadge.className"
:title="imageComfyHost || 'Image edit sidecar'"
>
<span class="h-2 w-2 rounded-full" :class="imageComfyBadge.dot" />
{{ imageComfyBadge.label }}
</span>
<span class="hidden md:block text-zinc-400">{{ userName }}</span> <span class="hidden md:block text-zinc-400">{{ userName }}</span>
<button type="button" class="rounded-full border border-white/10 px-3 py-1 text-zinc-300 hover:border-amber-300/50 hover:text-white" @click="settingsOpen = true"> <button type="button" class="rounded-full border border-white/10 px-3 py-1 text-zinc-300 hover:border-amber-300/50 hover:text-white" @click="settingsOpen = true">
Settings Settings
@@ -29,8 +37,28 @@
<main class="mx-auto grid max-w-7xl gap-6 px-4 py-6 sm:px-6 lg:grid-cols-[minmax(0,1fr)_minmax(0,1fr)]"> <main class="mx-auto grid max-w-7xl gap-6 px-4 py-6 sm:px-6 lg:grid-cols-[minmax(0,1fr)_minmax(0,1fr)]">
<section class="glass rounded-3xl p-5 sm:p-6 space-y-5"> <section class="glass rounded-3xl p-5 sm:p-6 space-y-5">
<div> <div>
<div class="mb-3 flex flex-wrap gap-2">
<button
type="button"
class="rounded-full border px-3 py-1 text-xs font-medium"
:class="studioMode === 'video' ? 'border-amber-300/70 bg-amber-400/10 text-amber-100' : 'border-white/10 text-zinc-400 hover:text-white'"
@click="studioMode = 'video'"
>
Video
</button>
<button
type="button"
class="rounded-full border px-3 py-1 text-xs font-medium"
:class="studioMode === 'edit' ? 'border-amber-300/70 bg-amber-400/10 text-amber-100' : 'border-white/10 text-zinc-400 hover:text-white'"
@click="studioMode = 'edit'"
>
Image edit
</button>
</div>
<h2 class="font-display text-2xl font-bold">Input</h2> <h2 class="font-display text-2xl font-bold">Input</h2>
<p class="text-sm text-zinc-400">Drop a still, describe the motion, then relay the job to the local desktop.</p> <p class="text-sm text-zinc-400">{{ studioMode === 'edit'
? 'Drop a still and describe the edit. This relays to the sidecar Qwen Comfy box once the workflow is mapped.'
: 'Drop a still, describe the motion, then relay the job to the local desktop.' }}</p>
</div> </div>
<div <div
@@ -39,7 +67,7 @@
@dragover.prevent="dragging = true" @dragover.prevent="dragging = true"
@dragleave.prevent="dragging = false" @dragleave.prevent="dragging = false"
@drop.prevent="onDrop" @drop.prevent="onDrop"
@click="pickerOpen = true" @click="openPicker('main')"
> >
<div v-if="preview && !hideInputPreview" class="relative"> <div v-if="preview && !hideInputPreview" class="relative">
<img :src="preview" alt="Source still" class="max-h-72 w-full rounded-xl object-contain bg-black/40"> <img :src="preview" alt="Source still" class="max-h-72 w-full rounded-xl object-contain bg-black/40">
@@ -59,8 +87,8 @@
<div> <div>
<div class="mb-2 flex flex-wrap items-center justify-between gap-2"> <div class="mb-2 flex flex-wrap items-center justify-between gap-2">
<label class="text-sm font-medium text-zinc-300">Motion & scene prompt</label> <label class="text-sm font-medium text-zinc-300">{{ studioMode === 'edit' ? 'Edit prompt' : 'Motion & scene prompt' }}</label>
<label class="flex cursor-pointer items-center gap-2 text-xs text-zinc-400"> <label v-if="studioMode === 'video'" class="flex cursor-pointer items-center gap-2 text-xs text-zinc-400">
<span class="relative inline-flex h-5 w-9 shrink-0 items-center"> <span class="relative inline-flex h-5 w-9 shrink-0 items-center">
<input v-model="shotScriptMode" type="checkbox" class="peer sr-only" role="switch"> <input v-model="shotScriptMode" type="checkbox" class="peer sr-only" role="switch">
<span class="absolute inset-0 rounded-full bg-zinc-700 transition peer-checked:bg-amber-400" /> <span class="absolute inset-0 rounded-full bg-zinc-700 transition peer-checked:bg-amber-400" />
@@ -69,18 +97,20 @@
Shot script Shot script
</label> </label>
</div> </div>
<p v-if="shotScriptMode" class="mb-2 text-xs text-zinc-500"> <p v-if="shotScriptMode && studioMode === 'video'" class="mb-2 text-xs text-zinc-500">
Paste a block. Text before the first marker is shot 1. Write <span class="font-medium text-zinc-300">shot 2</span>, <span class="font-medium text-zinc-300">shot 3</span>, and so on to queue extensions. Every shot uses the duration slider. Paste a block. Text before the first marker is shot 1. Write <span class="font-medium text-zinc-300">shot 2</span>, <span class="font-medium text-zinc-300">shot 3</span>, and so on to queue extensions. Every shot uses the duration slider.
</p> </p>
<textarea <textarea
v-model="prompt" v-model="prompt"
:rows="shotScriptMode ? 12 : 6" :rows="shotScriptMode ? 12 : 6"
class="w-full rounded-2xl border border-white/10 bg-zinc-950/70 px-4 py-3 text-sm outline-none ring-amber-300/40 placeholder:text-zinc-600 focus:ring-2" class="w-full rounded-2xl border border-white/10 bg-zinc-950/70 px-4 py-3 text-sm outline-none ring-amber-300/40 placeholder:text-zinc-600 focus:ring-2"
:placeholder="shotScriptMode :placeholder="studioMode === 'edit'
? 'Make the lighting warmer. Replace the background with a rainy street at night. Keep the face unchanged.'
: (shotScriptMode
? 'A woman walks into the village.\n\nshot 2\nShe crosses the stone bridge at dusk.\n\nshot 3\nShe opens the shop door and steps inside.' ? 'A woman walks into the village.\n\nshot 2\nShe crosses the stone bridge at dusk.\n\nshot 3\nShe opens the shop door and steps inside.'
: 'Camera slowly pushes in. Soft daylight, fabric ripples, dust motes in the air. Audio: quiet room tone, distant birds.'" : 'Camera slowly pushes in. Soft daylight, fabric ripples, dust motes in the air. Audio: quiet room tone, distant birds.')"
/> />
<div v-if="shotScriptMode" class="mt-3 space-y-2"> <div v-if="shotScriptMode && studioMode === 'video'" class="mt-3 space-y-2">
<p v-if="!parsedShots.length" class="text-xs text-zinc-500">Add a first shot, then <span class="text-zinc-300">shot 2</span> for an extension.</p> <p v-if="!parsedShots.length" class="text-xs text-zinc-500">Add a first shot, then <span class="text-zinc-300">shot 2</span> for an extension.</p>
<div <div
v-for="(shot, index) in parsedShots" v-for="(shot, index) in parsedShots"
@@ -96,7 +126,7 @@
</div> </div>
</div> </div>
<div v-if="!shotScriptMode" class="space-y-3"> <div v-if="studioMode === 'video' && !shotScriptMode" class="space-y-3">
<div <div
v-for="(item, index) in extensionQueue" v-for="(item, index) in extensionQueue"
:key="item.id" :key="item.id"
@@ -171,7 +201,7 @@
<span class="absolute left-0.5 top-0.5 h-5 w-5 rounded-full bg-white transition peer-checked:translate-x-5" /> <span class="absolute left-0.5 top-0.5 h-5 w-5 rounded-full bg-white transition peer-checked:translate-x-5" />
</span> </span>
</label> </label>
<label class="flex cursor-pointer items-center justify-between gap-3 rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3 text-sm"> <label v-if="studioMode === 'video'" class="flex cursor-pointer items-center justify-between gap-3 rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3 text-sm">
<span> <span>
<span class="block font-medium text-zinc-200">Generate sound</span> <span class="block font-medium text-zinc-200">Generate sound</span>
<span class="mt-0.5 block text-xs text-zinc-500">Off skips the audio VAE so Comfy encodes a silent clip.</span> <span class="mt-0.5 block text-xs text-zinc-500">Off skips the audio VAE so Comfy encodes a silent clip.</span>
@@ -182,7 +212,7 @@
<span class="absolute left-0.5 top-0.5 h-5 w-5 rounded-full bg-white transition peer-checked:translate-x-5" /> <span class="absolute left-0.5 top-0.5 h-5 w-5 rounded-full bg-white transition peer-checked:translate-x-5" />
</span> </span>
</label> </label>
<label class="flex cursor-pointer items-center justify-between gap-3 rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3 text-sm"> <label v-if="studioMode === 'video'" class="flex cursor-pointer items-center justify-between gap-3 rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3 text-sm">
<span> <span>
<span class="block font-medium text-zinc-200">Autoplay</span> <span class="block font-medium text-zinc-200">Autoplay</span>
<span class="mt-0.5 block text-xs text-zinc-500">Off leaves finished clips paused until you press play.</span> <span class="mt-0.5 block text-xs text-zinc-500">Off leaves finished clips paused until you press play.</span>
@@ -197,7 +227,79 @@
</div> </div>
</div> </div>
<details class="rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3"> <div v-if="studioMode === 'video'" class="space-y-3">
<div>
<p class="mb-2 text-xs uppercase tracking-wider text-zinc-500">MiniMax graph</p>
<div class="grid grid-cols-2 gap-2">
<button
type="button"
class="rounded-xl border px-3 py-2 text-left text-sm"
:class="videoWorkflow === 'v1' ? 'border-amber-300 bg-amber-400/10 text-amber-100' : 'border-white/10 hover:border-white/20'"
@click="videoWorkflow = 'v1'"
>
<span class="block font-semibold">v1 · validated</span>
<span class="text-xs text-zinc-400">First-frame image to video</span>
</button>
<button
type="button"
class="rounded-xl border px-3 py-2 text-left text-sm"
:class="videoWorkflow === 'v2' ? 'border-amber-300 bg-amber-400/10 text-amber-100' : 'border-white/10 hover:border-white/20'"
@click="videoWorkflow = 'v2'"
>
<span class="block font-semibold">v2 · identity refs</span>
<span class="text-xs text-zinc-400">Optional lock with up to 4 stills</span>
</button>
</div>
</div>
<div v-if="videoWorkflow === 'v2'" class="rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3 space-y-3">
<label class="flex cursor-pointer items-center justify-between gap-3 text-sm">
<span>
<span class="block font-medium text-zinc-200">Use identity references</span>
<span class="mt-0.5 block text-xs text-zinc-500">Off uses first-frame only. On sends extra stills into Ref2VA.</span>
</span>
<span class="relative inline-flex h-6 w-11 shrink-0 items-center">
<input v-model="useIdentityRefs" type="checkbox" class="peer sr-only" role="switch">
<span class="absolute inset-0 rounded-full bg-zinc-700 transition peer-checked:bg-amber-400" />
<span class="absolute left-0.5 top-0.5 h-5 w-5 rounded-full bg-white transition peer-checked:translate-x-5" />
</span>
</label>
<div v-if="useIdentityRefs" class="grid grid-cols-2 gap-2 sm:grid-cols-4">
<div
v-for="slot in 4"
:key="slot"
class="relative min-h-24 overflow-hidden rounded-xl border border-dashed border-white/15 bg-zinc-950/60"
>
<button
type="button"
class="block w-full text-left"
@click="openPicker(slot - 1)"
>
<img
v-if="identityPreviews[slot - 1]"
:src="identityPreviews[slot - 1]"
alt=""
class="h-24 w-full object-cover"
>
<span v-else class="flex h-24 items-center justify-center px-2 text-center text-[11px] text-zinc-500">
Ref {{ slot }}
</span>
</button>
<button
v-if="identityRefs[slot - 1]"
type="button"
class="absolute right-1 top-1 rounded-full bg-zinc-950/80 px-2 py-0.5 text-[10px] text-zinc-300"
@click="clearIdentityRef(slot - 1)"
>
Remove
</button>
</div>
</div>
<p v-if="useIdentityRefs" class="text-xs text-zinc-500">Extra stills lock people and objects. The first-frame still above stays the opening frame. Identity refs are not reused on queued extensions.</p>
</div>
</div>
<details v-if="studioMode === 'video'" class="rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3">
<summary class="cursor-pointer text-sm font-medium">Generation settings</summary> <summary class="cursor-pointer text-sm font-medium">Generation settings</summary>
<div class="mt-4 space-y-4"> <div class="mt-4 space-y-4">
<div> <div>
@@ -296,14 +398,47 @@
</div> </div>
</details> </details>
<details v-else class="rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3">
<summary class="cursor-pointer text-sm font-medium">Image edit settings</summary>
<div class="mt-4 space-y-4">
<p class="text-xs text-zinc-500">
Sidecar: <span class="text-zinc-200">{{ imageComfyHost || 'not configured' }}</span>
· Qwen Image Edit Lightning (4-step LoRA)
</p>
<label class="block text-sm">
<span class="mb-1 block text-zinc-400">Negative prompt</span>
<textarea v-model="editNegative" rows="2" class="w-full rounded-xl border border-white/10 bg-zinc-950 px-3 py-2 text-sm" />
</label>
<div class="grid gap-4 sm:grid-cols-2">
<label class="block text-sm">
<span class="mb-1 block text-zinc-400">Steps · {{ editSteps }}</span>
<input v-model.number="editSteps" type="range" min="1" max="8" step="1" class="w-full">
</label>
<label class="block text-sm">
<span class="mb-1 block text-zinc-400">CFG · {{ editCfg }}</span>
<input v-model.number="editCfg" type="range" min="0.5" max="4" step="0.1" class="w-full">
</label>
<label class="block text-sm">
<span class="mb-1 block text-zinc-400">Denoise · {{ editDenoise }}</span>
<input v-model.number="editDenoise" type="range" min="0.05" max="1" step="0.05" class="w-full">
</label>
<label class="block text-sm">
<span class="mb-1 block text-zinc-400">Seed</span>
<input v-model="seedInput" class="w-full rounded-xl border border-white/10 bg-zinc-950 px-3 py-2 text-sm" placeholder="random">
</label>
</div>
<p class="text-xs text-zinc-500">Lower denoise keeps more of the original still. Lightning is tuned for 4 steps, CFG 1, denoise around 0.75.</p>
</div>
</details>
<div class="flex flex-wrap gap-3"> <div class="flex flex-wrap gap-3">
<button <button
type="button" type="button"
class="flex-1 rounded-2xl bg-amber-400 px-5 py-3 font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50" class="flex-1 rounded-2xl bg-amber-400 px-5 py-3 font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50"
:disabled="busy || !file || !prompt.trim() || !folderId || !queueReady" :disabled="busy || !file || !prompt.trim() || !folderId || (studioMode !== 'edit' && !queueReady)"
@click="generate" @click="studioMode === 'edit' ? editImage() : generate()"
> >
{{ generateLabel }} {{ studioMode === 'edit' ? (busy ? 'Editing…' : 'Edit image') : generateLabel }}
</button> </button>
<button <button
v-if="busy" v-if="busy"
@@ -375,8 +510,15 @@
:class="{ 'sensitive-glow': awaitingReveal || lockedSave }" :class="{ 'sensitive-glow': awaitingReveal || lockedSave }"
:style="{ aspectRatio: `${playerSize.width} / ${playerSize.height}` }" :style="{ aspectRatio: `${playerSize.width} / ${playerSize.height}` }"
> >
<img
v-if="editResultUrl && !lockedSave"
:src="editResultUrl"
alt="Edited still"
class="h-full w-full bg-black object-contain"
:class="{ 'opacity-0': awaitingReveal }"
>
<video <video
v-if="videoUrl && !lockedSave" v-else-if="videoUrl && !lockedSave"
ref="player" ref="player"
:src="videoUrl" :src="videoUrl"
class="h-full w-full bg-black" class="h-full w-full bg-black"
@@ -404,12 +546,28 @@
<span class="text-sm text-amber-100/80">Unlock that folder in the library to view this clip</span> <span class="text-sm text-amber-100/80">Unlock that folder in the library to view this clip</span>
</div> </div>
<div <div
v-else-if="!videoUrl" v-else-if="!videoUrl && !editResultUrl"
class="flex h-full min-h-[12rem] items-center justify-center text-sm text-zinc-500" class="flex h-full min-h-[12rem] items-center justify-center text-sm text-zinc-500"
> >
Generated MP4 appears here {{ studioMode === 'edit' ? 'Edited still appears here' : 'Generated MP4 appears here' }}
</div> </div>
</div> </div>
<div v-if="editResultUrl && !lockedSave && !awaitingReveal" class="mt-4 flex flex-wrap gap-3">
<a
:href="editResultUrl"
:download="`${outputDownloadName || 'aigen-edit'}.png`"
class="inline-flex rounded-2xl border border-white/15 px-4 py-2 text-sm hover:border-amber-300/50"
>
Download still
</a>
<button
type="button"
class="inline-flex rounded-2xl border border-amber-300/40 px-4 py-2 text-sm text-amber-100 hover:border-amber-300/70"
@click="useEditAsInput"
>
Use as input still
</button>
</div>
<div v-if="videoUrl && !lockedSave" class="mt-4 flex flex-wrap gap-3"> <div v-if="videoUrl && !lockedSave" class="mt-4 flex flex-wrap gap-3">
<a <a
v-if="!awaitingReveal" v-if="!awaitingReveal"
@@ -646,6 +804,7 @@
:stills="stills" :stills="stills"
:folder-id="folderId" :folder-id="folderId"
:hide-input-preview="hideInputPreview" :hide-input-preview="hideInputPreview"
:heading="pickerHeading"
@close="pickerOpen = false" @close="pickerOpen = false"
@updated="applyLibrary" @updated="applyLibrary"
@error="toast" @error="toast"
@@ -712,6 +871,7 @@ interface LibraryClip {
familyId?: string familyId?: string
parentClipId?: string parentClipId?: string
chainIndex?: number chainIndex?: number
workflow?: 'v1' | 'v2'
} }
interface RetryDraft { interface RetryDraft {
@@ -737,6 +897,7 @@ interface RetryDraft {
samplerName?: string samplerName?: string
scheduler?: string scheduler?: string
extensions?: { prompt: string; duration: number }[] extensions?: { prompt: string; duration: number }[]
workflow?: 'v1' | 'v2'
} }
interface QueuedExtension { interface QueuedExtension {
@@ -790,6 +951,11 @@ const hideInputPreview = ref(false)
const autoplayEnabled = ref(false) const autoplayEnabled = ref(false)
const JOB_STORE = 'aigen-active-job' const JOB_STORE = 'aigen-active-job'
const busy = ref(false) const busy = ref(false)
const studioMode = ref('video')
const imageComfyOk = ref(false)
const imageComfyBusy = ref(false)
const imageComfyConfigured = ref(false)
const imageComfyHost = ref('')
const progress = ref(0) const progress = ref(0)
const statusMessage = ref('Waiting for a still.') const statusMessage = ref('Waiting for a still.')
const statusBusy = ref(false) const statusBusy = ref(false)
@@ -804,6 +970,11 @@ const activeChainPlan = ref<{ label: string; prompt: string; duration: number }[
const expandedFamilies = ref<Record<string, boolean>>({}) const expandedFamilies = ref<Record<string, boolean>>({})
const elapsedMs = ref(0) const elapsedMs = ref(0)
const videoUrl = ref('') const videoUrl = ref('')
const editResultUrl = ref('')
const editSteps = ref(4)
const editCfg = ref(1)
const editDenoise = ref(0.75)
const editNegative = ref('blurry, low quality, deformed, bad anatomy, watermark, text, extra fingers')
const jobId = ref('') const jobId = ref('')
const currentClipId = ref('') const currentClipId = ref('')
const extendOpen = ref(false) const extendOpen = ref(false)
@@ -838,6 +1009,15 @@ const renameInput = ref<HTMLInputElement | null>(null)
const unlockPassword = ref('') const unlockPassword = ref('')
const settingsOpen = ref(false) const settingsOpen = ref(false)
const pickerOpen = ref(false) const pickerOpen = ref(false)
const pickerSlot = ref<'main' | number>('main')
const videoWorkflow = ref('v1')
const useIdentityRefs = ref(false)
const identityRefs = ref<(File | null)[]>([null, null, null, null])
const identityPreviews = ref(['', '', '', ''])
const pickerHeading = computed(() => {
if (pickerSlot.value === 'main') return 'Choose an image'
return `Choose identity ref ${Number(pickerSlot.value) + 1}`
})
let timer: ReturnType<typeof setInterval> | null = null let timer: ReturnType<typeof setInterval> | null = null
let source: EventSource | null = null let source: EventSource | null = null
let pollTimer: ReturnType<typeof setInterval> | null = null let pollTimer: ReturnType<typeof setInterval> | null = null
@@ -875,6 +1055,35 @@ const comfyBadge = computed(() => {
} }
}) })
const imageComfyBadge = computed(() => {
if (!imageComfyConfigured.value) {
return {
label: 'Edit sidecar unset',
className: 'border-white/10 text-zinc-500',
dot: 'bg-zinc-500'
}
}
if (!imageComfyOk.value) {
return {
label: 'Edit Comfy offline',
className: 'border-red-400/30 text-red-300',
dot: 'bg-red-400'
}
}
if (imageComfyBusy.value) {
return {
label: 'Edit Comfy busy',
className: 'border-amber-400/40 text-amber-200',
dot: 'bg-amber-400'
}
}
return {
label: 'Edit Comfy online',
className: 'border-emerald-400/30 text-emerald-300',
dot: 'bg-emerald-400'
}
})
function snap32(value: number) { function snap32(value: number) {
return Math.max(32, Math.round(value / 32) * 32) return Math.max(32, Math.round(value / 32) * 32)
} }
@@ -1060,9 +1269,16 @@ async function loadLibrary() {
applyLibrary(data) applyLibrary(data)
} }
function applyComfyHealth(health: { comfy?: { ok?: boolean; busy?: boolean } } | null | undefined) { function applyComfyHealth(health: {
comfy?: { ok?: boolean; busy?: boolean }
imageComfy?: { ok?: boolean; busy?: boolean; configured?: boolean; host?: string }
} | null | undefined) {
comfyOk.value = Boolean(health?.comfy?.ok) comfyOk.value = Boolean(health?.comfy?.ok)
comfyBusy.value = Boolean(health?.comfy?.ok && health?.comfy?.busy) comfyBusy.value = Boolean(health?.comfy?.ok && health?.comfy?.busy)
imageComfyConfigured.value = Boolean(health?.imageComfy?.configured)
imageComfyOk.value = Boolean(health?.imageComfy?.ok)
imageComfyBusy.value = Boolean(health?.imageComfy?.ok && health?.imageComfy?.busy)
imageComfyHost.value = String(health?.imageComfy?.host || '')
} }
async function pollComfyHealth() { async function pollComfyHealth() {
@@ -1075,6 +1291,8 @@ onMounted(async () => {
hideInputPreview.value = localStorage.getItem('aigen-hide-input-preview') === 'true' hideInputPreview.value = localStorage.getItem('aigen-hide-input-preview') === 'true'
withSound.value = localStorage.getItem('aigen-generate-sound') !== 'false' withSound.value = localStorage.getItem('aigen-generate-sound') !== 'false'
autoplayEnabled.value = localStorage.getItem('aigen-autoplay') === 'true' autoplayEnabled.value = localStorage.getItem('aigen-autoplay') === 'true'
videoWorkflow.value = localStorage.getItem('aigen-video-workflow') === 'v2' ? 'v2' : 'v1'
useIdentityRefs.value = localStorage.getItem('aigen-use-identity-refs') === 'true'
} catch { /* ignore */ } } catch { /* ignore */ }
applyInstancePrivacy() applyInstancePrivacy()
window.addEventListener('pointerdown', onSitePointerDown, true) window.addEventListener('pointerdown', onSitePointerDown, true)
@@ -1129,6 +1347,18 @@ watch(autoplayEnabled, (value) => {
} }
}) })
watch(videoWorkflow, (value) => {
try {
localStorage.setItem('aigen-video-workflow', value)
} catch { /* ignore */ }
})
watch(useIdentityRefs, (value) => {
try {
localStorage.setItem('aigen-use-identity-refs', String(value))
} catch { /* ignore */ }
})
watch(turbo, (on) => { watch(turbo, (on) => {
if (!cfgTouched.value) cfg.value = on ? CFG_TURBO : CFG_QUALITY if (!cfgTouched.value) cfg.value = on ? CFG_TURBO : CFG_QUALITY
}) })
@@ -1221,10 +1451,42 @@ function readFile(next: File) {
img.src = preview.value img.src = preview.value
} }
function openPicker(slot: 'main' | number = 'main') {
pickerSlot.value = slot
pickerOpen.value = true
}
function setIdentityRef(index: number, next: File) {
const prev = identityPreviews.value[index]
if (prev) URL.revokeObjectURL(prev)
const copies = identityRefs.value.slice()
copies[index] = next
identityRefs.value = copies
const previews = identityPreviews.value.slice()
previews[index] = URL.createObjectURL(next)
identityPreviews.value = previews
}
function clearIdentityRef(index: number) {
const prev = identityPreviews.value[index]
if (prev) URL.revokeObjectURL(prev)
const copies = identityRefs.value.slice()
copies[index] = null
identityRefs.value = copies
const previews = identityPreviews.value.slice()
previews[index] = ''
identityPreviews.value = previews
}
function onPickedImage(next: File, hidden = false) { function onPickedImage(next: File, hidden = false) {
const slot = pickerSlot.value
pickerOpen.value = false
if (slot !== 'main') {
setIdentityRef(Number(slot), next)
return
}
if (hidden && showPrivacyToggles.value) hideInputPreview.value = true if (hidden && showPrivacyToggles.value) hideInputPreview.value = true
readFile(next) readFile(next)
pickerOpen.value = false
} }
async function persistDroppedStill(next: File) { async function persistDroppedStill(next: File) {
@@ -1364,6 +1626,7 @@ async function loadDraft(draft: RetryDraft) {
withSound.value = draft.sound !== false withSound.value = draft.sound !== false
if (draft.hideInput && showPrivacyToggles.value) hideInputPreview.value = true if (draft.hideInput && showPrivacyToggles.value) hideInputPreview.value = true
duration.value = clampDuration(Number(draft.duration)) duration.value = clampDuration(Number(draft.duration))
if (draft.workflow === 'v2') videoWorkflow.value = 'v2'
extensionQueue.value = (draft.extensions || []).map(item => ({ extensionQueue.value = (draft.extensions || []).map(item => ({
id: crypto.randomUUID(), id: crypto.randomUUID(),
prompt: String(item.prompt || ''), prompt: String(item.prompt || ''),
@@ -1643,6 +1906,73 @@ function removeQueuedExtension(id: string) {
extensionQueue.value = extensionQueue.value.filter(item => item.id !== id) extensionQueue.value = extensionQueue.value.filter(item => item.id !== id)
} }
async function editImage() {
if (!file.value || !prompt.value.trim()) return
if (!folderId.value) {
toast('Choose a library folder before editing.')
return
}
statusBusy.value = false
busy.value = true
listenGen += 1
stopListen()
settledUi = false
progress.value = 2
videoUrl.value = ''
editResultUrl.value = ''
currentClipId.value = ''
closeExtend()
awaitingReveal.value = false
concealOutput.value = hideInputPreview.value
lockedSave.value = false
watchingLibrary.value = false
statusMessage.value = 'Checking sidecar ComfyUI...'
maxStep.value = editSteps.value
startTimer()
try {
const body = new FormData()
body.append('image', file.value)
body.append('prompt', prompt.value.trim())
body.append('negative', editNegative.value)
body.append('steps', String(editSteps.value))
body.append('cfg', String(editCfg.value))
body.append('denoise', String(editDenoise.value))
body.append('seed', seedInput.value || 'random')
body.append('folderId', folderId.value)
body.append('hideInput', String(showPrivacyToggles.value && hideInputPreview.value))
body.append('name', clipName.value.trim())
outputDownloadName.value = clipFileName(clipName.value.trim() || prompt.value.split(/[.!?\n]/)[0] || 'aigen-edit')
const started = await $fetch<{ jobId: string; steps: number; hideThumbnail?: boolean; folderLocked?: boolean }>('/api/edit', {
method: 'POST',
body
})
jobId.value = started.jobId
maxStep.value = started.steps
persistActiveJob(started.jobId, started.hideThumbnail === true || hideInputPreview.value, started.folderLocked === true)
listen(started.jobId, started.hideThumbnail === true || hideInputPreview.value, started.folderLocked === true)
} catch (error: any) {
settledUi = true
const message = error?.data?.statusMessage || error?.statusMessage || error?.message || 'Image edit failed'
statusMessage.value = message
toast(message)
busy.value = false
statusBusy.value = false
stopTimer()
void pollComfyHealth()
}
}
async function useEditAsInput() {
if (!editResultUrl.value) return
try {
const blob = await $fetch<Blob>(editResultUrl.value, { responseType: 'blob' })
readFile(new File([blob], `${outputDownloadName.value || 'edited'}.png`, { type: blob.type || 'image/png' }))
toast('Edited still is now the input')
} catch {
toast('Could not load that still as input')
}
}
async function generate() { async function generate() {
if (!file.value || !prompt.value.trim()) return if (!file.value || !prompt.value.trim()) return
if (!queueReady.value) { if (!queueReady.value) {
@@ -1653,6 +1983,11 @@ async function generate() {
toast('Choose a library folder before generating.') toast('Choose a library folder before generating.')
return return
} }
const identityFiles = identityRefs.value.filter((item): item is File => Boolean(item))
if (videoWorkflow.value === 'v2' && useIdentityRefs.value && !identityFiles.length) {
toast('Add at least one identity reference still, or turn references off.')
return
}
const shots = shotScriptMode.value ? parsedShots.value : null const shots = shotScriptMode.value ? parsedShots.value : null
const initialPrompt = (shots?.[0]?.prompt || prompt.value).trim() const initialPrompt = (shots?.[0]?.prompt || prompt.value).trim()
const queued = shots const queued = shots
@@ -1665,6 +2000,7 @@ async function generate() {
settledUi = false settledUi = false
progress.value = 1 progress.value = 1
videoUrl.value = '' videoUrl.value = ''
editResultUrl.value = ''
currentClipId.value = '' currentClipId.value = ''
closeExtend() closeExtend()
awaitingReveal.value = false awaitingReveal.value = false
@@ -1700,6 +2036,13 @@ async function generate() {
body.append('hideInput', String(showPrivacyToggles.value && hideInputPreview.value)) body.append('hideInput', String(showPrivacyToggles.value && hideInputPreview.value))
body.append('sound', String(withSound.value)) body.append('sound', String(withSound.value))
body.append('name', clipName.value.trim()) body.append('name', clipName.value.trim())
body.append('workflow', videoWorkflow.value)
body.append('useIdentityRefs', String(videoWorkflow.value === 'v2' && useIdentityRefs.value))
if (videoWorkflow.value === 'v2' && useIdentityRefs.value) {
for (const ref of identityFiles) {
body.append('refImage', ref)
}
}
if (activeDraftId) body.append('draftId', activeDraftId) if (activeDraftId) body.append('draftId', activeDraftId)
if (queued.length) { if (queued.length) {
body.append('extensions', JSON.stringify(queued)) body.append('extensions', JSON.stringify(queued))
@@ -1780,12 +2123,34 @@ function applyJobEvent(payload: Record<string, any>, hidden: boolean, folderLock
return return
} }
if (complete && (payload.clipId || payload.filename)) { if (complete && (payload.clipId || payload.filename || payload.stillId)) {
settledUi = true settledUi = true
const dest = folders.value.find(folder => folder.id === folderId.value) const dest = folders.value.find(folder => folder.id === folderId.value)
const locked = dest const locked = dest
? Boolean(dest.protected && !dest.unlocked) ? Boolean(dest.protected && !dest.unlocked)
: (folderLocked || payload.folderLocked === true) : (folderLocked || payload.folderLocked === true)
if (payload.stillId) {
if (locked) {
editResultUrl.value = ''
lockedSave.value = true
browseFolderId.value = folderId.value
statusMessage.value = 'Saved to the locked folder. Unlock it to view.'
} else {
editResultUrl.value = `/api/library/stills/${payload.stillId}`
awaitingReveal.value = Boolean(payload.hideThumbnail || hidden)
concealOutput.value = awaitingReveal.value
lockedSave.value = false
statusMessage.value = 'Edit finished'
}
busy.value = false
statusBusy.value = false
stopTimer()
stopListen()
clearActiveJob()
loadLibrary().catch(() => null)
void pollComfyHealth()
return
}
const keepLibraryView = watchingLibrary.value && Boolean(currentClipId.value) && payload.clipId !== currentClipId.value const keepLibraryView = watchingLibrary.value && Boolean(currentClipId.value) && payload.clipId !== currentClipId.value
if (keepLibraryView) { if (keepLibraryView) {
busy.value = false busy.value = false
+3 -2
View File
@@ -110,9 +110,10 @@ function launchCommand() {
function startComfy() { function startComfy() {
const command = launchCommand() const command = launchCommand()
const child = spawn(command, { shell: true, detached: true, stdio: 'ignore', windowsHide: true }) const cwd = process.env.COMFY_LAUNCH_CWD || undefined
const child = spawn(command, { shell: true, detached: true, stdio: 'ignore', windowsHide: true, cwd })
child.unref() child.unref()
return { started: true, command } return { started: true, command, cwd: cwd || null }
} }
function sharedRoots() { function sharedRoots() {
+138
View File
@@ -0,0 +1,138 @@
"""Install scripts/comfy-host-agent.mjs on the sidecar image-Comfy PC over SSH.
Reads AIGEN_IMAGE_SSH_HOST, AIGEN_IMAGE_SSH_USER, and AIGEN_IMAGE_SSH_KEY
(default: %USERPROFILE%\\.ssh\\aigen_image_comfy) or AIGEN_IMAGE_SSH_PASS.
Installs the host agent only. Does not start Comfy or load models.
"""
from __future__ import annotations
import os
import sys
import time
from pathlib import Path
import paramiko
ROOT = Path(__file__).resolve().parents[1]
AGENT = ROOT / "scripts" / "comfy-host-agent.mjs"
REMOTE_DIR = r"C:\Users\comfyapi\aigen-agent"
REMOTE_AGENT = REMOTE_DIR + r"\comfy-host-agent.mjs"
REMOTE_START = REMOTE_DIR + r"\start.cmd"
NODE_ZIP = "https://nodejs.org/dist/v22.18.0/node-v22.18.0-win-x64.zip"
def connect():
host = os.environ.get("AIGEN_IMAGE_SSH_HOST", "192.168.77.101")
user = os.environ.get("AIGEN_IMAGE_SSH_USER", "comfyapi")
password = os.environ.get("AIGEN_IMAGE_SSH_PASS")
key_path = os.environ.get("AIGEN_IMAGE_SSH_KEY") or str(Path.home() / ".ssh" / "aigen_image_comfy")
if not Path(key_path).exists():
key_path = os.environ.get("AIGEN_IMAGE_SSH_KEY")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
kwargs = {
"hostname": host,
"username": user,
"timeout": 15,
"allow_agent": False,
"look_for_keys": False,
}
if key_path:
kwargs["key_filename"] = key_path
elif password:
kwargs["password"] = password
else:
raise SystemExit("Set AIGEN_IMAGE_SSH_PASS or AIGEN_IMAGE_SSH_KEY")
client.connect(**kwargs)
return client, host, user
def run(client, cmd, timeout=120):
stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
out = stdout.read().decode("utf-8", "replace")
err = stderr.read().decode("utf-8", "replace")
code = stdout.channel.recv_exit_status()
return code, out, err
def main():
if not AGENT.exists():
raise SystemExit(f"missing {AGENT}")
token = os.environ.get("IMAGE_COMFY_CONTROL_TOKEN") or os.urandom(24).hex()
token_file = Path(os.environ.get("TEMP", ".")) / "image_comfy_control_token"
token_file.write_text(token, encoding="ascii")
pubkey = Path.home() / ".ssh" / "aigen_image_comfy.pub"
start_cmd = "\r\n".join([
"@echo off",
f"cd /d {REMOTE_DIR}",
"set COMFY_CONTROL_PORT=8199",
"set COMFY_PROXY_PORT=8198",
"set COMFY_HOST=http://127.0.0.1:8188",
f"set COMFY_CONTROL_TOKEN={token}",
r"set COMFY_LAUNCH_CWD=C:\ComfyUI-QwenEdit\ComfyUI",
r"set COMFY_LAUNCH_CMD=C:\ComfyUI-QwenEdit\python\python.exe C:\ComfyUI-QwenEdit\ComfyUI\main.py --listen 0.0.0.0 --port 8188",
r'"C:\Users\comfyapi\aigen-agent\node.exe" comfy-host-agent.mjs',
"",
])
client, host, user = connect()
print("connected", user, host)
code, out, err = run(client, f'mkdir "{REMOTE_DIR}" 2>nul & echo dir-ok')
print("mkdir", code, out.strip() or err.strip())
sftp = client.open_sftp()
with sftp.file(REMOTE_AGENT, "wb") as handle:
handle.write(AGENT.read_bytes())
with sftp.file(REMOTE_START, "w") as handle:
handle.write(start_cmd)
if pubkey.exists():
run(client, r'if not exist "%USERPROFILE%\.ssh" mkdir "%USERPROFILE%\.ssh"')
existing = ""
try:
with sftp.file(r"C:\Users\comfyapi\.ssh\authorized_keys", "r") as handle:
existing = handle.read().decode("utf-8", "replace")
except FileNotFoundError:
existing = ""
key_text = pubkey.read_text(encoding="ascii").strip()
if key_text not in existing:
with sftp.file(r"C:\Users\comfyapi\.ssh\authorized_keys", "a") as handle:
handle.write(("\n" if existing and not existing.endswith("\n") else "") + key_text + "\n")
print("authorized_keys updated")
sftp.close()
code, out, err = run(client, f'if exist "{REMOTE_DIR}\\node.exe" echo node-present')
if "node-present" not in out:
print("downloading node")
code, out, err = run(
client,
f'curl.exe -L --retry 3 -o "{REMOTE_DIR}\\node.zip" {NODE_ZIP}',
timeout=180,
)
print("curl", code, (err or out)[-400:])
code, out, err = run(
client,
f'"C:\\Program Files\\7-Zip\\7z.exe" x -y -o"{REMOTE_DIR}\\node-dist" "{REMOTE_DIR}\\node.zip"',
timeout=120,
)
print("unzip", code, out[-400:] or err[-400:])
run(client, f'copy /Y "{REMOTE_DIR}\\node-dist\\node-v22.18.0-win-x64\\node.exe" "{REMOTE_DIR}\\node.exe"')
run(client, r'powershell -NoProfile -Command "Get-Process node -ErrorAction SilentlyContinue | Where-Object { $_.Path -like \'*aigen-agent*\' } | Stop-Process -Force"')
code, out, err = run(
client,
r'cmd /c start "" /MIN C:\Users\comfyapi\aigen-agent\start.cmd',
)
print("start", code, out.strip(), err.strip())
run(
client,
r'schtasks /Create /TN AigenImageComfyAgent /TR "C:\Users\comfyapi\aigen-agent\start.cmd" /SC ONLOGON /F',
)
client.close()
print("token-file", str(token_file))
print("wait for agent")
time.sleep(2)
if __name__ == "__main__":
main()
+201
View File
@@ -0,0 +1,201 @@
import { imageComfyConfigured, uploadImageEdit, queueImagePrompt, waitForImageEdit, downloadEditedImage } from '~/server/utils/imageComfy'
import { ensureImageComfyReady } from '~/server/utils/imageComfyLifecycle'
import { buildEditWorkflow } from '~/server/utils/imageWorkflow'
import { imageDimensions } from '~/server/utils/resolution'
function parseSteps(raw: unknown) {
const value = Number(raw)
if (!Number.isFinite(value)) return 4
return Math.min(20, Math.max(1, Math.round(value)))
}
function parseCfg(raw: unknown) {
const value = Number(raw)
if (!Number.isFinite(value)) return 1
return Math.min(8, Math.max(0.5, Math.round(value * 10) / 10))
}
function parseDenoise(raw: unknown) {
const value = Number(raw)
if (!Number.isFinite(value)) return 0.75
return Math.min(1, Math.max(0.05, Math.round(value * 100) / 100))
}
export default defineEventHandler(async (event) => {
const form = await readMultipartFormData(event).catch(() => null)
const fields: Record<string, string> = {}
let image: { filename: string; data: Buffer; type?: string } | null = null
for (const part of form || []) {
if (part.name === 'image' && part.filename && part.data?.length) {
image = { filename: part.filename, data: part.data, type: part.type }
} else if (part.name && part.data) {
fields[part.name] = part.data.toString('utf8')
}
}
const prompt = (fields.prompt || '').trim()
if (!image) {
throw createError({ statusCode: 400, statusMessage: 'An input image is required' })
}
if (!prompt) {
throw createError({ statusCode: 400, statusMessage: 'An edit prompt is required' })
}
if (!imageComfyConfigured()) {
throw createError({
statusCode: 503,
statusMessage: 'Image edit ComfyUI is not configured yet. Set IMAGE_COMFY_HOST on this instance.'
})
}
const ownerKey = libraryOwnerKey(event)
const library = publicLibrary(event)
const folderId = library.folders.some(folder => folder.id === fields.folderId)
? fields.folderId
: library.folders[0]?.id
if (!folderId) {
throw createError({ statusCode: 400, statusMessage: 'Create a library folder before editing' })
}
assertFolderExists(event, folderId)
const destFolder = library.folders.find(folder => folder.id === folderId)
const folderLocked = Boolean(destFolder?.protected && !destFolder.unlocked)
const hideInput = fields.hideInput === 'true'
const steps = parseSteps(fields.steps)
const cfg = parseCfg(fields.cfg)
const denoise = parseDenoise(fields.denoise)
const seed = fields.seed && fields.seed !== 'random'
? Number(fields.seed)
: Math.floor(Math.random() * 2_147_483_647)
const size = imageDimensions(image.data)
const job = createJob()
job.kind = 'edit'
job.maxStep = steps
job.hideThumbnail = hideInput
job.library = {
ownerKey,
folderId,
hideThumbnail: hideInput,
hideInput,
folderLocked,
name: (fields.name || '').trim().slice(0, 80),
prompt,
aspect: 'auto',
width: size?.width || 0,
height: size?.height || 0,
steps,
turbo: true,
seed,
cfg
}
emitJob(job, { type: 'status', message: 'Checking sidecar ComfyUI...', progress: 2 })
void runEdit(job, {
image,
prompt,
negative: (fields.negative || '').trim(),
steps,
seed,
cfg,
denoise
}).catch(async (error) => {
if (job.status === 'error' || job.status === 'cancelled') return
const message = error instanceof Error ? error.message : String(error)
job.status = 'error'
job.error = message
emitJob(job, { type: 'error', error: message, message })
})
return {
jobId: job.id,
seed,
steps,
hideThumbnail: hideInput,
folderLocked
}
})
async function runEdit(
job: ReturnType<typeof createJob>,
params: {
image: { filename: string; data: Buffer; type?: string }
prompt: string
negative: string
steps: number
seed: number
cfg: number
denoise: number
}
) {
await ensureImageComfyReady((status) => {
emitJob(job, {
type: status.state === 'busy' ? 'busy' : 'status',
message: status.message,
progress: status.state === 'online' ? Math.max(job.progress, 6) : Math.max(job.progress, 3),
busy: status.state === 'busy'
})
})
if (job.status === 'cancelled') throw new Error('Job interrupted.')
emitJob(job, { type: 'status', message: 'Uploading still to Qwen Edit...', progress: 8 })
const uploaded = await uploadImageEdit(params.image, job.id)
if (job.status === 'cancelled') throw new Error('Job interrupted.')
emitJob(job, { type: 'status', message: 'Queueing Qwen Image Edit Lightning...', progress: 12 })
const graph = buildEditWorkflow({
prompt: params.prompt,
negative: params.negative,
imageName: uploaded.name,
steps: params.steps,
seed: params.seed,
cfg: params.cfg,
denoise: params.denoise,
filenamePrefix: `aigen_edit_${job.id.slice(0, 8)}`
})
const queued = await queueImagePrompt(graph, job.clientId)
job.promptId = queued.prompt_id
job.status = 'running'
emitJob(job, { type: 'status', message: 'Editing still...', progress: 18, maxStep: params.steps })
const output = await waitForImageEdit({
promptId: queued.prompt_id,
clientId: job.clientId,
onProgress: (event) => {
emitJob(job, {
type: 'status',
message: event.message,
progress: event.progress,
step: event.step,
maxStep: event.maxStep || params.steps,
node: event.node
})
},
isCancelled: () => job.status === 'cancelled'
})
emitJob(job, { type: 'status', message: 'Saving edited still...', progress: 94 })
const buffer = await downloadEditedImage(output)
const size = imageDimensions(buffer)
const still = await saveStill({
ownerKey: job.library!.ownerKey,
folderId: job.library!.folderId,
filename: job.library?.name ? `${job.library.name}.png` : output.filename,
data: buffer,
width: size?.width || 0,
height: size?.height || 0,
hideInput: job.library?.hideInput
})
job.stillId = still.id
job.status = 'complete'
emitJob(job, {
type: 'complete',
message: 'Edit finished',
progress: 100,
stillId: still.id,
filename: output.filename,
subfolder: output.subfolder,
mediaType: 'image',
hideThumbnail: job.hideThumbnail,
folderLocked: job.library?.folderLocked
})
}
+5 -3
View File
@@ -100,7 +100,8 @@ export default defineEventHandler(async (event) => {
extendSourceClipId: source.id, extendSourceClipId: source.id,
familyId: source.familyId || crypto.randomUUID(), familyId: source.familyId || crypto.randomUUID(),
parentClipId: source.id, parentClipId: source.id,
chainIndex: (source.chainIndex || 0) + 1 chainIndex: (source.chainIndex || 0) + 1,
workflow: parseVideoWorkflow(source.workflow)
} }
emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 }) emitJob(job, { type: 'status', message: 'Extracting last frame & stitching extension...', progress: 2 })
@@ -163,7 +164,6 @@ async function runExtend(
copyFileSync(params.sourcePath, params.part1Path) copyFileSync(params.sourcePath, params.part1Path)
await extractLastFrame(params.part1Path, params.framePath) await extractLastFrame(params.part1Path, params.framePath)
const frame = readFileSync(params.framePath) const frame = readFileSync(params.framePath)
job.library!.thumb = frame
const sound = await probeHasAudio(params.part1Path) const sound = await probeHasAudio(params.part1Path)
await ensureComfyReady((status) => { await ensureComfyReady((status) => {
@@ -202,7 +202,9 @@ async function runExtend(
samplerName: params.samplerName, samplerName: params.samplerName,
scheduler: params.scheduler, scheduler: params.scheduler,
filenamePrefix: comfyFilenamePrefix(), filenamePrefix: comfyFilenamePrefix(),
sound sound,
workflow: job.library?.workflow,
duration: job.library?.duration
}) })
const queued = await queuePrompt(graph, job.clientId) const queued = await queuePrompt(graph, job.clientId)
+41 -5
View File
@@ -83,9 +83,14 @@ export default defineEventHandler(async (event) => {
const fields: Record<string, string> = {} const fields: Record<string, string> = {}
let image: { filename: string; data: Buffer; type?: string } | null = null let image: { filename: string; data: Buffer; type?: string } | null = null
const referenceImages: { filename: string; data: Buffer; type?: string }[] = []
for (const part of form) { for (const part of form) {
if (part.name === 'image' && part.filename && part.data?.length) { if (part.name === 'image' && part.filename && part.data?.length) {
image = { filename: part.filename, data: part.data, type: part.type } image = { filename: part.filename, data: part.data, type: part.type }
} else if (part.name === 'refImage' && part.filename && part.data?.length) {
if (referenceImages.length < 4) {
referenceImages.push({ filename: part.filename, data: part.data, type: part.type })
}
} else if (part.name && part.data) { } else if (part.name && part.data) {
fields[part.name] = part.data.toString('utf8') fields[part.name] = part.data.toString('utf8')
} }
@@ -99,6 +104,8 @@ export default defineEventHandler(async (event) => {
throw createError({ statusCode: 400, statusMessage: 'A motion prompt is required' }) throw createError({ statusCode: 400, statusMessage: 'A motion prompt is required' })
} }
const extensions = parseExtensions(fields.extensions) const extensions = parseExtensions(fields.extensions)
const workflow = parseVideoWorkflow(fields.workflow)
const useIdentityRefs = workflow === 'v2' && fields.useIdentityRefs === 'true' && referenceImages.length > 0
const { width, height } = resolveOutputSize(fields.aspect, image.data, Number(fields.width), Number(fields.height)) const { width, height } = resolveOutputSize(fields.aspect, image.data, Number(fields.width), Number(fields.height))
const turbo = fields.turbo !== 'false' const turbo = fields.turbo !== 'false'
@@ -170,7 +177,8 @@ export default defineEventHandler(async (event) => {
chainStep: 1, chainStep: 1,
chainTotal, chainTotal,
chainLabel: extensions.length ? 'Initial' : undefined, chainLabel: extensions.length ? 'Initial' : undefined,
familyId: crypto.randomUUID() familyId: crypto.randomUUID(),
workflow
} }
emitChainJob(job, { type: 'status', message: 'Checking ComfyUI...', progress: 1 }) emitChainJob(job, { type: 'status', message: 'Checking ComfyUI...', progress: 1 })
@@ -188,7 +196,11 @@ export default defineEventHandler(async (event) => {
fps, fps,
samplerName, samplerName,
scheduler, scheduler,
extensions extensions,
workflow,
duration: durationSeconds,
useIdentityRefs,
referenceImages
}).catch(async (error) => { }).catch(async (error) => {
removeExtendTemp(job.library?.extendTmpDir) removeExtendTemp(job.library?.extendTmpDir)
if (isComfyBusyTimeout(error) && job.library) { if (isComfyBusyTimeout(error) && job.library) {
@@ -215,7 +227,8 @@ export default defineEventHandler(async (event) => {
fps: job.library.fps, fps: job.library.fps,
samplerName: job.library.samplerName, samplerName: job.library.samplerName,
scheduler: job.library.scheduler, scheduler: job.library.scheduler,
extensions: job.library.extensions extensions: job.library.extensions,
workflow: job.library.workflow
}) })
job.library.draftId = draft.id job.library.draftId = draft.id
job.status = 'deferred' job.status = 'deferred'
@@ -268,6 +281,10 @@ type GenerateParams = {
samplerName: string samplerName: string
scheduler: string scheduler: string
extensions: { prompt: string; duration: number }[] extensions: { prompt: string; duration: number }[]
workflow: 'v1' | 'v2'
duration: number
useIdentityRefs: boolean
referenceImages: { filename: string; data: Buffer; type?: string }[]
} }
async function queueMiniMax( async function queueMiniMax(
@@ -290,6 +307,17 @@ async function queueMiniMax(
: 'Queueing MiniMax H3 job...' : 'Queueing MiniMax H3 job...'
emitChainJob(job, { type: 'status', message: uploading, progress: 4 }) emitChainJob(job, { type: 'status', message: uploading, progress: 4 })
const uploaded = await uploadImage(params.image, job.id) const uploaded = await uploadImage(params.image, job.id)
const referenceNames: string[] = []
const chainIndex = job.library?.chainIndex || 0
if (chainIndex === 0) {
for (const [index, ref] of (params.referenceImages || []).entries()) {
const next = await uploadImage({
...ref,
filename: `ref${index + 1}_${ref.filename || 'identity.png'}`
}, job.id)
referenceNames.push(next.name)
}
}
if (job.library) { if (job.library) {
job.library.imageName = uploaded.name job.library.imageName = uploaded.name
job.library.imageSubfolder = uploaded.subfolder job.library.imageSubfolder = uploaded.subfolder
@@ -311,7 +339,11 @@ async function queueMiniMax(
samplerName: params.samplerName, samplerName: params.samplerName,
scheduler: params.scheduler, scheduler: params.scheduler,
filenamePrefix: comfyFilenamePrefix(), filenamePrefix: comfyFilenamePrefix(),
sound: params.sound sound: params.sound,
workflow: params.workflow,
duration: params.duration,
useIdentityRefs: chainIndex === 0 && params.useIdentityRefs,
referenceImageNames: chainIndex === 0 ? referenceNames : []
}) })
const queued = await queuePrompt(graph, job.clientId) const queued = await queuePrompt(graph, job.clientId)
@@ -432,7 +464,11 @@ async function runGeneration(
fps: params.fps, fps: params.fps,
samplerName: params.samplerName, samplerName: params.samplerName,
scheduler: params.scheduler, scheduler: params.scheduler,
persist: isLast persist: isLast,
workflow: params.workflow,
duration: ext.duration,
useIdentityRefs: false,
referenceImages: []
}) })
if (!isLast) { if (!isLast) {
+23 -3
View File
@@ -1,11 +1,23 @@
import { checkComfyHttp, fetchQueue } from '~/server/utils/comfyLifecycle' import { checkComfyHttp, fetchQueue } from '~/server/utils/comfyLifecycle'
import { comfyConfigured } from '~/server/utils/comfy'
import { imageComfyConfigured } from '~/server/utils/imageComfy'
import { checkImageComfyHttp, fetchImageQueue } from '~/server/utils/imageComfyLifecycle'
export default defineEventHandler(async () => { export default defineEventHandler(async () => {
const [comfy, queue] = await Promise.all([ const videoConfigured = comfyConfigured()
checkComfyHttp().catch(() => ({ ok: false, host: '', stats: null })), const imageConfigured = imageComfyConfigured()
fetchQueue() const [comfy, queue, imageComfy, imageQueue] = await Promise.all([
videoConfigured
? checkComfyHttp().catch(() => ({ ok: false, host: '', stats: null }))
: Promise.resolve({ ok: false, host: '', stats: null }),
videoConfigured ? fetchQueue() : Promise.resolve({ running: 0, pending: 0 }),
imageConfigured
? checkImageComfyHttp().catch(() => ({ ok: false, host: '', stats: null }))
: Promise.resolve({ ok: false, host: '', stats: null }),
imageConfigured ? fetchImageQueue() : Promise.resolve({ running: 0, pending: 0 })
]) ])
const busy = Boolean(comfy.ok && queue.running > 0) const busy = Boolean(comfy.ok && queue.running > 0)
const imageBusy = Boolean(imageComfy.ok && imageQueue.running > 0)
return { return {
ok: true, ok: true,
service: useRuntimeConfig().public.instanceName || 'aigen', service: useRuntimeConfig().public.instanceName || 'aigen',
@@ -16,6 +28,14 @@ export default defineEventHandler(async () => {
queueRunning: queue.running, queueRunning: queue.running,
queuePending: queue.pending, queuePending: queue.pending,
stats: comfy.ok ? comfy.stats : undefined stats: comfy.ok ? comfy.stats : undefined
},
imageComfy: {
configured: imageConfigured,
ok: imageComfy.ok,
busy: imageBusy,
host: imageComfy.host || '',
queueRunning: imageQueue.running,
queuePending: imageQueue.pending
} }
} }
}) })
+3 -1
View File
@@ -8,7 +8,9 @@ export default defineEventHandler(async (event) => {
removeExtendTemp(job.library?.extendTmpDir) removeExtendTemp(job.library?.extendTmpDir)
} }
if (jobId) deletePendingJob(jobId) if (jobId) deletePendingJob(jobId)
const ok = await interruptComfy() const ok = job?.kind === 'edit'
? await interruptImageComfy()
: await interruptComfy()
if (job) { if (job) {
emitJob(job, { type: 'error', error: 'Job interrupted.', message: 'Job interrupted.' }) emitJob(job, { type: 'error', error: 'Job interrupted.', message: 'Job interrupted.' })
} }
@@ -0,0 +1,456 @@
{
"92": {
"inputs": {
"filename_prefix": "video/MiniMax_H3",
"format": "auto",
"format.codec": "auto",
"codec": "auto",
"video": [
"172",
0
]
},
"class_type": "SaveVideo",
"_meta": {
"title": "Save Video"
}
},
"114": {
"inputs": {
"image": "Mokker (1).jpeg"
},
"class_type": "LoadImage",
"_meta": {
"title": "Main Image (first_frame / ref)"
}
},
"127": {
"inputs": {
"image": [
"128",
0
]
},
"class_type": "GetImageSize",
"_meta": {
"title": "Get Image Size"
}
},
"128": {
"inputs": {
"upscale_method": "lanczos",
"megapixels": 0.98,
"resolution_steps": 32,
"image": [
"114",
0
]
},
"class_type": "ImageScaleToTotalPixels",
"_meta": {
"title": "Scale Image to Total Pixels"
}
},
"140": {
"inputs": {
"image": "crop IMG_8768.jpg"
},
"class_type": "LoadImage",
"_meta": {
"title": "Identity Ref 1 (optional)"
}
},
"141": {
"inputs": {
"image": ""
},
"class_type": "LoadImage",
"_meta": {
"title": "Identity Ref 2 (optional)"
}
},
"142": {
"inputs": {
"image": ""
},
"class_type": "LoadImage",
"_meta": {
"title": "Identity Ref 3 (optional)"
}
},
"143": {
"inputs": {
"image": ""
},
"class_type": "LoadImage",
"_meta": {
"title": "Identity Ref 4 (optional)"
}
},
"150": {
"inputs": {
"value": true
},
"class_type": "PrimitiveBoolean",
"_meta": {
"title": "Use Identity References (Ref2VA)"
}
},
"155": {
"inputs": {
"prompt": "<Picture 1> is the exact facial identity, hairstyle, body proportions and clothing of the main character. Preserve this identity with high fidelity even when the subject temporarily leaves the frame and returns.\n\n[describe the action, camera and audio here]",
"width": [
"127",
0
],
"height": [
"127",
1
],
"length": 241,
"ref_image_size": "match",
"clip": [
"170",
0
],
"vae": [
"161",
0
],
"audio_vae": [
"162",
0
],
"ref_images.ref_image_0": [
"128",
0
],
"ref_images.ref_image_1": [
"140",
0
],
"ref_images.ref_image_2": [
"141",
0
],
"ref_images.ref_image_3": [
"142",
0
],
"ref_images.ref_image_4": [
"143",
0
]
},
"class_type": "MiniMaxH3ReferenceToVideo",
"_meta": {
"title": "Reference to Video (MiniMax H3) – Identity Locked"
}
},
"161": {
"inputs": {
"vae_name": "minimax_h3_video_vae_fp16.safetensors"
},
"class_type": "VAELoader",
"_meta": {
"title": "Load VAE"
}
},
"162": {
"inputs": {
"vae_name": "minimax_h3_audio_vae_fp32.safetensors"
},
"class_type": "VAELoader",
"_meta": {
"title": "Load VAE"
}
},
"163": {
"inputs": {
"samples": [
"167",
0
],
"vae": [
"162",
0
]
},
"class_type": "VAEDecodeAudio",
"_meta": {
"title": "VAE Decode Audio"
}
},
"164": {
"inputs": {
"samples": [
"167",
0
],
"vae": [
"161",
0
]
},
"class_type": "VAEDecode",
"_meta": {
"title": "VAE Decode"
}
},
"165": {
"inputs": {
"sampler_name": "res_multistep"
},
"class_type": "KSamplerSelect",
"_meta": {
"title": "KSamplerSelect"
}
},
"166": {
"inputs": {
"scheduler": "simple",
"steps": [
"178",
0
],
"denoise": 1,
"model": [
"177",
0
]
},
"class_type": "BasicScheduler",
"_meta": {
"title": "BasicScheduler"
}
},
"167": {
"inputs": {
"noise": [
"171",
0
],
"guider": [
"168",
0
],
"sampler": [
"165",
0
],
"sigmas": [
"166",
0
],
"latent_image": [
"173",
1
]
},
"class_type": "SamplerCustomAdvanced",
"_meta": {
"title": "SamplerCustomAdvanced"
}
},
"168": {
"inputs": {
"model": [
"177",
0
],
"conditioning": [
"173",
0
]
},
"class_type": "BasicGuider",
"_meta": {
"title": "Basic Guider"
}
},
"169": {
"inputs": {
"unet_name": "minimax_h3_fl2va_pruned_int8_convrot.safetensors",
"weight_dtype": "default"
},
"class_type": "UNETLoader",
"_meta": {
"title": "Load Diffusion Model"
}
},
"170": {
"inputs": {
"clip_name": "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors",
"type": "minimax",
"device": "default"
},
"class_type": "CLIPLoader",
"_meta": {
"title": "Load CLIP"
}
},
"171": {
"inputs": {
"noise_seed": 1
},
"class_type": "RandomNoise",
"_meta": {
"title": "RandomNoise"
}
},
"172": {
"inputs": {
"fps": 24,
"bit_depth": 8,
"color_space": "sRGB",
"images": [
"164",
0
],
"audio": [
"163",
0
]
},
"class_type": "CreateVideo",
"_meta": {
"title": "Create Video"
}
},
"173": {
"inputs": {
"prompt": "Vaporwave title sequence look: pink and blue gradient palette, VHS tracking artifacts, Greek statue motifs, chrome palm trees, RGB chromatic aberration, lo-fi retro atmosphere, mood languid and nostalgic.\n\nTimeline:\n[0s-1s] VHS static opens the frame, the title \"COMFYUI\" appears with RGB split and a slight horizontal jitter.\n[1s-2.5s] Hard cut, a Greek plaster bust close-up, pink-purple gradient sky, a pixelated sun.\n[2.5s-4s] Clean \"STARRING\" credits appear, \"LATENT\" and \"CONTROLNET\" each shown exactly once.\n[4s-5s] Final card \"DIRECTED BY COMFYUI\" holds, one VHS tracking glitch settling into stability.\n\nHard cuts only, transitions landing with tape jumps, no push-ins, no dissolves.\n\nAudio: lo-fi vaporwave score, slow drum machine with soft bass, VHS tape-noise sample joins at 2.5s, melody fading for the last 1s.\n\nAll text must be clearly legible, do not misspell English, no Chinese characters, do not repeat names or job titles, no soft dissolves, no subtitle bars.",
"width": [
"127",
0
],
"height": [
"127",
1
],
"length": [
"174",
1
],
"clip": [
"170",
0
],
"vae": [
"161",
0
],
"first_frame": [
"128",
0
]
},
"class_type": "MiniMaxH3ImageToVideo",
"_meta": {
"title": "MiniMax H3 Image to Video"
}
},
"174": {
"inputs": {
"expression": "max(5, round(a * 24)) + (5 - (max(5, round(a * 24)) % 17)) % 17",
"values.a": [
"175",
0
]
},
"class_type": "ComfyMathExpression",
"_meta": {
"title": "Math Expression"
}
},
"175": {
"inputs": {
"value": 2
},
"class_type": "PrimitiveFloat",
"_meta": {
"title": "Float (duration)"
}
},
"176": {
"inputs": {
"lora_name": "minimax_h3_fl2v_turbo_8step_v1.0_comfyui_bf16.safetensors",
"strength_model": 1,
"model": [
"169",
0
]
},
"class_type": "LoraLoaderModelOnly",
"_meta": {
"title": "Load LoRA"
}
},
"177": {
"inputs": {
"switch": [
"181",
0
],
"on_false": [
"169",
0
],
"on_true": [
"176",
0
]
},
"class_type": "ComfySwitchNode",
"_meta": {
"title": "If/Else Switch (Model)"
}
},
"178": {
"inputs": {
"switch": [
"181",
0
],
"on_false": [
"179",
0
],
"on_true": [
"180",
0
]
},
"class_type": "ComfySwitchNode",
"_meta": {
"title": "If/Else Switch (Steps)"
}
},
"179": {
"inputs": {
"value": 20
},
"class_type": "PrimitiveInt",
"_meta": {
"title": "Int"
}
},
"180": {
"inputs": {
"value": 6
},
"class_type": "PrimitiveInt",
"_meta": {
"title": "Int"
}
},
"181": {
"inputs": {
"value": false
},
"class_type": "PrimitiveBoolean",
"_meta": {
"title": "Boolean (Enable Lightning LoRA)"
}
}
}
+98
View File
@@ -0,0 +1,98 @@
{
"1": {
"inputs": {
"unet_name": "qwen_image_edit_2509_fp8_e4m3fn.safetensors",
"weight_dtype": "default"
},
"class_type": "UNETLoader",
"_meta": { "title": "Load Qwen Edit UNET" }
},
"11": {
"inputs": {
"model": ["1", 0],
"lora_name": "Qwen-Image-Edit-2509-Lightning-4steps-V1.0-bf16.safetensors",
"strength_model": 1
},
"class_type": "LoraLoaderModelOnly",
"_meta": { "title": "Lightning LoRA" }
},
"2": {
"inputs": {
"clip_name": "qwen_2.5_vl_7b_fp8_scaled.safetensors",
"type": "qwen_image",
"device": "default"
},
"class_type": "CLIPLoader",
"_meta": { "title": "Load Qwen CLIP" }
},
"3": {
"inputs": {
"vae_name": "qwen_image_vae.safetensors"
},
"class_type": "VAELoader",
"_meta": { "title": "Load Qwen VAE" }
},
"4": {
"inputs": {
"image": "example.png"
},
"class_type": "LoadImage",
"_meta": { "title": "Load input still" }
},
"5": {
"inputs": {
"text": "",
"clip": ["2", 0]
},
"class_type": "CLIPTextEncode",
"_meta": { "title": "Positive prompt" }
},
"6": {
"inputs": {
"text": "blurry, low quality, deformed, bad anatomy, watermark, text, extra fingers",
"clip": ["2", 0]
},
"class_type": "CLIPTextEncode",
"_meta": { "title": "Negative prompt" }
},
"7": {
"inputs": {
"pixels": ["4", 0],
"vae": ["3", 0]
},
"class_type": "VAEEncode",
"_meta": { "title": "Encode still" }
},
"8": {
"inputs": {
"seed": 42,
"steps": 4,
"cfg": 1,
"sampler_name": "euler",
"scheduler": "simple",
"denoise": 0.75,
"model": ["11", 0],
"positive": ["5", 0],
"negative": ["6", 0],
"latent_image": ["7", 0]
},
"class_type": "KSampler",
"_meta": { "title": "Sample edit" }
},
"9": {
"inputs": {
"samples": ["8", 0],
"vae": ["3", 0]
},
"class_type": "VAEDecode",
"_meta": { "title": "Decode still" }
},
"10": {
"inputs": {
"filename_prefix": "Qwen_Edit",
"images": ["9", 0]
},
"class_type": "SaveImage",
"_meta": { "title": "Save edited still" }
}
}
+6
View File
@@ -4,6 +4,12 @@ export function setComfyHostOverride(url: string) {
comfyHostOverride = String(url || '').replace(/\/$/, '') comfyHostOverride = String(url || '').replace(/\/$/, '')
} }
export function comfyConfigured() {
if (comfyHostOverride) return true
const config = useRuntimeConfig()
return Boolean(String(config.comfyHost || process.env.COMFY_HOST || '').trim())
}
function comfyBase() { function comfyBase() {
if (comfyHostOverride) return comfyHostOverride if (comfyHostOverride) return comfyHostOverride
const config = useRuntimeConfig() const config = useRuntimeConfig()
+13 -7
View File
@@ -1,6 +1,6 @@
import { execFile, spawn } from 'node:child_process' import { execFile, spawn } from 'node:child_process'
import { promisify } from 'node:util' import { promisify } from 'node:util'
import { getComfyHost, setComfyHostOverride } from '~/server/utils/comfy' import { getComfyHost, setComfyHostOverride, comfyConfigured } from '~/server/utils/comfy'
const execFileAsync = promisify(execFile) const execFileAsync = promisify(execFile)
@@ -48,20 +48,26 @@ function settings() {
} }
} }
export async function checkComfyHttp(timeoutMs = 2500, host = getComfyHost()) { export async function checkComfyHttp(timeoutMs = 2500, host?: string) {
let resolved = host || ''
try { try {
const res = await fetch(`${host}/system_stats`, { signal: AbortSignal.timeout(timeoutMs) }) if (!resolved) {
if (!res.ok) return { ok: false as const, host, status: res.status, stats: null } if (!comfyConfigured()) return { ok: false as const, host: '', status: 0, stats: null }
resolved = getComfyHost()
}
const res = await fetch(`${resolved}/system_stats`, { signal: AbortSignal.timeout(timeoutMs) })
if (!res.ok) return { ok: false as const, host: resolved, status: res.status, stats: null }
const stats = await res.json().catch(() => null) const stats = await res.json().catch(() => null)
return { ok: true as const, host, status: res.status, stats } return { ok: true as const, host: resolved, status: res.status, stats }
} catch { } catch {
return { ok: false as const, host, status: 0, stats: null } return { ok: false as const, host: resolved, status: 0, stats: null }
} }
} }
export async function fetchQueue() { export async function fetchQueue() {
const host = getComfyHost() if (!comfyConfigured()) return { running: 0, pending: 0 }
try { try {
const host = getComfyHost()
const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(2500) }) const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(2500) })
if (!res.ok) return { running: 0, pending: 0 } if (!res.ok) return { running: 0, pending: 0 }
const payload = await res.json() as { queue_running?: unknown[]; queue_pending?: unknown[] } const payload = await res.json() as { queue_running?: unknown[]; queue_pending?: unknown[] }
+255
View File
@@ -0,0 +1,255 @@
let imageComfyHostOverride = ''
export function setImageComfyHostOverride(url: string) {
imageComfyHostOverride = String(url || '').replace(/\/$/, '')
}
export function getImageComfyHost() {
if (imageComfyHostOverride) return imageComfyHostOverride
const config = useRuntimeConfig()
let host = String(config.imageComfyHost || process.env.IMAGE_COMFY_HOST || '').trim().replace(/\/$/, '')
const port = String(config.imageComfyPort || process.env.IMAGE_COMFY_PORT || '').trim()
if (!host) {
throw createError({ statusCode: 500, statusMessage: 'IMAGE_COMFY_HOST is not configured' })
}
if (!/^https?:\/\//i.test(host)) host = `http://${host}`
try {
const url = new URL(host)
if (port && !url.port) url.port = port
return url.origin
} catch {
return port ? `${host}:${port}` : host
}
}
export function imageComfyConfigured() {
const config = useRuntimeConfig()
return Boolean(String(config.imageComfyHost || process.env.IMAGE_COMFY_HOST || '').trim())
}
export async function imageComfyFetch(path: string, init?: RequestInit) {
const host = getImageComfyHost()
try {
return await fetch(`${host}${path}`, init)
} catch (error) {
throw createError({
statusCode: 502,
statusMessage: `Image ComfyUI host unreachable (${host})`,
data: { cause: error instanceof Error ? error.message : String(error) }
})
}
}
export function imageComfyWsUrl(clientId: string) {
return `${getImageComfyHost().replace(/^http/, 'ws')}/ws?clientId=${encodeURIComponent(clientId)}`
}
function imageInputFilename(original: string, jobId?: string) {
const raw = String(original || 'still.png')
const dot = raw.lastIndexOf('.')
const ext = (dot >= 0 ? raw.slice(dot) : '.png').replace(/[^.a-zA-Z0-9]/g, '') || '.png'
const base = (dot >= 0 ? raw.slice(0, dot) : raw).replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 48) || 'still'
const id = String(jobId || crypto.randomUUID()).replace(/-/g, '').slice(0, 10)
return `aigen_edit_${id}_${base}${ext}`
}
export async function uploadImageEdit(file: { filename: string; data: Buffer; type?: string }, jobId?: string) {
const body = new FormData()
const blob = new Blob([new Uint8Array(file.data)], { type: file.type || 'application/octet-stream' })
const filename = imageInputFilename(file.filename, jobId)
body.append('image', blob, filename)
body.append('overwrite', 'true')
body.append('type', 'input')
const res = await imageComfyFetch('/upload/image', { method: 'POST', body })
if (!res.ok) {
throw createError({ statusCode: 502, statusMessage: `Image upload failed (${res.status})` })
}
const uploaded = (await res.json()) as { name: string; subfolder?: string; type?: string }
return {
name: uploaded.name || filename,
subfolder: uploaded.subfolder || '',
type: uploaded.type || 'input'
}
}
export async function queueImagePrompt(graph: unknown, clientId: string) {
const res = await imageComfyFetch('/prompt', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ prompt: graph, client_id: clientId })
})
const payload = await res.json().catch(() => ({}))
if (!res.ok) {
const message = (payload as { error?: { message?: string } }).error?.message
|| (payload as { node_errors?: unknown }).node_errors
|| `Queue failed (${res.status})`
throw createError({ statusCode: 502, statusMessage: String(message), data: payload })
}
return payload as { prompt_id: string; number?: number }
}
export async function interruptImageComfy() {
if (!imageComfyConfigured()) return false
const res = await imageComfyFetch('/interrupt', { method: 'POST' })
return res.ok
}
export async function fetchImageHistory(promptId: string) {
const res = await imageComfyFetch(`/history/${encodeURIComponent(promptId)}`)
if (!res.ok) return null
return (await res.json()) as Record<string, unknown>
}
function isStillFile(item: { filename?: string } | null | undefined) {
if (!item?.filename) return false
return /\.(png|jpe?g|webp)$/i.test(item.filename)
}
function findStill(value: unknown, depth = 0): { filename: string; subfolder: string; type: string } | null {
if (!value || typeof value !== 'object' || depth > 8) return null
if (Array.isArray(value)) {
const files = value.filter((item): item is { filename?: string; subfolder?: string; type?: string } => Boolean(item && typeof item === 'object'))
const match = files.find(isStillFile)
if (match?.filename) {
return {
filename: String(match.filename),
subfolder: String(match.subfolder || ''),
type: String(match.type || 'output')
}
}
for (const item of value) {
const nested = findStill(item, depth + 1)
if (nested) return nested
}
return null
}
const record = value as { filename?: string; subfolder?: string; type?: string; images?: unknown }
if (record.filename && isStillFile(record)) {
return {
filename: String(record.filename),
subfolder: String(record.subfolder || ''),
type: String(record.type || 'output')
}
}
for (const nested of Object.values(value as Record<string, unknown>)) {
const found = findStill(nested, depth + 1)
if (found) return found
}
return null
}
export function extractEditedImage(history: Record<string, unknown> | null, promptId: string) {
if (!history) return null
const wrapped = history[promptId] as { outputs?: Record<string, unknown> } | undefined
if (wrapped) return findStill(wrapped.outputs || {}) || findStill(wrapped)
if ((history as { outputs?: unknown }).outputs) {
return findStill((history as { outputs?: unknown }).outputs) || findStill(history)
}
return findStill(history)
}
export async function downloadEditedImage(image: { filename: string; subfolder: string; type: string }) {
const params = new URLSearchParams({
filename: image.filename,
subfolder: image.subfolder || '',
type: image.type || 'output'
})
const res = await imageComfyFetch(`/view?${params.toString()}`)
if (!res.ok) {
throw createError({ statusCode: 502, statusMessage: `Failed to fetch edited still (${res.status})` })
}
return Buffer.from(await res.arrayBuffer())
}
function historyError(history: Record<string, unknown> | null, promptId: string) {
const entry = history?.[promptId] as { status?: { status_str?: string; completed?: boolean } } | undefined
const status = entry?.status?.status_str
if (status === 'interrupted') return 'Job interrupted.'
if (status === 'error') return 'Image ComfyUI reported an execution error'
if (entry?.status?.completed && !extractEditedImage(history, promptId)) {
return 'Image ComfyUI finished without an output still'
}
return null
}
export async function waitForImageEdit(opts: {
promptId: string
clientId: string
timeoutMs?: number
onProgress?: (event: { message: string; progress: number; step?: number; maxStep?: number; node?: string | null }) => void
isCancelled?: () => boolean
}) {
const timeoutMs = opts.timeoutMs || 180_000
const started = Date.now()
let settled = false
let lastError: string | null = null
let result: { filename: string; subfolder: string; type: string } | null = null
const finish = (image: { filename: string; subfolder: string; type: string } | null, error?: string) => {
if (settled) return
settled = true
lastError = error || null
result = image
}
let ws: WebSocket | null = null
try {
ws = new WebSocket(imageComfyWsUrl(opts.clientId))
ws.addEventListener('message', (event) => {
const payload = JSON.parse(String(event.data || '{}')) as {
type?: string
data?: { value?: number; max?: number; node?: string | null; prompt_id?: string }
}
if (payload.type === 'progress' && payload.data) {
const max = Math.max(1, Number(payload.data.max || 4))
const value = Number(payload.data.value || 0)
opts.onProgress?.({
message: 'Sampling edit...',
progress: Math.min(90, 20 + Math.round((value / max) * 60)),
step: value,
maxStep: max,
node: payload.data.node || '8'
})
}
if (payload.type === 'executing' && payload.data?.node) {
opts.onProgress?.({
message: payload.data.node === '8' ? 'Sampling edit...' : 'Running Qwen Edit...',
progress: payload.data.node === '10' ? 92 : 30,
node: payload.data.node
})
}
if (payload.type === 'execution_error') {
finish(null, 'Image ComfyUI reported an execution error')
}
})
} catch {
ws = null
}
while (!settled && Date.now() - started < timeoutMs) {
if (opts.isCancelled?.()) {
finish(null, 'Job interrupted.')
break
}
const history = await fetchImageHistory(opts.promptId)
const image = extractEditedImage(history, opts.promptId)
if (image) {
finish(image)
break
}
const error = historyError(history, opts.promptId)
if (error) {
finish(null, error)
break
}
await new Promise(resolve => setTimeout(resolve, 700))
}
try { ws?.close() } catch { /* ignore */ }
if (result) return result
throw createError({
statusCode: 502,
statusMessage: lastError || 'Image edit timed out waiting for Qwen Edit to finish'
})
}
+218
View File
@@ -0,0 +1,218 @@
import { getImageComfyHost, setImageComfyHostOverride, imageComfyConfigured } from '~/server/utils/imageComfy'
export type ImageComfyState = 'online' | 'booting' | 'busy' | 'starting' | 'offline'
export interface ImageComfyStatus {
state: ImageComfyState
message: string
host: string
processRunning?: boolean
queueRunning?: number
queuePending?: number
}
type StatusFn = (status: ImageComfyStatus) => void
let gate: Promise<unknown> = Promise.resolve()
let lastStartAt = 0
function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms))
}
function log(event: string, detail: Record<string, unknown> = {}) {
console.log(JSON.stringify({
src: 'image-comfy-lifecycle',
event,
at: new Date().toISOString(),
...detail
}))
}
function settings() {
const config = useRuntimeConfig()
return {
host: getImageComfyHost(),
launchCmd: String(config.imageComfyLaunchCmd || process.env.IMAGE_COMFY_LAUNCH_CMD || '').trim(),
controlUrl: String(config.imageComfyControlUrl || process.env.IMAGE_COMFY_CONTROL_URL || '').replace(/\/$/, ''),
controlToken: String(config.imageComfyControlToken || process.env.IMAGE_COMFY_CONTROL_TOKEN || ''),
healthTimeoutMs: 2500,
bootPollAttempts: Number(process.env.IMAGE_COMFY_BOOT_POLL_ATTEMPTS || 20),
bootPollDelayMs: 3000,
startTimeoutMs: Number(process.env.IMAGE_COMFY_START_TIMEOUT_MS || 180_000),
busyWaitMs: Number(process.env.IMAGE_COMFY_BUSY_WAIT_MS || 180_000)
}
}
export async function checkImageComfyHttp(timeoutMs = 2500, host?: string) {
let resolved = host || ''
try {
if (!resolved) {
if (!imageComfyConfigured()) return { ok: false as const, host: '', status: 0, stats: null }
resolved = getImageComfyHost()
}
const res = await fetch(`${resolved}/system_stats`, { signal: AbortSignal.timeout(timeoutMs) })
if (!res.ok) return { ok: false as const, host: resolved, status: res.status, stats: null }
const stats = await res.json().catch(() => null)
return { ok: true as const, host: resolved, status: res.status, stats }
} catch {
return { ok: false as const, host: resolved, status: 0, stats: null }
}
}
export async function fetchImageQueue() {
try {
const host = getImageComfyHost()
const res = await fetch(`${host}/queue`, { signal: AbortSignal.timeout(2500) })
if (!res.ok) return { running: 0, pending: 0 }
const payload = await res.json() as { queue_running?: unknown[]; queue_pending?: unknown[] }
return {
running: Array.isArray(payload.queue_running) ? payload.queue_running.length : 0,
pending: Array.isArray(payload.queue_pending) ? payload.queue_pending.length : 0
}
} catch {
return { running: 0, pending: 0 }
}
}
async function controlRequest(path: string, method = 'GET') {
const { controlUrl, controlToken } = settings()
if (!controlUrl) return null
try {
const res = await fetch(`${controlUrl}${path}`, {
method,
headers: {
Accept: 'application/json',
...(controlToken ? { Authorization: `Bearer ${controlToken}` } : {})
},
signal: AbortSignal.timeout(5000)
})
if (!res.ok) return { ok: false, status: res.status }
return await res.json() as {
ok?: boolean
http?: boolean
process?: boolean
started?: boolean
already?: boolean
port?: number | null
proxyPort?: number | null
}
} catch {
return null
}
}
async function waitWhileBusy(onStatus: StatusFn) {
const { busyWaitMs, host } = settings()
const started = Date.now()
while (Date.now() - started < busyWaitMs) {
const queue = await fetchImageQueue()
if (queue.running === 0) return
onStatus({
state: 'busy',
message: `Image ComfyUI is busy with another prompt (${queue.running} running, ${queue.pending} queued). Waiting up to 3 minutes.`,
host,
queueRunning: queue.running,
queuePending: queue.pending
})
await sleep(2000)
}
throw createError({
statusCode: 409,
statusMessage: 'Image ComfyUI is still busy after 3 minutes.',
data: { code: 'IMAGE_COMFY_BUSY' }
})
}
function hostWithPort(host: string, port: number) {
try {
const url = new URL(host)
url.port = String(port)
return url.origin
} catch {
return host
}
}
async function startImageComfy() {
const now = Date.now()
if (now - lastStartAt < 15_000) {
log('start-skipped', { reason: 'recent-start' })
return { started: false, reason: 'recent-start' as const }
}
lastStartAt = now
const remote = await controlRequest('/start', 'POST')
if (remote?.ok || remote?.started) {
log('start-remote', { port: remote.port || null })
return { started: true, reason: 'remote' as const }
}
return { started: false, reason: 'no-launcher' as const }
}
async function ensureUnlocked(onStatus: StatusFn) {
const cfg = settings()
const host = cfg.host
onStatus({ state: 'offline', message: 'Checking image ComfyUI...', host })
const health = await checkImageComfyHttp(cfg.healthTimeoutMs)
if (health.ok) {
await waitWhileBusy(onStatus)
onStatus({ state: 'online', message: 'Image ComfyUI online', host })
return
}
const remote = await controlRequest('/status')
if (remote?.http && remote.proxyPort) {
const proxyUrl = hostWithPort(host, remote.proxyPort)
const viaProxy = await checkImageComfyHttp(cfg.healthTimeoutMs, proxyUrl)
if (viaProxy.ok) {
setImageComfyHostOverride(proxyUrl)
await waitWhileBusy(onStatus)
onStatus({ state: 'online', message: 'Image ComfyUI online', host: proxyUrl, processRunning: true })
return
}
}
onStatus({
state: 'starting',
message: 'Image ComfyUI is offline. Starting the sidecar instance...',
host,
processRunning: Boolean(remote?.process)
})
const started = await startImageComfy()
if (!started.started && started.reason === 'no-launcher') {
throw createError({
statusCode: 503,
statusMessage: 'Image ComfyUI is offline and the sidecar host agent did not start it. Confirm the agent on 192.168.77.101:8199 is running.'
})
}
const deadline = Date.now() + cfg.startTimeoutMs
let attempt = 0
while (Date.now() < deadline) {
attempt += 1
onStatus({
state: 'starting',
message: `Starting image ComfyUI... waiting for /system_stats (${attempt})`,
host
})
const next = await checkImageComfyHttp()
if (next.ok) {
await waitWhileBusy(onStatus)
onStatus({ state: 'online', message: 'Image ComfyUI online', host, processRunning: true })
return
}
await sleep(2000)
}
throw createError({
statusCode: 503,
statusMessage: 'Started image ComfyUI but the API never answered. Headless Qwen Edit can take a couple of minutes to load.'
})
}
export function ensureImageComfyReady(onStatus: StatusFn = () => undefined) {
const run = gate.then(() => ensureUnlocked(onStatus))
gate = run.then(() => undefined, () => undefined)
return run
}
+52
View File
@@ -0,0 +1,52 @@
import template from '../assets/workflow_qwen_edit.json'
export const DEFAULT_EDIT_NEGATIVE = 'blurry, low quality, deformed, bad anatomy, watermark, text, extra fingers'
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
type WorkflowGraph = Record<string, WorkflowNode>
export interface EditParams {
prompt: string
negative?: string
imageName: string
steps: number
seed: number
cfg: number
denoise: number
samplerName?: string
scheduler?: string
filenamePrefix?: string
}
export function buildEditWorkflow(params: EditParams) {
const graph = structuredClone(template) as WorkflowGraph
if (graph['4']) graph['4'].inputs.image = params.imageName
if (graph['5']) graph['5'].inputs.text = params.prompt
if (graph['6']) graph['6'].inputs.text = (params.negative || DEFAULT_EDIT_NEGATIVE).trim() || DEFAULT_EDIT_NEGATIVE
if (graph['8']) {
graph['8'].inputs.seed = params.seed
graph['8'].inputs.steps = params.steps
graph['8'].inputs.cfg = params.cfg
graph['8'].inputs.denoise = params.denoise
graph['8'].inputs.sampler_name = params.samplerName || 'euler'
graph['8'].inputs.scheduler = params.scheduler || 'simple'
}
if (graph['10'] && params.filenamePrefix) {
graph['10'].inputs.filename_prefix = params.filenamePrefix
}
return graph
}
export const EDIT_NODE_LABELS: Record<string, string> = {
'1': 'Loading Qwen Edit UNET',
'11': 'Applying Lightning LoRA',
'2': 'Loading Qwen CLIP',
'3': 'Loading Qwen VAE',
'4': 'Loading input still',
'5': 'Encoding prompt',
'6': 'Encoding negative',
'7': 'Encoding still',
'8': 'Sampling edit',
'9': 'Decoding still',
'10': 'Saving still'
}
+5
View File
@@ -11,6 +11,7 @@ export interface JobEvent {
subfolder?: string subfolder?: string
mediaType?: string mediaType?: string
clipId?: string clipId?: string
stillId?: string
hideThumbnail?: boolean hideThumbnail?: boolean
error?: string error?: string
elapsedMs?: number elapsedMs?: number
@@ -28,6 +29,7 @@ export interface JobEvent {
export interface Job { export interface Job {
id: string id: string
kind?: 'video' | 'edit'
promptId?: string promptId?: string
clientId: string clientId: string
status: JobStatus status: JobStatus
@@ -38,6 +40,7 @@ export interface Job {
startedAt: number startedAt: number
video?: { filename: string; subfolder: string; type: string } video?: { filename: string; subfolder: string; type: string }
clipId?: string clipId?: string
stillId?: string
hideThumbnail?: boolean hideThumbnail?: boolean
library?: { library?: {
ownerKey: string ownerKey: string
@@ -75,6 +78,7 @@ export interface Job {
extendSourceClipId?: string extendSourceClipId?: string
familyId?: string familyId?: string
parentClipId?: string parentClipId?: string
workflow?: 'v1' | 'v2'
} }
error?: string error?: string
socketReady?: boolean socketReady?: boolean
@@ -132,6 +136,7 @@ export function jobSnapshot(job: Job) {
subfolder: job.library?.folderLocked ? undefined : job.video?.subfolder, subfolder: job.library?.folderLocked ? undefined : job.video?.subfolder,
mediaType: job.library?.folderLocked ? undefined : job.video?.type, mediaType: job.library?.folderLocked ? undefined : job.video?.type,
clipId: job.clipId, clipId: job.clipId,
stillId: job.stillId,
hideThumbnail: job.hideThumbnail, hideThumbnail: job.hideThumbnail,
error: job.error, error: job.error,
folderLocked: job.library?.folderLocked, folderLocked: job.library?.folderLocked,
+11 -4
View File
@@ -38,6 +38,7 @@ export interface LibraryClip {
familyId?: string familyId?: string
parentClipId?: string parentClipId?: string
chainIndex?: number chainIndex?: number
workflow?: 'v1' | 'v2'
} }
export interface LibraryStill { export interface LibraryStill {
@@ -89,6 +90,7 @@ export interface RetryDraft {
samplerName?: string samplerName?: string
scheduler?: string scheduler?: string
extensions?: QueuedExtension[] extensions?: QueuedExtension[]
workflow?: 'v1' | 'v2'
} }
interface Catalog { interface Catalog {
@@ -715,6 +717,7 @@ export async function saveRetryDraft(params: {
samplerName?: string samplerName?: string
scheduler?: string scheduler?: string
extensions?: QueuedExtension[] extensions?: QueuedExtension[]
workflow?: 'v1' | 'v2'
}) { }) {
return mutate(params.ownerKey, (catalog) => { return mutate(params.ownerKey, (catalog) => {
const existing = params.id ? catalog.drafts.find(item => item.id === params.id) : undefined const existing = params.id ? catalog.drafts.find(item => item.id === params.id) : undefined
@@ -740,7 +743,8 @@ export async function saveRetryDraft(params: {
fps: params.fps, fps: params.fps,
samplerName: params.samplerName, samplerName: params.samplerName,
scheduler: params.scheduler, scheduler: params.scheduler,
extensions: params.extensions?.length ? params.extensions : undefined extensions: params.extensions?.length ? params.extensions : undefined,
workflow: params.workflow
} }
catalog.drafts = [draft, ...catalog.drafts.filter(item => item.id !== draft.id)].slice(0, 20) catalog.drafts = [draft, ...catalog.drafts.filter(item => item.id !== draft.id)].slice(0, 20)
return draft return draft
@@ -809,6 +813,7 @@ export async function saveClip(params: {
familyId?: string familyId?: string
parentClipId?: string parentClipId?: string
chainIndex?: number chainIndex?: number
workflow?: 'v1' | 'v2'
}) { }) {
const catalog = readCatalog(params.ownerKey) const catalog = readCatalog(params.ownerKey)
const folder = catalog.folders.find(item => item.id === params.folderId) || catalog.folders[0] const folder = catalog.folders.find(item => item.id === params.folderId) || catalog.folders[0]
@@ -833,7 +838,8 @@ export async function saveClip(params: {
scheduler: params.scheduler, scheduler: params.scheduler,
familyId: params.familyId, familyId: params.familyId,
parentClipId: params.parentClipId, parentClipId: params.parentClipId,
chainIndex: params.chainIndex chainIndex: params.chainIndex,
workflow: params.workflow
} }
mkdirSync(clipDir(params.ownerKey, clip.id), { recursive: true }) mkdirSync(clipDir(params.ownerKey, clip.id), { recursive: true })
const videoPath = clipVideoPath(params.ownerKey, clip.id) const videoPath = clipVideoPath(params.ownerKey, clip.id)
@@ -852,12 +858,13 @@ export async function saveClip(params: {
} else { } else {
await extractVideoThumb(clipVideoPath(params.ownerKey, clip.id), join(clipDir(params.ownerKey, clip.id), 'thumb.jpg')) await extractVideoThumb(clipVideoPath(params.ownerKey, clip.id), join(clipDir(params.ownerKey, clip.id), 'thumb.jpg'))
} }
if (params.thumb?.length) { const saveInputStill = Boolean(params.thumb?.length) && !params.parentClipId && !(params.chainIndex || 0)
if (saveInputStill) {
await saveStill({ await saveStill({
ownerKey: params.ownerKey, ownerKey: params.ownerKey,
folderId: folder.id, folderId: folder.id,
filename: `${clip.id}.png`, filename: `${clip.id}.png`,
data: params.thumb, data: params.thumb!,
width: params.width, width: params.width,
height: params.height, height: params.height,
hideInput: params.hideInput hideInput: params.hideInput
+2 -1
View File
@@ -191,7 +191,8 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
scheduler: job.library.scheduler, scheduler: job.library.scheduler,
familyId: job.library.familyId || (job.library.familyId = crypto.randomUUID()), familyId: job.library.familyId || (job.library.familyId = crypto.randomUUID()),
parentClipId: job.library.parentClipId, parentClipId: job.library.parentClipId,
chainIndex: job.library.chainIndex chainIndex: job.library.chainIndex,
workflow: job.library.workflow
}) })
job.clipId = clip.id job.clipId = clip.id
job.savedPromptId = job.promptId job.savedPromptId = job.promptId
+130 -26
View File
@@ -1,5 +1,8 @@
// Nitro bundles this JSON into the production server output. // Nitro bundles these JSON graphs into the production server output.
import workflowTemplate from '../assets/workflow_minimax_video.json' import workflowV1 from '../assets/workflow_minimax_video.json'
import workflowV2 from '../assets/workflow_minimax_video_v2.json'
export type VideoWorkflowId = 'v1' | 'v2'
export interface GenerateParams { export interface GenerateParams {
prompt: string prompt: string
@@ -16,6 +19,10 @@ export interface GenerateParams {
scheduler: string scheduler: string
filenamePrefix?: string filenamePrefix?: string
sound?: boolean sound?: boolean
workflow?: VideoWorkflowId
duration?: number
useIdentityRefs?: boolean
referenceImageNames?: string[]
} }
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } } type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
@@ -23,6 +30,7 @@ type WorkflowGraph = Record<string, WorkflowNode>
const PROMPT_CLASSES = new Set([ const PROMPT_CLASSES = new Set([
'MiniMaxH3ImageToVideo', 'MiniMaxH3ImageToVideo',
'MiniMaxH3ReferenceToVideo',
'CLIPTextEncode', 'CLIPTextEncode',
'CLIPTextEncodeQwen3VL', 'CLIPTextEncodeQwen3VL',
'TextEncodeQwen3VL' 'TextEncodeQwen3VL'
@@ -30,6 +38,7 @@ const PROMPT_CLASSES = new Set([
const SEED_CLASSES = new Set(['RandomNoise', 'KSampler', 'SamplerMiniMax', 'KSamplerAdvanced']) const SEED_CLASSES = new Set(['RandomNoise', 'KSampler', 'SamplerMiniMax', 'KSamplerAdvanced'])
const IMAGE_CLASSES = new Set(['LoadImage']) const IMAGE_CLASSES = new Set(['LoadImage'])
const LORA_CLASSES = new Set(['LoraLoaderModelOnly', 'LoraLoader']) const LORA_CLASSES = new Set(['LoraLoaderModelOnly', 'LoraLoader'])
const V2_REF_LOADERS = ['140', '141', '142', '143'] as const
function entriesOf(graph: WorkflowGraph, classType: string) { function entriesOf(graph: WorkflowGraph, classType: string) {
return Object.entries(graph).filter(([, node]) => node.class_type === classType) return Object.entries(graph).filter(([, node]) => node.class_type === classType)
@@ -47,13 +56,24 @@ function titleOf(node: WorkflowNode) {
return node._meta?.title || '' return node._meta?.title || ''
} }
function templateFor(id: VideoWorkflowId) {
return (id === 'v2' ? workflowV2 : workflowV1) as WorkflowGraph
}
export function parseVideoWorkflow(raw: unknown): VideoWorkflowId {
return String(raw || '').trim() === 'v2' ? 'v2' : 'v1'
}
export function buildWorkflow(params: GenerateParams) { export function buildWorkflow(params: GenerateParams) {
const graph = structuredClone(workflowTemplate) as WorkflowGraph const version = params.workflow === 'v2' ? 'v2' : 'v1'
const graph = structuredClone(templateFor(version))
const width = snap32(params.width) const width = snap32(params.width)
const height = snap32(params.height) const height = snap32(params.height)
const refs = (params.referenceImageNames || []).map(name => String(name || '').trim()).filter(Boolean).slice(0, 4)
const useRefs = version === 'v2' && params.useIdentityRefs === true && refs.length > 0
for (const [, node] of Object.entries(graph)) { for (const [id, node] of Object.entries(graph)) {
if (IMAGE_CLASSES.has(node.class_type) && 'image' in node.inputs) { if (IMAGE_CLASSES.has(node.class_type) && 'image' in node.inputs && id === '114') {
node.inputs.image = params.imageName node.inputs.image = params.imageName
} }
@@ -64,9 +84,14 @@ export function buildWorkflow(params: GenerateParams) {
if (node.class_type === 'MiniMaxH3ImageToVideo') { if (node.class_type === 'MiniMaxH3ImageToVideo') {
node.inputs.prompt = params.prompt node.inputs.prompt = params.prompt
node.inputs.width = width if (typeof node.inputs.width === 'number') node.inputs.width = width
node.inputs.height = height if (typeof node.inputs.height === 'number') node.inputs.height = height
node.inputs.length = params.length if (typeof node.inputs.length === 'number') node.inputs.length = params.length
}
if (node.class_type === 'MiniMaxH3ReferenceToVideo') {
node.inputs.prompt = params.prompt
if (!Array.isArray(node.inputs.length)) node.inputs.length = ['174', 1]
} }
if (node.class_type === 'CreateVideo' && 'fps' in node.inputs) { if (node.class_type === 'CreateVideo' && 'fps' in node.inputs) {
@@ -93,7 +118,7 @@ export function buildWorkflow(params: GenerateParams) {
if ('seed' in node.inputs) node.inputs.seed = params.seed if ('seed' in node.inputs) node.inputs.seed = params.seed
} }
if (LORA_CLASSES.has(node.class_type)) { if (LORA_CLASSES.has(node.class_type) && !/identity|ref 1/i.test(titleOf(node))) {
node.inputs.strength_model = params.turbo ? 1 : 0 node.inputs.strength_model = params.turbo ? 1 : 0
} }
@@ -101,11 +126,22 @@ export function buildWorkflow(params: GenerateParams) {
node.inputs.filename_prefix = params.filenamePrefix || node.inputs.filename_prefix || 'video/MiniMax_H3' node.inputs.filename_prefix = params.filenamePrefix || node.inputs.filename_prefix || 'video/MiniMax_H3'
} }
if (node.class_type === 'PrimitiveBoolean') { if (node.class_type === 'PrimitiveBoolean' && /lightning|turbo/i.test(titleOf(node))) {
node.inputs.value = params.turbo node.inputs.value = params.turbo
} }
if (node.class_type === 'ComfySwitchNode' && typeof node.inputs.switch === 'boolean') { if (node.class_type === 'PrimitiveBoolean' && /identity|ref2va|reference/i.test(titleOf(node))) {
node.inputs.value = useRefs
}
if (node.class_type === 'PrimitiveFloat' && /duration/i.test(titleOf(node))) {
const seconds = typeof params.duration === 'number' && params.duration > 0
? params.duration
: params.length / Math.max(params.fps, 1)
node.inputs.value = seconds
}
if (node.class_type === 'ComfySwitchNode' && typeof node.inputs.switch === 'boolean' && /lightning|turbo|lora/i.test(titleOf(node))) {
node.inputs.switch = params.turbo node.inputs.switch = params.turbo
} }
@@ -114,7 +150,7 @@ export function buildWorkflow(params: GenerateParams) {
} }
} }
if (graph['128']) { if (version === 'v1' && graph['128']?.class_type === 'ImageScaleToTotalPixels') {
graph['128'] = { graph['128'] = {
class_type: 'ImageScale', class_type: 'ImageScale',
inputs: { inputs: {
@@ -128,16 +164,18 @@ export function buildWorkflow(params: GenerateParams) {
} }
} }
graph['105:cfg'] = { if (version === 'v1') {
class_type: 'FluxGuidance', graph['105:cfg'] = {
inputs: { class_type: 'FluxGuidance',
guidance: params.cfg, inputs: {
conditioning: ['105:104', 0] guidance: params.cfg,
}, conditioning: ['105:104', 0]
_meta: { title: 'Guidance scale' } },
} _meta: { title: 'Guidance scale' }
if (graph['105:16']?.inputs) { }
graph['105:16'].inputs.conditioning = ['105:cfg', 0] if (graph['105:16']?.inputs) {
graph['105:16'].inputs.conditioning = ['105:cfg', 0]
}
} }
for (const [, node] of entriesOf(graph, 'ComfySwitchNode')) { for (const [, node] of entriesOf(graph, 'ComfySwitchNode')) {
@@ -152,15 +190,69 @@ export function buildWorkflow(params: GenerateParams) {
} }
} }
if (version === 'v2') applyV2IdentityPath(graph, params, useRefs, refs)
if (params.sound === false) { if (params.sound === false) {
delete graph['105:23'] delete graph['105:23']
delete graph['105:24'] delete graph['105:24']
if (graph['105:91']?.inputs) delete graph['105:91'].inputs.audio if (graph['105:91']?.inputs) delete graph['105:91'].inputs.audio
delete graph['163']
if (graph['172']?.inputs) delete graph['172'].inputs.audio
} }
return graph return graph
} }
function applyV2IdentityPath(
graph: WorkflowGraph,
params: GenerateParams,
useRefs: boolean,
refs: string[]
) {
if (useRefs && graph['155'] && graph['167'] && graph['168']) {
graph['167'].inputs.latent_image = ['155', 1]
graph['168'].inputs.conditioning = ['155', 0]
graph['155'].inputs.prompt = params.prompt
graph['155'].inputs.length = ['174', 1]
delete graph['173']
V2_REF_LOADERS.forEach((id, index) => {
const name = refs[index]
const inputKey = `ref_images.ref_image_${index + 1}`
if (name && graph[id]) {
graph[id].inputs.image = name
} else {
delete graph[id]
if (graph['155']?.inputs) delete graph['155'].inputs[inputKey]
}
})
graph['v2:cfg'] = {
class_type: 'FluxGuidance',
inputs: {
guidance: params.cfg,
conditioning: ['155', 0]
},
_meta: { title: 'Guidance scale' }
}
graph['168'].inputs.conditioning = ['v2:cfg', 0]
return
}
delete graph['155']
delete graph['150']
for (const id of V2_REF_LOADERS) delete graph[id]
if (graph['173'] && graph['168']) {
graph['v2:cfg'] = {
class_type: 'FluxGuidance',
inputs: {
guidance: params.cfg,
conditioning: ['173', 0]
},
_meta: { title: 'Guidance scale' }
}
graph['168'].inputs.conditioning = ['v2:cfg', 0]
}
}
const LABEL_OVERRIDES: Record<string, string> = { const LABEL_OVERRIDES: Record<string, string> = {
'114': 'Loading image', '114': 'Loading image',
'128': 'Scaling image', '128': 'Scaling image',
@@ -168,6 +260,14 @@ const LABEL_OVERRIDES: Record<string, string> = {
'119': 'Scaling image', '119': 'Scaling image',
'120': 'Reading image size', '120': 'Reading image size',
'92': 'Saving MP4', '92': 'Saving MP4',
'140': 'Loading identity ref 1',
'141': 'Loading identity ref 2',
'142': 'Loading identity ref 3',
'143': 'Loading identity ref 4',
'150': 'Identity references',
'155': 'Reference to video',
'172': 'Encoding video',
'173': 'Image to video',
'105:6': 'Loading MiniMax H3 model', '105:6': 'Loading MiniMax H3 model',
'105:11': 'Loading video VAE', '105:11': 'Loading video VAE',
'105:13': 'Loading Qwen3-VL encoder', '105:13': 'Loading Qwen3-VL encoder',
@@ -177,6 +277,7 @@ const LABEL_OVERRIDES: Record<string, string> = {
'105:14': 'Sampling', '105:14': 'Sampling',
'105:16': 'Preparing guider', '105:16': 'Preparing guider',
'105:cfg': 'Applying guidance', '105:cfg': 'Applying guidance',
'v2:cfg': 'Applying guidance',
'105:17': 'Selecting sampler', '105:17': 'Selecting sampler',
'105:23': 'Decoding audio', '105:23': 'Decoding audio',
'105:24': 'Loading audio VAE', '105:24': 'Loading audio VAE',
@@ -192,13 +293,16 @@ const LABEL_OVERRIDES: Record<string, string> = {
'105:126': 'Selecting turbo LoRA' '105:126': 'Selecting turbo LoRA'
} }
function labelsFrom(template: WorkflowGraph) {
return Object.entries(template).map(([id, node]) => [id, node._meta?.title || `Node ${id}`] as const)
}
export const NODE_LABELS: Record<string, string> = { export const NODE_LABELS: Record<string, string> = {
...Object.fromEntries( ...Object.fromEntries(labelsFrom(workflowV1 as WorkflowGraph)),
Object.entries(workflowTemplate as WorkflowGraph).map(([id, node]) => [id, node._meta?.title || `Node ${id}`]) ...Object.fromEntries(labelsFrom(workflowV2 as WorkflowGraph)),
),
...LABEL_OVERRIDES ...LABEL_OVERRIDES
} }
export function isEncodingNode(node: string) { export function isEncodingNode(node: string) {
return node === '92' || node === '105:91' || /encoding|saving mp4/i.test(NODE_LABELS[node] || '') return node === '92' || node === '105:91' || node === '172' || /encoding|saving mp4|create video/i.test(NODE_LABELS[node] || '')
} }