From 13ecec949ca0ac29d17e17c5f7732cce3285fa8f Mon Sep 17 00:00:00 2001 From: Towsty Date: Thu, 27 Aug 2026 09:06:50 -0500 Subject: [PATCH] Run image edit on Beast with Flux.2 Klein and stop routing jobs to Sidecar. Co-authored-by: Cursor --- pages/index.vue | 143 +++-- server/api/edit.post.ts | 102 ++- server/api/health.get.ts | 43 +- server/api/interrupt.post.ts | 5 +- server/assets/workflow_flux2_klein_edit.json | 613 +++++++++++++++++++ server/utils/imageComfy.ts | 12 +- server/utils/imageComfyLifecycle.ts | 81 +-- server/utils/imageWorkflow.ts | 112 ++-- 8 files changed, 899 insertions(+), 212 deletions(-) create mode 100644 server/assets/workflow_flux2_klein_edit.json diff --git a/pages/index.vue b/pages/index.vue index 922eb4f..b8233a4 100644 --- a/pages/index.vue +++ b/pages/index.vue @@ -6,7 +6,7 @@
A

{{ instanceName }}

-

MiniMax H3 · Qwen Edit · ComfyUI

+

MiniMax H3 · Flux.2 Klein · ComfyUI

@@ -20,7 +20,7 @@ {{ imageComfyBadge.label }} @@ -57,7 +57,7 @@

Input

{{ studioMode === 'edit' - ? 'Drop a still and describe the edit. Beast handles it when free; Sidecar only if Beast is busy.' + ? 'Drop the still to edit. An optional second still applies image 2 onto image 1. Both run on Beast; files are removed from the desktop after save.' : 'Drop a still, describe the motion, then relay the job to the local desktop.' }}

@@ -80,11 +80,52 @@
-

Choose an image

+

{{ studioMode === 'edit' ? 'Image 1' : 'Choose an image' }}

Browse the library or upload a new PNG, JPG, or WEBP

+
+
+

Image 2 · optional

+ +
+ +

+ {{ editRefFile + ? 'Two-image branch on. Image 2 is applied onto image 1.' + : 'One-image branch on. Add a second still to transfer details from image 2.' }} +

+
+
@@ -428,13 +469,11 @@ Image edit settings

- Beast first, Sidecar overflow. + Flux.2 Klein 9B on Beast. Sidecar is off. {{ imageComfyBadge.label }} - · Qwen Image Edit u2 (Lightning)

- Beast {{ imageBeastHost || 'unset' }}{{ imageBeastOk ? (imageBeastBusy ? ' · busy' : ' · free') : ' · offline' }} - · Sidecar {{ imageSidecarHost || 'unset' }}{{ imageSidecarOk ? (imageSidecarBusy ? ' · busy' : ' · ready') : (imageSidecarConfigured ? ' · offline' : ' · unset') }} + Beast {{ imageBeastHost || imageComfyHost || 'unset' }}{{ comfyOk ? (comfyBusy ? ' · busy' : ' · free') : ' · offline' }}

@@ -515,8 +550,8 @@ · step {{ step }}/{{ maxStep || defaultVideoSteps(turbo, videoWorkflow) }}

-

Video generation continues in the background.

-

Image edit continues in the background.

+

Video is using Beast. This edit waits in the Comfy queue.

+

Image edit is using Beast. Video waits until that finishes.

@@ -1084,14 +1119,15 @@ const editProgress = ref(0) const editStatusMessage = ref('Waiting for a still.') const editStatusBusy = ref(false) const editStep = ref(0) -const editMaxStep = ref(4) +const editMaxStep = ref(20) const editElapsedMs = ref(0) const editAwaitingReveal = ref(false) const editLockedSave = ref(false) -const editSteps = ref(4) +const editSteps = ref(20) const editCfg = ref(1) -const editDenoise = ref(1) -const editNegative = ref('blurry, low quality, deformed, bad anatomy, watermark, text, extra fingers') +const editNegative = ref('') +const editRefFile = ref(null) +const editRefPreview = ref('') const videoJobId = ref('') const editJobId = ref('') const currentClipId = ref('') @@ -1128,13 +1164,14 @@ const renameInput = ref(null) const unlockPassword = ref('') const settingsOpen = ref(false) const pickerOpen = ref(false) -const pickerSlot = ref<'main' | number>('main') +const pickerSlot = ref<'main' | 'editRef' | number>('main') const videoWorkflow = ref('v1') const useIdentityRefs = ref(false) const identityRefs = ref<(File | null)[]>([null, null, null, null]) const identityPreviews = ref(['', '', '', '']) const pickerHeading = computed(() => { - if (pickerSlot.value === 'main') return 'Choose an image' + if (pickerSlot.value === 'main') return studioMode.value === 'edit' ? 'Choose image 1' : 'Choose an image' + if (pickerSlot.value === 'editRef') return 'Choose image 2' return `Choose Picture ${Number(pickerSlot.value) + 2}` }) let videoTimer: ReturnType | null = null @@ -1183,43 +1220,29 @@ const comfyBadge = computed(() => { }) const imageComfyBadge = computed(() => { - if (!imageComfyConfigured.value) { + if (!imageComfyConfigured.value && !comfyOk.value) { return { label: 'Edit unset', className: 'border-white/10 text-zinc-500', dot: 'bg-zinc-500' } } - if (imageBeastOk.value && !imageBeastBusy.value) { + if (comfyOk.value && !comfyBusy.value) { return { - label: 'Beast · edit ready', + label: 'Beast · Klein ready', className: 'border-emerald-400/30 text-emerald-300', dot: 'bg-emerald-400' } } - if (imageBeastBusy.value && imageSidecarOk.value && !imageSidecarBusy.value) { + if (comfyBusy.value) { return { - label: 'Sidecar · overflow ready', - className: 'border-amber-400/40 text-amber-200', - dot: 'bg-amber-400' - } - } - if (imageSidecarOk.value && !imageSidecarBusy.value) { - return { - label: 'Sidecar · edit ready', - className: 'border-emerald-400/30 text-emerald-300', - dot: 'bg-emerald-400' - } - } - if (imageComfyBusy.value) { - return { - label: 'Edit boxes busy', + label: 'Beast · busy', className: 'border-amber-400/40 text-amber-200', dot: 'bg-amber-400' } } return { - label: 'Edit Comfy offline', + label: 'Beast offline', className: 'border-red-400/30 text-red-300', dot: 'bg-red-400' } @@ -1282,14 +1305,12 @@ const videoGenerateDisabled = computed(() => { const editBlocked = computed(() => { if (editBusy.value) return true if (!file.value || !prompt.value.trim() || !folderId.value) return true - if (!imageComfyConfigured.value) return true - if (!imageComfyOk.value && !imageSidecarConfigured.value) return true + if (!comfyOk.value && !imageComfyOk.value) return true return false }) const editBlockReason = computed(() => { - if (!imageComfyConfigured.value) return 'Image edit is not configured. Set COMFY_HOST and IMAGE_COMFY_HOST.' - if (!imageComfyOk.value && !imageSidecarConfigured.value) return 'Beast is offline and Sidecar is not configured.' - if (!imageComfyOk.value && imageSidecarConfigured.value && !imageSidecarOk.value) return 'Beast and Sidecar are both offline.' + if (!imageComfyConfigured.value && !comfyOk.value) return 'Image edit is not configured. Set COMFY_HOST.' + if (!comfyOk.value && !imageComfyOk.value) return 'Beast is offline.' return '' }) const generateLabel = computed(() => { @@ -1310,7 +1331,9 @@ const composedIdentityPrompt = computed(() => { }) const promptPlaceholder = computed(() => { if (studioMode.value === 'edit') { - return 'Make the lighting warmer. Replace the background with a rainy street at night. Keep the face unchanged.' + return editRefFile.value + ? 'Apply the dress in image 2 to the person in image 1. Preserve the exact features, proportions, pose, and setting of image 1.' + : 'Make the lighting warmer. Replace the background with a rainy street at night. Keep the face unchanged.' } if (identityPrompting.value) { return shotScriptMode.value @@ -1676,11 +1699,23 @@ function readFile(next: File, opts?: { keepThumbnailPref?: boolean }) { img.src = preview.value } -function openPicker(slot: 'main' | number = 'main') { +function openPicker(slot: 'main' | 'editRef' | number = 'main') { pickerSlot.value = slot pickerOpen.value = true } +function setEditRef(next: File) { + if (editRefPreview.value) URL.revokeObjectURL(editRefPreview.value) + editRefFile.value = next + editRefPreview.value = URL.createObjectURL(next) +} + +function clearEditRef() { + if (editRefPreview.value) URL.revokeObjectURL(editRefPreview.value) + editRefFile.value = null + editRefPreview.value = '' +} + function setIdentityRef(index: number, next: File) { const prev = identityPreviews.value[index] if (prev) URL.revokeObjectURL(prev) @@ -1706,6 +1741,10 @@ function clearIdentityRef(index: number) { function onPickedImage(next: File, hidden = false) { const slot = pickerSlot.value pickerOpen.value = false + if (slot === 'editRef') { + setEditRef(next) + return + } if (slot !== 'main') { setIdentityRef(Number(slot), next) return @@ -2280,17 +2319,17 @@ async function editImage() { editResultUrl.value = '' editAwaitingReveal.value = false editLockedSave.value = false - editStatusMessage.value = 'Choosing Beast or Sidecar...' + editStatusMessage.value = 'Checking Beast ComfyUI...' editMaxStep.value = editSteps.value startTimer('edit') try { const body = new FormData() body.append('image', file.value) + if (editRefFile.value) body.append('image2', editRefFile.value) body.append('prompt', prompt.value.trim()) body.append('negative', editNegative.value) body.append('steps', String(editSteps.value)) body.append('cfg', String(editCfg.value)) - body.append('denoise', String(editDenoise.value)) body.append('seed', seedInput.value || 'random') body.append('folderId', folderId.value) body.append('hideInput', String(showPrivacyToggles.value && hideInputPreview.value)) diff --git a/server/api/edit.post.ts b/server/api/edit.post.ts index cde8dbb..e4e0c74 100644 --- a/server/api/edit.post.ts +++ b/server/api/edit.post.ts @@ -1,12 +1,13 @@ -import { imageComfyConfigured, uploadImageEdit, queueImagePrompt, waitForImageEdit, downloadEditedImage, purgeImageComfyArtifacts, withImageComfyHost } from '~/server/utils/imageComfy' -import { pickImageEditHost } from '~/server/utils/imageComfyLifecycle' +import { ensureComfyReady } from '~/server/utils/comfyLifecycle' +import { getComfyHost, comfyConfigured, uploadImage, queuePrompt, purgeComfyArtifacts } from '~/server/utils/comfy' +import { withImageComfyHost, waitForImageEdit, downloadEditedImage } from '~/server/utils/imageComfy' import { buildEditWorkflow } from '~/server/utils/imageWorkflow' import { imageDimensions } from '~/server/utils/resolution' function parseSteps(raw: unknown) { const value = Number(raw) - if (!Number.isFinite(value)) return 4 - return Math.min(20, Math.max(1, Math.round(value))) + if (!Number.isFinite(value)) return 20 + return Math.min(40, Math.max(1, Math.round(value))) } function parseCfg(raw: unknown) { @@ -15,19 +16,16 @@ function parseCfg(raw: unknown) { return Math.min(8, Math.max(0.5, Math.round(value * 10) / 10)) } -function parseDenoise(raw: unknown) { - const value = Number(raw) - if (!Number.isFinite(value)) return 0.75 - return Math.min(1, Math.max(0.05, Math.round(value * 100) / 100)) -} - export default defineEventHandler(async (event) => { const form = await readMultipartFormData(event).catch(() => null) const fields: Record = {} let image: { filename: string; data: Buffer; type?: string } | null = null + let reference: { filename: string; data: Buffer; type?: string } | null = null for (const part of form || []) { if (part.name === 'image' && part.filename && part.data?.length) { image = { filename: part.filename, data: part.data, type: part.type } + } else if ((part.name === 'image2' || part.name === 'refImage') && part.filename && part.data?.length) { + reference = { filename: part.filename, data: part.data, type: part.type } } else if (part.name && part.data) { fields[part.name] = part.data.toString('utf8') } @@ -40,10 +38,10 @@ export default defineEventHandler(async (event) => { if (!prompt) { throw createError({ statusCode: 400, statusMessage: 'An edit prompt is required' }) } - if (!imageComfyConfigured()) { + if (!comfyConfigured()) { throw createError({ statusCode: 503, - statusMessage: 'Image edit ComfyUI is not configured yet. Set COMFY_HOST for Beast and IMAGE_COMFY_HOST for Sidecar overflow.' + statusMessage: 'Beast ComfyUI is not configured. Set COMFY_HOST.' }) } @@ -62,12 +60,32 @@ export default defineEventHandler(async (event) => { const hideInput = fields.hideInput === 'true' const steps = parseSteps(fields.steps) const cfg = parseCfg(fields.cfg) - const denoise = parseDenoise(fields.denoise) const seed = fields.seed && fields.seed !== 'random' ? Number(fields.seed) : Math.floor(Math.random() * 2_147_483_647) const size = imageDimensions(image.data) + const still = isPipelineFrameFilename(image.filename) + ? null + : await saveStill({ + ownerKey, + folderId, + filename: image.filename, + data: image.data, + width: size?.width || 0, + height: size?.height || 0, + hideInput + }) + if (reference && !isPipelineFrameFilename(reference.filename)) { + await saveStill({ + ownerKey, + folderId, + filename: reference.filename, + data: reference.data, + hideInput + }) + } + const job = createJob() job.kind = 'edit' job.maxStep = steps @@ -86,18 +104,19 @@ export default defineEventHandler(async (event) => { steps, turbo: true, seed, - cfg + cfg, + stillId: still?.id } - emitJob(job, { type: 'status', message: 'Choosing Beast or Sidecar...', progress: 2 }) + emitJob(job, { type: 'status', message: 'Checking Beast ComfyUI...', progress: 2 }) void runEdit(job, { image, + reference, prompt, negative: (fields.negative || '').trim(), steps, seed, - cfg, - denoise + cfg }).catch(async (error) => { if (job.status === 'error' || job.status === 'cancelled') return const message = error instanceof Error ? error.message : String(error) @@ -111,7 +130,8 @@ export default defineEventHandler(async (event) => { seed, steps, hideThumbnail: hideInput, - folderLocked + folderLocked, + dual: Boolean(reference) } }) @@ -119,15 +139,15 @@ async function runEdit( job: ReturnType, params: { image: { filename: string; data: Buffer; type?: string } + reference: { filename: string; data: Buffer; type?: string } | null prompt: string negative: string steps: number seed: number cfg: number - denoise: number } ) { - const target = await pickImageEditHost((status) => { + await ensureComfyReady((status) => { emitJob(job, { type: status.state === 'busy' ? 'busy' : 'status', message: status.message, @@ -135,41 +155,56 @@ async function runEdit( busy: status.state === 'busy' }) }) - job.imageComfyHost = target.host if (job.status === 'cancelled') throw new Error('Job interrupted.') + job.imageComfyHost = getComfyHost() - await withImageComfyHost(target.host, async () => { + await withImageComfyHost(job.imageComfyHost, async () => { + const dual = Boolean(params.reference) emitJob(job, { type: 'status', - message: `Uploading still to ${target.label}...`, + message: dual ? 'Uploading both stills to Beast...' : 'Uploading still to Beast...', progress: 8 }) - const uploaded = await uploadImageEdit(params.image, job.id) + const uploaded = await uploadImage(params.image, job.id) + const uploadedRef = params.reference + ? await uploadImage({ + ...params.reference, + filename: `ref_${params.reference.filename || 'image2.png'}` + }, job.id) + : null if (job.status === 'cancelled') throw new Error('Job interrupted.') emitJob(job, { type: 'status', - message: `Queueing Qwen Image Edit on ${target.label}...`, + message: dual + ? 'Queueing Flux.2 Klein two-image edit on Beast...' + : 'Queueing Flux.2 Klein edit on Beast...', progress: 12 }) const graph = buildEditWorkflow({ prompt: params.prompt, negative: params.negative, imageName: uploaded.name, + referenceImageName: uploadedRef?.name, steps: params.steps, seed: params.seed, cfg: params.cfg, - denoise: params.denoise, filenamePrefix: `aigen_edit_${job.id.slice(0, 8)}` }) - const queued = await queueImagePrompt(graph, job.clientId) + const queued = await queuePrompt(graph, job.clientId) job.promptId = queued.prompt_id job.status = 'running' - emitJob(job, { type: 'status', message: `Editing still on ${target.label}...`, progress: 18, maxStep: params.steps }) + emitJob(job, { + type: 'status', + message: dual ? 'Applying image 2 onto image 1...' : 'Editing still on Beast...', + progress: 18, + maxStep: params.steps + }) const output = await waitForImageEdit({ promptId: queued.prompt_id, clientId: job.clientId, + timeoutMs: 10 * 60 * 1000, onProgress: (event) => { emitJob(job, { type: 'status', @@ -196,16 +231,17 @@ async function runEdit( hideInput: job.library?.hideInput }) job.stillId = still?.id - await purgeImageComfyArtifacts({ - output, - inputName: uploaded.name, - inputSubfolder: uploaded.subfolder, + await purgeComfyArtifacts({ + video: { filename: output.filename, subfolder: output.subfolder, type: output.type }, + imageName: uploaded.name, + imageSubfolder: uploaded.subfolder, + extraImageNames: uploadedRef?.name ? [uploadedRef.name] : [], promptId: job.promptId }) job.status = 'complete' emitJob(job, { type: 'complete', - message: `Edit finished on ${target.label}`, + message: 'Edit finished on Beast', progress: 100, stillId: still?.id, filename: output.filename, diff --git a/server/api/health.get.ts b/server/api/health.get.ts index f92231a..c84b1e1 100644 --- a/server/api/health.get.ts +++ b/server/api/health.get.ts @@ -1,14 +1,13 @@ import { checkComfyHttp, fetchQueue } from '~/server/utils/comfyLifecycle' import { comfyConfigured } from '~/server/utils/comfy' -import { imageComfyConfigured, getBeastImageHost, getSidecarImageHost, sameImageHost } from '~/server/utils/imageComfy' +import { imageComfyConfigured, getBeastImageHost } from '~/server/utils/imageComfy' import { checkImageComfyHttp, fetchQueueAt } from '~/server/utils/imageComfyLifecycle' export default defineEventHandler(async () => { const videoConfigured = comfyConfigured() const imageConfigured = imageComfyConfigured() const beastHost = getBeastImageHost() - const sidecarHost = getSidecarImageHost() - const [comfy, queue, beast, beastQueue, sidecar, sidecarQueue] = await Promise.all([ + const [comfy, queue, beast, beastQueue] = await Promise.all([ videoConfigured ? checkComfyHttp().catch(() => ({ ok: false, host: '', stats: null })) : Promise.resolve({ ok: false, host: '', stats: null }), @@ -16,21 +15,13 @@ export default defineEventHandler(async () => { beastHost ? checkImageComfyHttp(2500, beastHost).catch(() => ({ ok: false, host: beastHost, stats: null })) : Promise.resolve({ ok: false, host: beastHost, stats: null }), - beastHost ? fetchQueueAt(beastHost) : Promise.resolve({ running: 0, pending: 0 }), - sidecarHost && (!beastHost || !sameImageHost(beastHost, sidecarHost)) - ? checkImageComfyHttp(2500, sidecarHost).catch(() => ({ ok: false, host: sidecarHost, stats: null })) - : Promise.resolve({ ok: false, host: sidecarHost, stats: null }), - sidecarHost && (!beastHost || !sameImageHost(beastHost, sidecarHost)) - ? fetchQueueAt(sidecarHost) - : Promise.resolve({ running: 0, pending: 0 }) + beastHost ? fetchQueueAt(beastHost) : Promise.resolve({ running: 0, pending: 0 }) ]) const busy = Boolean(comfy.ok && queue.running > 0) const beastBusy = Boolean(beast.ok && beastQueue.running > 0) - const sidecarBusy = Boolean(sidecar.ok && sidecarQueue.running > 0) - const sidecarConfigured = Boolean(sidecarHost && (!beastHost || !sameImageHost(beastHost, sidecarHost))) - const imageOk = Boolean(beast.ok || sidecar.ok) - const imageBusy = Boolean(imageConfigured && ((beastBusy && (!sidecarConfigured || sidecarBusy || !sidecar.ok)) || (!beast.ok && sidecarBusy))) - const preferredHost = beast.ok && !beastBusy ? beastHost : (sidecar.ok ? sidecarHost : beastHost) + const imageOk = Boolean(beast.ok || comfy.ok) + const imageBusy = Boolean(imageConfigured && (beastBusy || busy)) + const preferredHost = beastHost || comfy.host || '' return { ok: true, service: useRuntimeConfig().public.instanceName || 'aigen', @@ -46,23 +37,23 @@ export default defineEventHandler(async () => { configured: imageConfigured, ok: imageOk, busy: imageBusy, - host: preferredHost || '', - box: beast.ok && !beastBusy ? 'beast' : (sidecar.ok ? 'sidecar' : (beast.ok ? 'beast' : '')), - queueRunning: (beast.ok && !beastBusy ? beastQueue.running : sidecarQueue.running) || 0, - queuePending: (beast.ok && !beastBusy ? beastQueue.pending : sidecarQueue.pending) || 0, + host: preferredHost, + box: imageOk ? 'beast' : '', + queueRunning: (beastQueue.running || queue.running) || 0, + queuePending: (beastQueue.pending || queue.pending) || 0, beast: { label: 'Beast', configured: Boolean(beastHost), - ok: Boolean(beast.ok), - busy: beastBusy, - host: beastHost || '' + ok: Boolean(beast.ok || comfy.ok), + busy: beastBusy || busy, + host: preferredHost }, sidecar: { label: 'Sidecar', - configured: sidecarConfigured, - ok: Boolean(sidecar.ok), - busy: sidecarBusy, - host: sidecarHost || '' + configured: false, + ok: false, + busy: false, + host: '' } } } diff --git a/server/api/interrupt.post.ts b/server/api/interrupt.post.ts index f92e9fa..6ed1b36 100644 --- a/server/api/interrupt.post.ts +++ b/server/api/interrupt.post.ts @@ -8,10 +8,7 @@ export default defineEventHandler(async (event) => { removeExtendTemp(job.library?.extendTmpDir) } if (jobId) deletePendingJob(jobId) - const kind = job?.kind || (body?.kind === 'edit' ? 'edit' : 'video') - const ok = kind === 'edit' - ? await interruptImageComfy(job?.imageComfyHost) - : await interruptComfy() + const ok = await interruptComfy() if (job) { emitJob(job, { type: 'error', error: 'Job interrupted.', message: 'Job interrupted.' }) } diff --git a/server/assets/workflow_flux2_klein_edit.json b/server/assets/workflow_flux2_klein_edit.json new file mode 100644 index 0000000..52e6cea --- /dev/null +++ b/server/assets/workflow_flux2_klein_edit.json @@ -0,0 +1,613 @@ +{ + "9": { + "inputs": { + "filename_prefix": "Flux2-Klein-4b-base", + "images": [ + "75:65", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "76": { + "inputs": { + "image": "" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "81": { + "inputs": { + "image": "" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "94": { + "inputs": { + "filename_prefix": "Flux2-Klein-4b-base", + "images": [ + "92:104", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "75:61": { + "inputs": { + "sampler_name": "euler" + }, + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + } + }, + "75:62": { + "inputs": { + "steps": 20, + "width": [ + "75:100", + 0 + ], + "height": [ + "75:100", + 1 + ] + }, + "class_type": "Flux2Scheduler", + "_meta": { + "title": "Flux2Scheduler" + } + }, + "75:63": { + "inputs": { + "cfg": 1, + "model": [ + "75:131", + 0 + ], + "positive": [ + "75:124", + 0 + ], + "negative": [ + "75:122", + 0 + ] + }, + "class_type": "CFGGuider", + "_meta": { + "title": "CFG Guider" + } + }, + "75:64": { + "inputs": { + "noise": [ + "75:73", + 0 + ], + "guider": [ + "75:63", + 0 + ], + "sampler": [ + "75:61", + 0 + ], + "sigmas": [ + "75:62", + 0 + ], + "latent_image": [ + "75:66", + 0 + ] + }, + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + } + }, + "75:65": { + "inputs": { + "samples": [ + "75:64", + 0 + ], + "vae": [ + "75:72", + 0 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "75:73": { + "inputs": { + "noise_seed": 202848876233972 + }, + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + } + }, + "75:70": { + "inputs": { + "unet_name": "flux-2-klein-base-9b-fp8.safetensors", + "weight_dtype": "default" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "75:71": { + "inputs": { + "clip_name": "qwen_3_8b_fp8mixed.safetensors", + "type": "flux2", + "device": "default" + }, + "class_type": "CLIPLoader", + "_meta": { + "title": "Load CLIP" + } + }, + "75:74": { + "inputs": { + "text": "Placeholder prompt \n", + "clip": [ + "75:131", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Positive Prompt)" + } + }, + "75:67": { + "inputs": { + "text": "", + "clip": [ + "75:71", + 0 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Negative Prompt)" + } + }, + "75:72": { + "inputs": { + "vae_name": "full_encoder_small_decoder.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "75:66": { + "inputs": { + "width": [ + "75:100", + 0 + ], + "height": [ + "75:100", + 1 + ], + "batch_size": 1 + }, + "class_type": "EmptyFlux2LatentImage", + "_meta": { + "title": "Empty Flux 2 Latent" + } + }, + "75:100": { + "inputs": { + "image": [ + "76", + 0 + ] + }, + "class_type": "GetImageSize", + "_meta": { + "title": "Get Image Size" + } + }, + "75:122": { + "inputs": { + "conditioning": [ + "75:67", + 0 + ], + "latent": [ + "75:123", + 0 + ] + }, + "class_type": "ReferenceLatent", + "_meta": { + "title": "Set Reference Latent" + } + }, + "75:123": { + "inputs": { + "pixels": [ + "76", + 0 + ], + "vae": [ + "75:72", + 0 + ] + }, + "class_type": "VAEEncode", + "_meta": { + "title": "VAE Encode" + } + }, + "75:124": { + "inputs": { + "conditioning": [ + "75:74", + 0 + ], + "latent": [ + "75:123", + 0 + ] + }, + "class_type": "ReferenceLatent", + "_meta": { + "title": "Set Reference Latent" + } + }, + "75:131": { + "inputs": { + "lora_name": "klein_snofs_v1_4.safetensors", + "strength_model": 1, + "strength_clip": 1, + "model": [ + "75:70", + 0 + ], + "clip": [ + "75:71", + 0 + ] + }, + "class_type": "LoraLoader", + "_meta": { + "title": "Load LoRA (Model and CLIP)" + } + }, + "92:102": { + "inputs": { + "sampler_name": "euler" + }, + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + } + }, + "92:103": { + "inputs": { + "noise": [ + "92:105", + 0 + ], + "guider": [ + "92:114", + 0 + ], + "sampler": [ + "92:102", + 0 + ], + "sigmas": [ + "92:115", + 0 + ], + "latent_image": [ + "92:109", + 0 + ] + }, + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + } + }, + "92:104": { + "inputs": { + "samples": [ + "92:103", + 0 + ], + "vae": [ + "92:107", + 0 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "92:105": { + "inputs": { + "noise_seed": 10865292823982 + }, + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + } + }, + "92:106": { + "inputs": { + "unet_name": "flux-2-klein-base-9b-fp8.safetensors", + "weight_dtype": "default" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "92:107": { + "inputs": { + "vae_name": "full_encoder_small_decoder.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "92:108": { + "inputs": { + "image": [ + "76", + 0 + ] + }, + "class_type": "GetImageSize", + "_meta": { + "title": "Get Image Size" + } + }, + "92:109": { + "inputs": { + "width": [ + "92:108", + 0 + ], + "height": [ + "92:108", + 1 + ], + "batch_size": 1 + }, + "class_type": "EmptyFlux2LatentImage", + "_meta": { + "title": "Empty Flux 2 Latent" + } + }, + "92:85": { + "inputs": { + "upscale_method": "lanczos", + "megapixels": 1, + "resolution_steps": 1, + "image": [ + "81", + 0 + ] + }, + "class_type": "ImageScaleToTotalPixels", + "_meta": { + "title": "Scale Image to Total Pixels" + } + }, + "92:111": { + "inputs": { + "clip_name": "qwen_3_8b_fp8mixed.safetensors", + "type": "flux2", + "device": "default" + }, + "class_type": "CLIPLoader", + "_meta": { + "title": "Load CLIP" + } + }, + "92:113": { + "inputs": { + "text": "Placeholder prompt multi imagee ", + "clip": [ + "92:132", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Positive Prompt)" + } + }, + "92:87": { + "inputs": { + "text": "", + "clip": [ + "92:111", + 0 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode ( Negative Prompt)" + } + }, + "92:114": { + "inputs": { + "cfg": 1, + "model": [ + "92:132", + 0 + ], + "positive": [ + "92:130", + 0 + ], + "negative": [ + "92:128", + 0 + ] + }, + "class_type": "CFGGuider", + "_meta": { + "title": "CFG Guider" + } + }, + "92:115": { + "inputs": { + "steps": 20, + "width": [ + "92:108", + 0 + ], + "height": [ + "92:108", + 1 + ] + }, + "class_type": "Flux2Scheduler", + "_meta": { + "title": "Flux2Scheduler" + } + }, + "92:125": { + "inputs": { + "conditioning": [ + "92:87", + 0 + ], + "latent": [ + "92:126", + 0 + ] + }, + "class_type": "ReferenceLatent", + "_meta": { + "title": "Set Reference Latent" + } + }, + "92:126": { + "inputs": { + "pixels": [ + "76", + 0 + ], + "vae": [ + "92:107", + 0 + ] + }, + "class_type": "VAEEncode", + "_meta": { + "title": "VAE Encode" + } + }, + "92:127": { + "inputs": { + "conditioning": [ + "92:113", + 0 + ], + "latent": [ + "92:126", + 0 + ] + }, + "class_type": "ReferenceLatent", + "_meta": { + "title": "Set Reference Latent" + } + }, + "92:128": { + "inputs": { + "conditioning": [ + "92:125", + 0 + ], + "latent": [ + "92:129", + 0 + ] + }, + "class_type": "ReferenceLatent", + "_meta": { + "title": "Set Reference Latent" + } + }, + "92:129": { + "inputs": { + "pixels": [ + "92:85", + 0 + ], + "vae": [ + "92:107", + 0 + ] + }, + "class_type": "VAEEncode", + "_meta": { + "title": "VAE Encode" + } + }, + "92:130": { + "inputs": { + "conditioning": [ + "92:127", + 0 + ], + "latent": [ + "92:129", + 0 + ] + }, + "class_type": "ReferenceLatent", + "_meta": { + "title": "Set Reference Latent" + } + }, + "92:132": { + "inputs": { + "lora_name": "klein_snofs_v1_4.safetensors", + "strength_model": 1, + "strength_clip": 1, + "model": [ + "92:106", + 0 + ], + "clip": [ + "92:111", + 0 + ] + }, + "class_type": "LoraLoader", + "_meta": { + "title": "Load LoRA (Model and CLIP)" + } + } +} \ No newline at end of file diff --git a/server/utils/imageComfy.ts b/server/utils/imageComfy.ts index 45ce27d..12ad676 100644 --- a/server/utils/imageComfy.ts +++ b/server/utils/imageComfy.ts @@ -55,7 +55,7 @@ export function getImageComfyHost() { } export function imageComfyConfigured() { - return Boolean(getBeastImageHost() || getSidecarImageHost()) + return Boolean(getBeastImageHost()) } export function sameImageHost(a: string, b: string) { @@ -246,17 +246,17 @@ export async function waitForImageEdit(opts: { const max = Math.max(1, Number(payload.data.max || 4)) const value = Number(payload.data.value || 0) opts.onProgress?.({ - message: 'Sampling edit...', + message: 'Sampling Flux.2 Klein...', progress: Math.min(90, 20 + Math.round((value / max) * 60)), step: value, maxStep: max, - node: payload.data.node || '8' + node: payload.data.node || null }) } if (payload.type === 'executing' && payload.data?.node) { opts.onProgress?.({ - message: payload.data.node === '8' ? 'Sampling edit...' : 'Running Qwen Edit...', - progress: payload.data.node === '10' ? 92 : 30, + message: 'Running Flux.2 Klein...', + progress: payload.data.node === '9' || payload.data.node === '94' ? 92 : 30, node: payload.data.node }) } @@ -291,7 +291,7 @@ export async function waitForImageEdit(opts: { if (result) return result throw createError({ statusCode: 502, - statusMessage: lastError || 'Image edit timed out waiting for Qwen Edit to finish' + statusMessage: lastError || 'Image edit timed out waiting for Flux.2 Klein to finish' }) } diff --git a/server/utils/imageComfyLifecycle.ts b/server/utils/imageComfyLifecycle.ts index 85295e2..27582a2 100644 --- a/server/utils/imageComfyLifecycle.ts +++ b/server/utils/imageComfyLifecycle.ts @@ -1,4 +1,4 @@ -import { getImageComfyHost, setImageComfyHostOverride, imageComfyConfigured, getBeastImageHost, getSidecarImageHost, sameImageHost, withImageComfyHost, type ImageEditBox } from '~/server/utils/imageComfy' +import { getImageComfyHost, setImageComfyHostOverride, imageComfyConfigured, getBeastImageHost, withImageComfyHost, type ImageEditBox } from '~/server/utils/imageComfy' export type ImageComfyState = 'online' | 'booting' | 'busy' | 'starting' | 'offline' @@ -231,67 +231,40 @@ export function ensureImageComfyReady(onStatus: StatusFn = () => undefined) { export async function pickImageEditHost(onStatus: StatusFn = () => undefined): Promise { const beast = getBeastImageHost() - const sidecar = getSidecarImageHost() - if (!beast && !sidecar) { + if (!beast) { throw createError({ statusCode: 503, - statusMessage: 'No image ComfyUI host is configured. Set COMFY_HOST (Beast) and/or IMAGE_COMFY_HOST (Sidecar).' + statusMessage: 'Beast ComfyUI is not configured. Set COMFY_HOST.' }) } - if (beast) { - const health = await checkImageComfyHttp(2500, beast) - if (health.ok) { - const queue = await fetchQueueAt(beast) - if (queue.running === 0) { - onStatus({ - state: 'online', - message: 'Beast is free. Editing on this machine.', - host: beast, - box: 'beast', - queueRunning: 0, - queuePending: queue.pending - }) - return { host: beast, box: 'beast', label: 'Beast' } - } - const canOverflow = Boolean(sidecar && !sameImageHost(beast, sidecar)) - onStatus({ - state: 'busy', - message: canOverflow - ? 'Beast is busy. Rolling over to Sidecar...' - : `Beast is busy (${queue.running} running). Waiting for a free slot...`, - host: beast, - box: 'beast', - queueRunning: queue.running, - queuePending: queue.pending - }) - if (!canOverflow) { - await withImageComfyHost(beast, () => waitWhileBusy(onStatus)) - return { host: beast, box: 'beast', label: 'Beast' } - } - } else if (sidecar && !sameImageHost(beast, sidecar)) { - onStatus({ - state: 'offline', - message: 'Beast is offline. Trying Sidecar...', - host: beast, - box: 'beast' - }) - } + const health = await checkImageComfyHttp(2500, beast) + if (!health.ok) { + throw createError({ + statusCode: 503, + statusMessage: 'Beast is offline. Image edit runs on the 5080 only.' + }) } - - if (sidecar && (!beast || !sameImageHost(beast, sidecar))) { + const queue = await fetchQueueAt(beast) + if (queue.running === 0) { onStatus({ - state: 'starting', - message: 'Checking Sidecar for this edit...', - host: sidecar, - box: 'sidecar' + state: 'online', + message: 'Beast is free. Editing on this machine.', + host: beast, + box: 'beast', + queueRunning: 0, + queuePending: queue.pending }) - await withImageComfyHost(sidecar, () => ensureImageComfyReady(onStatus)) - return { host: sidecar, box: 'sidecar', label: 'Sidecar' } + return { host: beast, box: 'beast', label: 'Beast' } } - - throw createError({ - statusCode: 503, - statusMessage: 'Beast is unavailable and Sidecar is not configured.' + onStatus({ + state: 'busy', + message: `Beast is busy (${queue.running} running). Waiting for a free slot...`, + host: beast, + box: 'beast', + queueRunning: queue.running, + queuePending: queue.pending }) + await withImageComfyHost(beast, () => waitWhileBusy(onStatus)) + return { host: beast, box: 'beast', label: 'Beast' } } diff --git a/server/utils/imageWorkflow.ts b/server/utils/imageWorkflow.ts index 3bf7296..42f889a 100644 --- a/server/utils/imageWorkflow.ts +++ b/server/utils/imageWorkflow.ts @@ -1,61 +1,99 @@ -import template from '../assets/workflow_qwen_edit.json' +import template from '../assets/workflow_flux2_klein_edit.json' -export const DEFAULT_EDIT_NEGATIVE = 'blurry, low quality, deformed, bad anatomy, watermark, text, extra fingers' +const BYPASS = 4 +const IMAGE1 = '76' +const IMAGE2 = '81' +const SINGLE_SAVE = '9' +const DUAL_SAVE = '94' +const SINGLE_PROMPT = '75:74' +const SINGLE_NEGATIVE = '75:67' +const SINGLE_NOISE = '75:73' +const SINGLE_SCHEDULER = '75:62' +const SINGLE_CFG = '75:63' +const DUAL_PROMPT = '92:113' +const DUAL_NEGATIVE = '92:87' +const DUAL_NOISE = '92:105' +const DUAL_SCHEDULER = '92:115' +const DUAL_CFG = '92:114' -type WorkflowNode = { class_type: string; inputs: Record; _meta?: { title?: string } } +type WorkflowNode = { class_type: string; inputs: Record; mode?: number; _meta?: { title?: string } } type WorkflowGraph = Record export interface EditParams { prompt: string negative?: string imageName: string + referenceImageName?: string steps: number seed: number cfg: number - denoise: number - samplerName?: string - scheduler?: string filenamePrefix?: string } export function buildEditWorkflow(params: EditParams) { const graph = structuredClone(template) as WorkflowGraph - const loadImage = nodeByClass(graph, 'LoadImage') || graph['4'] - const qwenPrompt = nodeByClass(graph, 'TextEncodeQwenImageEdit') || graph['17'] - const sampler = nodeByClass(graph, 'KSampler') || graph['8'] - const save = nodeByClass(graph, 'SaveImage') || graph['10'] - if (loadImage) loadImage.inputs.image = params.imageName - if (qwenPrompt) { - if ('prompt' in qwenPrompt.inputs) qwenPrompt.inputs.prompt = params.prompt - if ('text' in qwenPrompt.inputs) qwenPrompt.inputs.text = params.prompt + const dual = Boolean(params.referenceImageName) + const prompt = String(params.prompt || '').trim() + const negative = String(params.negative || '') + const prefix = params.filenamePrefix || 'aigen-edit' + + setImage(graph, IMAGE1, params.imageName) + setImage(graph, IMAGE2, params.referenceImageName || '') + setText(graph, SINGLE_PROMPT, prompt) + setText(graph, DUAL_PROMPT, prompt) + setText(graph, SINGLE_NEGATIVE, negative) + setText(graph, DUAL_NEGATIVE, negative) + setInput(graph, SINGLE_NOISE, 'noise_seed', params.seed) + setInput(graph, DUAL_NOISE, 'noise_seed', params.seed) + setInput(graph, SINGLE_SCHEDULER, 'steps', params.steps) + setInput(graph, DUAL_SCHEDULER, 'steps', params.steps) + setInput(graph, SINGLE_CFG, 'cfg', params.cfg) + setInput(graph, DUAL_CFG, 'cfg', params.cfg) + setInput(graph, SINGLE_SAVE, 'filename_prefix', prefix) + setInput(graph, DUAL_SAVE, 'filename_prefix', prefix) + + for (const [id, node] of Object.entries(graph)) { + if (!node) continue + if (id === IMAGE1) { + delete node.mode + continue + } + const dualNode = id === IMAGE2 || id === DUAL_SAVE || id.startsWith('92:') + const singleNode = id === SINGLE_SAVE || id.startsWith('75:') + if (dual ? singleNode : dualNode) node.mode = BYPASS + else delete node.mode } - if (sampler) { - sampler.inputs.seed = params.seed - sampler.inputs.steps = params.steps - sampler.inputs.cfg = params.cfg - sampler.inputs.denoise = params.denoise - sampler.inputs.sampler_name = params.samplerName || 'euler' - sampler.inputs.scheduler = params.scheduler || 'simple' - } - if (save && params.filenamePrefix) save.inputs.filename_prefix = params.filenamePrefix + return graph } -function nodeByClass(graph: WorkflowGraph, classType: string, index = 0) { - const matches = Object.values(graph).filter(node => node?.class_type === classType) - return matches[index] +function setImage(graph: WorkflowGraph, id: string, name: string) { + if (graph[id]) graph[id].inputs.image = name +} + +function setText(graph: WorkflowGraph, id: string, text: string) { + if (graph[id]) graph[id].inputs.text = text +} + +function setInput(graph: WorkflowGraph, id: string, key: string, value: unknown) { + if (graph[id]) graph[id].inputs[key] = value } export const EDIT_NODE_LABELS: Record = { - '1': 'Loading Qwen Edit UNET', - '11': 'Applying Lightning LoRA', - '2': 'Loading Qwen CLIP', - '3': 'Loading Qwen VAE', - '4': 'Loading input still', - '17': 'Encoding Qwen edit prompt', - '19': 'Preparing negative', - '7': 'Encoding still', - '8': 'Sampling edit', - '9': 'Decoding still', - '10': 'Saving still' + '76': 'Loading image 1', + '81': 'Loading image 2', + '75:70': 'Loading Flux.2 Klein', + '75:71': 'Loading CLIP', + '75:72': 'Loading VAE', + '75:74': 'Encoding prompt', + '75:64': 'Sampling edit', + '75:65': 'Decoding still', + '9': 'Saving still', + '92:106': 'Loading Flux.2 Klein', + '92:111': 'Loading CLIP', + '92:107': 'Loading VAE', + '92:113': 'Encoding prompt', + '92:103': 'Sampling two-image edit', + '92:104': 'Decoding still', + '94': 'Saving still' }