import editTemplate from '../assets/klein_v2_edit.json' import composeTemplate from '../assets/klein_v2_compose.json' import refineTemplate from '../assets/klein_v2_refine.json' import generateTemplate from '../assets/klein_v2_generate.json' import kreaGenerateTemplate from '../assets/krea_v2_generate.json' import kreaEditTemplate from '../assets/krea_v2_edit.json' import kreaComposeTemplate from '../assets/krea_v2_compose.json' import kreaRefineTemplate from '../assets/krea_v2_refine.json' import { IMAGE_SCALE_TO_TOTAL_PIXELS } from '~/server/utils/comfy' import { imageComfyFetch } from '~/server/utils/imageComfy' import { applyImageV2UserLoras } from '~/server/utils/loras' import { type LoraStackItem } from '~/utils/loras' import { clampImageCfg, clampImageScaleMegapixels, clampImageSteps } from '~/utils/generationPresets' import { isXaigenStudio } from '~/utils/videoModels' import { IMAGE_V2_KREA_CFG, IMAGE_V2_KREA_CLIP, IMAGE_V2_KREA_CLIP_TYPE, IMAGE_V2_KREA_COMPOSE_DENOISE, IMAGE_V2_KREA_CONCEPT_LORA, IMAGE_V2_KREA_EDIT_DENOISE, IMAGE_V2_KREA_STEPS, IMAGE_V2_KREA_UNET_CANDIDATES, IMAGE_V2_KREA_VAE, IMAGE_V2_DENOISE_DEFAULT, IMAGE_V2_GENERATE_HEIGHT, IMAGE_V2_GENERATE_WIDTH, clampImageV2Denoise, clampImageV2Size, composeImageV2Prompt, imageV2WorkflowFile, mergeLegacyImageV2Loras, parseImageV2Engine, type ImageV2Engine, type ImageV2Mode, type ImageV2Task } from '~/utils/imageV2' type WorkflowNode = { class_type: string; inputs: Record; _meta?: { title?: string } } type WorkflowGraph = Record const LOAD_A = '1' const LOAD_B = '22' const LOAD_MASK = '30' const SCHEDULER_DENOISE = '17' const PROMPT = '9' const NEGATIVE = '10' const NOISE = '15' const SCHEDULER = '17' const CFG = '18' const SAVE = '21' export const IMAGE_V2_EDIT_WORKFLOW = 'klein_v2_edit.json' export const IMAGE_V2_COMPOSE_WORKFLOW = 'klein_v2_compose.json' export const IMAGE_V2_REFINE_WORKFLOW = 'klein_v2_refine.json' export const IMAGE_V2_GENERATE_WORKFLOW = 'klein_v2_generate.json' export const IMAGE_V2_KREA_GENERATE_WORKFLOW = 'krea_v2_generate.json' export const IMAGE_V2_KREA_EDIT_WORKFLOW = 'krea_v2_edit.json' export const IMAGE_V2_KREA_COMPOSE_WORKFLOW = 'krea_v2_compose.json' export const IMAGE_V2_KREA_REFINE_WORKFLOW = 'krea_v2_refine.json' export interface ImageV2BuildParams { mode: ImageV2Mode task: ImageV2Task engine?: ImageV2Engine prompt: string negative?: string imageAName?: string imageBName?: string maskName?: string strength?: number width?: number height?: number aspect?: string turbo?: boolean snofsModel?: number snofsClip?: number consistencyModel?: number consistencyClip?: number steps: number cfg: number seed: number megapixels?: number filenamePrefix?: string kreaUnetName?: string kreaClipName?: string kreaVaeName?: string kreaConceptLora?: string loraStack?: LoraStackItem[] } function setInput(graph: WorkflowGraph, id: string, key: string, value: unknown) { if (graph[id]) graph[id].inputs[key] = value } function loadImageNames(graph: WorkflowGraph) { return Object.entries(graph) .filter(([, node]) => node.class_type === 'LoadImage') .map(([id, node]) => ({ id, title: String(node._meta?.title || id), image: String(node.inputs.image || '') })) } function patchScaleMegapixels(graph: WorkflowGraph, megapixels: number) { const mp = clampImageScaleMegapixels(megapixels) for (const node of Object.values(graph)) { if (node.class_type !== IMAGE_SCALE_TO_TOTAL_PIXELS) continue node.inputs.megapixels = mp node.inputs.upscale_method = 'lanczos' } } function linkSource(value: unknown): string | null { return Array.isArray(value) && typeof value[0] === 'string' ? value[0] : null } function stripScaleNodes(graph: WorkflowGraph) { for (const [id, node] of Object.entries(graph)) { if (node.class_type !== IMAGE_SCALE_TO_TOTAL_PIXELS) continue const src = Array.isArray(node.inputs.image) ? node.inputs.image : null for (const [otherId, other] of Object.entries(graph)) { if (otherId === id) continue for (const [key, value] of Object.entries(other.inputs)) { if (linkSource(value) !== id || !Array.isArray(value)) continue other.inputs[key] = src ? [src[0], value[1] === 0 ? src[1] : value[1]] : value } } delete graph[id] } } function applyScaleMegapixels(graph: WorkflowGraph, megapixels?: number) { const mp = Number(megapixels) if (Number.isFinite(mp) && mp > 0) { patchScaleMegapixels(graph, mp) return } stripScaleNodes(graph) } function graphHasMaskInput(graph: WorkflowGraph) { return Object.values(graph).some((node) => { const mask = node.inputs?.mask return mask !== undefined && mask !== null && mask !== '' }) } function modelBasename(name: string) { return String(name || '').replace(/\\/g, '/').split('/').pop() || '' } function modelIdentity(name: string) { return modelBasename(name).toLowerCase() } async function listComfyModels(folder: string) { const res = await imageComfyFetch(`/models/${encodeURIComponent(folder)}`, { signal: AbortSignal.timeout(8000) }).catch(() => null) if (!res || !res.ok) return [] as string[] const payload = await res.json().catch(() => null) if (Array.isArray(payload)) return payload.map(item => String(item || '')).filter(Boolean) return [] as string[] } function pickComfyModel(available: string[], candidates: string[]) { for (const wanted of candidates) { const id = modelIdentity(wanted) const exact = available.find(name => modelIdentity(name) === id) if (exact) return exact const stem = id.replace(/\.(safetensors|sft|ckpt|pt|bin)$/i, '') const prefix = available.find(name => modelIdentity(name).startsWith(stem)) if (prefix) return prefix } return '' } export function configuredKreaConceptLora() { const config = useRuntimeConfig() return String( config.kreaConceptLora || (config.public as { kreaConceptLora?: string } | undefined)?.kreaConceptLora || IMAGE_V2_KREA_CONCEPT_LORA || '' ).trim() } export async function resolveKreaGenerateAssets() { const [unets, clips, vaes, clipInfo] = await Promise.all([ listComfyModels('diffusion_models'), listComfyModels('text_encoders').then(async (names) => names.length ? names : listComfyModels('clip')), listComfyModels('vae'), imageComfyFetch('/object_info/CLIPLoader', { signal: AbortSignal.timeout(8000) }) .then(async (res) => res.ok ? res.json() : null) .catch(() => null) ]) const unet = pickComfyModel(unets, [...IMAGE_V2_KREA_UNET_CANDIDATES]) if (!unet) { throw createError({ statusCode: 503, statusMessage: 'Krea diffusion model is missing on Beast Comfy (need krea2_turbo_mxfp8, nvfp4, or fp8_scaled).' }) } const clip = pickComfyModel(clips, [IMAGE_V2_KREA_CLIP]) if (!clip) { throw createError({ statusCode: 503, statusMessage: `Krea text encoder is missing on Beast Comfy (${IMAGE_V2_KREA_CLIP}).` }) } const vae = pickComfyModel(vaes, [IMAGE_V2_KREA_VAE]) if (!vae) { throw createError({ statusCode: 503, statusMessage: `Qwen image VAE is missing on Beast Comfy (${IMAGE_V2_KREA_VAE}).` }) } const rec = clipInfo && typeof clipInfo === 'object' ? clipInfo as Record : {} const body = (rec.CLIPLoader && typeof rec.CLIPLoader === 'object' ? rec.CLIPLoader : rec) as { input?: { required?: { type?: unknown } } } const typeSpec = body.input?.required?.type const types = Array.isArray(typeSpec) && Array.isArray(typeSpec[0]) ? (typeSpec[0] as unknown[]).map(item => String(item || '')) : [] if (types.length && !types.includes(IMAGE_V2_KREA_CLIP_TYPE)) { throw createError({ statusCode: 503, statusMessage: 'Beast Comfy CLIPLoader has no krea2 type. Update Comfy before using Krea.' }) } return { unet, clip, vae, conceptLora: configuredKreaConceptLora() } } function graphFileNames(graph: WorkflowGraph) { const names: string[] = [] for (const node of Object.values(graph)) { for (const key of ['unet_name', 'clip_name', 'vae_name', 'lora_name']) { const value = node.inputs[key] if (typeof value === 'string' && value.trim()) names.push(value) } } return names } function assertKreaShared(graph: WorkflowGraph) { if (Object.values(graph).some(node => node.class_type === 'EmptyFlux2LatentImage' || node.class_type === 'Flux2Scheduler' || node.class_type === 'ReferenceLatent')) { throw createError({ statusCode: 500, statusMessage: 'Krea graph contains a Klein latent, scheduler, or reference node. Refusing to run.' }) } const clip = graph['5'] if (!clip || clip.class_type !== 'CLIPLoader' || String(clip.inputs.type || '') !== IMAGE_V2_KREA_CLIP_TYPE) { throw createError({ statusCode: 500, statusMessage: 'Krea graph CLIPLoader type must be krea2.' }) } const sampler = graph['15'] if (!sampler || sampler.class_type !== 'KSampler') { throw createError({ statusCode: 500, statusMessage: 'Krea graph is missing KSampler.' }) } const banned = graphFileNames(graph).filter((name) => { const id = modelIdentity(name) return /flux-?2-klein|klein_snofs|flux2-klein-9b-consistency|qwen_3_8b|full_encoder_small_decoder/i.test(id) }) if (banned.length) { throw createError({ statusCode: 500, statusMessage: 'Krea graph includes a Klein model or LoRA. Refusing to run.' }) } } function assertKreaGenerateGraph(graph: WorkflowGraph) { const loaders = loadImageNames(graph) if (loaders.length) { throw createError({ statusCode: 500, statusMessage: 'Krea generate graph has a required LoadImage. Refusing to run.' }) } const latent = Object.values(graph).find(node => node.class_type === 'EmptyLatentImage') if (!latent) { throw createError({ statusCode: 500, statusMessage: 'Krea generate graph is missing EmptyLatentImage.' }) } assertKreaShared(graph) } function assertKreaItiGraph(graph: WorkflowGraph, mode: ImageV2Mode) { const loaders = loadImageNames(graph) if (!loaders.length) { throw createError({ statusCode: 500, statusMessage: 'Krea image-to-image graph is missing LoadImage.' }) } if (!Object.values(graph).some(node => node.class_type === 'VAEEncode')) { throw createError({ statusCode: 500, statusMessage: 'Krea image-to-image graph is missing VAEEncode.' }) } if (mode === 'compose' && !Object.values(graph).some(node => node.class_type === 'ImageStitch')) { throw createError({ statusCode: 500, statusMessage: 'Krea compose graph is missing ImageStitch. Refusing to drop still B.' }) } assertKreaShared(graph) } export function assertImageV2Graph(graph: WorkflowGraph, mode: ImageV2Mode, imageBName?: string, engine: ImageV2Engine = 'flux') { const loaders = loadImageNames(graph) if (mode === 'generate' && engine === 'krea') { assertKreaGenerateGraph(graph) } else if (engine === 'krea') { assertKreaItiGraph(graph, mode) } else if (mode === 'generate') { if (loaders.length) { throw createError({ statusCode: 500, statusMessage: 'Generate graph has a required LoadImage. Refusing to run an edit fallback.' }) } const latent = Object.values(graph).find(node => node.class_type === 'EmptyFlux2LatentImage') if (!latent) { throw createError({ statusCode: 500, statusMessage: 'Generate graph is missing EmptyFlux2LatentImage.' }) } } if (mode === 'refine') { const mask = graph[LOAD_MASK] if (!mask || mask.class_type !== 'LoadImage' || !String(mask.inputs.image || '').trim()) { throw createError({ statusCode: 500, statusMessage: 'Refine job is missing the mask image. Refusing to run.' }) } if (!graphHasMaskInput(graph)) { throw createError({ statusCode: 500, statusMessage: 'Refine graph has no mask input. Refusing to run.' }) } if (!('denoise' in (graph[SCHEDULER]?.inputs || {})) && !('denoise' in (graph['15']?.inputs || {}))) { throw createError({ statusCode: 500, statusMessage: 'Refine graph has no denoise on the sampler. Refusing to run.' }) } } if (mode === 'compose') { if (loaders.length < 2) { throw createError({ statusCode: 500, statusMessage: 'Compose graph has no second image input. Refusing to run a one-image fallback.' }) } const b = graph[LOAD_B] if (!b || b.class_type !== 'LoadImage' || !String(b.inputs.image || '').trim()) { throw createError({ statusCode: 500, statusMessage: 'Compose job is missing Load Image B. Refusing to run.' }) } } if (mode !== 'refine' && mode !== 'generate' && imageBName && loaders.length < 2) { throw createError({ statusCode: 500, statusMessage: 'image_b was sent but the executed graph has no second image input.' }) } const promptNode = graph[PROMPT] if (!promptNode || promptNode.class_type !== 'CLIPTextEncode') { throw createError({ statusCode: 500, statusMessage: 'v2 graph is missing the positive CLIPTextEncode node.' }) } if (Array.isArray(promptNode.inputs.text)) { throw createError({ statusCode: 500, statusMessage: 'v2 prompt is a subgraph link. Refusing to run with a leftover widget prompt.' }) } if (!String(promptNode.inputs.text || '').trim()) { throw createError({ statusCode: 400, statusMessage: 'v2 prompt was not patched onto the graph.' }) } } function imageV2UserStack(params: ImageV2BuildParams, engine: ImageV2Engine) { return mergeLegacyImageV2Loras(params.loraStack, { engine, mode: params.mode, xaigen: isXaigenStudio(), snofsModel: params.snofsModel, snofsClip: params.snofsClip, consistencyModel: params.consistencyModel, consistencyClip: params.consistencyClip, kreaConcept: params.kreaConceptLora }) } function requireKreaModels(params: ImageV2BuildParams) { if (!params.kreaUnetName || !params.kreaClipName || !params.kreaVaeName) { throw createError({ statusCode: 503, statusMessage: 'Krea models were not resolved. Refusing to fall back to Flux.' }) } } function buildKreaItiWorkflow(params: ImageV2BuildParams) { requireKreaModels(params) const refine = params.mode === 'refine' const compose = params.mode === 'compose' if (refine && !String(params.maskName || '').trim()) { throw createError({ statusCode: 400, statusMessage: 'Refine requires a mask. Refusing to fall back to Edit.' }) } if (compose && !String(params.imageBName || '').trim()) { throw createError({ statusCode: 400, statusMessage: 'Compose requires still B. Refusing to fall back to Edit.' }) } const graph = structuredClone( refine ? kreaRefineTemplate : compose ? kreaComposeTemplate : kreaEditTemplate ) as WorkflowGraph const prompt = composeImageV2Prompt(params.mode, params.task, params.prompt) const negative = String(params.negative || '') const steps = clampImageSteps(params.steps, IMAGE_V2_KREA_STEPS) const cfg = clampImageCfg(params.cfg, IMAGE_V2_KREA_CFG) const denoise = clampImageV2Denoise( params.strength, refine ? IMAGE_V2_DENOISE_DEFAULT : compose ? IMAGE_V2_KREA_COMPOSE_DENOISE : IMAGE_V2_KREA_EDIT_DENOISE ) const workflowFile = imageV2WorkflowFile(params.mode, 'krea') setInput(graph, LOAD_A, 'image', params.imageAName || '') if (compose) setInput(graph, LOAD_B, 'image', params.imageBName || '') if (refine) setInput(graph, LOAD_MASK, 'image', params.maskName || '') setInput(graph, '4', 'unet_name', params.kreaUnetName) setInput(graph, '5', 'clip_name', params.kreaClipName) setInput(graph, '5', 'type', IMAGE_V2_KREA_CLIP_TYPE) setInput(graph, '6', 'vae_name', params.kreaVaeName) setInput(graph, PROMPT, 'text', prompt) setInput(graph, NEGATIVE, 'text', negative) setInput(graph, '15', 'seed', params.seed) setInput(graph, '15', 'steps', steps) setInput(graph, '15', 'cfg', cfg) setInput(graph, '15', 'sampler_name', 'euler') setInput(graph, '15', 'scheduler', 'simple') setInput(graph, '15', 'denoise', denoise) setInput(graph, SAVE, 'filename_prefix', params.filenamePrefix || `v2-krea-${params.mode}`) applyScaleMegapixels(graph, params.megapixels) const userLoras = applyImageV2UserLoras(graph, imageV2UserStack(params, 'krea'), 'krea') assertImageV2Graph(graph, params.mode, compose ? params.imageBName : undefined, 'krea') const loaders = loadImageNames(graph) console.log(JSON.stringify({ src: 'image-v2', engine: 'krea', workflow: workflowFile, mode: params.mode, task: params.task, canvas: { id: LOAD_A, file: graph[LOAD_A]?.inputs.image }, mask: refine ? { id: LOAD_MASK, file: graph[LOAD_MASK]?.inputs.image } : undefined, strength: denoise, loadImage: Object.fromEntries(loaders.map(item => [item.id, { title: item.title, file: item.image }])), loras: { user: userLoras }, steps, cfg, seed: params.seed, megapixels: Number(params.megapixels) > 0 ? clampImageScaleMegapixels(params.megapixels) : 0 })) return { graph, workflowFile, loaders, prompt, strength: denoise } } function buildKreaGenerateWorkflow(params: ImageV2BuildParams) { requireKreaModels(params) const graph = structuredClone(kreaGenerateTemplate) as WorkflowGraph const prompt = composeImageV2Prompt('generate', 't2i', params.prompt) const negative = String(params.negative || '') const size = clampImageV2Size(params.width, params.height, params.aspect) const steps = clampImageSteps(params.steps, IMAGE_V2_KREA_STEPS) const cfg = clampImageCfg(params.cfg, IMAGE_V2_KREA_CFG) const workflowFile = IMAGE_V2_KREA_GENERATE_WORKFLOW setInput(graph, '4', 'unet_name', params.kreaUnetName) setInput(graph, '5', 'clip_name', params.kreaClipName) setInput(graph, '5', 'type', IMAGE_V2_KREA_CLIP_TYPE) setInput(graph, '6', 'vae_name', params.kreaVaeName) setInput(graph, '14', 'width', size.width) setInput(graph, '14', 'height', size.height) setInput(graph, PROMPT, 'text', prompt) setInput(graph, NEGATIVE, 'text', negative) setInput(graph, '15', 'seed', params.seed) setInput(graph, '15', 'steps', steps) setInput(graph, '15', 'cfg', cfg) setInput(graph, '15', 'sampler_name', 'euler') setInput(graph, '15', 'scheduler', 'simple') setInput(graph, SAVE, 'filename_prefix', params.filenamePrefix || 'v2-krea') const userLoras = applyImageV2UserLoras(graph, imageV2UserStack(params, 'krea'), 'krea') assertImageV2Graph(graph, 'generate', undefined, 'krea') const loaders = loadImageNames(graph) console.log(JSON.stringify({ src: 'image-v2', engine: 'krea', workflow: workflowFile, mode: 'generate', task: 't2i', size: { width: size.width, height: size.height }, models: { unet: params.kreaUnetName, clip: params.kreaClipName, vae: params.kreaVaeName }, loras: { user: userLoras }, steps, cfg, seed: params.seed })) return { graph, workflowFile, loaders, prompt, strength: undefined } } export function buildImageV2Workflow(params: ImageV2BuildParams) { const engine = parseImageV2Engine(params.engine, 'flux') if (engine === 'krea') { return params.mode === 'generate' ? buildKreaGenerateWorkflow(params) : buildKreaItiWorkflow(params) } const compose = params.mode === 'compose' const refine = params.mode === 'refine' const generate = params.mode === 'generate' if (refine && !String(params.maskName || '').trim()) { throw createError({ statusCode: 400, statusMessage: 'Refine requires a mask. Refusing to fall back to Edit.' }) } const graph = structuredClone( generate ? generateTemplate : refine ? refineTemplate : compose ? composeTemplate : editTemplate ) as WorkflowGraph const prompt = composeImageV2Prompt(params.mode, params.task, params.prompt) const negative = String(params.negative || '') const steps = clampImageSteps(params.steps, 24) const cfg = clampImageCfg(params.cfg, 4) const strength = refine ? clampImageV2Denoise(params.strength, IMAGE_V2_DENOISE_DEFAULT) : undefined const size = generate ? clampImageV2Size(params.width, params.height, params.aspect) : null const workflowFile = imageV2WorkflowFile(params.mode, engine) if (!generate) setInput(graph, LOAD_A, 'image', params.imageAName || '') if (compose) setInput(graph, LOAD_B, 'image', params.imageBName || '') if (refine) { setInput(graph, LOAD_MASK, 'image', params.maskName || '') setInput(graph, SCHEDULER_DENOISE, 'denoise', strength) } if (generate && size) { setInput(graph, '14', 'width', size.width) setInput(graph, '14', 'height', size.height) setInput(graph, SCHEDULER, 'width', size.width) setInput(graph, SCHEDULER, 'height', size.height) } setInput(graph, PROMPT, 'text', prompt) setInput(graph, NEGATIVE, 'text', negative) setInput(graph, NOISE, 'noise_seed', params.seed) setInput(graph, SCHEDULER, 'steps', steps) setInput(graph, CFG, 'cfg', cfg) setInput(graph, SAVE, 'filename_prefix', params.filenamePrefix || (generate ? 'v2-generate' : 'v2')) if (!generate) applyScaleMegapixels(graph, params.megapixels) const userLoras = applyImageV2UserLoras(graph, imageV2UserStack(params, 'flux'), 'flux') const guider = Object.values(graph).find(node => node.class_type === 'CFGGuider' || node.class_type === 'BasicGuider') const modelFeed = Array.isArray(guider?.inputs.model) ? String(guider.inputs.model[0] || '') : '' assertImageV2Graph(graph, params.mode, generate || refine ? undefined : params.imageBName, engine) const loaders = loadImageNames(graph) console.log(JSON.stringify({ src: 'image-v2', engine, workflow: workflowFile, mode: params.mode, task: params.task, canvas: generate ? undefined : { id: LOAD_A, file: graph[LOAD_A]?.inputs.image }, mask: refine ? { id: LOAD_MASK, file: graph[LOAD_MASK]?.inputs.image } : undefined, size: generate ? { width: size?.width ?? IMAGE_V2_GENERATE_WIDTH, height: size?.height ?? IMAGE_V2_GENERATE_HEIGHT } : undefined, strength, turbo: params.turbo === true, loadImage: Object.fromEntries(loaders.map(item => [item.id, { title: item.title, file: item.image }])), loras: { user: userLoras, modelFeed }, steps, cfg, seed: params.seed, megapixels: generate ? undefined : (Number(params.megapixels) > 0 ? clampImageScaleMegapixels(params.megapixels) : 0) })) return { graph, workflowFile, loaders, prompt, strength } } export const IMAGE_V2_NODE_LABELS: Record = { '1': 'Loading image A', '22': 'Loading image B', '30': 'Loading mask', '2': 'Scaling image A', '23': 'Scaling image B', '27': 'Stitching stills', '31': 'Resizing mask', '4': 'Loading diffusion model', '5': 'Loading CLIP', '6': 'Loading VAE', '7': 'Applying SNOFS', '8': 'Applying Consistency', '70': 'Applying LoRA', '71': 'Applying LoRA', '72': 'Applying LoRA', '73': 'Applying LoRA', '74': 'Applying LoRA', '75': 'Applying LoRA', '76': 'Applying LoRA', '77': 'Applying LoRA', '9': 'Encoding prompt', '11': 'Encoding image A', '24': 'Encoding image B', '33': 'Applying mask', '14': 'Building empty latent', '15': 'Sampling', '19': 'Sampling', '20': 'Decoding still', '21': 'Saving still' }