661 lines
25 KiB
TypeScript
661 lines
25 KiB
TypeScript
import editTemplate from '../assets/klein_v2_edit.json'
|
|
import composeTemplate from '../assets/klein_v2_compose.json'
|
|
import refineTemplate from '../assets/klein_v2_refine.json'
|
|
import generateTemplate from '../assets/klein_v2_generate.json'
|
|
import kreaGenerateTemplate from '../assets/krea_v2_generate.json'
|
|
import kreaEditTemplate from '../assets/krea_v2_edit.json'
|
|
import kreaComposeTemplate from '../assets/krea_v2_compose.json'
|
|
import kreaRefineTemplate from '../assets/krea_v2_refine.json'
|
|
import { IMAGE_SCALE_TO_TOTAL_PIXELS } from '~/server/utils/comfy'
|
|
import { imageComfyFetch } from '~/server/utils/imageComfy'
|
|
import { cachedComfyLoraNames } from '~/server/utils/loras'
|
|
import { resolveComfyLoraName, loraIdentityKey } from '~/utils/loras'
|
|
import { clampImageCfg, clampImageScaleMegapixels, clampImageSteps } from '~/utils/generationPresets'
|
|
import { isXaigenStudio } from '~/utils/videoModels'
|
|
import {
|
|
IMAGE_V2_CONSISTENCY_CLIP,
|
|
IMAGE_V2_CONSISTENCY_LORA,
|
|
IMAGE_V2_CONSISTENCY_MODEL,
|
|
IMAGE_V2_KREA_CFG,
|
|
IMAGE_V2_KREA_CLIP,
|
|
IMAGE_V2_KREA_CLIP_TYPE,
|
|
IMAGE_V2_KREA_COMPOSE_DENOISE,
|
|
IMAGE_V2_KREA_CONCEPT_LORA,
|
|
IMAGE_V2_KREA_EDIT_DENOISE,
|
|
IMAGE_V2_KREA_STEPS,
|
|
IMAGE_V2_KREA_UNET_CANDIDATES,
|
|
IMAGE_V2_KREA_VAE,
|
|
IMAGE_V2_SNOFS_CLIP,
|
|
IMAGE_V2_SNOFS_LORA,
|
|
IMAGE_V2_SNOFS_MODEL,
|
|
IMAGE_V2_DENOISE_DEFAULT,
|
|
IMAGE_V2_GENERATE_HEIGHT,
|
|
IMAGE_V2_GENERATE_WIDTH,
|
|
clampImageV2Denoise,
|
|
clampImageV2Size,
|
|
clampImageV2Strength,
|
|
composeImageV2Prompt,
|
|
imageV2WorkflowFile,
|
|
parseImageV2Engine,
|
|
type ImageV2Engine,
|
|
type ImageV2Mode,
|
|
type ImageV2Task
|
|
} from '~/utils/imageV2'
|
|
|
|
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
|
|
type WorkflowGraph = Record<string, WorkflowNode>
|
|
|
|
const LOAD_A = '1'
|
|
const LOAD_B = '22'
|
|
const LOAD_MASK = '30'
|
|
const SCHEDULER_DENOISE = '17'
|
|
const SCALE_A = '2'
|
|
const SCALE_B = '23'
|
|
const PROMPT = '9'
|
|
const NEGATIVE = '10'
|
|
const NOISE = '15'
|
|
const SCHEDULER = '17'
|
|
const CFG = '18'
|
|
const SAVE = '21'
|
|
const UNET = '4'
|
|
const CLIP = '5'
|
|
const SNOFS = '7'
|
|
const CONSISTENCY = '8'
|
|
|
|
export const IMAGE_V2_EDIT_WORKFLOW = 'klein_v2_edit.json'
|
|
export const IMAGE_V2_COMPOSE_WORKFLOW = 'klein_v2_compose.json'
|
|
export const IMAGE_V2_REFINE_WORKFLOW = 'klein_v2_refine.json'
|
|
export const IMAGE_V2_GENERATE_WORKFLOW = 'klein_v2_generate.json'
|
|
export const IMAGE_V2_KREA_GENERATE_WORKFLOW = 'krea_v2_generate.json'
|
|
export const IMAGE_V2_KREA_EDIT_WORKFLOW = 'krea_v2_edit.json'
|
|
export const IMAGE_V2_KREA_COMPOSE_WORKFLOW = 'krea_v2_compose.json'
|
|
export const IMAGE_V2_KREA_REFINE_WORKFLOW = 'krea_v2_refine.json'
|
|
|
|
export interface ImageV2BuildParams {
|
|
mode: ImageV2Mode
|
|
task: ImageV2Task
|
|
engine?: ImageV2Engine
|
|
prompt: string
|
|
negative?: string
|
|
imageAName?: string
|
|
imageBName?: string
|
|
maskName?: string
|
|
strength?: number
|
|
width?: number
|
|
height?: number
|
|
turbo?: boolean
|
|
snofsModel?: number
|
|
snofsClip?: number
|
|
consistencyModel?: number
|
|
consistencyClip?: number
|
|
steps: number
|
|
cfg: number
|
|
seed: number
|
|
megapixels?: number
|
|
filenamePrefix?: string
|
|
kreaUnetName?: string
|
|
kreaClipName?: string
|
|
kreaVaeName?: string
|
|
kreaConceptLora?: string
|
|
}
|
|
|
|
function setInput(graph: WorkflowGraph, id: string, key: string, value: unknown) {
|
|
if (graph[id]) graph[id].inputs[key] = value
|
|
}
|
|
|
|
function loadImageNames(graph: WorkflowGraph) {
|
|
return Object.entries(graph)
|
|
.filter(([, node]) => node.class_type === 'LoadImage')
|
|
.map(([id, node]) => ({
|
|
id,
|
|
title: String(node._meta?.title || id),
|
|
image: String(node.inputs.image || '')
|
|
}))
|
|
}
|
|
|
|
function resolveRequiredLora(wanted: string, label: string) {
|
|
const names = cachedComfyLoraNames('image')
|
|
const resolved = names.length ? resolveComfyLoraName(wanted, names) : wanted
|
|
const hit = names.some(name => loraIdentityKey(name) === loraIdentityKey(wanted) || loraIdentityKey(name) === loraIdentityKey(resolved))
|
|
if (names.length && !hit) {
|
|
throw createError({ statusCode: 503, statusMessage: `Missing ${label} LoRA (${wanted}) on Beast Comfy` })
|
|
}
|
|
return resolved
|
|
}
|
|
|
|
function patchScaleMegapixels(graph: WorkflowGraph, megapixels: number) {
|
|
const mp = clampImageScaleMegapixels(megapixels)
|
|
for (const node of Object.values(graph)) {
|
|
if (node.class_type !== IMAGE_SCALE_TO_TOTAL_PIXELS) continue
|
|
node.inputs.megapixels = mp
|
|
node.inputs.upscale_method = 'lanczos'
|
|
}
|
|
}
|
|
|
|
function graphHasMaskInput(graph: WorkflowGraph) {
|
|
return Object.values(graph).some((node) => {
|
|
const mask = node.inputs?.mask
|
|
return mask !== undefined && mask !== null && mask !== ''
|
|
})
|
|
}
|
|
|
|
function bypassLoraNode(graph: WorkflowGraph, id: string, modelFrom: string, clipFrom: string) {
|
|
delete graph[id]
|
|
for (const node of Object.values(graph)) {
|
|
for (const [key, value] of Object.entries(node.inputs)) {
|
|
if (!Array.isArray(value) || value[0] !== id) continue
|
|
node.inputs[key] = value[1] === 1 ? [clipFrom, 1] : [modelFrom, 0]
|
|
}
|
|
}
|
|
}
|
|
|
|
function bypassLoraToBaseLoaders(graph: WorkflowGraph, id: string) {
|
|
delete graph[id]
|
|
for (const node of Object.values(graph)) {
|
|
for (const [key, value] of Object.entries(node.inputs)) {
|
|
if (!Array.isArray(value) || value[0] !== id) continue
|
|
node.inputs[key] = value[1] === 1 ? [CLIP, 0] : [UNET, 0]
|
|
}
|
|
}
|
|
}
|
|
|
|
function modelBasename(name: string) {
|
|
return String(name || '').replace(/\\/g, '/').split('/').pop() || ''
|
|
}
|
|
|
|
function modelIdentity(name: string) {
|
|
return modelBasename(name).toLowerCase()
|
|
}
|
|
|
|
async function listComfyModels(folder: string) {
|
|
const res = await imageComfyFetch(`/models/${encodeURIComponent(folder)}`, {
|
|
signal: AbortSignal.timeout(8000)
|
|
}).catch(() => null)
|
|
if (!res || !res.ok) return [] as string[]
|
|
const payload = await res.json().catch(() => null)
|
|
if (Array.isArray(payload)) return payload.map(item => String(item || '')).filter(Boolean)
|
|
return [] as string[]
|
|
}
|
|
|
|
function pickComfyModel(available: string[], candidates: string[]) {
|
|
for (const wanted of candidates) {
|
|
const id = modelIdentity(wanted)
|
|
const exact = available.find(name => modelIdentity(name) === id)
|
|
if (exact) return exact
|
|
const stem = id.replace(/\.(safetensors|sft|ckpt|pt|bin)$/i, '')
|
|
const prefix = available.find(name => modelIdentity(name).startsWith(stem))
|
|
if (prefix) return prefix
|
|
}
|
|
return ''
|
|
}
|
|
|
|
export function configuredKreaConceptLora() {
|
|
const config = useRuntimeConfig()
|
|
return String(
|
|
config.kreaConceptLora
|
|
|| (config.public as { kreaConceptLora?: string } | undefined)?.kreaConceptLora
|
|
|| IMAGE_V2_KREA_CONCEPT_LORA
|
|
|| ''
|
|
).trim()
|
|
}
|
|
|
|
export async function resolveKreaGenerateAssets() {
|
|
const [unets, clips, vaes, clipInfo] = await Promise.all([
|
|
listComfyModels('diffusion_models'),
|
|
listComfyModels('text_encoders').then(async (names) => names.length ? names : listComfyModels('clip')),
|
|
listComfyModels('vae'),
|
|
imageComfyFetch('/object_info/CLIPLoader', { signal: AbortSignal.timeout(8000) })
|
|
.then(async (res) => res.ok ? res.json() : null)
|
|
.catch(() => null)
|
|
])
|
|
const unet = pickComfyModel(unets, [...IMAGE_V2_KREA_UNET_CANDIDATES])
|
|
if (!unet) {
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: 'Krea diffusion model is missing on Beast Comfy (need krea2_turbo_mxfp8, nvfp4, or fp8_scaled).'
|
|
})
|
|
}
|
|
const clip = pickComfyModel(clips, [IMAGE_V2_KREA_CLIP])
|
|
if (!clip) {
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: `Krea text encoder is missing on Beast Comfy (${IMAGE_V2_KREA_CLIP}).`
|
|
})
|
|
}
|
|
const vae = pickComfyModel(vaes, [IMAGE_V2_KREA_VAE])
|
|
if (!vae) {
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: `Qwen image VAE is missing on Beast Comfy (${IMAGE_V2_KREA_VAE}).`
|
|
})
|
|
}
|
|
const rec = clipInfo && typeof clipInfo === 'object' ? clipInfo as Record<string, unknown> : {}
|
|
const body = (rec.CLIPLoader && typeof rec.CLIPLoader === 'object' ? rec.CLIPLoader : rec) as {
|
|
input?: { required?: { type?: unknown } }
|
|
}
|
|
const typeSpec = body.input?.required?.type
|
|
const types = Array.isArray(typeSpec) && Array.isArray(typeSpec[0])
|
|
? (typeSpec[0] as unknown[]).map(item => String(item || ''))
|
|
: []
|
|
if (types.length && !types.includes(IMAGE_V2_KREA_CLIP_TYPE)) {
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: 'Beast Comfy CLIPLoader has no krea2 type. Update Comfy before using Krea.'
|
|
})
|
|
}
|
|
return { unet, clip, vae, conceptLora: configuredKreaConceptLora() }
|
|
}
|
|
|
|
function graphFileNames(graph: WorkflowGraph) {
|
|
const names: string[] = []
|
|
for (const node of Object.values(graph)) {
|
|
for (const key of ['unet_name', 'clip_name', 'vae_name', 'lora_name']) {
|
|
const value = node.inputs[key]
|
|
if (typeof value === 'string' && value.trim()) names.push(value)
|
|
}
|
|
}
|
|
return names
|
|
}
|
|
|
|
function assertKreaShared(graph: WorkflowGraph) {
|
|
if (Object.values(graph).some(node => node.class_type === 'EmptyFlux2LatentImage' || node.class_type === 'Flux2Scheduler' || node.class_type === 'ReferenceLatent')) {
|
|
throw createError({ statusCode: 500, statusMessage: 'Krea graph contains a Klein latent, scheduler, or reference node. Refusing to run.' })
|
|
}
|
|
const clip = graph['5']
|
|
if (!clip || clip.class_type !== 'CLIPLoader' || String(clip.inputs.type || '') !== IMAGE_V2_KREA_CLIP_TYPE) {
|
|
throw createError({ statusCode: 500, statusMessage: 'Krea graph CLIPLoader type must be krea2.' })
|
|
}
|
|
const sampler = graph['15']
|
|
if (!sampler || sampler.class_type !== 'KSampler') {
|
|
throw createError({ statusCode: 500, statusMessage: 'Krea graph is missing KSampler.' })
|
|
}
|
|
const banned = graphFileNames(graph).filter((name) => {
|
|
const id = modelIdentity(name)
|
|
return /flux-?2-klein|klein_snofs|flux2-klein-9b-consistency|qwen_3_8b|full_encoder_small_decoder/i.test(id)
|
|
})
|
|
if (banned.length) {
|
|
throw createError({
|
|
statusCode: 500,
|
|
statusMessage: 'Krea graph includes a Klein model or LoRA. Refusing to run.'
|
|
})
|
|
}
|
|
}
|
|
|
|
function assertKreaGenerateGraph(graph: WorkflowGraph) {
|
|
const loaders = loadImageNames(graph)
|
|
if (loaders.length) {
|
|
throw createError({
|
|
statusCode: 500,
|
|
statusMessage: 'Krea generate graph has a required LoadImage. Refusing to run.'
|
|
})
|
|
}
|
|
const latent = Object.values(graph).find(node => node.class_type === 'EmptyLatentImage')
|
|
if (!latent) {
|
|
throw createError({ statusCode: 500, statusMessage: 'Krea generate graph is missing EmptyLatentImage.' })
|
|
}
|
|
assertKreaShared(graph)
|
|
}
|
|
|
|
function assertKreaItiGraph(graph: WorkflowGraph, mode: ImageV2Mode) {
|
|
const loaders = loadImageNames(graph)
|
|
if (!loaders.length) {
|
|
throw createError({ statusCode: 500, statusMessage: 'Krea image-to-image graph is missing LoadImage.' })
|
|
}
|
|
if (!Object.values(graph).some(node => node.class_type === 'VAEEncode')) {
|
|
throw createError({ statusCode: 500, statusMessage: 'Krea image-to-image graph is missing VAEEncode.' })
|
|
}
|
|
if (mode === 'compose' && !Object.values(graph).some(node => node.class_type === 'ImageStitch')) {
|
|
throw createError({ statusCode: 500, statusMessage: 'Krea compose graph is missing ImageStitch. Refusing to drop still B.' })
|
|
}
|
|
assertKreaShared(graph)
|
|
}
|
|
|
|
export function assertImageV2Graph(graph: WorkflowGraph, mode: ImageV2Mode, imageBName?: string, engine: ImageV2Engine = 'flux') {
|
|
const loaders = loadImageNames(graph)
|
|
if (mode === 'generate' && engine === 'krea') {
|
|
assertKreaGenerateGraph(graph)
|
|
} else if (engine === 'krea') {
|
|
assertKreaItiGraph(graph, mode)
|
|
} else if (mode === 'generate') {
|
|
if (loaders.length) {
|
|
throw createError({
|
|
statusCode: 500,
|
|
statusMessage: 'Generate graph has a required LoadImage. Refusing to run an edit fallback.'
|
|
})
|
|
}
|
|
const latent = Object.values(graph).find(node => node.class_type === 'EmptyFlux2LatentImage')
|
|
if (!latent) {
|
|
throw createError({ statusCode: 500, statusMessage: 'Generate graph is missing EmptyFlux2LatentImage.' })
|
|
}
|
|
}
|
|
if (mode === 'refine') {
|
|
const mask = graph[LOAD_MASK]
|
|
if (!mask || mask.class_type !== 'LoadImage' || !String(mask.inputs.image || '').trim()) {
|
|
throw createError({
|
|
statusCode: 500,
|
|
statusMessage: 'Refine job is missing the mask image. Refusing to run.'
|
|
})
|
|
}
|
|
if (!graphHasMaskInput(graph)) {
|
|
throw createError({
|
|
statusCode: 500,
|
|
statusMessage: 'Refine graph has no mask input. Refusing to run.'
|
|
})
|
|
}
|
|
if (!('denoise' in (graph[SCHEDULER]?.inputs || {})) && !('denoise' in (graph['15']?.inputs || {}))) {
|
|
throw createError({
|
|
statusCode: 500,
|
|
statusMessage: 'Refine graph has no denoise on the sampler. Refusing to run.'
|
|
})
|
|
}
|
|
}
|
|
if (mode === 'compose') {
|
|
if (loaders.length < 2) {
|
|
throw createError({
|
|
statusCode: 500,
|
|
statusMessage: 'Compose graph has no second image input. Refusing to run a one-image fallback.'
|
|
})
|
|
}
|
|
const b = graph[LOAD_B]
|
|
if (!b || b.class_type !== 'LoadImage' || !String(b.inputs.image || '').trim()) {
|
|
throw createError({
|
|
statusCode: 500,
|
|
statusMessage: 'Compose job is missing Load Image B. Refusing to run.'
|
|
})
|
|
}
|
|
}
|
|
if (mode !== 'refine' && mode !== 'generate' && imageBName && loaders.length < 2) {
|
|
throw createError({
|
|
statusCode: 500,
|
|
statusMessage: 'image_b was sent but the executed graph has no second image input.'
|
|
})
|
|
}
|
|
const promptNode = graph[PROMPT]
|
|
if (!promptNode || promptNode.class_type !== 'CLIPTextEncode') {
|
|
throw createError({ statusCode: 500, statusMessage: 'v2 graph is missing the positive CLIPTextEncode node.' })
|
|
}
|
|
if (Array.isArray(promptNode.inputs.text)) {
|
|
throw createError({ statusCode: 500, statusMessage: 'v2 prompt is a subgraph link. Refusing to run with a leftover widget prompt.' })
|
|
}
|
|
if (!String(promptNode.inputs.text || '').trim()) {
|
|
throw createError({ statusCode: 400, statusMessage: 'v2 prompt was not patched onto the graph.' })
|
|
}
|
|
}
|
|
|
|
function applyKreaConceptLora(graph: WorkflowGraph, loraName: string, model: number, clip: number) {
|
|
graph['7'] = {
|
|
class_type: 'LoraLoader',
|
|
inputs: {
|
|
lora_name: loraName,
|
|
strength_model: model,
|
|
strength_clip: clip,
|
|
model: ['4', 0],
|
|
clip: ['5', 0]
|
|
},
|
|
_meta: { title: 'Concept LoRA' }
|
|
}
|
|
setInput(graph, PROMPT, 'clip', ['7', 1])
|
|
setInput(graph, NEGATIVE, 'clip', ['7', 1])
|
|
setInput(graph, '15', 'model', ['7', 0])
|
|
}
|
|
|
|
function applyKreaConceptIfNeeded(graph: WorkflowGraph, params: ImageV2BuildParams) {
|
|
const conceptName = isXaigenStudio() ? String(params.kreaConceptLora || '').trim() : ''
|
|
const snofsModel = conceptName ? clampImageV2Strength(params.snofsModel, 0) : 0
|
|
const snofsClip = conceptName ? clampImageV2Strength(params.snofsClip, 0) : 0
|
|
if (!conceptName || (snofsModel <= 0 && snofsClip <= 0)) return { snofsModel: 0, snofsClip: 0 }
|
|
const resolved = resolveRequiredLora(conceptName, 'Krea concept')
|
|
if (/klein_snofs|flux2-klein-9b-consistency|flux-?2-klein/i.test(loraIdentityKey(resolved))) {
|
|
throw createError({ statusCode: 500, statusMessage: 'Refusing to load a Klein LoRA into the Krea graph.' })
|
|
}
|
|
applyKreaConceptLora(graph, resolved, snofsModel, snofsClip)
|
|
return { snofsModel, snofsClip, name: resolved }
|
|
}
|
|
|
|
function requireKreaModels(params: ImageV2BuildParams) {
|
|
if (!params.kreaUnetName || !params.kreaClipName || !params.kreaVaeName) {
|
|
throw createError({
|
|
statusCode: 503,
|
|
statusMessage: 'Krea models were not resolved. Refusing to fall back to Flux.'
|
|
})
|
|
}
|
|
}
|
|
|
|
function buildKreaItiWorkflow(params: ImageV2BuildParams) {
|
|
requireKreaModels(params)
|
|
const refine = params.mode === 'refine'
|
|
const compose = params.mode === 'compose'
|
|
if (refine && !String(params.maskName || '').trim()) {
|
|
throw createError({ statusCode: 400, statusMessage: 'Refine requires a mask. Refusing to fall back to Edit.' })
|
|
}
|
|
if (compose && !String(params.imageBName || '').trim()) {
|
|
throw createError({ statusCode: 400, statusMessage: 'Compose requires still B. Refusing to fall back to Edit.' })
|
|
}
|
|
const graph = structuredClone(
|
|
refine ? kreaRefineTemplate : compose ? kreaComposeTemplate : kreaEditTemplate
|
|
) as WorkflowGraph
|
|
const prompt = composeImageV2Prompt(params.mode, params.task, params.prompt)
|
|
const negative = String(params.negative || '')
|
|
const steps = clampImageSteps(params.steps, IMAGE_V2_KREA_STEPS)
|
|
const cfg = clampImageCfg(params.cfg, IMAGE_V2_KREA_CFG)
|
|
const denoise = clampImageV2Denoise(
|
|
params.strength,
|
|
refine ? IMAGE_V2_DENOISE_DEFAULT : compose ? IMAGE_V2_KREA_COMPOSE_DENOISE : IMAGE_V2_KREA_EDIT_DENOISE
|
|
)
|
|
const workflowFile = imageV2WorkflowFile(params.mode, 'krea')
|
|
|
|
setInput(graph, LOAD_A, 'image', params.imageAName || '')
|
|
if (compose) setInput(graph, LOAD_B, 'image', params.imageBName || '')
|
|
if (refine) setInput(graph, LOAD_MASK, 'image', params.maskName || '')
|
|
setInput(graph, '4', 'unet_name', params.kreaUnetName)
|
|
setInput(graph, '5', 'clip_name', params.kreaClipName)
|
|
setInput(graph, '5', 'type', IMAGE_V2_KREA_CLIP_TYPE)
|
|
setInput(graph, '6', 'vae_name', params.kreaVaeName)
|
|
setInput(graph, PROMPT, 'text', prompt)
|
|
setInput(graph, NEGATIVE, 'text', negative)
|
|
setInput(graph, '15', 'seed', params.seed)
|
|
setInput(graph, '15', 'steps', steps)
|
|
setInput(graph, '15', 'cfg', cfg)
|
|
setInput(graph, '15', 'sampler_name', 'euler')
|
|
setInput(graph, '15', 'scheduler', 'simple')
|
|
setInput(graph, '15', 'denoise', denoise)
|
|
setInput(graph, SAVE, 'filename_prefix', params.filenamePrefix || `aigen-v2-krea-${params.mode}`)
|
|
patchScaleMegapixels(graph, params.megapixels ?? 1)
|
|
const concept = applyKreaConceptIfNeeded(graph, params)
|
|
|
|
assertImageV2Graph(graph, params.mode, compose ? params.imageBName : undefined, 'krea')
|
|
const loaders = loadImageNames(graph)
|
|
console.log(JSON.stringify({
|
|
src: 'image-v2',
|
|
engine: 'krea',
|
|
workflow: workflowFile,
|
|
mode: params.mode,
|
|
task: params.task,
|
|
canvas: { id: LOAD_A, file: graph[LOAD_A]?.inputs.image },
|
|
mask: refine ? { id: LOAD_MASK, file: graph[LOAD_MASK]?.inputs.image } : undefined,
|
|
strength: denoise,
|
|
loadImage: Object.fromEntries(loaders.map(item => [item.id, { title: item.title, file: item.image }])),
|
|
loras: concept.name ? { concept: { name: concept.name, model: concept.snofsModel, clip: concept.snofsClip } } : { loaded: false },
|
|
steps,
|
|
cfg,
|
|
seed: params.seed,
|
|
megapixels: clampImageScaleMegapixels(params.megapixels ?? 1)
|
|
}))
|
|
return { graph, workflowFile, loaders, prompt, strength: denoise }
|
|
}
|
|
|
|
function buildKreaGenerateWorkflow(params: ImageV2BuildParams) {
|
|
requireKreaModels(params)
|
|
const graph = structuredClone(kreaGenerateTemplate) as WorkflowGraph
|
|
const prompt = composeImageV2Prompt('generate', 't2i', params.prompt)
|
|
const negative = String(params.negative || '')
|
|
const size = clampImageV2Size(params.width, params.height)
|
|
const steps = clampImageSteps(params.steps, IMAGE_V2_KREA_STEPS)
|
|
const cfg = clampImageCfg(params.cfg, IMAGE_V2_KREA_CFG)
|
|
const workflowFile = IMAGE_V2_KREA_GENERATE_WORKFLOW
|
|
|
|
setInput(graph, '4', 'unet_name', params.kreaUnetName)
|
|
setInput(graph, '5', 'clip_name', params.kreaClipName)
|
|
setInput(graph, '5', 'type', IMAGE_V2_KREA_CLIP_TYPE)
|
|
setInput(graph, '6', 'vae_name', params.kreaVaeName)
|
|
setInput(graph, '14', 'width', size.width)
|
|
setInput(graph, '14', 'height', size.height)
|
|
setInput(graph, PROMPT, 'text', prompt)
|
|
setInput(graph, NEGATIVE, 'text', negative)
|
|
setInput(graph, '15', 'seed', params.seed)
|
|
setInput(graph, '15', 'steps', steps)
|
|
setInput(graph, '15', 'cfg', cfg)
|
|
setInput(graph, '15', 'sampler_name', 'euler')
|
|
setInput(graph, '15', 'scheduler', 'simple')
|
|
setInput(graph, SAVE, 'filename_prefix', params.filenamePrefix || 'aigen-v2-krea')
|
|
const concept = applyKreaConceptIfNeeded(graph, params)
|
|
|
|
assertImageV2Graph(graph, 'generate', undefined, 'krea')
|
|
const loaders = loadImageNames(graph)
|
|
console.log(JSON.stringify({
|
|
src: 'image-v2',
|
|
engine: 'krea',
|
|
workflow: workflowFile,
|
|
mode: 'generate',
|
|
task: 't2i',
|
|
size: { width: size.width, height: size.height },
|
|
models: {
|
|
unet: params.kreaUnetName,
|
|
clip: params.kreaClipName,
|
|
vae: params.kreaVaeName
|
|
},
|
|
loras: {
|
|
concept: concept.name
|
|
? { name: concept.name, model: concept.snofsModel, clip: concept.snofsClip }
|
|
: { loaded: false, model: 0, clip: 0 }
|
|
},
|
|
steps,
|
|
cfg,
|
|
seed: params.seed
|
|
}))
|
|
return { graph, workflowFile, loaders, prompt, strength: undefined }
|
|
}
|
|
|
|
export function buildImageV2Workflow(params: ImageV2BuildParams) {
|
|
const engine = parseImageV2Engine(params.engine, 'flux')
|
|
if (engine === 'krea') {
|
|
return params.mode === 'generate' ? buildKreaGenerateWorkflow(params) : buildKreaItiWorkflow(params)
|
|
}
|
|
const compose = params.mode === 'compose'
|
|
const refine = params.mode === 'refine'
|
|
const generate = params.mode === 'generate'
|
|
if (refine && !String(params.maskName || '').trim()) {
|
|
throw createError({ statusCode: 400, statusMessage: 'Refine requires a mask. Refusing to fall back to Edit.' })
|
|
}
|
|
const graph = structuredClone(
|
|
generate ? generateTemplate : refine ? refineTemplate : compose ? composeTemplate : editTemplate
|
|
) as WorkflowGraph
|
|
const prompt = composeImageV2Prompt(params.mode, params.task, params.prompt)
|
|
const negative = String(params.negative || '')
|
|
const conceptAllowed = isXaigenStudio()
|
|
const snofsModel = conceptAllowed ? clampImageV2Strength(params.snofsModel, IMAGE_V2_SNOFS_MODEL) : 0
|
|
const snofsClip = conceptAllowed ? clampImageV2Strength(params.snofsClip, IMAGE_V2_SNOFS_CLIP) : 0
|
|
const consistencyModel = clampImageV2Strength(
|
|
params.consistencyModel,
|
|
generate ? 0 : IMAGE_V2_CONSISTENCY_MODEL
|
|
)
|
|
const consistencyClip = clampImageV2Strength(
|
|
params.consistencyClip,
|
|
generate ? 0 : IMAGE_V2_CONSISTENCY_CLIP
|
|
)
|
|
const steps = clampImageSteps(params.steps, 24)
|
|
const cfg = clampImageCfg(params.cfg, 4)
|
|
const strength = refine ? clampImageV2Denoise(params.strength, IMAGE_V2_DENOISE_DEFAULT) : undefined
|
|
const size = generate ? clampImageV2Size(params.width, params.height) : null
|
|
const workflowFile = imageV2WorkflowFile(params.mode, engine)
|
|
|
|
if (!generate) setInput(graph, LOAD_A, 'image', params.imageAName || '')
|
|
if (compose) setInput(graph, LOAD_B, 'image', params.imageBName || '')
|
|
if (refine) {
|
|
setInput(graph, LOAD_MASK, 'image', params.maskName || '')
|
|
setInput(graph, SCHEDULER_DENOISE, 'denoise', strength)
|
|
}
|
|
if (generate && size) {
|
|
setInput(graph, '14', 'width', size.width)
|
|
setInput(graph, '14', 'height', size.height)
|
|
setInput(graph, SCHEDULER, 'width', size.width)
|
|
setInput(graph, SCHEDULER, 'height', size.height)
|
|
}
|
|
setInput(graph, PROMPT, 'text', prompt)
|
|
setInput(graph, NEGATIVE, 'text', negative)
|
|
setInput(graph, NOISE, 'noise_seed', params.seed)
|
|
setInput(graph, SCHEDULER, 'steps', steps)
|
|
setInput(graph, CFG, 'cfg', cfg)
|
|
setInput(graph, SAVE, 'filename_prefix', params.filenamePrefix || (generate ? 'aigen-v2-generate' : 'aigen-v2'))
|
|
if (!generate) patchScaleMegapixels(graph, params.megapixels ?? 1)
|
|
|
|
if (conceptAllowed) {
|
|
setInput(graph, SNOFS, 'lora_name', resolveRequiredLora(IMAGE_V2_SNOFS_LORA, 'SNOFS'))
|
|
setInput(graph, SNOFS, 'strength_model', snofsModel)
|
|
setInput(graph, SNOFS, 'strength_clip', snofsClip)
|
|
} else {
|
|
bypassLoraToBaseLoaders(graph, SNOFS)
|
|
}
|
|
if (generate && consistencyModel <= 0 && consistencyClip <= 0) {
|
|
if (graph[SNOFS]) bypassLoraNode(graph, CONSISTENCY, SNOFS, SNOFS)
|
|
else bypassLoraToBaseLoaders(graph, CONSISTENCY)
|
|
} else {
|
|
setInput(graph, CONSISTENCY, 'lora_name', resolveRequiredLora(IMAGE_V2_CONSISTENCY_LORA, 'Consistency'))
|
|
setInput(graph, CONSISTENCY, 'strength_model', consistencyModel)
|
|
setInput(graph, CONSISTENCY, 'strength_clip', consistencyClip)
|
|
}
|
|
|
|
assertImageV2Graph(graph, params.mode, generate || refine ? undefined : params.imageBName, engine)
|
|
|
|
const loaders = loadImageNames(graph)
|
|
console.log(JSON.stringify({
|
|
src: 'image-v2',
|
|
engine,
|
|
workflow: workflowFile,
|
|
mode: params.mode,
|
|
task: params.task,
|
|
canvas: generate ? undefined : { id: LOAD_A, file: graph[LOAD_A]?.inputs.image },
|
|
mask: refine ? { id: LOAD_MASK, file: graph[LOAD_MASK]?.inputs.image } : undefined,
|
|
size: generate ? { width: size?.width ?? IMAGE_V2_GENERATE_WIDTH, height: size?.height ?? IMAGE_V2_GENERATE_HEIGHT } : undefined,
|
|
strength,
|
|
turbo: params.turbo === true,
|
|
loadImage: Object.fromEntries(loaders.map(item => [item.id, { title: item.title, file: item.image }])),
|
|
loras: {
|
|
snofs: { name: graph[SNOFS]?.inputs.lora_name, model: snofsModel, clip: snofsClip },
|
|
consistency: graph[CONSISTENCY]
|
|
? { name: graph[CONSISTENCY]?.inputs.lora_name, model: consistencyModel, clip: consistencyClip }
|
|
: { loaded: false, model: 0, clip: 0 }
|
|
},
|
|
steps,
|
|
cfg,
|
|
seed: params.seed,
|
|
megapixels: generate ? undefined : clampImageScaleMegapixels(params.megapixels ?? 1)
|
|
}))
|
|
|
|
return { graph, workflowFile, loaders, prompt, strength }
|
|
}
|
|
|
|
export const IMAGE_V2_NODE_LABELS: Record<string, string> = {
|
|
'1': 'Loading image A',
|
|
'22': 'Loading image B',
|
|
'30': 'Loading mask',
|
|
'2': 'Scaling image A',
|
|
'23': 'Scaling image B',
|
|
'27': 'Stitching stills',
|
|
'31': 'Resizing mask',
|
|
'4': 'Loading diffusion model',
|
|
'5': 'Loading CLIP',
|
|
'6': 'Loading VAE',
|
|
'7': 'Applying Concept LoRA',
|
|
'8': 'Applying Consistency',
|
|
'9': 'Encoding prompt',
|
|
'11': 'Encoding image A',
|
|
'24': 'Encoding image B',
|
|
'33': 'Applying mask',
|
|
'14': 'Building empty latent',
|
|
'15': 'Sampling',
|
|
'19': 'Sampling',
|
|
'20': 'Decoding still',
|
|
'21': 'Saving still'
|
|
}
|