Add Image v2 as a sibling of Image edit, with separate Klein graphs.

Keep the v1 Image tab on its dual-branch template. The host agent follows Desktop's live port so Coolify stays on :8198.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-08-28 21:22:37 -05:00
co-authored by Cursor
parent 55f8cb1def
commit 21fe2412f9
18 changed files with 1678 additions and 67 deletions
+255
View File
@@ -0,0 +1,255 @@
import { addStudioJob, kickStudioQueue, listStudioJobs, videoJobsBusy } from '~/server/utils/studioQueue'
import { comfyConfigured } from '~/server/utils/comfy'
import { imageDimensions } from '~/server/utils/resolution'
import { clampImageCfg, clampImageScaleMegapixels, clampImageSteps } from '~/utils/generationPresets'
import {
IMAGE_V2_CFG_DEFAULT,
IMAGE_V2_CONSISTENCY_CLIP,
IMAGE_V2_CONSISTENCY_MODEL,
IMAGE_V2_SNOFS_CLIP,
IMAGE_V2_SNOFS_MODEL,
IMAGE_V2_STEPS_DEFAULT,
IMAGE_V2_TURBO_CFG,
IMAGE_V2_TURBO_STEPS,
clampImageV2Strength,
parseImageV2Mode,
parseImageV2Task,
type ImageV2Mode,
type ImageV2Task
} from '~/utils/imageV2'
import { getStill, rememberInputStill, stillPath } from '~/server/utils/library'
import { existsSync, readFileSync } from 'node:fs'
type ImageFile = { filename: string; data: Buffer; type?: string }
function parseBool(raw: unknown) {
return raw === true || raw === 'true' || raw === '1' || raw === 1
}
function isHttpUrl(value: string) {
return /^https?:\/\//i.test(value)
}
async function fileFromUrl(url: string): Promise<ImageFile> {
const res = await fetch(url, { signal: AbortSignal.timeout(20_000) })
if (!res.ok) {
throw createError({ statusCode: 400, statusMessage: `Could not fetch image (${res.status})` })
}
const mime = String(res.headers.get('content-type') || 'image/png').split(';')[0]
if (!/^image\//i.test(mime)) {
throw createError({ statusCode: 400, statusMessage: 'image_a / image_b URL must be an image' })
}
const data = Buffer.from(await res.arrayBuffer())
if (data.length > 40 * 1024 * 1024) {
throw createError({ statusCode: 400, statusMessage: 'Image is too large' })
}
const ext = mime.includes('jpeg') || mime.includes('jpg') ? '.jpg' : mime.includes('webp') ? '.webp' : '.png'
return { filename: `remote${ext}`, data, type: mime }
}
function fileFromStill(ownerKey: string, id: string): ImageFile {
const still = getStill(ownerKey, id)
const path = stillPath(ownerKey, still.id)
if (!existsSync(path)) {
throw createError({ statusCode: 400, statusMessage: 'That still is missing from the library' })
}
return {
filename: still.filename || `${still.id}.png`,
data: readFileSync(path),
type: 'image/png'
}
}
async function resolveImageRef(ownerKey: string, raw: unknown, uploaded: ImageFile | null) {
if (uploaded) return uploaded
const value = String(raw || '').trim()
if (!value || value === 'null') return null
if (isHttpUrl(value)) return fileFromUrl(value)
return fileFromStill(ownerKey, value)
}
function readMultipart(parts: Array<{ name?: string; filename?: string; type?: string; data?: Buffer }> | null) {
const fields: Record<string, string> = {}
let imageA: ImageFile | null = null
let imageB: ImageFile | null = null
for (const part of parts || []) {
if ((part.name === 'image_a' || part.name === 'image') && part.filename && part.data?.length) {
imageA = { filename: part.filename, data: part.data, type: part.type }
} else if ((part.name === 'image_b' || part.name === 'image2') && part.filename && part.data?.length) {
imageB = { filename: part.filename, data: part.data, type: part.type }
} else if (part.name && part.data) {
fields[part.name] = part.data.toString('utf8')
}
}
return { fields, imageA, imageB }
}
export default defineEventHandler(async (event) => {
const contentType = String(getHeader(event, 'content-type') || '')
let fields: Record<string, unknown> = {}
let uploadedA: ImageFile | null = null
let uploadedB: ImageFile | null = null
if (contentType.includes('multipart/form-data')) {
const form = await readMultipartFormData(event).catch(() => null)
const parsed = readMultipart(form || [])
fields = parsed.fields
uploadedA = parsed.imageA
uploadedB = parsed.imageB
} else {
fields = await readBody<Record<string, unknown>>(event).catch(() => ({}))
}
const mode = parseImageV2Mode(fields.mode)
if (!mode) {
throw createError({ statusCode: 400, statusMessage: 'mode must be edit or compose' })
}
const task = parseImageV2Task(fields.task, 'scene')
const prompt = String(fields.prompt || '').trim()
if (!prompt) {
throw createError({ statusCode: 400, statusMessage: 'A prompt is required' })
}
if (!comfyConfigured()) {
throw createError({
statusCode: 503,
statusMessage: 'Beast ComfyUI is not configured. Set COMFY_HOST.'
})
}
if (mode === 'edit' && (task === 'identity' || task === 'outfit' || task === 'face_lock')) {
throw createError({
statusCode: 400,
statusMessage: `${task} requires Compose and image B. Edit is one image only.`
})
}
const ownerKey = libraryOwnerKey(event)
const imageA = await resolveImageRef(ownerKey, fields.image_a, uploadedA)
const imageB = await resolveImageRef(ownerKey, fields.image_b, uploadedB)
if (!imageA) {
throw createError({ statusCode: 400, statusMessage: 'image_a is required' })
}
if (mode === 'edit' && imageB) {
throw createError({
statusCode: 400,
statusMessage: 'Edit mode takes one image. Use Compose for two stills.'
})
}
if (mode === 'compose' && !imageB) {
throw createError({
statusCode: 400,
statusMessage: 'Compose requires image_b. Refusing to fall back to one-image generation.'
})
}
const library = publicLibrary(event)
const folderId = library.folders.some(folder => folder.id === String(fields.folderId || ''))
? String(fields.folderId)
: library.folders[0]?.id
if (!folderId) {
throw createError({ statusCode: 400, statusMessage: 'Create a library folder before generating' })
}
assertFolderExists(event, folderId)
const destFolder = library.folders.find(folder => folder.id === folderId)
const folderLocked = Boolean(destFolder?.protected && !destFolder.unlocked)
const hideInput = parseBool(fields.hideInput)
const hideThumbnail = parseBool(fields.hideThumbnail)
const turbo = parseBool(fields.turbo)
const steps = turbo ? IMAGE_V2_TURBO_STEPS : clampImageSteps(fields.steps, IMAGE_V2_STEPS_DEFAULT)
const cfg = turbo ? IMAGE_V2_TURBO_CFG : clampImageCfg(fields.cfg, IMAGE_V2_CFG_DEFAULT)
const megapixels = clampImageScaleMegapixels(fields.megapixels ?? fields.scaleMegapixels, 1)
const seed = fields.seed && String(fields.seed) !== 'random'
? Number(fields.seed)
: Math.floor(Math.random() * 2_147_483_647)
const size = imageDimensions(imageA.data)
const clipName = String(fields.name || '').trim().slice(0, 80)
const v2Mode = mode as ImageV2Mode
const v2Task = (mode === 'edit' ? 'scene' : task) as ImageV2Task
const still = await rememberInputStill({
ownerKey,
folderId,
filename: imageA.filename,
data: imageA.data,
width: size?.width || 0,
height: size?.height || 0,
hideInput
})
const savedRef = imageB
? await rememberInputStill({
ownerKey,
folderId,
filename: imageB.filename,
data: imageB.data,
hideInput
})
: null
const studio = await addStudioJob({
ownerKey,
familyId: crypto.randomUUID(),
kind: 'edit',
payload: {
prompt,
name: clipName,
folderId,
aspect: 'auto',
width: size?.width || 0,
height: size?.height || 0,
steps,
turbo,
seed,
cfg,
fps: 24,
samplerName: 'euler',
scheduler: 'simple',
duration: 0,
sound: false,
workflow: 'v1',
useIdentityRefs: false,
stillId: still?.id,
stillFilename: still?.filename,
hideThumbnail,
hideInput,
folderLocked,
referenceStillIds: [null, null, null, null],
extensions: [],
queueAutoRun: false,
negative: String(fields.negative || '').trim(),
referenceStillId: savedRef?.id,
referenceStillFilename: savedRef?.filename,
scaleToTotalPixels: true,
scaleMegapixels: megapixels,
imagePipeline: 'v2',
v2Mode,
v2Task,
snofsModel: clampImageV2Strength(fields.snofs_model, IMAGE_V2_SNOFS_MODEL),
snofsClip: clampImageV2Strength(fields.snofs_clip, IMAGE_V2_SNOFS_CLIP),
consistencyModel: clampImageV2Strength(fields.consistency_model, IMAGE_V2_CONSISTENCY_MODEL),
consistencyClip: clampImageV2Strength(fields.consistency_clip, IMAGE_V2_CONSISTENCY_CLIP)
}
})
await kickStudioQueue()
let latest = listStudioJobs(ownerKey).find(item => item.id === studio.id)
if (!latest?.liveJobId && !(await videoJobsBusy())) {
await kickStudioQueue()
latest = listStudioJobs(ownerKey).find(item => item.id === studio.id)
}
const liveJobId = latest?.liveJobId || ''
return {
jobId: liveJobId || studio.id,
studioJobId: studio.id,
queued: !liveJobId,
seed,
steps,
cfg,
mode: v2Mode,
task: v2Task,
workflow: v2Mode === 'compose' ? 'klein_v2_compose.json' : 'klein_v2_edit.json',
hideThumbnail,
folderLocked
}
})
+210
View File
@@ -0,0 +1,210 @@
{
"1": {
"inputs": { "image": "" },
"class_type": "LoadImage",
"_meta": { "title": "Load Image A" }
},
"22": {
"inputs": { "image": "" },
"class_type": "LoadImage",
"_meta": { "title": "Load Image B" }
},
"2": {
"inputs": {
"upscale_method": "lanczos",
"megapixels": 1,
"resolution_steps": 1,
"image": ["1", 0]
},
"class_type": "ImageScaleToTotalPixels",
"_meta": { "title": "Scale Image A" }
},
"23": {
"inputs": {
"upscale_method": "lanczos",
"megapixels": 1,
"resolution_steps": 1,
"image": ["22", 0]
},
"class_type": "ImageScaleToTotalPixels",
"_meta": { "title": "Scale Image B" }
},
"3": {
"inputs": { "image": ["2", 0] },
"class_type": "GetImageSize",
"_meta": { "title": "Get Image Size" }
},
"4": {
"inputs": {
"unet_name": "flux-2-klein-base-9b-fp8.safetensors",
"weight_dtype": "default"
},
"class_type": "UNETLoader",
"_meta": { "title": "Load Flux.2 Klein 9B Base" }
},
"5": {
"inputs": {
"clip_name": "qwen_3_8b_fp8mixed.safetensors",
"type": "flux2",
"device": "default"
},
"class_type": "CLIPLoader",
"_meta": { "title": "Load Qwen 3 8B CLIP" }
},
"6": {
"inputs": { "vae_name": "full_encoder_small_decoder.safetensors" },
"class_type": "VAELoader",
"_meta": { "title": "Load Klein VAE" }
},
"7": {
"inputs": {
"lora_name": "klein_snofs_v1_4.safetensors",
"strength_model": 0.65,
"strength_clip": 0.35,
"model": ["4", 0],
"clip": ["5", 0]
},
"class_type": "LoraLoader",
"_meta": { "title": "SNOFS" }
},
"8": {
"inputs": {
"lora_name": "Flux2-Klein-9B-consistency-V2.safetensors",
"strength_model": 0.7,
"strength_clip": 0.7,
"model": ["7", 0],
"clip": ["7", 1]
},
"class_type": "LoraLoader",
"_meta": { "title": "Consistency" }
},
"9": {
"inputs": {
"text": "",
"clip": ["8", 1]
},
"class_type": "CLIPTextEncode",
"_meta": { "title": "Positive Prompt" }
},
"10": {
"inputs": {
"text": "",
"clip": ["8", 1]
},
"class_type": "CLIPTextEncode",
"_meta": { "title": "Negative Prompt" }
},
"11": {
"inputs": {
"pixels": ["2", 0],
"vae": ["6", 0]
},
"class_type": "VAEEncode",
"_meta": { "title": "VAE Encode A" }
},
"24": {
"inputs": {
"pixels": ["23", 0],
"vae": ["6", 0]
},
"class_type": "VAEEncode",
"_meta": { "title": "VAE Encode B" }
},
"12": {
"inputs": {
"conditioning": ["9", 0],
"latent": ["11", 0]
},
"class_type": "ReferenceLatent",
"_meta": { "title": "Reference Latent A+" }
},
"25": {
"inputs": {
"conditioning": ["12", 0],
"latent": ["24", 0]
},
"class_type": "ReferenceLatent",
"_meta": { "title": "Reference Latent B+" }
},
"13": {
"inputs": {
"conditioning": ["10", 0],
"latent": ["11", 0]
},
"class_type": "ReferenceLatent",
"_meta": { "title": "Reference Latent A-" }
},
"26": {
"inputs": {
"conditioning": ["13", 0],
"latent": ["24", 0]
},
"class_type": "ReferenceLatent",
"_meta": { "title": "Reference Latent B-" }
},
"14": {
"inputs": {
"width": ["3", 0],
"height": ["3", 1],
"batch_size": 1
},
"class_type": "EmptyFlux2LatentImage",
"_meta": { "title": "Empty Flux 2 Latent" }
},
"15": {
"inputs": { "noise_seed": 1 },
"class_type": "RandomNoise",
"_meta": { "title": "RandomNoise" }
},
"16": {
"inputs": { "sampler_name": "euler" },
"class_type": "KSamplerSelect",
"_meta": { "title": "KSamplerSelect" }
},
"17": {
"inputs": {
"steps": 24,
"width": ["3", 0],
"height": ["3", 1]
},
"class_type": "Flux2Scheduler",
"_meta": { "title": "Flux2Scheduler" }
},
"18": {
"inputs": {
"cfg": 4,
"model": ["8", 0],
"positive": ["25", 0],
"negative": ["26", 0]
},
"class_type": "CFGGuider",
"_meta": { "title": "CFG Guider" }
},
"19": {
"inputs": {
"noise": ["15", 0],
"guider": ["18", 0],
"sampler": ["16", 0],
"sigmas": ["17", 0],
"latent_image": ["14", 0]
},
"class_type": "SamplerCustomAdvanced",
"_meta": { "title": "SamplerCustomAdvanced" }
},
"20": {
"inputs": {
"samples": ["19", 0],
"vae": ["6", 0]
},
"class_type": "VAEDecode",
"_meta": { "title": "VAE Decode" }
},
"21": {
"inputs": {
"filename_prefix": "aigen-v2",
"images": ["20", 0]
},
"class_type": "SaveImage",
"_meta": { "title": "Save Image" }
}
}
+171
View File
@@ -0,0 +1,171 @@
{
"1": {
"inputs": { "image": "" },
"class_type": "LoadImage",
"_meta": { "title": "Load Image A" }
},
"2": {
"inputs": {
"upscale_method": "lanczos",
"megapixels": 1,
"resolution_steps": 1,
"image": ["1", 0]
},
"class_type": "ImageScaleToTotalPixels",
"_meta": { "title": "Scale Image A" }
},
"3": {
"inputs": { "image": ["2", 0] },
"class_type": "GetImageSize",
"_meta": { "title": "Get Image Size" }
},
"4": {
"inputs": {
"unet_name": "flux-2-klein-base-9b-fp8.safetensors",
"weight_dtype": "default"
},
"class_type": "UNETLoader",
"_meta": { "title": "Load Flux.2 Klein 9B Base" }
},
"5": {
"inputs": {
"clip_name": "qwen_3_8b_fp8mixed.safetensors",
"type": "flux2",
"device": "default"
},
"class_type": "CLIPLoader",
"_meta": { "title": "Load Qwen 3 8B CLIP" }
},
"6": {
"inputs": { "vae_name": "full_encoder_small_decoder.safetensors" },
"class_type": "VAELoader",
"_meta": { "title": "Load Klein VAE" }
},
"7": {
"inputs": {
"lora_name": "klein_snofs_v1_4.safetensors",
"strength_model": 0.65,
"strength_clip": 0.35,
"model": ["4", 0],
"clip": ["5", 0]
},
"class_type": "LoraLoader",
"_meta": { "title": "SNOFS" }
},
"8": {
"inputs": {
"lora_name": "Flux2-Klein-9B-consistency-V2.safetensors",
"strength_model": 0.7,
"strength_clip": 0.7,
"model": ["7", 0],
"clip": ["7", 1]
},
"class_type": "LoraLoader",
"_meta": { "title": "Consistency" }
},
"9": {
"inputs": {
"text": "",
"clip": ["8", 1]
},
"class_type": "CLIPTextEncode",
"_meta": { "title": "Positive Prompt" }
},
"10": {
"inputs": {
"text": "",
"clip": ["8", 1]
},
"class_type": "CLIPTextEncode",
"_meta": { "title": "Negative Prompt" }
},
"11": {
"inputs": {
"pixels": ["2", 0],
"vae": ["6", 0]
},
"class_type": "VAEEncode",
"_meta": { "title": "VAE Encode A" }
},
"12": {
"inputs": {
"conditioning": ["9", 0],
"latent": ["11", 0]
},
"class_type": "ReferenceLatent",
"_meta": { "title": "Reference Latent A+" }
},
"13": {
"inputs": {
"conditioning": ["10", 0],
"latent": ["11", 0]
},
"class_type": "ReferenceLatent",
"_meta": { "title": "Reference Latent A-" }
},
"14": {
"inputs": {
"width": ["3", 0],
"height": ["3", 1],
"batch_size": 1
},
"class_type": "EmptyFlux2LatentImage",
"_meta": { "title": "Empty Flux 2 Latent" }
},
"15": {
"inputs": { "noise_seed": 1 },
"class_type": "RandomNoise",
"_meta": { "title": "RandomNoise" }
},
"16": {
"inputs": { "sampler_name": "euler" },
"class_type": "KSamplerSelect",
"_meta": { "title": "KSamplerSelect" }
},
"17": {
"inputs": {
"steps": 24,
"width": ["3", 0],
"height": ["3", 1]
},
"class_type": "Flux2Scheduler",
"_meta": { "title": "Flux2Scheduler" }
},
"18": {
"inputs": {
"cfg": 4,
"model": ["8", 0],
"positive": ["12", 0],
"negative": ["13", 0]
},
"class_type": "CFGGuider",
"_meta": { "title": "CFG Guider" }
},
"19": {
"inputs": {
"noise": ["15", 0],
"guider": ["18", 0],
"sampler": ["16", 0],
"sigmas": ["17", 0],
"latent_image": ["14", 0]
},
"class_type": "SamplerCustomAdvanced",
"_meta": { "title": "SamplerCustomAdvanced" }
},
"20": {
"inputs": {
"samples": ["19", 0],
"vae": ["6", 0]
},
"class_type": "VAEDecode",
"_meta": { "title": "VAE Decode" }
},
"21": {
"inputs": {
"filename_prefix": "aigen-v2",
"images": ["20", 0]
},
"class_type": "SaveImage",
"_meta": { "title": "Save Image" }
}
}
+1
View File
@@ -0,0 +1 @@
export { default } from '../../api/v2/generate.post'
+2 -2
View File
@@ -15,8 +15,8 @@ export function comfyConfigured() {
function comfyBase() {
if (comfyHostOverride) return comfyHostOverride
const config = useRuntimeConfig()
let host = String(config.comfyHost || process.env.COMFY_HOST || '').trim().replace(/\/$/, '')
const port = String(config.comfyPort || process.env.COMFY_PORT || '').trim()
let host = String(process.env.COMFY_HOST || config.comfyHost || '').trim().replace(/\/$/, '')
const port = String(process.env.COMFY_PORT || config.comfyPort || '').trim()
if (!host) {
throw createError({ statusCode: 500, statusMessage: 'COMFY_HOST is not configured' })
}
+39 -2
View File
@@ -14,6 +14,18 @@ import {
type ImagePresetSettings,
type VideoPresetSettings
} from '~/utils/generationPresets'
import {
IMAGE_V2_CFG_DEFAULT,
IMAGE_V2_CONSISTENCY_CLIP,
IMAGE_V2_CONSISTENCY_MODEL,
IMAGE_V2_SNOFS_CLIP,
IMAGE_V2_SNOFS_MODEL,
IMAGE_V2_STEPS_DEFAULT,
clampImageV2Strength,
parseImageV2Mode,
parseImageV2Task,
type ImageV2PresetSettings
} from '~/utils/imageV2'
import { coerceVideoWorkflow, isXaigenStudio, parseVideoWorkflow } from '~/utils/videoModels'
const writeChains = new Map<string, Promise<unknown>>()
@@ -96,6 +108,31 @@ function sanitizeImageSettings(raw: unknown): ImagePresetSettings {
}
}
function sanitizeImageV2Settings(raw: unknown): ImageV2PresetSettings {
const rec = raw && typeof raw === 'object' ? raw as Record<string, unknown> : {}
const mode = parseImageV2Mode(rec.mode) || 'edit'
const turbo = rec.turbo === true
return {
mode,
task: mode === 'compose' ? parseImageV2Task(rec.task, 'scene') : 'scene',
negative: String(rec.negative || '').slice(0, 2000),
snofsModel: clampImageV2Strength(rec.snofsModel ?? rec.snofs_model, IMAGE_V2_SNOFS_MODEL),
snofsClip: clampImageV2Strength(rec.snofsClip ?? rec.snofs_clip, IMAGE_V2_SNOFS_CLIP),
consistencyModel: clampImageV2Strength(rec.consistencyModel ?? rec.consistency_model, IMAGE_V2_CONSISTENCY_MODEL),
consistencyClip: clampImageV2Strength(rec.consistencyClip ?? rec.consistency_clip, IMAGE_V2_CONSISTENCY_CLIP),
steps: turbo ? 8 : clampImageSteps(rec.steps, IMAGE_V2_STEPS_DEFAULT),
cfg: turbo ? 1 : clampImageCfg(rec.cfg, IMAGE_V2_CFG_DEFAULT),
megapixels: clampImageScaleMegapixels(rec.megapixels ?? rec.scaleMegapixels, 1),
turbo
}
}
function sanitizeSettings(kind: GenerationPresetKind, raw: unknown) {
if (kind === 'image') return sanitizeImageSettings(raw)
if (kind === 'imagev2') return sanitizeImageV2Settings(raw)
return sanitizeVideoSettings(raw)
}
function normalizeStoredPreset(raw: unknown): GenerationPreset | null {
if (!raw || typeof raw !== 'object') return null
const rec = raw as Record<string, unknown>
@@ -111,7 +148,7 @@ function normalizeStoredPreset(raw: unknown): GenerationPreset | null {
createdAt,
updatedAt: Number(rec.updatedAt) || createdAt,
loraStack: normalizeLoraStack(rec.loraStack),
settings: kind === 'image' ? sanitizeImageSettings(rec.settings) : sanitizeVideoSettings(rec.settings)
settings: sanitizeSettings(kind, rec.settings)
}
}
@@ -147,7 +184,7 @@ export async function saveGenerationPreset(owner: string, input: {
const kind = parseGenerationPresetKind(input.kind)
const xaigen = isXaigenStudio()
const loraStack = filterLoraStackForStudio(input.loraStack, xaigen)
const settings = kind === 'image' ? sanitizeImageSettings(input.settings) : sanitizeVideoSettings(input.settings)
const settings = sanitizeSettings(kind, input.settings)
const now = Date.now()
return mutate(owner, (presets) => {
const existing = presets.find(item => item.kind === kind && item.name.toLowerCase() === name.toLowerCase())
+168
View File
@@ -0,0 +1,168 @@
import { createJob, emitJob, type Job } from '~/server/utils/jobs'
import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
import { assertImageScaleToTotalPixelsNode, getComfyHost, uploadImage, queuePrompt, purgeComfyArtifacts } from '~/server/utils/comfy'
import { withImageComfyHost, waitForImageEdit, downloadEditedImage } from '~/server/utils/imageComfy'
import { buildImageV2Workflow } from '~/server/utils/imageWorkflowV2'
import { ensureComfyLoraNames } from '~/server/utils/loras'
import { imageDimensions } from '~/server/utils/resolution'
import { emitChainJob } from '~/server/utils/watch'
import { saveStill } from '~/server/utils/library'
import type { ImageV2Mode, ImageV2Task } from '~/utils/imageV2'
import type { EditImageFile } from '~/server/utils/imageChain'
export type EditV2RunParams = {
mode: ImageV2Mode
task: ImageV2Task
image: EditImageFile
reference: EditImageFile | null
prompt: string
negative: string
steps: number
seed: number
cfg: number
snofsModel: number
snofsClip: number
consistencyModel: number
consistencyClip: number
megapixels: number
}
export async function runEditV2(job: Job, params: EditV2RunParams) {
const library = job.library
if (!library) throw new Error('Edit job is missing library metadata')
if (params.mode === 'compose' && !params.reference) {
throw new Error('Compose requires image B. Refusing to fall back to one-image generation.')
}
if (params.mode === 'edit' && params.reference) {
throw new Error('Edit mode takes one image. Use Compose for two stills.')
}
try {
await ensureComfyReady((status) => {
emitChainJob(job, {
type: status.state === 'busy' ? 'busy' : 'status',
message: status.message,
progress: status.state === 'online' ? Math.max(job.progress, 6) : Math.max(job.progress, 3),
busy: status.state === 'busy'
})
}, { skipBusyWait: true })
if (job.status === 'cancelled') throw new Error('Job interrupted.')
job.imageComfyHost = getComfyHost()
await withImageComfyHost(job.imageComfyHost, async () => {
emitChainJob(job, {
type: 'status',
message: params.mode === 'compose' ? 'Uploading stills A and B to Beast...' : 'Uploading still A to Beast...',
progress: 8
})
const uploaded = await uploadImage(params.image, job.id)
const uploadedRef = params.reference
? await uploadImage({
...params.reference,
filename: `ref_${params.reference.filename || 'image_b.png'}`
}, job.id)
: null
if (job.status === 'cancelled') throw new Error('Job interrupted.')
emitChainJob(job, {
type: 'status',
message: params.mode === 'compose'
? `Queueing Klein v2 compose (${params.task}) on Beast...`
: 'Queueing Klein v2 edit on Beast...',
progress: 12
})
await ensureComfyLoraNames('image')
await assertImageScaleToTotalPixelsNode()
const built = buildImageV2Workflow({
mode: params.mode,
task: params.task,
prompt: params.prompt,
negative: params.negative,
imageAName: uploaded.name,
imageBName: uploadedRef?.name,
snofsModel: params.snofsModel,
snofsClip: params.snofsClip,
consistencyModel: params.consistencyModel,
consistencyClip: params.consistencyClip,
steps: params.steps,
cfg: params.cfg,
seed: params.seed,
megapixels: params.megapixels,
filenamePrefix: `aigen_v2_${job.id.slice(0, 8)}`
})
const queued = await queuePrompt(built.graph, job.clientId)
job.promptId = queued.prompt_id
job.status = 'running'
emitChainJob(job, {
type: 'status',
message: `Running ${built.workflowFile}...`,
progress: 18,
maxStep: params.steps
})
const output = await waitForImageEdit({
promptId: queued.prompt_id,
clientId: job.clientId,
timeoutMs: 10 * 60 * 1000,
onProgress: (event) => {
emitChainJob(job, {
type: 'status',
message: event.message,
progress: event.progress,
step: event.step,
maxStep: event.maxStep || params.steps,
node: event.node
})
},
isCancelled: () => job.status === 'cancelled'
})
emitChainJob(job, { type: 'status', message: 'Saving Klein v2 still...', progress: 94 })
const buffer = await downloadEditedImage(output)
const size = imageDimensions(buffer)
const still = await saveStill({
ownerKey: library.ownerKey,
folderId: library.folderId,
filename: output.filename,
data: buffer,
width: size?.width || 0,
height: size?.height || 0,
hideInput: job.hideThumbnail === true,
role: 'output',
name: library.name || undefined,
prompt: built.prompt
})
job.stillId = still?.id
await purgeComfyArtifacts({
video: { filename: output.filename, subfolder: output.subfolder, type: output.type },
imageName: uploaded.name,
imageSubfolder: uploaded.subfolder,
extraImageNames: uploadedRef?.name ? [uploadedRef.name] : [],
promptId: job.promptId
})
job.status = 'complete'
emitChainJob(job, {
type: 'complete',
message: 'Klein v2 finished on Beast',
progress: 100,
stillId: still?.id,
filename: output.filename,
subfolder: output.subfolder,
mediaType: 'image',
hideThumbnail: job.hideThumbnail,
folderLocked: library.folderLocked
})
})
} catch (error) {
if (job.status !== 'error' && job.status !== 'cancelled') {
const message = error instanceof Error ? error.message : String(error)
job.status = 'error'
job.error = message
emitJob(job, { type: 'error', error: message, message })
}
} finally {
const { onLiveVideoSettled } = await import('~/server/utils/studioQueue')
await onLiveVideoSettled(job)
}
}
+1 -1
View File
@@ -292,7 +292,7 @@ export async function waitForImageEdit(opts: {
if (payload.type === 'executing' && payload.data?.node) {
opts.onProgress?.({
message: 'Running Flux.2 Klein...',
progress: payload.data.node === '9' || payload.data.node === '94' ? 92 : 30,
progress: payload.data.node === '9' || payload.data.node === '94' || payload.data.node === '21' ? 92 : 30,
node: payload.data.node
})
}
+193
View File
@@ -0,0 +1,193 @@
import editTemplate from '../assets/klein_v2_edit.json'
import composeTemplate from '../assets/klein_v2_compose.json'
import { IMAGE_SCALE_TO_TOTAL_PIXELS } from '~/server/utils/comfy'
import { cachedComfyLoraNames } from '~/server/utils/loras'
import { resolveComfyLoraName, loraIdentityKey } from '~/utils/loras'
import { clampImageCfg, clampImageScaleMegapixels, clampImageSteps } from '~/utils/generationPresets'
import {
IMAGE_V2_CONSISTENCY_CLIP,
IMAGE_V2_CONSISTENCY_LORA,
IMAGE_V2_CONSISTENCY_MODEL,
IMAGE_V2_SNOFS_CLIP,
IMAGE_V2_SNOFS_LORA,
IMAGE_V2_SNOFS_MODEL,
clampImageV2Strength,
composeImageV2Prompt,
type ImageV2Mode,
type ImageV2Task
} from '~/utils/imageV2'
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
type WorkflowGraph = Record<string, WorkflowNode>
const LOAD_A = '1'
const LOAD_B = '22'
const SCALE_A = '2'
const SCALE_B = '23'
const PROMPT = '9'
const NEGATIVE = '10'
const NOISE = '15'
const SCHEDULER = '17'
const CFG = '18'
const SAVE = '21'
const SNOFS = '7'
const CONSISTENCY = '8'
export const IMAGE_V2_EDIT_WORKFLOW = 'klein_v2_edit.json'
export const IMAGE_V2_COMPOSE_WORKFLOW = 'klein_v2_compose.json'
export interface ImageV2BuildParams {
mode: ImageV2Mode
task: ImageV2Task
prompt: string
negative?: string
imageAName: string
imageBName?: string
snofsModel?: number
snofsClip?: number
consistencyModel?: number
consistencyClip?: number
steps: number
cfg: number
seed: number
megapixels?: number
filenamePrefix?: string
}
function setInput(graph: WorkflowGraph, id: string, key: string, value: unknown) {
if (graph[id]) graph[id].inputs[key] = value
}
function loadImageNames(graph: WorkflowGraph) {
return Object.entries(graph)
.filter(([, node]) => node.class_type === 'LoadImage')
.map(([id, node]) => ({
id,
title: String(node._meta?.title || id),
image: String(node.inputs.image || '')
}))
}
function resolveRequiredLora(wanted: string, label: string) {
const names = cachedComfyLoraNames('image')
const resolved = names.length ? resolveComfyLoraName(wanted, names) : wanted
const hit = names.some(name => loraIdentityKey(name) === loraIdentityKey(wanted) || loraIdentityKey(name) === loraIdentityKey(resolved))
if (names.length && !hit) {
throw createError({ statusCode: 503, statusMessage: `Missing ${label} LoRA (${wanted}) on Beast Comfy` })
}
return resolved
}
function patchScaleMegapixels(graph: WorkflowGraph, megapixels: number) {
const mp = clampImageScaleMegapixels(megapixels)
for (const node of Object.values(graph)) {
if (node.class_type !== IMAGE_SCALE_TO_TOTAL_PIXELS) continue
node.inputs.megapixels = mp
node.inputs.upscale_method = 'lanczos'
}
}
export function assertImageV2Graph(graph: WorkflowGraph, mode: ImageV2Mode, imageBName?: string) {
const loaders = loadImageNames(graph)
if (mode === 'compose') {
if (loaders.length < 2) {
throw createError({
statusCode: 500,
statusMessage: 'Compose graph has no second image input. Refusing to run a one-image fallback.'
})
}
const b = graph[LOAD_B]
if (!b || b.class_type !== 'LoadImage' || !String(b.inputs.image || '').trim()) {
throw createError({
statusCode: 500,
statusMessage: 'Compose job is missing Load Image B. Refusing to run.'
})
}
}
if (imageBName && loaders.length < 2) {
throw createError({
statusCode: 500,
statusMessage: 'image_b was sent but the executed graph has no second image input.'
})
}
const promptNode = graph[PROMPT]
if (!promptNode || promptNode.class_type !== 'CLIPTextEncode') {
throw createError({ statusCode: 500, statusMessage: 'v2 graph is missing the positive CLIPTextEncode node.' })
}
if (Array.isArray(promptNode.inputs.text)) {
throw createError({ statusCode: 500, statusMessage: 'v2 prompt is a subgraph link. Refusing to run with a leftover widget prompt.' })
}
if (!String(promptNode.inputs.text || '').trim()) {
throw createError({ statusCode: 400, statusMessage: 'v2 prompt was not patched onto the graph.' })
}
}
export function buildImageV2Workflow(params: ImageV2BuildParams) {
const compose = params.mode === 'compose'
const graph = structuredClone(compose ? composeTemplate : editTemplate) as WorkflowGraph
const prompt = composeImageV2Prompt(params.mode, params.task, params.prompt)
const negative = String(params.negative || '')
const snofsModel = clampImageV2Strength(params.snofsModel, IMAGE_V2_SNOFS_MODEL)
const snofsClip = clampImageV2Strength(params.snofsClip, IMAGE_V2_SNOFS_CLIP)
const consistencyModel = clampImageV2Strength(params.consistencyModel, IMAGE_V2_CONSISTENCY_MODEL)
const consistencyClip = clampImageV2Strength(params.consistencyClip, IMAGE_V2_CONSISTENCY_CLIP)
const steps = clampImageSteps(params.steps, 24)
const cfg = clampImageCfg(params.cfg, 4)
const workflowFile = compose ? IMAGE_V2_COMPOSE_WORKFLOW : IMAGE_V2_EDIT_WORKFLOW
setInput(graph, LOAD_A, 'image', params.imageAName)
if (compose) setInput(graph, LOAD_B, 'image', params.imageBName || '')
setInput(graph, PROMPT, 'text', prompt)
setInput(graph, NEGATIVE, 'text', negative)
setInput(graph, NOISE, 'noise_seed', params.seed)
setInput(graph, SCHEDULER, 'steps', steps)
setInput(graph, CFG, 'cfg', cfg)
setInput(graph, SAVE, 'filename_prefix', params.filenamePrefix || 'aigen-v2')
patchScaleMegapixels(graph, params.megapixels ?? 1)
setInput(graph, SNOFS, 'lora_name', resolveRequiredLora(IMAGE_V2_SNOFS_LORA, 'SNOFS'))
setInput(graph, SNOFS, 'strength_model', snofsModel)
setInput(graph, SNOFS, 'strength_clip', snofsClip)
setInput(graph, CONSISTENCY, 'lora_name', resolveRequiredLora(IMAGE_V2_CONSISTENCY_LORA, 'Consistency'))
setInput(graph, CONSISTENCY, 'strength_model', consistencyModel)
setInput(graph, CONSISTENCY, 'strength_clip', consistencyClip)
assertImageV2Graph(graph, params.mode, params.imageBName)
const loaders = loadImageNames(graph)
console.log(JSON.stringify({
src: 'image-v2',
workflow: workflowFile,
mode: params.mode,
task: params.task,
loadImage: Object.fromEntries(loaders.map(item => [item.id, { title: item.title, file: item.image }])),
loras: {
snofs: { name: graph[SNOFS]?.inputs.lora_name, model: snofsModel, clip: snofsClip },
consistency: { name: graph[CONSISTENCY]?.inputs.lora_name, model: consistencyModel, clip: consistencyClip }
},
steps,
cfg,
seed: params.seed,
megapixels: clampImageScaleMegapixels(params.megapixels ?? 1)
}))
return { graph, workflowFile, loaders, prompt }
}
export const IMAGE_V2_NODE_LABELS: Record<string, string> = {
'1': 'Loading image A',
'22': 'Loading image B',
'2': 'Scaling image A',
'23': 'Scaling image B',
'4': 'Loading Flux.2 Klein 9B',
'5': 'Loading CLIP',
'6': 'Loading VAE',
'7': 'Applying SNOFS',
'8': 'Applying Consistency',
'9': 'Encoding prompt',
'11': 'Encoding image A',
'24': 'Encoding image B',
'19': 'Sampling Klein v2',
'20': 'Decoding still',
'21': 'Saving still'
}
+69
View File
@@ -51,6 +51,13 @@ export interface StudioJobPayload {
referenceStillFilename?: string
scaleToTotalPixels?: boolean
scaleMegapixels?: number
imagePipeline?: 'v1' | 'v2'
v2Mode?: 'edit' | 'compose'
v2Task?: 'scene' | 'identity' | 'outfit' | 'face_lock'
snofsModel?: number
snofsClip?: number
consistencyModel?: number
consistencyClip?: number
}
export interface StudioJob {
@@ -210,6 +217,7 @@ export function summarizeStudioJob(job: StudioJob) {
liveJobId: job.liveJobId,
stillId: job.payload.stillId,
workflow: job.payload.workflow,
imagePipeline: job.payload.imagePipeline || 'v1',
duration: job.payload.duration,
hideThumbnail: job.payload.hideThumbnail,
folderLocked: job.payload.folderLocked === true,
@@ -665,6 +673,7 @@ async function startStudioEditJob(item: StudioJob) {
const { stillPath } = await import('~/server/utils/library')
const { existsSync, readFileSync } = await import('node:fs')
const { createEditLiveJob, runEdit } = await import('~/server/utils/imageChain')
const { runEditV2 } = await import('~/server/utils/imageChainV2')
const payload = item.payload
if (!payload.stillId || !existsSync(stillPath(item.ownerKey, payload.stillId))) {
@@ -684,6 +693,66 @@ async function startStudioEditJob(item: StudioJob) {
}
: null
if (payload.imagePipeline === 'v2') {
const mode = payload.v2Mode === 'compose' ? 'compose' : 'edit'
if (mode === 'compose' && !reference) {
throw new Error('Compose requires image B. Refusing to fall back to one-image generation.')
}
if (mode === 'edit' && reference) {
throw new Error('Edit mode takes one image. Use Compose for two stills.')
}
live = createEditLiveJob({
steps: payload.steps,
hideThumbnail: payload.hideThumbnail,
library: {
ownerKey: item.ownerKey,
folderId: payload.folderId,
hideThumbnail: payload.hideThumbnail,
hideInput: payload.hideInput,
folderLocked: payload.folderLocked,
name: payload.name,
prompt: payload.prompt,
aspect: payload.aspect || 'auto',
width: payload.width,
height: payload.height,
steps: payload.steps,
turbo: payload.turbo === true,
seed: payload.seed || Math.floor(Math.random() * 2_147_483_647),
cfg: payload.cfg,
stillId: payload.stillId,
stillFilename: payload.stillFilename,
familyId: item.familyId,
chainIndex: 0,
chainStep: 1,
chainTotal: 1
}
})
await markStudioLive(item.ownerKey, item.id, live.id)
void runEditV2(live, {
mode,
task: payload.v2Task || 'scene',
image,
reference,
prompt: payload.prompt,
negative: payload.negative || '',
steps: payload.steps,
seed: live.library?.seed || payload.seed,
cfg: payload.cfg,
snofsModel: payload.snofsModel ?? 0.65,
snofsClip: payload.snofsClip ?? 0.35,
consistencyModel: payload.consistencyModel ?? 0.7,
consistencyClip: payload.consistencyClip ?? 0.7,
megapixels: payload.scaleMegapixels ?? 1
}).catch((error) => {
const message = error instanceof Error ? error.message : String(error)
if (live && live.status !== 'error' && live.status !== 'cancelled' && live.status !== 'deferred') {
live.status = 'error'
live.error = message
}
})
return
}
const passes = payload.passes || []
live = createEditLiveJob({
steps: payload.steps,