From f51c3772e5093cc988162e510e5588ac132f20dd Mon Sep 17 00:00:00 2001 From: Towsty Date: Fri, 28 Aug 2026 16:21:32 -0500 Subject: [PATCH] Load only the LoRAs the user picked on Klein; do not require snofs. Co-authored-by: Cursor --- components/LoraSelect.vue | 9 +- server/api/extend.post.ts | 3 +- server/assets/workflow_flux2_klein_edit.json | 8 +- server/utils/imageChain.ts | 2 + server/utils/imageWorkflow.ts | 5 +- server/utils/loras.ts | 144 +++++++++++++++++-- server/utils/videoChain.ts | 3 +- server/utils/workflow.ts | 4 +- utils/loras.ts | 39 ++++- 9 files changed, 192 insertions(+), 25 deletions(-) diff --git a/components/LoraSelect.vue b/components/LoraSelect.vue index b6ef3af..76906bf 100644 --- a/components/LoraSelect.vue +++ b/components/LoraSelect.vue @@ -87,6 +87,7 @@ import { clampLoraStrength, formatLoraStrength, loraDisplayName, + loraIdentityKey, makeLoraStackItem, type LoraStackItem } from '~/utils/loras' @@ -111,8 +112,8 @@ const props = withDefaults(defineProps<{ const model = defineModel({ default: () => [] }) const availableNames = computed(() => { - const taken = new Set(model.value.map(item => item.name.replace(/\\/g, '/').toLowerCase())) - return props.names.filter(name => !taken.has(name.replace(/\\/g, '/').toLowerCase())) + const taken = new Set(model.value.map(item => loraIdentityKey(item.name))) + return props.names.filter(name => !taken.has(loraIdentityKey(name))) }) function onAdd(event: Event) { @@ -124,8 +125,8 @@ function onAdd(event: Event) { function addLora(name: string) { const item = makeLoraStackItem(name) if (!item || model.value.length >= LORA_STACK_MAX) return - const taken = new Set(model.value.map(entry => entry.name.replace(/\\/g, '/').toLowerCase())) - if (taken.has(item.name.replace(/\\/g, '/').toLowerCase())) return + const taken = new Set(model.value.map(entry => loraIdentityKey(entry.name))) + if (taken.has(loraIdentityKey(item.name))) return model.value = [...model.value, item] } diff --git a/server/api/extend.post.ts b/server/api/extend.post.ts index 0f4c2d9..9400a60 100644 --- a/server/api/extend.post.ts +++ b/server/api/extend.post.ts @@ -1,7 +1,7 @@ import { copyFileSync, existsSync, readFileSync } from 'node:fs' import { join } from 'node:path' import { composeShotPrompt } from '~/utils/globalLocks' -import { parsePostedLoraStack, listStudioLoras, persistLoraFields } from '~/server/utils/loras' +import { parsePostedLoraStack, listStudioLoras, persistLoraFields, ensureComfyLoraNames } from '~/server/utils/loras' import { readLoraStack } from '~/utils/loras' import { clampVideoCfg } from '~/utils/generationPresets' import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels' @@ -213,6 +213,7 @@ async function runExtend( } emitJob(job, { type: 'status', message: 'Queueing extension on MiniMax H3...', progress: 6 }) await waitForComfySocket(job, 4000) + await ensureComfyLoraNames('video') const graph = buildWorkflow({ prompt: params.prompt, diff --git a/server/assets/workflow_flux2_klein_edit.json b/server/assets/workflow_flux2_klein_edit.json index 52e6cea..c6197a3 100644 --- a/server/assets/workflow_flux2_klein_edit.json +++ b/server/assets/workflow_flux2_klein_edit.json @@ -278,7 +278,7 @@ }, "75:131": { "inputs": { - "lora_name": "klein_snofs_v1_4.safetensors", + "lora_name": "", "strength_model": 1, "strength_clip": 1, "model": [ @@ -292,7 +292,7 @@ }, "class_type": "LoraLoader", "_meta": { - "title": "Load LoRA (Model and CLIP)" + "title": "User LoRA (optional)" } }, "92:102": { @@ -593,7 +593,7 @@ }, "92:132": { "inputs": { - "lora_name": "klein_snofs_v1_4.safetensors", + "lora_name": "", "strength_model": 1, "strength_clip": 1, "model": [ @@ -607,7 +607,7 @@ }, "class_type": "LoraLoader", "_meta": { - "title": "Load LoRA (Model and CLIP)" + "title": "User LoRA (optional)" } } } \ No newline at end of file diff --git a/server/utils/imageChain.ts b/server/utils/imageChain.ts index 37ced86..a4f4039 100644 --- a/server/utils/imageChain.ts +++ b/server/utils/imageChain.ts @@ -3,6 +3,7 @@ import { ensureComfyReady } from '~/server/utils/comfyLifecycle' import { getComfyHost, uploadImage, queuePrompt, purgeComfyArtifacts } from '~/server/utils/comfy' import { withImageComfyHost, waitForImageEdit, downloadEditedImage } from '~/server/utils/imageComfy' import { buildEditWorkflow } from '~/server/utils/imageWorkflow' +import { ensureComfyLoraNames } from '~/server/utils/loras' import { imageDimensions } from '~/server/utils/resolution' import { emitChainJob } from '~/server/utils/watch' import { saveStill, stillChainName } from '~/server/utils/library' @@ -91,6 +92,7 @@ export async function runEdit(job: Job, params: EditRunParams) { : 'Queueing Flux.2 Klein edit on Beast...', progress: 12 }) + await ensureComfyLoraNames('image') const graph = buildEditWorkflow({ prompt, negative: params.negative, diff --git a/server/utils/imageWorkflow.ts b/server/utils/imageWorkflow.ts index d849997..8ebbe71 100644 --- a/server/utils/imageWorkflow.ts +++ b/server/utils/imageWorkflow.ts @@ -1,5 +1,5 @@ import template from '../assets/workflow_flux2_klein_edit.json' -import { applyUserLoraToGraph } from '~/server/utils/loras' +import { applyOptionalLoraLoaders } from '~/server/utils/loras' import type { LoraStackItem } from '~/utils/loras' const IMAGE1 = '76' @@ -65,7 +65,8 @@ export function buildEditWorkflow(params: EditParams) { if (dual ? singleNode : dualNode) delete graph[id] } - applyUserLoraToGraph(graph, params.loraStack || params.loraName) + // User LoRAs only. Empty stack bypasses the optional LoraLoader (no snofs default). + applyOptionalLoraLoaders(graph, params.loraStack || params.loraName, 'image') return graph } diff --git a/server/utils/loras.ts b/server/utils/loras.ts index eb5333a..a3792ec 100644 --- a/server/utils/loras.ts +++ b/server/utils/loras.ts @@ -2,15 +2,18 @@ import { comfyConfigured, comfyFetch, getComfyHost } from '~/server/utils/comfy' import { getBeastImageHost, imageComfyFetch, sameImageHost } from '~/server/utils/imageComfy' import { XAIGEN_LORA_MESSAGE, + MINIMAX_TURBO_LORA, filterLorasForStudio, isXaigenOnlyLora, + loraIdentityKey, normalizeLoraName, normalizeLoraStack, persistLoraFields, + resolveComfyLoraName, type LoraKind, type LoraStackItem } from '~/utils/loras' -import { isXaigenStudio } from '~/utils/videoModels' +import { isXaigenStudio, LTX_DISTILLED_LORA } from '~/utils/videoModels' type WorkflowNode = { class_type: string; inputs: Record; _meta?: { title?: string } } type WorkflowGraph = Record @@ -28,6 +31,7 @@ const OBJECT_INFO_NODES = [ ] const MODEL_FEED_CLASSES = new Set(['BasicGuider', 'CFGGuider', 'KSampler', 'KSamplerAdvanced']) const CACHE_MS = 20_000 +const SYSTEM_LORA_PREFERRED = [MINIMAX_TURBO_LORA, LTX_DISTILLED_LORA] type LoraCache = { at: number @@ -150,11 +154,10 @@ async function loadLoraCache(): Promise { const image = imageHost && !same ? await discoverFromHost('image').catch(() => ({ image: [] as string[], video: [] as string[] })) : video - const xaigen = isXaigenStudio() const next: LoraCache = { at: Date.now(), - image: filterLorasForStudio(uniqueNames([...image.image, ...video.image]), xaigen), - video: filterLorasForStudio(uniqueNames([...video.video, ...image.video]), xaigen) + image: uniqueNames([...image.image, ...video.image]), + video: uniqueNames([...video.video, ...image.video]) } cache = next return next @@ -166,21 +169,73 @@ async function loadLoraCache(): Promise { export async function listStudioLoras() { try { - return await loadLoraCache() + const listed = await loadLoraCache() + const xaigen = isXaigenStudio() + return { + at: listed.at, + image: filterLorasForStudio(listed.image, xaigen), + video: filterLorasForStudio(listed.video, xaigen) + } } catch { return { at: 0, image: [] as string[], video: [] as string[] } } } +/** Unfiltered Comfy filenames for graph system LoRAs (MiniMax turbo, LTX distilled). Reuses the listing cache. */ +export function cachedComfyLoraNames(kind?: LoraKind) { + if (!cache) return [] as string[] + if (kind === 'image') return uniqueNames([...cache.image, ...cache.video]) + if (kind === 'video') return uniqueNames([...cache.video, ...cache.image]) + return uniqueNames([...cache.image, ...cache.video]) +} + +export async function ensureComfyLoraNames(kind?: LoraKind) { + try { + await loadLoraCache() + } catch { + /* keep whatever was cached */ + } + return cachedComfyLoraNames(kind) +} + +export function resolveGraphLoraNames(graph: WorkflowGraph, kind?: LoraKind) { + const names = cachedComfyLoraNames(kind) + for (const node of Object.values(graph)) { + if (node.class_type !== LORA_LOADER && node.class_type !== LORA_MODEL_ONLY) continue + const current = String(node.inputs.lora_name || '').trim() + if (!current) continue + const preferred = SYSTEM_LORA_PREFERRED.find(item => loraIdentityKey(item) === loraIdentityKey(current)) || current + node.inputs.lora_name = names.length ? resolveComfyLoraName(preferred, names) : preferred + } +} + +function allowedLoraNames(kind?: LoraKind) { + return filterLorasForStudio(cachedComfyLoraNames(kind), isXaigenStudio()) +} + +function resolveUserLoraName(name: string, kind?: LoraKind) { + const allowed = allowedLoraNames(kind) + if (!allowed.length) return name + return resolveComfyLoraName(name, allowed) +} + export function assertLoraAllowed(raw: unknown, kind: LoraKind) { const name = normalizeLoraName(raw) if (!name) return '' if (isXaigenOnlyLora(name) && !isXaigenStudio()) { throw createError({ statusCode: 400, statusMessage: XAIGEN_LORA_MESSAGE }) } - const known = cache?.[kind] - if (known?.length && !known.includes(name) && !known.some(item => item.replace(/\\/g, '/') === name)) { - throw createError({ statusCode: 400, statusMessage: `Unknown ${kind} LoRA` }) + const allowed = allowedLoraNames(kind) + if (allowed.length) { + const identity = loraIdentityKey(name) + const listed = allowed.some((item) => { + const path = item.replace(/\\/g, '/') + return path === name || path.toLowerCase() === name.toLowerCase() || loraIdentityKey(item) === identity + }) + if (!listed) { + throw createError({ statusCode: 400, statusMessage: `Unknown ${kind} LoRA` }) + } + return resolveUserLoraName(name, kind) } return name } @@ -225,10 +280,11 @@ function linkSource(value: unknown): string | null { } function alreadyHasLora(graph: WorkflowGraph, name: string) { - const wanted = name.replace(/\\/g, '/').toLowerCase() + const wanted = loraIdentityKey(name) + if (!wanted) return false return Object.values(graph).some((node) => { if (node.class_type !== LORA_LOADER && node.class_type !== LORA_MODEL_ONLY) return false - return String(node.inputs.lora_name || '').replace(/\\/g, '/').toLowerCase() === wanted + return loraIdentityKey(String(node.inputs.lora_name || '')) === wanted }) } @@ -260,8 +316,74 @@ function findModelFeed(graph: WorkflowGraph) { return '' } +function bypassLoraNode(graph: WorkflowGraph, id: string) { + const node = graph[id] + if (!node) return + const model = Array.isArray(node.inputs.model) ? node.inputs.model : null + const clip = Array.isArray(node.inputs.clip) ? node.inputs.clip : null + for (const [otherId, other] of Object.entries(graph)) { + if (otherId === id) continue + for (const [key, value] of Object.entries(other.inputs)) { + const src = linkSource(value) + if (src !== id || !Array.isArray(value)) continue + const slot = value[1] + if (slot === 0 && model) other.inputs[key] = [model[0], model[1]] + else if (slot === 1 && clip) other.inputs[key] = [clip[0], clip[1]] + } + } + delete graph[id] +} + +/** Optional Klein-style LoraLoader: user stack only. Empty stack bypasses model/CLIP around the loader. */ +export function applyOptionalLoraLoaders(graph: WorkflowGraph, stack?: unknown, kind: LoraKind = 'image') { + const loaders = Object.entries(graph).filter(([, node]) => node.class_type === LORA_LOADER) + const xaigen = isXaigenStudio() + const items = normalizeLoraStack(stack) + .filter(item => xaigen || !isXaigenOnlyLora(item.name)) + .map(item => ({ + ...item, + name: resolveUserLoraName(item.name, kind) + })) + .filter(item => item.name && (xaigen || !isXaigenOnlyLora(item.name))) + if (!loaders.length) { + applyUserLoraToGraph(graph, items) + return + } + if (!items.length) { + for (const [id] of loaders) bypassLoraNode(graph, id) + return + } + for (const [id] of loaders) { + const first = items[0] + graph[id].inputs.lora_name = first.name + graph[id].inputs.strength_model = first.strengthModel + graph[id].inputs.strength_clip = first.strengthClip + graph[id]._meta = { title: items.length === 1 ? 'User LoRA' : 'User LoRA 1' } + let sourceId = id + for (const [index, item] of items.slice(1).entries()) { + const nodeId = `user:lora:${id}:${index + 1}` + injectAfter(graph, sourceId, nodeId, { + class_type: LORA_LOADER, + inputs: { + lora_name: item.name, + strength_model: item.strengthModel, + strength_clip: item.strengthClip, + model: [sourceId, 0], + clip: [sourceId, 1] + }, + _meta: { title: `User LoRA ${index + 2}` } + }) + sourceId = nodeId + } + } +} + export function applyUserLoraToGraph(graph: WorkflowGraph, stack?: unknown) { - const items = normalizeLoraStack(stack).filter(item => !alreadyHasLora(graph, item.name)) + const xaigen = isXaigenStudio() + const items = normalizeLoraStack(stack) + .filter(item => xaigen || !isXaigenOnlyLora(item.name)) + .map(item => ({ ...item, name: resolveUserLoraName(item.name) })) + .filter(item => item.name && (xaigen || !isXaigenOnlyLora(item.name)) && !alreadyHasLora(graph, item.name)) if (!items.length) return const clipLoaders = Object.entries(graph).filter(([, node]) => node.class_type === LORA_LOADER) diff --git a/server/utils/videoChain.ts b/server/utils/videoChain.ts index 59aa052..27774eb 100644 --- a/server/utils/videoChain.ts +++ b/server/utils/videoChain.ts @@ -21,7 +21,7 @@ import { updateShotQueue } from '~/server/utils/shotQueue' import { composeShotPrompt, allowIdentityRefs, type PermanenceRef } from '~/utils/globalLocks' -import { persistLoraFields } from '~/server/utils/loras' +import { persistLoraFields, ensureComfyLoraNames } from '~/server/utils/loras' import { readLoraStack, resolveLoraStack } from '~/utils/loras' import type { LoraStackItem } from '~/utils/loras' @@ -180,6 +180,7 @@ export async function queueMiniMax( } emitChainJob(job, { type: 'status', message: queueing, progress: 6 }) await waitForComfySocket(job, 4000) + await ensureComfyLoraNames('video') const shotIndex = job.library?.chainIndex || 0 const composedPrompt = composeShotPrompt({ diff --git a/server/utils/workflow.ts b/server/utils/workflow.ts index 4c233d2..a913e96 100644 --- a/server/utils/workflow.ts +++ b/server/utils/workflow.ts @@ -3,7 +3,7 @@ import workflowV1 from '../assets/workflow_minimax_video.json' import workflowV2 from '../assets/workflow_minimax_video_v2.json' import workflowLtx from '../assets/workflow_ltx_video.json' import { buildIdentityPrompt } from '~/utils/identityPrompt' -import { applyUserLoraToGraph } from '~/server/utils/loras' +import { applyUserLoraToGraph, resolveGraphLoraNames } from '~/server/utils/loras' import type { LoraStackItem } from '~/utils/loras' import { isLtxWorkflow, @@ -225,6 +225,7 @@ export function buildWorkflow(params: GenerateParams) { if (graph['105:104']?.inputs) delete graph['105:104'].inputs.audio_vae } + resolveGraphLoraNames(graph, 'video') applyUserLoraToGraph(graph, params.loraStack || params.loraName) return graph } @@ -298,6 +299,7 @@ function buildLtxWorkflow(params: GenerateParams, version: VideoWorkflowId) { if (graph['12']?.inputs) graph['12'].inputs.latent = ['8', 0] if (graph['14']?.inputs) graph['14'].inputs.latent_image = ['8', 0] } + resolveGraphLoraNames(graph, 'video') applyUserLoraToGraph(graph, params.loraStack || params.loraName) return graph } diff --git a/utils/loras.ts b/utils/loras.ts index 5384938..01d7809 100644 --- a/utils/loras.ts +++ b/utils/loras.ts @@ -16,16 +16,52 @@ export const LORA_STRENGTH_DEFAULT = 1 export const LORA_STRENGTH_STEP = 0.05 export const LORA_STACK_MAX = 8 +/** Graph system LoRA for MiniMax turbo. Not a user pick. */ +export const MINIMAX_TURBO_LORA = 'minimax_h3_fl2v_turbo_8step_v1.0_comfyui_bf16.safetensors' + export function loraBasename(name: string) { const trimmed = String(name || '').trim().replace(/\\/g, '/') const parts = trimmed.split('/').filter(Boolean) return parts[parts.length - 1] || trimmed } +export function loraIdentityKey(name: string) { + return loraBasename(name).replace(/^xaigen-?/i, '').toLowerCase() +} + export function isXaigenOnlyLora(name: string) { return /^xaigen/i.test(loraBasename(name)) } +/** Pick the real Comfy filename: exact path, then same basename ignoring `xaigen-`, then suffix/contains. */ +export function resolveComfyLoraName(wanted: string, available: string[]) { + const target = String(wanted || '').trim().replace(/\\/g, '/') + if (!target) return '' + if (!available.length) return target + const targetLower = target.toLowerCase() + const exact = available.find(item => item.replace(/\\/g, '/') === target) + if (exact) return exact + const exactCi = available.find(item => item.replace(/\\/g, '/').toLowerCase() === targetLower) + if (exactCi) return exactCi + + const identity = loraIdentityKey(target) + const identityHit = available.find(item => loraIdentityKey(item) === identity) + if (identityHit) return identityHit + + const stem = identity.replace(/\.(safetensors|sft|ckpt|pt|bin)$/i, '') + if (!stem) return target + const containsHits = available.filter((item) => { + const base = loraBasename(item).toLowerCase() + return base.includes(stem) || loraIdentityKey(item).includes(stem) + }) + if (!containsHits.length) return target + const suffix = containsHits.find((item) => { + const base = loraBasename(item).toLowerCase() + return base.endsWith(identity) || base.endsWith(stem) || loraIdentityKey(item).endsWith(identity) + }) + return suffix || containsHits[0] +} + export function loraDisplayName(name: string) { return loraBasename(name).replace(/\.(safetensors|sft|ckpt|pt|bin)$/i, '') } @@ -106,7 +142,7 @@ export function normalizeLoraStack(raw: unknown): LoraStackItem[] { if (out.length >= LORA_STACK_MAX) break const item = parseStackEntry(entry) if (!item) continue - const key = loraKey(item.name) + const key = loraIdentityKey(item.name) if (seen.has(key)) continue seen.add(key) out.push(item) @@ -159,6 +195,7 @@ export function loraStacksEqual(a?: unknown, b?: unknown) { )) } +/** User LoRA picker only. `xaigen-*` names are visible only on xAIGen. */ export function filterLorasForStudio(names: string[], xaigenStudio: boolean) { const seen = new Set() const out: string[] = []