From bd82f5bd5f7ec7a810122160e04a621744ec32b6 Mon Sep 17 00:00:00 2001 From: Towsty Date: Thu, 10 Sep 2026 20:13:53 -0500 Subject: [PATCH] Add native H3 identity graph and Studio 2 prompt contracts --- server/assets/studio2_minimax_native.json | 338 ++++++++++++++++++++++ shared/studio2/contracts.mjs | 47 +++ shared/studio2/graphs.mjs | 33 +++ tests/studio2.test.mjs | 20 ++ 4 files changed, 438 insertions(+) create mode 100644 server/assets/studio2_minimax_native.json create mode 100644 shared/studio2/contracts.mjs create mode 100644 shared/studio2/graphs.mjs create mode 100644 tests/studio2.test.mjs diff --git a/server/assets/studio2_minimax_native.json b/server/assets/studio2_minimax_native.json new file mode 100644 index 0000000..d961b44 --- /dev/null +++ b/server/assets/studio2_minimax_native.json @@ -0,0 +1,338 @@ +{ + "92": { + "inputs": { + "filename_prefix": "video/MiniMax_H3", + "format": "auto", + "codec": "auto", + "video": [ + "172", + 0 + ] + }, + "class_type": "SaveVideo", + "_meta": { + "title": "Save Video" + } + }, + "155": { + "inputs": { + "prompt": "", + "width": 1344, + "height": 768, + "length": [ + "174", + 1 + ], + "ref_image_size": "max", + "clip": [ + "170", + 0 + ], + "vae": [ + "161", + 0 + ], + "audio_vae": [ + "162", + 0 + ], + "ref_images.ref_image_0": [ + "hero", + 0 + ] + }, + "class_type": "MiniMaxH3ReferenceToVideo", + "_meta": { + "title": "Reference to Video (MiniMax H3) – Identity Locked" + } + }, + "161": { + "inputs": { + "vae_name": "minimax_h3_video_vae_fp16.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "162": { + "inputs": { + "vae_name": "minimax_h3_audio_vae_fp32.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "163": { + "inputs": { + "samples": [ + "167", + 0 + ], + "vae": [ + "162", + 0 + ] + }, + "class_type": "VAEDecodeAudio", + "_meta": { + "title": "VAE Decode Audio" + } + }, + "164": { + "inputs": { + "samples": [ + "167", + 0 + ], + "vae": [ + "161", + 0 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "165": { + "inputs": { + "sampler_name": "res_multistep" + }, + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + } + }, + "166": { + "inputs": { + "scheduler": "simple", + "steps": [ + "178", + 0 + ], + "denoise": 1, + "model": [ + "177", + 0 + ] + }, + "class_type": "BasicScheduler", + "_meta": { + "title": "BasicScheduler" + } + }, + "167": { + "inputs": { + "noise": [ + "171", + 0 + ], + "guider": [ + "168", + 0 + ], + "sampler": [ + "165", + 0 + ], + "sigmas": [ + "166", + 0 + ], + "latent_image": [ + "155", + 1 + ] + }, + "class_type": "SamplerCustomAdvanced", + "_meta": { + "title": "SamplerCustomAdvanced" + } + }, + "168": { + "inputs": { + "model": [ + "177", + 0 + ], + "conditioning": [ + "guidance", + 0 + ] + }, + "class_type": "BasicGuider", + "_meta": { + "title": "Basic Guider" + } + }, + "169": { + "inputs": { + "unet_name": "minimax_h3_fl2va_pruned_int8_convrot.safetensors", + "weight_dtype": "default" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "170": { + "inputs": { + "clip_name": "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", + "type": "minimax", + "device": "default" + }, + "class_type": "CLIPLoader", + "_meta": { + "title": "Load CLIP" + } + }, + "171": { + "inputs": { + "noise_seed": 1 + }, + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + } + }, + "172": { + "inputs": { + "fps": 24, + "bit_depth": 8, + "color_space": "sRGB", + "images": [ + "164", + 0 + ], + "audio": [ + "163", + 0 + ] + }, + "class_type": "CreateVideo", + "_meta": { + "title": "Create Video" + } + }, + "174": { + "inputs": { + "expression": "max(5, round(a * 24)) + (5 - (max(5, round(a * 24)) % 17)) % 17", + "values.a": [ + "175", + 0 + ] + }, + "class_type": "ComfyMathExpression", + "_meta": { + "title": "Math Expression" + } + }, + "175": { + "inputs": { + "value": 2 + }, + "class_type": "PrimitiveFloat", + "_meta": { + "title": "Float (duration)" + } + }, + "176": { + "inputs": { + "lora_name": "minimax_h3_fl2v_turbo_8step_v1.0_comfyui_bf16.safetensors", + "strength_model": 1, + "model": [ + "169", + 0 + ] + }, + "class_type": "LoraLoaderModelOnly", + "_meta": { + "title": "Load LoRA" + } + }, + "177": { + "inputs": { + "switch": [ + "181", + 0 + ], + "on_false": [ + "169", + 0 + ], + "on_true": [ + "176", + 0 + ] + }, + "class_type": "ComfySwitchNode", + "_meta": { + "title": "If/Else Switch (Model)" + } + }, + "178": { + "inputs": { + "switch": [ + "181", + 0 + ], + "on_false": [ + "179", + 0 + ], + "on_true": [ + "180", + 0 + ] + }, + "class_type": "ComfySwitchNode", + "_meta": { + "title": "If/Else Switch (Steps)" + } + }, + "179": { + "inputs": { + "value": 20 + }, + "class_type": "PrimitiveInt", + "_meta": { + "title": "Int" + } + }, + "180": { + "inputs": { + "value": 6 + }, + "class_type": "PrimitiveInt", + "_meta": { + "title": "Int" + } + }, + "181": { + "inputs": { + "value": false + }, + "class_type": "PrimitiveBoolean", + "_meta": { + "title": "Boolean (Enable Lightning LoRA)" + } + }, + "hero": { + "class_type": "LoadImage", + "inputs": { + "image": "" + } + }, + "guidance": { + "class_type": "FluxGuidance", + "inputs": { + "conditioning": [ + "155", + 0 + ], + "guidance": 1.5 + } + } +} diff --git a/shared/studio2/contracts.mjs b/shared/studio2/contracts.mjs new file mode 100644 index 0000000..eb67334 --- /dev/null +++ b/shared/studio2/contracts.mjs @@ -0,0 +1,47 @@ +export const identityLock = ' is identity. Face, hair, glasses, outfit fully preserved. Start frame is composition only. Do not copy facial drift from the start frame.' +export const cameraFamilies = ['static', 'dolly in', 'dolly out', 'pan left', 'pan right', 'tilt up', 'tilt down', 'zoom in', 'zoom out', 'orbit left', 'orbit right'] +export function sectionsFrom(value = {}) { + if (typeof value === 'string') value = { action: value } + const sections = Object.fromEntries(['action','camera','soundscape','dialogue','music','locks'].map(key => [key, String(value[key] || '')])) + const chip = value.cameraChip + if (chip && cameraFamilies.includes(chip.family)) sections.cameraChip = { family: chip.family, amplitude: ['subtle','moderate','large'].includes(chip.amplitude) ? chip.amplitude : 'subtle', speed: ['slow','medium','fast'].includes(chip.speed) ? chip.speed : 'slow' } + return sections +} +export function compilePrompt(value, video = false) { + const s = sectionsFrom(value), c = s.cameraChip + const camera = c ? (c.family === 'static' ? 'Static shot. Camera locked on a tripod.' : `Camera ${c.family}, ${c.amplitude} movement, ${c.speed} speed.`) : s.camera + return [s.action, camera && `Camera: ${camera}`, s.soundscape && `Soundscape: ${s.soundscape}`, s.dialogue && `Dialogue: (S1) [English]${s.dialogue}`, s.music && `Music: ${s.music}`, s.locks && `Locks: ${s.locks}`, video ? identityLock : ''].filter(Boolean).join('\n\n') +} +export function restoreInput(item) { + const saved = item.studio2 || item + return { ...saved, promptSections: sectionsFrom(saved.promptSections || { action: item.prompt || saved.compiledPrompt || '' }) } +} +export function validateRequest(raw, xaigen = false) { + const fail = message => { throw Object.assign(new Error(message), {statusCode:400}) } + const mode = raw.mode || 'generate', engine = raw.engine || 'flux' + if (!['generate','edit','compose','refine','iterate','video','extend'].includes(mode)) fail('Unknown task.') + const video = ['video','extend'].includes(mode) + if (!(video ? ['minimax','ltx'] : ['flux','krea']).includes(engine)) fail('Engine does not support this task.') + if (engine === 'ltx' && !xaigen) fail('LTX is available only in xAIGen.') + if (!raw.folderId) fail('Choose a folder.') + if ((video || ['edit','compose','iterate'].includes(mode)) && !raw.identityStillId) fail('Choose a sticky hero still.') + if (mode === 'compose' && !raw.imageBId) fail('Compose requires a second still.') + if (mode === 'refine' && !raw.maskId) fail('Refine requires a mask.') + if (['edit','compose','refine'].includes(mode) && !raw.imageAId) fail('Choose the source still.') + if (mode === 'extend' && !raw.startClipId) fail('Choose the video to extend.') + const promptSections = sectionsFrom(raw.promptSections || {action:raw.prompt || ''}) + if (!promptSections.action.trim()) fail('Enter an action prompt.') + const number=(key, fallback, min, max) => { const n=Number(raw.settings?.[key] ?? fallback); if (!Number.isFinite(n)||nmax) fail(`Invalid ${key}.`); return n } + return { mode, engine, folderId:String(raw.folderId), identityStillId:raw.identityStillId || '', imageAId:raw.imageAId || '', imageBId:raw.imageBId || '', maskId:raw.maskId || '', startClipId:raw.startClipId || '', endStillId:raw.endStillId || '', + startFrameSource: raw.startClipId ? {kind:'previous-last-frame',clipId:raw.startClipId} : {kind:'still',stillId:raw.imageAId || ''}, + promptSections, compiledPrompt:compilePrompt(promptSections,video), + settings: { width:number('width',1344,256,4096),height:number('height',768,256,4096),steps:number('steps',video?8:24,1,100),cfg:number('cfg',video?1.5:4,0,20),seed:number('seed',Math.floor(Math.random()*2147483647),0,2147483647),duration:number('duration',5,1,15),turbo:raw.settings?.turbo !== false, loraStack:Array.isArray(raw.settings?.loraStack)?raw.settings.loraStack:[] }, + guides: (Array.isArray(raw.guides)?raw.guides:[]).slice(0,8).map(g=>{ if (!g.stillId || !Number.isInteger(g.frame) || g.frame<1) fail('Each guide requires a still and positive frame index.'); return {stillId:String(g.stillId),frame:g.frame} }), + batch: (Array.isArray(raw.batch)?raw.batch:[]).slice(0,30).map(s=>sectionsFrom(typeof s==='string'?{action:s}:s)) } +} +export function scopedFile(file, prefix) { + const clean = s => String(s || '').replaceAll('\\','/') + const path = [clean(file.subfolder),clean(file.filename)].filter(Boolean).join('/') + const p=clean(prefix).replace(/\/$/,'') + return !!p && !path.split('/').some(s=>s==='..'||s==='.') && !path.startsWith('/') && path.startsWith(p + '/') +} diff --git a/shared/studio2/graphs.mjs b/shared/studio2/graphs.mjs new file mode 100644 index 0000000..e52e9b9 --- /dev/null +++ b/shared/studio2/graphs.mjs @@ -0,0 +1,33 @@ +export function nativeVideoGraph(template, request, files, prefix) { + if (!files.hero) throw new Error('Identity still is required; never substitute the start frame.') + const g=structuredClone(template), s=request.settings + g.hero.inputs.image=files.hero + Object.assign(g['155'].inputs,{prompt:request.compiledPrompt,width:s.width,height:s.height,length:Math.ceil((s.duration*24-5)/17)*17+5,ref_image_size:'max'}) + g['155'].inputs['ref_images.ref_image_0']=['hero',0] + let conditioning=['155',0] + for (const [name,image,frame] of [['start',files.start,0],['end',files.end,-1],...(files.guides||[]).map((f,i)=>[`guide${i}`,f.image,f.frame])]) { + if (!image) continue + g[`load_${name}`]={class_type:'LoadImage',inputs:{image}} + g[name]={class_type:'MiniMaxH3AddGuide',inputs:{positive:conditioning,latent:['155',1],vae:['161',0],image:[`load_${name}`,0],frame_idx:frame}} + conditioning=[name,0] + } + g.guidance.inputs={conditioning,guidance:s.cfg} + g['171'].inputs.noise_seed=s.seed + g['179'].inputs.value=s.steps;g['180'].inputs.value=s.steps;g['181'].inputs.value=s.turbo + g['92'].inputs.filename_prefix=prefix+'/video' + g.anchor_frame={class_type:'ImageFromBatch',inputs:{image:['164',0],batch_index:-1,length:1}} + g.anchor_save={class_type:'SaveImage',inputs:{images:['anchor_frame',0],filename_prefix:prefix+'/anchor'}} + return g +} +export function attachHeroReference(g, image) { + if (!image) return g + const guider = g['18']; if (!guider) throw new Error('Klein reference conditioning is missing.') + g.hero_load={class_type:'LoadImage',inputs:{image}} + g.hero_encode={class_type:'VAEEncode',inputs:{pixels:['hero_load',0],vae:['6',0]}} + for (const name of ['positive','negative']) { + if (!guider.inputs[name]) continue + g[`hero_${name}`]={class_type:'ReferenceLatent',inputs:{conditioning:guider.inputs[name],latent:['hero_encode',0]}} + guider.inputs[name]=[`hero_${name}`,0] + } + return g +} diff --git a/tests/studio2.test.mjs b/tests/studio2.test.mjs new file mode 100644 index 0000000..a52ce47 --- /dev/null +++ b/tests/studio2.test.mjs @@ -0,0 +1,20 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import {validateRequest,compilePrompt,restoreInput,scopedFile} from '../shared/studio2/contracts.mjs' +import {nativeVideoGraph} from '../shared/studio2/graphs.mjs' +const template=JSON.parse(fs.readFileSync(new URL('../server/assets/studio2_minimax_native.json',import.meta.url))) +const base={folderId:'folder',mode:'video',engine:'minimax',identityStillId:'hero',promptSections:{action:'walk'}} +test('ten shots retain hero as Picture 1 while each start frame changes',()=>{ + for(let i=0;i<10;i++) {const r=validateRequest({...base,startClipId:'clip'+i});const g=nativeVideoGraph(template,r,{hero:'hero.png',start:`last${i}.png`},'preview/job');assert.equal(g.hero.inputs.image,'hero.png');assert.deepEqual(g['155'].inputs['ref_images.ref_image_0'],['hero',0]);assert.equal(g.load_start.inputs.image,`last${i}.png`);assert.equal(r.identityStillId,'hero');assert.equal(g['155'].inputs.ref_image_size,'max')} +}) +test('compose without B and refine without mask are 400',()=>{ + for(const mode of ['compose','refine']) assert.throws(()=>validateRequest({...base,mode,engine:'flux',imageAId:'a'}),e=>e.statusCode===400) +}) +test('Use as input round-trips camera chip and never invents one for flat text',()=>{ + const r=validateRequest({...base,promptSections:{action:'walk',cameraChip:{family:'dolly in',amplitude:'subtle',speed:'slow'}}}); assert.deepEqual(restoreInput({studio2:r}).promptSections,r.promptSections);assert.equal(restoreInput({prompt:'old text'}).promptSections.cameraChip,undefined);assert.match(compilePrompt(r.promptSections),/dolly in/) +}) +test('purge accepts only exact preview path boundary',()=>{ + assert.equal(scopedFile({subfolder:'video/MiniMax_H3_preview/job',filename:'a.png'},'video/MiniMax_H3_preview'),true) + for(const subfolder of ['video/MiniMax_H3','video/MiniMax_H3_preview_other','video/MiniMax_H3_preview/../MiniMax_H3']) assert.equal(scopedFile({subfolder,filename:'a.png'},'video/MiniMax_H3_preview'),false) +})