Sample Klein edits from source and bind Studio 2 extensions correctly

This commit is contained in:
Towsty
2026-09-11 12:23:34 -05:00
parent 8fe69272c3
commit 28b919ecee
11 changed files with 118 additions and 27 deletions
+9 -7
View File
@@ -36,10 +36,12 @@
<div class="flex justify-between"><h2 class="text-xl font-semibold">Prompt</h2><button class="text-sm text-amber-200" @click="advanced=!advanced">{{ advanced?'Simple':'Advanced' }}</button></div> <div class="flex justify-between"><h2 class="text-xl font-semibold">Prompt</h2><button class="text-sm text-amber-200" @click="advanced=!advanced">{{ advanced?'Simple':'Advanced' }}</button></div>
<label class="block">Action<textarea v-model="form.promptSections.action" rows="4" class="mt-1 w-full rounded-xl bg-zinc-900 p-3" placeholder="What happens in this shot?" /></label> <label class="block">Action<textarea v-model="form.promptSections.action" rows="4" class="mt-1 w-full rounded-xl bg-zinc-900 p-3" placeholder="What happens in this shot?" /></label>
<Studio2PromptControls v-if="video" v-model="form.promptSections" /> <Studio2PromptControls v-if="video" v-model="form.promptSections" />
<label class="block">Locks<textarea v-model="form.promptSections.locks" rows="2" class="mt-1 w-full rounded-xl bg-zinc-900 p-3" /></label> <label v-if="video" class="block">Locks<textarea v-model="form.promptSections.locks" rows="2" class="mt-1 w-full rounded-xl bg-zinc-900 p-3" /></label>
<details class="rounded-xl border border-white/10 p-3"><summary>{{ video?'More shots':'Iteration prompts' }}</summary><textarea v-model="batchText" rows="5" class="mt-2 w-full rounded-xl bg-zinc-900 p-3" placeholder="One action prompt per line. Hero stays attached to every output." /></details> <details class="rounded-xl border border-white/10 p-3"><summary>{{ video?'More shots':'Iteration prompts' }}</summary><textarea v-model="batchText" rows="5" class="mt-2 w-full rounded-xl bg-zinc-900 p-3" placeholder="One action prompt per line. Hero stays attached to every output." /></details>
<button class="text-sm text-amber-200" @click="settingsOpen=true">Generation settings</button> <button class="text-sm text-amber-200" @click="settingsOpen=true">Generation settings</button>
<p v-if="error" role="alert" class="text-sm text-red-300">{{ error }}</p> <p v-if="error" role="alert" class="text-sm text-red-300">{{ error }}</p>
<p v-if="form.mode==='extend'" class="text-sm text-amber-200">{{ form.startClipId ? 'Extend · '+(clips.find((c:any)=>c.id===form.startClipId)?.name || form.startClipId)+' · last frame' : 'Select the clip in the family strip or Previous clip.' }}</p>
<p v-if="!video && form.engine==='krea'" class="text-sm text-zinc-400">Krea does not attach the hero identity reference.</p>
<button :disabled="submitting" @click="queue" class="w-full rounded-2xl bg-amber-300 p-4 font-semibold text-black disabled:opacity-50">{{ submitting?'Queuing…':'Queue' }}</button> <button :disabled="submitting" @click="queue" class="w-full rounded-2xl bg-amber-300 p-4 font-semibold text-black disabled:opacity-50">{{ submitting?'Queuing…':'Queue' }}</button>
<Studio2JobProgress v-for="job in folderJobs.slice(0,6)" :key="job.id" :job="job" :now="now" @select="focusJob(job)" @cancel="cancel(job)" /> <Studio2JobProgress v-for="job in folderJobs.slice(0,6)" :key="job.id" :job="job" :now="now" @select="focusJob(job)" @cancel="cancel(job)" />
<details v-if="advanced"><summary>Compiled prompt</summary><pre class="whitespace-pre-wrap text-xs text-zinc-400">{{ compiled }}</pre></details> <details v-if="advanced"><summary>Compiled prompt</summary><pre class="whitespace-pre-wrap text-xs text-zinc-400">{{ compiled }}</pre></details>
@@ -55,7 +57,7 @@
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import {compilePrompt,sectionsFrom} from '~/shared/studio2/contracts.mjs' import {compilePrompt,sectionsFrom} from '~/shared/studio2/contracts.mjs'
import {aspectOptions,resolveSize,bindClip,restoreAsset,extendSelected} from '~/shared/studio2/input.mjs' import {aspectOptions,resolveSize,bindClip,restoreAsset,extendSelected,changeSource} from '~/shared/studio2/input.mjs'
import {folderAssets,newestOutput,assetLabel} from '~/shared/studio2/viewer.mjs' import {folderAssets,newestOutput,assetLabel} from '~/shared/studio2/viewer.mjs'
import {applyUploadedStill} from '~/shared/studio2/upload.mjs' import {applyUploadedStill} from '~/shared/studio2/upload.mjs'
const uploading=ref(false),uploadMessage=ref('') const uploading=ref(false),uploadMessage=ref('')
@@ -76,14 +78,14 @@ const sourceAsset=computed(()=>form.startClipId?clips.value.find((c:any)=>c.id==
const sourceSize=computed(()=>measured[sourceAsset.value?.id] || sourceAsset.value) const sourceSize=computed(()=>measured[sourceAsset.value?.id] || sourceAsset.value)
const resolvedSize=computed(()=>resolveSize(form.settings.aspect,sourceSize.value)) const resolvedSize=computed(()=>resolveSize(form.settings.aspect,sourceSize.value))
watch(()=>sourceAsset.value?.id,id=>{const asset=sourceAsset.value;if(!import.meta.client||!asset||measured[id])return;const media=asset.id===form.startClipId?document.createElement('video'):new Image();if(media.tagName==='VIDEO')(media as HTMLVideoElement).preload='metadata';const done=()=>{measured[asset.id]={width:(media as HTMLVideoElement).videoWidth||(media as HTMLImageElement).naturalWidth,height:(media as HTMLVideoElement).videoHeight||(media as HTMLImageElement).naturalHeight}};media.addEventListener(media.tagName==='VIDEO'?'loadedmetadata':'load',done,{once:true});media.src=media.tagName==='VIDEO'?`/api/library/clips/${asset.id}/video`:stillUrl(asset.id)}) watch(()=>sourceAsset.value?.id,id=>{const asset=sourceAsset.value;if(!import.meta.client||!asset||measured[id])return;const media=asset.id===form.startClipId?document.createElement('video'):new Image();if(media.tagName==='VIDEO')(media as HTMLVideoElement).preload='metadata';const done=()=>{measured[asset.id]={width:(media as HTMLVideoElement).videoWidth||(media as HTMLImageElement).naturalWidth,height:(media as HTMLVideoElement).videoHeight||(media as HTMLImageElement).naturalHeight}};media.addEventListener(media.tagName==='VIDEO'?'loadedmetadata':'load',done,{once:true});media.src=media.tagName==='VIDEO'?`/api/library/clips/${asset.id}/video`:stillUrl(asset.id)})
function chooseTask(task:string){if(task==='extend'){Object.assign(form,extendSelected(form,selected.value?.kind==='clip'?selected.value:null));return}const was=video.value;form.mode=task;form.startClipId='';if(video.value!==was){form.engine=video.value?'minimax':'flux';form.settings.steps=video.value?8:24;form.settings.cfg=video.value?1.5:4}} function chooseTask(task:string){if(task==='extend'){Object.assign(form,extendSelected(form,selected.value?.kind==='clip'?selected.value:null,useRuntimeConfig().public.studio==='xaigen'));return}const was=video.value;form.mode=task;form.startClipId='';if(video.value!==was){form.engine=video.value?'minimax':'flux';form.settings.steps=video.value?8:24;form.settings.cfg=video.value?1.5:4}}
function chooseClip(id:string){if(!id){form.startClipId='';return}const clip=clips.value.find((c:any)=>c.id===id);if(clip)Object.assign(form,bindClip(form,clip))} function chooseClip(id:string){if(!id){form.startClipId='';return}const clip=clips.value.find((c:any)=>c.id===id);if(clip)Object.assign(form,bindClip(form,clip,useRuntimeConfig().public.studio==='xaigen'))}
function selectOutput(item:any){selected.value=item} function selectOutput(item:any){selected.value=item;if(form.mode==='extend' && item?.kind==='clip')Object.assign(form,extendSelected(form,item,useRuntimeConfig().public.studio==='xaigen'))}
function sourceChanged(){form.startClipId='';if(form.mode==='extend')form.mode='video';form.settings.aspect='auto'} function sourceChanged(){Object.assign(form,changeSource(form))}
function restore(item:any){Object.assign(form,restoreAsset(form,item));form.guides=form.guides||[];batchText.value=''} function restore(item:any){Object.assign(form,restoreAsset(form,item));form.guides=form.guides||[];batchText.value=''}
function focusJob(job:any){selectOutput(job.outputs?.at(-1)||null)} function focusJob(job:any){selectOutput(job.outputs?.at(-1)||null)}
async function loadLoras(){loraNames.value=await $fetch('/api/loras').catch(()=>({image:[],video:[]}))} async function loadLoras(){loraNames.value=await $fetch('/api/loras').catch(()=>({image:[],video:[]}))}
async function queue(){submitting.value=true;error.value='';try{if(form.settings.seedMode!=='locked')form.settings.seed=randomSeed();const queued=await $fetch<any>('/api/studio-2/jobs',{method:'POST',body:{...form,settings:{...form.settings,...resolvedSize.value},batch:batchText.value.split('\n').map(s=>s.trim()).filter(Boolean)}});form.settings.seed=queued.seed;form.settings.width=queued.width;form.settings.height=queued.height;await refresh()}catch(e:any){error.value=e.data?.statusMessage||e.message}finally{submitting.value=false}} async function queue(){submitting.value=true;error.value='';try{if(form.settings.seedMode!=='locked')form.settings.seed=randomSeed();const queued=await $fetch<any>('/api/studio-2/jobs',{method:'POST',body:{...form,promptSections:video.value?form.promptSections:{action:form.promptSections.action},settings:{...form.settings,...resolvedSize.value},batch:batchText.value.split('\n').map(s=>s.trim()).filter(Boolean)}});form.settings.seed=queued.seed;form.settings.width=queued.width;form.settings.height=queued.height;await refresh()}catch(e:any){error.value=e.data?.statusMessage||e.message}finally{submitting.value=false}}
async function cancel(job:any){await $fetch('/api/studio-2/cancel',{method:'POST',body:{id:job.id}});await refresh()} async function cancel(job:any){await $fetch('/api/studio-2/cancel',{method:'POST',body:{id:job.id}});await refresh()}
async function upload(event:Event,target:'source'|'hero'){ async function upload(event:Event,target:'source'|'hero'){
const input=event.target as HTMLInputElement,file=input.files?.[0];if(!file)return const input=event.target as HTMLInputElement,file=input.files?.[0];if(!file)return
+3
View File
@@ -8,6 +8,9 @@ const props=defineProps<{metadata?:any}>()
const rows=computed(()=>{const m=props.metadata||{},s=m.settings||{};const time=(v:any)=>v?new Date(v).toLocaleString():null;return [ const rows=computed(()=>{const m=props.metadata||{},s=m.settings||{};const time=(v:any)=>v?new Date(v).toLocaleString():null;return [
{label:'Kind / mode',value:[m.kind,m.mode].filter(Boolean).join(' / ')}, {label:'Kind / mode',value:[m.kind,m.mode].filter(Boolean).join(' / ')},
{label:'Engine',value:m.engine},{label:'Graph ID',value:m.graphId}, {label:'Engine',value:m.engine},{label:'Graph ID',value:m.graphId},
{label:'Sample latent',value:m.sampleLatent},{label:'Edit denoise',value:m.sampleDenoise},
{label:'Source still',value:m.sourceStillId || m.imageAId || 'None'},
{label:'Hero reference attached',value:m.heroReferenceAttached==null?'Not recorded':m.heroReferenceAttached?'Yes':'No'},
{label:'Seed',value:s.seed},{label:'Steps',value:s.steps},{label:'CFG',value:s.cfg}, {label:'Seed',value:s.seed},{label:'Steps',value:s.steps},{label:'CFG',value:s.cfg},
{label:'Size',value:s.width&&s.height?`${s.width} × ${s.height}`:null},{label:'Duration',value:s.duration!=null?`${s.duration}s`:null}, {label:'Size',value:s.width&&s.height?`${s.width} × ${s.height}`:null},{label:'Duration',value:s.duration!=null?`${s.duration}s`:null},
{label:'LoRAs',value:s.loraStack?.length?s.loraStack.map((l:any)=>`${l.name} · ${l.strengthModel}`).join('\n'):'None'}, {label:'LoRAs',value:s.loraStack?.length?s.loraStack.map((l:any)=>`${l.name} · ${l.strengthModel}`).join('\n'):'None'},
+15 -4
View File
@@ -5,7 +5,7 @@ import { queueSeeds } from '~/shared/studio2/seed.mjs';
import { readFileSync, mkdirSync, existsSync, unlinkSync } from 'node:fs'; import { readFileSync, mkdirSync, existsSync, unlinkSync } from 'node:fs';
import { join } from 'node:path'; import { join } from 'node:path';
import template from '../../assets/studio2_minimax_native.json'; import template from '../../assets/studio2_minimax_native.json';
import { nativeVideoGraph, attachHeroReference, applyResolvedImageSize } from '~/shared/studio2/graphs.mjs'; import { nativeVideoGraph, attachHeroReference, applyResolvedImageSize, sampleKleinSource } from '~/shared/studio2/graphs.mjs';
import { compilePrompt, scopedFile } from '~/shared/studio2/contracts.mjs'; import { compilePrompt, scopedFile } from '~/shared/studio2/contracts.mjs';
import { createJob, restoreJob, getJob, emitJob, type Job } from '../jobs'; import { createJob, restoreJob, getJob, emitJob, type Job } from '../jobs';
import { markStudioLive, onLiveVideoSettled, type StudioJob } from '../studioQueue'; import { markStudioLive, onLiveVideoSettled, type StudioJob } from '../studioQueue';
@@ -68,7 +68,15 @@ async function prepareGraph(r: any) {
const load = async (id: string, label: string) => id ? upload(r, label, readFileSync(stillPath(r.owner, getStill(r.owner, id).id))) : ''; const load = async (id: string, label: string) => id ? upload(r, label, readFileSync(stillPath(r.owner, getStill(r.owner, id).id))) : '';
await resolveRequestSize(r.owner,q); await resolveRequestSize(r.owner,q);
saveRecord(r); saveRecord(r);
const hero = q.lockFace === false && q.lockOutfit === false ? '' : await load(q.identityStillId, 'hero'), a = await load(q.imageAId, 'source'), b = await load(q.imageBId, 'compose'), mask = await load(q.maskId, 'mask'); const video=['video','extend'].includes(q.mode);
const sourceId=q.imageAId || (!video && q.engine==='flux' && ['edit','iterate'].includes(q.mode) ? q.identityStillId : '');
const usesSourceLatent=!video && q.engine==='flux' && ['edit','iterate'].includes(q.mode) && !!sourceId;
const a=await load(sourceId,'source'),b=await load(q.imageBId,'compose'),mask=await load(q.maskId,'mask');
const attachHero=(video || q.engine==='flux') && (q.lockFace!==false || q.lockOutfit!==false) && !(!video && q.mode!=='generate' && sourceId && sourceId===q.identityStillId);
const hero=attachHero ? await load(q.identityStillId,'hero') : '';
r.sourceStillId=sourceId || null;
r.heroReferenceAttached=!!hero || (!video && q.engine==='flux' && q.mode!=='generate' && !!sourceId && sourceId===q.identityStillId);
let graph: any; let graph: any;
if (['video', 'extend'].includes(q.mode)) { if (['video', 'extend'].includes(q.mode)) {
if (q.engine !== 'minimax') if (q.engine !== 'minimax')
@@ -97,9 +105,12 @@ async function prepareGraph(r: any) {
const assets = found ? { kreaUnetName: found.unet, kreaClipName: found.clip, kreaVaeName: found.vae, kreaConceptLora: found.conceptLora } : {}; const assets = found ? { kreaUnetName: found.unet, kreaClipName: found.clip, kreaVaeName: found.vae, kreaConceptLora: found.conceptLora } : {};
graph = buildImageV2Workflow({ ...s, ...assets, engine: q.engine, mode, task: 'scene', prompt: q.compiledPrompt, imageAName: a, imageBName: mode === 'compose' ? b : undefined, maskName: mode === 'refine' ? mask : undefined, filenamePrefix: prefix + '/image' }).graph; graph = buildImageV2Workflow({ ...s, ...assets, engine: q.engine, mode, task: 'scene', prompt: q.compiledPrompt, imageAName: a, imageBName: mode === 'compose' ? b : undefined, maskName: mode === 'refine' ? mask : undefined, filenamePrefix: prefix + '/image' }).graph;
applyResolvedImageSize(graph,s); applyResolvedImageSize(graph,s);
if (usesSourceLatent) sampleKleinSource(graph,.65);
if (q.engine === 'flux') if (q.engine === 'flux')
attachHeroReference(graph, hero); attachHeroReference(graph, hero);
else if (hero && ['edit', 'compose', 'iterate'].includes(q.mode)) r.sampleLatent=usesSourceLatent?'source image latent':q.mode==='refine'?'masked source latent':'empty latent';
r.sampleDenoise=usesSourceLatent ? .65 : null;
if (q.engine !== 'flux' && hero && ['edit', 'compose', 'iterate'].includes(q.mode))
throw new Error('Krea native hero-reference binding is not supported by this graph. Select Klein explicitly; no fallback is performed.'); throw new Error('Krea native hero-reference binding is not supported by this graph. Select Klein explicitly; no fallback is performed.');
await ensureComfyLoraNames('image'); await ensureComfyLoraNames('image');
resolveGraphLoraNames(graph, 'image'); resolveGraphLoraNames(graph, 'image');
@@ -212,7 +223,7 @@ async function run(r: any, job: Job) {
saveRecord(r); saveRecord(r);
r.savedAt = Date.now(); r.savedAt = Date.now();
r.wallTime = (r.savedAt - r.startedAt) / 1000; r.wallTime = (r.savedAt - r.startedAt) / 1000;
const metadata = { ...structuredClone(q), id: r.id, kind: video ? 'video' : 'image', graphId: r.graphId, promptId: r.promptId, queuedAt: r.queuedAt, startedAt: r.startedAt, savedAt: r.savedAt, gpuSeconds: r.gpuSeconds, wallTime: r.wallTime, outputWidth: asset.width || s.width, outputHeight: asset.height || s.height, purgeResult: 'Left on host' }; const metadata = { ...structuredClone(q), id: r.id, sourceStillId:r.sourceStillId,heroReferenceAttached:r.heroReferenceAttached,sampleLatent:r.sampleLatent,sampleDenoise:r.sampleDenoise,kind: video ? 'video' : 'image', graphId: r.graphId, promptId: r.promptId, queuedAt: r.queuedAt, startedAt: r.startedAt, savedAt: r.savedAt, gpuSeconds: r.gpuSeconds, wallTime: r.wallTime, outputWidth: asset.width || s.width, outputHeight: asset.height || s.height, purgeResult: 'Left on host' };
await attachStudio2Metadata(r.owner, asset.id, metadata); await attachStudio2Metadata(r.owner, asset.id, metadata);
if (!r.outputs.some((o:any)=>o.id===asset.id)) r.outputs.push({id:asset.id,kind:video?'clip':'still',studio2:metadata}); if (!r.outputs.some((o:any)=>o.id===asset.id)) r.outputs.push({id:asset.id,kind:video?'clip':'still',studio2:metadata});
saveRecord(r); saveRecord(r);
+15 -6
View File
@@ -10,7 +10,15 @@ export function appearanceLock({lockFace = true, lockOutfit = true} = {}) {
return 'The start frame is the only appearance reference for person and clothing, as well as pose, place, and camera. Do not use the hero as an appearance reference.' return 'The start frame is the only appearance reference for person and clothing, as well as pose, place, and camera. Do not use the hero as an appearance reference.'
} }
export function compilePrompt(value, video = false, locks = {}) { export function compilePrompt(value, video = false, locks = {}) {
return [compileSections(value), video ? appearanceLock(locks) : ''].filter(Boolean).join('\n\n') if (video) return [compileSections(value),appearanceLock(locks)].filter(Boolean).join('\n\n')
const hasImage=!!(locks.imageAId || locks.identityStillId)
const attached=hasImage && locks.engine !== 'krea'
const reference=locks.mode!=='generate' && locks.identityStillId && locks.imageAId && locks.identityStillId!==locks.imageAId ? 'second (hero) reference' : 'image reference'
return [sectionsFrom(value).action,
attached && locks.lockFace !== false ? `Preserve the person’s face, glasses, hair, and body identity from the ${reference}.` : '',
attached && locks.lockOutfit !== false ? `Preserve the wardrobe from the ${reference}.` : '',
attached && locks.lockOutfit === false ? 'Clothing may follow the source and requested scene; do not restore the hero outfit.' : ''
].filter(Boolean).join('\n\n')
} }
export function restoreInput(item) { export function restoreInput(item) {
const saved = structuredClone(item.studio2 || item) const saved = structuredClone(item.studio2 || item)
@@ -28,12 +36,13 @@ export function validateRequest(raw, xaigen = false) {
if (Array.isArray(raw.batch) && raw.batch.length>30) fail('A batch supports up to 31 shots including the first prompt.') if (Array.isArray(raw.batch) && raw.batch.length>30) fail('A batch supports up to 31 shots including the first prompt.')
if (Array.isArray(raw.guides) && raw.guides.length>8) fail('Use up to eight guide stills.') if (Array.isArray(raw.guides) && raw.guides.length>8) fail('Use up to eight guide stills.')
if (!raw.folderId) fail('Choose a folder.') if (!raw.folderId) fail('Choose a folder.')
if ((video || ['edit','compose','iterate'].includes(mode)) && (raw.lockFace !== false || raw.lockOutfit !== false) && !raw.identityStillId) fail('Choose a sticky hero still.') if ((video || ['edit','compose','iterate'].includes(mode)) && (raw.lockFace !== false || raw.lockOutfit !== false) && !raw.identityStillId && (video || !raw.imageAId)) fail('Choose a sticky hero still.')
if (mode === 'compose' && !raw.imageBId) fail('Compose requires a second still.') if (mode === 'compose' && !raw.imageBId) fail('Compose requires a second still.')
if (mode === 'refine' && !raw.maskId) fail('Refine requires a mask.') if (mode === 'refine' && !raw.maskId) fail('Refine requires a mask.')
if (['edit','compose','refine'].includes(mode) && !raw.imageAId) fail('Choose the source still.') if (['edit','compose','refine'].includes(mode) && !raw.imageAId && !(mode==='edit' && engine==='flux' && raw.identityStillId)) fail('Choose the source still.')
if (mode === 'extend' && !raw.startClipId) fail('Choose the video to extend.') if (mode === 'extend' && !raw.startClipId) fail('Select the clip in the family strip or Previous clip.')
const promptSections = sectionsFrom(raw.promptSections || {action:raw.prompt || ''}) const allSections = sectionsFrom(raw.promptSections || {action:raw.prompt || ''})
const promptSections = video ? allSections : {action:allSections.action}
if (!promptSections.action.trim()) fail('Enter an action prompt.') if (!promptSections.action.trim()) fail('Enter an action prompt.')
const number=(key, fallback, min, max) => { const n=Number(raw.settings?.[key] ?? fallback); if (!Number.isFinite(n)||n<min||n>max) fail(`Invalid ${key}.`); return n } const number=(key, fallback, min, max) => { const n=Number(raw.settings?.[key] ?? fallback); if (!Number.isFinite(n)||n<min||n>max) fail(`Invalid ${key}.`); return n }
for (const key of ['steps']) if (!Number.isInteger(number(key,key==='steps'?24:1,0,2147483647))) fail(`Invalid ${key}.`) for (const key of ['steps']) if (!Number.isInteger(number(key,key==='steps'?24:1,0,2147483647))) fail(`Invalid ${key}.`)
@@ -51,7 +60,7 @@ export function validateRequest(raw, xaigen = false) {
promptSections, compiledPrompt:compilePrompt(promptSections,video,raw), promptSections, compiledPrompt:compilePrompt(promptSections,video,raw),
settings: { aspect:['auto','16:9','9:16','1:1'].includes(raw.settings?.aspect)?raw.settings.aspect:'auto', fps:number('fps',24,1,60), width:number('width',1344,256,4096),height:number('height',768,256,4096),steps:number('steps',video?8:24,1,100),cfg:number('cfg',video?1.5:4,0,20),seedMode:raw.settings?.seedMode==='locked'?'locked':'random',seed:shotSeeds[0],duration:number('duration',5,1,15),turbo:raw.settings?.turbo !== false, loraStack:Array.isArray(raw.settings?.loraStack)?raw.settings.loraStack:[] }, settings: { aspect:['auto','16:9','9:16','1:1'].includes(raw.settings?.aspect)?raw.settings.aspect:'auto', fps:number('fps',24,1,60), width:number('width',1344,256,4096),height:number('height',768,256,4096),steps:number('steps',video?8:24,1,100),cfg:number('cfg',video?1.5:4,0,20),seedMode:raw.settings?.seedMode==='locked'?'locked':'random',seed:shotSeeds[0],duration:number('duration',5,1,15),turbo:raw.settings?.turbo !== false, loraStack:Array.isArray(raw.settings?.loraStack)?raw.settings.loraStack:[] },
guides: (Array.isArray(raw.guides)?raw.guides:[]).slice(0,8).map(g=>{ if (!g.stillId || !Number.isInteger(g.frame) || g.frame<1 || g.frame>=frames) fail('Each guide requires a still and positive frame index.'); return {stillId:String(g.stillId),frame:g.frame} }), guides: (Array.isArray(raw.guides)?raw.guides:[]).slice(0,8).map(g=>{ if (!g.stillId || !Number.isInteger(g.frame) || g.frame<1 || g.frame>=frames) fail('Each guide requires a still and positive frame index.'); return {stillId:String(g.stillId),frame:g.frame} }),
batch: (Array.isArray(raw.batch)?raw.batch:[]).slice(0,30).map(s=>{const next=sectionsFrom({...promptSections,...(typeof s==='string'?{action:s}:s)});if(!next.action.trim())fail('Every shot needs an action prompt.');return next}) } batch: (Array.isArray(raw.batch)?raw.batch:[]).slice(0,30).map(s=>{const next=sectionsFrom({...promptSections,...(typeof s==='string'?{action:s}:s)});if(!next.action.trim())fail('Every shot needs an action prompt.');return video?next:{action:next.action}}) }
} }
export function scopedFile(file, prefix) { export function scopedFile(file, prefix) {
const clean = s => String(s || '').replaceAll('\\','/') const clean = s => String(s || '').replaceAll('\\','/')
+10
View File
@@ -34,6 +34,16 @@ export function attachHeroReference(g, image) {
} }
return g return g
} }
export function sampleKleinSource(graph, denoise = .65) {
if (!graph['11'] || !graph['19'] || !graph['17']) throw new Error('Klein source sampling nodes are missing.')
graph['19'].inputs.latent_image=['11',0]
delete graph['14']
const steps=graph['17'].inputs.steps
graph['17'].inputs.steps=Math.ceil(steps/denoise)
graph.source_sigmas={class_type:'SplitSigmasDenoise',inputs:{sigmas:['17',0],denoise}}
graph['19'].inputs.sigmas=['source_sigmas',1]
return graph
}
/** The shared image builders have their own defaults and megapixel scalers. /** The shared image builders have their own defaults and megapixel scalers.
* Studio 2 must send its already resolved canvas size through those nodes. */ * Studio 2 must send its already resolved canvas size through those nodes. */
export function applyResolvedImageSize(graph, {width,height}) { export function applyResolvedImageSize(graph, {width,height}) {
+16 -5
View File
@@ -7,21 +7,28 @@ export function resolveSize(aspect, source, fallback = {width:1344,height:768})
const factor = Math.min(1344/Math.max(source.width,source.height),768/Math.min(source.width,source.height)) const factor = Math.min(1344/Math.max(source.width,source.height),768/Math.min(source.width,source.height))
return {width:Math.max(256,Math.round(source.width*factor/32)*32),height:Math.max(256,Math.round(source.height*factor/32)*32)} return {width:Math.max(256,Math.round(source.width*factor/32)*32),height:Math.max(256,Math.round(source.height*factor/32)*32)}
} }
export function bindClip(form, clip) { export function bindClip(form, clip, xaigen = false) {
if (!clip?.id) return form if (!clip?.id) return form
const saved = clip.studio2 || {}, settings = saved.settings || clip.settings || clip const saved = clip.studio2 || clip.job || {}, settings = saved.settings || clip.settings || clip
const next = {...form, startClipId:clip.id, settings:{...form.settings}} const next = {...form, startClipId:clip.id, settings:{...form.settings}}
next.settings.aspect=['auto','16:9','9:16','1:1'].includes(settings.aspect)?settings.aspect:'auto' next.settings.aspect=['auto','16:9','9:16','1:1'].includes(settings.aspect)?settings.aspect:'auto'
for (const key of ['duration','aspect','width','height','fps','loraStack','turbo','steps','cfg']) { for (const key of ['duration','aspect','width','height','fps','loraStack','turbo','steps','cfg']) {
if (settings[key] != null && key!=='aspect') next.settings[key] = structuredClone(settings[key]) if (settings[key] != null && key!=='aspect') next.settings[key] = structuredClone(settings[key])
} }
next.engine = saved.engine || clip.engine || 'minimax' next.engine = xaigen && (saved.engine || clip.engine)==='ltx' ? 'ltx' : 'minimax'
if (saved.identityStillId) next.identityStillId = saved.identityStillId if (saved.identityStillId) next.identityStillId = saved.identityStillId
// Missing legacy metadata must not reset a user's current choice. // Missing legacy metadata must not reset a user's current choice.
for (const key of ['lockFace','lockOutfit']) if (typeof saved[key] === 'boolean') next[key] = saved[key] for (const key of ['lockFace','lockOutfit']) if (typeof saved[key] === 'boolean') next[key] = saved[key]
return next return next
} }
export function restoreAsset(form, item) { export function restoreAsset(form, item) {
if (item.kind==='clip' || item.studio2?.kind==='video') {
const next=bindClip(form,item,form.engine==='ltx')
const seed=item.studio2?.settings?.seed ?? item.settings?.seed ?? item.seed
if (seed != null) next.settings.seed=seed
next.settings.seedMode='random'
return {...next,mode:'extend'}
}
const restored = restoreInput(item) const restored = restoreInput(item)
const next = {...form,...restored,settings:{...form.settings,...restored.settings}} const next = {...form,...restored,settings:{...form.settings,...restored.settings}}
next.settings.seedMode='random' next.settings.seedMode='random'
@@ -31,7 +38,11 @@ export function restoreAsset(form, item) {
next.settings.aspect=item.studio2?.settings?.aspect || item.settings?.aspect || 'auto' next.settings.aspect=item.studio2?.settings?.aspect || item.settings?.aspect || 'auto'
return next return next
} }
export function extendSelected(form, clip) { export function extendSelected(form, clip, xaigen = false) {
// Switching task on the already bound clip must preserve the user's overrides. // Switching task on the already bound clip must preserve the user's overrides.
return {...(clip?.id && form.startClipId !== clip.id ? bindClip(form,clip) : form),mode:'extend'} const next=clip?.id && form.startClipId !== clip.id ? bindClip(form,clip,xaigen) : form
return {...next,mode:'extend',engine:xaigen && next.engine==='ltx'?'ltx':'minimax'}
}
export function changeSource(form) {
return form.mode==='extend' ? form : {...form,startClipId:'',settings:{...form.settings,aspect:'auto'}}
} }
+1 -1
View File
@@ -3,6 +3,6 @@ export function applyUploadedStill(form, response, target) {
if (!still?.id) throw new Error('The upload did not return a saved still. Please try again.') if (!still?.id) throw new Error('The upload did not return a saved still. Please try again.')
const next={...form,settings:{...form.settings}} const next={...form,settings:{...form.settings}}
if (target==='hero') next.identityStillId=still.id if (target==='hero') next.identityStillId=still.id
else {next.imageAId=still.id;next.startClipId='';if(next.mode==='extend')next.mode='video';next.settings.aspect='auto'} else {next.imageAId=still.id;if(next.mode!=='extend'){next.startClipId='';next.settings.aspect='auto'}}
return next return next
} }
+2 -2
View File
@@ -37,8 +37,8 @@ test('appearance roles distinguish wardrobe lock and both-off graph excludes her
assert.equal(graph.hero,undefined);assert.equal(graph['155'].inputs['ref_images.ref_image_0'],undefined);assert.equal(graph.load_start.inputs.image,'start.png');assert.equal(r.identityStillId,'hero') assert.equal(graph.hero,undefined);assert.equal(graph['155'].inputs['ref_images.ref_image_0'],undefined);assert.equal(graph.load_start.inputs.image,'start.png');assert.equal(r.identityStillId,'hero')
}) })
test('sound/dialogue/music compile without empty invented audio',()=>{ test('sound/dialogue/music compile without empty invented audio',()=>{
const empty=compilePrompt({});assert.doesNotMatch(empty,/Soundscape:|<d>/);assert.match(empty,/non_diegetic_music: N\/A/) const empty=compilePrompt({},true);assert.doesNotMatch(empty,/Soundscape:|<d>/);assert.match(empty,/non_diegetic_music: N\/A/)
const text=compilePrompt({soundscapeChips:['footsteps'],dialogue:{mode:'spoken off-screen',line:'Hello'}});assert.match(text,/lips remain closed/);assert.match(text,/<Subject 1> \(S1\) says <d>\[English\]Hello<\/d>/) const text=compilePrompt({soundscapeChips:['footsteps'],dialogue:{mode:'spoken off-screen',line:'Hello'}},true);assert.match(text,/lips remain closed/);assert.match(text,/<Subject 1> \(S1\) says <d>\[English\]Hello<\/d>/)
}) })
test('aspect sizing follows source and named presets',()=>{ test('aspect sizing follows source and named presets',()=>{
assert.deepEqual(resolveSize('9:16',{width:1344,height:768}),{width:768,height:1344});const size=resolveSize('auto',{width:768,height:1024});assert.equal(size.width/size.height,.75) assert.deepEqual(resolveSize('9:16',{width:1344,height:768}),{width:768,height:1344});const size=resolveSize('auto',{width:768,height:1024});assert.equal(size.width/size.height,.75)
+45
View File
@@ -0,0 +1,45 @@
import test from 'node:test'
import assert from 'node:assert/strict'
import fs from 'node:fs'
import ts from 'typescript'
import {validateRequest,compilePrompt} from '../shared/studio2/contracts.mjs'
import {sampleKleinSource,attachHeroReference,applyResolvedImageSize} from '../shared/studio2/graphs.mjs'
import {extendSelected,changeSource,restoreAsset} from '../shared/studio2/input.mjs'
const template=JSON.parse(fs.readFileSync(new URL('../server/assets/klein_v2_edit.json',import.meta.url)))
const source=fs.readFileSync(new URL('../server/utils/studio2/runner.ts',import.meta.url),'utf8')
const tree=ts.createSourceFile('runner.ts',source,ts.ScriptTarget.Latest,true)
const fn=tree.statements.find(n=>ts.isFunctionDeclaration(n)&&n.name.text==='prepareGraph')
const js=ts.transpileModule(fn.getText(tree),{compilerOptions:{target:ts.ScriptTarget.ES2022}}).outputText
async function prepare(request){
const uploads=[]
const scope={useRuntimeConfig:()=>({comfyFilenamePrefix:'preview'}),upload:async(r,label)=>{uploads.push(label);return label+'.png'},readFileSync:()=>Buffer.from('fixture'),stillPath:()=>'',getStill:(_,id)=>({id}),resolveRequestSize:async()=>{},saveRecord(){},
buildImageV2Workflow:params=>{const graph=structuredClone(template);graph['1'].inputs.image=params.imageAName;return {graph}},sampleKleinSource,attachHeroReference,applyResolvedImageSize,ensureComfyLoraNames:async()=>{},resolveGraphLoraNames(){}}
const run=new Function(...Object.keys(scope),`${js};return prepareGraph`)(...Object.values(scope))
const record={request,owner:'o',id:'job',index:0};return {graph:await run(record),record,uploads}
}
for(const sourceId of ['hero','source','']) test(`Klein edit samples source with hero ${sourceId || 'as sole source'}`,async()=>{
const q=validateRequest({folderId:'f',mode:'edit',engine:'flux',imageAId:sourceId,identityStillId:'hero',lockFace:true,lockOutfit:false,prompt:'she at a pool'})
const {graph,record,uploads}=await prepare(q)
assert.deepEqual(graph['19'].inputs.latent_image,['11',0]);assert.equal(graph['14'],undefined)
assert.deepEqual(graph['19'].inputs.sigmas,['source_sigmas',1]);assert.equal(graph.source_sigmas.inputs.denoise,.65)
assert.equal(record.sourceStillId,sourceId||'hero');assert.equal(record.sampleLatent,'source image latent')
assert.equal(uploads.includes('hero'),sourceId==='source');assert.equal(!!graph.hero_load,sourceId==='source')
assert.match(q.compiledPrompt,/face, glasses, hair/);assert.match(q.compiledPrompt,/do not restore the hero outfit/)
})
test('image requests omit video sections while form keeps them',()=>{
const sections={action:'sitting in a restaurant',cameraBeats:[{moves:['hold']}],soundscapeChips:['footsteps'],dialogue:{mode:'whisper',line:'Hello'},musicChip:'low underscore'}
const q=validateRequest({folderId:'f',mode:'edit',engine:'flux',identityStillId:'hero',promptSections:sections})
assert.doesNotMatch(q.compiledPrompt,/tripod|hold|soundscape|music|Hello|Picture 1/i)
assert.deepEqual(q.promptSections,{action:sections.action});assert.equal(sections.cameraBeats.length,1)
assert.equal(compilePrompt(sections,false,{engine:'krea',identityStillId:'hero'}),sections.action)
})
test('Extend normalizes engine and inherits clip without replacing action or camera',()=>{
const form={mode:'edit',engine:'flux',imageAId:'still',promptSections:{action:'New action',cameraBeats:[{moves:['hold']}]},settings:{loraStack:[]}}
const clip={id:'clip',kind:'clip',studio2:{engine:'minimax',identityStillId:'hero',lockOutfit:false,promptSections:{action:'Old action'},settings:{duration:8,aspect:'1:1',width:960,height:960,fps:24,turbo:false,steps:8,cfg:1.5,loraStack:[{name:'a',strengthModel:.6,strengthClip:.3}]}}}
const next=extendSelected(form,clip);assert.equal(next.engine,'minimax');assert.equal(next.startClipId,'clip');assert.deepEqual(next.settings,clip.studio2.settings)
assert.deepEqual(next.promptSections,form.promptSections);assert.equal(next.lockOutfit,false)
assert.equal(changeSource({...next,imageAId:'new-still'}).startClipId,'clip');assert.equal(changeSource(next).mode,'extend')
assert.deepEqual(restoreAsset(form,clip).promptSections,form.promptSections)
assert.equal(extendSelected({...form,startClipId:'clip'},null).startClipId,'clip')
assert.throws(()=>validateRequest({folderId:'f',mode:'extend',engine:'minimax',identityStillId:'hero',prompt:'Go'}),/Select the clip in the family strip or Previous clip/)
})
+1 -1
View File
@@ -4,7 +4,7 @@ import {applyUploadedStill} from '../shared/studio2/upload.mjs'
test('uploaded source is selected without changing hero or output',()=>{ test('uploaded source is selected without changing hero or output',()=>{
const form={imageAId:'old',identityStillId:'hero',selected:'result',startClipId:'clip',mode:'extend',settings:{aspect:'16:9'}} const form={imageAId:'old',identityStillId:'hero',selected:'result',startClipId:'clip',mode:'extend',settings:{aspect:'16:9'}}
const next=applyUploadedStill(form,{still:{id:'uploaded'}},'source') const next=applyUploadedStill(form,{still:{id:'uploaded'}},'source')
assert.equal(next.imageAId,'uploaded');assert.equal(next.identityStillId,'hero');assert.equal(next.selected,'result');assert.equal(next.startClipId,'');assert.equal(next.settings.aspect,'auto') assert.equal(next.imageAId,'uploaded');assert.equal(next.identityStillId,'hero');assert.equal(next.selected,'result');assert.equal(next.startClipId,'clip');assert.equal(next.mode,'extend');assert.equal(next.settings.aspect,'16:9')
}) })
test('uploaded hero changes identity only and missing saves are reported',()=>{ test('uploaded hero changes identity only and missing saves are reported',()=>{
const form={identityStillId:'old',imageAId:'source',settings:{aspect:'1:1'}} const form={identityStillId:'old',imageAId:'source',settings:{aspect:'1:1'}}
+1 -1
View File
@@ -12,7 +12,7 @@ test('compose without B and refine without mask are 400',()=>{
for(const mode of ['compose','refine']) assert.throws(()=>validateRequest({...base,mode,engine:'flux',imageAId:'a'}),e=>e.statusCode===400) for(const mode of ['compose','refine']) assert.throws(()=>validateRequest({...base,mode,engine:'flux',imageAId:'a'}),e=>e.statusCode===400)
}) })
test('Use as input round-trips camera chip and never invents one for flat text',()=>{ test('Use as input round-trips camera chip and never invents one for flat text',()=>{
const r=validateRequest({...base,promptSections:{action:'walk',cameraChip:{family:'dolly in',amplitude:'subtle',speed:'slow'}}}); assert.deepEqual(restoreInput({studio2:r}).promptSections,r.promptSections);assert.equal(restoreInput({prompt:'old text'}).promptSections.cameraChip,undefined);assert.match(compilePrompt(r.promptSections),/dolly in/) const r=validateRequest({...base,promptSections:{action:'walk',cameraChip:{family:'dolly in',amplitude:'subtle',speed:'slow'}}}); assert.deepEqual(restoreInput({studio2:r}).promptSections,r.promptSections);assert.equal(restoreInput({prompt:'old text'}).promptSections.cameraChip,undefined);assert.match(compilePrompt(r.promptSections,true),/dolly in/)
}) })
test('purge accepts only exact preview path boundary',()=>{ test('purge accepts only exact preview path boundary',()=>{
assert.equal(scopedFile({subfolder:'video/MiniMax_H3_preview/job',filename:'a.png'},'video/MiniMax_H3_preview'),true) assert.equal(scopedFile({subfolder:'video/MiniMax_H3_preview/job',filename:'a.png'},'video/MiniMax_H3_preview'),true)