Model camera beats and independent hero identity locks

shared/studio2/contracts.mjs

@@ -1,21 +1,19 @@
-export const identityLock = '<Picture 1> is identity. Face, hair, glasses, outfit fully preserved. Start frame is composition only. Do not copy facial drift from the start frame.'
+import { sectionsFrom, compileSections } from './prompt.mjs'
+export { sectionsFrom } from './prompt.mjs'
 export const cameraFamilies = ['static','dolly in','dolly out','pan left','pan right','tilt up','tilt down','zoom in','zoom out','orbit left','orbit right']
-export function sectionsFrom(value = {}) {
+export const identityLock = '<Picture 1> / hero is identity for face, hair, glasses, body, and wardrobe. Keep the person and clothes fully preserved. Start frame is composition, pose, place, and camera only. Do not copy facial drift from the start frame.'
-  value = value || {}
+export function appearanceLock({lockFace = true, lockOutfit = true} = {}) {
-  if (typeof value === 'string') value = { action: value }
+  if (lockFace && lockOutfit) return identityLock
-  const sections = Object.fromEntries(['action','camera','soundscape','dialogue','music','locks'].map(key => [key, String(value[key] || '')]))
+  if (lockFace) return '<Picture 1> / hero is identity for face, hair, glasses, and body only — not wardrobe. The start frame is composition, pose, place, and clothing. Do not restore an earlier outfit. Do not reset clothes, straps, or print to the hero still.'
-  const chip = value.cameraChip
+  if (lockOutfit) return '<Picture 1> / hero is wardrobe only. Preserve clothes from the hero. Face, hair, glasses, body, pose, place, and camera follow the start frame.'
-  if (chip && cameraFamilies.includes(chip.family)) sections.cameraChip = { family: chip.family, amplitude: ['subtle','moderate','large'].includes(chip.amplitude) ? chip.amplitude : 'subtle', speed: ['slow','medium','fast'].includes(chip.speed) ? chip.speed : 'slow' }
+  return 'The start frame is the only appearance reference for person and clothing, as well as pose, place, and camera. Do not use the hero as an appearance reference.'
-  return sections
 }
-export function compilePrompt(value, video = false) {
+export function compilePrompt(value, video = false, locks = {}) {
-  const s = sectionsFrom(value), c = s.cameraChip
+  return [compileSections(value), video ? appearanceLock(locks) : ''].filter(Boolean).join('\n\n')
-  const camera = c ? (c.family === 'static' ? 'Static shot. Camera locked on a tripod.' : `Camera ${c.family}, ${c.amplitude} movement, ${c.speed} speed.`) : s.camera
-  return [s.action, camera && `Camera: ${camera}`, s.soundscape && `Soundscape: ${s.soundscape}`, s.dialogue && `Dialogue: (S1) <d>[English]${s.dialogue}</d>`, s.music && `Music: ${s.music}`, s.locks && `Locks: ${s.locks}`, video ? identityLock : ''].filter(Boolean).join('\n\n')
 }
 export function restoreInput(item) {
-  const saved = item.studio2 || item
+  const saved = structuredClone(item.studio2 || item)
-  return { ...saved, promptSections: sectionsFrom(saved.promptSections || { action: item.prompt || saved.compiledPrompt || '' }) }
+  return { ...saved, lockFace:saved.lockFace !== false, lockOutfit:saved.lockOutfit !== false, hideInput:!!saved.hideInput, settings:structuredClone(saved.settings || {}), promptSections: sectionsFrom(saved.promptSections || { action: item.prompt || saved.compiledPrompt || '' }) }
 }
 export function validateRequest(raw, xaigen = false) {
   raw = raw || {}
@@ -29,7 +27,7 @@ export function validateRequest(raw, xaigen = false) {
   if (Array.isArray(raw.batch) && raw.batch.length>30) fail('A batch supports up to 31 shots including the first prompt.')
   if (Array.isArray(raw.guides) && raw.guides.length>8) fail('Use up to eight guide stills.')
   if (!raw.folderId) fail('Choose a folder.')
-  if ((video || ['edit','compose','iterate'].includes(mode)) && !raw.identityStillId) fail('Choose a sticky hero still.')
+  if ((video || ['edit','compose','iterate'].includes(mode)) && (raw.lockFace !== false || raw.lockOutfit !== false) && !raw.identityStillId) fail('Choose a sticky hero still.')
   if (mode === 'compose' && !raw.imageBId) fail('Compose requires a second still.')
   if (mode === 'refine' && !raw.maskId) fail('Refine requires a mask.')
   if (['edit','compose','refine'].includes(mode) && !raw.imageAId) fail('Choose the source still.')
@@ -39,11 +37,17 @@ export function validateRequest(raw, xaigen = false) {
   const number=(key, fallback, min, max) => { const n=Number(raw.settings?.[key] ?? fallback); if (!Number.isFinite(n)||n<min||n>max) fail(`Invalid ${key}.`); return n }
   for (const key of ['steps','seed']) if (!Number.isInteger(number(key,key==='steps'?24:1,0,2147483647))) fail(`Invalid ${key}.`)
   for (const key of ['width','height']) if (number(key,key==='width'?1344:768,256,4096)%32) fail('Dimensions must be multiples of 32.')
-  const frames=Math.ceil((number('duration',5,1,15)*24-5)/17)*17+5
+  const duration=number('duration',5,1,15)
-  return { mode, engine, folderId:String(raw.folderId), identityStillId:raw.identityStillId || '', imageAId:raw.imageAId || '', imageBId:raw.imageBId || '', maskId:raw.maskId || '', startClipId:raw.startClipId || '', endStillId:raw.endStillId || '',
+  if (video) {
+    if (promptSections.cameraBeats.slice(0,-1).some(b=>b.durationSec == null)) fail('Only the last camera beat can run to the end.')
+    if (promptSections.cameraBeats.reduce((sum,b)=>sum+(b.durationSec || 0),0)>duration) fail('Camera beats exceed the clip duration.')
+    if (raw.lockFace === false && raw.lockOutfit === false && !raw.imageAId && !raw.startClipId) fail('Choose a start still or previous clip when both identity locks are off.')
+  }
+  const frames=Math.ceil((duration*number('fps',24,1,60)-5)/17)*17+5
+  return { mode, engine, lockFace:raw.lockFace !== false, lockOutfit:raw.lockOutfit !== false, hideInput:!!raw.hideInput, folderId:String(raw.folderId), identityStillId:raw.identityStillId || '', imageAId:raw.imageAId || '', imageBId:raw.imageBId || '', maskId:raw.maskId || '', startClipId:raw.startClipId || '', endStillId:raw.endStillId || '',
     startFrameSource: raw.startClipId ? {kind:'previous-last-frame',clipId:raw.startClipId} : {kind:'still',stillId:raw.imageAId || ''},
-    promptSections, compiledPrompt:compilePrompt(promptSections,video),
+    promptSections, compiledPrompt:compilePrompt(promptSections,video,raw),
-    settings: { width:number('width',1344,256,4096),height:number('height',768,256,4096),steps:number('steps',video?8:24,1,100),cfg:number('cfg',video?1.5:4,0,20),seed:number('seed',Math.floor(Math.random()*2147483647),0,2147483647),duration:number('duration',5,1,15),turbo:raw.settings?.turbo !== false, loraStack:Array.isArray(raw.settings?.loraStack)?raw.settings.loraStack:[] },
+    settings: { aspect:['auto','16:9','9:16','1:1'].includes(raw.settings?.aspect)?raw.settings.aspect:'auto', fps:number('fps',24,1,60), width:number('width',1344,256,4096),height:number('height',768,256,4096),steps:number('steps',video?8:24,1,100),cfg:number('cfg',video?1.5:4,0,20),seed:number('seed',Math.floor(Math.random()*2147483647),0,2147483647),duration:number('duration',5,1,15),turbo:raw.settings?.turbo !== false, loraStack:Array.isArray(raw.settings?.loraStack)?raw.settings.loraStack:[] },
     guides: (Array.isArray(raw.guides)?raw.guides:[]).slice(0,8).map(g=>{ if (!g.stillId || !Number.isInteger(g.frame) || g.frame<1 || g.frame>=frames) fail('Each guide requires a still and positive frame index.'); return {stillId:String(g.stillId),frame:g.frame} }),
     batch: (Array.isArray(raw.batch)?raw.batch:[]).slice(0,30).map(s=>{const next=sectionsFrom({...promptSections,...(typeof s==='string'?{action:s}:s)});if(!next.action.trim())fail('Every shot needs an action prompt.');return next}) }
 }

shared/studio2/graphs.mjs

@@ -1,9 +1,10 @@
 export function nativeVideoGraph(template, request, files, prefix) {
-  if (!files.hero) throw new Error('Identity still is required; never substitute the start frame.')
+  const appearance = request.lockFace !== false || request.lockOutfit !== false
+  if (appearance && !files.hero) throw new Error('Identity still is required; never substitute the start frame.')
   const g=structuredClone(template), s=request.settings
-  g.hero.inputs.image=files.hero
+  if (appearance) { g.hero.inputs.image=files.hero; g['155'].inputs['ref_images.ref_image_0']=['hero',0] }
-  Object.assign(g['155'].inputs,{prompt:request.compiledPrompt,width:s.width,height:s.height,length:Math.ceil((s.duration*24-5)/17)*17+5,ref_image_size:'max'})
+  else { delete g.hero; delete g['155'].inputs['ref_images.ref_image_0'] }
-  g['155'].inputs['ref_images.ref_image_0']=['hero',0]
+  Object.assign(g['155'].inputs,{prompt:request.compiledPrompt,width:s.width,height:s.height,length:Math.ceil((s.duration*(s.fps || 24)-5)/17)*17+5,ref_image_size:'max'})
   let conditioning=['155',0]
   for (const [name,image,frame] of [['start',files.start,0],['end',files.end,-1],...(files.guides||[]).map((f,i)=>[`guide${i}`,f.image,f.frame])]) {
     if (!image) continue
@@ -13,6 +14,7 @@ export function nativeVideoGraph(template, request, files, prefix) {
   }
   g.guidance.inputs={conditioning,guidance:s.cfg}
   g['171'].inputs.noise_seed=s.seed
+  g['172'].inputs.fps=s.fps || 24
   g['179'].inputs.value=s.steps;g['180'].inputs.value=s.steps;g['181'].inputs.value=s.turbo
   g['92'].inputs.filename_prefix=prefix+'/video'
   g.anchor_frame={class_type:'ImageFromBatch',inputs:{image:['164',0],batch_index:g['155'].inputs.length-1,length:1}}

shared/studio2/input.mjs

@@ -0,0 +1,31 @@
+import { restoreInput } from './contracts.mjs'
+export const aspectOptions = [{id:'auto',label:'Automatic',hint:'Match start or hero still'},{id:'16:9',label:'16:9',hint:'1344 × 768 · padded'},{id:'9:16',label:'9:16',hint:'768 × 1344 · padded'},{id:'1:1',label:'1:1',hint:'960 × 960 · padded'}]
+export function resolveSize(aspect, source, fallback = {width:1344,height:768}) {
+  const presets = {'16:9':[1344,768],'9:16':[768,1344],'1:1':[960,960]}
+  if (presets[aspect]) return {width:presets[aspect][0],height:presets[aspect][1]}
+  if (!source?.width || !source?.height) return {width:fallback.width || 1344,height:fallback.height || 768}
+  const factor = Math.min(1344/Math.max(source.width,source.height),768/Math.min(source.width,source.height))
+  return {width:Math.max(256,Math.round(source.width*factor/32)*32),height:Math.max(256,Math.round(source.height*factor/32)*32)}
+}
+export function bindClip(form, clip) {
+  if (!clip?.id) return form
+  const saved = clip.studio2 || {}, settings = saved.settings || clip.settings || clip
+  const next = {...form, startClipId:clip.id, settings:{...form.settings}}
+  for (const key of ['duration','aspect','width','height','fps','loraStack','turbo','steps','cfg']) {
+    if (settings[key] != null) next.settings[key] = structuredClone(settings[key])
+  }
+  next.engine = saved.engine || clip.engine || 'minimax'
+  if (saved.identityStillId) next.identityStillId = saved.identityStillId
+  // Missing legacy metadata must not reset a user's current choice.
+  for (const key of ['lockFace','lockOutfit']) if (typeof saved[key] === 'boolean') next[key] = saved[key]
+  return next
+}
+export function restoreAsset(form, item) {
+  const restored = restoreInput(item)
+  const next = {...form,...restored,settings:{...form.settings,...restored.settings}}
+  return item.kind === 'clip' || item.studio2?.kind === 'video' ? {...bindClip(next,item),mode:'extend'} : next
+}
+export function extendSelected(form, clip) {
+  // Switching task on the already bound clip must preserve the user's overrides.
+  return {...(clip?.id && form.startClipId !== clip.id ? bindClip(form,clip) : form),mode:'extend'}
+}

shared/studio2/prompt.mjs

@@ -0,0 +1,40 @@
+export const cameraMoves = ['hold','dolly-in','dolly-out','pan-left','pan-right','tilt-up','tilt-down','zoom-in','zoom-out','orbit-left','orbit-right']
+export const soundscapeOptions = ['quiet room','distant birds','fabric rustle','footsteps','soft cloth','city hush','rain on glass','room tone only','no extra SFX']
+export function normalizeBeat(beat = {}) {
+  let moves = [...new Set((beat.moves || []).filter(m => cameraMoves.includes(m)))]
+  if (moves.includes('hold') || !moves.length) moves = ['hold']
+  const duration = Number(beat.durationSec)
+  return { moves, amplitude: ['subtle','moderate','large'].includes(beat.amplitude) ? beat.amplitude : 'subtle', speed: ['slow','medium','fast'].includes(beat.speed) ? beat.speed : 'slow', durationSec: beat.durationSec == null ? null : Number.isFinite(duration) ? Math.max(.5, Math.round(duration * 2) / 2) : null }
+}
+export function toggleMove(beat, move) {
+  if (!cameraMoves.includes(move)) return normalizeBeat(beat)
+  if (move === 'hold') return normalizeBeat({...beat,moves:['hold']})
+  const moves = beat.moves.filter(m => m !== 'hold')
+  return normalizeBeat({...beat,moves:moves.includes(move) ? moves.filter(m => m !== move) : [...moves,move]})
+}
+export function sectionsFrom(value = {}) {
+  if (typeof value === 'string') value = {action:value}
+  value ||= {}
+  const s = Object.fromEntries(['action','camera','soundscape','music','locks'].map(k => [k,typeof value[k] === 'string' ? value[k] : '']))
+  const legacy = value.cameraChip
+  s.cameraBeats = Array.isArray(value.cameraBeats) ? value.cameraBeats.map(normalizeBeat) : legacy ? [normalizeBeat({...legacy,moves:[legacy.family === 'static' ? 'hold' : legacy.family.replaceAll(' ','-')]})] : []
+  s.soundscapeChips = [...new Set((value.soundscapeChips || []).filter(x => soundscapeOptions.includes(x)))]
+  s.soundscapeLevel = value.soundscapeLevel === 'crush' ? 'crush' : 'normal'
+  const d = typeof value.dialogue === 'object' && value.dialogue ? value.dialogue : {mode:value.dialogue ? 'spoken to camera' : 'none',line:value.dialogue || ''}
+  s.dialogue = {mode:['none','spoken to camera','spoken off-screen','whisper'].includes(d.mode) ? d.mode : 'none',language:String(d.language || 'English'),speaker:String(d.speaker || 'S1'),line:String(d.line || '')}
+  s.musicChip = ['N/A','none','low underscore'].includes(value.musicChip) ? value.musicChip : 'N/A'
+  return s
+}
+export function compileSections(value) {
+  const s = sectionsFrom(value)
+  const camera = s.cameraBeats.length ? ['One continuous take.', ...s.cameraBeats.map((b,i) => {
+    const move = b.moves[0] === 'hold' ? 'hold the camera locked on a tripod' : b.moves.map(m=>m.replaceAll('-',' ')).join(' and ') + (b.moves.length > 1 ? ' simultaneously' : '')
+    const timing = b.durationSec == null ? 'through the end of the remaining clip' : i === 0 ? `for the first ${b.durationSec} seconds` : `for ${b.durationSec} seconds`
+    return `${i ? 'Then' : 'First'}, ${move}, ${{subtle:'small',moderate:'medium',large:'large'}[b.amplitude]} amplitude, ${b.speed} speed, ${timing}.`
+  }), 'No cuts. No new setup.'].join(' ') : s.camera
+  const sound = [s.soundscapeChips.length ? `${s.soundscapeChips.join(', ')}.${s.soundscapeLevel === 'crush' ? ' Crushed soundscape dynamics.' : ''}` : '',s.soundscape].filter(Boolean).join(' ')
+  const d=s.dialogue, line=d.line.trim()
+  const dialogue = d.mode !== 'none' && line ? `${d.mode === 'spoken off-screen' ? 'Off-screen voice; on-screen lips remain closed. ' : ''}<Subject 1> (${d.speaker}) ${d.mode === 'whisper' ? 'whispers' : 'says'} <d>[${d.language}]${line}</d>` : ''
+  const music = s.music || (s.musicChip === 'N/A' ? 'non_diegetic_music: N/A' : s.musicChip === 'none' ? 'No non-diegetic music.' : 'A low non-diegetic musical underscore.')
+  return [s.action,camera && `Camera: ${camera}`,sound && `Soundscape: ${sound}`,dialogue,music,s.locks && `Locks: ${s.locks}`].filter(Boolean).join('\n\n')
+}

tests/studio2-controls.test.mjs

@@ -0,0 +1,45 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import {compilePrompt,validateRequest,restoreInput} from '../shared/studio2/contracts.mjs'
+import {normalizeBeat,toggleMove} from '../shared/studio2/prompt.mjs'
+import {bindClip,restoreAsset,resolveSize,extendSelected} from '../shared/studio2/input.mjs'
+import {nativeVideoGraph} from '../shared/studio2/graphs.mjs'
+const base={mode:'video',engine:'minimax',folderId:'f',identityStillId:'hero',promptSections:{action:'Walk forward'}}
+const beats=[normalizeBeat({moves:['tilt-down','dolly-out'],durationSec:2}),normalizeBeat({moves:['pan-left'],durationSec:null})]
+test('sequential and simultaneous beats compile as one continuous take',()=>{
+ const text=compilePrompt({cameraBeats:beats},true)
+ assert.match(text,/tilt down and dolly out simultaneously/);assert.match(text,/first 2 seconds/);assert.match(text,/Then, pan left/);assert.match(text,/No cuts\. No new setup\./);assert.doesNotMatch(text,/\[Shot|Shot 2|00:02/)
+})
+test('hold is exclusive; toggling moves supports simultaneous motion',()=>{
+ const mixed=normalizeBeat({moves:['hold','pan-left']});assert.deepEqual(mixed.moves,['hold'])
+ const moved=toggleMove(toggleMove(mixed,'tilt-down'),'dolly-out');assert.deepEqual(moved.moves,['tilt-down','dolly-out']);assert.deepEqual(toggleMove(moved,'hold').moves,['hold'])
+})
+test('Use as input restores camera, sound, dialogue, music, LoRAs, aspect and locks',()=>{
+ const request=validateRequest({...base,lockFace:true,lockOutfit:false,hideInput:true,promptSections:{action:'Walk',cameraBeats:beats,soundscapeChips:['footsteps'],dialogue:{mode:'whisper',line:'Hello',language:'English'},musicChip:'none'},settings:{aspect:'9:16',width:768,height:1344,loraStack:[{name:'example.safetensors',strengthModel:.8,strengthClip:.4}]}})
+ const restored=restoreInput({studio2:request});assert.deepEqual(restored,request);assert.equal(restored.lockOutfit,false);assert.equal(restored.hideInput,true)
+})
+test('extend binds completed clip, inherits settings and leaves delta prompt alone',()=>{
+ const form={...base,lockFace:true,lockOutfit:false,settings:{duration:5,width:1344,height:768},promptSections:{action:'Turn',cameraBeats:beats}}
+ const clip={id:'done',kind:'clip',studio2:{...base,lockFace:true,lockOutfit:false,settings:{duration:8,width:768,height:1344,aspect:'9:16',fps:24,steps:8,cfg:1.5,turbo:false,loraStack:[{name:'a',strengthModel:1,strengthClip:.5}]}}}
+ const bound=bindClip(form,clip);assert.equal(bound.startClipId,'done');assert.deepEqual(bound.settings,clip.studio2.settings);assert.equal(bound.promptSections.action,'Turn');assert.equal(bound.identityStillId,'hero');assert.equal(bound.lockOutfit,false)
+ assert.equal(restoreAsset(form,clip).startClipId,'done');assert.equal(bindClip(form,{id:'legacy'}).lockOutfit,false)
+ const overridden={...bound,lockOutfit:false,settings:{...bound.settings,cfg:2.5}}
+ const extend=extendSelected(overridden,{...clip,studio2:{...clip.studio2,lockOutfit:true}})
+ assert.equal(extend.lockOutfit,false);assert.equal(extend.settings.cfg,2.5);assert.equal(extend.startClipId,'done')
+})
+test('appearance roles distinguish wardrobe lock and both-off graph excludes hero',()=>{
+ assert.match(compilePrompt({},true,{lockOutfit:false}),/Do not restore an earlier outfit/)
+ assert.match(compilePrompt({},true),/person and clothes fully preserved/)
+ const r=validateRequest({...base,lockFace:false,lockOutfit:false,imageAId:'start'})
+ const template=JSON.parse(fs.readFileSync(new URL('../server/assets/studio2_minimax_native.json',import.meta.url)))
+ const graph=nativeVideoGraph(template,r,{hero:'ignored.png',start:'start.png'},'preview')
+ assert.equal(graph.hero,undefined);assert.equal(graph['155'].inputs['ref_images.ref_image_0'],undefined);assert.equal(graph.load_start.inputs.image,'start.png');assert.equal(r.identityStillId,'hero')
+})
+test('sound/dialogue/music compile without empty invented audio',()=>{
+ const empty=compilePrompt({});assert.doesNotMatch(empty,/Soundscape:|<d>/);assert.match(empty,/non_diegetic_music: N\/A/)
+ const text=compilePrompt({soundscapeChips:['footsteps'],dialogue:{mode:'spoken off-screen',line:'Hello'}});assert.match(text,/lips remain closed/);assert.match(text,/<Subject 1> \(S1\) says <d>\[English\]Hello<\/d>/)
+})
+test('aspect sizing follows source and named presets',()=>{
+ assert.deepEqual(resolveSize('9:16',{width:1344,height:768}),{width:768,height:1344});const size=resolveSize('auto',{width:768,height:1024});assert.equal(size.width/size.height,.75)
+})