From bf5873269019542cafbe2042fcbd7ce78be71e00 Mon Sep 17 00:00:00 2001 From: Towsty Date: Fri, 11 Sep 2026 07:07:26 -0500 Subject: [PATCH] Separate Studio 2 source and canvas and apply size and seed choices --- components/studio2/Bench.vue | 64 ++++++++++++++++-------------- server/api/studio-2/jobs.post.ts | 4 +- server/utils/studio2/runner.ts | 11 +++-- server/utils/studio2/size.ts | 25 ++++++++++++ shared/studio2/contracts.mjs | 8 ++-- shared/studio2/graphs.mjs | 14 +++++++ shared/studio2/input.mjs | 10 ++++- shared/studio2/seed.mjs | 17 ++++++++ shared/studio2/viewer.mjs | 14 +++++++ tests/studio2-canvas-seed.test.mjs | 34 ++++++++++++++++ tests/studio2-runner.test.mjs | 3 +- 11 files changed, 163 insertions(+), 41 deletions(-) create mode 100644 server/utils/studio2/size.ts create mode 100644 shared/studio2/seed.mjs create mode 100644 shared/studio2/viewer.mjs create mode 100644 tests/studio2-canvas-seed.test.mjs diff --git a/components/studio2/Bench.vue b/components/studio2/Bench.vue index c52d428..1473bc6 100644 --- a/components/studio2/Bench.vue +++ b/components/studio2/Bench.vue @@ -4,7 +4,7 @@

Studio 2

One hero. Every shot.

- + Still loaded · preview hidden Sticky hero identity LibraryStudio 1 @@ -15,23 +15,22 @@
-
+

Input preview · {{ inputStill ? assetLabel(inputStill) : form.startClipId ? 'Previous clip · last frame' : 'No start still' }}

Still loaded · preview hidden

Selected source still

Previous clip loaded · its last frame will be used

Choose a source / start still

- - - + + + - +
-
Add Guide stills
+
Add Guide stills

Prompt

@@ -48,7 +47,7 @@
@@ -57,42 +56,47 @@ diff --git a/server/api/studio-2/jobs.post.ts b/server/api/studio-2/jobs.post.ts index ebf5eec..666add0 100644 --- a/server/api/studio-2/jobs.post.ts +++ b/server/api/studio-2/jobs.post.ts @@ -1,4 +1,5 @@ import { validateRequest } from '~/shared/studio2/contracts.mjs'; +import { resolveRequestSize } from '../../utils/studio2/size'; import { parsePostedLoraStack, assertImageV2LoraStack } from '../../utils/loras'; import { saveRecord, readRecord } from '../../utils/studio2/store'; import { addStudioJob, kickStudioQueue, type StudioJobPayload } from '../../utils/studioQueue'; @@ -22,10 +23,11 @@ export default defineEventHandler(async (event) => { assertFolderAccess(event, getStill(owner, id).folderId); if (request.startClipId) assertFolderAccess(event, getClip(owner, request.startClipId).folderId); + await resolveRequestSize(owner,request); const id = crypto.randomUUID(), record = { id, owner, request, state: 'waiting', queuedAt: Date.now(), familyId: crypto.randomUUID(), outputs: [], purgeResult: 'Not yet saved' }; saveRecord(record); const row = await addStudioJob({ ownerKey: owner, kind: ['video', 'extend'].includes(request.mode) ? 'video' : 'edit', familyId: record.familyId, payload: { ...request.settings, studio2Id: id, prompt: request.compiledPrompt, folderId: request.folderId, extensions: [], referenceStillIds: [], useIdentityRefs: false, queueAutoRun: true } as StudioJobPayload }); saveRecord({ ...readRecord(owner, id), queueId: row.id }); kickStudioQueue(); - return { id }; + return { id, seed:request.settings.seed, width:request.settings.width, height:request.settings.height }; }); diff --git a/server/utils/studio2/runner.ts b/server/utils/studio2/runner.ts index 0a12d3c..aae292e 100644 --- a/server/utils/studio2/runner.ts +++ b/server/utils/studio2/runner.ts @@ -1,10 +1,11 @@ import { watchProgress } from './progress'; import { fitStill } from './media'; -import { resolveSize } from '~/shared/studio2/input.mjs'; +import { resolveRequestSize } from './size'; +import { queueSeeds } from '~/shared/studio2/seed.mjs'; import { readFileSync, mkdirSync, existsSync, unlinkSync } from 'node:fs'; import { join } from 'node:path'; import template from '../../assets/studio2_minimax_native.json'; -import { nativeVideoGraph, attachHeroReference } from '~/shared/studio2/graphs.mjs'; +import { nativeVideoGraph, attachHeroReference, applyResolvedImageSize } from '~/shared/studio2/graphs.mjs'; import { compilePrompt, scopedFile } from '~/shared/studio2/contracts.mjs'; import { createJob, restoreJob, getJob, emitJob, type Job } from '../jobs'; import { markStudioLive, onLiveVideoSettled, type StudioJob } from '../studioQueue'; @@ -65,8 +66,7 @@ async function purge(r: any) { async function prepareGraph(r: any) { const q = r.request, s = q.settings, prefix = String(useRuntimeConfig().comfyFilenamePrefix).replace(/\/$/, '') + `/studio2/${r.id}/${r.index}`; const load = async (id: string, label: string) => id ? upload(r, label, readFileSync(stillPath(r.owner, getStill(r.owner, id).id))) : ''; - const source = q.startClipId ? getClip(r.owner,q.startClipId) : (q.imageAId || q.identityStillId) ? getStill(r.owner,q.imageAId || q.identityStillId) : null; - Object.assign(s,resolveSize(s.aspect,source,s)); + await resolveRequestSize(r.owner,q); saveRecord(r); const hero = q.lockFace === false && q.lockOutfit === false ? '' : await load(q.identityStillId, 'hero'), a = await load(q.imageAId, 'source'), b = await load(q.imageBId, 'compose'), mask = await load(q.maskId, 'mask'); let graph: any; @@ -96,6 +96,7 @@ async function prepareGraph(r: any) { const found = q.engine === 'krea' ? await resolveKreaGenerateAssets() : null; const assets = found ? { kreaUnetName: found.unet, kreaClipName: found.clip, kreaVaeName: found.vae, kreaConceptLora: found.conceptLora } : {}; graph = buildImageV2Workflow({ ...s, ...assets, engine: q.engine, mode, task: 'scene', prompt: q.compiledPrompt, imageAName: a, imageBName: mode === 'compose' ? b : undefined, maskName: mode === 'refine' ? mask : undefined, filenamePrefix: prefix + '/image' }).graph; + applyResolvedImageSize(graph,s); if (q.engine === 'flux') attachHeroReference(graph, hero); else if (hero && ['edit', 'compose', 'iterate'].includes(q.mode)) @@ -112,6 +113,7 @@ async function run(r: any, job: Job) { try { const prompts = r.prompts || [r.request.promptSections, ...r.request.batch]; r.prompts = prompts; + r.request.shotSeeds ||= queueSeeds(r.request.settings,prompts.length); saveRecord(r); for (r.index = r.index || 0; r.index < prompts.length; r.index++) { if (job.status === 'cancelled') @@ -122,6 +124,7 @@ async function run(r: any, job: Job) { throw new Error('Cancelled'); await new Promise(resolve => setTimeout(resolve, 2000)); } + r.request.settings.seed = r.request.shotSeeds[r.index]; r.files = []; r.progress = null; r.request.promptSections = prompts[r.index]; diff --git a/server/utils/studio2/size.ts b/server/utils/studio2/size.ts new file mode 100644 index 0000000..0c8a860 --- /dev/null +++ b/server/utils/studio2/size.ts @@ -0,0 +1,25 @@ +import { readFileSync } from 'node:fs' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { getClip, getStill, clipVideoPath, stillPath } from '../library' +import { imageDimensions } from '../resolution' +import { resolveSize } from '~/shared/studio2/input.mjs' +const exec = promisify(execFile) +export async function resolveRequestSize(owner: string, request: any) { + const video=['video','extend'].includes(request.mode) + let dimensions:any + if (video && request.startClipId) { + getClip(owner,request.startClipId) + const result=await exec('ffprobe',['-v','error','-select_streams','v:0','-show_entries','stream=width,height','-of','json',clipVideoPath(owner,request.startClipId)],{windowsHide:true,timeout:15000}) + dimensions=JSON.parse(result.stdout).streams?.[0] + } else { + const id=request.imageAId || request.identityStillId + if (id) { + getStill(owner,id) + dimensions=imageDimensions(readFileSync(stillPath(owner,id))) + if (!dimensions) throw new Error('Cannot measure the source still.') + } + } + if (video && request.startClipId && !dimensions?.width) throw new Error('Cannot measure the previous clip.') + Object.assign(request.settings,resolveSize(request.settings.aspect,dimensions)) +} diff --git a/shared/studio2/contracts.mjs b/shared/studio2/contracts.mjs index ad47e5a..a359d6b 100644 --- a/shared/studio2/contracts.mjs +++ b/shared/studio2/contracts.mjs @@ -1,3 +1,4 @@ +import { queueSeeds } from './seed.mjs' import { sectionsFrom, compileSections } from './prompt.mjs' export { sectionsFrom } from './prompt.mjs' export const cameraFamilies = ['static','dolly in','dolly out','pan left','pan right','tilt up','tilt down','zoom in','zoom out','orbit left','orbit right'] @@ -35,7 +36,7 @@ export function validateRequest(raw, xaigen = false) { const promptSections = sectionsFrom(raw.promptSections || {action:raw.prompt || ''}) if (!promptSections.action.trim()) fail('Enter an action prompt.') const number=(key, fallback, min, max) => { const n=Number(raw.settings?.[key] ?? fallback); if (!Number.isFinite(n)||nmax) fail(`Invalid ${key}.`); return n } - for (const key of ['steps','seed']) if (!Number.isInteger(number(key,key==='steps'?24:1,0,2147483647))) fail(`Invalid ${key}.`) + for (const key of ['steps']) if (!Number.isInteger(number(key,key==='steps'?24:1,0,2147483647))) fail(`Invalid ${key}.`) for (const key of ['width','height']) if (number(key,key==='width'?1344:768,256,4096)%32) fail('Dimensions must be multiples of 32.') const duration=number('duration',5,1,15) if (video) { @@ -44,10 +45,11 @@ export function validateRequest(raw, xaigen = false) { if (raw.lockFace === false && raw.lockOutfit === false && !raw.imageAId && !raw.startClipId) fail('Choose a start still or previous clip when both identity locks are off.') } const frames=Math.ceil((duration*number('fps',24,1,60)-5)/17)*17+5 - return { mode, engine, lockFace:raw.lockFace !== false, lockOutfit:raw.lockOutfit !== false, hideInput:!!raw.hideInput, folderId:String(raw.folderId), identityStillId:raw.identityStillId || '', imageAId:raw.imageAId || '', imageBId:raw.imageBId || '', maskId:raw.maskId || '', startClipId:raw.startClipId || '', endStillId:raw.endStillId || '', + const shotSeeds=queueSeeds(raw.settings,1+(Array.isArray(raw.batch)?raw.batch.length:0)) + return { mode, engine, shotSeeds, lockFace:raw.lockFace !== false, lockOutfit:raw.lockOutfit !== false, hideInput:!!raw.hideInput, folderId:String(raw.folderId), identityStillId:raw.identityStillId || '', imageAId:raw.imageAId || '', imageBId:raw.imageBId || '', maskId:raw.maskId || '', startClipId:raw.startClipId || '', endStillId:raw.endStillId || '', startFrameSource: raw.startClipId ? {kind:'previous-last-frame',clipId:raw.startClipId} : {kind:'still',stillId:raw.imageAId || ''}, promptSections, compiledPrompt:compilePrompt(promptSections,video,raw), - settings: { aspect:['auto','16:9','9:16','1:1'].includes(raw.settings?.aspect)?raw.settings.aspect:'auto', fps:number('fps',24,1,60), width:number('width',1344,256,4096),height:number('height',768,256,4096),steps:number('steps',video?8:24,1,100),cfg:number('cfg',video?1.5:4,0,20),seed:number('seed',Math.floor(Math.random()*2147483647),0,2147483647),duration:number('duration',5,1,15),turbo:raw.settings?.turbo !== false, loraStack:Array.isArray(raw.settings?.loraStack)?raw.settings.loraStack:[] }, + settings: { aspect:['auto','16:9','9:16','1:1'].includes(raw.settings?.aspect)?raw.settings.aspect:'auto', fps:number('fps',24,1,60), width:number('width',1344,256,4096),height:number('height',768,256,4096),steps:number('steps',video?8:24,1,100),cfg:number('cfg',video?1.5:4,0,20),seedMode:raw.settings?.seedMode==='locked'?'locked':'random',seed:shotSeeds[0],duration:number('duration',5,1,15),turbo:raw.settings?.turbo !== false, loraStack:Array.isArray(raw.settings?.loraStack)?raw.settings.loraStack:[] }, guides: (Array.isArray(raw.guides)?raw.guides:[]).slice(0,8).map(g=>{ if (!g.stillId || !Number.isInteger(g.frame) || g.frame<1 || g.frame>=frames) fail('Each guide requires a still and positive frame index.'); return {stillId:String(g.stillId),frame:g.frame} }), batch: (Array.isArray(raw.batch)?raw.batch:[]).slice(0,30).map(s=>{const next=sectionsFrom({...promptSections,...(typeof s==='string'?{action:s}:s)});if(!next.action.trim())fail('Every shot needs an action prompt.');return next}) } } diff --git a/shared/studio2/graphs.mjs b/shared/studio2/graphs.mjs index 5adf8ab..4a5e17a 100644 --- a/shared/studio2/graphs.mjs +++ b/shared/studio2/graphs.mjs @@ -34,3 +34,17 @@ export function attachHeroReference(g, image) { } return g } +/** The shared image builders have their own defaults and megapixel scalers. + * Studio 2 must send its already resolved canvas size through those nodes. */ +export function applyResolvedImageSize(graph, {width,height}) { + for (const node of Object.values(graph)) { + if (node.class_type === 'ImageScaleToTotalPixels') { + node.class_type='ImageScale' + node.inputs={image:node.inputs.image,upscale_method:'lanczos',width,height,crop:'disabled'} + } else { + if (typeof node.inputs?.width === 'number') node.inputs.width=width + if (typeof node.inputs?.height === 'number') node.inputs.height=height + } + } + return graph +} diff --git a/shared/studio2/input.mjs b/shared/studio2/input.mjs index 93dd430..c95b951 100644 --- a/shared/studio2/input.mjs +++ b/shared/studio2/input.mjs @@ -11,8 +11,9 @@ export function bindClip(form, clip) { if (!clip?.id) return form const saved = clip.studio2 || {}, settings = saved.settings || clip.settings || clip const next = {...form, startClipId:clip.id, settings:{...form.settings}} + next.settings.aspect=['auto','16:9','9:16','1:1'].includes(settings.aspect)?settings.aspect:'auto' for (const key of ['duration','aspect','width','height','fps','loraStack','turbo','steps','cfg']) { - if (settings[key] != null) next.settings[key] = structuredClone(settings[key]) + if (settings[key] != null && key!=='aspect') next.settings[key] = structuredClone(settings[key]) } next.engine = saved.engine || clip.engine || 'minimax' if (saved.identityStillId) next.identityStillId = saved.identityStillId @@ -23,7 +24,12 @@ export function bindClip(form, clip) { export function restoreAsset(form, item) { const restored = restoreInput(item) const next = {...form,...restored,settings:{...form.settings,...restored.settings}} - return item.kind === 'clip' || item.studio2?.kind === 'video' ? {...bindClip(next,item),mode:'extend'} : next + next.settings.seedMode='random' + if (item.kind === 'clip' || item.studio2?.kind === 'video') return {...bindClip(next,item),mode:'extend'} + next.imageAId=item.id + next.startClipId='' + next.settings.aspect=item.studio2?.settings?.aspect || item.settings?.aspect || 'auto' + return next } export function extendSelected(form, clip) { // Switching task on the already bound clip must preserve the user's overrides. diff --git a/shared/studio2/seed.mjs b/shared/studio2/seed.mjs new file mode 100644 index 0000000..f79fb7e --- /dev/null +++ b/shared/studio2/seed.mjs @@ -0,0 +1,17 @@ +let previous +export function randomSeed() { + const bytes = new Uint32Array(1) + globalThis.crypto.getRandomValues(bytes) + let seed = 2 + bytes[0] % 2147483646 + if (seed === previous) seed = seed === 2147483647 ? 2 : seed + 1 + previous = seed + return seed +} +export function queueSeeds(settings = {}, count = 1) { + if (settings.seedMode === 'locked') { + const seed=Number(settings.seed) + if (!Number.isInteger(seed) || seed<0 || seed>2147483647) throw Object.assign(new Error('Invalid seed.'),{statusCode:400}) + return Array(count).fill(seed) + } + return Array.from({length:count},()=>randomSeed()) +} diff --git a/shared/studio2/viewer.mjs b/shared/studio2/viewer.mjs new file mode 100644 index 0000000..8a80765 --- /dev/null +++ b/shared/studio2/viewer.mjs @@ -0,0 +1,14 @@ +export function folderAssets(library, jobs, folderId) { + const assets=new Map() + for (const kind of ['still','clip']) for (const asset of library[kind==='still'?'stills':'clips'] || []) { + if (asset.folderId===folderId) assets.set(asset.id,{...asset,kind}) + } + for (const job of jobs) if (job.request.folderId===folderId) for (const asset of job.outputs || []) assets.set(asset.id,{...assets.get(asset.id),...asset}) + return [...assets.values()].sort((a,b)=>(a.createdAt || a.studio2?.savedAt || 0)-(b.createdAt || b.studio2?.savedAt || 0)) +} +export function newestOutput(jobs, seen, folderId) { + const fresh=jobs.filter(j=>j.request.folderId===folderId).flatMap(j=>j.outputs || []).filter(a=>!seen.has(a.id)) + return fresh.sort((a,b)=>(a.studio2?.savedAt || 0)-(b.studio2?.savedAt || 0)).at(-1) || null +} +export function selectResult(state, asset) { return {...state,selected:asset} } +export function assetLabel(asset) { return asset?.name || asset?.originalName || (asset ? `Still ${asset.id.slice(0,8)}` : 'None') } diff --git a/tests/studio2-canvas-seed.test.mjs b/tests/studio2-canvas-seed.test.mjs new file mode 100644 index 0000000..2b1fa53 --- /dev/null +++ b/tests/studio2-canvas-seed.test.mjs @@ -0,0 +1,34 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import {folderAssets,newestOutput,selectResult} from '../shared/studio2/viewer.mjs' +import {restoreAsset,resolveSize} from '../shared/studio2/input.mjs' +import {validateRequest} from '../shared/studio2/contracts.mjs' +import {applyResolvedImageSize} from '../shared/studio2/graphs.mjs' +const base={mode:'generate',engine:'flux',folderId:'f',prompt:'Walk'} +test('completed outputs join all folder assets and select only the output canvas',()=>{ + const source={id:'source',folderId:'f'},output={id:'result',kind:'still',studio2:{savedAt:2}} + const jobs=[{request:{folderId:'f'},outputs:[output]}],library={stills:[source,{id:'other',folderId:'f'}],clips:[{id:'video',folderId:'f'}]} + const strip=folderAssets(library,jobs,'f');assert.equal(strip.length,4) + const state=selectResult({imageAId:'source',startClipId:'existing'},newestOutput(jobs,new Set(),'f')) + assert.equal(state.selected.id,'result');assert.equal(state.imageAId,'source');assert.equal(state.startClipId,'existing') + assert.equal(newestOutput(jobs,new Set(['result']),'f'),null) +}) +test('Use as input binds the selected still, not its original parent input',()=>{ + const next=restoreAsset({settings:{}},{id:'output',kind:'still',studio2:{imageAId:'old-parent',settings:{aspect:'1:1',seed:1}}}) + assert.equal(next.imageAId,'output');assert.equal(next.startClipId,'');assert.equal(next.settings.aspect,'1:1');assert.equal(next.settings.seedMode,'random') +}) +test('resolved aspect survives image graph default and megapixel scaling',()=>{ + for(const aspect of ['auto','1:1','16:9','9:16']) { + const size=resolveSize(aspect,{width:1024,height:1024}) + const request=validateRequest({...base,settings:{aspect,...size}}) + const graph=applyResolvedImageSize({latent:{inputs:{width:1344,height:768}},scale:{class_type:'ImageScaleToTotalPixels',inputs:{image:['load',0],megapixels:1}}},request.settings) + assert.equal(graph.latent.inputs.width,size.width);assert.equal(graph.latent.inputs.height,size.height) + assert.equal(graph.scale.inputs.crop,'disabled');assert.equal(graph.scale.inputs.width,size.width) + if(['auto','1:1'].includes(aspect))assert.equal(size.width,size.height) + } +}) +test('random Queues and shots differ; typed seed including 1 remains locked',()=>{ + const a=validateRequest({...base,batch:['Turn','Sit']}),b=validateRequest(base) + assert.notEqual(a.settings.seed,b.settings.seed);assert.equal(new Set(a.shotSeeds).size,3);assert.ok(a.shotSeeds.every(s=>s!==1)) + for(const seed of [1,987]){const raw={...base,batch:['Turn'],settings:{seedMode:'locked',seed}};assert.deepEqual(validateRequest(raw).shotSeeds,[seed,seed]);assert.equal(validateRequest(raw).settings.seed,seed)} +}) diff --git a/tests/studio2-runner.test.mjs b/tests/studio2-runner.test.mjs index 8956ef1..5259405 100644 --- a/tests/studio2-runner.test.mjs +++ b/tests/studio2-runner.test.mjs @@ -1,3 +1,4 @@ +import {queueSeeds} from '../shared/studio2/seed.mjs' import test from 'node:test' import assert from 'node:assert/strict' import {readFileSync,mkdtempSync,mkdirSync,writeFileSync,existsSync} from 'node:fs' @@ -16,7 +17,7 @@ function fixture(mode='video') { const job={status:'running',clientId:'client',library:{}} let id=0 const scope={ - watchProgress:()=>()=>{},compilePrompt,saveRecord(){},update(r,state){r.state=state;phases.push(state)},acquireSharedGpu:async()=>true,ensureComfyReady:async()=>{},emitJob(){}, + queueSeeds,watchProgress:()=>()=>{},compilePrompt,saveRecord(){},update(r,state){r.state=state;phases.push(state)},acquireSharedGpu:async()=>true,ensureComfyReady:async()=>{},emitJob(){}, prepareGraph:async r=>{queued.push({hero:r.request.identityStillId,start:r.request.startClipId,action:r.request.promptSections.action});return {}},queuePrompt:async()=>({prompt_id:`p${++id}`}), fetchHistory:async p=>({[p]:{status:{status_str:'success'},outputs:{}}}),comfyFetch:()=>{throw new Error('Unexpected network call')}, extractVideo:()=>({filename:'video.mp4',subfolder:'preview/studio2/r',type:'output'}),extractEditedImage:()=>({filename:'image.png',subfolder:'preview/studio2/r',type:'output'}),