Add Studio 2 caption jobs via Qwen2.5-VL GGUF on the host agent.
Exclusive llama-server load/unload on the 5080, queued Describe UI with resultText, Copy, and Use as prompt. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
@@ -11,7 +11,7 @@
|
||||
<option v-for="f in library.folders" :key="f.id" :value="f.id">{{ f.name }}</option>
|
||||
</select>
|
||||
</label>
|
||||
<div v-if="isClassic && form.mode!=='music'" class="flex flex-wrap items-center gap-3 text-sm text-zinc-300" aria-label="Identity locks">
|
||||
<div v-if="isClassic && form.mode!=='music' && form.mode!=='caption'" class="flex flex-wrap items-center gap-3 text-sm text-zinc-300" aria-label="Identity locks">
|
||||
<label class="inline-flex items-center gap-1.5"><input v-model="form.lockFace" type="checkbox" class="accent-amber-400">Lock face</label>
|
||||
<label class="inline-flex items-center gap-1.5"><input v-model="form.lockOutfit" type="checkbox" class="accent-amber-400">Lock outfit</label>
|
||||
</div>
|
||||
@@ -45,7 +45,7 @@
|
||||
>{{ task }}</button>
|
||||
</nav>
|
||||
|
||||
<div v-if="form.mode!=='music'" class="min-w-0">
|
||||
<div v-if="form.mode!=='music' && form.mode!=='caption'" class="min-w-0">
|
||||
<label v-if="isClassic" class="display-body block text-zinc-300">Model
|
||||
<select v-model="form.engine" class="display-control mt-1 w-full max-w-md border border-white/10 bg-zinc-900 px-3 py-2 outline-none ring-amber-300/40 focus:ring-2">
|
||||
<option v-for="opt in engineOptions" :key="opt.value" :value="opt.value" :disabled="opt.disabled">{{ opt.label }}{{ opt.hint ? ' — '+opt.hint : '' }}</option>
|
||||
@@ -68,7 +68,8 @@
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<p v-else class="display-body text-zinc-400">Model · YuE2</p>
|
||||
<p v-else-if="form.mode==='music'" class="display-body text-zinc-400">Model · YuE2</p>
|
||||
<p v-else class="display-body text-zinc-400">Model · Qwen2.5-VL caption</p>
|
||||
</div>
|
||||
|
||||
<div class="grid min-w-0 gap-6 xl:grid-cols-[minmax(0,3fr)_minmax(24rem,2fr)] 2xl:gap-8">
|
||||
@@ -103,6 +104,7 @@
|
||||
<button v-if="selected.kind!=='track'" class="rounded-full border border-white/20 px-4 py-2 text-sm outline-none ring-amber-300/40 focus-visible:ring-2" @click="restore(selected)">Use as input</button>
|
||||
<button v-if="selected.kind==='still'" class="rounded-full border border-white/20 px-4 py-2 text-sm outline-none ring-amber-300/40 focus-visible:ring-2" @click="Object.assign(form,useAsSource(form,selected))">Use as source still</button>
|
||||
<button v-if="selected.kind==='still'" class="rounded-full border border-white/20 px-4 py-2 text-sm outline-none ring-amber-300/40 focus-visible:ring-2" @click="Object.assign(form,useAsHero(form,selected))">Use as hero</button>
|
||||
<button v-if="selected.kind==='still' || (selected.kind!=='track' && selected.kind!=='clip' && selected.id)" class="rounded-full border border-amber-300/40 px-4 py-2 text-sm text-amber-100 outline-none ring-amber-300/40 focus-visible:ring-2" :disabled="submitting" @click="describeSelected">Describe</button>
|
||||
<button v-if="selected.kind!=='track'" class="rounded-full border border-white/20 px-4 py-2 text-sm outline-none ring-amber-300/40 focus-visible:ring-2" @click="detailsOpen=true">Details</button>
|
||||
<button class="rounded-full border border-red-400/40 px-4 py-2 text-sm text-red-200 outline-none ring-amber-300/40 focus-visible:ring-2" @click="deleteOutput">Delete</button>
|
||||
<VideoUpscaleControl v-if="selected.kind==='clip'" :clip-id="selected.id" queue-only @complete="refresh" />
|
||||
@@ -139,7 +141,7 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section v-if="form.mode!=='music'" aria-label="Source" class="display-card">
|
||||
<section v-if="form.mode!=='music' && form.mode!=='caption'" aria-label="Source" class="display-card">
|
||||
<div class="mb-3 flex flex-wrap items-start justify-between gap-3">
|
||||
<h2 class="text-xs uppercase tracking-wider text-zinc-400">Source</h2>
|
||||
<div v-if="isStudio" class="flex flex-wrap gap-4 text-sm text-zinc-300" aria-label="Identity locks">
|
||||
@@ -194,6 +196,18 @@
|
||||
<input v-model="music.name" class="mt-1 w-full rounded-xl bg-zinc-900 p-3 outline-none ring-amber-300/40 focus:ring-2" placeholder="Optional track name">
|
||||
</label>
|
||||
</template>
|
||||
<template v-else-if="form.mode==='caption'">
|
||||
<p class="text-sm text-zinc-400">Qwen2.5-VL · image-to-text · exclusive GPU · unloads after one shot</p>
|
||||
<div class="display-card space-y-3 p-3">
|
||||
<Studio2AssetSlot label="Still to describe" :asset="captionStill" :hidden="slotHidden(captionStillId)" upload :disabled="uploading" @pick="picker='captionStillId'" @upload="uploadCaption($event)" />
|
||||
<label class="block text-sm text-zinc-300">Caption style
|
||||
<select v-model="captionStyle" class="mt-1 w-full rounded-xl border border-white/10 bg-zinc-900 p-3 outline-none ring-amber-300/40 focus:ring-2">
|
||||
<option v-for="style in captionStyles" :key="style" :value="style">{{ style }}</option>
|
||||
</select>
|
||||
</label>
|
||||
<p v-if="captionStyle==='delta'" class="text-xs text-zinc-500">Delta describes only changes from the locked look (bob / pink tank / blue shorts / Slytherin robe / barefoot).</p>
|
||||
</div>
|
||||
</template>
|
||||
<template v-else>
|
||||
<label class="block">Prompt<textarea v-model="form.promptSections.action" rows="4" class="mt-1 w-full rounded-xl bg-zinc-900 p-3 outline-none ring-amber-300/40 focus:ring-2" placeholder="Describe the image or the shot." /></label>
|
||||
<Studio2PromptControls v-if="video" v-model="form.promptSections" />
|
||||
@@ -223,6 +237,25 @@
|
||||
<progress class="h-2 w-full accent-amber-300" max="100" :value="job.progress || 0" />
|
||||
<p class="text-xs text-zinc-400">{{ job.message }}</p>
|
||||
</div>
|
||||
<div v-for="job in captionJobs" :key="job.id" class="space-y-2 rounded-xl border border-amber-300/25 bg-zinc-900 p-3 text-sm" aria-live="polite" role="status">
|
||||
<div class="flex justify-between gap-3">
|
||||
<div class="flex min-w-0 items-start gap-3">
|
||||
<img v-if="job.request?.stillId || job.request?.imageAId" :src="stillUrl(job.request.stillId || job.request.imageAId)" class="h-14 w-14 shrink-0 rounded object-cover" alt="Caption source">
|
||||
<div class="min-w-0">
|
||||
<strong class="block truncate">Describe · {{ job.request?.captionStyle || 'caption' }}</strong>
|
||||
<span class="capitalize text-amber-200">{{ job.state==='waiting'?'queued':job.state }}</span>
|
||||
</div>
|
||||
</div>
|
||||
<button v-if="!['complete','failed','cancelled'].includes(job.state)" type="button" class="shrink-0 text-zinc-300" @click="cancel(job)">Cancel</button>
|
||||
</div>
|
||||
<p class="text-xs text-zinc-400">{{ elapsedCaption(job) }}</p>
|
||||
<p v-if="job.resultText" class="whitespace-pre-wrap text-zinc-200">{{ job.resultText }}</p>
|
||||
<div v-if="job.resultText" class="flex flex-wrap gap-2">
|
||||
<button type="button" class="rounded-full border border-white/20 px-3 py-1.5 text-xs outline-none ring-amber-300/40 focus-visible:ring-2" @click="copyCaption(job.resultText)">Copy</button>
|
||||
<button type="button" class="rounded-full border border-amber-300/40 px-3 py-1.5 text-xs text-amber-100 outline-none ring-amber-300/40 focus-visible:ring-2" @click="useCaptionAsPrompt(job.resultText)">Use as prompt</button>
|
||||
</div>
|
||||
<p v-if="job.error" role="alert" class="text-red-300">{{ job.error }}</p>
|
||||
</div>
|
||||
<div v-for="job in musicJobs" :key="job.id" class="space-y-2 rounded-xl border border-amber-300/25 bg-zinc-900 p-3 text-sm" aria-live="polite" role="status">
|
||||
<div class="flex justify-between gap-3">
|
||||
<strong class="truncate">{{ job.name || 'YuE2' }}</strong>
|
||||
@@ -233,7 +266,7 @@
|
||||
<p v-if="job.lastError || job.live?.error" role="alert" class="text-red-300">{{ job.lastError || job.live?.error }}</p>
|
||||
</div>
|
||||
<Studio2JobProgress v-for="job in benchJobs" :key="job.id" :job="job" :now="now" @select="focusJob(job)" @cancel="cancel(job)" />
|
||||
<button v-if="folderJobs.some(j=>['complete','failed','cancelled'].includes(j.state)) || musicJobs.some(j=>['complete','done','error','cancelled'].includes(j.status))" type="button" class="text-sm text-zinc-400 outline-none ring-amber-300/40 focus-visible:ring-2" @click="clearFinished">Clear finished</button>
|
||||
<button v-if="folderJobs.some(j=>['complete','failed','cancelled'].includes(j.state)) || musicJobs.some(j=>['complete','done','error','cancelled'].includes(j.status)) || captionJobs.some(j=>['complete','failed','cancelled'].includes(j.state))" type="button" class="text-sm text-zinc-400 outline-none ring-amber-300/40 focus-visible:ring-2" @click="clearFinished">Clear finished</button>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
@@ -288,9 +321,10 @@
|
||||
import {createLoraMemory,loraUse} from '~/shared/studio2/lora-memory.mjs'
|
||||
import {filterLorasForStudio,filterLorasForUse} from '~/utils/loras'
|
||||
import {compilePrompt,sectionsFrom} from '~/shared/studio2/contracts.mjs'
|
||||
import {CAPTION_STYLES} from '~/shared/studio2/caption.mjs'
|
||||
import {aspectOptions,resolveSize,bindClip,changeSource,restoreAll,useAsHero,useAsSource} from '~/shared/studio2/input.mjs'
|
||||
import {folderAssets,newestOutput,assetLabel,reconcileSelection,previewHidden} from '~/shared/studio2/viewer.mjs'
|
||||
import {queueJobs} from '~/shared/studio2/queue-view.mjs'
|
||||
import {queueJobs,elapsedSeconds} from '~/shared/studio2/queue-view.mjs'
|
||||
import {gpuStatus} from '~/shared/studio2/status.mjs'
|
||||
import {applyUploadedStill} from '~/shared/studio2/upload.mjs'
|
||||
import {parseBatch,stylePrompt} from '~/shared/studio2/styles.mjs'
|
||||
@@ -312,12 +346,15 @@ const liveMusic=ref<Record<string,any>>({})
|
||||
const form=reactive<any>({mode:'generate',engine:'flux',imageStyles:{positive:[],negative:[]},lockFace:true,lockOutfit:true,hideInput:xaigen,folderId:'',identityStillId:'',imageAId:'',imageBId:'',maskId:'',startClipId:'',endStillId:'',guides:[],promptSections:sectionsFrom({}),settings:{aspect:'auto',fps:24,steps:24,cfg:4,seed:null,seedMode:'random',duration:5,turbo:true,loraStack:[]}})
|
||||
const music=reactive({tags:DEFAULT_MUSIC_TAGS,lyrics:DEFAULT_MUSIC_LYRICS,duration:60,seed:'',name:''})
|
||||
const tagsPlaceholder=DEFAULT_MUSIC_TAGS,lyricsPlaceholder=DEFAULT_MUSIC_LYRICS
|
||||
const captionStyles=CAPTION_STYLES
|
||||
const captionStyle=ref('descriptive')
|
||||
const captionStillId=ref('')
|
||||
let loraMemory=createLoraMemory(xaigen)
|
||||
const currentLoraUse=computed(()=>loraUse(form))
|
||||
const pickerLoras=computed(()=>!xaigen && currentLoraUse.value==='ltx'?[]:filterLorasForUse(filterLorasForStudio([...loraNames.value.image,...loraNames.value.video],xaigen),currentLoraUse.value))
|
||||
watch(()=>[currentLoraUse.value,form.settings.loraStack],()=>loraMemory.sync(form),{deep:true,flush:'sync'})
|
||||
const route=useRoute()
|
||||
const tasks=['generate','edit','compose','refine','video','extend','music']
|
||||
const tasks=['generate','edit','compose','refine','video','extend','music','caption']
|
||||
const settingsOpen=ref(false),detailsOpen=ref(false),submitting=ref(false),error=ref(''),batchText=ref('')
|
||||
const video=computed(()=>['video','extend'].includes(form.mode))
|
||||
const kreaSticky=computed(()=>['edit','compose','iterate'].includes(form.mode))
|
||||
@@ -353,11 +390,13 @@ const clips=computed(()=>library.value.clips.filter((s:any)=>s.folderId===form.f
|
||||
const compiled=computed(()=>compilePrompt(form.promptSections,video.value,form))
|
||||
const stillUrl=(id:string)=>`/api/library/stills/${id}`
|
||||
const folderJobs=computed(()=>jobs.value.filter(j=>j.request.folderId===form.folderId))
|
||||
const benchJobs=computed(()=>queueJobs(folderJobs.value))
|
||||
const captionJobs=computed(()=>folderJobs.value.filter(j=>j.request?.mode==='caption'))
|
||||
const benchJobs=computed(()=>queueJobs(folderJobs.value.filter(j=>j.request?.mode!=='caption')))
|
||||
const musicJobs=computed(()=>studioQueue.value
|
||||
.filter((j:any)=>j.kind==='music' && j.folderId===form.folderId && !['complete','done'].includes(j.status))
|
||||
.map((j:any)=>({...j,live:j.liveJobId?liveMusic.value[j.liveJobId]:null})))
|
||||
const gpu=computed(()=>gpuStatus(jobs.value,healthState.value,upscales.value,musicJobs.value))
|
||||
const gpu=computed(()=>gpuStatus(jobs.value,healthState.value,upscales.value,musicJobs.value,captionJobs.value))
|
||||
const captionStill=computed(()=>stills.value.find((s:any)=>s.id===captionStillId.value))
|
||||
const sourceSlot=computed(()=>{const clip=clips.value.find((a:any)=>a.id===form.startClipId);return form.mode==='extend'?(clip?{...clip,kind:'clip'}:null):inputStill.value})
|
||||
const pickerAssets=computed(()=>picker.value==='startClipId'?clips.value.map((a:any)=>({...a,kind:'clip'})):stills.value.map((a:any)=>({...a,kind:'still'})))
|
||||
const inputStill=computed(()=>stills.value.find((s:any)=>s.id===form.imageAId))
|
||||
@@ -366,6 +405,42 @@ const sourceAsset=computed(()=>form.startClipId?clips.value.find((c:any)=>c.id==
|
||||
const sourceSize=computed(()=>measured[sourceAsset.value?.id] || sourceAsset.value)
|
||||
const resolvedSize=computed(()=>resolveSize(form.settings.aspect,sourceSize.value,undefined,form.engine))
|
||||
|
||||
function elapsedCaption(job:any){
|
||||
const seconds=elapsedSeconds(job,now.value)
|
||||
return `${Math.floor(seconds/60)}m ${seconds%60}s ${job.startedAt||job.state==='rendering'||job.state==='saving'?'elapsed':'waiting'}`
|
||||
}
|
||||
async function copyCaption(text:string){
|
||||
try{await navigator.clipboard.writeText(text)}catch{error.value='Could not copy caption.'}
|
||||
}
|
||||
function useCaptionAsPrompt(text:string){
|
||||
const current=String(form.promptSections.action||'').trim()
|
||||
form.promptSections.action=current?`${current}\n\n${text}`:text
|
||||
if(form.mode==='caption' || form.mode==='music')form.mode='generate'
|
||||
}
|
||||
async function queueCaption(stillId:string){
|
||||
if(!stillId){error.value='Choose a still to describe.';return false}
|
||||
try{
|
||||
await $fetch('/api/studio-2/caption',{method:'POST',body:{folderId:form.folderId,stillId,sourcePath:stillId,captionStyle:captionStyle.value}})
|
||||
await refresh()
|
||||
return true
|
||||
}catch(e:any){error.value=e.data?.statusMessage||e.message;return false}
|
||||
}
|
||||
function describeSelected(){
|
||||
const id=selected.value?.kind==='still'?selected.value.id:(selected.value?.kind==='clip'? '' : (selected.value?.id || form.imageAId || form.identityStillId))
|
||||
if(!id){error.value='Select a still to describe.';return}
|
||||
captionStillId.value=id
|
||||
submitting.value=true;error.value=''
|
||||
void queueCaption(id).finally(()=>{submitting.value=false})
|
||||
}
|
||||
async function uploadCaption(event:Event){
|
||||
const input=event.target as HTMLInputElement,file=input.files?.[0];if(!file)return
|
||||
uploading.value=true;uploadMessage.value='';error.value=''
|
||||
const body=new FormData();body.append('image',file);body.append('folderId',form.folderId);body.append('hideInput',String(xaigen))
|
||||
try{const response=await $fetch<any>('/api/studio-2/stills',{method:'POST',body});library.value=response;captionStillId.value=response.still.id;measured[response.still.id]={width:response.still.width,height:response.still.height};uploadMessage.value=`Loaded ${response.still.name || file.name} for caption.`}
|
||||
catch(e:any){error.value=e.data?.statusMessage||e.message}
|
||||
finally{uploading.value=false;input.value=''}
|
||||
}
|
||||
|
||||
function isHidden(asset:any){
|
||||
if(!xaigen || !asset?.id)return false
|
||||
if(isClassic.value){
|
||||
@@ -396,6 +471,11 @@ watch(()=>sourceAsset.value?.id,id=>{const asset=sourceAsset.value;if(!import.me
|
||||
|
||||
function chooseTask(task:string){
|
||||
if(task==='music'){form.mode='music';return}
|
||||
if(task==='caption'){
|
||||
form.mode='caption'
|
||||
captionStillId.value=selected.value?.kind==='still'?selected.value.id:form.imageAId||form.identityStillId||captionStillId.value
|
||||
return
|
||||
}
|
||||
if(task==='extend'){
|
||||
form.mode='extend'
|
||||
form.engine=xaigen && form.engine==='ltx'?'ltx':'minimax'
|
||||
@@ -414,7 +494,7 @@ function selectOutput(item:any){selected.value=item;if(form.mode==='extend' && i
|
||||
function restore(item:any){Object.assign(form,restoreAll(form,item,xaigen,loraMemory));form.guides=form.guides||[];batchText.value=''}
|
||||
function focusJob(job:any){selectOutput(job.outputs?.at(-1)||null)}
|
||||
async function loadLoras(fresh=false){try{loraNames.value=await $fetch('/api/studio-2/loras'+(fresh?'?fresh=1':''))}catch{error.value='Could not refresh LoRAs. The last loaded list is still available.'}}
|
||||
function pickAsset(asset:any){const key=picker.value;if(key.startsWith('guide:'))form.guides[Number(key.split(':')[1])].stillId=asset?.id||'';else if(key==='startClipId')chooseClip(asset?.id||'');else form[key]=asset?.id||'';picker.value=''}
|
||||
function pickAsset(asset:any){const key=picker.value;if(key.startsWith('guide:'))form.guides[Number(key.split(':')[1])].stillId=asset?.id||'';else if(key==='startClipId')chooseClip(asset?.id||'');else if(key==='captionStillId')captionStillId.value=asset?.id||'';else form[key]=asset?.id||'';picker.value=''}
|
||||
async function clearFinished(){await $fetch('/api/studio-2/clear-finished',{method:'POST'});await refresh()}
|
||||
async function deleteOutput(){
|
||||
if(!selected.value)return
|
||||
@@ -432,6 +512,10 @@ async function queue(){
|
||||
await refresh()
|
||||
return
|
||||
}
|
||||
if(form.mode==='caption'){
|
||||
await queueCaption(captionStillId.value || form.imageAId || form.identityStillId)
|
||||
return
|
||||
}
|
||||
if(form.settings.seedMode!=='locked')form.settings.seed=randomSeed()
|
||||
const queued=await $fetch<any>('/api/studio-2/jobs',{method:'POST',body:{...form,mode:form.mode==='edit' && parseBatch(batchText.value).length?'iterate':form.mode,hideInput:xaigen,hideOutput:false,promptSections:video.value?form.promptSections:{action:form.promptSections.action},settings:{...form.settings,...resolvedSize.value},batch:video.value || form.mode==='edit'?parseBatch(batchText.value):[]}})
|
||||
form.settings.seed=queued.seed;form.settings.width=queued.width;form.settings.height=queued.height
|
||||
@@ -486,6 +570,7 @@ onMounted(async()=>{
|
||||
const asset=[...library.value.stills,...library.value.clips,...(library.value.tracks||[])].find((a:any)=>a.id===id)
|
||||
if(asset){selected.value={...asset,kind:route.query.clip?'clip':route.query.track?'track':'still'};if(route.query.use==='1'&&selected.value.kind!=='track')restore(selected.value);if(xaigen)toggleReveal(asset.id)}
|
||||
if(route.query.task==='music')form.mode='music'
|
||||
if(route.query.task==='caption')form.mode='caption'
|
||||
const poll=async()=>{try{await refresh();const health=await $fetch<any>('/api/health');healthState.value=health}catch(e:any){healthState.value=null;error.value=e.data?.statusMessage||e.message}if(!disposed)timer=setTimeout(poll,3000)};void poll()
|
||||
})
|
||||
onBeforeUnmount(()=>{disposed=true;clearTimeout(timer);clearInterval(clock)})
|
||||
|
||||
Reference in New Issue
Block a user