Keep YuE2 jobs live on the bench and wire target length to semantic tokens.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-15 20:53:07 -05:00
parent 2ebc3066e6
commit 5694104226
13 changed files with 92 additions and 29 deletions
@@ -49,7 +49,9 @@
      <span aria-hidden="true">{{ isHidden(selected) ? 'Show' : 'Hide' }}</span>
     </button>
     <p v-if="selected && isHidden(selected)" class="text-zinc-400">{{ selected.kind==='track' ? 'Track loaded · preview hidden' : selected.role==='input' ? 'Still loaded · preview hidden' : 'Output hidden' }}</p>
-     <audio v-else-if="selected?.kind==='track'" :key="selected.id" :src="`/api/library/tracks/${selected.id}/audio`" controls class="w-full max-w-xl px-6" />
+     <div v-else-if="selected?.kind==='track'" class="w-full max-w-xl px-6 py-4">
+      <AudioPlayer :src="`/api/library/tracks/${selected.id}/audio`" :download-name="`${selected.name || 'track'}.${selected.ext || 'wav'}`" :label="selected.name || 'Track'" />
+     </div>
     <video v-else-if="selected?.kind==='clip'" :key="selected.id" :src="`/api/library/clips/${selected.id}/video`" controls class="max-h-[65vh] w-full object-contain" />
     <img v-else-if="selected" :src="stillUrl(selected.id)" class="max-h-[65vh] max-w-full object-contain" alt="Current output">
     <p v-else class="text-zinc-500">Select a result to view it here</p>
@@ -128,8 +130,9 @@
      <textarea v-model="music.lyrics" rows="8" class="mt-1 w-full rounded-xl bg-zinc-900 p-3 outline-none ring-amber-300/40 focus:ring-2" :placeholder="lyricsPlaceholder" />
     </label>
     <div class="grid grid-cols-2 gap-3">
-      <label class="block text-sm">Duration (sec)
-       <input v-model.number="music.duration" type="number" min="30" max="150" class="mt-1 w-full rounded-xl bg-zinc-900 p-3 outline-none ring-amber-300/40 focus:ring-2">
+      <label class="block text-sm">Target length (sec)
+       <input v-model.number="music.duration" type="number" min="15" max="150" class="mt-1 w-full rounded-xl bg-zinc-900 p-3 outline-none ring-amber-300/40 focus:ring-2">
+       <span class="mt-1 block text-[11px] text-zinc-500">Caps semantic tokens (~25/s). Actual length follows the model end token — not an FFmpeg trim.</span>
      </label>
      <label class="block text-sm">Seed
       <input v-model="music.seed" placeholder="random" class="mt-1 w-full rounded-xl bg-zinc-900 p-3 outline-none ring-amber-300/40 focus:ring-2">
@@ -167,13 +170,14 @@
     <progress class="h-2 w-full accent-amber-300" max="100" :value="job.progress || 0" />
     <p class="text-xs text-zinc-400">{{ job.message }}</p>
    </div>
-    <div v-for="job in musicJobs" :key="job.id" class="space-y-2 rounded-xl border border-white/10 p-3 text-sm" aria-live="polite">
+    <div v-for="job in musicJobs" :key="job.id" class="space-y-2 rounded-xl border border-amber-300/25 bg-zinc-900 p-3 text-sm" aria-live="polite" role="status">
     <div class="flex justify-between gap-3">
      <strong class="truncate">{{ job.name || 'YuE2' }}</strong>
-      <span class="capitalize text-amber-200">{{ job.status }}</span>
+      <span class="capitalize text-amber-200">{{ job.live?.status || job.status }}</span>
     </div>
-     <p class="text-xs text-zinc-400">Music · {{ job.payload?.duration || '—' }}s</p>
-     <p v-if="job.lastError" role="alert" class="text-red-300">{{ job.lastError }}</p>
+     <p class="text-xs text-zinc-400">YuE2 · target {{ job.duration || '—' }}s<span v-if="job.live?.message"> · {{ job.live.message }}</span></p>
+     <progress v-if="job.live?.progress" class="h-2 w-full accent-amber-300" max="100" :value="job.live.progress || 0" />
+     <p v-if="job.lastError || job.live?.error" role="alert" class="text-red-300">{{ job.lastError || job.live?.error }}</p>
    </div>
    <Studio2JobProgress v-for="job in benchJobs" :key="job.id" :job="job" :now="now" @select="focusJob(job)" @cancel="cancel(job)" />
    <button v-if="folderJobs.some(j=>['complete','failed','cancelled'].includes(j.state)) || musicJobs.some(j=>['complete','done','error','cancelled'].includes(j.status))" type="button" class="text-sm text-zinc-400 outline-none ring-amber-300/40 focus-visible:ring-2" @click="clearFinished">Clear finished</button>
@@ -256,9 +260,9 @@ const uploading=ref(false),uploadMessage=ref('')
 const revealed=ref<Set<string>>(new Set())
 const measured=reactive<Record<string,{width:number;height:number}>>({}),seenOutputs=new Set<string>()
 const loraNames=ref<any>({image:[],video:[]}),now=ref(Date.now()),healthState=ref<any>(null),upscales=ref<any[]>([])
-const gpu=computed(()=>gpuStatus(jobs.value,healthState.value,upscales.value))
 const library=ref<any>({folders:[],stills:[],clips:[],tracks:[]}),jobs=ref<any[]>([]),selected=ref<any>(null)
 const studioQueue=ref<any[]>([])
+const liveMusic=ref<Record<string,any>>({})
 const form=reactive<any>({mode:'generate',engine:'flux',imageStyles:{positive:[],negative:[]},lockFace:true,lockOutfit:true,hideInput:xaigen,folderId:'',identityStillId:'',imageAId:'',imageBId:'',maskId:'',startClipId:'',endStillId:'',guides:[],promptSections:sectionsFrom({}),settings:{aspect:'auto',fps:24,steps:24,cfg:4,seed:null,seedMode:'random',duration:5,turbo:true,loraStack:[]}})
 const music=reactive({tags:DEFAULT_MUSIC_TAGS,lyrics:DEFAULT_MUSIC_LYRICS,duration:60,seed:'',name:''})
 const tagsPlaceholder=DEFAULT_MUSIC_TAGS,lyricsPlaceholder=DEFAULT_MUSIC_LYRICS
@@ -276,7 +280,10 @@ const compiled=computed(()=>compilePrompt(form.promptSections,video.value,form))
 const stillUrl=(id:string)=>`/api/library/stills/${id}`
 const folderJobs=computed(()=>jobs.value.filter(j=>j.request.folderId===form.folderId))
 const benchJobs=computed(()=>queueJobs(folderJobs.value))
-const musicJobs=computed(()=>studioQueue.value.filter((j:any)=>j.kind==='music' && j.payload?.folderId===form.folderId && !['complete','done'].includes(j.status)))
+const musicJobs=computed(()=>studioQueue.value
+ .filter((j:any)=>j.kind==='music' && j.folderId===form.folderId && !['complete','done'].includes(j.status))
+ .map((j:any)=>({...j,live:j.liveJobId?liveMusic.value[j.liveJobId]:null})))
+const gpu=computed(()=>gpuStatus(jobs.value,healthState.value,upscales.value,musicJobs.value))
 const sourceSlot=computed(()=>{const clip=clips.value.find((a:any)=>a.id===form.startClipId);return form.mode==='extend'?(clip?{...clip,kind:'clip'}:null):inputStill.value})
 const pickerAssets=computed(()=>picker.value==='startClipId'?clips.value.map((a:any)=>({...a,kind:'clip'})):stills.value.map((a:any)=>({...a,kind:'still'})))
 const inputStill=computed(()=>stills.value.find((s:any)=>s.id===form.imageAId))
@@ -365,6 +372,14 @@ async function refresh(){
 ])
 jobs.value=nextJobs;library.value=nextLibrary;upscales.value=nextUpscales
 studioQueue.value=Array.isArray(queue)?queue:(queue?.jobs||[])
+ const musicRows=studioQueue.value.filter((j:any)=>j.kind==='music' && j.liveJobId && !['complete','done'].includes(j.status))
+ const liveEntries=await Promise.all(musicRows.map(async(j:any)=>{
+  try{return [j.liveJobId,await $fetch<any>(`/api/generate/${j.liveJobId}`)] as const}
+  catch{return [j.liveJobId,null] as const}
+ }))
+ const nextLive:Record<string,any>={}
+ for(const [id,snap] of liveEntries){if(snap)nextLive[id]=snap}
+ liveMusic.value=nextLive
 const fresh=newestOutput(nextJobs,seenOutputs,form.folderId)
 for(const job of nextJobs)for(const output of job.outputs||[])seenOutputs.add(output.id)
 const existingFresh=family.value.find(a=>a.id===fresh?.id)
@@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url'

 export function validateYue2Request(body) {
  const duration = body.duration ?? 60
-  if (!Number.isInteger(duration) || duration < 30 || duration > 150) throw new Error('YuE2 duration must be 30–150 seconds.')
+  if (!Number.isInteger(duration) || duration < 15 || duration > 150) throw new Error('YuE2 target length must be 15–150 seconds.')
  if (!Number.isInteger(body.seed) || body.seed < 0 || body.seed > 2147483647) throw new Error('Invalid seed.')
  if (!/^[a-zA-Z0-9-]{12,80}$/.test(body.id || '')) throw new Error('Invalid job ID.')
  const tags = String(body.tags || '').trim()
@@ -97,8 +97,10 @@ def main():
    root = Path(cli.root).resolve()
    output = Path(cli.request).resolve().parent
    duration = int(request.get('duration', 60))
-    if not 30 <= duration <= 150:
-        raise ValueError('Duration must be 30–150 seconds.')
+    if not 15 <= duration <= 150:
+        raise ValueError('Target length must be 15–150 seconds.')
+    # VAE downsampling_ratio 1920 @ 48 kHz → 25 semantic / latent frames per second.
+    max_tokens = max(200, min(9000, duration * 25))
    lyrics = normalize_lyrics(request['lyrics'])
    style = ' '.join(str(request['tags']).split())
    seed = int(request['seed'])
@@ -111,9 +113,11 @@ def main():
    import soundfile as sf
    attention_backend, pipeline_backend = resolve_attention_backend(torch)
    emit(stage='loading', progress=1, message='Loading YuE2', model=model, vae=vae,
-         duration=duration, attention_backend=attention_backend, pipeline_backend=pipeline_backend,
+         duration=duration, maxTokens=max_tokens, attention_backend=attention_backend,
+         pipeline_backend=pipeline_backend,
         cudaAllocConf=os.environ.get('PYTORCH_CUDA_ALLOC_CONF'))
    from yue2 import YuE2Pipeline
+    from yue2.protocol import Sampling
    if not torch.cuda.is_available():
        raise RuntimeError('YuE2 requires a CUDA GPU; CPU fallback is disabled.')
    patch_graph_attention(attention_backend)
@@ -129,6 +133,8 @@ def main():
    )
    cot = 'full'
    pipe_kwargs = dict(style=style, lyrics=lyrics, cot=cot, seed=seed)
+    # Cap semantic length from target seconds. Do not FFmpeg-trim after decode.
+    semantic_sampling = Sampling(max_tokens=max_tokens, min_tokens=min(200, max_tokens))
    # YuE2Pipeline.__init__ always calls set_per_process_memory_fraction; skip it on this 16GB host.
    _set_fraction = torch.cuda.set_per_process_memory_fraction
    torch.cuda.set_per_process_memory_fraction = lambda *args, **kwargs: None
@@ -153,8 +159,9 @@ def main():
            plan = pipe.plan(**pipe_kwargs)
        free_cuda(torch)
        memory_snapshot(torch, 'memory-after-plan')
-        emit(stage='semantic', message='Generating semantic tokens', progress=25, cot=cot)
-        semantic = pipe.generate_semantic(plan)
+        emit(stage='semantic', message='Generating semantic tokens', progress=25, cot=cot,
+             maxTokens=max_tokens, targetSeconds=duration)
+        semantic = pipe.generate_semantic(plan, sampling=semantic_sampling)
        free_cuda(torch)
        memory_snapshot(torch, 'memory-after-semantic')
        emit(stage='synthesize', message='Synthesizing acoustic latents', progress=55, cot=cot)
@@ -48,11 +48,16 @@ export default defineEventHandler(async () => {
      host: comfy.host || '',
      queueRunning: queue.running,
      queuePending: queue.pending,
-      asleep: Boolean(control?.asleep) && !comfyReachable,
+      asleep: Boolean(control?.asleep) && !comfyReachable && !control?.yue2?.busy,
      processUp: Boolean(control?.process ?? control?.processUp),
      lastActivityAt: control?.lastActivityAt || null,
      stats: comfy.ok ? comfy.stats : undefined
    },
+    yue2: {
+      configured: Boolean(control?.yue2?.configured),
+      busy: Boolean(control?.yue2?.busy),
+      backend: control?.yue2?.backend || 'yue2'
+    },
    imageComfy: {
      configured: imageConfigured,
      ok: imageOk,
@@ -65,7 +65,7 @@ export default defineEventHandler(async (event) => {
    const duration = optionalNumber(body.duration)
    if (duration != null) {
      if (row.kind === 'music') {
-        if (!Number.isInteger(duration) || duration < 30 || duration > 150) throw createError({ statusCode: 400, statusMessage: 'YuE2 duration must be 30–150 seconds.' })
+        if (!Number.isInteger(duration) || duration < 15 || duration > 150) throw createError({ statusCode: 400, statusMessage: 'YuE2 target length must be 15–150 seconds.' })
        payload.duration = duration
        payload.musicEngine = 'yue2'
      } else payload.duration = Math.min(120, Math.max(0.5, duration))
@@ -240,6 +240,8 @@ export function summarizeStudioJob(job: StudioJob) {
    familyId: job.familyId,
    shotQueueId: job.shotQueueId,
    liveJobId: job.liveJobId,
+    folderId: job.payload.folderId,
+    musicEngine: job.payload.musicEngine,
    stillId: job.payload.stillId,
    workflow: job.payload.workflow,
    imagePipeline: job.payload.imagePipeline || 'v1',
@@ -881,6 +883,10 @@ function pendingAlive(job: StudioJob) {
  if (job.liveJobId) {
    const pending = readPendingJob(job.liveJobId)
    if (pending?.promptId && freshEnough(pending.startedAt)) return true
+    if (job.kind === 'music') {
+      const root = join(String(useRuntimeConfig().libraryDir || process.env.LIBRARY_DIR || '/data/library'), 'yue2-pending', `${job.liveJobId}.json`)
+      if (existsSync(root)) return true
+    }
  }
  if (!job.shotQueueId) return false
  return listPendingJobs().some(pending => (
@@ -11,6 +11,11 @@ function settings() {
    token: String(config.comfyControlToken || process.env.COMFY_CONTROL_TOKEN || '') }
 }
 export function yue2Configured() { return Boolean(settings().url) }
+function pendingRoot() { return join(String(useRuntimeConfig().libraryDir || process.env.LIBRARY_DIR || '/data/library'), 'yue2-pending') }
+export function yue2PendingAlive(jobId: string) {
+  if (!jobId) return false
+  return existsSync(join(pendingRoot(), `${jobId}.json`))
+}
 async function request(path: string, body?: unknown) {
  const { url, token } = settings()
  if (!url) throw new Error('YuE2 host is not configured. Set COMFY_CONTROL_URL.')
@@ -26,7 +31,6 @@ async function request(path: string, body?: unknown) {
  }
  return response
 }
-function pendingRoot() { return join(String(useRuntimeConfig().libraryDir || process.env.LIBRARY_DIR || '/data/library'), 'yue2-pending') }
 function persist(job: Job) {
  mkdirSync(pendingRoot(), { recursive: true })
  const path = join(pendingRoot(), `${job.id}.json`)
@@ -1,7 +1,13 @@
-export function gpuStatus(jobs, health, upscales=[]) {
+export function gpuStatus(jobs, health, upscales=[], musicJobs=[]) {
 if (!health) return {label:'Status unavailable',tone:'unknown',detail:'Checking GPU connection…'}
 const upscale=upscales.find(j=>j.status==='running')
 if(upscale)return {label:'Upscaling',tone:'busy',detail:upscale.name || 'Video upscale'}
+ const music=musicJobs.find(j=>['waiting','running','held'].includes(j.status) || j.live?.status==='running' || j.live?.musicActivity?.running)
+ if(music){
+  const stage=music.live?.message || music.live?.stage || music.waitReason || 'YuE2'
+  return {label:music.status==='waiting'?'Queued':'Generating',tone:'busy',detail:String(stage)}
+ }
+ if(health.yue2?.busy)return {label:'Generating',tone:'busy',detail:'YuE2 is using the GPU'}
 const job=jobs.find(j=>['waking','submitting','rendering','saving'].includes(j.state))
 if(job)return {label:{waking:'Waking up',submitting:'Submitting',rendering:'Generating',saving:'Saving'}[job.state],tone:'busy',detail:job.request?.promptSections?.action || 'Generation in progress'}
 if(health.comfy?.busy || health.imageComfy?.busy)return {label:'Busy',tone:'busy',detail:'GPU is working on another job'}
@@ -10,6 +10,8 @@ test('waiting records never replace the idle GPU status after completion',()=>{
 assert.deepEqual(gpuStatus([waiting,{...running,state:'complete'}],health),{label:'Ready',tone:'ready',detail:'No job running'});
 assert.equal(gpuStatus([waiting],health,[{status:'running',name:'Upscale current clip'}]).label,'Upscaling');
 assert.equal(gpuStatus([waiting],{comfy:{asleep:true}}).label,'Sleeping');
+ assert.equal(gpuStatus([waiting],{comfy:{asleep:true},yue2:{busy:true}}).label,'Generating');
+ assert.equal(gpuStatus([waiting],{comfy:{asleep:true}},[],[{status:'running',name:'song',live:{message:'Planning'}}]).label,'Generating');
 });
 test('Unicode video titles produce valid response headers and retain UTF-8 filename',()=>{
 for(const title of ['A scene… “hello” — 🎥.mp4','日本語.mp4','bad\r\nheader".mp4']){
@@ -32,6 +32,12 @@ class WorkerTests(unittest.TestCase):
        self.assertNotEqual(attention, 'flash')
        self.assertTrue(os.environ.get('PYTORCH_CUDA_ALLOC_CONF', '').startswith('expandable_segments'))

+    def test_target_length_maps_to_semantic_max_tokens(self):
+        # 48000 / 1920 = 25 latent frames per second.
+        self.assertEqual(max(200, min(9000, 30 * 25)), 750)
+        self.assertEqual(max(200, min(9000, 15 * 25)), 375)
+        self.assertEqual(max(200, min(9000, 150 * 25)), 3750)
+
    def test_cuda_oom_detection(self):
        self.assertTrue(worker.is_cuda_oom(RuntimeError('CUDA out of memory. Tried to allocate 2.49 GiB')))
        self.assertFalse(worker.is_cuda_oom(RuntimeError('bad lyrics')))
@@ -18,7 +18,8 @@ test('YuE2 defaults are 60 seconds and allow multiple lyric sections', () => {
    '[Verse]\nA\n[Chorus]\nB'
  )
  assert.throws(() => validateYue2Request({ ...request, lyrics: '' }))
-  assert.throws(() => validateYue2Request({ ...request, duration: 20 }))
+  assert.throws(() => validateYue2Request({ ...request, duration: 10 }))
+  assert.deepEqual(validateYue2Request({ ...request, duration: 15 }).duration, 15)
  assert.throws(() => validateYue2Request({ ...request, id: '../escape' }))
 })

@@ -298,11 +298,12 @@ export function trackSettingsRows(track: {
  lyricsStrength?: number
 }) {
  const rows: SettingsRow[] = []
+  const engine = String(track.engine || '')
+  const isYue2 = engine === 'yue2' || engine === 'yue-2'
  if (track.engine) {
-    const engine = String(track.engine)
    rows.push({
      label: 'Engine',
-      value: engine === 'yue2' || engine === 'yue-2'
+      value: isYue2
        ? 'YuE2'
        : engine === 'ace-step-1.5'
          ? 'ACE-Step 1.5 (legacy)'
@@ -318,14 +319,17 @@ export function trackSettingsRows(track: {
  else if (track.lyrics?.trim()) rows.push({ label: 'Lyrics', value: track.lyrics.trim() })
  if (typeof track.duration === 'number' && track.duration > 0) {
    const rounded = Math.round(track.duration * 10) / 10
-    rows.push({ label: 'Duration', value: Number.isInteger(rounded) ? `${rounded}s` : `${rounded.toFixed(1)}s` })
+    rows.push({
+      label: isYue2 ? 'Length' : 'Duration',
+      value: Number.isInteger(rounded) ? `${rounded}s` : `${rounded.toFixed(1)}s`
+    })
  }
  if (track.yueProfile != null) rows.push({ label: 'YuEGP profile', value: String(track.yueProfile) })
-  if (track.yueProfile == null && track.steps != null) rows.push({ label: 'Steps', value: String(track.steps) })
-  if (track.yueProfile == null && track.cfg != null) rows.push({ label: 'CFG', value: formatNum(track.cfg) })
-  if (track.samplerName) rows.push({ label: 'Sampler', value: track.samplerName })
-  if (track.scheduler) rows.push({ label: 'Scheduler', value: track.scheduler })
-  if (track.lyricsStrength != null) rows.push({ label: 'Lyric strength', value: formatNum(track.lyricsStrength) })
+  if (!isYue2 && track.yueProfile == null && track.steps != null) rows.push({ label: 'Steps', value: String(track.steps) })
+  if (!isYue2 && track.yueProfile == null && track.cfg != null) rows.push({ label: 'CFG', value: formatNum(track.cfg) })
+  if (!isYue2 && track.samplerName) rows.push({ label: 'Sampler', value: track.samplerName })
+  if (!isYue2 && track.scheduler) rows.push({ label: 'Scheduler', value: track.scheduler })
+  if (!isYue2 && track.lyricsStrength != null) rows.push({ label: 'Lyric strength', value: formatNum(track.lyricsStrength) })
  if (track.seed != null) rows.push({ label: 'Seed', value: String(track.seed) })
  return rows
 }
@@ -1,6 +1,13 @@
-export const MUSIC_DURATION_MIN = 30
+export const MUSIC_DURATION_MIN = 15
 export const MUSIC_DURATION_MAX = 150
 export const MUSIC_DURATION_DEFAULT = 60
+/** YuE2 VAE downsampling: 48000 / 1920 = 25 semantic tokens per second of audio. */
+export const YUE2_TOKENS_PER_SECOND = 25
+
+export function yue2MaxTokens(durationSec: number) {
+  const seconds = clampMusicDuration(durationSec)
+  return Math.max(200, Math.min(9000, seconds * YUE2_TOKENS_PER_SECOND))
+}
 export const MUSIC_STEPS_MIN = 8
 export const MUSIC_STEPS_MAX = 100
 export const MUSIC_STEPS_DEFAULT = 50