towsty/aigen
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add ACE-Step 1.5 and YuE on Music, and put Video/Image/Music cards back.

Browse Source 
Recover finished Comfy audio when queueing hangs so a completed track is not stuck on Queueing.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-09-03 07:42:54 -05:00
co-authored by Cursor
parent ce85e7f513
commit 78b158471b
17 changed files with 951 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/InfoTip.vue
+109
View File
@@ -0,0 +1,109 @@
<template>
<div
ref="root"
class="relative inline-flex shrink-0"
@mouseenter="open"
@mouseleave="scheduleClose"
@focusin="open"
@focusout="onFocusOut"
>
<button
type="button"
class="flex h-6 w-6 items-center justify-center rounded-full border border-white/15 text-zinc-400 hover:border-amber-300/50 hover:text-amber-200"
:aria-label="ariaLabel"
:aria-expanded="show"
@click.prevent.stop="show = !show"
>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" class="h-3.5 w-3.5" aria-hidden="true">
<circle cx="12" cy="12" r="10" />
<path d="M12 16v-4M12 8h.01" stroke-linecap="round" />
</svg>
</button>
<Teleport to="body">
<div
v-if="show"
ref="panel"
class="fixed z-[80] max-h-[min(20rem,50vh)] w-[min(28rem,calc(100vw-1.5rem))] overflow-y-auto overscroll-contain whitespace-pre-wrap break-words rounded-xl border border-white/10 bg-zinc-950 p-3 text-xs leading-relaxed text-zinc-300 shadow-2xl"
:style="panelStyle"
@mouseenter="open"
@mouseleave="scheduleClose"
@click.stop
>
{{ text }}
</div>
</Teleport>
</div>
</template>
<script setup lang="ts">
defineProps<{
text: string
ariaLabel: string
}>()
const root = ref<HTMLElement | null>(null)
const panel = ref<HTMLElement | null>(null)
const show = ref(false)
const panelStyle = ref<Record<string, string>>({})
let closeTimer: ReturnType<typeof setTimeout> | null = null
function place() {
const el = root.value
if (!el) return
const rect = el.getBoundingClientRect()
const width = Math.min(448, window.innerWidth - 24)
const maxHeight = Math.min(320, window.innerHeight * 0.5)
let left = rect.right - width
left = Math.min(Math.max(12, left), window.innerWidth - width - 12)
const below = rect.bottom + 8
const above = rect.top - 8 - maxHeight
const top = below + maxHeight <= window.innerHeight - 12 ? below : Math.max(12, above)
panelStyle.value = {
top: `${Math.round(top)}px`,
left: `${Math.round(left)}px`,
width: `${Math.round(width)}px`
}
}
function open() {
if (closeTimer) {
clearTimeout(closeTimer)
closeTimer = null
}
show.value = true
nextTick(place)
}
function close() {
show.value = false
}
function scheduleClose() {
if (closeTimer) clearTimeout(closeTimer)
closeTimer = setTimeout(close, 160)
}
function onFocusOut(event: FocusEvent) {
const next = event.relatedTarget as Node | null
if (root.value?.contains(next) || panel.value?.contains(next)) return
scheduleClose()
}
watch(show, (value) => {
if (typeof window === 'undefined') return
if (value) {
window.addEventListener('resize', place)
window.addEventListener('scroll', place, true)
return
}
window.removeEventListener('resize', place)
window.removeEventListener('scroll', place, true)
})
onBeforeUnmount(() => {
if (closeTimer) clearTimeout(closeTimer)
if (typeof window === 'undefined') return
window.removeEventListener('resize', place)
window.removeEventListener('scroll', place, true)
})
</script>
components/StudioKindCards.vue
+48
View File
@@ -0,0 +1,48 @@
<template>
<div class="mb-4 grid grid-cols-3 gap-3">
<NuxtLink
to="/?studio=video"
class="rounded-3xl border px-4 py-4 text-left transition"
:class="current === 'video' ? 'border-amber-300 bg-amber-400/10 text-amber-50' : 'border-white/10 text-zinc-400 hover:border-white/20 hover:text-zinc-200'"
>
<svg class="h-10 w-10" viewBox="0 0 40 40" fill="none" aria-hidden="true">
<rect x="4" y="10" width="24" height="20" rx="3" stroke="currentColor" stroke-width="2" />
<path d="M28 16 36 12v16l-8-4V16Z" stroke="currentColor" stroke-width="2" stroke-linejoin="round" />
<circle cx="12" cy="20" r="2" fill="currentColor" />
</svg>
<span class="mt-3 block font-display text-lg font-bold text-zinc-100">Video</span>
<span class="mt-0.5 block text-xs text-zinc-500">MiniMax H3 · LTX</span>
</NuxtLink>
<NuxtLink
to="/?studio=image"
class="rounded-3xl border px-4 py-4 text-left transition"
:class="current === 'image' ? 'border-amber-300 bg-amber-400/10 text-amber-50' : 'border-white/10 text-zinc-400 hover:border-white/20 hover:text-zinc-200'"
>
<svg class="h-10 w-10" viewBox="0 0 40 40" fill="none" aria-hidden="true">
<rect x="6" y="8" width="28" height="24" rx="3" stroke="currentColor" stroke-width="2" />
<path d="m10 26 7-8 6 7 3-3 4 4" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" />
<circle cx="15" cy="15" r="2.5" fill="currentColor" />
</svg>
<span class="mt-3 block font-display text-lg font-bold text-zinc-100">Image</span>
<span class="mt-0.5 block text-xs text-zinc-500">Flux · Krea</span>
</NuxtLink>
<NuxtLink
to="/music"
class="rounded-3xl border px-4 py-4 text-left transition"
:class="current === 'music' ? 'border-amber-300 bg-amber-400/10 text-amber-50' : 'border-white/10 text-zinc-400 hover:border-white/20 hover:text-zinc-200'"
>
<svg class="h-10 w-10" viewBox="0 0 40 40" fill="none" aria-hidden="true">
<circle cx="20" cy="20" r="12" stroke="currentColor" stroke-width="2" />
<path d="M18 14v12l10-4V12l-10 2Z" stroke="currentColor" stroke-width="2" stroke-linejoin="round" />
</svg>
<span class="mt-3 block font-display text-lg font-bold text-zinc-100">Music</span>
<span class="mt-0.5 block text-xs text-zinc-500">ACE-Step · YuE</span>
</NuxtLink>
</div>
</template>
<script setup lang="ts">
defineProps<{
current: 'video' | 'image' | 'music'
}>()
</script>
nuxt.config.ts
+2
View File
@@ -59,6 +59,8 @@ export default defineNuxtConfig({
purgeComfyOutputs: process.env.PURGE_COMFY_OUTPUTS !== 'false', purgeComfyOutputs: process.env.PURGE_COMFY_OUTPUTS !== 'false',
comfyFilenamePrefix: process.env.COMFY_FILENAME_PREFIX || 'video/MiniMax_H3', comfyFilenamePrefix: process.env.COMFY_FILENAME_PREFIX || 'video/MiniMax_H3',
comfyAudioPrefix: process.env.COMFY_AUDIO_PREFIX || 'audio/AIGen', comfyAudioPrefix: process.env.COMFY_AUDIO_PREFIX || 'audio/AIGen',
yueStageA: process.env.YUE_STAGE_A || '',
yueStageB: process.env.YUE_STAGE_B || '',
kreaConceptLora: process.env.NUXT_KREA_CONCEPT_LORA || '', kreaConceptLora: process.env.NUXT_KREA_CONCEPT_LORA || '',
public: { public: {
authEnabled: process.env.NUXT_PUBLIC_AUTH_ENABLED === 'true', authEnabled: process.env.NUXT_PUBLIC_AUTH_ENABLED === 'true',
pages/index.vue
+15 -42
View File
@@ -7,50 +7,11 @@
@gpu="pollComfyHealth" @gpu="pollComfyHealth"
/> />
<main class="mx-auto grid min-w-0 max-w-7xl gap-6 px-4 py-6 sm:px-6 lg:grid-cols-[minmax(0,1fr)_minmax(0,1fr)] lg:items-stretch"> <main class="mx-auto min-w-0 max-w-7xl space-y-6 px-4 py-6 sm:px-6">
<StudioKindCards :current="imageStudio ? 'image' : 'video'" />
<div class="grid min-w-0 gap-6 lg:grid-cols-[minmax(0,1fr)_minmax(0,1fr)] lg:items-stretch">
<section id="studio-input" class="glass min-w-0 max-w-full space-y-5 rounded-3xl p-5 sm:p-6"> <section id="studio-input" class="glass min-w-0 max-w-full space-y-5 rounded-3xl p-5 sm:p-6">
<div> <div>
<div class="mb-4 grid grid-cols-3 gap-3">
<button
type="button"
class="rounded-3xl border px-4 py-4 text-left transition"
:class="studioMode === 'video' ? 'border-amber-300 bg-amber-400/10 text-amber-50' : 'border-white/10 text-zinc-400 hover:border-white/20 hover:text-zinc-200'"
@click="selectStudioKind('video')"
>
<svg class="h-10 w-10" viewBox="0 0 40 40" fill="none" aria-hidden="true">
<rect x="4" y="10" width="24" height="20" rx="3" stroke="currentColor" stroke-width="2" />
<path d="M28 16 36 12v16l-8-4V16Z" stroke="currentColor" stroke-width="2" stroke-linejoin="round" />
<circle cx="12" cy="20" r="2" fill="currentColor" />
</svg>
<span class="mt-3 block font-display text-lg font-bold text-zinc-100">Video</span>
<span class="mt-0.5 block text-xs text-zinc-500">MiniMax H3 · LTX</span>
</button>
<button
type="button"
class="rounded-3xl border px-4 py-4 text-left transition"
:class="imageStudio ? 'border-amber-300 bg-amber-400/10 text-amber-50' : 'border-white/10 text-zinc-400 hover:border-white/20 hover:text-zinc-200'"
@click="selectStudioKind('image')"
>
<svg class="h-10 w-10" viewBox="0 0 40 40" fill="none" aria-hidden="true">
<rect x="6" y="8" width="28" height="24" rx="3" stroke="currentColor" stroke-width="2" />
<path d="m10 26 7-8 6 7 3-3 4 4" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" />
<circle cx="15" cy="15" r="2.5" fill="currentColor" />
</svg>
<span class="mt-3 block font-display text-lg font-bold text-zinc-100">Image</span>
<span class="mt-0.5 block text-xs text-zinc-500">Flux · Krea</span>
</button>
<NuxtLink
to="/music"
class="rounded-3xl border border-white/10 px-4 py-4 text-left text-zinc-400 transition hover:border-white/20 hover:text-zinc-200"
>
<svg class="h-10 w-10" viewBox="0 0 40 40" fill="none" aria-hidden="true">
<circle cx="20" cy="20" r="12" stroke="currentColor" stroke-width="2" />
<path d="M18 14v12l10-4V12l-10 2Z" stroke="currentColor" stroke-width="2" stroke-linejoin="round" />
</svg>
<span class="mt-3 block font-display text-lg font-bold text-zinc-100">Music</span>
<span class="mt-0.5 block text-xs text-zinc-500">ACE-Step · vocals</span>
</NuxtLink>
</div>
<h2 class="font-display text-2xl font-bold">Input</h2> <h2 class="font-display text-2xl font-bold">Input</h2>
<p class="text-sm text-zinc-400">{{ studioMode === 'editv2' <p class="text-sm text-zinc-400">{{ studioMode === 'editv2'
? (v2Mode === 'generate' ? (v2Mode === 'generate'
@@ -2108,6 +2069,7 @@
</template> </template>
</div> </div>
</section> </section>
</div>
</main> </main>
<ImageBatchModal <ImageBatchModal
@@ -2611,6 +2573,12 @@ function selectStudioKind(kind: 'video' | 'image') {
studioMode.value = 'editv2' studioMode.value = 'editv2'
outputFocus.value = 'edit' outputFocus.value = 'edit'
} }
function applyStudioQueryFromRoute() {
const studio = String(route.query.studio || '').toLowerCase()
if (studio === 'image') selectStudioKind('image')
else if (studio === 'video') selectStudioKind('video')
}
const imageComfyOk = ref(false) const imageComfyOk = ref(false)
const imageComfyBusy = ref(false) const imageComfyBusy = ref(false)
const imageComfyConfigured = ref(false) const imageComfyConfigured = ref(false)
@@ -3990,6 +3958,7 @@ onMounted(async () => {
if (route.query.clip || route.query.still) applyIncomingLibraryItem() if (route.query.clip || route.query.still) applyIncomingLibraryItem()
else applyIncomingDirectorScript() else applyIncomingDirectorScript()
await applyIncomingQueuedJob() await applyIncomingQueuedJob()
applyStudioQueryFromRoute()
window.addEventListener('keydown', onStudioKey) window.addEventListener('keydown', onStudioKey)
}) })
@@ -4020,6 +3989,10 @@ watch(studioMode, () => {
} }
}) })
watch(() => route.query.studio, () => {
applyStudioQueryFromRoute()
})
watch([browseFolderId, libraryFilter], () => { watch([browseFolderId, libraryFilter], () => {
selectedKeys.value = [] selectedKeys.value = []
lastSelectedKey.value = '' lastSelectedKey.value = ''
pages/music.vue
+152 -8
View File
@@ -1,12 +1,14 @@
<template> <template>
<div class="min-h-dvh min-w-0"> <div class="min-h-dvh min-w-0">
<AppHeader <AppHeader
subtitle="ACE-Step · vocals" :subtitle="headerSubtitle"
current="music" current="music"
:queue-count="queueCount" :queue-count="queueCount"
/> />
<main class="mx-auto grid min-w-0 max-w-7xl gap-6 px-4 py-6 sm:px-6 lg:grid-cols-[minmax(0,1fr)_minmax(0,1fr)] lg:items-stretch"> <main class="mx-auto min-w-0 max-w-7xl space-y-6 px-4 py-6 sm:px-6">
<StudioKindCards current="music" />
<div class="grid min-w-0 gap-6 lg:grid-cols-[minmax(0,1fr)_minmax(0,1fr)] lg:items-stretch">
<section class="glass min-w-0 max-w-full space-y-5 rounded-3xl p-5 sm:p-6"> <section class="glass min-w-0 max-w-full space-y-5 rounded-3xl p-5 sm:p-6">
<div> <div>
<h2 class="font-display text-2xl font-bold">Music Generator</h2> <h2 class="font-display text-2xl font-bold">Music Generator</h2>
@@ -15,6 +17,56 @@
</p> </p>
</div> </div>
<div>
<p class="mb-2 text-xs uppercase tracking-wider text-zinc-500">Engine</p>
<div class="grid grid-cols-2 gap-2">
<button
type="button"
class="rounded-xl border px-3 py-2 text-left text-sm"
:class="engineFamily === 'ace' ? 'border-amber-300 bg-amber-400/10 text-amber-100' : 'border-white/10 hover:border-white/20'"
@click="selectEngineFamily('ace')"
>
<span class="block font-semibold">ACE-Step</span>
<span class="text-xs text-zinc-400">Native Comfy checkpoint. Fast vocals.</span>
</button>
<div
class="cursor-pointer rounded-xl border px-3 py-2 text-left text-sm"
:class="engineFamily === 'yue' ? 'border-amber-300 bg-amber-400/10 text-amber-100' : 'border-white/10 hover:border-white/20'"
role="button"
tabindex="0"
@click="selectEngineFamily('yue')"
@keydown.enter.prevent="selectEngineFamily('yue')"
>
<span class="flex items-center gap-2">
<span class="font-semibold">YuE</span>
<InfoTip
aria-label="What YuE does"
text="YuE is a separate lyrics-to-song engine (stage-1 7B + stage-2 1B + xcodec). It needs the ComfyUI_YuE custom node on Beast and is slower and heavier than ACE-Step. Use it when you want a full sung track from lyrics rather than ACE-Step’s tag-conditioned sampler."
/>
</span>
<span class="text-xs text-zinc-400">Lyrics-to-song. Custom node.</span>
</div>
</div>
</div>
<div v-if="engineFamily === 'ace'" class="flex items-center justify-between gap-3 rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3 text-sm">
<span class="min-w-0">
<span class="flex items-center gap-2">
<span class="block font-medium text-zinc-200">ACE-Step 1.5</span>
<InfoTip
aria-label="What ACE-Step 1.5 does"
:text="ace15Info"
/>
</span>
<span class="mt-0.5 block text-xs text-zinc-500">Turbo AIO checkpoint. Different graph than v1, usually 8 steps.</span>
</span>
<label class="relative inline-flex h-6 w-11 shrink-0 cursor-pointer items-center" title="Toggle ACE-Step 1.5">
<input v-model="ace15" type="checkbox" class="peer sr-only" role="switch">
<span class="absolute inset-0 rounded-full bg-zinc-700 transition peer-checked:bg-amber-400" />
<span class="absolute left-0.5 top-0.5 h-5 w-5 rounded-full bg-white transition peer-checked:translate-x-5" />
</label>
</div>
<p v-if="error" class="rounded-2xl border border-red-400/30 bg-red-400/10 px-4 py-3 text-sm text-red-200">{{ error }}</p> <p v-if="error" class="rounded-2xl border border-red-400/30 bg-red-400/10 px-4 py-3 text-sm text-red-200">{{ error }}</p>
<label class="block text-sm"> <label class="block text-sm">
@@ -66,9 +118,9 @@
:max="durationMax" :max="durationMax"
step="5" step="5"
> >
<span class="mt-1 block text-[11px] text-zinc-500">{{ durationMin }}–{{ durationMax }} seconds</span> <span class="mt-1 block text-[11px] text-zinc-500">{{ engineFamily === 'yue' ? 'YuE uses this as stage-A length' : `${durationMin}–${durationMax} seconds` }}</span>
</label> </label>
<label class="block text-sm"> <label v-if="engineFamily !== 'yue'" class="block text-sm">
<span class="mb-1 block font-medium text-zinc-300">Steps · {{ steps }}</span> <span class="mb-1 block font-medium text-zinc-300">Steps · {{ steps }}</span>
<input <input
v-model.number="steps" v-model.number="steps"
@@ -78,7 +130,7 @@
:max="stepsMax" :max="stepsMax"
step="1" step="1"
> >
<span class="mt-1 block text-[11px] text-zinc-500">30–50 is the usual range</span> <span class="mt-1 block text-[11px] text-zinc-500">{{ ace15 ? '8 is the usual 1.5 turbo range' : '30–50 is the usual range' }}</span>
</label> </label>
</div> </div>
@@ -167,6 +219,7 @@
<NuxtLink to="/library" class="text-amber-200 hover:text-amber-100">Open Library</NuxtLink> <NuxtLink to="/library" class="text-amber-200 hover:text-amber-100">Open Library</NuxtLink>
</p> </p>
</section> </section>
</div>
</main> </main>
<QueuedJobModal :job-id="inspectJobId" @close="inspectJobId = ''" /> <QueuedJobModal :job-id="inspectJobId" @close="inspectJobId = ''" />
</div> </div>
@@ -176,13 +229,18 @@
import { import {
DEFAULT_MUSIC_LYRICS, DEFAULT_MUSIC_LYRICS,
DEFAULT_MUSIC_TAGS, DEFAULT_MUSIC_TAGS,
MUSIC_CFG_DEFAULT,
MUSIC_CFG_DEFAULT_15,
MUSIC_DURATION_DEFAULT, MUSIC_DURATION_DEFAULT,
MUSIC_DURATION_MAX, MUSIC_DURATION_MAX,
MUSIC_DURATION_MIN, MUSIC_DURATION_MIN,
MUSIC_STEPS_DEFAULT, MUSIC_STEPS_DEFAULT,
MUSIC_STEPS_DEFAULT_15,
MUSIC_STEPS_MAX, MUSIC_STEPS_MAX,
MUSIC_STEPS_MIN, MUSIC_STEPS_MIN,
MUSIC_STRUCTURE_CHIPS MUSIC_STRUCTURE_CHIPS,
parseMusicEngine,
type MusicEngine
} from '~/utils/music' } from '~/utils/music'
import { ensureGenerationNotifyPermission, notifyGenerationComplete } from '~/utils/generationNotify' import { ensureGenerationNotifyPermission, notifyGenerationComplete } from '~/utils/generationNotify'
import type { LibraryFolder } from '~/utils/libraryBrowse' import type { LibraryFolder } from '~/utils/libraryBrowse'
@@ -197,6 +255,8 @@ const durationMax = MUSIC_DURATION_MAX
const stepsMin = MUSIC_STEPS_MIN const stepsMin = MUSIC_STEPS_MIN
const stepsMax = MUSIC_STEPS_MAX const stepsMax = MUSIC_STEPS_MAX
const ace15Info = 'Turns on ACE-Step 1.5 Turbo, the all-in-one checkpoint already on Beast. 1.5 uses a different Comfy graph than v1: a 1.5 text encoder (tags, BPM, key, duration) and EmptyAceStep1.5LatentAudio, usually with 8 sampler steps and CFG 1. Songs often come back faster and cleaner. Lyrics-strength from v1 does not apply. Leave this off to stay on ACE-Step v1 3.5B, which is the graph that already generated on this GPU.'
const tags = ref(DEFAULT_MUSIC_TAGS) const tags = ref(DEFAULT_MUSIC_TAGS)
const lyrics = ref(DEFAULT_MUSIC_LYRICS) const lyrics = ref(DEFAULT_MUSIC_LYRICS)
const instrumental = ref(false) const instrumental = ref(false)
@@ -207,6 +267,8 @@ const name = ref('')
const folderId = ref('') const folderId = ref('')
const folders = ref<LibraryFolder[]>([]) const folders = ref<LibraryFolder[]>([])
const lyricsEl = ref<HTMLTextAreaElement | null>(null) const lyricsEl = ref<HTMLTextAreaElement | null>(null)
const ace15 = ref(false)
const engineFamily = ref<'ace' | 'yue'>('ace')
const busy = ref(false) const busy = ref(false)
const error = ref('') const error = ref('')
const status = ref('') const status = ref('')
@@ -226,8 +288,17 @@ let source: EventSource | null = null
let listenGen = 0 let listenGen = 0
let settled = false let settled = false
let queuePoll: ReturnType<typeof setInterval> | null = null let queuePoll: ReturnType<typeof setInterval> | null = null
let recoverPoll: ReturnType<typeof setInterval> | null = null
const blocked = computed(() => !tags.value.trim() || (!instrumental.value && !lyrics.value.trim()) || !folderId.value) const blocked = computed(() => !tags.value.trim() || (!instrumental.value && !lyrics.value.trim()) || !folderId.value)
const headerSubtitle = computed(() => {
if (engineFamily.value === 'yue') return 'YuE · lyrics-to-song'
return ace15.value ? 'ACE-Step 1.5 · vocals' : 'ACE-Step · vocals'
})
const selectedEngine = computed<MusicEngine>(() => {
if (engineFamily.value === 'yue') return 'yue'
return ace15.value ? 'ace-step-1.5' : 'ace-step'
})
const blockReason = computed(() => { const blockReason = computed(() => {
if (!folderId.value) return 'Choose a library folder before generating.' if (!folderId.value) return 'Choose a library folder before generating.'
if (!tags.value.trim()) return 'Add genre and style tags.' if (!tags.value.trim()) return 'Add genre and style tags.'
@@ -235,6 +306,32 @@ const blockReason = computed(() => {
return '' return ''
}) })
function selectEngineFamily(family: 'ace' | 'yue') {
engineFamily.value = family
if (family === 'ace' && ace15.value && steps.value === MUSIC_STEPS_DEFAULT) {
steps.value = MUSIC_STEPS_DEFAULT_15
}
if (family === 'ace' && !ace15.value && steps.value === MUSIC_STEPS_DEFAULT_15) {
steps.value = MUSIC_STEPS_DEFAULT
}
}
function applyEngine(engine: MusicEngine) {
if (engine === 'yue') {
engineFamily.value = 'yue'
ace15.value = false
return
}
engineFamily.value = 'ace'
ace15.value = engine === 'ace-step-1.5'
}
watch(ace15, (on) => {
if (engineFamily.value !== 'ace') return
if (on && (steps.value === MUSIC_STEPS_DEFAULT || steps.value >= 30)) steps.value = MUSIC_STEPS_DEFAULT_15
if (!on && steps.value === MUSIC_STEPS_DEFAULT_15) steps.value = MUSIC_STEPS_DEFAULT
})
function insertChip(chip: string) { function insertChip(chip: string) {
const el = lyricsEl.value const el = lyricsEl.value
const token = `${chip}\n` const token = `${chip}\n`
@@ -273,6 +370,26 @@ function stopQueuePoll() {
} }
} }
function stopRecoverPoll() {
if (recoverPoll) {
clearInterval(recoverPoll)
recoverPoll = null
}
}
async function tryRecover() {
if (settled || !busy.value || queued.value) return
try {
const recovered = await $fetch<Record<string, any>>('/api/generate/recover', {
method: 'POST',
body: { jobId: jobId.value || undefined }
})
if (recovered?.trackId) applyEvent(recovered)
} catch {
/* still running */
}
}
async function refreshStudioQueue() { async function refreshStudioQueue() {
const data = await $fetch<{ const data = await $fetch<{
jobs?: Array<{ jobs?: Array<{
@@ -341,6 +458,7 @@ function applyEvent(payload: Record<string, any>) {
stopListen() stopListen()
queued.value = false queued.value = false
studioJobId.value = '' studioJobId.value = ''
stopRecoverPoll()
stopQueuePoll() stopQueuePoll()
queuePoll = setInterval(() => { void refreshStudioQueue() }, 8000) queuePoll = setInterval(() => { void refreshStudioQueue() }, 8000)
void notifyGenerationComplete('music') void notifyGenerationComplete('music')
@@ -352,6 +470,7 @@ function applyEvent(payload: Record<string, any>) {
status.value = error.value status.value = error.value
queued.value = false queued.value = false
stopListen() stopListen()
stopRecoverPoll()
stopQueuePoll() stopQueuePoll()
queuePoll = setInterval(() => { void refreshStudioQueue() }, 8000) queuePoll = setInterval(() => { void refreshStudioQueue() }, 8000)
} }
@@ -375,6 +494,9 @@ function listen(id: string) {
listen(id) listen(id)
}, 2000) }, 2000)
} }
if (!recoverPoll) {
recoverPoll = setInterval(() => { void tryRecover() }, 8000)
}
} }
async function generate() { async function generate() {
@@ -392,7 +514,11 @@ async function generate() {
trackId.value = '' trackId.value = ''
jobId.value = '' jobId.value = ''
studioJobId.value = '' studioJobId.value = ''
status.value = 'Queueing ACE-Step…' status.value = engineFamily.value === 'yue'
? 'Queueing YuE…'
: ace15.value
? 'Queueing ACE-Step 1.5…'
: 'Queueing ACE-Step…'
try { try {
const started = await $fetch<{ jobId: string; studioJobId?: string; queued?: boolean; name?: string }>( const started = await $fetch<{ jobId: string; studioJobId?: string; queued?: boolean; name?: string }>(
'/api/generate/music', '/api/generate/music',
@@ -404,9 +530,11 @@ async function generate() {
duration: duration.value, duration: duration.value,
steps: steps.value, steps: steps.value,
seed: seed.value || 'random', seed: seed.value || 'random',
cfg: selectedEngine.value === 'ace-step-1.5' ? MUSIC_CFG_DEFAULT_15 : MUSIC_CFG_DEFAULT,
instrumental: instrumental.value, instrumental: instrumental.value,
folderId: folderId.value, folderId: folderId.value,
name: name.value.trim() name: name.value.trim(),
engine: selectedEngine.value
} }
} }
) )
@@ -422,6 +550,9 @@ async function generate() {
} }
jobId.value = started.jobId jobId.value = started.jobId
listen(started.jobId) listen(started.jobId)
stopRecoverPoll()
recoverPoll = setInterval(() => { void tryRecover() }, 8000)
window.setTimeout(() => { void tryRecover() }, 12000)
} catch (err: any) { } catch (err: any) {
busy.value = false busy.value = false
queued.value = false queued.value = false
@@ -448,6 +579,7 @@ function applyIncomingMusic() {
steps?: number steps?: number
seed?: string seed?: string
folderId?: string folderId?: string
engine?: string
} }
if (typeof loaded.tags === 'string') tags.value = loaded.tags if (typeof loaded.tags === 'string') tags.value = loaded.tags
if (typeof loaded.lyrics === 'string') lyrics.value = loaded.lyrics if (typeof loaded.lyrics === 'string') lyrics.value = loaded.lyrics
@@ -457,6 +589,7 @@ function applyIncomingMusic() {
if (typeof loaded.seed === 'string') seed.value = loaded.seed if (typeof loaded.seed === 'string') seed.value = loaded.seed
if (typeof loaded.name === 'string') name.value = loaded.name if (typeof loaded.name === 'string') name.value = loaded.name
if (loaded.folderId && folders.value.some(folder => folder.id === loaded.folderId)) folderId.value = loaded.folderId if (loaded.folderId && folders.value.some(folder => folder.id === loaded.folderId)) folderId.value = loaded.folderId
if (loaded.engine) applyEngine(parseMusicEngine(loaded.engine))
status.value = 'Loaded from the queue' status.value = 'Loaded from the queue'
} catch { /* ignore */ } } catch { /* ignore */ }
} }
@@ -476,6 +609,7 @@ function applyPreset() {
if (typeof settings.duration === 'number') duration.value = settings.duration if (typeof settings.duration === 'number') duration.value = settings.duration
if (typeof settings.steps === 'number') steps.value = settings.steps if (typeof settings.steps === 'number') steps.value = settings.steps
if (typeof settings.seed === 'number') seed.value = String(settings.seed) if (typeof settings.seed === 'number') seed.value = String(settings.seed)
if (settings.engine) applyEngine(settings.engine)
name.value = preset.name name.value = preset.name
} }
@@ -491,11 +625,21 @@ onMounted(async () => {
audioUrl.value = `/api/library/tracks/${query.track}/audio` audioUrl.value = `/api/library/tracks/${query.track}/audio`
trackId.value = query.track trackId.value = query.track
status.value = 'Loaded from library' status.value = 'Loaded from library'
return
}
if (!audioUrl.value) {
try {
const recovered = await $fetch<Record<string, any>>('/api/generate/recover', { method: 'POST', body: {} })
if (recovered?.trackId && /recovered/i.test(String(recovered.message || ''))) {
applyEvent({ ...recovered, type: 'complete' })
}
} catch { /* nothing waiting in ComfyUI */ }
} }
}) })
onBeforeUnmount(() => { onBeforeUnmount(() => {
stopListen() stopListen()
stopRecoverPoll()
stopQueuePoll() stopQueuePoll()
}) })
</script> </script>
server/api/generate/music.post.ts
+14 -4
View File
@@ -5,6 +5,10 @@ import {
clampMusicCfg, clampMusicCfg,
clampMusicDuration, clampMusicDuration,
clampMusicSteps, clampMusicSteps,
MUSIC_CFG_DEFAULT,
MUSIC_CFG_DEFAULT_15,
MUSIC_STEPS_DEFAULT,
MUSIC_STEPS_DEFAULT_15,
parseMusicEngine, parseMusicEngine,
parseMusicSeed, parseMusicSeed,
sanitizeMusicLyrics, sanitizeMusicLyrics,
@@ -57,14 +61,20 @@ export default defineEventHandler(async (event) => {
const destFolder = library.folders.find(folder => folder.id === folderId) const destFolder = library.folders.find(folder => folder.id === folderId)
const folderLocked = Boolean(destFolder?.protected && !destFolder.unlocked) const folderLocked = Boolean(destFolder?.protected && !destFolder.unlocked)
const engine = parseMusicEngine(body.engine)
const duration = clampMusicDuration(body.duration) const duration = clampMusicDuration(body.duration)
const steps = clampMusicSteps(body.steps) const steps = clampMusicSteps(
const cfg = clampMusicCfg(body.cfg) body.steps,
engine === 'ace-step-1.5' ? MUSIC_STEPS_DEFAULT_15 : MUSIC_STEPS_DEFAULT
)
const cfg = clampMusicCfg(
body.cfg,
engine === 'ace-step-1.5' ? MUSIC_CFG_DEFAULT_15 : MUSIC_CFG_DEFAULT
)
const lyricsStrength = clampLyricsStrength(body.lyricsStrength) const lyricsStrength = clampLyricsStrength(body.lyricsStrength)
const seed = parseMusicSeed(body.seed) const seed = parseMusicSeed(body.seed)
const engine = parseMusicEngine(body.engine)
const clipName = allocateOutputName(ownerKey, folderId, String(body.name || '').trim(), { const clipName = allocateOutputName(ownerKey, folderId, String(body.name || '').trim(), {
engine: engine === 'ace-step-1.5' ? 'ace15' : 'ace' engine: engine === 'ace-step-1.5' ? 'ace15' : engine === 'yue' ? 'yue' : 'ace'
}) })
const samplerName = String(body.sampler_name || 'euler') const samplerName = String(body.sampler_name || 'euler')
server/api/generate/recover.post.ts
+73 -22
View File
@@ -16,11 +16,78 @@ function recentLibraryClip(event: Parameters<typeof assertLibraryOwner>[0]) {
} }
} }
function recentLibraryTrack(event: Parameters<typeof assertLibraryOwner>[0]) {
const { catalog } = assertLibraryOwner(event)
const newest = catalog.tracks?.[0]
if (!newest || Date.now() - newest.createdAt > RECENT_CLIP_MS) return null
return {
type: 'complete' as const,
status: 'complete' as const,
message: 'Track ready',
progress: 100,
trackId: newest.id,
filename: newest.comfyFilename,
folderLocked: false
}
}
async function recoverFinishedMedia(event: Parameters<typeof assertLibraryOwner>[0], opts?: { tags?: string }) {
const { owner } = assertLibraryOwner(event)
const library = publicLibrary(event)
const folderId = library.folders.find(folder => folder.unlocked)?.id
const tracks = folderId ? await importMissingComfyAudio(owner, folderId) : []
const want = String(opts?.tags || '').trim()
const { catalog } = assertLibraryOwner(event)
const match = [...tracks].reverse().find((track) => {
if (!want) return true
const tags = String(track.tags || '').trim()
return tags === want || tags.includes(want) || want.includes(tags)
}) || (want ? catalog.tracks.find((track) => {
const tags = String(track.tags || '').trim()
return Date.now() - track.createdAt < RECENT_CLIP_MS && (tags === want || tags.includes(want) || want.includes(tags))
}) : null)
if (match) {
return {
type: 'complete' as const,
status: 'complete' as const,
message: 'Recovered track from ComfyUI',
progress: 100,
trackId: match.id,
filename: match.comfyFilename,
folderLocked: false
}
}
if (opts?.tags) return null
const videos = folderId ? await importMissingComfyVideos(owner, folderId) : []
const newestVideo = videos[videos.length - 1]
if (newestVideo) {
return {
type: 'complete' as const,
status: 'complete' as const,
message: 'Recovered video from ComfyUI',
progress: 100,
clipId: newestVideo.id,
filename: newestVideo.comfyFilename,
hideThumbnail: newestVideo.hideThumbnail
}
}
return recentLibraryTrack(event) || recentLibraryClip(event)
}
export default defineEventHandler(async (event) => { export default defineEventHandler(async (event) => {
const body = await readBody<{ jobId?: string }>(event).catch(() => ({} as { jobId?: string })) const body = await readBody<{ jobId?: string }>(event).catch(() => ({} as { jobId?: string }))
const jobId = String(body?.jobId || '') const jobId = String(body?.jobId || '')
const live = jobId ? getJob(jobId) : undefined const live = jobId ? getJob(jobId) : undefined
if (live) return jobSnapshot(live) if (live) {
if (live.status === 'complete' || live.status === 'error' || live.trackId || live.clipId) {
return jobSnapshot(live)
}
if (live.kind === 'music' && Date.now() - live.startedAt > 15_000) {
const recovered = await recoverFinishedMedia(event, { tags: live.library?.tags })
if (recovered?.trackId) return recovered
}
return jobSnapshot(live)
}
if (jobId) { if (jobId) {
const pending = readPendingJob(jobId) const pending = readPendingJob(jobId)
@@ -29,28 +96,12 @@ export default defineEventHandler(async (event) => {
if (done) return done if (done) return done
return jobSnapshot(ensurePendingWatch(pending)) return jobSnapshot(ensurePendingWatch(pending))
} }
const saved = recentLibraryClip(event) const recovered = await recoverFinishedMedia(event)
if (saved) return saved if (recovered) return recovered
throw createError({ statusCode: 404, statusMessage: 'Job not found' }) throw createError({ statusCode: 404, statusMessage: 'Job not found' })
} }
const { owner } = assertLibraryOwner(event) const recovered = await recoverFinishedMedia(event)
const library = publicLibrary(event) if (recovered) return recovered
const folderId = library.folders.find(folder => folder.unlocked)?.id throw createError({ statusCode: 404, statusMessage: 'No finished ComfyUI output to recover' })
const imported = folderId ? await importMissingComfyVideos(owner, folderId) : []
const newest = imported[imported.length - 1]
if (newest) {
return {
type: 'complete',
status: 'complete',
message: 'Recovered video from ComfyUI',
progress: 100,
clipId: newest.id,
filename: newest.comfyFilename,
hideThumbnail: newest.hideThumbnail
}
}
const saved = recentLibraryClip(event)
if (saved) return saved
throw createError({ statusCode: 404, statusMessage: 'No finished ComfyUI video to recover' })
}) })
server/assets/workflow_ace_step_15.json
+101
View File
@@ -0,0 +1,101 @@
{
"14": {
"inputs": {
"ckpt_name": "ace_step_1.5_turbo_aio.safetensors"
},
"class_type": "CheckpointLoaderSimple",
"_meta": {
"title": "Load ACE-Step 1.5"
}
},
"40": {
"inputs": {
"tags": "modern synthpop, 120 bpm, female vocals",
"lyrics": "[Verse 1]\n\n[Chorus]\n",
"seed": 0,
"bpm": 120,
"duration": 60,
"timesignature": "4",
"language": "en",
"keyscale": "C major",
"generate_audio_codes": true,
"cfg_scale": 2,
"temperature": 0.85,
"top_p": 0.9,
"top_k": 0,
"min_p": 0,
"clip": ["14", 1]
},
"class_type": "TextEncodeAceStepAudio1.5",
"_meta": {
"title": "Tags & Lyrics 1.5"
}
},
"47": {
"inputs": {
"conditioning": ["40", 0]
},
"class_type": "ConditioningZeroOut",
"_meta": {
"title": "Negative (zeroed)"
}
},
"17": {
"inputs": {
"seconds": 60,
"batch_size": 1
},
"class_type": "EmptyAceStep1.5LatentAudio",
"_meta": {
"title": "Audio Length 1.5"
}
},
"45": {
"inputs": {
"shift": 3,
"model": ["14", 0]
},
"class_type": "ModelSamplingAuraFlow",
"_meta": {
"title": "Model Sampling"
}
},
"3": {
"inputs": {
"seed": 0,
"steps": 8,
"cfg": 1,
"sampler_name": "euler",
"scheduler": "simple",
"denoise": 1,
"model": ["45", 0],
"positive": ["40", 0],
"negative": ["47", 0],
"latent_image": ["17", 0]
},
"class_type": "KSampler",
"_meta": {
"title": "Audio Sampler"
}
},
"16": {
"inputs": {
"samples": ["3", 0],
"vae": ["14", 2]
},
"class_type": "VAEDecodeAudio",
"_meta": {
"title": "VAE Decode Audio"
}
},
"19": {
"inputs": {
"filename_prefix": "audio/AIGen",
"audio": ["16", 0]
},
"class_type": "SaveAudio",
"_meta": {
"title": "Save Audio"
}
}
}
server/assets/workflow_yue.json
+75
View File
@@ -0,0 +1,75 @@
{
"15": {
"inputs": {
"stage_A_repo": "m-a-p/YuE-s1-7B-anneal-en-cot",
"xcodec_ckpt": "ckpt_00360000.pth",
"quantization_model": "fp16",
"use_mmgp": true,
"stage1_cache_size": 16384,
"exllamav2_cache_mode": "FP16",
"mmgp_profile": 2
},
"class_type": "YUE_Stage_A_Loader",
"_meta": {
"title": "YuE Stage A Loader"
}
},
"8": {
"inputs": {
"genres_prompt": "inspiring female uplifting pop airy vocal electronic bright",
"lyrics_prompt": "[verse]\n\n[chorus]\n",
"seed": 0,
"run_n_segment": 2,
"repetition_penalty": 1.1,
"prompt_start_time": 0,
"prompt_end_time": 30,
"max_new_tokens": 3000,
"use_dual_tracks_prompt": false,
"use_audio_prompt": false,
"offload_model": true,
"stage1_no_guidance": true,
"model": ["15", 0]
},
"class_type": "YUE_Stage_A_Sampler",
"_meta": {
"title": "YuE Stage A"
}
},
"16": {
"inputs": {
"stage_B_repo": "m-a-p/YuE-s2-1B-general",
"stage2_cache_size": 8192,
"stage2_batch_size": 2,
"exllamav2_cache_mode": "FP16",
"use_mmgp": true,
"info": ["8", 1]
},
"class_type": "YUE_Stage_B_Loader",
"_meta": {
"title": "YuE Stage B Loader"
}
},
"9": {
"inputs": {
"vocal_decoder_ckpt": "decoder_131000.pth",
"inst_decoder_ckpt": "decoder_151000.pth",
"rescale": true,
"stage1_set": ["8", 0],
"model": ["16", 0]
},
"class_type": "YUE_Stage_B_Sampler",
"_meta": {
"title": "YuE Stage B"
}
},
"19": {
"inputs": {
"filename_prefix": "audio/AIGen",
"audio": ["9", 0]
},
"class_type": "SaveAudio",
"_meta": {
"title": "Save Audio"
}
}
}
server/utils/comfy.ts
+108 -2
View File
@@ -171,11 +171,12 @@ function comfyQueueMessage(payload: unknown, status: number) {
return message || `Queue failed (${status})` return message || `Queue failed (${status})`
} }
export async function queuePrompt(graph: unknown, clientId: string) { export async function queuePrompt(graph: unknown, clientId: string, opts?: { timeoutMs?: number }) {
const res = await comfyFetch('/prompt', { const res = await comfyFetch('/prompt', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ prompt: graph, client_id: clientId }) body: JSON.stringify({ prompt: graph, client_id: clientId }),
...(opts?.timeoutMs ? { signal: AbortSignal.timeout(opts.timeoutMs) } : {})
}) })
const payload = await res.json().catch(() => ({})) const payload = await res.json().catch(() => ({}))
if (!res.ok) { if (!res.ok) {
@@ -295,6 +296,111 @@ export function extractAudio(history: Record<string, unknown> | null, promptId:
return findAudio(history) return findAudio(history)
} }
export function comfyAudioPrefix() {
return String(useRuntimeConfig().comfyAudioPrefix || process.env.COMFY_AUDIO_PREFIX || 'audio/AIGen')
}
export function isOurComfyAudio(audio: { filename: string; subfolder?: string }) {
const prefix = comfyAudioPrefix().replace(/\/$/, '')
const parts = prefix.split('/')
const namePrefix = parts[parts.length - 1] || 'AIGen'
const sub = parts.length > 1 ? parts.slice(0, -1).join('/') : 'audio'
const nameOk = String(audio.filename || '').startsWith(namePrefix)
const subOk = !audio.subfolder || audio.subfolder === sub
return nameOk && subOk
}
export function extractAceMetaFromHistory(entry: unknown) {
const prompt = (entry as { prompt?: unknown[] })?.prompt
const graph = Array.isArray(prompt) ? prompt[2] : null
const meta = {
tags: '',
lyrics: '',
duration: 0,
steps: 0,
seed: 0,
cfg: 0,
lyricsStrength: 0.9,
samplerName: '',
scheduler: '',
engine: 'ace-step'
}
if (!graph || typeof graph !== 'object') return meta
for (const node of Object.values(graph as Record<string, { class_type?: string; inputs?: Record<string, unknown> }>)) {
const inputs = node?.inputs || {}
if (node?.class_type === 'TextEncodeAceStepAudio' || node?.class_type === 'TextEncodeAceStepAudio1.5') {
meta.tags = String(inputs.tags || '')
meta.lyrics = String(inputs.lyrics || '')
meta.lyricsStrength = Number(inputs.lyrics_strength || meta.lyricsStrength)
}
if (node?.class_type === 'YUE_Stage_A_Sampler') {
meta.tags = String(inputs.genres_prompt || '')
meta.lyrics = String(inputs.lyrics_prompt || '')
meta.engine = 'yue'
meta.duration = Number(inputs.prompt_end_time || meta.duration)
meta.seed = Number(inputs.seed || meta.seed)
}
if (node?.class_type === 'EmptyAceStepLatentAudio' || node?.class_type === 'EmptyAceStep1.5LatentAudio') {
meta.duration = Number(inputs.seconds || 0)
}
if (node?.class_type === 'KSampler') {
meta.steps = Number(inputs.steps || 0)
meta.seed = Number(inputs.seed || 0)
meta.cfg = Number(inputs.cfg || 0)
meta.samplerName = String(inputs.sampler_name || '')
meta.scheduler = String(inputs.scheduler || '')
}
if (node?.class_type === 'CheckpointLoaderSimple') {
const ckpt = String(inputs.ckpt_name || '')
if (/1\.5|turbo_aio/i.test(ckpt)) meta.engine = 'ace-step-1.5'
}
}
return meta
}
function historyCompletedAt(entry: unknown) {
const messages = (entry as { status?: { messages?: Array<[string, { timestamp?: number }]> } })?.status?.messages
const last = messages?.find(item => item[0] === 'execution_success')
return Number(last?.[1]?.timestamp || 0)
}
export function findHistoryAudio(
history: Record<string, unknown> | null,
opts?: { promptId?: string; tags?: string; since?: number }
) {
if (!history) return null
if (opts?.promptId) {
const audio = extractAudio(history, opts.promptId)
if (audio) return { promptId: opts.promptId, audio }
}
const want = String(opts?.tags || '').trim()
const matches: {
promptId: string
audio: NonNullable<ReturnType<typeof extractAudio>>
ts: number
tags: string
}[] = []
for (const [id, entry] of Object.entries(history)) {
const audio = extractAudio({ [id]: entry as Record<string, unknown> }, id)
if (!audio || !isOurComfyAudio(audio)) continue
const ts = historyCompletedAt(entry)
if (opts?.since && ts && ts < opts.since) continue
const meta = extractAceMetaFromHistory(entry)
matches.push({ promptId: id, audio, ts: historyCompletedAt(entry), tags: meta.tags })
}
if (!matches.length) return null
const tagged = want
? matches.filter((item) => {
const tags = item.tags.trim()
return tags === want || tags.includes(want) || want.includes(tags)
})
: matches
const pool = want ? tagged : matches
if (!pool.length) return null
pool.sort((a, b) => b.ts - a.ts)
return { promptId: pool[0].promptId, audio: pool[0].audio }
}
export function extractVideo(history: Record<string, unknown> | null, promptId: string) { export function extractVideo(history: Record<string, unknown> | null, promptId: string) {
if (!history) return null if (!history) return null
const wrapped = history[promptId] as { outputs?: Record<string, unknown> } | undefined const wrapped = history[promptId] as { outputs?: Record<string, unknown> } | undefined
server/utils/generationPresets.ts
+3 -1
View File
@@ -40,6 +40,7 @@ import {
clampMusicCfg, clampMusicCfg,
clampMusicDuration, clampMusicDuration,
clampMusicSteps, clampMusicSteps,
parseMusicEngine,
sanitizeMusicLyrics, sanitizeMusicLyrics,
sanitizeMusicTags sanitizeMusicTags
} from '~/utils/music' } from '~/utils/music'
@@ -173,7 +174,8 @@ function sanitizeMusicSettings(raw: unknown): MusicPresetSettings {
duration: clampMusicDuration(rec.duration), duration: clampMusicDuration(rec.duration),
steps: clampMusicSteps(rec.steps), steps: clampMusicSteps(rec.steps),
seed: typeof rec.seed === 'number' && Number.isFinite(rec.seed) ? Math.max(0, Math.floor(rec.seed)) : undefined, seed: typeof rec.seed === 'number' && Number.isFinite(rec.seed) ? Math.max(0, Math.floor(rec.seed)) : undefined,
cfg: rec.cfg == null ? undefined : clampMusicCfg(rec.cfg) cfg: rec.cfg == null ? undefined : clampMusicCfg(rec.cfg),
engine: parseMusicEngine(rec.engine)
} }
} }
server/utils/library.ts
+47
View File
@@ -19,6 +19,7 @@ import {
type KeepPromptLists type KeepPromptLists
} from '~/utils/keepPrompts' } from '~/utils/keepPrompts'
import { familyBaseName, nextClipPartName, clipPartLabel } from '~/utils/libraryBrowse' import { familyBaseName, nextClipPartName, clipPartLabel } from '~/utils/libraryBrowse'
import { musicExtension } from '~/utils/music'
export { familyBaseName, nextClipPartName, clipPartLabel } export { familyBaseName, nextClipPartName, clipPartLabel }
@@ -1170,6 +1171,7 @@ export async function maintainLibrary(owner: string, folderId?: string) {
const busy = Boolean(live && live.running > 0) const busy = Boolean(live && live.running > 0)
if (!busy) { if (!busy) {
await importMissingComfyVideos(owner, folderId).catch(() => []) await importMissingComfyVideos(owner, folderId).catch(() => [])
await importMissingComfyAudio(owner, folderId).catch(() => [])
} }
await ensureClipThumbnails(owner, { skipHistory: busy }) await ensureClipThumbnails(owner, { skipHistory: busy })
if (!busy) await sweepOwnerExtensionStills(owner) if (!busy) await sweepOwnerExtensionStills(owner)
@@ -2043,3 +2045,48 @@ export async function importMissingComfyVideos(owner: string, folderId?: string)
} }
return imported return imported
} }
export async function importMissingComfyAudio(owner: string, folderId?: string) {
const catalog = readCatalog(owner)
const folder = catalog.folders.find(item => item.id === folderId) || catalog.folders[0]
if (!folder) return []
const known = new Set((catalog.tracks || []).map(track => track.comfyFilename).filter(Boolean) as string[])
const history = await fetchHistoryAll()
const found: {
promptId: string
audio: { filename: string; subfolder: string; type: string }
meta: ReturnType<typeof extractAceMetaFromHistory>
}[] = []
for (const [promptId, entry] of Object.entries(history)) {
const audio = extractAudio({ [promptId]: entry as Record<string, unknown> }, promptId)
if (!audio || known.has(audio.filename) || !isOurComfyAudio(audio)) continue
found.push({ promptId, audio, meta: extractAceMetaFromHistory(entry) })
}
found.sort((a, b) => a.audio.filename.localeCompare(b.audio.filename, undefined, { numeric: true }))
const imported = []
for (const item of found) {
const buffer = await downloadComfyAudio(item.audio)
const ext = musicExtension(item.audio.filename)
const track = await saveTrack({
ownerKey: owner,
folderId: folder.id,
name: clipTitle({ name: '', prompt: item.meta.tags || 'Recovered track' }),
tags: item.meta.tags || 'Recovered from ComfyUI',
lyrics: item.meta.lyrics || '',
duration: item.meta.duration,
steps: item.meta.steps,
seed: item.meta.seed,
cfg: item.meta.cfg,
instrumental: !String(item.meta.lyrics || '').trim(),
audio: buffer,
ext,
engine: item.meta.engine,
samplerName: item.meta.samplerName,
scheduler: item.meta.scheduler,
lyricsStrength: item.meta.lyricsStrength,
comfyFilename: item.audio.filename
})
imported.push(track)
}
return imported
}
server/utils/musicChain.ts
+56 -15
View File
@@ -1,9 +1,9 @@
import { createJob, emitJob, type Job } from '~/server/utils/jobs' import { createJob, emitJob, type Job } from '~/server/utils/jobs'
import { extractAudio, fetchHistory, queuePrompt } from '~/server/utils/comfy' import { extractAudio, fetchHistory, fetchHistoryAll, findHistoryAudio, queuePrompt } from '~/server/utils/comfy'
import { comfyWsUrl } from '~/server/utils/comfy' import { comfyWsUrl } from '~/server/utils/comfy'
import { ensureComfyReady } from '~/server/utils/comfyLifecycle' import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
import { downloadComfyAudio, saveTrack } from '~/server/utils/library' import { downloadComfyAudio, saveTrack } from '~/server/utils/library'
import { buildMusicWorkflow, musicFilenamePrefix } from '~/server/utils/musicWorkflow' import { buildMusicWorkflow, musicFilenamePrefix, assertMusicEngineNodes } from '~/server/utils/musicWorkflow'
import { NODE_LABELS } from '~/server/utils/workflow' import { NODE_LABELS } from '~/server/utils/workflow'
import { musicExtension } from '~/utils/music' import { musicExtension } from '~/utils/music'
import type { MusicEngine } from '~/utils/music' import type { MusicEngine } from '~/utils/music'
@@ -131,13 +131,35 @@ function watchMusicJob(job: Job): Promise<void> {
resolve() resolve()
} }
const adoptPromptId = (id: unknown) => {
const promptId = String(id || '').trim()
if (promptId && !job.promptId) job.promptId = promptId
}
const loadJobAudio = async () => {
if (job.promptId) {
const scoped = await fetchHistory(job.promptId)
const fromPrompt = extractAudio(scoped, job.promptId)
if (fromPrompt) return fromPrompt
}
const found = findHistoryAudio(await fetchHistoryAll(), {
promptId: job.promptId,
tags: job.library?.tags,
since: job.startedAt - 60_000
})
if (found) {
job.promptId = found.promptId
return found.audio
}
return null
}
const succeed = async () => { const succeed = async () => {
if (settled || finishing || !job.promptId) return false if (settled || finishing) return false
finishing = true finishing = true
job.saving = true job.saving = true
try { try {
const history = await fetchHistory(job.promptId) const audio = await loadJobAudio()
const audio = extractAudio(history, job.promptId)
if (!audio) { if (!audio) {
finishing = false finishing = false
job.saving = false job.saving = false
@@ -205,14 +227,20 @@ function watchMusicJob(job: Job): Promise<void> {
} }
const pollHistory = async () => { const pollHistory = async () => {
if (settled || finishing || !job.promptId) return if (settled || finishing) return
try { try {
const history = await fetchHistory(job.promptId) const history = await fetchHistoryAll()
if (extractAudio(history, job.promptId)) { const found = findHistoryAudio(history, {
promptId: job.promptId,
tags: job.library?.tags,
since: job.startedAt - 60_000
})
if (found) {
job.promptId = found.promptId
await succeed() await succeed()
return return
} }
const entry = (history?.[job.promptId] || history) as { const entry = (job.promptId ? history?.[job.promptId] : undefined) as {
status?: { status_str?: string; completed?: boolean } status?: { status_str?: string; completed?: boolean }
} | undefined } | undefined
const status = entry?.status?.status_str const status = entry?.status?.status_str
@@ -242,6 +270,7 @@ function watchMusicJob(job: Job): Promise<void> {
} }
const type = String(payload.type || '') const type = String(payload.type || '')
const data = payload.data || {} const data = payload.data || {}
adoptPromptId(data.prompt_id)
if (type === 'status') markActivity() if (type === 'status') markActivity()
if (type === 'progress') { if (type === 'progress') {
const value = Number(data.value || 0) const value = Number(data.value || 0)
@@ -259,7 +288,7 @@ function watchMusicJob(job: Job): Promise<void> {
} }
if (type === 'executing') { if (type === 'executing') {
const node = data.node == null ? '' : String(data.node) const node = data.node == null ? '' : String(data.node)
if (!node && job.promptId) void succeed() if (!node) void succeed()
else if (node) { else if (node) {
emitLocal({ emitLocal({
type: 'status', type: 'status',
@@ -327,9 +356,11 @@ export async function startMusicJob(params: MusicJobParams) {
queuePending: status.queuePending queuePending: status.queuePending
}) })
}) })
job.status = 'uploading' job.status = 'running'
emitJob(job, { type: 'status', message: 'Queueing ACE-Step...', progress: 6 }) const engineLabel = params.engine === 'yue' ? 'YuE' : params.engine === 'ace-step-1.5' ? 'ACE-Step 1.5' : 'ACE-Step'
emitJob(job, { type: 'status', message: `Queueing ${engineLabel}...`, progress: 6 })
await waitForSocket(job, 4000) await waitForSocket(job, 4000)
await assertMusicEngineNodes(params.engine)
const graph = buildMusicWorkflow({ const graph = buildMusicWorkflow({
tags: params.tags, tags: params.tags,
lyrics: params.instrumental ? '' : params.lyrics, lyrics: params.instrumental ? '' : params.lyrics,
@@ -344,10 +375,20 @@ export async function startMusicJob(params: MusicJobParams) {
engine: params.engine, engine: params.engine,
vocalMultiplier: params.instrumental ? 0.2 : 1 vocalMultiplier: params.instrumental ? 0.2 : 1
}) })
const queued = await queuePrompt(graph, job.clientId) try {
job.promptId = queued.prompt_id const queued = await queuePrompt(graph, job.clientId, { timeoutMs: 25_000 })
job.status = 'running' if (queued?.prompt_id) job.promptId = queued.prompt_id
emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 }) emitJob(job, { type: 'status', message: 'Job queued on ComfyUI', progress: 8 })
} catch (queueError: unknown) {
const statusCode = Number((queueError as { statusCode?: number; status?: number })?.statusCode
|| (queueError as { status?: number })?.status || 0)
if (statusCode >= 400) throw queueError
emitJob(job, {
type: 'status',
message: 'Waiting for ComfyUI to pick up the job...',
progress: 8
})
}
await done await done
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : String(error) const message = error instanceof Error ? error.message : String(error)
server/utils/musicWorkflow.ts
+117 -13
View File
@@ -1,5 +1,8 @@
import workflowAceStep from '../assets/workflow_ace_step.json' import workflowAceStep from '../assets/workflow_ace_step.json'
import workflowAceStep15 from '../assets/workflow_ace_step_15.json'
import workflowYue from '../assets/workflow_yue.json'
import type { MusicEngine } from '~/utils/music' import type { MusicEngine } from '~/utils/music'
import { parseMusicBpm } from '~/utils/music'
export type MusicWorkflowParams = { export type MusicWorkflowParams = {
tags: string tags: string
@@ -15,6 +18,7 @@ export type MusicWorkflowParams = {
checkpoint?: string checkpoint?: string
engine?: MusicEngine engine?: MusicEngine
vocalMultiplier?: number vocalMultiplier?: number
bpm?: number
} }
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } } type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
@@ -22,23 +26,38 @@ type WorkflowGraph = Record<string, WorkflowNode>
const ACE_V1_CKPT = 'ace_step_v1_3.5b.safetensors' const ACE_V1_CKPT = 'ace_step_v1_3.5b.safetensors'
const ACE_15_CKPT = 'ace_step_1.5_turbo_aio.safetensors' const ACE_15_CKPT = 'ace_step_1.5_turbo_aio.safetensors'
const YUE_S1_DEFAULT = 'C:\\Users\\ianjm\\AppData\\Local\\Comfy-Desktop\\ComfyUI-Shared\\models\\YuE\\YuE-s1-7B-anneal-en-cot'
const YUE_S2_DEFAULT = 'C:\\Users\\ianjm\\AppData\\Local\\Comfy-Desktop\\ComfyUI-Shared\\models\\YuE\\YuE-s2-1B-general'
const YUE_NODE = 'YUE_Stage_A_Loader'
export function musicFilenamePrefix() { export function musicFilenamePrefix() {
return String(useRuntimeConfig().comfyAudioPrefix || process.env.COMFY_AUDIO_PREFIX || 'audio/AIGen') return String(useRuntimeConfig().comfyAudioPrefix || process.env.COMFY_AUDIO_PREFIX || 'audio/AIGen')
} }
export function buildMusicWorkflow(params: MusicWorkflowParams): WorkflowGraph { function yueStageARepo() {
return String(process.env.YUE_STAGE_A || useRuntimeConfig().yueStageA || YUE_S1_DEFAULT)
}
function yueStageBRepo() {
return String(process.env.YUE_STAGE_B || useRuntimeConfig().yueStageB || YUE_S2_DEFAULT)
}
function applyPrefix(graph: WorkflowGraph, prefix: string) {
for (const node of Object.values(graph)) {
if ((node.class_type === 'SaveAudio' || node.class_type === 'SaveAudioMP3') && 'filename_prefix' in node.inputs) {
node.inputs.filename_prefix = prefix
}
}
}
function buildAceV1Workflow(params: MusicWorkflowParams): WorkflowGraph {
const graph = structuredClone(workflowAceStep as WorkflowGraph) const graph = structuredClone(workflowAceStep as WorkflowGraph)
const engine = params.engine || 'ace-step'
const checkpoint = params.checkpoint
|| (engine === 'ace-step-1.5' ? ACE_15_CKPT : ACE_V1_CKPT)
const prefix = params.filenamePrefix || musicFilenamePrefix() const prefix = params.filenamePrefix || musicFilenamePrefix()
const sampler = params.samplerName || 'euler' const sampler = params.samplerName || 'euler'
const scheduler = params.scheduler || 'simple' const scheduler = params.scheduler || 'simple'
for (const node of Object.values(graph)) { for (const node of Object.values(graph)) {
if (node.class_type === 'CheckpointLoaderSimple' && 'ckpt_name' in node.inputs) { if (node.class_type === 'CheckpointLoaderSimple' && 'ckpt_name' in node.inputs) {
node.inputs.ckpt_name = checkpoint node.inputs.ckpt_name = params.checkpoint || ACE_V1_CKPT
} }
if (node.class_type === 'TextEncodeAceStepAudio') { if (node.class_type === 'TextEncodeAceStepAudio') {
node.inputs.tags = params.tags node.inputs.tags = params.tags
@@ -55,14 +74,99 @@ export function buildMusicWorkflow(params: MusicWorkflowParams): WorkflowGraph {
node.inputs.sampler_name = sampler node.inputs.sampler_name = sampler
node.inputs.scheduler = scheduler node.inputs.scheduler = scheduler
} }
if (node.class_type === 'SaveAudio' && 'filename_prefix' in node.inputs) {
node.inputs.filename_prefix = prefix
} }
} applyPrefix(graph, prefix)
return graph return graph
} }
export const MUSIC_NODE_LABELS: Record<string, string> = Object.fromEntries( function buildAce15Workflow(params: MusicWorkflowParams): WorkflowGraph {
Object.entries(workflowAceStep as WorkflowGraph).map(([id, node]) => [id, node._meta?.title || `Node ${id}`]) const graph = structuredClone(workflowAceStep15 as WorkflowGraph)
) const prefix = params.filenamePrefix || musicFilenamePrefix()
const sampler = params.samplerName || 'euler'
const scheduler = params.scheduler || 'simple'
const bpm = params.bpm || parseMusicBpm(params.tags)
for (const node of Object.values(graph)) {
if (node.class_type === 'CheckpointLoaderSimple' && 'ckpt_name' in node.inputs) {
node.inputs.ckpt_name = params.checkpoint || ACE_15_CKPT
}
if (node.class_type === 'TextEncodeAceStepAudio1.5') {
node.inputs.tags = params.tags
node.inputs.lyrics = params.lyrics
node.inputs.seed = params.seed
node.inputs.bpm = bpm
node.inputs.duration = params.duration
}
if (node.class_type === 'EmptyAceStep1.5LatentAudio' && 'seconds' in node.inputs) {
node.inputs.seconds = params.duration
}
if (node.class_type === 'KSampler') {
node.inputs.seed = params.seed
node.inputs.steps = params.steps
node.inputs.cfg = params.cfg
node.inputs.sampler_name = sampler
node.inputs.scheduler = scheduler
}
}
applyPrefix(graph, prefix)
return graph
}
function buildYueWorkflow(params: MusicWorkflowParams): WorkflowGraph {
const graph = structuredClone(workflowYue as WorkflowGraph)
const prefix = params.filenamePrefix || musicFilenamePrefix()
const tags = params.tags
const lyrics = params.lyrics || '[instrumental]\n'
const segments = Math.max(1, Math.min(10, lyrics.split(/\[[^\]]+\]/g).filter(part => part.trim()).length || 2))
for (const node of Object.values(graph)) {
if (node.class_type === 'YUE_Stage_A_Loader') {
node.inputs.stage_A_repo = yueStageARepo()
}
if (node.class_type === 'YUE_Stage_B_Loader') {
node.inputs.stage_B_repo = yueStageBRepo()
}
if (node.class_type === 'YUE_Stage_A_Sampler') {
node.inputs.genres_prompt = tags.replace(/,/g, ' ').replace(/\s+/g, ' ').trim()
node.inputs.lyrics_prompt = lyrics
node.inputs.seed = params.seed
node.inputs.prompt_end_time = params.duration
node.inputs.run_n_segment = segments
}
}
applyPrefix(graph, prefix)
return graph
}
export function buildMusicWorkflow(params: MusicWorkflowParams): WorkflowGraph {
const engine = params.engine || 'ace-step'
if (engine === 'yue') return buildYueWorkflow(params)
if (engine === 'ace-step-1.5') return buildAce15Workflow(params)
return buildAceV1Workflow(params)
}
export async function assertMusicEngineNodes(engine: MusicEngine | undefined) {
const { comfyHasClassType } = await import('~/server/utils/comfy')
if (engine === 'yue') {
const present = await comfyHasClassType(YUE_NODE)
if (present === false) {
throw createError({
statusCode: 400,
statusMessage: 'YuE is not installed on Beast. Clone ComfyUI_YuE into custom_nodes and restart Comfy, then try again.'
})
}
}
if (engine === 'ace-step-1.5') {
const present = await comfyHasClassType('TextEncodeAceStepAudio1.5')
if (present === false) {
throw createError({
statusCode: 400,
statusMessage: 'This ComfyUI build does not have ACE-Step 1.5 nodes. Update Comfy Desktop, or turn the 1.5 toggle off.'
})
}
}
}
export const MUSIC_NODE_LABELS: Record<string, string> = Object.fromEntries([
...Object.entries(workflowAceStep as WorkflowGraph).map(([id, node]) => [id, node._meta?.title || `Node ${id}`]),
...Object.entries(workflowAceStep15 as WorkflowGraph).map(([id, node]) => [id, node._meta?.title || `Node ${id}`]),
...Object.entries(workflowYue as WorkflowGraph).map(([id, node]) => [id, node._meta?.title || `Node ${id}`])
])
utils/generationPresets.ts
+1
View File
@@ -12,6 +12,7 @@ export type MusicPresetSettings = {
steps?: number steps?: number
seed?: number seed?: number
cfg?: number cfg?: number
engine?: import('~/utils/music').MusicEngine
} }
export type VideoPresetSettings = { export type VideoPresetSettings = {
utils/libraryGeneration.ts
+6 -1
View File
@@ -295,7 +295,12 @@ export function trackSettingsRows(track: {
lyricsStrength?: number lyricsStrength?: number
}) { }) {
const rows: SettingsRow[] = [] const rows: SettingsRow[] = []
if (track.engine) rows.push({ label: 'Engine', value: track.engine === 'ace-step-1.5' ? 'ACE-Step 1.5' : 'ACE-Step' }) if (track.engine) {
rows.push({
label: 'Engine',
value: track.engine === 'ace-step-1.5' ? 'ACE-Step 1.5' : track.engine === 'yue' ? 'YuE' : 'ACE-Step'
})
}
if (track.tags?.trim()) rows.push({ label: 'Tags', value: track.tags.trim() }) if (track.tags?.trim()) rows.push({ label: 'Tags', value: track.tags.trim() })
if (track.instrumental) rows.push({ label: 'Instrumental', value: 'On' }) if (track.instrumental) rows.push({ label: 'Instrumental', value: 'On' })
else if (track.lyrics?.trim()) rows.push({ label: 'Lyrics', value: track.lyrics.trim() }) else if (track.lyrics?.trim()) rows.push({ label: 'Lyrics', value: track.lyrics.trim() })
utils/music.ts
+23 -1
View File
@@ -4,7 +4,9 @@ export const MUSIC_DURATION_DEFAULT = 60
export const MUSIC_STEPS_MIN = 8 export const MUSIC_STEPS_MIN = 8
export const MUSIC_STEPS_MAX = 100 export const MUSIC_STEPS_MAX = 100
export const MUSIC_STEPS_DEFAULT = 50 export const MUSIC_STEPS_DEFAULT = 50
export const MUSIC_STEPS_DEFAULT_15 = 8
export const MUSIC_CFG_DEFAULT = 5 export const MUSIC_CFG_DEFAULT = 5
export const MUSIC_CFG_DEFAULT_15 = 1
export const MUSIC_LYRICS_STRENGTH_DEFAULT = 0.9 export const MUSIC_LYRICS_STRENGTH_DEFAULT = 0.9
export const MUSIC_TAGS_MAX = 2000 export const MUSIC_TAGS_MAX = 2000
export const MUSIC_LYRICS_MAX = 8000 export const MUSIC_LYRICS_MAX = 8000
@@ -31,14 +33,34 @@ export const DEFAULT_MUSIC_LYRICS = `[Verse 1]
[Outro] [Outro]
` `
export type MusicEngine = 'ace-step' | 'ace-step-1.5' export type MusicEngine = 'ace-step' | 'ace-step-1.5' | 'yue'
export function parseMusicEngine(raw: unknown): MusicEngine { export function parseMusicEngine(raw: unknown): MusicEngine {
const value = String(raw || '').trim().toLowerCase() const value = String(raw || '').trim().toLowerCase()
if (value === 'ace-step-1.5' || value === 'ace15' || value === 'v1.5') return 'ace-step-1.5' if (value === 'ace-step-1.5' || value === 'ace15' || value === 'v1.5') return 'ace-step-1.5'
if (value === 'yue' || value === 'yue-s1' || value === 'lyrics2song') return 'yue'
return 'ace-step' return 'ace-step'
} }
export function isAceMusicEngine(engine: MusicEngine | string | undefined) {
return parseMusicEngine(engine) !== 'yue'
}
export function musicEngineLabel(engine: MusicEngine | string | undefined) {
const value = parseMusicEngine(engine)
if (value === 'ace-step-1.5') return 'ACE-Step 1.5'
if (value === 'yue') return 'YuE'
return 'ACE-Step'
}
export function parseMusicBpm(tags: unknown, fallback = 120) {
const match = String(tags || '').match(/(\d{2,3})\s*bpm/i)
if (!match) return fallback
const value = Number(match[1])
if (!Number.isFinite(value)) return fallback
return Math.min(300, Math.max(10, Math.round(value)))
}
export function clampMusicDuration(raw: unknown, fallback = MUSIC_DURATION_DEFAULT) { export function clampMusicDuration(raw: unknown, fallback = MUSIC_DURATION_DEFAULT) {
const value = Number(raw) const value = Number(raw)
if (!Number.isFinite(value)) return fallback if (!Number.isFinite(value)) return fallback