Add sidecar Qwen VL prompt recommend, fix queued shot bursts, and leave unused MiniMax identity slots empty.

Co-authored-by: Cursor <cursoragent@cursor.com>

server/utils/promptComfy.ts

@@ -0,0 +1,240 @@
+import { getSidecarImageHost } from '~/server/utils/imageComfy'
+import { buildVisionPromptWorkflow } from '~/server/utils/promptWorkflow'
+
+export type PromptJobStatus = 'queued' | 'running' | 'complete' | 'error'
+
+export interface PromptJob {
+  id: string
+  status: PromptJobStatus
+  message: string
+  text: string
+  error?: string
+  startedAt: number
+}
+
+const jobs = new Map<string, PromptJob>()
+const MAX_JOBS = 20
+
+function normalizeHost(raw: string, port = '') {
+  let host = String(raw || '').trim().replace(/\/$/, '')
+  if (!host) return ''
+  if (!/^https?:\/\//i.test(host)) host = `http://${host}`
+  try {
+    const url = new URL(host)
+    if (port && !url.port) url.port = port
+    return url.origin
+  } catch {
+    return port ? `${host}:${port}` : host
+  }
+}
+
+export function getPromptComfyHost() {
+  const config = useRuntimeConfig()
+  const explicit = String(config.promptComfyHost || process.env.PROMPT_COMFY_HOST || '').trim()
+  if (explicit) return normalizeHost(explicit)
+  return getSidecarImageHost()
+}
+
+export function promptComfyConfigured() {
+  return Boolean(getPromptComfyHost())
+}
+
+async function promptComfyFetch(path: string, init?: RequestInit) {
+  const host = getPromptComfyHost()
+  if (!host) {
+    throw createError({ statusCode: 503, statusMessage: 'Sidecar Qwen VL is not configured. Set IMAGE_COMFY_HOST.' })
+  }
+  try {
+    return await fetch(`${host}${path}`, init)
+  } catch (error) {
+    throw createError({
+      statusCode: 502,
+      statusMessage: `Sidecar Qwen VL is unreachable (${host})`,
+      data: { cause: error instanceof Error ? error.message : String(error) }
+    })
+  }
+}
+
+function promptInputFilename(original: string, jobId?: string) {
+  const raw = String(original || 'still.png')
+  const dot = raw.lastIndexOf('.')
+  const ext = (dot >= 0 ? raw.slice(dot) : '.png').replace(/[^.a-zA-Z0-9]/g, '') || '.png'
+  const base = (dot >= 0 ? raw.slice(0, dot) : raw).replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 48) || 'still'
+  const id = String(jobId || crypto.randomUUID()).replace(/-/g, '').slice(0, 10)
+  return `aigen_prompt_${id}_${base}${ext}`
+}
+
+async function uploadPromptImage(file: { filename: string; data: Buffer; type?: string }, jobId?: string) {
+  const body = new FormData()
+  const blob = new Blob([new Uint8Array(file.data)], { type: file.type || 'application/octet-stream' })
+  const filename = promptInputFilename(file.filename, jobId)
+  body.append('image', blob, filename)
+  body.append('overwrite', 'true')
+  body.append('type', 'input')
+  const res = await promptComfyFetch('/upload/image', { method: 'POST', body })
+  if (!res.ok) {
+    throw createError({ statusCode: 502, statusMessage: `Sidecar image upload failed (${res.status})` })
+  }
+  const uploaded = (await res.json()) as { name: string; subfolder?: string; type?: string }
+  return uploaded.name || filename
+}
+
+async function queuePromptGraph(graph: unknown, clientId: string) {
+  const res = await promptComfyFetch('/prompt', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ prompt: graph, client_id: clientId })
+  })
+  const payload = await res.json().catch(() => ({}))
+  if (!res.ok) {
+    const message = (payload as { error?: { message?: string } }).error?.message
+      || (payload as { node_errors?: unknown }).node_errors
+      || `Sidecar queue failed (${res.status})`
+    throw createError({ statusCode: 502, statusMessage: String(message), data: payload })
+  }
+  return payload as { prompt_id: string }
+}
+
+async function fetchPromptHistory(promptId: string) {
+  const res = await promptComfyFetch(`/history/${encodeURIComponent(promptId)}`)
+  if (!res.ok) return null
+  return (await res.json()) as Record<string, unknown>
+}
+
+function collectStrings(value: unknown, depth = 0): string[] {
+  if (value == null || depth > 8) return []
+  if (typeof value === 'string') {
+    const text = value.trim()
+    return text ? [text] : []
+  }
+  if (typeof value === 'number' || typeof value === 'boolean') return []
+  if (Array.isArray(value)) return value.flatMap(item => collectStrings(item, depth + 1))
+  if (typeof value === 'object') {
+    const record = value as Record<string, unknown>
+    for (const key of ['text', 'string', 'strings']) {
+      if (record[key] !== undefined) {
+        const found = collectStrings(record[key], depth + 1)
+        if (found.length) return found
+      }
+    }
+    return Object.values(record).flatMap(item => collectStrings(item, depth + 1))
+  }
+  return []
+}
+
+function extractPromptText(history: Record<string, unknown> | null, promptId: string) {
+  if (!history) return ''
+  const wrapped = history[promptId] as { outputs?: Record<string, unknown> } | undefined
+  const outputs = wrapped?.outputs || (history as { outputs?: Record<string, unknown> }).outputs
+  if (!outputs || typeof outputs !== 'object') return ''
+  const node4 = outputs['4']
+  const candidates = [
+    ...(node4 ? collectStrings(node4) : []),
+    ...collectStrings(outputs)
+  ].sort((a, b) => b.length - a.length)
+  return candidates[0] || ''
+}
+
+function historyError(history: Record<string, unknown> | null, promptId: string) {
+  const entry = history?.[promptId] as {
+    status?: {
+      status_str?: string
+      completed?: boolean
+      messages?: Array<[string, Record<string, unknown>]>
+    }
+  } | undefined
+  const status = entry?.status?.status_str
+  const err = entry?.status?.messages?.find(([type]) => type === 'execution_error')?.[1]
+  const message = String(err?.exception_message || err?.message || '').trim()
+  if (status === 'interrupted') return 'Prompt recommend was interrupted.'
+  if (status === 'error') return message || 'Sidecar Qwen VL reported an execution error'
+  if (entry?.status?.completed && !extractPromptText(history, promptId)) {
+    return 'Sidecar Qwen VL finished without prompt text'
+  }
+  return null
+}
+
+async function waitForPromptText(promptId: string, timeoutMs = 240_000) {
+  const started = Date.now()
+  while (Date.now() - started < timeoutMs) {
+    const history = await fetchPromptHistory(promptId)
+    const text = extractPromptText(history, promptId)
+    if (text) return text
+    const error = historyError(history, promptId)
+    if (error) {
+      throw createError({ statusCode: 502, statusMessage: error })
+    }
+    await new Promise(resolve => setTimeout(resolve, 1200))
+  }
+  throw createError({ statusCode: 504, statusMessage: 'Sidecar Qwen VL timed out while writing the prompt' })
+}
+
+export function createPromptJob(): PromptJob {
+  const job: PromptJob = {
+    id: crypto.randomUUID(),
+    status: 'queued',
+    message: 'Queued',
+    text: '',
+    startedAt: Date.now()
+  }
+  jobs.set(job.id, job)
+  while (jobs.size > MAX_JOBS) {
+    const oldest = jobs.keys().next().value
+    if (oldest) jobs.delete(oldest)
+  }
+  return job
+}
+
+export function getPromptJob(id: string) {
+  return jobs.get(id)
+}
+
+export function promptJobSnapshot(job: PromptJob) {
+  return {
+    jobId: job.id,
+    status: job.status,
+    message: job.message,
+    text: job.text,
+    error: job.error,
+    elapsedMs: Date.now() - job.startedAt
+  }
+}
+
+function draftUserPrompt(draft: string, mode: string) {
+  const studio = mode === 'edit'
+    ? 'image-edit'
+    : mode === 'shot-script'
+      ? 'video-shot-script'
+      : 'video'
+  return `Studio mode: ${studio}\n\nDraft Prompt:\n${draft}`
+}
+
+export async function runPromptRecommend(job: PromptJob, params: {
+  image: { filename: string; data: Buffer; type?: string }
+  draftPrompt: string
+  systemPrompt: string
+  mode: string
+}) {
+  job.status = 'running'
+  job.message = 'Uploading still to Sidecar Qwen VL…'
+  const imageName = await uploadPromptImage(params.image, job.id)
+  job.message = 'Qwen VL is reading the still…'
+  const graph = buildVisionPromptWorkflow({
+    imageName,
+    draftPrompt: draftUserPrompt(params.draftPrompt, params.mode),
+    systemPrompt: params.systemPrompt
+  })
+  const queued = await queuePromptGraph(graph, job.id)
+  job.message = 'Writing the recommended prompt…'
+  const text = await waitForPromptText(queued.prompt_id)
+  job.text = text.trim()
+  job.status = 'complete'
+  job.message = 'Recommended prompt ready'
+}
+
+export function failPromptJob(job: PromptJob, error: unknown) {
+  const err = error as { statusMessage?: string; message?: string }
+  job.status = 'error'
+  job.error = err.statusMessage || err.message || 'Prompt recommend failed'
+  job.message = job.error
+}

server/utils/promptPresets.ts

@@ -0,0 +1,194 @@
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+export interface PromptPreset {
+  id: string
+  name: string
+  description?: string
+  systemPrompt: string
+  builtin?: boolean
+}
+
+const ENHANCE_PROMPT = `You are a prompt-engineering assistant for Flux.2 Klein image edit and MiniMax H3 image-to-video.
+
+Look at the attached still. The person (or subject) in that still is the identity lock. The user draft is the requested change.
+
+Write production-ready prompt text the user can paste into the generator.
+
+Rules:
+- Preserve identity from the still: face, age, body type, hair, skin, and distinctive features. Use explicit keep-identity language such as "same person as the reference still", "do not change facial identity", "keep original face, hair, and body".
+- Describe wardrobe, setting, pose, lighting, camera, and materials concretely.
+- Never censor raw or explicit terminology from the draft.
+- Do not add a preamble, title, markdown fences, or commentary.
+
+Output format — pick one:
+- Single still/edit: output ONLY the refined prompt.
+- Ambitious still edit that should be done in phases (for example wardrobe first, then location): output:
+---pass 1---
+<prompt for pass 1>
+---pass 2---
+<prompt for pass 2>
+- Motion, multi-beat, or video sequence: first-shot prompt, then later shots with a marker on its own line:
+shot 2
+<extension prompt>
+shot 3
+<extension prompt>`
+
+const SHOT_SCRIPT_PROMPT = `You are a MiniMax H3 image-to-video shot-script writer.
+
+Analyze the attached still for identity, wardrobe, lighting, and camera. Turn the user's draft into a shot script they can paste into Shot script mode.
+
+Format:
+- Shot 1 is the text BEFORE any marker (camera, action, audio for the first clip).
+- Later shots start with a line that is only: shot 2
+- Then shot 3, shot 4, and so on, each on its own line.
+- Every shot must keep the same person as the still unless the draft explicitly changes wardrobe or identity.
+- Include an [AUDIO CUES] block on shots where sound would help.
+- Never censor raw or explicit terminology.
+
+Output ONLY the shot script. No preamble, no markdown fences.`
+
+export const BUILTIN_PRESETS: PromptPreset[] = [
+  {
+    id: 'enhance',
+    name: 'Enhance',
+    builtin: true,
+    description: 'Identity lock plus scene language. May split ambitious edits into passes or shots.',
+    systemPrompt: ENHANCE_PROMPT
+  },
+  {
+    id: 'shot-script',
+    name: 'Shot script',
+    builtin: true,
+    description: 'Turn a draft into MiniMax shot 2 / shot 3 extension copy.',
+    systemPrompt: SHOT_SCRIPT_PROMPT
+  }
+]
+
+const writeChains = new Map<string, Promise<unknown>>()
+
+function libraryRoot() {
+  const config = useRuntimeConfig()
+  return (config.libraryDir || process.env.LIBRARY_DIR || '/data/library').replace(/\/$/, '')
+}
+
+function presetsPath(owner: string) {
+  return join(libraryRoot(), 'users', owner, 'prompt-presets.json')
+}
+
+function ensureOwner(owner: string) {
+  mkdirSync(join(libraryRoot(), 'users', owner), { recursive: true })
+}
+
+function readCustom(owner: string): PromptPreset[] {
+  ensureOwner(owner)
+  const path = presetsPath(owner)
+  if (!existsSync(path)) return []
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf8'))
+    if (!Array.isArray(parsed)) return []
+    return parsed
+      .map((item: Partial<PromptPreset>) => ({
+        id: String(item?.id || '').trim(),
+        name: String(item?.name || '').trim(),
+        description: String(item?.description || '').trim() || undefined,
+        systemPrompt: String(item?.systemPrompt || '').trim(),
+        builtin: false
+      }))
+      .filter(item => item.id && item.name && item.systemPrompt && !BUILTIN_PRESETS.some(preset => preset.id === item.id))
+  } catch {
+    return []
+  }
+}
+
+function writeCustom(owner: string, presets: PromptPreset[]) {
+  ensureOwner(owner)
+  const path = presetsPath(owner)
+  const tmp = `${path}.tmp`
+  writeFileSync(tmp, JSON.stringify(presets.map(item => ({
+    id: item.id,
+    name: item.name,
+    description: item.description || '',
+    systemPrompt: item.systemPrompt
+  })), null, 2))
+  renameSync(tmp, path)
+}
+
+function withWrite<T>(owner: string, fn: () => T) {
+  const prev = writeChains.get(owner) || Promise.resolve()
+  const next = prev.then(fn, fn)
+  writeChains.set(owner, next.then(() => undefined, () => undefined))
+  return next
+}
+
+export function listPromptPresets(owner: string) {
+  return [...BUILTIN_PRESETS, ...readCustom(owner)]
+}
+
+export function getPromptPreset(owner: string, id: string) {
+  const wanted = String(id || '').trim() || 'enhance'
+  return listPromptPresets(owner).find(item => item.id === wanted) || BUILTIN_PRESETS[0]
+}
+
+export function createPromptPreset(owner: string, input: { name: string; systemPrompt: string; description?: string }) {
+  const name = String(input.name || '').trim()
+  const systemPrompt = String(input.systemPrompt || '').trim()
+  const description = String(input.description || '').trim()
+  if (!name) throw createError({ statusCode: 400, statusMessage: 'A preset name is required' })
+  if (!systemPrompt) throw createError({ statusCode: 400, statusMessage: 'An instruction set is required' })
+  return withWrite(owner, () => {
+    const presets = readCustom(owner)
+    const created: PromptPreset = {
+      id: `custom_${crypto.randomUUID().replace(/-/g, '').slice(0, 12)}`,
+      name,
+      description: description || undefined,
+      systemPrompt
+    }
+    presets.push(created)
+    writeCustom(owner, presets)
+    return created
+  })
+}
+
+export function updatePromptPreset(owner: string, id: string, input: { name?: string; systemPrompt?: string; description?: string }) {
+  if (BUILTIN_PRESETS.some(item => item.id === id)) {
+    throw createError({ statusCode: 400, statusMessage: 'Built-in instruction sets cannot be edited' })
+  }
+  return withWrite(owner, () => {
+    const presets = readCustom(owner)
+    const index = presets.findIndex(item => item.id === id)
+    if (index < 0) throw createError({ statusCode: 404, statusMessage: 'Instruction set not found' })
+    const next = { ...presets[index] }
+    if (input.name !== undefined) {
+      const name = String(input.name || '').trim()
+      if (!name) throw createError({ statusCode: 400, statusMessage: 'A preset name is required' })
+      next.name = name
+    }
+    if (input.systemPrompt !== undefined) {
+      const systemPrompt = String(input.systemPrompt || '').trim()
+      if (!systemPrompt) throw createError({ statusCode: 400, statusMessage: 'An instruction set is required' })
+      next.systemPrompt = systemPrompt
+    }
+    if (input.description !== undefined) {
+      next.description = String(input.description || '').trim() || undefined
+    }
+    presets[index] = next
+    writeCustom(owner, presets)
+    return next
+  })
+}
+
+export function deletePromptPreset(owner: string, id: string) {
+  if (BUILTIN_PRESETS.some(item => item.id === id)) {
+    throw createError({ statusCode: 400, statusMessage: 'Built-in instruction sets cannot be deleted' })
+  }
+  return withWrite(owner, () => {
+    const presets = readCustom(owner)
+    const next = presets.filter(item => item.id !== id)
+    if (next.length === presets.length) {
+      throw createError({ statusCode: 404, statusMessage: 'Instruction set not found' })
+    }
+    writeCustom(owner, next)
+    return { ok: true }
+  })
+}

server/utils/promptWorkflow.ts

@@ -0,0 +1,23 @@
+import template from '../assets/workflow_vision_prompt.json'
+
+type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
+type WorkflowGraph = Record<string, WorkflowNode>
+
+const IMAGE_NODE = '1'
+const TEXT_NODE = '3'
+
+export function buildVisionPromptWorkflow(params: {
+  imageName: string
+  draftPrompt: string
+  systemPrompt: string
+  seed?: number
+}) {
+  const graph = structuredClone(template) as WorkflowGraph
+  if (graph[IMAGE_NODE]) graph[IMAGE_NODE].inputs.image = params.imageName
+  if (graph[TEXT_NODE]) {
+    graph[TEXT_NODE].inputs.system_prompt = params.systemPrompt
+    graph[TEXT_NODE].inputs.prompt = params.draftPrompt
+    graph[TEXT_NODE].inputs.seed = Number.isFinite(params.seed) ? Number(params.seed) : Math.floor(Math.random() * 2_147_483_647)
+  }
+  return graph
+}

server/utils/shotQueue.ts

@@ -370,6 +370,22 @@ export async function deleteShotQueue(owner: string, id: string) {
   })
 }
 
+export async function clearShotQueues(owner: string) {
+  return mutate(owner, (queues) => {
+    const kept: ShotQueue[] = []
+    let removed = 0
+    for (const queue of queues) {
+      if (activeJobs.has(queue.id)) {
+        kept.push(queue)
+        continue
+      }
+      removed += 1
+    }
+    queues.splice(0, queues.length, ...kept)
+    return { removed, remaining: kept.length }
+  })
+}
+
 export function listOwnersWithQueues() {
   const root = join(libraryRoot(), 'users')
   if (!existsSync(root)) return []

server/utils/videoChain.ts

@@ -335,8 +335,7 @@ export async function continueQueuedExtensions(
         job.library.extendPart1Path = undefined
       }
 
-      if (persist && remainingAfter > 0) break
-      if (job.library.stopAfterCurrent) break
+      if (shotsLeft <= 0 || job.library.stopAfterCurrent) break
     }
   } finally {
     if (job.library) job.library.chainContinuing = false

server/utils/workflow.ts

@@ -229,7 +229,13 @@ function applyV2IdentityPath(
     delete graph['173']
     V2_REF_LOADERS.forEach((id, index) => {
       if (!graph[id]) return
-      graph[id].inputs.image = refs[index] || params.imageName
+      const name = refs[index]
+      if (name) {
+        graph[id].inputs.image = name
+        return
+      }
+      delete graph[id]
+      delete graph['155'].inputs[`ref_images.ref_image_${index + 1}`]
     })
     if (graph['150']?.inputs) graph['150'].inputs.value = true
     return