Add sidecar Qwen VL prompt recommend, fix queued shot bursts, and leave unused MiniMax identity slots empty.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-27 17:04:43 -05:00
parent 2bb30395d8
commit df5ac24622
23 changed files with 1068 additions and 25 deletions
@@ -0,0 +1,240 @@
+import { getSidecarImageHost } from '~/server/utils/imageComfy'
+import { buildVisionPromptWorkflow } from '~/server/utils/promptWorkflow'
+
+export type PromptJobStatus = 'queued' | 'running' | 'complete' | 'error'
+
+export interface PromptJob {
+  id: string
+  status: PromptJobStatus
+  message: string
+  text: string
+  error?: string
+  startedAt: number
+}
+
+const jobs = new Map<string, PromptJob>()
+const MAX_JOBS = 20
+
+function normalizeHost(raw: string, port = '') {
+  let host = String(raw || '').trim().replace(/\/$/, '')
+  if (!host) return ''
+  if (!/^https?:\/\//i.test(host)) host = `http://${host}`
+  try {
+    const url = new URL(host)
+    if (port && !url.port) url.port = port
+    return url.origin
+  } catch {
+    return port ? `${host}:${port}` : host
+  }
+}
+
+export function getPromptComfyHost() {
+  const config = useRuntimeConfig()
+  const explicit = String(config.promptComfyHost || process.env.PROMPT_COMFY_HOST || '').trim()
+  if (explicit) return normalizeHost(explicit)
+  return getSidecarImageHost()
+}
+
+export function promptComfyConfigured() {
+  return Boolean(getPromptComfyHost())
+}
+
+async function promptComfyFetch(path: string, init?: RequestInit) {
+  const host = getPromptComfyHost()
+  if (!host) {
+    throw createError({ statusCode: 503, statusMessage: 'Sidecar Qwen VL is not configured. Set IMAGE_COMFY_HOST.' })
+  }
+  try {
+    return await fetch(`${host}${path}`, init)
+  } catch (error) {
+    throw createError({
+      statusCode: 502,
+      statusMessage: `Sidecar Qwen VL is unreachable (${host})`,
+      data: { cause: error instanceof Error ? error.message : String(error) }
+    })
+  }
+}
+
+function promptInputFilename(original: string, jobId?: string) {
+  const raw = String(original || 'still.png')
+  const dot = raw.lastIndexOf('.')
+  const ext = (dot >= 0 ? raw.slice(dot) : '.png').replace(/[^.a-zA-Z0-9]/g, '') || '.png'
+  const base = (dot >= 0 ? raw.slice(0, dot) : raw).replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 48) || 'still'
+  const id = String(jobId || crypto.randomUUID()).replace(/-/g, '').slice(0, 10)
+  return `aigen_prompt_${id}_${base}${ext}`
+}
+
+async function uploadPromptImage(file: { filename: string; data: Buffer; type?: string }, jobId?: string) {
+  const body = new FormData()
+  const blob = new Blob([new Uint8Array(file.data)], { type: file.type || 'application/octet-stream' })
+  const filename = promptInputFilename(file.filename, jobId)
+  body.append('image', blob, filename)
+  body.append('overwrite', 'true')
+  body.append('type', 'input')
+  const res = await promptComfyFetch('/upload/image', { method: 'POST', body })
+  if (!res.ok) {
+    throw createError({ statusCode: 502, statusMessage: `Sidecar image upload failed (${res.status})` })
+  }
+  const uploaded = (await res.json()) as { name: string; subfolder?: string; type?: string }
+  return uploaded.name || filename
+}
+
+async function queuePromptGraph(graph: unknown, clientId: string) {
+  const res = await promptComfyFetch('/prompt', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ prompt: graph, client_id: clientId })
+  })
+  const payload = await res.json().catch(() => ({}))
+  if (!res.ok) {
+    const message = (payload as { error?: { message?: string } }).error?.message
+      || (payload as { node_errors?: unknown }).node_errors
+      || `Sidecar queue failed (${res.status})`
+    throw createError({ statusCode: 502, statusMessage: String(message), data: payload })
+  }
+  return payload as { prompt_id: string }
+}
+
+async function fetchPromptHistory(promptId: string) {
+  const res = await promptComfyFetch(`/history/${encodeURIComponent(promptId)}`)
+  if (!res.ok) return null
+  return (await res.json()) as Record<string, unknown>
+}
+
+function collectStrings(value: unknown, depth = 0): string[] {
+  if (value == null || depth > 8) return []
+  if (typeof value === 'string') {
+    const text = value.trim()
+    return text ? [text] : []
+  }
+  if (typeof value === 'number' || typeof value === 'boolean') return []
+  if (Array.isArray(value)) return value.flatMap(item => collectStrings(item, depth + 1))
+  if (typeof value === 'object') {
+    const record = value as Record<string, unknown>
+    for (const key of ['text', 'string', 'strings']) {
+      if (record[key] !== undefined) {
+        const found = collectStrings(record[key], depth + 1)
+        if (found.length) return found
+      }
+    }
+    return Object.values(record).flatMap(item => collectStrings(item, depth + 1))
+  }
+  return []
+}
+
+function extractPromptText(history: Record<string, unknown> | null, promptId: string) {
+  if (!history) return ''
+  const wrapped = history[promptId] as { outputs?: Record<string, unknown> } | undefined
+  const outputs = wrapped?.outputs || (history as { outputs?: Record<string, unknown> }).outputs
+  if (!outputs || typeof outputs !== 'object') return ''
+  const node4 = outputs['4']
+  const candidates = [
+    ...(node4 ? collectStrings(node4) : []),
+    ...collectStrings(outputs)
+  ].sort((a, b) => b.length - a.length)
+  return candidates[0] || ''
+}
+
+function historyError(history: Record<string, unknown> | null, promptId: string) {
+  const entry = history?.[promptId] as {
+    status?: {
+      status_str?: string
+      completed?: boolean
+      messages?: Array<[string, Record<string, unknown>]>
+    }
+  } | undefined
+  const status = entry?.status?.status_str
+  const err = entry?.status?.messages?.find(([type]) => type === 'execution_error')?.[1]
+  const message = String(err?.exception_message || err?.message || '').trim()
+  if (status === 'interrupted') return 'Prompt recommend was interrupted.'
+  if (status === 'error') return message || 'Sidecar Qwen VL reported an execution error'
+  if (entry?.status?.completed && !extractPromptText(history, promptId)) {
+    return 'Sidecar Qwen VL finished without prompt text'
+  }
+  return null
+}
+
+async function waitForPromptText(promptId: string, timeoutMs = 240_000) {
+  const started = Date.now()
+  while (Date.now() - started < timeoutMs) {
+    const history = await fetchPromptHistory(promptId)
+    const text = extractPromptText(history, promptId)
+    if (text) return text
+    const error = historyError(history, promptId)
+    if (error) {
+      throw createError({ statusCode: 502, statusMessage: error })
+    }
+    await new Promise(resolve => setTimeout(resolve, 1200))
+  }
+  throw createError({ statusCode: 504, statusMessage: 'Sidecar Qwen VL timed out while writing the prompt' })
+}
+
+export function createPromptJob(): PromptJob {
+  const job: PromptJob = {
+    id: crypto.randomUUID(),
+    status: 'queued',
+    message: 'Queued',
+    text: '',
+    startedAt: Date.now()
+  }
+  jobs.set(job.id, job)
+  while (jobs.size > MAX_JOBS) {
+    const oldest = jobs.keys().next().value
+    if (oldest) jobs.delete(oldest)
+  }
+  return job
+}
+
+export function getPromptJob(id: string) {
+  return jobs.get(id)
+}
+
+export function promptJobSnapshot(job: PromptJob) {
+  return {
+    jobId: job.id,
+    status: job.status,
+    message: job.message,
+    text: job.text,
+    error: job.error,
+    elapsedMs: Date.now() - job.startedAt
+  }
+}
+
+function draftUserPrompt(draft: string, mode: string) {
+  const studio = mode === 'edit'
+    ? 'image-edit'
+    : mode === 'shot-script'
+      ? 'video-shot-script'
+      : 'video'
+  return `Studio mode: ${studio}\n\nDraft Prompt:\n${draft}`
+}
+
+export async function runPromptRecommend(job: PromptJob, params: {
+  image: { filename: string; data: Buffer; type?: string }
+  draftPrompt: string
+  systemPrompt: string
+  mode: string
+}) {
+  job.status = 'running'
+  job.message = 'Uploading still to Sidecar Qwen VL…'
+  const imageName = await uploadPromptImage(params.image, job.id)
+  job.message = 'Qwen VL is reading the still…'
+  const graph = buildVisionPromptWorkflow({
+    imageName,
+    draftPrompt: draftUserPrompt(params.draftPrompt, params.mode),
+    systemPrompt: params.systemPrompt
+  })
+  const queued = await queuePromptGraph(graph, job.id)
+  job.message = 'Writing the recommended prompt…'
+  const text = await waitForPromptText(queued.prompt_id)
+  job.text = text.trim()
+  job.status = 'complete'
+  job.message = 'Recommended prompt ready'
+}
+
+export function failPromptJob(job: PromptJob, error: unknown) {
+  const err = error as { statusMessage?: string; message?: string }
+  job.status = 'error'
+  job.error = err.statusMessage || err.message || 'Prompt recommend failed'
+  job.message = job.error
+}