Add image-to-text Describe on the bench via exclusive llama.cpp caption jobs.

server/api/generate/caption.post.ts

@@ -0,0 +1,50 @@
+import { validateCaptionBody } from '~/shared/caption.mjs'
+import { addStudioJob, kickStudioQueue, type StudioJobPayload } from '~/server/utils/studioQueue'
+import { getStill } from '~/server/utils/library'
+
+export default defineEventHandler(async (event) => {
+  const { owner } = assertLibraryOwner(event)
+  const raw = await readBody(event)
+  let body: ReturnType<typeof validateCaptionBody>
+  try {
+    body = validateCaptionBody(raw)
+  } catch (e: any) {
+    throw createError({ statusCode: e.statusCode || 400, statusMessage: e.message })
+  }
+  assertFolderAccess(event, body.folderId)
+  const still = getStill(owner, body.stillId)
+  assertFolderAccess(event, still.folderId)
+
+  const row = await addStudioJob({
+    ownerKey: owner,
+    kind: 'caption',
+    familyId: crypto.randomUUID(),
+    payload: {
+      prompt: `Describe · ${body.captionStyle}`,
+      name: `Describe · ${body.captionStyle}`,
+      folderId: body.folderId,
+      stillId: still.id,
+      captionStyle: body.captionStyle,
+      aspect: 'auto',
+      width: 0,
+      height: 0,
+      steps: 1,
+      turbo: true,
+      seed: 0,
+      cfg: 0,
+      fps: 24,
+      samplerName: 'euler',
+      scheduler: 'simple',
+      duration: 0,
+      sound: false,
+      workflow: 'minimax',
+      useIdentityRefs: false,
+      hideThumbnail: false,
+      referenceStillIds: [],
+      extensions: [],
+      queueAutoRun: true
+    } as StudioJobPayload
+  })
+  kickStudioQueue()
+  return { id: row.id, liveJobId: row.liveJobId, captionStyle: body.captionStyle, stillId: still.id }
+})

server/api/health.get.ts

@@ -48,7 +48,7 @@ export default defineEventHandler(async () => {
       host: comfy.host || '',
       queueRunning: queue.running,
       queuePending: queue.pending,
-      asleep: Boolean(control?.asleep) && !comfyReachable && !control?.yue2?.busy,
+      asleep: Boolean(control?.asleep) && !comfyReachable && !control?.yue2?.busy && !control?.caption?.busy,
       processUp: Boolean(control?.process ?? control?.processUp),
       lastActivityAt: control?.lastActivityAt || null,
       stats: comfy.ok ? comfy.stats : undefined
@@ -58,6 +58,11 @@ export default defineEventHandler(async () => {
       busy: Boolean(control?.yue2?.busy),
       backend: control?.yue2?.backend || 'yue2'
     },
+    caption: {
+      configured: Boolean(control?.caption?.configured),
+      busy: Boolean(control?.caption?.busy),
+      backend: control?.caption?.backend || 'llama.cpp'
+    },
     imageComfy: {
       configured: imageConfigured,
       ok: imageOk,

server/api/studio-queue.get.ts

@@ -10,13 +10,14 @@ export default defineEventHandler((event) => {
       || job.status === 'running'
       || job.status === 'held'
       || (job.status === 'error' && Date.now() - job.updatedAt < 60 * 60 * 1000)
+      || (job.kind === 'caption' && job.status === 'complete' && Date.now() - job.updatedAt < 60 * 60 * 1000)
     ))
   const claimed = new Set(active.map(job => job.shotQueueId).filter(Boolean) as string[])
   const queues = listShotQueues(owner)
   const byId = new Map(queues.map(queue => [queue.id, queue]))
   const jobs = active.map((job) => {
     const shots = job.shotQueueId ? byId.get(job.shotQueueId) || null : null
-    const kind = job.kind === 'edit' ? 'edit' : job.kind === 'music' ? 'music' : 'video'
+    const kind = job.kind === 'edit' ? 'edit' : job.kind === 'music' ? 'music' : job.kind === 'caption' ? 'caption' : 'video'
     const shotSummary = shots ? summarizeQueue(shots) : null
     const plannedShots = shotSummary
       ? undefined