towsty/aigen
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add image-to-text Describe on the bench via exclusive llama.cpp caption jobs.

Browse Source
This commit is contained in:
Towsty
2026-09-30 22:50:19 -05:00
parent 9481791ac5
commit 15ef891b26
15 changed files with 937 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/utils/studioQueue.ts
+61 -9
View File
@@ -13,7 +13,7 @@ import { imageV2StackSpecials } from '~/utils/imageV2'
import { allowIdentityRefs, type PermanenceRef } from '~/utils/globalLocks'
export type StudioJobStatus = 'waiting' | 'running' | 'held' | 'complete' | 'error' | 'cancelled'
export type StudioJobKind = 'video' | 'edit' | 'music'
export type StudioJobKind = 'video' | 'edit' | 'music' | 'caption'
export interface StudioJobPayload {
studio2Id?: string
@@ -42,6 +42,7 @@ export interface StudioJobPayload {
useIdentityRefs: boolean
stillId?: string
stillFilename?: string
captionStyle?: string
hideThumbnail: boolean
hideInput?: boolean
folderLocked?: boolean
@@ -105,6 +106,7 @@ export interface StudioJob {
resumeAutoRun?: boolean
lastError?: string
waitReason?: string
resultText?: string
}
type StudioQueueStore = {
@@ -226,6 +228,7 @@ export function listStudioJobs(owner: string) {
export function studioJobKind(job: Pick<StudioJob, 'kind'> | { kind?: string }) {
if (job.kind === 'edit') return 'edit'
if (job.kind === 'music') return 'music'
if (job.kind === 'caption') return 'caption'
return 'video'
}
@@ -245,6 +248,8 @@ export function summarizeStudioJob(job: StudioJob) {
folderId: job.payload.folderId,
musicEngine: job.payload.musicEngine,
stillId: job.payload.stillId,
captionStyle: job.payload.captionStyle,
resultText: job.resultText,
workflow: job.payload.workflow,
imagePipeline: job.payload.imagePipeline || 'v1',
duration: job.payload.duration,
@@ -290,7 +295,7 @@ function failZombieLiveJob(job: Job, error: string) {
function sweepStaleLiveJobs() {
const now = Date.now()
for (const job of listJobs()) {
if (job.studio2 || job.yueGp || job.yue2 || job.upscale) continue
if (job.studio2 || job.yueGp || job.yue2 || job.upscale || job.caption) continue
if (job.saving) continue
if (job.status === 'queued' && !job.promptId && now - job.startedAt >= QUEUED_GRACE_MS) {
failZombieLiveJob(job, 'Job never started')
@@ -310,12 +315,12 @@ async function reapZombieLiveJobs() {
const { fetchHistory } = await import('~/server/utils/comfy')
for (const job of listJobs()) {
// Never interrupt download/stitch/library save — Comfy is idle then by design.
if (job.studio2 || job.yueGp || job.yue2 || job.upscale) continue
if (job.studio2 || job.yueGp || job.yue2 || job.upscale || job.caption) continue
if (job.saving) continue
if (job.library?.chainContinuing) continue
if (job.status !== 'running' && job.status !== 'uploading' && job.status !== 'queued') continue
if (jobIsLocallySubmitting(job)) continue
const zombieMs = (job.kind === 'edit' || job.kind === 'music') ? 45_000 : ZOMBIE_EMPTY_COMFY_MS
const zombieMs = (job.kind === 'edit' || job.kind === 'music' || job.kind === 'caption') ? 45_000 : ZOMBIE_EMPTY_COMFY_MS
if (jobAgeMs(job) < zombieMs) continue
if (!job.promptId) {
if (jobAgeMs(job) >= QUEUED_GRACE_MS) failZombieLiveJob(job, 'Job never started on ComfyUI')
@@ -355,7 +360,7 @@ async function reapZombieLiveJobs() {
}
function liveJobOwnsGpu(job: Job) {
if ((job.studio2 || job.yueGp || job.yue2 || job.upscale) && ['running', 'queued', 'uploading'].includes(job.status)) return true
if ((job.studio2 || job.yueGp || job.yue2 || job.upscale || job.caption) && ['running', 'queued', 'uploading'].includes(job.status)) return true
if (job.library?.stopAfterCurrent) return false
if (job.library?.chainContinuing) return true
if (job.saving) return true
@@ -375,7 +380,7 @@ function liveJobOwnsGpu(job: Job) {
*/
function clearDeadGpuClaimsForForceStart() {
for (const live of listJobs()) {
if (live.studio2 || live.yueGp || live.yue2 || live.upscale) continue
if (live.studio2 || live.yueGp || live.yue2 || live.upscale || live.caption) continue
if (live.saving) continue
if (jobIsLocallySubmitting(live)) continue
if (live.status !== 'running' && live.status !== 'queued' && live.status !== 'uploading') {
@@ -469,8 +474,11 @@ export async function addStudioJob(params: {
kind?: StudioJobKind
}) {
const now = Date.now()
const kind = params.kind === 'edit' ? 'edit' : params.kind === 'music' ? 'music' : 'video'
const shotCount = kind === 'music'
const kind = params.kind === 'edit' ? 'edit'
: params.kind === 'music' ? 'music'
: params.kind === 'caption' ? 'caption'
: 'video'
const shotCount = kind === 'music' || kind === 'caption'
? 1
: kind === 'edit'
? 1 + (params.payload.passes?.length || 0)
@@ -606,6 +614,10 @@ export async function clearStuckStudioWork(owner: string) {
const { cancelYue2Job } = await import('./yue2')
await cancelYue2Job(job)
}
if (job.caption) {
const { cancelCaptionJob } = await import('./caption')
await cancelCaptionJob(job)
}
job.status = 'cancelled'
job.error = 'Cleared by force reset'
if (job.library) {
@@ -678,6 +690,11 @@ async function stopLiveGeneration(liveJobId?: string, shotQueueId?: string) {
await cancelYue2Job(live)
return
}
if (live?.caption) {
const { cancelCaptionJob } = await import('./caption')
await cancelCaptionJob(live)
return
}
if (live) {
live.status = 'cancelled'
if (live.library) {
@@ -889,6 +906,10 @@ function pendingAlive(job: StudioJob) {
const root = join(String(useRuntimeConfig().libraryDir || process.env.LIBRARY_DIR || '/data/library'), 'yue2-pending', `${job.liveJobId}.json`)
if (existsSync(root)) return true
}
if (job.kind === 'caption') {
const root = join(String(useRuntimeConfig().libraryDir || process.env.LIBRARY_DIR || '/data/library'), 'caption-pending', `${job.liveJobId}.json`)
if (existsSync(root)) return true
}
}
if (!job.shotQueueId) return false
return listPendingJobs().some(pending => (
@@ -1397,6 +1418,32 @@ async function startStudioExtendJob(item: StudioJob) {
}
}
async function startStudioCaptionJob(item: StudioJob) {
let live: import('~/server/utils/jobs').Job | undefined
try {
const { startCaptionJob } = await import('./caption')
const style = String(item.payload.captionStyle || 'descriptive')
const stillId = String(item.payload.stillId || '')
if (!stillId) throw new Error('Caption requires a still.')
live = await startCaptionJob({
ownerKey: item.ownerKey,
folderId: item.payload.folderId,
stillId,
captionStyle: style,
name: item.payload.name
})
await markStudioLive(item.ownerKey, item.id, live.id)
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
if (live && (live.status === 'queued' || live.status === 'uploading' || live.status === 'running')) {
live.status = 'error'
live.error = message
}
await parkStudioOnStartFailure(item.ownerKey, item.id, message)
kickStudioQueue()
}
}
async function startStudioMusicJob(item: StudioJob) {
let live: import('~/server/utils/jobs').Job | undefined
try {
@@ -1439,6 +1486,10 @@ export async function startStudioJob(item: StudioJob) {
}
async function startStudioJobReserved(item: StudioJob) {
if (studioJobKind(item) === 'caption') {
await startStudioCaptionJob(item)
return
}
if (item.payload.studio2Id) {
try {
const { startStudio2Job } = await import('./studio2/runner')
@@ -1703,7 +1754,7 @@ export async function onLiveVideoSettled(job: Job) {
return
}
const remaining = remainingStudioShots(job)
const wakeFail = !job.studio2 && !job.yueGp && !job.yue2 && !job.upscale && job.status === 'error' && remaining > 0 && isTransientComfyError(job.error)
const wakeFail = !job.studio2 && !job.yueGp && !job.yue2 && !job.upscale && !job.caption && job.status === 'error' && remaining > 0 && isTransientComfyError(job.error)
const failed = (job.status === 'error' || job.status === 'cancelled') && !wakeFail
await mutateStore(owner, (store) => {
@@ -1777,6 +1828,7 @@ export async function onLiveVideoSettled(job: Job) {
: failed
? 'error'
: 'complete'
if (job.caption && job.resultText) row.resultText = job.resultText
clearStudioRowSlot(row, status, failed ? job.error : undefined)
syncPausedFlag(store)
})