towsty/aigen
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Wire Qwen Enhance prompt toggle through Studio 2 and the runner.

Browse Source 
Queue PE first when enabled, fail closed on parse_ok, free VRAM, then run the existing T2I or Edit graph with the rewrite.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-09-26 10:25:44 -05:00
co-authored by Cursor
parent e0292c8d2a
commit 2842855c7b
9 changed files with 153 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/utils/studio2/runner.ts
+110 -40
View File
@@ -9,7 +9,9 @@ import { readFileSync, writeFileSync, mkdirSync, existsSync, unlinkSync } from '
import { join } from 'node:path';
import template from '../../assets/studio2_minimax_native.json';
import qwen21Template from '../../assets/studio2_qwen21_t2i.json'
import qwen21EditTemplate from '../../assets/studio2_qwen21_edit.json';
import qwen21EditTemplate from '../../assets/studio2_qwen21_edit.json'
import qwen21PeT2iTemplate from '../../assets/studio2_qwen21_pe_t2i.json'
import qwen21PeEditTemplate from '../../assets/studio2_qwen21_pe_edit.json';
import { nativeVideoGraph, attachHeroReference, applyResolvedImageSize } from '~/shared/studio2/graphs.mjs';
import { compilePrompt, scopedFile } from '~/shared/studio2/contracts.mjs';
import { resolveQwen21Size } from '~/shared/studio2/input.mjs';
@@ -17,7 +19,7 @@ import { createJob, restoreJob, getJob, emitJob, type Job } from '../jobs';
import { markStudioLive, onLiveVideoSettled, type StudioJob } from '../studioQueue';
import { readRecord, saveRecord, studio2Root, records } from './store';
import { ensureComfyReady } from '../comfyLifecycle';
import { comfyFetch, queuePrompt, fetchHistory, extractVideo } from '../comfy';
import { comfyFetch, queuePrompt, fetchHistory, extractVideo, freeComfyVram } from '../comfy';
import { extractEditedImage } from '../imageComfy';
import { buildImageV2Workflow, resolveKreaGenerateAssets } from '../imageWorkflowV2';
import { resolveGraphLoraNames, applyUserLoraToGraph, ensureComfyLoraNames } from '../loras';
@@ -37,6 +39,107 @@ function update(r: any, state: string) { r.state = state; r.updatedAt = Date.now
// extend/video start frame (which is either the exact anchor PNG or an already-canvas-sized
// extracted last frame). Fitting these every hop is what destroys identity across chained extends.
const NEVER_FIT = new Set(['hero', 'start']);
function previewAnyText(outputs: Record<string, any> | undefined, nodeId: string): string {
const node = outputs?.[nodeId]
if (!node || typeof node !== 'object') return ''
const raw = node.text ?? node.STRING ?? node.source
if (Array.isArray(raw)) return String(raw[0] ?? '')
if (raw == null) return ''
return String(raw)
}
function harvestQwen21Pe(history: Record<string, any> | null | undefined, promptId: string, edit: boolean) {
const outputs = history?.[promptId]?.outputs as Record<string, any> | undefined
const positive_prompt = previewAnyText(outputs, '30').trim()
const wh_ratio = previewAnyText(outputs, '31').trim()
const ratio_follow = edit ? previewAnyText(outputs, '32').trim() : ''
const thinking = previewAnyText(outputs, edit ? '33' : '32').trim()
const parseRaw = previewAnyText(outputs, edit ? '34' : '33').trim()
const parse_ok = /^(true|1|yes)$/i.test(parseRaw)
return { positive_prompt, wh_ratio, ratio_follow, thinking, parse_ok }
}
async function waitPromptHistory(r: any, job: Job, promptId: string) {
let history: any = null, missingSince = 0
for (;;) {
try { history = await fetchHistory(promptId) }
catch { history = null }
const e = history?.[promptId]
if (job.status === 'cancelled') throw new Error('Cancelled')
if (['error', 'interrupted'].includes(e?.status?.status_str))
throw new Error('Comfy generation failed or was interrupted; inspect the host error log.')
if (!e) {
try {
const response = await comfyFetch('/queue', { signal: AbortSignal.timeout(5000) })
if (response.ok) {
const q = await response.json() as any
const present = [...(q.queue_running || []), ...(q.queue_pending || [])].some((entry: any) => entry[1] === promptId)
if (present) missingSince = 0
else if (!missingSince) missingSince = Date.now()
else if (Date.now() - missingSince > 60000) throw new Error('PROMPT_MISSING')
}
} catch (error: any) {
if (error.message === 'PROMPT_MISSING')
throw new Error('The saved Comfy prompt is no longer in history or the queue. It was not resubmitted.')
}
}
if (e?.status?.completed || e?.status?.status_str === 'success') break
if (job.status === 'cancelled') throw new Error('Cancelled')
await new Promise(resolve => setTimeout(resolve, 1500))
}
return history
}
async function runQwen21PromptEnhance(r: any, job: Job) {
const q = r.request, s = q.settings
if (!q.enhancePrompt || q.engine !== 'qwen21' || !['generate', 'edit'].includes(q.mode)) return
const edit = q.mode === 'edit'
if (edit && !q.imageAId) throw new Error('Choose the source still.')
update(r, 'enhancing')
emitJob(job, { type: 'status', message: 'Enhancing prompt…' })
let prompt = String(q.compiledPrompt || '')
if (edit && !/<image1>/i.test(prompt))
prompt = `Keep the subject in <image1>. ${prompt}`.trim()
const graph = structuredClone(edit ? qwen21PeEditTemplate : qwen21PeT2iTemplate)
graph['9'].inputs.prompt = prompt
graph['9'].inputs.seed = s.seed
if (edit) {
const a = await upload(r, 'pe-source', readFileSync(stillPath(r.owner, getStill(r.owner, q.imageAId).id)))
graph['10'].inputs.image = a
graph['9'].inputs.image_1 = ['10', 0]
if (q.imageBId) {
const b = await upload(r, 'pe-compose', readFileSync(stillPath(r.owner, getStill(r.owner, q.imageBId).id)))
graph['11'] = {
inputs: { image: b },
class_type: 'LoadImage',
_meta: { title: 'image_2 optional' }
}
graph['9'].inputs.image_2 = ['11', 0]
}
}
r.peGraphId = edit ? 'studio2_qwen21_pe_edit.json' : 'studio2_qwen21_pe_t2i.json'
saveRecord(r)
const queued = await queuePrompt(graph, job.clientId)
r.pePromptId = queued.prompt_id
saveRecord(r)
const history = await waitPromptHistory(r, job, queued.prompt_id)
const result = harvestQwen21Pe(history, queued.prompt_id, edit)
if (!result.parse_ok || !result.positive_prompt)
throw new Error('Prompt enhance failed to parse a rewrite. Try again or turn Enhance prompt off.')
r.request.promptRaw = q.compiledPrompt
r.request.compiledPrompt = result.positive_prompt
r.request.enhance = {
wh_ratio: result.wh_ratio || '',
...(edit ? { ratio_follow: result.ratio_follow || '' } : {}),
parse_ok: true,
thinking: result.thinking || ''
}
r.pePromptId = ''
saveRecord(r)
await freeComfyVram()
}
async function upload(r: any, name: string, data: Buffer): Promise<string> {
const prefix = String(useRuntimeConfig().comfyFilenamePrefix).replace(/\/$/, '') + `/studio2/${r.id}/${r.index}`;
let fitted = false;
@@ -234,56 +337,23 @@ async function run(r: any, job: Job) {
r.request.compiledPrompt = compilePrompt(prompts[r.index], ['video', 'extend'].includes(r.request.mode), r.request);
update(r, 'waking');
await ensureComfyReady(message => emitJob(job, { type: 'status', message }));
if (job.status === 'cancelled')
throw new Error('Cancelled');
await runQwen21PromptEnhance(r, job);
if (job.status === 'cancelled')
throw new Error('Cancelled');
const graph = await prepareGraph(r);
if (job.status === 'cancelled')
throw new Error('Cancelled');
update(r, 'submitting');
emitJob(job, { type: 'status', message: 'Generating…' });
const queued = await queuePrompt(graph, job.clientId);
job.promptId = queued.prompt_id;
r.promptId = queued.prompt_id;
r.renderStartedAt = Date.now();
update(r, 'rendering');
}
let history: any, missingSince = 0;
for (;;) {
try {
history = await fetchHistory(r.promptId);
}
catch {
history = null;
}
const e = history?.[r.promptId];
if (job.status === 'cancelled')
throw new Error('Cancelled');
if (['error', 'interrupted'].includes(e?.status?.status_str))
throw new Error('Comfy generation failed or was interrupted; inspect the host error log.');
if (!e) {
try {
const response = await comfyFetch('/queue', { signal: AbortSignal.timeout(5000) });
if (response.ok) {
const q = await response.json() as any;
const present = [...(q.queue_running || []), ...(q.queue_pending || [])].some((entry: any) => entry[1] === r.promptId);
if (present)
missingSince = 0;
else if (!missingSince)
missingSince = Date.now();
else if (Date.now() - missingSince > 60000)
throw new Error('PROMPT_MISSING');
}
}
catch (error: any) {
if (error.message === 'PROMPT_MISSING')
throw new Error('The saved Comfy prompt is no longer in history or the queue. It was not resubmitted.');
}
}
if (e?.status?.completed || e?.status?.status_str === 'success')
break;
if (job.status === 'cancelled')
throw new Error('Cancelled');
await new Promise(resolve => setTimeout(resolve, 1500));
}
const history = await waitPromptHistory(r, job, r.promptId);
const messages = history[r.promptId]?.status?.messages || [];
const executionStart = messages.find((m: any) => m[0] === 'execution_start')?.[1]?.timestamp;
const executionEnd = messages.find((m: any) => m[0] === 'execution_success')?.[1]?.timestamp;