diff --git a/docs/qwen21.md b/docs/qwen21.md index 65fdf5b..ee30044 100644 --- a/docs/qwen21.md +++ b/docs/qwen21.md @@ -35,7 +35,7 @@ Restart Comfy **only when idle** (`COMFY_CONTROL_URL/status` → `gpu.busy=false - Engine key: `qwen21` · UI label: **Qwen 2.1** - **Generate** (T2I) and **Edit** (same checkpoint, second graph). Compose / Iterate / Video / Extend / Music stay disabled unless a graph exists. -- Edit requires Start still → `images.image_1`; prompt slots use `` / ``. If the user omits ``, the runner prepends `Keep the identity, face, body, and pose of the person in . Apply only this change:` then their text. Hero is Klein-only and is ignored (never image_1). +- Edit maps Photos roles → graph sockets: Photo to change → `images.image_1`, Outfit / object or Extra → `image_2`. The inspector never shows ``; the runner injects tags into the TextEncode string on submit (and expands Mention phrases like “this photo” / “the outfit photo”). If `` is still missing, prepend the keep-identity stanza. Klein hero-ref is off for Qwen. - Face / outfit locks stay Klein semantics — they do not drive Qwen slots. - Sampler defaults: euler / simple / cfg **1** / steps **25**. Edit uses `QwenImage21Cache` (device auto, dtype int8). T2I uses `ModelSamplingAuraFlow` shift **3.1**. - Default Generate canvas follows the bench Aspect control on the Qwen-safe ~1 MP grid (`EmptyLatentImage`): 1:1 → 1024×1024, 16:9 → 1536×864, 9:16 → 864×1536 (and the other table rows). PE `wh_ratio` is advisory only and never sizes the canvas. Do not use native 2K bins on 16 GB. diff --git a/server/utils/studio2/runner.ts b/server/utils/studio2/runner.ts index dc61274..85c9c21 100644 --- a/server/utils/studio2/runner.ts +++ b/server/utils/studio2/runner.ts @@ -18,6 +18,7 @@ import { nativeVideoGraph, attachHeroReference, applyResolvedImageSize } from '~ import { compilePrompt, scopedFile } from '~/shared/studio2/contracts.mjs'; import { resolveQwen21Size } from '~/shared/studio2/qwen21-size.mjs'; import { ensureQwen21EditPrompt, stitchQwen21PeEditPrompt } from '~/shared/studio2/qwen21-edit.mjs'; +import { applyPhotosToRequest, injectQwenEditMentionTags } from '~/shared/studio2/photos.mjs'; import { qwen21PeRefusal } from '~/shared/studio2/qwen21-pe.mjs'; import { QWEN21_TURBO_LORA, QWEN21_TURBO_SIGMAS, QWEN21_TURBO_STEPS } from '~/shared/studio2/qwen21-turbo.mjs'; import { createJob, restoreJob, getJob, emitJob, type Job } from '../jobs'; @@ -96,25 +97,38 @@ async function waitPromptHistory(r: any, job: Job, promptId: string) { return history } +function finalizeQwen21EditSample(rawPrompt: string, request: any) { + const photos = Array.isArray(request?.photos) ? request.photos : [] + const tagged = injectQwenEditMentionTags(String(rawPrompt || ''), photos) + return ensureQwen21EditPrompt(tagged, { hasImage2: !!request?.imageBId }) +} + async function runQwen21PromptEnhance(r: any, job: Job) { const q = r.request, s = q.settings if (!q.enhancePrompt || q.engine !== 'qwen21' || !['generate', 'edit'].includes(q.mode)) return const edit = q.mode === 'edit' + if (Array.isArray(q.photos) && q.photos.length) { + const bound = applyPhotosToRequest(q) + q.imageAId = bound.imageAId + q.imageBId = bound.imageBId + q.identityStillId = bound.identityStillId + q.photos = bound.photos + } // Edit → PE-I2I only (pe_i2i). Generate → PE-T2I only. Never cross-feed. if (edit && !q.imageAId) - throw new Error('Qwen Edit requires Start still (image_1). Hero is ignored.') + throw new Error('Add a photo and mark it Photo to change.') update(r, 'enhancing') emitJob(job, { type: 'status', message: 'Enhancing prompt…' }) // promptRaw = exactly what they typed (compiled from sections before PE). const userPrompt = String(q.compiledPrompt || '') r.request.promptRaw = userPrompt - // Edit PE gets the raw edit instruction + Start still — not the keep stanza. + // Edit PE gets the raw edit instruction + canvas still — not the keep stanza. const peInput = userPrompt const graph = structuredClone(edit ? qwen21PeEditTemplate : qwen21PeT2iTemplate) graph['9'].inputs.prompt = peInput graph['9'].inputs.seed = s.seed if (edit) { - // Same Start still pixels as Edit image_1 — never Hero. + // Photo to change → image_1. Outfit/extra → image_2 when present. const a = await upload(r, 'pe-source', readFileSync(stillPath(r.owner, getStill(r.owner, q.imageAId).id))) graph['10'].inputs.image = a graph['9'].inputs.image_1 = ['10', 0] @@ -143,7 +157,7 @@ async function runQwen21PromptEnhance(r: any, job: Job) { const refusal = qwen21PeRefusal(userPrompt, result) if (refusal.refused) { // Fail open: keep the job, send the raw brief (Edit keep-identity stanza still applies). - const compiled = edit ? ensureQwen21EditPrompt(userPrompt) : userPrompt + const compiled = edit ? finalizeQwen21EditSample(userPrompt, q) : userPrompt r.request.compiledPrompt = compiled r.request.prompt = compiled r.request.enhance = { @@ -160,8 +174,9 @@ async function runQwen21PromptEnhance(r: any, job: Job) { // Edit only: stanza + raw first; PE chunk only if it is an edit directive (never PE-alone). // Generate keeps the long observer paragraph as the product — no describe sanitize. const stitched = stitchQwen21PeEditPrompt(userPrompt, result.positive_prompt) - r.request.compiledPrompt = stitched.prompt - r.request.prompt = stitched.prompt + const compiled = finalizeQwen21EditSample(stitched.prompt, q) + r.request.compiledPrompt = compiled + r.request.prompt = compiled r.request.enhance = { used: !stitched.skippedAsDescribe, wh_ratio: result.wh_ratio || '', @@ -223,10 +238,18 @@ async function upload(r: any, name: string, data: Buffer): Promise { } async function prepareGraph(r: any) { const q = r.request, s = q.settings, prefix = String(useRuntimeConfig().comfyFilenamePrefix).replace(/\/$/, '') + `/studio2/${r.id}/${r.index}`; + // Photos roles → graph sockets (imageAId / imageBId / identityStillId). Payload is source of truth. + if (Array.isArray(q.photos) && q.photos.length) { + const bound = applyPhotosToRequest(q) + q.imageAId = bound.imageAId + q.imageBId = bound.imageBId + q.identityStillId = bound.identityStillId + q.photos = bound.photos + } const load = async (id: string, label: string) => id ? upload(r, label, readFileSync(stillPath(r.owner, getStill(r.owner, id).id))) : ''; const video=['video','extend'].includes(q.mode); const identityPlan=kleinIdentityPlan(q); - // Qwen Edit: Start still (imageAId) only — never Hero / identityStillId. + // Qwen Edit: Photo to change (imageAId) → image_1. Never Klein hero-ref. const sourceId = q.engine === 'qwen21' ? (q.imageAId || '') : (identityPlan ? identityPlan.sourceId : q.imageAId); @@ -304,12 +327,12 @@ async function prepareGraph(r: any) { } const negative = turbo ? '' : stylePrompt(q.imageStyles, true) if (q.mode === 'edit') { - // Start still → image_1 only. Never Hero. Never T2I / EmptyLatent. + // Photo to change → image_1. Outfit/extra → image_2. Never T2I / EmptyLatent. if (!q.imageAId || !a) - throw new Error('Qwen Edit requires Start still (image_1). Hero is ignored.') - // promptRaw stays typed; prompt/compiledPrompt = what TextEncode samples. + throw new Error('Add a photo and mark it Photo to change.') + // promptRaw stays typed; prompt/compiledPrompt = what TextEncode samples (tags injected silently). if (q.promptRaw == null) q.promptRaw = String(q.compiledPrompt || '') - const prompt = ensureQwen21EditPrompt(String(q.compiledPrompt || '')) + const prompt = finalizeQwen21EditSample(String(q.compiledPrompt || ''), q) q.compiledPrompt = prompt q.prompt = prompt q.task = 'edit' diff --git a/shared/studio2/qwen21-edit.mjs b/shared/studio2/qwen21-edit.mjs index e9ef749..c90f33b 100644 --- a/shared/studio2/qwen21-edit.mjs +++ b/shared/studio2/qwen21-edit.mjs @@ -20,11 +20,16 @@ export function buildQwen21EditKeepStanza(rawInstruction) { } /** Enhance off / refusal: ensure the sample prompt is the keep stanza, never PE-only. */ -export function ensureQwen21EditPrompt(prompt) { +export function ensureQwen21EditPrompt(prompt, options = {}) { const text = String(prompt || '').trim() if (!text) return text - if (/Keep the identity, face, body, pose, and likeness of the person in /i.test(text)) return text - return buildQwen21EditKeepStanza(text) + let next = /Keep the identity, face, body, pose, and likeness of the person in /i.test(text) + ? text + : buildQwen21EditKeepStanza(text) + if (options.hasImage2 && !//i.test(next)) { + next = `${next}\nUse the garment or object in .`.trim() + } + return next } /** PE rewrite looks like a T2I observer caption, not an edit directive. */ diff --git a/tests/studio2-bench.test.mjs b/tests/studio2-bench.test.mjs index 83d5c00..e7f4316 100644 --- a/tests/studio2-bench.test.mjs +++ b/tests/studio2-bench.test.mjs @@ -9,6 +9,7 @@ import {compilePrompt,validateRequest} from '../shared/studio2/contracts.mjs' import {cachedLoras} from '../shared/studio2/lora-cache.mjs' import {resolveQwen21Size} from '../shared/studio2/qwen21-size.mjs' import {ensureQwen21EditPrompt,stitchQwen21PeEditPrompt,QWEN21_EDIT_KEEP_CHANGE} from '../shared/studio2/qwen21-edit.mjs' +import {injectQwenEditMentionTags} from '../shared/studio2/photos.mjs' import {qwen21PeRefusal} from '../shared/studio2/qwen21-pe.mjs' test('deleted library files never return from job outputs; selection falls back or clears',()=>{ @@ -112,7 +113,10 @@ test('qwen21 supports Generate and Edit with 25/1 defaults and no locks',()=>{ assert.throws(()=>validateRequest({mode:'iterate',engine:'qwen21',folderId:'f',imageAId:'s',promptSections:{action:'x'}}),/Generate and Edit/) assert.equal(ensureQwen21EditPrompt('put her in a red leather jacket'),`${QWEN21_EDIT_KEEP_CHANGE} put her in a red leather jacket`) assert.match(ensureQwen21EditPrompt('put her in a red leather jacket'),//) + assert.match(ensureQwen21EditPrompt('put her in that dress',{hasImage2:true}),//) assert.equal(ensureQwen21EditPrompt('Keep the identity, face, body, pose, and likeness of the person in .\nDo not invent a different person.\nApply only this change: x'),'Keep the identity, face, body, pose, and likeness of the person in .\nDo not invent a different person.\nApply only this change: x') + assert.match(injectQwenEditMentionTags('Put her in the outfit photo',[{stillId:'a',role:'change'},{stillId:'b',role:'outfit'}]),//) + assert.match(injectQwenEditMentionTags('Change this photo',[{stillId:'a',role:'change'}]),//) const stitchedKeep=stitchQwen21PeEditPrompt('put her in a red leather jacket','Change the outfit on the person in to a red leather jacket') assert.match(stitchedKeep.prompt,/Keep the identity/) assert.match(stitchedKeep.prompt,//) @@ -192,6 +196,8 @@ test('custom negative chips survive normalizeStyles and bench exposes Load actio const runner=readFileSync(new URL('../server/utils/studio2/runner.ts',import.meta.url),'utf8') assert.match(runner,/ensureQwen21EditPrompt/) assert.match(runner,/stitchQwen21PeEditPrompt/) + assert.match(runner,/injectQwenEditMentionTags/) + assert.match(runner,/applyPhotosToRequest/) assert.match(runner,/skippedAsDescribe/) assert.match(runner,/promptRaw/) assert.match(runner,/images\.image_1/)