Add extension-frame refine and lossless anchor options.

Prefer clip anchor PNGs for handoffs, optionally run Bilateral+Flux cleanup before I2V, and save decoded last-frame anchors so sequential extensions accumulate less compression damage.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Towsty
2026-09-04 07:55:46 -05:00
co-authored by Cursor
parent 5917d5bb37
commit f41ac7a826
14 changed files with 674 additions and 14 deletions
+89 -1
View File
@@ -1109,6 +1109,41 @@
<span class="mt-0.5 block text-xs text-zinc-500">Overnight set-and-forget. Leave this off to edit prompts on Queue (under this job) and process the next shot when you are ready.</span>
</span>
</label>
<div v-if="studioMode === 'video'" class="space-y-3">
<label
class="flex cursor-pointer items-start gap-3 rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3"
title="Runs a low-denoise Bilateral + Flux cleanup pass on the handoff frame to strip compression artifacts."
>
<input v-model="refineExtensionFrame" type="checkbox" class="mt-1">
<span>
<span class="block text-sm font-medium text-zinc-200">Refine Extension Frame</span>
<span class="mt-0.5 block text-xs text-zinc-500">Runs a low-denoise Bilateral + Flux cleanup pass on the handoff frame to strip compression artifacts.</span>
</span>
</label>
<div
v-if="refineExtensionFrame"
class="rounded-2xl border border-white/10 bg-zinc-950/40 p-3"
>
<Thumbwheel
v-model="refinementDenoise"
label="Refinement Denoise"
:min="REFINEMENT_DENOISE_MIN"
:max="REFINEMENT_DENOISE_MAX"
:step="0.01"
:digits="2"
/>
</div>
<label
class="flex cursor-pointer items-start gap-3 rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3"
title="Saves an uncompressed PNG of the final frame alongside the MP4 for cleaner future extensions."
>
<input v-model="saveLosslessAnchor" type="checkbox" class="mt-1">
<span>
<span class="block text-sm font-medium text-zinc-200">Save Lossless Anchor Frame</span>
<span class="mt-0.5 block text-xs text-zinc-500">Saves an uncompressed PNG of the final frame alongside the MP4 for cleaner future extensions.</span>
</span>
</label>
</div>
<div class="flex flex-wrap gap-3">
<button
type="button"
@@ -1458,6 +1493,38 @@
@insert-trigger="insertLoraTrigger($event, 'extend')"
@update-trigger="saveLoraTrigger"
/>
<div class="space-y-3">
<label
class="flex cursor-pointer items-start gap-3 rounded-xl border border-white/10 bg-zinc-950/40 px-3 py-2.5"
title="Runs a low-denoise Bilateral + Flux cleanup pass on the handoff frame to strip compression artifacts."
>
<input v-model="refineExtensionFrame" type="checkbox" class="mt-1">
<span>
<span class="block text-sm font-medium text-zinc-200">Refine Extension Frame</span>
<span class="mt-0.5 block text-xs text-zinc-500">Bilateral + Flux cleanup on the handoff frame.</span>
</span>
</label>
<div v-if="refineExtensionFrame" class="rounded-xl border border-white/10 bg-zinc-950/40 p-3">
<Thumbwheel
v-model="refinementDenoise"
label="Refinement Denoise"
:min="REFINEMENT_DENOISE_MIN"
:max="REFINEMENT_DENOISE_MAX"
:step="0.01"
:digits="2"
/>
</div>
<label
class="flex cursor-pointer items-start gap-3 rounded-xl border border-white/10 bg-zinc-950/40 px-3 py-2.5"
title="Saves an uncompressed PNG of the final frame alongside the MP4 for cleaner future extensions."
>
<input v-model="saveLosslessAnchor" type="checkbox" class="mt-1">
<span>
<span class="block text-sm font-medium text-zinc-200">Save Lossless Anchor Frame</span>
<span class="mt-0.5 block text-xs text-zinc-500">Keep a PNG anchor with this extension for cleaner next hops.</span>
</span>
</label>
</div>
<button
type="button"
class="w-full rounded-2xl bg-amber-400 px-4 py-2.5 text-sm font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50"
@@ -2324,6 +2391,12 @@ import {
defaultVideoDurationPref,
setDefaultVideoDurationPref
} from '~/utils/videoDuration'
import {
clampRefinementDenoise,
REFINEMENT_DENOISE_DEFAULT,
REFINEMENT_DENOISE_MAX,
REFINEMENT_DENOISE_MIN
} from '~/utils/extensionFrame'
const runtimeConfig = useRuntimeConfig()
const route = useRoute()
@@ -2758,6 +2831,9 @@ const renameDraft = ref('')
const renameInput = ref<HTMLInputElement | null>(null)
const unlockPassword = ref('')
const queueAutoRun = ref(false)
const refineExtensionFrame = ref(true)
const saveLosslessAnchor = ref(false)
const refinementDenoise = ref(REFINEMENT_DENOISE_DEFAULT)
const queuePendingCount = ref(0)
const activeQueueId = ref('')
const liveQueue = ref<{
@@ -5452,6 +5528,9 @@ async function restoreStudioPayload(kind: string, payload: {
referenceStillIds?: Array<string | null>
extensions?: { prompt: string; promptPre?: string; promptPost?: string; duration: number; permanenceRefs?: PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[]
queueAutoRun?: boolean
refineExtensionFrame?: boolean
saveLosslessAnchor?: boolean
refinementDenoise?: number
promptPre?: string
promptPost?: string
globalLocks?: string
@@ -5520,6 +5599,9 @@ async function restoreStudioPayload(kind: string, payload: {
readLoraStack(item)
))
queueAutoRun.value = payload.queueAutoRun === true
refineExtensionFrame.value = payload.refineExtensionFrame !== false
saveLosslessAnchor.value = payload.saveLosslessAnchor === true
refinementDenoise.value = clampRefinementDenoise(payload.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
globalLocks.value = payload.globalLocks || ''
familyPermanenceRefs.value = payload.permanenceRefs || []
const shotRefs: Record<number, PermanenceRef[]> = {}
@@ -6113,7 +6195,10 @@ async function generateExtension() {
promptPre: extendPromptPre.value,
promptPost: extendPromptPost.value,
duration: extendDuration.value,
loraStack: extendLoraStack.value
loraStack: extendLoraStack.value,
refineExtensionFrame: refineExtensionFrame.value,
saveLosslessAnchor: saveLosslessAnchor.value,
refinementDenoise: clampRefinementDenoise(refinementDenoise.value)
}
})
if (queuedOffLiveOutput(started, 'video')) {
@@ -7391,6 +7476,9 @@ async function generate() {
}))))
body.append('queueAutoRun', String(queueAutoRun.value))
}
body.append('refineExtensionFrame', String(refineExtensionFrame.value))
body.append('saveLosslessAnchor', String(saveLosslessAnchor.value))
body.append('refinementDenoise', String(clampRefinementDenoise(refinementDenoise.value)))
const downloadName = clipFileName(clipName.value.trim() || autoOutputName.value)
const chainPlan = plannedChain.value.map(part => ({ ...part }))
console.info('[aigen generate payload]', {
+21 -1
View File
@@ -6,9 +6,26 @@ import { clipPartDuration, clampVideoDuration } from '~/utils/videoDuration'
import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
import { composePromptParts } from '~/utils/promptParts'
import { nextFamilyPartName } from '~/server/utils/library'
import {
clampRefinementDenoise,
parseRefineExtensionFrame,
parseSaveLosslessAnchor,
REFINEMENT_DENOISE_DEFAULT
} from '~/utils/extensionFrame'
export default defineEventHandler(async (event) => {
const body = await readBody<{ clipId?: string; prompt?: string; promptPre?: string; promptPost?: string; duration?: number; lora?: unknown; loraStack?: unknown }>(event).catch(() => ({}))
const body = await readBody<{
clipId?: string
prompt?: string
promptPre?: string
promptPost?: string
duration?: number
lora?: unknown
loraStack?: unknown
refineExtensionFrame?: boolean | string
saveLosslessAnchor?: boolean | string
refinementDenoise?: number | string
}>(event).catch(() => ({}))
const clipId = String(body?.clipId || '').trim()
const prompt = String(body?.prompt || '').trim()
if (!clipId) {
@@ -87,6 +104,9 @@ export default defineEventHandler(async (event) => {
extensions: [],
queueAutoRun: false,
extendFromClipId: clipId,
refineExtensionFrame: parseRefineExtensionFrame(body?.refineExtensionFrame, true),
saveLosslessAnchor: parseSaveLosslessAnchor(body?.saveLosslessAnchor, false),
refinementDenoise: clampRefinementDenoise(body?.refinementDenoise, REFINEMENT_DENOISE_DEFAULT),
globalLocks: source.globalLocks,
permanenceRefs: source.permanenceRefs,
...persistLoraFields(loraStack)
+12
View File
@@ -6,6 +6,12 @@ import { defaultVideoSteps, isLtxWorkflow, isTextToVideo, LTX_DISABLED_MESSAGE,
import { allowIdentityRefs, normalizePermanenceRefs, resolveGlobalLocks, type PermanenceRef } from '~/utils/globalLocks'
import { composePromptParts, ingestPromptParts, persistPromptWrappers } from '~/utils/promptParts'
import type { QueuedExtension } from '~/server/utils/library'
import {
clampRefinementDenoise,
parseRefineExtensionFrame,
parseSaveLosslessAnchor,
REFINEMENT_DENOISE_DEFAULT
} from '~/utils/extensionFrame'
function parseDuration(raw: unknown) {
const seconds = Number(raw)
@@ -250,6 +256,9 @@ export default defineEventHandler(async (event) => {
const chainTotal = 1 + extensions.length
const familyId = crypto.randomUUID()
const queueAutoRun = fields.queueAutoRun === 'true'
const refineExtensionFrame = parseRefineExtensionFrame(fields.refineExtensionFrame, true)
const saveLosslessAnchor = parseSaveLosslessAnchor(fields.saveLosslessAnchor, false)
const refinementDenoise = clampRefinementDenoise(fields.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
const studio = await addStudioJob({
ownerKey,
familyId,
@@ -282,6 +291,9 @@ export default defineEventHandler(async (event) => {
referenceStillIds,
extensions,
queueAutoRun,
refineExtensionFrame,
saveLosslessAnchor,
refinementDenoise,
globalLocks: globalLocks || undefined,
permanenceRefs: permanenceRefs.length ? permanenceRefs : undefined,
shotPermanenceRefs: shotPermanenceRefs.some(item => item.length) ? shotPermanenceRefs : undefined,
+165
View File
@@ -0,0 +1,165 @@
{
"1": {
"inputs": { "image": "last_frame.png" },
"class_type": "LoadImage",
"_meta": { "title": "Handoff Frame" }
},
"2": {
"inputs": {
"images": ["1", 0],
"size": 2,
"sigma_color": 0.12,
"sigma_space": 1.5
},
"class_type": "BilateralFilterImage",
"_meta": { "title": "Bilateral Deblock" }
},
"3": {
"inputs": {
"unet_name": "flux-2-klein-base-9b-fp8.safetensors",
"weight_dtype": "default"
},
"class_type": "UNETLoader",
"_meta": { "title": "Load Flux.2 Klein" }
},
"4": {
"inputs": {
"clip_name": "qwen_3_8b_fp8mixed.safetensors",
"type": "flux2",
"device": "default"
},
"class_type": "CLIPLoader",
"_meta": { "title": "Load CLIP" }
},
"5": {
"inputs": { "vae_name": "full_encoder_small_decoder.safetensors" },
"class_type": "VAELoader",
"_meta": { "title": "Load Klein VAE" }
},
"6": {
"inputs": {
"lora_name": "Flux2-Klein-9B-consistency-V2.safetensors",
"strength_model": 0.55,
"strength_clip": 0.55,
"model": ["3", 0],
"clip": ["4", 0]
},
"class_type": "LoraLoader",
"_meta": { "title": "Consistency" }
},
"7": {
"inputs": {
"text": "photoreal still frame, natural high-frequency texture, clean edges, no compression blocks, no mosquito noise",
"clip": ["6", 1]
},
"class_type": "CLIPTextEncode",
"_meta": { "title": "Positive" }
},
"8": {
"inputs": {
"text": "compression artifacts, blockiness, banding, blur, smear, plastic skin",
"clip": ["6", 1]
},
"class_type": "CLIPTextEncode",
"_meta": { "title": "Negative" }
},
"9": {
"inputs": {
"guidance": 1.8,
"conditioning": ["7", 0]
},
"class_type": "FluxGuidance",
"_meta": { "title": "Flux Guidance" }
},
"10": {
"inputs": {
"pixels": ["2", 0],
"vae": ["5", 0]
},
"class_type": "VAEEncode",
"_meta": { "title": "Encode Filtered" }
},
"11": {
"inputs": {
"conditioning": ["9", 0],
"latent": ["10", 0]
},
"class_type": "ReferenceLatent",
"_meta": { "title": "Reference +" }
},
"12": {
"inputs": {
"conditioning": ["8", 0],
"latent": ["10", 0]
},
"class_type": "ReferenceLatent",
"_meta": { "title": "Reference -" }
},
"13": {
"inputs": { "noise_seed": 1 },
"class_type": "RandomNoise",
"_meta": { "title": "Noise" }
},
"14": {
"inputs": { "sampler_name": "euler" },
"class_type": "KSamplerSelect",
"_meta": { "title": "Sampler" }
},
"15": {
"inputs": {
"scheduler": "simple",
"steps": 14,
"denoise": 0.18,
"model": ["6", 0]
},
"class_type": "BasicScheduler",
"_meta": { "title": "Scheduler" }
},
"16": {
"inputs": {
"cfg": 1,
"model": ["6", 0],
"positive": ["11", 0],
"negative": ["12", 0]
},
"class_type": "CFGGuider",
"_meta": { "title": "CFG 1.0" }
},
"17": {
"inputs": {
"noise": ["13", 0],
"guider": ["16", 0],
"sampler": ["14", 0],
"sigmas": ["15", 0],
"latent_image": ["10", 0]
},
"class_type": "SamplerCustomAdvanced",
"_meta": { "title": "Img2Img Cleanup" }
},
"18": {
"inputs": {
"samples": ["17", 0],
"vae": ["5", 0]
},
"class_type": "VAEDecode",
"_meta": { "title": "Decode" }
},
"19": {
"inputs": {
"image1": ["1", 0],
"image2": ["18", 0],
"blend_factor": 0.85,
"blend_mode": "normal"
},
"class_type": "ImageBlend",
"_meta": { "title": "Blend 0.85" }
},
"20": {
"inputs": {
"filename_prefix": "aigen/extension_refine",
"images": ["19", 0]
},
"class_type": "SaveImage",
"_meta": { "title": "Save Refined Frame" }
}
}
+65 -6
View File
@@ -1,4 +1,4 @@
import { copyFileSync, existsSync, readFileSync } from 'node:fs'
import { copyFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { composeShotPrompt } from '~/utils/globalLocks'
import { composePromptParts } from '~/utils/promptParts'
@@ -16,12 +16,22 @@ import {
nextFamilyPartName,
removeExtendTemp
} from '~/server/utils/library'
import { extractLastFrame, probeHasAudio } from '~/server/utils/ffmpeg'
import { probeHasAudio } from '~/server/utils/ffmpeg'
import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
import { waitForComfySocket, watchComfyJob } from '~/server/utils/watch'
import { comfyFilenamePrefix, queuePrompt, uploadImage } from '~/server/utils/comfy'
import { buildWorkflow } from '~/server/utils/workflow'
import { writePendingJob } from '~/server/utils/pending'
import {
refineExtensionHandoffFrame,
resolveExtensionHandoffFrame
} from '~/server/utils/extensionFrame'
import {
clampRefinementDenoise,
parseRefineExtensionFrame,
parseSaveLosslessAnchor,
REFINEMENT_DENOISE_DEFAULT
} from '~/utils/extensionFrame'
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
@@ -63,6 +73,9 @@ export async function beginExtendFromClip(params: {
loraStack?: unknown
folderLocked?: boolean
name?: string
refineExtensionFrame?: boolean
saveLosslessAnchor?: boolean
refinementDenoise?: number
}) {
const source = getClip(params.ownerKey, params.clipId)
const sourcePath = clipVideoPath(params.ownerKey, source.id)
@@ -111,6 +124,9 @@ export async function beginExtendFromClip(params: {
const width = source.width || 1344
const height = source.height || 768
const clipName = String(params.name || '').trim() || nextFamilyPartName(params.ownerKey, source)
const refineExtensionFrame = parseRefineExtensionFrame(params.refineExtensionFrame, true)
const saveLosslessAnchor = parseSaveLosslessAnchor(params.saveLosslessAnchor, false)
const refinementDenoise = clampRefinementDenoise(params.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
const job = createJob()
job.kind = 'video'
@@ -149,6 +165,9 @@ export async function beginExtendFromClip(params: {
workflow,
globalLocks: source.globalLocks,
permanenceRefs: source.permanenceRefs,
refineExtensionFrame,
saveLosslessAnchor,
refinementDenoise,
...loraFields
}
@@ -169,6 +188,9 @@ export async function beginExtendFromClip(params: {
fps,
samplerName,
scheduler,
refineExtensionFrame,
saveLosslessAnchor,
refinementDenoise,
...loraFields
}).catch((error) => {
removeExtendTemp(job.library?.extendTmpDir)
@@ -198,13 +220,21 @@ async function runExtend(
fps: number
samplerName: string
scheduler: string
refineExtensionFrame?: boolean
saveLosslessAnchor?: boolean
refinementDenoise?: number
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
}
) {
copyFileSync(params.sourcePath, params.part1Path)
await extractLastFrame(params.sourcePath, params.framePath)
const frame = readFileSync(params.framePath)
const handoff = await resolveExtensionHandoffFrame({
ownerKey: job.library!.ownerKey,
sourceClipId: job.library?.extendSourceClipId,
sourceVideoPath: params.sourcePath,
destPath: params.framePath
})
let frame = readFileSync(handoff.path)
if (!frame.length || frame.length < 64) {
throw new Error('Could not extract the last frame of the clip being extended')
}
@@ -223,8 +253,36 @@ async function runExtend(
})
const done = watchComfyJob(job)
if (params.refineExtensionFrame !== false) {
emitJob(job, {
type: 'status',
message: handoff.source === 'anchor'
? 'Refining lossless anchor frame...'
: 'Refining extracted handoff frame...',
progress: 3
})
try {
frame = await refineExtensionHandoffFrame({
frame,
denoise: params.refinementDenoise,
seed: params.seed,
clientId: job.clientId,
jobId: job.id,
onStatus: (message) => emitJob(job, { type: 'status', message, progress: 4 })
})
writeFileSync(params.framePath, frame)
} catch (error) {
const detail = error instanceof Error ? error.message : String(error)
emitJob(job, {
type: 'status',
message: `Frame refine skipped (${detail}). Using source frame.`,
progress: 4
})
}
}
job.status = 'uploading'
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 4 })
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 5 })
const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
if (job.library) {
job.library.imageName = uploaded.name
@@ -251,7 +309,8 @@ async function runExtend(
sound,
workflow: job.library?.workflow,
duration: job.library?.duration,
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName,
saveLosslessAnchor: params.saveLosslessAnchor === true || job.library?.saveLosslessAnchor === true
})
const queued = await queuePrompt(graph, job.clientId)
+152
View File
@@ -0,0 +1,152 @@
import { copyFileSync, existsSync, statSync, writeFileSync } from 'node:fs'
import refineTemplate from '../assets/extension_frame_refine.json'
import { clipAnchorFramePath, clipVideoPath, downloadComfyImage } from '~/server/utils/library'
import { extractLastFrame } from '~/server/utils/ffmpeg'
import { comfyFilenamePrefix, fetchHistory, queuePrompt, uploadImage } from '~/server/utils/comfy'
import { extractEditedImage } from '~/server/utils/imageComfy'
import {
clampRefinementDenoise,
REFINEMENT_DENOISE_DEFAULT
} from '~/utils/extensionFrame'
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
type WorkflowGraph = Record<string, WorkflowNode>
const MIN_ANCHOR_BYTES = 64
export function isValidAnchorFile(path: string) {
try {
return existsSync(path) && statSync(path).size >= MIN_ANCHOR_BYTES
} catch {
return false
}
}
export async function resolveExtensionHandoffFrame(opts: {
ownerKey: string
sourceClipId?: string
sourceVideoPath: string
destPath: string
}) {
const anchorPath = opts.sourceClipId
? clipAnchorFramePath(opts.ownerKey, opts.sourceClipId)
: ''
if (anchorPath && isValidAnchorFile(anchorPath)) {
copyFileSync(anchorPath, opts.destPath)
return { path: opts.destPath, source: 'anchor' as const }
}
await extractLastFrame(opts.sourceVideoPath, opts.destPath)
if (!isValidAnchorFile(opts.destPath)) {
throw new Error('Could not extract the last frame of the clip being extended')
}
return { path: opts.destPath, source: 'extract' as const }
}
export async function persistClipAnchorFrame(opts: {
ownerKey: string
clipId: string
fromPath?: string
fromBuffer?: Buffer
videoPath?: string
}) {
const dest = clipAnchorFramePath(opts.ownerKey, opts.clipId)
if (opts.fromBuffer?.length && opts.fromBuffer.length >= MIN_ANCHOR_BYTES) {
writeFileSync(dest, opts.fromBuffer)
return dest
}
if (opts.fromPath && isValidAnchorFile(opts.fromPath)) {
copyFileSync(opts.fromPath, dest)
return dest
}
const video = opts.videoPath || clipVideoPath(opts.ownerKey, opts.clipId)
if (!existsSync(video)) return ''
await extractLastFrame(video, dest)
return isValidAnchorFile(dest) ? dest : ''
}
function buildRefineGraph(opts: {
imageName: string
denoise: number
seed: number
filenamePrefix: string
}) {
const graph = structuredClone(refineTemplate) as WorkflowGraph
if (graph['1']?.inputs) graph['1'].inputs.image = opts.imageName
if (graph['13']?.inputs) graph['13'].inputs.noise_seed = opts.seed
if (graph['15']?.inputs) graph['15'].inputs.denoise = opts.denoise
if (graph['20']?.inputs) graph['20'].inputs.filename_prefix = opts.filenamePrefix
return graph
}
async function waitForRefineImage(promptId: string, timeoutMs = 120_000) {
const started = Date.now()
while (Date.now() - started < timeoutMs) {
const history = await fetchHistory(promptId)
const image = extractEditedImage(history, promptId)
if (image) return image
const entry = history?.[promptId] as { status?: { status_str?: string } } | undefined
const status = entry?.status?.status_str
if (status === 'error' || status === 'interrupted') {
throw new Error(status === 'interrupted' ? 'Frame refine interrupted' : 'Frame refine failed on ComfyUI')
}
await new Promise(resolve => setTimeout(resolve, 700))
}
throw new Error('Frame refine timed out waiting for ComfyUI')
}
/**
* Bilateral + low-denoise Flux cleanup before the video I2V start frame.
* Runs as its own Comfy prompt so MiniMax does not share VRAM with Klein.
* When refine is off, callers skip this and upload the PNG directly (fast path).
*/
export async function refineExtensionHandoffFrame(opts: {
frame: Buffer
denoise?: number
seed?: number
clientId: string
jobId: string
onStatus?: (message: string) => void
}) {
const denoise = clampRefinementDenoise(opts.denoise, REFINEMENT_DENOISE_DEFAULT)
opts.onStatus?.('Refining extension handoff frame...')
const uploaded = await uploadImage({
filename: 'last_frame.png',
data: opts.frame,
type: 'image/png'
}, opts.jobId)
const prefix = `${comfyFilenamePrefix()}_extrefine`
const graph = buildRefineGraph({
imageName: uploaded.name,
denoise,
seed: opts.seed ?? Math.floor(Math.random() * 2_147_483_647),
filenamePrefix: `aigen/extension_refine/${prefix}`
})
const queued = await queuePrompt(graph, opts.clientId)
const image = await waitForRefineImage(queued.prompt_id)
const buffer = await downloadComfyImage(image)
if (!buffer.length || buffer.length < MIN_ANCHOR_BYTES) {
throw new Error('Frame refine finished without an output still')
}
return buffer
}
export function extractAnchorImageFromHistory(history: Record<string, unknown> | null, promptId: string) {
if (!history || !promptId) return null
const entry = history[promptId] as { outputs?: Record<string, unknown> } | undefined
const outputs = entry?.outputs
if (!outputs || typeof outputs !== 'object') return extractEditedImage(history, promptId)
const anchor = outputs.aigen_anchor_save as { images?: Array<{ filename?: string; subfolder?: string; type?: string }> } | undefined
const file = anchor?.images?.[0]
if (file?.filename) {
return {
filename: file.filename,
subfolder: file.subfolder || '',
type: file.type || 'output'
}
}
return extractEditedImage(history, promptId)
}
export async function downloadAnchorImage(image: { filename: string; subfolder: string; type: string }) {
return downloadComfyImage(image)
}
+3
View File
@@ -101,6 +101,9 @@ export interface Job {
queueAutoRun?: boolean
queueBudget?: number
stopAfterCurrent?: boolean
refineExtensionFrame?: boolean
saveLosslessAnchor?: boolean
refinementDenoise?: number
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
+11 -2
View File
@@ -790,6 +790,11 @@ export function clipLastFramePath(owner: string, id: string) {
return join(clipDir(owner, id), 'last-frame.png')
}
/** Lossless PNG handoff frame for cleaner sequential extensions (not the preview cache). */
export function clipAnchorFramePath(owner: string, id: string) {
return join(clipDir(owner, id), 'anchor-last.png')
}
export function clipFirstFramePath(owner: string, id: string) {
return join(clipDir(owner, id), 'first-frame.jpg')
}
@@ -2009,9 +2014,9 @@ async function downloadViaHostAgent(file: { filename: string; subfolder: string;
async function downloadComfyMedia(
file: { filename: string; subfolder: string; type: string },
kind: 'video' | 'audio'
kind: 'video' | 'audio' | 'image'
) {
const extras = kind === 'video' ? ['video', ''] : ['audio', '']
const extras = kind === 'video' ? ['video', ''] : kind === 'audio' ? ['audio', ''] : ['']
const subfolders = [...new Set([file.subfolder, ...extras])]
let lastStatus = 0
let lastError = ''
@@ -2049,6 +2054,10 @@ export async function downloadComfyAudio(audio: { filename: string; subfolder: s
return downloadComfyMedia(audio, 'audio')
}
export async function downloadComfyImage(image: { filename: string; subfolder: string; type: string }) {
return downloadComfyMedia(image, 'image')
}
export async function importMissingComfyVideos(owner: string, folderId?: string) {
const catalog = readCatalog(owner)
const folder = catalog.folders.find(item => item.id === folderId) || catalog.folders[0]
+35
View File
@@ -56,6 +56,9 @@ export interface PendingJob {
queueAutoRun?: boolean
queueBudget?: number
stopAfterCurrent?: boolean
refineExtensionFrame?: boolean
saveLosslessAnchor?: boolean
refinementDenoise?: number
loraName?: string
loraStack?: import('~/utils/loras').LoraStackItem[]
shotLoras?: string[]
@@ -155,6 +158,9 @@ export function pendingFromJob(job: Job, overrides: Partial<PendingJob> = {}): P
queueAutoRun: library.queueAutoRun,
queueBudget: library.queueBudget,
stopAfterCurrent: library.stopAfterCurrent,
refineExtensionFrame: library.refineExtensionFrame,
saveLosslessAnchor: library.saveLosslessAnchor,
refinementDenoise: library.refinementDenoise,
...overrides
}
}
@@ -203,6 +209,9 @@ export function libraryFromPending(pending: PendingJob): NonNullable<Job['librar
queueAutoRun: pending.queueAutoRun,
queueBudget: pending.queueBudget,
stopAfterCurrent: pending.stopAfterCurrent,
refineExtensionFrame: pending.refineExtensionFrame,
saveLosslessAnchor: pending.saveLosslessAnchor,
refinementDenoise: pending.refinementDenoise,
globalLocks: pending.globalLocks,
permanenceRefs: pending.permanenceRefs,
shotPermanenceRefs: pending.shotPermanenceRefs,
@@ -342,6 +351,32 @@ export async function completePendingIfReady(pending: PendingJob) {
loraStack: pending.loraStack,
segmentFirstFrame
})
if (pending.saveLosslessAnchor === true) {
try {
const { persistClipAnchorFrame, extractAnchorImageFromHistory, downloadAnchorImage } = await import('~/server/utils/extensionFrame')
let saved = false
const anchorImage = extractAnchorImageFromHistory(history, pending.promptId)
if (anchorImage) {
const png = await downloadAnchorImage(anchorImage)
if (png.length >= 64) {
await persistClipAnchorFrame({
ownerKey: pending.ownerKey,
clipId: clip.id,
fromBuffer: png
})
saved = true
}
}
if (!saved) {
await persistClipAnchorFrame({
ownerKey: pending.ownerKey,
clipId: clip.id
})
}
} catch {
/* optional */
}
}
await purgeComfyArtifacts({
video,
imageName: pending.imageName,
+10 -1
View File
@@ -69,6 +69,9 @@ export interface StudioJobPayload {
maskStillFilename?: string
refineStrength?: number
extendFromClipId?: string
refineExtensionFrame?: boolean
saveLosslessAnchor?: boolean
refinementDenoise?: number
lyrics?: string
instrumental?: boolean
lyricsStrength?: number
@@ -1156,7 +1159,10 @@ async function startStudioExtendJob(item: StudioJob) {
duration: item.payload.duration,
loraStack: item.payload.loraStack || item.payload.loraName,
folderLocked: item.payload.folderLocked,
name: item.payload.name || item.name
name: item.payload.name || item.name,
refineExtensionFrame: item.payload.refineExtensionFrame,
saveLosslessAnchor: item.payload.saveLosslessAnchor,
refinementDenoise: item.payload.refinementDenoise
})
await markStudioLive(item.ownerKey, item.id, live.id)
} catch (error) {
@@ -1294,6 +1300,9 @@ export async function startStudioJob(item: StudioJob) {
useIdentityRefs: identity,
queueAutoRun: extensions.length ? payload.queueAutoRun : false,
queueBudget: extensions.length && payload.queueAutoRun ? extensions.length : 0,
refineExtensionFrame: payload.refineExtensionFrame !== false,
saveLosslessAnchor: payload.saveLosslessAnchor === true,
refinementDenoise: payload.refinementDenoise,
globalLocks: payload.globalLocks,
permanenceRefs: payload.permanenceRefs,
shotPermanenceRefs: payload.shotPermanenceRefs,
+33 -3
View File
@@ -7,9 +7,13 @@ import { comfyFilenamePrefix, queuePrompt, uploadImage } from '~/server/utils/co
import { buildWorkflow } from '~/server/utils/workflow'
import { isLtxWorkflow, isTextToVideo, parseVideoWorkflow, workflowForExtension, type VideoWorkflowId } from '~/utils/videoModels'
import { clipVideoPath, clipTitle, deleteRetryDraft, extendTempDir, getClip, nextFamilyPartName, removeExtendTemp, stillPath } from '~/server/utils/library'
import { extractLastFrame, probeHasAudio } from '~/server/utils/ffmpeg'
import { probeHasAudio } from '~/server/utils/ffmpeg'
import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
import { onLiveVideoSettled, onStudioQueueBurstStarted } from '~/server/utils/studioQueue'
import {
refineExtensionHandoffFrame,
resolveExtensionHandoffFrame
} from '~/server/utils/extensionFrame'
import {
finishQueueBurst,
getShotQueue,
@@ -216,6 +220,7 @@ export async function queueMiniMax(
duration: params.duration,
useIdentityRefs: params.useIdentityRefs,
referenceImageNames: params.useIdentityRefs ? referenceNames : [],
saveLosslessAnchor: job.library?.saveLosslessAnchor === true,
...persistLoraFields(params.loraStack || params.loraName)
})
@@ -373,7 +378,12 @@ export async function continueQueuedExtensions(
const parentVideo = parentId ? clipVideoPath(job.library.ownerKey, parentId) : ''
const extractSource = parentVideo && existsSync(parentVideo) ? parentVideo : part1Path
try {
await extractLastFrame(extractSource, framePath)
await resolveExtensionHandoffFrame({
ownerKey: job.library.ownerKey,
sourceClipId: parentId,
sourceVideoPath: extractSource,
destPath: framePath
})
} catch (error) {
const detail = error instanceof Error ? error.message : String(error)
throw new Error(detail.includes('last frame')
@@ -385,10 +395,30 @@ export async function continueQueuedExtensions(
job.library.stopAfterCurrent = true
break
}
const frame = readFileSync(framePath)
let frame = readFileSync(framePath)
if (!frame.length || frame.length < 64) {
throw new Error('Could not extract the last frame for the next extension: the frame file was empty')
}
if (job.library.refineExtensionFrame !== false) {
emitChainJob(job, { type: 'status', message: 'Refining extension handoff frame...', progress: 2 })
try {
frame = await refineExtensionHandoffFrame({
frame,
denoise: job.library.refinementDenoise,
clientId: job.clientId,
jobId: job.id,
onStatus: (message) => emitChainJob(job, { type: 'status', message, progress: 3 })
})
writeFileSync(framePath, frame)
} catch (error) {
const detail = error instanceof Error ? error.message : String(error)
emitChainJob(job, {
type: 'status',
message: `Frame refine skipped (${detail}). Using source frame.`,
progress: 3
})
}
}
job.library.extendPart1Path = part1Path
job.library.prompt = composePromptParts(wrappers.promptPre, ext.prompt, wrappers.promptPost)
job.library.promptMid = ext.prompt
+28
View File
@@ -275,6 +275,34 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
loraStack: job.library.loraStack,
segmentFirstFrame
})
if (job.library.saveLosslessAnchor === true) {
try {
const { persistClipAnchorFrame, extractAnchorImageFromHistory, downloadAnchorImage } = await import('~/server/utils/extensionFrame')
let saved = false
if (job.promptId && history) {
const anchorImage = extractAnchorImageFromHistory(history, job.promptId)
if (anchorImage) {
const png = await downloadAnchorImage(anchorImage)
if (png.length >= 64) {
await persistClipAnchorFrame({
ownerKey: job.library.ownerKey,
clipId: clip.id,
fromBuffer: png
})
saved = true
}
}
}
if (!saved) {
await persistClipAnchorFrame({
ownerKey: job.library.ownerKey,
clipId: clip.id
})
}
} catch {
/* Anchor is optional; extension can still extract from MP4. */
}
}
job.clipId = clip.id
job.savedPromptId = job.promptId
job.hideThumbnail = clip.hideThumbnail
+28
View File
@@ -40,6 +40,7 @@ export interface GenerateParams {
referenceImageNames?: string[]
loraName?: string
loraStack?: LoraStackItem[]
saveLosslessAnchor?: boolean
}
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
@@ -228,9 +229,34 @@ export function buildWorkflow(params: GenerateParams) {
resolveGraphLoraNames(graph, 'video')
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
attachLosslessAnchorSave(graph, params, version === 'v2' ? '164' : '105:90')
return graph
}
function attachLosslessAnchorSave(graph: WorkflowGraph, params: GenerateParams, decodeNodeId: string) {
if (!params.saveLosslessAnchor) return
if (!graph[decodeNodeId]) return
const batchIndex = Math.max(0, Math.floor(Number(params.length) || 1) - 1)
const prefix = String(params.filenamePrefix || 'video/MiniMax_H3').replace(/\/+$/, '')
graph.aigen_anchor_pick = {
class_type: 'ImageFromBatch',
inputs: {
image: [decodeNodeId, 0],
batch_index: batchIndex,
length: 1
},
_meta: { title: 'Last Decoded Frame' }
}
graph.aigen_anchor_save = {
class_type: 'SaveImage',
inputs: {
filename_prefix: `${prefix}_last`,
images: ['aigen_anchor_pick', 0]
},
_meta: { title: 'Save Lossless Anchor' }
}
}
function promptForNode(classType: string, useRefs: boolean, actionPrompt: string, graphPrompt: string) {
if (classType === 'MiniMaxH3ImageToVideo') return useRefs ? '' : actionPrompt
if (classType === 'MiniMaxH3TextToVideo') return actionPrompt
@@ -302,6 +328,8 @@ function buildLtxWorkflow(params: GenerateParams, version: VideoWorkflowId) {
}
resolveGraphLoraNames(graph, 'video')
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
// LTX VAEDecode is typically node 15
attachLosslessAnchorSave(graph, params, '15')
return graph
}
+22
View File
@@ -0,0 +1,22 @@
export const REFINEMENT_DENOISE_MIN = 0.1
export const REFINEMENT_DENOISE_MAX = 0.25
export const REFINEMENT_DENOISE_DEFAULT = 0.18
export function clampRefinementDenoise(raw: unknown, fallback = REFINEMENT_DENOISE_DEFAULT) {
const n = Number(raw)
if (!Number.isFinite(n)) return fallback
return Math.min(REFINEMENT_DENOISE_MAX, Math.max(REFINEMENT_DENOISE_MIN, Math.round(n * 100) / 100))
}
export function parseRefineExtensionFrame(raw: unknown, fallback = true) {
if (raw === undefined || raw === null || raw === '') return fallback
if (typeof raw === 'boolean') return raw
const value = String(raw).trim().toLowerCase()
if (value === 'true' || value === '1' || value === 'yes') return true
if (value === 'false' || value === '0' || value === 'no') return false
return fallback
}
export function parseSaveLosslessAnchor(raw: unknown, fallback = false) {
return parseRefineExtensionFrame(raw, fallback)
}