Add extension-frame refine and lossless anchor options.
Prefer clip anchor PNGs for handoffs, optionally run Bilateral+Flux cleanup before I2V, and save decoded last-frame anchors so sequential extensions accumulate less compression damage. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -6,9 +6,26 @@ import { clipPartDuration, clampVideoDuration } from '~/utils/videoDuration'
|
||||
import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
|
||||
import { composePromptParts } from '~/utils/promptParts'
|
||||
import { nextFamilyPartName } from '~/server/utils/library'
|
||||
import {
|
||||
clampRefinementDenoise,
|
||||
parseRefineExtensionFrame,
|
||||
parseSaveLosslessAnchor,
|
||||
REFINEMENT_DENOISE_DEFAULT
|
||||
} from '~/utils/extensionFrame'
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
const body = await readBody<{ clipId?: string; prompt?: string; promptPre?: string; promptPost?: string; duration?: number; lora?: unknown; loraStack?: unknown }>(event).catch(() => ({}))
|
||||
const body = await readBody<{
|
||||
clipId?: string
|
||||
prompt?: string
|
||||
promptPre?: string
|
||||
promptPost?: string
|
||||
duration?: number
|
||||
lora?: unknown
|
||||
loraStack?: unknown
|
||||
refineExtensionFrame?: boolean | string
|
||||
saveLosslessAnchor?: boolean | string
|
||||
refinementDenoise?: number | string
|
||||
}>(event).catch(() => ({}))
|
||||
const clipId = String(body?.clipId || '').trim()
|
||||
const prompt = String(body?.prompt || '').trim()
|
||||
if (!clipId) {
|
||||
@@ -87,6 +104,9 @@ export default defineEventHandler(async (event) => {
|
||||
extensions: [],
|
||||
queueAutoRun: false,
|
||||
extendFromClipId: clipId,
|
||||
refineExtensionFrame: parseRefineExtensionFrame(body?.refineExtensionFrame, true),
|
||||
saveLosslessAnchor: parseSaveLosslessAnchor(body?.saveLosslessAnchor, false),
|
||||
refinementDenoise: clampRefinementDenoise(body?.refinementDenoise, REFINEMENT_DENOISE_DEFAULT),
|
||||
globalLocks: source.globalLocks,
|
||||
permanenceRefs: source.permanenceRefs,
|
||||
...persistLoraFields(loraStack)
|
||||
|
||||
@@ -6,6 +6,12 @@ import { defaultVideoSteps, isLtxWorkflow, isTextToVideo, LTX_DISABLED_MESSAGE,
|
||||
import { allowIdentityRefs, normalizePermanenceRefs, resolveGlobalLocks, type PermanenceRef } from '~/utils/globalLocks'
|
||||
import { composePromptParts, ingestPromptParts, persistPromptWrappers } from '~/utils/promptParts'
|
||||
import type { QueuedExtension } from '~/server/utils/library'
|
||||
import {
|
||||
clampRefinementDenoise,
|
||||
parseRefineExtensionFrame,
|
||||
parseSaveLosslessAnchor,
|
||||
REFINEMENT_DENOISE_DEFAULT
|
||||
} from '~/utils/extensionFrame'
|
||||
|
||||
function parseDuration(raw: unknown) {
|
||||
const seconds = Number(raw)
|
||||
@@ -250,6 +256,9 @@ export default defineEventHandler(async (event) => {
|
||||
const chainTotal = 1 + extensions.length
|
||||
const familyId = crypto.randomUUID()
|
||||
const queueAutoRun = fields.queueAutoRun === 'true'
|
||||
const refineExtensionFrame = parseRefineExtensionFrame(fields.refineExtensionFrame, true)
|
||||
const saveLosslessAnchor = parseSaveLosslessAnchor(fields.saveLosslessAnchor, false)
|
||||
const refinementDenoise = clampRefinementDenoise(fields.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
|
||||
const studio = await addStudioJob({
|
||||
ownerKey,
|
||||
familyId,
|
||||
@@ -282,6 +291,9 @@ export default defineEventHandler(async (event) => {
|
||||
referenceStillIds,
|
||||
extensions,
|
||||
queueAutoRun,
|
||||
refineExtensionFrame,
|
||||
saveLosslessAnchor,
|
||||
refinementDenoise,
|
||||
globalLocks: globalLocks || undefined,
|
||||
permanenceRefs: permanenceRefs.length ? permanenceRefs : undefined,
|
||||
shotPermanenceRefs: shotPermanenceRefs.some(item => item.length) ? shotPermanenceRefs : undefined,
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
{
|
||||
"1": {
|
||||
"inputs": { "image": "last_frame.png" },
|
||||
"class_type": "LoadImage",
|
||||
"_meta": { "title": "Handoff Frame" }
|
||||
},
|
||||
"2": {
|
||||
"inputs": {
|
||||
"images": ["1", 0],
|
||||
"size": 2,
|
||||
"sigma_color": 0.12,
|
||||
"sigma_space": 1.5
|
||||
},
|
||||
"class_type": "BilateralFilterImage",
|
||||
"_meta": { "title": "Bilateral Deblock" }
|
||||
},
|
||||
"3": {
|
||||
"inputs": {
|
||||
"unet_name": "flux-2-klein-base-9b-fp8.safetensors",
|
||||
"weight_dtype": "default"
|
||||
},
|
||||
"class_type": "UNETLoader",
|
||||
"_meta": { "title": "Load Flux.2 Klein" }
|
||||
},
|
||||
"4": {
|
||||
"inputs": {
|
||||
"clip_name": "qwen_3_8b_fp8mixed.safetensors",
|
||||
"type": "flux2",
|
||||
"device": "default"
|
||||
},
|
||||
"class_type": "CLIPLoader",
|
||||
"_meta": { "title": "Load CLIP" }
|
||||
},
|
||||
"5": {
|
||||
"inputs": { "vae_name": "full_encoder_small_decoder.safetensors" },
|
||||
"class_type": "VAELoader",
|
||||
"_meta": { "title": "Load Klein VAE" }
|
||||
},
|
||||
"6": {
|
||||
"inputs": {
|
||||
"lora_name": "Flux2-Klein-9B-consistency-V2.safetensors",
|
||||
"strength_model": 0.55,
|
||||
"strength_clip": 0.55,
|
||||
"model": ["3", 0],
|
||||
"clip": ["4", 0]
|
||||
},
|
||||
"class_type": "LoraLoader",
|
||||
"_meta": { "title": "Consistency" }
|
||||
},
|
||||
"7": {
|
||||
"inputs": {
|
||||
"text": "photoreal still frame, natural high-frequency texture, clean edges, no compression blocks, no mosquito noise",
|
||||
"clip": ["6", 1]
|
||||
},
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Positive" }
|
||||
},
|
||||
"8": {
|
||||
"inputs": {
|
||||
"text": "compression artifacts, blockiness, banding, blur, smear, plastic skin",
|
||||
"clip": ["6", 1]
|
||||
},
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": { "title": "Negative" }
|
||||
},
|
||||
"9": {
|
||||
"inputs": {
|
||||
"guidance": 1.8,
|
||||
"conditioning": ["7", 0]
|
||||
},
|
||||
"class_type": "FluxGuidance",
|
||||
"_meta": { "title": "Flux Guidance" }
|
||||
},
|
||||
"10": {
|
||||
"inputs": {
|
||||
"pixels": ["2", 0],
|
||||
"vae": ["5", 0]
|
||||
},
|
||||
"class_type": "VAEEncode",
|
||||
"_meta": { "title": "Encode Filtered" }
|
||||
},
|
||||
"11": {
|
||||
"inputs": {
|
||||
"conditioning": ["9", 0],
|
||||
"latent": ["10", 0]
|
||||
},
|
||||
"class_type": "ReferenceLatent",
|
||||
"_meta": { "title": "Reference +" }
|
||||
},
|
||||
"12": {
|
||||
"inputs": {
|
||||
"conditioning": ["8", 0],
|
||||
"latent": ["10", 0]
|
||||
},
|
||||
"class_type": "ReferenceLatent",
|
||||
"_meta": { "title": "Reference -" }
|
||||
},
|
||||
"13": {
|
||||
"inputs": { "noise_seed": 1 },
|
||||
"class_type": "RandomNoise",
|
||||
"_meta": { "title": "Noise" }
|
||||
},
|
||||
"14": {
|
||||
"inputs": { "sampler_name": "euler" },
|
||||
"class_type": "KSamplerSelect",
|
||||
"_meta": { "title": "Sampler" }
|
||||
},
|
||||
"15": {
|
||||
"inputs": {
|
||||
"scheduler": "simple",
|
||||
"steps": 14,
|
||||
"denoise": 0.18,
|
||||
"model": ["6", 0]
|
||||
},
|
||||
"class_type": "BasicScheduler",
|
||||
"_meta": { "title": "Scheduler" }
|
||||
},
|
||||
"16": {
|
||||
"inputs": {
|
||||
"cfg": 1,
|
||||
"model": ["6", 0],
|
||||
"positive": ["11", 0],
|
||||
"negative": ["12", 0]
|
||||
},
|
||||
"class_type": "CFGGuider",
|
||||
"_meta": { "title": "CFG 1.0" }
|
||||
},
|
||||
"17": {
|
||||
"inputs": {
|
||||
"noise": ["13", 0],
|
||||
"guider": ["16", 0],
|
||||
"sampler": ["14", 0],
|
||||
"sigmas": ["15", 0],
|
||||
"latent_image": ["10", 0]
|
||||
},
|
||||
"class_type": "SamplerCustomAdvanced",
|
||||
"_meta": { "title": "Img2Img Cleanup" }
|
||||
},
|
||||
"18": {
|
||||
"inputs": {
|
||||
"samples": ["17", 0],
|
||||
"vae": ["5", 0]
|
||||
},
|
||||
"class_type": "VAEDecode",
|
||||
"_meta": { "title": "Decode" }
|
||||
},
|
||||
"19": {
|
||||
"inputs": {
|
||||
"image1": ["1", 0],
|
||||
"image2": ["18", 0],
|
||||
"blend_factor": 0.85,
|
||||
"blend_mode": "normal"
|
||||
},
|
||||
"class_type": "ImageBlend",
|
||||
"_meta": { "title": "Blend 0.85" }
|
||||
},
|
||||
"20": {
|
||||
"inputs": {
|
||||
"filename_prefix": "aigen/extension_refine",
|
||||
"images": ["19", 0]
|
||||
},
|
||||
"class_type": "SaveImage",
|
||||
"_meta": { "title": "Save Refined Frame" }
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { copyFileSync, existsSync, readFileSync } from 'node:fs'
|
||||
import { copyFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { composeShotPrompt } from '~/utils/globalLocks'
|
||||
import { composePromptParts } from '~/utils/promptParts'
|
||||
@@ -16,12 +16,22 @@ import {
|
||||
nextFamilyPartName,
|
||||
removeExtendTemp
|
||||
} from '~/server/utils/library'
|
||||
import { extractLastFrame, probeHasAudio } from '~/server/utils/ffmpeg'
|
||||
import { probeHasAudio } from '~/server/utils/ffmpeg'
|
||||
import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
|
||||
import { waitForComfySocket, watchComfyJob } from '~/server/utils/watch'
|
||||
import { comfyFilenamePrefix, queuePrompt, uploadImage } from '~/server/utils/comfy'
|
||||
import { buildWorkflow } from '~/server/utils/workflow'
|
||||
import { writePendingJob } from '~/server/utils/pending'
|
||||
import {
|
||||
refineExtensionHandoffFrame,
|
||||
resolveExtensionHandoffFrame
|
||||
} from '~/server/utils/extensionFrame'
|
||||
import {
|
||||
clampRefinementDenoise,
|
||||
parseRefineExtensionFrame,
|
||||
parseSaveLosslessAnchor,
|
||||
REFINEMENT_DENOISE_DEFAULT
|
||||
} from '~/utils/extensionFrame'
|
||||
|
||||
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
|
||||
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
|
||||
@@ -63,6 +73,9 @@ export async function beginExtendFromClip(params: {
|
||||
loraStack?: unknown
|
||||
folderLocked?: boolean
|
||||
name?: string
|
||||
refineExtensionFrame?: boolean
|
||||
saveLosslessAnchor?: boolean
|
||||
refinementDenoise?: number
|
||||
}) {
|
||||
const source = getClip(params.ownerKey, params.clipId)
|
||||
const sourcePath = clipVideoPath(params.ownerKey, source.id)
|
||||
@@ -111,6 +124,9 @@ export async function beginExtendFromClip(params: {
|
||||
const width = source.width || 1344
|
||||
const height = source.height || 768
|
||||
const clipName = String(params.name || '').trim() || nextFamilyPartName(params.ownerKey, source)
|
||||
const refineExtensionFrame = parseRefineExtensionFrame(params.refineExtensionFrame, true)
|
||||
const saveLosslessAnchor = parseSaveLosslessAnchor(params.saveLosslessAnchor, false)
|
||||
const refinementDenoise = clampRefinementDenoise(params.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
|
||||
|
||||
const job = createJob()
|
||||
job.kind = 'video'
|
||||
@@ -149,6 +165,9 @@ export async function beginExtendFromClip(params: {
|
||||
workflow,
|
||||
globalLocks: source.globalLocks,
|
||||
permanenceRefs: source.permanenceRefs,
|
||||
refineExtensionFrame,
|
||||
saveLosslessAnchor,
|
||||
refinementDenoise,
|
||||
...loraFields
|
||||
}
|
||||
|
||||
@@ -169,6 +188,9 @@ export async function beginExtendFromClip(params: {
|
||||
fps,
|
||||
samplerName,
|
||||
scheduler,
|
||||
refineExtensionFrame,
|
||||
saveLosslessAnchor,
|
||||
refinementDenoise,
|
||||
...loraFields
|
||||
}).catch((error) => {
|
||||
removeExtendTemp(job.library?.extendTmpDir)
|
||||
@@ -198,13 +220,21 @@ async function runExtend(
|
||||
fps: number
|
||||
samplerName: string
|
||||
scheduler: string
|
||||
refineExtensionFrame?: boolean
|
||||
saveLosslessAnchor?: boolean
|
||||
refinementDenoise?: number
|
||||
loraName?: string
|
||||
loraStack?: import('~/utils/loras').LoraStackItem[]
|
||||
}
|
||||
) {
|
||||
copyFileSync(params.sourcePath, params.part1Path)
|
||||
await extractLastFrame(params.sourcePath, params.framePath)
|
||||
const frame = readFileSync(params.framePath)
|
||||
const handoff = await resolveExtensionHandoffFrame({
|
||||
ownerKey: job.library!.ownerKey,
|
||||
sourceClipId: job.library?.extendSourceClipId,
|
||||
sourceVideoPath: params.sourcePath,
|
||||
destPath: params.framePath
|
||||
})
|
||||
let frame = readFileSync(handoff.path)
|
||||
if (!frame.length || frame.length < 64) {
|
||||
throw new Error('Could not extract the last frame of the clip being extended')
|
||||
}
|
||||
@@ -223,8 +253,36 @@ async function runExtend(
|
||||
})
|
||||
|
||||
const done = watchComfyJob(job)
|
||||
if (params.refineExtensionFrame !== false) {
|
||||
emitJob(job, {
|
||||
type: 'status',
|
||||
message: handoff.source === 'anchor'
|
||||
? 'Refining lossless anchor frame...'
|
||||
: 'Refining extracted handoff frame...',
|
||||
progress: 3
|
||||
})
|
||||
try {
|
||||
frame = await refineExtensionHandoffFrame({
|
||||
frame,
|
||||
denoise: params.refinementDenoise,
|
||||
seed: params.seed,
|
||||
clientId: job.clientId,
|
||||
jobId: job.id,
|
||||
onStatus: (message) => emitJob(job, { type: 'status', message, progress: 4 })
|
||||
})
|
||||
writeFileSync(params.framePath, frame)
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : String(error)
|
||||
emitJob(job, {
|
||||
type: 'status',
|
||||
message: `Frame refine skipped (${detail}). Using source frame.`,
|
||||
progress: 4
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
job.status = 'uploading'
|
||||
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 4 })
|
||||
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 5 })
|
||||
const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
|
||||
if (job.library) {
|
||||
job.library.imageName = uploaded.name
|
||||
@@ -251,7 +309,8 @@ async function runExtend(
|
||||
sound,
|
||||
workflow: job.library?.workflow,
|
||||
duration: job.library?.duration,
|
||||
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName
|
||||
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName,
|
||||
saveLosslessAnchor: params.saveLosslessAnchor === true || job.library?.saveLosslessAnchor === true
|
||||
})
|
||||
|
||||
const queued = await queuePrompt(graph, job.clientId)
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
import { copyFileSync, existsSync, statSync, writeFileSync } from 'node:fs'
|
||||
import refineTemplate from '../assets/extension_frame_refine.json'
|
||||
import { clipAnchorFramePath, clipVideoPath, downloadComfyImage } from '~/server/utils/library'
|
||||
import { extractLastFrame } from '~/server/utils/ffmpeg'
|
||||
import { comfyFilenamePrefix, fetchHistory, queuePrompt, uploadImage } from '~/server/utils/comfy'
|
||||
import { extractEditedImage } from '~/server/utils/imageComfy'
|
||||
import {
|
||||
clampRefinementDenoise,
|
||||
REFINEMENT_DENOISE_DEFAULT
|
||||
} from '~/utils/extensionFrame'
|
||||
|
||||
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
|
||||
type WorkflowGraph = Record<string, WorkflowNode>
|
||||
|
||||
const MIN_ANCHOR_BYTES = 64
|
||||
|
||||
export function isValidAnchorFile(path: string) {
|
||||
try {
|
||||
return existsSync(path) && statSync(path).size >= MIN_ANCHOR_BYTES
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveExtensionHandoffFrame(opts: {
|
||||
ownerKey: string
|
||||
sourceClipId?: string
|
||||
sourceVideoPath: string
|
||||
destPath: string
|
||||
}) {
|
||||
const anchorPath = opts.sourceClipId
|
||||
? clipAnchorFramePath(opts.ownerKey, opts.sourceClipId)
|
||||
: ''
|
||||
if (anchorPath && isValidAnchorFile(anchorPath)) {
|
||||
copyFileSync(anchorPath, opts.destPath)
|
||||
return { path: opts.destPath, source: 'anchor' as const }
|
||||
}
|
||||
await extractLastFrame(opts.sourceVideoPath, opts.destPath)
|
||||
if (!isValidAnchorFile(opts.destPath)) {
|
||||
throw new Error('Could not extract the last frame of the clip being extended')
|
||||
}
|
||||
return { path: opts.destPath, source: 'extract' as const }
|
||||
}
|
||||
|
||||
export async function persistClipAnchorFrame(opts: {
|
||||
ownerKey: string
|
||||
clipId: string
|
||||
fromPath?: string
|
||||
fromBuffer?: Buffer
|
||||
videoPath?: string
|
||||
}) {
|
||||
const dest = clipAnchorFramePath(opts.ownerKey, opts.clipId)
|
||||
if (opts.fromBuffer?.length && opts.fromBuffer.length >= MIN_ANCHOR_BYTES) {
|
||||
writeFileSync(dest, opts.fromBuffer)
|
||||
return dest
|
||||
}
|
||||
if (opts.fromPath && isValidAnchorFile(opts.fromPath)) {
|
||||
copyFileSync(opts.fromPath, dest)
|
||||
return dest
|
||||
}
|
||||
const video = opts.videoPath || clipVideoPath(opts.ownerKey, opts.clipId)
|
||||
if (!existsSync(video)) return ''
|
||||
await extractLastFrame(video, dest)
|
||||
return isValidAnchorFile(dest) ? dest : ''
|
||||
}
|
||||
|
||||
function buildRefineGraph(opts: {
|
||||
imageName: string
|
||||
denoise: number
|
||||
seed: number
|
||||
filenamePrefix: string
|
||||
}) {
|
||||
const graph = structuredClone(refineTemplate) as WorkflowGraph
|
||||
if (graph['1']?.inputs) graph['1'].inputs.image = opts.imageName
|
||||
if (graph['13']?.inputs) graph['13'].inputs.noise_seed = opts.seed
|
||||
if (graph['15']?.inputs) graph['15'].inputs.denoise = opts.denoise
|
||||
if (graph['20']?.inputs) graph['20'].inputs.filename_prefix = opts.filenamePrefix
|
||||
return graph
|
||||
}
|
||||
|
||||
async function waitForRefineImage(promptId: string, timeoutMs = 120_000) {
|
||||
const started = Date.now()
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
const history = await fetchHistory(promptId)
|
||||
const image = extractEditedImage(history, promptId)
|
||||
if (image) return image
|
||||
const entry = history?.[promptId] as { status?: { status_str?: string } } | undefined
|
||||
const status = entry?.status?.status_str
|
||||
if (status === 'error' || status === 'interrupted') {
|
||||
throw new Error(status === 'interrupted' ? 'Frame refine interrupted' : 'Frame refine failed on ComfyUI')
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 700))
|
||||
}
|
||||
throw new Error('Frame refine timed out waiting for ComfyUI')
|
||||
}
|
||||
|
||||
/**
|
||||
* Bilateral + low-denoise Flux cleanup before the video I2V start frame.
|
||||
* Runs as its own Comfy prompt so MiniMax does not share VRAM with Klein.
|
||||
* When refine is off, callers skip this and upload the PNG directly (fast path).
|
||||
*/
|
||||
export async function refineExtensionHandoffFrame(opts: {
|
||||
frame: Buffer
|
||||
denoise?: number
|
||||
seed?: number
|
||||
clientId: string
|
||||
jobId: string
|
||||
onStatus?: (message: string) => void
|
||||
}) {
|
||||
const denoise = clampRefinementDenoise(opts.denoise, REFINEMENT_DENOISE_DEFAULT)
|
||||
opts.onStatus?.('Refining extension handoff frame...')
|
||||
const uploaded = await uploadImage({
|
||||
filename: 'last_frame.png',
|
||||
data: opts.frame,
|
||||
type: 'image/png'
|
||||
}, opts.jobId)
|
||||
const prefix = `${comfyFilenamePrefix()}_extrefine`
|
||||
const graph = buildRefineGraph({
|
||||
imageName: uploaded.name,
|
||||
denoise,
|
||||
seed: opts.seed ?? Math.floor(Math.random() * 2_147_483_647),
|
||||
filenamePrefix: `aigen/extension_refine/${prefix}`
|
||||
})
|
||||
const queued = await queuePrompt(graph, opts.clientId)
|
||||
const image = await waitForRefineImage(queued.prompt_id)
|
||||
const buffer = await downloadComfyImage(image)
|
||||
if (!buffer.length || buffer.length < MIN_ANCHOR_BYTES) {
|
||||
throw new Error('Frame refine finished without an output still')
|
||||
}
|
||||
return buffer
|
||||
}
|
||||
|
||||
export function extractAnchorImageFromHistory(history: Record<string, unknown> | null, promptId: string) {
|
||||
if (!history || !promptId) return null
|
||||
const entry = history[promptId] as { outputs?: Record<string, unknown> } | undefined
|
||||
const outputs = entry?.outputs
|
||||
if (!outputs || typeof outputs !== 'object') return extractEditedImage(history, promptId)
|
||||
const anchor = outputs.aigen_anchor_save as { images?: Array<{ filename?: string; subfolder?: string; type?: string }> } | undefined
|
||||
const file = anchor?.images?.[0]
|
||||
if (file?.filename) {
|
||||
return {
|
||||
filename: file.filename,
|
||||
subfolder: file.subfolder || '',
|
||||
type: file.type || 'output'
|
||||
}
|
||||
}
|
||||
return extractEditedImage(history, promptId)
|
||||
}
|
||||
|
||||
export async function downloadAnchorImage(image: { filename: string; subfolder: string; type: string }) {
|
||||
return downloadComfyImage(image)
|
||||
}
|
||||
@@ -101,6 +101,9 @@ export interface Job {
|
||||
queueAutoRun?: boolean
|
||||
queueBudget?: number
|
||||
stopAfterCurrent?: boolean
|
||||
refineExtensionFrame?: boolean
|
||||
saveLosslessAnchor?: boolean
|
||||
refinementDenoise?: number
|
||||
loraName?: string
|
||||
loraStack?: import('~/utils/loras').LoraStackItem[]
|
||||
shotLoras?: string[]
|
||||
|
||||
+11
-2
@@ -790,6 +790,11 @@ export function clipLastFramePath(owner: string, id: string) {
|
||||
return join(clipDir(owner, id), 'last-frame.png')
|
||||
}
|
||||
|
||||
/** Lossless PNG handoff frame for cleaner sequential extensions (not the preview cache). */
|
||||
export function clipAnchorFramePath(owner: string, id: string) {
|
||||
return join(clipDir(owner, id), 'anchor-last.png')
|
||||
}
|
||||
|
||||
export function clipFirstFramePath(owner: string, id: string) {
|
||||
return join(clipDir(owner, id), 'first-frame.jpg')
|
||||
}
|
||||
@@ -2009,9 +2014,9 @@ async function downloadViaHostAgent(file: { filename: string; subfolder: string;
|
||||
|
||||
async function downloadComfyMedia(
|
||||
file: { filename: string; subfolder: string; type: string },
|
||||
kind: 'video' | 'audio'
|
||||
kind: 'video' | 'audio' | 'image'
|
||||
) {
|
||||
const extras = kind === 'video' ? ['video', ''] : ['audio', '']
|
||||
const extras = kind === 'video' ? ['video', ''] : kind === 'audio' ? ['audio', ''] : ['']
|
||||
const subfolders = [...new Set([file.subfolder, ...extras])]
|
||||
let lastStatus = 0
|
||||
let lastError = ''
|
||||
@@ -2049,6 +2054,10 @@ export async function downloadComfyAudio(audio: { filename: string; subfolder: s
|
||||
return downloadComfyMedia(audio, 'audio')
|
||||
}
|
||||
|
||||
export async function downloadComfyImage(image: { filename: string; subfolder: string; type: string }) {
|
||||
return downloadComfyMedia(image, 'image')
|
||||
}
|
||||
|
||||
export async function importMissingComfyVideos(owner: string, folderId?: string) {
|
||||
const catalog = readCatalog(owner)
|
||||
const folder = catalog.folders.find(item => item.id === folderId) || catalog.folders[0]
|
||||
|
||||
@@ -56,6 +56,9 @@ export interface PendingJob {
|
||||
queueAutoRun?: boolean
|
||||
queueBudget?: number
|
||||
stopAfterCurrent?: boolean
|
||||
refineExtensionFrame?: boolean
|
||||
saveLosslessAnchor?: boolean
|
||||
refinementDenoise?: number
|
||||
loraName?: string
|
||||
loraStack?: import('~/utils/loras').LoraStackItem[]
|
||||
shotLoras?: string[]
|
||||
@@ -155,6 +158,9 @@ export function pendingFromJob(job: Job, overrides: Partial<PendingJob> = {}): P
|
||||
queueAutoRun: library.queueAutoRun,
|
||||
queueBudget: library.queueBudget,
|
||||
stopAfterCurrent: library.stopAfterCurrent,
|
||||
refineExtensionFrame: library.refineExtensionFrame,
|
||||
saveLosslessAnchor: library.saveLosslessAnchor,
|
||||
refinementDenoise: library.refinementDenoise,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
@@ -203,6 +209,9 @@ export function libraryFromPending(pending: PendingJob): NonNullable<Job['librar
|
||||
queueAutoRun: pending.queueAutoRun,
|
||||
queueBudget: pending.queueBudget,
|
||||
stopAfterCurrent: pending.stopAfterCurrent,
|
||||
refineExtensionFrame: pending.refineExtensionFrame,
|
||||
saveLosslessAnchor: pending.saveLosslessAnchor,
|
||||
refinementDenoise: pending.refinementDenoise,
|
||||
globalLocks: pending.globalLocks,
|
||||
permanenceRefs: pending.permanenceRefs,
|
||||
shotPermanenceRefs: pending.shotPermanenceRefs,
|
||||
@@ -342,6 +351,32 @@ export async function completePendingIfReady(pending: PendingJob) {
|
||||
loraStack: pending.loraStack,
|
||||
segmentFirstFrame
|
||||
})
|
||||
if (pending.saveLosslessAnchor === true) {
|
||||
try {
|
||||
const { persistClipAnchorFrame, extractAnchorImageFromHistory, downloadAnchorImage } = await import('~/server/utils/extensionFrame')
|
||||
let saved = false
|
||||
const anchorImage = extractAnchorImageFromHistory(history, pending.promptId)
|
||||
if (anchorImage) {
|
||||
const png = await downloadAnchorImage(anchorImage)
|
||||
if (png.length >= 64) {
|
||||
await persistClipAnchorFrame({
|
||||
ownerKey: pending.ownerKey,
|
||||
clipId: clip.id,
|
||||
fromBuffer: png
|
||||
})
|
||||
saved = true
|
||||
}
|
||||
}
|
||||
if (!saved) {
|
||||
await persistClipAnchorFrame({
|
||||
ownerKey: pending.ownerKey,
|
||||
clipId: clip.id
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
/* optional */
|
||||
}
|
||||
}
|
||||
await purgeComfyArtifacts({
|
||||
video,
|
||||
imageName: pending.imageName,
|
||||
|
||||
@@ -69,6 +69,9 @@ export interface StudioJobPayload {
|
||||
maskStillFilename?: string
|
||||
refineStrength?: number
|
||||
extendFromClipId?: string
|
||||
refineExtensionFrame?: boolean
|
||||
saveLosslessAnchor?: boolean
|
||||
refinementDenoise?: number
|
||||
lyrics?: string
|
||||
instrumental?: boolean
|
||||
lyricsStrength?: number
|
||||
@@ -1156,7 +1159,10 @@ async function startStudioExtendJob(item: StudioJob) {
|
||||
duration: item.payload.duration,
|
||||
loraStack: item.payload.loraStack || item.payload.loraName,
|
||||
folderLocked: item.payload.folderLocked,
|
||||
name: item.payload.name || item.name
|
||||
name: item.payload.name || item.name,
|
||||
refineExtensionFrame: item.payload.refineExtensionFrame,
|
||||
saveLosslessAnchor: item.payload.saveLosslessAnchor,
|
||||
refinementDenoise: item.payload.refinementDenoise
|
||||
})
|
||||
await markStudioLive(item.ownerKey, item.id, live.id)
|
||||
} catch (error) {
|
||||
@@ -1294,6 +1300,9 @@ export async function startStudioJob(item: StudioJob) {
|
||||
useIdentityRefs: identity,
|
||||
queueAutoRun: extensions.length ? payload.queueAutoRun : false,
|
||||
queueBudget: extensions.length && payload.queueAutoRun ? extensions.length : 0,
|
||||
refineExtensionFrame: payload.refineExtensionFrame !== false,
|
||||
saveLosslessAnchor: payload.saveLosslessAnchor === true,
|
||||
refinementDenoise: payload.refinementDenoise,
|
||||
globalLocks: payload.globalLocks,
|
||||
permanenceRefs: payload.permanenceRefs,
|
||||
shotPermanenceRefs: payload.shotPermanenceRefs,
|
||||
|
||||
@@ -7,9 +7,13 @@ import { comfyFilenamePrefix, queuePrompt, uploadImage } from '~/server/utils/co
|
||||
import { buildWorkflow } from '~/server/utils/workflow'
|
||||
import { isLtxWorkflow, isTextToVideo, parseVideoWorkflow, workflowForExtension, type VideoWorkflowId } from '~/utils/videoModels'
|
||||
import { clipVideoPath, clipTitle, deleteRetryDraft, extendTempDir, getClip, nextFamilyPartName, removeExtendTemp, stillPath } from '~/server/utils/library'
|
||||
import { extractLastFrame, probeHasAudio } from '~/server/utils/ffmpeg'
|
||||
import { probeHasAudio } from '~/server/utils/ffmpeg'
|
||||
import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
|
||||
import { onLiveVideoSettled, onStudioQueueBurstStarted } from '~/server/utils/studioQueue'
|
||||
import {
|
||||
refineExtensionHandoffFrame,
|
||||
resolveExtensionHandoffFrame
|
||||
} from '~/server/utils/extensionFrame'
|
||||
import {
|
||||
finishQueueBurst,
|
||||
getShotQueue,
|
||||
@@ -216,6 +220,7 @@ export async function queueMiniMax(
|
||||
duration: params.duration,
|
||||
useIdentityRefs: params.useIdentityRefs,
|
||||
referenceImageNames: params.useIdentityRefs ? referenceNames : [],
|
||||
saveLosslessAnchor: job.library?.saveLosslessAnchor === true,
|
||||
...persistLoraFields(params.loraStack || params.loraName)
|
||||
})
|
||||
|
||||
@@ -373,7 +378,12 @@ export async function continueQueuedExtensions(
|
||||
const parentVideo = parentId ? clipVideoPath(job.library.ownerKey, parentId) : ''
|
||||
const extractSource = parentVideo && existsSync(parentVideo) ? parentVideo : part1Path
|
||||
try {
|
||||
await extractLastFrame(extractSource, framePath)
|
||||
await resolveExtensionHandoffFrame({
|
||||
ownerKey: job.library.ownerKey,
|
||||
sourceClipId: parentId,
|
||||
sourceVideoPath: extractSource,
|
||||
destPath: framePath
|
||||
})
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : String(error)
|
||||
throw new Error(detail.includes('last frame')
|
||||
@@ -385,10 +395,30 @@ export async function continueQueuedExtensions(
|
||||
job.library.stopAfterCurrent = true
|
||||
break
|
||||
}
|
||||
const frame = readFileSync(framePath)
|
||||
let frame = readFileSync(framePath)
|
||||
if (!frame.length || frame.length < 64) {
|
||||
throw new Error('Could not extract the last frame for the next extension: the frame file was empty')
|
||||
}
|
||||
if (job.library.refineExtensionFrame !== false) {
|
||||
emitChainJob(job, { type: 'status', message: 'Refining extension handoff frame...', progress: 2 })
|
||||
try {
|
||||
frame = await refineExtensionHandoffFrame({
|
||||
frame,
|
||||
denoise: job.library.refinementDenoise,
|
||||
clientId: job.clientId,
|
||||
jobId: job.id,
|
||||
onStatus: (message) => emitChainJob(job, { type: 'status', message, progress: 3 })
|
||||
})
|
||||
writeFileSync(framePath, frame)
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : String(error)
|
||||
emitChainJob(job, {
|
||||
type: 'status',
|
||||
message: `Frame refine skipped (${detail}). Using source frame.`,
|
||||
progress: 3
|
||||
})
|
||||
}
|
||||
}
|
||||
job.library.extendPart1Path = part1Path
|
||||
job.library.prompt = composePromptParts(wrappers.promptPre, ext.prompt, wrappers.promptPost)
|
||||
job.library.promptMid = ext.prompt
|
||||
|
||||
@@ -275,6 +275,34 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
|
||||
loraStack: job.library.loraStack,
|
||||
segmentFirstFrame
|
||||
})
|
||||
if (job.library.saveLosslessAnchor === true) {
|
||||
try {
|
||||
const { persistClipAnchorFrame, extractAnchorImageFromHistory, downloadAnchorImage } = await import('~/server/utils/extensionFrame')
|
||||
let saved = false
|
||||
if (job.promptId && history) {
|
||||
const anchorImage = extractAnchorImageFromHistory(history, job.promptId)
|
||||
if (anchorImage) {
|
||||
const png = await downloadAnchorImage(anchorImage)
|
||||
if (png.length >= 64) {
|
||||
await persistClipAnchorFrame({
|
||||
ownerKey: job.library.ownerKey,
|
||||
clipId: clip.id,
|
||||
fromBuffer: png
|
||||
})
|
||||
saved = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!saved) {
|
||||
await persistClipAnchorFrame({
|
||||
ownerKey: job.library.ownerKey,
|
||||
clipId: clip.id
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
/* Anchor is optional; extension can still extract from MP4. */
|
||||
}
|
||||
}
|
||||
job.clipId = clip.id
|
||||
job.savedPromptId = job.promptId
|
||||
job.hideThumbnail = clip.hideThumbnail
|
||||
|
||||
@@ -40,6 +40,7 @@ export interface GenerateParams {
|
||||
referenceImageNames?: string[]
|
||||
loraName?: string
|
||||
loraStack?: LoraStackItem[]
|
||||
saveLosslessAnchor?: boolean
|
||||
}
|
||||
|
||||
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
|
||||
@@ -228,9 +229,34 @@ export function buildWorkflow(params: GenerateParams) {
|
||||
|
||||
resolveGraphLoraNames(graph, 'video')
|
||||
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
|
||||
attachLosslessAnchorSave(graph, params, version === 'v2' ? '164' : '105:90')
|
||||
return graph
|
||||
}
|
||||
|
||||
function attachLosslessAnchorSave(graph: WorkflowGraph, params: GenerateParams, decodeNodeId: string) {
|
||||
if (!params.saveLosslessAnchor) return
|
||||
if (!graph[decodeNodeId]) return
|
||||
const batchIndex = Math.max(0, Math.floor(Number(params.length) || 1) - 1)
|
||||
const prefix = String(params.filenamePrefix || 'video/MiniMax_H3').replace(/\/+$/, '')
|
||||
graph.aigen_anchor_pick = {
|
||||
class_type: 'ImageFromBatch',
|
||||
inputs: {
|
||||
image: [decodeNodeId, 0],
|
||||
batch_index: batchIndex,
|
||||
length: 1
|
||||
},
|
||||
_meta: { title: 'Last Decoded Frame' }
|
||||
}
|
||||
graph.aigen_anchor_save = {
|
||||
class_type: 'SaveImage',
|
||||
inputs: {
|
||||
filename_prefix: `${prefix}_last`,
|
||||
images: ['aigen_anchor_pick', 0]
|
||||
},
|
||||
_meta: { title: 'Save Lossless Anchor' }
|
||||
}
|
||||
}
|
||||
|
||||
function promptForNode(classType: string, useRefs: boolean, actionPrompt: string, graphPrompt: string) {
|
||||
if (classType === 'MiniMaxH3ImageToVideo') return useRefs ? '' : actionPrompt
|
||||
if (classType === 'MiniMaxH3TextToVideo') return actionPrompt
|
||||
@@ -302,6 +328,8 @@ function buildLtxWorkflow(params: GenerateParams, version: VideoWorkflowId) {
|
||||
}
|
||||
resolveGraphLoraNames(graph, 'video')
|
||||
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
|
||||
// LTX VAEDecode is typically node 15
|
||||
attachLosslessAnchorSave(graph, params, '15')
|
||||
return graph
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user