Preserve extension quality with original segments and reliable refinement

This commit is contained in:
Towsty
2026-09-08 06:25:40 -05:00
parent c2b6032e49
commit 7665285a0d
16 changed files with 222 additions and 98 deletions
+1 -1
View File
@@ -105,7 +105,7 @@ export default defineEventHandler(async (event) => {
queueAutoRun: false,
extendFromClipId: clipId,
refineExtensionFrame: parseRefineExtensionFrame(body?.refineExtensionFrame, true),
saveLosslessAnchor: parseSaveLosslessAnchor(body?.saveLosslessAnchor, false),
saveLosslessAnchor: parseSaveLosslessAnchor(body?.saveLosslessAnchor, true),
refinementDenoise: clampRefinementDenoise(body?.refinementDenoise, REFINEMENT_DENOISE_DEFAULT),
globalLocks: source.globalLocks,
permanenceRefs: source.permanenceRefs,
+1 -1
View File
@@ -257,7 +257,7 @@ export default defineEventHandler(async (event) => {
const familyId = crypto.randomUUID()
const queueAutoRun = fields.queueAutoRun === 'true'
const refineExtensionFrame = parseRefineExtensionFrame(fields.refineExtensionFrame, true)
const saveLosslessAnchor = parseSaveLosslessAnchor(fields.saveLosslessAnchor, false)
const saveLosslessAnchor = parseSaveLosslessAnchor(fields.saveLosslessAnchor, true)
const refinementDenoise = clampRefinementDenoise(fields.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
const studio = await addStudioJob({
ownerKey,
+5 -15
View File
@@ -4,16 +4,6 @@
"class_type": "LoadImage",
"_meta": { "title": "Handoff Frame" }
},
"2": {
"inputs": {
"images": ["1", 0],
"size": 2,
"sigma_color": 0.12,
"sigma_space": 1.5
},
"class_type": "BilateralFilterImage",
"_meta": { "title": "Bilateral Deblock" }
},
"3": {
"inputs": {
"unet_name": "flux-2-klein-base-9b-fp8.safetensors",
@@ -38,7 +28,7 @@
},
"6": {
"inputs": {
"lora_name": "Flux2-Klein-9B-consistency-V2.safetensors",
"lora_name": "klein-consistency-V2.safetensors",
"strength_model": 0.55,
"strength_clip": 0.55,
"model": ["3", 0],
@@ -49,7 +39,7 @@
},
"7": {
"inputs": {
"text": "photoreal still frame, natural high-frequency texture, clean edges, no compression blocks, no mosquito noise",
"text": "Preserve the source image, its exact subject, composition, colors and art style. Subtle cleanup of compression artifacts only. Retain fine texture and edges.",
"clip": ["6", 1]
},
"class_type": "CLIPTextEncode",
@@ -73,7 +63,7 @@
},
"10": {
"inputs": {
"pixels": ["2", 0],
"pixels": ["1", 0],
"vae": ["5", 0]
},
"class_type": "VAEEncode",
@@ -148,11 +138,11 @@
"inputs": {
"image1": ["1", 0],
"image2": ["18", 0],
"blend_factor": 0.85,
"blend_factor": 0.35,
"blend_mode": "normal"
},
"class_type": "ImageBlend",
"_meta": { "title": "Blend 0.85" }
"_meta": { "title": "Conservative cleanup blend" }
},
"20": {
"inputs": {
+7 -9
View File
@@ -125,7 +125,7 @@ export async function beginExtendFromClip(params: {
const height = source.height || 768
const clipName = String(params.name || '').trim() || nextFamilyPartName(params.ownerKey, source)
const refineExtensionFrame = parseRefineExtensionFrame(params.refineExtensionFrame, true)
const saveLosslessAnchor = parseSaveLosslessAnchor(params.saveLosslessAnchor, false)
const saveLosslessAnchor = parseSaveLosslessAnchor(params.saveLosslessAnchor, true)
const refinementDenoise = clampRefinementDenoise(params.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
const job = createJob()
@@ -252,7 +252,6 @@ async function runExtend(
})
})
const done = watchComfyJob(job)
if (params.refineExtensionFrame !== false) {
emitJob(job, {
type: 'status',
@@ -268,19 +267,18 @@ async function runExtend(
seed: params.seed,
clientId: job.clientId,
jobId: job.id,
onQueued: (promptId) => { job.promptId = promptId },
onStatus: (message) => emitJob(job, { type: 'status', message, progress: 4 })
})
writeFileSync(params.framePath, frame)
} catch (error) {
const detail = error instanceof Error ? error.message : String(error)
emitJob(job, {
type: 'status',
message: `Frame refine skipped (${detail}). Using source frame.`,
progress: 4
})
throw new Error(`Extension frame refinement failed: ${error instanceof Error ? error.message : String(error)}. Retry after fixing refinement, or explicitly turn it off.`)
}
}
if (job.status === 'cancelled' || job.status === 'error') throw new Error('Extension stopped before video generation')
job.promptId = undefined
const done = watchComfyJob(job)
job.status = 'uploading'
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 5 })
const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
@@ -310,7 +308,7 @@ async function runExtend(
workflow: job.library?.workflow,
duration: job.library?.duration,
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName,
saveLosslessAnchor: params.saveLosslessAnchor === true || job.library?.saveLosslessAnchor === true
saveLosslessAnchor: (params.saveLosslessAnchor ?? job.library?.saveLosslessAnchor) !== false
})
const queued = await queuePrompt(graph, job.clientId)
+14 -6
View File
@@ -1,8 +1,10 @@
import { videoSourcePaths } from './videoSources'
import { copyFileSync, existsSync, statSync, writeFileSync } from 'node:fs'
import refineTemplate from '../assets/extension_frame_refine.json'
import { clipAnchorFramePath, clipVideoPath, downloadComfyImage } from '~/server/utils/library'
import { extractLastFrame } from '~/server/utils/ffmpeg'
import { comfyFilenamePrefix, fetchHistory, queuePrompt, uploadImage } from '~/server/utils/comfy'
import { ensureComfyLoraNames, resolveGraphLoraNames } from '~/server/utils/loras'
import { extractEditedImage } from '~/server/utils/imageComfy'
import {
clampRefinementDenoise,
@@ -35,7 +37,8 @@ export async function resolveExtensionHandoffFrame(opts: {
copyFileSync(anchorPath, opts.destPath)
return { path: opts.destPath, source: 'anchor' as const }
}
await extractLastFrame(opts.sourceVideoPath, opts.destPath)
const originalSources = videoSourcePaths(opts.sourceVideoPath)
await extractLastFrame(originalSources[originalSources.length - 1], opts.destPath)
if (!isValidAnchorFile(opts.destPath)) {
throw new Error('Could not extract the last frame of the clip being extended')
}
@@ -60,11 +63,12 @@ export async function persistClipAnchorFrame(opts: {
}
const video = opts.videoPath || clipVideoPath(opts.ownerKey, opts.clipId)
if (!existsSync(video)) return ''
await extractLastFrame(video, dest)
const originalSources = videoSourcePaths(video)
await extractLastFrame(originalSources[originalSources.length - 1], dest)
return isValidAnchorFile(dest) ? dest : ''
}
function buildRefineGraph(opts: {
export function buildRefineGraph(opts: {
imageName: string
denoise: number
seed: number
@@ -95,7 +99,7 @@ async function waitForRefineImage(promptId: string, timeoutMs = 120_000) {
}
/**
* Bilateral + low-denoise Flux cleanup before the video I2V start frame.
* Low-denoise Flux cleanup before the video I2V start frame.
* Runs as its own Comfy prompt so MiniMax does not share VRAM with Klein.
* When refine is off, callers skip this and upload the PNG directly (fast path).
*/
@@ -106,6 +110,7 @@ export async function refineExtensionHandoffFrame(opts: {
clientId: string
jobId: string
onStatus?: (message: string) => void
onQueued?: (promptId: string) => void
}) {
const denoise = clampRefinementDenoise(opts.denoise, REFINEMENT_DENOISE_DEFAULT)
opts.onStatus?.('Refining extension handoff frame...')
@@ -121,7 +126,10 @@ export async function refineExtensionHandoffFrame(opts: {
seed: opts.seed ?? Math.floor(Math.random() * 2_147_483_647),
filenamePrefix: `aigen/extension_refine/${prefix}`
})
await ensureComfyLoraNames('image')
resolveGraphLoraNames(graph, 'image')
const queued = await queuePrompt(graph, opts.clientId)
opts.onQueued?.(queued.prompt_id)
const image = await waitForRefineImage(queued.prompt_id)
const buffer = await downloadComfyImage(image)
if (!buffer.length || buffer.length < MIN_ANCHOR_BYTES) {
@@ -134,7 +142,7 @@ export function extractAnchorImageFromHistory(history: Record<string, unknown> |
if (!history || !promptId) return null
const entry = history[promptId] as { outputs?: Record<string, unknown> } | undefined
const outputs = entry?.outputs
if (!outputs || typeof outputs !== 'object') return extractEditedImage(history, promptId)
if (!outputs || typeof outputs !== 'object') return null
const anchor = outputs.aigen_anchor_save as { images?: Array<{ filename?: string; subfolder?: string; type?: string }> } | undefined
const file = anchor?.images?.[0]
if (file?.filename) {
@@ -144,7 +152,7 @@ export function extractAnchorImageFromHistory(history: Record<string, unknown> |
type: file.type || 'output'
}
}
return extractEditedImage(history, promptId)
return null
}
export async function downloadAnchorImage(image: { filename: string; subfolder: string; type: string }) {
+47 -46
View File
@@ -284,48 +284,6 @@ async function assertStitchLength(part1Path: string, part2Path: string, outputPa
}
}
async function xfadeStitch(part1Path: string, part2Path: string, outputPath: string) {
const [duration1, duration2, size, audio1, audio2] = await Promise.all([
probeDuration(part1Path),
probeDuration(part2Path),
probeSize(part1Path),
probeHasAudio(part1Path),
probeHasAudio(part2Path)
])
const fade = Math.min(XFADE_DURATION, Math.max(0.04, duration1 - 0.08), Math.max(0.04, duration2 - 0.08))
const offset = Math.max(0, Number((duration1 - fade - 0.04).toFixed(3)))
const prepare1 = `scale=${size.width}:${size.height}:flags=bicubic,format=yuv420p,setpts=PTS-STARTPTS`
const prepare2 = `select='gte(n,1)',setpts=PTS-STARTPTS,scale=${size.width}:${size.height}:flags=bicubic,format=yuv420p`
const videoFilter = `[0:v]${prepare1}[v0];[1:v]${prepare2}[v1];[v0][v1]xfade=transition=fade:duration=${fade.toFixed(3)}:offset=${offset},setpts=PTS-STARTPTS[v]`
const runVideo = async (withAudio: boolean) => {
const args = [
'-y',
'-i', part1Path,
'-i', part2Path,
'-filter_complex', withAudio
? `${videoFilter};[0:a]asetpts=PTS-STARTPTS[a0];[1:a]asetpts=PTS-STARTPTS[a1];[a0][a1]acrossfade=d=${fade.toFixed(3)}:c1=tri:c2=tri[a]`
: videoFilter,
'-map', '[v]',
...(withAudio ? ['-map', '[a]'] : []),
...encodeArgs(outputPath, withAudio)
]
await runFfmpeg(args, 300000)
}
if (audio1 && audio2) {
try {
await runVideo(true)
} catch {
await runVideo(false)
}
} else {
await runVideo(false)
}
if (!existsSync(outputPath)) {
throw new Error('FFmpeg did not write the blended video')
}
await assertStitchLength(part1Path, part2Path, outputPath, fade)
}
async function concatReencode(part1Path: string, part2Path: string, outputPath: string) {
const audio1 = await probeHasAudio(part1Path)
const audio2 = await probeHasAudio(part2Path)
@@ -366,18 +324,61 @@ export async function concatMp4(part1Path: string, part2Path: string, outputPath
}
}
export async function stitchExtension(params: { part1Path: string; part2: Buffer; tmpDir: string }) {
export function buildSourceAssembly(durations: number[], width: number, height: number, fps: number, audio: boolean, blend: boolean) {
const filters: string[] = []
durations.forEach((_, index) => {
filters.push(`[${index}:v]${index ? "select='gte(n,1)'," : ''}setpts=PTS-STARTPTS,scale=${width}:${height}:flags=lanczos,fps=${fps},format=yuv420p,settb=AVTB[v${index}]`)
if (audio) filters.push(`[${index}:a]aresample=48000,asetpts=PTS-STARTPTS[a${index}]`)
})
let duration = durations[0]
let video = 'v0', sound = 'a0'
for (let index = 1; index < durations.length; index++) {
const fade = blend ? Math.min(XFADE_DURATION, Math.max(0.04, duration - 0.08), Math.max(0.04, durations[index] - 0.08)) : 0
const nextVideo = `joinv${index}`, nextAudio = `joina${index}`
const offset = Math.max(0, duration - fade)
filters.push(`[${video}][v${index}]` + (blend
? `xfade=transition=fade:duration=${fade.toFixed(6)}:offset=${offset.toFixed(6)}`
: 'concat=n=2:v=1:a=0') + `[${nextVideo}]`)
if (audio) filters.push(`[${sound}][a${index}]` + (blend
? `acrossfade=d=${fade.toFixed(6)}:c1=tri:c2=tri`
: 'concat=n=2:v=0:a=1') + `[${nextAudio}]`)
duration += durations[index] - 1 / fps - fade
video = nextVideo; sound = nextAudio
}
return { filter: filters.join(';'), video, sound, duration }
}
async function assembleOriginalSources(paths: string[], outputPath: string, blend: boolean) {
const [durations, size, audioFlags, frameRate] = await Promise.all([
Promise.all(paths.map(probeDuration)), probeSize(paths[0]),
Promise.all(paths.map(probeHasAudio)),
runFfprobe(['-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=r_frame_rate', '-of', 'default=noprint_wrappers=1:nokey=1', paths[0]])
])
const [numerator, denominator = 1] = frameRate.split('/').map(Number)
const fps = numerator / denominator
if (!Number.isFinite(fps) || fps <= 0) throw new Error('Cannot read the source video frame rate')
const audio = audioFlags.every(Boolean)
const graph = buildSourceAssembly(durations, size.width, size.height, fps, audio, blend)
await runFfmpeg(['-y', ...paths.flatMap(path => ['-i', path]), '-filter_complex_threads', '1',
'-filter_complex', graph.filter, '-map', `[${graph.video}]`, ...(audio ? ['-map', `[${graph.sound}]`] : []),
...encodeArgs(outputPath, audio)], 600000)
const actual = await probeDuration(outputPath)
if (Math.abs(actual - graph.duration) > Math.max(0.5, graph.duration * 0.02)) throw new Error('Assembled video length does not match the original segments')
}
export async function stitchExtension(params: { part1Path: string; part2: Buffer; tmpDir: string; sourcePaths?: string[] }) {
const part2Path = join(params.tmpDir, `part2_${Date.now()}.mp4`)
const finalPath = join(params.tmpDir, 'final_stitched_video.mp4')
writeFileSync(part2Path, params.part2)
if (!existsSync(part2Path) || !params.part2.length) {
throw new Error('Extension segment was missing before stitch')
}
const sources = [...(params.sourcePaths?.length ? params.sourcePaths : [params.part1Path]), part2Path]
try {
await xfadeStitch(params.part1Path, part2Path, finalPath)
await assembleOriginalSources(sources, finalPath, true)
} catch (error) {
console.error('[stitch] xfade failed, concatenating:', (error as Error).message)
await concatReencode(params.part1Path, part2Path, finalPath)
console.error('[stitch] source crossfade failed; joining the same original sources:', (error as Error).message)
await assembleOriginalSources(sources, finalPath, false)
}
const stitched = readFileSync(finalPath)
for (const extra of [part2Path, join(params.tmpDir, 'concat_list.txt'), join(params.tmpDir, 'last_frame.png')]) {
+4
View File
@@ -1,3 +1,4 @@
import { preserveVideoSources } from './videoSources'
import { createHash } from 'node:crypto'
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync, statSync, createReadStream, openSync, readSync, closeSync } from 'node:fs'
import { writeFile } from 'node:fs/promises'
@@ -1689,6 +1690,8 @@ export async function saveClip(params: {
promptPre?: string
promptPost?: string
segmentFirstFrame?: Buffer
sourceSegments?: string[]
originalSegment?: Buffer
}) {
const catalog = readCatalog(params.ownerKey)
const folder = catalog.folders.find(item => item.id === params.folderId) || catalog.folders[0]
@@ -1729,6 +1732,7 @@ export async function saveClip(params: {
const videoPath = clipVideoPath(params.ownerKey, clip.id)
await writeFile(videoPath, params.video)
await remuxFaststart(videoPath)
if (params.originalSegment) preserveVideoSources(videoPath, params.sourceSegments || [], params.originalSegment)
if (params.segmentFirstFrame?.length && params.segmentFirstFrame.length >= 64) {
const firstPath = clipFirstFramePath(params.ownerKey, clip.id)
writeFileSync(firstPath, params.segmentFirstFrame)
+12 -3
View File
@@ -1,3 +1,4 @@
import { videoSourcePaths } from './videoSources'
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import type { Job } from '~/server/utils/jobs'
@@ -295,6 +296,11 @@ export async function completePendingIfReady(pending: PendingJob) {
const video = extractVideo(history, pending.promptId)
if (!video) return null
let buffer = await downloadComfyVideo(video)
const originalSegment = buffer
const parentVideo = pending.parentClipId ? clipVideoPath(pending.ownerKey, pending.parentClipId) : ''
const sourceSegments = pending.extendPart1Path
? videoSourcePaths(parentVideo && existsSync(parentVideo) ? parentVideo : pending.extendPart1Path)
: []
if (inFlight()) return null
const afterDownload = alreadySaved()
if (afterDownload) return afterDownload
@@ -313,13 +319,13 @@ export async function completePendingIfReady(pending: PendingJob) {
buffer = await stitchExtension({
part1Path: pending.extendPart1Path,
part2: buffer,
sourcePaths: sourceSegments,
tmpDir: pending.extendTmpDir
})
} catch (error) {
removeExtendTemp(pending.extendTmpDir)
throw error
}
removeExtendTemp(pending.extendTmpDir)
}
const afterStitch = alreadySaved()
if (afterStitch) return afterStitch
@@ -349,9 +355,12 @@ export async function completePendingIfReady(pending: PendingJob) {
permanenceRefs: mergePermanenceRefs(pending.permanenceRefs, pending.shotPermanenceRefs?.[pending.chainIndex || 0]),
loraName: pending.loraName,
loraStack: pending.loraStack,
segmentFirstFrame
segmentFirstFrame,
originalSegment,
sourceSegments
})
if (pending.saveLosslessAnchor === true) {
if (pending.extendTmpDir) removeExtendTemp(pending.extendTmpDir)
if (pending.saveLosslessAnchor !== false) {
try {
const { persistClipAnchorFrame, extractAnchorImageFromHistory, downloadAnchorImage } = await import('~/server/utils/extensionFrame')
let saved = false
+1 -1
View File
@@ -1503,7 +1503,7 @@ async function startStudioJobReserved(item: StudioJob) {
queueAutoRun: extensions.length ? payload.queueAutoRun : false,
queueBudget: extensions.length && payload.queueAutoRun ? extensions.length : 0,
refineExtensionFrame: payload.refineExtensionFrame !== false,
saveLosslessAnchor: payload.saveLosslessAnchor === true,
saveLosslessAnchor: payload.saveLosslessAnchor !== false,
refinementDenoise: payload.refinementDenoise,
globalLocks: payload.globalLocks,
permanenceRefs: payload.permanenceRefs,
+4 -7
View File
@@ -221,7 +221,7 @@ export async function queueMiniMax(
duration: params.duration,
useIdentityRefs: params.useIdentityRefs,
referenceImageNames: params.useIdentityRefs ? referenceNames : [],
saveLosslessAnchor: job.library?.saveLosslessAnchor === true,
saveLosslessAnchor: job.library?.saveLosslessAnchor !== false,
...persistLoraFields(params.loraStack || params.loraName)
})
@@ -414,18 +414,15 @@ export async function continueQueuedExtensions(
denoise: job.library.refinementDenoise,
clientId: job.clientId,
jobId: job.id,
onQueued: (promptId) => { job.promptId = promptId },
onStatus: (message) => emitChainJob(job, { type: 'status', message, progress: 3 })
})
writeFileSync(framePath, frame)
} catch (error) {
const detail = error instanceof Error ? error.message : String(error)
emitChainJob(job, {
type: 'status',
message: `Frame refine skipped (${detail}). Using source frame.`,
progress: 3
})
throw new Error(`Extension frame refinement failed: ${error instanceof Error ? error.message : String(error)}. Retry after fixing refinement, or explicitly turn it off.`)
}
}
assertJobActive(job)
job.library.extendPart1Path = part1Path
job.library.prompt = composePromptParts(wrappers.promptPre, ext.prompt, wrappers.promptPost)
job.library.promptMid = ext.prompt
+23
View File
@@ -0,0 +1,23 @@
import { existsSync, mkdirSync, readdirSync, linkSync, copyFileSync, writeFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
/** Each saved clip owns its sources, so deleting a parent cannot break extensions. */
export function videoSourcePaths(videoPath: string) {
const dir = join(dirname(videoPath), 'sources')
if (existsSync(dir)) {
const paths = readdirSync(dir).filter(name => /^\d{6}\.mp4$/.test(name)).sort().map(name => join(dir, name))
if (paths.length) return paths
}
// Legacy clips become a fixed baseline; their past compression is irreversible.
return [videoPath]
}
export function preserveVideoSources(videoPath: string, parents: string[], segment: Buffer) {
const dir = join(dirname(videoPath), 'sources')
mkdirSync(dir, { recursive: true })
parents.forEach((source, index) => {
const dest = join(dir, `${String(index).padStart(6, '0')}.mp4`)
try { linkSync(source, dest) } catch { copyFileSync(source, dest) }
})
writeFileSync(join(dir, `${String(parents.length).padStart(6, '0')}.mp4`), segment)
}
+11 -2
View File
@@ -1,3 +1,4 @@
import { videoSourcePaths } from './videoSources'
import { existsSync } from 'node:fs'
import type { Job, JobEvent } from '~/server/utils/jobs'
import { getJob, restoreJob } from '~/server/utils/jobs'
@@ -221,6 +222,11 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
})
if (job.library) {
let buffer = await downloadComfyVideo(video)
const originalSegment = buffer
const parentVideo = job.library.parentClipId ? clipVideoPath(job.library.ownerKey, job.library.parentClipId) : ''
const sourceSegments = job.library.extendPart1Path
? videoSourcePaths(parentVideo && existsSync(parentVideo) ? parentVideo : job.library.extendPart1Path)
: []
let segmentFirstFrame: Buffer | undefined
if (stitching) {
if (!existsSync(job.library.extendPart1Path!)) {
@@ -236,6 +242,7 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
buffer = await stitchExtension({
part1Path: job.library.extendPart1Path!,
part2: buffer,
sourcePaths: sourceSegments,
tmpDir: job.library.extendTmpDir!
})
}
@@ -273,9 +280,11 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
permanenceRefs: clipRefsFromJob(job.library),
loraName: job.library.loraName,
loraStack: job.library.loraStack,
segmentFirstFrame
segmentFirstFrame,
originalSegment,
sourceSegments
})
if (job.library.saveLosslessAnchor === true) {
if (job.library.saveLosslessAnchor !== false) {
try {
const { persistClipAnchorFrame, extractAnchorImageFromHistory, downloadAnchorImage } = await import('~/server/utils/extensionFrame')
let saved = false