Add extension-frame refine and lossless anchor options.
Prefer clip anchor PNGs for handoffs, optionally run Bilateral+Flux cleanup before I2V, and save decoded last-frame anchors so sequential extensions accumulate less compression damage. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+89
-1
@@ -1109,6 +1109,41 @@
|
|||||||
<span class="mt-0.5 block text-xs text-zinc-500">Overnight set-and-forget. Leave this off to edit prompts on Queue (under this job) and process the next shot when you are ready.</span>
|
<span class="mt-0.5 block text-xs text-zinc-500">Overnight set-and-forget. Leave this off to edit prompts on Queue (under this job) and process the next shot when you are ready.</span>
|
||||||
</span>
|
</span>
|
||||||
</label>
|
</label>
|
||||||
|
<div v-if="studioMode === 'video'" class="space-y-3">
|
||||||
|
<label
|
||||||
|
class="flex cursor-pointer items-start gap-3 rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3"
|
||||||
|
title="Runs a low-denoise Bilateral + Flux cleanup pass on the handoff frame to strip compression artifacts."
|
||||||
|
>
|
||||||
|
<input v-model="refineExtensionFrame" type="checkbox" class="mt-1">
|
||||||
|
<span>
|
||||||
|
<span class="block text-sm font-medium text-zinc-200">Refine Extension Frame</span>
|
||||||
|
<span class="mt-0.5 block text-xs text-zinc-500">Runs a low-denoise Bilateral + Flux cleanup pass on the handoff frame to strip compression artifacts.</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<div
|
||||||
|
v-if="refineExtensionFrame"
|
||||||
|
class="rounded-2xl border border-white/10 bg-zinc-950/40 p-3"
|
||||||
|
>
|
||||||
|
<Thumbwheel
|
||||||
|
v-model="refinementDenoise"
|
||||||
|
label="Refinement Denoise"
|
||||||
|
:min="REFINEMENT_DENOISE_MIN"
|
||||||
|
:max="REFINEMENT_DENOISE_MAX"
|
||||||
|
:step="0.01"
|
||||||
|
:digits="2"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<label
|
||||||
|
class="flex cursor-pointer items-start gap-3 rounded-2xl border border-white/10 bg-zinc-950/40 px-4 py-3"
|
||||||
|
title="Saves an uncompressed PNG of the final frame alongside the MP4 for cleaner future extensions."
|
||||||
|
>
|
||||||
|
<input v-model="saveLosslessAnchor" type="checkbox" class="mt-1">
|
||||||
|
<span>
|
||||||
|
<span class="block text-sm font-medium text-zinc-200">Save Lossless Anchor Frame</span>
|
||||||
|
<span class="mt-0.5 block text-xs text-zinc-500">Saves an uncompressed PNG of the final frame alongside the MP4 for cleaner future extensions.</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
<div class="flex flex-wrap gap-3">
|
<div class="flex flex-wrap gap-3">
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
@@ -1458,6 +1493,38 @@
|
|||||||
@insert-trigger="insertLoraTrigger($event, 'extend')"
|
@insert-trigger="insertLoraTrigger($event, 'extend')"
|
||||||
@update-trigger="saveLoraTrigger"
|
@update-trigger="saveLoraTrigger"
|
||||||
/>
|
/>
|
||||||
|
<div class="space-y-3">
|
||||||
|
<label
|
||||||
|
class="flex cursor-pointer items-start gap-3 rounded-xl border border-white/10 bg-zinc-950/40 px-3 py-2.5"
|
||||||
|
title="Runs a low-denoise Bilateral + Flux cleanup pass on the handoff frame to strip compression artifacts."
|
||||||
|
>
|
||||||
|
<input v-model="refineExtensionFrame" type="checkbox" class="mt-1">
|
||||||
|
<span>
|
||||||
|
<span class="block text-sm font-medium text-zinc-200">Refine Extension Frame</span>
|
||||||
|
<span class="mt-0.5 block text-xs text-zinc-500">Bilateral + Flux cleanup on the handoff frame.</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<div v-if="refineExtensionFrame" class="rounded-xl border border-white/10 bg-zinc-950/40 p-3">
|
||||||
|
<Thumbwheel
|
||||||
|
v-model="refinementDenoise"
|
||||||
|
label="Refinement Denoise"
|
||||||
|
:min="REFINEMENT_DENOISE_MIN"
|
||||||
|
:max="REFINEMENT_DENOISE_MAX"
|
||||||
|
:step="0.01"
|
||||||
|
:digits="2"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<label
|
||||||
|
class="flex cursor-pointer items-start gap-3 rounded-xl border border-white/10 bg-zinc-950/40 px-3 py-2.5"
|
||||||
|
title="Saves an uncompressed PNG of the final frame alongside the MP4 for cleaner future extensions."
|
||||||
|
>
|
||||||
|
<input v-model="saveLosslessAnchor" type="checkbox" class="mt-1">
|
||||||
|
<span>
|
||||||
|
<span class="block text-sm font-medium text-zinc-200">Save Lossless Anchor Frame</span>
|
||||||
|
<span class="mt-0.5 block text-xs text-zinc-500">Keep a PNG anchor with this extension for cleaner next hops.</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="w-full rounded-2xl bg-amber-400 px-4 py-2.5 text-sm font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50"
|
class="w-full rounded-2xl bg-amber-400 px-4 py-2.5 text-sm font-semibold text-zinc-950 transition hover:bg-amber-300 disabled:cursor-not-allowed disabled:opacity-50"
|
||||||
@@ -2324,6 +2391,12 @@ import {
|
|||||||
defaultVideoDurationPref,
|
defaultVideoDurationPref,
|
||||||
setDefaultVideoDurationPref
|
setDefaultVideoDurationPref
|
||||||
} from '~/utils/videoDuration'
|
} from '~/utils/videoDuration'
|
||||||
|
import {
|
||||||
|
clampRefinementDenoise,
|
||||||
|
REFINEMENT_DENOISE_DEFAULT,
|
||||||
|
REFINEMENT_DENOISE_MAX,
|
||||||
|
REFINEMENT_DENOISE_MIN
|
||||||
|
} from '~/utils/extensionFrame'
|
||||||
|
|
||||||
const runtimeConfig = useRuntimeConfig()
|
const runtimeConfig = useRuntimeConfig()
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
@@ -2758,6 +2831,9 @@ const renameDraft = ref('')
|
|||||||
const renameInput = ref<HTMLInputElement | null>(null)
|
const renameInput = ref<HTMLInputElement | null>(null)
|
||||||
const unlockPassword = ref('')
|
const unlockPassword = ref('')
|
||||||
const queueAutoRun = ref(false)
|
const queueAutoRun = ref(false)
|
||||||
|
const refineExtensionFrame = ref(true)
|
||||||
|
const saveLosslessAnchor = ref(false)
|
||||||
|
const refinementDenoise = ref(REFINEMENT_DENOISE_DEFAULT)
|
||||||
const queuePendingCount = ref(0)
|
const queuePendingCount = ref(0)
|
||||||
const activeQueueId = ref('')
|
const activeQueueId = ref('')
|
||||||
const liveQueue = ref<{
|
const liveQueue = ref<{
|
||||||
@@ -5452,6 +5528,9 @@ async function restoreStudioPayload(kind: string, payload: {
|
|||||||
referenceStillIds?: Array<string | null>
|
referenceStillIds?: Array<string | null>
|
||||||
extensions?: { prompt: string; promptPre?: string; promptPost?: string; duration: number; permanenceRefs?: PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[]
|
extensions?: { prompt: string; promptPre?: string; promptPost?: string; duration: number; permanenceRefs?: PermanenceRef[]; loraName?: string; loraStack?: LoraStackItem[] }[]
|
||||||
queueAutoRun?: boolean
|
queueAutoRun?: boolean
|
||||||
|
refineExtensionFrame?: boolean
|
||||||
|
saveLosslessAnchor?: boolean
|
||||||
|
refinementDenoise?: number
|
||||||
promptPre?: string
|
promptPre?: string
|
||||||
promptPost?: string
|
promptPost?: string
|
||||||
globalLocks?: string
|
globalLocks?: string
|
||||||
@@ -5520,6 +5599,9 @@ async function restoreStudioPayload(kind: string, payload: {
|
|||||||
readLoraStack(item)
|
readLoraStack(item)
|
||||||
))
|
))
|
||||||
queueAutoRun.value = payload.queueAutoRun === true
|
queueAutoRun.value = payload.queueAutoRun === true
|
||||||
|
refineExtensionFrame.value = payload.refineExtensionFrame !== false
|
||||||
|
saveLosslessAnchor.value = payload.saveLosslessAnchor === true
|
||||||
|
refinementDenoise.value = clampRefinementDenoise(payload.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
|
||||||
globalLocks.value = payload.globalLocks || ''
|
globalLocks.value = payload.globalLocks || ''
|
||||||
familyPermanenceRefs.value = payload.permanenceRefs || []
|
familyPermanenceRefs.value = payload.permanenceRefs || []
|
||||||
const shotRefs: Record<number, PermanenceRef[]> = {}
|
const shotRefs: Record<number, PermanenceRef[]> = {}
|
||||||
@@ -6113,7 +6195,10 @@ async function generateExtension() {
|
|||||||
promptPre: extendPromptPre.value,
|
promptPre: extendPromptPre.value,
|
||||||
promptPost: extendPromptPost.value,
|
promptPost: extendPromptPost.value,
|
||||||
duration: extendDuration.value,
|
duration: extendDuration.value,
|
||||||
loraStack: extendLoraStack.value
|
loraStack: extendLoraStack.value,
|
||||||
|
refineExtensionFrame: refineExtensionFrame.value,
|
||||||
|
saveLosslessAnchor: saveLosslessAnchor.value,
|
||||||
|
refinementDenoise: clampRefinementDenoise(refinementDenoise.value)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
if (queuedOffLiveOutput(started, 'video')) {
|
if (queuedOffLiveOutput(started, 'video')) {
|
||||||
@@ -7391,6 +7476,9 @@ async function generate() {
|
|||||||
}))))
|
}))))
|
||||||
body.append('queueAutoRun', String(queueAutoRun.value))
|
body.append('queueAutoRun', String(queueAutoRun.value))
|
||||||
}
|
}
|
||||||
|
body.append('refineExtensionFrame', String(refineExtensionFrame.value))
|
||||||
|
body.append('saveLosslessAnchor', String(saveLosslessAnchor.value))
|
||||||
|
body.append('refinementDenoise', String(clampRefinementDenoise(refinementDenoise.value)))
|
||||||
const downloadName = clipFileName(clipName.value.trim() || autoOutputName.value)
|
const downloadName = clipFileName(clipName.value.trim() || autoOutputName.value)
|
||||||
const chainPlan = plannedChain.value.map(part => ({ ...part }))
|
const chainPlan = plannedChain.value.map(part => ({ ...part }))
|
||||||
console.info('[aigen generate payload]', {
|
console.info('[aigen generate payload]', {
|
||||||
|
|||||||
@@ -6,9 +6,26 @@ import { clipPartDuration, clampVideoDuration } from '~/utils/videoDuration'
|
|||||||
import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
|
import { defaultVideoSteps, isLtxWorkflow, LTX_DISABLED_MESSAGE, ltxWorkflowEnabled, parseVideoWorkflow } from '~/utils/videoModels'
|
||||||
import { composePromptParts } from '~/utils/promptParts'
|
import { composePromptParts } from '~/utils/promptParts'
|
||||||
import { nextFamilyPartName } from '~/server/utils/library'
|
import { nextFamilyPartName } from '~/server/utils/library'
|
||||||
|
import {
|
||||||
|
clampRefinementDenoise,
|
||||||
|
parseRefineExtensionFrame,
|
||||||
|
parseSaveLosslessAnchor,
|
||||||
|
REFINEMENT_DENOISE_DEFAULT
|
||||||
|
} from '~/utils/extensionFrame'
|
||||||
|
|
||||||
export default defineEventHandler(async (event) => {
|
export default defineEventHandler(async (event) => {
|
||||||
const body = await readBody<{ clipId?: string; prompt?: string; promptPre?: string; promptPost?: string; duration?: number; lora?: unknown; loraStack?: unknown }>(event).catch(() => ({}))
|
const body = await readBody<{
|
||||||
|
clipId?: string
|
||||||
|
prompt?: string
|
||||||
|
promptPre?: string
|
||||||
|
promptPost?: string
|
||||||
|
duration?: number
|
||||||
|
lora?: unknown
|
||||||
|
loraStack?: unknown
|
||||||
|
refineExtensionFrame?: boolean | string
|
||||||
|
saveLosslessAnchor?: boolean | string
|
||||||
|
refinementDenoise?: number | string
|
||||||
|
}>(event).catch(() => ({}))
|
||||||
const clipId = String(body?.clipId || '').trim()
|
const clipId = String(body?.clipId || '').trim()
|
||||||
const prompt = String(body?.prompt || '').trim()
|
const prompt = String(body?.prompt || '').trim()
|
||||||
if (!clipId) {
|
if (!clipId) {
|
||||||
@@ -87,6 +104,9 @@ export default defineEventHandler(async (event) => {
|
|||||||
extensions: [],
|
extensions: [],
|
||||||
queueAutoRun: false,
|
queueAutoRun: false,
|
||||||
extendFromClipId: clipId,
|
extendFromClipId: clipId,
|
||||||
|
refineExtensionFrame: parseRefineExtensionFrame(body?.refineExtensionFrame, true),
|
||||||
|
saveLosslessAnchor: parseSaveLosslessAnchor(body?.saveLosslessAnchor, false),
|
||||||
|
refinementDenoise: clampRefinementDenoise(body?.refinementDenoise, REFINEMENT_DENOISE_DEFAULT),
|
||||||
globalLocks: source.globalLocks,
|
globalLocks: source.globalLocks,
|
||||||
permanenceRefs: source.permanenceRefs,
|
permanenceRefs: source.permanenceRefs,
|
||||||
...persistLoraFields(loraStack)
|
...persistLoraFields(loraStack)
|
||||||
|
|||||||
@@ -6,6 +6,12 @@ import { defaultVideoSteps, isLtxWorkflow, isTextToVideo, LTX_DISABLED_MESSAGE,
|
|||||||
import { allowIdentityRefs, normalizePermanenceRefs, resolveGlobalLocks, type PermanenceRef } from '~/utils/globalLocks'
|
import { allowIdentityRefs, normalizePermanenceRefs, resolveGlobalLocks, type PermanenceRef } from '~/utils/globalLocks'
|
||||||
import { composePromptParts, ingestPromptParts, persistPromptWrappers } from '~/utils/promptParts'
|
import { composePromptParts, ingestPromptParts, persistPromptWrappers } from '~/utils/promptParts'
|
||||||
import type { QueuedExtension } from '~/server/utils/library'
|
import type { QueuedExtension } from '~/server/utils/library'
|
||||||
|
import {
|
||||||
|
clampRefinementDenoise,
|
||||||
|
parseRefineExtensionFrame,
|
||||||
|
parseSaveLosslessAnchor,
|
||||||
|
REFINEMENT_DENOISE_DEFAULT
|
||||||
|
} from '~/utils/extensionFrame'
|
||||||
|
|
||||||
function parseDuration(raw: unknown) {
|
function parseDuration(raw: unknown) {
|
||||||
const seconds = Number(raw)
|
const seconds = Number(raw)
|
||||||
@@ -250,6 +256,9 @@ export default defineEventHandler(async (event) => {
|
|||||||
const chainTotal = 1 + extensions.length
|
const chainTotal = 1 + extensions.length
|
||||||
const familyId = crypto.randomUUID()
|
const familyId = crypto.randomUUID()
|
||||||
const queueAutoRun = fields.queueAutoRun === 'true'
|
const queueAutoRun = fields.queueAutoRun === 'true'
|
||||||
|
const refineExtensionFrame = parseRefineExtensionFrame(fields.refineExtensionFrame, true)
|
||||||
|
const saveLosslessAnchor = parseSaveLosslessAnchor(fields.saveLosslessAnchor, false)
|
||||||
|
const refinementDenoise = clampRefinementDenoise(fields.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
|
||||||
const studio = await addStudioJob({
|
const studio = await addStudioJob({
|
||||||
ownerKey,
|
ownerKey,
|
||||||
familyId,
|
familyId,
|
||||||
@@ -282,6 +291,9 @@ export default defineEventHandler(async (event) => {
|
|||||||
referenceStillIds,
|
referenceStillIds,
|
||||||
extensions,
|
extensions,
|
||||||
queueAutoRun,
|
queueAutoRun,
|
||||||
|
refineExtensionFrame,
|
||||||
|
saveLosslessAnchor,
|
||||||
|
refinementDenoise,
|
||||||
globalLocks: globalLocks || undefined,
|
globalLocks: globalLocks || undefined,
|
||||||
permanenceRefs: permanenceRefs.length ? permanenceRefs : undefined,
|
permanenceRefs: permanenceRefs.length ? permanenceRefs : undefined,
|
||||||
shotPermanenceRefs: shotPermanenceRefs.some(item => item.length) ? shotPermanenceRefs : undefined,
|
shotPermanenceRefs: shotPermanenceRefs.some(item => item.length) ? shotPermanenceRefs : undefined,
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
{
|
||||||
|
"1": {
|
||||||
|
"inputs": { "image": "last_frame.png" },
|
||||||
|
"class_type": "LoadImage",
|
||||||
|
"_meta": { "title": "Handoff Frame" }
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"inputs": {
|
||||||
|
"images": ["1", 0],
|
||||||
|
"size": 2,
|
||||||
|
"sigma_color": 0.12,
|
||||||
|
"sigma_space": 1.5
|
||||||
|
},
|
||||||
|
"class_type": "BilateralFilterImage",
|
||||||
|
"_meta": { "title": "Bilateral Deblock" }
|
||||||
|
},
|
||||||
|
"3": {
|
||||||
|
"inputs": {
|
||||||
|
"unet_name": "flux-2-klein-base-9b-fp8.safetensors",
|
||||||
|
"weight_dtype": "default"
|
||||||
|
},
|
||||||
|
"class_type": "UNETLoader",
|
||||||
|
"_meta": { "title": "Load Flux.2 Klein" }
|
||||||
|
},
|
||||||
|
"4": {
|
||||||
|
"inputs": {
|
||||||
|
"clip_name": "qwen_3_8b_fp8mixed.safetensors",
|
||||||
|
"type": "flux2",
|
||||||
|
"device": "default"
|
||||||
|
},
|
||||||
|
"class_type": "CLIPLoader",
|
||||||
|
"_meta": { "title": "Load CLIP" }
|
||||||
|
},
|
||||||
|
"5": {
|
||||||
|
"inputs": { "vae_name": "full_encoder_small_decoder.safetensors" },
|
||||||
|
"class_type": "VAELoader",
|
||||||
|
"_meta": { "title": "Load Klein VAE" }
|
||||||
|
},
|
||||||
|
"6": {
|
||||||
|
"inputs": {
|
||||||
|
"lora_name": "Flux2-Klein-9B-consistency-V2.safetensors",
|
||||||
|
"strength_model": 0.55,
|
||||||
|
"strength_clip": 0.55,
|
||||||
|
"model": ["3", 0],
|
||||||
|
"clip": ["4", 0]
|
||||||
|
},
|
||||||
|
"class_type": "LoraLoader",
|
||||||
|
"_meta": { "title": "Consistency" }
|
||||||
|
},
|
||||||
|
"7": {
|
||||||
|
"inputs": {
|
||||||
|
"text": "photoreal still frame, natural high-frequency texture, clean edges, no compression blocks, no mosquito noise",
|
||||||
|
"clip": ["6", 1]
|
||||||
|
},
|
||||||
|
"class_type": "CLIPTextEncode",
|
||||||
|
"_meta": { "title": "Positive" }
|
||||||
|
},
|
||||||
|
"8": {
|
||||||
|
"inputs": {
|
||||||
|
"text": "compression artifacts, blockiness, banding, blur, smear, plastic skin",
|
||||||
|
"clip": ["6", 1]
|
||||||
|
},
|
||||||
|
"class_type": "CLIPTextEncode",
|
||||||
|
"_meta": { "title": "Negative" }
|
||||||
|
},
|
||||||
|
"9": {
|
||||||
|
"inputs": {
|
||||||
|
"guidance": 1.8,
|
||||||
|
"conditioning": ["7", 0]
|
||||||
|
},
|
||||||
|
"class_type": "FluxGuidance",
|
||||||
|
"_meta": { "title": "Flux Guidance" }
|
||||||
|
},
|
||||||
|
"10": {
|
||||||
|
"inputs": {
|
||||||
|
"pixels": ["2", 0],
|
||||||
|
"vae": ["5", 0]
|
||||||
|
},
|
||||||
|
"class_type": "VAEEncode",
|
||||||
|
"_meta": { "title": "Encode Filtered" }
|
||||||
|
},
|
||||||
|
"11": {
|
||||||
|
"inputs": {
|
||||||
|
"conditioning": ["9", 0],
|
||||||
|
"latent": ["10", 0]
|
||||||
|
},
|
||||||
|
"class_type": "ReferenceLatent",
|
||||||
|
"_meta": { "title": "Reference +" }
|
||||||
|
},
|
||||||
|
"12": {
|
||||||
|
"inputs": {
|
||||||
|
"conditioning": ["8", 0],
|
||||||
|
"latent": ["10", 0]
|
||||||
|
},
|
||||||
|
"class_type": "ReferenceLatent",
|
||||||
|
"_meta": { "title": "Reference -" }
|
||||||
|
},
|
||||||
|
"13": {
|
||||||
|
"inputs": { "noise_seed": 1 },
|
||||||
|
"class_type": "RandomNoise",
|
||||||
|
"_meta": { "title": "Noise" }
|
||||||
|
},
|
||||||
|
"14": {
|
||||||
|
"inputs": { "sampler_name": "euler" },
|
||||||
|
"class_type": "KSamplerSelect",
|
||||||
|
"_meta": { "title": "Sampler" }
|
||||||
|
},
|
||||||
|
"15": {
|
||||||
|
"inputs": {
|
||||||
|
"scheduler": "simple",
|
||||||
|
"steps": 14,
|
||||||
|
"denoise": 0.18,
|
||||||
|
"model": ["6", 0]
|
||||||
|
},
|
||||||
|
"class_type": "BasicScheduler",
|
||||||
|
"_meta": { "title": "Scheduler" }
|
||||||
|
},
|
||||||
|
"16": {
|
||||||
|
"inputs": {
|
||||||
|
"cfg": 1,
|
||||||
|
"model": ["6", 0],
|
||||||
|
"positive": ["11", 0],
|
||||||
|
"negative": ["12", 0]
|
||||||
|
},
|
||||||
|
"class_type": "CFGGuider",
|
||||||
|
"_meta": { "title": "CFG 1.0" }
|
||||||
|
},
|
||||||
|
"17": {
|
||||||
|
"inputs": {
|
||||||
|
"noise": ["13", 0],
|
||||||
|
"guider": ["16", 0],
|
||||||
|
"sampler": ["14", 0],
|
||||||
|
"sigmas": ["15", 0],
|
||||||
|
"latent_image": ["10", 0]
|
||||||
|
},
|
||||||
|
"class_type": "SamplerCustomAdvanced",
|
||||||
|
"_meta": { "title": "Img2Img Cleanup" }
|
||||||
|
},
|
||||||
|
"18": {
|
||||||
|
"inputs": {
|
||||||
|
"samples": ["17", 0],
|
||||||
|
"vae": ["5", 0]
|
||||||
|
},
|
||||||
|
"class_type": "VAEDecode",
|
||||||
|
"_meta": { "title": "Decode" }
|
||||||
|
},
|
||||||
|
"19": {
|
||||||
|
"inputs": {
|
||||||
|
"image1": ["1", 0],
|
||||||
|
"image2": ["18", 0],
|
||||||
|
"blend_factor": 0.85,
|
||||||
|
"blend_mode": "normal"
|
||||||
|
},
|
||||||
|
"class_type": "ImageBlend",
|
||||||
|
"_meta": { "title": "Blend 0.85" }
|
||||||
|
},
|
||||||
|
"20": {
|
||||||
|
"inputs": {
|
||||||
|
"filename_prefix": "aigen/extension_refine",
|
||||||
|
"images": ["19", 0]
|
||||||
|
},
|
||||||
|
"class_type": "SaveImage",
|
||||||
|
"_meta": { "title": "Save Refined Frame" }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import { copyFileSync, existsSync, readFileSync } from 'node:fs'
|
import { copyFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||||||
import { join } from 'node:path'
|
import { join } from 'node:path'
|
||||||
import { composeShotPrompt } from '~/utils/globalLocks'
|
import { composeShotPrompt } from '~/utils/globalLocks'
|
||||||
import { composePromptParts } from '~/utils/promptParts'
|
import { composePromptParts } from '~/utils/promptParts'
|
||||||
@@ -16,12 +16,22 @@ import {
|
|||||||
nextFamilyPartName,
|
nextFamilyPartName,
|
||||||
removeExtendTemp
|
removeExtendTemp
|
||||||
} from '~/server/utils/library'
|
} from '~/server/utils/library'
|
||||||
import { extractLastFrame, probeHasAudio } from '~/server/utils/ffmpeg'
|
import { probeHasAudio } from '~/server/utils/ffmpeg'
|
||||||
import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
|
import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
|
||||||
import { waitForComfySocket, watchComfyJob } from '~/server/utils/watch'
|
import { waitForComfySocket, watchComfyJob } from '~/server/utils/watch'
|
||||||
import { comfyFilenamePrefix, queuePrompt, uploadImage } from '~/server/utils/comfy'
|
import { comfyFilenamePrefix, queuePrompt, uploadImage } from '~/server/utils/comfy'
|
||||||
import { buildWorkflow } from '~/server/utils/workflow'
|
import { buildWorkflow } from '~/server/utils/workflow'
|
||||||
import { writePendingJob } from '~/server/utils/pending'
|
import { writePendingJob } from '~/server/utils/pending'
|
||||||
|
import {
|
||||||
|
refineExtensionHandoffFrame,
|
||||||
|
resolveExtensionHandoffFrame
|
||||||
|
} from '~/server/utils/extensionFrame'
|
||||||
|
import {
|
||||||
|
clampRefinementDenoise,
|
||||||
|
parseRefineExtensionFrame,
|
||||||
|
parseSaveLosslessAnchor,
|
||||||
|
REFINEMENT_DENOISE_DEFAULT
|
||||||
|
} from '~/utils/extensionFrame'
|
||||||
|
|
||||||
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
|
const SAMPLERS = new Set(['res_multistep', 'euler', 'dpmpp_2m'])
|
||||||
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
|
const SCHEDULERS = new Set(['simple', 'ddim_uniform', 'sgm_uniform'])
|
||||||
@@ -63,6 +73,9 @@ export async function beginExtendFromClip(params: {
|
|||||||
loraStack?: unknown
|
loraStack?: unknown
|
||||||
folderLocked?: boolean
|
folderLocked?: boolean
|
||||||
name?: string
|
name?: string
|
||||||
|
refineExtensionFrame?: boolean
|
||||||
|
saveLosslessAnchor?: boolean
|
||||||
|
refinementDenoise?: number
|
||||||
}) {
|
}) {
|
||||||
const source = getClip(params.ownerKey, params.clipId)
|
const source = getClip(params.ownerKey, params.clipId)
|
||||||
const sourcePath = clipVideoPath(params.ownerKey, source.id)
|
const sourcePath = clipVideoPath(params.ownerKey, source.id)
|
||||||
@@ -111,6 +124,9 @@ export async function beginExtendFromClip(params: {
|
|||||||
const width = source.width || 1344
|
const width = source.width || 1344
|
||||||
const height = source.height || 768
|
const height = source.height || 768
|
||||||
const clipName = String(params.name || '').trim() || nextFamilyPartName(params.ownerKey, source)
|
const clipName = String(params.name || '').trim() || nextFamilyPartName(params.ownerKey, source)
|
||||||
|
const refineExtensionFrame = parseRefineExtensionFrame(params.refineExtensionFrame, true)
|
||||||
|
const saveLosslessAnchor = parseSaveLosslessAnchor(params.saveLosslessAnchor, false)
|
||||||
|
const refinementDenoise = clampRefinementDenoise(params.refinementDenoise, REFINEMENT_DENOISE_DEFAULT)
|
||||||
|
|
||||||
const job = createJob()
|
const job = createJob()
|
||||||
job.kind = 'video'
|
job.kind = 'video'
|
||||||
@@ -149,6 +165,9 @@ export async function beginExtendFromClip(params: {
|
|||||||
workflow,
|
workflow,
|
||||||
globalLocks: source.globalLocks,
|
globalLocks: source.globalLocks,
|
||||||
permanenceRefs: source.permanenceRefs,
|
permanenceRefs: source.permanenceRefs,
|
||||||
|
refineExtensionFrame,
|
||||||
|
saveLosslessAnchor,
|
||||||
|
refinementDenoise,
|
||||||
...loraFields
|
...loraFields
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,6 +188,9 @@ export async function beginExtendFromClip(params: {
|
|||||||
fps,
|
fps,
|
||||||
samplerName,
|
samplerName,
|
||||||
scheduler,
|
scheduler,
|
||||||
|
refineExtensionFrame,
|
||||||
|
saveLosslessAnchor,
|
||||||
|
refinementDenoise,
|
||||||
...loraFields
|
...loraFields
|
||||||
}).catch((error) => {
|
}).catch((error) => {
|
||||||
removeExtendTemp(job.library?.extendTmpDir)
|
removeExtendTemp(job.library?.extendTmpDir)
|
||||||
@@ -198,13 +220,21 @@ async function runExtend(
|
|||||||
fps: number
|
fps: number
|
||||||
samplerName: string
|
samplerName: string
|
||||||
scheduler: string
|
scheduler: string
|
||||||
|
refineExtensionFrame?: boolean
|
||||||
|
saveLosslessAnchor?: boolean
|
||||||
|
refinementDenoise?: number
|
||||||
loraName?: string
|
loraName?: string
|
||||||
loraStack?: import('~/utils/loras').LoraStackItem[]
|
loraStack?: import('~/utils/loras').LoraStackItem[]
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
copyFileSync(params.sourcePath, params.part1Path)
|
copyFileSync(params.sourcePath, params.part1Path)
|
||||||
await extractLastFrame(params.sourcePath, params.framePath)
|
const handoff = await resolveExtensionHandoffFrame({
|
||||||
const frame = readFileSync(params.framePath)
|
ownerKey: job.library!.ownerKey,
|
||||||
|
sourceClipId: job.library?.extendSourceClipId,
|
||||||
|
sourceVideoPath: params.sourcePath,
|
||||||
|
destPath: params.framePath
|
||||||
|
})
|
||||||
|
let frame = readFileSync(handoff.path)
|
||||||
if (!frame.length || frame.length < 64) {
|
if (!frame.length || frame.length < 64) {
|
||||||
throw new Error('Could not extract the last frame of the clip being extended')
|
throw new Error('Could not extract the last frame of the clip being extended')
|
||||||
}
|
}
|
||||||
@@ -223,8 +253,36 @@ async function runExtend(
|
|||||||
})
|
})
|
||||||
|
|
||||||
const done = watchComfyJob(job)
|
const done = watchComfyJob(job)
|
||||||
|
if (params.refineExtensionFrame !== false) {
|
||||||
|
emitJob(job, {
|
||||||
|
type: 'status',
|
||||||
|
message: handoff.source === 'anchor'
|
||||||
|
? 'Refining lossless anchor frame...'
|
||||||
|
: 'Refining extracted handoff frame...',
|
||||||
|
progress: 3
|
||||||
|
})
|
||||||
|
try {
|
||||||
|
frame = await refineExtensionHandoffFrame({
|
||||||
|
frame,
|
||||||
|
denoise: params.refinementDenoise,
|
||||||
|
seed: params.seed,
|
||||||
|
clientId: job.clientId,
|
||||||
|
jobId: job.id,
|
||||||
|
onStatus: (message) => emitJob(job, { type: 'status', message, progress: 4 })
|
||||||
|
})
|
||||||
|
writeFileSync(params.framePath, frame)
|
||||||
|
} catch (error) {
|
||||||
|
const detail = error instanceof Error ? error.message : String(error)
|
||||||
|
emitJob(job, {
|
||||||
|
type: 'status',
|
||||||
|
message: `Frame refine skipped (${detail}). Using source frame.`,
|
||||||
|
progress: 4
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
job.status = 'uploading'
|
job.status = 'uploading'
|
||||||
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 4 })
|
emitJob(job, { type: 'status', message: 'Uploading last frame to ComfyUI...', progress: 5 })
|
||||||
const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
|
const uploaded = await uploadImage({ filename: 'last_frame.png', data: frame, type: 'image/png' }, job.id)
|
||||||
if (job.library) {
|
if (job.library) {
|
||||||
job.library.imageName = uploaded.name
|
job.library.imageName = uploaded.name
|
||||||
@@ -251,7 +309,8 @@ async function runExtend(
|
|||||||
sound,
|
sound,
|
||||||
workflow: job.library?.workflow,
|
workflow: job.library?.workflow,
|
||||||
duration: job.library?.duration,
|
duration: job.library?.duration,
|
||||||
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName
|
loraStack: params.loraStack || job.library?.loraStack || params.loraName || job.library?.loraName,
|
||||||
|
saveLosslessAnchor: params.saveLosslessAnchor === true || job.library?.saveLosslessAnchor === true
|
||||||
})
|
})
|
||||||
|
|
||||||
const queued = await queuePrompt(graph, job.clientId)
|
const queued = await queuePrompt(graph, job.clientId)
|
||||||
|
|||||||
@@ -0,0 +1,152 @@
|
|||||||
|
import { copyFileSync, existsSync, statSync, writeFileSync } from 'node:fs'
|
||||||
|
import refineTemplate from '../assets/extension_frame_refine.json'
|
||||||
|
import { clipAnchorFramePath, clipVideoPath, downloadComfyImage } from '~/server/utils/library'
|
||||||
|
import { extractLastFrame } from '~/server/utils/ffmpeg'
|
||||||
|
import { comfyFilenamePrefix, fetchHistory, queuePrompt, uploadImage } from '~/server/utils/comfy'
|
||||||
|
import { extractEditedImage } from '~/server/utils/imageComfy'
|
||||||
|
import {
|
||||||
|
clampRefinementDenoise,
|
||||||
|
REFINEMENT_DENOISE_DEFAULT
|
||||||
|
} from '~/utils/extensionFrame'
|
||||||
|
|
||||||
|
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
|
||||||
|
type WorkflowGraph = Record<string, WorkflowNode>
|
||||||
|
|
||||||
|
const MIN_ANCHOR_BYTES = 64
|
||||||
|
|
||||||
|
export function isValidAnchorFile(path: string) {
|
||||||
|
try {
|
||||||
|
return existsSync(path) && statSync(path).size >= MIN_ANCHOR_BYTES
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resolveExtensionHandoffFrame(opts: {
|
||||||
|
ownerKey: string
|
||||||
|
sourceClipId?: string
|
||||||
|
sourceVideoPath: string
|
||||||
|
destPath: string
|
||||||
|
}) {
|
||||||
|
const anchorPath = opts.sourceClipId
|
||||||
|
? clipAnchorFramePath(opts.ownerKey, opts.sourceClipId)
|
||||||
|
: ''
|
||||||
|
if (anchorPath && isValidAnchorFile(anchorPath)) {
|
||||||
|
copyFileSync(anchorPath, opts.destPath)
|
||||||
|
return { path: opts.destPath, source: 'anchor' as const }
|
||||||
|
}
|
||||||
|
await extractLastFrame(opts.sourceVideoPath, opts.destPath)
|
||||||
|
if (!isValidAnchorFile(opts.destPath)) {
|
||||||
|
throw new Error('Could not extract the last frame of the clip being extended')
|
||||||
|
}
|
||||||
|
return { path: opts.destPath, source: 'extract' as const }
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function persistClipAnchorFrame(opts: {
|
||||||
|
ownerKey: string
|
||||||
|
clipId: string
|
||||||
|
fromPath?: string
|
||||||
|
fromBuffer?: Buffer
|
||||||
|
videoPath?: string
|
||||||
|
}) {
|
||||||
|
const dest = clipAnchorFramePath(opts.ownerKey, opts.clipId)
|
||||||
|
if (opts.fromBuffer?.length && opts.fromBuffer.length >= MIN_ANCHOR_BYTES) {
|
||||||
|
writeFileSync(dest, opts.fromBuffer)
|
||||||
|
return dest
|
||||||
|
}
|
||||||
|
if (opts.fromPath && isValidAnchorFile(opts.fromPath)) {
|
||||||
|
copyFileSync(opts.fromPath, dest)
|
||||||
|
return dest
|
||||||
|
}
|
||||||
|
const video = opts.videoPath || clipVideoPath(opts.ownerKey, opts.clipId)
|
||||||
|
if (!existsSync(video)) return ''
|
||||||
|
await extractLastFrame(video, dest)
|
||||||
|
return isValidAnchorFile(dest) ? dest : ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildRefineGraph(opts: {
|
||||||
|
imageName: string
|
||||||
|
denoise: number
|
||||||
|
seed: number
|
||||||
|
filenamePrefix: string
|
||||||
|
}) {
|
||||||
|
const graph = structuredClone(refineTemplate) as WorkflowGraph
|
||||||
|
if (graph['1']?.inputs) graph['1'].inputs.image = opts.imageName
|
||||||
|
if (graph['13']?.inputs) graph['13'].inputs.noise_seed = opts.seed
|
||||||
|
if (graph['15']?.inputs) graph['15'].inputs.denoise = opts.denoise
|
||||||
|
if (graph['20']?.inputs) graph['20'].inputs.filename_prefix = opts.filenamePrefix
|
||||||
|
return graph
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForRefineImage(promptId: string, timeoutMs = 120_000) {
|
||||||
|
const started = Date.now()
|
||||||
|
while (Date.now() - started < timeoutMs) {
|
||||||
|
const history = await fetchHistory(promptId)
|
||||||
|
const image = extractEditedImage(history, promptId)
|
||||||
|
if (image) return image
|
||||||
|
const entry = history?.[promptId] as { status?: { status_str?: string } } | undefined
|
||||||
|
const status = entry?.status?.status_str
|
||||||
|
if (status === 'error' || status === 'interrupted') {
|
||||||
|
throw new Error(status === 'interrupted' ? 'Frame refine interrupted' : 'Frame refine failed on ComfyUI')
|
||||||
|
}
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 700))
|
||||||
|
}
|
||||||
|
throw new Error('Frame refine timed out waiting for ComfyUI')
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bilateral + low-denoise Flux cleanup before the video I2V start frame.
|
||||||
|
* Runs as its own Comfy prompt so MiniMax does not share VRAM with Klein.
|
||||||
|
* When refine is off, callers skip this and upload the PNG directly (fast path).
|
||||||
|
*/
|
||||||
|
export async function refineExtensionHandoffFrame(opts: {
|
||||||
|
frame: Buffer
|
||||||
|
denoise?: number
|
||||||
|
seed?: number
|
||||||
|
clientId: string
|
||||||
|
jobId: string
|
||||||
|
onStatus?: (message: string) => void
|
||||||
|
}) {
|
||||||
|
const denoise = clampRefinementDenoise(opts.denoise, REFINEMENT_DENOISE_DEFAULT)
|
||||||
|
opts.onStatus?.('Refining extension handoff frame...')
|
||||||
|
const uploaded = await uploadImage({
|
||||||
|
filename: 'last_frame.png',
|
||||||
|
data: opts.frame,
|
||||||
|
type: 'image/png'
|
||||||
|
}, opts.jobId)
|
||||||
|
const prefix = `${comfyFilenamePrefix()}_extrefine`
|
||||||
|
const graph = buildRefineGraph({
|
||||||
|
imageName: uploaded.name,
|
||||||
|
denoise,
|
||||||
|
seed: opts.seed ?? Math.floor(Math.random() * 2_147_483_647),
|
||||||
|
filenamePrefix: `aigen/extension_refine/${prefix}`
|
||||||
|
})
|
||||||
|
const queued = await queuePrompt(graph, opts.clientId)
|
||||||
|
const image = await waitForRefineImage(queued.prompt_id)
|
||||||
|
const buffer = await downloadComfyImage(image)
|
||||||
|
if (!buffer.length || buffer.length < MIN_ANCHOR_BYTES) {
|
||||||
|
throw new Error('Frame refine finished without an output still')
|
||||||
|
}
|
||||||
|
return buffer
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractAnchorImageFromHistory(history: Record<string, unknown> | null, promptId: string) {
|
||||||
|
if (!history || !promptId) return null
|
||||||
|
const entry = history[promptId] as { outputs?: Record<string, unknown> } | undefined
|
||||||
|
const outputs = entry?.outputs
|
||||||
|
if (!outputs || typeof outputs !== 'object') return extractEditedImage(history, promptId)
|
||||||
|
const anchor = outputs.aigen_anchor_save as { images?: Array<{ filename?: string; subfolder?: string; type?: string }> } | undefined
|
||||||
|
const file = anchor?.images?.[0]
|
||||||
|
if (file?.filename) {
|
||||||
|
return {
|
||||||
|
filename: file.filename,
|
||||||
|
subfolder: file.subfolder || '',
|
||||||
|
type: file.type || 'output'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return extractEditedImage(history, promptId)
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function downloadAnchorImage(image: { filename: string; subfolder: string; type: string }) {
|
||||||
|
return downloadComfyImage(image)
|
||||||
|
}
|
||||||
@@ -101,6 +101,9 @@ export interface Job {
|
|||||||
queueAutoRun?: boolean
|
queueAutoRun?: boolean
|
||||||
queueBudget?: number
|
queueBudget?: number
|
||||||
stopAfterCurrent?: boolean
|
stopAfterCurrent?: boolean
|
||||||
|
refineExtensionFrame?: boolean
|
||||||
|
saveLosslessAnchor?: boolean
|
||||||
|
refinementDenoise?: number
|
||||||
loraName?: string
|
loraName?: string
|
||||||
loraStack?: import('~/utils/loras').LoraStackItem[]
|
loraStack?: import('~/utils/loras').LoraStackItem[]
|
||||||
shotLoras?: string[]
|
shotLoras?: string[]
|
||||||
|
|||||||
+11
-2
@@ -790,6 +790,11 @@ export function clipLastFramePath(owner: string, id: string) {
|
|||||||
return join(clipDir(owner, id), 'last-frame.png')
|
return join(clipDir(owner, id), 'last-frame.png')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Lossless PNG handoff frame for cleaner sequential extensions (not the preview cache). */
|
||||||
|
export function clipAnchorFramePath(owner: string, id: string) {
|
||||||
|
return join(clipDir(owner, id), 'anchor-last.png')
|
||||||
|
}
|
||||||
|
|
||||||
export function clipFirstFramePath(owner: string, id: string) {
|
export function clipFirstFramePath(owner: string, id: string) {
|
||||||
return join(clipDir(owner, id), 'first-frame.jpg')
|
return join(clipDir(owner, id), 'first-frame.jpg')
|
||||||
}
|
}
|
||||||
@@ -2009,9 +2014,9 @@ async function downloadViaHostAgent(file: { filename: string; subfolder: string;
|
|||||||
|
|
||||||
async function downloadComfyMedia(
|
async function downloadComfyMedia(
|
||||||
file: { filename: string; subfolder: string; type: string },
|
file: { filename: string; subfolder: string; type: string },
|
||||||
kind: 'video' | 'audio'
|
kind: 'video' | 'audio' | 'image'
|
||||||
) {
|
) {
|
||||||
const extras = kind === 'video' ? ['video', ''] : ['audio', '']
|
const extras = kind === 'video' ? ['video', ''] : kind === 'audio' ? ['audio', ''] : ['']
|
||||||
const subfolders = [...new Set([file.subfolder, ...extras])]
|
const subfolders = [...new Set([file.subfolder, ...extras])]
|
||||||
let lastStatus = 0
|
let lastStatus = 0
|
||||||
let lastError = ''
|
let lastError = ''
|
||||||
@@ -2049,6 +2054,10 @@ export async function downloadComfyAudio(audio: { filename: string; subfolder: s
|
|||||||
return downloadComfyMedia(audio, 'audio')
|
return downloadComfyMedia(audio, 'audio')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function downloadComfyImage(image: { filename: string; subfolder: string; type: string }) {
|
||||||
|
return downloadComfyMedia(image, 'image')
|
||||||
|
}
|
||||||
|
|
||||||
export async function importMissingComfyVideos(owner: string, folderId?: string) {
|
export async function importMissingComfyVideos(owner: string, folderId?: string) {
|
||||||
const catalog = readCatalog(owner)
|
const catalog = readCatalog(owner)
|
||||||
const folder = catalog.folders.find(item => item.id === folderId) || catalog.folders[0]
|
const folder = catalog.folders.find(item => item.id === folderId) || catalog.folders[0]
|
||||||
|
|||||||
@@ -56,6 +56,9 @@ export interface PendingJob {
|
|||||||
queueAutoRun?: boolean
|
queueAutoRun?: boolean
|
||||||
queueBudget?: number
|
queueBudget?: number
|
||||||
stopAfterCurrent?: boolean
|
stopAfterCurrent?: boolean
|
||||||
|
refineExtensionFrame?: boolean
|
||||||
|
saveLosslessAnchor?: boolean
|
||||||
|
refinementDenoise?: number
|
||||||
loraName?: string
|
loraName?: string
|
||||||
loraStack?: import('~/utils/loras').LoraStackItem[]
|
loraStack?: import('~/utils/loras').LoraStackItem[]
|
||||||
shotLoras?: string[]
|
shotLoras?: string[]
|
||||||
@@ -155,6 +158,9 @@ export function pendingFromJob(job: Job, overrides: Partial<PendingJob> = {}): P
|
|||||||
queueAutoRun: library.queueAutoRun,
|
queueAutoRun: library.queueAutoRun,
|
||||||
queueBudget: library.queueBudget,
|
queueBudget: library.queueBudget,
|
||||||
stopAfterCurrent: library.stopAfterCurrent,
|
stopAfterCurrent: library.stopAfterCurrent,
|
||||||
|
refineExtensionFrame: library.refineExtensionFrame,
|
||||||
|
saveLosslessAnchor: library.saveLosslessAnchor,
|
||||||
|
refinementDenoise: library.refinementDenoise,
|
||||||
...overrides
|
...overrides
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -203,6 +209,9 @@ export function libraryFromPending(pending: PendingJob): NonNullable<Job['librar
|
|||||||
queueAutoRun: pending.queueAutoRun,
|
queueAutoRun: pending.queueAutoRun,
|
||||||
queueBudget: pending.queueBudget,
|
queueBudget: pending.queueBudget,
|
||||||
stopAfterCurrent: pending.stopAfterCurrent,
|
stopAfterCurrent: pending.stopAfterCurrent,
|
||||||
|
refineExtensionFrame: pending.refineExtensionFrame,
|
||||||
|
saveLosslessAnchor: pending.saveLosslessAnchor,
|
||||||
|
refinementDenoise: pending.refinementDenoise,
|
||||||
globalLocks: pending.globalLocks,
|
globalLocks: pending.globalLocks,
|
||||||
permanenceRefs: pending.permanenceRefs,
|
permanenceRefs: pending.permanenceRefs,
|
||||||
shotPermanenceRefs: pending.shotPermanenceRefs,
|
shotPermanenceRefs: pending.shotPermanenceRefs,
|
||||||
@@ -342,6 +351,32 @@ export async function completePendingIfReady(pending: PendingJob) {
|
|||||||
loraStack: pending.loraStack,
|
loraStack: pending.loraStack,
|
||||||
segmentFirstFrame
|
segmentFirstFrame
|
||||||
})
|
})
|
||||||
|
if (pending.saveLosslessAnchor === true) {
|
||||||
|
try {
|
||||||
|
const { persistClipAnchorFrame, extractAnchorImageFromHistory, downloadAnchorImage } = await import('~/server/utils/extensionFrame')
|
||||||
|
let saved = false
|
||||||
|
const anchorImage = extractAnchorImageFromHistory(history, pending.promptId)
|
||||||
|
if (anchorImage) {
|
||||||
|
const png = await downloadAnchorImage(anchorImage)
|
||||||
|
if (png.length >= 64) {
|
||||||
|
await persistClipAnchorFrame({
|
||||||
|
ownerKey: pending.ownerKey,
|
||||||
|
clipId: clip.id,
|
||||||
|
fromBuffer: png
|
||||||
|
})
|
||||||
|
saved = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!saved) {
|
||||||
|
await persistClipAnchorFrame({
|
||||||
|
ownerKey: pending.ownerKey,
|
||||||
|
clipId: clip.id
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* optional */
|
||||||
|
}
|
||||||
|
}
|
||||||
await purgeComfyArtifacts({
|
await purgeComfyArtifacts({
|
||||||
video,
|
video,
|
||||||
imageName: pending.imageName,
|
imageName: pending.imageName,
|
||||||
|
|||||||
@@ -69,6 +69,9 @@ export interface StudioJobPayload {
|
|||||||
maskStillFilename?: string
|
maskStillFilename?: string
|
||||||
refineStrength?: number
|
refineStrength?: number
|
||||||
extendFromClipId?: string
|
extendFromClipId?: string
|
||||||
|
refineExtensionFrame?: boolean
|
||||||
|
saveLosslessAnchor?: boolean
|
||||||
|
refinementDenoise?: number
|
||||||
lyrics?: string
|
lyrics?: string
|
||||||
instrumental?: boolean
|
instrumental?: boolean
|
||||||
lyricsStrength?: number
|
lyricsStrength?: number
|
||||||
@@ -1156,7 +1159,10 @@ async function startStudioExtendJob(item: StudioJob) {
|
|||||||
duration: item.payload.duration,
|
duration: item.payload.duration,
|
||||||
loraStack: item.payload.loraStack || item.payload.loraName,
|
loraStack: item.payload.loraStack || item.payload.loraName,
|
||||||
folderLocked: item.payload.folderLocked,
|
folderLocked: item.payload.folderLocked,
|
||||||
name: item.payload.name || item.name
|
name: item.payload.name || item.name,
|
||||||
|
refineExtensionFrame: item.payload.refineExtensionFrame,
|
||||||
|
saveLosslessAnchor: item.payload.saveLosslessAnchor,
|
||||||
|
refinementDenoise: item.payload.refinementDenoise
|
||||||
})
|
})
|
||||||
await markStudioLive(item.ownerKey, item.id, live.id)
|
await markStudioLive(item.ownerKey, item.id, live.id)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -1294,6 +1300,9 @@ export async function startStudioJob(item: StudioJob) {
|
|||||||
useIdentityRefs: identity,
|
useIdentityRefs: identity,
|
||||||
queueAutoRun: extensions.length ? payload.queueAutoRun : false,
|
queueAutoRun: extensions.length ? payload.queueAutoRun : false,
|
||||||
queueBudget: extensions.length && payload.queueAutoRun ? extensions.length : 0,
|
queueBudget: extensions.length && payload.queueAutoRun ? extensions.length : 0,
|
||||||
|
refineExtensionFrame: payload.refineExtensionFrame !== false,
|
||||||
|
saveLosslessAnchor: payload.saveLosslessAnchor === true,
|
||||||
|
refinementDenoise: payload.refinementDenoise,
|
||||||
globalLocks: payload.globalLocks,
|
globalLocks: payload.globalLocks,
|
||||||
permanenceRefs: payload.permanenceRefs,
|
permanenceRefs: payload.permanenceRefs,
|
||||||
shotPermanenceRefs: payload.shotPermanenceRefs,
|
shotPermanenceRefs: payload.shotPermanenceRefs,
|
||||||
|
|||||||
@@ -7,9 +7,13 @@ import { comfyFilenamePrefix, queuePrompt, uploadImage } from '~/server/utils/co
|
|||||||
import { buildWorkflow } from '~/server/utils/workflow'
|
import { buildWorkflow } from '~/server/utils/workflow'
|
||||||
import { isLtxWorkflow, isTextToVideo, parseVideoWorkflow, workflowForExtension, type VideoWorkflowId } from '~/utils/videoModels'
|
import { isLtxWorkflow, isTextToVideo, parseVideoWorkflow, workflowForExtension, type VideoWorkflowId } from '~/utils/videoModels'
|
||||||
import { clipVideoPath, clipTitle, deleteRetryDraft, extendTempDir, getClip, nextFamilyPartName, removeExtendTemp, stillPath } from '~/server/utils/library'
|
import { clipVideoPath, clipTitle, deleteRetryDraft, extendTempDir, getClip, nextFamilyPartName, removeExtendTemp, stillPath } from '~/server/utils/library'
|
||||||
import { extractLastFrame, probeHasAudio } from '~/server/utils/ffmpeg'
|
import { probeHasAudio } from '~/server/utils/ffmpeg'
|
||||||
import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
|
import { ensureComfyReady } from '~/server/utils/comfyLifecycle'
|
||||||
import { onLiveVideoSettled, onStudioQueueBurstStarted } from '~/server/utils/studioQueue'
|
import { onLiveVideoSettled, onStudioQueueBurstStarted } from '~/server/utils/studioQueue'
|
||||||
|
import {
|
||||||
|
refineExtensionHandoffFrame,
|
||||||
|
resolveExtensionHandoffFrame
|
||||||
|
} from '~/server/utils/extensionFrame'
|
||||||
import {
|
import {
|
||||||
finishQueueBurst,
|
finishQueueBurst,
|
||||||
getShotQueue,
|
getShotQueue,
|
||||||
@@ -216,6 +220,7 @@ export async function queueMiniMax(
|
|||||||
duration: params.duration,
|
duration: params.duration,
|
||||||
useIdentityRefs: params.useIdentityRefs,
|
useIdentityRefs: params.useIdentityRefs,
|
||||||
referenceImageNames: params.useIdentityRefs ? referenceNames : [],
|
referenceImageNames: params.useIdentityRefs ? referenceNames : [],
|
||||||
|
saveLosslessAnchor: job.library?.saveLosslessAnchor === true,
|
||||||
...persistLoraFields(params.loraStack || params.loraName)
|
...persistLoraFields(params.loraStack || params.loraName)
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -373,7 +378,12 @@ export async function continueQueuedExtensions(
|
|||||||
const parentVideo = parentId ? clipVideoPath(job.library.ownerKey, parentId) : ''
|
const parentVideo = parentId ? clipVideoPath(job.library.ownerKey, parentId) : ''
|
||||||
const extractSource = parentVideo && existsSync(parentVideo) ? parentVideo : part1Path
|
const extractSource = parentVideo && existsSync(parentVideo) ? parentVideo : part1Path
|
||||||
try {
|
try {
|
||||||
await extractLastFrame(extractSource, framePath)
|
await resolveExtensionHandoffFrame({
|
||||||
|
ownerKey: job.library.ownerKey,
|
||||||
|
sourceClipId: parentId,
|
||||||
|
sourceVideoPath: extractSource,
|
||||||
|
destPath: framePath
|
||||||
|
})
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const detail = error instanceof Error ? error.message : String(error)
|
const detail = error instanceof Error ? error.message : String(error)
|
||||||
throw new Error(detail.includes('last frame')
|
throw new Error(detail.includes('last frame')
|
||||||
@@ -385,10 +395,30 @@ export async function continueQueuedExtensions(
|
|||||||
job.library.stopAfterCurrent = true
|
job.library.stopAfterCurrent = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
const frame = readFileSync(framePath)
|
let frame = readFileSync(framePath)
|
||||||
if (!frame.length || frame.length < 64) {
|
if (!frame.length || frame.length < 64) {
|
||||||
throw new Error('Could not extract the last frame for the next extension: the frame file was empty')
|
throw new Error('Could not extract the last frame for the next extension: the frame file was empty')
|
||||||
}
|
}
|
||||||
|
if (job.library.refineExtensionFrame !== false) {
|
||||||
|
emitChainJob(job, { type: 'status', message: 'Refining extension handoff frame...', progress: 2 })
|
||||||
|
try {
|
||||||
|
frame = await refineExtensionHandoffFrame({
|
||||||
|
frame,
|
||||||
|
denoise: job.library.refinementDenoise,
|
||||||
|
clientId: job.clientId,
|
||||||
|
jobId: job.id,
|
||||||
|
onStatus: (message) => emitChainJob(job, { type: 'status', message, progress: 3 })
|
||||||
|
})
|
||||||
|
writeFileSync(framePath, frame)
|
||||||
|
} catch (error) {
|
||||||
|
const detail = error instanceof Error ? error.message : String(error)
|
||||||
|
emitChainJob(job, {
|
||||||
|
type: 'status',
|
||||||
|
message: `Frame refine skipped (${detail}). Using source frame.`,
|
||||||
|
progress: 3
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
job.library.extendPart1Path = part1Path
|
job.library.extendPart1Path = part1Path
|
||||||
job.library.prompt = composePromptParts(wrappers.promptPre, ext.prompt, wrappers.promptPost)
|
job.library.prompt = composePromptParts(wrappers.promptPre, ext.prompt, wrappers.promptPost)
|
||||||
job.library.promptMid = ext.prompt
|
job.library.promptMid = ext.prompt
|
||||||
|
|||||||
@@ -275,6 +275,34 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
|
|||||||
loraStack: job.library.loraStack,
|
loraStack: job.library.loraStack,
|
||||||
segmentFirstFrame
|
segmentFirstFrame
|
||||||
})
|
})
|
||||||
|
if (job.library.saveLosslessAnchor === true) {
|
||||||
|
try {
|
||||||
|
const { persistClipAnchorFrame, extractAnchorImageFromHistory, downloadAnchorImage } = await import('~/server/utils/extensionFrame')
|
||||||
|
let saved = false
|
||||||
|
if (job.promptId && history) {
|
||||||
|
const anchorImage = extractAnchorImageFromHistory(history, job.promptId)
|
||||||
|
if (anchorImage) {
|
||||||
|
const png = await downloadAnchorImage(anchorImage)
|
||||||
|
if (png.length >= 64) {
|
||||||
|
await persistClipAnchorFrame({
|
||||||
|
ownerKey: job.library.ownerKey,
|
||||||
|
clipId: clip.id,
|
||||||
|
fromBuffer: png
|
||||||
|
})
|
||||||
|
saved = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!saved) {
|
||||||
|
await persistClipAnchorFrame({
|
||||||
|
ownerKey: job.library.ownerKey,
|
||||||
|
clipId: clip.id
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* Anchor is optional; extension can still extract from MP4. */
|
||||||
|
}
|
||||||
|
}
|
||||||
job.clipId = clip.id
|
job.clipId = clip.id
|
||||||
job.savedPromptId = job.promptId
|
job.savedPromptId = job.promptId
|
||||||
job.hideThumbnail = clip.hideThumbnail
|
job.hideThumbnail = clip.hideThumbnail
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ export interface GenerateParams {
|
|||||||
referenceImageNames?: string[]
|
referenceImageNames?: string[]
|
||||||
loraName?: string
|
loraName?: string
|
||||||
loraStack?: LoraStackItem[]
|
loraStack?: LoraStackItem[]
|
||||||
|
saveLosslessAnchor?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
|
type WorkflowNode = { class_type: string; inputs: Record<string, unknown>; _meta?: { title?: string } }
|
||||||
@@ -228,9 +229,34 @@ export function buildWorkflow(params: GenerateParams) {
|
|||||||
|
|
||||||
resolveGraphLoraNames(graph, 'video')
|
resolveGraphLoraNames(graph, 'video')
|
||||||
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
|
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
|
||||||
|
attachLosslessAnchorSave(graph, params, version === 'v2' ? '164' : '105:90')
|
||||||
return graph
|
return graph
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function attachLosslessAnchorSave(graph: WorkflowGraph, params: GenerateParams, decodeNodeId: string) {
|
||||||
|
if (!params.saveLosslessAnchor) return
|
||||||
|
if (!graph[decodeNodeId]) return
|
||||||
|
const batchIndex = Math.max(0, Math.floor(Number(params.length) || 1) - 1)
|
||||||
|
const prefix = String(params.filenamePrefix || 'video/MiniMax_H3').replace(/\/+$/, '')
|
||||||
|
graph.aigen_anchor_pick = {
|
||||||
|
class_type: 'ImageFromBatch',
|
||||||
|
inputs: {
|
||||||
|
image: [decodeNodeId, 0],
|
||||||
|
batch_index: batchIndex,
|
||||||
|
length: 1
|
||||||
|
},
|
||||||
|
_meta: { title: 'Last Decoded Frame' }
|
||||||
|
}
|
||||||
|
graph.aigen_anchor_save = {
|
||||||
|
class_type: 'SaveImage',
|
||||||
|
inputs: {
|
||||||
|
filename_prefix: `${prefix}_last`,
|
||||||
|
images: ['aigen_anchor_pick', 0]
|
||||||
|
},
|
||||||
|
_meta: { title: 'Save Lossless Anchor' }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function promptForNode(classType: string, useRefs: boolean, actionPrompt: string, graphPrompt: string) {
|
function promptForNode(classType: string, useRefs: boolean, actionPrompt: string, graphPrompt: string) {
|
||||||
if (classType === 'MiniMaxH3ImageToVideo') return useRefs ? '' : actionPrompt
|
if (classType === 'MiniMaxH3ImageToVideo') return useRefs ? '' : actionPrompt
|
||||||
if (classType === 'MiniMaxH3TextToVideo') return actionPrompt
|
if (classType === 'MiniMaxH3TextToVideo') return actionPrompt
|
||||||
@@ -302,6 +328,8 @@ function buildLtxWorkflow(params: GenerateParams, version: VideoWorkflowId) {
|
|||||||
}
|
}
|
||||||
resolveGraphLoraNames(graph, 'video')
|
resolveGraphLoraNames(graph, 'video')
|
||||||
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
|
applyUserLoraToGraph(graph, params.loraStack || params.loraName)
|
||||||
|
// LTX VAEDecode is typically node 15
|
||||||
|
attachLosslessAnchorSave(graph, params, '15')
|
||||||
return graph
|
return graph
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
export const REFINEMENT_DENOISE_MIN = 0.1
|
||||||
|
export const REFINEMENT_DENOISE_MAX = 0.25
|
||||||
|
export const REFINEMENT_DENOISE_DEFAULT = 0.18
|
||||||
|
|
||||||
|
export function clampRefinementDenoise(raw: unknown, fallback = REFINEMENT_DENOISE_DEFAULT) {
|
||||||
|
const n = Number(raw)
|
||||||
|
if (!Number.isFinite(n)) return fallback
|
||||||
|
return Math.min(REFINEMENT_DENOISE_MAX, Math.max(REFINEMENT_DENOISE_MIN, Math.round(n * 100) / 100))
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseRefineExtensionFrame(raw: unknown, fallback = true) {
|
||||||
|
if (raw === undefined || raw === null || raw === '') return fallback
|
||||||
|
if (typeof raw === 'boolean') return raw
|
||||||
|
const value = String(raw).trim().toLowerCase()
|
||||||
|
if (value === 'true' || value === '1' || value === 'yes') return true
|
||||||
|
if (value === 'false' || value === '0' || value === 'no') return false
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseSaveLosslessAnchor(raw: unknown, fallback = false) {
|
||||||
|
return parseRefineExtensionFrame(raw, fallback)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user