Add local queued video upscale with Real-ESRGAN and RIFE

server/api/generate/active.get.ts

@@ -18,6 +18,7 @@ function livePayload(job: Job) {
 }
 
 function jobLooksLive(job: Job, comfyBusy: boolean) {
+  if (job.upscale) return false // Clip-level post-processing must not occupy the generation form.
   if (job.library?.stopAfterCurrent) return false
   if (jobIsLocallySubmitting(job)) return true
   if (job.status === 'running' && job.promptId) return comfyBusy

server/api/generate/recover.post.ts

@@ -82,7 +82,7 @@ export default defineEventHandler(async (event) => {
     if (live.status === 'complete' || live.status === 'error' || live.trackId || live.clipId) {
       return jobSnapshot(live)
     }
-    if (live.yueGp) return jobSnapshot(live)
+    if (live.yueGp || live.upscale) return jobSnapshot(live)
     if (live.kind === 'music' && Date.now() - live.startedAt > 15_000) {
       const recovered = await recoverFinishedMedia(event, { tags: live.library?.tags })
       if (recovered?.trackId) return recovered

server/api/interrupt.post.ts

@@ -28,9 +28,19 @@ export default defineEventHandler(async (event) => {
   const jobId = String(body?.jobId || '')
   const kind = String(body?.kind || '')
   const named = jobId ? getJob(jobId) : undefined
+  if (named?.upscale) {
+    const { owner } = assertLibraryOwner(event)
+    if (named.library?.ownerKey !== owner) throw createError({ statusCode: 404, statusMessage: 'Job not found' })
+    const { cancelUpscaleJob } = await import('~/server/utils/videoUpscale')
+    const { onLiveVideoSettled } = await import('~/server/utils/studioQueue')
+    await cancelUpscaleJob(named)
+    await onLiveVideoSettled(named)
+    return { ok: true, cancelled: 1 }
+  }
   const targets = named
     ? [named]
     : listJobs().filter((job) => {
+      if (job.upscale) return false
       if (job.status === 'complete' || job.status === 'error' || job.status === 'cancelled') return false
       if (kind === 'edit') return job.kind === 'edit'
       if (kind === 'music') return job.kind === 'music'
@@ -46,6 +56,7 @@ export default defineEventHandler(async (event) => {
   if (jobId) deletePendingJob(jobId)
   const { owner } = assertLibraryOwner(event)
   for (const row of listStudioJobs(owner)) {
+    if (row.payload.upscale) continue
     if (row.status !== 'running' && row.status !== 'held') continue
     if (kind === 'edit' && row.kind !== 'edit') continue
     if (kind === 'music' && row.kind !== 'music') continue

server/api/library/clips/[id]/upscale.get.ts

@@ -0,0 +1,24 @@
+import { basename, parse } from 'node:path'
+import { upscaleRecords } from '../../../../utils/videoUpscale'
+import { listStudioJobs } from '../../../../utils/studioQueue'
+import { listJobs } from '../../../../utils/jobs'
+
+export default defineEventHandler(event => {
+  const { owner } = assertLibraryOwner(event)
+  const clip = getClip(owner, String(getRouterParam(event, 'id') || ''))
+  assertFolderAccess(event, clip.folderId)
+  const rows = listStudioJobs(owner)
+  const active = rows.find(row => ['waiting', 'held', 'running'].includes(row.status) && row.payload.upscale?.sourceId === clip.id)
+  const records = upscaleRecords(owner, clip.id)
+  const latest = active || rows.filter(row => row.payload.upscale?.sourceId === clip.id).sort((a, b) => b.createdAt - a.createdAt)[0]
+  const record = records.find(r => !latest || r.id === latest.id)
+  const status = active?.status || record?.status || latest?.status
+  const labels: Record<string, string> = { waiting: 'queued', held: 'queued', complete: 'done', error: 'failed', cancelled: 'failed', running: 'running' }
+  const generationBusy = rows.some(row => row.status === 'running' && row.payload.extendFromClipId === clip.id)
+    || listJobs().some(job => job.library?.ownerKey === owner && job.clipId === clip.id && ['queued', 'uploading', 'running'].includes(job.status))
+  return { source: { id: clip.id, name: clip.name, upscaledFromClipId: clip.upscaledFromClipId, stem: parse(basename(clipVideoPath(owner, clip.id))).name, width: clip.width, height: clip.height },
+    busy: Boolean(active || record?.status === 'running' || generationBusy),
+    job: latest || record ? { id: latest?.id || record.id, status: labels[status || ''] || status,
+      message: record?.message || latest?.waitReason, error: record?.error || latest?.lastError || (status === 'cancelled' ? 'Cancelled. Original video is safe.' : undefined), progress: record?.progress || 0,
+      width: record?.width, height: record?.height, elapsedMs: record?.elapsedMs || (record?.createdAt ? Date.now() - record.createdAt : 0), outputClipId: record?.outputClipId } : null }
+})

server/api/library/clips/[id]/upscale.post.ts

@@ -0,0 +1,29 @@
+import { existsSync } from 'node:fs'
+import { upscaleOptions } from '~/shared/video-upscale.mjs'
+import { addStudioJob, listStudioJobs, kickStudioQueue, type StudioJobPayload } from '../../../../utils/studioQueue'
+import { listJobs } from '../../../../utils/jobs'
+
+const submitting = new Set<string>()
+export default defineEventHandler(async event => {
+  const { owner } = assertLibraryOwner(event)
+  const clip = getClip(owner, String(getRouterParam(event, 'id') || ''))
+  assertFolderAccess(event, clip.folderId)
+  const body = await readBody(event)
+  let options
+  try { options = upscaleOptions(body) } catch { throw createError({ statusCode: 400, statusMessage: 'Invalid upscale settings.' }) }
+  const key = `${owner}:${clip.id}`
+  if (submitting.has(key)) throw createError({ statusCode: 409, statusMessage: 'This clip is already being queued.' })
+  submitting.add(key)
+  try {
+    if (!existsSync(clipVideoPath(owner, clip.id))) throw createError({ statusCode: 404, statusMessage: 'Completed video file is missing.' })
+    if (listStudioJobs(owner).some(row => ['waiting', 'held', 'running'].includes(row.status) && (row.payload.upscale?.sourceId === clip.id || row.status === 'running' && row.payload.extendFromClipId === clip.id))
+      || listJobs().some(job => job.library?.ownerKey === owner && job.clipId === clip.id && ['running', 'queued', 'uploading'].includes(job.status))) throw createError({ statusCode: 409, statusMessage: 'This clip already has a generation or upscale in progress.' })
+    const payload = { ...clip, name: `Upscale · ${clip.name || 'Video'}`, prompt: clip.prompt || 'Upscale video', folderId: clip.folderId,
+      steps: 0, cfg: 0, seed: 0, turbo: false, fps: clip.fps || 24, duration: clip.duration || 0, sound: clip.sound !== false,
+      samplerName: '', scheduler: '', useIdentityRefs: false, referenceStillIds: [], extensions: [], queueAutoRun: true,
+      upscale: { ...options, sourceId: clip.id } } as StudioJobPayload
+    const job = await addStudioJob({ ownerKey: owner, payload, familyId: crypto.randomUUID(), kind: 'video' })
+    kickStudioQueue()
+    return { id: job.id, status: 'queued' }
+  } finally { submitting.delete(key) }
+})

server/api/studio-queue/[id].patch.ts

@@ -28,6 +28,7 @@ export default defineEventHandler(async (event) => {
   if (!current) {
     throw createError({ statusCode: 404, statusMessage: 'Queued job not found' })
   }
+  if (current.payload.upscale) throw createError({ statusCode: 409, statusMessage: 'Cancel this upscale and use the clip dialog to change settings.' })
   if (!EDITABLE.has(current.status)) {
     throw createError({ statusCode: 409, statusMessage: 'That job is already generating. Copy it or save a template.' })
   }