Retry library pulls after Comfy finishes and fall back to disk.

Long jobs were succeeding on Beast then failing the save step when /view briefly dropped; retries plus host-agent file serving keep finished MP4s from being orphaned. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-03 21:06:15 -05:00
parent d260abb0f8
commit ec887a320a
3 changed files with 172 additions and 30 deletions
@@ -1964,36 +1964,89 @@ export function renameClipFamily(owner: string, id: string, name: string) {
  })
 }

-export async function downloadComfyVideo(video: { filename: string; subfolder: string; type: string }) {
-  const subfolders = [...new Set([video.subfolder, 'video', ''])]
-  let lastStatus = 0
+function sleep(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+function downloadErrorText(error: unknown) {
+  if (!error) return 'Unknown download error'
+  if (typeof error === 'object') {
+    const rec = error as { statusMessage?: string; message?: string; data?: { cause?: string } }
+    const bits = [rec.statusMessage, rec.message, rec.data?.cause].filter(Boolean)
+    if (bits.length) return bits[0] as string
+  }
+  if (error instanceof Error) return error.message
+  return String(error)
+}
+
+async function downloadViaHostAgent(file: { filename: string; subfolder: string; type: string }) {
+  const config = useRuntimeConfig()
+  const controlUrl = String(config.comfyControlUrl || process.env.COMFY_CONTROL_URL || '').replace(/\/$/, '')
+  const token = String(config.comfyControlToken || process.env.COMFY_CONTROL_TOKEN || '')
+  if (!controlUrl) return null
+  const subfolders = [...new Set([file.subfolder, file.type === 'input' ? '' : 'video', file.type === 'input' ? '' : 'audio', ''])]
  for (const subfolder of subfolders) {
    const params = new URLSearchParams({
-      filename: video.filename,
+      filename: file.filename,
      subfolder,
-      type: video.type || 'output'
+      type: file.type || 'output'
    })
-    const res = await comfyFetch(`/view?${params.toString()}`)
-    lastStatus = res.status
-    if (res.ok) return Buffer.from(await res.arrayBuffer())
+    try {
+      const res = await fetch(`${controlUrl}/view?${params.toString()}`, {
+        headers: {
+          Accept: '*/*',
+          ...(token ? { Authorization: `Bearer ${token}` } : {})
+        },
+        signal: AbortSignal.timeout(5 * 60 * 1000)
+      })
+      if (res.ok) return Buffer.from(await res.arrayBuffer())
+    } catch {
+      /* try next subfolder / fall through */
+    }
  }
-  throw new Error(`Failed to fetch completed video from ComfyUI (${lastStatus})`)
+  return null
+}
+
+async function downloadComfyMedia(
+  file: { filename: string; subfolder: string; type: string },
+  kind: 'video' | 'audio'
+) {
+  const extras = kind === 'video' ? ['video', ''] : ['audio', '']
+  const subfolders = [...new Set([file.subfolder, ...extras])]
+  let lastStatus = 0
+  let lastError = ''
+  const attempts = 8
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    for (const subfolder of subfolders) {
+      const params = new URLSearchParams({
+        filename: file.filename,
+        subfolder,
+        type: file.type || 'output'
+      })
+      try {
+        const res = await comfyFetch(`/view?${params.toString()}`, {
+          signal: AbortSignal.timeout(5 * 60 * 1000)
+        })
+        lastStatus = res.status
+        if (res.ok) return Buffer.from(await res.arrayBuffer())
+      } catch (error) {
+        lastError = downloadErrorText(error)
+      }
+    }
+    const disk = await downloadViaHostAgent(file)
+    if (disk?.length) return disk
+    if (attempt < attempts) await sleep(1500 * attempt)
+  }
+  const detail = lastError || `HTTP ${lastStatus || 'n/a'}`
+  throw new Error(`Failed to fetch completed ${kind} from ComfyUI (${detail}). File may still be on the desktop — use Recover.`)
+}
+
+export async function downloadComfyVideo(video: { filename: string; subfolder: string; type: string }) {
+  return downloadComfyMedia(video, 'video')
 }

 export async function downloadComfyAudio(audio: { filename: string; subfolder: string; type: string }) {
-  const subfolders = [...new Set([audio.subfolder, 'audio', ''])]
-  let lastStatus = 0
-  for (const subfolder of subfolders) {
-    const params = new URLSearchParams({
-      filename: audio.filename,
-      subfolder,
-      type: audio.type || 'output'
-    })
-    const res = await comfyFetch(`/view?${params.toString()}`)
-    lastStatus = res.status
-    if (res.ok) return Buffer.from(await res.arrayBuffer())
-  }
-  throw new Error(`Failed to fetch completed audio from ComfyUI (${lastStatus})`)
+  return downloadComfyMedia(audio, 'audio')
 }

 export async function importMissingComfyVideos(owner: string, folderId?: string) {
@@ -19,7 +19,7 @@ function classifyError(message: string) {
  if (lower.includes('out of memory') || (lower.includes('cuda') && lower.includes('alloc')) || lower.includes('vram')) {
    return 'ComfyUI VRAM allocation failed. Try Turbo or a smaller aspect ratio.'
  }
-  if (lower.includes('econnrefused') || lower.includes('unreachable') || lower.includes('fetch failed')) {
+  if (lower.includes('econnrefused') || lower.includes('unreachable') || lower.includes('fetch failed') || lower.includes('connection dropped')) {
    return 'ComfyUI host connection dropped. Confirm the desktop instance is running and reachable on the LAN.'
  }
  if (lower.includes('timeout')) {
@@ -28,6 +28,17 @@ function classifyError(message: string) {
  return message
 }

+function errorText(error: unknown) {
+  if (!error) return 'Unknown error'
+  if (typeof error === 'object') {
+    const rec = error as { statusMessage?: string; message?: string; data?: { cause?: string } }
+    const bits = [rec.statusMessage, rec.message, rec.data?.cause].filter(Boolean)
+    if (bits.length) return String(bits[0])
+  }
+  if (error instanceof Error) return error.message
+  return String(error)
+}
+
 function sleep(ms: number) {
  return new Promise(resolve => setTimeout(resolve, ms))
 }
@@ -175,8 +186,23 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
      finishing = true
      job.saving = true
      try {
-        const history = await fetchHistory(job.promptId)
-        const video = extractVideo(history, job.promptId)
+        let history: Awaited<ReturnType<typeof fetchHistory>> | null = null
+        let video: ReturnType<typeof extractVideo> = null
+        for (let attempt = 1; attempt <= 6; attempt++) {
+          try {
+            history = await fetchHistory(job.promptId)
+            video = extractVideo(history, job.promptId)
+            if (video) break
+          } catch (error) {
+            if (attempt >= 6) throw error
+          }
+          emitLocal({
+            type: 'status',
+            message: `Waiting for ComfyUI history (${attempt}/6)...`,
+            progress: 96
+          })
+          await sleep(1000 * attempt)
+        }
        if (!video) {
          finishing = false
          job.saving = false
@@ -312,12 +338,20 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
        } catch (saveError) {
          job.saving = false
          removeExtendTemp(job.library?.extendTmpDir)
-          const message = saveError instanceof Error ? saveError.message : String(saveError)
+          const message = classifyError(errorText(saveError))
          job.status = 'error'
          job.error = persist
            ? `Video generated but library save failed: ${message}`
            : `Segment finished but could not be prepared: ${message}`
          emitChainJob(job, { type: 'error', error: job.error, message: job.error })
+          // Keep pending so a later Recover / container restart can still pull the MP4.
+          if (job.library && job.promptId) {
+            writePendingJob(pendingFromJob(job, {
+              promptId: job.promptId,
+              currentClipId: job.clipId || job.library.parentClipId
+            }))
+          }
+          await syncQueueFromJob(job).catch(() => null)
        }
        resolve()
        return true
@@ -406,9 +440,19 @@ export function watchComfyJob(job: Job, options: { persist?: boolean } = {}): Pr
      if (payload.type === 'executing') {
        const node = data.node === null || data.node === undefined ? null : String(data.node)
        if (node === null) {
-          const saved = await succeed()
-          if (!saved) {
-            emitLocal({ type: 'status', message: 'Waiting for ComfyUI to save the MP4...', progress: 96 })
+          try {
+            const saved = await succeed()
+            if (!saved) {
+              emitLocal({ type: 'status', message: 'Waiting for ComfyUI to save the MP4...', progress: 96 })
+            }
+          } catch (error) {
+            finishing = false
+            job.saving = false
+            emitLocal({
+              type: 'status',
+              message: `Save hiccup (${errorText(error)}). Retrying...`,
+              progress: 96
+            })
          }
          return
        }