Coordinate shared GPU access across studio instances

server/utils/studioQueue.ts

@@ -1,3 +1,4 @@
+import { withSharedGpuStart, sharedGpuWaitReason, maintainSharedGpu, acquireSharedGpu } from '~/server/utils/sharedGpu'
 import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { getJob, listJobs, emitJob, type Job } from '~/server/utils/jobs'
@@ -98,6 +99,7 @@ export interface StudioJob {
   pausedByUser?: boolean
   resumeAutoRun?: boolean
   lastError?: string
+  waitReason?: string
 }
 
 type StudioQueueStore = {
@@ -246,7 +248,8 @@ export function summarizeStudioJob(job: StudioJob) {
     holdForCutIn: job.holdForCutIn === true,
     pauseAfterCurrent: job.pauseAfterCurrent === true,
     pausedByUser: job.pausedByUser === true,
-    lastError: job.lastError
+    lastError: job.lastError,
+    waitReason: job.status === 'waiting' ? job.waitReason : undefined
   }
 }
 
@@ -415,14 +418,11 @@ export async function videoJobsBusy() {
   }))) return true
   const queue = await fetchLiveQueue()
   if (queue) {
-    if (queue.running > 0) return true
-    if (queue.pending > 0) {
-      if (listJobs().some(job => liveJobOwnsGpu(job) || jobIsLocallySubmitting(job))) return true
-      if (listPendingJobs().some((pending) => {
-        if (!pending.promptId || pending.stopAfterCurrent) return false
-        return Date.now() - pending.startedAt < PENDING_ORPHAN_MS
-      })) return true
-      return false
+    if (queue.running > 0 || queue.pending > 0) {
+      // Join the device queue while another instance is rendering, so it cannot
+      // reclaim every gap ahead of this waiting instance.
+      await acquireSharedGpu()
+      return true
     }
     return false
   }
@@ -865,7 +865,7 @@ function pendingAlive(job: StudioJob) {
 }
 
 function isTransientComfyError(error?: string) {
-  return /still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker|refused to start|API never answered|cold start|ECONNREFUSED|ETIMEDOUT|fetch failed|502|503/i.test(String(error || ''))
+  return /GPU is in use|GPU reservation|GPU coordinator|GPU_BUSY|still busy|COMFY_BUSY|offline|never answered|host agent|unreachable|asleep|Starting Comfy|ComfyUI \(1\)|instance picker|refused to start|API never answered|cold start|ECONNREFUSED|ETIMEDOUT|fetch failed|502|503/i.test(String(error || ''))
 }
 
 async function parkStudioOnStartFailure(owner: string, id: string, message: string) {
@@ -1010,11 +1010,12 @@ async function dispatchStudioQueue() {
   }
   await settleAttachedTerminalLives()
   if (await videoJobsBusy()) {
-    if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting'))) {
+    if (owners.some(owner => readJobs(owner).some(job => job.status === 'waiting' || (job.status === 'held' && !job.pausedByUser)))) {
       scheduleKickRetry()
     }
     return
   }
+  await maintainSharedGpu()
   // Re-read after videoJobsBusy — reap/settle may have rewritten disk rows.
   for (const owner of owners) {
     const store = readStore(owner)
@@ -1039,6 +1040,13 @@ async function dispatchStudioQueue() {
 }
 
 async function resumeHeldStudioJob(item: StudioJob) {
+  return withSharedGpuStart(() => resumeHeldStudioJobReserved(item), async () => {
+    scheduleKickRetry()
+    return false
+  })
+}
+
+async function resumeHeldStudioJobReserved(item: StudioJob) {
   const latest = readJobs(item.ownerKey).find(job => job.id === item.id)
   if (!latest || latest.status === 'cancelled') return false
   if (!latest.shotQueueId) {
@@ -1386,6 +1394,13 @@ async function startStudioMusicJob(item: StudioJob) {
 }
 
 export async function startStudioJob(item: StudioJob) {
+  return withSharedGpuStart(() => startStudioJobReserved(item), async () => {
+    await patchStudioJob(item.ownerKey, item.id, row => { row.waitReason = sharedGpuWaitReason() })
+    scheduleKickRetry()
+  })
+}
+
+async function startStudioJobReserved(item: StudioJob) {
   if (studioJobKind(item) === 'music') {
     await startStudioMusicJob(item)
     return