Merge 2.6.1-beta to main for release (#332)

Co-authored-by: Markus Schüttler <[email protected]>
Co-authored-by: kyle <[email protected]>
Co-authored-by: KyleHagy <[email protected]>
Co-authored-by: Daniel Hirsch <[email protected]>

Author: 4 people

.gitignore

@@ -23,5 +23,5 @@ llama-cpp-env/
 *-env/
 build-envs/
 portable-git/
-llm_cache/*
+llm_cache*/
 emb_model_cache/

OpenVINO/openvino_backend.py

@@ -25,8 +25,14 @@ def load_model(self, params: LLMParams, callback: Callable[[str], None] = None):
             model_path = path.abspath(path.join(model_base_path, model_name))
 
             enable_compile_cache = dict()
-            enable_compile_cache["CACHE_DIR"] = "llm_cache"
-            self._model = openvino_genai.LLMPipeline(model_path, environ.get("OPENVINO_DEVICE", "AUTO"), **enable_compile_cache)
+            device = environ.get("OPENVINO_DEVICE", "AUTO")
+            if device == "NPU":
+                enable_compile_cache["MAX_PROMPT_LEN"] = int(environ.get("MAX_PROMPT_LEN", 1024))
+                cache_postfix = f"NPU_{str(enable_compile_cache["MAX_PROMPT_LEN"])}"
+            else:
+                cache_postfix = device
+            enable_compile_cache["CACHE_DIR"] = f"llm_cache_{cache_postfix}"
+            self._model = openvino_genai.LLMPipeline(model_path, device, **enable_compile_cache)
             self._tokenizer = self._model.get_tokenizer()
 
             self._last_repo_id = model_repo_id

WebUI/build/build-config.json

@@ -34,7 +34,7 @@
     {
       "from": "../OpenVINO",
       "to": "OpenVINO",
-      "filter": ["!__pycache__/", "!.cache/", "!db/", "!llm_cache/", "!emb_model_cache/"]
+      "filter": ["!__pycache__/", "!.cache/", "!db/", "!llm_cache*/**", "!emb_model_cache/"]
     },
     {
       "from": "external/workflows",

WebUI/electron/main.ts

@@ -54,7 +54,6 @@ import type { ModelPaths } from '@/assets/js/store/models.ts'
 import type { IndexedDocument, EmbedInquiry } from '@/assets/js/store/textInference.ts'
 import { BackendServiceName } from '@/assets/js/store/backendServices.ts'
 import z from 'zod'
-import { ModelSchema } from '../src/types/shared.ts'
 
 // }
 // The built directory structure
@@ -412,6 +411,18 @@ function initEventHandle() {
     return appSize
   })
 
+  ipcMain.handle('zoomIn', (event: IpcMainInvokeEvent) => {
+    const win = BrowserWindow.fromWebContents(event.sender)
+    if (!win) return
+    win.webContents.setZoomLevel(win.webContents.getZoomLevel() + 1)
+  })
+
+  ipcMain.handle('zoomOut', (event: IpcMainInvokeEvent) => {
+    const win = BrowserWindow.fromWebContents(event.sender)
+    if (!win) return
+    win.webContents.setZoomLevel(win.webContents.getZoomLevel() - 1)
+  })
+
   ipcMain.on('openUrl', (_event, url: string) => {
     return shell.openExternal(url)
   })
@@ -735,24 +746,24 @@ function initEventHandle() {
   ipcMain.handle('startService', (_event: IpcMainInvokeEvent, serviceName: string) => {
     if (!serviceRegistry) {
       appLogger.warn('received start signal too early during aipg startup', 'electron-backend')
-      return
+      return 'failed'
     }
     const service = serviceRegistry.getService(serviceName)
     if (!service) {
       appLogger.warn(`Tried to start service ${serviceName} which is not known`, 'electron-backend')
-      return
+      return 'failed'
     }
     return service.start()
   })
   ipcMain.handle('stopService', (_event: IpcMainInvokeEvent, serviceName: string) => {
     if (!serviceRegistry) {
       appLogger.warn('received stop signal too early during aipg startup', 'electron-backend')
-      return
+      return 'failed'
     }
     const service = serviceRegistry.getService(serviceName)
     if (!service) {
       appLogger.warn(`Tried to stop service ${serviceName} which is not known`, 'electron-backend')
-      return
+      return 'failed'
     }
     return service.stop()
   })

WebUI/electron/preload.ts

@@ -75,6 +75,8 @@ contextBridge.exposeInMainWorld('electronAPI', {
   refreshLLMModles: () => ipcRenderer.invoke('refreshLLMModles'),
   refreshLora: () => ipcRenderer.invoke('refreshLora'),
   loadModels: () => ipcRenderer.invoke('loadModels'),
+  zoomIn: () => ipcRenderer.invoke('zoomIn'),
+  zoomOut: () => ipcRenderer.invoke('zoomOut'),
   getDownloadedDiffusionModels: () => ipcRenderer.invoke('getDownloadedDiffusionModels'),
   getDownloadedInpaintModels: () => ipcRenderer.invoke('getDownloadedInpaintModels'),
   getDownloadedLoras: () => ipcRenderer.invoke('getDownloadedLoras'),

WebUI/electron/subprocesses/aiBackendService.ts

@@ -31,6 +31,10 @@ export class AiBackendService extends LongLivedPythonApiService {
   async detectDevices() {
     const availableDevices = await detectLevelZeroDevices(this.python)
     this.appLogger.info(`detected devices: ${JSON.stringify(availableDevices, null, 2)}`, this.name)
+    if (availableDevices.length === 0) {
+      this.appLogger.error(`No devices detected`, this.name)
+      return
+    }
 
     let bestDeviceId: string
     try {
@@ -47,6 +51,10 @@ export class AiBackendService extends LongLivedPythonApiService {
     this.devices = availableDevices.map((d) => ({ ...d, selected: d.id === bestDeviceId }))
   }
 
+  getServiceForPipFreeze(): UvPipService {
+    return this.uvPip
+  }
+
   async *set_up(): AsyncIterable<SetupProgress> {
     this.setStatus('installing')
     this.appLogger.info('setting up service', this.name)
@@ -137,7 +145,7 @@ export class AiBackendService extends LongLivedPythonApiService {
       this.appLogger.warn(`Aborting set up of ${this.name} service environment`, this.name, true)
       this.setStatus('installationFailed')
 
-      const errorDetails = createEnhancedErrorDetails(e, `${currentStep} operation`)
+      const errorDetails = await createEnhancedErrorDetails(e, `${currentStep} operation`, this.uvPip)
 
       yield {
         serviceName: this.name,

WebUI/electron/subprocesses/comfyUIBackendService.ts

@@ -86,6 +86,10 @@ export class ComfyUiBackendService extends LongLivedPythonApiService {
     this.devices = availableDevices.map((d) => ({ ...d, selected: d.id == '0' }))
   }
 
+  getServiceForPipFreeze(): UvPipService {
+    return this.uvPip
+  }
+
   async *set_up(): AsyncIterable<SetupProgress> {
     this.appLogger.info('setting up service', this.name)
     this.setStatus('installing')
@@ -292,7 +296,7 @@ export class ComfyUiBackendService extends LongLivedPythonApiService {
       this.appLogger.warn(`Aborting set up of ${this.name} service environment`, this.name, true)
       this.setStatus('installationFailed')
 
-      const errorDetails = createEnhancedErrorDetails(e, `${currentStep} operation`)
+      const errorDetails = await createEnhancedErrorDetails(e, `${currentStep} operation`, this.uvPip)
 
       yield {
         serviceName: this.name,

WebUI/electron/subprocesses/llamaCppBackendService.ts

@@ -151,6 +151,10 @@ export class LlamaCppBackendService extends LongLivedPythonApiService {
     }
   }
 
+  getServiceForPipFreeze(): UvPipService {
+    return this.uvPip
+  }
+
   async *set_up(): AsyncIterable<SetupProgress> {
     this.setStatus('installing')
     this.appLogger.info('setting up service', this.name)
@@ -231,7 +235,7 @@ export class LlamaCppBackendService extends LongLivedPythonApiService {
       this.appLogger.warn(`Aborting set up of ${this.name} service environment`, this.name, true)
       this.setStatus('installationFailed')
 
-      const errorDetails = createEnhancedErrorDetails(e, `${currentStep} operation`)
+      const errorDetails = await createEnhancedErrorDetails(e, `${currentStep} operation`, this.uvPip)
 
       yield {
         serviceName: this.name,

WebUI/electron/subprocesses/ollamaBackendService.ts

@@ -3,7 +3,7 @@ import path from 'node:path'
 import * as filesystem from 'fs-extra'
 import { app, BrowserWindow, net } from 'electron'
 import { appLoggerInstance } from '../logging/logger.ts'
-import { ApiService, DeviceService, PythonService, createEnhancedErrorDetails } from './service.ts'
+import { ApiService, DeviceService, PythonService, createEnhancedErrorDetails, ErrorDetails } from './service.ts'
 import { promisify } from 'util'
 import { exec } from 'child_process'
 import { detectLevelZeroDevices } from './deviceDetection.ts'
@@ -45,6 +45,9 @@ export class OllamaBackendService implements ApiService {
   encapsulatedProcess: ChildProcess | null = null
   desiredStatus: BackendStatus = 'uninitializedStatus'
 
+  // Store last startup error details for persistence
+  private lastStartupErrorDetails: ErrorDetails | null = null
+
   // Logger
   readonly appLogger = appLoggerInstance
 
@@ -117,6 +120,7 @@ export class OllamaBackendService implements ApiService {
       isSetUp: this.isSetUp,
       isRequired: this.isRequired,
       devices: this.devices,
+      errorDetails: this.lastStartupErrorDetails,
     }
   }
 
@@ -220,7 +224,7 @@ export class OllamaBackendService implements ApiService {
       this.setStatus('installationFailed')
 
       // Create detailed error information for any type of error
-      const errorDetails = createEnhancedErrorDetails(e, `${currentStep} operation`)
+      const errorDetails = await createEnhancedErrorDetails(e, `${currentStep} operation`, this.aiBackend)
 
       yield {
         serviceName: this.name,
@@ -326,6 +330,8 @@ export class OllamaBackendService implements ApiService {
       throw new Error('Server currently stopping. Cannot start it.')
     }
     if (this.currentStatus === 'running') {
+      // Clear error on successful running status
+      this.clearLastStartupError()
       return 'running'
     }
     if (this.desiredStatus === 'running') {
@@ -343,12 +349,23 @@ export class OllamaBackendService implements ApiService {
         this.currentStatus = 'running'
         this.appLogger.info(`started server ${this.name} on ${this.baseUrl}`, this.name)
         this.isSetUp = true
+        // Clear error on successful startup
+        this.clearLastStartupError()
       } else {
         this.currentStatus = 'failed'
         this.desiredStatus = 'failed'
         this.isSetUp = false
         this.appLogger.error(`server ${this.name} failed to boot`, this.name)
         this.encapsulatedProcess?.kill()
+        
+        // Capture detailed error information for startup failure
+        const startupError = new Error(`Server ${this.name} failed to boot - health check timeout or early process exit`)
+        const errorDetails = await createEnhancedErrorDetails(
+          startupError,
+          'service startup',
+          this.aiBackend
+        )
+        this.setLastStartupError(errorDetails)
       }
     } catch (error) {
       this.appLogger.error(`failed to start server due to ${error}`, this.name)
@@ -357,6 +374,14 @@ export class OllamaBackendService implements ApiService {
       this.isSetUp = false
       this.encapsulatedProcess?.kill()
       this.encapsulatedProcess = null
+      
+      // Capture detailed error information for startup exception
+      const errorDetails = await createEnhancedErrorDetails(
+        error,
+        'service startup',
+        this.aiBackend
+      )
+      this.setLastStartupError(errorDetails)
     } finally {
       this.win.webContents.send('serviceInfoUpdate', this.get_info())
     }
@@ -382,13 +407,28 @@ export class OllamaBackendService implements ApiService {
     return 'stopped'
   }
 
+  // Error management methods for startup failures
+  setLastStartupError(errorDetails: ErrorDetails): void {
+    this.lastStartupErrorDetails = errorDetails
+  }
+
+  getLastStartupError(): ErrorDetails | null {
+    return this.lastStartupErrorDetails
+  }
+
+  clearLastStartupError(): void {
+    this.lastStartupErrorDetails = null
+  }
+
   async uninstall(): Promise<void> {
     await this.stop()
     this.appLogger.info(`removing Ollama service directory`, this.name)
     await filesystem.remove(this.serviceDir)
     this.appLogger.info(`removed Ollama service directory`, this.name)
     this.setStatus('notInstalled')
     this.isSetUp = false
+    // Clear startup errors when uninstalling
+    this.clearLastStartupError()
   }
 
   pipeProcessLogs(process: ChildProcess) {

WebUI/electron/subprocesses/openVINOBackendService.ts

@@ -20,6 +20,7 @@ export class OpenVINOBackendService extends LongLivedPythonApiService {
   readonly uvPip = new UvPipService(this.pythonEnvDir, serviceFolder)
   readonly python = this.uvPip.python
   devices: InferenceDevice[] = [{ id: 'AUTO', name: 'Use best device', selected: true }]
+  currentContextSize: number | null = null
 
   serviceIsSetUp(): boolean {
     return filesystem.existsSync(this.python.getExePath())
@@ -36,6 +37,10 @@ export class OpenVINOBackendService extends LongLivedPythonApiService {
     ]
   }
 
+  getServiceForPipFreeze(): UvPipService {
+    return this.uvPip
+  }
+
   async *set_up(): AsyncIterable<SetupProgress> {
     this.setStatus('installing')
     this.appLogger.info('setting up service', this.name)
@@ -111,7 +116,7 @@ export class OpenVINOBackendService extends LongLivedPythonApiService {
       this.appLogger.warn(`Aborting set up of ${this.name} service environment`, this.name, true)
       this.setStatus('installationFailed')
 
-      const errorDetails = createEnhancedErrorDetails(e, `${currentStep} operation`)
+      const errorDetails = await createEnhancedErrorDetails(e, `${currentStep} operation`, this.uvPip)
 
       yield {
         serviceName: this.name,
@@ -138,6 +143,35 @@ export class OpenVINOBackendService extends LongLivedPythonApiService {
     }
   }
 
+  async ensureBackendReadiness(
+    llmModelName: string,
+    embeddingModelName?: string,
+    contextSize?: number,
+  ): Promise<void> {
+    this.appLogger.info(
+      `Ensuring openVINO backend readiness for contextSize: ${contextSize}`,
+      this.name,
+    )
+
+    try {
+      // Handle context size
+      if (contextSize && contextSize !== this.currentContextSize) {
+        this.currentContextSize = contextSize
+        await this.stop()
+        await this.start()
+        this.appLogger.info(`openVINO ready with context size: ${contextSize}`, this.name)
+      } else {
+        this.appLogger.info(`openVINO already running with context size: ${contextSize}`, this.name)
+      }
+    } catch (error) {
+      this.appLogger.error(
+        `Failed to ensure backend readiness - LLM: ${llmModelName}, Embedding: ${embeddingModelName ?? 'none'}: ${error}`,
+        this.name,
+      )
+      throw error
+    }
+  }
+
   async spawnAPIProcess(): Promise<{
     process: ChildProcess
     didProcessExitEarlyTracker: Promise<boolean>
@@ -147,6 +181,7 @@ export class OpenVINOBackendService extends LongLivedPythonApiService {
       SYCL_ENABLE_DEFAULT_CONTEXTS: '1',
       SYCL_CACHE_PERSISTENT: '1',
       PYTHONIOENCODING: 'utf-8',
+      ...(this.currentContextSize ? { MAX_PROMPT_LEN: this.currentContextSize.toString() } : {}),
       ...openVinoDeviceSelectorEnv(this.devices.find((d) => d.selected)?.id),
     }