Merge pull request #57 from BobbyL2k/feat/llama-cpp-proxy

feat: Proxy llama.cpp API endpoints via `/llama-cpp/{name}/`

README.md

@@ -197,6 +197,7 @@ server:
   host: "0.0.0.0"                # Server host to bind to
   port: 8080                     # Server port to bind to
   allowed_origins: ["*"]         # Allowed CORS origins (default: all)
+  allowed_headers: ["*"]         # Allowed CORS headers (default: all)
   enable_swagger: false          # Enable Swagger UI for API docs
 
 backends:

docs/getting-started/configuration.md

@@ -17,6 +17,7 @@ server:
   host: "0.0.0.0"                # Server host to bind to
   port: 8080                     # Server port to bind to
   allowed_origins: ["*"]         # Allowed CORS origins (default: all)
+  allowed_headers: ["*"]         # Allowed CORS headers (default: all)
   enable_swagger: false          # Enable Swagger UI for API docs
 
 backends:
@@ -104,6 +105,7 @@ server:
   host: "0.0.0.0"         # Server host to bind to (default: "0.0.0.0")
   port: 8080              # Server port to bind to (default: 8080)
   allowed_origins: ["*"]  # CORS allowed origins (default: ["*"])
+  allowed_headers: ["*"]  # CORS allowed headers (default: ["*"])
   enable_swagger: false   # Enable Swagger UI (default: false)
 ```
 

pkg/config/config.go

@@ -57,6 +57,9 @@ type ServerConfig struct {
 	// Allowed origins for CORS (e.g., "http://localhost:3000")
 	AllowedOrigins []string `yaml:"allowed_origins"`
 
+	// Allowed headers for CORS (e.g., "Accept", "Authorization", "Content-Type", "X-CSRF-Token")
+	AllowedHeaders []string `yaml:"allowed_headers"`
+
 	// Enable Swagger UI for API documentation
 	EnableSwagger bool `yaml:"enable_swagger"`
 
@@ -136,6 +139,7 @@ func LoadConfig(configPath string) (AppConfig, error) {
 			Host:           "0.0.0.0",
 			Port:           8080,
 			AllowedOrigins: []string{"*"}, // Default to allow all origins
+			AllowedHeaders: []string{"*"}, // Default to allow all headers
 			EnableSwagger:  false,
 		},
 		Backends: BackendConfig{

pkg/manager/manager.go

@@ -263,19 +263,32 @@ func (im *instanceManager) loadInstance(name, path string) error {
 }
 
 // autoStartInstances starts instances that were running when persisted and have auto-restart enabled
+// For instances with auto-restart disabled, it sets their status to Stopped
 func (im *instanceManager) autoStartInstances() {
 	im.mu.RLock()
 	var instancesToStart []*instance.Process
+	var instancesToStop []*instance.Process
 	for _, inst := range im.instances {
 		if inst.IsRunning() && // Was running when persisted
 			inst.GetOptions() != nil &&
-			inst.GetOptions().AutoRestart != nil &&
-			*inst.GetOptions().AutoRestart {
-			instancesToStart = append(instancesToStart, inst)
+			inst.GetOptions().AutoRestart != nil {
+			if *inst.GetOptions().AutoRestart {
+				instancesToStart = append(instancesToStart, inst)
+			} else {
+				// Instance was running but auto-restart is disabled, mark as stopped
+				instancesToStop = append(instancesToStop, inst)
+			}
 		}
 	}
 	im.mu.RUnlock()
 
+	// Stop instances that have auto-restart disabled
+	for _, inst := range instancesToStop {
+		log.Printf("Instance %s was running but auto-restart is disabled, setting status to stopped", inst.Name)
+		inst.SetStatus(instance.Stopped)
+	}
+
+	// Start instances that have auto-restart enabled
 	for _, inst := range instancesToStart {
 		log.Printf("Auto-starting instance %s", inst.Name)
 		// Reset running state before starting (since Start() expects stopped instance)

pkg/manager/manager_test.go

@@ -209,3 +209,66 @@ func createTestManager() manager.InstanceManager {
 	}
 	return manager.NewInstanceManager(backendConfig, cfg)
 }
+
+func TestAutoRestartDisabledInstanceStatus(t *testing.T) {
+	tempDir := t.TempDir()
+
+	backendConfig := config.BackendConfig{
+		LlamaCpp: config.BackendSettings{
+			Command: "llama-server",
+		},
+	}
+
+	cfg := config.InstancesConfig{
+		PortRange:            [2]int{8000, 9000},
+		InstancesDir:         tempDir,
+		MaxInstances:         10,
+		TimeoutCheckInterval: 5,
+	}
+
+	// Create first manager and instance with auto-restart disabled
+	manager1 := manager.NewInstanceManager(backendConfig, cfg)
+
+	autoRestart := false
+	options := &instance.CreateInstanceOptions{
+		BackendType: backends.BackendTypeLlamaCpp,
+		AutoRestart: &autoRestart,
+		LlamaServerOptions: &llamacpp.LlamaServerOptions{
+			Model: "/path/to/model.gguf",
+			Port:  8080,
+		},
+	}
+
+	inst, err := manager1.CreateInstance("test-instance", options)
+	if err != nil {
+		t.Fatalf("CreateInstance failed: %v", err)
+	}
+
+	// Simulate instance being in running state when persisted
+	// (this would happen if the instance was running when llamactl was stopped)
+	inst.SetStatus(instance.Running)
+
+	// Shutdown first manager
+	manager1.Shutdown()
+
+	// Create second manager (simulating restart of llamactl)
+	manager2 := manager.NewInstanceManager(backendConfig, cfg)
+
+	// Get the loaded instance
+	loadedInst, err := manager2.GetInstance("test-instance")
+	if err != nil {
+		t.Fatalf("GetInstance failed: %v", err)
+	}
+
+	// The instance should be marked as Stopped, not Running
+	// because auto-restart is disabled
+	if loadedInst.IsRunning() {
+		t.Errorf("Expected instance with auto-restart disabled to be stopped after manager restart, but it was running")
+	}
+
+	if loadedInst.GetStatus() != instance.Stopped {
+		t.Errorf("Expected instance status to be Stopped, got %v", loadedInst.GetStatus())
+	}
+
+	manager2.Shutdown()
+}

pkg/server/handlers.go

@@ -131,11 +131,16 @@ func (h *Handler) ListInstances() http.HandlerFunc {
 			return
 		}
 
-		w.Header().Set("Content-Type", "application/json")
-		if err := json.NewEncoder(w).Encode(instances); err != nil {
+		// Marshal to bytes first to set Content-Length header
+		data, err := json.Marshal(instances)
+		if err != nil {
 			http.Error(w, "Failed to encode instances: "+err.Error(), http.StatusInternalServerError)
 			return
 		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+		w.Write(data)
 	}
 }
 
@@ -202,7 +207,7 @@ func (h *Handler) GetInstance() http.HandlerFunc {
 
 		inst, err := h.InstanceManager.GetInstance(name)
 		if err != nil {
-			http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError)
+			http.Error(w, "Invalid instance: "+err.Error(), http.StatusBadRequest)
 			return
 		}
 
@@ -475,29 +480,15 @@ func (h *Handler) ProxyToInstance() http.HandlerFunc {
 
 		// Strip the "/api/v1/instances/<name>/proxy" prefix from the request URL
 		prefix := fmt.Sprintf("/api/v1/instances/%s/proxy", name)
-		proxyPath := r.URL.Path[len(prefix):]
-
-		// Ensure the proxy path starts with "/"
-		if !strings.HasPrefix(proxyPath, "/") {
-			proxyPath = "/" + proxyPath
-		}
+		r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix)
 
 		// Update the last request time for the instance
 		inst.UpdateLastRequestTime()
 
-		// Modify the request to remove the proxy prefix
-		originalPath := r.URL.Path
-		r.URL.Path = proxyPath
-
 		// Set forwarded headers
 		r.Header.Set("X-Forwarded-Host", r.Header.Get("Host"))
 		r.Header.Set("X-Forwarded-Proto", "http")
 
-		// Restore original path for logging purposes
-		defer func() {
-			r.URL.Path = originalPath
-		}()
-
 		// Forward the request using the cached proxy
 		proxy.ServeHTTP(w, r)
 	}
@@ -580,12 +571,13 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc {
 		// Route to the appropriate inst based on instance name
 		inst, err := h.InstanceManager.GetInstance(modelName)
 		if err != nil {
-			http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError)
+			http.Error(w, "Invalid instance: "+err.Error(), http.StatusBadRequest)
 			return
 		}
 
 		if !inst.IsRunning() {
-			allowOnDemand := inst.GetOptions() != nil && inst.GetOptions().OnDemandStart != nil && *inst.GetOptions().OnDemandStart
+			options := inst.GetOptions()
+			allowOnDemand := options != nil && options.OnDemandStart != nil && *options.OnDemandStart
 			if !allowOnDemand {
 				http.Error(w, "Instance is not running", http.StatusServiceUnavailable)
 				return
@@ -634,6 +626,84 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc {
 	}
 }
 
+func (h *Handler) LlamaCppProxy(onDemandStart bool) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		// Get the instance name from the URL parameter
+		name := chi.URLParam(r, "name")
+		if name == "" {
+			http.Error(w, "Instance name cannot be empty", http.StatusBadRequest)
+			return
+		}
+
+		// Route to the appropriate inst based on instance name
+		inst, err := h.InstanceManager.GetInstance(name)
+		if err != nil {
+			http.Error(w, "Invalid instance: "+err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		options := inst.GetOptions()
+		if options == nil {
+			http.Error(w, "Cannot obtain Instance's options", http.StatusInternalServerError)
+			return
+		}
+
+		if options.BackendType != backends.BackendTypeLlamaCpp {
+			http.Error(w, "Instance is not a llama.cpp server.", http.StatusBadRequest)
+			return
+		}
+
+		if !inst.IsRunning() {
+
+			if !(onDemandStart && options.OnDemandStart != nil && *options.OnDemandStart) {
+				http.Error(w, "Instance is not running", http.StatusServiceUnavailable)
+				return
+			}
+
+			if h.InstanceManager.IsMaxRunningInstancesReached() {
+				if h.cfg.Instances.EnableLRUEviction {
+					err := h.InstanceManager.EvictLRUInstance()
+					if err != nil {
+						http.Error(w, "Cannot start Instance, failed to evict instance "+err.Error(), http.StatusInternalServerError)
+						return
+					}
+				} else {
+					http.Error(w, "Cannot start Instance, maximum number of instances reached", http.StatusConflict)
+					return
+				}
+			}
+
+			// If on-demand start is enabled, start the instance
+			if _, err := h.InstanceManager.StartInstance(name); err != nil {
+				http.Error(w, "Failed to start instance: "+err.Error(), http.StatusInternalServerError)
+				return
+			}
+
+			// Wait for the instance to become healthy before proceeding
+			if err := inst.WaitForHealthy(h.cfg.Instances.OnDemandStartTimeout); err != nil { // 2 minutes timeout
+				http.Error(w, "Instance failed to become healthy: "+err.Error(), http.StatusServiceUnavailable)
+				return
+			}
+		}
+
+		proxy, err := inst.GetProxy()
+		if err != nil {
+			http.Error(w, "Failed to get proxy: "+err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// Strip the "/llama-cpp/<name>" prefix from the request URL
+		prefix := fmt.Sprintf("/llama-cpp/%s", name)
+		r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix)
+
+		// Update the last request time for the instance
+		inst.UpdateLastRequestTime()
+
+		proxy.ServeHTTP(w, r)
+	}
+}
+
 // ParseCommandRequest represents the request body for command parsing
 type ParseCommandRequest struct {
 	Command string `json:"command"`
@@ -714,21 +784,21 @@ func (h *Handler) ParseMlxCommand() http.HandlerFunc {
 			writeError(w, http.StatusBadRequest, "invalid_request", "Invalid JSON body")
 			return
 		}
-		
+
 		if strings.TrimSpace(req.Command) == "" {
 			writeError(w, http.StatusBadRequest, "invalid_command", "Command cannot be empty")
 			return
 		}
-		
+
 		mlxOptions, err := mlx.ParseMlxCommand(req.Command)
 		if err != nil {
 			writeError(w, http.StatusBadRequest, "parse_error", err.Error())
 			return
 		}
-		
+
 		// Currently only support mlx_lm backend type
 		backendType := backends.BackendTypeMlxLm
-		
+
 		options := &instance.CreateInstanceOptions{
 			BackendType:      backendType,
 			MlxServerOptions: mlxOptions,

pkg/server/routes.go

@@ -20,7 +20,7 @@ func SetupRouter(handler *Handler) *chi.Mux {
 	r.Use(cors.Handler(cors.Options{
 		AllowedOrigins:   handler.cfg.Server.AllowedOrigins,
 		AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
-		AllowedHeaders:   []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"},
+		AllowedHeaders:   handler.cfg.Server.AllowedHeaders,
 		ExposedHeaders:   []string{"Link"},
 		AllowCredentials: false,
 		MaxAge:           300,
@@ -103,6 +103,51 @@ func SetupRouter(handler *Handler) *chi.Mux {
 
 	})
 
+	r.Route("/llama-cpp/{name}", func(r chi.Router) {
+
+		// Public Routes
+		// Allow llama-cpp server to serve its own WebUI if it is running.
+		// Don't auto start the server since it can be accessed without an API key
+		r.Get("/", handler.LlamaCppProxy(false))
+
+		// Private Routes
+		r.Group(func(r chi.Router) {
+
+			if authMiddleware != nil && handler.cfg.Auth.RequireInferenceAuth {
+				r.Use(authMiddleware.AuthMiddleware(KeyTypeInference))
+			}
+
+			// This handler auto start the server if it's not running
+			llamaCppHandler := handler.LlamaCppProxy(true)
+
+			// llama.cpp server specific proxy endpoints
+			r.Get("/props", llamaCppHandler)
+			// /slots endpoint is secured (see: https://github.com/ggml-org/llama.cpp/pull/15630)
+			r.Get("/slots", llamaCppHandler)
+			r.Post("/apply-template", llamaCppHandler)
+			r.Post("/completion", llamaCppHandler)
+			r.Post("/detokenize", llamaCppHandler)
+			r.Post("/embeddings", llamaCppHandler)
+			r.Post("/infill", llamaCppHandler)
+			r.Post("/metrics", llamaCppHandler)
+			r.Post("/props", llamaCppHandler)
+			r.Post("/reranking", llamaCppHandler)
+			r.Post("/tokenize", llamaCppHandler)
+
+			// OpenAI-compatible proxy endpoint
+			// Handles all POST requests to /v1/*, including:
+			//   - /v1/completions
+			//   - /v1/chat/completions
+			//   - /v1/embeddings
+			//   - /v1/rerank
+			//   - /v1/reranking
+			// llamaCppHandler is used here because some users of llama.cpp endpoints depend
+			// on "model" field being optional, and handler.OpenAIProxy requires it.
+			r.Post("/v1/*", llamaCppHandler)
+		})
+
+	})
+
 	// Serve WebUI files
 	if err := webui.SetupWebUI(r); err != nil {
 		fmt.Printf("Failed to set up WebUI: %v\n", err)

webui/src/lib/__tests__/api.test.ts

@@ -11,11 +11,13 @@ describe('API Error Handling', () => {
   })
 
   it('converts HTTP errors to meaningful messages', async () => {
-    mockFetch.mockResolvedValue({
+    const mockResponse = {
       ok: false,
       status: 409,
-      text: () => Promise.resolve('Instance already exists')
-    })
+      text: () => Promise.resolve('Instance already exists'),
+      clone: function() { return this }
+    }
+    mockFetch.mockResolvedValue(mockResponse)
 
     await expect(instancesApi.create('existing', {}))
       .rejects
@@ -23,11 +25,13 @@ describe('API Error Handling', () => {
   })
 
   it('handles empty error responses gracefully', async () => {
-    mockFetch.mockResolvedValue({
+    const mockResponse = {
       ok: false,
       status: 500,
-      text: () => Promise.resolve('')
-    })
+      text: () => Promise.resolve(''),
+      clone: function() { return this }
+    }
+    mockFetch.mockResolvedValue(mockResponse)
 
     await expect(instancesApi.list())
       .rejects

webui/src/lib/api.ts

@@ -49,11 +49,8 @@ async function apiCall<T>(
     } else {
       // Handle empty responses for JSON endpoints
       const contentLength = response.headers.get('content-length');
-      if (contentLength === '0' || contentLength === null) {
-        const text = await response.text();
-        if (text.trim() === '') {
-          return {} as T; // Return empty object for empty JSON responses
-        }
+      if (contentLength === '0') {
+        return {} as T; // Return empty object for empty JSON responses
       }
       const data = await response.json() as T;
       return data;

webui/src/lib/errorUtils.ts

@@ -26,7 +26,8 @@ export async function handleApiError(response: Response): Promise<void> {
   }
 
   if (!response.ok) {
-    const errorMessage = await parseErrorResponse(response)
+    // Clone the response before reading to avoid consuming the body stream
+    const errorMessage = await parseErrorResponse(response.clone())
     throw new Error(errorMessage)
   }
 }