From a3f9213f0474de92355f075e4054bdf0d160bce3 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Sat, 25 Oct 2025 23:44:21 +0200 Subject: [PATCH] Implement ensureInstanceRunning helper --- pkg/server/handlers.go | 45 ++++++++++++++++++++++++++++++++ pkg/server/handlers_backends.go | 44 +++---------------------------- pkg/server/handlers_openai.go | 46 ++++++++++++--------------------- pkg/server/openai.go | 13 ---------- 4 files changed, 66 insertions(+), 82 deletions(-) delete mode 100644 pkg/server/openai.go diff --git a/pkg/server/handlers.go b/pkg/server/handlers.go index 4ddbfea..52ef533 100644 --- a/pkg/server/handlers.go +++ b/pkg/server/handlers.go @@ -1,12 +1,26 @@ package server import ( + "encoding/json" + "fmt" "llamactl/pkg/config" + "llamactl/pkg/instance" "llamactl/pkg/manager" "net/http" "time" ) +type errorResponse struct { + Error string `json:"error"` + Details string `json:"details,omitempty"` +} + +func writeError(w http.ResponseWriter, status int, code, details string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(errorResponse{Error: code, Details: details}) +} + type Handler struct { InstanceManager manager.InstanceManager cfg config.AppConfig @@ -22,3 +36,34 @@ func NewHandler(im manager.InstanceManager, cfg config.AppConfig) *Handler { }, } } + +func (h *Handler) ensureInstanceRunning(inst *instance.Instance) error { + options := inst.GetOptions() + allowOnDemand := options != nil && options.OnDemandStart != nil && *options.OnDemandStart + if !allowOnDemand { + return fmt.Errorf("instance is not running and on-demand start is not enabled") + } + + if h.InstanceManager.IsMaxRunningInstancesReached() { + if h.cfg.Instances.EnableLRUEviction { + err := h.InstanceManager.EvictLRUInstance() + if err != nil { + return fmt.Errorf("cannot start instance, failed to evict instance: %w", err) + } + } else { + return fmt.Errorf("cannot start instance, maximum number of instances reached") + } + } + + // If on-demand start is enabled, start the instance + if _, err := h.InstanceManager.StartInstance(inst.Name); err != nil { + return fmt.Errorf("failed to start instance: %w", err) + } + + // Wait for the instance to become healthy before proceeding + if err := inst.WaitForHealthy(h.cfg.Instances.OnDemandStartTimeout); err != nil { + return fmt.Errorf("instance failed to become healthy: %w", err) + } + + return nil +} diff --git a/pkg/server/handlers_backends.go b/pkg/server/handlers_backends.go index 9433a93..a53b40b 100644 --- a/pkg/server/handlers_backends.go +++ b/pkg/server/handlers_backends.go @@ -18,17 +18,6 @@ type ParseCommandRequest struct { Command string `json:"command"` } -type errorResponse struct { - Error string `json:"error"` - Details string `json:"details,omitempty"` -} - -func writeError(w http.ResponseWriter, status int, code, details string) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(errorResponse{Error: code, Details: details}) -} - func (h *Handler) LlamaCppProxy(onDemandStart bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -60,35 +49,10 @@ func (h *Handler) LlamaCppProxy(onDemandStart bool) http.HandlerFunc { return } - if !inst.IsRemote() && !inst.IsRunning() { - - if !(onDemandStart && options.OnDemandStart != nil && *options.OnDemandStart) { - http.Error(w, "Instance is not running", http.StatusServiceUnavailable) - return - } - - if h.InstanceManager.IsMaxRunningInstancesReached() { - if h.cfg.Instances.EnableLRUEviction { - err := h.InstanceManager.EvictLRUInstance() - if err != nil { - http.Error(w, "Cannot start Instance, failed to evict instance "+err.Error(), http.StatusInternalServerError) - return - } - } else { - http.Error(w, "Cannot start Instance, maximum number of instances reached", http.StatusConflict) - return - } - } - - // If on-demand start is enabled, start the instance - if _, err := h.InstanceManager.StartInstance(validatedName); err != nil { - http.Error(w, "Failed to start instance: "+err.Error(), http.StatusInternalServerError) - return - } - - // Wait for the instance to become healthy before proceeding - if err := inst.WaitForHealthy(h.cfg.Instances.OnDemandStartTimeout); err != nil { // 2 minutes timeout - http.Error(w, "Instance failed to become healthy: "+err.Error(), http.StatusServiceUnavailable) + if !inst.IsRemote() && !inst.IsRunning() && onDemandStart { + err := h.ensureInstanceRunning(inst) + if err != nil { + http.Error(w, "Failed to ensure instance is running: "+err.Error(), http.StatusInternalServerError) return } } diff --git a/pkg/server/handlers_openai.go b/pkg/server/handlers_openai.go index 9ad3207..251a4b1 100644 --- a/pkg/server/handlers_openai.go +++ b/pkg/server/handlers_openai.go @@ -8,6 +8,20 @@ import ( "net/http" ) +// OpenAIListInstancesResponse represents the response structure for listing instances (models) in OpenAI format +type OpenAIListInstancesResponse struct { + Object string `json:"object"` + Data []OpenAIInstance `json:"data"` +} + +// OpenAIInstance represents a single instance (model) in OpenAI format +type OpenAIInstance struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + OwnedBy string `json:"owned_by"` +} + // OpenAIListInstances godoc // @Summary List instances in OpenAI-compatible format // @Description Returns a list of instances in a format compatible with OpenAI API @@ -97,35 +111,9 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc { } if !inst.IsRemote() && !inst.IsRunning() { - options := inst.GetOptions() - allowOnDemand := options != nil && options.OnDemandStart != nil && *options.OnDemandStart - if !allowOnDemand { - http.Error(w, "Instance is not running", http.StatusServiceUnavailable) - return - } - - if h.InstanceManager.IsMaxRunningInstancesReached() { - if h.cfg.Instances.EnableLRUEviction { - err := h.InstanceManager.EvictLRUInstance() - if err != nil { - http.Error(w, "Cannot start Instance, failed to evict instance "+err.Error(), http.StatusInternalServerError) - return - } - } else { - http.Error(w, "Cannot start Instance, maximum number of instances reached", http.StatusConflict) - return - } - } - - // If on-demand start is enabled, start the instance - if _, err := h.InstanceManager.StartInstance(validatedName); err != nil { - http.Error(w, "Failed to start instance: "+err.Error(), http.StatusInternalServerError) - return - } - - // Wait for the instance to become healthy before proceeding - if err := inst.WaitForHealthy(h.cfg.Instances.OnDemandStartTimeout); err != nil { // 2 minutes timeout - http.Error(w, "Instance failed to become healthy: "+err.Error(), http.StatusServiceUnavailable) + err := h.ensureInstanceRunning(inst) + if err != nil { + http.Error(w, "Failed to ensure instance is running: "+err.Error(), http.StatusInternalServerError) return } } diff --git a/pkg/server/openai.go b/pkg/server/openai.go deleted file mode 100644 index 98d1043..0000000 --- a/pkg/server/openai.go +++ /dev/null @@ -1,13 +0,0 @@ -package server - -type OpenAIListInstancesResponse struct { - Object string `json:"object"` - Data []OpenAIInstance `json:"data"` -} - -type OpenAIInstance struct { - ID string `json:"id"` - Object string `json:"object"` - Created int64 `json:"created"` - OwnedBy string `json:"owned_by"` -}