Mathis/llamactl
1
0
Fork 0
mirror of https://github.com/lordmathis/llamactl.git synced 2025-11-05 16:44:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove remote instance proxy handling from handlers

Browse Source
This commit is contained in:
Matúš Námešný
2025-10-25 14:07:11 +02:00
parent 6a973fae2d
commit ff719f3ef9
6 changed files with 13 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/server/handlers.go
View File

@@ -4,8 +4,6 @@ import (
"llamactl/pkg/config"
"llamactl/pkg/manager"
"net/http"
"net/http/httputil"
"sync"
"time"
)
@@ -13,8 +11,6 @@ type Handler struct {
InstanceManager manager.InstanceManager
cfg config.AppConfig
httpClient *http.Client
remoteProxies map[string]*httputil.ReverseProxy // Cache of remote proxies by instance name
remoteProxiesMu sync.RWMutex
}
func NewHandler(im manager.InstanceManager, cfg config.AppConfig) *Handler {
@@ -24,6 +20,5 @@ func NewHandler(im manager.InstanceManager, cfg config.AppConfig) *Handler {
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
remoteProxies: make(map[string]*httputil.ReverseProxy),
}
}

2
pkg/server/handlers_backends.go
View File

@@ -49,7 +49,7 @@ func (h *Handler) LlamaCppProxy(onDemandStart bool) http.HandlerFunc {
return
}
if !inst.IsRunning() {
if !inst.IsRemote() && !inst.IsRunning() {
if !(onDemandStart && options.OnDemandStart != nil && *options.OnDemandStart) {
http.Error(w, "Instance is not running", http.StatusServiceUnavailable)

80
pkg/server/handlers_instances.go
View File

@@ -7,8 +7,6 @@ import (
"llamactl/pkg/manager"
"llamactl/pkg/validation"
"net/http"
"net/http/httputil"
"net/url"
"strconv"
"strings"
@@ -375,12 +373,6 @@ func (h *Handler) ProxyToInstance() http.HandlerFunc {
return
}
// Check if this is a remote instance
if inst.IsRemote() {
h.RemoteInstanceProxy(w, r, validatedName, inst)
return
}
if !inst.IsRunning() {
http.Error(w, "Instance is not running", http.StatusServiceUnavailable)
return
@@ -393,9 +385,12 @@ func (h *Handler) ProxyToInstance() http.HandlerFunc {
return
}
// Strip the "/api/v1/instances/<name>/proxy" prefix from the request URL
prefix := fmt.Sprintf("/api/v1/instances/%s/proxy", validatedName)
r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix)
// Check if this is a remote instance
if !inst.IsRemote() {
// Strip the "/api/v1/instances/<name>/proxy" prefix from the request URL
prefix := fmt.Sprintf("/api/v1/instances/%s/proxy", validatedName)
r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix)
}
// Update the last request time for the instance
inst.UpdateLastRequestTime()
@@ -408,66 +403,3 @@ func (h *Handler) ProxyToInstance() http.HandlerFunc {
proxy.ServeHTTP(w, r)
}
}
// RemoteInstanceProxy proxies requests to a remote instance
func (h *Handler) RemoteInstanceProxy(w http.ResponseWriter, r *http.Request, name string, inst *instance.Instance) {
// Get the node name from instance options
options := inst.GetOptions()
if options == nil {
http.Error(w, "Instance has no options configured", http.StatusInternalServerError)
return
}
// Get the first node from the set
var nodeName string
for node := range options.Nodes {
nodeName = node
break
}
if nodeName == "" {
http.Error(w, "Instance has no node configured", http.StatusInternalServerError)
return
}
// Check if we have a cached proxy for this node
h.remoteProxiesMu.RLock()
proxy, exists := h.remoteProxies[nodeName]
h.remoteProxiesMu.RUnlock()
if !exists {
// Find node configuration
nodeConfig, exists := h.cfg.Nodes[nodeName]
if !exists {
http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError)
return
}
// Create reverse proxy to remote node
targetURL, err := url.Parse(nodeConfig.Address)
if err != nil {
http.Error(w, "Failed to parse node address: "+err.Error(), http.StatusInternalServerError)
return
}
proxy = httputil.NewSingleHostReverseProxy(targetURL)
// Modify request before forwarding
originalDirector := proxy.Director
apiKey := nodeConfig.APIKey // Capture for closure
proxy.Director = func(req *http.Request) {
originalDirector(req)
// Add API key if configured
if apiKey != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
}
}
// Cache the proxy by node name
h.remoteProxiesMu.Lock()
h.remoteProxies[nodeName] = proxy
h.remoteProxiesMu.Unlock()
}
// Forward the request using the cached proxy
proxy.ServeHTTP(w, r)
}

77
pkg/server/handlers_openai.go
View File

@@ -3,13 +3,9 @@ package server
import (
"bytes"
"encoding/json"
"fmt"
"io"
"llamactl/pkg/instance"
"llamactl/pkg/validation"
"net/http"
"net/http/httputil"
"net/url"
)
// OpenAIListInstances godoc
@@ -100,15 +96,7 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc {
return
}
// Check if this is a remote instance
if inst.IsRemote() {
// Restore the body for the remote proxy
r.Body = io.NopCloser(bytes.NewReader(bodyBytes))
h.RemoteOpenAIProxy(w, r, validatedName, inst)
return
}
if !inst.IsRunning() {
if !inst.IsRemote() && !inst.IsRunning() {
options := inst.GetOptions()
allowOnDemand := options != nil && options.OnDemandStart != nil && *options.OnDemandStart
if !allowOnDemand {
@@ -158,66 +146,3 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc {
proxy.ServeHTTP(w, r)
}
}
// RemoteOpenAIProxy proxies OpenAI-compatible requests to a remote instance
func (h *Handler) RemoteOpenAIProxy(w http.ResponseWriter, r *http.Request, modelName string, inst *instance.Instance) {
// Get the node name from instance options
options := inst.GetOptions()
if options == nil {
http.Error(w, "Instance has no options configured", http.StatusInternalServerError)
return
}
// Get the first node from the set
var nodeName string
for node := range options.Nodes {
nodeName = node
break
}
if nodeName == "" {
http.Error(w, "Instance has no node configured", http.StatusInternalServerError)
return
}
// Check if we have a cached proxy for this node
h.remoteProxiesMu.RLock()
proxy, exists := h.remoteProxies[nodeName]
h.remoteProxiesMu.RUnlock()
if !exists {
// Find node configuration
nodeConfig, exists := h.cfg.Nodes[nodeName]
if !exists {
http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError)
return
}
// Create reverse proxy to remote node
targetURL, err := url.Parse(nodeConfig.Address)
if err != nil {
http.Error(w, "Failed to parse node address: "+err.Error(), http.StatusInternalServerError)
return
}
proxy = httputil.NewSingleHostReverseProxy(targetURL)
// Modify request before forwarding
originalDirector := proxy.Director
apiKey := nodeConfig.APIKey // Capture for closure
proxy.Director = func(req *http.Request) {
originalDirector(req)
// Add API key if configured
if apiKey != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey))
}
}
// Cache the proxy
h.remoteProxiesMu.Lock()
h.remoteProxies[nodeName] = proxy
h.remoteProxiesMu.Unlock()
}
// Forward the request using the cached proxy
proxy.ServeHTTP(w, r)
}