From 2759be65a5d5058c5014b9c9e708f2fce00e30fa Mon Sep 17 00:00:00 2001 From: LordMathis Date: Tue, 30 Sep 2025 21:09:05 +0200 Subject: [PATCH 01/34] Add remote instance management functionality and configuration support --- pkg/config/config.go | 7 ++ pkg/instance/instance.go | 11 ++ pkg/instance/options.go | 2 + pkg/manager/manager.go | 20 ++++ pkg/manager/remote_ops.go | 228 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 268 insertions(+) create mode 100644 pkg/manager/remote_ops.go diff --git a/pkg/config/config.go b/pkg/config/config.go index 3701643..1d86f4c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -41,6 +41,7 @@ type AppConfig struct { Backends BackendConfig `yaml:"backends"` Instances InstancesConfig `yaml:"instances"` Auth AuthConfig `yaml:"auth"` + Nodes []NodeConfig `yaml:"nodes,omitempty"` Version string `yaml:"-"` CommitHash string `yaml:"-"` BuildTime string `yaml:"-"` @@ -125,6 +126,12 @@ type AuthConfig struct { ManagementKeys []string `yaml:"management_keys"` } +type NodeConfig struct { + Name string `yaml:"name"` + Address string `yaml:"address"` + APIKey string `yaml:"api_key,omitempty"` +} + // LoadConfig loads configuration with the following precedence: // 1. Hardcoded defaults // 2. Config file diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index 228f382..dee38ff 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -287,3 +287,14 @@ func (i *Process) UnmarshalJSON(data []byte) error { return nil } + +func (i *Process) IsRemote() bool { + i.mu.RLock() + defer i.mu.RUnlock() + + if i.options == nil { + return false + } + + return len(i.options.Nodes) > 0 +} diff --git a/pkg/instance/options.go b/pkg/instance/options.go index 62181dd..439f426 100644 --- a/pkg/instance/options.go +++ b/pkg/instance/options.go @@ -27,6 +27,8 @@ type CreateInstanceOptions struct { BackendType backends.BackendType `json:"backend_type"` BackendOptions map[string]any `json:"backend_options,omitempty"` + Nodes []string `json:"nodes,omitempty"` + // Backend-specific options LlamaServerOptions *llamacpp.LlamaServerOptions `json:"-"` MlxServerOptions *mlx.MlxServerOptions `json:"-"` diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 6999643..e160d6c 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -6,6 +6,7 @@ import ( "llamactl/pkg/config" "llamactl/pkg/instance" "log" + "net/http" "os" "path/filepath" "strings" @@ -29,6 +30,18 @@ type InstanceManager interface { Shutdown() } +type RemoteManager interface { + ListRemoteInstances(node *config.NodeConfig) ([]*instance.Process, error) + CreateRemoteInstance(node *config.NodeConfig, name string, options *instance.CreateInstanceOptions) (*instance.Process, error) + GetRemoteInstance(node *config.NodeConfig, name string) (*instance.Process, error) + UpdateRemoteInstance(node *config.NodeConfig, name string, options *instance.CreateInstanceOptions) (*instance.Process, error) + DeleteRemoteInstance(node *config.NodeConfig, name string) error + StartRemoteInstance(node *config.NodeConfig, name string) (*instance.Process, error) + StopRemoteInstance(node *config.NodeConfig, name string) (*instance.Process, error) + RestartRemoteInstance(node *config.NodeConfig, name string) (*instance.Process, error) + GetRemoteInstanceLogs(node *config.NodeConfig, name string) (string, error) +} + type instanceManager struct { mu sync.RWMutex instances map[string]*instance.Process @@ -42,6 +55,9 @@ type instanceManager struct { shutdownChan chan struct{} shutdownDone chan struct{} isShutdown bool + + // Remote instance management + httpClient *http.Client } // NewInstanceManager creates a new instance of InstanceManager. @@ -59,6 +75,10 @@ func NewInstanceManager(backendsConfig config.BackendConfig, instancesConfig con timeoutChecker: time.NewTicker(time.Duration(instancesConfig.TimeoutCheckInterval) * time.Minute), shutdownChan: make(chan struct{}), shutdownDone: make(chan struct{}), + + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, } // Load existing instances from disk diff --git a/pkg/manager/remote_ops.go b/pkg/manager/remote_ops.go new file mode 100644 index 0000000..5050737 --- /dev/null +++ b/pkg/manager/remote_ops.go @@ -0,0 +1,228 @@ +package manager + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "llamactl/pkg/config" + "llamactl/pkg/instance" + "net/http" +) + +// makeRemoteRequest is a helper function to make HTTP requests to a remote node +func (im *instanceManager) makeRemoteRequest(nodeConfig *config.NodeConfig, method, path string, body any) (*http.Response, error) { + var reqBody io.Reader + if body != nil { + jsonData, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request body: %w", err) + } + reqBody = bytes.NewBuffer(jsonData) + } + + url := fmt.Sprintf("%s%s", nodeConfig.Address, path) + req, err := http.NewRequest(method, url, reqBody) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + if nodeConfig.APIKey != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", nodeConfig.APIKey)) + } + + resp, err := im.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to execute request: %w", err) + } + + return resp, nil +} + +// parseRemoteResponse is a helper function to parse API responses +func parseRemoteResponse(resp *http.Response, result any) error { + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + if result != nil { + if err := json.Unmarshal(body, result); err != nil { + return fmt.Errorf("failed to unmarshal response: %w", err) + } + } + + return nil +} + +// ListRemoteInstances lists all instances on the remote node +func (im *instanceManager) ListRemoteInstances(nodeConfig *config.NodeConfig) ([]*instance.Process, error) { + resp, err := im.makeRemoteRequest(nodeConfig, "GET", "/api/v1/instances/", nil) + if err != nil { + return nil, err + } + + var instances []*instance.Process + if err := parseRemoteResponse(resp, &instances); err != nil { + return nil, err + } + + return instances, nil +} + +// CreateRemoteInstance creates a new instance on the remote node +func (im *instanceManager) CreateRemoteInstance(nodeConfig *config.NodeConfig, name string, options *instance.CreateInstanceOptions) (*instance.Process, error) { + path := fmt.Sprintf("/api/v1/instances/%s/", name) + payload := map[string]any{ + "options": options, + } + + resp, err := im.makeRemoteRequest(nodeConfig, "POST", path, payload) + if err != nil { + return nil, err + } + + var inst instance.Process + if err := parseRemoteResponse(resp, &inst); err != nil { + return nil, err + } + + return &inst, nil +} + +// GetRemoteInstance retrieves an instance by name from the remote node +func (im *instanceManager) GetRemoteInstance(nodeConfig *config.NodeConfig, name string) (*instance.Process, error) { + path := fmt.Sprintf("/api/v1/instances/%s/", name) + resp, err := im.makeRemoteRequest(nodeConfig, "GET", path, nil) + if err != nil { + return nil, err + } + + var inst instance.Process + if err := parseRemoteResponse(resp, &inst); err != nil { + return nil, err + } + + return &inst, nil +} + +// UpdateRemoteInstance updates an existing instance on the remote node +func (im *instanceManager) UpdateRemoteInstance(nodeConfig *config.NodeConfig, name string, options *instance.CreateInstanceOptions) (*instance.Process, error) { + path := fmt.Sprintf("/api/v1/instances/%s/", name) + payload := map[string]any{ + "options": options, + } + + resp, err := im.makeRemoteRequest(nodeConfig, "PUT", path, payload) + if err != nil { + return nil, err + } + + var inst instance.Process + if err := parseRemoteResponse(resp, &inst); err != nil { + return nil, err + } + + return &inst, nil +} + +// DeleteRemoteInstance deletes an instance from the remote node +func (im *instanceManager) DeleteRemoteInstance(nodeConfig *config.NodeConfig, name string) error { + path := fmt.Sprintf("/api/v1/instances/%s/", name) + resp, err := im.makeRemoteRequest(nodeConfig, "DELETE", path, nil) + if err != nil { + return err + } + + return parseRemoteResponse(resp, nil) +} + +// StartRemoteInstance starts an instance on the remote node +func (im *instanceManager) StartRemoteInstance(nodeConfig *config.NodeConfig, name string) (*instance.Process, error) { + path := fmt.Sprintf("/api/v1/instances/%s/start", name) + resp, err := im.makeRemoteRequest(nodeConfig, "POST", path, nil) + if err != nil { + return nil, err + } + + var inst instance.Process + if err := parseRemoteResponse(resp, &inst); err != nil { + return nil, err + } + + return &inst, nil +} + +// StopRemoteInstance stops an instance on the remote node +func (im *instanceManager) StopRemoteInstance(nodeConfig *config.NodeConfig, name string) (*instance.Process, error) { + path := fmt.Sprintf("/api/v1/instances/%s/stop", name) + resp, err := im.makeRemoteRequest(nodeConfig, "POST", path, nil) + if err != nil { + return nil, err + } + + var inst instance.Process + if err := parseRemoteResponse(resp, &inst); err != nil { + return nil, err + } + + return &inst, nil +} + +// RestartRemoteInstance restarts an instance on the remote node +func (im *instanceManager) RestartRemoteInstance(nodeConfig *config.NodeConfig, name string) (*instance.Process, error) { + path := fmt.Sprintf("/api/v1/instances/%s/restart", name) + resp, err := im.makeRemoteRequest(nodeConfig, "POST", path, nil) + if err != nil { + return nil, err + } + + var inst instance.Process + if err := parseRemoteResponse(resp, &inst); err != nil { + return nil, err + } + + return &inst, nil +} + +// GetRemoteInstanceLogs retrieves logs for an instance from the remote node +func (im *instanceManager) GetRemoteInstanceLogs(nodeConfig *config.NodeConfig, name string) (string, error) { + path := fmt.Sprintf("/api/v1/instances/%s/logs", name) + resp, err := im.makeRemoteRequest(nodeConfig, "GET", path, nil) + if err != nil { + return "", err + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + // Logs endpoint might return plain text or JSON + // Try to parse as JSON first (in case it's wrapped in a response object) + var logResponse struct { + Logs string `json:"logs"` + } + if err := json.Unmarshal(body, &logResponse); err == nil && logResponse.Logs != "" { + return logResponse.Logs, nil + } + + // Otherwise, return as plain text + return string(body), nil +} From e0f176de107f18154c7b2883f703fffe59f0498f Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 1 Oct 2025 20:25:06 +0200 Subject: [PATCH 02/34] Enhance instance manager to support remote instance management and update related tests --- cmd/server/main.go | 2 +- pkg/manager/manager.go | 31 ++++++++- pkg/manager/manager_test.go | 8 +-- pkg/manager/operations.go | 111 ++++++++++++++++++++++++--------- pkg/manager/operations_test.go | 2 +- pkg/manager/timeout_test.go | 2 +- 6 files changed, 116 insertions(+), 40 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index e245ebf..de080c7 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -58,7 +58,7 @@ func main() { } // Initialize the instance manager - instanceManager := manager.NewInstanceManager(cfg.Backends, cfg.Instances) + instanceManager := manager.NewInstanceManager(cfg.Backends, cfg.Instances, cfg.Nodes) // Create a new handler with the instance manager handler := server.NewHandler(instanceManager, cfg) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index e160d6c..c63391a 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -57,14 +57,23 @@ type instanceManager struct { isShutdown bool // Remote instance management - httpClient *http.Client + httpClient *http.Client + instanceNodeMap map[string]*config.NodeConfig // Maps instance name to its node config + nodeConfigMap map[string]*config.NodeConfig // Maps node name to node config for quick lookup } // NewInstanceManager creates a new instance of InstanceManager. -func NewInstanceManager(backendsConfig config.BackendConfig, instancesConfig config.InstancesConfig) InstanceManager { +func NewInstanceManager(backendsConfig config.BackendConfig, instancesConfig config.InstancesConfig, nodesConfig []config.NodeConfig) InstanceManager { if instancesConfig.TimeoutCheckInterval <= 0 { instancesConfig.TimeoutCheckInterval = 5 // Default to 5 minutes if not set } + + // Build node config map for quick lookup + nodeConfigMap := make(map[string]*config.NodeConfig) + for i := range nodesConfig { + nodeConfigMap[nodesConfig[i].Name] = &nodesConfig[i] + } + im := &instanceManager{ instances: make(map[string]*instance.Process), runningInstances: make(map[string]struct{}), @@ -79,6 +88,9 @@ func NewInstanceManager(backendsConfig config.BackendConfig, instancesConfig con httpClient: &http.Client{ Timeout: 30 * time.Second, }, + + instanceNodeMap: make(map[string]*config.NodeConfig), + nodeConfigMap: nodeConfigMap, } // Load existing instances from disk @@ -316,3 +328,18 @@ func (im *instanceManager) onStatusChange(name string, oldStatus, newStatus inst delete(im.runningInstances, name) } } + +// getNodeForInstance returns the node configuration for a remote instance +// Returns nil if the instance is not remote or the node is not found +func (im *instanceManager) getNodeForInstance(inst *instance.Process) *config.NodeConfig { + if !inst.IsRemote() { + return nil + } + + // Check if we have a cached mapping + if nodeConfig, exists := im.instanceNodeMap[inst.Name]; exists { + return nodeConfig + } + + return nil +} diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index c629c63..ed9bde5 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -34,7 +34,7 @@ func TestNewInstanceManager(t *testing.T) { TimeoutCheckInterval: 5, } - mgr := manager.NewInstanceManager(backendConfig, cfg) + mgr := manager.NewInstanceManager(backendConfig, cfg, nil) if mgr == nil { t.Fatal("NewInstanceManager returned nil") } @@ -69,7 +69,7 @@ func TestPersistence(t *testing.T) { } // Test instance persistence on creation - manager1 := manager.NewInstanceManager(backendConfig, cfg) + manager1 := manager.NewInstanceManager(backendConfig, cfg, nil) options := &instance.CreateInstanceOptions{ BackendType: backends.BackendTypeLlamaCpp, LlamaServerOptions: &llamacpp.LlamaServerOptions{ @@ -90,7 +90,7 @@ func TestPersistence(t *testing.T) { } // Test loading instances from disk - manager2 := manager.NewInstanceManager(backendConfig, cfg) + manager2 := manager.NewInstanceManager(backendConfig, cfg, nil) instances, err := manager2.ListInstances() if err != nil { t.Fatalf("ListInstances failed: %v", err) @@ -207,5 +207,5 @@ func createTestManager() manager.InstanceManager { DefaultRestartDelay: 5, TimeoutCheckInterval: 5, } - return manager.NewInstanceManager(backendConfig, cfg) + return manager.NewInstanceManager(backendConfig, cfg, nil) } diff --git a/pkg/manager/operations.go b/pkg/manager/operations.go index b3c0d13..0a53f84 100644 --- a/pkg/manager/operations.go +++ b/pkg/manager/operations.go @@ -75,26 +75,37 @@ func (im *instanceManager) CreateInstance(name string, options *instance.CreateI // GetInstance retrieves an instance by its name. func (im *instanceManager) GetInstance(name string) (*instance.Process, error) { im.mu.RLock() - defer im.mu.RUnlock() + inst, exists := im.instances[name] + im.mu.RUnlock() - instance, exists := im.instances[name] if !exists { return nil, fmt.Errorf("instance with name %s not found", name) } - return instance, nil + + // Check if instance is remote and delegate to remote operation + if node := im.getNodeForInstance(inst); node != nil { + return im.GetRemoteInstance(node, name) + } + + return inst, nil } // UpdateInstance updates the options of an existing instance and returns it. // If the instance is running, it will be restarted to apply the new options. func (im *instanceManager) UpdateInstance(name string, options *instance.CreateInstanceOptions) (*instance.Process, error) { im.mu.RLock() - instance, exists := im.instances[name] + inst, exists := im.instances[name] im.mu.RUnlock() if !exists { return nil, fmt.Errorf("instance with name %s not found", name) } + // Check if instance is remote and delegate to remote operation + if node := im.getNodeForInstance(inst); node != nil { + return im.UpdateRemoteInstance(node, name, options) + } + if options == nil { return nil, fmt.Errorf("instance options cannot be nil") } @@ -105,55 +116,63 @@ func (im *instanceManager) UpdateInstance(name string, options *instance.CreateI } // Check if instance is running before updating options - wasRunning := instance.IsRunning() + wasRunning := inst.IsRunning() // If the instance is running, stop it first if wasRunning { - if err := instance.Stop(); err != nil { + if err := inst.Stop(); err != nil { return nil, fmt.Errorf("failed to stop instance %s for update: %w", name, err) } } // Now update the options while the instance is stopped - instance.SetOptions(options) + inst.SetOptions(options) // If it was running before, start it again with the new options if wasRunning { - if err := instance.Start(); err != nil { + if err := inst.Start(); err != nil { return nil, fmt.Errorf("failed to start instance %s after update: %w", name, err) } } im.mu.Lock() defer im.mu.Unlock() - if err := im.persistInstance(instance); err != nil { + if err := im.persistInstance(inst); err != nil { return nil, fmt.Errorf("failed to persist updated instance %s: %w", name, err) } - return instance, nil + return inst, nil } // DeleteInstance removes stopped instance by its name. func (im *instanceManager) DeleteInstance(name string) error { im.mu.Lock() - defer im.mu.Unlock() + inst, exists := im.instances[name] + im.mu.Unlock() - instance, exists := im.instances[name] if !exists { return fmt.Errorf("instance with name %s not found", name) } - if instance.IsRunning() { + // Check if instance is remote and delegate to remote operation + if node := im.getNodeForInstance(inst); node != nil { + return im.DeleteRemoteInstance(node, name) + } + + if inst.IsRunning() { return fmt.Errorf("instance with name %s is still running, stop it before deleting", name) } - delete(im.ports, instance.GetPort()) + im.mu.Lock() + defer im.mu.Unlock() + + delete(im.ports, inst.GetPort()) delete(im.instances, name) // Delete the instance's config file if persistence is enabled - instancePath := filepath.Join(im.instancesConfig.InstancesDir, instance.Name+".json") + instancePath := filepath.Join(im.instancesConfig.InstancesDir, inst.Name+".json") if err := os.Remove(instancePath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to delete config file for instance %s: %w", instance.Name, err) + return fmt.Errorf("failed to delete config file for instance %s: %w", inst.Name, err) } return nil @@ -163,33 +182,39 @@ func (im *instanceManager) DeleteInstance(name string) error { // If the instance is already running, it returns an error. func (im *instanceManager) StartInstance(name string) (*instance.Process, error) { im.mu.RLock() - instance, exists := im.instances[name] + inst, exists := im.instances[name] maxRunningExceeded := len(im.runningInstances) >= im.instancesConfig.MaxRunningInstances && im.instancesConfig.MaxRunningInstances != -1 im.mu.RUnlock() if !exists { return nil, fmt.Errorf("instance with name %s not found", name) } - if instance.IsRunning() { - return instance, fmt.Errorf("instance with name %s is already running", name) + + // Check if instance is remote and delegate to remote operation + if node := im.getNodeForInstance(inst); node != nil { + return im.StartRemoteInstance(node, name) + } + + if inst.IsRunning() { + return inst, fmt.Errorf("instance with name %s is already running", name) } if maxRunningExceeded { return nil, MaxRunningInstancesError(fmt.Errorf("maximum number of running instances (%d) reached", im.instancesConfig.MaxRunningInstances)) } - if err := instance.Start(); err != nil { + if err := inst.Start(); err != nil { return nil, fmt.Errorf("failed to start instance %s: %w", name, err) } im.mu.Lock() defer im.mu.Unlock() - err := im.persistInstance(instance) + err := im.persistInstance(inst) if err != nil { return nil, fmt.Errorf("failed to persist instance %s: %w", name, err) } - return instance, nil + return inst, nil } func (im *instanceManager) IsMaxRunningInstancesReached() bool { @@ -206,49 +231,73 @@ func (im *instanceManager) IsMaxRunningInstancesReached() bool { // StopInstance stops a running instance and returns it. func (im *instanceManager) StopInstance(name string) (*instance.Process, error) { im.mu.RLock() - instance, exists := im.instances[name] + inst, exists := im.instances[name] im.mu.RUnlock() if !exists { return nil, fmt.Errorf("instance with name %s not found", name) } - if !instance.IsRunning() { - return instance, fmt.Errorf("instance with name %s is already stopped", name) + + // Check if instance is remote and delegate to remote operation + if node := im.getNodeForInstance(inst); node != nil { + return im.StopRemoteInstance(node, name) } - if err := instance.Stop(); err != nil { + if !inst.IsRunning() { + return inst, fmt.Errorf("instance with name %s is already stopped", name) + } + + if err := inst.Stop(); err != nil { return nil, fmt.Errorf("failed to stop instance %s: %w", name, err) } im.mu.Lock() defer im.mu.Unlock() - err := im.persistInstance(instance) + err := im.persistInstance(inst) if err != nil { return nil, fmt.Errorf("failed to persist instance %s: %w", name, err) } - return instance, nil + return inst, nil } // RestartInstance stops and then starts an instance, returning the updated instance. func (im *instanceManager) RestartInstance(name string) (*instance.Process, error) { - instance, err := im.StopInstance(name) + im.mu.RLock() + inst, exists := im.instances[name] + im.mu.RUnlock() + + if !exists { + return nil, fmt.Errorf("instance with name %s not found", name) + } + + // Check if instance is remote and delegate to remote operation + if node := im.getNodeForInstance(inst); node != nil { + return im.RestartRemoteInstance(node, name) + } + + inst, err := im.StopInstance(name) if err != nil { return nil, err } - return im.StartInstance(instance.Name) + return im.StartInstance(inst.Name) } // GetInstanceLogs retrieves the logs for a specific instance by its name. func (im *instanceManager) GetInstanceLogs(name string) (string, error) { im.mu.RLock() - _, exists := im.instances[name] + inst, exists := im.instances[name] im.mu.RUnlock() if !exists { return "", fmt.Errorf("instance with name %s not found", name) } + // Check if instance is remote and delegate to remote operation + if node := im.getNodeForInstance(inst); node != nil { + return im.GetRemoteInstanceLogs(node, name) + } + // TODO: Implement actual log retrieval logic return fmt.Sprintf("Logs for instance %s", name), nil } diff --git a/pkg/manager/operations_test.go b/pkg/manager/operations_test.go index 97358c5..da26742 100644 --- a/pkg/manager/operations_test.go +++ b/pkg/manager/operations_test.go @@ -75,7 +75,7 @@ func TestCreateInstance_ValidationAndLimits(t *testing.T) { MaxInstances: 1, // Very low limit for testing TimeoutCheckInterval: 5, } - limitedManager := manager.NewInstanceManager(backendConfig, cfg) + limitedManager := manager.NewInstanceManager(backendConfig, cfg, nil) _, err = limitedManager.CreateInstance("instance1", options) if err != nil { diff --git a/pkg/manager/timeout_test.go b/pkg/manager/timeout_test.go index 08d500c..31b4298 100644 --- a/pkg/manager/timeout_test.go +++ b/pkg/manager/timeout_test.go @@ -23,7 +23,7 @@ func TestTimeoutFunctionality(t *testing.T) { MaxInstances: 5, } - manager := manager.NewInstanceManager(backendConfig, cfg) + manager := manager.NewInstanceManager(backendConfig, cfg, nil) if manager == nil { t.Fatal("Manager should be initialized with timeout checker") } From 0188f823064d65c4167a965834afe8bc36dd1a93 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 1 Oct 2025 22:05:18 +0200 Subject: [PATCH 03/34] Implement remote instance creation and deletion in instance manager --- pkg/manager/operations.go | 66 ++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/pkg/manager/operations.go b/pkg/manager/operations.go index 0a53f84..b7fdda8 100644 --- a/pkg/manager/operations.go +++ b/pkg/manager/operations.go @@ -3,6 +3,7 @@ package manager import ( "fmt" "llamactl/pkg/backends" + "llamactl/pkg/config" "llamactl/pkg/instance" "llamactl/pkg/validation" "os" @@ -43,16 +44,44 @@ func (im *instanceManager) CreateInstance(name string, options *instance.CreateI im.mu.Lock() defer im.mu.Unlock() - // Check max instances limit after acquiring the lock - if len(im.instances) >= im.instancesConfig.MaxInstances && im.instancesConfig.MaxInstances != -1 { - return nil, fmt.Errorf("maximum number of instances (%d) reached", im.instancesConfig.MaxInstances) - } - - // Check if instance with this name already exists + // Check if instance with this name already exists (must be globally unique) if im.instances[name] != nil { return nil, fmt.Errorf("instance with name %s already exists", name) } + // Check if this is a remote instance + isRemote := len(options.Nodes) > 0 + var nodeConfig *config.NodeConfig + + if isRemote { + // Validate that the node exists + nodeName := options.Nodes[0] // Use first node for now + var exists bool + nodeConfig, exists = im.nodeConfigMap[nodeName] + if !exists { + return nil, fmt.Errorf("node %s not found", nodeName) + } + + // Create the remote instance + inst, err := im.CreateRemoteInstance(nodeConfig, name, options) + if err != nil { + return nil, err + } + + // Add to local tracking maps (but don't count towards limits) + im.instances[name] = inst + im.instanceNodeMap[name] = nodeConfig + + return inst, nil + } + + // Local instance creation + // Check max instances limit for local instances only + localInstanceCount := len(im.instances) - len(im.instanceNodeMap) + if localInstanceCount >= im.instancesConfig.MaxInstances && im.instancesConfig.MaxInstances != -1 { + return nil, fmt.Errorf("maximum number of instances (%d) reached", im.instancesConfig.MaxInstances) + } + // Assign and validate port for backend-specific options if err := im.assignAndValidatePort(options); err != nil { return nil, err @@ -156,7 +185,18 @@ func (im *instanceManager) DeleteInstance(name string) error { // Check if instance is remote and delegate to remote operation if node := im.getNodeForInstance(inst); node != nil { - return im.DeleteRemoteInstance(node, name) + err := im.DeleteRemoteInstance(node, name) + if err != nil { + return err + } + + // Clean up local tracking + im.mu.Lock() + delete(im.instances, name) + delete(im.instanceNodeMap, name) + im.mu.Unlock() + + return nil } if inst.IsRunning() { @@ -183,7 +223,6 @@ func (im *instanceManager) DeleteInstance(name string) error { func (im *instanceManager) StartInstance(name string) (*instance.Process, error) { im.mu.RLock() inst, exists := im.instances[name] - maxRunningExceeded := len(im.runningInstances) >= im.instancesConfig.MaxRunningInstances && im.instancesConfig.MaxRunningInstances != -1 im.mu.RUnlock() if !exists { @@ -199,6 +238,17 @@ func (im *instanceManager) StartInstance(name string) (*instance.Process, error) return inst, fmt.Errorf("instance with name %s is already running", name) } + // Check max running instances limit for local instances only + im.mu.RLock() + localRunningCount := 0 + for instName := range im.runningInstances { + if _, isRemote := im.instanceNodeMap[instName]; !isRemote { + localRunningCount++ + } + } + maxRunningExceeded := localRunningCount >= im.instancesConfig.MaxRunningInstances && im.instancesConfig.MaxRunningInstances != -1 + im.mu.RUnlock() + if maxRunningExceeded { return nil, MaxRunningInstancesError(fmt.Errorf("maximum number of running instances (%d) reached", im.instancesConfig.MaxRunningInstances)) } From 2ed67eb6721f8820988689836ea3b0f326bb954f Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 1 Oct 2025 22:17:19 +0200 Subject: [PATCH 04/34] Add remote instance proxying functionality to handler --- pkg/server/handlers.go | 150 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/pkg/server/handlers.go b/pkg/server/handlers.go index 594c273..5f93f14 100644 --- a/pkg/server/handlers.go +++ b/pkg/server/handlers.go @@ -16,6 +16,7 @@ import ( "os/exec" "strconv" "strings" + "time" "github.com/go-chi/chi/v5" ) @@ -23,12 +24,16 @@ import ( type Handler struct { InstanceManager manager.InstanceManager cfg config.AppConfig + httpClient *http.Client } func NewHandler(im manager.InstanceManager, cfg config.AppConfig) *Handler { return &Handler{ InstanceManager: im, cfg: cfg, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, } } @@ -461,6 +466,12 @@ func (h *Handler) ProxyToInstance() http.HandlerFunc { return } + // Check if this is a remote instance + if inst.IsRemote() { + h.RemoteInstanceProxy(w, r, name, inst) + return + } + if !inst.IsRunning() { http.Error(w, "Instance is not running", http.StatusServiceUnavailable) return @@ -503,6 +514,73 @@ func (h *Handler) ProxyToInstance() http.HandlerFunc { } } +// RemoteInstanceProxy proxies requests to a remote instance +func (h *Handler) RemoteInstanceProxy(w http.ResponseWriter, r *http.Request, name string, inst *instance.Process) { + // Get the node name from instance options + options := inst.GetOptions() + if options == nil || len(options.Nodes) == 0 { + http.Error(w, "Instance has no node configured", http.StatusInternalServerError) + return + } + + nodeName := options.Nodes[0] + var nodeConfig *config.NodeConfig + for i := range h.cfg.Nodes { + if h.cfg.Nodes[i].Name == nodeName { + nodeConfig = &h.cfg.Nodes[i] + break + } + } + + if nodeConfig == nil { + http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) + return + } + + // Strip the "/api/v1/instances//proxy" prefix from the request URL + prefix := fmt.Sprintf("/api/v1/instances/%s/proxy", name) + proxyPath := r.URL.Path[len(prefix):] + + // Build the remote URL + remoteURL := fmt.Sprintf("%s/api/v1/instances/%s/proxy%s", nodeConfig.Address, name, proxyPath) + + // Create a new request to the remote node + req, err := http.NewRequest(r.Method, remoteURL, r.Body) + if err != nil { + http.Error(w, "Failed to create remote request: "+err.Error(), http.StatusInternalServerError) + return + } + + // Copy headers + req.Header = r.Header.Clone() + + // Add API key if configured + if nodeConfig.APIKey != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", nodeConfig.APIKey)) + } + + // Forward the request + resp, err := h.httpClient.Do(req) + if err != nil { + http.Error(w, "Failed to proxy to remote instance: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + + // Copy response headers + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + + // Copy status code + w.WriteHeader(resp.StatusCode) + + // Copy response body + io.Copy(w, resp.Body) +} + // OpenAIListInstances godoc // @Summary List instances in OpenAI-compatible format // @Description Returns a list of instances in a format compatible with OpenAI API @@ -584,6 +662,12 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc { return } + // Check if this is a remote instance + if inst.IsRemote() { + h.RemoteOpenAIProxy(w, r, modelName, inst, bodyBytes) + return + } + if !inst.IsRunning() { allowOnDemand := inst.GetOptions() != nil && inst.GetOptions().OnDemandStart != nil && *inst.GetOptions().OnDemandStart if !allowOnDemand { @@ -634,6 +718,72 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc { } } +// RemoteOpenAIProxy proxies OpenAI-compatible requests to a remote instance +func (h *Handler) RemoteOpenAIProxy(w http.ResponseWriter, r *http.Request, modelName string, inst *instance.Process, bodyBytes []byte) { + // Get the node name from instance options + options := inst.GetOptions() + if options == nil || len(options.Nodes) == 0 { + http.Error(w, "Instance has no node configured", http.StatusInternalServerError) + return + } + + nodeName := options.Nodes[0] + var nodeConfig *config.NodeConfig + for i := range h.cfg.Nodes { + if h.cfg.Nodes[i].Name == nodeName { + nodeConfig = &h.cfg.Nodes[i] + break + } + } + + if nodeConfig == nil { + http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) + return + } + + // Build the remote URL - forward to the same OpenAI endpoint on the remote node + remoteURL := fmt.Sprintf("%s%s", nodeConfig.Address, r.URL.Path) + if r.URL.RawQuery != "" { + remoteURL += "?" + r.URL.RawQuery + } + + // Create a new request to the remote node + req, err := http.NewRequest(r.Method, remoteURL, bytes.NewReader(bodyBytes)) + if err != nil { + http.Error(w, "Failed to create remote request: "+err.Error(), http.StatusInternalServerError) + return + } + + // Copy headers + req.Header = r.Header.Clone() + + // Add API key if configured + if nodeConfig.APIKey != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", nodeConfig.APIKey)) + } + + // Forward the request + resp, err := h.httpClient.Do(req) + if err != nil { + http.Error(w, "Failed to proxy to remote instance: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + + // Copy response headers + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + + // Copy status code + w.WriteHeader(resp.StatusCode) + + // Copy response body + io.Copy(w, resp.Body) +} + // ParseCommandRequest represents the request body for command parsing type ParseCommandRequest struct { Command string `json:"command"` From 347c58e15f39d28908264649504c5408a7251713 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 1 Oct 2025 22:58:57 +0200 Subject: [PATCH 05/34] Enhance instance manager to persist remote instances and update tracking on modifications --- pkg/manager/operations.go | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/pkg/manager/operations.go b/pkg/manager/operations.go index b7fdda8..f1e5929 100644 --- a/pkg/manager/operations.go +++ b/pkg/manager/operations.go @@ -72,6 +72,11 @@ func (im *instanceManager) CreateInstance(name string, options *instance.CreateI im.instances[name] = inst im.instanceNodeMap[name] = nodeConfig + // Persist the remote instance locally for tracking across restarts + if err := im.persistInstance(inst); err != nil { + return nil, fmt.Errorf("failed to persist remote instance %s: %w", name, err) + } + return inst, nil } @@ -132,7 +137,24 @@ func (im *instanceManager) UpdateInstance(name string, options *instance.CreateI // Check if instance is remote and delegate to remote operation if node := im.getNodeForInstance(inst); node != nil { - return im.UpdateRemoteInstance(node, name, options) + updatedInst, err := im.UpdateRemoteInstance(node, name, options) + if err != nil { + return nil, err + } + + // Update local tracking + im.mu.Lock() + im.instances[name] = updatedInst + im.mu.Unlock() + + // Persist the updated remote instance locally + im.mu.Lock() + defer im.mu.Unlock() + if err := im.persistInstance(updatedInst); err != nil { + return nil, fmt.Errorf("failed to persist updated remote instance %s: %w", name, err) + } + + return updatedInst, nil } if options == nil { @@ -192,9 +214,15 @@ func (im *instanceManager) DeleteInstance(name string) error { // Clean up local tracking im.mu.Lock() + defer im.mu.Unlock() delete(im.instances, name) delete(im.instanceNodeMap, name) - im.mu.Unlock() + + // Delete the instance's config file if persistence is enabled + instancePath := filepath.Join(im.instancesConfig.InstancesDir, name+".json") + if err := os.Remove(instancePath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to delete config file for remote instance %s: %w", name, err) + } return nil } From c30053e51c2e2f020d21a6fa89a61dff96aef688 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 1 Oct 2025 22:59:45 +0200 Subject: [PATCH 06/34] Enhance instance loading to support remote instances and handle node configuration --- pkg/manager/manager.go | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index c63391a..354fbd2 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -270,24 +270,43 @@ func (im *instanceManager) loadInstance(name, path string) error { return fmt.Errorf("instance name mismatch: file=%s, instance.Name=%s", name, persistedInstance.Name) } - statusCallback := func(oldStatus, newStatus instance.InstanceStatus) { - im.onStatusChange(persistedInstance.Name, oldStatus, newStatus) + options := persistedInstance.GetOptions() + + // Check if this is a remote instance + isRemote := options != nil && len(options.Nodes) > 0 + + var statusCallback func(oldStatus, newStatus instance.InstanceStatus) + if !isRemote { + // Only set status callback for local instances + statusCallback = func(oldStatus, newStatus instance.InstanceStatus) { + im.onStatusChange(persistedInstance.Name, oldStatus, newStatus) + } } // Create new inst using NewInstance (handles validation, defaults, setup) - inst := instance.NewInstance(name, &im.backendsConfig, &im.instancesConfig, persistedInstance.GetOptions(), statusCallback) + inst := instance.NewInstance(name, &im.backendsConfig, &im.instancesConfig, options, statusCallback) // Restore persisted fields that NewInstance doesn't set inst.Created = persistedInstance.Created inst.SetStatus(persistedInstance.Status) - // Check for port conflicts and add to maps - if inst.GetPort() > 0 { - port := inst.GetPort() - if im.ports[port] { - return fmt.Errorf("port conflict: instance %s wants port %d which is already in use", name, port) + // Handle remote instance mapping + if isRemote { + nodeName := options.Nodes[0] + nodeConfig, exists := im.nodeConfigMap[nodeName] + if !exists { + return fmt.Errorf("node %s not found for remote instance %s", nodeName, name) + } + im.instanceNodeMap[name] = nodeConfig + } else { + // Check for port conflicts only for local instances + if inst.GetPort() > 0 { + port := inst.GetPort() + if im.ports[port] { + return fmt.Errorf("port conflict: instance %s wants port %d which is already in use", name, port) + } + im.ports[port] = true } - im.ports[port] = true } im.instances[name] = inst From da564565042e012268355bad51762a5055f72dfa Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 2 Oct 2025 22:51:41 +0200 Subject: [PATCH 07/34] Add node management endpoints to handle listing and retrieving node details --- pkg/server/handlers.go | 89 ++++++++++++++++++++++++++++++++++++++++-- pkg/server/routes.go | 9 +++++ 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/pkg/server/handlers.go b/pkg/server/handlers.go index 5f93f14..a0f9ff8 100644 --- a/pkg/server/handlers.go +++ b/pkg/server/handlers.go @@ -784,6 +784,12 @@ func (h *Handler) RemoteOpenAIProxy(w http.ResponseWriter, r *http.Request, mode io.Copy(w, resp.Body) } +// NodeResponse represents a sanitized node configuration for API responses +type NodeResponse struct { + Name string `json:"name"` + Address string `json:"address"` +} + // ParseCommandRequest represents the request body for command parsing type ParseCommandRequest struct { Command string `json:"command"` @@ -864,21 +870,21 @@ func (h *Handler) ParseMlxCommand() http.HandlerFunc { writeError(w, http.StatusBadRequest, "invalid_request", "Invalid JSON body") return } - + if strings.TrimSpace(req.Command) == "" { writeError(w, http.StatusBadRequest, "invalid_command", "Command cannot be empty") return } - + mlxOptions, err := mlx.ParseMlxCommand(req.Command) if err != nil { writeError(w, http.StatusBadRequest, "parse_error", err.Error()) return } - + // Currently only support mlx_lm backend type backendType := backends.BackendTypeMlxLm - + options := &instance.CreateInstanceOptions{ BackendType: backendType, MlxServerOptions: mlxOptions, @@ -943,3 +949,78 @@ func (h *Handler) ParseVllmCommand() http.HandlerFunc { } } } + +// ListNodes godoc +// @Summary List all configured nodes +// @Description Returns a list of all nodes configured in the server +// @Tags nodes +// @Security ApiKeyAuth +// @Produces json +// @Success 200 {array} NodeResponse "List of nodes" +// @Failure 500 {string} string "Internal Server Error" +// @Router /nodes [get] +func (h *Handler) ListNodes() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Convert to sanitized response format + nodeResponses := make([]NodeResponse, len(h.cfg.Nodes)) + for i, node := range h.cfg.Nodes { + nodeResponses[i] = NodeResponse{ + Name: node.Name, + Address: node.Address, + } + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(nodeResponses); err != nil { + http.Error(w, "Failed to encode nodes: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// GetNode godoc +// @Summary Get details of a specific node +// @Description Returns the details of a specific node by name +// @Tags nodes +// @Security ApiKeyAuth +// @Produces json +// @Param name path string true "Node Name" +// @Success 200 {object} NodeResponse "Node details" +// @Failure 400 {string} string "Invalid name format" +// @Failure 404 {string} string "Node not found" +// @Failure 500 {string} string "Internal Server Error" +// @Router /nodes/{name} [get] +func (h *Handler) GetNode() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Node name cannot be empty", http.StatusBadRequest) + return + } + + var nodeConfig *config.NodeConfig + for i := range h.cfg.Nodes { + if h.cfg.Nodes[i].Name == name { + nodeConfig = &h.cfg.Nodes[i] + break + } + } + + if nodeConfig == nil { + http.Error(w, "Node not found", http.StatusNotFound) + return + } + + // Convert to sanitized response format + nodeResponse := NodeResponse{ + Name: nodeConfig.Name, + Address: nodeConfig.Address, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(nodeResponse); err != nil { + http.Error(w, "Failed to encode node: "+err.Error(), http.StatusInternalServerError) + return + } + } +} diff --git a/pkg/server/routes.go b/pkg/server/routes.go index 6af6a5c..d14baec 100644 --- a/pkg/server/routes.go +++ b/pkg/server/routes.go @@ -60,6 +60,15 @@ func SetupRouter(handler *Handler) *chi.Mux { }) }) + // Node management endpoints + r.Route("/nodes", func(r chi.Router) { + r.Get("/", handler.ListNodes()) // List all nodes + + r.Route("/{name}", func(r chi.Router) { + r.Get("/", handler.GetNode()) + }) + }) + // Instance management endpoints r.Route("/instances", func(r chi.Router) { r.Get("/", handler.ListInstances()) // List all instances From 670f8ff81b954cdd2775ea78f7e04ba36c66e773 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 2 Oct 2025 23:11:20 +0200 Subject: [PATCH 08/34] Split up handlers --- pkg/server/handlers.go | 1002 ------------------------------ pkg/server/handlers_backends.go | 239 +++++++ pkg/server/handlers_instances.go | 477 ++++++++++++++ pkg/server/handlers_nodes.go | 90 +++ pkg/server/handlers_openai.go | 214 +++++++ pkg/server/handlers_system.go | 22 + 6 files changed, 1042 insertions(+), 1002 deletions(-) create mode 100644 pkg/server/handlers_backends.go create mode 100644 pkg/server/handlers_instances.go create mode 100644 pkg/server/handlers_nodes.go create mode 100644 pkg/server/handlers_openai.go create mode 100644 pkg/server/handlers_system.go diff --git a/pkg/server/handlers.go b/pkg/server/handlers.go index a0f9ff8..4ddbfea 100644 --- a/pkg/server/handlers.go +++ b/pkg/server/handlers.go @@ -1,24 +1,10 @@ package server import ( - "bytes" - "encoding/json" - "fmt" - "io" - "llamactl/pkg/backends" - "llamactl/pkg/backends/llamacpp" - "llamactl/pkg/backends/mlx" - "llamactl/pkg/backends/vllm" "llamactl/pkg/config" - "llamactl/pkg/instance" "llamactl/pkg/manager" "net/http" - "os/exec" - "strconv" - "strings" "time" - - "github.com/go-chi/chi/v5" ) type Handler struct { @@ -36,991 +22,3 @@ func NewHandler(im manager.InstanceManager, cfg config.AppConfig) *Handler { }, } } - -// VersionHandler godoc -// @Summary Get llamactl version -// @Description Returns the version of the llamactl command -// @Tags version -// @Security ApiKeyAuth -// @Produces text/plain -// @Success 200 {string} string "Version information" -// @Failure 500 {string} string "Internal Server Error" -// @Router /version [get] -func (h *Handler) VersionHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain") - fmt.Fprintf(w, "Version: %s\nCommit: %s\nBuild Time: %s\n", h.cfg.Version, h.cfg.CommitHash, h.cfg.BuildTime) - } -} - -// LlamaServerHelpHandler godoc -// @Summary Get help for llama server -// @Description Returns the help text for the llama server command -// @Tags backends -// @Security ApiKeyAuth -// @Produces text/plain -// @Success 200 {string} string "Help text" -// @Failure 500 {string} string "Internal Server Error" -// @Router /backends/llama-cpp/help [get] -func (h *Handler) LlamaServerHelpHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - helpCmd := exec.Command("llama-server", "--help") - output, err := helpCmd.CombinedOutput() - if err != nil { - http.Error(w, "Failed to get help: "+err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "text/plain") - w.Write(output) - } -} - -// LlamaServerVersionHandler godoc -// @Summary Get version of llama server -// @Description Returns the version of the llama server command -// @Tags backends -// @Security ApiKeyAuth -// @Produces text/plain -// @Success 200 {string} string "Version information" -// @Failure 500 {string} string "Internal Server Error" -// @Router /backends/llama-cpp/version [get] -func (h *Handler) LlamaServerVersionHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - versionCmd := exec.Command("llama-server", "--version") - output, err := versionCmd.CombinedOutput() - if err != nil { - http.Error(w, "Failed to get version: "+err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "text/plain") - w.Write(output) - } -} - -// LlamaServerListDevicesHandler godoc -// @Summary List available devices for llama server -// @Description Returns a list of available devices for the llama server -// @Tags backends -// @Security ApiKeyAuth -// @Produces text/plain -// @Success 200 {string} string "List of devices" -// @Failure 500 {string} string "Internal Server Error" -// @Router /backends/llama-cpp/devices [get] -func (h *Handler) LlamaServerListDevicesHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - listCmd := exec.Command("llama-server", "--list-devices") - output, err := listCmd.CombinedOutput() - if err != nil { - http.Error(w, "Failed to list devices: "+err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "text/plain") - w.Write(output) - } -} - -// ListInstances godoc -// @Summary List all instances -// @Description Returns a list of all instances managed by the server -// @Tags instances -// @Security ApiKeyAuth -// @Produces json -// @Success 200 {array} instance.Process "List of instances" -// @Failure 500 {string} string "Internal Server Error" -// @Router /instances [get] -func (h *Handler) ListInstances() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - instances, err := h.InstanceManager.ListInstances() - if err != nil { - http.Error(w, "Failed to list instances: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(instances); err != nil { - http.Error(w, "Failed to encode instances: "+err.Error(), http.StatusInternalServerError) - return - } - } -} - -// CreateInstance godoc -// @Summary Create and start a new instance -// @Description Creates a new instance with the provided configuration options -// @Tags instances -// @Security ApiKeyAuth -// @Accept json -// @Produces json -// @Param name path string true "Instance Name" -// @Param options body instance.CreateInstanceOptions true "Instance configuration options" -// @Success 201 {object} instance.Process "Created instance details" -// @Failure 400 {string} string "Invalid request body" -// @Failure 500 {string} string "Internal Server Error" -// @Router /instances/{name} [post] -func (h *Handler) CreateInstance() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) - return - } - - var options instance.CreateInstanceOptions - if err := json.NewDecoder(r.Body).Decode(&options); err != nil { - http.Error(w, "Invalid request body", http.StatusBadRequest) - return - } - - inst, err := h.InstanceManager.CreateInstance(name, &options) - if err != nil { - http.Error(w, "Failed to create instance: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusCreated) - if err := json.NewEncoder(w).Encode(inst); err != nil { - http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) - return - } - } -} - -// GetInstance godoc -// @Summary Get details of a specific instance -// @Description Returns the details of a specific instance by name -// @Tags instances -// @Security ApiKeyAuth -// @Produces json -// @Param name path string true "Instance Name" -// @Success 200 {object} instance.Process "Instance details" -// @Failure 400 {string} string "Invalid name format" -// @Failure 500 {string} string "Internal Server Error" -// @Router /instances/{name} [get] -func (h *Handler) GetInstance() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) - return - } - - inst, err := h.InstanceManager.GetInstance(name) - if err != nil { - http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(inst); err != nil { - http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) - return - } - } -} - -// UpdateInstance godoc -// @Summary Update an instance's configuration -// @Description Updates the configuration of a specific instance by name -// @Tags instances -// @Security ApiKeyAuth -// @Accept json -// @Produces json -// @Param name path string true "Instance Name" -// @Param options body instance.CreateInstanceOptions true "Instance configuration options" -// @Success 200 {object} instance.Process "Updated instance details" -// @Failure 400 {string} string "Invalid name format" -// @Failure 500 {string} string "Internal Server Error" -// @Router /instances/{name} [put] -func (h *Handler) UpdateInstance() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) - return - } - - var options instance.CreateInstanceOptions - if err := json.NewDecoder(r.Body).Decode(&options); err != nil { - http.Error(w, "Invalid request body", http.StatusBadRequest) - return - } - - inst, err := h.InstanceManager.UpdateInstance(name, &options) - if err != nil { - http.Error(w, "Failed to update instance: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(inst); err != nil { - http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) - return - } - } -} - -// StartInstance godoc -// @Summary Start a stopped instance -// @Description Starts a specific instance by name -// @Tags instances -// @Security ApiKeyAuth -// @Produces json -// @Param name path string true "Instance Name" -// @Success 200 {object} instance.Process "Started instance details" -// @Failure 400 {string} string "Invalid name format" -// @Failure 500 {string} string "Internal Server Error" -// @Router /instances/{name}/start [post] -func (h *Handler) StartInstance() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) - return - } - - inst, err := h.InstanceManager.StartInstance(name) - if err != nil { - // Check if error is due to maximum running instances limit - if _, ok := err.(manager.MaxRunningInstancesError); ok { - http.Error(w, err.Error(), http.StatusConflict) - return - } - - http.Error(w, "Failed to start instance: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(inst); err != nil { - http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) - return - } - } -} - -// StopInstance godoc -// @Summary Stop a running instance -// @Description Stops a specific instance by name -// @Tags instances -// @Security ApiKeyAuth -// @Produces json -// @Param name path string true "Instance Name" -// @Success 200 {object} instance.Process "Stopped instance details" -// @Failure 400 {string} string "Invalid name format" -// @Failure 500 {string} string "Internal Server Error" -// @Router /instances/{name}/stop [post] -func (h *Handler) StopInstance() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) - return - } - - inst, err := h.InstanceManager.StopInstance(name) - if err != nil { - http.Error(w, "Failed to stop instance: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(inst); err != nil { - http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) - return - } - } -} - -// RestartInstance godoc -// @Summary Restart a running instance -// @Description Restarts a specific instance by name -// @Tags instances -// @Security ApiKeyAuth -// @Produces json -// @Param name path string true "Instance Name" -// @Success 200 {object} instance.Process "Restarted instance details" -// @Failure 400 {string} string "Invalid name format" -// @Failure 500 {string} string "Internal Server Error" -// @Router /instances/{name}/restart [post] -func (h *Handler) RestartInstance() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) - return - } - - inst, err := h.InstanceManager.RestartInstance(name) - if err != nil { - http.Error(w, "Failed to restart instance: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(inst); err != nil { - http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) - return - } - } -} - -// DeleteInstance godoc -// @Summary Delete an instance -// @Description Stops and removes a specific instance by name -// @Tags instances -// @Security ApiKeyAuth -// @Param name path string true "Instance Name" -// @Success 204 "No Content" -// @Failure 400 {string} string "Invalid name format" -// @Failure 500 {string} string "Internal Server Error" -// @Router /instances/{name} [delete] -func (h *Handler) DeleteInstance() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) - return - } - - if err := h.InstanceManager.DeleteInstance(name); err != nil { - http.Error(w, "Failed to delete instance: "+err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) - } -} - -// GetInstanceLogs godoc -// @Summary Get logs from a specific instance -// @Description Returns the logs from a specific instance by name with optional line limit -// @Tags instances -// @Security ApiKeyAuth -// @Param name path string true "Instance Name" -// @Param lines query string false "Number of lines to retrieve (default: all lines)" -// @Produces text/plain -// @Success 200 {string} string "Instance logs" -// @Failure 400 {string} string "Invalid name format or lines parameter" -// @Failure 500 {string} string "Internal Server Error" -// @Router /instances/{name}/logs [get] -func (h *Handler) GetInstanceLogs() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) - return - } - - lines := r.URL.Query().Get("lines") - if lines == "" { - lines = "-1" - } - - num_lines, err := strconv.Atoi(lines) - if err != nil { - http.Error(w, "Invalid lines parameter: "+err.Error(), http.StatusBadRequest) - return - } - - inst, err := h.InstanceManager.GetInstance(name) - if err != nil { - http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError) - return - } - - logs, err := inst.GetLogs(num_lines) - if err != nil { - http.Error(w, "Failed to get logs: "+err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "text/plain") - w.Write([]byte(logs)) - } -} - -// ProxyToInstance godoc -// @Summary Proxy requests to a specific instance -// @Description Forwards HTTP requests to the llama-server instance running on a specific port -// @Tags instances -// @Security ApiKeyAuth -// @Param name path string true "Instance Name" -// @Success 200 "Request successfully proxied to instance" -// @Failure 400 {string} string "Invalid name format" -// @Failure 500 {string} string "Internal Server Error" -// @Failure 503 {string} string "Instance is not running" -// @Router /instances/{name}/proxy [get] -// @Router /instances/{name}/proxy [post] -func (h *Handler) ProxyToInstance() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) - return - } - - inst, err := h.InstanceManager.GetInstance(name) - if err != nil { - http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError) - return - } - - // Check if this is a remote instance - if inst.IsRemote() { - h.RemoteInstanceProxy(w, r, name, inst) - return - } - - if !inst.IsRunning() { - http.Error(w, "Instance is not running", http.StatusServiceUnavailable) - return - } - - // Get the cached proxy for this instance - proxy, err := inst.GetProxy() - if err != nil { - http.Error(w, "Failed to get proxy: "+err.Error(), http.StatusInternalServerError) - return - } - - // Strip the "/api/v1/instances//proxy" prefix from the request URL - prefix := fmt.Sprintf("/api/v1/instances/%s/proxy", name) - proxyPath := r.URL.Path[len(prefix):] - - // Ensure the proxy path starts with "/" - if !strings.HasPrefix(proxyPath, "/") { - proxyPath = "/" + proxyPath - } - - // Update the last request time for the instance - inst.UpdateLastRequestTime() - - // Modify the request to remove the proxy prefix - originalPath := r.URL.Path - r.URL.Path = proxyPath - - // Set forwarded headers - r.Header.Set("X-Forwarded-Host", r.Header.Get("Host")) - r.Header.Set("X-Forwarded-Proto", "http") - - // Restore original path for logging purposes - defer func() { - r.URL.Path = originalPath - }() - - // Forward the request using the cached proxy - proxy.ServeHTTP(w, r) - } -} - -// RemoteInstanceProxy proxies requests to a remote instance -func (h *Handler) RemoteInstanceProxy(w http.ResponseWriter, r *http.Request, name string, inst *instance.Process) { - // Get the node name from instance options - options := inst.GetOptions() - if options == nil || len(options.Nodes) == 0 { - http.Error(w, "Instance has no node configured", http.StatusInternalServerError) - return - } - - nodeName := options.Nodes[0] - var nodeConfig *config.NodeConfig - for i := range h.cfg.Nodes { - if h.cfg.Nodes[i].Name == nodeName { - nodeConfig = &h.cfg.Nodes[i] - break - } - } - - if nodeConfig == nil { - http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) - return - } - - // Strip the "/api/v1/instances//proxy" prefix from the request URL - prefix := fmt.Sprintf("/api/v1/instances/%s/proxy", name) - proxyPath := r.URL.Path[len(prefix):] - - // Build the remote URL - remoteURL := fmt.Sprintf("%s/api/v1/instances/%s/proxy%s", nodeConfig.Address, name, proxyPath) - - // Create a new request to the remote node - req, err := http.NewRequest(r.Method, remoteURL, r.Body) - if err != nil { - http.Error(w, "Failed to create remote request: "+err.Error(), http.StatusInternalServerError) - return - } - - // Copy headers - req.Header = r.Header.Clone() - - // Add API key if configured - if nodeConfig.APIKey != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", nodeConfig.APIKey)) - } - - // Forward the request - resp, err := h.httpClient.Do(req) - if err != nil { - http.Error(w, "Failed to proxy to remote instance: "+err.Error(), http.StatusBadGateway) - return - } - defer resp.Body.Close() - - // Copy response headers - for key, values := range resp.Header { - for _, value := range values { - w.Header().Add(key, value) - } - } - - // Copy status code - w.WriteHeader(resp.StatusCode) - - // Copy response body - io.Copy(w, resp.Body) -} - -// OpenAIListInstances godoc -// @Summary List instances in OpenAI-compatible format -// @Description Returns a list of instances in a format compatible with OpenAI API -// @Tags openai -// @Security ApiKeyAuth -// @Produces json -// @Success 200 {object} OpenAIListInstancesResponse "List of OpenAI-compatible instances" -// @Failure 500 {string} string "Internal Server Error" -// @Router /v1/models [get] -func (h *Handler) OpenAIListInstances() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - instances, err := h.InstanceManager.ListInstances() - if err != nil { - http.Error(w, "Failed to list instances: "+err.Error(), http.StatusInternalServerError) - return - } - - openaiInstances := make([]OpenAIInstance, len(instances)) - for i, inst := range instances { - openaiInstances[i] = OpenAIInstance{ - ID: inst.Name, - Object: "model", - Created: inst.Created, - OwnedBy: "llamactl", - } - } - - openaiResponse := OpenAIListInstancesResponse{ - Object: "list", - Data: openaiInstances, - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(openaiResponse); err != nil { - http.Error(w, "Failed to encode instances: "+err.Error(), http.StatusInternalServerError) - return - } - } -} - -// OpenAIProxy godoc -// @Summary OpenAI-compatible proxy endpoint -// @Description Handles all POST requests to /v1/*, routing to the appropriate instance based on the request body. Requires API key authentication via the `Authorization` header. -// @Tags openai -// @Security ApiKeyAuth -// @Accept json -// @Produces json -// @Success 200 "OpenAI response" -// @Failure 400 {string} string "Invalid request body or instance name" -// @Failure 500 {string} string "Internal Server Error" -// @Router /v1/ [post] -func (h *Handler) OpenAIProxy() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // Read the entire body first - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "Failed to read request body", http.StatusBadRequest) - return - } - r.Body.Close() - - // Parse the body to extract instance name - var requestBody map[string]any - if err := json.Unmarshal(bodyBytes, &requestBody); err != nil { - http.Error(w, "Invalid request body", http.StatusBadRequest) - return - } - - modelName, ok := requestBody["model"].(string) - if !ok || modelName == "" { - http.Error(w, "Instance name is required", http.StatusBadRequest) - return - } - - // Route to the appropriate inst based on instance name - inst, err := h.InstanceManager.GetInstance(modelName) - if err != nil { - http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError) - return - } - - // Check if this is a remote instance - if inst.IsRemote() { - h.RemoteOpenAIProxy(w, r, modelName, inst, bodyBytes) - return - } - - if !inst.IsRunning() { - allowOnDemand := inst.GetOptions() != nil && inst.GetOptions().OnDemandStart != nil && *inst.GetOptions().OnDemandStart - if !allowOnDemand { - http.Error(w, "Instance is not running", http.StatusServiceUnavailable) - return - } - - if h.InstanceManager.IsMaxRunningInstancesReached() { - if h.cfg.Instances.EnableLRUEviction { - err := h.InstanceManager.EvictLRUInstance() - if err != nil { - http.Error(w, "Cannot start Instance, failed to evict instance "+err.Error(), http.StatusInternalServerError) - return - } - } else { - http.Error(w, "Cannot start Instance, maximum number of instances reached", http.StatusConflict) - return - } - } - - // If on-demand start is enabled, start the instance - if _, err := h.InstanceManager.StartInstance(modelName); err != nil { - http.Error(w, "Failed to start instance: "+err.Error(), http.StatusInternalServerError) - return - } - - // Wait for the instance to become healthy before proceeding - if err := inst.WaitForHealthy(h.cfg.Instances.OnDemandStartTimeout); err != nil { // 2 minutes timeout - http.Error(w, "Instance failed to become healthy: "+err.Error(), http.StatusServiceUnavailable) - return - } - } - - proxy, err := inst.GetProxy() - if err != nil { - http.Error(w, "Failed to get proxy: "+err.Error(), http.StatusInternalServerError) - return - } - - // Update last request time for the instance - inst.UpdateLastRequestTime() - - // Recreate the request body from the bytes we read - r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) - r.ContentLength = int64(len(bodyBytes)) - - proxy.ServeHTTP(w, r) - } -} - -// RemoteOpenAIProxy proxies OpenAI-compatible requests to a remote instance -func (h *Handler) RemoteOpenAIProxy(w http.ResponseWriter, r *http.Request, modelName string, inst *instance.Process, bodyBytes []byte) { - // Get the node name from instance options - options := inst.GetOptions() - if options == nil || len(options.Nodes) == 0 { - http.Error(w, "Instance has no node configured", http.StatusInternalServerError) - return - } - - nodeName := options.Nodes[0] - var nodeConfig *config.NodeConfig - for i := range h.cfg.Nodes { - if h.cfg.Nodes[i].Name == nodeName { - nodeConfig = &h.cfg.Nodes[i] - break - } - } - - if nodeConfig == nil { - http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) - return - } - - // Build the remote URL - forward to the same OpenAI endpoint on the remote node - remoteURL := fmt.Sprintf("%s%s", nodeConfig.Address, r.URL.Path) - if r.URL.RawQuery != "" { - remoteURL += "?" + r.URL.RawQuery - } - - // Create a new request to the remote node - req, err := http.NewRequest(r.Method, remoteURL, bytes.NewReader(bodyBytes)) - if err != nil { - http.Error(w, "Failed to create remote request: "+err.Error(), http.StatusInternalServerError) - return - } - - // Copy headers - req.Header = r.Header.Clone() - - // Add API key if configured - if nodeConfig.APIKey != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", nodeConfig.APIKey)) - } - - // Forward the request - resp, err := h.httpClient.Do(req) - if err != nil { - http.Error(w, "Failed to proxy to remote instance: "+err.Error(), http.StatusBadGateway) - return - } - defer resp.Body.Close() - - // Copy response headers - for key, values := range resp.Header { - for _, value := range values { - w.Header().Add(key, value) - } - } - - // Copy status code - w.WriteHeader(resp.StatusCode) - - // Copy response body - io.Copy(w, resp.Body) -} - -// NodeResponse represents a sanitized node configuration for API responses -type NodeResponse struct { - Name string `json:"name"` - Address string `json:"address"` -} - -// ParseCommandRequest represents the request body for command parsing -type ParseCommandRequest struct { - Command string `json:"command"` -} - -// ParseLlamaCommand godoc -// @Summary Parse llama-server command -// @Description Parses a llama-server command string into instance options -// @Tags backends -// @Security ApiKeyAuth -// @Accept json -// @Produce json -// @Param request body ParseCommandRequest true "Command to parse" -// @Success 200 {object} instance.CreateInstanceOptions "Parsed options" -// @Failure 400 {object} map[string]string "Invalid request or command" -// @Failure 500 {object} map[string]string "Internal Server Error" -// @Router /backends/llama-cpp/parse-command [post] -func (h *Handler) ParseLlamaCommand() http.HandlerFunc { - type errorResponse struct { - Error string `json:"error"` - Details string `json:"details,omitempty"` - } - writeError := func(w http.ResponseWriter, status int, code, details string) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(errorResponse{Error: code, Details: details}) - } - return func(w http.ResponseWriter, r *http.Request) { - var req ParseCommandRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid_request", "Invalid JSON body") - return - } - if strings.TrimSpace(req.Command) == "" { - writeError(w, http.StatusBadRequest, "invalid_command", "Command cannot be empty") - return - } - llamaOptions, err := llamacpp.ParseLlamaCommand(req.Command) - if err != nil { - writeError(w, http.StatusBadRequest, "parse_error", err.Error()) - return - } - options := &instance.CreateInstanceOptions{ - BackendType: backends.BackendTypeLlamaCpp, - LlamaServerOptions: llamaOptions, - } - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(options); err != nil { - writeError(w, http.StatusInternalServerError, "encode_error", err.Error()) - } - } -} - -// ParseMlxCommand godoc -// @Summary Parse mlx_lm.server command -// @Description Parses MLX-LM server command string into instance options -// @Tags backends -// @Security ApiKeyAuth -// @Accept json -// @Produce json -// @Param request body ParseCommandRequest true "Command to parse" -// @Success 200 {object} instance.CreateInstanceOptions "Parsed options" -// @Failure 400 {object} map[string]string "Invalid request or command" -// @Router /backends/mlx/parse-command [post] -func (h *Handler) ParseMlxCommand() http.HandlerFunc { - type errorResponse struct { - Error string `json:"error"` - Details string `json:"details,omitempty"` - } - writeError := func(w http.ResponseWriter, status int, code, details string) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(errorResponse{Error: code, Details: details}) - } - return func(w http.ResponseWriter, r *http.Request) { - var req ParseCommandRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid_request", "Invalid JSON body") - return - } - - if strings.TrimSpace(req.Command) == "" { - writeError(w, http.StatusBadRequest, "invalid_command", "Command cannot be empty") - return - } - - mlxOptions, err := mlx.ParseMlxCommand(req.Command) - if err != nil { - writeError(w, http.StatusBadRequest, "parse_error", err.Error()) - return - } - - // Currently only support mlx_lm backend type - backendType := backends.BackendTypeMlxLm - - options := &instance.CreateInstanceOptions{ - BackendType: backendType, - MlxServerOptions: mlxOptions, - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(options); err != nil { - writeError(w, http.StatusInternalServerError, "encode_error", err.Error()) - } - } -} - -// ParseVllmCommand godoc -// @Summary Parse vllm serve command -// @Description Parses a vLLM serve command string into instance options -// @Tags backends -// @Security ApiKeyAuth -// @Accept json -// @Produce json -// @Param request body ParseCommandRequest true "Command to parse" -// @Success 200 {object} instance.CreateInstanceOptions "Parsed options" -// @Failure 400 {object} map[string]string "Invalid request or command" -// @Router /backends/vllm/parse-command [post] -func (h *Handler) ParseVllmCommand() http.HandlerFunc { - type errorResponse struct { - Error string `json:"error"` - Details string `json:"details,omitempty"` - } - writeError := func(w http.ResponseWriter, status int, code, details string) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(errorResponse{Error: code, Details: details}) - } - return func(w http.ResponseWriter, r *http.Request) { - var req ParseCommandRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid_request", "Invalid JSON body") - return - } - - if strings.TrimSpace(req.Command) == "" { - writeError(w, http.StatusBadRequest, "invalid_command", "Command cannot be empty") - return - } - - vllmOptions, err := vllm.ParseVllmCommand(req.Command) - if err != nil { - writeError(w, http.StatusBadRequest, "parse_error", err.Error()) - return - } - - backendType := backends.BackendTypeVllm - - options := &instance.CreateInstanceOptions{ - BackendType: backendType, - VllmServerOptions: vllmOptions, - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(options); err != nil { - writeError(w, http.StatusInternalServerError, "encode_error", err.Error()) - } - } -} - -// ListNodes godoc -// @Summary List all configured nodes -// @Description Returns a list of all nodes configured in the server -// @Tags nodes -// @Security ApiKeyAuth -// @Produces json -// @Success 200 {array} NodeResponse "List of nodes" -// @Failure 500 {string} string "Internal Server Error" -// @Router /nodes [get] -func (h *Handler) ListNodes() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // Convert to sanitized response format - nodeResponses := make([]NodeResponse, len(h.cfg.Nodes)) - for i, node := range h.cfg.Nodes { - nodeResponses[i] = NodeResponse{ - Name: node.Name, - Address: node.Address, - } - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(nodeResponses); err != nil { - http.Error(w, "Failed to encode nodes: "+err.Error(), http.StatusInternalServerError) - return - } - } -} - -// GetNode godoc -// @Summary Get details of a specific node -// @Description Returns the details of a specific node by name -// @Tags nodes -// @Security ApiKeyAuth -// @Produces json -// @Param name path string true "Node Name" -// @Success 200 {object} NodeResponse "Node details" -// @Failure 400 {string} string "Invalid name format" -// @Failure 404 {string} string "Node not found" -// @Failure 500 {string} string "Internal Server Error" -// @Router /nodes/{name} [get] -func (h *Handler) GetNode() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - name := chi.URLParam(r, "name") - if name == "" { - http.Error(w, "Node name cannot be empty", http.StatusBadRequest) - return - } - - var nodeConfig *config.NodeConfig - for i := range h.cfg.Nodes { - if h.cfg.Nodes[i].Name == name { - nodeConfig = &h.cfg.Nodes[i] - break - } - } - - if nodeConfig == nil { - http.Error(w, "Node not found", http.StatusNotFound) - return - } - - // Convert to sanitized response format - nodeResponse := NodeResponse{ - Name: nodeConfig.Name, - Address: nodeConfig.Address, - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(nodeResponse); err != nil { - http.Error(w, "Failed to encode node: "+err.Error(), http.StatusInternalServerError) - return - } - } -} diff --git a/pkg/server/handlers_backends.go b/pkg/server/handlers_backends.go new file mode 100644 index 0000000..5f55cd4 --- /dev/null +++ b/pkg/server/handlers_backends.go @@ -0,0 +1,239 @@ +package server + +import ( + "encoding/json" + "llamactl/pkg/backends" + "llamactl/pkg/backends/llamacpp" + "llamactl/pkg/backends/mlx" + "llamactl/pkg/backends/vllm" + "llamactl/pkg/instance" + "net/http" + "os/exec" + "strings" +) + +// ParseCommandRequest represents the request body for command parsing +type ParseCommandRequest struct { + Command string `json:"command"` +} + +// ParseLlamaCommand godoc +// @Summary Parse llama-server command +// @Description Parses a llama-server command string into instance options +// @Tags backends +// @Security ApiKeyAuth +// @Accept json +// @Produce json +// @Param request body ParseCommandRequest true "Command to parse" +// @Success 200 {object} instance.CreateInstanceOptions "Parsed options" +// @Failure 400 {object} map[string]string "Invalid request or command" +// @Failure 500 {object} map[string]string "Internal Server Error" +// @Router /backends/llama-cpp/parse-command [post] +func (h *Handler) ParseLlamaCommand() http.HandlerFunc { + type errorResponse struct { + Error string `json:"error"` + Details string `json:"details,omitempty"` + } + writeError := func(w http.ResponseWriter, status int, code, details string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(errorResponse{Error: code, Details: details}) + } + return func(w http.ResponseWriter, r *http.Request) { + var req ParseCommandRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "Invalid JSON body") + return + } + if strings.TrimSpace(req.Command) == "" { + writeError(w, http.StatusBadRequest, "invalid_command", "Command cannot be empty") + return + } + llamaOptions, err := llamacpp.ParseLlamaCommand(req.Command) + if err != nil { + writeError(w, http.StatusBadRequest, "parse_error", err.Error()) + return + } + options := &instance.CreateInstanceOptions{ + BackendType: backends.BackendTypeLlamaCpp, + LlamaServerOptions: llamaOptions, + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(options); err != nil { + writeError(w, http.StatusInternalServerError, "encode_error", err.Error()) + } + } +} + +// ParseMlxCommand godoc +// @Summary Parse mlx_lm.server command +// @Description Parses MLX-LM server command string into instance options +// @Tags backends +// @Security ApiKeyAuth +// @Accept json +// @Produce json +// @Param request body ParseCommandRequest true "Command to parse" +// @Success 200 {object} instance.CreateInstanceOptions "Parsed options" +// @Failure 400 {object} map[string]string "Invalid request or command" +// @Router /backends/mlx/parse-command [post] +func (h *Handler) ParseMlxCommand() http.HandlerFunc { + type errorResponse struct { + Error string `json:"error"` + Details string `json:"details,omitempty"` + } + writeError := func(w http.ResponseWriter, status int, code, details string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(errorResponse{Error: code, Details: details}) + } + return func(w http.ResponseWriter, r *http.Request) { + var req ParseCommandRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "Invalid JSON body") + return + } + + if strings.TrimSpace(req.Command) == "" { + writeError(w, http.StatusBadRequest, "invalid_command", "Command cannot be empty") + return + } + + mlxOptions, err := mlx.ParseMlxCommand(req.Command) + if err != nil { + writeError(w, http.StatusBadRequest, "parse_error", err.Error()) + return + } + + // Currently only support mlx_lm backend type + backendType := backends.BackendTypeMlxLm + + options := &instance.CreateInstanceOptions{ + BackendType: backendType, + MlxServerOptions: mlxOptions, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(options); err != nil { + writeError(w, http.StatusInternalServerError, "encode_error", err.Error()) + } + } +} + +// ParseVllmCommand godoc +// @Summary Parse vllm serve command +// @Description Parses a vLLM serve command string into instance options +// @Tags backends +// @Security ApiKeyAuth +// @Accept json +// @Produce json +// @Param request body ParseCommandRequest true "Command to parse" +// @Success 200 {object} instance.CreateInstanceOptions "Parsed options" +// @Failure 400 {object} map[string]string "Invalid request or command" +// @Router /backends/vllm/parse-command [post] +func (h *Handler) ParseVllmCommand() http.HandlerFunc { + type errorResponse struct { + Error string `json:"error"` + Details string `json:"details,omitempty"` + } + writeError := func(w http.ResponseWriter, status int, code, details string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(errorResponse{Error: code, Details: details}) + } + return func(w http.ResponseWriter, r *http.Request) { + var req ParseCommandRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "Invalid JSON body") + return + } + + if strings.TrimSpace(req.Command) == "" { + writeError(w, http.StatusBadRequest, "invalid_command", "Command cannot be empty") + return + } + + vllmOptions, err := vllm.ParseVllmCommand(req.Command) + if err != nil { + writeError(w, http.StatusBadRequest, "parse_error", err.Error()) + return + } + + backendType := backends.BackendTypeVllm + + options := &instance.CreateInstanceOptions{ + BackendType: backendType, + VllmServerOptions: vllmOptions, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(options); err != nil { + writeError(w, http.StatusInternalServerError, "encode_error", err.Error()) + } + } +} + +// LlamaServerHelpHandler godoc +// @Summary Get help for llama server +// @Description Returns the help text for the llama server command +// @Tags backends +// @Security ApiKeyAuth +// @Produces text/plain +// @Success 200 {string} string "Help text" +// @Failure 500 {string} string "Internal Server Error" +// @Router /backends/llama-cpp/help [get] +func (h *Handler) LlamaServerHelpHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + helpCmd := exec.Command("llama-server", "--help") + output, err := helpCmd.CombinedOutput() + if err != nil { + http.Error(w, "Failed to get help: "+err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/plain") + w.Write(output) + } +} + +// LlamaServerVersionHandler godoc +// @Summary Get version of llama server +// @Description Returns the version of the llama server command +// @Tags backends +// @Security ApiKeyAuth +// @Produces text/plain +// @Success 200 {string} string "Version information" +// @Failure 500 {string} string "Internal Server Error" +// @Router /backends/llama-cpp/version [get] +func (h *Handler) LlamaServerVersionHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + versionCmd := exec.Command("llama-server", "--version") + output, err := versionCmd.CombinedOutput() + if err != nil { + http.Error(w, "Failed to get version: "+err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/plain") + w.Write(output) + } +} + +// LlamaServerListDevicesHandler godoc +// @Summary List available devices for llama server +// @Description Returns a list of available devices for the llama server +// @Tags backends +// @Security ApiKeyAuth +// @Produces text/plain +// @Success 200 {string} string "List of devices" +// @Failure 500 {string} string "Internal Server Error" +// @Router /backends/llama-cpp/devices [get] +func (h *Handler) LlamaServerListDevicesHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + listCmd := exec.Command("llama-server", "--list-devices") + output, err := listCmd.CombinedOutput() + if err != nil { + http.Error(w, "Failed to list devices: "+err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/plain") + w.Write(output) + } +} diff --git a/pkg/server/handlers_instances.go b/pkg/server/handlers_instances.go new file mode 100644 index 0000000..e8b108c --- /dev/null +++ b/pkg/server/handlers_instances.go @@ -0,0 +1,477 @@ +package server + +import ( + "encoding/json" + "fmt" + "io" + "llamactl/pkg/config" + "llamactl/pkg/instance" + "llamactl/pkg/manager" + "net/http" + "strconv" + "strings" + + "github.com/go-chi/chi/v5" +) + +// ListInstances godoc +// @Summary List all instances +// @Description Returns a list of all instances managed by the server +// @Tags instances +// @Security ApiKeyAuth +// @Produces json +// @Success 200 {array} instance.Process "List of instances" +// @Failure 500 {string} string "Internal Server Error" +// @Router /instances [get] +func (h *Handler) ListInstances() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + instances, err := h.InstanceManager.ListInstances() + if err != nil { + http.Error(w, "Failed to list instances: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(instances); err != nil { + http.Error(w, "Failed to encode instances: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// CreateInstance godoc +// @Summary Create and start a new instance +// @Description Creates a new instance with the provided configuration options +// @Tags instances +// @Security ApiKeyAuth +// @Accept json +// @Produces json +// @Param name path string true "Instance Name" +// @Param options body instance.CreateInstanceOptions true "Instance configuration options" +// @Success 201 {object} instance.Process "Created instance details" +// @Failure 400 {string} string "Invalid request body" +// @Failure 500 {string} string "Internal Server Error" +// @Router /instances/{name} [post] +func (h *Handler) CreateInstance() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) + return + } + + var options instance.CreateInstanceOptions + if err := json.NewDecoder(r.Body).Decode(&options); err != nil { + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + + inst, err := h.InstanceManager.CreateInstance(name, &options) + if err != nil { + http.Error(w, "Failed to create instance: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + if err := json.NewEncoder(w).Encode(inst); err != nil { + http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// GetInstance godoc +// @Summary Get details of a specific instance +// @Description Returns the details of a specific instance by name +// @Tags instances +// @Security ApiKeyAuth +// @Produces json +// @Param name path string true "Instance Name" +// @Success 200 {object} instance.Process "Instance details" +// @Failure 400 {string} string "Invalid name format" +// @Failure 500 {string} string "Internal Server Error" +// @Router /instances/{name} [get] +func (h *Handler) GetInstance() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) + return + } + + inst, err := h.InstanceManager.GetInstance(name) + if err != nil { + http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(inst); err != nil { + http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// UpdateInstance godoc +// @Summary Update an instance's configuration +// @Description Updates the configuration of a specific instance by name +// @Tags instances +// @Security ApiKeyAuth +// @Accept json +// @Produces json +// @Param name path string true "Instance Name" +// @Param options body instance.CreateInstanceOptions true "Instance configuration options" +// @Success 200 {object} instance.Process "Updated instance details" +// @Failure 400 {string} string "Invalid name format" +// @Failure 500 {string} string "Internal Server Error" +// @Router /instances/{name} [put] +func (h *Handler) UpdateInstance() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) + return + } + + var options instance.CreateInstanceOptions + if err := json.NewDecoder(r.Body).Decode(&options); err != nil { + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + + inst, err := h.InstanceManager.UpdateInstance(name, &options) + if err != nil { + http.Error(w, "Failed to update instance: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(inst); err != nil { + http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// StartInstance godoc +// @Summary Start a stopped instance +// @Description Starts a specific instance by name +// @Tags instances +// @Security ApiKeyAuth +// @Produces json +// @Param name path string true "Instance Name" +// @Success 200 {object} instance.Process "Started instance details" +// @Failure 400 {string} string "Invalid name format" +// @Failure 500 {string} string "Internal Server Error" +// @Router /instances/{name}/start [post] +func (h *Handler) StartInstance() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) + return + } + + inst, err := h.InstanceManager.StartInstance(name) + if err != nil { + // Check if error is due to maximum running instances limit + if _, ok := err.(manager.MaxRunningInstancesError); ok { + http.Error(w, err.Error(), http.StatusConflict) + return + } + + http.Error(w, "Failed to start instance: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(inst); err != nil { + http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// StopInstance godoc +// @Summary Stop a running instance +// @Description Stops a specific instance by name +// @Tags instances +// @Security ApiKeyAuth +// @Produces json +// @Param name path string true "Instance Name" +// @Success 200 {object} instance.Process "Stopped instance details" +// @Failure 400 {string} string "Invalid name format" +// @Failure 500 {string} string "Internal Server Error" +// @Router /instances/{name}/stop [post] +func (h *Handler) StopInstance() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) + return + } + + inst, err := h.InstanceManager.StopInstance(name) + if err != nil { + http.Error(w, "Failed to stop instance: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(inst); err != nil { + http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// RestartInstance godoc +// @Summary Restart a running instance +// @Description Restarts a specific instance by name +// @Tags instances +// @Security ApiKeyAuth +// @Produces json +// @Param name path string true "Instance Name" +// @Success 200 {object} instance.Process "Restarted instance details" +// @Failure 400 {string} string "Invalid name format" +// @Failure 500 {string} string "Internal Server Error" +// @Router /instances/{name}/restart [post] +func (h *Handler) RestartInstance() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) + return + } + + inst, err := h.InstanceManager.RestartInstance(name) + if err != nil { + http.Error(w, "Failed to restart instance: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(inst); err != nil { + http.Error(w, "Failed to encode instance: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// DeleteInstance godoc +// @Summary Delete an instance +// @Description Stops and removes a specific instance by name +// @Tags instances +// @Security ApiKeyAuth +// @Param name path string true "Instance Name" +// @Success 204 "No Content" +// @Failure 400 {string} string "Invalid name format" +// @Failure 500 {string} string "Internal Server Error" +// @Router /instances/{name} [delete] +func (h *Handler) DeleteInstance() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) + return + } + + if err := h.InstanceManager.DeleteInstance(name); err != nil { + http.Error(w, "Failed to delete instance: "+err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusNoContent) + } +} + +// GetInstanceLogs godoc +// @Summary Get logs from a specific instance +// @Description Returns the logs from a specific instance by name with optional line limit +// @Tags instances +// @Security ApiKeyAuth +// @Param name path string true "Instance Name" +// @Param lines query string false "Number of lines to retrieve (default: all lines)" +// @Produces text/plain +// @Success 200 {string} string "Instance logs" +// @Failure 400 {string} string "Invalid name format or lines parameter" +// @Failure 500 {string} string "Internal Server Error" +// @Router /instances/{name}/logs [get] +func (h *Handler) GetInstanceLogs() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) + return + } + + lines := r.URL.Query().Get("lines") + if lines == "" { + lines = "-1" + } + + num_lines, err := strconv.Atoi(lines) + if err != nil { + http.Error(w, "Invalid lines parameter: "+err.Error(), http.StatusBadRequest) + return + } + + inst, err := h.InstanceManager.GetInstance(name) + if err != nil { + http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError) + return + } + + logs, err := inst.GetLogs(num_lines) + if err != nil { + http.Error(w, "Failed to get logs: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte(logs)) + } +} + +// ProxyToInstance godoc +// @Summary Proxy requests to a specific instance +// @Description Forwards HTTP requests to the llama-server instance running on a specific port +// @Tags instances +// @Security ApiKeyAuth +// @Param name path string true "Instance Name" +// @Success 200 "Request successfully proxied to instance" +// @Failure 400 {string} string "Invalid name format" +// @Failure 500 {string} string "Internal Server Error" +// @Failure 503 {string} string "Instance is not running" +// @Router /instances/{name}/proxy [get] +// @Router /instances/{name}/proxy [post] +func (h *Handler) ProxyToInstance() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Instance name cannot be empty", http.StatusBadRequest) + return + } + + inst, err := h.InstanceManager.GetInstance(name) + if err != nil { + http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError) + return + } + + // Check if this is a remote instance + if inst.IsRemote() { + h.RemoteInstanceProxy(w, r, name, inst) + return + } + + if !inst.IsRunning() { + http.Error(w, "Instance is not running", http.StatusServiceUnavailable) + return + } + + // Get the cached proxy for this instance + proxy, err := inst.GetProxy() + if err != nil { + http.Error(w, "Failed to get proxy: "+err.Error(), http.StatusInternalServerError) + return + } + + // Strip the "/api/v1/instances//proxy" prefix from the request URL + prefix := fmt.Sprintf("/api/v1/instances/%s/proxy", name) + proxyPath := r.URL.Path[len(prefix):] + + // Ensure the proxy path starts with "/" + if !strings.HasPrefix(proxyPath, "/") { + proxyPath = "/" + proxyPath + } + + // Update the last request time for the instance + inst.UpdateLastRequestTime() + + // Modify the request to remove the proxy prefix + originalPath := r.URL.Path + r.URL.Path = proxyPath + + // Set forwarded headers + r.Header.Set("X-Forwarded-Host", r.Header.Get("Host")) + r.Header.Set("X-Forwarded-Proto", "http") + + // Restore original path for logging purposes + defer func() { + r.URL.Path = originalPath + }() + + // Forward the request using the cached proxy + proxy.ServeHTTP(w, r) + } +} + +// RemoteInstanceProxy proxies requests to a remote instance +func (h *Handler) RemoteInstanceProxy(w http.ResponseWriter, r *http.Request, name string, inst *instance.Process) { + // Get the node name from instance options + options := inst.GetOptions() + if options == nil || len(options.Nodes) == 0 { + http.Error(w, "Instance has no node configured", http.StatusInternalServerError) + return + } + + nodeName := options.Nodes[0] + var nodeConfig *config.NodeConfig + for i := range h.cfg.Nodes { + if h.cfg.Nodes[i].Name == nodeName { + nodeConfig = &h.cfg.Nodes[i] + break + } + } + + if nodeConfig == nil { + http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) + return + } + + // Strip the "/api/v1/instances//proxy" prefix from the request URL + prefix := fmt.Sprintf("/api/v1/instances/%s/proxy", name) + proxyPath := r.URL.Path[len(prefix):] + + // Build the remote URL + remoteURL := fmt.Sprintf("%s/api/v1/instances/%s/proxy%s", nodeConfig.Address, name, proxyPath) + + // Create a new request to the remote node + req, err := http.NewRequest(r.Method, remoteURL, r.Body) + if err != nil { + http.Error(w, "Failed to create remote request: "+err.Error(), http.StatusInternalServerError) + return + } + + // Copy headers + req.Header = r.Header.Clone() + + // Add API key if configured + if nodeConfig.APIKey != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", nodeConfig.APIKey)) + } + + // Forward the request + resp, err := h.httpClient.Do(req) + if err != nil { + http.Error(w, "Failed to proxy to remote instance: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + + // Copy response headers + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + + // Copy status code + w.WriteHeader(resp.StatusCode) + + // Copy response body + io.Copy(w, resp.Body) +} diff --git a/pkg/server/handlers_nodes.go b/pkg/server/handlers_nodes.go new file mode 100644 index 0000000..7d1116f --- /dev/null +++ b/pkg/server/handlers_nodes.go @@ -0,0 +1,90 @@ +package server + +import ( + "encoding/json" + "llamactl/pkg/config" + "net/http" + + "github.com/go-chi/chi/v5" +) + +// NodeResponse represents a sanitized node configuration for API responses +type NodeResponse struct { + Name string `json:"name"` + Address string `json:"address"` +} + +// ListNodes godoc +// @Summary List all configured nodes +// @Description Returns a list of all nodes configured in the server +// @Tags nodes +// @Security ApiKeyAuth +// @Produces json +// @Success 200 {array} NodeResponse "List of nodes" +// @Failure 500 {string} string "Internal Server Error" +// @Router /nodes [get] +func (h *Handler) ListNodes() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Convert to sanitized response format + nodeResponses := make([]NodeResponse, len(h.cfg.Nodes)) + for i, node := range h.cfg.Nodes { + nodeResponses[i] = NodeResponse{ + Name: node.Name, + Address: node.Address, + } + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(nodeResponses); err != nil { + http.Error(w, "Failed to encode nodes: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// GetNode godoc +// @Summary Get details of a specific node +// @Description Returns the details of a specific node by name +// @Tags nodes +// @Security ApiKeyAuth +// @Produces json +// @Param name path string true "Node Name" +// @Success 200 {object} NodeResponse "Node details" +// @Failure 400 {string} string "Invalid name format" +// @Failure 404 {string} string "Node not found" +// @Failure 500 {string} string "Internal Server Error" +// @Router /nodes/{name} [get] +func (h *Handler) GetNode() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if name == "" { + http.Error(w, "Node name cannot be empty", http.StatusBadRequest) + return + } + + var nodeConfig *config.NodeConfig + for i := range h.cfg.Nodes { + if h.cfg.Nodes[i].Name == name { + nodeConfig = &h.cfg.Nodes[i] + break + } + } + + if nodeConfig == nil { + http.Error(w, "Node not found", http.StatusNotFound) + return + } + + // Convert to sanitized response format + nodeResponse := NodeResponse{ + Name: nodeConfig.Name, + Address: nodeConfig.Address, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(nodeResponse); err != nil { + http.Error(w, "Failed to encode node: "+err.Error(), http.StatusInternalServerError) + return + } + } +} diff --git a/pkg/server/handlers_openai.go b/pkg/server/handlers_openai.go new file mode 100644 index 0000000..ef4c29d --- /dev/null +++ b/pkg/server/handlers_openai.go @@ -0,0 +1,214 @@ +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "llamactl/pkg/config" + "llamactl/pkg/instance" + "net/http" +) + +// OpenAIListInstances godoc +// @Summary List instances in OpenAI-compatible format +// @Description Returns a list of instances in a format compatible with OpenAI API +// @Tags openai +// @Security ApiKeyAuth +// @Produces json +// @Success 200 {object} OpenAIListInstancesResponse "List of OpenAI-compatible instances" +// @Failure 500 {string} string "Internal Server Error" +// @Router /v1/models [get] +func (h *Handler) OpenAIListInstances() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + instances, err := h.InstanceManager.ListInstances() + if err != nil { + http.Error(w, "Failed to list instances: "+err.Error(), http.StatusInternalServerError) + return + } + + openaiInstances := make([]OpenAIInstance, len(instances)) + for i, inst := range instances { + openaiInstances[i] = OpenAIInstance{ + ID: inst.Name, + Object: "model", + Created: inst.Created, + OwnedBy: "llamactl", + } + } + + openaiResponse := OpenAIListInstancesResponse{ + Object: "list", + Data: openaiInstances, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(openaiResponse); err != nil { + http.Error(w, "Failed to encode instances: "+err.Error(), http.StatusInternalServerError) + return + } + } +} + +// OpenAIProxy godoc +// @Summary OpenAI-compatible proxy endpoint +// @Description Handles all POST requests to /v1/*, routing to the appropriate instance based on the request body. Requires API key authentication via the `Authorization` header. +// @Tags openai +// @Security ApiKeyAuth +// @Accept json +// @Produces json +// @Success 200 "OpenAI response" +// @Failure 400 {string} string "Invalid request body or instance name" +// @Failure 500 {string} string "Internal Server Error" +// @Router /v1/ [post] +func (h *Handler) OpenAIProxy() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Read the entire body first + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read request body", http.StatusBadRequest) + return + } + r.Body.Close() + + // Parse the body to extract instance name + var requestBody map[string]any + if err := json.Unmarshal(bodyBytes, &requestBody); err != nil { + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + + modelName, ok := requestBody["model"].(string) + if !ok || modelName == "" { + http.Error(w, "Instance name is required", http.StatusBadRequest) + return + } + + // Route to the appropriate inst based on instance name + inst, err := h.InstanceManager.GetInstance(modelName) + if err != nil { + http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError) + return + } + + // Check if this is a remote instance + if inst.IsRemote() { + h.RemoteOpenAIProxy(w, r, modelName, inst, bodyBytes) + return + } + + if !inst.IsRunning() { + allowOnDemand := inst.GetOptions() != nil && inst.GetOptions().OnDemandStart != nil && *inst.GetOptions().OnDemandStart + if !allowOnDemand { + http.Error(w, "Instance is not running", http.StatusServiceUnavailable) + return + } + + if h.InstanceManager.IsMaxRunningInstancesReached() { + if h.cfg.Instances.EnableLRUEviction { + err := h.InstanceManager.EvictLRUInstance() + if err != nil { + http.Error(w, "Cannot start Instance, failed to evict instance "+err.Error(), http.StatusInternalServerError) + return + } + } else { + http.Error(w, "Cannot start Instance, maximum number of instances reached", http.StatusConflict) + return + } + } + + // If on-demand start is enabled, start the instance + if _, err := h.InstanceManager.StartInstance(modelName); err != nil { + http.Error(w, "Failed to start instance: "+err.Error(), http.StatusInternalServerError) + return + } + + // Wait for the instance to become healthy before proceeding + if err := inst.WaitForHealthy(h.cfg.Instances.OnDemandStartTimeout); err != nil { // 2 minutes timeout + http.Error(w, "Instance failed to become healthy: "+err.Error(), http.StatusServiceUnavailable) + return + } + } + + proxy, err := inst.GetProxy() + if err != nil { + http.Error(w, "Failed to get proxy: "+err.Error(), http.StatusInternalServerError) + return + } + + // Update last request time for the instance + inst.UpdateLastRequestTime() + + // Recreate the request body from the bytes we read + r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) + r.ContentLength = int64(len(bodyBytes)) + + proxy.ServeHTTP(w, r) + } +} + +// RemoteOpenAIProxy proxies OpenAI-compatible requests to a remote instance +func (h *Handler) RemoteOpenAIProxy(w http.ResponseWriter, r *http.Request, modelName string, inst *instance.Process, bodyBytes []byte) { + // Get the node name from instance options + options := inst.GetOptions() + if options == nil || len(options.Nodes) == 0 { + http.Error(w, "Instance has no node configured", http.StatusInternalServerError) + return + } + + nodeName := options.Nodes[0] + var nodeConfig *config.NodeConfig + for i := range h.cfg.Nodes { + if h.cfg.Nodes[i].Name == nodeName { + nodeConfig = &h.cfg.Nodes[i] + break + } + } + + if nodeConfig == nil { + http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) + return + } + + // Build the remote URL - forward to the same OpenAI endpoint on the remote node + remoteURL := fmt.Sprintf("%s%s", nodeConfig.Address, r.URL.Path) + if r.URL.RawQuery != "" { + remoteURL += "?" + r.URL.RawQuery + } + + // Create a new request to the remote node + req, err := http.NewRequest(r.Method, remoteURL, bytes.NewReader(bodyBytes)) + if err != nil { + http.Error(w, "Failed to create remote request: "+err.Error(), http.StatusInternalServerError) + return + } + + // Copy headers + req.Header = r.Header.Clone() + + // Add API key if configured + if nodeConfig.APIKey != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", nodeConfig.APIKey)) + } + + // Forward the request + resp, err := h.httpClient.Do(req) + if err != nil { + http.Error(w, "Failed to proxy to remote instance: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + + // Copy response headers + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + + // Copy status code + w.WriteHeader(resp.StatusCode) + + // Copy response body + io.Copy(w, resp.Body) +} diff --git a/pkg/server/handlers_system.go b/pkg/server/handlers_system.go new file mode 100644 index 0000000..e3bb016 --- /dev/null +++ b/pkg/server/handlers_system.go @@ -0,0 +1,22 @@ +package server + +import ( + "fmt" + "net/http" +) + +// VersionHandler godoc +// @Summary Get llamactl version +// @Description Returns the version of the llamactl command +// @Tags version +// @Security ApiKeyAuth +// @Produces text/plain +// @Success 200 {string} string "Version information" +// @Failure 500 {string} string "Internal Server Error" +// @Router /version [get] +func (h *Handler) VersionHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprintf(w, "Version: %s\nCommit: %s\nBuild Time: %s\n", h.cfg.Version, h.cfg.CommitHash, h.cfg.BuildTime) + } +} From a491f294831d22a14bc431c6abeef58deb45c54f Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 2 Oct 2025 23:18:33 +0200 Subject: [PATCH 09/34] Add node selection functionality to InstanceSettingsCard and define Node API --- .../instance/InstanceSettingsCard.tsx | 52 ++++++++++++++++++- webui/src/lib/api.ts | 15 ++++++ webui/src/schemas/instanceOptions.ts | 3 ++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/webui/src/components/instance/InstanceSettingsCard.tsx b/webui/src/components/instance/InstanceSettingsCard.tsx index c85eda9..124986b 100644 --- a/webui/src/components/instance/InstanceSettingsCard.tsx +++ b/webui/src/components/instance/InstanceSettingsCard.tsx @@ -1,4 +1,4 @@ -import React from 'react' +import React, { useState, useEffect } from 'react' import type { CreateInstanceOptions } from '@/types/instance' import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card' import { Label } from '@/components/ui/label' @@ -7,6 +7,8 @@ import AutoRestartConfiguration from '@/components/instance/AutoRestartConfigura import NumberInput from '@/components/form/NumberInput' import CheckboxInput from '@/components/form/CheckboxInput' import EnvironmentVariablesInput from '@/components/form/EnvironmentVariablesInput' +import SelectInput from '@/components/form/SelectInput' +import { nodesApi, type NodeResponse } from '@/lib/api' interface InstanceSettingsCardProps { instanceName: string @@ -25,6 +27,42 @@ const InstanceSettingsCard: React.FC = ({ onNameChange, onChange }) => { + const [nodes, setNodes] = useState([]) + const [loadingNodes, setLoadingNodes] = useState(true) + + useEffect(() => { + const fetchNodes = async () => { + try { + const fetchedNodes = await nodesApi.list() + setNodes(fetchedNodes) + } catch (error) { + console.error('Failed to fetch nodes:', error) + } finally { + setLoadingNodes(false) + } + } + + fetchNodes() + }, []) + + const nodeOptions = [ + { value: '', label: 'Default (Main Node)' }, + ...nodes.map(node => ({ + value: node.name, + label: node.name + })) + ] + + const handleNodeChange = (value: string | undefined) => { + if (value) { + onChange('nodes', [value]) + } else { + onChange('nodes', undefined) + } + } + + const selectedNode = formData.nodes && formData.nodes.length > 0 ? formData.nodes[0] : '' + return ( @@ -50,6 +88,18 @@ const InstanceSettingsCard: React.FC = ({

+ {/* Node Selection */} + {!loadingNodes && nodes.length > 0 && ( + + )} + {/* Auto Restart Configuration */} apiCall("/nodes"), + + // GET /nodes/{name} + get: (name: string) => apiCall(`/nodes/${name}`), +}; + // Instance API functions export const instancesApi = { // GET /instances diff --git a/webui/src/schemas/instanceOptions.ts b/webui/src/schemas/instanceOptions.ts index 0af09c1..3cbf523 100644 --- a/webui/src/schemas/instanceOptions.ts +++ b/webui/src/schemas/instanceOptions.ts @@ -39,6 +39,9 @@ export const CreateInstanceOptionsSchema = z.object({ // Backend configuration backend_type: z.enum([BackendType.LLAMA_CPP, BackendType.MLX_LM, BackendType.VLLM]).optional(), backend_options: BackendOptionsSchema.optional(), + + // Node configuration + nodes: z.array(z.string()).optional(), }) // Re-export types and schemas from backend files From b728a7c6b218ca0af0ebdb016d29c0a5d167e3d5 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Fri, 3 Oct 2025 10:53:29 +0200 Subject: [PATCH 10/34] Fix fetchNodes call to ensure proper handling of promise --- webui/src/components/instance/InstanceSettingsCard.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/src/components/instance/InstanceSettingsCard.tsx b/webui/src/components/instance/InstanceSettingsCard.tsx index 124986b..369b8a6 100644 --- a/webui/src/components/instance/InstanceSettingsCard.tsx +++ b/webui/src/components/instance/InstanceSettingsCard.tsx @@ -42,7 +42,7 @@ const InstanceSettingsCard: React.FC = ({ } } - fetchNodes() + void fetchNodes() }, []) const nodeOptions = [ From 554796391b4b96294ab532b237a6c3b9090e3625 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Tue, 7 Oct 2025 18:05:30 +0200 Subject: [PATCH 11/34] Remove test config file --- llamactl.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 llamactl.yaml diff --git a/llamactl.yaml b/llamactl.yaml deleted file mode 100644 index 1c616eb..0000000 --- a/llamactl.yaml +++ /dev/null @@ -1,5 +0,0 @@ -auth: - management_keys: - - test-mgmt - inference_keys: - - test-inf From aae3f84d49a2addf7b07f588790023ad77e44714 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Tue, 7 Oct 2025 18:44:23 +0200 Subject: [PATCH 12/34] Implement caching for remote instance proxies and enhance proxy request handling --- pkg/server/handlers.go | 5 ++ pkg/server/handlers_instances.go | 103 ++++++++++++++++--------------- 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/pkg/server/handlers.go b/pkg/server/handlers.go index 4ddbfea..9e31df9 100644 --- a/pkg/server/handlers.go +++ b/pkg/server/handlers.go @@ -4,6 +4,8 @@ import ( "llamactl/pkg/config" "llamactl/pkg/manager" "net/http" + "net/http/httputil" + "sync" "time" ) @@ -11,6 +13,8 @@ type Handler struct { InstanceManager manager.InstanceManager cfg config.AppConfig httpClient *http.Client + remoteProxies map[string]*httputil.ReverseProxy // Cache of remote proxies by instance name + remoteProxiesMu sync.RWMutex } func NewHandler(im manager.InstanceManager, cfg config.AppConfig) *Handler { @@ -20,5 +24,6 @@ func NewHandler(im manager.InstanceManager, cfg config.AppConfig) *Handler { httpClient: &http.Client{ Timeout: 30 * time.Second, }, + remoteProxies: make(map[string]*httputil.ReverseProxy), } } diff --git a/pkg/server/handlers_instances.go b/pkg/server/handlers_instances.go index 6a90cba..90a00f9 100644 --- a/pkg/server/handlers_instances.go +++ b/pkg/server/handlers_instances.go @@ -3,11 +3,12 @@ package server import ( "encoding/json" "fmt" - "io" "llamactl/pkg/config" "llamactl/pkg/instance" "llamactl/pkg/manager" "net/http" + "net/http/httputil" + "net/url" "strconv" "strings" @@ -361,6 +362,12 @@ func (h *Handler) ProxyToInstance() http.HandlerFunc { return } + // Check if this is a remote instance + if inst.IsRemote() { + h.RemoteInstanceProxy(w, r, name, inst) + return + } + if !inst.IsRunning() { http.Error(w, "Instance is not running", http.StatusServiceUnavailable) return @@ -399,59 +406,53 @@ func (h *Handler) RemoteInstanceProxy(w http.ResponseWriter, r *http.Request, na } nodeName := options.Nodes[0] - var nodeConfig *config.NodeConfig - for i := range h.cfg.Nodes { - if h.cfg.Nodes[i].Name == nodeName { - nodeConfig = &h.cfg.Nodes[i] - break + + // Check if we have a cached proxy for this instance + h.remoteProxiesMu.RLock() + proxy, exists := h.remoteProxies[name] + h.remoteProxiesMu.RUnlock() + + if !exists { + // Find node configuration + var nodeConfig *config.NodeConfig + for i := range h.cfg.Nodes { + if h.cfg.Nodes[i].Name == nodeName { + nodeConfig = &h.cfg.Nodes[i] + break + } } - } - if nodeConfig == nil { - http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) - return - } - - // Strip the "/api/v1/instances//proxy" prefix from the request URL - prefix := fmt.Sprintf("/api/v1/instances/%s/proxy", name) - proxyPath := r.URL.Path[len(prefix):] - - // Build the remote URL - remoteURL := fmt.Sprintf("%s/api/v1/instances/%s/proxy%s", nodeConfig.Address, name, proxyPath) - - // Create a new request to the remote node - req, err := http.NewRequest(r.Method, remoteURL, r.Body) - if err != nil { - http.Error(w, "Failed to create remote request: "+err.Error(), http.StatusInternalServerError) - return - } - - // Copy headers - req.Header = r.Header.Clone() - - // Add API key if configured - if nodeConfig.APIKey != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", nodeConfig.APIKey)) - } - - // Forward the request - resp, err := h.httpClient.Do(req) - if err != nil { - http.Error(w, "Failed to proxy to remote instance: "+err.Error(), http.StatusBadGateway) - return - } - defer resp.Body.Close() - - // Copy response headers - for key, values := range resp.Header { - for _, value := range values { - w.Header().Add(key, value) + if nodeConfig == nil { + http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) + return } + + // Create reverse proxy to remote node + targetURL, err := url.Parse(nodeConfig.Address) + if err != nil { + http.Error(w, "Failed to parse node address: "+err.Error(), http.StatusInternalServerError) + return + } + + proxy = httputil.NewSingleHostReverseProxy(targetURL) + + // Modify request before forwarding + originalDirector := proxy.Director + apiKey := nodeConfig.APIKey // Capture for closure + proxy.Director = func(req *http.Request) { + originalDirector(req) + // Add API key if configured + if apiKey != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey)) + } + } + + // Cache the proxy + h.remoteProxiesMu.Lock() + h.remoteProxies[name] = proxy + h.remoteProxiesMu.Unlock() } - // Copy status code - w.WriteHeader(resp.StatusCode) - - // Copy response body - io.Copy(w, resp.Body) + // Forward the request using the cached proxy + proxy.ServeHTTP(w, r) } From 6298b03636fa407fa69f0c947a10980990b1595d Mon Sep 17 00:00:00 2001 From: LordMathis Date: Tue, 7 Oct 2025 18:57:08 +0200 Subject: [PATCH 13/34] Refactor RemoteOpenAIProxy to use cached proxies and restore request body handling --- pkg/server/handlers_instances.go | 8 +-- pkg/server/handlers_openai.go | 101 +++++++++++++++---------------- 2 files changed, 54 insertions(+), 55 deletions(-) diff --git a/pkg/server/handlers_instances.go b/pkg/server/handlers_instances.go index 90a00f9..8c325ae 100644 --- a/pkg/server/handlers_instances.go +++ b/pkg/server/handlers_instances.go @@ -407,9 +407,9 @@ func (h *Handler) RemoteInstanceProxy(w http.ResponseWriter, r *http.Request, na nodeName := options.Nodes[0] - // Check if we have a cached proxy for this instance + // Check if we have a cached proxy for this node h.remoteProxiesMu.RLock() - proxy, exists := h.remoteProxies[name] + proxy, exists := h.remoteProxies[nodeName] h.remoteProxiesMu.RUnlock() if !exists { @@ -447,9 +447,9 @@ func (h *Handler) RemoteInstanceProxy(w http.ResponseWriter, r *http.Request, na } } - // Cache the proxy + // Cache the proxy by node name h.remoteProxiesMu.Lock() - h.remoteProxies[name] = proxy + h.remoteProxies[nodeName] = proxy h.remoteProxiesMu.Unlock() } diff --git a/pkg/server/handlers_openai.go b/pkg/server/handlers_openai.go index 07196f0..eea8440 100644 --- a/pkg/server/handlers_openai.go +++ b/pkg/server/handlers_openai.go @@ -8,6 +8,8 @@ import ( "llamactl/pkg/config" "llamactl/pkg/instance" "net/http" + "net/http/httputil" + "net/url" ) // OpenAIListInstances godoc @@ -93,7 +95,9 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc { // Check if this is a remote instance if inst.IsRemote() { - h.RemoteOpenAIProxy(w, r, modelName, inst, bodyBytes) + // Restore the body for the remote proxy + r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) + h.RemoteOpenAIProxy(w, r, modelName, inst) return } @@ -149,7 +153,7 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc { } // RemoteOpenAIProxy proxies OpenAI-compatible requests to a remote instance -func (h *Handler) RemoteOpenAIProxy(w http.ResponseWriter, r *http.Request, modelName string, inst *instance.Process, bodyBytes []byte) { +func (h *Handler) RemoteOpenAIProxy(w http.ResponseWriter, r *http.Request, modelName string, inst *instance.Process) { // Get the node name from instance options options := inst.GetOptions() if options == nil || len(options.Nodes) == 0 { @@ -158,58 +162,53 @@ func (h *Handler) RemoteOpenAIProxy(w http.ResponseWriter, r *http.Request, mode } nodeName := options.Nodes[0] - var nodeConfig *config.NodeConfig - for i := range h.cfg.Nodes { - if h.cfg.Nodes[i].Name == nodeName { - nodeConfig = &h.cfg.Nodes[i] - break + + // Check if we have a cached proxy for this node + h.remoteProxiesMu.RLock() + proxy, exists := h.remoteProxies[nodeName] + h.remoteProxiesMu.RUnlock() + + if !exists { + // Find node configuration + var nodeConfig *config.NodeConfig + for i := range h.cfg.Nodes { + if h.cfg.Nodes[i].Name == nodeName { + nodeConfig = &h.cfg.Nodes[i] + break + } } - } - if nodeConfig == nil { - http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) - return - } - - // Build the remote URL - forward to the same OpenAI endpoint on the remote node - remoteURL := fmt.Sprintf("%s%s", nodeConfig.Address, r.URL.Path) - if r.URL.RawQuery != "" { - remoteURL += "?" + r.URL.RawQuery - } - - // Create a new request to the remote node - req, err := http.NewRequest(r.Method, remoteURL, bytes.NewReader(bodyBytes)) - if err != nil { - http.Error(w, "Failed to create remote request: "+err.Error(), http.StatusInternalServerError) - return - } - - // Copy headers - req.Header = r.Header.Clone() - - // Add API key if configured - if nodeConfig.APIKey != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", nodeConfig.APIKey)) - } - - // Forward the request - resp, err := h.httpClient.Do(req) - if err != nil { - http.Error(w, "Failed to proxy to remote instance: "+err.Error(), http.StatusBadGateway) - return - } - defer resp.Body.Close() - - // Copy response headers - for key, values := range resp.Header { - for _, value := range values { - w.Header().Add(key, value) + if nodeConfig == nil { + http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) + return } + + // Create reverse proxy to remote node + targetURL, err := url.Parse(nodeConfig.Address) + if err != nil { + http.Error(w, "Failed to parse node address: "+err.Error(), http.StatusInternalServerError) + return + } + + proxy = httputil.NewSingleHostReverseProxy(targetURL) + + // Modify request before forwarding + originalDirector := proxy.Director + apiKey := nodeConfig.APIKey // Capture for closure + proxy.Director = func(req *http.Request) { + originalDirector(req) + // Add API key if configured + if apiKey != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiKey)) + } + } + + // Cache the proxy + h.remoteProxiesMu.Lock() + h.remoteProxies[nodeName] = proxy + h.remoteProxiesMu.Unlock() } - // Copy status code - w.WriteHeader(resp.StatusCode) - - // Copy response body - io.Copy(w, resp.Body) + // Forward the request using the cached proxy + proxy.ServeHTTP(w, r) } From 01380e66410b7e916553ff0f6442e722db712317 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Tue, 7 Oct 2025 19:18:13 +0200 Subject: [PATCH 14/34] Update instance manager tests to use empty NodeConfig slice --- pkg/manager/manager_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index e9fa1bc..5110e52 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -34,7 +34,7 @@ func TestNewInstanceManager(t *testing.T) { TimeoutCheckInterval: 5, } - mgr := manager.NewInstanceManager(backendConfig, cfg, nil) + mgr := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) if mgr == nil { t.Fatal("NewInstanceManager returned nil") } @@ -69,7 +69,7 @@ func TestPersistence(t *testing.T) { } // Test instance persistence on creation - manager1 := manager.NewInstanceManager(backendConfig, cfg, nil) + manager1 := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) options := &instance.CreateInstanceOptions{ BackendType: backends.BackendTypeLlamaCpp, LlamaServerOptions: &llamacpp.LlamaServerOptions{ @@ -90,7 +90,7 @@ func TestPersistence(t *testing.T) { } // Test loading instances from disk - manager2 := manager.NewInstanceManager(backendConfig, cfg, nil) + manager2 := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) instances, err := manager2.ListInstances() if err != nil { t.Fatalf("ListInstances failed: %v", err) @@ -207,7 +207,7 @@ func createTestManager() manager.InstanceManager { DefaultRestartDelay: 5, TimeoutCheckInterval: 5, } - return manager.NewInstanceManager(backendConfig, cfg, nil) + return manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) } func TestAutoRestartDisabledInstanceStatus(t *testing.T) { @@ -227,7 +227,7 @@ func TestAutoRestartDisabledInstanceStatus(t *testing.T) { } // Create first manager and instance with auto-restart disabled - manager1 := manager.NewInstanceManager(backendConfig, cfg) + manager1 := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) autoRestart := false options := &instance.CreateInstanceOptions{ @@ -252,7 +252,7 @@ func TestAutoRestartDisabledInstanceStatus(t *testing.T) { manager1.Shutdown() // Create second manager (simulating restart of llamactl) - manager2 := manager.NewInstanceManager(backendConfig, cfg) + manager2 := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) // Get the loaded instance loadedInst, err := manager2.GetInstance("test-instance") From 2f1cf5acdc9f5c0ee844364c25613bf44ce9964b Mon Sep 17 00:00:00 2001 From: LordMathis Date: Tue, 7 Oct 2025 19:57:21 +0200 Subject: [PATCH 15/34] Refactor CreateRemoteInstance and UpdateRemoteInstance to directly use options parameter in API requests --- pkg/manager/remote_ops.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/manager/remote_ops.go b/pkg/manager/remote_ops.go index 5050737..e7b3dbb 100644 --- a/pkg/manager/remote_ops.go +++ b/pkg/manager/remote_ops.go @@ -83,11 +83,8 @@ func (im *instanceManager) ListRemoteInstances(nodeConfig *config.NodeConfig) ([ // CreateRemoteInstance creates a new instance on the remote node func (im *instanceManager) CreateRemoteInstance(nodeConfig *config.NodeConfig, name string, options *instance.CreateInstanceOptions) (*instance.Process, error) { path := fmt.Sprintf("/api/v1/instances/%s/", name) - payload := map[string]any{ - "options": options, - } - resp, err := im.makeRemoteRequest(nodeConfig, "POST", path, payload) + resp, err := im.makeRemoteRequest(nodeConfig, "POST", path, options) if err != nil { return nil, err } @@ -119,11 +116,8 @@ func (im *instanceManager) GetRemoteInstance(nodeConfig *config.NodeConfig, name // UpdateRemoteInstance updates an existing instance on the remote node func (im *instanceManager) UpdateRemoteInstance(nodeConfig *config.NodeConfig, name string, options *instance.CreateInstanceOptions) (*instance.Process, error) { path := fmt.Sprintf("/api/v1/instances/%s/", name) - payload := map[string]any{ - "options": options, - } - resp, err := im.makeRemoteRequest(nodeConfig, "PUT", path, payload) + resp, err := im.makeRemoteRequest(nodeConfig, "PUT", path, options) if err != nil { return nil, err } From 3418735204c34b1ee1530c0a05ebe5c4b4cd25e1 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Tue, 7 Oct 2025 20:27:31 +0200 Subject: [PATCH 16/34] Add stripNodesFromOptions function to prevent routing loops in remote requests --- pkg/manager/remote_ops.go | 21 ++++++++++++++++++ pkg/manager/remote_ops_test.go | 39 ++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 pkg/manager/remote_ops_test.go diff --git a/pkg/manager/remote_ops.go b/pkg/manager/remote_ops.go index e7b3dbb..49b24f1 100644 --- a/pkg/manager/remote_ops.go +++ b/pkg/manager/remote_ops.go @@ -10,10 +10,31 @@ import ( "net/http" ) +// stripNodesFromOptions creates a copy of the instance options without the Nodes field +// to prevent routing loops when sending requests to remote nodes +func (im *instanceManager) stripNodesFromOptions(options *instance.CreateInstanceOptions) *instance.CreateInstanceOptions { + if options == nil { + return nil + } + + // Create a copy of the options struct + optionsCopy := *options + + // Clear the Nodes field to prevent the remote node from trying to route further + optionsCopy.Nodes = nil + + return &optionsCopy +} + // makeRemoteRequest is a helper function to make HTTP requests to a remote node func (im *instanceManager) makeRemoteRequest(nodeConfig *config.NodeConfig, method, path string, body any) (*http.Response, error) { var reqBody io.Reader if body != nil { + // Strip nodes from CreateInstanceOptions to prevent routing loops + if options, ok := body.(*instance.CreateInstanceOptions); ok { + body = im.stripNodesFromOptions(options) + } + jsonData, err := json.Marshal(body) if err != nil { return nil, fmt.Errorf("failed to marshal request body: %w", err) diff --git a/pkg/manager/remote_ops_test.go b/pkg/manager/remote_ops_test.go new file mode 100644 index 0000000..94db40b --- /dev/null +++ b/pkg/manager/remote_ops_test.go @@ -0,0 +1,39 @@ +package manager + +import ( + "llamactl/pkg/backends" + "llamactl/pkg/instance" + "testing" +) + +func TestStripNodesFromOptions(t *testing.T) { + im := &instanceManager{} + + // Test nil case + if result := im.stripNodesFromOptions(nil); result != nil { + t.Errorf("Expected nil, got %+v", result) + } + + // Test main case: nodes should be stripped, other fields preserved + options := &instance.CreateInstanceOptions{ + BackendType: backends.BackendTypeLlamaCpp, + Nodes: []string{"node1", "node2"}, + Environment: map[string]string{"TEST": "value"}, + } + + result := im.stripNodesFromOptions(options) + + if result.Nodes != nil { + t.Errorf("Expected Nodes to be nil, got %+v", result.Nodes) + } + if result.BackendType != backends.BackendTypeLlamaCpp { + t.Errorf("Expected BackendType preserved") + } + if result.Environment["TEST"] != "value" { + t.Errorf("Expected Environment preserved") + } + // Original should not be modified + if len(options.Nodes) != 2 { + t.Errorf("Original options should not be modified") + } +} From 7f6725da9655026af7602c17efdc517de23d8f5d Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 8 Oct 2025 19:24:24 +0200 Subject: [PATCH 17/34] Refactor NodeConfig handling to use a map --- pkg/config/config.go | 3 +-- pkg/manager/manager.go | 7 ++++--- pkg/manager/manager_test.go | 12 ++++++------ pkg/manager/operations_test.go | 2 +- pkg/manager/timeout_test.go | 2 +- pkg/server/handlers_instances.go | 12 ++---------- pkg/server/handlers_nodes.go | 27 ++++++++------------------- pkg/server/handlers_openai.go | 12 ++---------- 8 files changed, 25 insertions(+), 52 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 77637b8..6b34b63 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -41,7 +41,7 @@ type AppConfig struct { Backends BackendConfig `yaml:"backends"` Instances InstancesConfig `yaml:"instances"` Auth AuthConfig `yaml:"auth"` - Nodes []NodeConfig `yaml:"nodes,omitempty"` + Nodes map[string]NodeConfig `yaml:"nodes,omitempty"` Version string `yaml:"-"` CommitHash string `yaml:"-"` BuildTime string `yaml:"-"` @@ -130,7 +130,6 @@ type AuthConfig struct { } type NodeConfig struct { - Name string `yaml:"name"` Address string `yaml:"address"` APIKey string `yaml:"api_key,omitempty"` } diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index a5a7138..37989db 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -63,15 +63,16 @@ type instanceManager struct { } // NewInstanceManager creates a new instance of InstanceManager. -func NewInstanceManager(backendsConfig config.BackendConfig, instancesConfig config.InstancesConfig, nodesConfig []config.NodeConfig) InstanceManager { +func NewInstanceManager(backendsConfig config.BackendConfig, instancesConfig config.InstancesConfig, nodesConfig map[string]config.NodeConfig) InstanceManager { if instancesConfig.TimeoutCheckInterval <= 0 { instancesConfig.TimeoutCheckInterval = 5 // Default to 5 minutes if not set } // Build node config map for quick lookup nodeConfigMap := make(map[string]*config.NodeConfig) - for i := range nodesConfig { - nodeConfigMap[nodesConfig[i].Name] = &nodesConfig[i] + for name := range nodesConfig { + nodeCopy := nodesConfig[name] + nodeConfigMap[name] = &nodeCopy } im := &instanceManager{ diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index 5110e52..e59e2eb 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -34,7 +34,7 @@ func TestNewInstanceManager(t *testing.T) { TimeoutCheckInterval: 5, } - mgr := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) + mgr := manager.NewInstanceManager(backendConfig, cfg, map[string]config.NodeConfig{}) if mgr == nil { t.Fatal("NewInstanceManager returned nil") } @@ -69,7 +69,7 @@ func TestPersistence(t *testing.T) { } // Test instance persistence on creation - manager1 := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) + manager1 := manager.NewInstanceManager(backendConfig, cfg, map[string]config.NodeConfig{}) options := &instance.CreateInstanceOptions{ BackendType: backends.BackendTypeLlamaCpp, LlamaServerOptions: &llamacpp.LlamaServerOptions{ @@ -90,7 +90,7 @@ func TestPersistence(t *testing.T) { } // Test loading instances from disk - manager2 := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) + manager2 := manager.NewInstanceManager(backendConfig, cfg, map[string]config.NodeConfig{}) instances, err := manager2.ListInstances() if err != nil { t.Fatalf("ListInstances failed: %v", err) @@ -207,7 +207,7 @@ func createTestManager() manager.InstanceManager { DefaultRestartDelay: 5, TimeoutCheckInterval: 5, } - return manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) + return manager.NewInstanceManager(backendConfig, cfg, map[string]config.NodeConfig{}) } func TestAutoRestartDisabledInstanceStatus(t *testing.T) { @@ -227,7 +227,7 @@ func TestAutoRestartDisabledInstanceStatus(t *testing.T) { } // Create first manager and instance with auto-restart disabled - manager1 := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) + manager1 := manager.NewInstanceManager(backendConfig, cfg, map[string]config.NodeConfig{}) autoRestart := false options := &instance.CreateInstanceOptions{ @@ -252,7 +252,7 @@ func TestAutoRestartDisabledInstanceStatus(t *testing.T) { manager1.Shutdown() // Create second manager (simulating restart of llamactl) - manager2 := manager.NewInstanceManager(backendConfig, cfg, []config.NodeConfig{}) + manager2 := manager.NewInstanceManager(backendConfig, cfg, map[string]config.NodeConfig{}) // Get the loaded instance loadedInst, err := manager2.GetInstance("test-instance") diff --git a/pkg/manager/operations_test.go b/pkg/manager/operations_test.go index da26742..fdeb44f 100644 --- a/pkg/manager/operations_test.go +++ b/pkg/manager/operations_test.go @@ -75,7 +75,7 @@ func TestCreateInstance_ValidationAndLimits(t *testing.T) { MaxInstances: 1, // Very low limit for testing TimeoutCheckInterval: 5, } - limitedManager := manager.NewInstanceManager(backendConfig, cfg, nil) + limitedManager := manager.NewInstanceManager(backendConfig, cfg, map[string]config.NodeConfig{}) _, err = limitedManager.CreateInstance("instance1", options) if err != nil { diff --git a/pkg/manager/timeout_test.go b/pkg/manager/timeout_test.go index 31b4298..55cd781 100644 --- a/pkg/manager/timeout_test.go +++ b/pkg/manager/timeout_test.go @@ -23,7 +23,7 @@ func TestTimeoutFunctionality(t *testing.T) { MaxInstances: 5, } - manager := manager.NewInstanceManager(backendConfig, cfg, nil) + manager := manager.NewInstanceManager(backendConfig, cfg, map[string]config.NodeConfig{}) if manager == nil { t.Fatal("Manager should be initialized with timeout checker") } diff --git a/pkg/server/handlers_instances.go b/pkg/server/handlers_instances.go index 8c325ae..88e974f 100644 --- a/pkg/server/handlers_instances.go +++ b/pkg/server/handlers_instances.go @@ -3,7 +3,6 @@ package server import ( "encoding/json" "fmt" - "llamactl/pkg/config" "llamactl/pkg/instance" "llamactl/pkg/manager" "net/http" @@ -414,15 +413,8 @@ func (h *Handler) RemoteInstanceProxy(w http.ResponseWriter, r *http.Request, na if !exists { // Find node configuration - var nodeConfig *config.NodeConfig - for i := range h.cfg.Nodes { - if h.cfg.Nodes[i].Name == nodeName { - nodeConfig = &h.cfg.Nodes[i] - break - } - } - - if nodeConfig == nil { + nodeConfig, exists := h.cfg.Nodes[nodeName] + if !exists { http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) return } diff --git a/pkg/server/handlers_nodes.go b/pkg/server/handlers_nodes.go index 7d1116f..98a4b43 100644 --- a/pkg/server/handlers_nodes.go +++ b/pkg/server/handlers_nodes.go @@ -2,7 +2,6 @@ package server import ( "encoding/json" - "llamactl/pkg/config" "net/http" "github.com/go-chi/chi/v5" @@ -10,26 +9,24 @@ import ( // NodeResponse represents a sanitized node configuration for API responses type NodeResponse struct { - Name string `json:"name"` Address string `json:"address"` } // ListNodes godoc // @Summary List all configured nodes -// @Description Returns a list of all nodes configured in the server +// @Description Returns a map of all nodes configured in the server (node name -> node config) // @Tags nodes // @Security ApiKeyAuth // @Produces json -// @Success 200 {array} NodeResponse "List of nodes" +// @Success 200 {object} map[string]NodeResponse "Map of nodes" // @Failure 500 {string} string "Internal Server Error" // @Router /nodes [get] func (h *Handler) ListNodes() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // Convert to sanitized response format - nodeResponses := make([]NodeResponse, len(h.cfg.Nodes)) - for i, node := range h.cfg.Nodes { - nodeResponses[i] = NodeResponse{ - Name: node.Name, + // Convert to sanitized response format (map of name -> NodeResponse) + nodeResponses := make(map[string]NodeResponse, len(h.cfg.Nodes)) + for name, node := range h.cfg.Nodes { + nodeResponses[name] = NodeResponse{ Address: node.Address, } } @@ -62,22 +59,14 @@ func (h *Handler) GetNode() http.HandlerFunc { return } - var nodeConfig *config.NodeConfig - for i := range h.cfg.Nodes { - if h.cfg.Nodes[i].Name == name { - nodeConfig = &h.cfg.Nodes[i] - break - } - } - - if nodeConfig == nil { + nodeConfig, exists := h.cfg.Nodes[name] + if !exists { http.Error(w, "Node not found", http.StatusNotFound) return } // Convert to sanitized response format nodeResponse := NodeResponse{ - Name: nodeConfig.Name, Address: nodeConfig.Address, } diff --git a/pkg/server/handlers_openai.go b/pkg/server/handlers_openai.go index eea8440..c6e56e9 100644 --- a/pkg/server/handlers_openai.go +++ b/pkg/server/handlers_openai.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "llamactl/pkg/config" "llamactl/pkg/instance" "net/http" "net/http/httputil" @@ -170,15 +169,8 @@ func (h *Handler) RemoteOpenAIProxy(w http.ResponseWriter, r *http.Request, mode if !exists { // Find node configuration - var nodeConfig *config.NodeConfig - for i := range h.cfg.Nodes { - if h.cfg.Nodes[i].Name == nodeName { - nodeConfig = &h.cfg.Nodes[i] - break - } - } - - if nodeConfig == nil { + nodeConfig, exists := h.cfg.Nodes[nodeName] + if !exists { http.Error(w, fmt.Sprintf("Node %s not found", nodeName), http.StatusInternalServerError) return } From 688b815ca7585357ccb7156c58d6111be5810809 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 8 Oct 2025 19:43:53 +0200 Subject: [PATCH 18/34] Add LocalNode configuration --- pkg/config/config.go | 24 ++++--- pkg/config/config_test.go | 129 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 7 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 6b34b63..d6ee420 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -37,14 +37,15 @@ type BackendConfig struct { // AppConfig represents the configuration for llamactl type AppConfig struct { - Server ServerConfig `yaml:"server"` - Backends BackendConfig `yaml:"backends"` - Instances InstancesConfig `yaml:"instances"` - Auth AuthConfig `yaml:"auth"` + Server ServerConfig `yaml:"server"` + Backends BackendConfig `yaml:"backends"` + Instances InstancesConfig `yaml:"instances"` + Auth AuthConfig `yaml:"auth"` + LocalNode string `yaml:"local_node,omitempty"` Nodes map[string]NodeConfig `yaml:"nodes,omitempty"` - Version string `yaml:"-"` - CommitHash string `yaml:"-"` - BuildTime string `yaml:"-"` + Version string `yaml:"-"` + CommitHash string `yaml:"-"` + BuildTime string `yaml:"-"` } // ServerConfig contains HTTP server configuration @@ -148,6 +149,10 @@ func LoadConfig(configPath string) (AppConfig, error) { AllowedHeaders: []string{"*"}, // Default to allow all headers EnableSwagger: false, }, + LocalNode: "main", + Nodes: map[string]NodeConfig{ + "main": {}, // Local node with empty config + }, Backends: BackendConfig{ LlamaCpp: BackendSettings{ Command: "llama-server", @@ -475,6 +480,11 @@ func loadEnvVars(cfg *AppConfig) { if managementKeys := os.Getenv("LLAMACTL_MANAGEMENT_KEYS"); managementKeys != "" { cfg.Auth.ManagementKeys = strings.Split(managementKeys, ",") } + + // Local node config + if localNode := os.Getenv("LLAMACTL_LOCAL_NODE"); localNode != "" { + cfg.LocalNode = localNode + } } // ParsePortRange parses port range from string formats like "8000-9000" or "8000,9000" diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index ad800ed..964708e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -510,3 +510,132 @@ func TestGetBackendSettings_InvalidBackendType(t *testing.T) { t.Errorf("Expected empty command for invalid backend, got %q", settings.Command) } } + +func TestLoadConfig_LocalNode(t *testing.T) { + t.Run("default local node", func(t *testing.T) { + cfg, err := config.LoadConfig("nonexistent-file.yaml") + if err != nil { + t.Fatalf("LoadConfig failed: %v", err) + } + + if cfg.LocalNode != "main" { + t.Errorf("Expected default local node 'main', got %q", cfg.LocalNode) + } + }) + + t.Run("local node from file", func(t *testing.T) { + tempDir := t.TempDir() + configFile := filepath.Join(tempDir, "test-config.yaml") + + configContent := ` +local_node: "worker1" +nodes: + worker1: + address: "" + worker2: + address: "http://192.168.1.10:8080" + api_key: "test-key" +` + + err := os.WriteFile(configFile, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to write test config file: %v", err) + } + + cfg, err := config.LoadConfig(configFile) + if err != nil { + t.Fatalf("LoadConfig failed: %v", err) + } + + if cfg.LocalNode != "worker1" { + t.Errorf("Expected local node 'worker1', got %q", cfg.LocalNode) + } + + // Verify nodes map (includes default "main" + worker1 + worker2) + if len(cfg.Nodes) != 3 { + t.Errorf("Expected 3 nodes (default main + worker1 + worker2), got %d", len(cfg.Nodes)) + } + + // Verify local node exists and is empty + localNode, exists := cfg.Nodes["worker1"] + if !exists { + t.Error("Expected local node 'worker1' to exist in nodes map") + } + if localNode.Address != "" { + t.Errorf("Expected local node address to be empty, got %q", localNode.Address) + } + if localNode.APIKey != "" { + t.Errorf("Expected local node api_key to be empty, got %q", localNode.APIKey) + } + + // Verify remote node + remoteNode, exists := cfg.Nodes["worker2"] + if !exists { + t.Error("Expected remote node 'worker2' to exist in nodes map") + } + if remoteNode.Address != "http://192.168.1.10:8080" { + t.Errorf("Expected remote node address 'http://192.168.1.10:8080', got %q", remoteNode.Address) + } + + // Verify default main node still exists + _, exists = cfg.Nodes["main"] + if !exists { + t.Error("Expected default 'main' node to still exist in nodes map") + } + }) + + t.Run("custom local node name in config", func(t *testing.T) { + tempDir := t.TempDir() + configFile := filepath.Join(tempDir, "test-config.yaml") + + configContent := ` +local_node: "primary" +nodes: + primary: + address: "" + worker1: + address: "http://192.168.1.10:8080" +` + + err := os.WriteFile(configFile, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to write test config file: %v", err) + } + + cfg, err := config.LoadConfig(configFile) + if err != nil { + t.Fatalf("LoadConfig failed: %v", err) + } + + if cfg.LocalNode != "primary" { + t.Errorf("Expected local node 'primary', got %q", cfg.LocalNode) + } + + // Verify nodes map includes default "main" + primary + worker1 + if len(cfg.Nodes) != 3 { + t.Errorf("Expected 3 nodes (default main + primary + worker1), got %d", len(cfg.Nodes)) + } + + localNode, exists := cfg.Nodes["primary"] + if !exists { + t.Error("Expected local node 'primary' to exist in nodes map") + } + if localNode.Address != "" { + t.Errorf("Expected local node address to be empty, got %q", localNode.Address) + } + }) + + t.Run("local node from environment variable", func(t *testing.T) { + os.Setenv("LLAMACTL_LOCAL_NODE", "custom-node") + defer os.Unsetenv("LLAMACTL_LOCAL_NODE") + + cfg, err := config.LoadConfig("nonexistent-file.yaml") + if err != nil { + t.Fatalf("LoadConfig failed: %v", err) + } + + if cfg.LocalNode != "custom-node" { + t.Errorf("Expected local node 'custom-node' from env var, got %q", cfg.LocalNode) + } + }) +} From 56b95d1243b072914ca7bcc5ee9b56b36c24b452 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 8 Oct 2025 19:52:39 +0200 Subject: [PATCH 19/34] Refactor InstanceSettingsCard and API types to use NodesMap --- .../instance/InstanceSettingsCard.tsx | 17 +++++++---------- webui/src/lib/api.ts | 7 ++++--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/webui/src/components/instance/InstanceSettingsCard.tsx b/webui/src/components/instance/InstanceSettingsCard.tsx index 369b8a6..cd5f1e1 100644 --- a/webui/src/components/instance/InstanceSettingsCard.tsx +++ b/webui/src/components/instance/InstanceSettingsCard.tsx @@ -8,7 +8,7 @@ import NumberInput from '@/components/form/NumberInput' import CheckboxInput from '@/components/form/CheckboxInput' import EnvironmentVariablesInput from '@/components/form/EnvironmentVariablesInput' import SelectInput from '@/components/form/SelectInput' -import { nodesApi, type NodeResponse } from '@/lib/api' +import { nodesApi, type NodesMap } from '@/lib/api' interface InstanceSettingsCardProps { instanceName: string @@ -27,7 +27,7 @@ const InstanceSettingsCard: React.FC = ({ onNameChange, onChange }) => { - const [nodes, setNodes] = useState([]) + const [nodes, setNodes] = useState({}) const [loadingNodes, setLoadingNodes] = useState(true) useEffect(() => { @@ -45,13 +45,10 @@ const InstanceSettingsCard: React.FC = ({ void fetchNodes() }, []) - const nodeOptions = [ - { value: '', label: 'Default (Main Node)' }, - ...nodes.map(node => ({ - value: node.name, - label: node.name - })) - ] + const nodeOptions = Object.keys(nodes).map(nodeName => ({ + value: nodeName, + label: nodeName + })) const handleNodeChange = (value: string | undefined) => { if (value) { @@ -89,7 +86,7 @@ const InstanceSettingsCard: React.FC = ({ {/* Node Selection */} - {!loadingNodes && nodes.length > 0 && ( + {!loadingNodes && Object.keys(nodes).length > 0 && ( ; + // Node API functions export const nodesApi = { - // GET /nodes - list: () => apiCall("/nodes"), + // GET /nodes - returns map of node name to NodeResponse + list: () => apiCall("/nodes"), // GET /nodes/{name} get: (name: string) => apiCall(`/nodes/${name}`), From 5d958ed283d1ee006e34d2b07313db7410d72464 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 18:38:33 +0200 Subject: [PATCH 20/34] Fix backend_options cleanup to exclude empty arrays in InstanceDialog --- webui/src/components/InstanceDialog.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/src/components/InstanceDialog.tsx b/webui/src/components/InstanceDialog.tsx index d9b731c..324f694 100644 --- a/webui/src/components/InstanceDialog.tsx +++ b/webui/src/components/InstanceDialog.tsx @@ -106,7 +106,7 @@ const InstanceDialog: React.FC = ({ // Clean up undefined values to avoid sending empty fields const cleanOptions: CreateInstanceOptions = {}; Object.entries(formData).forEach(([key, value]) => { - if (key === 'backend_options' && value && typeof value === 'object') { + if (key === 'backend_options' && value && typeof value === 'object' && !Array.isArray(value)) { // Handle backend_options specially - clean nested object const cleanBackendOptions: any = {}; Object.entries(value).forEach(([backendKey, backendValue]) => { @@ -118,7 +118,7 @@ const InstanceDialog: React.FC = ({ cleanBackendOptions[backendKey] = backendValue; } }); - + // Only include backend_options if it has content if (Object.keys(cleanBackendOptions).length > 0) { (cleanOptions as any)[key] = cleanBackendOptions; From 6c1a76691d9ca9af41538c7b2d0e7df8845700c8 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 18:49:36 +0200 Subject: [PATCH 21/34] Improve cleanup of options in InstanceDialog to skip empty strings and arrays --- webui/src/components/InstanceDialog.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/webui/src/components/InstanceDialog.tsx b/webui/src/components/InstanceDialog.tsx index 324f694..4a54f7a 100644 --- a/webui/src/components/InstanceDialog.tsx +++ b/webui/src/components/InstanceDialog.tsx @@ -123,8 +123,12 @@ const InstanceDialog: React.FC = ({ if (Object.keys(cleanBackendOptions).length > 0) { (cleanOptions as any)[key] = cleanBackendOptions; } - } else if (value !== undefined && value !== null && (typeof value !== 'string' || value.trim() !== "")) { - // Handle arrays - don't include empty arrays + } else if (value !== undefined && value !== null) { + // Skip empty strings + if (typeof value === 'string' && value.trim() === "") { + return; + } + // Skip empty arrays if (Array.isArray(value) && value.length === 0) { return; } From 8d9b0c062103066368449953c611cd5316d6fa73 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 18:56:12 +0200 Subject: [PATCH 22/34] Initialize timeProvider and logger in UnmarshalJSON for Process --- pkg/instance/instance.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index dee38ff..6d33f66 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -285,6 +285,14 @@ func (i *Process) UnmarshalJSON(data []byte) error { i.options = aux.Options } + // Initialize fields that are not serialized + if i.timeProvider == nil { + i.timeProvider = realTimeProvider{} + } + if i.logger == nil && i.globalInstanceSettings != nil { + i.logger = NewInstanceLogger(i.Name, i.globalInstanceSettings.LogsDir) + } + return nil } From e281708b20d0d156391700e520437137a2e02b26 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 18:56:23 +0200 Subject: [PATCH 23/34] Enhance auto-start logic to differentiate between remote and local instances --- pkg/manager/manager.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 37989db..56df75a 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -345,8 +345,18 @@ func (im *instanceManager) autoStartInstances() { log.Printf("Auto-starting instance %s", inst.Name) // Reset running state before starting (since Start() expects stopped instance) inst.SetStatus(instance.Stopped) - if err := inst.Start(); err != nil { - log.Printf("Failed to auto-start instance %s: %v", inst.Name, err) + + // Check if this is a remote instance + if node := im.getNodeForInstance(inst); node != nil { + // Remote instance - use StartRemoteInstance + if _, err := im.StartRemoteInstance(node, inst.Name); err != nil { + log.Printf("Failed to auto-start remote instance %s: %v", inst.Name, err) + } + } else { + // Local instance - call Start() directly + if err := inst.Start(); err != nil { + log.Printf("Failed to auto-start instance %s: %v", inst.Name, err) + } } } } From 9d5f01d4aeeb5dc85c66a00851edaf2e92a08ceb Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 19:13:58 +0200 Subject: [PATCH 24/34] Auto-select first node in InstanceSettingsCard if none is selected --- webui/src/components/instance/InstanceSettingsCard.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/webui/src/components/instance/InstanceSettingsCard.tsx b/webui/src/components/instance/InstanceSettingsCard.tsx index cd5f1e1..a89ee90 100644 --- a/webui/src/components/instance/InstanceSettingsCard.tsx +++ b/webui/src/components/instance/InstanceSettingsCard.tsx @@ -35,6 +35,12 @@ const InstanceSettingsCard: React.FC = ({ try { const fetchedNodes = await nodesApi.list() setNodes(fetchedNodes) + + // Auto-select first node if none selected + const nodeNames = Object.keys(fetchedNodes) + if (nodeNames.length > 0 && (!formData.nodes || formData.nodes.length === 0)) { + onChange('nodes', [nodeNames[0]]) + } } catch (error) { console.error('Failed to fetch nodes:', error) } finally { @@ -43,6 +49,7 @@ const InstanceSettingsCard: React.FC = ({ } void fetchNodes() + // eslint-disable-next-line react-hooks/exhaustive-deps }, []) const nodeOptions = Object.keys(nodes).map(nodeName => ({ From 9684a8a09bf609628132ffd43c77a84c4ac25470 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 19:34:52 +0200 Subject: [PATCH 25/34] Enhance instance management to preserve local state for remote instances --- pkg/manager/operations.go | 59 ++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/pkg/manager/operations.go b/pkg/manager/operations.go index f1e5929..3bce1e6 100644 --- a/pkg/manager/operations.go +++ b/pkg/manager/operations.go @@ -62,12 +62,20 @@ func (im *instanceManager) CreateInstance(name string, options *instance.CreateI return nil, fmt.Errorf("node %s not found", nodeName) } - // Create the remote instance - inst, err := im.CreateRemoteInstance(nodeConfig, name, options) + // Create the remote instance on the remote node + remoteInst, err := im.CreateRemoteInstance(nodeConfig, name, options) if err != nil { return nil, err } + // Create a local stub that preserves the Nodes field for tracking + // We keep the original options (with Nodes) so IsRemote() works correctly + inst := instance.NewInstance(name, &im.backendsConfig, &im.instancesConfig, options, nil) + + // Copy the status and creation time from the remote instance + inst.Status = remoteInst.Status + inst.Created = remoteInst.Created + // Add to local tracking maps (but don't count towards limits) im.instances[name] = inst im.instanceNodeMap[name] = nodeConfig @@ -137,24 +145,25 @@ func (im *instanceManager) UpdateInstance(name string, options *instance.CreateI // Check if instance is remote and delegate to remote operation if node := im.getNodeForInstance(inst); node != nil { - updatedInst, err := im.UpdateRemoteInstance(node, name, options) + remoteInst, err := im.UpdateRemoteInstance(node, name, options) if err != nil { return nil, err } - // Update local tracking + // Update the local instance's fields (preserving Nodes field) im.mu.Lock() - im.instances[name] = updatedInst + inst.SetOptions(options) // Update options with the original (including Nodes) + inst.Status = remoteInst.Status im.mu.Unlock() // Persist the updated remote instance locally im.mu.Lock() defer im.mu.Unlock() - if err := im.persistInstance(updatedInst); err != nil { + if err := im.persistInstance(inst); err != nil { return nil, fmt.Errorf("failed to persist updated remote instance %s: %w", name, err) } - return updatedInst, nil + return inst, nil } if options == nil { @@ -259,7 +268,17 @@ func (im *instanceManager) StartInstance(name string) (*instance.Process, error) // Check if instance is remote and delegate to remote operation if node := im.getNodeForInstance(inst); node != nil { - return im.StartRemoteInstance(node, name) + remoteInst, err := im.StartRemoteInstance(node, name) + if err != nil { + return nil, err + } + + // Update the local instance's status to match the remote + im.mu.Lock() + inst.Status = remoteInst.Status + im.mu.Unlock() + + return inst, nil } if inst.IsRunning() { @@ -318,7 +337,17 @@ func (im *instanceManager) StopInstance(name string) (*instance.Process, error) // Check if instance is remote and delegate to remote operation if node := im.getNodeForInstance(inst); node != nil { - return im.StopRemoteInstance(node, name) + remoteInst, err := im.StopRemoteInstance(node, name) + if err != nil { + return nil, err + } + + // Update the local instance's status to match the remote + im.mu.Lock() + inst.Status = remoteInst.Status + im.mu.Unlock() + + return inst, nil } if !inst.IsRunning() { @@ -351,7 +380,17 @@ func (im *instanceManager) RestartInstance(name string) (*instance.Process, erro // Check if instance is remote and delegate to remote operation if node := im.getNodeForInstance(inst); node != nil { - return im.RestartRemoteInstance(node, name) + remoteInst, err := im.RestartRemoteInstance(node, name) + if err != nil { + return nil, err + } + + // Update the local instance's status to match the remote + im.mu.Lock() + inst.Status = remoteInst.Status + im.mu.Unlock() + + return inst, nil } inst, err := im.StopInstance(name) From 8a16a195de23af364ce10ef01ccf0e1148211748 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 20:22:32 +0200 Subject: [PATCH 26/34] Fix getting remote instance logs --- pkg/manager/manager.go | 4 +-- pkg/manager/operations.go | 52 +++++++++++++++++++++++++------- pkg/manager/remote_ops.go | 4 +-- pkg/server/handlers_instances.go | 25 ++++++--------- 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 56df75a..b944ef3 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -26,7 +26,7 @@ type InstanceManager interface { StopInstance(name string) (*instance.Process, error) EvictLRUInstance() error RestartInstance(name string) (*instance.Process, error) - GetInstanceLogs(name string) (string, error) + GetInstanceLogs(name string, numLines int) (string, error) Shutdown() } @@ -39,7 +39,7 @@ type RemoteManager interface { StartRemoteInstance(node *config.NodeConfig, name string) (*instance.Process, error) StopRemoteInstance(node *config.NodeConfig, name string) (*instance.Process, error) RestartRemoteInstance(node *config.NodeConfig, name string) (*instance.Process, error) - GetRemoteInstanceLogs(node *config.NodeConfig, name string) (string, error) + GetRemoteInstanceLogs(node *config.NodeConfig, name string, numLines int) (string, error) } type instanceManager struct { diff --git a/pkg/manager/operations.go b/pkg/manager/operations.go index 3bce1e6..2d95ef1 100644 --- a/pkg/manager/operations.go +++ b/pkg/manager/operations.go @@ -13,15 +13,33 @@ import ( type MaxRunningInstancesError error // ListInstances returns a list of all instances managed by the instance manager. +// For remote instances, this fetches the live state from remote nodes and updates local stubs. func (im *instanceManager) ListInstances() ([]*instance.Process, error) { im.mu.RLock() - defer im.mu.RUnlock() - - instances := make([]*instance.Process, 0, len(im.instances)) + localInstances := make([]*instance.Process, 0, len(im.instances)) for _, inst := range im.instances { - instances = append(instances, inst) + localInstances = append(localInstances, inst) } - return instances, nil + im.mu.RUnlock() + + // Update remote instances with live state + for _, inst := range localInstances { + if node := im.getNodeForInstance(inst); node != nil { + remoteInst, err := im.GetRemoteInstance(node, inst.Name) + if err != nil { + // Log error but continue with stale data + // Don't fail the entire list operation due to one remote failure + continue + } + + // Update the local stub's status to reflect remote state + im.mu.Lock() + inst.Status = remoteInst.Status + im.mu.Unlock() + } + } + + return localInstances, nil } // CreateInstance creates a new instance with the given options and returns it. @@ -115,6 +133,7 @@ func (im *instanceManager) CreateInstance(name string, options *instance.CreateI } // GetInstance retrieves an instance by its name. +// For remote instances, this fetches the live state from the remote node and updates the local stub. func (im *instanceManager) GetInstance(name string) (*instance.Process, error) { im.mu.RLock() inst, exists := im.instances[name] @@ -124,9 +143,20 @@ func (im *instanceManager) GetInstance(name string) (*instance.Process, error) { return nil, fmt.Errorf("instance with name %s not found", name) } - // Check if instance is remote and delegate to remote operation + // Check if instance is remote and fetch live state if node := im.getNodeForInstance(inst); node != nil { - return im.GetRemoteInstance(node, name) + remoteInst, err := im.GetRemoteInstance(node, name) + if err != nil { + return nil, err + } + + // Update the local stub's status to reflect remote state + im.mu.Lock() + inst.Status = remoteInst.Status + im.mu.Unlock() + + // Return the local stub (preserving Nodes field) + return inst, nil } return inst, nil @@ -401,7 +431,7 @@ func (im *instanceManager) RestartInstance(name string) (*instance.Process, erro } // GetInstanceLogs retrieves the logs for a specific instance by its name. -func (im *instanceManager) GetInstanceLogs(name string) (string, error) { +func (im *instanceManager) GetInstanceLogs(name string, numLines int) (string, error) { im.mu.RLock() inst, exists := im.instances[name] im.mu.RUnlock() @@ -412,11 +442,11 @@ func (im *instanceManager) GetInstanceLogs(name string) (string, error) { // Check if instance is remote and delegate to remote operation if node := im.getNodeForInstance(inst); node != nil { - return im.GetRemoteInstanceLogs(node, name) + return im.GetRemoteInstanceLogs(node, name, numLines) } - // TODO: Implement actual log retrieval logic - return fmt.Sprintf("Logs for instance %s", name), nil + // Get logs from the local instance + return inst.GetLogs(numLines) } // getPortFromOptions extracts the port from backend-specific options diff --git a/pkg/manager/remote_ops.go b/pkg/manager/remote_ops.go index 49b24f1..40b2384 100644 --- a/pkg/manager/remote_ops.go +++ b/pkg/manager/remote_ops.go @@ -211,8 +211,8 @@ func (im *instanceManager) RestartRemoteInstance(nodeConfig *config.NodeConfig, } // GetRemoteInstanceLogs retrieves logs for an instance from the remote node -func (im *instanceManager) GetRemoteInstanceLogs(nodeConfig *config.NodeConfig, name string) (string, error) { - path := fmt.Sprintf("/api/v1/instances/%s/logs", name) +func (im *instanceManager) GetRemoteInstanceLogs(nodeConfig *config.NodeConfig, name string, numLines int) (string, error) { + path := fmt.Sprintf("/api/v1/instances/%s/logs?lines=%d", name, numLines) resp, err := im.makeRemoteRequest(nodeConfig, "GET", path, nil) if err != nil { return "", err diff --git a/pkg/server/handlers_instances.go b/pkg/server/handlers_instances.go index 88e974f..be3cf4a 100644 --- a/pkg/server/handlers_instances.go +++ b/pkg/server/handlers_instances.go @@ -308,23 +308,18 @@ func (h *Handler) GetInstanceLogs() http.HandlerFunc { } lines := r.URL.Query().Get("lines") - if lines == "" { - lines = "-1" + numLines := -1 // Default to all lines + if lines != "" { + parsedLines, err := strconv.Atoi(lines) + if err != nil { + http.Error(w, "Invalid lines parameter: "+err.Error(), http.StatusBadRequest) + return + } + numLines = parsedLines } - num_lines, err := strconv.Atoi(lines) - if err != nil { - http.Error(w, "Invalid lines parameter: "+err.Error(), http.StatusBadRequest) - return - } - - inst, err := h.InstanceManager.GetInstance(name) - if err != nil { - http.Error(w, "Failed to get instance: "+err.Error(), http.StatusInternalServerError) - return - } - - logs, err := inst.GetLogs(num_lines) + // Use the instance manager which handles both local and remote instances + logs, err := h.InstanceManager.GetInstanceLogs(name, numLines) if err != nil { http.Error(w, "Failed to get logs: "+err.Error(), http.StatusInternalServerError) return From b965b77c1898f6d65e205f79c7c2f73a5cb807c7 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 20:24:54 +0200 Subject: [PATCH 27/34] Prevent remote instances from using local proxy in GetProxy method --- pkg/instance/instance.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index 6d33f66..dcebef4 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -171,6 +171,11 @@ func (i *Process) GetProxy() (*httputil.ReverseProxy, error) { return nil, fmt.Errorf("instance %s has no options set", i.Name) } + // Remote instances should not use local proxy - they are handled by RemoteInstanceProxy + if len(i.options.Nodes) > 0 { + return nil, fmt.Errorf("instance %s is a remote instance and should not use local proxy", i.Name) + } + var host string var port int switch i.options.BackendType { From 2b950ee6497a5078b330e3c6486ceab7bb78de3a Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 20:39:21 +0200 Subject: [PATCH 28/34] Implement updateLocalInstanceFromRemote to preserve Nodes field when syncing remote instance data --- pkg/manager/operations.go | 61 +++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/pkg/manager/operations.go b/pkg/manager/operations.go index 2d95ef1..938e542 100644 --- a/pkg/manager/operations.go +++ b/pkg/manager/operations.go @@ -12,6 +12,37 @@ import ( type MaxRunningInstancesError error +// updateLocalInstanceFromRemote updates the local stub instance with data from the remote instance +// while preserving the Nodes field to maintain remote instance tracking +func (im *instanceManager) updateLocalInstanceFromRemote(localInst *instance.Process, remoteInst *instance.Process) { + if localInst == nil || remoteInst == nil { + return + } + + // Get the remote instance options + remoteOptions := remoteInst.GetOptions() + if remoteOptions == nil { + return + } + + // Preserve the Nodes field from the local instance + localOptions := localInst.GetOptions() + var preservedNodes []string + if localOptions != nil && len(localOptions.Nodes) > 0 { + preservedNodes = make([]string, len(localOptions.Nodes)) + copy(preservedNodes, localOptions.Nodes) + } + + // Create a copy of remote options and restore the Nodes field + updatedOptions := *remoteOptions + updatedOptions.Nodes = preservedNodes + + // Update the local instance with all remote data + localInst.SetOptions(&updatedOptions) + localInst.Status = remoteInst.Status + localInst.Created = remoteInst.Created +} + // ListInstances returns a list of all instances managed by the instance manager. // For remote instances, this fetches the live state from remote nodes and updates local stubs. func (im *instanceManager) ListInstances() ([]*instance.Process, error) { @@ -32,9 +63,9 @@ func (im *instanceManager) ListInstances() ([]*instance.Process, error) { continue } - // Update the local stub's status to reflect remote state + // Update the local stub with all remote data (preserving Nodes) im.mu.Lock() - inst.Status = remoteInst.Status + im.updateLocalInstanceFromRemote(inst, remoteInst) im.mu.Unlock() } } @@ -90,9 +121,8 @@ func (im *instanceManager) CreateInstance(name string, options *instance.CreateI // We keep the original options (with Nodes) so IsRemote() works correctly inst := instance.NewInstance(name, &im.backendsConfig, &im.instancesConfig, options, nil) - // Copy the status and creation time from the remote instance - inst.Status = remoteInst.Status - inst.Created = remoteInst.Created + // Update the local stub with all remote data (preserving Nodes) + im.updateLocalInstanceFromRemote(inst, remoteInst) // Add to local tracking maps (but don't count towards limits) im.instances[name] = inst @@ -150,9 +180,9 @@ func (im *instanceManager) GetInstance(name string) (*instance.Process, error) { return nil, err } - // Update the local stub's status to reflect remote state + // Update the local stub with all remote data (preserving Nodes) im.mu.Lock() - inst.Status = remoteInst.Status + im.updateLocalInstanceFromRemote(inst, remoteInst) im.mu.Unlock() // Return the local stub (preserving Nodes field) @@ -180,10 +210,9 @@ func (im *instanceManager) UpdateInstance(name string, options *instance.CreateI return nil, err } - // Update the local instance's fields (preserving Nodes field) + // Update the local stub with all remote data (preserving Nodes) im.mu.Lock() - inst.SetOptions(options) // Update options with the original (including Nodes) - inst.Status = remoteInst.Status + im.updateLocalInstanceFromRemote(inst, remoteInst) im.mu.Unlock() // Persist the updated remote instance locally @@ -303,9 +332,9 @@ func (im *instanceManager) StartInstance(name string) (*instance.Process, error) return nil, err } - // Update the local instance's status to match the remote + // Update the local stub with all remote data (preserving Nodes) im.mu.Lock() - inst.Status = remoteInst.Status + im.updateLocalInstanceFromRemote(inst, remoteInst) im.mu.Unlock() return inst, nil @@ -372,9 +401,9 @@ func (im *instanceManager) StopInstance(name string) (*instance.Process, error) return nil, err } - // Update the local instance's status to match the remote + // Update the local stub with all remote data (preserving Nodes) im.mu.Lock() - inst.Status = remoteInst.Status + im.updateLocalInstanceFromRemote(inst, remoteInst) im.mu.Unlock() return inst, nil @@ -415,9 +444,9 @@ func (im *instanceManager) RestartInstance(name string) (*instance.Process, erro return nil, err } - // Update the local instance's status to match the remote + // Update the local stub with all remote data (preserving Nodes) im.mu.Lock() - inst.Status = remoteInst.Status + im.updateLocalInstanceFromRemote(inst, remoteInst) im.mu.Unlock() return inst, nil From e7a6a7003e8e242f63e99a14363b3a1174a318d5 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 21:13:38 +0200 Subject: [PATCH 29/34] Skip remote instances in checkAllTimeouts and EvictLRUInstance methods --- pkg/manager/timeout.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/manager/timeout.go b/pkg/manager/timeout.go index 0ee9c11..50b1c10 100644 --- a/pkg/manager/timeout.go +++ b/pkg/manager/timeout.go @@ -12,6 +12,11 @@ func (im *instanceManager) checkAllTimeouts() { // Identify instances that should timeout for _, inst := range im.instances { + // Skip remote instances - they are managed by their respective nodes + if inst.IsRemote() { + continue + } + if inst.ShouldTimeout() { timeoutInstances = append(timeoutInstances, inst.Name) } @@ -40,6 +45,11 @@ func (im *instanceManager) EvictLRUInstance() error { continue } + // Skip remote instances - they are managed by their respective nodes + if inst.IsRemote() { + continue + } + if inst.GetOptions() != nil && inst.GetOptions().IdleTimeout != nil && *inst.GetOptions().IdleTimeout <= 0 { continue // Skip instances without idle timeout } From ab2770bdd919beb278fd55ea9042786773225cad Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 21:50:39 +0200 Subject: [PATCH 30/34] Add documentation for remote node deployment and configuration --- README.md | 7 ++++- docs/getting-started/configuration.md | 38 ++++++++++++++++++++------- docs/getting-started/installation.md | 8 ++++++ docs/user-guide/api-reference.md | 33 +++++++++++++++++++++++ docs/user-guide/managing-instances.md | 24 +++++++++++++---- docs/user-guide/troubleshooting.md | 24 +++++++++++++++++ 6 files changed, 118 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0f27290..b452ebe 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,12 @@ ### ⚡ Smart Operations - **Instance Monitoring**: Health checks, auto-restart, log management - **Smart Resource Management**: Idle timeout, LRU eviction, and configurable instance limits -- **Environment Variables**: Set custom environment variables per instance for advanced configuration +- **Environment Variables**: Set custom environment variables per instance for advanced configuration + +### 🔗 Remote Instance Deployment +- **Remote Node Support**: Deploy instances on remote hosts +- **Central Management**: Manage remote instances from a single dashboard +- **Seamless Routing**: Automatic request routing to remote instances ![Dashboard Screenshot](docs/images/dashboard.png) diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index be4fc6d..c43efc6 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -70,6 +70,10 @@ auth: inference_keys: [] # Keys for inference endpoints require_management_auth: true # Require auth for management endpoints management_keys: [] # Keys for management endpoints + +local_node: "main" # Name of the local node (default: "main") +nodes: # Node configuration for multi-node deployment + main: # Default local node (empty config) ``` ## Configuration Files @@ -235,18 +239,32 @@ auth: management_keys: [] # List of valid management API keys ``` -**Environment Variables:** -- `LLAMACTL_REQUIRE_INFERENCE_AUTH` - Require auth for OpenAI endpoints (true/false) -- `LLAMACTL_INFERENCE_KEYS` - Comma-separated inference API keys -- `LLAMACTL_REQUIRE_MANAGEMENT_AUTH` - Require auth for management endpoints (true/false) -- `LLAMACTL_MANAGEMENT_KEYS` - Comma-separated management API keys +**Environment Variables:** +- `LLAMACTL_REQUIRE_INFERENCE_AUTH` - Require auth for OpenAI endpoints (true/false) +- `LLAMACTL_INFERENCE_KEYS` - Comma-separated inference API keys +- `LLAMACTL_REQUIRE_MANAGEMENT_AUTH` - Require auth for management endpoints (true/false) +- `LLAMACTL_MANAGEMENT_KEYS` - Comma-separated management API keys -## Command Line Options +### Remote Node Configuration -View all available command line options: +llamactl supports remote node deployments. Configure remote nodes to deploy instances on remote hosts and manage them centrally. -```bash -llamactl --help +```yaml +local_node: "main" # Name of the local node (default: "main") +nodes: # Node configuration map + main: # Local node (empty address means local) + address: "" # Not used for local node + api_key: "" # Not used for local node + worker1: # Remote worker node + address: "http://192.168.1.10:8080" + api_key: "worker1-api-key" # Management API key for authentication ``` -You can also override configuration using command line flags when starting llamactl. +**Node Configuration Fields:** +- `local_node`: Specifies which node in the `nodes` map represents the local node +- `nodes`: Map of node configurations + - `address`: HTTP/HTTPS URL of the remote node (empty for local node) + - `api_key`: Management API key for authenticating with the remote node + +**Environment Variables:** +- `LLAMACTL_LOCAL_NODE` - Name of the local node diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index f64146f..04e0dfd 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -157,6 +157,12 @@ cd webui && npm ci && npm run build && cd .. go build -o llamactl ./cmd/server ``` +## Remote Node Installation + +For deployments with remote nodes: +- Install llamactl on each node using any of the methods above +- Configure API keys for authentication between nodes + ## Verification Verify your installation by checking the version: @@ -168,3 +174,5 @@ llamactl --version ## Next Steps Now that Llamactl is installed, continue to the [Quick Start](quick-start.md) guide to get your first instance running! + +For remote node deployments, see the [Configuration Guide](configuration.md) for node setup instructions. diff --git a/docs/user-guide/api-reference.md b/docs/user-guide/api-reference.md index 26e01e4..472cd0b 100644 --- a/docs/user-guide/api-reference.md +++ b/docs/user-guide/api-reference.md @@ -126,6 +126,7 @@ POST /api/v1/instances/{name} - `on_demand_start`: Start instance when receiving requests - `idle_timeout`: Idle timeout in minutes - `environment`: Environment variables as key-value pairs +- `nodes`: Array with single node name to deploy the instance to (for remote deployments) See [Managing Instances](managing-instances.md) for complete configuration options. @@ -405,6 +406,38 @@ curl -X DELETE -H "Authorization: Bearer your-api-key" \ http://localhost:8080/api/v1/instances/my-model ``` +### Remote Node Instance Example + +```bash +# Create instance on specific remote node +curl -X POST http://localhost:8080/api/v1/instances/remote-model \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-api-key" \ + -d '{ + "backend_type": "llama_cpp", + "backend_options": { + "model": "/models/llama-2-7b.gguf", + "gpu_layers": 32 + }, + "nodes": ["worker1"] + }' + +# Check status of remote instance +curl -H "Authorization: Bearer your-api-key" \ + http://localhost:8080/api/v1/instances/remote-model + +# Use remote instance with OpenAI-compatible API +curl -X POST http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-inference-api-key" \ + -d '{ + "model": "remote-model", + "messages": [ + {"role": "user", "content": "Hello from remote node!"} + ] + }' +``` + ### Using the Proxy Endpoint You can also directly proxy requests to the llama-server instance: diff --git a/docs/user-guide/managing-instances.md b/docs/user-guide/managing-instances.md index 824c4fe..b02de2d 100644 --- a/docs/user-guide/managing-instances.md +++ b/docs/user-guide/managing-instances.md @@ -39,26 +39,27 @@ Each instance is displayed as a card showing: 1. Click the **"Create Instance"** button on the dashboard 2. Enter a unique **Name** for your instance (only required field) -3. **Choose Backend Type**: +3. **Select Target Node**: Choose which node to deploy the instance to from the dropdown +4. **Choose Backend Type**: - **llama.cpp**: For GGUF models using llama-server - **MLX**: For MLX-optimized models (macOS only) - **vLLM**: For distributed serving and high-throughput inference -4. Configure model source: +5. Configure model source: - **For llama.cpp**: GGUF model path or HuggingFace repo - **For MLX**: MLX model path or identifier (e.g., `mlx-community/Mistral-7B-Instruct-v0.3-4bit`) - **For vLLM**: HuggingFace model identifier (e.g., `microsoft/DialoGPT-medium`) -5. Configure optional instance management settings: +6. Configure optional instance management settings: - **Auto Restart**: Automatically restart instance on failure - **Max Restarts**: Maximum number of restart attempts - **Restart Delay**: Delay in seconds between restart attempts - **On Demand Start**: Start instance when receiving a request to the OpenAI compatible endpoint - **Idle Timeout**: Minutes before stopping idle instance (set to 0 to disable) - **Environment Variables**: Set custom environment variables for the instance process -6. Configure backend-specific options: +7. Configure backend-specific options: - **llama.cpp**: Threads, context size, GPU layers, port, etc. - **MLX**: Temperature, top-p, adapter path, Python environment, etc. - **vLLM**: Tensor parallel size, GPU memory utilization, quantization, etc. -7. Click **"Create"** to save the instance +8. Click **"Create"** to save the instance ### Via API @@ -121,6 +122,18 @@ curl -X POST http://localhost:8080/api/instances/gemma-3-27b \ "gpu_layers": 32 } }' + +# Create instance on specific remote node +curl -X POST http://localhost:8080/api/instances/remote-llama \ + -H "Content-Type: application/json" \ + -d '{ + "backend_type": "llama_cpp", + "backend_options": { + "model": "/models/llama-7b.gguf", + "gpu_layers": 32 + }, + "nodes": ["worker1"] + }' ``` ## Start Instance @@ -227,3 +240,4 @@ Check the health status of your instances: ```bash curl http://localhost:8080/api/instances/{name}/proxy/health ``` + diff --git a/docs/user-guide/troubleshooting.md b/docs/user-guide/troubleshooting.md index 5608139..4b7a507 100644 --- a/docs/user-guide/troubleshooting.md +++ b/docs/user-guide/troubleshooting.md @@ -125,6 +125,30 @@ This helps determine if the issue is with llamactl or with the underlying llama. http://localhost:8080/api/v1/instances ``` +## Remote Node Issues + +### Node Configuration + +**Problem:** Remote instances not appearing or cannot be managed + +**Solutions:** +1. **Verify node configuration:** + ```yaml + local_node: "main" # Must match a key in nodes map + nodes: + main: + address: "" # Empty for local node + worker1: + address: "http://worker1.internal:8080" + api_key: "secure-key" # Must match worker1's management key + ``` + +2. **Test remote node connectivity:** + ```bash + curl -H "Authorization: Bearer remote-node-key" \ + http://remote-node:8080/api/v1/instances + ``` + ## Debugging and Logs ### Viewing Instance Logs From f61e8dad5cca2e4b4f85c498094f23e78f22c258 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 21:51:38 +0200 Subject: [PATCH 31/34] Add User Docs badge to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b452ebe..d9fea15 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # llamactl -![Build and Release](https://github.com/lordmathis/llamactl/actions/workflows/release.yaml/badge.svg) ![Go Tests](https://github.com/lordmathis/llamactl/actions/workflows/go_test.yaml/badge.svg) ![WebUI Tests](https://github.com/lordmathis/llamactl/actions/workflows/webui_test.yaml/badge.svg) +![Build and Release](https://github.com/lordmathis/llamactl/actions/workflows/release.yaml/badge.svg) ![Go Tests](https://github.com/lordmathis/llamactl/actions/workflows/go_test.yaml/badge.svg) ![WebUI Tests](https://github.com/lordmathis/llamactl/actions/workflows/webui_test.yaml/badge.svg) ![User Docs](https://github.com/lordmathis/llamactl/actions/workflows/docs.yaml/badge.svg) **Unified management and routing for llama.cpp, MLX and vLLM models with web dashboard.** From 73b9dd5bc752b017ce5eedb71d443594891da2d3 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 21:53:14 +0200 Subject: [PATCH 32/34] Rename workflows for consistency --- .github/workflows/{codeql.yml => codeql.yaml} | 0 .github/workflows/{docs.yml => docs.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{codeql.yml => codeql.yaml} (100%) rename .github/workflows/{docs.yml => docs.yaml} (100%) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yaml similarity index 100% rename from .github/workflows/codeql.yml rename to .github/workflows/codeql.yaml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yaml similarity index 100% rename from .github/workflows/docs.yml rename to .github/workflows/docs.yaml From 5436c28a1f94a4b7f952761af94591a4256cb972 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 22:10:40 +0200 Subject: [PATCH 33/34] Add instance name validation before deletion for security --- pkg/manager/operations.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/manager/operations.go b/pkg/manager/operations.go index 938e542..0d38a73 100644 --- a/pkg/manager/operations.go +++ b/pkg/manager/operations.go @@ -287,6 +287,10 @@ func (im *instanceManager) DeleteInstance(name string) error { delete(im.instanceNodeMap, name) // Delete the instance's config file if persistence is enabled + // Re-validate instance name for security (defense in depth) + if _, err := validation.ValidateInstanceName(name); err != nil { + return fmt.Errorf("invalid instance name for file deletion: %w", err) + } instancePath := filepath.Join(im.instancesConfig.InstancesDir, name+".json") if err := os.Remove(instancePath); err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to delete config file for remote instance %s: %w", name, err) @@ -306,6 +310,10 @@ func (im *instanceManager) DeleteInstance(name string) error { delete(im.instances, name) // Delete the instance's config file if persistence is enabled + // Re-validate instance name for security (defense in depth) + if _, err := validation.ValidateInstanceName(inst.Name); err != nil { + return fmt.Errorf("invalid instance name for file deletion: %w", err) + } instancePath := filepath.Join(im.instancesConfig.InstancesDir, inst.Name+".json") if err := os.Remove(instancePath); err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to delete config file for instance %s: %w", inst.Name, err) From 9ee0a184b3efd37aeca180a23db31f956d32b91e Mon Sep 17 00:00:00 2001 From: LordMathis Date: Thu, 9 Oct 2025 22:18:53 +0200 Subject: [PATCH 34/34] Re-validate instance name in DeleteInstance for improved security --- pkg/manager/operations.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/manager/operations.go b/pkg/manager/operations.go index 0d38a73..a8b5c3f 100644 --- a/pkg/manager/operations.go +++ b/pkg/manager/operations.go @@ -288,12 +288,13 @@ func (im *instanceManager) DeleteInstance(name string) error { // Delete the instance's config file if persistence is enabled // Re-validate instance name for security (defense in depth) - if _, err := validation.ValidateInstanceName(name); err != nil { + validatedName, err := validation.ValidateInstanceName(name) + if err != nil { return fmt.Errorf("invalid instance name for file deletion: %w", err) } - instancePath := filepath.Join(im.instancesConfig.InstancesDir, name+".json") + instancePath := filepath.Join(im.instancesConfig.InstancesDir, validatedName+".json") if err := os.Remove(instancePath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to delete config file for remote instance %s: %w", name, err) + return fmt.Errorf("failed to delete config file for remote instance %s: %w", validatedName, err) } return nil @@ -311,12 +312,13 @@ func (im *instanceManager) DeleteInstance(name string) error { // Delete the instance's config file if persistence is enabled // Re-validate instance name for security (defense in depth) - if _, err := validation.ValidateInstanceName(inst.Name); err != nil { + validatedName, err := validation.ValidateInstanceName(inst.Name) + if err != nil { return fmt.Errorf("invalid instance name for file deletion: %w", err) } - instancePath := filepath.Join(im.instancesConfig.InstancesDir, inst.Name+".json") + instancePath := filepath.Join(im.instancesConfig.InstancesDir, validatedName+".json") if err := os.Remove(instancePath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to delete config file for instance %s: %w", inst.Name, err) + return fmt.Errorf("failed to delete config file for instance %s: %w", validatedName, err) } return nil