diff --git a/cmd/server/main.go b/cmd/server/main.go index 7433c78..e245ebf 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -58,7 +58,7 @@ func main() { } // Initialize the instance manager - instanceManager := manager.NewInstanceManager(cfg.Instances) + instanceManager := manager.NewInstanceManager(cfg.Backends, cfg.Instances) // Create a new handler with the instance manager handler := server.NewHandler(instanceManager, cfg) diff --git a/pkg/config/config.go b/pkg/config/config.go index 1b873a5..3c476d5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,9 +17,6 @@ type BackendConfig struct { // Path to mlx_lm executable (MLX-LM backend) MLXLMExecutable string `yaml:"mlx_lm_executable"` - - // Optional: Default Python virtual environment path for MLX backends - MLXPythonPath string `yaml:"mlx_python_path,omitempty"` } // AppConfig represents the configuration for llamactl @@ -128,7 +125,6 @@ func LoadConfig(configPath string) (AppConfig, error) { Backends: BackendConfig{ LlamaExecutable: "llama-server", MLXLMExecutable: "mlx_lm.server", - MLXPythonPath: "", // Empty means use system Python }, Instances: InstancesConfig{ PortRange: [2]int{8000, 9000}, @@ -250,14 +246,10 @@ func loadEnvVars(cfg *AppConfig) { // Backend config if llamaExec := os.Getenv("LLAMACTL_LLAMA_EXECUTABLE"); llamaExec != "" { cfg.Backends.LlamaExecutable = llamaExec - cfg.Instances.LlamaExecutable = llamaExec // Keep for backward compatibility } if mlxLMExec := os.Getenv("LLAMACTL_MLX_LM_EXECUTABLE"); mlxLMExec != "" { cfg.Backends.MLXLMExecutable = mlxLMExec } - if mlxPython := os.Getenv("LLAMACTL_MLX_PYTHON_PATH"); mlxPython != "" { - cfg.Backends.MLXPythonPath = mlxPython - } if autoRestart := os.Getenv("LLAMACTL_DEFAULT_AUTO_RESTART"); autoRestart != "" { if b, err := strconv.ParseBool(autoRestart); err == nil { cfg.Instances.DefaultAutoRestart = b diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index fc5089c..c0e5060 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -31,9 +31,10 @@ func (realTimeProvider) Now() time.Time { // Process represents a running instance of the llama server type Process struct { - Name string `json:"name"` - options *CreateInstanceOptions `json:"-"` - globalSettings *config.InstancesConfig + Name string `json:"name"` + options *CreateInstanceOptions `json:"-"` + globalInstanceSettings *config.InstancesConfig + globalBackendSettings *config.BackendConfig // Status Status InstanceStatus `json:"status"` @@ -65,22 +66,23 @@ type Process struct { } // NewInstance creates a new instance with the given name, log path, and options -func NewInstance(name string, globalSettings *config.InstancesConfig, options *CreateInstanceOptions, onStatusChange func(oldStatus, newStatus InstanceStatus)) *Process { +func NewInstance(name string, globalBackendSettings *config.BackendConfig, globalInstanceSettings *config.InstancesConfig, options *CreateInstanceOptions, onStatusChange func(oldStatus, newStatus InstanceStatus)) *Process { // Validate and copy options - options.ValidateAndApplyDefaults(name, globalSettings) + options.ValidateAndApplyDefaults(name, globalInstanceSettings) // Create the instance logger - logger := NewInstanceLogger(name, globalSettings.LogsDir) + logger := NewInstanceLogger(name, globalInstanceSettings.LogsDir) return &Process{ - Name: name, - options: options, - globalSettings: globalSettings, - logger: logger, - timeProvider: realTimeProvider{}, - Created: time.Now().Unix(), - Status: Stopped, - onStatusChange: onStatusChange, + Name: name, + options: options, + globalInstanceSettings: globalInstanceSettings, + globalBackendSettings: globalBackendSettings, + logger: logger, + timeProvider: realTimeProvider{}, + Created: time.Now().Unix(), + Status: Stopped, + onStatusChange: onStatusChange, } } @@ -96,7 +98,13 @@ func (i *Process) GetPort() int { if i.options != nil { switch i.options.BackendType { case backends.BackendTypeLlamaCpp: - return i.options.LlamaServerOptions.Port + if i.options.LlamaServerOptions != nil { + return i.options.LlamaServerOptions.Port + } + case backends.BackendTypeMlxLm: + if i.options.MlxServerOptions != nil { + return i.options.MlxServerOptions.Port + } } } return 0 @@ -108,7 +116,13 @@ func (i *Process) GetHost() string { if i.options != nil { switch i.options.BackendType { case backends.BackendTypeLlamaCpp: - return i.options.LlamaServerOptions.Host + if i.options.LlamaServerOptions != nil { + return i.options.LlamaServerOptions.Host + } + case backends.BackendTypeMlxLm: + if i.options.MlxServerOptions != nil { + return i.options.MlxServerOptions.Host + } } } return "" @@ -124,7 +138,7 @@ func (i *Process) SetOptions(options *CreateInstanceOptions) { } // Validate and copy options - options.ValidateAndApplyDefaults(i.Name, i.globalSettings) + options.ValidateAndApplyDefaults(i.Name, i.globalInstanceSettings) i.options = options // Clear the proxy so it gets recreated with new options @@ -153,8 +167,15 @@ func (i *Process) GetProxy() (*httputil.ReverseProxy, error) { var port int switch i.options.BackendType { case backends.BackendTypeLlamaCpp: - host = i.options.LlamaServerOptions.Host - port = i.options.LlamaServerOptions.Port + if i.options.LlamaServerOptions != nil { + host = i.options.LlamaServerOptions.Host + port = i.options.LlamaServerOptions.Port + } + case backends.BackendTypeMlxLm: + if i.options.MlxServerOptions != nil { + host = i.options.MlxServerOptions.Host + port = i.options.MlxServerOptions.Port + } } targetURL, err := url.Parse(fmt.Sprintf("http://%s:%d", host, port)) @@ -215,7 +236,7 @@ func (i *Process) UnmarshalJSON(data []byte) error { // Handle options with validation and defaults if aux.Options != nil { - aux.Options.ValidateAndApplyDefaults(i.Name, i.globalSettings) + aux.Options.ValidateAndApplyDefaults(i.Name, i.globalInstanceSettings) i.options = aux.Options } diff --git a/pkg/instance/lifecycle.go b/pkg/instance/lifecycle.go index 28a65b9..04c5fba 100644 --- a/pkg/instance/lifecycle.go +++ b/pkg/instance/lifecycle.go @@ -9,6 +9,8 @@ import ( "runtime" "syscall" "time" + + "llamactl/pkg/backends" ) // Start starts the llama server instance and returns an error if it fails. @@ -41,7 +43,20 @@ func (i *Process) Start() error { args := i.options.BuildCommandArgs() i.ctx, i.cancel = context.WithCancel(context.Background()) - i.cmd = exec.CommandContext(i.ctx, "llama-server", args...) + + var executable string + + // Get executable from global configuration + switch i.options.BackendType { + case backends.BackendTypeLlamaCpp: + executable = i.globalBackendSettings.LlamaExecutable + case backends.BackendTypeMlxLm: + executable = i.globalBackendSettings.MLXLMExecutable + default: + return fmt.Errorf("unsupported backend type: %s", i.options.BackendType) + } + + i.cmd = exec.CommandContext(i.ctx, executable, args...) if runtime.GOOS != "windows" { setProcAttrs(i.cmd) @@ -175,9 +190,16 @@ func (i *Process) WaitForHealthy(timeout int) error { var host string var port int switch opts.BackendType { - case "llama-cpp": - host = opts.LlamaServerOptions.Host - port = opts.LlamaServerOptions.Port + case backends.BackendTypeLlamaCpp: + if opts.LlamaServerOptions != nil { + host = opts.LlamaServerOptions.Host + port = opts.LlamaServerOptions.Port + } + case backends.BackendTypeMlxLm: + if opts.MlxServerOptions != nil { + host = opts.MlxServerOptions.Host + port = opts.MlxServerOptions.Port + } } if host == "" { host = "localhost" diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 80652a8..6999643 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -35,6 +35,7 @@ type instanceManager struct { runningInstances map[string]struct{} ports map[int]bool instancesConfig config.InstancesConfig + backendsConfig config.BackendConfig // Timeout checker timeoutChecker *time.Ticker @@ -44,7 +45,7 @@ type instanceManager struct { } // NewInstanceManager creates a new instance of InstanceManager. -func NewInstanceManager(instancesConfig config.InstancesConfig) InstanceManager { +func NewInstanceManager(backendsConfig config.BackendConfig, instancesConfig config.InstancesConfig) InstanceManager { if instancesConfig.TimeoutCheckInterval <= 0 { instancesConfig.TimeoutCheckInterval = 5 // Default to 5 minutes if not set } @@ -53,6 +54,7 @@ func NewInstanceManager(instancesConfig config.InstancesConfig) InstanceManager runningInstances: make(map[string]struct{}), ports: make(map[int]bool), instancesConfig: instancesConfig, + backendsConfig: backendsConfig, timeoutChecker: time.NewTicker(time.Duration(instancesConfig.TimeoutCheckInterval) * time.Minute), shutdownChan: make(chan struct{}), @@ -241,7 +243,7 @@ func (im *instanceManager) loadInstance(name, path string) error { } // Create new inst using NewInstance (handles validation, defaults, setup) - inst := instance.NewInstance(name, &im.instancesConfig, persistedInstance.GetOptions(), statusCallback) + inst := instance.NewInstance(name, &im.backendsConfig, &im.instancesConfig, persistedInstance.GetOptions(), statusCallback) // Restore persisted fields that NewInstance doesn't set inst.Created = persistedInstance.Created