diff --git a/pkg/backends/builder.go b/pkg/backends/builder.go index 23c3bb1..d5b5c0c 100644 --- a/pkg/backends/builder.go +++ b/pkg/backends/builder.go @@ -1,6 +1,8 @@ package backends import ( + "fmt" + "llamactl/pkg/config" "reflect" "strconv" "strings" @@ -68,3 +70,24 @@ func BuildCommandArgs(options any, multipleFlags map[string]bool) []string { return args } + +// BuildDockerCommand builds a Docker command with the specified configuration and arguments +func BuildDockerCommand(backendConfig *config.BackendSettings, instanceArgs []string) (string, []string, error) { + // Start with configured Docker arguments (should include "run", "--rm", etc.) + dockerArgs := make([]string, len(backendConfig.Docker.Args)) + copy(dockerArgs, backendConfig.Docker.Args) + + // Add environment variables + for key, value := range backendConfig.Docker.Environment { + dockerArgs = append(dockerArgs, "-e", fmt.Sprintf("%s=%s", key, value)) + } + + // Add image name + dockerArgs = append(dockerArgs, backendConfig.Docker.Image) + + // Add backend args and instance args + dockerArgs = append(dockerArgs, backendConfig.Args...) + dockerArgs = append(dockerArgs, instanceArgs...) + + return "docker", dockerArgs, nil +} diff --git a/pkg/backends/llamacpp/llama.go b/pkg/backends/llamacpp/llama.go index 970077f..bca29e8 100644 --- a/pkg/backends/llamacpp/llama.go +++ b/pkg/backends/llamacpp/llama.go @@ -1,16 +1,34 @@ package llamacpp import ( - "context" "encoding/json" - "fmt" "llamactl/pkg/backends" - "llamactl/pkg/config" - "os/exec" "reflect" "strconv" ) +// multiValuedFlags defines flags that should be repeated for each value rather than comma-separated +// Used for both parsing (with underscores) and building (with dashes) +var multiValuedFlags = map[string]bool{ + // Parsing keys (with underscores) + "override_tensor": true, + "override_kv": true, + "lora": true, + "lora_scaled": true, + "control_vector": true, + "control_vector_scaled": true, + "dry_sequence_breaker": true, + "logit_bias": true, + // Building keys (with dashes) + "override-tensor": true, + "override-kv": true, + "lora-scaled": true, + "control-vector": true, + "control-vector-scaled": true, + "dry-sequence-breaker": true, + "logit-bias": true, +} + type LlamaServerOptions struct { // Common params VerbosePrompt bool `json:"verbose_prompt,omitempty"` @@ -320,67 +338,13 @@ func (o *LlamaServerOptions) UnmarshalJSON(data []byte) error { // BuildCommandArgs converts InstanceOptions to command line arguments func (o *LlamaServerOptions) BuildCommandArgs() []string { // Llama uses multiple flags for arrays by default (not comma-separated) - multipleFlags := map[string]bool{ - "override-tensor": true, - "override-kv": true, - "lora": true, - "lora-scaled": true, - "control-vector": true, - "control-vector-scaled": true, - "dry-sequence-breaker": true, - "logit-bias": true, - } - return backends.BuildCommandArgs(o, multipleFlags) + // Use package-level multiValuedFlags variable + return backends.BuildCommandArgs(o, multiValuedFlags) } -// BuildCommandArgsWithDocker converts InstanceOptions to command line arguments, -// handling Docker transformations if needed -func (o *LlamaServerOptions) BuildCommandArgsWithDocker(dockerImage string) []string { - args := o.BuildCommandArgs() - - // No special Docker transformations needed for llama-cpp - return args -} - -// BuildCommand creates the complete command for execution, handling Docker vs native execution -func (o *LlamaServerOptions) BuildCommand(ctx context.Context, backendConfig *config.BackendSettings) (*exec.Cmd, error) { - // Build instance-specific arguments using backend functions - var instanceArgs []string - if backendConfig.Docker != nil && backendConfig.Docker.Enabled { - // Use Docker-aware argument building - instanceArgs = o.BuildCommandArgsWithDocker(backendConfig.Docker.Image) - } else { - // Use regular argument building for native execution - instanceArgs = o.BuildCommandArgs() - } - - // Combine backend args with instance args - finalArgs := append(backendConfig.Args, instanceArgs...) - - // Choose Docker vs Native execution - if backendConfig.Docker != nil && backendConfig.Docker.Enabled { - return buildDockerCommand(ctx, backendConfig, finalArgs) - } else { - return exec.CommandContext(ctx, backendConfig.Command, finalArgs...), nil - } -} - -// buildDockerCommand builds a Docker command with the specified configuration and arguments -func buildDockerCommand(ctx context.Context, backendConfig *config.BackendSettings, args []string) (*exec.Cmd, error) { - // Start with configured Docker arguments (should include "run", "--rm", etc.) - dockerArgs := make([]string, len(backendConfig.Docker.Args)) - copy(dockerArgs, backendConfig.Docker.Args) - - // Add environment variables - for key, value := range backendConfig.Docker.Environment { - dockerArgs = append(dockerArgs, "-e", fmt.Sprintf("%s=%s", key, value)) - } - - // Add image and container arguments - dockerArgs = append(dockerArgs, backendConfig.Docker.Image) - dockerArgs = append(dockerArgs, args...) - - return exec.CommandContext(ctx, "docker", dockerArgs...), nil +func (o *LlamaServerOptions) BuildDockerArgs() []string { + // For llama, Docker args are the same as normal args + return o.BuildCommandArgs() } // ParseLlamaCommand parses a llama-server command string into LlamaServerOptions @@ -392,16 +356,7 @@ func buildDockerCommand(ctx context.Context, backendConfig *config.BackendSettin func ParseLlamaCommand(command string) (*LlamaServerOptions, error) { executableNames := []string{"llama-server"} var subcommandNames []string // Llama has no subcommands - multiValuedFlags := map[string]bool{ - "override_tensor": true, - "override_kv": true, - "lora": true, - "lora_scaled": true, - "control_vector": true, - "control_vector_scaled": true, - "dry_sequence_breaker": true, - "logit_bias": true, - } + // Use package-level multiValuedFlags variable var llamaOptions LlamaServerOptions if err := backends.ParseCommand(command, executableNames, subcommandNames, multiValuedFlags, &llamaOptions); err != nil { diff --git a/pkg/backends/mlx/mlx.go b/pkg/backends/mlx/mlx.go index 62d8e68..3b83681 100644 --- a/pkg/backends/mlx/mlx.go +++ b/pkg/backends/mlx/mlx.go @@ -1,11 +1,7 @@ package mlx import ( - "context" - "fmt" "llamactl/pkg/backends" - "llamactl/pkg/config" - "os/exec" ) type MlxServerOptions struct { @@ -40,56 +36,6 @@ func (o *MlxServerOptions) BuildCommandArgs() []string { return backends.BuildCommandArgs(o, multipleFlags) } -// BuildCommandArgsWithDocker converts to command line arguments, -// handling Docker transformations if needed -func (o *MlxServerOptions) BuildCommandArgsWithDocker(dockerImage string) []string { - args := o.BuildCommandArgs() - - // No special Docker transformations needed for MLX - return args -} - -// BuildCommand creates the complete command for execution, handling Docker vs native execution -func (o *MlxServerOptions) BuildCommand(ctx context.Context, backendConfig *config.BackendSettings) (*exec.Cmd, error) { - // Build instance-specific arguments using backend functions - var instanceArgs []string - if backendConfig.Docker != nil && backendConfig.Docker.Enabled { - // Use Docker-aware argument building - instanceArgs = o.BuildCommandArgsWithDocker(backendConfig.Docker.Image) - } else { - // Use regular argument building for native execution - instanceArgs = o.BuildCommandArgs() - } - - // Combine backend args with instance args - finalArgs := append(backendConfig.Args, instanceArgs...) - - // Choose Docker vs Native execution - if backendConfig.Docker != nil && backendConfig.Docker.Enabled { - return buildDockerCommand(ctx, backendConfig, finalArgs) - } else { - return exec.CommandContext(ctx, backendConfig.Command, finalArgs...), nil - } -} - -// buildDockerCommand builds a Docker command with the specified configuration and arguments -func buildDockerCommand(ctx context.Context, backendConfig *config.BackendSettings, args []string) (*exec.Cmd, error) { - // Start with configured Docker arguments (should include "run", "--rm", etc.) - dockerArgs := make([]string, len(backendConfig.Docker.Args)) - copy(dockerArgs, backendConfig.Docker.Args) - - // Add environment variables - for key, value := range backendConfig.Docker.Environment { - dockerArgs = append(dockerArgs, "-e", fmt.Sprintf("%s=%s", key, value)) - } - - // Add image and container arguments - dockerArgs = append(dockerArgs, backendConfig.Docker.Image) - dockerArgs = append(dockerArgs, args...) - - return exec.CommandContext(ctx, "docker", dockerArgs...), nil -} - // ParseMlxCommand parses a mlx_lm.server command string into MlxServerOptions // Supports multiple formats: // 1. Full command: "mlx_lm.server --model model/path" diff --git a/pkg/backends/vllm/vllm.go b/pkg/backends/vllm/vllm.go index b4436c2..d4fee25 100644 --- a/pkg/backends/vllm/vllm.go +++ b/pkg/backends/vllm/vllm.go @@ -1,15 +1,18 @@ package vllm import ( - "context" - "fmt" - "llamactl/pkg/config" - "os/exec" - "strings" - "llamactl/pkg/backends" ) +// multiValuedFlags defines flags that should be repeated for each value rather than comma-separated +var multiValuedFlags = map[string]bool{ + "api-key": true, + "allowed-origins": true, + "allowed-methods": true, + "allowed-headers": true, + "middleware": true, +} + type VllmServerOptions struct { // Basic connection options (auto-assigned by llamactl) Host string `json:"host,omitempty"` @@ -137,104 +140,37 @@ type VllmServerOptions struct { } // BuildCommandArgs converts VllmServerOptions to command line arguments -// Note: This does NOT include the "serve" subcommand, that's handled at the instance level -// For vLLM, the model parameter is passed as a positional argument, not a --model flag +// For vLLM native, model is a positional argument after "serve" func (o *VllmServerOptions) BuildCommandArgs() []string { var args []string - // Add model as positional argument if specified + // Add model as positional argument if specified (for native execution) if o.Model != "" { args = append(args, o.Model) } - // Create a copy of the options without the Model field to avoid including it as --model flag + // Create a copy without Model field to avoid --model flag optionsCopy := *o - optionsCopy.Model = "" // Clear model field so it won't be included as a flag + optionsCopy.Model = "" - multipleFlags := map[string]bool{ - "api-key": true, - "allowed-origins": true, - "allowed-methods": true, - "allowed-headers": true, - "middleware": true, - } + // Use package-level multipleFlags variable - // Build the rest of the arguments as flags - flagArgs := backends.BuildCommandArgs(&optionsCopy, multipleFlags) + flagArgs := backends.BuildCommandArgs(&optionsCopy, multiValuedFlags) args = append(args, flagArgs...) return args } -// BuildCommandArgsWithDocker converts VllmServerOptions to command line arguments, -// handling Docker transformations if needed -func (o *VllmServerOptions) BuildCommandArgsWithDocker(dockerImage string) []string { - args := o.BuildCommandArgs() +func (o *VllmServerOptions) BuildDockerArgs() []string { + var args []string - // Handle vLLM Docker image quirk - if isVLLMDocker(dockerImage) { - args = transformVLLMArgs(args) - } + // Use package-level multipleFlags variable + flagArgs := backends.BuildCommandArgs(o, multiValuedFlags) + args = append(args, flagArgs...) return args } -// isVLLMDocker checks if the Docker image is a vLLM image -func isVLLMDocker(image string) bool { - return strings.Contains(strings.ToLower(image), "vllm") -} - -// transformVLLMArgs converts vLLM arguments for Docker execution -// Convert: ["serve", "microsoft/DialoGPT-medium", "--flag", "value"] -// To: ["--model", "microsoft/DialoGPT-medium", "--flag", "value"] -func transformVLLMArgs(args []string) []string { - if len(args) >= 2 && args[0] == "serve" { - return append([]string{"--model", args[1]}, args[2:]...) - } - return args -} - -// BuildCommand creates the complete command for execution, handling Docker vs native execution -func (o *VllmServerOptions) BuildCommand(ctx context.Context, backendConfig *config.BackendSettings) (*exec.Cmd, error) { - // Build instance-specific arguments using backend functions - var instanceArgs []string - if backendConfig.Docker != nil && backendConfig.Docker.Enabled { - // Use Docker-aware argument building - instanceArgs = o.BuildCommandArgsWithDocker(backendConfig.Docker.Image) - } else { - // Use regular argument building for native execution - instanceArgs = o.BuildCommandArgs() - } - - // Combine backend args with instance args - finalArgs := append(backendConfig.Args, instanceArgs...) - - // Choose Docker vs Native execution - if backendConfig.Docker != nil && backendConfig.Docker.Enabled { - return buildDockerCommand(ctx, backendConfig, finalArgs) - } else { - return exec.CommandContext(ctx, backendConfig.Command, finalArgs...), nil - } -} - -// buildDockerCommand builds a Docker command with the specified configuration and arguments -func buildDockerCommand(ctx context.Context, backendConfig *config.BackendSettings, args []string) (*exec.Cmd, error) { - // Start with configured Docker arguments (should include "run", "--rm", etc.) - dockerArgs := make([]string, len(backendConfig.Docker.Args)) - copy(dockerArgs, backendConfig.Docker.Args) - - // Add environment variables - for key, value := range backendConfig.Docker.Environment { - dockerArgs = append(dockerArgs, "-e", fmt.Sprintf("%s=%s", key, value)) - } - - // Add image and container arguments - dockerArgs = append(dockerArgs, backendConfig.Docker.Image) - dockerArgs = append(dockerArgs, args...) - - return exec.CommandContext(ctx, "docker", dockerArgs...), nil -} - // ParseVllmCommand parses a vLLM serve command string into VllmServerOptions // Supports multiple formats: // 1. Full command: "vllm serve --model MODEL_NAME --other-args" diff --git a/pkg/instance/lifecycle.go b/pkg/instance/lifecycle.go index 3cace05..9f7243a 100644 --- a/pkg/instance/lifecycle.go +++ b/pkg/instance/lifecycle.go @@ -372,8 +372,13 @@ func (i *Process) buildCommand() (*exec.Cmd, error) { return nil, err } - // Delegate to the backend's BuildCommand method - return i.options.BuildCommand(i.ctx, backendConfig) + // Get the command to execute + cmd := i.options.GetCommand(backendConfig) + + // Build command arguments + args := i.options.BuildCommandArgs(backendConfig) + + return exec.Command(cmd, args...), nil } // getBackendConfig resolves the backend configuration for the current instance @@ -394,4 +399,3 @@ func (i *Process) getBackendConfig() (*config.BackendSettings, error) { settings := i.globalBackendSettings.GetBackendSettings(backendTypeStr) return &settings, nil } - diff --git a/pkg/instance/options.go b/pkg/instance/options.go index f575faa..d637d0b 100644 --- a/pkg/instance/options.go +++ b/pkg/instance/options.go @@ -1,7 +1,6 @@ package instance import ( - "context" "encoding/json" "fmt" "llamactl/pkg/backends" @@ -10,7 +9,6 @@ import ( "llamactl/pkg/backends/vllm" "llamactl/pkg/config" "log" - "os/exec" ) type CreateInstanceOptions struct { @@ -190,61 +188,54 @@ func (c *CreateInstanceOptions) ValidateAndApplyDefaults(name string, globalSett } } +func (c *CreateInstanceOptions) GetCommand(backendConfig *config.BackendSettings) string { + + if backendConfig.Docker != nil && backendConfig.Docker.Enabled && c.BackendType != backends.BackendTypeMlxLm { + return "docker" + } + + return backendConfig.Command +} + // BuildCommandArgs builds command line arguments for the backend -func (c *CreateInstanceOptions) BuildCommandArgs() []string { - switch c.BackendType { - case backends.BackendTypeLlamaCpp: - if c.LlamaServerOptions != nil { - return c.LlamaServerOptions.BuildCommandArgs() - } - case backends.BackendTypeMlxLm: - if c.MlxServerOptions != nil { - return c.MlxServerOptions.BuildCommandArgs() - } - case backends.BackendTypeVllm: - if c.VllmServerOptions != nil { - // No longer prepend "serve" - comes from backend config - return c.VllmServerOptions.BuildCommandArgs() - } - } - return []string{} -} +func (c *CreateInstanceOptions) BuildCommandArgs(backendConfig *config.BackendSettings) []string { -// BuildCommandArgsWithDocker builds command line arguments for the backend, -// handling Docker transformations if needed -func (c *CreateInstanceOptions) BuildCommandArgsWithDocker(dockerImage string) []string { - switch c.BackendType { - case backends.BackendTypeLlamaCpp: - if c.LlamaServerOptions != nil { - return c.LlamaServerOptions.BuildCommandArgsWithDocker(dockerImage) - } - case backends.BackendTypeMlxLm: - if c.MlxServerOptions != nil { - return c.MlxServerOptions.BuildCommandArgsWithDocker(dockerImage) - } - case backends.BackendTypeVllm: - if c.VllmServerOptions != nil { - return c.VllmServerOptions.BuildCommandArgsWithDocker(dockerImage) - } - } - return []string{} -} + var args []string -// BuildCommand builds the complete command for the backend, handling Docker vs native execution -func (c *CreateInstanceOptions) BuildCommand(ctx context.Context, backendConfig *config.BackendSettings) (*exec.Cmd, error) { - switch c.BackendType { - case backends.BackendTypeLlamaCpp: - if c.LlamaServerOptions != nil { - return c.LlamaServerOptions.BuildCommand(ctx, backendConfig) + if backendConfig.Docker != nil && backendConfig.Docker.Enabled && c.BackendType != backends.BackendTypeMlxLm { + // For Docker, start with Docker args + args = append(args, backendConfig.Docker.Args...) + + switch c.BackendType { + case backends.BackendTypeLlamaCpp: + if c.LlamaServerOptions != nil { + args = append(args, c.LlamaServerOptions.BuildDockerArgs()...) + } + case backends.BackendTypeVllm: + if c.VllmServerOptions != nil { + args = append(args, c.VllmServerOptions.BuildDockerArgs()...) + } } - case backends.BackendTypeMlxLm: - if c.MlxServerOptions != nil { - return c.MlxServerOptions.BuildCommand(ctx, backendConfig) - } - case backends.BackendTypeVllm: - if c.VllmServerOptions != nil { - return c.VllmServerOptions.BuildCommand(ctx, backendConfig) + + } else { + // For native execution, start with backend args + args = append(args, backendConfig.Args...) + + switch c.BackendType { + case backends.BackendTypeLlamaCpp: + if c.LlamaServerOptions != nil { + args = append(args, c.LlamaServerOptions.BuildCommandArgs()...) + } + case backends.BackendTypeMlxLm: + if c.MlxServerOptions != nil { + args = append(args, c.MlxServerOptions.BuildCommandArgs()...) + } + case backends.BackendTypeVllm: + if c.VllmServerOptions != nil { + args = append(args, c.VllmServerOptions.BuildCommandArgs()...) + } } } - return nil, fmt.Errorf("no backend options configured for type: %s", c.BackendType) + + return args }