From 76ac93bedc659e35c0455846a8f1de17d1aff8e3 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 24 Sep 2025 21:31:58 +0200 Subject: [PATCH] Implement Docker command handling for Llama, MLX, and vLLM backends --- pkg/backends/llamacpp/llama.go | 54 ++++++++++++++++++++++++ pkg/backends/mlx/mlx.go | 54 ++++++++++++++++++++++++ pkg/backends/vllm/vllm.go | 75 ++++++++++++++++++++++++++++++++++ pkg/instance/lifecycle.go | 55 +++++++++++++++++-------- 4 files changed, 222 insertions(+), 16 deletions(-) diff --git a/pkg/backends/llamacpp/llama.go b/pkg/backends/llamacpp/llama.go index f2a7d31..970077f 100644 --- a/pkg/backends/llamacpp/llama.go +++ b/pkg/backends/llamacpp/llama.go @@ -1,8 +1,12 @@ package llamacpp import ( + "context" "encoding/json" + "fmt" "llamactl/pkg/backends" + "llamactl/pkg/config" + "os/exec" "reflect" "strconv" ) @@ -329,6 +333,56 @@ func (o *LlamaServerOptions) BuildCommandArgs() []string { return backends.BuildCommandArgs(o, multipleFlags) } +// BuildCommandArgsWithDocker converts InstanceOptions to command line arguments, +// handling Docker transformations if needed +func (o *LlamaServerOptions) BuildCommandArgsWithDocker(dockerImage string) []string { + args := o.BuildCommandArgs() + + // No special Docker transformations needed for llama-cpp + return args +} + +// BuildCommand creates the complete command for execution, handling Docker vs native execution +func (o *LlamaServerOptions) BuildCommand(ctx context.Context, backendConfig *config.BackendSettings) (*exec.Cmd, error) { + // Build instance-specific arguments using backend functions + var instanceArgs []string + if backendConfig.Docker != nil && backendConfig.Docker.Enabled { + // Use Docker-aware argument building + instanceArgs = o.BuildCommandArgsWithDocker(backendConfig.Docker.Image) + } else { + // Use regular argument building for native execution + instanceArgs = o.BuildCommandArgs() + } + + // Combine backend args with instance args + finalArgs := append(backendConfig.Args, instanceArgs...) + + // Choose Docker vs Native execution + if backendConfig.Docker != nil && backendConfig.Docker.Enabled { + return buildDockerCommand(ctx, backendConfig, finalArgs) + } else { + return exec.CommandContext(ctx, backendConfig.Command, finalArgs...), nil + } +} + +// buildDockerCommand builds a Docker command with the specified configuration and arguments +func buildDockerCommand(ctx context.Context, backendConfig *config.BackendSettings, args []string) (*exec.Cmd, error) { + // Start with configured Docker arguments (should include "run", "--rm", etc.) + dockerArgs := make([]string, len(backendConfig.Docker.Args)) + copy(dockerArgs, backendConfig.Docker.Args) + + // Add environment variables + for key, value := range backendConfig.Docker.Environment { + dockerArgs = append(dockerArgs, "-e", fmt.Sprintf("%s=%s", key, value)) + } + + // Add image and container arguments + dockerArgs = append(dockerArgs, backendConfig.Docker.Image) + dockerArgs = append(dockerArgs, args...) + + return exec.CommandContext(ctx, "docker", dockerArgs...), nil +} + // ParseLlamaCommand parses a llama-server command string into LlamaServerOptions // Supports multiple formats: // 1. Full command: "llama-server --model file.gguf" diff --git a/pkg/backends/mlx/mlx.go b/pkg/backends/mlx/mlx.go index 3b83681..62d8e68 100644 --- a/pkg/backends/mlx/mlx.go +++ b/pkg/backends/mlx/mlx.go @@ -1,7 +1,11 @@ package mlx import ( + "context" + "fmt" "llamactl/pkg/backends" + "llamactl/pkg/config" + "os/exec" ) type MlxServerOptions struct { @@ -36,6 +40,56 @@ func (o *MlxServerOptions) BuildCommandArgs() []string { return backends.BuildCommandArgs(o, multipleFlags) } +// BuildCommandArgsWithDocker converts to command line arguments, +// handling Docker transformations if needed +func (o *MlxServerOptions) BuildCommandArgsWithDocker(dockerImage string) []string { + args := o.BuildCommandArgs() + + // No special Docker transformations needed for MLX + return args +} + +// BuildCommand creates the complete command for execution, handling Docker vs native execution +func (o *MlxServerOptions) BuildCommand(ctx context.Context, backendConfig *config.BackendSettings) (*exec.Cmd, error) { + // Build instance-specific arguments using backend functions + var instanceArgs []string + if backendConfig.Docker != nil && backendConfig.Docker.Enabled { + // Use Docker-aware argument building + instanceArgs = o.BuildCommandArgsWithDocker(backendConfig.Docker.Image) + } else { + // Use regular argument building for native execution + instanceArgs = o.BuildCommandArgs() + } + + // Combine backend args with instance args + finalArgs := append(backendConfig.Args, instanceArgs...) + + // Choose Docker vs Native execution + if backendConfig.Docker != nil && backendConfig.Docker.Enabled { + return buildDockerCommand(ctx, backendConfig, finalArgs) + } else { + return exec.CommandContext(ctx, backendConfig.Command, finalArgs...), nil + } +} + +// buildDockerCommand builds a Docker command with the specified configuration and arguments +func buildDockerCommand(ctx context.Context, backendConfig *config.BackendSettings, args []string) (*exec.Cmd, error) { + // Start with configured Docker arguments (should include "run", "--rm", etc.) + dockerArgs := make([]string, len(backendConfig.Docker.Args)) + copy(dockerArgs, backendConfig.Docker.Args) + + // Add environment variables + for key, value := range backendConfig.Docker.Environment { + dockerArgs = append(dockerArgs, "-e", fmt.Sprintf("%s=%s", key, value)) + } + + // Add image and container arguments + dockerArgs = append(dockerArgs, backendConfig.Docker.Image) + dockerArgs = append(dockerArgs, args...) + + return exec.CommandContext(ctx, "docker", dockerArgs...), nil +} + // ParseMlxCommand parses a mlx_lm.server command string into MlxServerOptions // Supports multiple formats: // 1. Full command: "mlx_lm.server --model model/path" diff --git a/pkg/backends/vllm/vllm.go b/pkg/backends/vllm/vllm.go index 7811c4c..b4436c2 100644 --- a/pkg/backends/vllm/vllm.go +++ b/pkg/backends/vllm/vllm.go @@ -1,6 +1,12 @@ package vllm import ( + "context" + "fmt" + "llamactl/pkg/config" + "os/exec" + "strings" + "llamactl/pkg/backends" ) @@ -160,6 +166,75 @@ func (o *VllmServerOptions) BuildCommandArgs() []string { return args } +// BuildCommandArgsWithDocker converts VllmServerOptions to command line arguments, +// handling Docker transformations if needed +func (o *VllmServerOptions) BuildCommandArgsWithDocker(dockerImage string) []string { + args := o.BuildCommandArgs() + + // Handle vLLM Docker image quirk + if isVLLMDocker(dockerImage) { + args = transformVLLMArgs(args) + } + + return args +} + +// isVLLMDocker checks if the Docker image is a vLLM image +func isVLLMDocker(image string) bool { + return strings.Contains(strings.ToLower(image), "vllm") +} + +// transformVLLMArgs converts vLLM arguments for Docker execution +// Convert: ["serve", "microsoft/DialoGPT-medium", "--flag", "value"] +// To: ["--model", "microsoft/DialoGPT-medium", "--flag", "value"] +func transformVLLMArgs(args []string) []string { + if len(args) >= 2 && args[0] == "serve" { + return append([]string{"--model", args[1]}, args[2:]...) + } + return args +} + +// BuildCommand creates the complete command for execution, handling Docker vs native execution +func (o *VllmServerOptions) BuildCommand(ctx context.Context, backendConfig *config.BackendSettings) (*exec.Cmd, error) { + // Build instance-specific arguments using backend functions + var instanceArgs []string + if backendConfig.Docker != nil && backendConfig.Docker.Enabled { + // Use Docker-aware argument building + instanceArgs = o.BuildCommandArgsWithDocker(backendConfig.Docker.Image) + } else { + // Use regular argument building for native execution + instanceArgs = o.BuildCommandArgs() + } + + // Combine backend args with instance args + finalArgs := append(backendConfig.Args, instanceArgs...) + + // Choose Docker vs Native execution + if backendConfig.Docker != nil && backendConfig.Docker.Enabled { + return buildDockerCommand(ctx, backendConfig, finalArgs) + } else { + return exec.CommandContext(ctx, backendConfig.Command, finalArgs...), nil + } +} + +// buildDockerCommand builds a Docker command with the specified configuration and arguments +func buildDockerCommand(ctx context.Context, backendConfig *config.BackendSettings, args []string) (*exec.Cmd, error) { + // Start with configured Docker arguments (should include "run", "--rm", etc.) + dockerArgs := make([]string, len(backendConfig.Docker.Args)) + copy(dockerArgs, backendConfig.Docker.Args) + + // Add environment variables + for key, value := range backendConfig.Docker.Environment { + dockerArgs = append(dockerArgs, "-e", fmt.Sprintf("%s=%s", key, value)) + } + + // Add image and container arguments + dockerArgs = append(dockerArgs, backendConfig.Docker.Image) + dockerArgs = append(dockerArgs, args...) + + return exec.CommandContext(ctx, "docker", dockerArgs...), nil +} + // ParseVllmCommand parses a vLLM serve command string into VllmServerOptions // Supports multiple formats: // 1. Full command: "vllm serve --model MODEL_NAME --other-args" diff --git a/pkg/instance/lifecycle.go b/pkg/instance/lifecycle.go index 9eab260..3cace05 100644 --- a/pkg/instance/lifecycle.go +++ b/pkg/instance/lifecycle.go @@ -11,6 +11,7 @@ import ( "time" "llamactl/pkg/backends" + "llamactl/pkg/config" ) // Start starts the llama server instance and returns an error if it fails. @@ -41,24 +42,14 @@ func (i *Process) Start() error { return fmt.Errorf("failed to create log files: %w", err) } - args := i.options.BuildCommandArgs() - i.ctx, i.cancel = context.WithCancel(context.Background()) - - var executable string - - // Get executable from global configuration - switch i.options.BackendType { - case backends.BackendTypeLlamaCpp: - executable = i.globalBackendSettings.LlamaExecutable - case backends.BackendTypeMlxLm: - executable = i.globalBackendSettings.MLXLMExecutable - case backends.BackendTypeVllm: - executable = i.globalBackendSettings.VllmExecutable - default: - return fmt.Errorf("unsupported backend type: %s", i.options.BackendType) + // Build command using backend-specific methods + cmd, cmdErr := i.buildCommand() + if cmdErr != nil { + return fmt.Errorf("failed to build command: %w", cmdErr) } - i.cmd = exec.CommandContext(i.ctx, executable, args...) + i.ctx, i.cancel = context.WithCancel(context.Background()) + i.cmd = cmd if runtime.GOOS != "windows" { setProcAttrs(i.cmd) @@ -372,3 +363,35 @@ func (i *Process) validateRestartConditions() (shouldRestart bool, maxRestarts i return true, maxRestarts, restartDelay } + +// buildCommand builds the command to execute using backend-specific logic +func (i *Process) buildCommand() (*exec.Cmd, error) { + // Get backend configuration + backendConfig, err := i.getBackendConfig() + if err != nil { + return nil, err + } + + // Delegate to the backend's BuildCommand method + return i.options.BuildCommand(i.ctx, backendConfig) +} + +// getBackendConfig resolves the backend configuration for the current instance +func (i *Process) getBackendConfig() (*config.BackendSettings, error) { + var backendTypeStr string + + switch i.options.BackendType { + case backends.BackendTypeLlamaCpp: + backendTypeStr = "llama-cpp" + case backends.BackendTypeMlxLm: + backendTypeStr = "mlx" + case backends.BackendTypeVllm: + backendTypeStr = "vllm" + default: + return nil, fmt.Errorf("unsupported backend type: %s", i.options.BackendType) + } + + settings := i.globalBackendSettings.GetBackendSettings(backendTypeStr) + return &settings, nil +} +