From 8d92f9b371bcd2d7d052f141b178bde8f70555b8 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 12 Nov 2025 23:11:22 +0100 Subject: [PATCH] Add ExtraArgs support for Llama, Mlx, and Vllm server options --- pkg/backends/builder.go | 19 +++++++++++++++++++ pkg/backends/llama.go | 23 ++++++++++++++++++++++- pkg/backends/mlx.go | 23 ++++++++++++++++++++++- pkg/backends/vllm.go | 22 ++++++++++++++++++++++ 4 files changed, 85 insertions(+), 2 deletions(-) diff --git a/pkg/backends/builder.go b/pkg/backends/builder.go index d224742..0d7402e 100644 --- a/pkg/backends/builder.go +++ b/pkg/backends/builder.go @@ -93,3 +93,22 @@ func BuildDockerCommand(backendConfig *config.BackendSettings, instanceArgs []st return "docker", dockerArgs, nil } + +// convertExtraArgsToFlags converts map[string]string to command flags +// Empty values become boolean flags: {"flag": ""} → ["--flag"] +// Non-empty values: {"flag": "value"} → ["--flag", "value"] +func convertExtraArgsToFlags(extraArgs map[string]string) []string { + var args []string + + for key, value := range extraArgs { + if value == "" { + // Boolean flag + args = append(args, "--"+key) + } else { + // Value flag + args = append(args, "--"+key, value) + } + } + + return args +} diff --git a/pkg/backends/llama.go b/pkg/backends/llama.go index 2b3372a..827bab7 100644 --- a/pkg/backends/llama.go +++ b/pkg/backends/llama.go @@ -187,6 +187,10 @@ type LlamaServerOptions struct { FIMQwen7BDefault bool `json:"fim_qwen_7b_default,omitempty"` FIMQwen7BSpec bool `json:"fim_qwen_7b_spec,omitempty"` FIMQwen14BSpec bool `json:"fim_qwen_14b_spec,omitempty"` + + // ExtraArgs are additional command line arguments. + // Example: {"verbose": "", "log-file": "/logs/llama.log"} + ExtraArgs map[string]string `json:"extra_args,omitempty"` } // UnmarshalJSON implements custom JSON unmarshaling to support multiple field names @@ -354,6 +358,18 @@ func (o *LlamaServerOptions) Validate() error { return validation.ValidationError(fmt.Errorf("invalid port range: %d", o.Port)) } + // Validate extra_args keys and values + for key, value := range o.ExtraArgs { + if err := validation.ValidateStringForInjection(key); err != nil { + return validation.ValidationError(fmt.Errorf("extra_args key %q: %w", key, err)) + } + if value != "" { + if err := validation.ValidateStringForInjection(value); err != nil { + return validation.ValidationError(fmt.Errorf("extra_args value for %q: %w", key, err)) + } + } + } + return nil } @@ -361,7 +377,12 @@ func (o *LlamaServerOptions) Validate() error { func (o *LlamaServerOptions) BuildCommandArgs() []string { // Llama uses multiple flags for arrays by default (not comma-separated) // Use package-level llamaMultiValuedFlags variable - return BuildCommandArgs(o, llamaMultiValuedFlags) + args := BuildCommandArgs(o, llamaMultiValuedFlags) + + // Append extra args at the end + args = append(args, convertExtraArgsToFlags(o.ExtraArgs)...) + + return args } func (o *LlamaServerOptions) BuildDockerArgs() []string { diff --git a/pkg/backends/mlx.go b/pkg/backends/mlx.go index 8911d0b..2ce927b 100644 --- a/pkg/backends/mlx.go +++ b/pkg/backends/mlx.go @@ -29,6 +29,10 @@ type MlxServerOptions struct { TopK int `json:"top_k,omitempty"` MinP float64 `json:"min_p,omitempty"` MaxTokens int `json:"max_tokens,omitempty"` + + // ExtraArgs are additional command line arguments. + // Example: {"verbose": "", "log-file": "/logs/mlx.log"} + ExtraArgs map[string]string `json:"extra_args,omitempty"` } func (o *MlxServerOptions) GetPort() int { @@ -57,13 +61,30 @@ func (o *MlxServerOptions) Validate() error { return validation.ValidationError(fmt.Errorf("invalid port range: %d", o.Port)) } + // Validate extra_args keys and values + for key, value := range o.ExtraArgs { + if err := validation.ValidateStringForInjection(key); err != nil { + return validation.ValidationError(fmt.Errorf("extra_args key %q: %w", key, err)) + } + if value != "" { + if err := validation.ValidateStringForInjection(value); err != nil { + return validation.ValidationError(fmt.Errorf("extra_args value for %q: %w", key, err)) + } + } + } + return nil } // BuildCommandArgs converts to command line arguments func (o *MlxServerOptions) BuildCommandArgs() []string { multipleFlags := map[string]struct{}{} // MLX doesn't currently have []string fields - return BuildCommandArgs(o, multipleFlags) + args := BuildCommandArgs(o, multipleFlags) + + // Append extra args at the end + args = append(args, convertExtraArgsToFlags(o.ExtraArgs)...) + + return args } func (o *MlxServerOptions) BuildDockerArgs() []string { diff --git a/pkg/backends/vllm.go b/pkg/backends/vllm.go index 34dce4c..c9676e0 100644 --- a/pkg/backends/vllm.go +++ b/pkg/backends/vllm.go @@ -142,6 +142,10 @@ type VllmServerOptions struct { OverridePoolingConfig string `json:"override_pooling_config,omitempty"` OverrideNeuronConfig string `json:"override_neuron_config,omitempty"` OverrideKVCacheALIGNSize int `json:"override_kv_cache_align_size,omitempty"` + + // ExtraArgs are additional command line arguments. + // Example: {"verbose": "", "log-file": "/logs/vllm.log"} + ExtraArgs map[string]string `json:"extra_args,omitempty"` } func (o *VllmServerOptions) GetPort() int { @@ -171,6 +175,18 @@ func (o *VllmServerOptions) Validate() error { return validation.ValidationError(fmt.Errorf("invalid port range: %d", o.Port)) } + // Validate extra_args keys and values + for key, value := range o.ExtraArgs { + if err := validation.ValidateStringForInjection(key); err != nil { + return validation.ValidationError(fmt.Errorf("extra_args key %q: %w", key, err)) + } + if value != "" { + if err := validation.ValidateStringForInjection(value); err != nil { + return validation.ValidationError(fmt.Errorf("extra_args value for %q: %w", key, err)) + } + } + } + return nil } @@ -193,6 +209,9 @@ func (o *VllmServerOptions) BuildCommandArgs() []string { flagArgs := BuildCommandArgs(&optionsCopy, vllmMultiValuedFlags) args = append(args, flagArgs...) + // Append extra args at the end + args = append(args, convertExtraArgsToFlags(o.ExtraArgs)...) + return args } @@ -203,6 +222,9 @@ func (o *VllmServerOptions) BuildDockerArgs() []string { flagArgs := BuildCommandArgs(o, vllmMultiValuedFlags) args = append(args, flagArgs...) + // Append extra args at the end + args = append(args, convertExtraArgsToFlags(o.ExtraArgs)...) + return args }