Mathis/llamactl
1
0
Fork 0
mirror of https://github.com/lordmathis/llamactl.git synced 2025-11-06 00:54:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add OnDemandStartTimeout configuration and update OpenAIProxy to use it

Browse Source
This commit is contained in:
Matúš Námešný
2025-08-20 14:25:43 +02:00
parent 496ab3aa5d
commit ddb54763f6
2 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/config/config.go
View File

@@ -70,6 +70,9 @@ type InstancesConfig struct {
// Default on-demand start setting for new instances // Default on-demand start setting for new instances
DefaultOnDemandStart bool `yaml:"default_on_demand_start"` DefaultOnDemandStart bool `yaml:"default_on_demand_start"`
// How long to wait for an instance to start on demand (in seconds)
OnDemandStartTimeout int `yaml:"on_demand_start_timeout,omitempty"`
// Interval for checking instance timeouts (in minutes) // Interval for checking instance timeouts (in minutes)
TimeoutCheckInterval int `yaml:"timeout_check_interval"` TimeoutCheckInterval int `yaml:"timeout_check_interval"`
} }
@@ -114,8 +117,9 @@ func LoadConfig(configPath string) (AppConfig, error) {
DefaultAutoRestart: true, DefaultAutoRestart: true,
DefaultMaxRestarts: 3, DefaultMaxRestarts: 3,
DefaultRestartDelay: 5, DefaultRestartDelay: 5,
DefaultOnDemandStart: false, DefaultOnDemandStart: true,
TimeoutCheckInterval: 5, // Check timeouts every 5 minutes OnDemandStartTimeout: 120, // 2 minutes
TimeoutCheckInterval: 5, // Check timeouts every 5 minutes
}, },
Auth: AuthConfig{ Auth: AuthConfig{
RequireInferenceAuth: true, RequireInferenceAuth: true,
@@ -230,6 +234,11 @@ func loadEnvVars(cfg *AppConfig) {
cfg.Instances.DefaultOnDemandStart = b cfg.Instances.DefaultOnDemandStart = b
} }
} }
if onDemandTimeout := os.Getenv("LLAMACTL_ON_DEMAND_START_TIMEOUT"); onDemandTimeout != "" {
if seconds, err := strconv.Atoi(onDemandTimeout); err == nil {
cfg.Instances.OnDemandStartTimeout = seconds
}
}
if timeoutCheckInterval := os.Getenv("LLAMACTL_TIMEOUT_CHECK_INTERVAL"); timeoutCheckInterval != "" { if timeoutCheckInterval := os.Getenv("LLAMACTL_TIMEOUT_CHECK_INTERVAL"); timeoutCheckInterval != "" {
if minutes, err := strconv.Atoi(timeoutCheckInterval); err == nil { if minutes, err := strconv.Atoi(timeoutCheckInterval); err == nil {
cfg.Instances.TimeoutCheckInterval = minutes cfg.Instances.TimeoutCheckInterval = minutes

2
pkg/server/handlers.go
View File

@@ -583,7 +583,7 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc {
} }
// Wait for the instance to become healthy before proceeding // Wait for the instance to become healthy before proceeding
if err := inst.WaitForHealthy(120); err != nil { // 2 minutes timeout if err := inst.WaitForHealthy(h.cfg.Instances.OnDemandStartTimeout); err != nil { // 2 minutes timeout
http.Error(w, "Instance failed to become healthy: "+err.Error(), http.StatusServiceUnavailable) http.Error(w, "Instance failed to become healthy: "+err.Error(), http.StatusServiceUnavailable)
return return
} }