From ddb54763f61d2f786b19dbbf80beae9127647641 Mon Sep 17 00:00:00 2001
From: LordMathis <matus@namesny.com>
Date: Wed, 20 Aug 2025 14:25:43 +0200
Subject: [PATCH] Add OnDemandStartTimeout configuration and update OpenAIProxy
 to use it

---
 pkg/config/config.go   | 13 +++++++++++--
 pkg/server/handlers.go |  2 +-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/pkg/config/config.go b/pkg/config/config.go
index 1e2694e..4486fa5 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -70,6 +70,9 @@ type InstancesConfig struct {
 	// Default on-demand start setting for new instances
 	DefaultOnDemandStart bool `yaml:"default_on_demand_start"`
 
+	// How long to wait for an instance to start on demand (in seconds)
+	OnDemandStartTimeout int `yaml:"on_demand_start_timeout,omitempty"`
+
 	// Interval for checking instance timeouts (in minutes)
 	TimeoutCheckInterval int `yaml:"timeout_check_interval"`
 }
@@ -114,8 +117,9 @@ func LoadConfig(configPath string) (AppConfig, error) {
 			DefaultAutoRestart:   true,
 			DefaultMaxRestarts:   3,
 			DefaultRestartDelay:  5,
-			DefaultOnDemandStart: false,
-			TimeoutCheckInterval: 5, // Check timeouts every 5 minutes
+			DefaultOnDemandStart: true,
+			OnDemandStartTimeout: 120, // 2 minutes
+			TimeoutCheckInterval: 5,   // Check timeouts every 5 minutes
 		},
 		Auth: AuthConfig{
 			RequireInferenceAuth:  true,
@@ -230,6 +234,11 @@ func loadEnvVars(cfg *AppConfig) {
 			cfg.Instances.DefaultOnDemandStart = b
 		}
 	}
+	if onDemandTimeout := os.Getenv("LLAMACTL_ON_DEMAND_START_TIMEOUT"); onDemandTimeout != "" {
+		if seconds, err := strconv.Atoi(onDemandTimeout); err == nil {
+			cfg.Instances.OnDemandStartTimeout = seconds
+		}
+	}
 	if timeoutCheckInterval := os.Getenv("LLAMACTL_TIMEOUT_CHECK_INTERVAL"); timeoutCheckInterval != "" {
 		if minutes, err := strconv.Atoi(timeoutCheckInterval); err == nil {
 			cfg.Instances.TimeoutCheckInterval = minutes
diff --git a/pkg/server/handlers.go b/pkg/server/handlers.go
index 843407e..6306a31 100644
--- a/pkg/server/handlers.go
+++ b/pkg/server/handlers.go
@@ -583,7 +583,7 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc {
 				}
 
 				// Wait for the instance to become healthy before proceeding
-				if err := inst.WaitForHealthy(120); err != nil { // 2 minutes timeout
+				if err := inst.WaitForHealthy(h.cfg.Instances.OnDemandStartTimeout); err != nil { // 2 minutes timeout
 					http.Error(w, "Instance failed to become healthy: "+err.Error(), http.StatusServiceUnavailable)
 					return
 				}