Merge pull request #24 from lordmathis/feat/max-running-instances

feat: Implement max running instances constraint and refactor instances status

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Server",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/server/main.go",
+        }
+    ]
+}

pkg/config/config.go

@@ -55,6 +55,9 @@ type InstancesConfig struct {
 	// Maximum number of instances that can be created
 	MaxInstances int `yaml:"max_instances"`
 
+	// Maximum number of instances that can be running at the same time
+	MaxRunningInstances int `yaml:"max_running_instances,omitempty"`
+
 	// Path to llama-server executable
 	LlamaExecutable string `yaml:"llama_executable"`
 
@@ -113,6 +116,7 @@ func LoadConfig(configPath string) (AppConfig, error) {
 			LogsDir:              filepath.Join(getDefaultDataDirectory(), "logs"),
 			AutoCreateDirs:       true,
 			MaxInstances:         -1, // -1 means unlimited
+			MaxRunningInstances:  -1, // -1 means unlimited
 			LlamaExecutable:      "llama-server",
 			DefaultAutoRestart:   true,
 			DefaultMaxRestarts:   3,
@@ -211,6 +215,11 @@ func loadEnvVars(cfg *AppConfig) {
 			cfg.Instances.MaxInstances = m
 		}
 	}
+	if maxRunning := os.Getenv("LLAMACTL_MAX_RUNNING_INSTANCES"); maxRunning != "" {
+		if m, err := strconv.Atoi(maxRunning); err == nil {
+			cfg.Instances.MaxRunningInstances = m
+		}
+	}
 	if llamaExec := os.Getenv("LLAMACTL_LLAMA_EXECUTABLE"); llamaExec != "" {
 		cfg.Instances.LlamaExecutable = llamaExec
 	}

pkg/instance/instance.go

@@ -82,7 +82,8 @@ type Process struct {
 	globalSettings *config.InstancesConfig
 
 	// Status
-	Running bool `json:"running"`
+	Status         InstanceStatus `json:"status"`
+	onStatusChange func(oldStatus, newStatus InstanceStatus)
 
 	// Creation time
 	Created int64 `json:"created,omitempty"` // Unix timestamp when the instance was created
@@ -193,7 +194,7 @@ func applyDefaultOptions(options *CreateInstanceOptions, globalSettings *config.
 }
 
 // NewInstance creates a new instance with the given name, log path, and options
-func NewInstance(name string, globalSettings *config.InstancesConfig, options *CreateInstanceOptions) *Process {
+func NewInstance(name string, globalSettings *config.InstancesConfig, options *CreateInstanceOptions, onStatusChange func(oldStatus, newStatus InstanceStatus)) *Process {
 	// Validate and copy options
 	optionsCopy := validateAndCopyOptions(name, options)
 	// Apply defaults
@@ -208,6 +209,8 @@ func NewInstance(name string, globalSettings *config.InstancesConfig, options *C
 		logger:         logger,
 		timeProvider:   realTimeProvider{},
 		Created:        time.Now().Unix(),
+		Status:         Stopped,
+		onStatusChange: onStatusChange,
 	}
 }
 
@@ -287,12 +290,12 @@ func (i *Process) MarshalJSON() ([]byte, error) {
 	temp := struct {
 		Name    string                 `json:"name"`
 		Options *CreateInstanceOptions `json:"options,omitempty"`
-		Running bool                   `json:"running"`
+		Status  InstanceStatus         `json:"status"`
 		Created int64                  `json:"created,omitempty"`
 	}{
 		Name:    i.Name,
 		Options: i.options,
-		Running: i.Running,
+		Status:  i.Status,
 		Created: i.Created,
 	}
 
@@ -305,7 +308,7 @@ func (i *Process) UnmarshalJSON(data []byte) error {
 	temp := struct {
 		Name    string                 `json:"name"`
 		Options *CreateInstanceOptions `json:"options,omitempty"`
-		Running bool                   `json:"running"`
+		Status  InstanceStatus         `json:"status"`
 		Created int64                  `json:"created,omitempty"`
 	}{}
 
@@ -315,7 +318,7 @@ func (i *Process) UnmarshalJSON(data []byte) error {
 
 	// Set the fields
 	i.Name = temp.Name
-	i.Running = temp.Running
+	i.Status = temp.Status
 	i.Created = temp.Created
 
 	// Handle options with validation but no defaults

pkg/instance/instance_test.go

@@ -24,12 +24,15 @@ func TestNewInstance(t *testing.T) {
 		},
 	}
 
-	instance := instance.NewInstance("test-instance", globalSettings, options)
+	// Mock onStatusChange function
+	mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+	instance := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
 
 	if instance.Name != "test-instance" {
 		t.Errorf("Expected name 'test-instance', got %q", instance.Name)
 	}
-	if instance.Running {
+	if instance.IsRunning() {
 		t.Error("New instance should not be running")
 	}
 
@@ -76,7 +79,10 @@ func TestNewInstance_WithRestartOptions(t *testing.T) {
 		},
 	}
 
-	instance := instance.NewInstance("test-instance", globalSettings, options)
+	// Mock onStatusChange function
+	mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+	instance := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
 	opts := instance.GetOptions()
 
 	// Check that explicit values override defaults
@@ -106,7 +112,10 @@ func TestSetOptions(t *testing.T) {
 		},
 	}
 
-	inst := instance.NewInstance("test-instance", globalSettings, initialOptions)
+	// Mock onStatusChange function
+	mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+	inst := instance.NewInstance("test-instance", globalSettings, initialOptions, mockOnStatusChange)
 
 	// Update options
 	newOptions := &instance.CreateInstanceOptions{
@@ -144,7 +153,10 @@ func TestGetProxy(t *testing.T) {
 		},
 	}
 
-	inst := instance.NewInstance("test-instance", globalSettings, options)
+	// Mock onStatusChange function
+	mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+	inst := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
 
 	// Get proxy for the first time
 	proxy1, err := inst.GetProxy()
@@ -180,7 +192,10 @@ func TestMarshalJSON(t *testing.T) {
 		},
 	}
 
-	instance := instance.NewInstance("test-instance", globalSettings, options)
+	// Mock onStatusChange function
+	mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+	instance := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
 
 	data, err := json.Marshal(instance)
 	if err != nil {
@@ -188,7 +203,7 @@ func TestMarshalJSON(t *testing.T) {
 	}
 
 	// Check that JSON contains expected fields
-	var result map[string]interface{}
+	var result map[string]any
 	err = json.Unmarshal(data, &result)
 	if err != nil {
 		t.Fatalf("JSON unmarshal failed: %v", err)
@@ -197,8 +212,8 @@ func TestMarshalJSON(t *testing.T) {
 	if result["name"] != "test-instance" {
 		t.Errorf("Expected name 'test-instance', got %v", result["name"])
 	}
-	if result["running"] != false {
+	if result["status"] != "stopped" {
-		t.Errorf("Expected running false, got %v", result["running"])
+		t.Errorf("Expected status 'stopped', got %v", result["status"])
 	}
 
 	// Check that options are included
@@ -218,7 +233,7 @@ func TestMarshalJSON(t *testing.T) {
 func TestUnmarshalJSON(t *testing.T) {
 	jsonData := `{
 		"name": "test-instance",
-		"running": true,
+		"status": "running",
 		"options": {
 			"model": "/path/to/model.gguf",
 			"port": 8080,
@@ -236,8 +251,8 @@ func TestUnmarshalJSON(t *testing.T) {
 	if inst.Name != "test-instance" {
 		t.Errorf("Expected name 'test-instance', got %q", inst.Name)
 	}
-	if !inst.Running {
+	if !inst.IsRunning() {
-		t.Error("Expected running to be true")
+		t.Error("Expected status to be running")
 	}
 
 	opts := inst.GetOptions()
@@ -303,7 +318,10 @@ func TestCreateInstanceOptionsValidation(t *testing.T) {
 				},
 			}
 
-			instance := instance.NewInstance("test", globalSettings, options)
+			// Mock onStatusChange function
+			mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+			instance := instance.NewInstance("test", globalSettings, options, mockOnStatusChange)
 			opts := instance.GetOptions()
 
 			if opts.MaxRestarts == nil {

pkg/instance/lifecycle.go

@@ -16,7 +16,7 @@ func (i *Process) Start() error {
 	i.mu.Lock()
 	defer i.mu.Unlock()
 
-	if i.Running {
+	if i.IsRunning() {
 		return fmt.Errorf("instance %s is already running", i.Name)
 	}
 
@@ -65,7 +65,7 @@ func (i *Process) Start() error {
 		return fmt.Errorf("failed to start instance %s: %w", i.Name, err)
 	}
 
-	i.Running = true
+	i.SetStatus(Running)
 
 	// Create channel for monitor completion signaling
 	i.monitorDone = make(chan struct{})
@@ -82,7 +82,7 @@ func (i *Process) Start() error {
 func (i *Process) Stop() error {
 	i.mu.Lock()
 
-	if !i.Running {
+	if !i.IsRunning() {
 		// Even if not running, cancel any pending restart
 		if i.restartCancel != nil {
 			i.restartCancel()
@@ -99,8 +99,8 @@ func (i *Process) Stop() error {
 		i.restartCancel = nil
 	}
 
-	// Set running to false first to signal intentional stop
+	// Set status to stopped first to signal intentional stop
-	i.Running = false
+	i.SetStatus(Stopped)
 
 	// Clean up the proxy
 	i.proxy = nil
@@ -145,7 +145,7 @@ func (i *Process) Stop() error {
 }
 
 func (i *Process) WaitForHealthy(timeout int) error {
-	if !i.Running {
+	if !i.IsRunning() {
 		return fmt.Errorf("instance %s is not running", i.Name)
 	}
 
@@ -227,12 +227,12 @@ func (i *Process) monitorProcess() {
 	i.mu.Lock()
 
 	// Check if the instance was intentionally stopped
-	if !i.Running {
+	if !i.IsRunning() {
 		i.mu.Unlock()
 		return
 	}
 
-	i.Running = false
+	i.SetStatus(Stopped)
 	i.logger.Close()
 
 	// Cancel any existing restart context since we're handling a new exit
@@ -257,6 +257,7 @@ func (i *Process) handleRestart() {
 	// Validate restart conditions and get safe parameters
 	shouldRestart, maxRestarts, restartDelay := i.validateRestartConditions()
 	if !shouldRestart {
+		i.SetStatus(Failed)
 		i.mu.Unlock()
 		return
 	}

pkg/instance/status.go

@@ -0,0 +1,70 @@
+package instance
+
+import (
+	"encoding/json"
+	"log"
+)
+
+// Enum for instance status
+type InstanceStatus int
+
+const (
+	Stopped InstanceStatus = iota
+	Running
+	Failed
+)
+
+var nameToStatus = map[string]InstanceStatus{
+	"stopped": Stopped,
+	"running": Running,
+	"failed":  Failed,
+}
+
+var statusToName = map[InstanceStatus]string{
+	Stopped: "stopped",
+	Running: "running",
+	Failed:  "failed",
+}
+
+func (p *Process) SetStatus(status InstanceStatus) {
+	oldStatus := p.Status
+	p.Status = status
+
+	if p.onStatusChange != nil {
+		p.onStatusChange(oldStatus, status)
+	}
+}
+
+func (p *Process) GetStatus() InstanceStatus {
+	return p.Status
+}
+
+// IsRunning returns true if the status is Running
+func (p *Process) IsRunning() bool {
+	return p.Status == Running
+}
+
+func (s InstanceStatus) MarshalJSON() ([]byte, error) {
+	name, ok := statusToName[s]
+	if !ok {
+		name = "stopped" // Default to "stopped" for unknown status
+	}
+	return json.Marshal(name)
+}
+
+// UnmarshalJSON implements json.Unmarshaler
+func (s *InstanceStatus) UnmarshalJSON(data []byte) error {
+	var str string
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+
+	status, ok := nameToStatus[str]
+	if !ok {
+		log.Printf("Unknown instance status: %s", str)
+		status = Stopped // Default to Stopped on unknown status
+	}
+
+	*s = status
+	return nil
+}

pkg/instance/timeout.go

@@ -13,7 +13,7 @@ func (i *Process) ShouldTimeout() bool {
 	i.mu.RLock()
 	defer i.mu.RUnlock()
 
-	if !i.Running || i.options.IdleTimeout == nil || *i.options.IdleTimeout <= 0 {
+	if !i.IsRunning() || i.options.IdleTimeout == nil || *i.options.IdleTimeout <= 0 {
 		return false
 	}
 

pkg/instance/timeout_test.go

@@ -42,7 +42,10 @@ func TestUpdateLastRequestTime(t *testing.T) {
 		},
 	}
 
-	inst := instance.NewInstance("test-instance", globalSettings, options)
+	// Mock onStatusChange function
+	mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+	inst := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
 
 	// Test that UpdateLastRequestTime doesn't panic
 	inst.UpdateLastRequestTime()
@@ -61,7 +64,10 @@ func TestShouldTimeout_NotRunning(t *testing.T) {
 		},
 	}
 
-	inst := instance.NewInstance("test-instance", globalSettings, options)
+	// Mock onStatusChange function
+	mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+	inst := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
 
 	// Instance is not running, should not timeout regardless of configuration
 	if inst.ShouldTimeout() {
@@ -85,6 +91,9 @@ func TestShouldTimeout_NoTimeoutConfigured(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			// Mock onStatusChange function
+			mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
 			options := &instance.CreateInstanceOptions{
 				IdleTimeout: tt.idleTimeout,
 				LlamaServerOptions: llamacpp.LlamaServerOptions{
@@ -92,9 +101,9 @@ func TestShouldTimeout_NoTimeoutConfigured(t *testing.T) {
 				},
 			}
 
-			inst := instance.NewInstance("test-instance", globalSettings, options)
+			inst := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
 			// Simulate running state
-			inst.Running = true
+			inst.SetStatus(instance.Running)
 
 			if inst.ShouldTimeout() {
 				t.Errorf("Instance with %s should not timeout", tt.name)
@@ -116,8 +125,11 @@ func TestShouldTimeout_WithinTimeLimit(t *testing.T) {
 		},
 	}
 
-	inst := instance.NewInstance("test-instance", globalSettings, options)
+	// Mock onStatusChange function
-	inst.Running = true
+	mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+	inst := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
+	inst.SetStatus(instance.Running)
 
 	// Update last request time to now
 	inst.UpdateLastRequestTime()
@@ -141,8 +153,11 @@ func TestShouldTimeout_ExceedsTimeLimit(t *testing.T) {
 		},
 	}
 
-	inst := instance.NewInstance("test-instance", globalSettings, options)
+	// Mock onStatusChange function
-	inst.Running = true
+	mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+	inst := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
+	inst.SetStatus(instance.Running)
 
 	// Use MockTimeProvider to simulate old last request time
 	mockTime := NewMockTimeProvider(time.Now())
@@ -184,7 +199,10 @@ func TestTimeoutConfiguration_Validation(t *testing.T) {
 				},
 			}
 
-			inst := instance.NewInstance("test-instance", globalSettings, options)
+			// Mock onStatusChange function
+			mockOnStatusChange := func(oldStatus, newStatus instance.InstanceStatus) {}
+
+			inst := instance.NewInstance("test-instance", globalSettings, options, mockOnStatusChange)
 			opts := inst.GetOptions()
 
 			if opts.IdleTimeout == nil || *opts.IdleTimeout != tt.expectedTimeout {

pkg/manager/manager.go

@@ -30,6 +30,7 @@ type InstanceManager interface {
 type instanceManager struct {
 	mu               sync.RWMutex
 	instances        map[string]*instance.Process
+	runningInstances map[string]struct{}
 	ports            map[int]bool
 	instancesConfig  config.InstancesConfig
 
@@ -47,6 +48,7 @@ func NewInstanceManager(instancesConfig config.InstancesConfig) InstanceManager
 	}
 	im := &instanceManager{
 		instances:        make(map[string]*instance.Process),
+		runningInstances: make(map[string]struct{}),
 		ports:            make(map[int]bool),
 		instancesConfig:  instancesConfig,
 
@@ -148,7 +150,7 @@ func (im *instanceManager) Shutdown() {
 	wg.Add(len(im.instances))
 
 	for name, inst := range im.instances {
-		if !inst.Running {
+		if !inst.IsRunning() {
 			wg.Done() // If instance is not running, just mark it as done
 			continue
 		}
@@ -227,12 +229,16 @@ func (im *instanceManager) loadInstance(name, path string) error {
 		return fmt.Errorf("instance name mismatch: file=%s, instance.Name=%s", name, persistedInstance.Name)
 	}
 
+	statusCallback := func(oldStatus, newStatus instance.InstanceStatus) {
+		im.onStatusChange(persistedInstance.Name, oldStatus, newStatus)
+	}
+
 	// Create new inst using NewInstance (handles validation, defaults, setup)
-	inst := instance.NewInstance(name, &im.instancesConfig, persistedInstance.GetOptions())
+	inst := instance.NewInstance(name, &im.instancesConfig, persistedInstance.GetOptions(), statusCallback)
 
 	// Restore persisted fields that NewInstance doesn't set
 	inst.Created = persistedInstance.Created
-	inst.Running = persistedInstance.Running
+	inst.SetStatus(persistedInstance.Status)
 
 	// Check for port conflicts and add to maps
 	if inst.GetOptions() != nil && inst.GetOptions().Port > 0 {
@@ -252,7 +258,7 @@ func (im *instanceManager) autoStartInstances() {
 	im.mu.RLock()
 	var instancesToStart []*instance.Process
 	for _, inst := range im.instances {
-		if inst.Running && // Was running when persisted
+		if inst.IsRunning() && // Was running when persisted
 			inst.GetOptions() != nil &&
 			inst.GetOptions().AutoRestart != nil &&
 			*inst.GetOptions().AutoRestart {
@@ -264,9 +270,20 @@ func (im *instanceManager) autoStartInstances() {
 	for _, inst := range instancesToStart {
 		log.Printf("Auto-starting instance %s", inst.Name)
 		// Reset running state before starting (since Start() expects stopped instance)
-		inst.Running = false
+		inst.SetStatus(instance.Stopped)
 		if err := inst.Start(); err != nil {
 			log.Printf("Failed to auto-start instance %s: %v", inst.Name, err)
 		}
 	}
 }
+
+func (im *instanceManager) onStatusChange(name string, oldStatus, newStatus instance.InstanceStatus) {
+	im.mu.Lock()
+	defer im.mu.Unlock()
+
+	if newStatus == instance.Running {
+		im.runningInstances[name] = struct{}{}
+	} else {
+		delete(im.runningInstances, name)
+	}
+}

pkg/manager/manager_test.go

@@ -59,7 +59,7 @@ func TestCreateInstance_Success(t *testing.T) {
 	if inst.Name != "test-instance" {
 		t.Errorf("Expected instance name 'test-instance', got %q", inst.Name)
 	}
-	if inst.Running {
+	if inst.GetStatus() != instance.Stopped {
 		t.Error("New instance should not be running")
 	}
 	if inst.GetOptions().Port != 8080 {
@@ -357,7 +357,7 @@ func TestTimeoutFunctionality(t *testing.T) {
 	inst.SetTimeProvider(mockTime)
 
 	// Set instance to running state so timeout logic can work
-	inst.Running = true
+	inst.SetStatus(instance.Running)
 
 	// Simulate instance being "running" for timeout check (without actual process)
 	// We'll test the ShouldTimeout logic directly
@@ -377,7 +377,7 @@ func TestTimeoutFunctionality(t *testing.T) {
 	}
 
 	// Reset running state to avoid shutdown issues
-	inst.Running = false
+	inst.SetStatus(instance.Stopped)
 
 	// Test that instance without timeout doesn't timeout
 	noTimeoutOptions := &instance.CreateInstanceOptions{
@@ -393,7 +393,7 @@ func TestTimeoutFunctionality(t *testing.T) {
 	}
 
 	noTimeoutInst.SetTimeProvider(mockTime)
-	noTimeoutInst.Running = true // Set to running for timeout check
+	noTimeoutInst.SetStatus(instance.Running) // Set to running for timeout check
 	noTimeoutInst.UpdateLastRequestTime()
 
 	// Even with time advanced, should not timeout
@@ -402,7 +402,7 @@ func TestTimeoutFunctionality(t *testing.T) {
 	}
 
 	// Reset running state to avoid shutdown issues
-	noTimeoutInst.Running = false
+	noTimeoutInst.SetStatus(instance.Stopped)
 }
 
 func TestConcurrentAccess(t *testing.T) {

pkg/manager/operations.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 )
 
+type MaxRunningInstancesError error
+
 // ListInstances returns a list of all instances managed by the instance manager.
 func (im *instanceManager) ListInstances() ([]*instance.Process, error) {
 	im.mu.RLock()
@@ -65,7 +67,11 @@ func (im *instanceManager) CreateInstance(name string, options *instance.CreateI
 		im.ports[options.Port] = true
 	}
 
-	inst := instance.NewInstance(name, &im.instancesConfig, options)
+	statusCallback := func(oldStatus, newStatus instance.InstanceStatus) {
+		im.onStatusChange(name, oldStatus, newStatus)
+	}
+
+	inst := instance.NewInstance(name, &im.instancesConfig, options, statusCallback)
 	im.instances[inst.Name] = inst
 	im.ports[options.Port] = true
 
@@ -109,7 +115,7 @@ func (im *instanceManager) UpdateInstance(name string, options *instance.CreateI
 	}
 
 	// Check if instance is running before updating options
-	wasRunning := instance.Running
+	wasRunning := instance.IsRunning()
 
 	// If the instance is running, stop it first
 	if wasRunning {
@@ -147,7 +153,7 @@ func (im *instanceManager) DeleteInstance(name string) error {
 		return fmt.Errorf("instance with name %s not found", name)
 	}
 
-	if instance.Running {
+	if instance.IsRunning() {
 		return fmt.Errorf("instance with name %s is still running, stop it before deleting", name)
 	}
 
@@ -173,10 +179,14 @@ func (im *instanceManager) StartInstance(name string) (*instance.Process, error)
 	if !exists {
 		return nil, fmt.Errorf("instance with name %s not found", name)
 	}
-	if instance.Running {
+	if instance.IsRunning() {
 		return instance, fmt.Errorf("instance with name %s is already running", name)
 	}
 
+	if len(im.runningInstances) >= im.instancesConfig.MaxRunningInstances && im.instancesConfig.MaxRunningInstances != -1 {
+		return nil, MaxRunningInstancesError(fmt.Errorf("maximum number of running instances (%d) reached", im.instancesConfig.MaxRunningInstances))
+	}
+
 	if err := instance.Start(); err != nil {
 		return nil, fmt.Errorf("failed to start instance %s: %w", name, err)
 	}
@@ -200,7 +210,7 @@ func (im *instanceManager) StopInstance(name string) (*instance.Process, error)
 	if !exists {
 		return nil, fmt.Errorf("instance with name %s not found", name)
 	}
-	if !instance.Running {
+	if !instance.IsRunning() {
 		return instance, fmt.Errorf("instance with name %s is already stopped", name)
 	}
 

pkg/server/handlers.go

@@ -272,6 +272,12 @@ func (h *Handler) StartInstance() http.HandlerFunc {
 
 		inst, err := h.InstanceManager.StartInstance(name)
 		if err != nil {
+			// Check if error is due to maximum running instances limit
+			if _, ok := err.(manager.MaxRunningInstancesError); ok {
+				http.Error(w, err.Error(), http.StatusConflict)
+				return
+			}
+
 			http.Error(w, "Failed to start instance: "+err.Error(), http.StatusInternalServerError)
 			return
 		}
@@ -451,7 +457,7 @@ func (h *Handler) ProxyToInstance() http.HandlerFunc {
 			return
 		}
 
-		if !inst.Running {
+		if !inst.IsRunning() {
 			http.Error(w, "Instance is not running", http.StatusServiceUnavailable)
 			return
 		}
@@ -574,7 +580,7 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc {
 			return
 		}
 
-		if !inst.Running {
+		if !inst.IsRunning() {
 			if inst.GetOptions().OnDemandStart != nil && *inst.GetOptions().OnDemandStart {
 				// If on-demand start is enabled, start the instance
 				if _, err := h.InstanceManager.StartInstance(modelName); err != nil {

webui/src/App.tsx

@@ -30,9 +30,9 @@ function App() {
 
   const handleSaveInstance = (name: string, options: CreateInstanceOptions) => {
     if (editingInstance) {
-      updateInstance(editingInstance.name, options);
+      void updateInstance(editingInstance.name, options);
     } else {
-      createInstance(name, options);
+      void createInstance(name, options);
     }
   };
 

webui/src/__tests__/App.test.tsx

@@ -46,8 +46,8 @@ function renderApp() {
 
 describe('App Component - Critical Business Logic Only', () => {
   const mockInstances: Instance[] = [
-    { name: 'test-instance-1', running: false, options: { model: 'model1.gguf' } },
+    { name: 'test-instance-1', status: 'stopped', options: { model: 'model1.gguf' } },
-    { name: 'test-instance-2', running: true, options: { model: 'model2.gguf' } }
+    { name: 'test-instance-2', status: 'running', options: { model: 'model2.gguf' } }
   ]
 
   beforeEach(() => {
@@ -81,7 +81,7 @@ describe('App Component - Critical Business Logic Only', () => {
       const user = userEvent.setup()
       const newInstance: Instance = {
         name: 'new-test-instance',
-        running: false,
+        status: 'stopped',
         options: { model: 'new-model.gguf' }
       }
       vi.mocked(instancesApi.create).mockResolvedValue(newInstance)
@@ -118,7 +118,7 @@ describe('App Component - Critical Business Logic Only', () => {
       const user = userEvent.setup()
       const updatedInstance: Instance = {
         name: 'test-instance-1',
-        running: false,
+        status: 'stopped',
         options: { model: 'updated-model.gguf' }
       }
       vi.mocked(instancesApi.update).mockResolvedValue(updatedInstance)

webui/src/components/HealthBadge.tsx

@@ -27,6 +27,8 @@ const HealthBadge: React.FC<HealthBadgeProps> = ({ health }) => {
         return <XCircle className="h-3 w-3" />;
       case "unknown":
         return <Loader2 className="h-3 w-3 animate-spin" />;
+      case "failed":
+        return <XCircle className="h-3 w-3" />;
     }
   };
 
@@ -40,6 +42,8 @@ const HealthBadge: React.FC<HealthBadgeProps> = ({ health }) => {
         return "destructive";
       case "unknown":
         return "secondary";
+      case "failed":
+        return "destructive";
     }
   };
 
@@ -53,6 +57,8 @@ const HealthBadge: React.FC<HealthBadgeProps> = ({ health }) => {
         return "Error";
       case "unknown":
         return "Unknown";
+      case "failed":
+        return "Failed";
     }
   };
 

webui/src/components/InstanceCard.tsx

@@ -24,7 +24,7 @@ function InstanceCard({
   editInstance,
 }: InstanceCardProps) {
   const [isLogsOpen, setIsLogsOpen] = useState(false);
-  const health = useInstanceHealth(instance.name, instance.running);
+  const health = useInstanceHealth(instance.name, instance.status);
 
   const handleStart = () => {
     startInstance(instance.name);
@@ -50,13 +50,15 @@ function InstanceCard({
     setIsLogsOpen(true);
   };
 
+  const running = instance.status === "running";
+
   return (
     <>
       <Card>
         <CardHeader className="pb-3">
           <div className="flex items-center justify-between">
             <CardTitle className="text-lg">{instance.name}</CardTitle>
-            {instance.running && <HealthBadge health={health} />}
+            {running && <HealthBadge health={health} />}
           </div>
         </CardHeader>
 
@@ -66,7 +68,7 @@ function InstanceCard({
               size="sm"
               variant="outline"
               onClick={handleStart}
-              disabled={instance.running}
+              disabled={running}
               title="Start instance"
               data-testid="start-instance-button"
             >
@@ -77,7 +79,7 @@ function InstanceCard({
               size="sm"
               variant="outline"
               onClick={handleStop}
-              disabled={!instance.running}
+              disabled={!running}
               title="Stop instance"
               data-testid="stop-instance-button"
             >
@@ -108,7 +110,7 @@ function InstanceCard({
               size="sm"
               variant="destructive"
               onClick={handleDelete}
-              disabled={instance.running}
+              disabled={running}
               title="Delete instance"
               data-testid="delete-instance-button"
             >
@@ -122,7 +124,7 @@ function InstanceCard({
         open={isLogsOpen}
         onOpenChange={setIsLogsOpen}
         instanceName={instance.name}
-        isRunning={instance.running}
+        isRunning={running}
       />
     </>
   );

webui/src/components/InstanceDialog.tsx

@@ -29,7 +29,6 @@ const InstanceDialog: React.FC<InstanceDialogProps> = ({
   instance,
 }) => {
   const isEditing = !!instance;
-  const isRunning = instance?.running || true; // Assume running if instance exists
 
   const [instanceName, setInstanceName] = useState("");
   const [formData, setFormData] = useState<CreateInstanceOptions>({});
@@ -114,6 +113,16 @@ const InstanceDialog: React.FC<InstanceDialogProps> = ({
   // Check if auto_restart is enabled
   const isAutoRestartEnabled = formData.auto_restart === true;
 
+  // Save button label logic
+  let saveButtonLabel = "Create Instance";
+  if (isEditing) {
+    if (instance?.status === "running") {
+      saveButtonLabel = "Update & Restart Instance";
+    } else {
+      saveButtonLabel = "Update Instance";
+    }
+  }
+
   return (
     <Dialog open={open} onOpenChange={onOpenChange}>
       <DialogContent className="sm:max-w-[600px] max-h-[80vh] overflow-hidden flex flex-col">
@@ -264,11 +273,7 @@ const InstanceDialog: React.FC<InstanceDialogProps> = ({
             disabled={!instanceName.trim() || !!nameError}
             data-testid="dialog-save-button"
           >
-            {isEditing
+            {saveButtonLabel}
-              ? isRunning
-                ? "Update & Restart Instance"
-                : "Update Instance"
-              : "Create Instance"}
           </Button>
         </DialogFooter>
       </DialogContent>

webui/src/components/__tests__/InstanceCard.test.tsx

@@ -17,13 +17,13 @@ describe('InstanceCard - Instance Actions and State', () => {
 
   const stoppedInstance: Instance = {
     name: 'test-instance',
-    running: false,
+    status: 'stopped',
     options: { model: 'test-model.gguf' }
   }
 
   const runningInstance: Instance = {
     name: 'running-instance',
-    running: true,
+    status: 'running',
     options: { model: 'running-model.gguf' }
   }
 
@@ -301,7 +301,7 @@ afterEach(() => {
     it('handles instance with minimal data', () => {
       const minimalInstance: Instance = {
         name: 'minimal',
-        running: false,
+        status: 'stopped',
         options: {}
       }
 
@@ -323,7 +323,7 @@ afterEach(() => {
     it('handles instance with undefined options', () => {
       const instanceWithoutOptions: Instance = {
         name: 'no-options',
-        running: true,
+        status: 'running',
         options: undefined
       }
 

webui/src/components/__tests__/InstanceList.test.tsx

@@ -44,9 +44,9 @@ describe('InstanceList - State Management and UI Logic', () => {
   const mockEditInstance = vi.fn()
 
   const mockInstances: Instance[] = [
-    { name: 'instance-1', running: false, options: { model: 'model1.gguf' } },
+    { name: 'instance-1', status: 'stopped', options: { model: 'model1.gguf' } },
-    { name: 'instance-2', running: true, options: { model: 'model2.gguf' } },
+    { name: 'instance-2', status: 'running', options: { model: 'model2.gguf' } },
-    { name: 'instance-3', running: false, options: { model: 'model3.gguf' } }
+    { name: 'instance-3', status: 'stopped', options: { model: 'model3.gguf' } }
   ]
 
   const DUMMY_API_KEY = 'test-api-key-123'

webui/src/components/__tests__/InstanceModal.test.tsx

@@ -134,7 +134,7 @@ afterEach(() => {
   describe('Edit Mode', () => {
     const mockInstance: Instance = {
       name: 'existing-instance',
-      running: false,
+      status: 'stopped',
       options: {
         model: 'test-model.gguf',
         gpu_layers: 10,
@@ -184,7 +184,7 @@ afterEach(() => {
     })
 
     it('shows correct button text for running vs stopped instances', () => {
-      const runningInstance: Instance = { ...mockInstance, running: true }
+      const runningInstance: Instance = { ...mockInstance, status: 'running' }
 
       const { rerender } = render(
         <InstanceDialog

webui/src/contexts/InstancesContext.tsx

@@ -113,8 +113,8 @@ export const InstancesProvider = ({ children }: InstancesProviderProps) => {
       setError(null)
       await instancesApi.start(name)
 
-      // Update only this instance's running status
+      // Update only this instance's status
-      updateInstanceInMap(name, { running: true })
+      updateInstanceInMap(name, { status: "running" })
     } catch (err) {
       setError(err instanceof Error ? err.message : 'Failed to start instance')
     }
@@ -125,8 +125,8 @@ export const InstancesProvider = ({ children }: InstancesProviderProps) => {
       setError(null)
       await instancesApi.stop(name)
 
-      // Update only this instance's running status
+      // Update only this instance's status
-      updateInstanceInMap(name, { running: false })
+      updateInstanceInMap(name, { status: "stopped" })
     } catch (err) {
       setError(err instanceof Error ? err.message : 'Failed to stop instance')
     }
@@ -137,8 +137,8 @@ export const InstancesProvider = ({ children }: InstancesProviderProps) => {
       setError(null)
       await instancesApi.restart(name)
 
-      // Update only this instance's running status
+      // Update only this instance's status
-      updateInstanceInMap(name, { running: true })
+      updateInstanceInMap(name, { status: "running" })
     } catch (err) {
       setError(err instanceof Error ? err.message : 'Failed to restart instance')
     }

webui/src/contexts/__tests__/InstancesContext.test.tsx

@@ -41,7 +41,7 @@ function TestComponent() {
       <div data-testid="instances-count">{instances.length}</div>
       {instances.map((instance) => (
         <div key={instance.name} data-testid={`instance-${instance.name}`}>
-          {instance.name}:{instance.running.toString()}
+          {instance.name}:{instance.status}
         </div>
       ))}
 
@@ -99,8 +99,8 @@ function renderWithProvider(children: ReactNode) {
 
 describe("InstancesContext", () => {
   const mockInstances: Instance[] = [
-    { name: "instance1", running: true, options: { model: "model1.gguf" } },
+    { name: "instance1", status: "running", options: { model: "model1.gguf" } },
-    { name: "instance2", running: false, options: { model: "model2.gguf" } },
+    { name: "instance2", status: "stopped", options: { model: "model2.gguf" } },
   ];
 
   beforeEach(() => {
@@ -132,10 +132,10 @@ describe("InstancesContext", () => {
         expect(screen.getByTestId("loading")).toHaveTextContent("false");
         expect(screen.getByTestId("instances-count")).toHaveTextContent("2");
         expect(screen.getByTestId("instance-instance1")).toHaveTextContent(
-          "instance1:true"
+          "instance1:running"
         );
         expect(screen.getByTestId("instance-instance2")).toHaveTextContent(
-          "instance2:false"
+          "instance2:stopped"
         );
       });
     });
@@ -158,7 +158,7 @@ describe("InstancesContext", () => {
     it("creates instance and adds it to state", async () => {
       const newInstance: Instance = {
         name: "new-instance",
-        running: false,
+        status: "stopped",
         options: { model: "test.gguf" },
       };
       vi.mocked(instancesApi.create).mockResolvedValue(newInstance);
@@ -181,7 +181,7 @@ describe("InstancesContext", () => {
       await waitFor(() => {
         expect(screen.getByTestId("instances-count")).toHaveTextContent("3");
         expect(screen.getByTestId("instance-new-instance")).toHaveTextContent(
-          "new-instance:false"
+          "new-instance:stopped"
         );
       });
     });
@@ -214,7 +214,7 @@ describe("InstancesContext", () => {
     it("updates instance and maintains it in state", async () => {
       const updatedInstance: Instance = {
         name: "instance1",
-        running: true,
+        status: "running",
         options: { model: "updated.gguf" },
       };
       vi.mocked(instancesApi.update).mockResolvedValue(updatedInstance);
@@ -251,7 +251,7 @@ describe("InstancesContext", () => {
         expect(screen.getByTestId("loading")).toHaveTextContent("false");
         // instance2 starts as not running
         expect(screen.getByTestId("instance-instance2")).toHaveTextContent(
-          "instance2:false"
+          "instance2:stopped"
         );
       });
 
@@ -262,7 +262,7 @@ describe("InstancesContext", () => {
         expect(instancesApi.start).toHaveBeenCalledWith("instance2");
         // The running state should be updated to true
         expect(screen.getByTestId("instance-instance2")).toHaveTextContent(
-          "instance2:true"
+          "instance2:running"
         );
       });
     });
@@ -276,7 +276,7 @@ describe("InstancesContext", () => {
         expect(screen.getByTestId("loading")).toHaveTextContent("false");
         // instance1 starts as running
         expect(screen.getByTestId("instance-instance1")).toHaveTextContent(
-          "instance1:true"
+          "instance1:running"
         );
       });
 
@@ -287,7 +287,7 @@ describe("InstancesContext", () => {
         expect(instancesApi.stop).toHaveBeenCalledWith("instance1");
         // The running state should be updated to false
         expect(screen.getByTestId("instance-instance1")).toHaveTextContent(
-          "instance1:false"
+          "instance1:stopped"
         );
       });
     });
@@ -383,7 +383,7 @@ describe("InstancesContext", () => {
       // Test that operations don't interfere with each other
       const newInstance: Instance = {
         name: "new-instance",
-        running: false,
+        status: "stopped",
         options: {},
       };
       vi.mocked(instancesApi.create).mockResolvedValue(newInstance);
@@ -411,7 +411,7 @@ describe("InstancesContext", () => {
         expect(screen.getByTestId("instances-count")).toHaveTextContent("3"); // Still 3
         // But the running state should change
         expect(screen.getByTestId("instance-instance2")).toHaveTextContent(
-          "instance2:true"
+          "instance2:running"
         );
       });
     });

webui/src/hooks/useInstanceHealth.ts

@@ -1,14 +1,19 @@
 // ui/src/hooks/useInstanceHealth.ts
 import { useState, useEffect } from 'react'
-import type { HealthStatus } from '@/types/instance'
+import type { HealthStatus, InstanceStatus } from '@/types/instance'
 import { healthService } from '@/lib/healthService'
 
-export function useInstanceHealth(instanceName: string, isRunning: boolean): HealthStatus | undefined {
+export function useInstanceHealth(instanceName: string, instanceStatus: InstanceStatus): HealthStatus | undefined {
   const [health, setHealth] = useState<HealthStatus | undefined>()
 
   useEffect(() => {
-    if (!isRunning) {
+    if (instanceStatus === "stopped") {
-      setHealth(undefined)
+      setHealth({ status: "unknown", lastChecked: new Date() })
+      return
+    }
+
+    if (instanceStatus === "failed") {
+      setHealth({ status: instanceStatus, lastChecked: new Date() })
       return
     }
 
@@ -17,9 +22,9 @@ export function useInstanceHealth(instanceName: string, isRunning: boolean): Hea
       setHealth(healthStatus)
     })
 
-    // Cleanup subscription on unmount or when running changes
+    // Cleanup subscription on unmount or when instanceStatus changes
     return unsubscribe
-  }, [instanceName, isRunning])
+  }, [instanceName, instanceStatus])
 
   return health
 }

webui/src/types/instance.ts

@@ -2,14 +2,16 @@ import type { CreateInstanceOptions } from '@/schemas/instanceOptions'
 
 export { type CreateInstanceOptions } from '@/schemas/instanceOptions'
 
+export type InstanceStatus = 'running' | 'stopped' | 'failed'
+
 export interface HealthStatus {
-  status: 'ok' | 'loading' | 'error' | 'unknown'
+  status: 'ok' | 'loading' | 'error' | 'unknown' | 'failed'
   message?: string
   lastChecked: Date
 }
 
 export interface Instance {
   name: string;
-  running: boolean;
+  status: InstanceStatus;
   options?: CreateInstanceOptions;
 }