diff --git a/README.md b/README.md index 42a3a6e..795317b 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,9 @@ instances: default_on_demand_start: true # Default on-demand start setting on_demand_start_timeout: 120 # Default on-demand start timeout in seconds timeout_check_interval: 5 # Idle instance timeout check in minutes + log_rotation_enabled: true # Enable log rotation (default: true) + log_rotation_max_size: 100 # Max log file size in MB before rotation (default: 100) + log_rotation_compress: false # Compress rotated log files (default: false) database: path: ~/.local/share/llamactl/llamactl.db # Database file path (platform dependent) diff --git a/docs/configuration.md b/docs/configuration.md index 41489c1..5254b4e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -230,6 +230,9 @@ instances: default_on_demand_start: true # Default on-demand start setting on_demand_start_timeout: 120 # Default on-demand start timeout in seconds timeout_check_interval: 5 # Default instance timeout check interval in minutes + log_rotation_enabled: true # Enable log rotation (default: true) + log_rotation_max_size: 100 # Max log file size in MB before rotation (default: 100) + log_rotation_compress: false # Compress rotated log files (default: false) ``` **Environment Variables:** @@ -246,6 +249,9 @@ instances: - `LLAMACTL_DEFAULT_ON_DEMAND_START` - Default on-demand start setting (true/false) - `LLAMACTL_ON_DEMAND_START_TIMEOUT` - Default on-demand start timeout in seconds - `LLAMACTL_TIMEOUT_CHECK_INTERVAL` - Default instance timeout check interval in minutes +- `LLAMACTL_LOG_ROTATION_ENABLED` - Enable log rotation (true/false) +- `LLAMACTL_LOG_ROTATION_MAX_SIZE` - Max log file size in MB +- `LLAMACTL_LOG_ROTATION_COMPRESS` - Compress rotated logs (true/false) ### Database Configuration diff --git a/go.mod b/go.mod index 4bbcc21..100d144 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module llamactl go 1.24.5 require ( + github.com/DeRuina/timberjack v1.3.9 github.com/go-chi/chi/v5 v5.2.2 github.com/go-chi/cors v1.2.2 github.com/golang-migrate/migrate/v4 v4.19.1 @@ -20,6 +21,7 @@ require ( github.com/go-openapi/spec v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/swaggo/files v1.0.1 // indirect golang.org/x/mod v0.29.0 // indirect diff --git a/go.sum b/go.sum index 589e961..9de8889 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ +github.com/DeRuina/timberjack v1.3.9 h1:6UXZ1I7ExPGTX/1UNYawR58LlOJUHKBPiYC7WQ91eBo= +github.com/DeRuina/timberjack v1.3.9/go.mod h1:RLoeQrwrCGIEF8gO5nV5b/gMD0QIy7bzQhBUgpp1EqE= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= @@ -20,6 +24,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/pkg/config/config.go b/pkg/config/config.go index 0f49f58..869ac67 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -96,9 +96,6 @@ type InstancesConfig struct { // Instance config directory override (relative to data_dir if not absolute) InstancesDir string `yaml:"configs_dir" json:"configs_dir"` - // Logs directory override (relative to data_dir if not absolute) - LogsDir string `yaml:"logs_dir" json:"logs_dir"` - // Automatically create the data directory if it doesn't exist AutoCreateDirs bool `yaml:"auto_create_dirs" json:"auto_create_dirs"` @@ -128,6 +125,18 @@ type InstancesConfig struct { // Interval for checking instance timeouts (in minutes) TimeoutCheckInterval int `yaml:"timeout_check_interval" json:"timeout_check_interval"` + + // Logs directory override (relative to data_dir if not absolute) + LogsDir string `yaml:"logs_dir" json:"logs_dir"` + + // Log rotation enabled + LogRotationEnabled bool `yaml:"log_rotation_enabled" default:"true"` + + // Maximum log file size in MB before rotation + LogRotationMaxSize int `yaml:"log_rotation_max_size" default:"100"` + + // Whether to compress rotated log files + LogRotationCompress bool `yaml:"log_rotation_compress" default:"false"` } // AuthConfig contains authentication settings @@ -205,10 +214,9 @@ func LoadConfig(configPath string) (AppConfig, error) { }, Instances: InstancesConfig{ PortRange: [2]int{8000, 9000}, - // NOTE: empty strings are set as placeholder values since InstancesDir and LogsDir + // NOTE: empty string is set as placeholder value since InstancesDir // should be relative path to DataDir if not explicitly set. InstancesDir: "", - LogsDir: "", AutoCreateDirs: true, MaxInstances: -1, // -1 means unlimited MaxRunningInstances: -1, // -1 means unlimited @@ -219,6 +227,10 @@ func LoadConfig(configPath string) (AppConfig, error) { DefaultOnDemandStart: true, OnDemandStartTimeout: 120, // 2 minutes TimeoutCheckInterval: 5, // Check timeouts every 5 minutes + LogsDir: "", // Will be set to data_dir/logs if empty + LogRotationEnabled: true, + LogRotationMaxSize: 100, + LogRotationCompress: false, }, Database: DatabaseConfig{ Path: "", // Will be set to data_dir/llamactl.db if empty @@ -549,6 +561,23 @@ func loadEnvVars(cfg *AppConfig) { cfg.Database.ConnMaxLifetime = d } } + + // Log rotation config + if logRotationEnabled := os.Getenv("LLAMACTL_LOG_ROTATION_ENABLED"); logRotationEnabled != "" { + if b, err := strconv.ParseBool(logRotationEnabled); err == nil { + cfg.Instances.LogRotationEnabled = b + } + } + if logRotationMaxSize := os.Getenv("LLAMACTL_LOG_ROTATION_MAX_SIZE"); logRotationMaxSize != "" { + if m, err := strconv.Atoi(logRotationMaxSize); err == nil { + cfg.Instances.LogRotationMaxSize = m + } + } + if logRotationCompress := os.Getenv("LLAMACTL_LOG_ROTATION_COMPRESS"); logRotationCompress != "" { + if b, err := strconv.ParseBool(logRotationCompress); err == nil { + cfg.Instances.LogRotationCompress = b + } + } } // ParsePortRange parses port range from string formats like "8000-9000" or "8000,9000" diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 5be2199..e90a24f 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -78,8 +78,8 @@ server: port: 9090 instances: port_range: [7000, 8000] - logs_dir: "/custom/logs" max_instances: 5 + logs_dir: "/custom/logs" llama_executable: "/usr/bin/llama-server" default_auto_restart: false default_max_restarts: 10 @@ -219,7 +219,6 @@ instances: } } - func TestParsePortRange(t *testing.T) { tests := []struct { name string @@ -248,7 +247,6 @@ func TestParsePortRange(t *testing.T) { } } - func TestGetBackendSettings_NewStructuredConfig(t *testing.T) { bc := &config.BackendConfig{ LlamaCpp: config.BackendSettings{ @@ -305,7 +303,6 @@ func TestGetBackendSettings_NewStructuredConfig(t *testing.T) { } } - func TestLoadConfig_BackendEnvironmentVariables(t *testing.T) { // Test that backend environment variables work correctly envVars := map[string]string{ @@ -375,7 +372,6 @@ func TestLoadConfig_BackendEnvironmentVariables(t *testing.T) { } } - func TestLoadConfig_LocalNode(t *testing.T) { t.Run("default local node", func(t *testing.T) { cfg, err := config.LoadConfig("nonexistent-file.yaml") diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index 465cd5e..a8faa1b 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -3,10 +3,11 @@ package instance import ( "encoding/json" "fmt" - "llamactl/pkg/config" "log" "net/http" "time" + + "llamactl/pkg/config" ) // Instance represents a running instance of llama server @@ -68,7 +69,16 @@ func New(name string, globalConfig *config.AppConfig, opts *Options, onStatusCha // Only create logger, proxy, and process for local instances if !instance.IsRemote() { - instance.logger = newLogger(name, globalInstanceSettings.LogsDir) + logRotationConfig := &LogRotationConfig{ + Enabled: globalInstanceSettings.LogRotationEnabled, + MaxSize: globalInstanceSettings.LogRotationMaxSize, + Compress: globalInstanceSettings.LogRotationCompress, + } + instance.logger = newLogger( + name, + globalInstanceSettings.LogsDir, + logRotationConfig, + ) instance.process = newProcess(instance) } diff --git a/pkg/instance/instance_test.go b/pkg/instance/instance_test.go index 122cc96..e7ee3e3 100644 --- a/pkg/instance/instance_test.go +++ b/pkg/instance/instance_test.go @@ -27,8 +27,8 @@ func TestNewInstance(t *testing.T) { }, }, Instances: config.InstancesConfig{ - LogsDir: "/tmp/test", DefaultAutoRestart: true, + LogsDir: "/tmp/test", DefaultMaxRestarts: 3, DefaultRestartDelay: 5, }, @@ -120,8 +120,8 @@ func TestSetOptions(t *testing.T) { }, }, Instances: config.InstancesConfig{ - LogsDir: "/tmp/test", DefaultAutoRestart: true, + LogsDir: "/tmp/test", DefaultMaxRestarts: 3, DefaultRestartDelay: 5, }, diff --git a/pkg/instance/logger.go b/pkg/instance/logger.go index 681875b..8575d7c 100644 --- a/pkg/instance/logger.go +++ b/pkg/instance/logger.go @@ -7,66 +7,117 @@ import ( "os" "strings" "sync" - "sync/atomic" "time" + + timber "github.com/DeRuina/timberjack" ) +// LogRotationConfig contains log rotation settings for instances +type LogRotationConfig struct { + Enabled bool + MaxSize int + Compress bool +} + type logger struct { name string logDir string - logFile atomic.Pointer[os.File] + logFile *timber.Logger logFilePath string mu sync.RWMutex + cfg *LogRotationConfig } -func newLogger(name string, logDir string) *logger { +func newLogger(name, logDir string, cfg *LogRotationConfig) *logger { return &logger{ name: name, logDir: logDir, + cfg: cfg, } } -// create creates and opens the log files for stdout and stderr -func (i *logger) create() error { - i.mu.Lock() - defer i.mu.Unlock() +func (l *logger) create() error { + l.mu.Lock() + defer l.mu.Unlock() - if i.logDir == "" { - return fmt.Errorf("logDir is empty for instance %s", i.name) + if l.logDir == "" { + return fmt.Errorf("logDir empty for instance %s", l.name) } - // Set up instance logs - logPath := i.logDir + "/" + i.name + ".log" - - i.logFilePath = logPath - if err := os.MkdirAll(i.logDir, 0755); err != nil { + if err := os.MkdirAll(l.logDir, 0755); err != nil { return fmt.Errorf("failed to create log directory: %w", err) } - logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - return fmt.Errorf("failed to create stdout log file: %w", err) + logPath := fmt.Sprintf("%s/%s.log", l.logDir, l.name) + l.logFilePath = logPath + + // Build the timber logger + t := &timber.Logger{ + Filename: logPath, + MaxSize: l.cfg.MaxSize, + MaxBackups: 0, // No limit on backups + // Compression: "gzip" if Compress is true, else "none" + Compression: func() string { + if l.cfg.Compress { + return "gzip" + } + return "none" + }(), + FileMode: 0644, + LocalTime: true, } - i.logFile.Store(logFile) + // If rotation is disabled, set MaxSize to 0 so no rotation occurs + if !l.cfg.Enabled { + t.MaxSize = 0 + } - // Write a startup marker to both files - timestamp := time.Now().Format("2006-01-02 15:04:05") - fmt.Fprintf(logFile, "\n=== Instance %s started at %s ===\n", i.name, timestamp) + l.logFile = t + + // Write a startup marker + ts := time.Now().Format("2006-01-02 15:04:05") + fmt.Fprintf(t, "\n=== Instance %s started at %s ===\n", l.name, ts) return nil } -// getLogs retrieves the last n lines of logs from the instance -func (i *logger) getLogs(num_lines int) (string, error) { - i.mu.RLock() - defer i.mu.RUnlock() +func (l *logger) readOutput(rc io.ReadCloser) { + defer rc.Close() + scanner := bufio.NewScanner(rc) + for scanner.Scan() { + line := scanner.Text() + if lg := l.logFile; lg != nil { + fmt.Fprintln(lg, line) + } + } +} - if i.logFilePath == "" { - return "", fmt.Errorf("log file not created for instance %s", i.name) +func (l *logger) close() { + l.mu.Lock() + defer l.mu.Unlock() + + lg := l.logFile + if lg == nil { + return } - file, err := os.Open(i.logFilePath) + ts := time.Now().Format("2006-01-02 15:04:05") + fmt.Fprintf(lg, "=== Instance %s stopped at %s ===\n\n", l.name, ts) + + _ = lg.Close() + l.logFile = nil +} + +// getLogs retrieves the last n lines of logs from the instance +func (l *logger) getLogs(num_lines int) (string, error) { + l.mu.RLock() + defer l.mu.RUnlock() + + if l.logFilePath == "" { + return "", fmt.Errorf("log file not created for instance %s", l.name) + } + + file, err := os.Open(l.logFilePath) if err != nil { return "", fmt.Errorf("failed to open log file: %w", err) } @@ -97,31 +148,3 @@ func (i *logger) getLogs(num_lines int) (string, error) { return strings.Join(lines[start:], "\n"), nil } - -// close closes the log files -func (i *logger) close() { - i.mu.Lock() - defer i.mu.Unlock() - - logFile := i.logFile.Swap(nil) - if logFile != nil { - timestamp := time.Now().Format("2006-01-02 15:04:05") - fmt.Fprintf(logFile, "=== Instance %s stopped at %s ===\n\n", i.name, timestamp) - logFile.Sync() // Ensure all buffered data is written to disk - logFile.Close() - } -} - -// readOutput reads from the given reader and writes lines to the log file -func (i *logger) readOutput(reader io.ReadCloser) { - defer reader.Close() - - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - line := scanner.Text() - // Use atomic load to avoid lock contention on every line - if logFile := i.logFile.Load(); logFile != nil { - fmt.Fprintln(logFile, line) - } - } -} diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index ba6a109..78ab04d 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -203,11 +203,11 @@ func createTestAppConfig(instancesDir string) *config.AppConfig { Instances: config.InstancesConfig{ PortRange: [2]int{8000, 9000}, InstancesDir: instancesDir, - LogsDir: instancesDir, MaxInstances: 10, MaxRunningInstances: 10, DefaultAutoRestart: true, DefaultMaxRestarts: 3, + LogsDir: instancesDir, DefaultRestartDelay: 5, TimeoutCheckInterval: 5, },