diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bb48f8b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,45 @@ +# Git and version control +.git/ +.gitignore + +# Documentation +*.md +docs/ + +# Development files +.vscode/ +.idea/ + +# Build artifacts +webui/node_modules/ +webui/dist/ +webui/.next/ +*.log +*.tmp + +# Data directories +data/ +models/ +logs/ + +# Test files +*_test.go +**/*_test.go + +# CI/CD +.github/ + +# Local configuration +llamactl.yaml +config.yaml +.env +.env.local + +# OS files +.DS_Store +Thumbs.db + +# Backup files +*.bak +*.backup +*~ \ No newline at end of file diff --git a/README.md b/README.md index dc68e4f..7b917e6 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,30 @@ sudo mv llamactl /usr/local/bin/ # Windows - Download from releases page ``` -### Option 2: Build from Source +### Option 2: Docker (No local backend installation required) + +```bash +# Clone repository and build Docker images +git clone https://github.com/lordmathis/llamactl.git +cd llamactl +mkdir -p data/llamacpp data/vllm models + +# Build and start llamactl with llama.cpp CUDA backend +docker-compose -f docker/docker-compose.yml up llamactl-llamacpp -d + +# Build and start llamactl with vLLM CUDA backend +docker-compose -f docker/docker-compose.yml up llamactl-vllm -d + +# Build from source using multi-stage build +docker build -f docker/Dockerfile.source -t llamactl:source . +``` + +**Features:** CUDA support, automatic latest release installation, no backend dependencies. +**Note:** Dockerfiles are configured for CUDA. Adapt base images for other platforms (CPU, ROCm, etc.). + +For detailed Docker setup and configuration, see the [Installation Guide](docs/getting-started/installation.md). + +### Option 3: Build from Source Requires Go 1.24+ and Node.js 22+ ```bash git clone https://github.com/lordmathis/llamactl.git @@ -147,9 +170,9 @@ pip install vllm # Or use Docker - no local installation required ``` -## Docker Support +## Backend Docker Support -llamactl supports running backends in Docker containers - perfect for production deployments without local backend installation. Simply enable Docker in your configuration: +llamactl can run backends in Docker containers: ```yaml backends: diff --git a/docker/Dockerfile.llamacpp b/docker/Dockerfile.llamacpp new file mode 100644 index 0000000..f06155b --- /dev/null +++ b/docker/Dockerfile.llamacpp @@ -0,0 +1,23 @@ +FROM ghcr.io/ggml-org/llama.cpp:server-cuda + +# Install curl for downloading llamactl +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* + +# Download and install the latest llamactl release +RUN LATEST_VERSION=$(curl -s https://api.github.com/repos/lordmathis/llamactl/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') && \ + curl -L "https://github.com/lordmathis/llamactl/releases/download/${LATEST_VERSION}/llamactl-${LATEST_VERSION}-linux-amd64.tar.gz" | tar -xz && \ + mv llamactl /usr/local/bin/ && \ + chmod +x /usr/local/bin/llamactl + +# Set working directory +RUN mkdir -p /data +WORKDIR /data + +# Expose the default llamactl port +EXPOSE 8080 + +ENV LLAMACTL_LLAMACPP_COMMAND=/app/llama-server +ENV LD_LIBRARY_PATH="/app:/usr/local/lib:/usr/lib" + +# Set llamactl as the entrypoint +ENTRYPOINT ["llamactl"] \ No newline at end of file diff --git a/docker/Dockerfile.source b/docker/Dockerfile.source new file mode 100644 index 0000000..2b4482f --- /dev/null +++ b/docker/Dockerfile.source @@ -0,0 +1,64 @@ +# WebUI build stage +FROM node:20-alpine AS webui-builder + +WORKDIR /webui + +# Copy webui package files +COPY webui/package*.json ./ + +# Install dependencies +RUN npm ci + +# Copy webui source +COPY webui/ ./ + +# Build webui +RUN npm run build + +# Go build stage +FROM golang:1.24-alpine AS builder + +# Install build dependencies +RUN apk add --no-cache git ca-certificates + +# Set working directory +WORKDIR /build + +# Copy go mod files +COPY go.mod go.sum ./ + +# Download dependencies +RUN go mod download + +# Copy source code +COPY cmd/ ./cmd/ +COPY pkg/ ./pkg/ +COPY apidocs/ ./apidocs/ +COPY webui/webui.go ./webui/ + +# Copy built webui from webui-builder +COPY --from=webui-builder /webui/dist ./webui/dist + +# Build the application +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags="-w -s" -o llamactl ./cmd/server + +# Final stage +FROM alpine:latest + +# Install runtime dependencies +RUN apk --no-cache add ca-certificates + +# Create data directory +RUN mkdir -p /data + +# Set working directory +WORKDIR /data + +# Copy binary from builder +COPY --from=builder /build/llamactl /usr/local/bin/llamactl + +# Expose the default port +EXPOSE 8080 + +# Set llamactl as the entrypoint +ENTRYPOINT ["llamactl"] \ No newline at end of file diff --git a/docker/Dockerfile.vllm b/docker/Dockerfile.vllm new file mode 100644 index 0000000..3886a07 --- /dev/null +++ b/docker/Dockerfile.vllm @@ -0,0 +1,20 @@ +FROM vllm/vllm-openai:latest + +# Install curl for downloading llamactl +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* + +# Download and install the latest llamactl release +RUN LATEST_VERSION=$(curl -s https://api.github.com/repos/lordmathis/llamactl/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') && \ + curl -L "https://github.com/lordmathis/llamactl/releases/download/${LATEST_VERSION}/llamactl-${LATEST_VERSION}-linux-amd64.tar.gz" | tar -xz && \ + mv llamactl /usr/local/bin/ && \ + chmod +x /usr/local/bin/llamactl + +# Set working directory +RUN mkdir -p /data +WORKDIR /data + +# Expose the default llamactl port +EXPOSE 8080 + +# Set llamactl as the entrypoint +ENTRYPOINT ["llamactl"] \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..d1f9e5b --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3.8' + +services: + llamactl-llamacpp: + build: + context: .. + dockerfile: docker/Dockerfile.llamacpp + image: llamactl:llamacpp-cuda + container_name: llamactl-llamacpp + ports: + - "8080:8080" + volumes: + - ./data/llamacpp:/data + - ./models:/models # Mount models directory + - ~/.cache/llama.cpp:/root/.cache/llama.cpp # Llama.cpp cache + environment: + # Set data directory for persistence + - LLAMACTL_DATA_DIR=/data + # Enable Docker mode for nested containers (if needed) + - LLAMACTL_LLAMACPP_DOCKER_ENABLED=false + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + restart: unless-stopped + + llamactl-vllm: + build: + context: .. + dockerfile: docker/Dockerfile.vllm + image: llamactl:vllm-cuda + container_name: llamactl-vllm + ports: + - "8081:8080" # Use different port to avoid conflicts + volumes: + - ./data/vllm:/data + - ./models:/models # Mount models directory + - ~/.cache/huggingface:/root/.cache/huggingface # HuggingFace cache + environment: + # Set data directory for persistence + - LLAMACTL_DATA_DIR=/data + # Enable Docker mode for nested containers (if needed) + - LLAMACTL_VLLM_DOCKER_ENABLED=false + # vLLM specific environment variables + - CUDA_VISIBLE_DEVICES=all + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + restart: unless-stopped diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 6f52fff..f64146f 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -71,7 +71,72 @@ sudo mv llamactl /usr/local/bin/ # Windows - Download from releases page ``` -### Option 2: Build from Source +### Option 2: Docker + +llamactl provides Dockerfiles for creating Docker images with backends pre-installed. The resulting images include the latest llamactl release with the respective backend. + +**Available Dockerfiles (CUDA):** +- **llamactl with llama.cpp CUDA**: `docker/Dockerfile.llamacpp` (based on `ghcr.io/ggml-org/llama.cpp:server-cuda`) +- **llamactl with vLLM CUDA**: `docker/Dockerfile.vllm` (based on `vllm/vllm-openai:latest`) +- **llamactl built from source**: `docker/Dockerfile.source` (multi-stage build with webui) + +**Note:** These Dockerfiles are configured for CUDA. For other platforms (CPU, ROCm, Vulkan, etc.), adapt the base image. For llama.cpp, see available tags at [llama.cpp Docker docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/docker.md). For vLLM, check [vLLM docs](https://docs.vllm.ai/en/v0.6.5/serving/deploying_with_docker.html). + +#### Using Docker Compose + +```bash +# Clone the repository +git clone https://github.com/lordmathis/llamactl.git +cd llamactl + +# Create directories for data and models +mkdir -p data/llamacpp data/vllm models + +# Start llamactl with llama.cpp backend +docker-compose -f docker/docker-compose.yml up llamactl-llamacpp -d + +# Or start llamactl with vLLM backend +docker-compose -f docker/docker-compose.yml up llamactl-vllm -d +``` + +Access the dashboard at: +- llamactl with llama.cpp: http://localhost:8080 +- llamactl with vLLM: http://localhost:8081 + +#### Using Docker Build and Run + +**llamactl with llama.cpp CUDA:** +```bash +docker build -f docker/Dockerfile.llamacpp -t llamactl:llamacpp-cuda . +docker run -d \ + --name llamactl-llamacpp \ + --gpus all \ + -p 8080:8080 \ + -v ~/.cache/llama.cpp:/root/.cache/llama.cpp \ + llamactl:llamacpp-cuda +``` + +**llamactl with vLLM CUDA:** +```bash +docker build -f docker/Dockerfile.vllm -t llamactl:vllm-cuda . +docker run -d \ + --name llamactl-vllm \ + --gpus all \ + -p 8080:8080 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + llamactl:vllm-cuda +``` + +**llamactl built from source:** +```bash +docker build -f docker/Dockerfile.source -t llamactl:source . +docker run -d \ + --name llamactl \ + -p 8080:8080 \ + llamactl:source +``` + +### Option 3: Build from Source Requirements: - Go 1.24 or later