diff --git a/pkg/handlers.go b/pkg/handlers.go index c38aff8..39743ce 100644 --- a/pkg/handlers.go +++ b/pkg/handlers.go @@ -1,8 +1,10 @@ package llamactl import ( + "bytes" "encoding/json" "fmt" + "io" "net/http" "os/exec" "strconv" @@ -456,9 +458,17 @@ func (h *Handler) ProxyToInstance() http.HandlerFunc { // OpenAIProxy godoc func (h *Handler) OpenAIProxy() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // Extract model name from request body + // Read the entire body first + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read request body", http.StatusBadRequest) + return + } + r.Body.Close() + + // Parse the body to extract model name var requestBody map[string]any - if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil { + if err := json.Unmarshal(bodyBytes, &requestBody); err != nil { http.Error(w, "Invalid request body", http.StatusBadRequest) return } @@ -487,6 +497,10 @@ func (h *Handler) OpenAIProxy() http.HandlerFunc { return } + // Recreate the request body from the bytes we read + r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) + r.ContentLength = int64(len(bodyBytes)) + proxy.ServeHTTP(w, r) } } diff --git a/pkg/routes.go b/pkg/routes.go index b6bcda9..d4ef89c 100644 --- a/pkg/routes.go +++ b/pkg/routes.go @@ -50,13 +50,15 @@ func SetupRouter(handler *Handler) *chi.Mux { }) }) - // OpenAI-compatible endpoints (model name in request body determines routing) - r.Post("/v1/", handler.OpenAIProxy()) // Proxy to OpenAI-compatible endpoints based on instance name in request body - // r.Post("/v1/completions", handler.OpenAICompletions()) - // r.Post("/v1/chat/completions", handler.OpenAIChatCompletions()) - // r.Post("/v1/embeddings", handler.OpenAIEmbeddings()) - // r.Post("/v1/rerank", handler.OpenAIRerank()) - // r.Post("/v1/reranking", handler.OpenAIReranking()) + // OpenAI-compatible proxy endpoint + // Handles all POST requests to /v1/*, including: + // - /v1/completions + // - /v1/chat/completions + // - /v1/embeddings + // - /v1/rerank + // - /v1/reranking + // The instance/model to use is determined by the request body. + r.Post("/v1/*", handler.OpenAIProxy()) // Serve WebUI files if err := webui.SetupWebUI(r); err != nil {