SGLang

docker run --rm -p 8080:8080 --env-file .env enterpilot/gomodel

docker run --rm -p 8080:8080 \
  -e GOMODEL_MASTER_KEY="change-me" \
  -e SGLANG_BASE_URL="http://host.docker.internal:30000/v1" \
  enterpilot/gomodel

make build
./bin/gomodel

curl -s http://localhost:8080/v1/chat/completions \
  -H "Authorization: Bearer change-me" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "sglang/Qwen/Qwen2.5-0.5B-Instruct",
    "messages": [{"role": "user", "content": "Reply with exactly ok."}],
    "max_tokens": 8
  }'

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8080/v1", api_key="change-me")

completion = client.chat.completions.create(
    model="sglang/Qwen/Qwen2.5-0.5B-Instruct",
    messages=[{"role": "user", "content": "Reply with exactly ok."}],
    max_tokens=8,
)

print(completion.choices[0].message.content)

import OpenAI from "openai";

const client = new OpenAI({
  baseURL: "http://localhost:8080/v1",
  apiKey: "change-me",
});

const completion = await client.chat.completions.create({
  model: "sglang/Qwen/Qwen2.5-0.5B-Instruct",
  messages: [{ role: "user", content: "Reply with exactly ok." }],
  max_tokens: 8,
});

console.log(completion.choices[0].message.content);

curl -s http://localhost:8080/p/sglang/generate \
  -H "Authorization: Bearer change-me" \
  -H "Content-Type: application/json" \
  -d '{
    "text": "Hello",
    "sampling_params": {"max_new_tokens": 8}
  }'

import httpx

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8080", api_key="change-me")
generation = client.post(
    "/p/sglang/generate",
    cast_to=httpx.Response,
    body={
        "text": "Hello",
        "sampling_params": {"max_new_tokens": 8},
    },
)

print(generation.json())

import OpenAI from "openai";

const client = new OpenAI({
  baseURL: "http://localhost:8080",
  apiKey: "change-me",
});

const generation = await client.post("/p/sglang/generate", {
  body: {
    text: "Hello",
    sampling_params: { max_new_tokens: 8 },
  },
});

console.log(generation);