PolyLLM Documentation

# .env — copy from .env.example and fill in your keys

# Free tier — no credit card required
GROQ_API_KEY=gsk_...          # console.groq.com       → 14,400 req/day
CEREBRAS_API_KEY=csk_...      # cloud.cerebras.ai      → 14,400 req/day
GEMINI_API_KEY=AIza...        # aistudio.google.com    → 250 req/day
SAMBANOVA_API_KEY=...         # cloud.sambanova.ai     → 20 req/day

# One key → 4 isolated model slots (200 req/day each = 800 RPD total)
OPENROUTER_API_KEY=sk-or-...  # openrouter.ai          → 800 req/day

# Optional paid-tier providers
MISTRAL_API_KEY=
DEEPSEEK_API_KEY=
TOGETHER_API_KEY=
FIREWORKS_API_KEY=

from polyai import PolyAI

ai = PolyAI.from_env()   # reads .env automatically

response = ai.chat(messages=[
    {"role": "system",  "content": "You are a helpful assistant."},
    {"role": "user",    "content": "Explain quantum entanglement simply."},
])

print(response.content)           # text reply
print(response.provider)          # e.g. "cerebras"
print(response.model)             # e.g. "llama3.1-8b"
print(response.input_tokens)      # tokens consumed
print(response.total_tokens)

# prefer="auto"    — default: balances speed, quality, and availability
# prefer="speed"   — routes to fastest provider (Cerebras 2450 TPS)
# prefer="quality" — routes to highest reasoning score (Groq / SambaNova)

fast   = ai.chat(messages=[...], prefer="speed")
smart  = ai.chat(messages=[...], prefer="quality")
normal = ai.chat(messages=[...], prefer="auto")

tools = [{
    "type": "function",
    "function": {
        "name": "get_weather",
        "description": "Get current weather",
        "parameters": {
            "type": "object",
            "properties": {"city": {"type": "string"}},
            "required": ["city"]
        }
    }
}]

response = ai.chat(
    messages=[{"role": "user", "content": "What's the weather in London?"}],
    tools=tools   # router auto-selects a tool-capable provider
)

if response.tool_calls:
    tc = response.tool_calls[0]
    print(tc.name)        # "get_weather"
    print(tc.arguments)   # '{"city": "London"}'

cap = ai.capacity(messages=[{"role": "user", "content": "Hello"}])

print(cap["route"]["provider"])       # which provider would be chosen
print(cap["route"]["wait_seconds"])   # seconds until available (0 = now)
print(cap["aggregate_remaining"])      # combined RPM / RPD across all providers

status = ai.status()                  # per-provider usage + reset times

python server.py
# PolyLLM Gateway started
# Providers (8): cerebras, gemini, groq, openrouter, …
# Dashboard: https://polyai.80.225.202.88.nip.io

curl https://polyai.80.225.202.88.nip.io/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "auto",
    "messages": [{"role": "user", "content": "Hello!"}],
    "prefer": "speed",
    "max_tokens": 512
  }'

import OpenAI from "openai";

const ai = new OpenAI({
  baseURL: "https://polyai.80.225.202.88.nip.io/v1",
  apiKey:  "any",   // server handles auth via .env
});

const res = await ai.chat.completions.create({
  model:    "auto",
  messages: [{ role: "user", content: "Hello!" }],
});

console.log(res.choices[0].message.content);
console.log(res.x_polyai_provider);  // "cerebras" — which provider answered

import openai

client = openai.OpenAI(
    base_url="https://polyai.80.225.202.88.nip.io/v1",
    api_key="any",
)

res = client.chat.completions.create(
    model="auto",
    messages=[{"role": "user", "content": "Hello!"}],
)
print(res.choices[0].message.content)

{
  "id":      "chatcmpl-4f3a…",
  "object":  "chat.completion",
  "created": 1748505600,
  "model":   "llama3.1-8b",        // actual model that answered
  "choices": [{
    "index":        0,
    "message": {
      "role":       "assistant",
      "content":    "Here is a quicksort implementation…",
      "tool_calls": []   // populated when model calls a function
    },
    "finish_reason": "stop"  // "stop" | "tool_calls" | "length"
  }],
  "usage": {
    "prompt_tokens":     36,
    "completion_tokens": 21,
    "total_tokens":      57
  },
  // ── PolyLLM extensions ────────────────────────────────────
  "x_polyai_provider": "cerebras",   // which provider answered
  "x_polyai_elapsed":  1.79          // wall-clock seconds
}

curl https://polyai.80.225.202.88.nip.io/v1/status

// Response:
{
  "total_providers": 8,
  "requests_served": 42,
  "providers": {
    "cerebras": {
      "model":          "llama3.1-8b",
      "context_window": 8192,
      "speed_tps":      2450,
      "tool_quality":   3,
      "supports_tools": true,
      "limits":  { "rpm": 30, "tpm": 60000, "rpd": 14400 },
      "remaining": { "rpm": 29, "rpd": 14358 },
      "resets_in": { "minute": "47s", "day": "6h 22m" }
    },
    // ... all other providers
  }
}

curl https://polyai.80.225.202.88.nip.io/v1/routing-log

// Response — last 30 decisions:
{
  "decisions": [{
    "seq":             5,
    "provider":        "cerebras",
    "model":           "llama3.1-8b",
    "input_tokens":    36,
    "output_tokens":   21,
    "elapsed_s":       1.79,
    "wait_seconds":    0,
    "routing_reason":  "Routed to cerebras | score=52.3 | wait=0s | …",
    "request_preview": "hi",
    "timestamp":       1748505612.3
  }]
}