Groq — Ultra-Fast LLM Inference

# Verify credentials
[ -n "$GROQ_API_KEY" ] && echo "GROQ_API_KEY set" || echo "NOT SET"

pip install groq
export GROQ_API_KEY="gsk_..."   # https://console.groq.com/keys

from groq import Groq

client = Groq()
response = client.chat.completions.create(
    model="llama-3.3-70b-versatile",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Explain quantum computing in one paragraph."}
    ],
    temperature=0.7,
    max_completion_tokens=512,
)
print(response.choices[0].message.content)

from groq import Groq

client = Groq()
stream = client.chat.completions.create(
    model="llama-3.3-70b-versatile",
    messages=[{"role": "user", "content": "Explain fast inference"}],
    stream=True,
)
for chunk in stream:
    content = chunk.choices[0].delta.content
    if content:
        print(content, end="")

from groq import Groq
import json

client = Groq()
response = client.chat.completions.create(
    model="llama-3.3-70b-versatile",
    messages=[
        {"role": "system", "content": "Output valid JSON only."},
        {"role": "user", "content": "List 3 programming languages with their paradigms."}
    ],
    response_format={"type": "json_object"},
    temperature=0,
)
data = json.loads(response.choices[0].message.content)

from groq import Groq
import json

client = Groq()
tools = [{
    "type": "function",
    "function": {
        "name": "get_weather",
        "description": "Get weather for a location",
        "parameters": {
            "type": "object",
            "properties": {
                "location": {"type": "string", "description": "City name"},
                "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
            },
            "required": ["location"],
            "additionalProperties": False
        }
    }
}]

response = client.chat.completions.create(
    model="llama-3.3-70b-versatile",
    messages=[{"role": "user", "content": "Weather in San Francisco?"}],
    tools=tools,
    tool_choice="auto",
)

message = response.choices[0].message
if message.tool_calls:
    call = message.tool_calls[0]
    args = json.loads(call.function.arguments)
    # Execute function, then send result back as role="tool" message
    result = {"temperature": 62, "condition": "foggy"}
    followup = client.chat.completions.create(
        model="llama-3.3-70b-versatile",
        messages=[
            {"role": "user", "content": "Weather in San Francisco?"},
            message,
            {"role": "tool", "tool_call_id": call.id, "content": json.dumps(result)}
        ],
    )
    print(followup.choices[0].message.content)

from groq import Groq

client = Groq()
response = client.chat.completions.create(
    model="meta-llama/llama-4-scout-17b-16e-instruct",
    messages=[{
        "role": "user",
        "content": [
            {"type": "text", "text": "Describe this image."},
            {"type": "image_url", "image_url": {"url": "https://example.com/photo.jpg"}}
        ]
    }],
    max_completion_tokens=1024,
)
print(response.choices[0].message.content)

from groq import Groq

client = Groq()

# Transcription
with open("audio.mp3", "rb") as f:
    result = client.audio.transcriptions.create(
        file=("audio.mp3", f),
        model="whisper-large-v3-turbo",
        language="en",
        response_format="verbose_json",
    )
print(result.text)

# Translation (any language -> English)
with open("french.mp3", "rb") as f:
    result = client.audio.translations.create(
        file=("french.mp3", f), model="whisper-large-v3",
    )
print(result.text)

from groq import Groq
from pathlib import Path

client = Groq()
speech = client.audio.speech.create(
    model="playai-tts",
    voice="Arista-PlayAI",
    input="Hello from Groq!",
)
Path("output.mp3").write_bytes(speech.content)

from openai import OpenAI
import os

client = OpenAI(
    base_url="https://api.groq.com/openai/v1",
    api_key=os.environ.get("GROQ_API_KEY"),
)
response = client.chat.completions.create(
    model="llama-3.3-70b-versatile",
    messages=[{"role": "user", "content": "Hello from OpenAI SDK!"}],
)
print(response.choices[0].message.content)

import time
from groq import Groq, RateLimitError

client = Groq()

def call_with_retry(messages, model="llama-3.3-70b-versatile", max_retries=3):
    for attempt in range(max_retries):
        try:
            return client.chat.completions.create(model=model, messages=messages)
        except RateLimitError:
            if attempt == max_retries - 1:
                raise
            time.sleep(2 ** attempt)