0
# Provider Configuration
1
2
Configuration classes, settings, and authentication for 100+ LLM providers supported by LiteLLM. This includes provider-specific parameters, custom endpoints, API key management, and advanced configuration options for enterprise deployments.
3
4
## Capabilities
5
6
### Base Configuration Class
7
8
All provider configurations inherit from the base configuration class with common parameters and validation.
9
10
```python { .api }
11
class BaseConfig:
12
"""
13
Base configuration class for all LLM providers.
14
15
Provides common parameters and validation for provider-specific configurations.
16
"""
17
18
# Core parameters available across providers
19
max_tokens: Optional[int] = None
20
temperature: Optional[float] = None
21
top_p: Optional[float] = None
22
stream: Optional[bool] = None
23
stop: Optional[Union[str, List[str]]] = None
24
user: Optional[str] = None
25
26
# LiteLLM specific parameters
27
timeout: Optional[float] = None
28
api_key: Optional[str] = None
29
api_base: Optional[str] = None
30
31
def __init__(self, **kwargs):
32
"""Initialize configuration with provider-specific parameters"""
33
pass
34
35
def get_config(self) -> Dict[str, Any]:
36
"""Get configuration dictionary for provider API calls"""
37
pass
38
```
39
40
### OpenAI Configuration
41
42
Complete configuration for OpenAI and OpenAI-compatible providers including Azure OpenAI.
43
44
```python { .api }
45
class OpenAIConfig(BaseConfig):
46
"""
47
Configuration for OpenAI API and compatible providers.
48
49
Supports all OpenAI API parameters including function calling,
50
response formatting, and advanced features.
51
"""
52
53
# Standard OpenAI parameters
54
frequency_penalty: Optional[float] = None
55
logit_bias: Optional[Dict[str, float]] = None
56
max_tokens: Optional[int] = None
57
n: Optional[int] = None
58
presence_penalty: Optional[float] = None
59
response_format: Optional[Dict[str, Any]] = None
60
seed: Optional[int] = None
61
stop: Optional[Union[str, List[str]]] = None
62
stream: Optional[bool] = None
63
temperature: Optional[float] = None
64
top_p: Optional[float] = None
65
user: Optional[str] = None
66
67
# Function calling parameters
68
functions: Optional[List[Dict[str, Any]]] = None
69
function_call: Optional[Union[str, Dict[str, Any]]] = None
70
tools: Optional[List[Dict[str, Any]]] = None
71
tool_choice: Optional[Union[str, Dict[str, Any]]] = None
72
73
# OpenAI-specific parameters
74
logprobs: Optional[bool] = None
75
top_logprobs: Optional[int] = None
76
77
# Advanced features
78
parallel_tool_calls: Optional[bool] = None
79
service_tier: Optional[Literal["auto", "default"]] = None
80
81
def __init__(
82
self,
83
frequency_penalty: Optional[float] = None,
84
logit_bias: Optional[Dict[str, float]] = None,
85
max_tokens: Optional[int] = None,
86
n: Optional[int] = None,
87
presence_penalty: Optional[float] = None,
88
response_format: Optional[Dict[str, Any]] = None,
89
seed: Optional[int] = None,
90
stop: Optional[Union[str, List[str]]] = None,
91
stream: Optional[bool] = None,
92
temperature: Optional[float] = None,
93
top_p: Optional[float] = None,
94
tools: Optional[List[Dict[str, Any]]] = None,
95
tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
96
user: Optional[str] = None,
97
**kwargs
98
):
99
"""
100
Initialize OpenAI configuration.
101
102
Args:
103
frequency_penalty (Optional[float]): Penalty for token frequency (-2.0 to 2.0)
104
presence_penalty (Optional[float]): Penalty for token presence (-2.0 to 2.0)
105
temperature (Optional[float]): Sampling temperature (0.0 to 2.0)
106
max_tokens (Optional[int]): Maximum tokens to generate
107
tools (Optional[List[Dict]]): Available function tools
108
response_format (Optional[Dict]): Response format specification
109
"""
110
```
111
112
class AzureOpenAIConfig(OpenAIConfig):
113
"""
114
Configuration for Azure OpenAI Service deployments.
115
116
Extends OpenAI configuration with Azure-specific parameters.
117
"""
118
119
# Azure-specific parameters
120
api_version: str = "2024-02-01"
121
azure_endpoint: Optional[str] = None
122
azure_deployment: Optional[str] = None
123
azure_ad_token: Optional[str] = None
124
azure_ad_token_provider: Optional[Callable[[], str]] = None
125
126
def __init__(
127
self,
128
api_version: str = "2024-02-01",
129
azure_endpoint: Optional[str] = None,
130
azure_deployment: Optional[str] = None,
131
azure_ad_token: Optional[str] = None,
132
**kwargs
133
):
134
"""
135
Initialize Azure OpenAI configuration.
136
137
Args:
138
api_version (str): Azure API version
139
azure_endpoint (Optional[str]): Azure endpoint URL
140
azure_deployment (Optional[str]): Deployment name
141
azure_ad_token (Optional[str]): Azure AD authentication token
142
"""
143
super().__init__(**kwargs)
144
```
145
146
### Anthropic Configuration
147
148
Configuration for Anthropic Claude models with provider-specific parameters.
149
150
```python { .api }
151
class AnthropicConfig(BaseConfig):
152
"""
153
Configuration for Anthropic Claude models.
154
155
Supports Anthropic-specific parameters including system prompts,
156
stop sequences, and advanced sampling options.
157
"""
158
159
# Required parameter for Anthropic
160
max_tokens: int
161
162
# Optional parameters
163
metadata: Optional[Dict[str, Any]] = None
164
stop_sequences: Optional[List[str]] = None
165
system: Optional[str] = None
166
temperature: Optional[float] = None
167
tool_choice: Optional[Dict[str, Any]] = None
168
tools: Optional[List[Dict[str, Any]]] = None
169
top_k: Optional[int] = None
170
top_p: Optional[float] = None
171
172
def __init__(
173
self,
174
max_tokens: int,
175
metadata: Optional[Dict[str, Any]] = None,
176
stop_sequences: Optional[List[str]] = None,
177
system: Optional[str] = None,
178
temperature: Optional[float] = None,
179
top_k: Optional[int] = None,
180
top_p: Optional[float] = None,
181
**kwargs
182
):
183
"""
184
Initialize Anthropic configuration.
185
186
Args:
187
max_tokens (int): Maximum tokens to generate (required)
188
system (Optional[str]): System prompt for Claude
189
temperature (Optional[float]): Sampling temperature (0.0 to 1.0)
190
top_k (Optional[int]): Top-k sampling parameter
191
top_p (Optional[float]): Nucleus sampling parameter
192
stop_sequences (Optional[List[str]]): Custom stop sequences
193
"""
194
```
195
196
### Google Configuration
197
198
Configuration for Google's AI models including Vertex AI and Gemini.
199
200
```python { .api }
201
class GoogleConfig(BaseConfig):
202
"""Configuration for Google AI models (Vertex AI, Gemini)"""
203
204
# Google-specific parameters
205
candidate_count: Optional[int] = None
206
max_output_tokens: Optional[int] = None
207
temperature: Optional[float] = None
208
top_p: Optional[float] = None
209
top_k: Optional[int] = None
210
211
# Safety settings
212
safety_settings: Optional[List[Dict[str, Any]]] = None
213
214
# Generation configuration
215
generation_config: Optional[Dict[str, Any]] = None
216
217
def __init__(
218
self,
219
candidate_count: Optional[int] = None,
220
max_output_tokens: Optional[int] = None,
221
temperature: Optional[float] = None,
222
top_p: Optional[float] = None,
223
top_k: Optional[int] = None,
224
safety_settings: Optional[List[Dict[str, Any]]] = None,
225
**kwargs
226
):
227
"""
228
Initialize Google AI configuration.
229
230
Args:
231
max_output_tokens (Optional[int]): Maximum output tokens
232
temperature (Optional[float]): Sampling temperature
233
top_p (Optional[float]): Nucleus sampling parameter
234
top_k (Optional[int]): Top-k sampling parameter
235
safety_settings (Optional[List[Dict]]): Content safety settings
236
"""
237
238
class VertexAIConfig(GoogleConfig):
239
"""
240
Configuration for Google Vertex AI deployments.
241
242
Extends Google configuration with Vertex AI specific parameters.
243
"""
244
245
# Vertex AI specific
246
project_id: Optional[str] = None
247
location: Optional[str] = None
248
credentials: Optional[str] = None
249
250
def __init__(
251
self,
252
project_id: Optional[str] = None,
253
location: Optional[str] = "us-central1",
254
credentials: Optional[str] = None,
255
**kwargs
256
):
257
"""
258
Initialize Vertex AI configuration.
259
260
Args:
261
project_id (Optional[str]): Google Cloud project ID
262
location (Optional[str]): Vertex AI location/region
263
credentials (Optional[str]): Service account credentials path
264
"""
265
super().__init__(**kwargs)
266
```
267
268
### Cohere Configuration
269
270
Configuration for Cohere models with provider-specific parameters.
271
272
```python { .api }
273
class CohereConfig(BaseConfig):
274
"""Configuration for Cohere models"""
275
276
# Cohere-specific parameters
277
max_tokens: Optional[int] = None
278
temperature: Optional[float] = None
279
k: Optional[int] = None # top-k
280
p: Optional[float] = None # top-p
281
frequency_penalty: Optional[float] = None
282
presence_penalty: Optional[float] = None
283
end_sequences: Optional[List[str]] = None
284
stop_sequences: Optional[List[str]] = None
285
return_likelihoods: Optional[str] = None
286
logit_bias: Optional[Dict[int, float]] = None
287
288
# Chat-specific parameters
289
chat_history: Optional[List[Dict[str, str]]] = None
290
conversation_id: Optional[str] = None
291
292
# Tool use parameters
293
tools: Optional[List[Dict[str, Any]]] = None
294
tool_results: Optional[List[Dict[str, Any]]] = None
295
296
def __init__(
297
self,
298
max_tokens: Optional[int] = None,
299
temperature: Optional[float] = None,
300
k: Optional[int] = None,
301
p: Optional[float] = None,
302
end_sequences: Optional[List[str]] = None,
303
**kwargs
304
):
305
"""
306
Initialize Cohere configuration.
307
308
Args:
309
max_tokens (Optional[int]): Maximum tokens to generate
310
temperature (Optional[float]): Sampling temperature
311
k (Optional[int]): Top-k sampling parameter
312
p (Optional[float]): Top-p sampling parameter
313
end_sequences (Optional[List[str]]): Stop sequences
314
"""
315
```
316
317
### AWS Bedrock Configuration
318
319
Configuration for AWS Bedrock models across different providers.
320
321
```python { .api }
322
class BedrockConfig(BaseConfig):
323
"""Configuration for AWS Bedrock models"""
324
325
# AWS Bedrock parameters
326
aws_access_key_id: Optional[str] = None
327
aws_secret_access_key: Optional[str] = None
328
aws_session_token: Optional[str] = None
329
aws_region_name: Optional[str] = None
330
331
# Model-specific parameters (varies by provider on Bedrock)
332
max_tokens_to_sample: Optional[int] = None
333
temperature: Optional[float] = None
334
top_p: Optional[float] = None
335
top_k: Optional[int] = None
336
stop_sequences: Optional[List[str]] = None
337
338
def __init__(
339
self,
340
aws_region_name: str = "us-east-1",
341
aws_access_key_id: Optional[str] = None,
342
aws_secret_access_key: Optional[str] = None,
343
max_tokens_to_sample: Optional[int] = None,
344
**kwargs
345
):
346
"""
347
Initialize AWS Bedrock configuration.
348
349
Args:
350
aws_region_name (str): AWS region for Bedrock
351
aws_access_key_id (Optional[str]): AWS access key
352
aws_secret_access_key (Optional[str]): AWS secret key
353
max_tokens_to_sample (Optional[int]): Maximum tokens to generate
354
"""
355
```
356
357
### Hugging Face Configuration
358
359
Configuration for Hugging Face models and inference endpoints.
360
361
```python { .api }
362
class HuggingFaceConfig(BaseConfig):
363
"""Configuration for Hugging Face models"""
364
365
# Hugging Face parameters
366
max_new_tokens: Optional[int] = None
367
temperature: Optional[float] = None
368
top_p: Optional[float] = None
369
top_k: Optional[int] = None
370
repetition_penalty: Optional[float] = None
371
do_sample: Optional[bool] = None
372
use_cache: Optional[bool] = None
373
374
# Generation parameters
375
num_return_sequences: Optional[int] = None
376
pad_token_id: Optional[int] = None
377
eos_token_id: Optional[int] = None
378
379
# Inference endpoint parameters
380
endpoint_url: Optional[str] = None
381
huggingface_api_key: Optional[str] = None
382
383
def __init__(
384
self,
385
max_new_tokens: Optional[int] = None,
386
temperature: Optional[float] = None,
387
top_p: Optional[float] = None,
388
do_sample: Optional[bool] = True,
389
**kwargs
390
):
391
"""
392
Initialize Hugging Face configuration.
393
394
Args:
395
max_new_tokens (Optional[int]): Maximum new tokens to generate
396
temperature (Optional[float]): Sampling temperature
397
top_p (Optional[float]): Nucleus sampling parameter
398
do_sample (Optional[bool]): Enable sampling vs greedy decoding
399
"""
400
```
401
402
### Custom Provider Registration
403
404
Register custom LLM providers with LiteLLM for proprietary or specialized models.
405
406
```python { .api }
407
def register_model(model_cost: Union[str, Dict[str, Any]]) -> None:
408
"""
409
Register custom model with cost information.
410
411
Args:
412
model_cost (Union[str, Dict]): Model cost configuration
413
Can be JSON string or dictionary with cost parameters
414
415
Example:
416
register_model({
417
"model_name": "custom-gpt-4",
418
"litellm_provider": "openai",
419
"model_cost": {
420
"input_cost_per_token": 0.00003,
421
"output_cost_per_token": 0.00006,
422
"litellm_provider": "openai",
423
"mode": "chat"
424
}
425
})
426
"""
427
428
def register_prompt_template(
429
model: str,
430
roles: Dict[str, str],
431
initial_prompt_value: str = "",
432
final_prompt_value: str = ""
433
) -> Dict[str, Any]:
434
"""
435
Register custom prompt template for model.
436
437
Args:
438
model (str): Model identifier
439
roles (Dict[str, str]): Role mapping for messages
440
initial_prompt_value (str): Template prefix
441
final_prompt_value (str): Template suffix
442
443
Returns:
444
Dict[str, Any]: Registered template configuration
445
446
Example:
447
register_prompt_template(
448
model="custom-model",
449
roles={
450
"system": "System: ",
451
"user": "Human: ",
452
"assistant": "Assistant: "
453
},
454
initial_prompt_value="<start>",
455
final_prompt_value="<end>"
456
)
457
"""
458
459
class CustomLLM:
460
"""
461
Base class for implementing custom LLM providers.
462
463
Implement this class to add support for proprietary or specialized models.
464
"""
465
466
def completion(
467
self,
468
model: str,
469
messages: List[Dict[str, Any]],
470
api_base: str,
471
model_response: ModelResponse,
472
print_verbose: Callable,
473
encoding: str,
474
api_key: str,
475
logging_obj: Any,
476
custom_prompt_dict: Dict[str, str] = {},
477
litellm_params: Dict[str, Any] = {},
478
logger_fn: Optional[Callable] = None,
479
headers: Dict[str, str] = {},
480
**kwargs
481
) -> ModelResponse:
482
"""
483
Implement completion for custom provider.
484
485
Args:
486
model (str): Model identifier
487
messages (List[Dict]): Chat messages
488
api_base (str): API base URL
489
model_response (ModelResponse): Response object to populate
490
print_verbose (Callable): Logging function
491
encoding (str): Text encoding
492
api_key (str): Provider API key
493
logging_obj (Any): Logging object
494
custom_prompt_dict (Dict): Custom prompt template
495
litellm_params (Dict): LiteLLM parameters
496
logger_fn (Optional[Callable]): Logger function
497
headers (Dict): HTTP headers
498
499
Returns:
500
ModelResponse: Populated response object
501
"""
502
pass
503
504
def streaming(
505
self,
506
model: str,
507
messages: List[Dict[str, Any]],
508
api_base: str,
509
model_response: ModelResponse,
510
print_verbose: Callable,
511
encoding: str,
512
api_key: str,
513
logging_obj: Any,
514
custom_prompt_dict: Dict[str, str] = {},
515
litellm_params: Dict[str, Any] = {},
516
logger_fn: Optional[Callable] = None,
517
headers: Dict[str, str] = {},
518
**kwargs
519
) -> Iterator[ModelResponseStream]:
520
"""
521
Implement streaming completion for custom provider.
522
523
Args:
524
Same as completion() method
525
526
Returns:
527
Iterator[ModelResponseStream]: Streaming response chunks
528
"""
529
pass
530
531
def register_custom_llm(custom_llm: CustomLLM, provider_name: str) -> None:
532
"""
533
Register custom LLM provider implementation.
534
535
Args:
536
custom_llm (CustomLLM): Custom provider implementation
537
provider_name (str): Name for the custom provider
538
"""
539
```
540
541
## Global Configuration Settings
542
543
```python { .api }
544
# Authentication configuration
545
litellm.api_key: Optional[str] = None
546
litellm.openai_key: Optional[str] = None
547
litellm.anthropic_key: Optional[str] = None
548
litellm.cohere_key: Optional[str] = None
549
litellm.replicate_key: Optional[str] = None
550
litellm.huggingface_key: Optional[str] = None
551
litellm.together_ai_key: Optional[str] = None
552
litellm.palm_key: Optional[str] = None
553
litellm.vertex_project: Optional[str] = None
554
litellm.vertex_location: Optional[str] = None
555
litellm.bedrock_aws_access_key_id: Optional[str] = None
556
litellm.bedrock_aws_secret_access_key: Optional[str] = None
557
litellm.bedrock_aws_region_name: Optional[str] = None
558
559
# Provider-specific API bases
560
litellm.openai_api_base: Optional[str] = None
561
litellm.anthropic_api_base: Optional[str] = None
562
litellm.cohere_api_base: Optional[str] = None
563
564
# Parameter handling
565
litellm.drop_params: bool = False
566
litellm.modify_params: bool = False
567
litellm.model_alias_map: Dict[str, str] = {}
568
569
# Default settings
570
litellm.request_timeout: float = 600
571
litellm.max_tokens: int = 256
572
litellm.temperature: Optional[float] = None
573
574
# Debugging and logging
575
litellm.set_verbose: bool = False
576
litellm.suppress_debug_info: bool = False
577
```
578
579
## Usage Examples
580
581
### Basic Provider Configuration
582
583
```python
584
import litellm
585
586
# Configure OpenAI
587
litellm.openai_key = "sk-your-openai-key"
588
589
response = litellm.completion(
590
model="gpt-4",
591
messages=[{"role": "user", "content": "Hello!"}]
592
)
593
594
# Configure Anthropic
595
litellm.anthropic_key = "your-anthropic-key"
596
597
response = litellm.completion(
598
model="claude-3-sonnet-20240229",
599
messages=[{"role": "user", "content": "Hello!"}]
600
)
601
602
# Configure multiple providers
603
litellm.api_key = "fallback-key"
604
litellm.cohere_key = "your-cohere-key"
605
litellm.huggingface_key = "your-hf-key"
606
```
607
608
### Azure OpenAI Configuration
609
610
```python
611
import litellm
612
613
# Method 1: Environment variables
614
import os
615
os.environ["AZURE_API_KEY"] = "your-azure-key"
616
os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com/"
617
os.environ["AZURE_API_VERSION"] = "2024-02-01"
618
619
response = litellm.completion(
620
model="azure/gpt-4",
621
messages=[{"role": "user", "content": "Hello!"}]
622
)
623
624
# Method 2: Direct parameters
625
response = litellm.completion(
626
model="azure/gpt-4-deployment",
627
messages=[{"role": "user", "content": "Hello!"}],
628
api_key="your-azure-key",
629
api_base="https://your-resource.openai.azure.com/",
630
api_version="2024-02-01"
631
)
632
633
# Method 3: Using configuration class
634
config = litellm.AzureOpenAIConfig(
635
api_version="2024-02-01",
636
azure_endpoint="https://your-resource.openai.azure.com/",
637
azure_deployment="gpt-4-deployment"
638
)
639
640
response = litellm.completion(
641
model="azure/gpt-4",
642
messages=[{"role": "user", "content": "Hello!"}],
643
**config.get_config()
644
)
645
```
646
647
### AWS Bedrock Configuration
648
649
```python
650
import litellm
651
652
# Configure AWS credentials
653
litellm.bedrock_aws_access_key_id = "your-access-key"
654
litellm.bedrock_aws_secret_access_key = "your-secret-key"
655
litellm.bedrock_aws_region_name = "us-east-1"
656
657
# Use Bedrock models
658
response = litellm.completion(
659
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
660
messages=[{"role": "user", "content": "Hello from Bedrock!"}]
661
)
662
663
# With provider-specific parameters
664
response = litellm.completion(
665
model="bedrock/anthropic.claude-v2",
666
messages=[{"role": "user", "content": "Hello!"}],
667
max_tokens_to_sample=1000,
668
temperature=0.7,
669
top_k=250,
670
top_p=1.0,
671
stop_sequences=["Human:"]
672
)
673
```
674
675
### Google Vertex AI Configuration
676
677
```python
678
import litellm
679
680
# Set up Vertex AI credentials
681
import os
682
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "path/to/service-account.json"
683
litellm.vertex_project = "your-project-id"
684
litellm.vertex_location = "us-central1"
685
686
# Use Vertex AI models
687
response = litellm.completion(
688
model="vertex_ai/gemini-pro",
689
messages=[{"role": "user", "content": "Hello from Vertex AI!"}]
690
)
691
692
# With Google-specific parameters
693
response = litellm.completion(
694
model="vertex_ai/gemini-pro",
695
messages=[{"role": "user", "content": "Hello!"}],
696
max_output_tokens=1024,
697
temperature=0.8,
698
top_p=0.95,
699
top_k=40,
700
safety_settings=[
701
{
702
"category": "HARM_CATEGORY_DANGEROUS_CONTENT",
703
"threshold": "BLOCK_MEDIUM_AND_ABOVE"
704
}
705
]
706
)
707
```
708
709
### Custom Provider Implementation
710
711
```python
712
import litellm
713
from litellm import CustomLLM, ModelResponse
714
import requests
715
716
class MyCustomProvider(CustomLLM):
717
def completion(self, model, messages, **kwargs):
718
# Implement your custom API call
719
api_base = kwargs.get("api_base", "https://api.example.com")
720
api_key = kwargs.get("api_key")
721
722
headers = {
723
"Authorization": f"Bearer {api_key}",
724
"Content-Type": "application/json"
725
}
726
727
# Convert LiteLLM format to your API format
728
custom_payload = {
729
"model": model,
730
"messages": messages,
731
"temperature": kwargs.get("temperature", 0.7),
732
"max_tokens": kwargs.get("max_tokens", 256)
733
}
734
735
# Make API request
736
response = requests.post(
737
f"{api_base}/completions",
738
headers=headers,
739
json=custom_payload,
740
timeout=kwargs.get("timeout", 30)
741
)
742
743
# Convert response to LiteLLM format
744
response_json = response.json()
745
746
# Create ModelResponse object
747
model_response = ModelResponse()
748
model_response.choices = [{
749
"message": {
750
"content": response_json["content"],
751
"role": "assistant"
752
},
753
"finish_reason": "stop",
754
"index": 0
755
}]
756
model_response.model = model
757
model_response.usage = {
758
"prompt_tokens": response_json.get("prompt_tokens", 0),
759
"completion_tokens": response_json.get("completion_tokens", 0),
760
"total_tokens": response_json.get("total_tokens", 0)
761
}
762
763
return model_response
764
765
def streaming(self, model, messages, **kwargs):
766
# Implement streaming if supported
767
# Return iterator of ModelResponseStream objects
768
pass
769
770
# Register custom provider
771
custom_provider = MyCustomProvider()
772
litellm.register_custom_llm(custom_provider, "my_provider")
773
774
# Use custom provider
775
response = litellm.completion(
776
model="my_provider/custom-model-v1",
777
messages=[{"role": "user", "content": "Hello!"}],
778
api_key="your-custom-key",
779
api_base="https://api.example.com"
780
)
781
```
782
783
### Model Registration and Cost Tracking
784
785
```python
786
import litellm
787
788
# Register custom model with cost information
789
litellm.register_model({
790
"model_name": "custom/my-model-v1",
791
"litellm_provider": "custom",
792
"model_cost": {
793
"input_cost_per_token": 0.00001,
794
"output_cost_per_token": 0.00003,
795
"litellm_provider": "custom",
796
"mode": "chat"
797
}
798
})
799
800
# Register custom prompt template
801
litellm.register_prompt_template(
802
model="custom/my-model-v1",
803
roles={
804
"system": "### System:\n",
805
"user": "### Human:\n",
806
"assistant": "### Assistant:\n"
807
},
808
initial_prompt_value="<conversation_start>",
809
final_prompt_value="### Assistant:\n"
810
)
811
812
# Use registered model
813
response = litellm.completion(
814
model="custom/my-model-v1",
815
messages=[
816
{"role": "system", "content": "You are helpful."},
817
{"role": "user", "content": "Hello!"}
818
]
819
)
820
821
# Check cost
822
cost = litellm.completion_cost(response)
823
print(f"Request cost: ${cost:.6f}")
824
```
825
826
### Environment Variable Configuration
827
828
```python
829
# Set up environment variables for multiple providers
830
import os
831
832
# OpenAI
833
os.environ["OPENAI_API_KEY"] = "sk-your-openai-key"
834
835
# Anthropic
836
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
837
838
# Cohere
839
os.environ["COHERE_API_KEY"] = "your-cohere-key"
840
841
842
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service-account.json"
843
os.environ["VERTEXAI_PROJECT"] = "your-project-id"
844
os.environ["VERTEXAI_LOCATION"] = "us-central1"
845
846
# AWS
847
os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key"
848
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-key"
849
os.environ["AWS_REGION_NAME"] = "us-east-1"
850
851
# Azure
852
os.environ["AZURE_API_KEY"] = "your-azure-key"
853
os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com/"
854
os.environ["AZURE_API_VERSION"] = "2024-02-01"
855
856
# Hugging Face
857
os.environ["HUGGINGFACE_API_KEY"] = "your-hf-token"
858
859
# Now all providers are configured and ready to use
860
import litellm
861
862
# Use any provider without additional configuration
863
openai_response = litellm.completion(
864
model="gpt-4",
865
messages=[{"role": "user", "content": "Hello OpenAI!"}]
866
)
867
868
anthropic_response = litellm.completion(
869
model="claude-3-sonnet-20240229",
870
messages=[{"role": "user", "content": "Hello Anthropic!"}]
871
)
872
873
bedrock_response = litellm.completion(
874
model="bedrock/anthropic.claude-v2",
875
messages=[{"role": "user", "content": "Hello Bedrock!"}]
876
)
877
```
878
879
### Advanced Configuration with Router
880
881
```python
882
from litellm import Router
883
884
# Configure router with multiple providers
885
model_list = [
886
# OpenAI deployment
887
{
888
"model_name": "gpt-4",
889
"litellm_params": {
890
"model": "gpt-4",
891
"api_key": os.environ["OPENAI_API_KEY"]
892
}
893
},
894
# Azure OpenAI deployment
895
{
896
"model_name": "gpt-4",
897
"litellm_params": {
898
"model": "azure/gpt-4-deployment",
899
"api_key": os.environ["AZURE_API_KEY"],
900
"api_base": os.environ["AZURE_API_BASE"],
901
"api_version": "2024-02-01"
902
}
903
},
904
# Anthropic deployment
905
{
906
"model_name": "claude-3",
907
"litellm_params": {
908
"model": "claude-3-sonnet-20240229",
909
"api_key": os.environ["ANTHROPIC_API_KEY"]
910
}
911
},
912
# Bedrock deployment
913
{
914
"model_name": "claude-bedrock",
915
"litellm_params": {
916
"model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
917
"aws_access_key_id": os.environ["AWS_ACCESS_KEY_ID"],
918
"aws_secret_access_key": os.environ["AWS_SECRET_ACCESS_KEY"],
919
"aws_region_name": "us-east-1"
920
}
921
}
922
]
923
924
router = Router(model_list=model_list)
925
926
# Router automatically handles provider-specific configurations
927
response = router.completion(
928
model="gpt-4",
929
messages=[{"role": "user", "content": "Hello!"}]
930
)
931
```