KServe Python SDK

pip install kserve              # Base package
pip install kserve[storage]     # With S3, GCS, Azure support
pip install kserve[llm]         # With OpenAI protocol support

from kserve import Model, ModelServer                    # Model serving
from kserve import InferenceRESTClient, InferenceGRPCClient  # Clients
from kserve import KServeClient                          # Kubernetes control
from kserve import InferRequest, InferResponse           # Protocol types

from kserve import Model, ModelServer

class MyModel(Model):
    def load(self):
        self.model = load_my_model()
        self.ready = True
    
    def predict(self, payload, headers=None):
        return {"predictions": self.model.predict(payload["instances"])}

if __name__ == "__main__":
    model = MyModel("my-model")
    model.load()
    ModelServer().start([model])

class Model:
    def load(self) -> None: ...                          # Load model artifacts
    def preprocess(self, body, headers=None): ...        # Transform input
    def predict(self, payload, headers=None): ...        # Run inference
    def postprocess(self, response, headers=None): ...   # Transform output
    def explain(self, payload, headers=None): ...        # Generate explanations

# REST Client
client = InferenceRESTClient(url="http://localhost:8080")
response = await client.infer(base_url=url, model_name="my-model", data={...})

# gRPC Client
client = InferenceGRPCClient(url="localhost:8081")
response = await client.infer(model_name="my-model", inputs=[...])

client = KServeClient()
client.create(inferenceservice)                          # Create resource
client.get(name, namespace)                              # Get resource
client.patch(name, inferenceservice, namespace)          # Update resource
client.delete(name, namespace)                           # Delete resource
client.wait_isvc_ready(name, namespace, timeout)         # Wait for ready

# Input tensor
input = InferInput(name="input-0", shape=[1, 4], datatype="FP32", data=[[...]])
input.set_data_from_numpy(array)                         # From NumPy
array = input.as_numpy()                                 # To NumPy

# Request/Response
request = InferRequest(model_name="model", infer_inputs=[input])
response = InferResponse(model_name="model", infer_outputs=[output])

# Start server with options
ModelServer(
    http_port=8080,
    grpc_port=8081,
    workers=4,
    enable_grpc=True,
    enable_docs_url=True
).start([model])

from kserve.errors import (
    InferenceError,      # Inference execution failure (500)
    InvalidInput,        # Invalid input data (400)
    ModelNotFound,       # Model doesn't exist (404)
    ModelNotReady,       # Model not initialized (503)
    UnsupportedProtocol, # Unknown protocol
    ServerNotReady       # Server not ready (503)
)

python model.py \
  --http_port 8080 \
  --grpc_port 8081 \
  --workers 4 \
  --max_threads 8 \
  --enable_grpc true \
  --enable_docs_url true \
  --log_config_file /path/to/config.yaml