0
# Protocol and Data Types
1
2
Standardized data structures for inference requests and responses with support for multiple protocols (REST v1/v2, gRPC v2) and data formats. These classes provide seamless protocol conversion and integration with popular data science libraries.
3
4
## Capabilities
5
6
### Inference Request
7
8
Container class for inference requests that supports multiple input tensors, parameters, and protocol conversion.
9
10
```python { .api }
11
class InferRequest:
12
def __init__(self,
13
model_name: str,
14
infer_inputs: List[InferInput],
15
request_id: Optional[str] = None,
16
raw_inputs=None,
17
from_grpc: Optional[bool] = False,
18
parameters: Optional[Union[Dict, MessageMap[str, InferParameter]]] = None,
19
request_outputs: Optional[List[RequestedOutput]] = None,
20
model_version: Optional[str] = None):
21
"""
22
Create inference request.
23
24
Args:
25
model_name: Name of model to query
26
infer_inputs: List of input tensors
27
request_id: Unique request identifier
28
raw_inputs: Raw input data
29
from_grpc: Whether request originated from gRPC
30
parameters: Request parameters as Dict or MessageMap
31
request_outputs: Requested outputs
32
model_version: Version of the model
33
"""
34
35
def as_dataframe(self) -> 'pandas.DataFrame':
36
"""
37
Convert request to pandas DataFrame.
38
39
Returns:
40
pandas.DataFrame: Request data as DataFrame
41
"""
42
43
def to_rest(self) -> Dict[str, Any]:
44
"""
45
Convert to REST API format.
46
47
Returns:
48
Dict[str, Any]: REST-formatted request
49
"""
50
51
def to_grpc(self):
52
"""
53
Convert to gRPC format.
54
55
Returns:
56
gRPC request object
57
"""
58
59
@classmethod
60
def from_grpc(cls, grpc_request) -> 'InferRequest':
61
"""
62
Create from gRPC request.
63
64
Args:
65
grpc_request: gRPC request object
66
67
Returns:
68
InferRequest: Converted request
69
"""
70
71
@classmethod
72
def from_bytes(cls, data: bytes) -> 'InferRequest':
73
"""
74
Create from serialized bytes.
75
76
Args:
77
data (bytes): Serialized request data
78
79
Returns:
80
InferRequest: Deserialized request
81
"""
82
83
# Properties
84
model_name: str # Target model name
85
inputs: List[InferInput] # Input tensors
86
request_id: Optional[str] # Request ID
87
parameters: Optional[Dict[str, Any]] # Request parameters
88
request_outputs: Optional[List['RequestedOutput']] # Output specifications
89
```
90
91
### Inference Response
92
93
Container class for inference responses with multiple output tensors and metadata.
94
95
```python { .api }
96
class InferResponse:
97
def __init__(self,
98
response_id: str,
99
model_name: str,
100
infer_outputs: List[InferOutput],
101
model_version: Optional[str] = None,
102
raw_outputs=None,
103
from_grpc: Optional[bool] = False,
104
parameters: Optional[Union[Dict, MessageMap[str, InferParameter]]] = None,
105
use_binary_outputs: Optional[bool] = False,
106
requested_outputs: Optional[List[RequestedOutput]] = None):
107
"""
108
Create inference response.
109
110
Args:
111
response_id: Response identifier
112
model_name: Name of model that generated response
113
infer_outputs: List of output tensors
114
model_version: Version of the model
115
raw_outputs: Raw output data
116
from_grpc: Whether response originated from gRPC
117
parameters: Response parameters as Dict or MessageMap
118
use_binary_outputs: Whether to use binary outputs
119
requested_outputs: List of requested outputs
120
"""
121
122
def to_rest(self) -> Dict[str, Any]:
123
"""
124
Convert to REST API format.
125
126
Returns:
127
Dict[str, Any]: REST-formatted response
128
"""
129
130
def to_grpc(self):
131
"""
132
Convert to gRPC format.
133
134
Returns:
135
gRPC response object
136
"""
137
138
@classmethod
139
def from_rest(cls, response: Dict[str, Any]) -> 'InferResponse':
140
"""
141
Create from REST response.
142
143
Args:
144
response (Dict[str, Any]): REST response data
145
146
Returns:
147
InferResponse: Converted response
148
"""
149
150
@classmethod
151
def from_grpc(cls, grpc_response) -> 'InferResponse':
152
"""
153
Create from gRPC response.
154
155
Args:
156
grpc_response: gRPC response object
157
158
Returns:
159
InferResponse: Converted response
160
"""
161
162
@classmethod
163
def from_bytes(cls, data: bytes) -> 'InferResponse':
164
"""
165
Create from serialized bytes.
166
167
Args:
168
data (bytes): Serialized response data
169
170
Returns:
171
InferResponse: Deserialized response
172
"""
173
174
# Properties
175
model_name: str # Source model name
176
outputs: List[InferOutput] # Output tensors
177
response_id: Optional[str] # Response ID
178
parameters: Optional[Dict[str, Any]] # Response parameters
179
```
180
181
### Input Tensor
182
183
Represents input tensor data with shape, datatype, and data payload.
184
185
```python { .api }
186
class InferInput:
187
def __init__(self,
188
name: str,
189
shape: List[int],
190
datatype: str,
191
parameters: Optional[Dict[str, Any]] = None):
192
"""
193
Create input tensor specification.
194
195
Args:
196
name (str): Input tensor name
197
shape (List[int]): Tensor shape dimensions
198
datatype (str): Data type (FP32, FP64, INT32, INT64, BOOL, BYTES)
199
parameters (Dict[str, Any], optional): Additional parameters
200
"""
201
202
def set_data_from_numpy(self, input_tensor: 'numpy.ndarray'):
203
"""
204
Set tensor data from numpy array.
205
206
Args:
207
input_tensor (numpy.ndarray): Input data as numpy array
208
"""
209
210
def as_numpy(self) -> 'numpy.ndarray':
211
"""
212
Get tensor data as numpy array.
213
214
Returns:
215
numpy.ndarray: Tensor data
216
"""
217
218
def as_string(self) -> List[str]:
219
"""
220
Get tensor data as list of strings.
221
222
Returns:
223
List[str]: String data
224
"""
225
226
def set_data_from_bytes(self, data: bytes):
227
"""
228
Set tensor data from raw bytes.
229
230
Args:
231
data (bytes): Raw tensor data
232
"""
233
234
# Properties
235
name: str # Tensor name
236
shape: List[int] # Tensor dimensions
237
datatype: str # Data type string
238
data: Any # Tensor data payload
239
parameters: Optional[Dict[str, Any]] # Additional parameters
240
```
241
242
### Output Tensor
243
244
Represents output tensor data with shape, datatype, and result payload.
245
246
```python { .api }
247
class InferOutput:
248
def __init__(self,
249
name: str,
250
shape: List[int],
251
datatype: str,
252
data: Any = None,
253
parameters: Optional[Dict[str, Any]] = None):
254
"""
255
Create output tensor specification.
256
257
Args:
258
name (str): Output tensor name
259
shape (List[int]): Tensor shape dimensions
260
datatype (str): Data type (FP32, FP64, INT32, INT64, BOOL, BYTES)
261
data (Any, optional): Tensor data payload
262
parameters (Dict[str, Any], optional): Additional parameters
263
"""
264
265
def as_numpy(self) -> 'numpy.ndarray':
266
"""
267
Get tensor data as numpy array.
268
269
Returns:
270
numpy.ndarray: Tensor data
271
"""
272
273
def set_data_from_numpy(self, output_tensor: 'numpy.ndarray'):
274
"""
275
Set tensor data from numpy array.
276
277
Args:
278
output_tensor (numpy.ndarray): Output data as numpy array
279
"""
280
281
# Properties
282
name: str # Tensor name
283
shape: List[int] # Tensor dimensions
284
datatype: str # Data type string
285
data: Any # Tensor data payload
286
parameters: Optional[Dict[str, Any]] # Additional parameters
287
```
288
289
### Requested Output
290
291
Specification for requested output tensors in inference requests.
292
293
```python { .api }
294
class RequestedOutput:
295
def __init__(self,
296
name: str,
297
parameters: Optional[Dict[str, Any]] = None,
298
binary_data: bool = False):
299
"""
300
Specify requested output tensor.
301
302
Args:
303
name (str): Output tensor name
304
parameters (Dict[str, Any], optional): Output parameters
305
binary_data (bool): Request binary data format
306
"""
307
308
# Properties
309
name: str # Output tensor name
310
parameters: Optional[Dict[str, Any]] # Output parameters
311
binary_data: bool # Binary data flag
312
```
313
314
## Usage Examples
315
316
### Creating Inference Requests
317
318
```python
319
from kserve import InferRequest, InferInput
320
import numpy as np
321
322
# Create input tensor
323
input_data = np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32)
324
input_tensor = InferInput(
325
name="input",
326
shape=[1, 4],
327
datatype="FP32"
328
)
329
input_tensor.set_data_from_numpy(input_data)
330
331
# Create inference request
332
request = InferRequest(
333
model_name="iris-classifier",
334
inputs=[input_tensor],
335
request_id="req-123"
336
)
337
338
# Convert to different formats
339
rest_format = request.to_rest()
340
grpc_format = request.to_grpc()
341
```
342
343
### Handling Multiple Inputs
344
345
```python
346
import numpy as np
347
from kserve import InferRequest, InferInput
348
349
# Create multiple input tensors
350
image_data = np.random.rand(1, 224, 224, 3).astype(np.float32)
351
metadata = np.array([["image1.jpg"]], dtype=object)
352
353
# Image input
354
image_input = InferInput("images", [1, 224, 224, 3], "FP32")
355
image_input.set_data_from_numpy(image_data)
356
357
# Metadata input
358
meta_input = InferInput("metadata", [1, 1], "BYTES")
359
meta_input.set_data_from_numpy(metadata)
360
361
# Create request with multiple inputs
362
request = InferRequest(
363
model_name="multimodal-classifier",
364
inputs=[image_input, meta_input],
365
parameters={"temperature": 0.8}
366
)
367
```
368
369
### Processing Inference Responses
370
371
```python
372
from kserve import InferResponse
373
374
async def process_response(response: InferResponse):
375
print(f"Model: {response.model_name}")
376
print(f"Response ID: {response.response_id}")
377
378
# Process each output tensor
379
for output in response.outputs:
380
print(f"Output '{output.name}':")
381
print(f" Shape: {output.shape}")
382
print(f" Type: {output.datatype}")
383
384
# Convert to numpy for processing
385
data = output.as_numpy()
386
print(f" Data: {data}")
387
388
# Additional processing based on output type
389
if output.name == "probabilities":
390
predicted_class = np.argmax(data)
391
confidence = np.max(data)
392
print(f" Predicted class: {predicted_class}")
393
print(f" Confidence: {confidence:.3f}")
394
```
395
396
### Protocol Conversion
397
398
```python
399
from kserve import InferRequest, InferResponse
400
401
# Convert between protocols
402
def convert_request_formats(request: InferRequest):
403
# To REST format
404
rest_request = request.to_rest()
405
print("REST format:", rest_request)
406
407
# To gRPC format
408
grpc_request = request.to_grpc()
409
print("gRPC format:", grpc_request)
410
411
# From REST format
412
reconstructed = InferRequest.from_rest(rest_request)
413
414
return reconstructed
415
416
# Response conversion
417
def convert_response_formats(response_data: dict):
418
# From REST response
419
response = InferResponse.from_rest(response_data)
420
421
# Convert to different formats
422
grpc_format = response.to_grpc()
423
rest_format = response.to_rest()
424
425
return response
426
```
427
428
### Working with DataFrames
429
430
```python
431
import pandas as pd
432
from kserve import InferRequest, InferInput
433
434
# Create DataFrame input
435
df = pd.DataFrame({
436
'feature1': [1.0, 2.0, 3.0],
437
'feature2': [4.0, 5.0, 6.0],
438
'feature3': [7.0, 8.0, 9.0]
439
})
440
441
# Convert DataFrame to input tensor
442
input_tensor = InferInput(
443
name="tabular_data",
444
shape=[3, 3],
445
datatype="FP32"
446
)
447
input_tensor.set_data_from_numpy(df.values.astype(np.float32))
448
449
# Create request
450
request = InferRequest("tabular-model", [input_tensor])
451
452
# Convert request to DataFrame
453
request_df = request.as_dataframe()
454
print(request_df.head())
455
```
456
457
### Binary Data Handling
458
459
```python
460
from kserve import InferInput, RequestedOutput
461
462
# Handle binary input data
463
binary_data = b"binary content"
464
binary_input = InferInput("binary_input", [len(binary_data)], "BYTES")
465
binary_input.set_data_from_bytes(binary_data)
466
467
# Request binary output
468
binary_output = RequestedOutput("embeddings", binary_data=True)
469
470
request = InferRequest(
471
model_name="binary-model",
472
inputs=[binary_input],
473
request_outputs=[binary_output]
474
)
475
```
476
477
## Types
478
479
```python { .api }
480
from typing import List, Dict, Any, Optional, Union
481
import numpy as np
482
import pandas as pd
483
484
TensorShape = List[int]
485
DataType = str # "FP32", "FP64", "INT32", "INT64", "BOOL", "BYTES"
486
TensorData = Union[np.ndarray, List[Any], bytes]
487
Parameters = Dict[str, Any]
488
RequestID = str
489
ModelName = str
490
```