0
# Reference Implementation
1
2
Complete reference implementation of ONNX operators for testing, validation, and educational purposes. This module provides a Python-based execution engine that implements all ONNX operators according to the specification.
3
4
## Capabilities
5
6
### Reference Evaluator
7
8
Complete reference implementation for executing ONNX models with all standard operators.
9
10
```python { .api }
11
class ReferenceEvaluator:
12
"""
13
Reference implementation for ONNX model execution.
14
15
Provides accurate, specification-compliant execution of ONNX models
16
primarily for testing and validation purposes.
17
"""
18
19
def __init__(self, model_proto, verbose=0, **kwargs):
20
"""
21
Initialize reference evaluator with ONNX model.
22
23
Parameters:
24
- model_proto: ModelProto to execute
25
- verbose: Verbosity level for debugging (0=silent, 1=basic, 2=detailed)
26
- **kwargs: Additional evaluator options
27
28
Raises:
29
RuntimeError: If model contains unsupported operators or is invalid
30
"""
31
32
def run(self, output_names, feed_inputs, attributes=None):
33
"""
34
Execute model and return specified outputs.
35
36
Parameters:
37
- output_names: List of output names to compute (None for all outputs)
38
- feed_inputs: Dictionary mapping input names to numpy arrays
39
- attributes: Optional dictionary of additional attributes
40
41
Returns:
42
list: List of output arrays corresponding to output_names
43
44
Raises:
45
RuntimeError: If execution fails due to invalid inputs or operators
46
ValueError: If input shapes or types are incompatible
47
"""
48
```
49
50
## Usage Examples
51
52
### Basic Model Execution
53
54
```python
55
import onnx
56
from onnx.reference import ReferenceEvaluator
57
import numpy as np
58
59
# Load an ONNX model
60
model = onnx.load_model("example_model.onnx")
61
62
# Create reference evaluator
63
evaluator = ReferenceEvaluator(model, verbose=1)
64
65
# Prepare input data
66
input_data = {
67
"input": np.random.randn(1, 3, 224, 224).astype(np.float32)
68
}
69
70
try:
71
# Execute model
72
outputs = evaluator.run(None, input_data) # None means all outputs
73
74
print(f"Model executed successfully!")
75
print(f"Number of outputs: {len(outputs)}")
76
77
for i, output in enumerate(outputs):
78
print(f"Output {i} shape: {output.shape}, dtype: {output.dtype}")
79
80
except Exception as e:
81
print(f"Execution failed: {e}")
82
```
83
84
### Comparing with Other Backends
85
86
```python
87
import onnx
88
from onnx.reference import ReferenceEvaluator
89
import numpy as np
90
91
def compare_backends(model_path, input_data):
92
"""Compare reference implementation with other backends."""
93
94
model = onnx.load_model(model_path)
95
96
# Reference implementation
97
ref_evaluator = ReferenceEvaluator(model)
98
ref_outputs = ref_evaluator.run(None, input_data)
99
100
print("Reference implementation results:")
101
for i, output in enumerate(ref_outputs):
102
print(f" Output {i}: mean={output.mean():.6f}, std={output.std():.6f}")
103
104
# You could compare with other backends here
105
# For example, if you have ONNX Runtime installed:
106
"""
107
import onnxruntime as ort
108
109
ort_session = ort.InferenceSession(model_path)
110
ort_outputs = ort_session.run(None, input_data)
111
112
print("ONNX Runtime results:")
113
for i, output in enumerate(ort_outputs):
114
print(f" Output {i}: mean={output.mean():.6f}, std={output.std():.6f}")
115
116
# Compare results
117
for i, (ref_out, ort_out) in enumerate(zip(ref_outputs, ort_outputs)):
118
max_diff = np.max(np.abs(ref_out - ort_out))
119
print(f"Output {i} max difference: {max_diff}")
120
"""
121
122
# Example usage
123
input_data = {"input": np.random.randn(1, 10).astype(np.float32)}
124
# compare_backends("simple_model.onnx", input_data)
125
```
126
127
### Debugging Model Execution
128
129
```python
130
import onnx
131
from onnx.reference import ReferenceEvaluator
132
import numpy as np
133
134
def debug_model_execution(model_path, input_data):
135
"""Debug model execution with detailed logging."""
136
137
model = onnx.load_model(model_path)
138
139
# Create evaluator with maximum verbosity
140
evaluator = ReferenceEvaluator(model, verbose=2)
141
142
try:
143
print("Starting model execution with detailed logging...")
144
outputs = evaluator.run(None, input_data)
145
146
print("Execution completed successfully!")
147
return outputs
148
149
except Exception as e:
150
print(f"Execution failed at: {e}")
151
print("This can help identify:")
152
print("- Which operator caused the failure")
153
print("- Input/output shape mismatches")
154
print("- Type conversion issues")
155
print("- Unsupported operator attributes")
156
return None
157
158
# Example debugging session
159
def create_debug_model():
160
"""Create a simple model for debugging demonstration."""
161
from onnx import helper, TensorProto
162
163
# Create a model with potential issues
164
X = helper.make_tensor_value_info('X', TensorProto.FLOAT, [2, 3])
165
Y = helper.make_tensor_value_info('Y', TensorProto.FLOAT, [2, 3])
166
167
# Create nodes that might have issues
168
relu_node = helper.make_node('Relu', ['X'], ['relu_out'])
169
170
# Intentionally problematic reshape (wrong dimensions)
171
reshape_node = helper.make_node('Reshape', ['relu_out'], ['Y'],
172
shape=[6]) # This will cause shape mismatch
173
174
graph = helper.make_graph([relu_node, reshape_node], 'debug_model',
175
[X], [Y])
176
return helper.make_model(graph)
177
178
# Test debugging
179
debug_model = create_debug_model()
180
test_input = {"X": np.array([[1, -2, 3], [4, -5, 6]], dtype=np.float32)}
181
182
print("=== Debugging Model Execution ===")
183
try:
184
evaluator = ReferenceEvaluator(debug_model, verbose=1)
185
result = evaluator.run(None, test_input)
186
print("Unexpected success!")
187
except Exception as e:
188
print(f"Expected error caught: {e}")
189
print("This demonstrates how the reference evaluator helps identify issues")
190
```
191
192
### Testing Operator Implementations
193
194
```python
195
import onnx
196
from onnx.reference import ReferenceEvaluator
197
from onnx import helper, TensorProto
198
import numpy as np
199
200
def test_operator_reference(op_type, inputs, attributes=None, **kwargs):
201
"""Test reference implementation of a specific operator."""
202
203
# Create minimal model with just the operator
204
input_infos = []
205
input_names = []
206
207
for i, input_array in enumerate(inputs):
208
input_name = f"input_{i}"
209
input_names.append(input_name)
210
input_info = helper.make_tensor_value_info(
211
input_name, TensorProto.FLOAT, list(input_array.shape)
212
)
213
input_infos.append(input_info)
214
215
# Create output info (shape will be inferred)
216
output_info = helper.make_tensor_value_info('output', TensorProto.FLOAT, [])
217
218
# Create node
219
node_attrs = attributes or {}
220
node = helper.make_node(op_type, input_names, ['output'], **node_attrs)
221
222
# Create graph and model
223
graph = helper.make_graph([node], f'{op_type}_test',
224
input_infos, [output_info])
225
model = helper.make_model(graph)
226
227
# Test with reference evaluator
228
evaluator = ReferenceEvaluator(model, **kwargs)
229
230
# Prepare input dictionary
231
feed_dict = {f"input_{i}": inp for i, inp in enumerate(inputs)}
232
233
try:
234
outputs = evaluator.run(['output'], feed_dict)
235
return outputs[0]
236
except Exception as e:
237
print(f"Operator {op_type} test failed: {e}")
238
return None
239
240
# Test various operators
241
def run_operator_tests():
242
"""Run tests for various operators."""
243
244
print("=== Testing Reference Implementation Operators ===")
245
246
# Test Add operator
247
a = np.array([[1, 2], [3, 4]], dtype=np.float32)
248
b = np.array([[5, 6], [7, 8]], dtype=np.float32)
249
250
add_result = test_operator_reference('Add', [a, b])
251
if add_result is not None:
252
print(f"Add result:\n{add_result}")
253
print(f"Expected:\n{a + b}")
254
print(f"Correct: {np.allclose(add_result, a + b)}")
255
256
print()
257
258
# Test Relu operator
259
x = np.array([[-1, 2], [-3, 4]], dtype=np.float32)
260
relu_result = test_operator_reference('Relu', [x])
261
if relu_result is not None:
262
print(f"Relu result:\n{relu_result}")
263
print(f"Expected:\n{np.maximum(0, x)}")
264
print(f"Correct: {np.allclose(relu_result, np.maximum(0, x))}")
265
266
print()
267
268
# Test Conv operator (more complex)
269
input_tensor = np.random.randn(1, 1, 5, 5).astype(np.float32)
270
weight_tensor = np.random.randn(1, 1, 3, 3).astype(np.float32)
271
272
conv_result = test_operator_reference('Conv', [input_tensor, weight_tensor],
273
attributes={'kernel_shape': [3, 3],
274
'pads': [1, 1, 1, 1]})
275
if conv_result is not None:
276
print(f"Conv output shape: {conv_result.shape}")
277
print("Conv operation completed successfully")
278
279
print()
280
281
# Run the operator tests
282
run_operator_tests()
283
```