0
# Neural Network Operations
1

2
Core neural network operations including activations, convolutions, pooling, normalization, and loss functions. These operations provide the fundamental building blocks for constructing and training neural networks.
3

4
## Capabilities
5

6
### Activation Functions
7

8
Non-linear activation functions that introduce non-linearity into neural networks.
9

10
```python { .api }
11
def relu(features, name=None):
12
    """
13
    Computes rectified linear: max(features, 0).
14
    
15
    Parameters:
16
    - features: A Tensor. Must be one of the following types: float32, float64, int32, uint8, int16, int8, int64, bfloat16, uint16, half, uint32, uint64
17
    - name: A name for the operation
18
    
19
    Returns:
20
    A Tensor. Has the same type as features
21
    """
22

23
def relu6(features, name=None):
24
    """
25
    Computes Rectified Linear 6: min(max(features, 0), 6).
26
    
27
    Parameters:
28
    - features: A Tensor with type float, double, int32, uint8, int16, or int8
29
    - name: A name for the operation
30
    
31
    Returns:
32
    A Tensor with the same type as features
33
    """
34

35
def elu(features, name=None):
36
    """
37
    Computes exponential linear: exp(features) - 1 if < 0, features otherwise.
38
    
39
    Parameters:
40
    - features: A Tensor. Must be one of the following types: half, bfloat16, float32, float64
41
    - name: A name for the operation
42
    
43
    Returns:
44
    A Tensor. Has the same type as features
45
    """
46

47
def sigmoid(x, name=None):
48
    """
49
    Computes sigmoid of x element-wise.
50
    
51
    Parameters:
52
    - x: A Tensor with type float16, float32, float64, complex64, or complex128
53
    - name: A name for the operation
54
    
55
    Returns:
56
    A Tensor with the same type as x
57
    """
58

59
def tanh(x, name=None):
60
    """
61
    Computes hyperbolic tangent of x element-wise.
62
    
63
    Parameters:
64
    - x: A Tensor. Must be one of the following types: bfloat16, half, float32, float64, complex64, complex128
65
    - name: A name for the operation
66
    
67
    Returns:
68
    A Tensor. Has the same type as x
69
    """
70

71
def softmax(logits, axis=None, name=None):
72
    """
73
    Computes softmax activations.
74
    
75
    Parameters:
76
    - logits: A non-empty Tensor. Must be one of the following types: half, bfloat16, float32, float64
77
    - axis: The dimension softmax would be performed on. The default is -1 which indicates the last dimension
78
    - name: A name for the operation
79
    
80
    Returns:
81
    A Tensor. Has the same type and shape as logits
82
    """
83

84
def log_softmax(logits, axis=None, name=None):
85
    """
86
    Computes log softmax activations.
87
    
88
    Parameters:
89
    - logits: A non-empty Tensor. Must be one of the following types: half, bfloat16, float32, float64
90
    - axis: The dimension softmax would be performed on. The default is -1 which indicates the last dimension
91
    - name: A name for the operation
92
    
93
    Returns:
94
    A Tensor. Has the same type and shape as logits
95
    """
96

97
def leaky_relu(features, alpha=0.2, name=None):
98
    """
99
    Compute the Leaky ReLU activation function.
100
    
101
    Parameters:
102
    - features: A Tensor representing preactivation values. Must be one of the following types: float16, float32, float64, int32, int64
103
    - alpha: Slope of the activation function at x < 0
104
    - name: A name for the operation
105
    
106
    Returns:
107
    The activation value
108
    """
109

110
def gelu(features, approximate=False, name=None):
111
    """
112
    Compute the Gaussian Error Linear Unit (GELU) activation function.
113
    
114
    Parameters:
115
    - features: A Tensor representing preactivation values
116
    - approximate: An optional bool. Defaults to False. Whether to enable approximation
117
    - name: A name for the operation
118
    
119
    Returns:
120
    A Tensor with the same type as features
121
    """
122

123
def swish(features, name=None):
124
    """
125
    Computes the Swish activation function: features * sigmoid(features).
126
    
127
    Parameters:
128
    - features: A Tensor representing preactivation values
129
    - name: A name for the operation
130
    
131
    Returns:
132
    The activation value
133
    """
134
```
135

136
### Convolution Operations
137

138
Convolution operations for processing spatial data like images.
139

140
```python { .api }
141
def conv2d(input, filters, strides, padding, use_cudnn_on_gpu=True, data_format="NHWC", 
142
           dilations=[1,1,1,1], name=None):
143
    """
144
    Computes a 2-D convolution given 4-D input and filter tensors.
145
    
146
    Parameters:
147
    - input: A Tensor. Must be one of the following types: half, bfloat16, float32, float64
148
    - filters: A Tensor. Must have the same type as input
149
    - strides: An int or list of ints that has length 1, 2 or 4
150
    - padding: Either the string "SAME" or "VALID" indicating the type of padding algorithm to use
151
    - use_cudnn_on_gpu: An optional bool. Defaults to True. Whether to use cuDNN on GPU when available
152
    - data_format: An optional string from: "NHWC", "NCHW". Defaults to "NHWC"
153
    - dilations: A list of ints. Defaults to [1, 1, 1, 1]. The dilation factor for each dimension of input
154
    - name: A name for the operation
155
    
156
    Returns:
157
    A Tensor. Has the same type as input
158
    """
159

160
def conv2d_transpose(input, filters, output_shape, strides, padding="SAME",
161
                     data_format="NHWC", dilations=None, name=None):
162
    """
163
    The transpose of conv2d.
164
    
165
    Parameters:
166
    - input: A 4-D Tensor of type float and shape [batch, height, width, in_channels] for NHWC data format
167
    - filters: A 4-D Tensor with the same type as input and shape [height, width, output_channels, in_channels]
168
    - output_shape: A 1-D Tensor representing the output shape of the deconvolution op
169
    - strides: An int or list of ints that has length 1, 2 or 4
170
    - padding: A string, either 'VALID' or 'SAME'
171
    - data_format: A string. 'NHWC' and 'NCHW' are supported
172
    - dilations: An int or list of ints that has length 1, 2 or 4, defaults to 1
173
    - name: Optional name for the returned tensor
174
    
175
    Returns:
176
    A Tensor with the same type as input
177
    """
178

179
def depthwise_conv2d(input, filter, strides, padding, data_format=None,
180
                     dilations=None, name=None):
181
    """
182
    Depthwise 2-D convolution.
183
    
184
    Parameters:
185
    - input: 4-D with shape according to data_format
186
    - filter: 4-D with shape [filter_height, filter_width, in_channels, channel_multiplier]
187
    - strides: 1-D of size 4. The stride of the sliding window for each dimension of input
188
    - padding: Controls how to pad the image before applying the convolution
189
    - data_format: The data format for input. Either "NHWC" (default) or "NCHW"
190
    - dilations: 1-D of size 2. The dilation rate in which we sample input values
191
    - name: A name for this operation
192
    
193
    Returns:
194
    A 4-D Tensor with shape according to data_format
195
    """
196

197
def separable_conv2d(input, depthwise_filter, pointwise_filter, strides,
198
                     padding, data_format=None, dilations=None, name=None):
199
    """
200
    2-D convolution with separable filters.
201
    
202
    Parameters:
203
    - input: 4-D Tensor with shape according to data_format
204
    - depthwise_filter: 4-D Tensor with shape [filter_height, filter_width, in_channels, channel_multiplier]
205
    - pointwise_filter: 4-D Tensor with shape [1, 1, channel_multiplier * in_channels, out_channels]
206
    - strides: 1-D of size 4. The stride of the sliding window for each dimension of input
207
    - padding: Controls how to pad the image before applying the convolution
208
    - data_format: The data format for input. Either "NHWC" (default) or "NCHW"
209
    - dilations: 1-D of size 2. The dilation rate in which we sample input values
210
    - name: A name for this operation
211
    
212
    Returns:
213
    A 4-D Tensor with shape according to data_format
214
    """
215
```
216

217
### Pooling Operations
218

219
Pooling operations for downsampling and feature extraction.
220

221
```python { .api }
222
def max_pool2d(input, ksize, strides, padding, data_format="NHWC", name=None):
223
    """
224
    Performs the max pooling on the input.
225
    
226
    Parameters:
227
    - input: A 4-D Tensor of the format specified by data_format
228
    - ksize: An int or list of ints that has length 1, 2 or 4
229
    - strides: An int or list of ints that has length 1, 2 or 4
230
    - padding: Either the string "SAME" or "VALID" indicating the type of padding algorithm to use
231
    - data_format: A string. 'NHWC' and 'NCHW' are supported
232
    - name: Optional name for the operation
233
    
234
    Returns:
235
    A Tensor of format specified by data_format
236
    """
237

238
def avg_pool2d(input, ksize, strides, padding, data_format="NHWC", name=None):
239
    """
240
    Performs the average pooling on the input.
241
    
242
    Parameters:
243
    - input: A 4-D Tensor of shape [batch, height, width, channels] and type float32, float64, qint8, quint8, or qint32
244
    - ksize: An int or list of ints that has length 1, 2 or 4
245
    - strides: An int or list of ints that has length 1, 2 or 4
246
    - padding: A string, either 'VALID' or 'SAME'
247
    - data_format: A string. 'NHWC' and 'NCHW' are supported
248
    - name: Optional name for the operation
249
    
250
    Returns:
251
    A Tensor with the same type as input
252
    """
253

254
def global_max_pool2d(input, data_format="NHWC", name=None):
255
    """
256
    Performs global max pooling on the input.
257
    
258
    Parameters:
259
    - input: A 4-D Tensor of the format specified by data_format
260
    - data_format: A string. 'NHWC' and 'NCHW' are supported
261
    - name: Optional name for the operation
262
    
263
    Returns:
264
    A Tensor of format specified by data_format
265
    """
266

267
def global_avg_pool2d(input, data_format="NHWC", name=None):
268
    """
269
    Performs global average pooling on the input.
270
    
271
    Parameters:
272
    - input: A 4-D Tensor of the format specified by data_format
273
    - data_format: A string. 'NHWC' and 'NCHW' are supported
274
    - name: Optional name for the operation
275
    
276
    Returns:
277
    A Tensor of format specified by data_format
278
    """
279
```
280

281
### Normalization
282

283
Normalization operations for training stability and performance.
284

285
```python { .api }
286
def batch_normalization(x, mean, variance, offset, scale, variance_epsilon, name=None):
287
    """
288
    Batch normalization.
289
    
290
    Parameters:
291
    - x: Input Tensor
292
    - mean: A mean Tensor
293
    - variance: A variance Tensor
294
    - offset: An offset Tensor, often denoted β in equations, or None
295
    - scale: A scale Tensor, often denoted γ in equations, or None
296
    - variance_epsilon: A small float number to avoid dividing by 0
297
    - name: A name for this operation
298
    
299
    Returns:
300
    the normalized, scaled, offset tensor
301
    """
302

303
def layer_normalization(inputs, begin_norm_axis=1, begin_params_axis=-1, name=None):
304
    """
305
    Applies layer normalization.
306
    
307
    Parameters:
308
    - inputs: A tensor with 2 or more dimensions, where the first dimension has batch_size
309
    - begin_norm_axis: The first normalization dimension: normalization will be performed along dimensions begin_norm_axis : rank(inputs)
310
    - begin_params_axis: Part of the standard interface, unused
311
    - name: A name for this operation
312
    
313
    Returns:
314
    A normalized Tensor with the same shape as inputs
315
    """
316

317
def local_response_normalization(input, depth_radius=5, bias=1, alpha=1, beta=0.5, name=None):
318
    """
319
    Local Response Normalization.
320
    
321
    Parameters:
322
    - input: A Tensor. Must be one of the following types: half, bfloat16, float32
323
    - depth_radius: An optional int. Defaults to 5. 0-D. Half-width of the 1-D normalization window
324
    - bias: An optional float. Defaults to 1. An offset (usually positive to avoid dividing by 0)
325
    - alpha: An optional float. Defaults to 1. A scale factor, usually positive
326
    - beta: An optional float. Defaults to 0.5. An exponent
327
    - name: A name for the operation
328
    
329
    Returns:
330
    A Tensor. Has the same type as input
331
    """
332

333
def l2_normalize(x, axis=None, epsilon=1e-12, name=None):
334
    """
335
    Normalizes along dimension axis using an L2 norm.
336
    
337
    Parameters:
338
    - x: A Tensor
339
    - axis: Dimension along which to normalize. A scalar or a vector of integers
340
    - epsilon: A lower bound value for the norm. Will use sqrt(epsilon) as the divisor if norm < sqrt(epsilon)
341
    - name: A name for this operation
342
    
343
    Returns:
344
    A Tensor with the same shape as x
345
    """
346
```
347

348
### Loss Functions
349

350
Loss functions for training neural networks.
351

352
```python { .api }
353
def softmax_cross_entropy_with_logits(labels, logits, axis=-1, name=None):
354
    """
355
    Computes softmax cross entropy between logits and labels.
356
    
357
    Parameters:
358
    - labels: Each vector along the class dimension should hold a valid probability distribution
359
    - logits: Per-label activations, typically a linear output
360
    - axis: The class dimension. Defaulted to -1 which is the last dimension
361
    - name: A name for the operation
362
    
363
    Returns:
364
    A Tensor that contains the softmax cross entropy loss
365
    """
366

367
def sparse_softmax_cross_entropy_with_logits(labels, logits, name=None):
368
    """
369
    Computes sparse softmax cross entropy between logits and labels.
370
    
371
    Parameters:
372
    - labels: Tensor of shape [...] and dtype int32 or int64
373
    - logits: Per-label activations of shape [..., num_classes] and dtype float16, float32, or float64
374
    - name: A name for the operation
375
    
376
    Returns:
377
    A Tensor of the same shape as labels and of the same type as logits with the softmax cross entropy loss
378
    """
379

380
def sigmoid_cross_entropy_with_logits(labels, logits, name=None):
381
    """
382
    Computes sigmoid cross entropy given logits.
383
    
384
    Parameters:
385
    - labels: A Tensor of the same type and shape as logits
386
    - logits: A Tensor of type float32 or float64
387
    - name: A name for the operation
388
    
389
    Returns:
390
    A Tensor of the same shape as logits with the componentwise logistic losses
391
    """
392

393
def l2_loss(t, name=None):
394
    """
395
    Computes half the L2 norm of a tensor without the sqrt.
396
    
397
    Parameters:
398
    - t: A Tensor. Must be one of the following types: half, bfloat16, float32, float64
399
    - name: A name for the operation
400
    
401
    Returns:
402
    A Tensor. Has the same type as t
403
    """
404

405
def mean_squared_error(y_true, y_pred):
406
    """
407
    Computes the mean squared error between labels and predictions.
408
    
409
    Parameters:
410
    - y_true: Ground truth values
411
    - y_pred: The predicted values
412
    
413
    Returns:
414
    Mean squared error values
415
    """
416

417
def mean_absolute_error(y_true, y_pred):
418
    """
419
    Computes the mean absolute error between labels and predictions.
420
    
421
    Parameters:
422
    - y_true: Ground truth values
423
    - y_pred: The predicted values
424
    
425
    Returns:
426
    Mean absolute error values
427
    """
428
```
429

430
### Dropout and Regularization
431

432
Operations for regularization and preventing overfitting.
433

434
```python { .api }
435
def dropout(x, rate, noise_shape=None, seed=None, training=None, name=None):
436
    """
437
    Computes dropout: randomly sets elements to zero to prevent overfitting.
438
    
439
    Parameters:
440
    - x: A floating point tensor
441
    - rate: A scalar Tensor with the same type as x. The probability that each element is discarded
442
    - noise_shape: A 1-D integer Tensor, representing the shape for randomly generated keep/drop flags
443
    - seed: A Python integer. Used to create random seeds
444
    - training: Either a Python boolean, or a TensorFlow boolean scalar tensor
445
    - name: A name for this operation
446
    
447
    Returns:
448
    A Tensor of the same shape of x
449
    """
450

451
def spatial_dropout(x, rate, data_format="channels_last", name=None):
452
    """
453
    Spatial 2D version of Dropout.
454
    
455
    Parameters:
456
    - x: A 4D tensor
457
    - rate: Float between 0 and 1. Fraction of the input units to drop
458
    - data_format: 'channels_first' or 'channels_last'
459
    - name: A name for this operation
460
    
461
    Returns:
462
    A tensor of the same shape as x
463
    """
464
```
465

466
## Usage Examples
467

468
```python
469
import tensorflow as tf
470

471
# Activation functions
472
x = tf.constant([-2.0, -1.0, 0.0, 1.0, 2.0])
473
relu_out = tf.nn.relu(x)           # [0.0, 0.0, 0.0, 1.0, 2.0]
474
sigmoid_out = tf.nn.sigmoid(x)     # [0.119, 0.269, 0.5, 0.731, 0.881]
475
tanh_out = tf.nn.tanh(x)          # [-0.964, -0.762, 0.0, 0.762, 0.964]
476

477
logits = tf.constant([[1.0, 2.0, 3.0], [1.0, 2.0, 3.0]])
478
softmax_out = tf.nn.softmax(logits)  # [[0.09, 0.244, 0.665], [0.09, 0.244, 0.665]]
479

480
# Convolution operations
481
input_data = tf.random.normal([1, 32, 32, 3])  # Batch, Height, Width, Channels
482
filters = tf.random.normal([5, 5, 3, 64])      # Height, Width, In_channels, Out_channels
483
conv_out = tf.nn.conv2d(input_data, filters, strides=[1, 1, 1, 1], padding='SAME')
484

485
# Pooling operations
486
max_pool_out = tf.nn.max_pool2d(conv_out, ksize=2, strides=2, padding='VALID')
487
avg_pool_out = tf.nn.avg_pool2d(conv_out, ksize=2, strides=2, padding='VALID')
488

489
# Loss functions
490
y_true = tf.constant([0, 1, 2])
491
y_pred = tf.constant([[0.1, 0.8, 0.1], [0.2, 0.7, 0.1], [0.1, 0.2, 0.7]])
492
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred)
493

494
# Dropout
495
training_data = tf.random.normal([32, 128])
496
dropped_out = tf.nn.dropout(training_data, rate=0.5, training=True)
497
```