0
# Neural Network Layers
1

2
Complete collection of neural network layer types for building deep learning models, including core layers, convolutional layers, recurrent layers, normalization, attention mechanisms, and preprocessing layers.
3

4
## Capabilities
5

6
### Core Layers
7

8
Fundamental layers for neural network construction including fully connected layers, embeddings, and basic transformations.
9

10
```python { .api }
11
class Dense:
12
    """
13
    Fully connected (dense) layer.
14
    
15
    Args:
16
        units (int): Number of output units
17
        activation (str or callable, optional): Activation function
18
        use_bias (bool): Whether to use bias terms
19
        kernel_initializer (str or callable): Weight initialization
20
        bias_initializer (str or callable): Bias initialization
21
        kernel_regularizer: Weight regularization
22
        bias_regularizer: Bias regularization
23
        kernel_constraint: Weight constraints
24
        bias_constraint: Bias constraints
25
    """
26
    def __init__(self, units, activation=None, use_bias=True, **kwargs): ...
27

28
class Embedding:
29
    """
30
    Turns positive integers into dense vectors of fixed size.
31
    
32
    Args:
33
        input_dim (int): Size of vocabulary
34
        output_dim (int): Size of dense vectors
35
        embeddings_initializer (str or callable): Embedding initialization
36
        embeddings_regularizer: Embedding regularization
37
        embeddings_constraint: Embedding constraints
38
        mask_zero (bool): Whether to mask zero values
39
        input_length (int, optional): Input sequence length
40
    """
41
    def __init__(self, input_dim, output_dim, **kwargs): ...
42

43
class Identity:
44
    """Identity layer (pass-through)."""
45
    def __init__(self, **kwargs): ...
46

47
class Lambda:
48
    """
49
    Wraps arbitrary expressions as a layer.
50
    
51
    Args:
52
        function: Function to be wrapped
53
        output_shape: Output shape of the function
54
        mask: Mask value to be passed to the function
55
        arguments (dict): Optional keyword arguments to pass to function
56
    """
57
    def __init__(self, function, output_shape=None, mask=None, arguments=None, **kwargs): ...
58
```
59

60
### Activation Layers
61

62
Dedicated activation layers that can be inserted between other layers.
63

64
```python { .api }
65
class Activation:
66
    """
67
    Applies activation function to input.
68
    
69
    Args:
70
        activation (str or callable): Activation function to use
71
    """
72
    def __init__(self, activation, **kwargs): ...
73

74
class ReLU:
75
    """
76
    Rectified Linear Unit activation.
77
    
78
    Args:
79
        max_value (float, optional): Maximum value to clip at
80
        negative_slope (float): Slope for negative values
81
        threshold (float): Threshold value for activation
82
    """
83
    def __init__(self, max_value=None, negative_slope=0.0, threshold=0.0, **kwargs): ...
84

85
class LeakyReLU:
86
    """
87
    Leaky Rectified Linear Unit.
88
    
89
    Args:
90
        negative_slope (float): Slope for negative values
91
    """
92
    def __init__(self, negative_slope=0.3, **kwargs): ...
93

94
class Softmax:
95
    """
96
    Softmax activation layer.
97
    
98
    Args:
99
        axis (int): Axis along which to apply softmax
100
    """
101
    def __init__(self, axis=-1, **kwargs): ...
102
```
103

104
### Convolutional Layers
105

106
Convolutional layers for processing grid-like data such as images, supporting 1D, 2D, and 3D convolutions.
107

108
```python { .api }
109
class Conv2D:
110
    """
111
    2D convolution layer.
112
    
113
    Args:
114
        filters (int): Number of output filters
115
        kernel_size (int or tuple): Size of convolution kernel
116
        strides (int or tuple): Convolution stride
117
        padding (str): Padding mode ('valid' or 'same')
118
        data_format (str): Data format ('channels_last' or 'channels_first')
119
        dilation_rate (int or tuple): Dilation rate
120
        groups (int): Number of groups for grouped convolution
121
        activation (str or callable, optional): Activation function
122
        use_bias (bool): Whether to use bias
123
        kernel_initializer: Kernel initialization
124
        bias_initializer: Bias initialization
125
    """
126
    def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', **kwargs): ...
127

128
class Conv1D:
129
    """1D convolution layer."""
130
    def __init__(self, filters, kernel_size, strides=1, padding='valid', **kwargs): ...
131

132
class Conv3D:
133
    """3D convolution layer."""
134
    def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding='valid', **kwargs): ...
135

136
class Conv2DTranspose:
137
    """
138
    Transposed 2D convolution layer (deconvolution).
139
    
140
    Args:
141
        filters (int): Number of output filters
142
        kernel_size (int or tuple): Size of convolution kernel
143
        strides (int or tuple): Convolution stride
144
        padding (str): Padding mode
145
        output_padding (int or tuple, optional): Output padding
146
    """
147
    def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', **kwargs): ...
148

149
class DepthwiseConv2D:
150
    """
151
    Depthwise 2D convolution.
152
    
153
    Args:
154
        kernel_size (int or tuple): Size of convolution kernel
155
        strides (int or tuple): Convolution stride
156
        padding (str): Padding mode
157
        depth_multiplier (int): Number of depthwise convolution output channels per input channel
158
    """
159
    def __init__(self, kernel_size, strides=(1, 1), padding='valid', depth_multiplier=1, **kwargs): ...
160

161
class SeparableConv2D:
162
    """
163
    Separable 2D convolution (depthwise followed by pointwise).
164
    
165
    Args:
166
        filters (int): Number of output filters
167
        kernel_size (int or tuple): Size of convolution kernel
168
        strides (int or tuple): Convolution stride
169
        padding (str): Padding mode
170
        depth_multiplier (int): Depthwise multiplier
171
    """
172
    def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', **kwargs): ...
173
```
174

175
### Pooling Layers
176

177
Pooling layers for downsampling and feature extraction including average pooling, max pooling, and global pooling.
178

179
```python { .api }
180
class MaxPooling2D:
181
    """
182
    Max pooling for 2D spatial data.
183
    
184
    Args:
185
        pool_size (int or tuple): Pooling window size
186
        strides (int or tuple, optional): Pooling stride
187
        padding (str): Padding mode
188
        data_format (str): Data format
189
    """
190
    def __init__(self, pool_size=(2, 2), strides=None, padding='valid', **kwargs): ...
191

192
class AveragePooling2D:
193
    """Average pooling for 2D spatial data."""
194
    def __init__(self, pool_size=(2, 2), strides=None, padding='valid', **kwargs): ...
195

196
class GlobalMaxPooling2D:
197
    """Global max pooling for 2D spatial data."""
198
    def __init__(self, data_format=None, keepdims=False, **kwargs): ...
199

200
class GlobalAveragePooling2D:
201
    """Global average pooling for 2D spatial data."""
202
    def __init__(self, data_format=None, keepdims=False, **kwargs): ...
203

204
# Similar 1D and 3D variants
205
class MaxPooling1D:
206
    def __init__(self, pool_size=2, strides=None, padding='valid', **kwargs): ...
207

208
class MaxPooling3D:
209
    def __init__(self, pool_size=(2, 2, 2), strides=None, padding='valid', **kwargs): ...
210
```
211

212
### Recurrent Layers
213

214
Recurrent neural network layers for sequence processing including LSTM, GRU, and simple RNN variants.
215

216
```python { .api }
217
class LSTM:
218
    """
219
    Long Short-Term Memory layer.
220
    
221
    Args:
222
        units (int): Number of recurrent units
223
        activation (str or callable): Activation function
224
        recurrent_activation (str or callable): Recurrent activation
225
        use_bias (bool): Whether to use bias
226
        kernel_initializer: Input weight initialization
227
        recurrent_initializer: Recurrent weight initialization
228
        bias_initializer: Bias initialization
229
        dropout (float): Dropout rate for inputs
230
        recurrent_dropout (float): Dropout rate for recurrent connections
231
        return_sequences (bool): Whether to return full sequences
232
        return_state (bool): Whether to return final state
233
        go_backwards (bool): Whether to process sequences backwards
234
        stateful (bool): Whether to maintain state between batches
235
        unroll (bool): Whether to unroll the network
236
    """
237
    def __init__(self, units, activation='tanh', recurrent_activation='sigmoid',
238
                 use_bias=True, dropout=0.0, recurrent_dropout=0.0,
239
                 return_sequences=False, return_state=False, **kwargs): ...
240

241
class GRU:
242
    """
243
    Gated Recurrent Unit layer.
244
    
245
    Args:
246
        units (int): Number of recurrent units
247
        activation (str or callable): Activation function
248
        recurrent_activation (str or callable): Recurrent activation
249
        use_bias (bool): Whether to use bias
250
        dropout (float): Dropout rate for inputs
251
        recurrent_dropout (float): Dropout rate for recurrent connections
252
        return_sequences (bool): Whether to return full sequences
253
        return_state (bool): Whether to return final state
254
    """
255
    def __init__(self, units, activation='tanh', recurrent_activation='sigmoid',
256
                 use_bias=True, dropout=0.0, recurrent_dropout=0.0,
257
                 return_sequences=False, return_state=False, **kwargs): ...
258

259
class SimpleRNN:
260
    """Simple recurrent neural network layer."""
261
    def __init__(self, units, activation='tanh', use_bias=True, dropout=0.0,
262
                 recurrent_dropout=0.0, return_sequences=False, return_state=False, **kwargs): ...
263

264
class Bidirectional:
265
    """
266
    Bidirectional wrapper for RNNs.
267
    
268
    Args:
269
        layer: RNN layer to wrap
270
        merge_mode (str): How to combine forward and backward outputs
271
        backward_layer: Optional separate backward layer
272
    """
273
    def __init__(self, layer, merge_mode='concat', backward_layer=None, **kwargs): ...
274

275
class TimeDistributed:
276
    """
277
    Applies a layer to every temporal slice of an input.
278
    
279
    Args:
280
        layer: Layer to be applied to each temporal slice
281
    """
282
    def __init__(self, layer, **kwargs): ...
283
```
284

285
### Normalization Layers
286

287
Normalization layers for stabilizing and accelerating training including batch normalization, layer normalization, and other variants.
288

289
```python { .api }
290
class BatchNormalization:
291
    """
292
    Batch normalization layer.
293
    
294
    Args:
295
        axis (int): Axis to normalize along
296
        momentum (float): Momentum for moving average
297
        epsilon (float): Small constant for numerical stability
298
        center (bool): Whether to add offset parameter
299
        scale (bool): Whether to add scale parameter
300
        beta_initializer: Offset parameter initialization
301
        gamma_initializer: Scale parameter initialization
302
    """
303
    def __init__(self, axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True, **kwargs): ...
304

305
class LayerNormalization:
306
    """
307
    Layer normalization layer.
308
    
309
    Args:
310
        axis (int or list): Axis/axes to normalize
311
        epsilon (float): Small constant for numerical stability
312
        center (bool): Whether to add offset parameter
313
        scale (bool): Whether to add scale parameter
314
    """
315
    def __init__(self, axis=-1, epsilon=1e-3, center=True, scale=True, **kwargs): ...
316

317
class GroupNormalization:
318
    """
319
    Group normalization layer.
320
    
321
    Args:
322
        groups (int): Number of groups for GroupNorm
323
        axis (int): Axis to normalize along
324
        epsilon (float): Small constant for numerical stability
325
        center (bool): Whether to add offset parameter
326
        scale (bool): Whether to add scale parameter
327
    """
328
    def __init__(self, groups=32, axis=-1, epsilon=1e-3, center=True, scale=True, **kwargs): ...
329
```
330

331
### Attention Layers
332

333
Attention mechanisms for handling sequential and structured data with focus on relevant information.
334

335
```python { .api }
336
class MultiHeadAttention:
337
    """
338
    Multi-head attention layer.
339
    
340
    Args:
341
        num_heads (int): Number of attention heads
342
        key_dim (int): Size of each attention head for query and key
343
        value_dim (int, optional): Size of each attention head for value
344
        dropout (float): Dropout rate
345
        use_bias (bool): Whether to use bias in linear transformations
346
        output_shape (int, optional): Output dimensionality
347
        attention_axes (int or tuple, optional): Axes over which attention is applied
348
    """
349
    def __init__(self, num_heads, key_dim, value_dim=None, dropout=0.0, **kwargs): ...
350

351
class Attention:
352
    """
353
    Basic attention layer.
354
    
355
    Args:
356
        use_scale (bool): Whether to scale attention scores
357
        score_mode (str): How to compute attention scores
358
        dropout (float): Dropout rate for attention weights
359
    """
360
    def __init__(self, use_scale=False, score_mode='dot', dropout=0.0, **kwargs): ...
361

362
class AdditiveAttention:
363
    """
364
    Additive attention mechanism.
365
    
366
    Args:
367
        use_scale (bool): Whether to use learnable scale parameter
368
        dropout (float): Dropout rate for attention weights
369
    """
370
    def __init__(self, use_scale=True, dropout=0.0, **kwargs): ...
371
```
372

373
### Regularization Layers
374

375
Regularization layers for preventing overfitting including dropout variants and noise injection.
376

377
```python { .api }
378
class Dropout:
379
    """
380
    Applies dropout to input.
381
    
382
    Args:
383
        rate (float): Fraction of input units to drop
384
        noise_shape (tuple, optional): Shape of binary dropout mask
385
        seed (int, optional): Random seed
386
    """
387
    def __init__(self, rate, noise_shape=None, seed=None, **kwargs): ...
388

389
class SpatialDropout2D:
390
    """
391
    Spatial dropout for 2D inputs.
392
    
393
    Args:
394
        rate (float): Fraction of input channels to drop
395
        data_format (str): Data format
396
    """
397
    def __init__(self, rate, data_format=None, **kwargs): ...
398

399
class GaussianNoise:
400
    """
401
    Applies Gaussian noise to input.
402
    
403
    Args:
404
        stddev (float): Standard deviation of noise
405
    """
406
    def __init__(self, stddev, **kwargs): ...
407

408
class GaussianDropout:
409
    """
410
    Applies multiplicative Gaussian noise.
411
    
412
    Args:
413
        rate (float): Drop probability
414
    """
415
    def __init__(self, rate, **kwargs): ...
416
```
417

418
### Reshaping Layers
419

420
Layers for manipulating tensor shapes and dimensions including flattening, reshaping, and padding.
421

422
```python { .api }
423
class Flatten:
424
    """
425
    Flattens input tensor.
426
    
427
    Args:
428
        data_format (str, optional): Data format
429
    """
430
    def __init__(self, data_format=None, **kwargs): ...
431

432
class Reshape:
433
    """
434
    Reshapes input to target shape.
435
    
436
    Args:
437
        target_shape (tuple): Target shape (excluding batch dimension)
438
    """
439
    def __init__(self, target_shape, **kwargs): ...
440

441
class Permute:
442
    """
443
    Permutes dimensions of input.
444
    
445
    Args:
446
        dims (tuple): Permutation pattern
447
    """
448
    def __init__(self, dims, **kwargs): ...
449

450
class RepeatVector:
451
    """
452
    Repeats input n times.
453
    
454
    Args:
455
        n (int): Number of repetitions
456
    """
457
    def __init__(self, n, **kwargs): ...
458

459
class ZeroPadding2D:
460
    """
461
    Zero-padding for 2D data.
462
    
463
    Args:
464
        padding (int or tuple): Padding specification
465
        data_format (str): Data format
466
    """
467
    def __init__(self, padding=(1, 1), data_format=None, **kwargs): ...
468

469
class Cropping2D:
470
    """
471
    Cropping for 2D data.
472
    
473
    Args:
474
        cropping (int or tuple): Cropping specification
475
        data_format (str): Data format
476
    """
477
    def __init__(self, cropping=((0, 0), (0, 0)), data_format=None, **kwargs): ...
478
```
479

480
### Merge Layers
481

482
Layers for combining multiple input tensors through various operations.
483

484
```python { .api }
485
class Add:
486
    """Element-wise addition of inputs."""
487
    def __init__(self, **kwargs): ...
488

489
class Subtract:
490
    """Element-wise subtraction of inputs."""
491
    def __init__(self, **kwargs): ...
492

493
class Multiply:
494
    """Element-wise multiplication of inputs."""
495
    def __init__(self, **kwargs): ...
496

497
class Average:
498
    """Element-wise average of inputs."""
499
    def __init__(self, **kwargs): ...
500

501
class Maximum:
502
    """Element-wise maximum of inputs."""
503
    def __init__(self, **kwargs): ...
504

505
class Minimum:
506
    """Element-wise minimum of inputs."""
507
    def __init__(self, **kwargs): ...
508

509
class Concatenate:
510
    """
511
    Concatenates inputs along specified axis.
512
    
513
    Args:
514
        axis (int): Concatenation axis
515
    """
516
    def __init__(self, axis=-1, **kwargs): ...
517

518
class Dot:
519
    """
520
    Computes dot product between inputs.
521
    
522
    Args:
523
        axes (int or tuple): Axes to compute dot product over
524
        normalize (bool): Whether to normalize inputs
525
    """
526
    def __init__(self, axes, normalize=False, **kwargs): ...
527

528
# Functional equivalents
529
def add(inputs): ...
530
def subtract(inputs): ...  
531
def multiply(inputs): ...
532
def concatenate(inputs, axis=-1): ...
533
```
534

535
## Usage Examples
536

537
### Building Convolutional Neural Network
538

539
```python
540
import keras
541
from keras import layers
542

543
model = keras.Sequential([
544
    layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),
545
    layers.BatchNormalization(),
546
    layers.MaxPooling2D((2, 2)),
547
    
548
    layers.Conv2D(64, (3, 3), activation='relu'),
549
    layers.BatchNormalization(),
550
    layers.MaxPooling2D((2, 2)),
551
    
552
    layers.Conv2D(64, (3, 3), activation='relu'),
553
    layers.GlobalAveragePooling2D(),
554
    layers.Dropout(0.5),
555
    
556
    layers.Dense(10, activation='softmax')
557
])
558
```
559

560
### Building Recurrent Neural Network
561

562
```python
563
import keras
564
from keras import layers
565

566
model = keras.Sequential([
567
    layers.Embedding(vocab_size, 128, input_length=max_length),
568
    layers.LSTM(64, return_sequences=True, dropout=0.2),
569
    layers.LSTM(32, dropout=0.2),
570
    layers.Dense(64, activation='relu'),
571
    layers.Dropout(0.5),
572
    layers.Dense(num_classes, activation='softmax')
573
])
574
```
575

576
### Building Attention-based Model
577

578
```python
579
import keras
580
from keras import layers
581

582
# Input
583
inputs = keras.Input(shape=(seq_length, embedding_dim))
584

585
# Multi-head attention
586
attention_output = layers.MultiHeadAttention(
587
    num_heads=8, 
588
    key_dim=64
589
)(inputs, inputs)
590

591
# Add & Norm
592
attention_output = layers.Add()([inputs, attention_output])
593
attention_output = layers.LayerNormalization()(attention_output)
594

595
# Feed Forward
596
ffn_output = layers.Dense(128, activation='relu')(attention_output)
597
ffn_output = layers.Dense(embedding_dim)(ffn_output)
598

599
# Add & Norm
600
outputs = layers.Add()([attention_output, ffn_output])
601
outputs = layers.LayerNormalization()(outputs)
602

603
# Classification head
604
outputs = layers.GlobalAveragePooling1D()(outputs)
605
outputs = layers.Dense(num_classes, activation='softmax')(outputs)
606

607
model = keras.Model(inputs=inputs, outputs=outputs)
608
```