0
# Framework Integrations
1
2
Integration modules for popular Python frameworks enabling seamless progress tracking within existing workflows. These integrations provide framework-specific callbacks and adapters while maintaining tqdm's consistent interface.
3
4
## Capabilities
5
6
### Keras Integration
7
8
Training callback for monitoring Keras model training progress with epoch and batch-level progress tracking.
9
10
```python { .api }
11
from tqdm.keras import TqdmCallback
12
13
class TqdmCallback(keras.callbacks.Callback):
14
"""
15
Keras callback for training progress with tqdm progress bars.
16
17
Provides epoch-level and optional batch-level progress tracking
18
with support for validation metrics and custom formatting.
19
"""
20
def __init__(self, epochs=None, data_size=None, batch_size=None,
21
verbose=1, tqdm_class=tqdm_auto, **tqdm_kwargs):
22
"""
23
Initialize Keras callback with progress tracking.
24
25
Parameters:
26
- epochs: Total number of training epochs (auto-detected if None)
27
- data_size: Size of training dataset (auto-detected if None)
28
- batch_size: Training batch size (auto-detected if None)
29
- verbose: Verbosity level (0=silent, 1=progress bar, 2=one line per epoch)
30
- tqdm_class: tqdm class to use (default: tqdm.auto.tqdm)
31
- **tqdm_kwargs: Additional arguments passed to tqdm constructor
32
"""
33
34
@staticmethod
35
def bar2callback(bar, pop=None, delta=lambda logs: 1):
36
"""
37
Convert tqdm progress bar to Keras callback function.
38
39
Parameters:
40
- bar: tqdm progress bar instance
41
- pop: List of keys to remove from logs before display
42
- delta: Function to calculate progress increment from logs
43
44
Returns:
45
Callback function compatible with Keras training
46
"""
47
```
48
49
### Dask Integration
50
51
Computation callback for tracking Dask task execution progress across distributed computing environments.
52
53
```python { .api }
54
from tqdm.dask import TqdmCallback
55
56
class TqdmCallback(Callback):
57
"""
58
Dask computation callback with tqdm progress tracking.
59
60
Monitors task execution across Dask workers and provides
61
real-time progress updates for complex computation graphs.
62
"""
63
def __init__(self, start=None, pretask=None, tqdm_class=tqdm_auto, **tqdm_kwargs):
64
"""
65
Initialize Dask callback with progress tracking.
66
67
Parameters:
68
- start: Callback for computation start (optional)
69
- pretask: Callback before each task (optional)
70
- tqdm_class: tqdm class to use (default: tqdm.auto.tqdm)
71
- **tqdm_kwargs: Additional arguments passed to tqdm constructor
72
"""
73
74
def display(self):
75
"""Display progress in notebook environments"""
76
```
77
78
### Pandas Integration
79
80
Class method integration enabling progress tracking for pandas DataFrame operations through monkey-patching.
81
82
```python { .api }
83
# Enable pandas integration
84
tqdm.pandas(**tqdm_kwargs)
85
86
# This enables methods like:
87
df.progress_apply(func) # Apply with progress
88
df.progress_map(func) # Map with progress
89
df.progress_applymap(func) # Element-wise apply with progress
90
df.groupby(col).progress_apply(func) # GroupBy operations with progress
91
```
92
93
## Usage Examples
94
95
### Keras Model Training
96
97
```python
98
from tqdm.keras import TqdmCallback
99
import tensorflow as tf
100
from tensorflow import keras
101
import numpy as np
102
103
# Create sample data
104
X_train = np.random.random((1000, 32))
105
y_train = np.random.randint(2, size=(1000, 1))
106
X_val = np.random.random((200, 32))
107
y_val = np.random.randint(2, size=(200, 1))
108
109
# Build model
110
model = keras.Sequential([
111
keras.layers.Dense(64, activation='relu', input_shape=(32,)),
112
keras.layers.Dense(32, activation='relu'),
113
keras.layers.Dense(1, activation='sigmoid')
114
])
115
116
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
117
118
# Train with tqdm callback
119
tqdm_callback = TqdmCallback(
120
epochs=10,
121
verbose=1,
122
desc="Training",
123
leave=True
124
)
125
126
history = model.fit(
127
X_train, y_train,
128
epochs=10,
129
batch_size=32,
130
validation_data=(X_val, y_val),
131
callbacks=[tqdm_callback],
132
verbose=0 # Disable default Keras progress
133
)
134
```
135
136
### Advanced Keras Integration
137
138
```python
139
from tqdm.keras import TqdmCallback
140
from tqdm.auto import tqdm
141
import tensorflow as tf
142
143
# Custom callback with detailed progress
144
class DetailedTqdmCallback(TqdmCallback):
145
def __init__(self, **kwargs):
146
super().__init__(**kwargs)
147
self.epoch_bar = None
148
self.batch_bar = None
149
150
def on_train_begin(self, logs=None):
151
super().on_train_begin(logs)
152
self.epoch_bar = tqdm(total=self.epochs, desc="Epochs", position=0)
153
154
def on_epoch_begin(self, epoch, logs=None):
155
super().on_epoch_begin(epoch, logs)
156
if self.data_size:
157
self.batch_bar = tqdm(
158
total=self.data_size // self.batch_size,
159
desc=f"Epoch {epoch+1}/{self.epochs}",
160
position=1,
161
leave=False
162
)
163
164
def on_batch_end(self, batch, logs=None):
165
if self.batch_bar:
166
self.batch_bar.update(1)
167
# Update with current metrics
168
if logs:
169
self.batch_bar.set_postfix({
170
'loss': f"{logs.get('loss', 0):.4f}",
171
'acc': f"{logs.get('accuracy', 0):.4f}"
172
})
173
174
def on_epoch_end(self, epoch, logs=None):
175
if self.batch_bar:
176
self.batch_bar.close()
177
if self.epoch_bar:
178
self.epoch_bar.update(1)
179
if logs:
180
self.epoch_bar.set_postfix({
181
'val_loss': f"{logs.get('val_loss', 0):.4f}",
182
'val_acc': f"{logs.get('val_accuracy', 0):.4f}"
183
})
184
185
# Use detailed callback
186
detailed_callback = DetailedTqdmCallback()
187
model.fit(X_train, y_train, epochs=5, callbacks=[detailed_callback])
188
```
189
190
### Dask Distributed Computing
191
192
```python
193
from tqdm.dask import TqdmCallback
194
import dask
195
import dask.array as da
196
from dask.distributed import Client
197
import numpy as np
198
199
# Setup Dask client
200
client = Client('localhost:8786') # Connect to Dask scheduler
201
202
# Create large array computation
203
x = da.random.random((10000, 10000), chunks=(1000, 1000))
204
y = da.random.random((10000, 10000), chunks=(1000, 1000))
205
206
# Matrix operations with progress tracking
207
with TqdmCallback(desc="Matrix Multiplication"):
208
result = da.dot(x, y)
209
computed_result = result.compute()
210
211
# Complex computation graph with progress
212
def process_chunk(chunk):
213
return np.sum(chunk ** 2)
214
215
with TqdmCallback(desc="Processing Chunks", leave=True):
216
chunks = [da.from_array(np.random.random((1000, 1000))) for _ in range(100)]
217
processed = [da.apply_gufunc(process_chunk, chunk, signature='(i,j)->()')
218
for chunk in chunks]
219
results = dask.compute(*processed)
220
221
client.close()
222
```
223
224
### Pandas DataFrame Operations
225
226
```python
227
import pandas as pd
228
import numpy as np
229
from tqdm import tqdm
230
231
# Enable pandas integration
232
tqdm.pandas(desc="Processing")
233
234
# Create sample DataFrame
235
df = pd.DataFrame({
236
'A': np.random.randn(100000),
237
'B': np.random.randn(100000),
238
'C': np.random.choice(['X', 'Y', 'Z'], 100000)
239
})
240
241
# Apply operations with progress bars
242
def complex_function(x):
243
# Simulate complex computation
244
return x ** 2 + np.sin(x) + np.log(abs(x) + 1)
245
246
# DataFrame operations with progress
247
result1 = df['A'].progress_apply(complex_function)
248
result2 = df.progress_apply(lambda row: row['A'] + row['B'], axis=1)
249
250
# GroupBy operations with progress
251
grouped_result = df.groupby('C').progress_apply(
252
lambda group: group['A'].mean() + group['B'].std()
253
)
254
255
# Element-wise operations with progress
256
df_processed = df[['A', 'B']].progress_applymap(lambda x: x * 2 if x > 0 else x / 2)
257
258
# Custom pandas integration with detailed progress
259
tqdm.pandas(desc="Custom Processing", unit="rows", leave=True)
260
261
def detailed_processing(row):
262
# Simulate complex row processing
263
result = row['A'] * row['B']
264
if row['C'] == 'X':
265
result *= 2
266
elif row['C'] == 'Y':
267
result += 10
268
return result
269
270
df['processed'] = df.progress_apply(detailed_processing, axis=1)
271
```
272
273
### Custom Integration Patterns
274
275
```python
276
import tensorflow as tf
277
from tqdm.auto import tqdm
278
import time
279
280
# Custom TensorFlow integration
281
class TqdmTensorFlowCallback(tf.keras.callbacks.Callback):
282
def __init__(self, **tqdm_kwargs):
283
super().__init__()
284
self.tqdm_kwargs = tqdm_kwargs
285
self.pbar = None
286
287
def on_train_begin(self, logs=None):
288
self.pbar = tqdm(total=self.params['epochs'], **self.tqdm_kwargs)
289
290
def on_epoch_end(self, epoch, logs=None):
291
if self.pbar:
292
self.pbar.update(1)
293
if logs:
294
# Update with metrics
295
postfix = {k: f"{v:.4f}" for k, v in logs.items()
296
if k in ['loss', 'accuracy', 'val_loss', 'val_accuracy']}
297
self.pbar.set_postfix(postfix)
298
299
def on_train_end(self, logs=None):
300
if self.pbar:
301
self.pbar.close()
302
303
# Custom PyTorch integration
304
class TqdmPyTorchTrainer:
305
def __init__(self, model, optimizer, criterion):
306
self.model = model
307
self.optimizer = optimizer
308
self.criterion = criterion
309
310
def train_epoch(self, dataloader, epoch):
311
self.model.train()
312
pbar = tqdm(dataloader, desc=f"Epoch {epoch}")
313
314
total_loss = 0
315
for batch_idx, (data, target) in enumerate(pbar):
316
self.optimizer.zero_grad()
317
output = self.model(data)
318
loss = self.criterion(output, target)
319
loss.backward()
320
self.optimizer.step()
321
322
total_loss += loss.item()
323
avg_loss = total_loss / (batch_idx + 1)
324
325
# Update progress bar with current metrics
326
pbar.set_postfix({
327
'loss': f"{loss.item():.4f}",
328
'avg_loss': f"{avg_loss:.4f}"
329
})
330
331
return avg_loss
332
333
# Generic framework integration helper
334
def create_framework_callback(framework_callback_class, tqdm_class=tqdm):
335
"""Factory function for creating framework-specific tqdm callbacks"""
336
337
class TqdmFrameworkCallback(framework_callback_class):
338
def __init__(self, total=None, desc=None, **tqdm_kwargs):
339
super().__init__()
340
self.total = total
341
self.desc = desc
342
self.tqdm_kwargs = tqdm_kwargs
343
self.pbar = None
344
345
def start_progress(self, total=None):
346
if self.pbar is None:
347
self.pbar = tqdm_class(
348
total=total or self.total,
349
desc=self.desc,
350
**self.tqdm_kwargs
351
)
352
353
def update_progress(self, n=1, **postfix):
354
if self.pbar:
355
self.pbar.update(n)
356
if postfix:
357
self.pbar.set_postfix(postfix)
358
359
def finish_progress(self):
360
if self.pbar:
361
self.pbar.close()
362
self.pbar = None
363
364
return TqdmFrameworkCallback
365
```
366
367
## Framework-Specific Considerations
368
369
### Threading and Multiprocessing
370
- Keras: Use `verbose=0` to disable default progress and avoid conflicts
371
- Dask: Progress tracking works across distributed workers
372
- Pandas: Thread-safe by default when using `tqdm.pandas()`
373
374
### Memory Usage
375
- Large datasets: Consider chunking or streaming approaches
376
- Deep learning: Monitor GPU memory alongside training progress
377
- Distributed computing: Progress aggregation across workers
378
379
### Error Handling
380
- Framework exceptions are propagated through tqdm callbacks
381
- Progress bars are automatically cleaned up on errors
382
- Use context managers for guaranteed cleanup
383
384
### Performance Considerations
385
- Minimal overhead: ~60ns per update
386
- Batch updates recommended for high-frequency operations
387
- Disable progress bars in production if performance critical