0
# Training and Evaluation
1
2
Core training functions and cross-validation capabilities for XGBoost model development. These functions provide the primary interface for training XGBoost models with extensive configuration options, evaluation metrics, and early stopping capabilities.
3
4
## Capabilities
5
6
### Model Training
7
8
Primary training function that builds XGBoost models using gradient boosting with support for custom objectives, evaluation metrics, and advanced training configurations.
9
10
```python { .api }
11
def train(params, dtrain, num_boost_round=10, evals=(), obj=None,
12
maximize=None, early_stopping_rounds=None, evals_result=None,
13
verbose_eval=True, xgb_model=None, callbacks=None, custom_metric=None):
14
"""
15
Train a booster with given parameters.
16
17
Parameters:
18
- params: Training parameters as dictionary (dict)
19
Common parameters:
20
- 'objective': Learning task ('reg:squarederror', 'binary:logistic', 'multi:softmax', etc.)
21
- 'eval_metric': Evaluation metric ('rmse', 'logloss', 'mlogloss', 'auc', etc.)
22
- 'max_depth': Maximum tree depth (int, default=6)
23
- 'learning_rate': Boosting learning rate (float, default=0.3)
24
- 'subsample': Fraction of observations to subsample (float, default=1)
25
- 'colsample_bytree': Fraction of features to subsample per tree (float, default=1)
26
- 'gamma': Minimum loss reduction required for split (float, default=0)
27
- 'min_child_weight': Minimum sum of instance weight in child (float, default=1)
28
- 'reg_alpha': L1 regularization term (float, default=0)
29
- 'reg_lambda': L2 regularization term (float, default=1)
30
- 'scale_pos_weight': Balancing weight for positive class (float, default=1)
31
- 'tree_method': Tree construction algorithm ('auto', 'exact', 'approx', 'hist')
32
- 'device': Device to use ('cpu', 'cuda', 'gpu')
33
- 'random_state': Random seed (int)
34
- dtrain: Training DMatrix (DMatrix)
35
- num_boost_round: Number of boosting iterations (int)
36
- evals: List of pairs (DMatrix, string) for evaluation during training (list)
37
- obj: Customized objective function (callable, optional)
38
Signature: obj(y_pred, y_true) -> (grad, hess)
39
- maximize: Whether to maximize evaluation metric (bool, optional)
40
- early_stopping_rounds: Stop training if evaluation doesn't improve (int, optional)
41
- evals_result: Dictionary to store evaluation results (dict, optional)
42
- verbose_eval: Whether to display evaluation results (bool or int)
43
- xgb_model: Existing model to continue training (Booster, optional)
44
- callbacks: List of callback functions (list, optional)
45
- custom_metric: Custom evaluation metric function (callable, optional)
46
Signature: custom_metric(y_pred, y_true) -> (eval_name, eval_result, is_higher_better)
47
48
Returns: Booster - Trained XGBoost model
49
"""
50
```
51
52
### Cross-Validation
53
54
Robust cross-validation functionality for model validation, hyperparameter tuning, and performance estimation with support for stratified sampling and custom evaluation metrics.
55
56
```python { .api }
57
def cv(params, dtrain, num_boost_round=10, nfold=3, stratified=False,
58
folds=None, metrics=(), obj=None, maximize=None,
59
early_stopping_rounds=None, fpreproc=None, as_pandas=True,
60
verbose_eval=None, show_stdv=True, seed=0, callbacks=None,
61
shuffle=True, custom_metric=None):
62
"""
63
Cross-validation with given parameters.
64
65
Parameters:
66
- params: Training parameters (same as train() function) (dict)
67
- dtrain: Training DMatrix (DMatrix)
68
- num_boost_round: Number of boosting iterations (int)
69
- nfold: Number of CV folds (int)
70
- stratified: Whether to use stratified sampling (bool)
71
- folds: Custom CV folds as sklearn splits (iterator, optional)
72
- metrics: Additional evaluation metrics (tuple of str)
73
- obj: Custom objective function (callable, optional)
74
- maximize: Whether to maximize evaluation metric (bool, optional)
75
- early_stopping_rounds: Stop if no improvement for N rounds (int, optional)
76
- fpreproc: Preprocessing function for each fold (callable, optional)
77
Signature: fpreproc(dtrain, dtest, params) -> (dtrain, dtest, params)
78
- as_pandas: Return results as pandas DataFrame (bool)
79
- verbose_eval: Control evaluation result display (bool or int, optional)
80
- show_stdv: Whether to display standard deviation (bool)
81
- seed: Random seed for fold assignment (int)
82
- callbacks: List of callback functions (list, optional)
83
- shuffle: Whether to shuffle data before splitting (bool)
84
- custom_metric: Custom evaluation metric (callable, optional)
85
86
Returns: dict or pandas.DataFrame - Cross-validation results
87
If as_pandas=True: DataFrame with metrics for each fold and iteration
88
If as_pandas=False: Dict with metric names as keys and lists of scores as values
89
"""
90
```
91
92
## Usage Examples
93
94
### Basic Model Training
95
96
```python
97
import xgboost as xgb
98
import numpy as np
99
from sklearn.datasets import make_classification
100
from sklearn.model_selection import train_test_split
101
102
# Create sample data
103
X, y = make_classification(n_samples=1000, n_features=20, n_classes=2,
104
n_informative=15, random_state=42)
105
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
106
random_state=42)
107
108
# Create DMatrix objects
109
dtrain = xgb.DMatrix(X_train, label=y_train)
110
dtest = xgb.DMatrix(X_test, label=y_test)
111
112
# Set training parameters
113
params = {
114
'objective': 'binary:logistic',
115
'eval_metric': 'logloss',
116
'max_depth': 6,
117
'learning_rate': 0.1,
118
'subsample': 0.8,
119
'colsample_bytree': 0.8,
120
'gamma': 0.1,
121
'min_child_weight': 1,
122
'reg_alpha': 0.1,
123
'reg_lambda': 1.0,
124
'random_state': 42
125
}
126
127
# Train model with evaluation
128
model = xgb.train(
129
params=params,
130
dtrain=dtrain,
131
num_boost_round=100,
132
evals=[(dtrain, 'train'), (dtest, 'test')],
133
early_stopping_rounds=10,
134
verbose_eval=10
135
)
136
137
print(f"Best iteration: {model.best_iteration}")
138
print(f"Best score: {model.best_score}")
139
140
# Make predictions
141
y_pred = model.predict(dtest)
142
y_pred_binary = (y_pred > 0.5).astype(int)
143
```
144
145
### Advanced Training with Custom Objective
146
147
```python
148
import numpy as np
149
150
def custom_objective(y_pred, y_true):
151
"""Custom focal loss objective for imbalanced classification."""
152
alpha = 0.25
153
gamma = 2.0
154
155
# Convert DMatrix labels to numpy array
156
y_true = y_true.get_label()
157
158
# Sigmoid activation
159
sigmoid = 1 / (1 + np.exp(-y_pred))
160
161
# Focal loss gradients and hessians
162
grad = alpha * (y_true - sigmoid) * ((1 - sigmoid) ** gamma) * gamma * np.log(sigmoid + 1e-8) + \
163
alpha * (y_true - sigmoid) * ((1 - sigmoid) ** (gamma - 1))
164
165
hess = alpha * gamma * ((1 - sigmoid) ** (gamma - 1)) * \
166
(gamma * (y_true - sigmoid) * np.log(sigmoid + 1e-8) + (y_true - sigmoid) + \
167
(1 - sigmoid))
168
169
return grad, hess
170
171
def custom_eval_metric(y_pred, y_true):
172
"""Custom F1 score evaluation metric."""
173
from sklearn.metrics import f1_score
174
175
y_true = y_true.get_label()
176
y_pred_binary = (y_pred > 0.5).astype(int)
177
f1 = f1_score(y_true, y_pred_binary)
178
179
return 'f1', f1, True # Name, value, higher_is_better
180
181
# Train with custom objective and metric
182
model_custom = xgb.train(
183
params={'max_depth': 6, 'learning_rate': 0.1},
184
dtrain=dtrain,
185
num_boost_round=100,
186
evals=[(dtrain, 'train'), (dtest, 'test')],
187
obj=custom_objective,
188
custom_metric=custom_eval_metric,
189
early_stopping_rounds=10,
190
verbose_eval=10
191
)
192
```
193
194
### Cross-Validation for Model Selection
195
196
```python
197
# Basic cross-validation
198
cv_params = {
199
'objective': 'binary:logistic',
200
'eval_metric': 'auc',
201
'max_depth': 6,
202
'learning_rate': 0.1,
203
'subsample': 0.8,
204
'colsample_bytree': 0.8,
205
'random_state': 42
206
}
207
208
cv_results = xgb.cv(
209
params=cv_params,
210
dtrain=dtrain,
211
num_boost_round=200,
212
nfold=5,
213
stratified=True,
214
early_stopping_rounds=10,
215
seed=42,
216
verbose_eval=10,
217
show_stdv=True
218
)
219
220
print(f"Best CV score: {cv_results.iloc[-1]['test-auc-mean']:.4f} ± {cv_results.iloc[-1]['test-auc-std']:.4f}")
221
print(f"Best iteration: {len(cv_results)}")
222
223
# Cross-validation with hyperparameter grid search
224
from sklearn.model_selection import ParameterGrid
225
226
param_grid = {
227
'max_depth': [3, 6, 9],
228
'learning_rate': [0.01, 0.1, 0.2],
229
'subsample': [0.8, 0.9, 1.0],
230
'colsample_bytree': [0.8, 0.9, 1.0]
231
}
232
233
best_score = 0
234
best_params = None
235
best_num_rounds = 0
236
237
for params in ParameterGrid(param_grid):
238
cv_params = {
239
'objective': 'binary:logistic',
240
'eval_metric': 'auc',
241
'random_state': 42,
242
**params
243
}
244
245
cv_results = xgb.cv(
246
params=cv_params,
247
dtrain=dtrain,
248
num_boost_round=100,
249
nfold=5,
250
stratified=True,
251
early_stopping_rounds=10,
252
seed=42,
253
verbose_eval=False,
254
show_stdv=False
255
)
256
257
score = cv_results.iloc[-1]['test-auc-mean']
258
if score > best_score:
259
best_score = score
260
best_params = params
261
best_num_rounds = len(cv_results)
262
263
print(f"Best parameters: {best_params}")
264
print(f"Best CV score: {best_score:.4f}")
265
print(f"Best number of rounds: {best_num_rounds}")
266
267
# Train final model with best parameters
268
final_params = {
269
'objective': 'binary:logistic',
270
'eval_metric': 'auc',
271
'random_state': 42,
272
**best_params
273
}
274
275
final_model = xgb.train(
276
params=final_params,
277
dtrain=dtrain,
278
num_boost_round=best_num_rounds,
279
evals=[(dtest, 'test')],
280
verbose_eval=False
281
)
282
```
283
284
### Multi-Class Classification Training
285
286
```python
287
from sklearn.datasets import make_classification
288
289
# Create multi-class data
290
X_multi, y_multi = make_classification(n_samples=1000, n_features=20,
291
n_classes=5, n_informative=15,
292
random_state=42)
293
X_train_multi, X_test_multi, y_train_multi, y_test_multi = \
294
train_test_split(X_multi, y_multi, test_size=0.2, random_state=42)
295
296
# Create DMatrix for multi-class
297
dtrain_multi = xgb.DMatrix(X_train_multi, label=y_train_multi)
298
dtest_multi = xgb.DMatrix(X_test_multi, label=y_test_multi)
299
300
# Multi-class parameters
301
multi_params = {
302
'objective': 'multi:softmax',
303
'eval_metric': 'mlogloss',
304
'num_class': 5, # Number of classes
305
'max_depth': 6,
306
'learning_rate': 0.1,
307
'random_state': 42
308
}
309
310
# Train multi-class model
311
multi_model = xgb.train(
312
params=multi_params,
313
dtrain=dtrain_multi,
314
num_boost_round=100,
315
evals=[(dtrain_multi, 'train'), (dtest_multi, 'test')],
316
early_stopping_rounds=10,
317
verbose_eval=10
318
)
319
320
# Get class probabilities instead of class predictions
321
multi_params_prob = multi_params.copy()
322
multi_params_prob['objective'] = 'multi:softprob'
323
324
multi_model_prob = xgb.train(
325
params=multi_params_prob,
326
dtrain=dtrain_multi,
327
num_boost_round=100,
328
early_stopping_rounds=10,
329
verbose_eval=False
330
)
331
332
# Predictions
333
y_pred_classes = multi_model.predict(dtest_multi)
334
y_pred_probs = multi_model_prob.predict(dtest_multi).reshape(-1, 5)
335
336
print(f"Predicted classes shape: {y_pred_classes.shape}")
337
print(f"Predicted probabilities shape: {y_pred_probs.shape}")
338
```
339
340
### Regression Training
341
342
```python
343
from sklearn.datasets import make_regression
344
from sklearn.metrics import mean_squared_error, r2_score
345
346
# Create regression data
347
X_reg, y_reg = make_regression(n_samples=1000, n_features=20, noise=0.1,
348
random_state=42)
349
X_train_reg, X_test_reg, y_train_reg, y_test_reg = \
350
train_test_split(X_reg, y_reg, test_size=0.2, random_state=42)
351
352
# Create DMatrix for regression
353
dtrain_reg = xgb.DMatrix(X_train_reg, label=y_train_reg)
354
dtest_reg = xgb.DMatrix(X_test_reg, label=y_test_reg)
355
356
# Regression parameters
357
reg_params = {
358
'objective': 'reg:squarederror',
359
'eval_metric': 'rmse',
360
'max_depth': 6,
361
'learning_rate': 0.1,
362
'subsample': 0.8,
363
'colsample_bytree': 0.8,
364
'random_state': 42
365
}
366
367
# Cross-validation for regression
368
reg_cv_results = xgb.cv(
369
params=reg_params,
370
dtrain=dtrain_reg,
371
num_boost_round=100,
372
nfold=5,
373
early_stopping_rounds=10,
374
seed=42,
375
verbose_eval=10
376
)
377
378
# Train final regression model
379
reg_model = xgb.train(
380
params=reg_params,
381
dtrain=dtrain_reg,
382
num_boost_round=len(reg_cv_results),
383
evals=[(dtrain_reg, 'train'), (dtest_reg, 'test')],
384
verbose_eval=False
385
)
386
387
# Evaluate regression performance
388
y_pred_reg = reg_model.predict(dtest_reg)
389
rmse = mean_squared_error(y_test_reg, y_pred_reg, squared=False)
390
r2 = r2_score(y_test_reg, y_pred_reg)
391
392
print(f"RMSE: {rmse:.4f}")
393
print(f"R²: {r2:.4f}")
394
```
395
396
### Training with Callbacks
397
398
```python
399
import xgboost as xgb
400
401
# Define custom callback
402
class CustomCallback:
403
def __init__(self):
404
self.history = {'train': [], 'test': []}
405
406
def after_iteration(self, model, epoch, evals_log):
407
"""Called after each training iteration."""
408
if evals_log:
409
for data_name, eval_results in evals_log.items():
410
for metric_name, metric_values in eval_results.items():
411
self.history[data_name].append(metric_values[-1])
412
413
# Custom logic, e.g., adaptive learning rate
414
if epoch > 50 and epoch % 10 == 0:
415
current_lr = model.get_xgb_params().get('learning_rate', 0.1)
416
new_lr = current_lr * 0.95
417
model.set_param('learning_rate', new_lr)
418
print(f"Epoch {epoch}: Reduced learning rate to {new_lr:.4f}")
419
420
return False # Continue training
421
422
# Train with callbacks
423
callback = CustomCallback()
424
evals_result = {}
425
426
model_with_callback = xgb.train(
427
params=params,
428
dtrain=dtrain,
429
num_boost_round=100,
430
evals=[(dtrain, 'train'), (dtest, 'test')],
431
evals_result=evals_result,
432
callbacks=[callback],
433
verbose_eval=10
434
)
435
436
print(f"Training history length: {len(callback.history['train'])}")
437
print(f"Final evaluation results keys: {list(evals_result.keys())}")
438
```
439
440
### Early Stopping and Model Selection
441
442
```python
443
# Training with detailed early stopping
444
evals_result_detailed = {}
445
446
model_early_stop = xgb.train(
447
params={
448
'objective': 'binary:logistic',
449
'eval_metric': ['logloss', 'auc'], # Multiple metrics
450
'max_depth': 6,
451
'learning_rate': 0.1,
452
'random_state': 42
453
},
454
dtrain=dtrain,
455
num_boost_round=500, # Large number, will stop early
456
evals=[(dtrain, 'train'), (dtest, 'validation')],
457
evals_result=evals_result_detailed,
458
early_stopping_rounds=20, # Stop if no improvement for 20 rounds
459
maximize=False, # Minimize logloss (first metric)
460
verbose_eval=25
461
)
462
463
print(f"Training stopped at iteration: {model_early_stop.best_iteration + 1}")
464
print(f"Best score: {model_early_stop.best_score:.4f}")
465
466
# Access detailed evaluation history
467
train_logloss = evals_result_detailed['train']['logloss']
468
val_logloss = evals_result_detailed['validation']['logloss']
469
train_auc = evals_result_detailed['train']['auc']
470
val_auc = evals_result_detailed['validation']['auc']
471
472
print(f"Final train logloss: {train_logloss[-1]:.4f}")
473
print(f"Final validation logloss: {val_logloss[-1]:.4f}")
474
print(f"Final train AUC: {train_auc[-1]:.4f}")
475
print(f"Final validation AUC: {val_auc[-1]:.4f}")
476
477
# Plot training curves
478
import matplotlib.pyplot as plt
479
480
plt.figure(figsize=(12, 5))
481
482
plt.subplot(1, 2, 1)
483
plt.plot(train_logloss, label='Train')
484
plt.plot(val_logloss, label='Validation')
485
plt.xlabel('Iteration')
486
plt.ylabel('Log Loss')
487
plt.title('Training Curves - Log Loss')
488
plt.legend()
489
490
plt.subplot(1, 2, 2)
491
plt.plot(train_auc, label='Train')
492
plt.plot(val_auc, label='Validation')
493
plt.xlabel('Iteration')
494
plt.ylabel('AUC')
495
plt.title('Training Curves - AUC')
496
plt.legend()
497
498
plt.tight_layout()
499
plt.show()
500
```
501
502
### Training Callbacks
503
504
Comprehensive callback system for customizing training behavior with built-in callbacks for early stopping, learning rate scheduling, evaluation monitoring, and checkpointing. Callbacks enable advanced training control and monitoring.
505
506
```python { .api }
507
import xgboost.callback as callback
508
509
class callback.TrainingCallback:
510
"""Abstract base class for training callbacks."""
511
512
def before_training(self, model):
513
"""Called before training starts. Parameters: model (Booster)"""
514
515
def after_training(self, model):
516
"""Called after training finishes. Parameters: model (Booster)"""
517
518
def before_iteration(self, model, epoch, evals_log):
519
"""Called before each iteration. Parameters: model (Booster), epoch (int), evals_log (dict)"""
520
521
def after_iteration(self, model, epoch, evals_log):
522
"""
523
Called after each iteration.
524
525
Parameters:
526
- model: Current model state (Booster)
527
- epoch: Current iteration number (int)
528
- evals_log: Evaluation history (dict)
529
530
Returns: bool - True to stop training, False to continue
531
"""
532
533
class callback.EarlyStopping:
534
def __init__(self, rounds, metric_name=None, data_name=None,
535
maximize=None, save_best=False, min_delta=0.0):
536
"""
537
Early stopping with best model saving support.
538
539
Parameters:
540
- rounds: Number of rounds with no improvement to stop (int)
541
- metric_name: Metric name for early stopping (str, optional)
542
- data_name: Dataset name for early stopping (str, optional)
543
- maximize: Whether to maximize metric (bool, optional, auto-detected)
544
- save_best: Save best model vs last model (bool, tree methods only)
545
- min_delta: Minimum change to qualify as improvement (float)
546
"""
547
548
class callback.LearningRateScheduler:
549
def __init__(self, learning_rates):
550
"""
551
Dynamic learning rate adjustment during training.
552
553
Parameters:
554
- learning_rates: Callable accepting epoch and returning learning rate,
555
or sequence of learning rates (callable or sequence)
556
"""
557
558
class callback.EvaluationMonitor:
559
def __init__(self, rank=0, period=1, show_stdv=False, logger=None):
560
"""
561
Print evaluation results at specified intervals.
562
563
Parameters:
564
- rank: Which worker prints results (int, default: 0)
565
- period: Epochs between printing (int, default: 1)
566
- show_stdv: Show standard deviation in CV (bool, default: False)
567
- logger: Custom logging function (callable, optional)
568
"""
569
570
class callback.TrainingCheckPoint:
571
def __init__(self, directory, name='model', as_pickle=False, interval=100):
572
"""
573
Periodic model checkpointing during training.
574
575
Parameters:
576
- directory: Output directory for checkpoints (str)
577
- name: Checkpoint file name pattern (str, default: 'model')
578
- as_pickle: Save as pickle vs model format (bool, default: False)
579
- interval: Checkpointing interval in rounds (int, default: 100)
580
"""
581
```
582
583
## Callback Usage Examples
584
585
### Custom Callback Development
586
587
```python
588
import xgboost as xgb
589
from xgboost.callback import TrainingCallback
590
591
class CustomCallback(TrainingCallback):
592
"""Custom callback for advanced training control."""
593
594
def __init__(self, log_file='training.log'):
595
self.log_file = log_file
596
self.iteration_times = []
597
self.best_score = float('inf')
598
self.plateau_count = 0
599
600
def before_training(self, model):
601
"""Initialize callback state before training."""
602
import time
603
self.start_time = time.time()
604
with open(self.log_file, 'w') as f:
605
f.write("Training started\n")
606
607
def after_iteration(self, model, epoch, evals_log):
608
"""Custom logic after each iteration."""
609
import time
610
current_time = time.time()
611
612
# Track iteration timing
613
if hasattr(self, 'iter_start'):
614
iter_time = current_time - self.iter_start
615
self.iteration_times.append(iter_time)
616
617
self.iter_start = current_time
618
619
# Check for validation score improvement
620
if evals_log:
621
for data_name, metrics in evals_log.items():
622
if 'logloss' in metrics:
623
current_score = metrics['logloss'][-1]
624
if current_score < self.best_score:
625
self.best_score = current_score
626
self.plateau_count = 0
627
else:
628
self.plateau_count += 1
629
630
# Dynamic learning rate adjustment based on plateau
631
if self.plateau_count > 5:
632
current_lr = model.get_xgb_params().get('learning_rate', 0.1)
633
new_lr = current_lr * 0.9
634
model.set_param('learning_rate', new_lr)
635
print(f"Epoch {epoch}: Reduced learning rate to {new_lr:.6f}")
636
self.plateau_count = 0
637
638
# Log progress
639
with open(self.log_file, 'a') as f:
640
f.write(f"Epoch {epoch}: Best score = {self.best_score:.6f}\n")
641
642
return False # Continue training
643
644
def after_training(self, model):
645
"""Cleanup and final logging."""
646
total_time = time.time() - self.start_time
647
avg_iter_time = sum(self.iteration_times) / len(self.iteration_times)
648
649
with open(self.log_file, 'a') as f:
650
f.write(f"Training completed in {total_time:.2f}s\n")
651
f.write(f"Average iteration time: {avg_iter_time:.4f}s\n")
652
f.write(f"Final best score: {self.best_score:.6f}\n")
653
654
# Use custom callback
655
custom_cb = CustomCallback('xgb_training.log')
656
model = xgb.train(params, dtrain, num_boost_round=100,
657
callbacks=[custom_cb])
658
```
659
660
### Combining Multiple Callbacks
661
662
```python
663
import xgboost as xgb
664
from xgboost import callback
665
import numpy as np
666
667
# Create multiple callbacks
668
early_stop = callback.EarlyStopping(rounds=10, save_best=True, min_delta=0.001)
669
checkpointer = callback.TrainingCheckPoint(directory='./checkpoints',
670
interval=25, as_pickle=False)
671
monitor = callback.EvaluationMonitor(period=5, show_stdv=True)
672
673
# Custom learning rate schedule
674
def cosine_schedule(epoch):
675
"""Cosine annealing learning rate schedule."""
676
max_epochs = 200
677
initial_lr = 0.1
678
min_lr = 0.001
679
680
if epoch >= max_epochs:
681
return min_lr
682
683
return min_lr + (initial_lr - min_lr) * \
684
(1 + np.cos(np.pi * epoch / max_epochs)) / 2
685
686
lr_scheduler = callback.LearningRateScheduler(cosine_schedule)
687
688
# Combine all callbacks
689
callbacks = [early_stop, checkpointer, monitor, lr_scheduler]
690
691
# Train with all callbacks
692
model = xgb.train(
693
params={'objective': 'binary:logistic', 'eval_metric': 'logloss'},
694
dtrain=dtrain,
695
num_boost_round=200,
696
evals=[(dtrain, 'train'), (dtest, 'validation')],
697
callbacks=callbacks,
698
verbose_eval=False # Monitor callback handles output
699
)
700
701
print(f"Training stopped at iteration: {model.best_iteration}")
702
print(f"Best validation score: {model.best_score}")
703
```
704
705
### Callback Integration with Scikit-learn Interface
706
707
```python
708
from xgboost import XGBClassifier, callback
709
from sklearn.model_selection import GridSearchCV
710
711
# Create callbacks for sklearn interface
712
early_stop_cb = callback.EarlyStopping(rounds=10, save_best=True)
713
checkpoint_cb = callback.TrainingCheckPoint('./sklearn_checkpoints', interval=50)
714
715
# Use with sklearn estimator
716
xgb_clf = XGBClassifier(
717
n_estimators=500,
718
max_depth=6,
719
learning_rate=0.1,
720
callbacks=[early_stop_cb, checkpoint_cb],
721
eval_metric='logloss'
722
)
723
724
# Callbacks work with hyperparameter tuning
725
param_grid = {
726
'max_depth': [3, 6, 9],
727
'learning_rate': [0.01, 0.1, 0.2]
728
}
729
730
# Note: Each grid search iteration re-initializes callbacks
731
grid_search = GridSearchCV(
732
xgb_clf, param_grid,
733
cv=5, scoring='roc_auc',
734
fit_params={'eval_set': [(X_test, y_test)]}
735
)
736
737
grid_search.fit(X_train, y_train)
738
```
739
740
### Advanced Callback Patterns
741
742
```python
743
class AdaptiveRegularizationCallback(TrainingCallback):
744
"""Dynamically adjust regularization based on overfitting detection."""
745
746
def __init__(self, patience=10, reg_step=0.1):
747
self.patience = patience
748
self.reg_step = reg_step
749
self.train_scores = []
750
self.val_scores = []
751
self.best_diff = float('inf')
752
self.wait_count = 0
753
754
def after_iteration(self, model, epoch, evals_log):
755
if evals_log and 'train' in evals_log and 'val' in evals_log:
756
# Assume both have the same metric
757
metric_name = list(evals_log['train'].keys())[0]
758
train_score = evals_log['train'][metric_name][-1]
759
val_score = evals_log['val'][metric_name][-1]
760
761
# Calculate overfitting gap
762
score_diff = abs(val_score - train_score)
763
764
if score_diff < self.best_diff:
765
self.best_diff = score_diff
766
self.wait_count = 0
767
else:
768
self.wait_count += 1
769
770
# Increase regularization if overfitting detected
771
if self.wait_count >= self.patience:
772
current_alpha = model.get_xgb_params().get('reg_alpha', 0)
773
current_lambda = model.get_xgb_params().get('reg_lambda', 0)
774
775
new_alpha = current_alpha + self.reg_step
776
new_lambda = current_lambda + self.reg_step
777
778
model.set_param('reg_alpha', new_alpha)
779
model.set_param('reg_lambda', new_lambda)
780
781
print(f"Epoch {epoch}: Increased regularization - "
782
f"alpha={new_alpha:.3f}, lambda={new_lambda:.3f}")
783
784
self.wait_count = 0
785
786
return False
787
788
# Multi-stage training with callback switching
789
class MultiStageCallback(TrainingCallback):
790
"""Switch between different training strategies."""
791
792
def __init__(self, stage_epochs=[50, 100]):
793
self.stage_epochs = stage_epochs
794
self.current_stage = 0
795
796
def after_iteration(self, model, epoch, evals_log):
797
# Switch training strategy at specified epochs
798
for i, stage_epoch in enumerate(self.stage_epochs):
799
if epoch == stage_epoch and self.current_stage == i:
800
self.current_stage += 1
801
802
if i == 0: # First stage: fast learning
803
model.set_param('learning_rate', 0.01) # Slower learning
804
model.set_param('max_depth', 8) # Deeper trees
805
print(f"Stage {i+1}: Switched to fine-tuning mode")
806
807
elif i == 1: # Second stage: regularization
808
model.set_param('reg_alpha', 0.1)
809
model.set_param('reg_lambda', 0.1)
810
print(f"Stage {i+2}: Added regularization")
811
812
return False
813
814
# Usage with multiple advanced callbacks
815
adaptive_reg_cb = AdaptiveRegularizationCallback(patience=15)
816
multi_stage_cb = MultiStageCallback(stage_epochs=[75, 150])
817
early_stop_cb = callback.EarlyStopping(rounds=25, save_best=True)
818
819
advanced_model = xgb.train(
820
params={'objective': 'reg:squarederror', 'eval_metric': 'rmse'},
821
dtrain=dtrain,
822
num_boost_round=200,
823
evals=[(dtrain, 'train'), (dtest, 'val')],
824
callbacks=[multi_stage_cb, adaptive_reg_cb, early_stop_cb]
825
)
826
```