0
# Model Cards
1

2
Automatic model card generation and metadata management for reproducibility and documentation. Model cards provide comprehensive documentation of model performance, training details, and usage guidelines.
3

4
## Capabilities
5

6
### SetFit Model Card Data
7

8
Dataclass for storing and managing model card metadata throughout the training and deployment lifecycle.
9

10
```python { .api }
11
class SetFitModelCardData:
12
    def __init__(
13
        self,
14
        language: Optional[Union[str, List[str]]] = None,
15
        license: Optional[str] = None,
16
        tags: Optional[List[str]] = None,
17
        model_name: Optional[str] = None,
18
        model_id: Optional[str] = None,
19
        dataset_name: Optional[str] = None,
20
        dataset_id: Optional[str] = None,
21
        dataset_revision: Optional[str] = None,
22
        task_name: Optional[str] = None,
23
        st_id: Optional[str] = None
24
    ):
25
        """
26
        Initialize model card data for SetFit models.
27

28
        Parameters:
29
        - language: Language(s) supported by the model
30
        - license: Model license (e.g., "MIT", "Apache-2.0")
31
        - tags: List of tags for model categorization
32
        - model_name: Human-readable model name
33
        - model_id: Unique model identifier (HuggingFace Hub ID)
34
        - dataset_name: Name of training dataset
35
        - dataset_id: Dataset identifier (HuggingFace Hub ID)
36
        - dataset_revision: Specific dataset revision/commit
37
        - task_name: ML task type (e.g., "text-classification")
38
        - st_id: Sentence Transformer model ID used as base
39
        """
40

41
    def set_best_model_step(self, step: int) -> None:
42
        """
43
        Record the training step where best performance was achieved.
44

45
        Parameters:
46
        - step: Training step number with best validation metrics
47
        """
48

49
    def set_widget_examples(self, examples: List[Dict[str, str]]) -> None:
50
        """
51
        Set example inputs for HuggingFace Hub model widget.
52

53
        Parameters:
54
        - examples: List of example inputs with expected outputs
55
        """
56

57
    def set_train_set_metrics(self, metrics: Dict[str, float]) -> None:
58
        """
59
        Record training set performance metrics.
60

61
        Parameters:
62
        - metrics: Dictionary of metric names and values
63
        """
64

65
    def set_label_examples(self, examples: Dict[str, List[str]]) -> None:
66
        """
67
        Set example texts for each label class.
68

69
        Parameters:
70
        - examples: Mapping from label names to example texts
71
        """
72

73
    def infer_dataset_id(self, dataset_name: str) -> Optional[str]:
74
        """
75
        Infer HuggingFace dataset ID from dataset name.
76

77
        Parameters:
78
        - dataset_name: Name of the dataset
79

80
        Returns:
81
        Inferred dataset ID or None if not found
82
        """
83

84
    def register_model(self, model_id: str, exists_ok: bool = False) -> None:
85
        """
86
        Register model with HuggingFace Hub.
87

88
        Parameters:
89
        - model_id: Model ID for registration
90
        - exists_ok: Whether to allow overwriting existing model
91
        """
92

93
    def infer_st_id(self, st_model: SentenceTransformer) -> str:
94
        """
95
        Infer sentence transformer model ID from model object.
96

97
        Parameters:
98
        - st_model: Sentence transformer model instance
99

100
        Returns:
101
        Model ID string
102
        """
103

104
    def set_st_id(self, st_id: str) -> None:
105
        """
106
        Set the sentence transformer model ID.
107

108
        Parameters:
109
        - st_id: Sentence transformer model identifier
110
        """
111

112
    def post_training_eval_results(self, results: Dict[str, Any]) -> None:
113
        """
114
        Record post-training evaluation results.
115

116
        Parameters:
117
        - results: Evaluation results including metrics and metadata
118
        """
119

120
    def to_dict(self) -> Dict[str, Any]:
121
        """
122
        Convert model card data to dictionary format.
123

124
        Returns:
125
        Dictionary representation of model card data
126
        """
127

128
    def to_yaml(self) -> str:
129
        """
130
        Convert model card data to YAML format for HuggingFace Hub.
131

132
        Returns:
133
        YAML string representation
134
        """
135
```
136

137
### Model Card Callback
138

139
Training callback that automatically tracks metrics and generates model cards during training.
140

141
```python { .api }
142
class ModelCardCallback:
143
    def __init__(self, trainer: "Trainer"):
144
        """
145
        Initialize model card callback for automatic tracking.
146

147
        Parameters:
148
        - trainer: SetFit trainer instance to monitor
149
        """
150

151
    def on_init_end(
152
        self,
153
        args: TrainingArguments,
154
        state: "TrainerState",
155
        control: "TrainerControl"
156
    ) -> None:
157
        """
158
        Called when trainer initialization ends.
159

160
        Parameters:
161
        - args: Training arguments
162
        - state: Current trainer state
163
        - control: Training control flags
164
        """
165

166
    def on_train_begin(
167
        self,
168
        args: TrainingArguments,
169
        state: "TrainerState", 
170
        control: "TrainerControl"
171
    ) -> None:
172
        """
173
        Called when training begins.
174

175
        Parameters:
176
        - args: Training arguments
177
        - state: Current trainer state
178
        - control: Training control flags
179
        """
180

181
    def on_evaluate(
182
        self,
183
        args: TrainingArguments,
184
        state: "TrainerState",
185
        control: "TrainerControl",
186
        model: SetFitModel,
187
        logs: Optional[Dict[str, float]] = None
188
    ) -> None:
189
        """
190
        Called after each evaluation.
191

192
        Parameters:
193
        - args: Training arguments
194
        - state: Current trainer state
195
        - control: Training control flags
196
        - model: Model being trained
197
        - logs: Evaluation metrics
198
        """
199

200
    def on_log(
201
        self,
202
        args: TrainingArguments,
203
        state: "TrainerState",
204
        control: "TrainerControl",
205
        model: SetFitModel,
206
        logs: Optional[Dict[str, float]] = None
207
    ) -> None:
208
        """
209
        Called when metrics are logged.
210

211
        Parameters:
212
        - args: Training arguments
213
        - state: Current trainer state
214
        - control: Training control flags
215
        - model: Model being trained
216
        - logs: Logged metrics
217
        """
218
```
219

220
### Model Card Generation Functions
221

222
Utility functions for generating model cards from trained models.
223

224
```python { .api }
225
def generate_model_card(model: SetFitModel) -> str:
226
    """
227
    Generate comprehensive model card for a SetFit model.
228

229
    Parameters:
230
    - model: Trained SetFit model with model card data
231

232
    Returns:
233
    Complete model card as markdown string
234
    """
235

236
def is_on_huggingface(repo_id: str, is_model: bool = True) -> bool:
237
    """
238
    Check if a repository exists on HuggingFace Hub.
239

240
    Parameters:
241
    - repo_id: Repository identifier to check
242
    - is_model: Whether to check model hub (True) or dataset hub (False)
243

244
    Returns:
245
    True if repository exists, False otherwise
246
    """
247
```
248

249
## Usage Examples
250

251
### Basic Model Card Creation
252

253
```python
254
from setfit import SetFitModel, SetFitTrainer, TrainingArguments, SetFitModelCardData
255
from datasets import load_dataset
256

257
# Create model card data
258
model_card_data = SetFitModelCardData(
259
    language="en",
260
    license="apache-2.0",
261
    tags=["setfit", "sentence-transformers", "text-classification", "few-shot"],
262
    model_name="SetFit Sentiment Classifier",
263
    dataset_name="emotion",
264
    task_name="text-classification"
265
)
266

267
# Initialize model with model card data
268
model = SetFitModel.from_pretrained(
269
    "sentence-transformers/all-MiniLM-L6-v2",
270
    model_card_data=model_card_data
271
)
272

273
# Load dataset
274
train_dataset = load_dataset("emotion", split="train[:100]")
275
eval_dataset = load_dataset("emotion", split="validation[:50]")
276

277
# Set up training with model card tracking
278
args = TrainingArguments(
279
    output_dir="./results",
280
    num_epochs=4,
281
    batch_size=16,
282
    eval_strategy="epoch",
283
    save_strategy="epoch",
284
    logging_steps=10,
285
    load_best_model_at_end=True,
286
    metric_for_best_model="eval_accuracy"
287
)
288

289
trainer = SetFitTrainer(
290
    model=model,
291
    args=args,
292
    train_dataset=train_dataset,
293
    eval_dataset=eval_dataset,
294
    column_mapping={"text": "text", "label": "label"}
295
)
296

297
# Train model (model card will be updated automatically)
298
trainer.train()
299

300
# Generate and display model card
301
model_card = model.generate_model_card()
302
print(model_card)
303

304
# Save model with model card
305
model.save_pretrained("./my-setfit-model")
306
```
307

308
### Comprehensive Model Card with Custom Metadata
309

310
```python
311
from setfit import SetFitModel, SetFitModelCardData, generate_model_card
312
import json
313

314
# Create detailed model card data
315
model_card_data = SetFitModelCardData(
316
    language=["en", "es", "fr"],  # Multi-language support
317
    license="mit",
318
    tags=[
319
        "setfit", 
320
        "sentence-transformers", 
321
        "text-classification",
322
        "few-shot-learning",
323
        "multilingual",
324
        "sentiment-analysis"
325
    ],
326
    model_name="Multilingual SetFit Sentiment Classifier",
327
    model_id="my-org/multilingual-setfit-sentiment",
328
    dataset_name="multilingual_sentiment",
329
    dataset_id="my-org/multilingual-sentiment-dataset",
330
    task_name="text-classification"
331
)
332

333
# Add widget examples for HuggingFace Hub
334
widget_examples = [
335
    {"text": "I love this product!"},
336
    {"text": "This is terrible quality."},
337
    {"text": "¡Me encanta este producto!"},  # Spanish
338
    {"text": "J'adore ce produit!"}        # French
339
]
340
model_card_data.set_widget_examples(widget_examples)
341

342
# Add label examples
343
label_examples = {
344
    "positive": [
345
        "This is amazing!",
346
        "I absolutely love it!",
347
        "Best purchase ever!"
348
    ],
349
    "negative": [
350
        "This is awful.",
351
        "Worst product I've bought.",
352
        "Complete waste of money."
353
    ],
354
    "neutral": [
355
        "It's okay.",
356
        "Average quality product.",
357
        "Nothing special about it."
358
    ]
359
}
360
model_card_data.set_label_examples(label_examples)
361

362
# Initialize model
363
model = SetFitModel.from_pretrained(
364
    "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
365
    model_card_data=model_card_data
366
)
367

368
# Simulate training metrics
369
training_metrics = {
370
    "final_train_accuracy": 0.94,
371
    "final_train_f1": 0.93,
372
    "final_eval_accuracy": 0.89,
373
    "final_eval_f1": 0.88,
374
    "training_time": "00:15:32",
375
    "total_steps": 200,
376
    "best_step": 180
377
}
378

379
# Update model card with training results
380
model_card_data.set_train_set_metrics(training_metrics)
381
model_card_data.set_best_model_step(180)
382

383
# Add post-training evaluation results
384
eval_results = {
385
    "test_accuracy": 0.87,
386
    "test_f1": 0.86,
387
    "test_precision": 0.88,
388
    "test_recall": 0.85,
389
    "per_class_metrics": {
390
        "positive": {"precision": 0.91, "recall": 0.89, "f1": 0.90},
391
        "negative": {"precision": 0.88, "recall": 0.87, "f1": 0.88},
392
        "neutral": {"precision": 0.85, "recall": 0.81, "f1": 0.83}
393
    },
394
    "confusion_matrix": [[45, 3, 2], [4, 41, 5], [7, 6, 37]]
395
}
396

397
model_card_data.post_training_eval_results(eval_results)
398

399
# Generate comprehensive model card
400
model_card = generate_model_card(model)
401
print("Generated Model Card:")
402
print("=" * 50)
403
print(model_card)
404

405
# Save model card to file
406
with open("MODEL_CARD.md", "w") as f:
407
    f.write(model_card)
408

409
print("\nModel card saved to MODEL_CARD.md")
410
```
411

412
### Automatic Model Card Generation with Callback
413

414
```python
415
from setfit import SetFitTrainer, ModelCardCallback, SetFitModelCardData
416
from datasets import load_dataset
417

418
# Set up model card data
419
model_card_data = SetFitModelCardData(
420
    language="en",
421
    license="apache-2.0",
422
    tags=["setfit", "text-classification", "emotion-detection"],
423
    model_name="SetFit Emotion Classifier",
424
    dataset_name="emotion",
425
    task_name="multi-class-classification"
426
)
427

428
# Initialize model
429
model = SetFitModel.from_pretrained(
430
    "sentence-transformers/all-MiniLM-L6-v2",
431
    model_card_data=model_card_data
432
)
433

434
# Load dataset
435
train_dataset = load_dataset("emotion", split="train[:200]")
436
eval_dataset = load_dataset("emotion", split="validation[:100]")
437

438
# Create trainer with model card callback
439
trainer = SetFitTrainer(
440
    model=model,
441
    train_dataset=train_dataset,
442
    eval_dataset=eval_dataset,
443
    args=TrainingArguments(
444
        output_dir="./emotion-classifier",
445
        num_epochs=3,
446
        batch_size=16,
447
        eval_strategy="epoch",
448
        logging_steps=20,
449
        save_strategy="epoch"
450
    ),
451
    callbacks=[ModelCardCallback]  # Add model card callback
452
)
453

454
# Train model - callback will automatically update model card
455
print("Training model with automatic model card generation...")
456
trainer.train()
457

458
# Model card is automatically generated and updated during training
459
final_model_card = model.generate_model_card()
460

461
print("Final Model Card:")
462
print(final_model_card)
463

464
# Save model with updated model card
465
model.save_pretrained("./emotion-classifier-final")
466
```
467

468
### Model Card for HuggingFace Hub Upload
469

470
```python
471
from setfit import SetFitModel, SetFitModelCardData
472
from huggingface_hub import HfApi
473
import os
474

475
# Create production-ready model card
476
model_card_data = SetFitModelCardData(
477
    language="en",
478
    license="apache-2.0",
479
    tags=[
480
        "setfit",
481
        "sentence-transformers", 
482
        "text-classification",
483
        "few-shot-learning",
484
        "pytorch",
485
        "scikit-learn"
486
    ],
487
    model_name="SetFit Binary Sentiment Classifier",
488
    model_id="my-username/setfit-sentiment-binary",
489
    dataset_name="imdb",
490
    dataset_id="imdb",
491
    task_name="binary-classification"
492
)
493

494
# Load trained model
495
model = SetFitModel.from_pretrained("./my-trained-model")
496
model.model_card_data = model_card_data
497

498
# Add comprehensive evaluation metrics
499
evaluation_metrics = {
500
    "accuracy": 0.922,
501
    "precision": 0.918,
502
    "recall": 0.925,
503
    "f1": 0.921,
504
    "roc_auc": 0.965,
505
    "samples_seen": 25000,
506
    "training_samples": 16,  # Few-shot learning
507
    "eval_samples": 25000
508
}
509

510
model_card_data.set_train_set_metrics(evaluation_metrics)
511

512
# Set widget examples for interactive testing
513
widget_examples = [
514
    {"text": "This movie is absolutely fantastic! I loved every minute of it."},
515
    {"text": "Boring and predictable. Waste of time."},
516
    {"text": "Not bad, but could be better."},
517
    {"text": "Outstanding performance by the lead actor!"}
518
]
519
model_card_data.set_widget_examples(widget_examples)
520

521
# Generate model card
522
model_card_content = generate_model_card(model)
523

524
# Save model locally with model card
525
save_path = "./setfit-sentiment-for-hub"
526
model.save_pretrained(save_path)
527

528
# Write model card to README.md
529
with open(f"{save_path}/README.md", "w") as f:
530
    f.write(model_card_content)
531

532
print(f"Model and model card saved to {save_path}")
533

534
# Upload to HuggingFace Hub (requires authentication)
535
# api = HfApi()
536
# api.upload_folder(
537
#     folder_path=save_path,
538
#     repo_id="my-username/setfit-sentiment-binary",
539
#     repo_type="model"
540
# )
541
# print("Model uploaded to HuggingFace Hub!")
542
```
543

544
### Custom Model Card Template
545

546
```python
547
from setfit import SetFitModelCardData
548
from jinja2 import Template
549

550
# Custom model card template
551
CUSTOM_TEMPLATE = """
552
# {{ model_name }}
553

554
{{ description }}
555

556
## Model Details
557

558
- **Model Type**: SetFit (Sentence Transformers + Classification Head)
559
- **Language**: {{ language }}
560
- **License**: {{ license }}
561
- **Base Model**: {{ st_id }}
562

563
## Training Data
564

565
- **Dataset**: {{ dataset_name }}
566
- **Training Samples**: {{ training_samples }}
567
- **Validation Samples**: {{ validation_samples }}
568

569
## Performance Metrics
570

571
| Metric | Score |
572
|--------|-------|
573
{% for metric, score in metrics.items() %}
574
| {{ metric.title() }} | {{ "%.3f"|format(score) }} |
575
{% endfor %}
576

577
## Usage
578

579
```python
580
from setfit import SetFitModel
581

582
# Load model
583
model = SetFitModel.from_pretrained("{{ model_id }}")
584

585
# Make predictions
586
predictions = model.predict([
587
    "Your text here"
588
])
589
```
590

591
## Training Configuration
592

593
- **Batch Size**: {{ batch_size }}  
594
- **Learning Rate**: {{ learning_rate }}
595
- **Epochs**: {{ num_epochs }}
596
- **Optimizer**: {{ optimizer }}
597

598
## Limitations and Bias
599

600
{{ limitations }}
601

602
## Citation
603

604
If you use this model, please cite:
605

606
```bibtex
607
@article{setfit2022,
608
  title={Efficient Few-Shot Learning Without Prompts},
609
  author={Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren},
610
  journal={arXiv preprint arXiv:2209.11055},
611
  year={2022}
612
}
613
```
614
"""
615

616
class CustomModelCardGenerator:
617
    def __init__(self, template_string=CUSTOM_TEMPLATE):
618
        self.template = Template(template_string)
619
    
620
    def generate(self, model_card_data, **kwargs):
621
        """Generate model card from template and data."""
622
        
623
        # Prepare template variables
624
        template_vars = {
625
            "model_name": model_card_data.model_name,
626
            "description": "Few-shot text classification model trained with SetFit framework",
627
            "language": model_card_data.language,
628
            "license": model_card_data.license,
629
            "st_id": model_card_data.st_id,
630
            "dataset_name": model_card_data.dataset_name,
631
            "model_id": model_card_data.model_id,
632
            "training_samples": kwargs.get("training_samples", "Unknown"),
633
            "validation_samples": kwargs.get("validation_samples", "Unknown"),
634
            "metrics": kwargs.get("metrics", {}),
635
            "batch_size": kwargs.get("batch_size", "16"),
636
            "learning_rate": kwargs.get("learning_rate", "2e-5"),
637
            "num_epochs": kwargs.get("num_epochs", "1"),
638
            "optimizer": kwargs.get("optimizer", "AdamW"),
639
            "limitations": kwargs.get("limitations", "This model may have biases present in the training data.")
640
        }
641
        
642
        return self.template.render(**template_vars)
643

644
# Usage example
645
model_card_data = SetFitModelCardData(
646
    model_name="Custom SetFit Classifier",
647
    language="en",
648
    license="mit",
649
    st_id="sentence-transformers/all-MiniLM-L6-v2",
650
    dataset_name="custom_dataset",
651
    model_id="username/custom-setfit-model"
652
)
653

654
generator = CustomModelCardGenerator()
655
custom_card = generator.generate(
656
    model_card_data,
657
    training_samples=500,
658
    validation_samples=100,
659
    metrics={
660
        "accuracy": 0.89,
661
        "f1": 0.87,
662
        "precision": 0.91,
663
        "recall": 0.84
664
    },
665
    batch_size=32,
666
    learning_rate="5e-5",
667
    num_epochs=3,
668
    limitations="Model trained on domain-specific data. May not generalize well to other domains."
669
)
670

671
print("Custom Model Card:")
672
print(custom_card)
673
```
674
675
### Model Card Validation and Quality Checks
676

677
```python
678
from setfit import SetFitModelCardData, is_on_huggingface
679
import re
680

681
def validate_model_card_data(model_card_data: SetFitModelCardData) -> Dict[str, List[str]]:
682
    """
683
    Validate model card data for completeness and quality.
684
    
685
    Returns:
686
    Dictionary with validation results and recommendations
687
    """
688
    errors = []
689
    warnings = []
690
    suggestions = []
691
    
692
    # Required fields check
693
    required_fields = ['model_name', 'language', 'license', 'task_name']
694
    for field in required_fields:
695
        if not getattr(model_card_data, field):
696
            errors.append(f"Missing required field: {field}")
697
    
698
    # License validation
699
    valid_licenses = ['apache-2.0', 'mit', 'cc-by-4.0', 'cc-by-sa-4.0', 'gpl-3.0']
700
    if model_card_data.license and model_card_data.license.lower() not in valid_licenses:
701
        warnings.append(f"Unusual license: {model_card_data.license}")
702
    
703
    # Model ID validation
704
    if model_card_data.model_id:
705
        if not re.match(r'^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$', model_card_data.model_id):
706
            errors.append("Model ID should follow format: username/model-name")
707
        
708
        # Check if model exists on HuggingFace Hub
709
        if is_on_huggingface(model_card_data.model_id):
710
            warnings.append(f"Model ID {model_card_data.model_id} already exists on HuggingFace Hub")
711
    
712
    # Dataset validation
713
    if model_card_data.dataset_id:
714
        if not is_on_huggingface(model_card_data.dataset_id, is_model=False):
715
            warnings.append(f"Dataset {model_card_data.dataset_id} not found on HuggingFace Hub")
716
    
717
    # Tag validation
718
    if model_card_data.tags:
719
        recommended_tags = ['setfit', 'sentence-transformers', 'text-classification']
720
        missing_recommended = [tag for tag in recommended_tags if tag not in model_card_data.tags]
721
        if missing_recommended:
722
            suggestions.append(f"Consider adding recommended tags: {missing_recommended}")
723
    else:
724
        suggestions.append("Add tags to improve model discoverability")
725
    
726
    # Language validation
727
    if model_card_data.language:
728
        if isinstance(model_card_data.language, str):
729
            if len(model_card_data.language) > 3:
730
                warnings.append("Language should use ISO 639-1 codes (e.g., 'en', 'fr')")
731
    
732
    return {
733
        "errors": errors,
734
        "warnings": warnings,
735
        "suggestions": suggestions,
736
        "is_valid": len(errors) == 0
737
    }
738

739
# Example validation
740
model_card_data = SetFitModelCardData(
741
    model_name="Test Model",
742
    language="english",  # Should be "en"
743
    license="custom-license",  # Not standard
744
    model_id="invalid-id-format",  # Invalid format
745
    tags=["classification"],  # Missing recommended tags
746
    dataset_id="nonexistent/dataset"
747
)
748

749
validation_results = validate_model_card_data(model_card_data)
750

751
print("Model Card Validation Results:")
752
print(f"Valid: {validation_results['is_valid']}")
753

754
if validation_results['errors']:
755
    print("\nErrors (must fix):")
756
    for error in validation_results['errors']:
757
        print(f"  ❌ {error}")
758

759
if validation_results['warnings']:
760
    print("\nWarnings (should review):")
761
    for warning in validation_results['warnings']:
762
        print(f"  ⚠️  {warning}")
763

764
if validation_results['suggestions']:
765
    print("\nSuggestions (recommended):")
766
    for suggestion in validation_results['suggestions']:
767
        print(f"  💡 {suggestion}")
768
```