0
# Utilities
1

2
ModelScope provides a comprehensive set of utility functions and classes for configuration management, logging, task constants, and various helper functions used throughout the ecosystem.
3

4
## Capabilities
5

6
### Task Constants
7

8
Comprehensive task type constants organized by domain.
9

10
```python { .api }
11
class Tasks:
12
    """
13
    Comprehensive task type constants for different AI domains.
14
    """
15
    
16
    # Computer Vision Tasks
17
    class CVTasks:
18
        IMAGE_CLASSIFICATION = 'image-classification'
19
        OBJECT_DETECTION = 'object-detection'
20
        SEMANTIC_SEGMENTATION = 'semantic-segmentation'
21
        INSTANCE_SEGMENTATION = 'instance-segmentation'
22
        FACE_DETECTION = 'face-detection'
23
        FACE_RECOGNITION = 'face-recognition'
24
        IMAGE_GENERATION = 'image-generation'
25
        IMAGE_INPAINTING = 'image-inpainting'
26
        IMAGE_DENOISING = 'image-denoising'
27
        IMAGE_SUPER_RESOLUTION = 'image-super-resolution'
28
        # Additional CV task constants available
29
    
30
    # Natural Language Processing Tasks
31
    class NLPTasks:
32
        TEXT_CLASSIFICATION = 'text-classification'
33
        TOKEN_CLASSIFICATION = 'token-classification'
34
        TEXT_GENERATION = 'text-generation'
35
        TRANSLATION = 'translation'
36
        SUMMARIZATION = 'summarization'
37
        QUESTION_ANSWERING = 'question-answering'
38
        SENTIMENT_ANALYSIS = 'sentiment-analysis'
39
        NAMED_ENTITY_RECOGNITION = 'named-entity-recognition'
40
        TEXT_RANKING = 'text-ranking'
41
        FILL_MASK = 'fill-mask'
42
        # Additional NLP task constants available
43
    
44
    # Audio Processing Tasks
45
    class AudioTasks:
46
        AUTOMATIC_SPEECH_RECOGNITION = 'automatic-speech-recognition'
47
        TEXT_TO_SPEECH = 'text-to-speech'
48
        AUDIO_CLASSIFICATION = 'audio-classification'
49
        SPEAKER_RECOGNITION = 'speaker-recognition'
50
        AUDIO_SEPARATION = 'audio-separation'
51
        # Additional audio task constants available
52
    
53
    # Multi-Modal Tasks
54
    class MultiModalTasks:
55
        IMAGE_CAPTIONING = 'image-captioning'
56
        VISUAL_QUESTION_ANSWERING = 'visual-question-answering'
57
        TEXT_TO_IMAGE_SYNTHESIS = 'text-to-image-synthesis'
58
        IMAGE_TEXT_MATCHING = 'image-text-matching'
59
        # Additional multi-modal task constants available
60
    
61
    # Scientific Computing Tasks
62
    class ScienceTasks:
63
        PROTEIN_STRUCTURE_PREDICTION = 'protein-structure-prediction'
64
        MOLECULAR_PROPERTY_PREDICTION = 'molecular-property-prediction'
65
        # Additional science task constants available
66
```
67

68
### Logging Utilities
69

70
Comprehensive logging configuration and management.
71

72
```python { .api }
73
def get_logger(
74
    log_file: str = None,
75
    log_level: int = INFO,
76
    file_mode: str = 'w',
77
    **kwargs
78
) -> Logger:
79
    """
80
    Create and configure logger for ModelScope applications.
81
    
82
    Parameters:
83
    - log_file: Path to log file (optional, logs to console if None)
84
    - log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
85
    - file_mode: File mode for log file ('w' for write, 'a' for append)
86
    - **kwargs: Additional logging configuration options
87
    
88
    Returns:
89
    Configured logger instance
90
    """
91
```
92

93
### Configuration Management
94

95
Utilities for reading and managing configuration files.
96

97
```python { .api }
98
def read_config(path: str, **kwargs) -> dict:
99
    """
100
    Read configuration from file (JSON, YAML, or Python).
101
    
102
    Parameters:
103
    - path: Path to configuration file
104
    - **kwargs: Additional configuration loading options
105
    
106
    Returns:
107
    Configuration dictionary
108
    """
109
```
110

111
### Hub Utilities
112

113
Utility functions for interacting with ModelScope Hub.
114

115
```python { .api }
116
def create_model_if_not_exist(
117
    model_name: str,
118
    model_type: str = 'model',
119
    **kwargs
120
):
121
    """
122
    Create model repository on ModelScope Hub if it doesn't exist.
123
    
124
    Parameters:
125
    - model_name: Name of the model repository
126
    - model_type: Type of repository ('model', 'dataset')
127
    - **kwargs: Additional repository creation options
128
    """
129
```
130

131
## Usage Examples
132

133
### Using Task Constants
134

135
```python
136
from modelscope import Tasks, pipeline
137

138
# Use task constants for pipeline creation
139
classifier = pipeline(Tasks.CVTasks.IMAGE_CLASSIFICATION, model='model_name')
140
detector = pipeline(Tasks.CVTasks.OBJECT_DETECTION, model='detection_model')
141

142
# NLP tasks
143
text_classifier = pipeline(Tasks.NLPTasks.TEXT_CLASSIFICATION, model='nlp_model')
144
generator = pipeline(Tasks.NLPTasks.TEXT_GENERATION, model='generation_model')
145

146
# Audio tasks
147
asr = pipeline(Tasks.AudioTasks.AUTOMATIC_SPEECH_RECOGNITION, model='speech_model')
148
tts = pipeline(Tasks.AudioTasks.TEXT_TO_SPEECH, model='tts_model')
149

150
# Multi-modal tasks
151
captioner = pipeline(Tasks.MultiModalTasks.IMAGE_CAPTIONING, model='caption_model')
152
vqa = pipeline(Tasks.MultiModalTasks.VISUAL_QUESTION_ANSWERING, model='vqa_model')
153

154
# List all available CV tasks
155
cv_tasks = [attr for attr in dir(Tasks.CVTasks) if not attr.startswith('_')]
156
print(f"Available CV tasks: {cv_tasks}")
157
```
158

159
### Logger Configuration
160

161
```python
162
from modelscope import get_logger
163
import logging
164

165
# Basic logger
166
logger = get_logger()
167
logger.info("Basic logging message")
168

169
# Logger with file output
170
file_logger = get_logger(
171
    log_file='modelscope.log',
172
    log_level=logging.DEBUG,
173
    file_mode='a'
174
)
175

176
file_logger.debug("Debug message")
177
file_logger.info("Info message")
178
file_logger.warning("Warning message")
179
file_logger.error("Error message")
180

181
# Custom logger configuration
182
custom_logger = get_logger(
183
    log_file='custom.log',
184
    log_level=logging.INFO,
185
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
186
    datefmt='%Y-%m-%d %H:%M:%S'
187
)
188

189
custom_logger.info("Custom formatted log message")
190
```
191

192
### Configuration File Management
193

194
```python
195
from modelscope import read_config
196

197
# Read JSON configuration
198
json_config = read_config('config.json')
199
print(f"JSON config: {json_config}")
200

201
# Read YAML configuration
202
yaml_config = read_config('config.yaml')
203
print(f"YAML config: {yaml_config}")
204

205
# Read Python configuration file
206
py_config = read_config('config.py')
207
print(f"Python config: {py_config}")
208

209
# Use configuration in model training
210
training_config = read_config('training_config.json')
211

212
from modelscope import EpochBasedTrainer, TrainingArgs
213

214
training_args = TrainingArgs(
215
    output_dir=training_config['output_dir'],
216
    max_epochs=training_config['max_epochs'],
217
    learning_rate=training_config['learning_rate'],
218
    train_batch_size=training_config['batch_size']
219
)
220
```
221

222
### Hub Utilities Usage
223

224
```python
225
from modelscope import create_model_if_not_exist, HubApi
226

227
# Create model repository if it doesn't exist
228
create_model_if_not_exist(
229
    model_name='my-custom-model',
230
    model_type='model',
231
    visibility=1,  # Public repository
232
    license='Apache-2.0'
233
)
234

235
# Use with Hub API
236
api = HubApi()
237
api.login('your_token')
238

239
# Check if model exists before creating
240
try:
241
    model_info = api.get_model_info('my-custom-model')
242
    print(f"Model exists: {model_info['name']}")
243
except:
244
    # Model doesn't exist, create it
245
    create_model_if_not_exist('my-custom-model')
246
    print("Model repository created")
247
```
248

249
### Device and Hardware Utilities
250

251
```python
252
from modelscope.utils.device import get_device, is_gpu_available
253

254
# Get optimal device
255
device = get_device()
256
print(f"Selected device: {device}")
257

258
# Check GPU availability
259
if is_gpu_available():
260
    print("GPU is available for acceleration")
261
    device = 'cuda'
262
else:
263
    print("Using CPU for computation")
264
    device = 'cpu'
265

266
# Use device in model loading
267
from modelscope import Model
268
model = Model.from_pretrained('model_name', device=device)
269
```
270

271
### File Utilities
272

273
```python
274
from modelscope.utils.file_utils import get_cache_dir, makedirs
275

276
# Get cache directory
277
cache_dir = get_cache_dir()
278
print(f"Cache directory: {cache_dir}")
279

280
# Create directories safely
281
output_dir = './output/experiments/run_1'
282
makedirs(output_dir, exist_ok=True)
283
print(f"Created directory: {output_dir}")
284

285
# File path utilities
286
from modelscope.utils.file_utils import is_local_path, get_file_extension
287

288
path1 = '/local/path/to/file.json'
289
path2 = 'https://remote.com/file.json'
290

291
print(f"Is local path: {is_local_path(path1)}")  # True
292
print(f"Is local path: {is_local_path(path2)}")  # False
293

294
print(f"File extension: {get_file_extension(path1)}")  # .json
295
```
296

297
### Import Utilities
298

299
```python
300
from modelscope.utils.import_utils import (
301
    is_torch_available,
302
    is_transformers_available,
303
    is_opencv_available
304
)
305

306
# Check for optional dependencies
307
if is_torch_available():
308
    print("PyTorch is available")
309
    from modelscope import TorchModel
310
    # Use PyTorch-specific functionality
311

312
if is_transformers_available():
313
    print("Transformers library is available")
314
    # Use HuggingFace transformers integration
315

316
if is_opencv_available():
317
    print("OpenCV is available")
318
    # Use computer vision functionality
319

320
# Conditional imports based on availability
321
def get_model_class():
322
    if is_torch_available():
323
        from modelscope import TorchModel
324
        return TorchModel
325
    else:
326
        from modelscope import Model
327
        return Model
328

329
ModelClass = get_model_class()
330
model = ModelClass.from_pretrained('model_name')
331
```
332

333
### Configuration Templates
334

335
```python
336
from modelscope import read_config, get_logger
337

338
# Create configuration template
339
def create_training_config(
340
    model_name: str,
341
    output_dir: str,
342
    max_epochs: int = 10,
343
    learning_rate: float = 1e-5
344
) -> dict:
345
    """Create standardized training configuration."""
346
    
347
    config = {
348
        'model': {
349
            'name': model_name,
350
            'type': 'classification'
351
        },
352
        'training': {
353
            'output_dir': output_dir,
354
            'max_epochs': max_epochs,
355
            'learning_rate': learning_rate,
356
            'batch_size': 32,
357
            'eval_strategy': 'epoch',
358
            'save_strategy': 'epoch',
359
            'logging_steps': 100
360
        },
361
        'data': {
362
            'train_file': 'train.json',
363
            'eval_file': 'eval.json',
364
            'test_file': 'test.json'
365
        }
366
    }
367
    
368
    return config
369

370
# Use configuration template
371
config = create_training_config(
372
    model_name='damo/nlp_structbert_base_chinese',
373
    output_dir='./output',
374
    max_epochs=5,
375
    learning_rate=2e-5
376
)
377

378
# Save configuration
379
import json
380
with open('training_config.json', 'w') as f:
381
    json.dump(config, f, indent=2)
382

383
# Load and use configuration
384
loaded_config = read_config('training_config.json')
385
logger = get_logger('training.log')
386
logger.info(f"Training configuration loaded: {loaded_config['model']['name']}")
387
```
388

389
### Utility Function Composition
390

391
```python
392
from modelscope import get_logger, read_config, Tasks
393

394
def setup_experiment(config_path: str, log_file: str = None):
395
    """
396
    Set up complete experiment environment.
397
    
398
    Parameters:
399
    - config_path: Path to experiment configuration
400
    - log_file: Optional log file path
401
    
402
    Returns:
403
    Tuple of (config, logger, task_type)
404
    """
405
    
406
    # Load configuration
407
    config = read_config(config_path)
408
    
409
    # Setup logging
410
    logger = get_logger(
411
        log_file=log_file,
412
        log_level=config.get('log_level', 'INFO')
413
    )
414
    
415
    # Determine task type
416
    task_type = config.get('task_type', Tasks.NLPTasks.TEXT_CLASSIFICATION)
417
    
418
    logger.info(f"Experiment setup complete")
419
    logger.info(f"Task type: {task_type}")
420
    logger.info(f"Configuration loaded from: {config_path}")
421
    
422
    return config, logger, task_type
423

424
# Use utility composition
425
config, logger, task = setup_experiment(
426
    'experiment_config.json',
427
    'experiment.log'
428
)
429

430
# Proceed with experiment using setup
431
from modelscope import pipeline
432
pipe = pipeline(task, model=config['model_name'])
433
```