0
# User Agent and Browser Emulation
1

2
Comprehensive browser fingerprinting and user agent management with support for multiple browsers, platforms, and device types. CloudScraper includes automatic fallback for executable environments and extensive browser database support.
3

4
## Capabilities
5

6
### User_Agent Class
7

8
Main class for managing user agent strings, HTTP headers, and TLS cipher suites to emulate different browsers and devices.
9

10
```python { .api }
11
class User_Agent:
12
    def __init__(self, browser=None, **kwargs):
13
        """
14
        Initialize user agent manager.
15
        
16
        Parameters:
17
        - browser: str|dict, browser configuration
18
        - **kwargs: additional configuration options
19
        """
20

21
    def loadUserAgent(self, **kwargs):
22
        """
23
        Load user agent configuration from browsers database.
24
        
25
        Parameters:
26
        - browser: str|dict, browser type and options
27
        - platform: str, target platform (windows, linux, darwin, android, ios)
28
        - desktop: bool, include desktop user agents (default: True)
29
        - mobile: bool, include mobile user agents (default: True)
30
        - custom: str, custom user agent string
31
        - allow_brotli: bool, enable Brotli compression
32
        """
33

34
    def filterAgents(self, user_agents: dict) -> dict:
35
        """
36
        Filter user agents based on platform and device preferences.
37
        
38
        Parameters:
39
        - user_agents: dict, complete user agents database
40
        
41
        Returns:
42
        dict: Filtered user agents matching criteria
43
        """
44

45
    def tryMatchCustom(self, user_agents: dict) -> bool:
46
        """
47
        Try to match custom user agent string with known browsers.
48
        
49
        Parameters:
50
        - user_agents: dict, user agents database
51
        
52
        Returns:
53
        bool: True if custom user agent was matched and configured
54
        """
55

56
    @property
57
    def headers(self) -> dict:
58
        """HTTP headers including User-Agent for the selected browser."""
59

60
    @property
61
    def cipherSuite(self) -> list:
62
        """TLS cipher suite configuration for the selected browser."""
63
    
64
    # Configuration attributes (set during initialization)
65
    browser: str         # Selected browser name
66
    platform: str        # Selected platform name
67
    desktop: bool        # Whether desktop user agents are enabled
68
    mobile: bool         # Whether mobile user agents are enabled
69
    custom: str          # Custom user agent string (if provided)
70
    platforms: list      # Available platforms ['linux', 'windows', 'darwin', 'android', 'ios']
71
    browsers: list       # Available browsers ['chrome', 'firefox']
72
```
73

74
### Basic Browser Selection
75

76
Simple browser configuration for common use cases:
77

78
```python
79
# Default browser (random selection)
80
scraper = cloudscraper.create_scraper()
81

82
# Specific browser
83
scraper = cloudscraper.create_scraper(browser='chrome')
84
scraper = cloudscraper.create_scraper(browser='firefox')
85

86
# Browser with platform
87
scraper = cloudscraper.create_scraper(
88
    browser={
89
        'browser': 'chrome',
90
        'platform': 'windows'
91
    }
92
)
93

94
# Check selected user agent
95
print(scraper.headers['User-Agent'])
96
# Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36
97
```
98

99
### Advanced Browser Configuration
100

101
Comprehensive browser fingerprinting with detailed device and platform options:
102

103
```python
104
# Desktop-only Chrome on Windows
105
desktop_scraper = cloudscraper.create_scraper(
106
    browser={
107
        'browser': 'chrome',
108
        'platform': 'windows', 
109
        'desktop': True,
110
        'mobile': False
111
    }
112
)
113

114
# Mobile-only Firefox on Android
115
mobile_scraper = cloudscraper.create_scraper(
116
    browser={
117
        'browser': 'firefox',
118
        'platform': 'android',
119
        'desktop': False,
120
        'mobile': True
121
    }
122
)
123

124
# Any Firefox on macOS (mobile or desktop)
125
mac_scraper = cloudscraper.create_scraper(
126
    browser={
127
        'browser': 'firefox',
128
        'platform': 'darwin',  # macOS
129
        'desktop': True,
130
        'mobile': True
131
    }
132
)
133
```
134

135
## Browser Types and Platforms
136

137
### Supported Browsers
138

139
CloudScraper supports comprehensive browser emulation:
140

141
```python
142
supported_browsers = ['chrome', 'firefox']
143

144
# Chrome variants across platforms
145
chrome_configs = [
146
    {'browser': 'chrome', 'platform': 'windows'},
147
    {'browser': 'chrome', 'platform': 'linux'},
148
    {'browser': 'chrome', 'platform': 'darwin'},   # macOS
149
    {'browser': 'chrome', 'platform': 'android'},
150
    {'browser': 'chrome', 'platform': 'ios'}
151
]
152

153
# Firefox variants
154
firefox_configs = [
155
    {'browser': 'firefox', 'platform': 'windows'},
156
    {'browser': 'firefox', 'platform': 'linux'},
157
    {'browser': 'firefox', 'platform': 'darwin'},
158
    {'browser': 'firefox', 'platform': 'android'},
159
    {'browser': 'firefox', 'platform': 'ios'}
160
]
161

162
# Test different browser configurations
163
for config in chrome_configs:
164
    scraper = cloudscraper.create_scraper(browser=config)
165
    print(f"{config['browser']} on {config['platform']}: {scraper.headers['User-Agent'][:50]}...")
166
```
167

168
### Platform-Specific Configuration
169

170
Target specific operating systems and devices:
171

172
```python
173
platforms = ['windows', 'linux', 'darwin', 'android', 'ios']
174

175
# Windows-specific browser emulation
176
windows_scraper = cloudscraper.create_scraper(
177
    browser={
178
        'browser': 'chrome',
179
        'platform': 'windows',
180
        'desktop': True,
181
        'mobile': False
182
    }
183
)
184

185
# Linux server environment
186
linux_scraper = cloudscraper.create_scraper(
187
    browser={
188
        'browser': 'firefox', 
189
        'platform': 'linux',
190
        'desktop': True,
191
        'mobile': False
192
    }
193
)
194

195
# iOS mobile device
196
ios_scraper = cloudscraper.create_scraper(
197
    browser={
198
        'browser': 'chrome',  # Chrome on iOS
199
        'platform': 'ios',
200
        'desktop': False,
201
        'mobile': True
202
    }
203
)
204

205
# Android device
206
android_scraper = cloudscraper.create_scraper(
207
    browser={
208
        'browser': 'chrome',
209
        'platform': 'android',
210
        'desktop': False,
211
        'mobile': True
212
    }
213
)
214
```
215

216
### Device Type Selection
217

218
Control desktop vs mobile device emulation:
219

220
```python
221
# Desktop only (no mobile user agents)
222
desktop_only = cloudscraper.create_scraper(
223
    browser={
224
        'browser': 'chrome',
225
        'desktop': True,
226
        'mobile': False
227
    }
228
)
229

230
# Mobile only (no desktop user agents)
231
mobile_only = cloudscraper.create_scraper(
232
    browser={
233
        'browser': 'chrome',
234
        'desktop': False,
235
        'mobile': True
236
    }
237
)
238

239
# Both desktop and mobile (default)
240
mixed_devices = cloudscraper.create_scraper(
241
    browser={
242
        'browser': 'chrome',
243
        'desktop': True,   # Default
244
        'mobile': True     # Default
245
    }
246
)
247

248
# Error: cannot disable both
249
try:
250
    invalid_scraper = cloudscraper.create_scraper(
251
        browser={
252
            'desktop': False,
253
            'mobile': False
254
        }
255
    )
256
except RuntimeError as e:
257
    print(f"Error: {e}")
258
    # "Sorry you can't have mobile and desktop disabled at the same time."
259
```
260

261
## Custom User Agents
262

263
### Custom User Agent Strings
264

265
Use custom user agent strings with automatic browser matching:
266

267
```python
268
# Custom user agent with automatic browser detection
269
custom_ua = "MyBot/1.0 (compatible; MSIE 9.0; Windows NT 6.1)"
270
scraper = cloudscraper.create_scraper(
271
    browser={'custom': custom_ua}
272
)
273

274
print(scraper.headers['User-Agent'])  # Uses custom user agent
275
print(scraper.cipherSuite)            # Automatically selected cipher suite
276

277
# Custom user agent with Chrome characteristics
278
chrome_custom = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) MyApp/1.0 Chrome/120.0.0.0 Safari/537.36"
279
scraper = cloudscraper.create_scraper(
280
    browser={'custom': chrome_custom}
281
)
282

283
# If custom UA matches known browser, uses browser's headers and ciphers
284
# Otherwise, uses generic headers and ciphers
285
```
286

287
### Fallback Behavior for Custom User Agents
288

289
Handle custom user agents that don't match known browsers:
290

291
```python
292
# Custom user agent that doesn't match any known browser
293
unknown_ua = "CustomScraper/1.0"
294
scraper = cloudscraper.create_scraper(
295
    browser={'custom': unknown_ua}
296
)
297

298
# Uses generic headers and cipher suite
299
print("Headers:")
300
for key, value in scraper.headers.items():
301
    print(f"  {key}: {value}")
302

303
print(f"\nCipher Suite: {scraper.cipherSuite}")
304

305
# Expected output uses fallback configuration:
306
# User-Agent: CustomScraper/1.0
307
# Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
308
# Accept-Language: en-US,en;q=0.9
309
# Accept-Encoding: gzip, deflate, br
310
```
311

312
## Browser Headers and Fingerprinting
313

314
### HTTP Headers by Browser
315

316
Different browsers send different HTTP headers:
317

318
```python
319
# Compare headers between browsers
320
browsers = ['chrome', 'firefox']
321

322
for browser in browsers:
323
    scraper = cloudscraper.create_scraper(browser=browser)
324
    print(f"\n{browser.upper()} Headers:")
325
    for key, value in scraper.headers.items():
326
        print(f"  {key}: {value}")
327

328
# Chrome typical headers:
329
# User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...
330
# Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
331
# Accept-Language: en-US,en;q=0.9
332
# Accept-Encoding: gzip, deflate, br
333

334
# Firefox typical headers:
335
# User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0
336
# Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
337
# Accept-Language: en-US,en;q=0.5
338
# Accept-Encoding: gzip, deflate, br
339
```
340

341
### TLS Cipher Suites
342

343
Browser-specific TLS cipher suite configurations:
344

345
```python
346
# Compare cipher suites between browsers
347
browsers = ['chrome', 'firefox']
348

349
for browser in browsers:
350
    scraper = cloudscraper.create_scraper(browser=browser)
351
    print(f"\n{browser.upper()} Cipher Suite:")
352
    print("  " + "\n  ".join(scraper.cipherSuite[:5]))  # First 5 ciphers
353
    print(f"  ... and {len(scraper.cipherSuite) - 5} more")
354

355
# Chrome uses different cipher preferences than Firefox
356
# This helps CloudScraper appear as the correct browser type
357
```
358

359
### Browser Fingerprint Consistency
360

361
Maintain consistent browser fingerprints across requests:
362

363
```python
364
# Consistent browser fingerprint
365
scraper = cloudscraper.create_scraper(
366
    browser={
367
        'browser': 'chrome',
368
        'platform': 'windows',
369
        'desktop': True,
370
        'mobile': False
371
    }
372
)
373

374
# All requests use the same user agent and TLS fingerprint
375
for i in range(5):
376
    response = scraper.get('https://httpbin.org/headers')
377
    headers = response.json()['headers']
378
    print(f"Request {i+1}: {headers['User-Agent'][:30]}...")
379
    # Same user agent every time for consistency
380
```
381

382
## User Agent Rotation
383

384
### Manual User Agent Rotation
385

386
Change user agents between sessions:
387

388
```python
389
def create_scraper_with_random_browser():
390
    """Create scraper with randomly selected browser configuration."""
391
    import random
392
    
393
    browsers = ['chrome', 'firefox']
394
    platforms = ['windows', 'linux', 'darwin']
395
    
396
    config = {
397
        'browser': random.choice(browsers),
398
        'platform': random.choice(platforms),
399
        'desktop': True,
400
        'mobile': random.choice([True, False])
401
    }
402
    
403
    scraper = cloudscraper.create_scraper(browser=config)
404
    print(f"Created scraper: {config['browser']} on {config['platform']}")
405
    print(f"User-Agent: {scraper.headers['User-Agent'][:50]}...")
406
    
407
    return scraper
408

409
# Create multiple scrapers with different fingerprints
410
scrapers = []
411
for i in range(3):
412
    scraper = create_scraper_with_random_browser()
413
    scrapers.append(scraper)
414
    print()
415
```
416

417
### User Agent Refresh
418

419
Refresh user agent during long-running sessions:
420

421
```python
422
# User agent refresh capability
423
scraper = cloudscraper.create_scraper(browser='chrome')
424
print(f"Initial UA: {scraper.headers['User-Agent'][:50]}...")
425

426
# Refresh user agent (generates new one)
427
scraper.user_agent.loadUserAgent(browser='firefox')
428
scraper.headers = scraper.user_agent.headers
429

430
print(f"Refreshed UA: {scraper.headers['User-Agent'][:50]}...")
431

432
# Or create new scraper with different browser
433
old_cookies = scraper.cookies  # Preserve session cookies
434
scraper = cloudscraper.create_scraper(browser='firefox')
435
scraper.cookies.update(old_cookies)  # Restore cookies
436
```
437

438
## Executable Environment Support
439

440
### PyInstaller and Frozen Applications
441

442
CloudScraper includes comprehensive fallback for executable environments:
443

444
```python
445
# Automatic detection and fallback for executables
446
import sys
447

448
if getattr(sys, 'frozen', False):
449
    print("Running in executable environment")
450
    # CloudScraper automatically uses built-in fallback user agents
451
    scraper = cloudscraper.create_scraper()
452
else:
453
    print("Running in normal Python environment")
454
    # CloudScraper uses full browsers.json database
455
    scraper = cloudscraper.create_scraper()
456

457
# Works in both environments with appropriate fallbacks
458
response = scraper.get('https://httpbin.org/headers')
459
print(f"User-Agent: {response.json()['headers']['User-Agent'][:50]}...")
460
```
461

462
### Built-in Fallback User Agents
463

464
Comprehensive hardcoded user agents for when browsers.json is unavailable:
465

466
```python
467
# Fallback user agents are automatically used when needed
468
# Covers major browsers and platforms:
469

470
fallback_browsers = {
471
    'chrome_windows': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
472
    'chrome_linux': "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
473
    'chrome_mac': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
474
    'firefox_windows': "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
475
    'firefox_linux': "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0",
476
    'chrome_android': "Mozilla/5.0 (Linux; Android 10; SM-G973F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
477
    'chrome_ios': "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.0.0 Mobile/15E148 Safari/604.1"
478
}
479

480
# These are automatically used when browsers.json is not available
481
```
482

483
### Manual Executable Configuration
484

485
Manually configure for executable environments:
486

487
```python
488
# Force fallback mode for testing
489
import os
490

491
# Temporarily hide browsers.json to test fallback
492
browsers_json_path = 'cloudscraper/user_agent/browsers.json'
493
if os.path.exists(browsers_json_path):
494
    os.rename(browsers_json_path, browsers_json_path + '.backup')
495

496
try:
497
    # This will use fallback user agents
498
    scraper = cloudscraper.create_scraper(browser='chrome')
499
    print(f"Fallback UA: {scraper.headers['User-Agent']}")
500
    
501
finally:
502
    # Restore browsers.json
503
    if os.path.exists(browsers_json_path + '.backup'):
504
        os.rename(browsers_json_path + '.backup', browsers_json_path)
505
```
506

507
## Compression and Encoding
508

509
### Brotli Compression Support
510

511
Configure Brotli compression support based on browser capabilities:
512

513
```python
514
# Enable Brotli (default for browsers that support it)
515
scraper = cloudscraper.create_scraper(
516
    browser='chrome',
517
    allow_brotli=True  # Default
518
)
519
print(scraper.headers['Accept-Encoding'])
520
# "gzip, deflate, br" (includes Brotli)
521

522
# Disable Brotli
523
scraper = cloudscraper.create_scraper(
524
    browser='chrome',
525
    allow_brotli=False
526
)
527
print(scraper.headers['Accept-Encoding'])
528
# "gzip, deflate" (no Brotli)
529

530
# Brotli support varies by browser and version
531
firefox_scraper = cloudscraper.create_scraper(
532
    browser='firefox',
533
    allow_brotli=True
534
)
535
print(firefox_scraper.headers['Accept-Encoding'])
536
```
537

538
### Content Encoding Handling
539

540
Automatic handling of compressed responses:
541

542
```python
543
# CloudScraper automatically handles compressed responses
544
scraper = cloudscraper.create_scraper(allow_brotli=True)
545

546
# Request compressed content
547
response = scraper.get('https://httpbin.org/gzip')
548
print(f"Content-Encoding: {response.headers.get('Content-Encoding', 'none')}")
549
print(f"Content length: {len(response.text)} characters")
550

551
# Brotli compression (if supported)
552
response = scraper.get('https://httpbin.org/brotli')
553
print(f"Content-Encoding: {response.headers.get('Content-Encoding', 'none')}")
554
print(f"Decompressed: {len(response.text)} characters")
555
```
556

557
## User Agent Debugging and Testing
558

559
### User Agent Information
560

561
Inspect user agent configuration and selection:
562

563
```python
564
# Detailed user agent information
565
scraper = cloudscraper.create_scraper(
566
    browser={
567
        'browser': 'chrome',
568
        'platform': 'windows',
569
        'desktop': True,
570
        'mobile': False
571
    }
572
)
573

574
print("User Agent Configuration:")
575
print(f"  Browser: {scraper.user_agent.browser}")
576
print(f"  Platform: {scraper.user_agent.platform}")
577
print(f"  Desktop: {scraper.user_agent.desktop}")
578
print(f"  Mobile: {scraper.user_agent.mobile}")
579

580
print(f"\nSelected User-Agent:")
581
print(f"  {scraper.headers['User-Agent']}")
582

583
print(f"\nAll Headers:")
584
for key, value in scraper.headers.items():
585
    print(f"  {key}: {value}")
586

587
print(f"\nCipher Suite ({len(scraper.cipherSuite)} ciphers):")
588
for i, cipher in enumerate(scraper.cipherSuite[:3]):
589
    print(f"  {i+1}. {cipher}")
590
print(f"  ... and {len(scraper.cipherSuite) - 3} more")
591
```
592

593
### User Agent Testing
594

595
Test user agent selection and variation:
596

597
```python
598
def test_user_agent_variety(configurations, requests_per_config=3):
599
    """Test user agent variety across different configurations."""
600
    results = {}
601
    
602
    for config_name, config in configurations.items():
603
        print(f"\nTesting {config_name}:")
604
        user_agents = set()
605
        
606
        # Test multiple instances to see variety
607
        for i in range(requests_per_config):
608
            scraper = cloudscraper.create_scraper(browser=config)
609
            ua = scraper.headers['User-Agent']
610
            user_agents.add(ua)
611
            print(f"  {i+1}: {ua[:60]}...")
612
        
613
        results[config_name] = {
614
            'unique_user_agents': len(user_agents),
615
            'user_agents': list(user_agents)
616
        }
617
        print(f"  Unique UAs: {len(user_agents)}")
618
    
619
    return results
620

621
# Test different configurations
622
test_configs = {
623
    'chrome_any': {'browser': 'chrome'},
624
    'firefox_any': {'browser': 'firefox'},
625
    'chrome_windows': {'browser': 'chrome', 'platform': 'windows'},
626
    'chrome_mobile': {'browser': 'chrome', 'desktop': False, 'mobile': True},
627
    'random_browser': None  # Uses random selection
628
}
629

630
results = test_user_agent_variety(test_configs)
631

632
# Analyze variety
633
for config, result in results.items():
634
    if result['unique_user_agents'] > 1:
635
        print(f"\n{config} provides good variety ({result['unique_user_agents']} unique UAs)")
636
    else:
637
        print(f"\n{config} provides consistent fingerprint (1 unique UA)")
638
```
639

640
### Browser Fingerprint Validation
641

642
Validate that browser fingerprints are realistic:
643

644
```python
645
# Test browser fingerprint against real browser detection
646
def validate_browser_fingerprint(scraper, test_url='https://httpbin.org/headers'):
647
    """Validate browser fingerprint components."""
648
    response = scraper.get(test_url)
649
    headers = response.json()['headers']
650
    
651
    ua = headers.get('User-Agent', '')
652
    accept = headers.get('Accept', '')
653
    accept_encoding = headers.get('Accept-Encoding', '')
654
    accept_language = headers.get('Accept-Language', '')
655
    
656
    print("Browser Fingerprint Validation:")
657
    print(f"  User-Agent: {ua[:50]}...")
658
    print(f"  Accept: {accept}")
659
    print(f"  Accept-Encoding: {accept_encoding}")
660
    print(f"  Accept-Language: {accept_language}")
661
    
662
    # Basic validation checks
663
    checks = {
664
        'has_user_agent': bool(ua),
665
        'has_accept': bool(accept),
666
        'has_accept_encoding': bool(accept_encoding),
667
        'chrome_like': 'Chrome' in ua and 'Safari' in ua,
668
        'firefox_like': 'Firefox' in ua and 'Gecko' in ua,
669
        'supports_modern_encoding': 'gzip' in accept_encoding,
670
        'supports_brotli': 'br' in accept_encoding
671
    }
672
    
673
    print(f"\nValidation Results:")
674
    for check, result in checks.items():
675
        status = "✅" if result else "❌"
676
        print(f"  {status} {check}")
677
    
678
    return checks
679

680
# Validate different browser configurations  
681
browsers = ['chrome', 'firefox']
682
for browser in browsers:
683
    print(f"\n{'='*50}")
684
    print(f"Validating {browser.upper()} fingerprint")
685
    print('='*50)
686
    
687
    scraper = cloudscraper.create_scraper(browser=browser)
688
    validate_browser_fingerprint(scraper)
689
```