0
# Utilities
1

2
Helper functions for debugging HTTP requests and responses, constructing user-agent strings, enhanced URL encoding, and other common tasks for HTTP client development.
3

4
## Capabilities
5

6
### Request/Response Debugging
7

8
Tools for inspecting and debugging HTTP traffic by dumping request and response details.
9

10
```python { .api }
11
def dump_response(response, request_prefix=b'< ', response_prefix=b'> ', data_array=None):
12
    """
13
    Dump HTTP response details for debugging.
14
    
15
    Parameters:
16
    - response: Response object to dump
17
    - request_prefix: bytes, prefix for request lines (default: b'< ')
18
    - response_prefix: bytes, prefix for response lines (default: b'> ')
19
    - data_array: bytearray, optional array to append dump data
20
    
21
    Returns:
22
    bytes: formatted dump of response
23
    """
24

25
def dump_all(response, request_prefix=b'< ', response_prefix=b'> '):
26
    """
27
    Dump both request and response details.
28
    
29
    Parameters:
30
    - response: Response object to dump
31
    - request_prefix: bytes, prefix for request lines (default: b'< ')
32
    - response_prefix: bytes, prefix for response lines (default: b'> ')
33
    
34
    Returns:
35
    bytes: formatted dump of request and response
36
    """
37

38
class PrefixSettings:
39
    """
40
    Settings for customizing dump output prefixes.
41
    
42
    Parameters:
43
    - request: bytes, prefix for request lines
44
    - response: bytes, prefix for response lines
45
    """
46
    def __init__(self, request, response): ...
47
```
48

49
#### Usage Examples
50

51
```python
52
import requests
53
from requests_toolbelt.utils import dump_response, dump_all
54

55
# Basic response debugging
56
response = requests.get('https://httpbin.org/get?param=value')
57
print(dump_response(response).decode('utf-8'))
58

59
# Debug both request and response
60
response = requests.post(
61
    'https://httpbin.org/post',
62
    json={'key': 'value'},
63
    headers={'Custom-Header': 'test'}
64
)
65
print(dump_all(response).decode('utf-8'))
66

67
# Custom prefixes for better readability
68
response = requests.get('https://httpbin.org/headers')
69
debug_output = dump_all(
70
    response,
71
    request_prefix=b'REQUEST: ',
72
    response_prefix=b'RESPONSE: '
73
)
74
print(debug_output.decode('utf-8'))
75

76
# Save debug output to file
77
response = requests.get('https://api.example.com/data')
78
with open('debug_output.txt', 'wb') as f:
79
    f.write(dump_all(response))
80

81
# Append to existing data
82
data_buffer = bytearray()
83
response1 = requests.get('https://httpbin.org/get')
84
response2 = requests.post('https://httpbin.org/post', data={'test': 'data'})
85

86
dump_response(response1, data_array=data_buffer)
87
dump_response(response2, data_array=data_buffer)
88

89
with open('combined_debug.txt', 'wb') as f:
90
    f.write(data_buffer)
91
```
92

93
### User-Agent Construction
94

95
Build standardized, informative user-agent strings for HTTP requests.
96

97
```python { .api }
98
def user_agent(name, version, extras=None):
99
    """
100
    Create internet-friendly user-agent string.
101
    
102
    Parameters:
103
    - name: str, application name (e.g., 'my-scraper')
104
    - version: str, application version (e.g., '1.0.0')
105
    - extras: list, additional components as (name, version) tuples
106
    
107
    Returns:
108
    str: formatted user-agent string
109
    """
110

111
class UserAgentBuilder:
112
    """
113
    Builder class for constructing user-agent strings.
114
    
115
    Parameters:
116
    - name: str, application name
117
    - version: str, application version
118
    """
119
    def __init__(self, name, version): ...
120
    
121
    def include_extras(self, extras):
122
        """
123
        Add extra components to user-agent.
124
        
125
        Parameters:
126
        - extras: list of (name, version) tuples
127
        
128
        Returns:
129
        UserAgentBuilder: self for chaining
130
        """
131
    
132
    def include_implementation(self):
133
        """
134
        Include Python implementation details.
135
        
136
        Returns:
137
        UserAgentBuilder: self for chaining
138
        """
139
    
140
    def include_system(self):
141
        """
142
        Include system/platform information.
143
        
144
        Returns:
145
        UserAgentBuilder: self for chaining
146
        """
147
    
148
    def build(self):
149
        """
150
        Build the final user-agent string.
151
        
152
        Returns:
153
        str: complete user-agent string
154
        """
155
```
156

157
#### Usage Examples
158

159
```python
160
import requests
161
from requests_toolbelt.utils import user_agent
162
from requests_toolbelt.utils.user_agent import UserAgentBuilder
163

164
# Simple user-agent
165
ua = user_agent('my-app', '2.1.0')
166
print(ua)  # Output: my-app/2.1.0 CPython/3.9.0 Linux/5.4.0
167

168
# User-agent with extras
169
extras = [('requests', '2.28.0'), ('urllib3', '1.26.0')]
170
ua = user_agent('data-collector', '1.5.2', extras)
171
print(ua)
172

173
# Using builder for custom control
174
builder = UserAgentBuilder('web-scraper', '3.0.0')
175
ua = builder.include_extras([
176
    ('beautifulsoup4', '4.11.1'),
177
    ('lxml', '4.9.1')
178
]).include_implementation().include_system().build()
179

180
# Use with requests
181
session = requests.Session()
182
session.headers['User-Agent'] = user_agent('my-client', '1.0')
183

184
response = session.get('https://httpbin.org/user-agent')
185
print(response.json())
186

187
# Different user-agents for different services
188
github_ua = user_agent('github-client', '2.0', [('requests', '2.28.0')])
189
twitter_ua = user_agent('twitter-bot', '1.5', [('oauth', '1.1')])
190

191
github_session = requests.Session()
192
github_session.headers['User-Agent'] = github_ua
193

194
twitter_session = requests.Session()
195
twitter_session.headers['User-Agent'] = twitter_ua
196
```
197

198
### Enhanced URL Encoding
199

200
Improved URL encoding with better handling of lists and complex data structures.
201

202
```python { .api }
203
def urlencode(query, doseq=False, safe='', encoding=None, errors=None, quote_via=quote_plus):
204
    """
205
    Enhanced URL encoding with improved list handling.
206
    
207
    Parameters:
208
    - query: dict or list of tuples, data to encode
209
    - doseq: bool, handle sequences in values
210
    - safe: str, characters not to encode
211
    - encoding: str, character encoding
212
    - errors: str, error handling scheme
213
    - quote_via: callable, quoting function
214
    
215
    Returns:
216
    str: URL-encoded string
217
    """
218
```
219

220
#### Usage Examples
221

222
```python
223
from requests_toolbelt.utils.formdata import urlencode
224
import requests
225

226
# Enhanced list handling
227
data = {
228
    'tags': ['python', 'http', 'requests'],
229
    'categories': ['web', 'api'],  
230
    'user': 'john_doe'
231
}
232

233
# Standard encoding - lists become multiple parameters
234
encoded = urlencode(data, doseq=True)
235
print(encoded)  # tags=python&tags=http&tags=requests&categories=web&categories=api&user=john_doe
236

237
# Use with requests
238
response = requests.get('https://api.example.com/search', params=encoded)
239

240
# Complex data structures
241
complex_data = {
242
    'filters': {
243
        'status': ['active', 'pending'],
244
        'type': 'user'
245
    },
246
    'sort': 'created_date',
247
    'limit': 50
248
}
249

250
# Flatten and encode complex structures
251
flat_data = []
252
for key, value in complex_data.items():
253
    if isinstance(value, dict):
254
        for subkey, subvalue in value.items():
255
            full_key = f"{key}[{subkey}]"
256
            if isinstance(subvalue, list):
257
                for item in subvalue:
258
                    flat_data.append((full_key, item))
259
            else:
260
                flat_data.append((full_key, subvalue))
261
    else:
262
        flat_data.append((key, value))
263

264
encoded = urlencode(flat_data)
265
print(encoded)
266
```
267

268
### Deprecated Utilities
269

270
Legacy functions maintained for backward compatibility.
271

272
```python { .api }
273
def get_encodings_from_content(content):
274
    """
275
    Extract character encodings from HTML content.
276
    
277
    Parameters:
278
    - content: str, HTML content to analyze
279
    
280
    Returns:
281
    list: detected encodings
282
    
283
    Note: Deprecated - use charset-normalizer or chardet instead
284
    """
285

286
def get_unicode_from_response(response):
287
    """
288
    Extract unicode text from response.
289
    
290
    Parameters:
291
    - response: Response object
292
    
293
    Returns:
294
    str: decoded response text
295
    
296
    Note: Deprecated - use response.text instead
297
    """
298
```
299

300
### Debugging Workflow Examples
301

302
```python
303
import requests
304
from requests_toolbelt.utils import dump_all, user_agent
305

306
def debug_api_call(url, method='GET', **kwargs):
307
    """Helper function for debugging API calls."""
308
    
309
    # Setup session with debugging
310
    session = requests.Session()
311
    session.headers['User-Agent'] = user_agent('debug-client', '1.0')
312
    
313
    # Make request
314
    response = session.request(method, url, **kwargs)
315
    
316
    # Debug output
317
    print("=" * 50)
318
    print(f"DEBUG: {method} {url}")
319
    print("=" * 50)
320
    print(dump_all(response).decode('utf-8', errors='replace'))
321
    print("=" * 50)
322
    
323
    return response
324

325
# Usage
326
response = debug_api_call(
327
    'https://httpbin.org/post',
328
    method='POST',
329
    json={'test': 'data'},
330
    headers={'Custom-Header': 'debug-value'}
331
)
332

333
# Automated debugging for multiple requests
334
def debug_request_batch(requests_list):
335
    """Debug multiple requests and save output."""
336
    
337
    with open('debug_batch.txt', 'w') as f:
338
        for i, req in enumerate(requests_list):
339
            f.write(f"\\n{'='*60}\\n")
340
            f.write(f"REQUEST {i+1}: {req['method']} {req['url']}\\n")
341
            f.write(f"{'='*60}\\n")
342
            
343
            response = requests.request(**req)
344
            debug_output = dump_all(response).decode('utf-8', errors='replace')
345
            f.write(debug_output)
346

347
# Usage
348
requests_to_debug = [
349
    {'method': 'GET', 'url': 'https://httpbin.org/get'},
350
    {'method': 'POST', 'url': 'https://httpbin.org/post', 'json': {'key': 'value'}},
351
    {'method': 'PUT', 'url': 'https://httpbin.org/put', 'data': 'test data'}
352
]
353

354
debug_request_batch(requests_to_debug)
355
```