A utility belt for advanced users of python-requests
—
Specialized cookie jar implementations for privacy and session management, plus custom exception classes for better error handling and debugging in HTTP operations.
Specialized cookie jar that forgets cookies after use for enhanced privacy.
class ForgetfulCookieJar(RequestsCookieJar):
"""
Cookie jar that refuses to store cookies.
Inherits from requests.cookies.RequestsCookieJar but overrides set_cookie
to prevent any cookies from being stored.
"""
def set_cookie(self, *args, **kwargs):
"""Override to prevent cookies from being stored."""import requests
from requests_toolbelt.cookies.forgetful import ForgetfulCookieJar
# Session with forgetful cookies
session = requests.Session()
session.cookies = ForgetfulCookieJar()
# Requests try to set cookies but they are not stored
response1 = session.get('https://httpbin.org/cookies/set/session_id/abc123')
print(f"Cookies after first request: {dict(session.cookies)}") # Empty
# Subsequent requests have no cookies
response2 = session.get('https://httpbin.org/cookies')
print(f"Cookies after second request: {dict(session.cookies)}") # Empty
# Regular cookie jar comparison
regular_session = requests.Session()
regular_session.get('https://httpbin.org/cookies/set/session_id/xyz789')
print(f"Regular cookies persist: {dict(regular_session.cookies)}") # Has cookies
# ForgetfulCookieJar ensures no cookies are ever stored
session = requests.Session()
session.cookies = ForgetfulCookieJar()
# Multiple requests with different cookie-setting responses
session.get('https://httpbin.org/cookies/set/session_id/abc123')
session.get('https://httpbin.org/cookies/set/auth_token/xyz789')
session.get('https://httpbin.org/cookies/set/user_pref/dark_mode')
print(f"All cookies ignored: {dict(session.cookies)}") # Always emptyCustom exceptions for better error handling and debugging of HTTP operations.
class StreamingError(Exception):
"""
Exception raised during streaming operations.
Used in requests_toolbelt.downloadutils.stream when there are errors
in streaming uploads, downloads, or other data transfer operations.
"""
class VersionMismatchError(Exception):
"""
Exception raised when there's a version compatibility issue.
Used to indicate a version mismatch in the version of requests required.
The feature in use requires a newer version of Requests to function
appropriately but the version installed is not sufficient.
"""
class RequestsVersionTooOld(Warning):
"""
Warning issued when requests version is too old.
Used to indicate that the Requests version is too old. If the version
of Requests is too old to support a feature, this warning is issued.
"""import requests
from requests_toolbelt import StreamingIterator
from requests_toolbelt.exceptions import StreamingError, VersionMismatchError
import warnings
def safe_streaming_upload(url, data_iterator, total_size):
"""Upload with proper error handling."""
try:
stream = StreamingIterator(total_size, data_iterator)
response = requests.post(
url,
data=stream,
headers={'Content-Length': str(total_size)}
)
if response.status_code != 200:
raise StreamingError(
f"Upload failed with status {response.status_code}",
response=response
)
return response
except Exception as e:
if isinstance(e, StreamingError):
print(f"Streaming error: {e}")
if e.response:
print(f"Response status: {e.response.status_code}")
print(f"Response text: {e.response.text}")
else:
raise StreamingError(f"Unexpected error during streaming: {e}")
# Version checking
def check_requests_version():
"""Check if requests version is compatible."""
import requests
from packaging import version
current_version = version.parse(requests.__version__)
minimum_version = version.parse("2.0.1")
if current_version < minimum_version:
raise VersionMismatchError(
expected="requests>=2.0.1",
actual=f"requests=={requests.__version__}"
)
# Issue warning for very old versions
old_version = version.parse("2.10.0")
if current_version < old_version:
warnings.warn(
f"requests version {requests.__version__} is very old. "
"Some features may not work correctly.",
RequestsVersionTooOld
)
# Usage with error handling
try:
check_requests_version()
def data_generator():
for i in range(100):
yield f"Data chunk {i}\\n".encode('utf-8')
# Calculate total size
total = sum(len(f"Data chunk {i}\\n".encode('utf-8')) for i in range(100))
response = safe_streaming_upload(
'https://upload.example.com/data',
data_generator(),
total
)
print("Upload successful!")
except VersionMismatchError as e:
print(f"Version compatibility error: {e}")
print("Please upgrade requests: pip install --upgrade requests")
except StreamingError as e:
print(f"Upload failed: {e}")
except Exception as e:
print(f"Unexpected error: {e}")
# Warning handling
warnings.simplefilter('always', RequestsVersionTooOld)
try:
check_requests_version()
except VersionMismatchError:
print("Critical version mismatch - cannot continue")import requests
from requests_toolbelt.cookies.forgetful import ForgetfulCookieJar
class PrivacySession(requests.Session):
"""Session with enhanced privacy features."""
def __init__(self):
super().__init__()
# Use forgetful cookie jar
self.cookies = ForgetfulCookieJar()
# Privacy-focused headers
self.headers.update({
'DNT': '1', # Do Not Track
'User-Agent': 'Privacy-Focused-Client/1.0',
})
def request(self, method, url, **kwargs):
"""Override to add privacy protections."""
# Remove referer header for privacy
if 'headers' not in kwargs:
kwargs['headers'] = {}
kwargs['headers'].pop('Referer', None)
response = super().request(method, url, **kwargs)
# Clear sensitive cookies after each request
self.cookies.clear()
return response
# Usage
privacy_session = PrivacySession()
# Each request is independent - no cookie persistence
response1 = privacy_session.get('https://example.com/login')
response2 = privacy_session.get('https://example.com/profile') # No login cookies
# Temporary authentication session
class TemporaryAuthSession(requests.Session):
"""Session that maintains auth only for the current operation."""
def __init__(self):
super().__init__()
self.cookies = ForgetfulCookieJar()
def authenticated_request(self, method, url, auth_cookies=None, **kwargs):
"""Make request with temporary authentication."""
# Set auth cookies temporarily
if auth_cookies:
for name, value in auth_cookies.items():
self.cookies.set(name, value)
try:
response = self.request(method, url, **kwargs)
return response
finally:
# Always clear auth cookies after request
self.cookies.clear()
# Usage
temp_session = TemporaryAuthSession()
auth_cookies = {'session_token': 'abc123', 'csrf_token': 'xyz789'}
# Each authenticated request is isolated
response = temp_session.authenticated_request(
'POST',
'https://api.example.com/secure-endpoint',
auth_cookies=auth_cookies,
json={'data': 'sensitive'}
)
# No cookies persist after the operation
print(f"Cookies after request: {dict(temp_session.cookies)}") # Emptyfrom requests_toolbelt.exceptions import StreamingError, VersionMismatchError
import requests
import logging
# Configure logging for better error tracking
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class RobustHTTPClient:
"""HTTP client with comprehensive error handling."""
def __init__(self):
self.session = requests.Session()
def upload_with_retry(self, url, data, max_retries=3):
"""Upload with automatic retry on streaming errors."""
for attempt in range(max_retries):
try:
response = self.session.post(url, data=data)
if response.status_code >= 500:
raise StreamingError(
f"Server error: {response.status_code}",
response=response
)
return response
except StreamingError as e:
logger.warning(f"Attempt {attempt + 1} failed: {e}")
if attempt == max_retries - 1:
logger.error("All retry attempts exhausted")
raise
# Wait before retry
import time
time.sleep(2 ** attempt) # Exponential backoff
def safe_operation(self, operation_func, *args, **kwargs):
"""Wrap operations with comprehensive error handling."""
try:
return operation_func(*args, **kwargs)
except VersionMismatchError as e:
logger.error(f"Version compatibility issue: {e}")
raise
except StreamingError as e:
logger.error(f"Streaming operation failed: {e}")
if e.response:
logger.error(f"Response details: {e.response.status_code} - {e.response.text[:100]}")
raise
except Exception as e:
logger.exception("Unexpected error in HTTP operation")
raise StreamingError(f"Operation failed: {e}")
# Usage
client = RobustHTTPClient()
try:
# Robust upload operation
with open('important_data.json', 'rb') as f:
response = client.upload_with_retry(
'https://backup.example.com/upload',
f
)
print("Upload completed successfully")
except StreamingError as e:
print(f"Upload failed permanently: {e}")
# Implement fallback strategy
except VersionMismatchError as e:
print(f"System compatibility issue: {e}")
# Prompt user to update dependenciesInstall with Tessl CLI
npx tessl i tessl/pypi-requests-toolbelt