0
# HTTP Client and Server
1

2
Complete HTTP implementation with client agents, server resources, and web application framework. Twisted's web module provides both high-level and low-level APIs for HTTP operations, WSGI support, and extensive web service capabilities.
3

4
## Capabilities
5

6
### HTTP Client
7

8
HTTP client functionality with support for connection pooling, redirects, cookies, content encoding, and TLS.
9

10
```python { .api }
11
class client.Agent:
12
    """
13
    HTTP client agent for making requests.
14
    """
15
    def __init__(self, reactor, contextFactory=None, connectTimeout=None, bindAddress=None, pool=None):
16
        """
17
        Args:
18
            reactor: The reactor to use
19
            contextFactory: SSL context factory for HTTPS
20
            connectTimeout (float): Connection timeout in seconds
21
            bindAddress: Local address to bind connections
22
            pool: Connection pool to use
23
        """
24
    
25
    def request(self, method, uri, headers=None, bodyProducer=None):
26
        """
27
        Make an HTTP request.
28
        
29
        Args:
30
            method (bytes): HTTP method (b'GET', b'POST', etc.)
31
            uri (bytes): Request URI
32
            headers (Headers): HTTP headers
33
            bodyProducer: Body producer for request data
34
            
35
        Returns:
36
            Deferred[Response]: Fires with Response object
37
        """
38

39
class client.Response:
40
    """
41
    HTTP response object.
42
    
43
    Attributes:
44
    - version: HTTP version tuple (major, minor)
45
    - code: HTTP status code (int)
46
    - phrase: HTTP status phrase (bytes)
47
    - headers: Response headers (Headers object)
48
    - length: Content length or UNKNOWN_LENGTH
49
    """
50
    version = None
51
    code = None
52
    phrase = None
53
    headers = None
54
    length = None
55
    
56
    def deliverBody(self, protocol):
57
        """
58
        Deliver response body to a protocol.
59
        
60
        Args:
61
            protocol: Protocol to receive body data
62
        """
63

64
def client.readBody(response):
65
    """
66
    Read the entire body of an HTTP response.
67
    
68
    Args:
69
        response (Response): HTTP response object
70
        
71
    Returns:
72
        Deferred[bytes]: Complete response body
73
    """
74

75
class client.BrowserLikeRedirectAgent:
76
    """
77
    HTTP agent that follows redirects like a browser.
78
    """
79
    def __init__(self, agent, redirectLimit=20):
80
        """
81
        Args:
82
            agent: Underlying HTTP agent
83
            redirectLimit (int): Maximum number of redirects to follow
84
        """
85

86
class client.CookieAgent:
87
    """
88
    HTTP agent that automatically handles cookies.
89
    """
90
    def __init__(self, agent, cookieJar):
91
        """
92
        Args:
93
            agent: Underlying HTTP agent
94
            cookieJar: Cookie jar for storing cookies
95
        """
96

97
class client.ContentDecoderAgent:
98
    """
99
    HTTP agent that automatically decodes content (gzip, deflate).
100
    """
101
    def __init__(self, agent, decoders=None):
102
        """
103
        Args:
104
            agent: Underlying HTTP agent
105
            decoders: List of content decoders
106
        """
107

108
class client.ProxyAgent:
109
    """
110
    HTTP agent that routes requests through a proxy.
111
    """
112
    def __init__(self, endpoint, reactor=None, pool=None):
113
        """
114
        Args:
115
            endpoint: Proxy server endpoint
116
            reactor: The reactor to use
117
            pool: Connection pool
118
        """
119

120
class client.FileBodyProducer:
121
    """
122
    Body producer for uploading files.
123
    """
124
    def __init__(self, inputFile, cooperator=None):
125
        """
126
        Args:
127
            inputFile: File-like object to read from
128
            cooperator: Task cooperator for yielding control
129
        """
130

131
class client.HTTPConnectionPool:
132
    """
133
    Connection pool for HTTP connections.
134
    """
135
    def __init__(self, reactor, persistent=True, maxPersistentPerHost=10, 
136
                 cachedConnectionTimeout=240, retryAutomatically=True):
137
        """
138
        Args:
139
            reactor: The reactor to use
140
            persistent (bool): Whether to keep connections alive
141
            maxPersistentPerHost (int): Max connections per host
142
            cachedConnectionTimeout (int): Connection cache timeout
143
            retryAutomatically (bool): Retry failed connections
144
        """
145
```
146

147
**HTTP Client Usage Example**:
148

149
```python
150
from twisted.internet import reactor, defer
151
from twisted.web import client
152
from twisted.web.http_headers import Headers
153

154
@defer.inlineCallbacks
155
def makeRequest():
156
    agent = client.Agent(reactor)
157
    
158
    # Simple GET request
159
    response = yield agent.request(
160
        b'GET',
161
        b'https://httpbin.org/get',
162
        Headers({'User-Agent': ['Twisted HTTP Client']})
163
    )
164
    
165
    print(f"Status: {response.code}")
166
    body = yield client.readBody(response)
167
    print(f"Body: {body.decode()}")
168

169
# Enhanced agent with redirects and cookies
170
agent = client.Agent(reactor)
171
redirect_agent = client.BrowserLikeRedirectAgent(agent)
172
cookie_agent = client.CookieAgent(redirect_agent, cookielib.CookieJar())
173

174
makeRequest()
175
reactor.run()
176
```
177

178
### HTTP Server
179

180
HTTP server implementation with resource-based routing, session management, and WSGI support.
181

182
```python { .api }
183
class server.Site:
184
    """
185
    HTTP server site that serves resources.
186
    """
187
    def __init__(self, resource, timeout=60*60*12, logPath=None):
188
        """
189
        Args:
190
            resource: Root resource to serve
191
            timeout (int): Session timeout in seconds
192
            logPath: Path for access logs
193
        """
194
    
195
    def getResourceFor(self, request):
196
        """
197
        Get the resource for a request.
198
        
199
        Args:
200
            request: HTTP request object
201
            
202
        Returns:
203
            Resource: Resource to handle request
204
        """
205

206
class server.Request:
207
    """
208
    HTTP request object.
209
    
210
    Key Attributes:
211
    - method: HTTP method (bytes)
212
    - uri: Request URI (bytes)  
213
    - path: Request path (bytes)
214
    - args: Query parameters (dict)
215
    - headers: Request headers
216
    - content: Request body stream
217
    - cookies: Request cookies (list)
218
    - session: Session object
219
    - client: Client address
220
    """
221
    method = None
222
    uri = None
223
    path = None
224
    args = None
225
    headers = None
226
    content = None
227
    cookies = None
228
    
229
    def getSession(self):
230
        """Get or create session for this request."""
231
    
232
    def write(self, data):
233
        """
234
        Write data to response.
235
        
236
        Args:
237
            data (bytes): Data to write
238
        """
239
    
240
    def finish(self):
241
        """Finish the HTTP response."""
242
    
243
    def setHeader(self, name, value):
244
        """
245
        Set HTTP response header.
246
        
247
        Args:
248
            name (str): Header name
249
            value (str): Header value  
250
        """
251
    
252
    def setResponseCode(self, code, message=None):
253
        """
254
        Set HTTP response code.
255
        
256
        Args:
257
            code (int): HTTP status code
258
            message (str): Status message
259
        """
260
    
261
    def redirect(self, url):
262
        """
263
        Redirect to another URL.
264
        
265
        Args:
266
            url (bytes): Redirect URL
267
        """
268

269
class server.Session:
270
    """
271
    HTTP session for maintaining state across requests.
272
    
273
    Attributes:
274
    - uid: Unique session identifier
275
    - expireCallbacks: List of expiration callbacks
276
    """
277
    uid = None
278
    
279
    def expire(self):
280
        """Expire this session."""
281
    
282
    def touch(self):
283
        """Update session last access time."""
284

285
# Server constants
286
server.NOT_DONE_YET  # Constant indicating async response
287
```
288

289
**HTTP Server Usage Example**:
290

291
```python
292
from twisted.web import server, resource
293
from twisted.internet import reactor, endpoints
294

295
class HelloResource(resource.Resource):
296
    isLeaf = True
297
    
298
    def render_GET(self, request):
299
        request.setHeader(b'content-type', b'text/plain')
300
        return b'Hello, World!'
301
    
302
    def render_POST(self, request):
303
        data = request.content.read()
304
        request.setHeader(b'content-type', b'text/plain')
305
        return f'Received: {data.decode()}'.encode()
306

307
root = HelloResource()
308
site = server.Site(root)
309

310
endpoint = endpoints.TCP4ServerEndpoint(reactor, 8080)
311
endpoint.listen(site)
312

313
print("Server listening on port 8080")
314
reactor.run()
315
```
316

317
### Web Resources
318

319
Resource-based system for organizing web applications with hierarchical URL routing and content generation.
320

321
```python { .api }
322
class resource.Resource:
323
    """
324
    Base class for web resources.
325
    
326
    Attributes:
327
    - isLeaf: Whether this resource has children (bool)
328
    """
329
    isLeaf = False
330
    
331
    def render(self, request):
332
        """
333
        Render this resource for a request.
334
        
335
        Args:
336
            request: HTTP request object
337
            
338
        Returns:
339
            bytes or NOT_DONE_YET: Response data
340
        """
341
    
342
    def render_GET(self, request):
343
        """Handle GET requests."""
344
    
345
    def render_POST(self, request):
346
        """Handle POST requests."""
347
    
348
    def render_PUT(self, request):
349
        """Handle PUT requests."""
350
    
351
    def render_DELETE(self, request):
352
        """Handle DELETE requests."""
353
    
354
    def getChild(self, path, request):
355
        """
356
        Get child resource for path segment.
357
        
358
        Args:
359
            path (bytes): Path segment
360
            request: HTTP request
361
            
362
        Returns:
363
            Resource: Child resource
364
        """
365
    
366
    def putChild(self, path, child):
367
        """
368
        Add a child resource.
369
        
370
        Args:
371
            path (bytes): Path segment
372
            child (Resource): Child resource
373
        """
374

375
class resource.ErrorPage(resource.Resource):
376
    """
377
    Resource for generating error pages.
378
    """
379
    def __init__(self, status, brief, detail):
380
        """
381
        Args:
382
            status (int): HTTP status code
383
            brief (str): Brief error description
384
            detail (str): Detailed error description
385
        """
386

387
class resource.NoResource(resource.ErrorPage):
388
    """
389
    404 Not Found resource.
390
    """
391
    def __init__(self, message="Sorry. No such resource."):
392
        """
393
        Args:
394
            message (str): 404 error message
395
        """
396

397
class resource.ForbiddenResource(resource.ErrorPage):
398
    """
399
    403 Forbidden resource.
400
    """
401
    def __init__(self, message="Sorry, resource is forbidden."):
402
        """
403
        Args:
404
            message (str): 403 error message
405
        """
406

407
def resource.getChildForRequest(resource, request):
408
    """
409
    Get the appropriate child resource for a request.
410
    
411
    Args:
412
        resource: Parent resource
413
        request: HTTP request
414
        
415
    Returns:
416
        Resource: Child resource to handle request
417
    """
418
```
419

420
### Static File Serving
421

422
Resources for serving static files and directory listings.
423

424
```python { .api }
425
class static.File:
426
    """
427
    Resource for serving static files and directories.
428
    """
429
    def __init__(self, path, defaultType="text/html", ignoredExts=None, registry=None, allowExt=0):
430
        """
431
        Args:
432
            path (str): File system path to serve
433
            defaultType (str): Default MIME type
434
            ignoredExts: File extensions to ignore
435
            registry: MIME type registry
436
            allowExt (bool): Allow extension-based MIME detection
437
        """
438
    
439
    def directoryListing(self):
440
        """
441
        Get directory listing resource.
442
        
443
        Returns:
444
            Resource: Directory listing resource
445
        """
446

447
class static.DirectoryLister:
448
    """
449
    Resource for generating directory listings.
450
    """
451
    def __init__(self, pathname, dirs=None, contentTypes=None, contentEncodings=None, defaultType="text/html"):
452
        """
453
        Args:
454
            pathname (str): Directory path
455
            dirs: Subdirectories to list
456
            contentTypes: MIME type mappings
457
            contentEncodings: Content encoding mappings
458
            defaultType (str): Default MIME type
459
        """
460

461
class static.Data:
462
    """
463
    Resource for serving data from memory.
464
    """
465
    def __init__(self, data, type):
466
        """
467
        Args:
468
            data (bytes): Data to serve
469
            type (str): MIME type
470
        """
471
```
472

473
### WSGI Integration
474

475
Support for running WSGI applications within Twisted's web server.
476

477
```python { .api }
478
class wsgi.WSGIResource:
479
    """
480
    Resource for running WSGI applications.
481
    """
482
    def __init__(self, reactor, threadpool, application):
483
        """
484
        Args:
485
            reactor: The reactor to use
486
            threadpool: Thread pool for WSGI app
487
            application: WSGI application callable
488
        """
489
```
490

491
**WSGI Usage Example**:
492

493
```python
494
from twisted.web import wsgi, server
495
from twisted.python import threadpool
496
from twisted.internet import reactor, endpoints
497

498
# Simple WSGI application
499
def simple_app(environ, start_response):
500
    status = '200 OK'
501
    headers = [('Content-type', 'text/plain')]
502
    start_response(status, headers)
503
    return [b'Hello from WSGI!']
504

505
# Create WSGI resource
506
pool = threadpool.ThreadPool()
507
pool.start()
508
wsgi_resource = wsgi.WSGIResource(reactor, pool, simple_app)
509

510
# Serve it
511
site = server.Site(wsgi_resource)
512
endpoint = endpoints.TCP4ServerEndpoint(reactor, 8080)
513
endpoint.listen(site)
514
```
515

516
### HTTP Headers
517

518
Utilities for working with HTTP headers.
519

520
```python { .api }
521
class http_headers.Headers:
522
    """
523
    HTTP headers container.
524
    """
525
    def __init__(self, rawHeaders=None):
526
        """
527
        Args:
528
            rawHeaders: Dict of header name to list of values
529
        """
530
    
531
    def hasHeader(self, name):
532
        """
533
        Check if header exists.
534
        
535
        Args:
536
            name (str): Header name
537
            
538
        Returns:
539
            bool: True if header exists
540
        """
541
    
542
    def getHeader(self, name, default=None):
543
        """
544
        Get header value.
545
        
546
        Args:
547
            name (str): Header name
548
            default: Default value if not found
549
            
550
        Returns:
551
            bytes or default: Header value
552
        """
553
    
554
    def setHeader(self, name, value):
555
        """
556
        Set header value.
557
        
558
        Args:
559
            name (str): Header name
560
            value: Header value (str or list)
561
        """
562
    
563
    def addRawHeader(self, name, value):
564
        """
565
        Add raw header value.
566
        
567
        Args:
568
            name (bytes): Header name
569
            value (bytes): Header value
570
        """
571
```
572

573
### Template System
574

575
Basic template system for generating dynamic HTML content.
576

577
```python { .api }
578
class template.Element:
579
    """
580
    Base class for template elements.
581
    """
582
    def __init__(self, loader, tag=None):
583
        """
584
        Args:
585
            loader: Template loader
586
            tag: Root template tag
587
        """
588

589
class template.XMLFile:
590
    """
591
    Load template from XML file.
592
    """
593
    def __init__(self, path):
594
        """
595
        Args:
596
            path (str): Path to XML template file
597
        """
598

599
class template.XMLString:
600
    """
601
    Load template from XML string.
602
    """
603
    def __init__(self, template):
604
        """
605
        Args:
606
            template (str): XML template string
607
        """
608

609
def template.renderer(func):
610
    """
611
    Decorator for template renderer methods.
612
    
613
    Args:
614
        func: Renderer function
615
        
616
    Returns:
617
        Decorated renderer function
618
    """
619

620
def template.flatten(element):
621
    """
622
    Flatten template element to bytes.
623
    
624
    Args:
625
        element: Template element
626
        
627
    Returns:
628
        Deferred[bytes]: Rendered template
629
    """
630
```
631

632
### Web Utilities
633

634
Helper functions and utilities for web development.
635

636
```python { .api }
637
def util.redirectTo(URL, request):
638
    """
639
    Generate redirect response.
640
    
641
    Args:
642
        URL (bytes): Redirect URL
643
        request: HTTP request object
644
        
645
    Returns:
646
        bytes: Redirect response data
647
    """
648

649
class util.Redirect:
650
    """
651
    Resource that redirects to another URL.
652
    """
653
    def __init__(self, url):
654
        """
655
        Args:
656
            url (bytes): Redirect URL
657
        """
658

659
class util.ChildRedirector:
660
    """
661
    Resource that redirects based on path.
662
    """
663
    def __init__(self, url):
664
        """
665
        Args:
666
            url (bytes): Base redirect URL
667
        """
668
```
669

670
### Interface Types
671

672
Key interfaces and types used in Twisted's HTTP implementation.
673

674
```python { .api }
675
# From twisted.web.iweb
676
class IBodyProducer:
677
    """
678
    Interface for request body producers.
679
    
680
    Methods:
681
    - startProducing(consumer): Start producing data
682
    - stopProducing(): Stop producing data
683
    - pauseProducing(): Pause data production  
684
    - resumeProducing(): Resume data production
685
    """
686
    def startProducing(consumer): ...
687
    def stopProducing(): ...
688
    def pauseProducing(): ...
689
    def resumeProducing(): ...
690

691
class IResponse:
692
    """
693
    Interface for HTTP responses.
694
    
695
    Attributes:
696
    - version: HTTP version tuple
697
    - code: Status code
698
    - phrase: Status phrase
699
    - headers: Response headers
700
    - length: Content length
701
    """
702
    version = None
703
    code = None
704
    phrase = None  
705
    headers = None
706
    length = None
707

708
# From twisted.web.http_headers
709
class Headers:
710
    """
711
    HTTP header collection.
712
    
713
    Methods:
714
    - addRawHeader(name, value): Add header
715
    - setRawHeaders(name, values): Set header values
716
    - getRawHeaders(name): Get header values
717
    - hasHeader(name): Check if header exists
718
    """
719
    def __init__(self, rawHeaders=None): ...
720
    def addRawHeader(self, name, value): ...
721
    def setRawHeaders(self, name, values): ...
722
    def getRawHeaders(self, name, default=None): ...
723
    def hasHeader(self, name): ...
724

725
# Constants
726
NOT_DONE_YET = object()  # Indicates async response
727
UNKNOWN_LENGTH = -1      # Unknown content length
728
```
729

730
### Required Imports
731

732
Complete import statements for HTTP functionality:
733

734
```python
735
# Core HTTP functionality
736
from twisted.web import client, server, resource, static, http
737
from twisted.web.http_headers import Headers
738
from twisted.web.iweb import IBodyProducer, IResponse
739
from twisted.internet import defer, reactor, endpoints
740

741
# Common import patterns
742
from twisted.web.client import (
743
    Agent, Response, BrowserLikeRedirectAgent, CookieAgent, 
744
    ContentDecoderAgent, ProxyAgent, FileBodyProducer, readBody
745
)
746
from twisted.web.server import Site, Request, Session  
747
from twisted.web.resource import Resource, ErrorPage, NoResource
748
from twisted.web.static import File
749
```