0
# Paging
1

2
Iterator-based paging support for REST APIs that return large result sets, providing automatic page traversal, reset capability, and raw response access. The paging system enables efficient handling of paginated API responses with Python iterator protocol.
3

4
## Capabilities
5

6
### Paged Iterator
7

8
Main class for handling paginated REST responses with automatic page traversal.
9

10
```python { .api }
11
class Paged:
12
    def __init__(self, command, classes, raw_headers=None, **kwargs):
13
        """
14
        Initialize paged response container.
15
        
16
        Parameters:
17
        - command: Function to retrieve next page (callable)
18
        - classes: Dict of model classes for deserialization
19
        - raw_headers: Dict of headers to include in raw responses
20
        - kwargs: Additional initialization parameters
21
        """
22
    
23
    next_link: str = ""
24
    current_page: list = []
25
    
26
    def __iter__(self):
27
        """Return iterator (self)."""
28
    
29
    def __next__(self):
30
        """Get next item, advancing pages as needed."""
31
    
32
    # Python 2 compatibility
33
    next = __next__
34
```
35

36
### Page Navigation
37

38
Control page traversal and iteration state.
39

40
```python { .api }
41
def advance_page(self) -> list:
42
    """
43
    Force moving cursor to next page.
44
    
45
    Returns:
46
    Current page list
47
    
48
    Raises:
49
    StopIteration if no further pages available
50
    """
51

52
def reset(self):
53
    """Reset iterator to first page."""
54

55
def get(self, url: str) -> list:
56
    """
57
    Get arbitrary page by URL.
58
    
59
    This resets iterator and consumes it to return specific page.
60
    
61
    Parameters:
62
    - url: URL to specific page
63
    
64
    Returns:
65
    Page items list
66
    """
67
```
68

69
### Raw Response Access
70

71
Access underlying HTTP response data and headers.
72

73
```python { .api }
74
@property
75
def raw(self):
76
    """
77
    Get current page as ClientRawResponse.
78
    
79
    Returns:
80
    ClientRawResponse with current page data and response headers
81
    """
82
```
83

84
### Async Support
85

86
For Python 3.5+, async paging support is available through AsyncPagedMixin.
87

88
```python { .api }
89
class AsyncPagedMixin:
90
    """Mixin providing async iterator support for paging."""
91
    
92
    async def __aiter__(self):
93
        """Return async iterator."""
94
    
95
    async def __anext__(self):
96
        """Get next item asynchronously."""
97
```
98

99
## Usage Examples
100

101
### Basic Paging
102

103
```python
104
from msrest.paging import Paged
105
from msrest import ServiceClient, Configuration
106

107
# Assuming you have a function that gets the next page
108
def get_next_page(next_link):
109
    """Function to fetch next page from API."""
110
    if not next_link:
111
        # First page
112
        request = client.get('/users')
113
    else:
114
        # Subsequent pages
115
        request = client.get(next_link)
116
    
117
    response = client.send(request)
118
    return response
119

120
# Create configuration and client
121
config = Configuration(base_url='https://api.example.com')
122
client = ServiceClient(None, config)
123

124
# Create paged iterator
125
model_classes = {'User': User}  # Your model classes
126
users_paged = Paged(get_next_page, model_classes)
127

128
# Iterate through all pages automatically
129
for user in users_paged:
130
    print(f"User: {user.name} ({user.email})")
131
```
132

133
### Manual Page Control
134

135
```python
136
from msrest.paging import Paged
137

138
# Create paged iterator
139
users_paged = Paged(get_next_page, model_classes)
140

141
# Get first page manually
142
first_page = users_paged.advance_page()
143
print(f"First page has {len(first_page)} users")
144

145
# Check if more pages available
146
if users_paged.next_link:
147
    second_page = users_paged.advance_page()
148
    print(f"Second page has {len(second_page)} users")
149

150
# Reset to beginning
151
users_paged.reset()
152

153
# Start iteration from beginning again
154
for user in users_paged:
155
    print(user.name)
156
    break  # Just get first user
157
```
158

159
### Direct Page Access
160

161
```python
162
from msrest.paging import Paged
163

164
users_paged = Paged(get_next_page, model_classes)
165

166
# Get specific page by URL
167
specific_page_url = 'https://api.example.com/users?page=5'
168
page_5_users = users_paged.get(specific_page_url)
169

170
print(f"Page 5 has {len(page_5_users)} users")
171
for user in page_5_users:
172
    print(f"  - {user.name}")
173
```
174

175
### Raw Response Access
176

177
```python
178
from msrest.paging import Paged
179

180
# Include headers in raw responses
181
headers_to_capture = {'X-Total-Count': 'str', 'X-Page-Count': 'str'}
182
users_paged = Paged(get_next_page, model_classes, raw_headers=headers_to_capture)
183

184
# Get first page
185
users_paged.advance_page()
186

187
# Access raw response data
188
raw_response = users_paged.raw
189
print(f"Response status: {raw_response.response.status_code}")
190
print(f"Total count: {raw_response.headers.get('X-Total-Count')}")
191
print(f"Page count: {raw_response.headers.get('X-Page-Count')}")
192
print(f"Current page items: {len(raw_response.output)}")
193
```
194

195
### Custom Page Retrieval Function
196

197
```python
198
import json
199
from msrest.paging import Paged
200

201
def custom_page_fetcher(next_link):
202
    """Custom function to handle specific API pagination format."""
203
    
204
    if not next_link:
205
        # First request
206
        url = '/api/data'
207
        params = {'page': 1, 'per_page': 50}
208
    else:
209
        # Parse next_link to get page info
210
        from urllib.parse import urlparse, parse_qs
211
        parsed = urlparse(next_link)
212
        query_params = parse_qs(parsed.query)
213
        
214
        url = parsed.path
215
        params = {k: v[0] for k, v in query_params.items()}
216
    
217
    # Make request
218
    request = client.get(url, params=params)
219
    response = client.send(request)
220
    
221
    # Parse response to extract items and next link
222
    data = json.loads(response.text)
223
    
224
    # Assuming API returns: {"items": [...], "next_page": "url"}
225
    # Transform to msrest format
226
    response_with_items = type('Response', (), {
227
        'text': json.dumps(data['items']),
228
        'headers': response.headers,
229
        'status_code': response.status_code
230
    })()
231
    
232
    # Set next link for paging
233
    if 'next_page' in data:
234
        response_with_items.next_link = data['next_page']
235
    else:
236
        response_with_items.next_link = None
237
    
238
    return response_with_items
239

240
# Use custom fetcher
241
data_paged = Paged(custom_page_fetcher, {'DataItem': DataItem})
242

243
# Iterate through all pages
244
for item in data_paged:
245
    print(f"Item: {item.id}")
246
```
247

248
### Async Paging (Python 3.5+)
249

250
```python
251
import asyncio
252
from msrest.paging import Paged
253

254
async def async_page_fetcher(next_link):
255
    """Async function to fetch pages."""
256
    # Use async HTTP client here
257
    # This is a simplified example
258
    
259
    if not next_link:
260
        url = '/async/data'
261
    else:
262
        url = next_link
263
    
264
    # Async request (pseudo-code)
265
    async_response = await async_client.get(url)
266
    return async_response
267

268
# For Python 3.5+
269
class AsyncPaged(Paged):
270
    """Async version of Paged iterator."""
271
    
272
    async def __aiter__(self):
273
        return self
274
    
275
    async def __anext__(self):
276
        if self.current_page and self._current_page_iter_index < len(self.current_page):
277
            response = self.current_page[self._current_page_iter_index]
278
            self._current_page_iter_index += 1
279
            return response
280
        else:
281
            await self.advance_page_async()
282
            return await self.__anext__()
283
    
284
    async def advance_page_async(self):
285
        if self.next_link is None:
286
            raise StopAsyncIteration("End of paging")
287
        
288
        self._current_page_iter_index = 0
289
        self._response = await self._get_next(self.next_link)
290
        self._derserializer(self, self._response)
291
        return self.current_page
292

293
# Usage
294
async def process_async_pages():
295
    async_paged = AsyncPaged(async_page_fetcher, model_classes)
296
    
297
    async for item in async_paged:
298
        print(f"Async item: {item.name}")
299
        
300
        # Process only first 10 items
301
        if some_condition:
302
            break
303

304
# Run async function
305
asyncio.run(process_async_pages())
306
```
307

308
### Error Handling with Paging
309

310
```python
311
from msrest.paging import Paged
312
from msrest.exceptions import HttpOperationError
313

314
def robust_page_fetcher(next_link):
315
    """Page fetcher with error handling."""
316
    try:
317
        if not next_link:
318
            request = client.get('/data')
319
        else:
320
            request = client.get(next_link)
321
        
322
        response = client.send(request)
323
        return response
324
        
325
    except HttpOperationError as e:
326
        if e.response.status_code == 404:
327
            # No more pages
328
            return None
329
        else:
330
            # Re-raise other errors
331
            raise
332

333
data_paged = Paged(robust_page_fetcher, model_classes)
334

335
try:
336
    for item in data_paged:
337
        print(f"Item: {item.id}")
338
        
339
except StopIteration:
340
    print("Finished processing all pages")
341
except HttpOperationError as e:
342
    print(f"API error during paging: {e}")
343
```
344

345
### Paging with Authentication
346

347
```python
348
from msrest import ServiceClient, Configuration
349
from msrest.authentication import ApiKeyCredentials
350
from msrest.paging import Paged
351

352
# Setup authenticated client
353
config = Configuration(base_url='https://api.example.com')
354
config.credentials = ApiKeyCredentials(in_headers={'Authorization': 'Bearer your-token'})
355

356
with ServiceClient(None, config) as client:
357
    def auth_page_fetcher(next_link):
358
        """Authenticated page fetcher."""
359
        if not next_link:
360
            request = client.get('/protected/data')
361
        else:
362
            request = client.get(next_link)
363
        
364
        # Authentication is automatically handled by the client
365
        return client.send(request)
366
    
367
    # Create paged iterator with authenticated fetcher
368
    protected_data = Paged(auth_page_fetcher, model_classes)
369
    
370
    # Iterate through protected resources
371
    for item in protected_data:
372
        print(f"Protected item: {item.name}")
373
```
374

375
## Integration with Deserializer
376

377
The Paged class works closely with the Deserializer to convert raw response data into Python objects:
378

379
```python
380
from msrest.serialization import Deserializer
381
from msrest.paging import Paged
382

383
# The Paged class internally uses a deserializer
384
# You provide the model classes during initialization
385
model_classes = {
386
    'User': User,
387
    'Address': Address,
388
    'Organization': Organization
389
}
390

391
# Paged creates internal deserializer: Deserializer(model_classes)
392
users_paged = Paged(get_next_page, model_classes)
393

394
# Each page response is automatically deserialized using the appropriate model class
395
for user in users_paged:
396
    # user is already a User model instance
397
    print(f"User: {user.name}")
398
    if hasattr(user, 'address'):
399
        print(f"  Address: {user.address.city}")
400
```