0
# Page Iteration
1
2
Standardized pagination patterns for list operations with support for both HTTP/JSON and gRPC APIs, including async variants. The page iteration system provides consistent interfaces for traversing large result sets across different Google API transport protocols.
3
4
## Capabilities
5
6
### Base Iterator Classes
7
8
Abstract base classes that define the common interface for all page iterators.
9
10
```python { .api }
11
class Iterator:
12
"""
13
Abstract base class for iterating through API responses.
14
15
Args:
16
client: API client instance
17
item_to_value (Callable): Function to transform items from API responses
18
page_token (str, optional): Token for starting at specific page
19
max_results (int, optional): Maximum number of results to return
20
"""
21
def __init__(self, client, item_to_value, page_token=None, max_results=None): ...
22
23
def __iter__(self):
24
"""Return iterator instance."""
25
return self
26
27
def __next__(self):
28
"""
29
Get next item from iterator.
30
31
Returns:
32
Any: Next item from the API response
33
34
Raises:
35
StopIteration: When no more items available
36
"""
37
38
@property
39
def page_number(self):
40
"""
41
Current page number (0-indexed).
42
43
Returns:
44
int: Current page number
45
"""
46
47
@property
48
def next_page_token(self):
49
"""
50
Token for the next page of results.
51
52
Returns:
53
str or None: Next page token, None if no more pages
54
"""
55
56
@property
57
def num_results(self):
58
"""
59
Total number of results yielded so far.
60
61
Returns:
62
int: Count of results returned
63
"""
64
65
class AsyncIterator:
66
"""
67
Abstract base class for async iteration through API responses.
68
69
Args:
70
client: Async API client instance
71
item_to_value (Callable): Function to transform items from API responses
72
page_token (str, optional): Token for starting at specific page
73
max_results (int, optional): Maximum number of results to return
74
"""
75
def __init__(self, client, item_to_value, page_token=None, max_results=None): ...
76
77
def __aiter__(self):
78
"""Return async iterator instance."""
79
return self
80
81
async def __anext__(self):
82
"""
83
Get next item from async iterator.
84
85
Returns:
86
Any: Next item from the API response
87
88
Raises:
89
StopAsyncIteration: When no more items available
90
"""
91
92
@property
93
def page_number(self):
94
"""Current page number (0-indexed)."""
95
96
@property
97
def next_page_token(self):
98
"""Token for the next page of results."""
99
100
@property
101
def num_results(self):
102
"""Total number of results yielded so far."""
103
```
104
105
### HTTP/REST Iterator Classes
106
107
Iterators for HTTP/JSON-based APIs with RESTful pagination patterns.
108
109
```python { .api }
110
class HTTPIterator(Iterator):
111
"""
112
Iterator for HTTP/JSON API list responses.
113
114
Args:
115
client: HTTP API client instance
116
api_request (Callable): Function to make HTTP API requests
117
path (str): API endpoint path template
118
item_to_value (Callable): Function to extract items from response
119
page_token (str, optional): Starting page token
120
max_results (int, optional): Maximum results to return
121
extra_params (dict, optional): Additional query parameters
122
"""
123
def __init__(self, client, api_request, path, item_to_value, page_token=None, max_results=None, extra_params=None): ...
124
125
@property
126
def started(self):
127
"""
128
Check if iteration has started.
129
130
Returns:
131
bool: True if iteration has begun
132
"""
133
134
def _update_state(self, response):
135
"""
136
Update iterator state from API response.
137
138
Args:
139
response (dict): API response data
140
"""
141
```
142
143
### gRPC Iterator Classes
144
145
Iterators for gRPC-based APIs with protobuf message pagination.
146
147
```python { .api }
148
class GRPCIterator(Iterator):
149
"""
150
Iterator for gRPC list responses using protobuf messages.
151
152
Args:
153
client: gRPC client instance
154
method (Callable): gRPC method to call for pagination
155
request: Initial request protobuf message
156
items_field (str): Field name containing items in response
157
request_token_field (str): Field name for page token in request (default: "page_token")
158
response_token_field (str): Field name for next page token in response (default: "next_page_token")
159
"""
160
def __init__(self, client, method, request, items_field, request_token_field="page_token", response_token_field="next_page_token"): ...
161
162
@property
163
def started(self):
164
"""Check if iteration has started."""
165
166
def _update_state(self, response):
167
"""Update iterator state from gRPC response."""
168
169
class AsyncGRPCIterator(AsyncIterator):
170
"""
171
Async iterator for gRPC list responses.
172
173
Args:
174
client: Async gRPC client instance
175
method (Callable): Async gRPC method for pagination
176
request: Initial request protobuf message
177
items_field (str): Field name containing items in response
178
request_token_field (str): Page token field in request
179
response_token_field (str): Next page token field in response
180
"""
181
def __init__(self, client, method, request, items_field, request_token_field="page_token", response_token_field="next_page_token"): ...
182
183
@property
184
def started(self):
185
"""Check if async iteration has started."""
186
187
async def _update_state(self, response):
188
"""Update iterator state from async gRPC response."""
189
```
190
191
### Page Container Classes
192
193
Classes representing individual pages of results within an iterator.
194
195
```python { .api }
196
class Page:
197
"""
198
Single page of results from a paginated API response.
199
200
Args:
201
parent (Iterator): Parent iterator instance
202
response: Raw API response for this page
203
item_to_value (Callable): Function to transform response items
204
"""
205
def __init__(self, parent, response, item_to_value): ...
206
207
def __iter__(self):
208
"""Iterate over items in this page."""
209
210
@property
211
def num_items(self):
212
"""
213
Number of items in this page.
214
215
Returns:
216
int: Count of items in current page
217
"""
218
219
@property
220
def remaining(self):
221
"""
222
Number of items remaining in this page.
223
224
Returns:
225
int: Count of unprocessed items in page
226
"""
227
228
@property
229
def response(self):
230
"""
231
Raw API response for this page.
232
233
Returns:
234
Any: Original API response object
235
"""
236
```
237
238
### Utility Functions
239
240
Helper functions for customizing page iteration behavior.
241
242
```python { .api }
243
def _item_to_value_identity(iterator, item):
244
"""
245
Default item transformation function that returns items unchanged.
246
247
Args:
248
iterator: Iterator instance (unused)
249
item: Item from API response
250
251
Returns:
252
Any: Item unchanged
253
"""
254
255
def _do_nothing_page_start(iterator, page, response):
256
"""
257
Default page start handler that performs no action.
258
259
Args:
260
iterator: Iterator instance
261
page: Page instance
262
response: API response for the page
263
"""
264
```
265
266
## Usage Examples
267
268
### Basic HTTP API Pagination
269
270
```python
271
from google.api_core import page_iterator
272
import requests
273
274
class HTTPClient:
275
def __init__(self, base_url):
276
self.base_url = base_url
277
278
def api_request(self, method, path, **kwargs):
279
"""Make HTTP API request."""
280
url = f"{self.base_url}{path}"
281
response = requests.request(method, url, **kwargs)
282
response.raise_for_status()
283
return response.json()
284
285
def extract_items(response):
286
"""Extract items from API response."""
287
return response.get("items", [])
288
289
# Create HTTP iterator
290
client = HTTPClient("https://api.example.com")
291
iterator = page_iterator.HTTPIterator(
292
client=client,
293
api_request=lambda path, **kwargs: client.api_request("GET", path, params=kwargs),
294
path="/users",
295
item_to_value=lambda iterator, item: item,
296
max_results=100
297
)
298
299
# Iterate through all results
300
users = []
301
for user in iterator:
302
users.append(user)
303
print(f"User: {user['name']}")
304
305
print(f"Total users retrieved: {len(users)}")
306
print(f"Pages processed: {iterator.page_number + 1}")
307
```
308
309
### gRPC API Pagination
310
311
```python
312
from google.api_core import page_iterator
313
import grpc
314
from my_api_pb2 import ListUsersRequest
315
from my_api_pb2_grpc import UserServiceStub
316
317
# Create gRPC client
318
channel = grpc.insecure_channel("api.example.com:443")
319
client = UserServiceStub(channel)
320
321
# Create initial request
322
request = ListUsersRequest(
323
page_size=50,
324
filter="status:active"
325
)
326
327
# Create gRPC iterator
328
iterator = page_iterator.GRPCIterator(
329
client=client,
330
method=client.ListUsers,
331
request=request,
332
items_field="users" # Field containing user list in response
333
)
334
335
# Iterate through users
336
for user in iterator:
337
print(f"User ID: {user.id}, Name: {user.name}")
338
339
print(f"Total results: {iterator.num_results}")
340
```
341
342
### Async Pagination
343
344
```python
345
import asyncio
346
from google.api_core import page_iterator_async
347
import aiohttp
348
349
class AsyncHTTPClient:
350
def __init__(self, base_url):
351
self.base_url = base_url
352
353
async def api_request(self, path, **params):
354
"""Make async HTTP API request."""
355
async with aiohttp.ClientSession() as session:
356
url = f"{self.base_url}{path}"
357
async with session.get(url, params=params) as response:
358
response.raise_for_status()
359
return await response.json()
360
361
async def async_pagination_example():
362
client = AsyncHTTPClient("https://api.example.com")
363
364
# Create async iterator
365
iterator = page_iterator_async.AsyncHTTPIterator(
366
client=client,
367
api_request=client.api_request,
368
path="/products",
369
item_to_value=lambda iterator, item: item,
370
max_results=200
371
)
372
373
# Async iteration
374
products = []
375
async for product in iterator:
376
products.append(product)
377
print(f"Product: {product['name']}")
378
379
print(f"Total products: {len(products)}")
380
381
asyncio.run(async_pagination_example())
382
```
383
384
### Custom Item Transformation
385
386
```python
387
from google.api_core import page_iterator
388
from datetime import datetime
389
390
def transform_user_item(iterator, raw_user):
391
"""Transform raw API user data into custom format."""
392
return {
393
"id": raw_user["user_id"],
394
"name": raw_user["display_name"],
395
"email": raw_user["email_address"],
396
"created": datetime.fromisoformat(raw_user["created_at"]),
397
"active": raw_user["status"] == "active"
398
}
399
400
# Use custom transformation
401
iterator = page_iterator.HTTPIterator(
402
client=client,
403
api_request=api_request_func,
404
path="/users",
405
item_to_value=transform_user_item,
406
max_results=1000
407
)
408
409
# Get transformed user objects
410
for user in iterator:
411
print(f"User {user['name']} ({user['email']}) - Active: {user['active']}")
412
```
413
414
### Page-by-Page Processing
415
416
```python
417
from google.api_core import page_iterator
418
419
# Create iterator
420
iterator = page_iterator.HTTPIterator(
421
client=client,
422
api_request=api_request_func,
423
path="/large-dataset",
424
item_to_value=lambda iterator, item: item
425
)
426
427
# Process one page at a time
428
for page in iterator._page_iter:
429
print(f"Processing page {iterator.page_number + 1}")
430
print(f"Page contains {page.num_items} items")
431
432
# Process all items in current page
433
page_items = list(page)
434
435
# Custom page processing logic
436
process_page_batch(page_items)
437
438
print(f"Completed page {iterator.page_number + 1}")
439
```
440
441
### Iterator with Error Handling
442
443
```python
444
from google.api_core import page_iterator
445
from google.api_core import exceptions
446
from google.api_core import retry
447
448
# Configure retry for API requests
449
retry_config = retry.Retry(
450
predicate=retry.if_exception_type(
451
exceptions.ServiceUnavailable,
452
exceptions.InternalServerError
453
),
454
initial=1.0,
455
maximum=60.0
456
)
457
458
@retry_config
459
def robust_api_request(path, **params):
460
"""API request with retry logic."""
461
response = requests.get(f"https://api.example.com{path}", params=params)
462
if response.status_code >= 500:
463
raise exceptions.ServiceUnavailable("Service unavailable")
464
response.raise_for_status()
465
return response.json()
466
467
# Create iterator with robust error handling
468
iterator = page_iterator.HTTPIterator(
469
client=client,
470
api_request=robust_api_request,
471
path="/reliable-endpoint",
472
item_to_value=lambda iterator, item: item
473
)
474
475
# Iterate with error handling
476
try:
477
items = list(iterator)
478
print(f"Successfully retrieved {len(items)} items")
479
except exceptions.GoogleAPIError as e:
480
print(f"API error during pagination: {e}")
481
except Exception as e:
482
print(f"Unexpected error: {e}")
483
```