0
# Resilience and Reliability
1
2
Circuit breaker implementation for regional failover and retry mechanisms with configurable backoff strategies. Provides automatic failure detection and recovery for improved reliability in distributed cloud environments.
3
4
## Capabilities
5
6
### Circuit Breaker Implementation
7
8
Circuit breaker pattern implementation to prevent cascading failures and enable automatic failover to backup regions.
9
10
```python { .api }
11
class CircuitBreaker:
12
def __init__(self, breaker_setting):
13
"""
14
Create circuit breaker.
15
16
Args:
17
breaker_setting: RegionBreakerProfile configuration object
18
"""
19
20
def before_requests(self) -> tuple[int, bool]:
21
"""
22
Check circuit breaker state before making request.
23
24
Returns:
25
tuple: (generation_id, use_backup_region)
26
generation_id (int): Current generation for tracking
27
use_backup_region (bool): Whether to use backup region
28
"""
29
30
def after_requests(self, before: int, success: bool) -> None:
31
"""
32
Record request result after completion.
33
34
Args:
35
before (int): Generation ID from before_requests()
36
success (bool): Whether the request succeeded
37
"""
38
```
39
40
### Request Counter
41
42
Counter for tracking request statistics used by the circuit breaker.
43
44
```python { .api }
45
class Counter:
46
def __init__(self):
47
"""Create request counter."""
48
49
def on_success(self) -> None:
50
"""Record successful request."""
51
52
def on_failure(self) -> None:
53
"""Record failed request."""
54
55
def clear(self) -> None:
56
"""Reset all counters."""
57
58
def get_failure_rate(self) -> float:
59
"""
60
Calculate failure rate.
61
62
Returns:
63
float: Failure rate (0.0 to 1.0)
64
"""
65
```
66
67
### Circuit Breaker States
68
69
```python { .api }
70
STATE_CLOSED: int = 0 # Circuit breaker is closed (normal operation)
71
STATE_HALF_OPEN: int = 1 # Circuit breaker is half-open (testing recovery)
72
STATE_OPEN: int = 2 # Circuit breaker is open (using backup region)
73
```
74
75
### Retry Mechanisms
76
77
No-operation retry policy that performs no retries.
78
79
```python { .api }
80
class NoopRetryer:
81
def send_request(self, fn):
82
"""
83
Execute function without retry.
84
85
Args:
86
fn: Function to execute
87
88
Returns:
89
Result of function execution
90
"""
91
```
92
93
Standard retry policy with exponential backoff and configurable retry conditions.
94
95
```python { .api }
96
class StandardRetryer:
97
def __init__(self, max_attempts: int = 3, backoff_fn = None, logger = None):
98
"""
99
Create standard retryer.
100
101
Args:
102
max_attempts (int): Maximum number of attempts. Default: 3
103
backoff_fn: Backoff function taking attempt number, returns sleep seconds
104
logger: Logger instance for retry logging
105
"""
106
107
def send_request(self, fn):
108
"""
109
Execute function with retry logic.
110
111
Args:
112
fn: Function to execute
113
114
Returns:
115
Result of successful function execution
116
117
Raises:
118
TencentCloudSDKException: If all retry attempts fail
119
"""
120
121
@staticmethod
122
def should_retry(resp, err) -> bool:
123
"""
124
Determine if request should be retried.
125
126
Args:
127
resp: Response object (if any)
128
err: Exception object (if any)
129
130
Returns:
131
bool: True if request should be retried
132
"""
133
134
@staticmethod
135
def backoff(n: int) -> int:
136
"""
137
Default exponential backoff function.
138
139
Args:
140
n (int): Attempt number (0-based)
141
142
Returns:
143
int: Sleep time in seconds (2^n)
144
"""
145
146
def on_retry(self, n: int, sleep: int, resp, err) -> None:
147
"""
148
Called before each retry attempt.
149
150
Args:
151
n (int): Attempt number
152
sleep (int): Sleep time before retry
153
resp: Response object (if any)
154
err: Exception object (if any)
155
"""
156
```
157
158
## Usage Examples
159
160
### Basic Circuit Breaker
161
162
```python
163
from tencentcloud.common.circuit_breaker import CircuitBreaker
164
from tencentcloud.common.profile.client_profile import RegionBreakerProfile
165
166
# Create breaker configuration
167
breaker_config = RegionBreakerProfile(
168
backup_endpoint="ap-shanghai.tencentcloudapi.com",
169
max_fail_num=5,
170
max_fail_percent=0.75,
171
window_interval=300, # 5 minutes
172
timeout=60, # 1 minute
173
max_requests=5
174
)
175
176
# Create circuit breaker
177
breaker = CircuitBreaker(breaker_config)
178
179
# Use circuit breaker around requests
180
def make_request():
181
generation, use_backup = breaker.before_requests()
182
183
try:
184
# Make API request (use backup endpoint if use_backup is True)
185
if use_backup:
186
endpoint = "ap-shanghai.tencentcloudapi.com"
187
else:
188
endpoint = "cvm.tencentcloudapi.com"
189
190
# Simulate API call
191
response = api_call(endpoint)
192
breaker.after_requests(generation, True) # Success
193
return response
194
195
except Exception as e:
196
breaker.after_requests(generation, False) # Failure
197
raise e
198
```
199
200
### Standard Retry Configuration
201
202
```python
203
from tencentcloud.common.retry import StandardRetryer
204
import logging
205
206
# Create logger for retry events
207
logger = logging.getLogger("retry")
208
logger.setLevel(logging.DEBUG)
209
210
# Create retryer with custom backoff
211
def custom_backoff(attempt):
212
"""Custom backoff: 1s, 2s, 4s, then cap at 10s"""
213
return min(2 ** attempt, 10)
214
215
retryer = StandardRetryer(
216
max_attempts=5,
217
backoff_fn=custom_backoff,
218
logger=logger
219
)
220
221
# Use retryer
222
def api_call():
223
# Simulate API call that might fail
224
import random
225
if random.random() < 0.7: # 70% chance of failure
226
raise TencentCloudSDKException("RequestLimitExceeded", "Rate limited")
227
return {"success": True}
228
229
try:
230
result = retryer.send_request(api_call)
231
print("Success:", result)
232
except Exception as e:
233
print("All retries failed:", e)
234
```
235
236
### No Retry Policy
237
238
```python
239
from tencentcloud.common.retry import NoopRetryer
240
241
# Create no-retry policy
242
retryer = NoopRetryer()
243
244
# Use no-retry retryer (executes once only)
245
def api_call():
246
return {"data": "response"}
247
248
result = retryer.send_request(api_call)
249
print(result)
250
```
251
252
### Integration with Client Profile
253
254
```python
255
from tencentcloud.common.credential import Credential
256
from tencentcloud.common.common_client import CommonClient
257
from tencentcloud.common.profile.client_profile import ClientProfile, RegionBreakerProfile
258
from tencentcloud.common.retry import StandardRetryer
259
import logging
260
261
# Setup logging
262
logging.basicConfig(level=logging.DEBUG)
263
logger = logging.getLogger("sdk_retry")
264
265
# Create credentials
266
cred = Credential("your-secret-id", "your-secret-key")
267
268
# Configure circuit breaker
269
breaker_profile = RegionBreakerProfile(
270
backup_endpoint="ap-shanghai.tencentcloudapi.com",
271
max_fail_num=3,
272
max_fail_percent=0.6,
273
window_interval=180,
274
timeout=30
275
)
276
277
# Configure retry policy
278
retryer = StandardRetryer(
279
max_attempts=4,
280
backoff_fn=lambda n: min(1.5 ** n, 30), # 1.5s, 2.25s, 3.4s, 30s
281
logger=logger
282
)
283
284
# Create client profile with resilience features
285
client_profile = ClientProfile()
286
client_profile.disable_region_breaker = False
287
client_profile.region_breaker_profile = breaker_profile
288
client_profile.retryer = retryer
289
290
# Create client
291
client = CommonClient("cvm", "2017-03-12", cred, "ap-guangzhou", client_profile)
292
293
# API calls will automatically use circuit breaker and retry
294
try:
295
response = client.call_json("DescribeInstances", {"Limit": 10})
296
print("Success:", len(response["Response"]["InstanceSet"]))
297
except Exception as e:
298
print("Final failure:", e)
299
```
300
301
### Custom Retry Conditions
302
303
```python
304
from tencentcloud.common.retry import StandardRetryer
305
from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
306
307
class CustomRetryer(StandardRetryer):
308
@staticmethod
309
def should_retry(resp, err):
310
"""Custom retry logic"""
311
if not err:
312
return False
313
314
if isinstance(err, TencentCloudSDKException):
315
# Retry on network errors and rate limiting
316
retryable_codes = [
317
"ClientNetworkError",
318
"ServerNetworkError",
319
"RequestLimitExceeded",
320
"InternalError"
321
]
322
return err.get_code() in retryable_codes
323
324
return False
325
326
# Use custom retryer
327
custom_retryer = CustomRetryer(max_attempts=5)
328
329
def api_call():
330
# Your API call logic
331
pass
332
333
result = custom_retryer.send_request(api_call)
334
```
335
336
### Monitoring Circuit Breaker State
337
338
```python
339
from tencentcloud.common.circuit_breaker import CircuitBreaker, STATE_CLOSED, STATE_HALF_OPEN, STATE_OPEN
340
341
class MonitoredCircuitBreaker(CircuitBreaker):
342
def before_requests(self):
343
generation, use_backup = super().before_requests()
344
345
# Log state changes
346
state_names = {
347
STATE_CLOSED: "CLOSED",
348
STATE_HALF_OPEN: "HALF_OPEN",
349
STATE_OPEN: "OPEN"
350
}
351
352
print(f"Circuit breaker state: {state_names[self.state]}")
353
if use_backup:
354
print("Using backup region due to circuit breaker")
355
356
return generation, use_backup
357
358
# Use monitored circuit breaker
359
breaker = MonitoredCircuitBreaker(breaker_config)
360
```