0
# Synchronization
1

2
Automated synchronization operations that apply calculated differences to update target datasets. Supports creation, modification, and deletion of records with comprehensive error handling and status tracking.
3

4
## Capabilities
5

6
### Sync Operations
7

8
Core methods for performing synchronization between Adapter instances.
9

10
```python { .api }
11
def sync_from(self, source: "Adapter", diff_class: Type[Diff] = Diff,
12
              flags: DiffSyncFlags = DiffSyncFlags.NONE,
13
              callback: Optional[Callable[[str, int, int], None]] = None,
14
              diff: Optional[Diff] = None) -> Diff:
15
    """
16
    Synchronize data from the given source DiffSync object into the current DiffSync object.
17
    
18
    Args:
19
        source: Object to sync data from into this one
20
        diff_class: Diff or subclass thereof to use to calculate the diffs to use for synchronization
21
        flags: Flags influencing the behavior of this sync
22
        callback: Function with parameters (stage, current, total), called at intervals as sync proceeds
23
        diff: An existing diff to be used rather than generating a completely new diff
24
    
25
    Returns:
26
        Diff between origin object and source
27
        
28
    Raises:
29
        DiffClassMismatch: The provided diff's class does not match the diff_class
30
    """
31
```
32

33
```python { .api }
34
def sync_to(self, target: "Adapter", diff_class: Type[Diff] = Diff,
35
            flags: DiffSyncFlags = DiffSyncFlags.NONE,
36
            callback: Optional[Callable[[str, int, int], None]] = None,
37
            diff: Optional[Diff] = None) -> Diff:
38
    """
39
    Synchronize data from the current DiffSync object into the given target DiffSync object.
40
    
41
    Args:
42
        target: Object to sync data into from this one
43
        diff_class: Diff or subclass thereof to use to calculate the diffs to use for synchronization
44
        flags: Flags influencing the behavior of this sync
45
        callback: Function with parameters (stage, current, total), called at intervals as sync proceeds
46
        diff: An existing diff that will be used when determining what needs to be synced
47
    
48
    Returns:
49
        Diff between origin object and target
50
        
51
    Raises:
52
        DiffClassMismatch: The provided diff's class does not match the diff_class
53
    """
54
```
55

56
#### Basic Sync Example
57

58
```python
59
from diffsync import Adapter, DiffSyncModel, DiffSyncFlags
60

61
# Create source and target adapters
62
source = SourceNetworkAdapter(name="network_source")
63
target = TargetNetworkAdapter(name="network_target")
64

65
# Load their respective data
66
source.load()
67
target.load()
68

69
# Synchronize target to match source
70
diff = target.sync_from(source)
71

72
# Check results
73
if diff.has_diffs():
74
    summary = diff.summary()
75
    print(f"Applied {summary['create']} creates, {summary['update']} updates, {summary['delete']} deletes")
76
else:
77
    print("No changes were needed")
78
```
79

80
### Sync Completion Callback
81

82
Hook for post-synchronization processing.
83

84
```python { .api }
85
def sync_complete(self, source: "Adapter", diff: Diff,
86
                  flags: DiffSyncFlags = DiffSyncFlags.NONE,
87
                  logger: Optional[structlog.BoundLogger] = None) -> None:
88
    """
89
    Callback triggered after a sync_from operation has completed and updated the model data of this instance.
90
    
91
    Note that this callback is **only** triggered if the sync actually resulted in data changes.
92
    If there are no detected changes, this callback will **not** be called.
93
    
94
    The default implementation does nothing, but a subclass could use this, for example,
95
    to perform bulk updates to a backend (such as a file) that doesn't readily support incremental updates.
96
    
97
    Args:
98
        source: The DiffSync whose data was used to update this instance
99
        diff: The Diff calculated prior to the sync operation
100
        flags: Any flags that influenced the sync
101
        logger: Logging context for the sync
102
    """
103
```
104

105
#### Sync Completion Example
106

107
```python
108
class FileBasedAdapter(Adapter):
109
    device = Device
110
    top_level = ["device"]
111
    
112
    def __init__(self, filename, **kwargs):
113
        super().__init__(**kwargs)
114
        self.filename = filename
115
    
116
    def sync_complete(self, source, diff, flags, logger):
117
        # After sync is complete, save all data to file
118
        print(f"Sync completed, saving {len(self)} objects to {self.filename}")
119
        
120
        data = self.dict()
121
        with open(self.filename, 'w') as f:
122
            json.dump(data, f, indent=2)
123
        
124
        # Log what was changed
125
        summary = diff.summary()
126
        logger.info(f"Saved changes to file", 
127
                   created=summary['create'],
128
                   updated=summary['update'], 
129
                   deleted=summary['delete'])
130
```
131

132
### Error Handling and Status Tracking
133

134
Synchronization operations include comprehensive error handling and status tracking through model status management.
135

136
#### Error Handling with Flags
137

138
```python
139
from diffsync import DiffSyncFlags
140

141
# Continue sync even if individual operations fail
142
diff = target.sync_from(source, flags=DiffSyncFlags.CONTINUE_ON_FAILURE)
143

144
# Check for any failures
145
for element in diff.get_children():
146
    if element.action:
147
        # Get the synchronized object to check its status
148
        try:
149
            obj = target.get(element.type, element.keys)
150
            status, message = obj.get_status()
151
            if status != DiffSyncStatus.SUCCESS:
152
                print(f"Failed to {element.action} {element.type} {element.name}: {message}")
153
        except ObjectNotFound:
154
            print(f"Object {element.type} {element.name} was not found after sync")
155
```
156

157
#### Custom Error Handling in Models
158

159
```python
160
class NetworkDevice(DiffSyncModel):
161
    _modelname = "device"
162
    _identifiers = ("name",)
163
    _attributes = ("ip_address", "os_version")
164
    
165
    name: str
166
    ip_address: str
167
    os_version: str
168
    
169
    @classmethod
170
    def create(cls, adapter, ids, attrs):
171
        device = super().create(adapter, ids, attrs)
172
        try:
173
            # Attempt to configure device on network
174
            configure_device(device.name, device.ip_address, device.os_version)
175
            device.set_status(DiffSyncStatus.SUCCESS, "Device configured successfully")
176
        except NetworkError as e:
177
            device.set_status(DiffSyncStatus.ERROR, f"Network configuration failed: {e}")
178
        except ValidationError as e:
179
            device.set_status(DiffSyncStatus.FAILURE, f"Invalid configuration: {e}")
180
        return device
181
    
182
    def update(self, attrs):
183
        old_values = {k: getattr(self, k) for k in attrs.keys()}
184
        device = super().update(attrs)
185
        
186
        try:
187
            # Apply changes to actual device
188
            if 'ip_address' in attrs:
189
                change_device_ip(self.name, old_values['ip_address'], self.ip_address)
190
            if 'os_version' in attrs:
191
                upgrade_device_os(self.name, self.os_version)
192
            device.set_status(DiffSyncStatus.SUCCESS, "Device updated successfully")
193
        except Exception as e:
194
            # Rollback changes
195
            for key, value in old_values.items():
196
                setattr(self, key, value)
197
            device.set_status(DiffSyncStatus.ERROR, f"Update failed, rolled back: {e}")
198
        return device
199
    
200
    def delete(self):
201
        try:
202
            # Remove device from network
203
            remove_device(self.name)
204
            device = super().delete()
205
            device.set_status(DiffSyncStatus.SUCCESS, "Device removed successfully")
206
        except Exception as e:
207
            device = super().delete()
208
            device.set_status(DiffSyncStatus.ERROR, f"Device removal failed: {e}")
209
        return device
210
```
211

212
### Advanced Sync Features
213

214
#### Progress Monitoring
215

216
```python
217
def sync_progress_callback(stage, current, total):
218
    if stage == "sync":
219
        percentage = (current / total) * 100 if total > 0 else 0
220
        print(f"Syncing: {current}/{total} elements ({percentage:.1f}%)")
221

222
# Monitor sync progress
223
diff = target.sync_from(source, callback=sync_progress_callback)
224
```
225

226
#### Using Pre-calculated Diffs
227

228
```python
229
# Calculate diff first
230
diff = target.diff_from(source)
231

232
# Review and potentially modify diff
233
print("About to apply these changes:")
234
print(diff.str())
235

236
if input("Continue? (y/n): ").lower() == 'y':
237
    # Apply the pre-calculated diff
238
    result_diff = target.sync_from(source, diff=diff)
239
else:
240
    print("Sync cancelled")
241
```
242

243
#### Selective Synchronization with Flags
244

245
```python
246
# Only create new objects, don't delete existing ones
247
diff = target.sync_from(source, flags=DiffSyncFlags.SKIP_UNMATCHED_DST)
248

249
# Only update existing objects, don't create or delete
250
diff = target.sync_from(source, flags=DiffSyncFlags.SKIP_UNMATCHED_BOTH)
251

252
# Continue even if some operations fail, and log unchanged records
253
diff = target.sync_from(source, 
254
                       flags=DiffSyncFlags.CONTINUE_ON_FAILURE | DiffSyncFlags.LOG_UNCHANGED_RECORDS)
255
```
256

257
### Bidirectional Synchronization
258

259
Example of implementing bidirectional synchronization between two data sources.
260

261
```python
262
def bidirectional_sync(adapter1, adapter2):
263
    """Perform bidirectional synchronization between two adapters."""
264
    
265
    # Sync adapter1 changes to adapter2
266
    print("Syncing adapter1 -> adapter2")
267
    diff1to2 = adapter2.sync_from(adapter1)
268
    
269
    # Sync adapter2 changes to adapter1
270
    print("Syncing adapter2 -> adapter1") 
271
    diff2to1 = adapter1.sync_from(adapter2)
272
    
273
    # Report results
274
    summary1to2 = diff1to2.summary()
275
    summary2to1 = diff2to1.summary()
276
    
277
    print(f"adapter1 -> adapter2: {summary1to2}")
278
    print(f"adapter2 -> adapter1: {summary2to1}")
279
    
280
    return diff1to2, diff2to1
281

282
# Example usage
283
source_db = DatabaseAdapter(name="source_db")
284
target_api = APIAdapter(name="target_api")
285

286
source_db.load()
287
target_api.load()
288

289
diffs = bidirectional_sync(source_db, target_api)
290
```
291

292
### Hierarchical Synchronization
293

294
DiffSync automatically handles hierarchical data structures during synchronization.
295

296
```python
297
class Site(DiffSyncModel):
298
    _modelname = "site"
299
    _identifiers = ("name",)
300
    _attributes = ("address", "contact")
301
    _children = {"device": "devices"}
302
    
303
    name: str
304
    address: str
305
    contact: str
306
    devices: List[str] = []
307

308
class Device(DiffSyncModel):
309
    _modelname = "device"
310
    _identifiers = ("site", "name")
311
    _attributes = ("model", "ip_address")
312
    _children = {"interface": "interfaces"}
313
    
314
    site: str
315
    name: str
316
    model: str
317
    ip_address: str
318
    interfaces: List[str] = []
319

320
class Interface(DiffSyncModel):
321
    _modelname = "interface"
322
    _identifiers = ("device_site", "device_name", "name")
323
    _attributes = ("description", "vlan")
324
    
325
    device_site: str
326
    device_name: str
327
    name: str
328
    description: str
329
    vlan: int
330

331
# Synchronization will automatically handle the hierarchy:
332
# 1. Sync sites first
333
# 2. Then sync devices within each site
334
# 3. Finally sync interfaces within each device
335
diff = target.sync_from(source)
336
```
337

338
## Types
339

340
```python { .api }
341
from typing import Optional, Callable, Type
342
from diffsync.diff import Diff
343
from diffsync.enum import DiffSyncFlags
344
import structlog
345

346
# Progress callback function type
347
ProgressCallback = Callable[[str, int, int], None]
348

349
# Logger type for sync completion callbacks
350
SyncLogger = structlog.BoundLogger
351
```