0
# Data Migration
1

2
The data migration system enables schema evolution by transforming serialized data from older versions to newer formats. This is essential for production systems that need to handle changes to data structures without breaking compatibility with existing persisted data.
3

4
## Core Migration Framework
5

6
### JacksonMigration
7

8
The base class for all data migrations provides the framework for version-aware transformations.
9

10
```scala { .api }
11
/**
12
 * Data migration of old formats to current format.
13
 * Used when deserializing data of older version than the currentVersion.
14
 * Implement transformation of JSON structure in the transform method.
15
 */
16
abstract class JacksonMigration {
17
  
18
  /**
19
   * Define current version used when serializing new data.
20
   * The first version, when no migration was used, is always 1.
21
   * 
22
   * @return current version number
23
   */
24
  def currentVersion: Int
25
  
26
  /**
27
   * Define the supported forward version this migration can read.
28
   * Must be greater or equal than currentVersion.
29
   * 
30
   * @return supported forward version number
31
   */
32
  def supportedForwardVersion: Int = currentVersion
33
  
34
  /**
35
   * Override if you have changed the class name. Return current class name.
36
   * 
37
   * @param fromVersion the version of the old data
38
   * @param className the old class name
39
   * @return current class name
40
   */
41
  def transformClassName(fromVersion: Int, className: String): String = className
42
  
43
  /**
44
   * Implement the transformation of old JSON structure to new JSON structure.
45
   * The JsonNode is mutable so you can add, remove fields, or change values.
46
   * Cast to specific sub-classes such as ObjectNode and ArrayNode to get access to mutators.
47
   * 
48
   * @param fromVersion the version of the old data
49
   * @param json the old JSON data
50
   * @return transformed JSON data
51
   */
52
  def transform(fromVersion: Int, json: JsonNode): JsonNode
53
}
54
```
55

56
## Basic Migration Example
57

58
### Simple Field Rename
59

60
```scala
61
import com.fasterxml.jackson.databind.JsonNode
62
import com.fasterxml.jackson.databind.node.ObjectNode
63
import akka.serialization.jackson.JacksonMigration
64

65
class UserMigration extends JacksonMigration {
66
  override def currentVersion: Int = 2
67
  
68
  override def transform(fromVersion: Int, json: JsonNode): JsonNode = {
69
    val obj = json.asInstanceOf[ObjectNode]
70
    
71
    fromVersion match {
72
      case 1 =>
73
        // Rename 'name' field to 'fullName' in version 2
74
        if (obj.has("name")) {
75
          val nameValue = obj.remove("name")
76
          obj.set("fullName", nameValue)
77
        }
78
        obj
79
      case _ => obj
80
    }
81
  }
82
}
83
```
84

85
### Class Name Change Migration
86

87
```scala
88
class OrderMigration extends JacksonMigration {
89
  override def currentVersion: Int = 2
90
  
91
  override def transformClassName(fromVersion: Int, className: String): String = {
92
    fromVersion match {
93
      case 1 if className == "com.example.PurchaseOrder" => "com.example.Order"
94
      case _ => className
95
    }
96
  }
97
  
98
  override def transform(fromVersion: Int, json: JsonNode): JsonNode = {
99
    val obj = json.asInstanceOf[ObjectNode]
100
    
101
    fromVersion match {
102
      case 1 =>
103
        // Convert old 'items' array to new 'orderItems' structure
104
        if (obj.has("items")) {
105
          val items = obj.remove("items")
106
          obj.set("orderItems", items)
107
        }
108
        obj
109
      case _ => obj
110
    }
111
  }
112
}
113
```
114

115
## Advanced Migration Patterns
116

117
### Forward Compatibility Migration
118

119
```scala
120
class EventMigration extends JacksonMigration {
121
  override def currentVersion: Int = 3
122
  override def supportedForwardVersion: Int = 4 // Can read version 4 data
123
  
124
  override def transform(fromVersion: Int, json: JsonNode): JsonNode = {
125
    val obj = json.asInstanceOf[ObjectNode]
126
    
127
    fromVersion match {
128
      case 1 =>
129
        // Migrate from version 1 to current format
130
        migrateFromV1(obj)
131
      case 2 =>
132
        // Migrate from version 2 to current format  
133
        migrateFromV2(obj)
134
      case 4 =>
135
        // Downcast from future version 4 to current version 3
136
        downcastFromV4(obj)
137
      case _ => obj
138
    }
139
  }
140
  
141
  private def migrateFromV1(obj: ObjectNode): ObjectNode = {
142
    // Add required fields that didn't exist in v1
143
    if (!obj.has("timestamp")) {
144
      obj.put("timestamp", System.currentTimeMillis())
145
    }
146
    obj
147
  }
148
  
149
  private def migrateFromV2(obj: ObjectNode): ObjectNode = {
150
    // Convert string timestamp to long
151
    if (obj.has("timestamp") && obj.get("timestamp").isTextual) {
152
      val timestampStr = obj.remove("timestamp").asText()
153
      obj.put("timestamp", timestampStr.toLong)
154
    }
155
    obj
156
  }
157
  
158
  private def downcastFromV4(obj: ObjectNode): ObjectNode = {
159
    // Remove fields that don't exist in current version
160
    obj.remove("futureFeatureFlag")
161
    obj
162
  }
163
}
164
```
165

166
### Complex Nested Structure Migration
167

168
```scala
169
import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode}
170

171
class OrderEventMigration extends JacksonMigration {
172
  override def currentVersion: Int = 3
173
  
174
  override def transform(fromVersion: Int, json: JsonNode): JsonNode = {
175
    val obj = json.asInstanceOf[ObjectNode]
176
    
177
    fromVersion match {
178
      case 1 => migrateFromV1(obj)
179
      case 2 => migrateFromV2(obj)
180
      case _ => obj
181
    }
182
  }
183
  
184
  private def migrateFromV1(obj: ObjectNode): ObjectNode = {
185
    // Migrate nested address structure
186
    if (obj.has("customerInfo")) {
187
      val customerInfo = obj.get("customerInfo").asInstanceOf[ObjectNode]
188
      
189
      // Split 'address' string into structured object
190
      if (customerInfo.has("address") && customerInfo.get("address").isTextual) {
191
        val addressStr = customerInfo.remove("address").asText()
192
        val addressParts = addressStr.split(",")
193
        
194
        val addressObj = obj.objectNode()
195
        if (addressParts.length > 0) addressObj.put("street", addressParts(0).trim)
196
        if (addressParts.length > 1) addressObj.put("city", addressParts(1).trim)
197
        if (addressParts.length > 2) addressObj.put("state", addressParts(2).trim)
198
        
199
        customerInfo.set("address", addressObj)
200
      }
201
    }
202
    
203
    obj
204
  }
205
  
206
  private def migrateFromV2(obj: ObjectNode): ObjectNode = {
207
    // Convert items array to map structure
208
    if (obj.has("items") && obj.get("items").isArray) {
209
      val items = obj.remove("items").asInstanceOf[ArrayNode]
210
      val itemsMap = obj.objectNode()
211
      
212
      items.forEach { item =>
213
        val itemObj = item.asInstanceOf[ObjectNode]
214
        if (itemObj.has("id") && itemObj.has("quantity")) {
215
          val id = itemObj.get("id").asText()
216
          itemsMap.set(id, itemObj)
217
        }
218
      }
219
      
220
      obj.set("itemsById", itemsMap)
221
    }
222
    
223
    obj
224
  }
225
}
226
```
227

228
## Configuration and Registration
229

230
### Registering Migrations
231

232
Configure migrations in your `application.conf`:
233

234
```hocon
235
akka.serialization.jackson.migrations {
236
  "com.example.User" = "com.example.UserMigration"
237
  "com.example.Order" = "com.example.OrderMigration"
238
  "com.example.OrderEvent" = "com.example.OrderEventMigration"
239
}
240
```
241

242
### Per-Binding Migration Configuration
243

244
```hocon
245
akka.serialization.jackson {
246
  # Global migrations
247
  migrations {
248
    "com.example.User" = "com.example.UserMigration"
249
  }
250
  
251
  # Binding-specific migrations
252
  jackson-json {
253
    migrations {
254
      "com.example.JsonOnlyEvent" = "com.example.JsonEventMigration"
255
    }
256
  }
257
  
258
  jackson-cbor {
259
    migrations {
260
      "com.example.CborOnlyEvent" = "com.example.CborEventMigration"
261
    }
262
  }
263
}
264
```
265

266
## Migration Process Flow
267

268
### Serialization with Version
269

270
When serializing new data:
271
1. The current version from the migration is added to the manifest
272
2. Data is serialized normally using the current class structure
273

274
### Deserialization with Migration
275

276
When deserializing data:
277
1. Version is extracted from the manifest
278
2. If version < currentVersion: Apply forward migration  
279
3. If version == currentVersion: Deserialize normally
280
4. If version <= supportedForwardVersion: Apply backward migration
281
5. If version > supportedForwardVersion: Throw exception
282

283
### Example Migration Flow
284

285
```scala
286
// Version 1 data (old format)
287
{
288
  "name": "John Doe",
289
  "age": 30
290
}
291

292
// After migration to version 2 (current format)  
293
{
294
  "fullName": "John Doe",
295
  "age": 30,
296
  "createdAt": "2023-01-01T00:00:00Z"
297
}
298
```
299

300
## Best Practices
301

302
### Migration Design
303

304
1. **Always increment versions** - Never reuse version numbers
305
2. **Test migrations thoroughly** - Include edge cases and malformed data
306
3. **Keep migrations simple** - Complex logic can introduce bugs
307
4. **Handle missing fields gracefully** - Old data may not have new required fields
308
5. **Document migration rationale** - Explain why changes were made
309

310
### Performance Considerations
311

312
1. **Minimize object creation** - Reuse JsonNode instances when possible
313
2. **Use batch migrations** - For large datasets, consider migration tools
314
3. **Monitor migration performance** - Track deserialization time for migrated data
315
4. **Cache migration instances** - Migrations are created once per class
316

317
### Backward Compatibility
318

319
1. **Support multiple versions** - Don't immediately drop support for old versions
320
2. **Use supportedForwardVersion carefully** - Test forward compatibility thoroughly
321
3. **Provide rollback plans** - Consider how to handle version downgrades
322
4. **Version your migration classes** - Migrations themselves may need updates
323

324
### Error Handling
325

326
```scala
327
override def transform(fromVersion: Int, json: JsonNode): JsonNode = {
328
  try {
329
    val obj = json.asInstanceOf[ObjectNode]
330
    fromVersion match {
331
      case 1 => migrateFromV1(obj)
332
      case _ => obj
333
    }
334
  } catch {
335
    case e: Exception =>
336
      // Log error but don't fail deserialization
337
      logger.warn(s"Migration failed for version $fromVersion", e)
338
      json // Return original data
339
  }
340
}
341
```
342

343
## Testing Migrations
344

345
### Unit Testing
346

347
```scala
348
import com.fasterxml.jackson.databind.ObjectMapper
349
import org.scalatest.flatspec.AnyFlatSpec
350

351
class UserMigrationTest extends AnyFlatSpec {
352
  val migration = new UserMigration()
353
  val mapper = new ObjectMapper()
354
  
355
  "UserMigration" should "rename name field to fullName" in {
356
    val oldJson = """{"name": "John Doe", "age": 30}"""
357
    val jsonNode = mapper.readTree(oldJson)
358
    
359
    val migrated = migration.transform(1, jsonNode)
360
    
361
    assert(!migrated.has("name"))
362
    assert(migrated.has("fullName"))
363
    assert(migrated.get("fullName").asText() == "John Doe")
364
  }
365
}
366
```
367

368
### Integration Testing
369

370
```scala
371
// Test end-to-end serialization with migration
372
val user = User("John Doe", 30) // Old format
373
val serialized = serialize(user) // With version 1
374

375
// Simulate version upgrade
376
val deserializedUser = deserialize[User](serialized) // Should apply migration
377
assert(deserializedUser.fullName == "John Doe")
378
```