0
# MessagePack Integration
1

2
Direct integration with upack for binary MessagePack serialization. MessagePack provides efficient binary serialization that is more compact than JSON while maintaining the same data model.
3

4
## Capabilities
5

6
### MessagePack AST Writing
7

8
Writes Scala values directly to upack AST types for binary serialization.
9

10
```scala { .api }
11
/**
12
 * Write the given Scala value as a MessagePack struct (upack.Msg)
13
 * @param t Value to serialize
14
 * @return upack.Msg representation
15
 */
16
def writeMsg[T: Writer](t: T): upack.Msg
17
```
18

19
**Usage Examples:**
20

21
```scala
22
import upickle.default._
23
import upack._
24

25
case class Person(name: String, age: Int)
26
val person = Person("Alice", 30)
27

28
// Convert to upack AST
29
val msgAst = writeMsg(person)
30
// Result: upack.Obj("name" -> upack.Str("Alice"), "age" -> upack.Int32(30))
31

32
// Convert to binary  
33
val binary = upack.write(msgAst)
34

35
// Convert back from binary
36
val parsed = upack.read(binary)
37

38
// Transform to JSON for inspection
39
val jsonAst = upack.transform(msgAst, ujson.Value)
40
val jsonString = ujson.write(jsonAst)
41
```
42

43
### upack.Msg Reader and Writer
44

45
Built-in support for reading and writing upack.Msg and its subtypes.
46

47
```scala { .api }
48
/**
49
 * Reader for generic upack.Msg
50
 */
51
implicit val MsgValueR: Reader[upack.Msg]
52

53
/**
54
 * Writer for generic upack.Msg
55
 */
56
implicit val MsgValueW: Writer[upack.Msg]
57
```
58

59
**Usage Examples:**
60

61
```scala
62
import upickle.default._
63
import upack._
64

65
// Create MessagePack data
66
val msgData = upack.Obj(
67
  "users" -> upack.Arr(
68
    upack.Obj("name" -> upack.Str("Alice"), "age" -> upack.Int32(30)),
69
    upack.Obj("name" -> upack.Str("Bob"), "age" -> upack.Int32(25))
70
  ),
71
  "count" -> upack.Int32(2)
72
)
73

74
// Serialize to binary using uPickle
75
val binary = writeBinary(msgData)
76

77
// Deserialize back to upack.Msg
78
val parsed = readBinary[upack.Msg](binary)
79

80
// Access values
81
val users = parsed.obj("users").arr
82
val firstUser = users(0).obj  
83
val userName = firstUser("name").str  // "Alice"
84
val userAge = firstUser("age").int32  // 30
85
```
86

87
### Binary Serialization Functions
88

89
Direct binary serialization using MessagePack format.
90

91
```scala { .api }
92
/**
93
 * Reads the given MessagePack input into a Scala value
94
 * @param s MessagePack binary input (Array[Byte], InputStream, etc.)
95
 * @param trace Enable tracing for debugging
96
 * @return Deserialized Scala value of type T
97
 */
98
def readBinary[T: Reader](s: upack.Readable, trace: Boolean = false): T
99

100
/**
101
 * Write the given Scala value as a MessagePack binary
102
 * @param t Value to serialize
103
 * @param sortKeys Whether to sort object keys alphabetically
104
 * @return MessagePack byte array
105
 */
106
def writeBinary[T: Writer](t: T, sortKeys: Boolean = false): Array[Byte]
107
```
108

109
**Usage Examples:**
110

111
```scala
112
import upickle.default._
113

114
case class Product(id: Int, name: String, price: Double, tags: List[String])
115

116
val product = Product(123, "Laptop", 999.99, List("electronics", "computers"))
117

118
// Binary serialization
119
val binary = writeBinary(product)
120
println(s"JSON size: ${write(product).getBytes.length} bytes")
121
println(s"MessagePack size: ${binary.length} bytes")
122

123
// Binary deserialization
124
val parsed = readBinary[Product](binary)
125
assert(parsed == product)
126

127
// Sorted keys for deterministic output
128
val sortedBinary = writeBinary(product, sortKeys = true)
129
```
130

131
### MessagePack Type Mapping
132

133
Understanding how Scala types map to MessagePack types.
134

135
**Usage Examples:**
136

137
```scala
138
import upickle.default._
139
import upack._
140

141
// Primitive type mappings
142
val intMsg = writeMsg(42)         // upack.Int32(42)
143
val longMsg = writeMsg(42L)       // upack.Int64(42L) 
144
val floatMsg = writeMsg(3.14f)    // upack.Float32(3.14f)
145
val doubleMsg = writeMsg(3.14)    // upack.Float64(3.14)
146
val stringMsg = writeMsg("hello") // upack.Str("hello")
147
val boolMsg = writeMsg(true)      // upack.True
148
val nullMsg = writeMsg[Option[Int]](None) // upack.Null
149

150
// Collection type mappings
151
val arrayMsg = writeMsg(Array(1, 2, 3))           // upack.Arr(Int32(1), Int32(2), Int32(3))
152
val listMsg = writeMsg(List("a", "b"))            // upack.Arr(Str("a"), Str("b"))
153
val mapMsg = writeMsg(Map("key" -> "value"))      // upack.Obj("key" -> Str("value"))
154

155
// Binary data mapping (byte arrays are stored efficiently)
156
val binaryData = Array[Byte](1, 2, 3, 4, 5)
157
val binaryMsg = writeMsg(binaryData)              // upack.Binary(Array(1, 2, 3, 4, 5))
158
```
159

160
### Format Conversion
161

162
Converting between JSON and MessagePack formats.
163

164
**Usage Examples:**
165

166
```scala
167
import upickle.default._
168
import ujson._
169
import upack._
170

171
case class Data(id: Int, values: List[Double], metadata: Map[String, String])
172
val data = Data(42, List(1.1, 2.2, 3.3), Map("version" -> "1.0", "type" -> "test"))
173

174
// JSON to MessagePack conversion
175
val jsonString = write(data)
176
val parsedData = read[Data](jsonString)
177
val msgPackBinary = writeBinary(parsedData)
178

179
// MessagePack to JSON conversion  
180
val fromBinary = readBinary[Data](msgPackBinary)
181
val backToJson = write(fromBinary)
182

183
// Direct AST conversion
184
val jsonAst = writeJs(data)         // ujson.Value
185
val msgAst = writeMsg(data)         // upack.Msg
186

187
// Transform between AST types
188
val jsonFromMsg = upack.transform(msgAst, ujson.Value)
189
val msgFromJson = ujson.transform(jsonAst, upack.Msg)
190

191
// Size comparison
192
println(s"JSON string: ${jsonString.getBytes("UTF-8").length} bytes")
193
println(s"MessagePack binary: ${msgPackBinary.length} bytes")
194
```
195

196
### Streaming MessagePack
197

198
Efficient streaming for large MessagePack data.
199

200
**Usage Examples:**
201

202
```scala
203
import upickle.default._
204
import java.io.{FileInputStream, FileOutputStream, ByteArrayOutputStream}
205

206
case class LogEntry(timestamp: Long, level: String, message: String)
207

208
// Write large dataset to MessagePack file
209
def writeLogFile(entries: Iterator[LogEntry], filename: String): Unit = {
210
  val output = new FileOutputStream(filename)
211
  try {
212
    entries.foreach { entry =>
213
      val binary = writeBinary(entry) 
214
      output.write(binary.length)  // Write length prefix
215
      output.write(binary)         // Write data
216
    }
217
  } finally {
218
    output.close()
219
  }
220
}
221

222
// Read MessagePack file
223
def readLogFile(filename: String): Iterator[LogEntry] = {
224
  val input = new FileInputStream(filename)
225
  new Iterator[LogEntry] {
226
    def hasNext: Boolean = input.available() > 0
227
    
228
    def next(): LogEntry = {
229
      val length = input.read()
230
      val buffer = new Array[Byte](length)
231
      input.read(buffer)
232
      readBinary[LogEntry](buffer)
233
    }
234
  }
235
}
236

237
// Streaming with upack directly
238
def streamLargeArray[T: Writer](items: Iterator[T]): Array[Byte] = {
239
  val output = new ByteArrayOutputStream()
240
  val writer = new upack.MsgPackWriter(output)
241
  
242
  writer.visitArray(items.size, -1)
243
  items.foreach { item =>
244
    writer.visitValue(transform(item).transform(writer), -1)
245
  }
246
  writer.visitEnd(-1)
247
  
248
  output.toByteArray
249
}
250
```
251

252
### Direct upack Operations
253

254
Core upack module functions for MessagePack processing and manipulation.
255

256
```scala { .api }
257
/**
258
 * Read MessagePack input into upack.Msg AST
259
 * @param s MessagePack binary input
260
 * @param trace Enable tracing for debugging
261
 * @return upack.Msg AST representation
262
 */
263
def upack.read(s: upack.Readable, trace: Boolean = false): upack.Msg
264

265
/**
266
 * Write upack.Msg to MessagePack binary
267
 * @param t upack.Msg to serialize
268
 * @return MessagePack byte array
269
 */
270
def upack.write(t: upack.Msg): Array[Byte]
271

272
/**
273
 * Write upack.Msg directly to OutputStream
274
 * @param t upack.Msg to serialize
275
 * @param out OutputStream to write to
276
 */
277
def upack.writeTo(t: upack.Msg, out: java.io.OutputStream): Unit
278

279
/**
280
 * Write upack.Msg to byte array
281
 * @param t upack.Msg to serialize  
282
 * @return MessagePack byte array
283
 */
284
def upack.writeToByteArray(t: upack.Msg): Array[Byte]
285
```
286

287
**Usage Examples:**
288

289
```scala
290
import upack._
291

292
// Create MessagePack AST directly
293
val msgData = upack.Obj(
294
  "user" -> upack.Str("Alice"),
295
  "score" -> upack.Int32(85),
296
  "active" -> upack.True,
297
  "metadata" -> upack.Obj(
298
    "version" -> upack.Float64(1.2),
299
    "tags" -> upack.Arr(upack.Str("premium"), upack.Str("verified"))
300
  )
301
)
302

303
// Write to binary
304
val binary = upack.write(msgData)
305
println(s"MessagePack size: ${binary.length} bytes")
306

307
// Read back from binary
308
val parsed = upack.read(binary)
309

310
// Access values
311
val userName = parsed.obj("user").str                    // "Alice"
312
val userScore = parsed.obj("score").int32               // 85
313
val isActive = parsed.obj("active") == upack.True       // true
314
val version = parsed.obj("metadata").obj("version").float64 // 1.2
315

316
// Write to file
317
val outputStream = new java.io.FileOutputStream("data.msgpack")
318
try {
319
  upack.writeTo(msgData, outputStream)
320
} finally {
321
  outputStream.close()
322
}
323

324
// Read from file
325
val inputStream = new java.io.FileInputStream("data.msgpack")
326
val fileData = try {
327
  upack.read(inputStream)
328
} finally {
329
  inputStream.close()
330
}
331
```
332

333
### MessagePack Validation
334

335
Validate MessagePack input without parsing to check format correctness.
336

337
```scala { .api }
338
/**
339
 * Validate MessagePack format without full parsing
340
 * @param s MessagePack input to validate
341
 * @throws upack.UpackException if invalid MessagePack
342
 */
343
def upack.validate(s: upack.Readable): Unit
344
```
345

346
**Usage Examples:**
347

348
```scala
349
import upack._
350

351
// Validate MessagePack byte arrays
352
try {
353
  val binaryData = Array[Byte](-108, 4, 85, 115, 101, 114, -91, 65, 108, 105, 99, 101)
354
  upack.validate(binaryData)
355
  println("Valid MessagePack")
356
} catch {
357
  case e: upack.UpackException => println(s"Invalid MessagePack: ${e.getMessage}")
358
}
359

360
// Validate before expensive parsing
361
def safeParseMessagePack(input: Array[Byte]): Option[upack.Msg] = {
362
  try {
363
    upack.validate(input)
364
    Some(upack.read(input))
365
  } catch {
366
    case _: upack.UpackException => None
367
  }
368
}
369

370
// Validate large files
371
def validateMessagePackFile(filename: String): Boolean = {
372
  val inputStream = new java.io.FileInputStream(filename)
373
  try {
374
    upack.validate(inputStream)
375
    true
376
  } catch {
377
    case _: upack.UpackException => false
378
  } finally {
379
    inputStream.close()
380
  }
381
}
382
```
383

384
### Low-level MessagePack Processing
385

386
Advanced functions for custom MessagePack processing workflows.
387

388
```scala { .api }
389
/**
390
 * Transform MessagePack using custom visitor
391
 * @param t MessagePack input
392
 * @param v Custom visitor for processing
393
 * @return Processed result
394
 */
395
def upack.transform[T](t: upack.Readable, v: upickle.core.Visitor[_, T]): T
396

397
/**
398
 * Copy upack.Msg creating a deep clone
399
 * @param t upack.Msg to copy
400
 * @return Deep copy of the input
401
 */
402
def upack.copy(t: upack.Msg): upack.Msg
403
```
404

405
**Usage Examples:**
406

407
```scala
408
import upack._
409
import upickle.core._
410

411
// Custom visitor to analyze MessagePack structure
412
class MessagePackAnalyzer extends Visitor[Any, Map[String, Int]] {
413
  private var typeCounts = Map.empty[String, Int]
414
  
415
  private def increment(typeName: String): Unit = {
416
    typeCounts = typeCounts.updated(typeName, typeCounts.getOrElse(typeName, 0) + 1)
417
  }
418
  
419
  override def visitArray(length: Int, index: Int) = new ArrVisitor[Any, Map[String, Int]] {
420
    def subVisitor = MessagePackAnalyzer.this
421
    def visitValue(v: Any, index: Int): Unit = increment("array_element")
422
    def visitEnd(index: Int) = { increment("array"); typeCounts }
423
  }
424
  
425
  override def visitObject(length: Int, jsonableKeys: Boolean, index: Int) = new ObjVisitor[Any, Map[String, Int]] {
426
    def subVisitor = MessagePackAnalyzer.this
427
    def visitKey(index: Int) = MessagePackAnalyzer.this
428
    def visitKeyValue(s: Any): Unit = increment("object_key")
429
    def visitValue(v: Any, index: Int): Unit = increment("object_value")
430
    def visitEnd(index: Int) = { increment("object"); typeCounts }
431
  }
432
  
433
  override def visitString(s: CharSequence, index: Int) = { increment("string"); typeCounts }
434
  override def visitInt32(i: Int, index: Int) = { increment("int32"); typeCounts }
435
  override def visitInt64(i: Long, index: Int) = { increment("int64"); typeCounts }
436
  override def visitFloat64(d: Double, index: Int) = { increment("float64"); typeCounts }
437
  override def visitBool(b: Boolean, index: Int) = { increment("boolean"); typeCounts }
438
  override def visitNull(index: Int) = { increment("null"); typeCounts }
439
}
440

441
// Analyze MessagePack structure
442
val binaryData = upack.write(upack.Obj(
443
  "numbers" -> upack.Arr(upack.Int32(1), upack.Int64(2L), upack.Float64(3.14)),
444
  "text" -> upack.Str("hello"),
445
  "flag" -> upack.True
446
))
447

448
val analysis = upack.transform(binaryData, new MessagePackAnalyzer())
449
println("MessagePack structure analysis:")
450
analysis.foreach { case (typeName, count) =>
451
  println(s"  $typeName: $count")
452
}
453

454
// Deep copy MessagePack values
455
val original = upack.Obj("data" -> upack.Arr(upack.Int32(1), upack.Int32(2)))
456
val copied = upack.copy(original)
457

458
// Modify copy without affecting original
459
copied.obj("data").arr :+ upack.Int32(3)
460
```
461

462
### MessagePack Extensions
463

464
Working with MessagePack extension types for custom data.
465

466
**Usage Examples:**
467

468
```scala
469
import upickle.default._
470
import upack._
471

472
// Custom extension type for timestamps
473
case class Timestamp(epochMillis: Long)
474

475
implicit val timestampRW: ReadWriter[Timestamp] = readwriter[Long].bimap(
476
  (ts: Timestamp) => ts.epochMillis,
477
  (millis: Long) => Timestamp(millis)
478
)
479

480
// Serialize with extension type
481
val timestamp = Timestamp(System.currentTimeMillis())
482
val binary = writeBinary(timestamp)
483

484
// Custom binary data handling
485
case class BinaryData(contentType: String, data: Array[Byte])
486

487
implicit val binaryDataRW: ReadWriter[BinaryData] = macroRW
488

489
val pdfData = BinaryData("application/pdf", loadPdfBytes())
490
val binaryPdf = writeBinary(pdfData)
491
val parsedPdf = readBinary[BinaryData](binaryPdf)
492

493
// Efficient binary arrays
494
val imageBytes: Array[Byte] = loadImageBytes()
495
val msgWithImage = writeMsg(Map(
496
  "filename" -> "photo.jpg",
497
  "size" -> imageBytes.length,
498
  "data" -> imageBytes  // Stored efficiently as binary
499
))
500

501
// Working with MessagePack extension types directly
502
val extData = upack.Ext(42.toByte, Array[Byte](1, 2, 3, 4))
503
val extBinary = upack.write(extData)
504
val parsedExt = upack.read(extBinary).asInstanceOf[upack.Ext]
505
println(s"Extension type: ${parsedExt.tag}, data: ${parsedExt.data.mkString(",")}")
506
```
507

508
## Types
509

510
```scala { .api }
511
/**
512
 * upack.Msg is the base type for all MessagePack AST nodes
513
 */
514
sealed trait upack.Msg
515

516
/**
517
 * Specific upack message types
518
 */
519
case class upack.Obj(value: Map[upack.Msg, upack.Msg]) extends upack.Msg
520
case class upack.Arr(value: Seq[upack.Msg]) extends upack.Msg
521
case class upack.Str(value: String) extends upack.Msg
522
case class upack.Int32(value: Int) extends upack.Msg
523
case class upack.Int64(value: Long) extends upack.Msg
524
case class upack.UInt64(value: Long) extends upack.Msg
525
case class upack.Float32(value: Float) extends upack.Msg
526
case class upack.Float64(value: Double) extends upack.Msg
527
case class upack.Binary(value: Array[Byte]) extends upack.Msg
528
case object upack.True extends upack.Msg
529
case object upack.False extends upack.Msg
530
case object upack.Null extends upack.Msg
531

532
/**
533
 * upack.Readable represents sources that can provide MessagePack data
534
 */
535
trait upack.Readable {
536
  def transform[T](f: upack.Visitor[_, T]): T
537
}
538

539
/**
540
 * Writer for MessagePack binary output
541
 */
542
class upack.MsgPackWriter(out: java.io.OutputStream) extends upack.Visitor[Any, Unit]
543
```