0
# Serialization and Persistence
1

2
Comprehensive encoding, decoding, and log processing capabilities for histogram data persistence and exchange. HdrHistogram provides multiple formats and tools for storing, transmitting, and processing histogram data.
3

4
## EncodableHistogram
5

6
Abstract base class for histograms that support encoding and decoding operations.
7

8
```java { .api }
9
public abstract class EncodableHistogram implements Serializable {
10
    
11
    // Encoding methods
12
    abstract int getNeededByteBufferCapacity();
13
    abstract int encodeIntoByteBuffer(ByteBuffer targetBuffer);
14
    abstract int encodeIntoCompressedByteBuffer(ByteBuffer targetBuffer, int compressionLevel);
15
    
16
    // Metadata methods
17
    abstract long getStartTimeStamp();
18
    abstract void setStartTimeStamp(long startTimeStamp);
19
    abstract long getEndTimeStamp();
20
    abstract void setEndTimeStamp(long endTimeStamp);
21
    abstract String getTag();
22
    abstract void setTag(String tag);
23
    abstract double getMaxValueAsDouble();
24
    
25
    // Static decoding methods
26
    static EncodableHistogram decodeFromByteBuffer(ByteBuffer buffer, 
27
                                                  long minBarForHighestTrackableValue);
28
    static EncodableHistogram decodeFromCompressedByteBuffer(ByteBuffer buffer, 
29
                                                            long minBarForHighestTrackableValue);
30
}
31
```
32

33
## Binary Encoding and Decoding
34

35
### Encoding Histograms to ByteBuffer
36

37
```java
38
// Encode histogram to binary format
39
Histogram histogram = createSampleHistogram();
40

41
// Calculate required buffer size
42
int bufferSize = histogram.getNeededByteBufferCapacity();
43
ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
44

45
// Encode histogram
46
int bytesWritten = histogram.encodeIntoByteBuffer(buffer);
47
System.out.printf("Encoded histogram in %d bytes%n", bytesWritten);
48

49
// Prepare buffer for reading
50
buffer.flip();
51

52
// The buffer now contains the encoded histogram
53
byte[] histogramData = new byte[bytesWritten];
54
buffer.get(histogramData);
55
```
56

57
### Compressed Encoding
58

59
```java
60
// Encode with compression for smaller size
61
Histogram histogram = createSampleHistogram();
62

63
int bufferSize = histogram.getNeededByteBufferCapacity();
64
ByteBuffer compressedBuffer = ByteBuffer.allocate(bufferSize);
65

66
// Compress with deflate compression (0-9 compression levels)
67
int compressedSize = histogram.encodeIntoCompressedByteBuffer(compressedBuffer, 6);
68
System.out.printf("Compressed histogram to %d bytes (level 6)%n", compressedSize);
69

70
// Compare compression ratios
71
int uncompressedSize = histogram.getNeededByteBufferCapacity();
72
double compressionRatio = (double) compressedSize / uncompressedSize;
73
System.out.printf("Compression ratio: %.2f (%.1f%% reduction)%n", 
74
    compressionRatio, (1.0 - compressionRatio) * 100);
75
```
76

77
### Decoding Histograms from ByteBuffer
78

79
```java
80
// Decode standard encoded histogram
81
ByteBuffer encodedBuffer = getEncodedHistogramBuffer();
82
Histogram decoded = Histogram.decodeFromByteBuffer(encodedBuffer, 1000);
83

84
System.out.printf("Decoded histogram: %d samples, max=%d%n",
85
    decoded.getTotalCount(), decoded.getMaxValue());
86

87
// Decode compressed histogram  
88
ByteBuffer compressedBuffer = getCompressedHistogramBuffer();
89
Histogram decompressed = Histogram.decodeFromCompressedByteBuffer(compressedBuffer, 1000);
90

91
// Verify integrity
92
if (decoded.equals(decompressed)) {
93
    System.out.println("Compression/decompression preserves data integrity");
94
}
95
```
96

97
### Polymorphic Decoding
98

99
```java
100
// Decode without knowing specific histogram type
101
ByteBuffer unknownHistogramBuffer = getHistogramBuffer();
102

103
// Returns appropriate type (Histogram or DoubleHistogram)
104
EncodableHistogram histogram = EncodableHistogram.decodeFromByteBuffer(unknownHistogramBuffer, 1000);
105

106
if (histogram instanceof Histogram) {
107
    Histogram intHistogram = (Histogram) histogram;
108
    System.out.printf("Integer histogram: P95=%d%n", 
109
        intHistogram.getValueAtPercentile(95.0));
110
} else if (histogram instanceof DoubleHistogram) {
111
    DoubleHistogram doubleHistogram = (DoubleHistogram) histogram;
112
    System.out.printf("Double histogram: P95=%.3f%n", 
113
        doubleHistogram.getValueAtPercentile(95.0));
114
}
115
```
116

117
## Base64 String Encoding
118

119
### Base64 Compression and Encoding
120

121
```java
122
// Encode histogram as Base64 compressed string
123
public String encodeHistogramToBase64(AbstractHistogram histogram) {
124
    int bufferSize = histogram.getNeededByteBufferCapacity();
125
    ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
126
    
127
    // Encode with maximum compression
128
    int compressedSize = histogram.encodeIntoCompressedByteBuffer(buffer, 9);
129
    
130
    // Extract compressed bytes
131
    buffer.flip();
132
    byte[] compressedBytes = new byte[compressedSize];
133
    buffer.get(compressedBytes);
134
    
135
    // Convert to Base64
136
    return Base64Helper.printBase64Binary(compressedBytes);
137
}
138

139
// Decode histogram from Base64 string
140
public Histogram decodeHistogramFromBase64(String base64String) {
141
    // Parse Base64 to bytes
142
    byte[] compressedBytes = Base64Helper.parseBase64Binary(base64String);
143
    
144
    // Create buffer and decode
145
    ByteBuffer buffer = ByteBuffer.wrap(compressedBytes);
146
    return Histogram.decodeFromCompressedByteBuffer(buffer, 1000);
147
}
148
```
149

150
### Convenient String Serialization
151

152
```java
153
// Using built-in string serialization
154
Histogram histogram = createSampleHistogram();
155

156
// Encode to compressed byte buffer, then Base64
157
int bufferSize = histogram.getNeededByteBufferCapacity();
158
ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
159
int compressedSize = histogram.encodeIntoCompressedByteBuffer(buffer, 9);
160

161
buffer.flip();
162
byte[] compressedBytes = new byte[compressedSize];
163
buffer.get(compressedBytes);
164
String encoded = Base64Helper.printBase64Binary(compressedBytes);
165

166
System.out.println("Encoded histogram: " + encoded);
167

168
// Direct decoding from string
169
Histogram restored = Histogram.fromString(encoded);
170

171
// Verify restoration
172
System.out.printf("Original: count=%d, P95=%d%n", 
173
    histogram.getTotalCount(), histogram.getValueAtPercentile(95.0));
174
System.out.printf("Restored: count=%d, P95=%d%n", 
175
    restored.getTotalCount(), restored.getValueAtPercentile(95.0));
176
```
177

178
## Histogram Log Processing
179

180
### HistogramLogWriter
181

182
Class for writing histograms to log files in standard format.
183

184
```java { .api }
185
public class HistogramLogWriter {
186
    
187
    // Constructors
188
    public HistogramLogWriter(String fileName) throws FileNotFoundException;
189
    public HistogramLogWriter(File outputFile) throws FileNotFoundException;
190
    public HistogramLogWriter(PrintStream outputStream);
191
    
192
    // Writing methods
193
    void outputComment(String comment);
194
    void outputLegend();
195
    void outputStartTime(long startTime);
196
    void outputBaseTime(long baseTime);
197
    void outputIntervalHistogram(EncodableHistogram histogram);
198
    
199
    // Utility methods
200
    void close();
201
}
202
```
203

204
### Writing Histogram Logs
205

206
```java
207
// Create log writer
208
try (FileOutputStream fos = new FileOutputStream("performance.hlog");
209
     PrintStream ps = new PrintStream(fos)) {
210
    
211
    HistogramLogWriter logWriter = new HistogramLogWriter(ps);
212
    
213
    // Write log header
214
    logWriter.outputComment("Performance measurement log");
215
    logWriter.outputComment("Application: MyService v1.2.3");
216
    logWriter.outputStartTime(System.currentTimeMillis());
217
    logWriter.outputLegend();
218
    
219
    // Simulate interval recording
220
    Recorder recorder = new Recorder(3);
221
    long startTime = System.currentTimeMillis();
222
    
223
    for (int interval = 0; interval < 60; interval++) {  // 60 intervals
224
        // Simulate load for 1 second
225
        simulateLoad(recorder, 1000);
226
        
227
        // Get interval histogram  
228
        Histogram intervalHist = recorder.getIntervalHistogram();
229
        
230
        // Set timestamps for this interval
231
        long endTime = System.currentTimeMillis();
232
        intervalHist.setStartTimeStamp(startTime);
233
        intervalHist.setEndTimeStamp(endTime);
234
        intervalHist.setTag("interval-" + interval);
235
        
236
        // Write to log
237
        logWriter.outputIntervalHistogram(intervalHist);
238
        
239
        startTime = endTime;
240
    }
241
    
242
    logWriter.outputComment("Log completed successfully");
243
}
244
```
245

246
### HistogramLogReader
247

248
Class for reading histogram logs and processing interval data.
249

250
```java { .api }
251
public class HistogramLogReader implements Iterable<EncodableHistogram> {
252
    
253
    // Constructors
254
    public HistogramLogReader(String fileName) throws FileNotFoundException;
255
    public HistogramLogReader(File inputFile) throws FileNotFoundException;
256
    public HistogramLogReader(InputStream inputStream);
257
    
258
    // Reading methods
259
    boolean hasNext();
260
    EncodableHistogram nextIntervalHistogram();
261
    EncodableHistogram nextIntervalHistogram(double startTimeSec, double endTimeSec);
262
    
263
    // Iterator support
264
    Iterator<EncodableHistogram> iterator();
265
    
266
    // Utility methods
267
    void close();
268
}
269
```
270

271
### Reading and Processing Histogram Logs
272

273
```java
274
// Read histogram log file
275
try (HistogramLogReader logReader = new HistogramLogReader("performance.hlog")) {
276
    
277
    System.out.println("Processing histogram log...");
278
    
279
    long totalSamples = 0;
280
    double maxP99 = 0;
281
    List<Double> p95Values = new ArrayList<>();
282
    
283
    // Process each interval histogram
284
    for (EncodableHistogram histogram : logReader) {
285
        if (histogram instanceof Histogram) {
286
            Histogram intHist = (Histogram) histogram;
287
            
288
            totalSamples += intHist.getTotalCount();
289
            long p95 = intHist.getValueAtPercentile(95.0);
290
            long p99 = intHist.getValueAtPercentile(99.0);
291
            
292
            p95Values.add((double) p95);
293
            maxP99 = Math.max(maxP99, p99);
294
            
295
            System.out.printf("Interval %s: count=%d, P95=%d, P99=%d%n",
296
                intHist.getTag(), intHist.getTotalCount(), p95, p99);
297
        }
298
    }
299
    
300
    // Generate summary statistics
301
    double avgP95 = p95Values.stream().mapToDouble(Double::doubleValue).average().orElse(0);
302
    
303
    System.out.printf("Summary:%n");
304
    System.out.printf("  Total samples: %d%n", totalSamples);
305
    System.out.printf("  Average P95: %.1f%n", avgP95);
306
    System.out.printf("  Maximum P99: %.1f%n", maxP99);
307
}
308
```
309

310
### Time-Range Filtering
311

312
```java
313
// Read histograms within specific time range
314
try (HistogramLogReader logReader = new HistogramLogReader("performance.hlog")) {
315
    
316
    // Define time range (seconds since epoch)
317
    double startTime = System.currentTimeMillis() / 1000.0 - 3600;  // Last hour
318
    double endTime = System.currentTimeMillis() / 1000.0;
319
    
320
    System.out.printf("Analyzing histograms from %.0f to %.0f%n", startTime, endTime);
321
    
322
    EncodableHistogram histogram;
323
    while ((histogram = logReader.nextIntervalHistogram(startTime, endTime)) != null) {
324
        System.out.printf("Processing histogram: %s (start=%.0f, end=%.0f)%n",
325
            histogram.getTag(),
326
            histogram.getStartTimeStamp() / 1000.0,
327
            histogram.getEndTimeStamp() / 1000.0);
328
            
329
        analyzeHistogram(histogram);
330
    }
331
}
332
```
333

334
## Advanced Log Processing
335

336
### HistogramLogProcessor
337

338
Configurable processor for advanced log analysis and transformation.
339

340
```java { .api }
341
public class HistogramLogProcessor {
342
    
343
    // Constructor
344
    public HistogramLogProcessor();
345
    
346
    // Event handler interfaces
347
    interface EventHandler {
348
        void handleHistogramInterval(EncodableHistogram histogram);
349
    }
350
    
351
    // Processing methods
352
    void processLogFile(String inputFileName, EventHandler handler);
353
    void processLogFile(File inputFile, EventHandler handler);
354
}
355
```
356

357
### Custom Log Processing
358

359
```java
360
// Custom histogram log processor
361
HistogramLogProcessor processor = new HistogramLogProcessor();
362

363
// Define custom processing logic
364
HistogramLogProcessor.EventHandler customHandler = histogram -> {
365
    if (histogram instanceof Histogram) {
366
        Histogram h = (Histogram) histogram;
367
        
368
        // Custom analysis logic
369
        long p99 = h.getValueAtPercentile(99.0);
370
        if (p99 > 10000) {  // Alert if P99 > 10ms
371
            System.err.printf("ALERT: High P99 latency %d μs in interval %s%n",
372
                p99, h.getTag());
373
        }
374
        
375
        // Custom metrics extraction
376
        extractCustomMetrics(h);
377
    }
378
};
379

380
// Process log with custom handler
381
processor.processLogFile("performance.hlog", customHandler);
382
```
383

384
### Log Aggregation and Analysis
385

386
```java
387
public class HistogramLogAnalyzer {
388
    
389
    public void analyzeLongTermTrends(String logFileName) {
390
        Map<String, List<Long>> dailyP95s = new HashMap<>();
391
        
392
        try (HistogramLogReader reader = new HistogramLogReader(logFileName)) {
393
            for (EncodableHistogram histogram : reader) {
394
                if (histogram instanceof Histogram) {
395
                    Histogram h = (Histogram) histogram;
396
                    
397
                    // Extract date from timestamp
398
                    LocalDate date = Instant.ofEpochMilli(h.getStartTimeStamp())
399
                        .atZone(ZoneId.systemDefault()).toLocalDate();
400
                    String dateKey = date.toString();
401
                    
402
                    // Collect P95 values by date
403
                    dailyP95s.computeIfAbsent(dateKey, k -> new ArrayList<>())
404
                             .add(h.getValueAtPercentile(95.0));
405
                }
406
            }
407
        }
408
        
409
        // Analyze daily trends
410
        dailyP95s.entrySet().stream()
411
                 .sorted(Map.Entry.comparingByKey())
412
                 .forEach(entry -> {
413
                     String date = entry.getKey();
414
                     List<Long> p95s = entry.getValue();
415
                     
416
                     double avgP95 = p95s.stream().mapToLong(Long::longValue).average().orElse(0);
417
                     long maxP95 = p95s.stream().mapToLong(Long::longValue).max().orElse(0);
418
                     
419
                     System.out.printf("%s: avg=%.1f μs, max=%d μs, intervals=%d%n",
420
                         date, avgP95, maxP95, p95s.size());
421
                 });
422
    }
423
    
424
    public void detectAnomalies(String logFileName, double thresholdMultiplier) {
425
        List<Double> p95Values = new ArrayList<>();
426
        
427
        // First pass: collect all P95 values
428
        try (HistogramLogReader reader = new HistogramLogReader(logFileName)) {
429
            for (EncodableHistogram histogram : reader) {
430
                if (histogram instanceof Histogram) {
431
                    Histogram h = (Histogram) histogram;
432
                    p95Values.add((double) h.getValueAtPercentile(95.0));
433
                }
434
            }
435
        }
436
        
437
        // Calculate baseline statistics
438
        double mean = p95Values.stream().mapToDouble(Double::doubleValue).average().orElse(0);
439
        double stdDev = calculateStandardDeviation(p95Values, mean);
440
        double anomalyThreshold = mean + (thresholdMultiplier * stdDev);
441
        
442
        System.out.printf("Anomaly detection: threshold=%.1f (mean=%.1f, stddev=%.1f)%n",
443
            anomalyThreshold, mean, stdDev);
444
        
445
        // Second pass: detect anomalies
446
        try (HistogramLogReader reader = new HistogramLogReader(logFileName)) {
447
            for (EncodableHistogram histogram : reader) {
448
                if (histogram instanceof Histogram) {
449
                    Histogram h = (Histogram) histogram;
450
                    long p95 = h.getValueAtPercentile(95.0);
451
                    
452
                    if (p95 > anomalyThreshold) {
453
                        System.out.printf("ANOMALY: %s P95=%d (%.1fx baseline)%n",
454
                            h.getTag(), p95, p95 / mean);
455
                    }
456
                }
457
            }
458
        }
459
    }
460
}
461
```
462

463
## Serialization Best Practices
464

465
### Metadata Management
466

467
```java
468
public void prepareHistogramForSerialization(EncodableHistogram histogram, 
469
                                           String tag, 
470
                                           long startTime, 
471
                                           long endTime) {
472
    // Set comprehensive metadata
473
    histogram.setTag(tag);
474
    histogram.setStartTimeStamp(startTime);
475
    histogram.setEndTimeStamp(endTime);
476
    
477
    System.out.printf("Prepared histogram for serialization:%n");
478
    System.out.printf("  Tag: %s%n", histogram.getTag());
479
    System.out.printf("  Time range: %d - %d (%d ms duration)%n",
480
        startTime, endTime, endTime - startTime);
481
    System.out.printf("  Max value: %.0f%n", histogram.getMaxValueAsDouble());
482
}
483
```
484

485
### Compression Level Optimization
486

487
```java
488
public void analyzeCompressionLevels(AbstractHistogram histogram) {
489
    System.out.println("Compression Level Analysis:");
490
    
491
    int uncompressedSize = histogram.getNeededByteBufferCapacity();
492
    System.out.printf("Uncompressed: %d bytes%n", uncompressedSize);
493
    
494
    for (int level = 0; level <= 9; level++) {
495
        ByteBuffer buffer = ByteBuffer.allocate(uncompressedSize);
496
        
497
        long startTime = System.nanoTime();
498
        int compressedSize = histogram.encodeIntoCompressedByteBuffer(buffer, level);
499
        long endTime = System.nanoTime();
500
        
501
        double ratio = (double) compressedSize / uncompressedSize;
502
        double timeMs = (endTime - startTime) / 1e6;
503
        
504
        System.out.printf("Level %d: %d bytes (%.2f ratio) in %.2f ms%n",
505
            level, compressedSize, ratio, timeMs);
506
    }
507
}
508
```
509

510
### Batch Processing Optimization
511

512
```java
513
public class BatchHistogramProcessor {
514
    
515
    public void processBatch(List<AbstractHistogram> histograms, String outputFile) {
516
        try (FileOutputStream fos = new FileOutputStream(outputFile);
517
             BufferedOutputStream bos = new BufferedOutputStream(fos, 65536);
518
             PrintStream ps = new PrintStream(bos)) {
519
            
520
            HistogramLogWriter writer = new HistogramLogWriter(ps);
521
            
522
            // Write batch header
523
            writer.outputComment("Batch processing: " + histograms.size() + " histograms");
524
            writer.outputLegend();
525
            
526
            // Process histograms with progress tracking
527
            for (int i = 0; i < histograms.size(); i++) {
528
                AbstractHistogram histogram = histograms.get(i);
529
                
530
                // Set batch metadata
531
                histogram.setTag("batch-" + i);
532
                
533
                writer.outputIntervalHistogram(histogram);
534
                
535
                // Progress reporting
536
                if (i % 100 == 0) {
537
                    System.out.printf("Processed %d/%d histograms%n", i, histograms.size());
538
                }
539
            }
540
            
541
            writer.outputComment("Batch processing completed");
542
            
543
        } catch (IOException e) {
544
            throw new RuntimeException("Failed to process batch", e);
545
        }
546
    }
547
}
548
```
549

550
## Memory-Efficient Serialization
551

552
### Streaming Serialization
553

554
```java
555
public class StreamingHistogramSerializer {
556
    
557
    public void serializeStream(Iterator<AbstractHistogram> histograms, 
558
                               OutputStream output) throws IOException {
559
        
560
        try (BufferedOutputStream buffered = new BufferedOutputStream(output, 32768);
561
             PrintStream ps = new PrintStream(buffered)) {
562
            
563
            HistogramLogWriter writer = new HistogramLogWriter(ps);
564
            writer.outputLegend();
565
            
566
            while (histograms.hasNext()) {
567
                AbstractHistogram histogram = histograms.next();
568
                writer.outputIntervalHistogram(histogram);
569
                
570
                // Force periodic flushing to manage memory
571
                if (Math.random() < 0.01) {  // 1% chance
572
                    ps.flush();
573
                }
574
            }
575
        }
576
    }
577
}
578
```
579

580
### Format Comparison
581

582
| Format | Size | Speed | Use Case |
583
|--------|------|-------|----------|
584
| Uncompressed Binary | Large | Fastest | Real-time systems |
585
| Compressed Binary (Level 1) | Medium | Fast | Network transmission |
586
| Compressed Binary (Level 6) | Small | Medium | Storage optimization |
587
| Compressed Binary (Level 9) | Smallest | Slow | Archive storage |
588
| Base64 Compressed | Small + Text | Medium | JSON/XML embedding |