0
# Memory and Utilities
1
2
Memory management utilities, specialized collections, and helper classes for efficient runtime operations including memory pools, hash sets, binary data processing, and performance optimization utilities.
3
4
## Capabilities
5
6
### Memory Management
7
8
Memory management utilities for efficient allocation, pooling, and manipulation of memory segments and buffers.
9
10
```java { .api }
11
/**
12
* Memory pool for efficient segment management
13
* Provides lazy allocation and reuse of memory segments for high-performance operations
14
*/
15
class LazyMemorySegmentPool {
16
/** Create memory pool with specified capacity */
17
LazyMemorySegmentPool(int numberOfPages, int pageSize);
18
19
/** Get next available memory segment */
20
MemorySegment nextSegment();
21
22
/** Return memory segments to pool */
23
void returnAll(List<MemorySegment> memory);
24
25
/** Get number of available pages */
26
int getNumberOfAvailablePages();
27
28
/** Get page size */
29
int getPageSize();
30
31
/** Check if pool is destroyed */
32
boolean isDestroyed();
33
34
/** Destroy the pool and release resources */
35
void destroy();
36
}
37
38
/**
39
* External buffer for large data
40
* Manages spilling to disk when memory is insufficient
41
*/
42
class ResettableExternalBuffer {
43
ResettableExternalBuffer(
44
MemoryManager memManager,
45
IOManager ioManager,
46
int pageSize,
47
int maxMemoryPages
48
);
49
50
/** Add data to buffer */
51
void add(byte[] data) throws IOException;
52
53
/** Get iterator over buffered data */
54
Iterator<byte[]> iterator() throws IOException;
55
56
/** Reset buffer for reuse */
57
void reset();
58
59
/** Get current size */
60
long size();
61
62
/** Check if data spilled to disk */
63
boolean isSpilled();
64
65
/** Close and cleanup */
66
void close() throws IOException;
67
}
68
69
/** Memory segment utilities */
70
class SegmentsUtil {
71
/** Allocate memory segments */
72
static List<MemorySegment> allocatePages(MemoryManager memManager, int numPages);
73
74
/** Copy data between segments */
75
static void copyMemory(
76
MemorySegment source, int sourceOffset,
77
MemorySegment target, int targetOffset,
78
int length
79
);
80
81
/** Compare memory segments */
82
static int compareMemory(
83
MemorySegment seg1, int offset1,
84
MemorySegment seg2, int offset2,
85
int length
86
);
87
88
/** Free memory segments */
89
static void freePages(MemoryManager memManager, List<MemorySegment> pages);
90
}
91
```
92
93
### Specialized Hash Collections
94
95
High-performance hash set implementations for primitive types, optimized for memory usage and access patterns.
96
97
```java { .api }
98
/** Hash set for byte values */
99
class ByteHashSet {
100
ByteHashSet();
101
ByteHashSet(int initialCapacity);
102
103
/** Add byte value */
104
boolean add(byte value);
105
106
/** Check if contains value */
107
boolean contains(byte value);
108
109
/** Remove value */
110
boolean remove(byte value);
111
112
/** Get current size */
113
int size();
114
115
/** Check if empty */
116
boolean isEmpty();
117
118
/** Clear all values */
119
void clear();
120
121
/** Get iterator over values */
122
Iterator<Byte> iterator();
123
}
124
125
/** Hash set for integer values */
126
class IntHashSet {
127
IntHashSet();
128
IntHashSet(int initialCapacity);
129
130
/** Add integer value */
131
boolean add(int value);
132
133
/** Check if contains value */
134
boolean contains(int value);
135
136
/** Remove value */
137
boolean remove(int value);
138
139
/** Get current size */
140
int size();
141
142
/** Check if empty */
143
boolean isEmpty();
144
145
/** Clear all values */
146
void clear();
147
148
/** Get values as array */
149
int[] toArray();
150
}
151
152
/** Hash set for long values */
153
class LongHashSet {
154
LongHashSet();
155
LongHashSet(int initialCapacity);
156
157
/** Add long value */
158
boolean add(long value);
159
160
/** Check if contains value */
161
boolean contains(long value);
162
163
/** Remove value */
164
boolean remove(long value);
165
166
/** Get current size */
167
int size();
168
169
/** Get values as array */
170
long[] toArray();
171
}
172
173
/** Hash set for float values */
174
class FloatHashSet {
175
FloatHashSet();
176
FloatHashSet(int initialCapacity);
177
178
/** Add float value */
179
boolean add(float value);
180
181
/** Check if contains value */
182
boolean contains(float value);
183
184
/** Remove value */
185
boolean remove(float value);
186
187
/** Get current size */
188
int size();
189
}
190
191
/** Hash set for double values */
192
class DoubleHashSet {
193
DoubleHashSet();
194
DoubleHashSet(int initialCapacity);
195
196
/** Add double value */
197
boolean add(double value);
198
199
/** Check if contains value */
200
boolean contains(double value);
201
202
/** Remove value */
203
boolean remove(double value);
204
205
/** Get current size */
206
int size();
207
}
208
209
/** Hash set for objects */
210
class ObjectHashSet<T> {
211
ObjectHashSet();
212
ObjectHashSet(int initialCapacity);
213
214
/** Add object */
215
boolean add(T value);
216
217
/** Check if contains object */
218
boolean contains(T value);
219
220
/** Remove object */
221
boolean remove(T value);
222
223
/** Get current size */
224
int size();
225
226
/** Get iterator */
227
Iterator<T> iterator();
228
}
229
```
230
231
### Cache Implementation
232
233
LRU cache implementation for efficient caching with automatic eviction of least recently used items.
234
235
```java { .api }
236
/**
237
* LRU cache implementation
238
* Automatically evicts least recently used items when capacity is exceeded
239
*/
240
class LRUMap<K, V> extends LinkedHashMap<K, V> {
241
/** Create LRU map with maximum capacity */
242
LRUMap(int maxCapacity);
243
244
/** Put key-value pair */
245
V put(K key, V value);
246
247
/** Get value by key */
248
V get(Object key);
249
250
/** Get maximum capacity */
251
int getMaxCapacity();
252
253
/** Check if should remove eldest entry */
254
protected boolean removeEldestEntry(Map.Entry<K, V> eldest);
255
}
256
```
257
258
### Stream Processing Utilities
259
260
Utilities for stream record collection and processing, supporting efficient data flow in streaming operations.
261
262
```java { .api }
263
/**
264
* Collector for stream records
265
* Efficiently collects and forwards stream records with proper timestamp handling
266
*/
267
class StreamRecordCollector<T> implements Collector<T>, Output<StreamRecord<T>> {
268
StreamRecordCollector(Output<StreamRecord<T>> output);
269
270
/** Collect element */
271
void collect(T record);
272
273
/** Collect element with timestamp */
274
void collect(T record, long timestamp);
275
276
/** Emit stream record */
277
void collect(StreamRecord<T> record);
278
279
/** Close collector */
280
void close();
281
282
/** Set timestamp */
283
void setTimestamp(long timestamp);
284
285
/** Emit watermark */
286
void emitWatermark(Watermark mark);
287
}
288
```
289
290
### Algorithm Utilities
291
292
Implementation of common algorithms used in data processing including bin packing for resource allocation.
293
294
```java { .api }
295
/**
296
* Bin packing algorithms
297
* Provides efficient algorithms for packing items into bins with capacity constraints
298
*/
299
class BinPacking {
300
/** First fit algorithm */
301
static List<List<Integer>> firstFit(List<Integer> items, int binCapacity);
302
303
/** Best fit algorithm */
304
static List<List<Integer>> bestFit(List<Integer> items, int binCapacity);
305
306
/** First fit decreasing algorithm */
307
static List<List<Integer>> firstFitDecreasing(List<Integer> items, int binCapacity);
308
309
/** Best fit decreasing algorithm */
310
static List<List<Integer>> bestFitDecreasing(List<Integer> items, int binCapacity);
311
312
/** Calculate bin utilization */
313
static double calculateUtilization(List<List<Integer>> bins, int binCapacity);
314
}
315
```
316
317
### Data Processing Utilities
318
319
Utilities for common data processing tasks including JSON handling and UTF-8 string operations.
320
321
```java { .api }
322
/** JSON processing utilities */
323
class JsonUtils {
324
/** Parse JSON string to object */
325
static Object parseJson(String jsonString) throws Exception;
326
327
/** Convert object to JSON string */
328
static String toJsonString(Object obj) throws Exception;
329
330
/** Parse JSON to specific type */
331
static <T> T parseJson(String jsonString, Class<T> clazz) throws Exception;
332
333
/** Convert to pretty JSON string */
334
static String toPrettyJsonString(Object obj) throws Exception;
335
336
/** Check if string is valid JSON */
337
static boolean isValidJson(String jsonString);
338
}
339
340
/** UTF-8 string utilities */
341
class StringUtf8Utils {
342
/** Convert string to UTF-8 bytes */
343
static byte[] toUtf8Bytes(String str);
344
345
/** Convert UTF-8 bytes to string */
346
static String fromUtf8Bytes(byte[] bytes);
347
348
/** Get UTF-8 byte length of string */
349
static int getUtf8ByteLength(String str);
350
351
/** Encode string with UTF-8 */
352
static String encodeUtf8(String str);
353
354
/** Decode UTF-8 string */
355
static String decodeUtf8(String encodedStr);
356
357
/** Check if string is valid UTF-8 */
358
static boolean isValidUtf8(byte[] bytes);
359
}
360
```
361
362
### Performance Monitoring Utilities
363
364
Utilities for performance monitoring, metrics collection, and runtime statistics gathering.
365
366
```java { .api }
367
/** Performance metrics collector */
368
class MetricsCollector {
369
/** Record timing metric */
370
void recordTiming(String name, long durationMs);
371
372
/** Record counter metric */
373
void recordCounter(String name, long value);
374
375
/** Record gauge metric */
376
void recordGauge(String name, double value);
377
378
/** Get metric value */
379
Optional<Number> getMetric(String name);
380
381
/** Get all metrics */
382
Map<String, Number> getAllMetrics();
383
384
/** Reset all metrics */
385
void reset();
386
}
387
388
/** Runtime statistics utilities */
389
class RuntimeStatsUtils {
390
/** Get memory usage statistics */
391
static MemoryUsageStats getMemoryUsage();
392
393
/** Get thread statistics */
394
static ThreadStats getThreadStats();
395
396
/** Get garbage collection statistics */
397
static GCStats getGCStats();
398
399
/** Get system load */
400
static double getSystemLoad();
401
402
/** Format statistics as string */
403
static String formatStats(Object stats);
404
}
405
```
406
407
### Compression and Encoding Utilities
408
409
Utilities for data compression, encoding, and efficient binary data manipulation.
410
411
```java { .api }
412
/** Compression utilities */
413
class CompressionUtils {
414
/** Compress data using specified algorithm */
415
static byte[] compress(byte[] data, CompressionType type) throws IOException;
416
417
/** Decompress data */
418
static byte[] decompress(byte[] compressedData, CompressionType type) throws IOException;
419
420
/** Get compression ratio */
421
static double getCompressionRatio(byte[] original, byte[] compressed);
422
423
/** Check if data is compressed */
424
static boolean isCompressed(byte[] data, CompressionType type);
425
426
/** Compression type enumeration */
427
enum CompressionType {
428
GZIP, DEFLATE, LZ4, SNAPPY
429
}
430
}
431
432
/** Encoding utilities */
433
class EncodingUtils {
434
/** Base64 encode */
435
static String base64Encode(byte[] data);
436
437
/** Base64 decode */
438
static byte[] base64Decode(String encoded);
439
440
/** URL encode */
441
static String urlEncode(String str);
442
443
/** URL decode */
444
static String urlDecode(String encoded);
445
446
/** Hex encode */
447
static String hexEncode(byte[] data);
448
449
/** Hex decode */
450
static byte[] hexDecode(String hex);
451
}
452
```
453
454
### Configuration Utilities
455
456
Utilities for handling configuration, properties, and runtime parameters.
457
458
```java { .api }
459
/** Configuration utilities */
460
class ConfigUtils {
461
/** Load configuration from properties file */
462
static Configuration loadFromFile(String filePath) throws IOException;
463
464
/** Load configuration from resource */
465
static Configuration loadFromResource(String resourcePath) throws IOException;
466
467
/** Merge configurations */
468
static Configuration merge(Configuration base, Configuration override);
469
470
/** Get configuration subset */
471
static Configuration subset(Configuration config, String prefix);
472
473
/** Convert to properties */
474
static Properties toProperties(Configuration config);
475
}
476
477
/** Environment utilities */
478
class EnvironmentUtils {
479
/** Get environment variable */
480
static Optional<String> getEnv(String name);
481
482
/** Get system property */
483
static Optional<String> getSystemProperty(String name);
484
485
/** Get runtime information */
486
static RuntimeInfo getRuntimeInfo();
487
488
/** Check if running in cluster mode */
489
static boolean isClusterMode();
490
491
/** Get Flink configuration */
492
static Configuration getFlinkConfiguration();
493
}
494
```
495
496
### File and IO Utilities
497
498
Utilities for file operations, IO handling, and resource management.
499
500
```java { .api }
501
/** File utilities */
502
class FileUtils {
503
/** Read file to string */
504
static String readFileToString(Path filePath) throws IOException;
505
506
/** Write string to file */
507
static void writeStringToFile(Path filePath, String content) throws IOException;
508
509
/** Copy file */
510
static void copyFile(Path source, Path target) throws IOException;
511
512
/** Delete directory recursively */
513
static void deleteDirectory(Path directory) throws IOException;
514
515
/** Create directories */
516
static void createDirectories(Path path) throws IOException;
517
518
/** Check if file exists */
519
static boolean exists(Path path);
520
521
/** Get file size */
522
static long getFileSize(Path path) throws IOException;
523
}
524
525
/** Resource utilities */
526
class ResourceUtils {
527
/** Get resource as stream */
528
static InputStream getResourceAsStream(String resourcePath);
529
530
/** Get resource as string */
531
static String getResourceAsString(String resourcePath) throws IOException;
532
533
/** Check if resource exists */
534
static boolean resourceExists(String resourcePath);
535
536
/** Get resource URL */
537
static URL getResourceURL(String resourcePath);
538
539
/** List resources in package */
540
static List<String> listResources(String packagePath) throws IOException;
541
}
542
```
543
544
## Usage Examples
545
546
```java
547
// Memory management
548
LazyMemorySegmentPool pool = new LazyMemorySegmentPool(1000, 4096);
549
MemorySegment segment = pool.nextSegment();
550
// Use segment for operations
551
List<MemorySegment> segments = Arrays.asList(segment);
552
pool.returnAll(segments);
553
554
// Specialized hash sets
555
IntHashSet intSet = new IntHashSet();
556
intSet.add(42);
557
intSet.add(100);
558
boolean contains = intSet.contains(42); // true
559
560
LongHashSet longSet = new LongHashSet(1000);
561
longSet.add(1000000L);
562
563
// LRU cache
564
LRUMap<String, Integer> cache = new LRUMap<>(100);
565
cache.put("key1", 42);
566
cache.put("key2", 100);
567
Integer value = cache.get("key1");
568
569
// Stream record collection
570
StreamRecordCollector<String> collector = new StreamRecordCollector<>(output);
571
collector.collect("data", System.currentTimeMillis());
572
573
// JSON utilities
574
String json = JsonUtils.toJsonString(myObject);
575
MyClass obj = JsonUtils.parseJson(json, MyClass.class);
576
577
// UTF-8 string operations
578
byte[] utf8Bytes = StringUtf8Utils.toUtf8Bytes("Hello, 世界!");
579
String restored = StringUtf8Utils.fromUtf8Bytes(utf8Bytes);
580
581
// Compression
582
byte[] data = "Large data to compress".getBytes();
583
byte[] compressed = CompressionUtils.compress(data, CompressionType.GZIP);
584
byte[] decompressed = CompressionUtils.decompress(compressed, CompressionType.GZIP);
585
586
// File operations
587
String content = FileUtils.readFileToString(Paths.get("data.txt"));
588
FileUtils.writeStringToFile(Paths.get("output.txt"), "Processed data");
589
590
// External buffer for large data
591
ResettableExternalBuffer buffer = new ResettableExternalBuffer(
592
memManager, ioManager, 4096, 1000);
593
buffer.add("large data chunk".getBytes());
594
Iterator<byte[]> iterator = buffer.iterator();
595
```