Memory management utilities, specialized collections, and helper classes for efficient runtime operations including memory pools, hash sets, binary data processing, and performance optimization utilities.
Memory management utilities for efficient allocation, pooling, and manipulation of memory segments and buffers.
/**
* Memory pool for efficient segment management
* Provides lazy allocation and reuse of memory segments for high-performance operations
*/
class LazyMemorySegmentPool {
/** Create memory pool with specified capacity */
LazyMemorySegmentPool(int numberOfPages, int pageSize);
/** Get next available memory segment */
MemorySegment nextSegment();
/** Return memory segments to pool */
void returnAll(List<MemorySegment> memory);
/** Get number of available pages */
int getNumberOfAvailablePages();
/** Get page size */
int getPageSize();
/** Check if pool is destroyed */
boolean isDestroyed();
/** Destroy the pool and release resources */
void destroy();
}
/**
* External buffer for large data
* Manages spilling to disk when memory is insufficient
*/
class ResettableExternalBuffer {
ResettableExternalBuffer(
MemoryManager memManager,
IOManager ioManager,
int pageSize,
int maxMemoryPages
);
/** Add data to buffer */
void add(byte[] data) throws IOException;
/** Get iterator over buffered data */
Iterator<byte[]> iterator() throws IOException;
/** Reset buffer for reuse */
void reset();
/** Get current size */
long size();
/** Check if data spilled to disk */
boolean isSpilled();
/** Close and cleanup */
void close() throws IOException;
}
/** Memory segment utilities */
class SegmentsUtil {
/** Allocate memory segments */
static List<MemorySegment> allocatePages(MemoryManager memManager, int numPages);
/** Copy data between segments */
static void copyMemory(
MemorySegment source, int sourceOffset,
MemorySegment target, int targetOffset,
int length
);
/** Compare memory segments */
static int compareMemory(
MemorySegment seg1, int offset1,
MemorySegment seg2, int offset2,
int length
);
/** Free memory segments */
static void freePages(MemoryManager memManager, List<MemorySegment> pages);
}High-performance hash set implementations for primitive types, optimized for memory usage and access patterns.
/** Hash set for byte values */
class ByteHashSet {
ByteHashSet();
ByteHashSet(int initialCapacity);
/** Add byte value */
boolean add(byte value);
/** Check if contains value */
boolean contains(byte value);
/** Remove value */
boolean remove(byte value);
/** Get current size */
int size();
/** Check if empty */
boolean isEmpty();
/** Clear all values */
void clear();
/** Get iterator over values */
Iterator<Byte> iterator();
}
/** Hash set for integer values */
class IntHashSet {
IntHashSet();
IntHashSet(int initialCapacity);
/** Add integer value */
boolean add(int value);
/** Check if contains value */
boolean contains(int value);
/** Remove value */
boolean remove(int value);
/** Get current size */
int size();
/** Check if empty */
boolean isEmpty();
/** Clear all values */
void clear();
/** Get values as array */
int[] toArray();
}
/** Hash set for long values */
class LongHashSet {
LongHashSet();
LongHashSet(int initialCapacity);
/** Add long value */
boolean add(long value);
/** Check if contains value */
boolean contains(long value);
/** Remove value */
boolean remove(long value);
/** Get current size */
int size();
/** Get values as array */
long[] toArray();
}
/** Hash set for float values */
class FloatHashSet {
FloatHashSet();
FloatHashSet(int initialCapacity);
/** Add float value */
boolean add(float value);
/** Check if contains value */
boolean contains(float value);
/** Remove value */
boolean remove(float value);
/** Get current size */
int size();
}
/** Hash set for double values */
class DoubleHashSet {
DoubleHashSet();
DoubleHashSet(int initialCapacity);
/** Add double value */
boolean add(double value);
/** Check if contains value */
boolean contains(double value);
/** Remove value */
boolean remove(double value);
/** Get current size */
int size();
}
/** Hash set for objects */
class ObjectHashSet<T> {
ObjectHashSet();
ObjectHashSet(int initialCapacity);
/** Add object */
boolean add(T value);
/** Check if contains object */
boolean contains(T value);
/** Remove object */
boolean remove(T value);
/** Get current size */
int size();
/** Get iterator */
Iterator<T> iterator();
}LRU cache implementation for efficient caching with automatic eviction of least recently used items.
/**
* LRU cache implementation
* Automatically evicts least recently used items when capacity is exceeded
*/
class LRUMap<K, V> extends LinkedHashMap<K, V> {
/** Create LRU map with maximum capacity */
LRUMap(int maxCapacity);
/** Put key-value pair */
V put(K key, V value);
/** Get value by key */
V get(Object key);
/** Get maximum capacity */
int getMaxCapacity();
/** Check if should remove eldest entry */
protected boolean removeEldestEntry(Map.Entry<K, V> eldest);
}Utilities for stream record collection and processing, supporting efficient data flow in streaming operations.
/**
* Collector for stream records
* Efficiently collects and forwards stream records with proper timestamp handling
*/
class StreamRecordCollector<T> implements Collector<T>, Output<StreamRecord<T>> {
StreamRecordCollector(Output<StreamRecord<T>> output);
/** Collect element */
void collect(T record);
/** Collect element with timestamp */
void collect(T record, long timestamp);
/** Emit stream record */
void collect(StreamRecord<T> record);
/** Close collector */
void close();
/** Set timestamp */
void setTimestamp(long timestamp);
/** Emit watermark */
void emitWatermark(Watermark mark);
}Implementation of common algorithms used in data processing including bin packing for resource allocation.
/**
* Bin packing algorithms
* Provides efficient algorithms for packing items into bins with capacity constraints
*/
class BinPacking {
/** First fit algorithm */
static List<List<Integer>> firstFit(List<Integer> items, int binCapacity);
/** Best fit algorithm */
static List<List<Integer>> bestFit(List<Integer> items, int binCapacity);
/** First fit decreasing algorithm */
static List<List<Integer>> firstFitDecreasing(List<Integer> items, int binCapacity);
/** Best fit decreasing algorithm */
static List<List<Integer>> bestFitDecreasing(List<Integer> items, int binCapacity);
/** Calculate bin utilization */
static double calculateUtilization(List<List<Integer>> bins, int binCapacity);
}Utilities for common data processing tasks including JSON handling and UTF-8 string operations.
/** JSON processing utilities */
class JsonUtils {
/** Parse JSON string to object */
static Object parseJson(String jsonString) throws Exception;
/** Convert object to JSON string */
static String toJsonString(Object obj) throws Exception;
/** Parse JSON to specific type */
static <T> T parseJson(String jsonString, Class<T> clazz) throws Exception;
/** Convert to pretty JSON string */
static String toPrettyJsonString(Object obj) throws Exception;
/** Check if string is valid JSON */
static boolean isValidJson(String jsonString);
}
/** UTF-8 string utilities */
class StringUtf8Utils {
/** Convert string to UTF-8 bytes */
static byte[] toUtf8Bytes(String str);
/** Convert UTF-8 bytes to string */
static String fromUtf8Bytes(byte[] bytes);
/** Get UTF-8 byte length of string */
static int getUtf8ByteLength(String str);
/** Encode string with UTF-8 */
static String encodeUtf8(String str);
/** Decode UTF-8 string */
static String decodeUtf8(String encodedStr);
/** Check if string is valid UTF-8 */
static boolean isValidUtf8(byte[] bytes);
}Utilities for performance monitoring, metrics collection, and runtime statistics gathering.
/** Performance metrics collector */
class MetricsCollector {
/** Record timing metric */
void recordTiming(String name, long durationMs);
/** Record counter metric */
void recordCounter(String name, long value);
/** Record gauge metric */
void recordGauge(String name, double value);
/** Get metric value */
Optional<Number> getMetric(String name);
/** Get all metrics */
Map<String, Number> getAllMetrics();
/** Reset all metrics */
void reset();
}
/** Runtime statistics utilities */
class RuntimeStatsUtils {
/** Get memory usage statistics */
static MemoryUsageStats getMemoryUsage();
/** Get thread statistics */
static ThreadStats getThreadStats();
/** Get garbage collection statistics */
static GCStats getGCStats();
/** Get system load */
static double getSystemLoad();
/** Format statistics as string */
static String formatStats(Object stats);
}Utilities for data compression, encoding, and efficient binary data manipulation.
/** Compression utilities */
class CompressionUtils {
/** Compress data using specified algorithm */
static byte[] compress(byte[] data, CompressionType type) throws IOException;
/** Decompress data */
static byte[] decompress(byte[] compressedData, CompressionType type) throws IOException;
/** Get compression ratio */
static double getCompressionRatio(byte[] original, byte[] compressed);
/** Check if data is compressed */
static boolean isCompressed(byte[] data, CompressionType type);
/** Compression type enumeration */
enum CompressionType {
GZIP, DEFLATE, LZ4, SNAPPY
}
}
/** Encoding utilities */
class EncodingUtils {
/** Base64 encode */
static String base64Encode(byte[] data);
/** Base64 decode */
static byte[] base64Decode(String encoded);
/** URL encode */
static String urlEncode(String str);
/** URL decode */
static String urlDecode(String encoded);
/** Hex encode */
static String hexEncode(byte[] data);
/** Hex decode */
static byte[] hexDecode(String hex);
}Utilities for handling configuration, properties, and runtime parameters.
/** Configuration utilities */
class ConfigUtils {
/** Load configuration from properties file */
static Configuration loadFromFile(String filePath) throws IOException;
/** Load configuration from resource */
static Configuration loadFromResource(String resourcePath) throws IOException;
/** Merge configurations */
static Configuration merge(Configuration base, Configuration override);
/** Get configuration subset */
static Configuration subset(Configuration config, String prefix);
/** Convert to properties */
static Properties toProperties(Configuration config);
}
/** Environment utilities */
class EnvironmentUtils {
/** Get environment variable */
static Optional<String> getEnv(String name);
/** Get system property */
static Optional<String> getSystemProperty(String name);
/** Get runtime information */
static RuntimeInfo getRuntimeInfo();
/** Check if running in cluster mode */
static boolean isClusterMode();
/** Get Flink configuration */
static Configuration getFlinkConfiguration();
}Utilities for file operations, IO handling, and resource management.
/** File utilities */
class FileUtils {
/** Read file to string */
static String readFileToString(Path filePath) throws IOException;
/** Write string to file */
static void writeStringToFile(Path filePath, String content) throws IOException;
/** Copy file */
static void copyFile(Path source, Path target) throws IOException;
/** Delete directory recursively */
static void deleteDirectory(Path directory) throws IOException;
/** Create directories */
static void createDirectories(Path path) throws IOException;
/** Check if file exists */
static boolean exists(Path path);
/** Get file size */
static long getFileSize(Path path) throws IOException;
}
/** Resource utilities */
class ResourceUtils {
/** Get resource as stream */
static InputStream getResourceAsStream(String resourcePath);
/** Get resource as string */
static String getResourceAsString(String resourcePath) throws IOException;
/** Check if resource exists */
static boolean resourceExists(String resourcePath);
/** Get resource URL */
static URL getResourceURL(String resourcePath);
/** List resources in package */
static List<String> listResources(String packagePath) throws IOException;
}// Memory management
LazyMemorySegmentPool pool = new LazyMemorySegmentPool(1000, 4096);
MemorySegment segment = pool.nextSegment();
// Use segment for operations
List<MemorySegment> segments = Arrays.asList(segment);
pool.returnAll(segments);
// Specialized hash sets
IntHashSet intSet = new IntHashSet();
intSet.add(42);
intSet.add(100);
boolean contains = intSet.contains(42); // true
LongHashSet longSet = new LongHashSet(1000);
longSet.add(1000000L);
// LRU cache
LRUMap<String, Integer> cache = new LRUMap<>(100);
cache.put("key1", 42);
cache.put("key2", 100);
Integer value = cache.get("key1");
// Stream record collection
StreamRecordCollector<String> collector = new StreamRecordCollector<>(output);
collector.collect("data", System.currentTimeMillis());
// JSON utilities
String json = JsonUtils.toJsonString(myObject);
MyClass obj = JsonUtils.parseJson(json, MyClass.class);
// UTF-8 string operations
byte[] utf8Bytes = StringUtf8Utils.toUtf8Bytes("Hello, 世界!");
String restored = StringUtf8Utils.fromUtf8Bytes(utf8Bytes);
// Compression
byte[] data = "Large data to compress".getBytes();
byte[] compressed = CompressionUtils.compress(data, CompressionType.GZIP);
byte[] decompressed = CompressionUtils.decompress(compressed, CompressionType.GZIP);
// File operations
String content = FileUtils.readFileToString(Paths.get("data.txt"));
FileUtils.writeStringToFile(Paths.get("output.txt"), "Processed data");
// External buffer for large data
ResettableExternalBuffer buffer = new ResettableExternalBuffer(
memManager, ioManager, 4096, 1000);
buffer.add("large data chunk".getBytes());
Iterator<byte[]> iterator = buffer.iterator();