0
# Memory Management
1
2
Memory allocation and management supporting both heap and off-heap memory with object pooling for large allocations and debugging capabilities. The memory management system provides efficient allocation strategies optimized for Spark's big data processing workloads.
3
4
## Capabilities
5
6
### Memory Allocator Interface
7
8
Core interface for memory allocation with debugging support and multiple implementation strategies.
9
10
```java { .api }
11
public interface MemoryAllocator {
12
public abstract MemoryBlock allocate(long size) throws OutOfMemoryError;
13
public abstract void free(MemoryBlock memory);
14
15
public static final boolean MEMORY_DEBUG_FILL_ENABLED;
16
public static final byte MEMORY_DEBUG_FILL_CLEAN_VALUE;
17
public static final byte MEMORY_DEBUG_FILL_FREED_VALUE;
18
public static final MemoryAllocator UNSAFE;
19
public static final MemoryAllocator HEAP;
20
}
21
```
22
23
### Memory Location Representation
24
25
Base class representing a memory location with object reference and offset for unified memory addressing.
26
27
```java { .api }
28
public class MemoryLocation {
29
public MemoryLocation(Object obj, long offset);
30
public MemoryLocation();
31
public void setObjAndOffset(Object newObj, long newOffset);
32
public final Object getBaseObject();
33
public final long getBaseOffset();
34
}
35
```
36
37
### Memory Block Operations
38
39
Represents a contiguous memory block with fixed size, extending MemoryLocation with size information and utility operations.
40
41
```java { .api }
42
public class MemoryBlock extends MemoryLocation {
43
public MemoryBlock(Object obj, long offset, long length);
44
public static MemoryBlock fromLongArray(final long[] array);
45
public long size();
46
public void fill(byte value);
47
public int pageNumber;
48
49
public static final int NO_PAGE_NUMBER = -1;
50
public static final int FREED_IN_TMM_PAGE_NUMBER = -2;
51
public static final int FREED_IN_ALLOCATOR_PAGE_NUMBER = -3;
52
}
53
```
54
55
### Heap Memory Allocator
56
57
Memory allocator using JVM heap with object pooling for large allocations to improve performance and reduce garbage collection pressure.
58
59
```java { .api }
60
public class HeapMemoryAllocator implements MemoryAllocator {
61
public MemoryBlock allocate(long size) throws OutOfMemoryError;
62
public void free(MemoryBlock memory);
63
}
64
```
65
66
### Unsafe Memory Allocator
67
68
Memory allocator using off-heap memory via Unsafe for maximum performance and to avoid JVM heap limitations.
69
70
```java { .api }
71
public class UnsafeMemoryAllocator implements MemoryAllocator {
72
public MemoryBlock allocate(long size) throws OutOfMemoryError;
73
public void free(MemoryBlock memory);
74
}
75
```
76
77
## Usage Examples
78
79
### Basic Memory Allocation
80
81
```java
82
import org.apache.spark.unsafe.memory.MemoryAllocator;
83
import org.apache.spark.unsafe.memory.MemoryBlock;
84
85
// Use heap allocator for smaller allocations
86
MemoryAllocator heapAllocator = MemoryAllocator.HEAP;
87
MemoryBlock heapBlock = heapAllocator.allocate(1024);
88
89
// Use unsafe allocator for larger, high-performance allocations
90
MemoryAllocator unsafeAllocator = MemoryAllocator.UNSAFE;
91
MemoryBlock unsafeBlock = unsafeAllocator.allocate(1024 * 1024);
92
93
// Clean up
94
heapAllocator.free(heapBlock);
95
unsafeAllocator.free(unsafeBlock);
96
```
97
98
### Working with Memory Blocks
99
100
```java
101
import org.apache.spark.unsafe.memory.MemoryBlock;
102
import org.apache.spark.unsafe.Platform;
103
104
// Create memory block from long array
105
long[] data = {1L, 2L, 3L, 4L, 5L};
106
MemoryBlock block = MemoryBlock.fromLongArray(data);
107
108
// Access memory block properties
109
Object baseObject = block.getBaseObject();
110
long baseOffset = block.getBaseOffset();
111
long size = block.size();
112
113
// Fill block with specific value
114
block.fill((byte) 0x42);
115
116
// Direct memory access through Platform
117
long value = Platform.getLong(baseObject, baseOffset);
118
Platform.putLong(baseObject, baseOffset + 8, 999L);
119
```
120
121
### Memory Location Management
122
123
```java
124
import org.apache.spark.unsafe.memory.MemoryLocation;
125
126
// Create memory location
127
byte[] array = new byte[1000];
128
MemoryLocation location = new MemoryLocation(array, 0);
129
130
// Update location
131
location.setObjAndOffset(array, 100);
132
133
// Access location properties
134
Object obj = location.getBaseObject();
135
long offset = location.getBaseOffset();
136
```
137
138
### Page Number Management
139
140
```java
141
import org.apache.spark.unsafe.memory.MemoryBlock;
142
import org.apache.spark.unsafe.memory.MemoryAllocator;
143
144
MemoryBlock block = MemoryAllocator.UNSAFE.allocate(1024);
145
146
// Set page number for TaskMemoryManager integration
147
block.pageNumber = 42;
148
149
// Check page status
150
if (block.pageNumber == MemoryBlock.NO_PAGE_NUMBER) {
151
// Block is not managed by TaskMemoryManager
152
}
153
154
// Free and check status
155
MemoryAllocator.UNSAFE.free(block);
156
if (block.pageNumber == MemoryBlock.FREED_IN_ALLOCATOR_PAGE_NUMBER) {
157
// Block has been freed
158
}
159
```
160
161
### Debug Fill Operations
162
163
```java
164
import org.apache.spark.unsafe.memory.MemoryAllocator;
165
import org.apache.spark.unsafe.memory.MemoryBlock;
166
167
// When MEMORY_DEBUG_FILL_ENABLED is true, allocated memory is filled
168
// with MEMORY_DEBUG_FILL_CLEAN_VALUE and freed memory with
169
// MEMORY_DEBUG_FILL_FREED_VALUE for debugging purposes
170
171
if (MemoryAllocator.MEMORY_DEBUG_FILL_ENABLED) {
172
MemoryBlock block = MemoryAllocator.UNSAFE.allocate(1024);
173
// Block is filled with 0xa5 (MEMORY_DEBUG_FILL_CLEAN_VALUE)
174
175
MemoryAllocator.UNSAFE.free(block);
176
// Block is filled with 0x5a (MEMORY_DEBUG_FILL_FREED_VALUE)
177
}
178
```