0
# Array Operations
1

2
Optimized byte and long array operations supporting both on-heap and off-heap memory without bounds checking for maximum performance. These operations are designed for high-throughput data processing in distributed computing environments.
3

4
## Capabilities
5

6
### Byte Array Methods
7

8
Utility methods for optimized byte array operations, comparisons, and memory-efficient manipulations.
9

10
```java { .api }
11
public class ByteArrayMethods {
12
    public static long nextPowerOf2(long num);
13
    public static int roundNumberOfBytesToNearestWord(int numBytes);
14
    public static long roundNumberOfBytesToNearestWord(long numBytes);
15
    public static boolean arrayEquals(Object leftBase, long leftOffset, Object rightBase, long rightOffset, long length);
16
    public static boolean contains(byte[] arr, byte[] sub);
17
    public static boolean startsWith(byte[] array, byte[] target);
18
    public static boolean endsWith(byte[] array, byte[] target);
19
    public static boolean matchAt(byte[] arr, byte[] sub, int pos);
20
    
21
    public static final int MAX_ROUNDED_ARRAY_LENGTH;
22
}
23
```
24

25
### Long Array Operations
26

27
High-performance array of long values supporting both on-heap and off-heap memory without bounds checking.
28

29
```java { .api }
30
public final class LongArray {
31
    public LongArray(MemoryBlock memory);
32
    public MemoryBlock memoryBlock();
33
    public Object getBaseObject();
34
    public long getBaseOffset();
35
    public long size();
36
    public void zeroOut();
37
    public void set(int index, long value);
38
    public long get(int index);
39
}
40
```
41

42
### Key-Value Iterator
43

44
Abstract iterator interface for key-value pairs with I/O exception handling capabilities.
45

46
```java { .api }
47
public abstract class KVIterator<K, V> {
48
    public abstract boolean next() throws IOException;
49
    public abstract K getKey();
50
    public abstract V getValue();
51
    public abstract void close();
52
}
53
```
54

55
## Usage Examples
56

57
### Byte Array Operations
58

59
```java
60
import org.apache.spark.unsafe.array.ByteArrayMethods;
61

62
// Power of 2 calculations for memory alignment
63
long size = ByteArrayMethods.nextPowerOf2(1000); // Returns 1024
64
int rounded = ByteArrayMethods.roundNumberOfBytesToNearestWord(15); // Aligns to word boundary
65

66
// Array comparison and search operations
67
byte[] data = "Hello World".getBytes();
68
byte[] pattern = "World".getBytes();
69

70
boolean hasPattern = ByteArrayMethods.contains(data, pattern);
71
boolean startsWithHello = ByteArrayMethods.startsWith(data, "Hello".getBytes());
72
boolean endsWithWorld = ByteArrayMethods.endsWith(data, "World".getBytes());
73
boolean matchesAtPosition = ByteArrayMethods.matchAt(data, pattern, 6);
74

75
// High-performance array equality check
76
byte[] array1 = "test data".getBytes();
77
byte[] array2 = "test data".getBytes();
78
boolean equal = ByteArrayMethods.arrayEquals(
79
    array1, 0,           // Left base and offset
80
    array2, 0,           // Right base and offset  
81
    array1.length        // Length to compare
82
);
83
```
84

85
### Long Array Usage
86

87
```java
88
import org.apache.spark.unsafe.array.LongArray;
89
import org.apache.spark.unsafe.memory.MemoryAllocator;
90
import org.apache.spark.unsafe.memory.MemoryBlock;
91

92
// Create long array from allocated memory
93
MemoryAllocator allocator = MemoryAllocator.UNSAFE;
94
MemoryBlock memory = allocator.allocate(8 * 1000); // 1000 longs
95
LongArray array = new LongArray(memory);
96

97
// Array operations
98
long arraySize = array.size(); // Number of elements
99
array.zeroOut(); // Fill with zeros
100

101
// Set and get values (no bounds checking for performance)
102
for (int i = 0; i < arraySize; i++) {
103
    array.set(i, i * 2L);
104
}
105

106
long value = array.get(500); // Get value at index 500
107

108
// Access underlying memory
109
Object baseObject = array.getBaseObject();
110
long baseOffset = array.getBaseOffset();
111
MemoryBlock underlyingMemory = array.memoryBlock();
112

113
// Clean up
114
allocator.free(memory);
115
```
116

117
### Memory-Based Array Equality
118

119
```java
120
import org.apache.spark.unsafe.array.ByteArrayMethods;
121
import org.apache.spark.unsafe.Platform;
122

123
// Create two memory regions
124
long addr1 = Platform.allocateMemory(1000);
125
long addr2 = Platform.allocateMemory(1000);
126

127
// Fill with test data
128
Platform.setMemory(addr1, (byte) 0x42, 1000);
129
Platform.setMemory(addr2, (byte) 0x42, 1000);
130

131
// High-performance comparison
132
boolean equal = ByteArrayMethods.arrayEquals(
133
    null, addr1,    // Left base (null for off-heap) and offset
134
    null, addr2,    // Right base and offset
135
    1000            // Length
136
);
137

138
// Clean up
139
Platform.freeMemory(addr1);
140
Platform.freeMemory(addr2);
141
```
142

143
### KV Iterator Implementation
144

145
```java
146
import org.apache.spark.unsafe.KVIterator;
147
import java.io.IOException;
148
import java.util.List;
149

150
// Example implementation for iterating over key-value pairs
151
public class ListKVIterator extends KVIterator<String, Integer> {
152
    private final List<Map.Entry<String, Integer>> entries;
153
    private int position = -1;
154
    
155
    public ListKVIterator(List<Map.Entry<String, Integer>> entries) {
156
        this.entries = entries;
157
    }
158
    
159
    @Override
160
    public boolean next() throws IOException {
161
        position++;
162
        return position < entries.size();
163
    }
164
    
165
    @Override
166
    public String getKey() {
167
        return entries.get(position).getKey();
168
    }
169
    
170
    @Override
171
    public Integer getValue() {
172
        return entries.get(position).getValue();
173
    }
174
    
175
    @Override
176
    public void close() {
177
        // Cleanup resources if needed
178
    }
179
}
180

181
// Usage
182
KVIterator<String, Integer> iterator = new ListKVIterator(keyValuePairs);
183
try {
184
    while (iterator.next()) {
185
        String key = iterator.getKey();
186
        Integer value = iterator.getValue();
187
        // Process key-value pair
188
    }
189
} finally {
190
    iterator.close();
191
}
192
```
193

194
### Word Alignment Operations
195

196
```java
197
import org.apache.spark.unsafe.array.ByteArrayMethods;
198

199
// Ensure memory allocations are word-aligned for optimal performance
200
int requestedSize = 123;
201
int alignedSize = ByteArrayMethods.roundNumberOfBytesToNearestWord(requestedSize);
202

203
// Use aligned size for memory allocation
204
long address = Platform.allocateMemory(alignedSize);
205

206
// Check maximum safe array length
207
int maxLength = ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH;
208
if (requestedSize <= maxLength) {
209
    // Safe to allocate
210
}
211
```