0
# Apache Spark KVStore
1

2
A local key/value store abstraction library for Apache Spark applications that provides automatic serialization, indexing, and key management features for storing application data with support for both in-memory and LevelDB-backed storage implementations.
3

4
## Package Information
5

6
- **Package Name**: spark-kvstore_2.12
7
- **Package Type**: maven
8
- **Language**: Java
9
- **Installation**: `org.apache.spark:spark-kvstore_2.12:3.0.1`
10

11
## Core Imports
12

13
```java
14
import org.apache.spark.util.kvstore.KVStore;
15
import org.apache.spark.util.kvstore.InMemoryStore;
16
import org.apache.spark.util.kvstore.LevelDB;
17
import org.apache.spark.util.kvstore.KVIndex;
18
import org.apache.spark.util.kvstore.KVStoreView;
19
import org.apache.spark.util.kvstore.KVStoreIterator;
20
import org.apache.spark.util.kvstore.KVStoreSerializer;
21
import org.apache.spark.util.kvstore.KVTypeInfo;
22
```
23

24
## Basic Usage
25

26
```java
27
import org.apache.spark.util.kvstore.*;
28
import java.io.File;
29

30
// Create an in-memory store
31
KVStore store = new InMemoryStore();
32

33
// Or create a persistent LevelDB store
34
KVStore levelDbStore = new LevelDB(new File("./mystore"));
35

36
// Define a data class with indexing
37
class Person {
38
    @KVIndex
39
    public String id;
40
    
41
    @KVIndex("age")
42
    public int age;
43
    
44
    @KVIndex("name")
45
    public String name;
46
    
47
    public Person(String id, String name, int age) {
48
        this.id = id;
49
        this.name = name;
50
        this.age = age;
51
    }
52
}
53

54
// Write data to the store
55
Person person = new Person("p1", "Alice", 30);
56
store.write(person);
57

58
// Read data back
59
Person retrieved = store.read(Person.class, "p1");
60

61
// Query by index
62
KVStoreView<Person> adults = store.view(Person.class).index("age").first(18);
63
for (Person adult : adults) {
64
    System.out.println(adult.name + " is " + adult.age + " years old");
65
}
66

67
// Clean up
68
store.close();
69
```
70

71
## Architecture
72

73
Apache Spark KVStore is built around several key components:
74

75
- **Store Interface**: `KVStore` provides the main abstraction for all storage operations
76
- **Storage Backends**: In-memory (`InMemoryStore`) and persistent (`LevelDB`) implementations
77
- **Indexing System**: Annotation-based automatic indexing with `@KVIndex` for efficient queries
78
- **Serialization**: JSON-based serialization with GZIP compression using Jackson
79
- **View System**: Configurable iteration and querying with `KVStoreView` and `KVStoreIterator`
80
- **Type Management**: Reflection-based type introspection and metadata caching
81

82
## Capabilities
83

84
### Core Store Operations
85

86
Basic CRUD operations for storing and retrieving objects with automatic key management and serialization.
87

88
```java { .api }
89
public interface KVStore extends Closeable {
90
    <T> T read(Class<T> klass, Object naturalKey) throws Exception;
91
    void write(Object value) throws Exception;
92
    void delete(Class<?> type, Object naturalKey) throws Exception;
93
    long count(Class<?> type) throws Exception;
94
}
95
```
96

97
[Core Operations](./core-operations.md)
98

99
### Data Querying and Views
100

101
Configurable iteration and querying system with index-based filtering, sorting, and pagination.
102

103
```java { .api }
104
public abstract class KVStoreView<T> implements Iterable<T> {
105
    public KVStoreView<T> index(String name);
106
    public KVStoreView<T> first(Object value);
107
    public KVStoreView<T> last(Object value);
108
    public KVStoreView<T> max(long max);
109
    public KVStoreView<T> skip(long n);
110
}
111
```
112

113
[Querying and Views](./querying-views.md)
114

115
### Indexing and Annotations
116

117
Automatic index creation and management using field and method annotations for efficient data access.
118

119
```java { .api }
120
@Retention(RetentionPolicy.RUNTIME)
121
@Target({ElementType.FIELD, ElementType.METHOD})
122
public @interface KVIndex {
123
    String value() default NATURAL_INDEX_NAME;
124
    String parent() default "";
125
    boolean copy() default false;
126
}
127
```
128

129
[Indexing](./indexing.md)
130

131
### Storage Implementations
132

133
In-memory and persistent storage backends with different performance and durability characteristics.
134

135
```java { .api }
136
public class InMemoryStore implements KVStore {
137
    public InMemoryStore();
138
}
139

140
public class LevelDB implements KVStore {
141
    public LevelDB(File path) throws Exception;
142
    public LevelDB(File path, KVStoreSerializer serializer) throws Exception;
143
}
144
```
145

146
[Storage Implementations](./storage-implementations.md)
147

148
### Serialization
149

150
JSON-based object serialization with compression and custom ObjectMapper support.
151

152
```java { .api }
153
public class KVStoreSerializer {
154
    public KVStoreSerializer();
155
    public byte[] serialize(Object o) throws Exception;
156
    public <T> T deserialize(byte[] data, Class<T> klass) throws Exception;
157
}
158
```
159

160
[Serialization](./serialization.md)
161

162
### Type Information and Metadata
163

164
Reflection-based type introspection for managing indexed fields and metadata about stored object types.
165

166
```java { .api }
167
public class KVTypeInfo {
168
    public KVTypeInfo(Class<?> type);
169
    public Accessor getAccessor(String indexName);
170
    public String getParentIndexName(String indexName);
171
    public Accessor getParentAccessor(String indexName);
172
}
173
```
174

175
[Type Information](./type-information.md)
176

177
## Types
178

179
```java { .api }
180
public interface KVStore extends Closeable {
181
    <T> T getMetadata(Class<T> klass) throws Exception;
182
    void setMetadata(Object value) throws Exception;
183
    <T> T read(Class<T> klass, Object naturalKey) throws Exception;
184
    void write(Object value) throws Exception;
185
    void delete(Class<?> type, Object naturalKey) throws Exception;
186
    <T> KVStoreView<T> view(Class<T> type) throws Exception;
187
    long count(Class<?> type) throws Exception;
188
    long count(Class<?> type, String index, Object indexedValue) throws Exception;
189
    <T> boolean removeAllByIndexValues(Class<T> klass, String index, Collection<?> indexValues) throws Exception;
190
}
191

192
public interface KVStoreIterator<T> extends Iterator<T>, Closeable {
193
    List<T> next(int max);
194
    boolean skip(long n);
195
}
196

197
public abstract class KVStoreView<T> implements Iterable<T> {
198
    public KVStoreView<T> reverse();
199
    public KVStoreView<T> index(String name);
200
    public KVStoreView<T> parent(Object value);
201
    public KVStoreView<T> first(Object value);
202
    public KVStoreView<T> last(Object value);
203
    public KVStoreView<T> max(long max);
204
    public KVStoreView<T> skip(long n);
205
    public KVStoreIterator<T> closeableIterator() throws Exception;
206
}
207

208
public class KVTypeInfo {
209
    public KVTypeInfo(Class<?> type);
210
    public Accessor getAccessor(String indexName);
211
    public String getParentIndexName(String indexName);
212
    public Accessor getParentAccessor(String indexName);
213
    
214
    public interface Accessor {
215
        Object get(Object object) throws ReflectiveOperationException;
216
        Class<?> getType();
217
    }
218
}
219

220
public class UnsupportedStoreVersionException extends IOException {
221
}
222
```