tessl/maven-org-apache-spark--spark-kvstore

A local key/value store abstraction library for Apache Spark applications that provides automatic serialization, indexing, and key management features.

—

Pending

Quality

Pending

Does it follow best practices?

Impact

Pending

No eval scenarios have been run

Securityby

Pending

The risk profile of this skill

Overview

Eval results

Files

Apache Spark KVStore

Name: tessl/maven-org-apache-spark--spark-kvstore
Author: tessl

A local key/value store abstraction library for Apache Spark applications that provides automatic serialization, indexing, and key management features for storing application data with support for both in-memory and LevelDB-backed storage implementations.

Package Information

Package Name: spark-kvstore_2.12
Package Type: maven
Language: Java
Installation: org.apache.spark:spark-kvstore_2.12:3.0.1

Core Imports

import org.apache.spark.util.kvstore.KVStore;
import org.apache.spark.util.kvstore.InMemoryStore;
import org.apache.spark.util.kvstore.LevelDB;
import org.apache.spark.util.kvstore.KVIndex;
import org.apache.spark.util.kvstore.KVStoreView;
import org.apache.spark.util.kvstore.KVStoreIterator;
import org.apache.spark.util.kvstore.KVStoreSerializer;
import org.apache.spark.util.kvstore.KVTypeInfo;

Basic Usage

import org.apache.spark.util.kvstore.*;
import java.io.File;

// Create an in-memory store
KVStore store = new InMemoryStore();

// Or create a persistent LevelDB store
KVStore levelDbStore = new LevelDB(new File("./mystore"));

// Define a data class with indexing
class Person {
    @KVIndex
    public String id;
    
    @KVIndex("age")
    public int age;
    
    @KVIndex("name")
    public String name;
    
    public Person(String id, String name, int age) {
        this.id = id;
        this.name = name;
        this.age = age;
    }
}

// Write data to the store
Person person = new Person("p1", "Alice", 30);
store.write(person);

// Read data back
Person retrieved = store.read(Person.class, "p1");

// Query by index
KVStoreView<Person> adults = store.view(Person.class).index("age").first(18);
for (Person adult : adults) {
    System.out.println(adult.name + " is " + adult.age + " years old");
}

// Clean up
store.close();

Architecture

Apache Spark KVStore is built around several key components:

Store Interface: KVStore provides the main abstraction for all storage operations
Storage Backends: In-memory (InMemoryStore) and persistent (LevelDB) implementations
Indexing System: Annotation-based automatic indexing with @KVIndex for efficient queries
Serialization: JSON-based serialization with GZIP compression using Jackson
View System: Configurable iteration and querying with KVStoreView and KVStoreIterator
Type Management: Reflection-based type introspection and metadata caching

Capabilities

Core Store Operations

Basic CRUD operations for storing and retrieving objects with automatic key management and serialization.

public interface KVStore extends Closeable {
    <T> T read(Class<T> klass, Object naturalKey) throws Exception;
    void write(Object value) throws Exception;
    void delete(Class<?> type, Object naturalKey) throws Exception;
    long count(Class<?> type) throws Exception;
}

Core Operations

Data Querying and Views

Configurable iteration and querying system with index-based filtering, sorting, and pagination.

public abstract class KVStoreView<T> implements Iterable<T> {
    public KVStoreView<T> index(String name);
    public KVStoreView<T> first(Object value);
    public KVStoreView<T> last(Object value);
    public KVStoreView<T> max(long max);
    public KVStoreView<T> skip(long n);
}

Querying and Views

Indexing and Annotations

Automatic index creation and management using field and method annotations for efficient data access.

@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD, ElementType.METHOD})
public @interface KVIndex {
    String value() default NATURAL_INDEX_NAME;
    String parent() default "";
    boolean copy() default false;
}

Indexing

Storage Implementations

In-memory and persistent storage backends with different performance and durability characteristics.

public class InMemoryStore implements KVStore {
    public InMemoryStore();
}

public class LevelDB implements KVStore {
    public LevelDB(File path) throws Exception;
    public LevelDB(File path, KVStoreSerializer serializer) throws Exception;
}

Storage Implementations

Serialization

JSON-based object serialization with compression and custom ObjectMapper support.

public class KVStoreSerializer {
    public KVStoreSerializer();
    public byte[] serialize(Object o) throws Exception;
    public <T> T deserialize(byte[] data, Class<T> klass) throws Exception;
}

Serialization

Type Information and Metadata

Reflection-based type introspection for managing indexed fields and metadata about stored object types.

public class KVTypeInfo {
    public KVTypeInfo(Class<?> type);
    public Accessor getAccessor(String indexName);
    public String getParentIndexName(String indexName);
    public Accessor getParentAccessor(String indexName);
}

Type Information

Types

public interface KVStore extends Closeable {
    <T> T getMetadata(Class<T> klass) throws Exception;
    void setMetadata(Object value) throws Exception;
    <T> T read(Class<T> klass, Object naturalKey) throws Exception;
    void write(Object value) throws Exception;
    void delete(Class<?> type, Object naturalKey) throws Exception;
    <T> KVStoreView<T> view(Class<T> type) throws Exception;
    long count(Class<?> type) throws Exception;
    long count(Class<?> type, String index, Object indexedValue) throws Exception;
    <T> boolean removeAllByIndexValues(Class<T> klass, String index, Collection<?> indexValues) throws Exception;
}

public interface KVStoreIterator<T> extends Iterator<T>, Closeable {
    List<T> next(int max);
    boolean skip(long n);
}

public abstract class KVStoreView<T> implements Iterable<T> {
    public KVStoreView<T> reverse();
    public KVStoreView<T> index(String name);
    public KVStoreView<T> parent(Object value);
    public KVStoreView<T> first(Object value);
    public KVStoreView<T> last(Object value);
    public KVStoreView<T> max(long max);
    public KVStoreView<T> skip(long n);
    public KVStoreIterator<T> closeableIterator() throws Exception;
}

public class KVTypeInfo {
    public KVTypeInfo(Class<?> type);
    public Accessor getAccessor(String indexName);
    public String getParentIndexName(String indexName);
    public Accessor getParentAccessor(String indexName);
    
    public interface Accessor {
        Object get(Object object) throws ReflectiveOperationException;
        Class<?> getType();
    }
}

public class UnsupportedStoreVersionException extends IOException {
}