or run

npx @tessl/cli init

Version

Tile

Overview

Evals

Files

docs

byte-utilities.md data-format-system.md index.md schema-system.md stream-processing.md structured-records.md

tile.json

tessl/maven-co-cask-cdap--cdap-api-common

Core API classes and utilities for CDAP application development, providing common data schema definitions, data format abstractions, stream event handling, and byte manipulation utilities

Workspace: tessl
Visibility: Public
Created: 3 months ago
Last updated: 3 months ago
Describes: pkg:maven/co.cask.cdap/cdap-api-common@5.1.x

To install, run

npx @tessl/cli install tessl/maven-co-cask-cdap--cdap-api-common@5.1.0

CDAP API Common

Core API classes and utilities for CDAP (Cask Data Application Platform) application development. Provides essential building blocks including data schema definitions, data format abstractions, stream event handling, and comprehensive byte manipulation utilities for big data processing scenarios.

Package Information

Package Name: co.cask.cdap:cdap-api-common
Package Type: Maven
Language: Java
Version: 5.1.2
Status: Experimental (marked with @Beta annotation)
Installation: Add to Maven dependencies:

<dependency>
    <groupId>co.cask.cdap</groupId>
    <artifactId>cdap-api-common</artifactId>
    <version>5.1.2</version>
</dependency>

Core Imports

import co.cask.cdap.api.data.schema.Schema;
import co.cask.cdap.api.data.format.StructuredRecord;
import co.cask.cdap.api.common.Bytes;
import co.cask.cdap.api.flow.flowlet.StreamEvent;
import co.cask.cdap.api.data.format.RecordFormat;
import co.cask.cdap.api.stream.StreamEventDecoder;
import co.cask.cdap.api.macro.InvalidMacroException;

Basic Usage

import co.cask.cdap.api.data.schema.Schema;
import co.cask.cdap.api.data.format.StructuredRecord;
import co.cask.cdap.api.common.Bytes;
import java.time.LocalDate;

// Define a schema for structured data
Schema personSchema = Schema.recordOf("Person",
    Schema.Field.of("name", Schema.of(Schema.Type.STRING)),
    Schema.Field.of("age", Schema.of(Schema.Type.INT)),
    Schema.Field.of("birthDate", Schema.of(Schema.LogicalType.DATE))
);

// Create structured records
StructuredRecord person = StructuredRecord.builder(personSchema)
    .set("name", "John Doe")
    .set("age", 30)
    .setDate("birthDate", LocalDate.of(1993, 5, 15))
    .build();

// Access record data
String name = person.get("name");
Integer age = person.get("age");
LocalDate birthDate = person.getDate("birthDate");

// Byte manipulation for data processing
byte[] nameBytes = Bytes.toBytes("John Doe");
String recoveredName = Bytes.toString(nameBytes);
long timestamp = System.currentTimeMillis();
byte[] timestampBytes = Bytes.toBytes(timestamp);

Architecture

CDAP API Common is built around several key components:

Schema System: Type-safe schema definitions supporting primitive types, complex types (arrays, maps, records, unions), and logical types for dates and timestamps
Structured Records: Runtime data containers that enforce schema compliance with builder pattern construction
Stream Processing: Event-driven data processing with headers, body, and timestamp support
Byte Utilities: Comprehensive byte array manipulation optimized for big data scenarios
Format Abstractions: Pluggable data format conversion system for various input/output formats

Capabilities

Data Schema Definition

Comprehensive schema system supporting primitive types, complex nested structures, logical types for dates/timestamps, and schema compatibility checking. Provides type safety and validation for data processing pipelines.

// Schema creation methods
public static Schema of(Schema.Type type);
public static Schema of(Schema.LogicalType logicalType);
public static Schema recordOf(String name, Schema.Field... fields);
public static Schema arrayOf(Schema componentSchema);
public static Schema mapOf(Schema keySchema, Schema valueSchema);
public static Schema unionOf(Schema... schemas);
public static Schema nullableOf(Schema schema);

// Schema parsing
public static Schema parseJson(String schemaJson) throws IOException;
public static Schema parseSQL(String schemaString) throws IOException;

Schema System

Structured Data Records

Type-safe record instances that conform to defined schemas, with builder pattern construction and specialized accessors for date/time logical types. Essential for data pipeline processing and validation.

public static StructuredRecord.Builder builder(Schema schema);

// Builder methods
public StructuredRecord.Builder set(String fieldName, Object value);
public StructuredRecord.Builder setDate(String fieldName, LocalDate date);
public StructuredRecord.Builder setTime(String fieldName, LocalTime time);
public StructuredRecord.Builder setTimestamp(String fieldName, ZonedDateTime timestamp);
public StructuredRecord build();

// Data access methods  
public <T> T get(String fieldName);
public LocalDate getDate(String fieldName);
public LocalTime getTime(String fieldName);
public ZonedDateTime getTimestamp(String fieldName);

Structured Records

Stream Event Processing

Event-driven data processing capabilities for handling streaming data with headers, body content, and timestamps. Supports custom decoders for converting stream events into structured key-value pairs.

public class StreamEvent extends StreamEventData {
    public StreamEvent(Map<String, String> headers, ByteBuffer body, long timestamp);
    public long getTimestamp();
    public Map<String, String> getHeaders();
    public ByteBuffer getBody();
}

public interface StreamEventDecoder<K, V> {
    DecodeResult<K, V> decode(StreamEvent event, DecodeResult<K, V> result);
}

Stream Processing

Byte Array Utilities

Comprehensive byte manipulation utilities optimized for big data processing, including conversions between primitive types and byte arrays, array operations, comparisons, and hash computations.

// Type conversions
public static byte[] toBytes(String s);
public static byte[] toBytes(long val);
public static byte[] toBytes(int val);
public static String toString(byte[] b);
public static long toLong(byte[] bytes);
public static int toInt(byte[] bytes);

// Array operations
public static byte[] add(byte[] a, byte[] b);
public static byte[] concat(byte[]... arrays);
public static int compareTo(byte[] left, byte[] right);
public static boolean equals(byte[] left, byte[] right);
public static int hashCode(byte[] b);

Byte Utilities

Data Format Abstractions

Pluggable format system for converting data between different representations, with built-in support for common formats and extensible architecture for custom formats.

public abstract class RecordFormat<FROM, TO> {
    public abstract TO read(FROM input) throws UnexpectedFormatException;
    public void initialize(FormatSpecification formatSpec);
    public Schema getSchema();
}

public class FormatSpecification {
    public FormatSpecification(String name, Schema schema, Map<String, String> settings);
    public String getName();
    public Schema getSchema();
    public Map<String, String> getSettings();
}

Data Format System

Types

Core Schema Types

public enum Schema.Type {
    NULL, BOOLEAN, INT, LONG, FLOAT, DOUBLE, BYTES, STRING,
    ENUM, ARRAY, MAP, RECORD, UNION;
    
    public boolean isSimpleType();
}

public enum Schema.LogicalType {
    DATE, TIMESTAMP_MILLIS, TIMESTAMP_MICROS, TIME_MILLIS, TIME_MICROS;
    
    public String getToken();
    public static LogicalType fromToken(String token);
}

public static final class Schema.Field {
    public static Field of(String name, Schema schema);
    public String getName();
    public Schema getSchema();
}

Exception Types

public class UnexpectedFormatException extends RuntimeException {
    public UnexpectedFormatException(String message);
    public UnexpectedFormatException(String message, Throwable cause);
}

public class UnsupportedTypeException extends Exception {
    public UnsupportedTypeException(String message);
    public UnsupportedTypeException(String message, Throwable cause);
}

public class InvalidMacroException extends RuntimeException {
    public InvalidMacroException(String message);
    public InvalidMacroException(String message, Throwable cause);
}

Utility Types

@Beta
public @interface Beta {
    // Marks experimental APIs
}

public final class SchemaHash {
    public SchemaHash(Schema schema);
    public byte[] toByteArray();
    public String toString(); // Hex representation
}

public final class Formats {
    public static final String AVRO = "avro";
    public static final String CSV = "csv"; 
    public static final String TSV = "tsv";
    public static final String TEXT = "text";
    public static final String COMBINED_LOG_FORMAT = "clf";
    public static final String GROK = "grok";
    public static final String SYSLOG = "syslog";
}