tessl/maven-org-apache-flink--flink-streaming-core
Apache Flink streaming core library providing fundamental building blocks for scalable stream data processing.
—
Pending
stream-operators.mddocs/
Stream Operators
Stream operators are the internal execution components that implement the actual stream processing logic in Flink. They handle element processing, state management, and coordination with the runtime system.
Core Operator Interfaces
StreamOperator<OUT>
Base interface for all streaming operators.
public interface StreamOperator<OUT> extends Serializable {
// Lifecycle methods
void setup(Output<StreamRecord<OUT>> output, RuntimeContext runtimeContext);
void open(Configuration parameters) throws Exception;
void close() throws Exception;
// Configuration
void setChainingStrategy(ChainingStrategy strategy);
ChainingStrategy getChainingStrategy();
}OneInputStreamOperator<IN, OUT>
Interface for operators that process a single input stream.
public interface OneInputStreamOperator<IN, OUT> extends StreamOperator<OUT> {
void processElement(StreamRecord<IN> element) throws Exception;
void processWatermark(Watermark mark) throws Exception;
}TwoInputStreamOperator<IN1, IN2, OUT>
Interface for operators that process two input streams (e.g., for connected streams).
public interface TwoInputStreamOperator<IN1, IN2, OUT> extends StreamOperator<OUT> {
void processElement1(StreamRecord<IN1> element) throws Exception;
void processElement2(StreamRecord<IN2> element) throws Exception;
void processWatermark1(Watermark mark) throws Exception;
void processWatermark2(Watermark mark) throws Exception;
}Function-based Operators
StreamMap<IN, OUT>
Operator that applies a MapFunction to each element.
public class StreamMap<IN, OUT> extends AbstractUdfStreamOperator<OUT, MapFunction<IN, OUT>>
implements OneInputStreamOperator<IN, OUT> {
public StreamMap(MapFunction<IN, OUT> mapper);
public void processElement(StreamRecord<IN> element) throws Exception;
}StreamFlatMap<IN, OUT>
Operator that applies a FlatMapFunction to each element.
public class StreamFlatMap<IN, OUT> extends AbstractUdfStreamOperator<OUT, FlatMapFunction<IN, OUT>>
implements OneInputStreamOperator<IN, OUT> {
public StreamFlatMap(FlatMapFunction<IN, OUT> flatMapper);
public void processElement(StreamRecord<IN> element) throws Exception;
}StreamFilter<T>
Operator that filters elements based on a FilterFunction.
public class StreamFilter<T> extends AbstractUdfStreamOperator<T, FilterFunction<T>>
implements OneInputStreamOperator<T, T> {
public StreamFilter(FilterFunction<T> filter);
public void processElement(StreamRecord<T> element) throws Exception;
}Keyed Stream Operators
StreamGroupedReduce<T>
Operator for keyed reduce operations.
public class StreamGroupedReduce<T> extends AbstractUdfStreamOperator<T, ReduceFunction<T>>
implements OneInputStreamOperator<T, T> {
public StreamGroupedReduce(ReduceFunction<T> reducer, TypeSerializer<T> serializer);
public void processElement(StreamRecord<T> element) throws Exception;
}StreamGroupedFold<IN, OUT>
Operator for keyed fold operations.
public class StreamGroupedFold<IN, OUT> extends AbstractUdfStreamOperator<OUT, FoldFunction<IN, OUT>>
implements OneInputStreamOperator<IN, OUT> {
public StreamGroupedFold(FoldFunction<IN, OUT> folder, OUT initialValue, TypeSerializer<OUT> serializer);
public void processElement(StreamRecord<IN> element) throws Exception;
}Source and Sink Operators
StreamSource<OUT>
Operator that wraps a SourceFunction for data ingestion.
public class StreamSource<OUT> extends AbstractUdfStreamOperator<OUT, SourceFunction<OUT>>
implements StreamOperator<OUT> {
public StreamSource(SourceFunction<OUT> sourceFunction);
public void run(Object lockingObject, Output<StreamRecord<OUT>> collector) throws Exception;
}StreamSink<IN>
Operator that wraps a SinkFunction for data output.
public class StreamSink<IN> extends AbstractUdfStreamOperator<Object, SinkFunction<IN>>
implements OneInputStreamOperator<IN, Object> {
public StreamSink(SinkFunction<IN> sinkFunction);
public void processElement(StreamRecord<IN> element) throws Exception;
}Connected Stream Operators
CoStreamMap<IN1, IN2, OUT>
Operator for applying CoMapFunction to connected streams.
public class CoStreamMap<IN1, IN2, OUT> extends AbstractUdfStreamOperator<OUT, CoMapFunction<IN1, IN2, OUT>>
implements TwoInputStreamOperator<IN1, IN2, OUT> {
public CoStreamMap(CoMapFunction<IN1, IN2, OUT> mapper);
public void processElement1(StreamRecord<IN1> element) throws Exception;
public void processElement2(StreamRecord<IN2> element) throws Exception;
}CoStreamFlatMap<IN1, IN2, OUT>
Operator for applying CoFlatMapFunction to connected streams.
public class CoStreamFlatMap<IN1, IN2, OUT> extends AbstractUdfStreamOperator<OUT, CoFlatMapFunction<IN1, IN2, OUT>>
implements TwoInputStreamOperator<IN1, IN2, OUT> {
public CoStreamFlatMap(CoFlatMapFunction<IN1, IN2, OUT> flatMapper);
public void processElement1(StreamRecord<IN1> element) throws Exception;
public void processElement2(StreamRecord<IN2> element) throws Exception;
}CoStreamReduce<T>
Operator for applying CoReduceFunction to connected streams.
public class CoStreamReduce<T> extends AbstractUdfStreamOperator<T, CoReduceFunction<T, T, T>>
implements TwoInputStreamOperator<T, T, T> {
public CoStreamReduce(CoReduceFunction<T, T, T> reducer);
public void processElement1(StreamRecord<T> element) throws Exception;
public void processElement2(StreamRecord<T> element) throws Exception;
}Windowing Operators
WindowingOperator<T>
Operator that implements windowing logic.
public class WindowingOperator<T> extends OneInputStreamOperator<T, StreamWindow<T>> {
public WindowingOperator(WindowingHelper<T> helper, TypeSerializer<T> serializer);
public void processElement(StreamRecord<T> element) throws Exception;
}Partitioning Operators
StreamPartition<T>
Base class for partitioning operators that control data distribution.
public abstract class StreamPartition<T> implements StreamOperator<T> {
// Partitioning logic implementation
}ShufflePartitioner<T>
Randomly distributes elements across parallel instances.
public class ShufflePartitioner<T> extends StreamPartition<T> {
// Random partitioning implementation
}RebalancePartitioner<T>
Round-robin distribution of elements.
public class RebalancePartitioner<T> extends StreamPartition<T> {
// Round-robin partitioning implementation
}BroadcastPartitioner<T>
Broadcasts elements to all parallel instances.
public class BroadcastPartitioner<T> extends StreamPartition<T> {
// Broadcast partitioning implementation
}Base Classes
AbstractStreamOperator<OUT>
Abstract base class providing common operator functionality.
public abstract class AbstractStreamOperator<OUT> implements StreamOperator<OUT> {
protected Output<StreamRecord<OUT>> output;
protected RuntimeContext runtimeContext;
protected ChainingStrategy chainingStrategy;
public void setup(Output<StreamRecord<OUT>> output, RuntimeContext runtimeContext);
public void open(Configuration parameters) throws Exception;
public void close() throws Exception;
public void setChainingStrategy(ChainingStrategy strategy);
public ChainingStrategy getChainingStrategy();
}AbstractUdfStreamOperator<OUT, F>
Base class for operators that wrap user-defined functions.
public abstract class AbstractUdfStreamOperator<OUT, F extends Function>
extends AbstractStreamOperator<OUT> {
protected final F userFunction;
public AbstractUdfStreamOperator(F userFunction);
public F getUserFunction();
}Types
// Operator output interface
public interface Output<T> {
void collect(T record);
void emitWatermark(Watermark mark);
void close();
}
// Chaining strategy for operator connections
public enum ChainingStrategy {
ALWAYS, // Always try to chain with neighbors
NEVER, // Never chain with neighbors
HEAD // Can be chained to but not chain to others
}
// Stream record wrapper
public class StreamRecord<T> {
public T getValue();
public long getTimestamp();
public boolean hasTimestamp();
public StreamRecord<T> replace(T element);
public StreamRecord<T> replace(T element, long timestamp);
}
// Watermark for event time processing
public class Watermark {
public long getTimestamp();
public static final Watermark MAX_WATERMARK;
}Install with Tessl CLI
npx tessl i tessl/maven-org-apache-flink--flink-streaming-core