Essential parser and lexer base classes that provide the fundamental parsing infrastructure for language recognition. These classes form the backbone of ANTLR4's parsing system.
Abstract base class for parsers and lexers providing common recognition functionality.
/**
* Base class for parsers and lexers providing shared recognition functionality
*/
abstract class Recognizer<TSymbol> {
/** Current parser/lexer state */
state: number;
/** Remove all error listeners */
removeErrorListeners(): void;
/** Add an error listener for handling recognition errors */
addErrorListener(listener: ErrorListener<TSymbol>): void;
/** Get the current error listener */
getErrorListener(): ErrorListener<TSymbol>;
/** Get array of literal token names (e.g., ['=', '+', 'class']) */
getLiteralNames(): string[];
/** Get array of symbolic token names (e.g., ['ASSIGN', 'PLUS', 'CLASS']) */
getSymbolicNames(): string[];
}Base parser class for parsing token streams into parse trees.
/**
* Base parser class for parsing token streams
*/
class Parser extends Recognizer<Token> {
/** End-of-file token type constant */
static readonly EOF: number;
/** Input token stream */
readonly _input: TokenStream;
/** Current rule context */
_ctx: ParserRuleContext;
/** Parser ATN simulator for prediction */
_interp: ParserATNSimulator;
/** Error recovery strategy */
_errHandler: ErrorStrategy;
/** Parse tree listeners */
_parseListeners?: any[];
/** Whether EOF was matched */
matchedEOF: boolean;
/** Whether to build parse trees (default: true) */
buildParseTrees: boolean;
/** Count of syntax errors encountered */
syntaxErrorsCount: number;
/**
* Create parser with token stream input
* @param input - Token stream to parse
*/
constructor(input: TokenStream);
/**
* Match expected token type and consume it
* @param ttype - Expected token type
* @returns The matched token
* @throws RecognitionException if token doesn't match
*/
match(ttype: number): Token;
/**
* Match any token (wildcard)
* @returns The matched token
*/
matchWildcard(): Token;
/**
* Get all registered parse tree listeners
* @returns Array of parse tree listeners
*/
getParseListeners(): ParseTreeListener[];
/**
* Add a parse tree listener
* @param listener - Listener to add
*/
addParseListener(listener: ParseTreeListener): void;
/**
* Remove a specific parse tree listener
* @param listener - Listener to remove
*/
removeParseListener(listener: ParseTreeListener): void;
/** Remove all parse tree listeners */
removeParseListeners(): void;
/**
* Consume the current token and advance input
* @returns The consumed token
*/
consume(): Token;
/**
* Enter a parse rule
* @param localctx - Local rule context
* @param state - Parser state
* @param ruleIndex - Index of the rule
*/
enterRule(localctx: ParserRuleContext, state: number, ruleIndex: number): void;
/** Exit the current rule */
exitRule(): void;
/** Trigger exit rule event for listeners */
triggerExitRuleEvent(): void;
/**
* Enter an alternative within a rule
* @param localctx - Local rule context
* @param altNum - Alternative number
*/
enterOuterAlt(localctx: ParserRuleContext, altNum: number): void;
/**
* Enter a left-recursive rule
* @param localctx - Local rule context
* @param state - Parser state
* @param ruleIndex - Rule index
* @param precedence - Precedence level
*/
enterRecursionRule(localctx: ParserRuleContext, state: number, ruleIndex: number, precedence: number): void;
/**
* Push a new recursion context
* @param localctx - Local rule context
* @param state - Parser state
* @param ruleIndex - Rule index
*/
pushNewRecursionContext(localctx: ParserRuleContext, state: number, ruleIndex: number): void;
/**
* Unroll recursion contexts
* @param parentCtx - Parent context to unroll to
*/
unrollRecursionContexts(parentCtx: ParserRuleContext): void;
/**
* Test precedence predicate
* @param localctx - Local rule context
* @param precedence - Required precedence
* @returns True if precedence is satisfied
*/
precpred(localctx: ParserRuleContext, precedence: number): boolean;
/**
* Get the rule invocation stack as strings
* @returns Array of rule names in call stack
*/
getRuleInvocationStack(): string[];
/** Debug method to dump DFA information */
dumpDFA(): void;
/**
* Get set of expected tokens at current position
* @returns Set of expected token types
*/
getExpectedTokens(): IntervalSet;
/**
* Get the token stream being parsed
* @returns Current token stream
*/
getTokenStream(): TokenStream;
/** Reset parser state */
reset(): void;
/**
* Set a new token stream
* @param input - New token stream to parse
*/
setTokenStream(input: TokenStream): void;
/**
* Notify error listeners of a syntax error
* @param msg - Error message
* @param offendingToken - Token that caused the error
* @param err - Recognition exception (optional)
*/
notifyErrorListeners(msg: string, offendingToken: Token, err: RecognitionException | undefined): void;
/**
* Get the current token being examined
* @returns Current token
*/
getCurrentToken(): Token;
/**
* Get the context that invoked the specified rule
* @param ruleIndex - Rule index to find
* @returns Parser rule context or null
*/
getInvokingContext(ruleIndex: number): ParserRuleContext;
}Base lexer class for tokenizing character input into tokens.
/**
* Base lexer class for tokenizing input
*/
class Lexer extends Recognizer<number> {
/** Default lexer mode constant */
static readonly DEFAULT_MODE: number;
/** Default token channel constant */
static readonly DEFAULT_TOKEN_CHANNEL: number;
/** Hidden channel constant */
static readonly HIDDEN: number;
/** Minimum character value constant */
static readonly MIN_CHAR_VALUE: number;
/** Maximum character value constant */
static readonly MAX_CHAR_VALUE: number;
/** MORE action constant - continue building token */
static readonly MORE: number;
/** SKIP action constant - discard token */
static readonly SKIP: number;
/** Input character stream */
_input: CharStream;
/** Lexer ATN simulator */
_interp: LexerATNSimulator;
/** Current token text */
text: string;
/** Current line number */
line: number;
/** Current column position */
column: number;
/** Token start character index */
_tokenStartCharIndex: number;
/** Token start line */
_tokenStartLine: number;
/** Token start column */
_tokenStartColumn: number;
/** Current token type */
_type: number;
/**
* Create lexer with character stream input
* @param input - Character stream to tokenize
*/
constructor(input: CharStream);
/** Reset lexer state */
reset(): void;
/**
* Generate the next token from input
* @returns Next token from input stream
*/
nextToken(): Token;
/** Skip the current token (don't emit it) */
skip(): void;
/** Continue building the current token (MORE action) */
more(): void;
/**
* Set the current lexer mode
* @param m - Mode to set
*/
setMode(m: number): void;
/**
* Get the current lexer mode
* @returns Current mode
*/
getMode(): number;
/**
* Get the mode stack
* @returns Array of modes in stack
*/
getModeStack(): number[];
/**
* Push a new mode onto the mode stack
* @param m - Mode to push
*/
pushMode(m: number): void;
/**
* Pop the top mode from the mode stack
* @returns The popped mode
*/
popMode(): number;
/**
* Emit a specific token
* @param token - Token to emit
*/
emitToken(token: Token): void;
/**
* Emit the current token being built
* @returns The emitted token
*/
emit(): Token;
/**
* Emit an end-of-file token
* @returns EOF token
*/
emitEOF(): Token;
/**
* Get all tokens from input until EOF
* @returns Array of all tokens
*/
getAllTokens(): Token[];
}Token representation classes for lexical units.
/**
* Base token class representing a lexical unit
*/
class Token {
/** Invalid token type constant */
static readonly INVALID_TYPE: number;
/** End-of-file token constant */
static readonly EOF: number;
/** Default channel constant */
static readonly DEFAULT_CHANNEL: number;
/** Hidden channel constant */
static readonly HIDDEN_CHANNEL: number;
/** Token index in stream */
tokenIndex: number;
/** Line number where token appears */
line: number;
/** Column position where token starts */
column: number;
/** Channel this token belongs to */
channel: number;
/** Text of the token */
text: string;
/** Token type */
type: number;
/** Start character index in input */
start: number;
/** Stop character index in input */
stop: number;
/**
* Create a copy of this token
* @returns Cloned token
*/
clone(): Token;
/**
* Create a copy with different token type
* @param type - New token type
* @returns Cloned token with new type
*/
cloneWithType(type: number): Token;
/**
* Get the token source that created this token
* @returns Token source
*/
getTokenSource(): TokenSource;
/**
* Get the input stream this token came from
* @returns Character stream
*/
getInputStream(): CharStream;
}
/**
* Standard implementation of Token
*/
class CommonToken extends Token {
/**
* Create a common token
* @param source - Token source and input stream pair
* @param type - Token type
* @param channel - Token channel
* @param start - Start character index
* @param stop - Stop character index
*/
constructor(source: [TokenSource, InputStream], type: number, channel: number, start: number, stop: number);
/**
* Create a copy of this token
* @returns Cloned CommonToken
*/
clone(): CommonToken;
/**
* Create a copy with different token type
* @param type - New token type
* @returns Cloned token with new type
*/
cloneWithType(type: number): CommonToken;
/**
* String representation of token
* @returns Token string
*/
toString(): string;
}
/**
* Interface for token sources (base class for lexers)
*/
class TokenSource {
// Empty base class - implemented by lexers
}Creating a custom lexer:
import { Lexer, CharStream, CommonToken } from "antlr4";
class MyLexer extends Lexer {
constructor(input: CharStream) {
super(input);
// Initialize lexer-specific properties
}
// Override nextToken to provide custom tokenization logic
nextToken(): Token {
// Custom tokenization logic here
return super.nextToken();
}
}Creating a custom parser:
import { Parser, TokenStream, ParserRuleContext } from "antlr4";
class MyParser extends Parser {
constructor(input: TokenStream) {
super(input);
// Initialize parser-specific properties
}
// Define parse rules
expression(): ParserRuleContext {
const localctx = new ParserRuleContext(this._ctx, this.state);
this.enterRule(localctx, 0, 0); // rule index 0
try {
// Parse rule logic here
return localctx;
} finally {
this.exitRule();
}
}
}Working with tokens:
import { CommonToken, Token } from "antlr4";
// Create a token
const token = new CommonToken([null, null], 1, Token.DEFAULT_CHANNEL, 0, 4);
token.text = "hello";
token.line = 1;
token.column = 0;
// Clone token with different type
const newToken = token.cloneWithType(2);
console.log(token.toString()); // Token information