Low-level compiler internals for tokenizing CoffeeScript source code into tokens and parsing tokens into Abstract Syntax Tree (AST) nodes.
Tokenizes CoffeeScript source code into an array of tokens representing lexical elements.
/**
* Tokenize CoffeeScript source code into lexical tokens
* @param code - CoffeeScript source code to tokenize
* @param options - Tokenization options
* @returns Array of tokens representing the source code
*/
function tokens(code: string, options?: TokenizeOptions): Token[];
interface TokenizeOptions {
/** Source filename for error reporting */
filename?: string;
/** Apply rewriter transformations to tokens */
rewrite?: boolean;
}
interface Token {
/** Token type (e.g., 'IDENTIFIER', 'NUMBER', 'STRING', 'CALL_START') */
0: string;
/** Token value/content */
1: string;
/** Location data for source mapping */
2: LocationData;
/** Additional token metadata */
[key: string]: any;
}
interface LocationData {
first_line: number;
last_line: number;
first_column: number;
last_column: number;
}Usage Examples:
const CoffeeScript = require('coffeescript');
// Basic tokenization
const tokens = CoffeeScript.tokens('square = (x) -> x * x');
console.log(tokens);
// Output: Array of token objects
// Tokenize with filename
const tokens = CoffeeScript.tokens(`
add = (a, b) -> a + b
result = add 3, 4
`, { filename: 'math.coffee' });
// Examine token structure
tokens.forEach((token, index) => {
console.log(`Token ${index}:`, {
type: token[0],
value: token[1],
location: token[2]
});
});CoffeeScript tokenization produces various token types representing different language elements:
// Identifiers and literals
'IDENTIFIER' // variable names, function names
'NUMBER' // numeric literals
'STRING' // string literals
'BOOL' // boolean literals
'NULL' // null literal
'UNDEFINED' // undefined literal
// Operators
'+' // arithmetic operators
'COMPARE' // comparison operators (==, !=, <, >, etc.)
'LOGIC' // logical operators (and, or, not)
'MATH' // mathematical operators (+, -, *, /, etc.)
// Punctuation and structure
'(' // parentheses
'CALL_START' // function call start
'CALL_END' // function call end
'INDENT' // indentation increase
'OUTDENT' // indentation decrease
'TERMINATOR' // line terminator
// Keywords
'IF' // conditional statements
'FOR' // loop constructs
'WHILE' // while loops
'CLASS' // class definitions
'FUNC_EXIST' // function existence check (?.)Parses CoffeeScript source code or token array into Abstract Syntax Tree (AST) nodes.
/**
* Parse CoffeeScript source or tokens into AST nodes
* @param source - CoffeeScript source code or token array
* @param options - Parsing options
* @returns Root Block node containing parsed AST
*/
function nodes(source: string | Token[], options?: ParseOptions): Block;
interface ParseOptions {
/** Source filename for error reporting */
filename?: string;
}Usage Examples:
const CoffeeScript = require('coffeescript');
// Parse from source code
const ast = CoffeeScript.nodes('square = (x) -> x * x');
console.log(ast.constructor.name); // 'Block'
// Parse from tokens
const tokens = CoffeeScript.tokens('add = (a, b) -> a + b');
const ast = CoffeeScript.nodes(tokens);
// Examine AST structure
console.log(ast.expressions.length); // Number of top-level expressions
ast.expressions.forEach((expr, index) => {
console.log(`Expression ${index}:`, expr.constructor.name);
});The parser generates various AST node types representing different language constructs:
/**
* Base class for all AST nodes
*/
class Base {
/** Compile node to JavaScript */
compile(options?: CompileOptions): string;
/** Compile node to source fragments */
compileToFragments(options?: CompileOptions): Fragment[];
/** String representation of node */
toString(): string;
}
/**
* Block of statements/expressions
*/
class Block extends Base {
expressions: Base[];
}
/**
* Literal values (numbers, strings, booleans)
*/
class Literal extends Base {
value: string;
}
/**
* Variable and property access
*/
class Value extends Base {
base: Base;
properties: Access[];
}
/**
* Function definitions
*/
class Code extends Base {
params: Param[];
body: Block;
bound: boolean; // arrow function (=>)
}
/**
* Function calls
*/
class Call extends Base {
variable: Base;
args: Base[];
}
/**
* Assignment expressions
*/
class Assign extends Base {
variable: Base;
value: Base;
context: string; // '=', '||=', '&&=', etc.
}
/**
* Binary and unary operations
*/
class Op extends Base {
operator: string;
first: Base;
second?: Base; // undefined for unary ops
}
/**
* Class definitions
*/
class Class extends Base {
variable: Value;
parent?: Value;
body: Block;
}const CoffeeScript = require('coffeescript');
// Analyze complex CoffeeScript code
const source = `
class Calculator
constructor: (@name) ->
add: (a, b) =>
result = a + b
console.log "#{@name}: #{a} + #{b} = #{result}"
result
square: (x) -> x * x
calc = new Calculator "MyCalc"
sum = calc.add 3, 4
squared = calc.square 5
`;
// Get tokens
const tokens = CoffeeScript.tokens(source);
console.log(`Generated ${tokens.length} tokens`);
// Get AST
const ast = CoffeeScript.nodes(source);
console.log(`AST has ${ast.expressions.length} top-level expressions`);
// Find specific node types
function findNodesByType(node, type) {
let results = [];
if (node.constructor.name === type) {
results.push(node);
}
// Recursively search child nodes
for (let key in node) {
if (node[key] && typeof node[key] === 'object') {
if (Array.isArray(node[key])) {
node[key].forEach(child => {
if (child && child.constructor) {
results = results.concat(findNodesByType(child, type));
}
});
} else if (node[key].constructor) {
results = results.concat(findNodesByType(node[key], type));
}
}
}
return results;
}
// Find all function definitions
const codeFunctions = findNodesByType(ast, 'Code');
console.log(`Found ${codeFunctions.length} function definitions`);
// Find all assignments
const assignments = findNodesByType(ast, 'Assign');
console.log(`Found ${assignments.length} assignments`);CoffeeScript applies token rewriting to transform tokens before parsing:
const CoffeeScript = require('coffeescript');
// Without rewriting (raw tokens)
const rawTokens = CoffeeScript.tokens('a + b', { rewrite: false });
// With rewriting (default, transforms for easier parsing)
const rewrittenTokens = CoffeeScript.tokens('a + b', { rewrite: true });
console.log('Raw tokens:', rawTokens.length);
console.log('Rewritten tokens:', rewrittenTokens.length);Tokenization and parsing functions throw SyntaxError for invalid CoffeeScript:
const CoffeeScript = require('coffeescript');
try {
CoffeeScript.tokens('invalid -> -> syntax');
} catch (error) {
console.log(error.name); // 'SyntaxError'
console.log(error.message); // Error description
console.log(error.location); // Location data
}
try {
CoffeeScript.nodes('class without body');
} catch (error) {
console.log(error.filename); // Source filename (if provided)
console.log(error.location); // Error location in source
}const CoffeeScript = require('coffeescript');
function analyzeTokens(source) {
const tokens = CoffeeScript.tokens(source);
const analysis = {
identifiers: [],
numbers: [],
strings: [],
keywords: []
};
tokens.forEach(token => {
const [type, value] = token;
switch (type) {
case 'IDENTIFIER':
analysis.identifiers.push(value);
break;
case 'NUMBER':
analysis.numbers.push(value);
break;
case 'STRING':
analysis.strings.push(value);
break;
default:
if (['IF', 'FOR', 'WHILE', 'CLASS', 'TRY'].includes(type)) {
analysis.keywords.push(type);
}
}
});
return analysis;
}
// Analyze CoffeeScript source
const analysis = analyzeTokens(`
numbers = [1, 2, 3, 4, 5]
total = 0
for num in numbers
total += num
console.log "Total: #{total}"
`);
console.log('Identifiers:', analysis.identifiers);
console.log('Numbers:', analysis.numbers);
console.log('Strings:', analysis.strings);
console.log('Keywords:', analysis.keywords);