0
# Compilation and Parsing
1

2
Core compilation functionality for transforming RapydScript source code (.pyj files) into optimized JavaScript. The compilation system provides complete parsing, AST manipulation, and code generation capabilities.
3

4
## Capabilities
5

6
### Compiler Factory
7

8
Creates a compiler instance with all compilation facilities in an isolated VM context.
9

10
```javascript { .api }
11
/**
12
 * Creates a compiler instance with all compilation facilities
13
 * @returns Compiler instance with parsing and output capabilities
14
 */
15
function create_compiler(): CompilerInstance;
16

17
interface CompilerInstance {
18
  parse: (code: string, options: ParseOptions) => AST_Toplevel;
19
  OutputStream: new (options: OutputOptions) => OutputStream;
20
  DefaultsError: typeof Error;
21
  SyntaxError: typeof Error;
22
  ImportError: typeof Error;
23
  string_template: (template: string, vars: object) => string;
24
  tokenizer: (code: string, options: object) => TokenStream;
25
  ALL_KEYWORDS: string[];
26
  IDENTIFIER_PAT: RegExp;
27
  NATIVE_CLASSES: string[];
28
}
29
```
30

31
**Usage Example:**
32

33
```javascript
34
const { create_compiler } = require("rapydscript-ng");
35
const RapydScript = create_compiler();
36

37
// Access all compiler facilities
38
const ast = RapydScript.parse(code, options);
39
const output = new RapydScript.OutputStream({ beautify: true });
40
```
41

42
### Parse Function
43

44
Main parsing function that converts RapydScript source code into an Abstract Syntax Tree.
45

46
```javascript { .api }
47
/**
48
 * Parse RapydScript source code into an AST
49
 * @param code - RapydScript source code string
50
 * @param options - Parsing configuration options
51
 * @returns AST_Toplevel root node
52
 * @throws SyntaxError on parsing failures
53
 */
54
function parse(code: string, options: ParseOptions): AST_Toplevel;
55

56
interface ParseOptions {
57
  /** Source filename for error reporting */
58
  filename?: string;
59
  /** Existing toplevel AST to extend */
60
  toplevel?: AST_Toplevel;
61
  /** Base directory for relative imports */
62
  basedir?: string;
63
  /** Standard library directory path */
64
  libdir?: string;
65
  /** Additional import search directories */
66
  import_dirs?: string[];
67
  /** Remove assert statements during parsing */
68
  discard_asserts?: boolean;
69
  /** Cache directory for compiled modules */
70
  module_cache_dir?: string;
71
}
72
```
73

74
**Usage Examples:**
75

76
```javascript
77
// Basic parsing
78
const ast = RapydScript.parse(`
79
def hello(name):
80
    print(f"Hello, {name}!")
81
`, { filename: 'hello.pyj' });
82

83
// Advanced parsing with options
84
const ast = RapydScript.parse(sourceCode, {
85
  filename: 'main.pyj',
86
  basedir: '/path/to/project',
87
  libdir: '/path/to/rapydscript/lib',
88
  import_dirs: ['/custom/modules'],
89
  discard_asserts: true
90
});
91
```
92

93
### Output Stream
94

95
Output stream class that manages JavaScript code generation with formatting and optimization options.
96

97
```javascript { .api }
98
/**
99
 * Creates an output stream for JavaScript code generation
100
 * @param options - Output formatting and optimization options
101
 */
102
class OutputStream {
103
  constructor(options: OutputOptions);
104
  
105
  /** Print text to the output stream */
106
  print(text: string): void;
107
  
108
  /** Execute function with increased indentation */
109
  with_indent(body: () => void): void;
110
  
111
  /** Get the complete generated output */
112
  get(): string;
113
  
114
  /** Get the complete generated output (alias for get) */
115
  toString(): string;
116
}
117

118
interface OutputOptions {
119
  /** Generate formatted, readable JavaScript */
120
  beautify?: boolean;
121
  /** Wrap output in private scope (function wrapper) */
122
  private_scope?: boolean;
123
  /** Exclude base library functions from output */
124
  omit_baselib?: boolean;
125
  /** Target JavaScript version (5 or 6) */
126
  js_version?: number;
127
  /** Preserve docstrings as __doc__ attributes */
128
  keep_docstrings?: boolean;
129
  /** Remove assert statements from output */
130
  discard_asserts?: boolean;
131
  /** Cache directory for compiled modules */
132
  module_cache_dir?: string;
133
  /** Pre-loaded base library code */
134
  baselib_plain?: string;
135
  /** Comment preservation strategy */
136
  comments?: boolean | Function;
137
}
138
```
139

140
**Usage Examples:**
141

142
```javascript
143
// Basic output generation
144
const output = new RapydScript.OutputStream({ 
145
  beautify: true,
146
  private_scope: true 
147
});
148
ast.print(output);
149
const javascript = output.get();
150

151
// Minified output for production
152
const minified = new RapydScript.OutputStream({ 
153
  beautify: false,
154
  omit_baselib: true,
155
  js_version: 6 
156
});
157
ast.print(minified);
158
const compactJs = minified.get();
159
```
160

161
### Error Classes
162

163
Exception classes used during compilation for different types of errors.
164

165
```javascript { .api }
166
/**
167
 * Thrown when invalid compiler options are provided
168
 */
169
class DefaultsError extends Error {
170
  constructor(message: string);
171
}
172

173
/**
174
 * Thrown when parsing fails due to syntax errors
175
 */
176
class SyntaxError extends Error {
177
  constructor(message: string, filename?: string, line?: number, col?: number);
178
}
179

180
/**
181
 * Thrown when module imports cannot be resolved
182
 */
183
class ImportError extends Error {
184
  constructor(message: string, filename?: string);
185
}
186
```
187

188
### Utility Functions
189

190
Helper functions for string templating and lexical analysis.
191

192
```javascript { .api }
193
/**
194
 * Simple string templating with variable substitution
195
 * @param template - Template string with {variable} placeholders
196
 * @param vars - Object containing variable values
197
 * @returns Interpolated string
198
 */
199
function string_template(template: string, vars: object): string;
200

201
/**
202
 * Tokenize RapydScript source code into a token stream
203
 * @param code - Source code to tokenize
204
 * @param options - Tokenization options
205
 * @returns Token stream for parsing
206
 */
207
function tokenizer(code: string, options: object): TokenStream;
208

209
/**
210
 * Access to compile-time decorators registry
211
 * @returns Object containing available compile-time decorators
212
 */
213
const compile_time_decorators: object;
214
```
215

216
### Parser Constants
217

218
Constants used by the parser for keyword recognition and identifier validation.
219

220
```javascript { .api }
221
/** Array of all reserved keywords in RapydScript */
222
const ALL_KEYWORDS: string[];
223

224
/** Regular expression pattern for valid identifiers */
225
const IDENTIFIER_PAT: RegExp;
226

227
/** List of built-in JavaScript classes recognized by the compiler */
228
const NATIVE_CLASSES: string[];
229
```
230

231
**Usage Examples:**
232

233
```javascript
234
// Check if a word is a reserved keyword
235
const isKeyword = RapydScript.ALL_KEYWORDS.includes('def'); // true
236

237
// Validate identifier names
238
const isValidId = RapydScript.IDENTIFIER_PAT.test('my_variable'); // true
239

240
// Check for native JavaScript classes
241
const isNative = RapydScript.NATIVE_CLASSES.includes('Array'); // true
242
```
243

244
### Complete Compilation Workflow
245

246
```javascript
247
const { create_compiler } = require("rapydscript-ng");
248
const fs = require("fs");
249

250
// Create compiler instance
251
const RapydScript = create_compiler();
252

253
// Read source file
254
const sourceCode = fs.readFileSync("app.pyj", "utf-8");
255

256
try {
257
  // Parse into AST
258
  const ast = RapydScript.parse(sourceCode, {
259
    filename: "app.pyj",
260
    basedir: __dirname,
261
    libdir: "/usr/local/lib/rapydscript/lib"
262
  });
263
  
264
  // Generate JavaScript
265
  const output = new RapydScript.OutputStream({
266
    beautify: true,
267
    private_scope: false,
268
    js_version: 5
269
  });
270
  
271
  ast.print(output);
272
  const javascript = output.get();
273
  
274
  // Write output
275
  fs.writeFileSync("app.js", javascript);
276
  
277
} catch (error) {
278
  if (error instanceof RapydScript.SyntaxError) {
279
    console.error("Syntax error:", error.message);
280
  } else if (error instanceof RapydScript.ImportError) {
281
    console.error("Import error:", error.message);
282
  } else {
283
    throw error;
284
  }
285
}
286
```