CtrlK
BlogDocsLog inGet started
Tessl Logo
PluginPublic

mcclowes/language-design

Use when designing language features - covers lexer, parser, AST, and interpreter patterns

Quality

95%

Does it follow best practices?

Run evals on this skill

Adds up to 20 points to the overall score

View guide
SecuritybySnyk

Passed

No findings from the security scan

Overview
Quality
Evals
Security
Files

lexer.mdreferences/

Lexer — full implementation with helper methods

The core nextToken dispatch loop lives in SKILL.md. This fleshes out the helper methods it calls — character classification, the per-token scanners, and the cursor primitives.

class Lexer {
  private source: string;
  private pos = 0;
  private line = 1;
  private column = 1;
  private tokenStart = 0;

  constructor(source: string) {
    this.source = source;
  }

  nextToken(): Token {
    this.skipWhitespace();
    this.tokenStart = this.pos;

    if (this.isAtEnd()) return this.makeToken(TokenType.EOF);

    const char = this.advance();

    if (this.isDigit(char)) return this.number();
    if (this.isAlpha(char)) return this.identifier();
    if (char === '"') return this.string();

    switch (char) {
      case '+': return this.makeToken(TokenType.PLUS);
      case '-': return this.makeToken(TokenType.MINUS);
      case '*': return this.makeToken(TokenType.STAR);
      case '/':
        // Project-specific: /> is the pipe operator
        if (this.match('>')) return this.makeToken(TokenType.PIPE);
        return this.makeToken(TokenType.SLASH);
    }

    throw new LexerError(`Unexpected character: ${char}`, this.line, this.column);
  }

  // --- Token scanners ---

  private number(): Token {
    while (this.isDigit(this.peek())) this.advance();
    if (this.peek() === "." && this.isDigit(this.peekNext())) {
      this.advance(); // consume the '.'
      while (this.isDigit(this.peek())) this.advance();
    }
    return this.makeToken(TokenType.NUMBER);
  }

  private identifier(): Token {
    while (this.isAlphaNumeric(this.peek())) this.advance();
    const text = this.source.slice(this.tokenStart, this.pos);
    return this.makeToken(KEYWORDS[text] ?? TokenType.IDENT);
  }

  private string(): Token {
    while (this.peek() !== '"' && !this.isAtEnd()) {
      if (this.peek() === "\n") this.line++;
      this.advance();
    }
    if (this.isAtEnd()) {
      throw new LexerError("Unterminated string", this.line, this.column);
    }
    this.advance(); // closing quote
    return this.makeToken(TokenType.STRING);
  }

  // --- Character classification ---

  private isDigit(c: string): boolean {
    return c >= "0" && c <= "9";
  }

  private isAlpha(c: string): boolean {
    return (c >= "a" && c <= "z") || (c >= "A" && c <= "Z") || c === "_";
  }

  private isAlphaNumeric(c: string): boolean {
    return this.isAlpha(c) || this.isDigit(c);
  }

  // --- Cursor primitives ---

  private skipWhitespace(): void {
    while (!this.isAtEnd()) {
      const c = this.peek();
      if (c === " " || c === "\t" || c === "\r") {
        this.advance();
      } else if (c === "\n") {
        this.line++;
        this.column = 1;
        this.advance();
      } else {
        break;
      }
    }
  }

  private advance(): string {
    const c = this.source[this.pos++];
    this.column++;
    return c;
  }

  private match(expected: string): boolean {
    if (this.isAtEnd() || this.source[this.pos] !== expected) return false;
    this.pos++;
    this.column++;
    return true;
  }

  private peek(): string {
    return this.isAtEnd() ? "\0" : this.source[this.pos];
  }

  private peekNext(): string {
    return this.pos + 1 >= this.source.length ? "\0" : this.source[this.pos + 1];
  }

  private isAtEnd(): boolean {
    return this.pos >= this.source.length;
  }

  private makeToken(type: TokenType): Token {
    return {
      type,
      value: this.source.slice(this.tokenStart, this.pos),
      line: this.line,
      column: this.column,
    };
  }
}

README.md

SKILL.md

tile.json