or run

npx @tessl/cli init

Version

Tile

Overview

Evals

Files

docs

error-handling.md index.md token-types.md tokenization.md type-predicates.md utilities.md

tile.json

Token Types and Interfaces

Complete type definitions for all CSS token types following the W3C CSS Syntax Level 3 specification, including enums, interfaces, and utility types.

Capabilities

Token Type Enum

Enumeration of all possible CSS token types as defined by the W3C specification.

enum TokenType {
  /** Comment token type */
  Comment = 'comment',
  /** At-keyword token type (@media, @import, etc.) */
  AtKeyword = 'at-keyword-token',
  /** Bad string token type (unclosed string) */
  BadString = 'bad-string-token',
  /** Bad URL token type (malformed URL) */
  BadURL = 'bad-url-token',
  /** CDC token type (-->) */
  CDC = 'CDC-token',
  /** CDO token type (<!--) */
  CDO = 'CDO-token',
  /** Colon token type (:) */
  Colon = 'colon-token',
  /** Comma token type (,) */
  Comma = 'comma-token',
  /** Delimiter token type (any single character) */
  Delim = 'delim-token',
  /** Dimension token type (10px, 5em, etc.) */
  Dimension = 'dimension-token',
  /** End of file token type */
  EOF = 'EOF-token',
  /** Function token type (calc(, rgb(, etc.) */
  Function = 'function-token',
  /** Hash token type (#id, #ff0000, etc.) */
  Hash = 'hash-token',
  /** Identifier token type */
  Ident = 'ident-token',
  /** Number token type */
  Number = 'number-token',
  /** Percentage token type (50%, etc.) */
  Percentage = 'percentage-token',
  /** Semicolon token type (;) */
  Semicolon = 'semicolon-token',
  /** String token type ("text", 'text') */
  String = 'string-token',
  /** URL token type */
  URL = 'url-token',
  /** Whitespace token type */
  Whitespace = 'whitespace-token',
  /** Open parenthesis token type (() */
  OpenParen = '(-token',
  /** Close parenthesis token type ()) */
  CloseParen = ')-token',
  /** Open square bracket token type ([) */
  OpenSquare = '[-token',
  /** Close square bracket token type (]) */
  CloseSquare = ']-token',
  /** Open curly bracket token type ({) */
  OpenCurly = '{-token',
  /** Close curly bracket token type (}) */
  CloseCurly = '}-token',
  /** Unicode range token type (U+0000-00FF) */
  UnicodeRange = 'unicode-range-token'
}

Supporting Enums

Additional enums for token properties and classifications.

enum NumberType {
  /** Integer number type */
  Integer = 'integer',
  /** Decimal number type */
  Number = 'number'
}

enum HashType {
  /** Hash token not starting with identifier sequence */
  Unrestricted = 'unrestricted',
  /** Hash token starting with identifier sequence (valid ID selector) */
  ID = 'id'
}

Core Token Types

Base interface and union types for all tokens. Important: Tokens are implemented as tuple-like arrays, not objects.

/**
 * Base token interface - tokens are implemented as tuple-like arrays
 * @template T - Token type
 * @template U - Token data
 */
interface Token<T extends TokenType, U> extends Array<T | string | number | U> {
  /** The type of token */
  0: T;
  /** The token representation (string used when stringifying) */
  1: string;
  /** Start position of representation */
  2: number;
  /** End position of representation */
  3: number;
  /** Extra data (parsed value, unescaped, unquoted, converted, etc.) */
  4: U;
}

/**
 * Union of all possible CSS token types
 */
type CSSToken = TokenAtKeyword | TokenBadString | TokenBadURL | TokenCDC | TokenCDO |
  TokenColon | TokenComma | TokenComment | TokenDelim | TokenDimension | TokenEOF |
  TokenFunction | TokenHash | TokenIdent | TokenNumber | TokenPercentage |
  TokenSemicolon | TokenString | TokenURL | TokenWhitespace | TokenOpenParen |
  TokenCloseParen | TokenOpenSquare | TokenCloseSquare | TokenOpenCurly |
  TokenCloseCurly | TokenUnicodeRange;

/**
 * Union of numeric token types (Number, Dimension, Percentage)
 */
type NumericToken = TokenNumber | TokenDimension | TokenPercentage;

Specific Token Interfaces

Individual token type interfaces with their specific data structures. All tokens follow the tuple pattern.

/** At-keyword token (@media, @import, etc.) */
interface TokenAtKeyword extends Token<TokenType.AtKeyword, { value: string }> {}

/** Bad string token (unclosed string) */
interface TokenBadString extends Token<TokenType.BadString, undefined> {}

/** Bad URL token (malformed URL) */
interface TokenBadURL extends Token<TokenType.BadURL, undefined> {}

/** CDC token (-->) */
interface TokenCDC extends Token<TokenType.CDC, undefined> {}

/** CDO token (<!--) */
interface TokenCDO extends Token<TokenType.CDO, undefined> {}

/** Colon token (:) */
interface TokenColon extends Token<TokenType.Colon, undefined> {}

/** Comma token (,) */
interface TokenComma extends Token<TokenType.Comma, undefined> {}

/** Comment token */
interface TokenComment extends Token<TokenType.Comment, undefined> {}

/** Delimiter token (any single character) */
interface TokenDelim extends Token<TokenType.Delim, { value: string }> {}

/** Dimension token (10px, 5em, etc.) */
interface TokenDimension extends Token<TokenType.Dimension, {
  value: number;
  unit: string;
  type: NumberType;
  signCharacter?: '+' | '-';
}> {}

/** End of file token */
interface TokenEOF extends Token<TokenType.EOF, undefined> {}

/** Function token (calc(, rgb(, etc.) */
interface TokenFunction extends Token<TokenType.Function, { value: string }> {}

/** Hash token (#id, #ff0000, etc.) */
interface TokenHash extends Token<TokenType.Hash, {
  value: string;
  type: HashType;
}> {}

/** Identifier token */
interface TokenIdent extends Token<TokenType.Ident, { value: string }> {}

/** Number token */
interface TokenNumber extends Token<TokenType.Number, {
  value: number;
  type: NumberType;
  signCharacter?: '+' | '-';
}> {}

/** Percentage token (50%, etc.) */
interface TokenPercentage extends Token<TokenType.Percentage, {
  value: number;
  signCharacter?: '+' | '-';
}> {}

/** Semicolon token (;) */
interface TokenSemicolon extends Token<TokenType.Semicolon, undefined> {}

/** String token ("text", 'text') */
interface TokenString extends Token<TokenType.String, { value: string }> {}

/** URL token */
interface TokenURL extends Token<TokenType.URL, { value: string }> {}

/** Whitespace token */
interface TokenWhitespace extends Token<TokenType.Whitespace, undefined> {}

/** Open parenthesis token (() */
interface TokenOpenParen extends Token<TokenType.OpenParen, undefined> {}

/** Close parenthesis token ()) */
interface TokenCloseParen extends Token<TokenType.CloseParen, undefined> {}

/** Open square bracket token ([) */
interface TokenOpenSquare extends Token<TokenType.OpenSquare, undefined> {}

/** Close square bracket token (]) */
interface TokenCloseSquare extends Token<TokenType.CloseSquare, undefined> {}

/** Open curly bracket token ({) */
interface TokenOpenCurly extends Token<TokenType.OpenCurly, undefined> {}

/** Close curly bracket token (}) */
interface TokenCloseCurly extends Token<TokenType.CloseCurly, undefined> {}

/** Unicode range token (U+0000-00FF) */
interface TokenUnicodeRange extends Token<TokenType.UnicodeRange, {
  startOfRange: number;
  endOfRange: number;
}> {}

Token Mirror Functions

Utility functions for getting corresponding bracket/parenthesis tokens.

/**
 * Gets mirror variant of bracket/paren tokens
 * @param token - Input token
 * @returns Corresponding closing/opening token or null if not applicable
 */
function mirrorVariant(token: CSSToken): CSSToken | null;

/**
 * Gets mirror variant type of token types
 * @param type - Input token type
 * @returns Corresponding closing/opening token type or null if not applicable
 */
function mirrorVariantType(type: TokenType): TokenType | null;

Usage Examples:

import { tokenize, mirrorVariant, mirrorVariantType, TokenType } from "@csstools/css-tokenizer";

const tokens = tokenize({ css: "body { color: red; }" });

// Find mirror variants of bracket tokens
tokens.forEach(token => {
  const mirror = mirrorVariant(token);
  if (mirror) {
    console.log(`${token[0]} mirrors to ${mirror[0]}`);
  }
});

// Get mirror types directly
const openCurlyMirror = mirrorVariantType(TokenType.OpenCurly);
console.log(openCurlyMirror); // '}' token type

const openParenMirror = mirrorVariantType(TokenType.OpenParen);
console.log(openParenMirror); // ')' token type

Token Structure and Usage

Understanding Token Structure

Tokens are tuple-like arrays with a specific structure:

// Example: TokenIdent for "foo"
const identToken: TokenIdent = [
  'ident-token',    // [0] - Token type
  'foo',            // [1] - String representation
  0,                // [2] - Start position
  3,                // [3] - End position
  { value: 'foo' }  // [4] - Parsed data
];

// Example: TokenDimension for "10px"
const dimensionToken: TokenDimension = [
  'dimension-token',           // [0] - Token type
  '10px',                      // [1] - String representation
  0,                           // [2] - Start position
  4,                           // [3] - End position
  {                            // [4] - Parsed data
    value: 10,
    unit: 'px',
    type: NumberType.Integer,
    signCharacter: undefined
  }
];

Accessing Token Data

import { tokenize, TokenType } from "@csstools/css-tokenizer";

const tokens = tokenize({ css: ".foo { width: 10px; }" });

tokens.forEach((token) => {
  const tokenType = token[0];              // Token type
  const representation = token[1];         // String representation
  const startPos = token[2];              // Start position
  const endPos = token[3];                // End position
  const data = token[4];                  // Parsed data

  if (tokenType === TokenType.Ident) {
    console.log('Identifier:', data.value);
  } else if (tokenType === TokenType.Dimension) {
    console.log(`Dimension: ${data.value}${data.unit}`);
  }
});

Important Notes

Use Parsed Data: Always use token[4] (parsed data) for analysis, not token[1] (string representation)
Stringification: token[1] is used when converting tokens back to CSS
Type Safety: Use type predicates for safe access to token data
Mutation: When mutating tokens, update both the parsed data and string representation