0
# Utilities
1

2
Utility functions and the bundled Prism instance for advanced usage, custom implementations, and direct access to tokenization functionality.
3

4
## Capabilities
5

6
### Bundled Prism Instance
7

8
Pre-configured Prism.js instance with common programming languages bundled for optimal performance and no global namespace pollution.
9

10
```typescript { .api }
11
/**
12
 * Pre-configured Prism.js instance with bundled language definitions
13
 * Includes common programming languages without polluting global namespace
14
 */
15
declare const Prism: PrismLib;
16

17
interface PrismLib {
18
  /** Object containing grammar definitions for supported languages */
19
  languages: Record<string, PrismGrammar>;
20
  /** Tokenize text using a specific grammar */
21
  tokenize(text: string, grammar: PrismGrammar): (string | PrismToken)[];
22
  /** Hook system for extending Prism functionality */
23
  hooks: {
24
    /** Run hooks for specific events */
25
    run(name: string, env: EnvConfig): void;
26
  };
27
}
28

29
type PrismGrammar = import("prismjs").Grammar;
30
```
31

32
**Supported Languages:**
33
- **Web**: markup (HTML), jsx, tsx, javascript (via js-extras)
34
- **Systems**: rust, go, cpp, swift, kotlin, objectivec
35
- **Functional**: reason
36
- **Scripting**: python
37
- **Data**: json, yaml, graphql
38
- **Documentation**: markdown
39

40
**Usage Examples:**
41

42
```typescript
43
import { Prism } from "prism-react-renderer";
44

45
// Check if a language is supported
46
function isLanguageSupported(language: string): boolean {
47
  return language.toLowerCase() in Prism.languages;
48
}
49

50
// Get available languages
51
const availableLanguages = Object.keys(Prism.languages);
52
console.log("Supported languages:", availableLanguages);
53

54
// Direct tokenization
55
const code = "const greeting = 'Hello, World!';";
56
const grammar = Prism.languages.javascript;
57
const tokens = Prism.tokenize(code, grammar);
58

59
// Use with custom hook implementations
60
function useCustomTokenizer(code: string, language: string) {
61
  const grammar = Prism.languages[language.toLowerCase()];
62
  
63
  if (!grammar) {
64
    console.warn(`Grammar not found for language: ${language}`);
65
    return [code]; // Return as plain text
66
  }
67
  
68
  return Prism.tokenize(code, grammar);
69
}
70

71
// Language fallback handling
72
function getGrammarWithFallback(language: string): PrismGrammar | null {
73
  const normalizedLang = language.toLowerCase();
74
  
75
  // Direct match
76
  if (Prism.languages[normalizedLang]) {
77
    return Prism.languages[normalizedLang];
78
  }
79
  
80
  // Common aliases
81
  const aliases: Record<string, string> = {
82
    'js': 'javascript',
83
    'ts': 'typescript',  
84
    'py': 'python',
85
    'md': 'markdown',
86
    'yml': 'yaml'
87
  };
88
  
89
  const aliasTarget = aliases[normalizedLang];
90
  if (aliasTarget && Prism.languages[aliasTarget]) {
91
    return Prism.languages[aliasTarget];
92
  }
93
  
94
  return null;
95
}
96
```
97

98
### normalizeTokens Function
99

100
Utility function that converts Prism's raw token output into a normalized 2D array structure organized by lines.
101

102
```typescript { .api }
103
/**
104
 * Converts Prism tokens into normalized Token structure grouped by lines
105
 * Handles nested tokens, newlines, and empty lines consistently
106
 * @param tokens - Raw tokens from Prism.tokenize()
107
 * @returns 2D array of normalized tokens organized by lines
108
 */
109
declare function normalizeTokens(tokens: (PrismToken | string)[]): Token[][];
110

111
interface Token {
112
  /** Array of token type classifications */
113
  types: string[];
114
  /** Token content/text */
115
  content: string;
116
  /** Whether this token represents an empty line */
117
  empty?: boolean;
118
}
119
```
120

121
**Usage Examples:**
122

123
```typescript
124
import { normalizeTokens, Prism } from "prism-react-renderer";
125

126
// Basic normalization
127
const code = `function greet(name) {
128
  console.log('Hello, ' + name);
129
}`;
130

131
const grammar = Prism.languages.javascript;
132
const rawTokens = Prism.tokenize(code, grammar);
133
const normalizedTokens = normalizeTokens(rawTokens);
134

135
console.log(normalizedTokens);
136
// [
137
//   [{ types: ['keyword'], content: 'function' }, { types: ['plain'], content: ' ' }, ...],
138
//   [{ types: ['plain'], content: '  ' }, { types: ['builtin'], content: 'console' }, ...],
139
//   [{ types: ['punctuation'], content: '}' }]
140
// ]
141

142
// Handle empty lines
143
const codeWithEmptyLines = `const a = 1;
144

145
const b = 2;`;
146

147
const tokensWithEmpty = normalizeTokens(Prism.tokenize(codeWithEmptyLines, grammar));
148
// Second array will contain: [{ types: ['plain'], content: '\n', empty: true }]
149

150
// Custom rendering with normalized tokens
151
function renderTokens(tokens: Token[][]) {
152
  return tokens.map((line, lineIndex) => (
153
    <div key={lineIndex} className="code-line">
154
      {line.map((token, tokenIndex) => (
155
        <span
156
          key={tokenIndex}
157
          className={token.types.join(' ')}
158
          data-empty={token.empty || undefined}
159
        >
160
          {token.content}
161
        </span>
162
      ))}
163
    </div>
164
  ));
165
}
166
```
167

168

169
## Advanced Usage Patterns
170

171
### Custom Language Support
172

173
For languages not included in the bundle, you can extend Prism:
174

175
```typescript
176
import { Prism } from "prism-react-renderer";
177

178
// Note: This requires additional Prism language definitions
179
// You would need to import and register the language manually
180

181
// Example of how to check and handle unsupported languages
182
function highlightWithFallback(code: string, language: string) {
183
  const grammar = Prism.languages[language.toLowerCase()];
184
  
185
  if (!grammar) {
186
    // Fallback to plain text
187
    return normalizeTokens([code]);
188
  }
189
  
190
  const tokens = Prism.tokenize(code, grammar);
191
  return normalizeTokens(tokens);
192
}
193
```
194

195
### Performance Optimization
196

197
```typescript
198
import { useTokenize, Prism } from "prism-react-renderer";
199
import { useMemo } from "react";
200

201
function OptimizedHighlighter({ code, language }) {
202
  // Grammar lookup is memoized
203
  const grammar = useMemo(() => {
204
    return Prism.languages[language.toLowerCase()];
205
  }, [language]);
206
  
207
  // useTokenize already includes memoization
208
  const tokens = useTokenize({
209
    prism: Prism,
210
    code,
211
    language,
212
    grammar
213
  });
214
  
215
  return (
216
    <div>
217
      {tokens.map((line, i) => (
218
        <div key={i}>
219
          {line.map((token, j) => (
220
            <span key={j} className={token.types.join(' ')}>
221
              {token.content}
222
            </span>
223
          ))}
224
        </div>
225
      ))}
226
    </div>
227
  );
228
}
229
```
230

231
## Supporting Types
232

233
```typescript { .api }
234
type Language = string;
235
type PrismGrammar = import("prismjs").Grammar;
236
type PrismLib = typeof import("prismjs");
237
```