0
# Testing
1
2
TSLint provides a comprehensive testing framework for validating custom rules with markup-based test files and automated test runners.
3
4
## Testing Framework Overview
5
6
TSLint's testing system uses a markup format that allows you to specify expected lint failures directly in test source code using comments.
7
8
### Core Testing Functions
9
10
```typescript { .api }
11
import { Test } from 'tslint';
12
13
// Run tests in a directory
14
function runTest(testDirectory: string, rulesDirectory?: string | string[]): TestResult
15
16
// Run tests matching patterns
17
function runTests(patterns: string[], rulesDirectory?: string | string[]): TestResult[]
18
19
// Handle single test result output
20
function consoleTestResultHandler(testResult: TestResult, logger: Logger): void
21
22
// Handle multiple test results output
23
function consoleTestResultsHandler(testResults: TestResult[], logger: Logger): void
24
```
25
26
### Test Result Interfaces
27
28
```typescript { .api }
29
interface Logger {
30
log(message: string): void;
31
error(message: string): void;
32
}
33
34
interface TestResult {
35
directory: string;
36
results: {
37
[fileName: string]: TestOutput | SkippedTest;
38
};
39
}
40
41
interface TestOutput {
42
skipped: false;
43
errorsFromLinter: LintError[];
44
errorsFromMarkup: LintError[];
45
fixesFromLinter: string;
46
fixesFromMarkup: string;
47
markupFromLinter: string;
48
markupFromMarkup: string;
49
}
50
51
interface SkippedTest {
52
skipped: true;
53
requirement: string;
54
}
55
56
interface LintError {
57
endPos: {
58
col: number;
59
line: number;
60
};
61
message: string;
62
startPos: {
63
col: number;
64
line: number;
65
};
66
}
67
```
68
69
## Test File Structure
70
71
### Test Directory Layout
72
73
```
74
test/
75
├── rules/
76
│ ├── my-rule/
77
│ │ ├── test.ts.lint # Test file with markup
78
│ │ ├── test.ts.fix # Expected fix result (optional)
79
│ │ └── tslint.json # Rule configuration
80
│ └── another-rule/
81
│ ├── valid.ts.lint
82
│ ├── invalid.ts.lint
83
│ └── tslint.json
84
└── custom-rules/ # Custom rules directory
85
├── myRuleRule.ts
86
└── anotherRuleRule.ts
87
```
88
89
### Test Configuration (tslint.json)
90
91
```json
92
{
93
"rules": {
94
"my-rule": [true, "option1", "option2"],
95
"semicolon": true
96
}
97
}
98
```
99
100
## Test Markup Format
101
102
### Basic Markup Syntax
103
104
Test files use special comments to indicate expected failures:
105
106
```typescript
107
// In test.ts.lint file
108
function example() {
109
console.log('test');
110
~~~~~~~~~~~~~~~ [error message here]
111
}
112
113
let unused = 5;
114
~~~~~~ [Variable 'unused' is never used]
115
116
const noSemicolon = true
117
~ [Missing semicolon]
118
```
119
120
**Markup Rules:**
121
- `~` characters mark the span where an error is expected
122
- `[message]` specifies the expected error message
123
- Line must align with the code position
124
- Multiple errors per line are supported
125
126
### Advanced Markup Features
127
128
#### Multi-line Errors
129
130
```typescript
131
function longFunction(
132
parameter1: string,
133
~~~~~~~~~~~~~~~~~~~ [Parameter should be on single line]
134
parameter2: number
135
~~~~~~~~~~~~~~~~~~
136
) {
137
return parameter1 + parameter2;
138
}
139
```
140
141
#### Multiple Errors on Same Line
142
143
```typescript
144
let a = 1, b = 2;
145
~ [Variable 'a' should use const]
146
~ [Variable 'b' should use const]
147
```
148
149
#### Error Message Escaping
150
151
```typescript
152
const message = 'Don\'t use quotes';
153
~~~~~~~~~~~~~~~~~ [Don't use single quotes inside strings]
154
155
// Escape brackets in messages
156
const test = `template ${variable}`;
157
~~~~~~~~~~~ [Prefer 'string' over \`template\`]
158
```
159
160
### Fix Testing
161
162
For rules that provide auto-fixes, create a `.fix` file with the expected result:
163
164
**test.ts.lint:**
165
```typescript
166
const value = "double";
167
~~~~~~~~ [Use single quotes]
168
```
169
170
**test.ts.fix:**
171
```typescript
172
const value = 'double';
173
```
174
175
## Running Tests
176
177
### Command Line Testing
178
179
```bash
180
# Run all tests in test directory
181
tslint --test test/rules/
182
183
# Run specific rule tests
184
tslint --test test/rules/my-rule/
185
186
# Run tests with custom rules directory
187
tslint --test test/rules/ --rules-dir custom-rules/
188
```
189
190
### Programmatic Testing
191
192
```typescript
193
import { Test } from 'tslint';
194
195
// Run single test directory
196
const testResult = Test.runTest('test/rules/my-rule', 'custom-rules');
197
198
// Handle results
199
Test.consoleTestResultHandler(testResult, console);
200
201
// Check if tests passed
202
const passed = Object.values(testResult.results).every(result => {
203
if (result.skipped) return true;
204
return result.errorsFromLinter.length === 0;
205
});
206
207
console.log(`Tests ${passed ? 'PASSED' : 'FAILED'}`);
208
```
209
210
### Batch Testing
211
212
```typescript
213
import { Test } from 'tslint';
214
import * as glob from 'glob';
215
216
// Run multiple test directories
217
const testDirs = glob.sync('test/rules/*');
218
const testResults = Test.runTests(testDirs, 'custom-rules');
219
220
// Process all results
221
Test.consoleTestResultsHandler(testResults, console);
222
223
// Generate summary
224
const summary = testResults.reduce((acc, result) => {
225
Object.values(result.results).forEach(test => {
226
if (test.skipped) {
227
acc.skipped++;
228
} else if (test.errorsFromLinter.length > 0) {
229
acc.failed++;
230
} else {
231
acc.passed++;
232
}
233
});
234
return acc;
235
}, { passed: 0, failed: 0, skipped: 0 });
236
237
console.log(`Summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped`);
238
```
239
240
## Advanced Testing Patterns
241
242
### Testing Rule Options
243
244
```typescript
245
// test/rules/my-rule/default-options.ts.lint
246
console.log('test');
247
~~~~~~~~~~~~~~~~~~~ [Default message]
248
249
// test/rules/my-rule/custom-options.ts.lint
250
console.log('test');
251
~~~~~~~~~~~~~~~~~~~ [Custom message with option]
252
253
// test/rules/my-rule/tslint.json
254
{
255
"rules": {
256
"my-rule": [true, { "customMessage": "Custom message with option" }]
257
}
258
}
259
```
260
261
### Testing Type-Aware Rules
262
263
Type-aware rules require a `tsconfig.json` in the test directory:
264
265
```
266
test/rules/my-typed-rule/
267
├── test.ts.lint
268
├── tslint.json
269
├── tsconfig.json
270
└── helper.ts # Additional files for type checking
271
```
272
273
**tsconfig.json:**
274
```json
275
{
276
"compilerOptions": {
277
"target": "es2017",
278
"module": "commonjs",
279
"strict": true
280
},
281
"files": [
282
"test.ts",
283
"helper.ts"
284
]
285
}
286
```
287
288
**test.ts.lint:**
289
```typescript
290
import { Helper } from './helper';
291
292
const helper: Helper = new Helper();
293
const result: string = helper.getValue(); // Type info available
294
~~~~~~~~~~~~~~~ [Expected number, got string]
295
```
296
297
### Testing JavaScript Rules
298
299
Test JavaScript-specific rules by using `.js.lint` extension:
300
301
```
302
test/rules/my-js-rule/
303
├── test.js.lint
304
└── tslint.json
305
```
306
307
**tslint.json:**
308
```json
309
{
310
"jsRules": {
311
"my-js-rule": true
312
}
313
}
314
```
315
316
### Conditional Testing
317
318
Skip tests based on conditions using special markup:
319
320
```typescript
321
// test-conditional.ts.lint
322
/* tslint:disable-file */ // Skip entire file
323
324
// Or skip with reason
325
/// <reference path="./skipThis.d.ts" />
326
/* tslint:disable */
327
// This test is skipped because TypeScript version < 3.0
328
/* tslint:enable */
329
```
330
331
## Custom Test Runners
332
333
### Building a Test Runner
334
335
```typescript
336
import { Test, TestResult } from 'tslint';
337
import * as fs from 'fs';
338
import * as path from 'path';
339
340
class CustomTestRunner {
341
private rulesDirectory: string;
342
343
constructor(rulesDirectory: string) {
344
this.rulesDirectory = rulesDirectory;
345
}
346
347
async runAllTests(): Promise<boolean> {
348
const testDirs = this.findTestDirectories();
349
const results: TestResult[] = [];
350
351
for (const testDir of testDirs) {
352
const result = Test.runTest(testDir, this.rulesDirectory);
353
results.push(result);
354
}
355
356
return this.processResults(results);
357
}
358
359
private findTestDirectories(): string[] {
360
const testRoot = './test/rules';
361
362
return fs.readdirSync(testRoot)
363
.map(dir => path.join(testRoot, dir))
364
.filter(dir => fs.statSync(dir).isDirectory());
365
}
366
367
private processResults(results: TestResult[]): boolean {
368
let allPassed = true;
369
370
results.forEach(result => {
371
console.log(`\nTesting ${result.directory}:`);
372
373
Object.entries(result.results).forEach(([fileName, testOutput]) => {
374
if (testOutput.skipped) {
375
console.log(` ${fileName}: SKIPPED (${testOutput.reason})`);
376
} else {
377
const errors = testOutput.errorsFromLinter;
378
if (errors.length === 0) {
379
console.log(` ${fileName}: PASSED`);
380
} else {
381
console.log(` ${fileName}: FAILED`);
382
errors.forEach(error => {
383
console.log(` ${error.message} at ${error.startPos.line}:${error.startPos.col}`);
384
});
385
allPassed = false;
386
}
387
}
388
});
389
});
390
391
return allPassed;
392
}
393
}
394
395
// Usage
396
const runner = new CustomTestRunner('./custom-rules');
397
runner.runAllTests().then(passed => {
398
process.exit(passed ? 0 : 1);
399
});
400
```
401
402
### Continuous Integration Testing
403
404
```typescript
405
import { Test } from 'tslint';
406
import * as fs from 'fs';
407
408
interface CITestResult {
409
success: boolean;
410
summary: {
411
total: number;
412
passed: number;
413
failed: number;
414
skipped: number;
415
};
416
failures: Array<{
417
rule: string;
418
file: string;
419
errors: string[];
420
}>;
421
}
422
423
function runCITests(): CITestResult {
424
const testResults = Test.runTests(['test/rules/*'], 'src/rules');
425
426
const result: CITestResult = {
427
success: true,
428
summary: { total: 0, passed: 0, failed: 0, skipped: 0 },
429
failures: []
430
};
431
432
testResults.forEach(testResult => {
433
const ruleName = path.basename(testResult.directory);
434
435
Object.entries(testResult.results).forEach(([fileName, output]) => {
436
result.summary.total++;
437
438
if (output.skipped) {
439
result.summary.skipped++;
440
} else if (output.errorsFromLinter.length > 0) {
441
result.summary.failed++;
442
result.success = false;
443
444
result.failures.push({
445
rule: ruleName,
446
file: fileName,
447
errors: output.errorsFromLinter.map(e => e.message)
448
});
449
} else {
450
result.summary.passed++;
451
}
452
});
453
});
454
455
return result;
456
}
457
458
// Generate CI report
459
const ciResult = runCITests();
460
fs.writeFileSync('test-results.json', JSON.stringify(ciResult, null, 2));
461
462
if (!ciResult.success) {
463
console.error('Tests failed!');
464
process.exit(1);
465
}
466
```
467
468
### Test Coverage Analysis
469
470
```typescript
471
import { Test, TestResult } from 'tslint';
472
import * as fs from 'fs';
473
import * as glob from 'glob';
474
475
interface CoverageReport {
476
rulesCovered: string[];
477
rulesUntested: string[];
478
coveragePercentage: number;
479
testStats: {
480
totalTests: number;
481
passingTests: number;
482
failingTests: number;
483
};
484
}
485
486
function analyzeCoverage(): CoverageReport {
487
// Find all available rules
488
const ruleFiles = glob.sync('src/rules/*Rule.ts');
489
const availableRules = ruleFiles.map(file =>
490
path.basename(file, 'Rule.ts').replace(/([A-Z])/g, '-$1').toLowerCase().substring(1)
491
);
492
493
// Find tested rules
494
const testDirs = glob.sync('test/rules/*');
495
const testedRules = testDirs.map(dir => path.basename(dir));
496
497
// Run tests to get statistics
498
const testResults = Test.runTests(testDirs, 'src/rules');
499
500
const testStats = testResults.reduce((stats, result) => {
501
Object.values(result.results).forEach(output => {
502
stats.totalTests++;
503
if (!output.skipped) {
504
if (output.errorsFromLinter.length === 0) {
505
stats.passingTests++;
506
} else {
507
stats.failingTests++;
508
}
509
}
510
});
511
return stats;
512
}, { totalTests: 0, passingTests: 0, failingTests: 0 });
513
514
const rulesUntested = availableRules.filter(rule => !testedRules.includes(rule));
515
516
return {
517
rulesCovered: testedRules,
518
rulesUntested,
519
coveragePercentage: Math.round((testedRules.length / availableRules.length) * 100),
520
testStats
521
};
522
}
523
524
// Generate coverage report
525
const coverage = analyzeCoverage();
526
console.log(`Rule Coverage: ${coverage.coveragePercentage}% (${coverage.rulesCovered.length}/${coverage.rulesCovered.length + coverage.rulesUntested.length})`);
527
console.log(`Untested rules: ${coverage.rulesUntested.join(', ')}`);
528
```
529
530
## Best Practices
531
532
### Test Development Guidelines
533
534
1. **Comprehensive Coverage**: Test all code paths and edge cases
535
2. **Clear Test Names**: Use descriptive file names for different scenarios
536
3. **Proper Markup**: Ensure markup exactly matches expected error positions
537
4. **Test Fixes**: Always test auto-fix functionality when applicable
538
5. **Type-aware Testing**: Use TypeScript projects for type-dependent rules
539
6. **Edge Cases**: Test boundary conditions and invalid inputs
540
7. **Configuration Testing**: Test different rule option combinations
541
542
### Test Organization
543
544
```
545
test/
546
├── rules/
547
│ ├── rule-name/
548
│ │ ├── basic/
549
│ │ │ ├── test.ts.lint
550
│ │ │ └── tslint.json
551
│ │ ├── with-options/
552
│ │ │ ├── test.ts.lint
553
│ │ │ └── tslint.json
554
│ │ ├── edge-cases/
555
│ │ │ ├── empty-file.ts.lint
556
│ │ │ ├── large-file.ts.lint
557
│ │ │ └── tslint.json
558
│ │ └── fixes/
559
│ │ ├── test.ts.lint
560
│ │ ├── test.ts.fix
561
│ │ └── tslint.json
562
```
563
564
### Debugging Failed Tests
565
566
```typescript
567
import { Test } from 'tslint';
568
569
function debugFailedTest(testPath: string) {
570
const result = Test.runTest(testPath, 'custom-rules');
571
572
Object.entries(result.results).forEach(([fileName, output]) => {
573
if (!output.skipped && output.errorsFromLinter.length > 0) {
574
console.log(`\nDebugging ${fileName}:`);
575
console.log('Expected errors from markup:');
576
output.errorsFromMarkup.forEach(error => {
577
console.log(` ${error.startPos.line}:${error.startPos.col} - ${error.message}`);
578
});
579
580
console.log('Actual errors from linter:');
581
output.errorsFromLinter.forEach(error => {
582
console.log(` ${error.startPos.line}:${error.startPos.col} - ${error.message}`);
583
});
584
585
console.log('Differences:');
586
// Compare and show differences
587
const expectedMessages = new Set(output.errorsFromMarkup.map(e => e.message));
588
const actualMessages = new Set(output.errorsFromLinter.map(e => e.message));
589
590
expectedMessages.forEach(msg => {
591
if (!actualMessages.has(msg)) {
592
console.log(` Missing: ${msg}`);
593
}
594
});
595
596
actualMessages.forEach(msg => {
597
if (!expectedMessages.has(msg)) {
598
console.log(` Unexpected: ${msg}`);
599
}
600
});
601
}
602
});
603
}
604
```