0
# Validation and Utilities
1

2
Validation functions and utility methods for working with sitemap data, URL normalization, and error handling. These functions provide the foundation for data validation and processing throughout the library.
3

4
## Capabilities
5

6
### Validation Functions
7

8
Core validation functionality that ensures sitemap data conforms to specifications.
9

10
```typescript { .api }
11
/**
12
 * Verifies all data passed in will comply with sitemap spec
13
 * @param conf - Sitemap item to validate
14
 * @param level - Logging/error level for validation failures
15
 * @param errorHandler - Custom error handling function
16
 * @returns Validated sitemap item
17
 */
18
function validateSMIOptions(
19
  conf: SitemapItem,
20
  level?: ErrorLevel,
21
  errorHandler?: ErrorHandler
22
): SitemapItem;
23

24
/**
25
 * Custom error handler function type
26
 * @param error - The validation error
27
 * @param level - Error level that triggered the handler
28
 */
29
type ErrorHandler = (error: Error, level: ErrorLevel) => void;
30

31
enum ErrorLevel {
32
  /** Validation will be skipped and nothing logged or thrown */
33
  SILENT = 'silent',
34
  
35
  /** If an invalid value is encountered, a console.warn will be called */
36
  WARN = 'warn',
37
  
38
  /** An Error will be thrown on encountering invalid data */
39
  THROW = 'throw'
40
}
41
```
42

43
**Usage Examples:**
44

45
```typescript
46
import { validateSMIOptions, ErrorLevel } from "sitemap";
47

48
// Validate with default warning level
49
const validatedItem = validateSMIOptions({
50
  url: "https://example.com/page",
51
  changefreq: "daily",
52
  priority: 0.8,
53
  img: [],
54
  video: [],
55
  links: []
56
});
57

58
// Validate with strict error handling
59
const strictItem = validateSMIOptions(
60
  {
61
    url: "https://example.com/page",
62
    changefreq: "invalid-freq", // This will throw
63
    priority: 2.0, // This will also throw
64
    img: [],
65
    video: [],
66
    links: []
67
  },
68
  ErrorLevel.THROW
69
);
70

71
// Validate with custom error handler
72
const customValidatedItem = validateSMIOptions(
73
  someItem,
74
  ErrorLevel.WARN,
75
  (error, level) => {
76
    console.log(`Validation ${level}: ${error.message}`);
77
    // Log to external service, etc.
78
  }
79
);
80
```
81

82
### URL Normalization
83

84
Functions for normalizing and processing URLs and sitemap items.
85

86
```typescript { .api }
87
/**
88
 * Converts the passed in sitemap entry into one capable of being consumed by SitemapItem
89
 * @param elem - The string or object to be converted
90
 * @param hostname - Base hostname for relative URLs
91
 * @param lastmodDateOnly - Whether to include only date in lastmod
92
 * @returns Normalized sitemap item
93
 */
94
function normalizeURL(
95
  elem: string | SitemapItemLoose,
96
  hostname?: string,
97
  lastmodDateOnly?: boolean
98
): SitemapItem;
99
```
100

101
**Usage Examples:**
102

103
```typescript
104
import { normalizeURL } from "sitemap";
105

106
// Normalize a simple URL string
107
const normalized = normalizeURL(
108
  "/relative/path",
109
  "https://example.com"
110
);
111
// Result: { url: "https://example.com/relative/path", img: [], video: [], links: [] }
112

113
// Normalize complex sitemap item
114
const complexItem = normalizeURL(
115
  {
116
    url: "/product/123",
117
    lastmod: "2023-01-01T12:30:45.123Z",
118
    img: "https://example.com/image.jpg", // String converted to Img object
119
    video: {
120
      thumbnail_loc: "https://example.com/thumb.jpg",
121
      title: "Product Video",
122
      description: "A great product demo"
123
    },
124
    priority: 0.8
125
  },
126
  "https://example.com",
127
  true // lastmodDateOnly
128
);
129
// Result: Full SitemapItem with normalized URLs and date-only lastmod
130
```
131

132
### Stream Utilities
133

134
Utility functions for working with streams and data processing.
135

136
```typescript { .api }
137
/**
138
 * Combines multiple streams into one
139
 * @param streams - Array of readable streams to combine
140
 * @param options - Transform options for the combined stream
141
 * @returns Single readable stream containing all data
142
 */
143
function mergeStreams(
144
  streams: Readable[],
145
  options?: TransformOptions
146
): Readable;
147

148
/**
149
 * Takes a stream likely from fs.createReadStream and returns a stream of sitemap items
150
 * @param stream - Stream of line-separated URLs or JSON
151
 * @param opts.isJSON - Whether stream contains JSON (auto-detected if undefined)
152
 * @returns Stream of sitemap options
153
 */
154
function lineSeparatedURLsToSitemapOptions(
155
  stream: Readable,
156
  opts?: { isJSON?: boolean }
157
): Readable;
158

159
/**
160
 * Wraps node's ReadLine in a stream for processing line-by-line input
161
 */
162
class ReadlineStream extends Readable {
163
  constructor(options: ReadlineStreamOptions);
164
  
165
  /** Internal readline interface */
166
  private _source: Interface;
167
}
168

169
interface ReadlineStreamOptions extends ReadableOptions {
170
  /** Input stream to read lines from */
171
  input: Readable;
172
}
173
```
174

175
**Usage Examples:**
176

177
```typescript
178
import { 
179
  mergeStreams, 
180
  lineSeparatedURLsToSitemapOptions,
181
  ReadlineStream 
182
} from "sitemap";
183
import { createReadStream } from "fs";
184

185
// Merge multiple URL files into one stream
186
const stream1 = createReadStream("urls1.txt");
187
const stream2 = createReadStream("urls2.txt");
188
const stream3 = createReadStream("urls3.txt");
189

190
const combinedStream = mergeStreams([stream1, stream2, stream3]);
191

192
// Convert line-separated URLs to sitemap options
193
const urlStream = lineSeparatedURLsToSitemapOptions(
194
  createReadStream("urls.txt")
195
);
196

197
urlStream.on('data', (item) => {
198
  console.log("Parsed item:", item);
199
});
200

201
// Process JSON lines
202
const jsonStream = lineSeparatedURLsToSitemapOptions(
203
  createReadStream("urls.jsonl"),
204
  { isJSON: true }
205
);
206

207
// Manual ReadlineStream usage
208
const readline = new ReadlineStream({
209
  input: createReadStream("large-file.txt")
210
});
211

212
readline.on('data', (line) => {
213
  console.log("Line:", line);
214
});
215
```
216

217
### Array Utilities
218

219
Utility functions for array processing.
220

221
```typescript { .api }
222
/**
223
 * Splits array into chunks of specified size (based on lodash implementation)
224
 * @param array - Array to split into chunks
225
 * @param size - Size of each chunk (default: 1)
226
 * @returns Array of chunks
227
 */
228
function chunk(array: any[], size?: number): any[];
229
```
230

231
**Usage Examples:**
232

233
```typescript
234
import { chunk } from "sitemap";
235

236
// Split URLs into batches for processing
237
const allUrls = Array.from({ length: 1000 }, (_, i) => `/page-${i}`);
238
const batches = chunk(allUrls, 100);
239

240
console.log(`Split ${allUrls.length} URLs into ${batches.length} batches`);
241

242
// Process each batch
243
for (const batch of batches) {
244
  console.log(`Processing batch of ${batch.length} URLs`);
245
  // Process batch...
246
}
247
```
248

249
## Type Guards and Validators
250

251
Functions for type checking and validation of sitemap data.
252

253
```typescript { .api }
254
/**
255
 * Type guard for change frequency values
256
 * @param freq - String to validate
257
 * @returns True if valid change frequency
258
 */
259
function isValidChangeFreq(freq: string): freq is EnumChangefreq;
260

261
/**
262
 * Type guard for yes/no values
263
 * @param yn - String to validate  
264
 * @returns True if valid yes/no value
265
 */
266
function isValidYesNo(yn: string): yn is EnumYesNo;
267

268
/**
269
 * Type guard for allow/deny values
270
 * @param ad - String to validate
271
 * @returns True if valid allow/deny value
272
 */
273
function isAllowDeny(ad: string): ad is EnumAllowDeny;
274

275
/**
276
 * Type guard for video price types
277
 * @param pt - String to validate
278
 * @returns True if valid price type
279
 */
280
function isPriceType(pt: string | PriceType): pt is PriceType;
281

282
/**
283
 * Type guard for video resolutions
284
 * @param res - String to validate
285
 * @returns True if valid resolution
286
 */
287
function isResolution(res: string): res is Resolution;
288
```
289

290
**Usage Examples:**
291

292
```typescript
293
import { 
294
  isValidChangeFreq, 
295
  isValidYesNo, 
296
  isPriceType 
297
} from "sitemap";
298

299
// Validate user input
300
function processChangeFreq(input: string) {
301
  if (isValidChangeFreq(input)) {
302
    // TypeScript knows input is EnumChangefreq
303
    console.log("Valid change frequency:", input);
304
    return input;
305
  } else {
306
    throw new Error(`Invalid change frequency: ${input}`);
307
  }
308
}
309

310
// Validate video metadata
311
function processVideoData(data: any) {
312
  if (data.family_friendly && !isValidYesNo(data.family_friendly)) {
313
    throw new Error("Invalid family_friendly value");
314
  }
315
  
316
  if (data.price && data.price_type && !isPriceType(data.price_type)) {
317
    throw new Error("Invalid price type");
318
  }
319
  
320
  return data;
321
}
322
```
323

324
## Error Handling Patterns
325

326
### Custom Error Handlers
327

328
```typescript
329
import { validateSMIOptions, ErrorLevel } from "sitemap";
330

331
// Collect all validation errors
332
const validationErrors: Error[] = [];
333

334
const errorCollector: ErrorHandler = (error, level) => {
335
  validationErrors.push(error);
336
  
337
  if (level === ErrorLevel.THROW) {
338
    throw error;
339
  }
340
};
341

342
// Validate multiple items
343
const items = [...]; // Array of sitemap items
344
const validatedItems = items.map(item => 
345
  validateSMIOptions(item, ErrorLevel.WARN, errorCollector)
346
);
347

348
console.log(`Validation complete. Found ${validationErrors.length} issues.`);
349
```
350

351
### Batch Processing with Error Recovery
352

353
```typescript
354
import { normalizeURL, validateSMIOptions, ErrorLevel } from "sitemap";
355

356
function processSitemapBatch(rawItems: any[], hostname: string) {
357
  const successful: SitemapItem[] = [];
358
  const failed: { item: any; error: Error }[] = [];
359
  
360
  for (const rawItem of rawItems) {
361
    try {
362
      const normalized = normalizeURL(rawItem, hostname);
363
      const validated = validateSMIOptions(normalized, ErrorLevel.THROW);
364
      successful.push(validated);
365
    } catch (error) {
366
      failed.push({ item: rawItem, error: error as Error });
367
    }
368
  }
369
  
370
  return { successful, failed };
371
}
372
```
373

374
## Constants and Enums
375

376
```typescript { .api }
377
/** Array of all valid change frequency values */
378
const CHANGEFREQ: EnumChangefreq[];
379

380
/** Regex validators for various sitemap attributes */
381
const validators: { [index: string]: RegExp };
382

383
enum EnumChangefreq {
384
  DAILY = 'daily',
385
  MONTHLY = 'monthly', 
386
  ALWAYS = 'always',
387
  HOURLY = 'hourly',
388
  WEEKLY = 'weekly',
389
  YEARLY = 'yearly',
390
  NEVER = 'never'
391
}
392

393
enum EnumYesNo {
394
  YES = 'YES',
395
  NO = 'NO',
396
  Yes = 'Yes', 
397
  No = 'No',
398
  yes = 'yes',
399
  no = 'no'
400
}
401

402
enum EnumAllowDeny {
403
  ALLOW = 'allow',
404
  DENY = 'deny'
405
}
406

407
type PriceType = 'rent' | 'purchase' | 'RENT' | 'PURCHASE';
408
type Resolution = 'HD' | 'hd' | 'sd' | 'SD';
409
```