0
# Data Loading
1

2
Vega's data loading system provides comprehensive support for loading data from multiple sources, automatic type inference, format detection, and both synchronous and asynchronous data access patterns.
3

4
## Capabilities
5

6
### Core Data Loading
7

8
Main data loader factory and data reading utilities.
9

10
```typescript { .api }
11
/**
12
 * Create a data loader with specified options
13
 * @param options - Loader configuration options
14
 * @returns Configured loader instance
15
 */
16
function loader(options?: LoaderOptions): Loader;
17

18
/**
19
 * Read and parse data from string with format specification
20
 * @param data - Raw data string
21
 * @param schema - Format specification
22
 * @param dateParse - Optional custom date parsing function
23
 * @returns Array of parsed data objects
24
 */
25
function read(data: string, schema: Format, dateParse?: (dateString: string) => Date): object[];
26

27
interface LoaderOptions {
28
  /** Base URL for relative paths */
29
  baseURL?: string;
30
  
31
  /** Default file format when not specified */
32
  defaultProtocol?: string;
33
  
34
  /** HTTP request options */
35
  http?: RequestInit;
36
  
37
  /** File system access options */
38
  file?: FileOptions;
39
  
40
  /** Custom format parsers */
41
  formatParsers?: { [format: string]: FormatParser };
42
}
43

44
interface FileOptions {
45
  /** Local file system access mode */
46
  mode?: 'file' | 'http';
47
  
48
  /** File encoding */
49
  encoding?: string;
50
}
51

52
interface FormatParser {
53
  /** Parse function for the format */
54
  parse: (data: string, format: Format) => any[];
55
  
56
  /** MIME type for the format */
57
  type?: string;
58
}
59
```
60

61
### Loader Interface
62

63
Complete data loader interface for HTTP and file system access.
64

65
```typescript { .api }
66
/**
67
 * Data loader interface for accessing external data sources
68
 */
69
interface Loader {
70
  /**
71
   * Load data from URI
72
   * @param uri - Data source URI
73
   * @param options - Optional loading context and options
74
   * @returns Promise resolving to loaded data string
75
   */
76
  load: (uri: string, options?: LoaderOptionsWithContext) => Promise<string>;
77
  
78
  /**
79
   * Sanitize and validate URI for security
80
   * @param uri - URI to sanitize
81
   * @param options - Loading context
82
   * @returns Promise resolving to sanitized URI object
83
   */
84
  sanitize: (uri: string, options: LoaderOptionsWithContext) => Promise<{ href: string }>;
85
  
86
  /**
87
   * HTTP-specific loading method
88
   * @param uri - HTTP URL
89
   * @param options - HTTP request options
90
   * @returns Promise resolving to response text
91
   */
92
  http: (uri: string, options?: Partial<RequestInit>) => Promise<string>;
93
  
94
  /**
95
   * File system loading method
96
   * @param filename - Local file path
97
   * @returns Promise resolving to file contents
98
   */
99
  file: (filename: string) => Promise<string>;
100
  
101
  /**
102
   * Add custom protocol handler
103
   * @param protocol - Protocol name (e.g., 'custom:')
104
   * @param handler - Handler function
105
   */
106
  addProtocolHandler?: (protocol: string, handler: ProtocolHandler) => void;
107
}
108

109
interface LoaderOptionsWithContext {
110
  /** Request context object */
111
  context?: any;
112
  
113
  /** HTTP headers */
114
  headers?: { [key: string]: string };
115
  
116
  /** Request credentials mode */
117
  credentials?: RequestCredentials;
118
  
119
  /** Request method */
120
  method?: string;
121
  
122
  /** Request body */
123
  body?: BodyInit;
124
}
125

126
type ProtocolHandler = (uri: string, options: LoaderOptionsWithContext) => Promise<string>;
127
```
128

129
### Type Inference
130

131
Automatic type detection and inference for loaded data.
132

133
```typescript { .api }
134
/**
135
 * Infer data type for a field from sample values
136
 * @param values - Array of sample values
137
 * @param field - Optional field name for context
138
 * @returns Type inference result
139
 */
140
function inferType(values: readonly any[], field?: string): TypeInference;
141

142
/**
143
 * Infer types for multiple fields from sample data
144
 * @param values - Array of data objects
145
 * @param fields - Array of field names to analyze
146
 * @returns Object mapping field names to type inference results
147
 */
148
function inferTypes(values: readonly any[], fields: readonly string[]): { [field: string]: TypeInference };
149

150
interface TypeInference {
151
  /** Inferred data type */
152
  type: DataType;
153
  
154
  /** Confidence level (0-1) */
155
  confidence?: number;
156
  
157
  /** Sample parsed values */
158
  samples?: any[];
159
  
160
  /** Format hint for parsing */
161
  format?: string;
162
}
163

164
type DataType = 'number' | 'integer' | 'date' | 'boolean' | 'string';
165
```
166

167
### Format Handling
168

169
Data format detection, parsing, and response type handling.
170

171
```typescript { .api }
172
/**
173
 * Get format specification from URI or data
174
 * @param name - File name or URI
175
 * @param data - Optional data sample
176
 * @returns Format specification object
177
 */
178
function format(name: string, data?: string): Format;
179

180
/**
181
 * Get available format specifications
182
 * @returns Object mapping format names to specifications
183
 */
184
function formats(): { [name: string]: Format };
185

186
/**
187
 * Get appropriate response type for format
188
 * @param type - Format type
189
 * @returns Response type ('text', 'json', 'arrayBuffer', etc.)
190
 */
191
function responseType(type: string): ResponseType;
192

193
interface Format {
194
  /** Format type identifier */
195
  type: string;
196
  
197
  /** Parse function for the format */
198
  parse?: (data: string, format: Format) => any[];
199
  
200
  /** Field delimiter (CSV, TSV, etc.) */
201
  delimiter?: string;
202
  
203
  /** Header row handling */
204
  header?: boolean;
205
  
206
  /** Skip rows from beginning */
207
  skipRows?: number;
208
  
209
  /** Property path for nested data (JSON) */
210
  property?: string;
211
  
212
  /** Feature property path (GeoJSON) */
213
  feature?: string;
214
  
215
  /** Mesh property path (TopoJSON) */
216
  mesh?: string;
217
  
218
  /** Custom field parsing functions */
219
  parsers?: { [field: string]: (value: string) => any };
220
}
221

222
type ResponseType = 'text' | 'json' | 'arrayBuffer' | 'blob';
223
```
224

225
### Type Parsers
226

227
Built-in and custom type parsing functions.
228

229
```typescript { .api }
230
/**
231
 * Registry of built-in type parsing functions
232
 */
233
const typeParsers: {
234
  /** Parse as number */
235
  number: (value: string) => number | null;
236
  
237
  /** Parse as integer */
238
  integer: (value: string) => number | null;
239
  
240
  /** Parse as boolean */
241
  boolean: (value: string) => boolean | null;
242
  
243
  /** Parse as date */
244
  date: (value: string) => Date | null;
245
  
246
  /** Parse as string (identity) */
247
  string: (value: string) => string;
248
  
249
  /** Auto-detect and parse */
250
  auto: (value: string) => any;
251
};
252

253
/**
254
 * Add custom type parser
255
 * @param type - Type name
256
 * @param parser - Parser function
257
 */
258
function addTypeParser(type: string, parser: (value: string) => any): void;
259
```
260

261
## Built-in Format Support
262

263
### CSV/TSV Formats
264

265
```typescript { .api }
266
interface CSVFormat extends Format {
267
  type: 'csv' | 'tsv';
268
  
269
  /** Field delimiter character */
270
  delimiter: ',' | '\t' | string;
271
  
272
  /** Quote character */
273
  quote?: string;
274
  
275
  /** Escape character */
276
  escape?: string;
277
  
278
  /** Header row present */
279
  header: boolean;
280
  
281
  /** Comment line prefix */
282
  comment?: string;
283
}
284
```
285

286
### JSON Formats
287

288
```typescript { .api }
289
interface JSONFormat extends Format {
290
  type: 'json';
291
  
292
  /** Property path to extract array data */
293
  property?: string;
294
  
295
  /** Automatically parse date strings */
296
  autoParseDate?: boolean;
297
}
298
```
299

300
### GeoJSON/TopoJSON Formats
301

302
```typescript { .api }
303
interface GeoFormat extends Format {
304
  type: 'geojson' | 'topojson';
305
  
306
  /** Feature collection property */
307
  feature?: string;
308
  
309
  /** Mesh property (TopoJSON) */
310
  mesh?: string;
311
}
312
```
313

314
### DSV (Delimiter-Separated Values)
315

316
```typescript { .api }
317
interface DSVFormat extends Format {
318
  type: 'dsv';
319
  
320
  /** Custom delimiter */
321
  delimiter: string;
322
  
323
  /** Header handling */
324
  header: boolean;
325
}
326
```
327

328
## Usage Examples
329

330
### Basic Data Loading
331

332
```typescript
333
import { loader, read, format } from "vega";
334

335
// Create loader
336
const dataLoader = loader({
337
  baseURL: 'https://api.example.com/',
338
  http: { credentials: 'same-origin' }
339
});
340

341
// Load CSV data
342
dataLoader.load('data/sales.csv').then(data => {
343
  const fmt = format('sales.csv');
344
  const parsed = read(data, fmt);
345
  console.log(parsed);
346
});
347
```
348

349
### Async/Await Pattern
350

351
```typescript
352
import { loader, read, format } from "vega";
353

354
async function loadData() {
355
  const dataLoader = loader();
356
  
357
  try {
358
    // Load multiple data sources
359
    const [csvData, jsonData] = await Promise.all([
360
      dataLoader.load('data.csv'),
361
      dataLoader.load('config.json')
362
    ]);
363
    
364
    // Parse CSV
365
    const csvFormat = format('data.csv');
366
    const csvRecords = read(csvData, csvFormat);
367
    
368
    // Parse JSON
369
    const jsonFormat = format('config.json');
370
    const jsonRecords = read(jsonData, jsonFormat);
371
    
372
    return { csv: csvRecords, json: jsonRecords };
373
  } catch (error) {
374
    console.error('Loading failed:', error);
375
  }
376
}
377
```
378

379
### Custom Format Parser
380

381
```typescript
382
import { loader, formats } from "vega";
383

384
// Add custom format
385
const customFormat = {
386
  type: 'custom',
387
  parse: (data, format) => {
388
    return data.split('\n').map(line => {
389
      const [id, value] = line.split(':');
390
      return { id: parseInt(id), value: parseFloat(value) };
391
    });
392
  }
393
};
394

395
// Register format
396
formats()['custom'] = customFormat;
397

398
// Use custom format
399
const dataLoader = loader();
400
dataLoader.load('data.custom').then(data => {
401
  const parsed = read(data, customFormat);
402
  console.log(parsed);
403
});
404
```
405

406
### Type Inference
407

408
```typescript
409
import { inferType, inferTypes, typeParsers } from "vega";
410

411
const sampleData = [
412
  { name: 'Alice', age: '25', active: 'true', joined: '2023-01-15' },
413
  { name: 'Bob', age: '30', active: 'false', joined: '2023-02-20' },
414
  { name: 'Carol', age: '28', active: 'true', joined: '2023-03-10' }
415
];
416

417
// Infer types for all fields
418
const fieldTypes = inferTypes(sampleData, ['name', 'age', 'active', 'joined']);
419
console.log(fieldTypes);
420
// {
421
//   name: { type: 'string', confidence: 1.0 },
422
//   age: { type: 'integer', confidence: 1.0 },
423
//   active: { type: 'boolean', confidence: 1.0 },
424
//   joined: { type: 'date', confidence: 0.9 }
425
// }
426

427
// Infer single field type
428
const ageType = inferType(['25', '30', '28'], 'age');
429
console.log(ageType); // { type: 'integer', confidence: 1.0 }
430
```
431

432
### Advanced CSV Loading
433

434
```typescript
435
import { loader, read } from "vega";
436

437
const csvFormat = {
438
  type: 'csv',
439
  delimiter: ',',
440
  header: true,
441
  parsers: {
442
    'price': typeParsers.number,
443
    'date': (d) => new Date(d),
444
    'active': typeParsers.boolean
445
  }
446
};
447

448
const dataLoader = loader();
449
dataLoader.load('products.csv').then(data => {
450
  const products = read(data, csvFormat);
451
  console.log(products);
452
  // [
453
  //   { name: 'Widget', price: 19.99, date: Date, active: true },
454
  //   { name: 'Gadget', price: 29.99, date: Date, active: false }
455
  // ]
456
});
457
```
458

459
### HTTP Request Configuration
460

461
```typescript
462
import { loader } from "vega";
463

464
const dataLoader = loader({
465
  http: {
466
    method: 'POST',
467
    headers: {
468
      'Authorization': 'Bearer token123',
469
      'Content-Type': 'application/json'
470
    },
471
    credentials: 'include'
472
  }
473
});
474

475
// Load with custom headers
476
dataLoader.load('https://api.example.com/data', {
477
  headers: { 'X-Custom': 'value' }
478
}).then(data => {
479
  console.log(data);
480
});
481
```
482

483
### File System Loading
484

485
```typescript
486
import { loader } from "vega";
487

488
// Configure for local file access
489
const fileLoader = loader({
490
  file: {
491
    mode: 'file',
492
    encoding: 'utf8'
493
  }
494
});
495

496
// Load local file
497
fileLoader.file('./local-data.json').then(content => {
498
  const data = JSON.parse(content);
499
  console.log(data);
500
});
501
```
502

503
### GeoJSON Loading
504

505
```typescript
506
import { loader, read } from "vega";
507

508
const geoFormat = {
509
  type: 'geojson',
510
  feature: 'features'
511
};
512

513
const dataLoader = loader();
514
dataLoader.load('countries.geojson').then(data => {
515
  const features = read(data, geoFormat);
516
  console.log(features); // Array of GeoJSON features
517
});
518
```
519

520
### Custom Protocol Handler
521

522
```typescript
523
import { loader } from "vega";
524

525
const dataLoader = loader();
526

527
// Add custom protocol
528
if (dataLoader.addProtocolHandler) {
529
  dataLoader.addProtocolHandler('memory:', (uri, options) => {
530
    // Custom in-memory data access
531
    const key = uri.substring(7); // Remove 'memory:' prefix
532
    return Promise.resolve(memoryStore[key] || '');
533
  });
534
}
535

536
// Use custom protocol
537
dataLoader.load('memory:cached-data').then(data => {
538
  console.log('Loaded from memory:', data);
539
});
540
```
541

542
### Error Handling
543

544
```typescript
545
import { loader, read, format } from "vega";
546

547
async function safeLoadData(uri: string) {
548
  const dataLoader = loader();
549
  
550
  try {
551
    const data = await dataLoader.load(uri);
552
    const fmt = format(uri);
553
    return read(data, fmt);
554
  } catch (error) {
555
    if (error.message.includes('network')) {
556
      console.error('Network error:', error);
557
      return []; // Return empty array as fallback
558
    } else if (error.message.includes('parse')) {
559
      console.error('Parse error:', error);
560
      return null; // Indicate parse failure
561
    } else {
562
      throw error; // Re-throw unknown errors
563
    }
564
  }
565
}
566
```