Data Loading

/**
 * Create a data loader with specified options
 * @param options - Loader configuration options
 * @returns Configured loader instance
 */
function loader(options?: LoaderOptions): Loader;

/**
 * Read and parse data from string with format specification
 * @param data - Raw data string
 * @param schema - Format specification
 * @param dateParse - Optional custom date parsing function
 * @returns Array of parsed data objects
 */
function read(data: string, schema: Format, dateParse?: (dateString: string) => Date): object[];

interface LoaderOptions {
  /** Base URL for relative paths */
  baseURL?: string;
  
  /** Default file format when not specified */
  defaultProtocol?: string;
  
  /** HTTP request options */
  http?: RequestInit;
  
  /** File system access options */
  file?: FileOptions;
  
  /** Custom format parsers */
  formatParsers?: { [format: string]: FormatParser };
}

interface FileOptions {
  /** Local file system access mode */
  mode?: 'file' | 'http';
  
  /** File encoding */
  encoding?: string;
}

interface FormatParser {
  /** Parse function for the format */
  parse: (data: string, format: Format) => any[];
  
  /** MIME type for the format */
  type?: string;
}

/**
 * Data loader interface for accessing external data sources
 */
interface Loader {
  /**
   * Load data from URI
   * @param uri - Data source URI
   * @param options - Optional loading context and options
   * @returns Promise resolving to loaded data string
   */
  load: (uri: string, options?: LoaderOptionsWithContext) => Promise<string>;
  
  /**
   * Sanitize and validate URI for security
   * @param uri - URI to sanitize
   * @param options - Loading context
   * @returns Promise resolving to sanitized URI object
   */
  sanitize: (uri: string, options: LoaderOptionsWithContext) => Promise<{ href: string }>;
  
  /**
   * HTTP-specific loading method
   * @param uri - HTTP URL
   * @param options - HTTP request options
   * @returns Promise resolving to response text
   */
  http: (uri: string, options?: Partial<RequestInit>) => Promise<string>;
  
  /**
   * File system loading method
   * @param filename - Local file path
   * @returns Promise resolving to file contents
   */
  file: (filename: string) => Promise<string>;
  
  /**
   * Add custom protocol handler
   * @param protocol - Protocol name (e.g., 'custom:')
   * @param handler - Handler function
   */
  addProtocolHandler?: (protocol: string, handler: ProtocolHandler) => void;
}

interface LoaderOptionsWithContext {
  /** Request context object */
  context?: any;
  
  /** HTTP headers */
  headers?: { [key: string]: string };
  
  /** Request credentials mode */
  credentials?: RequestCredentials;
  
  /** Request method */
  method?: string;
  
  /** Request body */
  body?: BodyInit;
}

type ProtocolHandler = (uri: string, options: LoaderOptionsWithContext) => Promise<string>;

/**
 * Infer data type for a field from sample values
 * @param values - Array of sample values
 * @param field - Optional field name for context
 * @returns Type inference result
 */
function inferType(values: readonly any[], field?: string): TypeInference;

/**
 * Infer types for multiple fields from sample data
 * @param values - Array of data objects
 * @param fields - Array of field names to analyze
 * @returns Object mapping field names to type inference results
 */
function inferTypes(values: readonly any[], fields: readonly string[]): { [field: string]: TypeInference };

interface TypeInference {
  /** Inferred data type */
  type: DataType;
  
  /** Confidence level (0-1) */
  confidence?: number;
  
  /** Sample parsed values */
  samples?: any[];
  
  /** Format hint for parsing */
  format?: string;
}

type DataType = 'number' | 'integer' | 'date' | 'boolean' | 'string';

/**
 * Get format specification from URI or data
 * @param name - File name or URI
 * @param data - Optional data sample
 * @returns Format specification object
 */
function format(name: string, data?: string): Format;

/**
 * Get available format specifications
 * @returns Object mapping format names to specifications
 */
function formats(): { [name: string]: Format };

/**
 * Get appropriate response type for format
 * @param type - Format type
 * @returns Response type ('text', 'json', 'arrayBuffer', etc.)
 */
function responseType(type: string): ResponseType;

interface Format {
  /** Format type identifier */
  type: string;
  
  /** Parse function for the format */
  parse?: (data: string, format: Format) => any[];
  
  /** Field delimiter (CSV, TSV, etc.) */
  delimiter?: string;
  
  /** Header row handling */
  header?: boolean;
  
  /** Skip rows from beginning */
  skipRows?: number;
  
  /** Property path for nested data (JSON) */
  property?: string;
  
  /** Feature property path (GeoJSON) */
  feature?: string;
  
  /** Mesh property path (TopoJSON) */
  mesh?: string;
  
  /** Custom field parsing functions */
  parsers?: { [field: string]: (value: string) => any };
}

type ResponseType = 'text' | 'json' | 'arrayBuffer' | 'blob';

/**
 * Registry of built-in type parsing functions
 */
const typeParsers: {
  /** Parse as number */
  number: (value: string) => number | null;
  
  /** Parse as integer */
  integer: (value: string) => number | null;
  
  /** Parse as boolean */
  boolean: (value: string) => boolean | null;
  
  /** Parse as date */
  date: (value: string) => Date | null;
  
  /** Parse as string (identity) */
  string: (value: string) => string;
  
  /** Auto-detect and parse */
  auto: (value: string) => any;
};

/**
 * Add custom type parser
 * @param type - Type name
 * @param parser - Parser function
 */
function addTypeParser(type: string, parser: (value: string) => any): void;

interface CSVFormat extends Format {
  type: 'csv' | 'tsv';
  
  /** Field delimiter character */
  delimiter: ',' | '\t' | string;
  
  /** Quote character */
  quote?: string;
  
  /** Escape character */
  escape?: string;
  
  /** Header row present */
  header: boolean;
  
  /** Comment line prefix */
  comment?: string;
}

interface JSONFormat extends Format {
  type: 'json';
  
  /** Property path to extract array data */
  property?: string;
  
  /** Automatically parse date strings */
  autoParseDate?: boolean;
}

interface GeoFormat extends Format {
  type: 'geojson' | 'topojson';
  
  /** Feature collection property */
  feature?: string;
  
  /** Mesh property (TopoJSON) */
  mesh?: string;
}

interface DSVFormat extends Format {
  type: 'dsv';
  
  /** Custom delimiter */
  delimiter: string;
  
  /** Header handling */
  header: boolean;
}

import { loader, read, format } from "vega";

// Create loader
const dataLoader = loader({
  baseURL: 'https://api.example.com/',
  http: { credentials: 'same-origin' }
});

// Load CSV data
dataLoader.load('data/sales.csv').then(data => {
  const fmt = format('sales.csv');
  const parsed = read(data, fmt);
  console.log(parsed);
});

import { loader, read, format } from "vega";

async function loadData() {
  const dataLoader = loader();
  
  try {
    // Load multiple data sources
    const [csvData, jsonData] = await Promise.all([
      dataLoader.load('data.csv'),
      dataLoader.load('config.json')
    ]);
    
    // Parse CSV
    const csvFormat = format('data.csv');
    const csvRecords = read(csvData, csvFormat);
    
    // Parse JSON
    const jsonFormat = format('config.json');
    const jsonRecords = read(jsonData, jsonFormat);
    
    return { csv: csvRecords, json: jsonRecords };
  } catch (error) {
    console.error('Loading failed:', error);
  }
}

import { loader, formats } from "vega";

// Add custom format
const customFormat = {
  type: 'custom',
  parse: (data, format) => {
    return data.split('\n').map(line => {
      const [id, value] = line.split(':');
      return { id: parseInt(id), value: parseFloat(value) };
    });
  }
};

// Register format
formats()['custom'] = customFormat;

// Use custom format
const dataLoader = loader();
dataLoader.load('data.custom').then(data => {
  const parsed = read(data, customFormat);
  console.log(parsed);
});

import { inferType, inferTypes, typeParsers } from "vega";

const sampleData = [
  { name: 'Alice', age: '25', active: 'true', joined: '2023-01-15' },
  { name: 'Bob', age: '30', active: 'false', joined: '2023-02-20' },
  { name: 'Carol', age: '28', active: 'true', joined: '2023-03-10' }
];

// Infer types for all fields
const fieldTypes = inferTypes(sampleData, ['name', 'age', 'active', 'joined']);
console.log(fieldTypes);
// {
//   name: { type: 'string', confidence: 1.0 },
//   age: { type: 'integer', confidence: 1.0 },
//   active: { type: 'boolean', confidence: 1.0 },
//   joined: { type: 'date', confidence: 0.9 }
// }

// Infer single field type
const ageType = inferType(['25', '30', '28'], 'age');
console.log(ageType); // { type: 'integer', confidence: 1.0 }

import { loader, read } from "vega";

const csvFormat = {
  type: 'csv',
  delimiter: ',',
  header: true,
  parsers: {
    'price': typeParsers.number,
    'date': (d) => new Date(d),
    'active': typeParsers.boolean
  }
};

const dataLoader = loader();
dataLoader.load('products.csv').then(data => {
  const products = read(data, csvFormat);
  console.log(products);
  // [
  //   { name: 'Widget', price: 19.99, date: Date, active: true },
  //   { name: 'Gadget', price: 29.99, date: Date, active: false }
  // ]
});

import { loader } from "vega";

const dataLoader = loader({
  http: {
    method: 'POST',
    headers: {
      'Authorization': 'Bearer token123',
      'Content-Type': 'application/json'
    },
    credentials: 'include'
  }
});

// Load with custom headers
dataLoader.load('https://api.example.com/data', {
  headers: { 'X-Custom': 'value' }
}).then(data => {
  console.log(data);
});

import { loader } from "vega";

// Configure for local file access
const fileLoader = loader({
  file: {
    mode: 'file',
    encoding: 'utf8'
  }
});

// Load local file
fileLoader.file('./local-data.json').then(content => {
  const data = JSON.parse(content);
  console.log(data);
});

import { loader, read } from "vega";

const geoFormat = {
  type: 'geojson',
  feature: 'features'
};

const dataLoader = loader();
dataLoader.load('countries.geojson').then(data => {
  const features = read(data, geoFormat);
  console.log(features); // Array of GeoJSON features
});

import { loader } from "vega";

const dataLoader = loader();

// Add custom protocol
if (dataLoader.addProtocolHandler) {
  dataLoader.addProtocolHandler('memory:', (uri, options) => {
    // Custom in-memory data access
    const key = uri.substring(7); // Remove 'memory:' prefix
    return Promise.resolve(memoryStore[key] || '');
  });
}

// Use custom protocol
dataLoader.load('memory:cached-data').then(data => {
  console.log('Loaded from memory:', data);
});

import { loader, read, format } from "vega";

async function safeLoadData(uri: string) {
  const dataLoader = loader();
  
  try {
    const data = await dataLoader.load(uri);
    const fmt = format(uri);
    return read(data, fmt);
  } catch (error) {
    if (error.message.includes('network')) {
      console.error('Network error:', error);
      return []; // Return empty array as fallback
    } else if (error.message.includes('parse')) {
      console.error('Parse error:', error);
      return null; // Indicate parse failure
    } else {
      throw error; // Re-throw unknown errors
    }
  }
}