Language Model

/**
 * Create a language model instance
 * @param {Object} options - Configuration options
 * @param {string} [options.name] - Dataset name for identification
 * @param {Object} [options.dataset] - Pre-existing dataset with name and files
 * @param {string[]} [options.files] - Training document filenames (without .txt extension)
 * @param {boolean} [options.bootstrap=false] - Use built-in default training data
 * @returns {Promise<LanguageModel>} Language model API with prediction and training methods
 */
async function Language(options = {});

const { Language } = require('next-token-prediction');

// Bootstrap with default training data
const defaultModel = await Language({
  bootstrap: true
});

// Use pre-existing dataset
const Dataset = require('./training/datasets/OpenSourceBooks');
const bookModel = await Language({
  dataset: Dataset
});

// Train on custom files
const customModel = await Language({
  name: 'my-dataset',
  files: ['document1', 'document2', 'document3']
});

/**
 * Language model instance with prediction and training capabilities
 */
interface LanguageModel {
  // High-level prediction methods
  complete(query: string): string;

  // Full transformer API access
  getTokenPrediction(token: string): TokenPredictionResult;
  getTokenSequencePrediction(input: string, sequenceLength?: number): SequencePredictionResult;
  getCompletions(input: string): CompletionsResult;

  // Training and model management
  train(dataset: Dataset): Promise<void>;
  createContext(embeddings: EmbeddingsObject): void;
  ingest(text: string): void;
  fromTrainingData(trainingData: TrainingData): TransformerAPI;
  fromFiles(files: string[]): Promise<TransformerAPI>;
}

/**
 * Get the highest-ranked completion for input text
 * @param {string} query - Input text to complete
 * @returns {string} Best completion prediction
 */
complete(query);

// Simple completion
const result1 = model.complete('The weather today is');
// Returns: "beautiful" (or other highest-ranked prediction)

// Phrase completion
const result2 = model.complete('JavaScript is a programming');
// Returns: "language" (or similar contextual completion)

/**
 * Create transformer from pre-computed training data
 * @param {TrainingData} trainingData - Object with text and embeddings
 * @returns {TransformerAPI} Transformer instance ready for predictions
 */
fromTrainingData(trainingData);

/**
 * Create transformer from text files with full training process
 * @param {string[]} files - Document filenames (without .txt extension)
 * @returns {Promise<TransformerAPI>} Trained transformer instance
 */
fromFiles(files);

// Using pre-computed embeddings
const trainingData = {
  text: "Combined document text...",
  embeddings: { /* pre-computed embeddings */ }
};
const transformer1 = model.fromTrainingData(trainingData);

// Training from files
const transformer2 = await model.fromFiles([
  'shakespeare-hamlet',
  'shakespeare-macbeth',
  'shakespeare-othello'
]);

/**
 * Configuration options for Language factory function
 */
interface LanguageOptions {
  name?: string;           // Dataset identifier
  dataset?: Dataset;       // Pre-existing dataset configuration
  files?: string[];        // Training document filenames
  bootstrap?: boolean;     // Use default training data
}

/**
 * Training dataset configuration
 */
interface Dataset {
  name: string;           // Dataset identifier
  files: string[];        // Document filenames without .txt extension
}

/**
 * Pre-computed training data structure
 */
interface TrainingData {
  text: string;           // Combined training text
  embeddings: EmbeddingsObject; // Token embedding vectors
}