Audio Processing

import { 
  ElevenLabsClient,
  type BodySpeechToSpeechV1SpeechToSpeechVoiceIdPost,
  type BodyAudioIsolationV1AudioIsolationPost,
  type CreateSoundEffectRequest,
  type DoDubbingRequest,
  type DubbingMetadataResponse,
  type GetDubbingResponse,
  type DeleteDubbingResponseModel
} from 'elevenlabs';

import * as fs from 'fs';

const client = new ElevenLabsClient();

// Convert speech from one voice to another
const transformedAudio = await client.speechToSpeech.convert(
  "21m00Tcm4TlvDq8ikWAM", // Target voice ID
  {
    audio: fs.createReadStream('original_speech.mp3'),
    model_id: "eleven_english_sts_v2",
    voice_settings: {
      stability: 0.5,
      similarity_boost: 0.8,
      style: 0.2,
      use_speaker_boost: true
    },
    enable_logging: true
  }
);

// Save the transformed audio
import { pipeline } from 'stream';
import { promisify } from 'util';

const pipelineAsync = promisify(pipeline);
await pipelineAsync(
  transformedAudio,
  fs.createWriteStream('transformed_speech.mp3')
);

// High-quality speech-to-speech with advanced settings
const professionalTransformation = await client.speechToSpeech.convert(
  "pNInz6obpgDQGcFmaJgB", // Professional voice
  {
    audio: fs.createReadStream('conference_recording.wav'),
    model_id: "eleven_english_sts_v2",
    voice_settings: {
      stability: 0.7,      // Higher stability for professional content
      similarity_boost: 0.9, // Maximum similarity to target voice
      style: 0.1,          // Minimal style changes
      use_speaker_boost: true,
      speed: 1.0
    },
    output_format: "mp3_44100_128",
    enable_logging: false, // Zero retention mode
    remove_background_noise: true
  }
);

interface BodySpeechToSpeechV1SpeechToSpeechVoiceIdPost {
  /** Input audio file (WAV, MP3, FLAC, etc.) */
  audio: File | fs.ReadStream | Blob;
  
  /** Model for speech-to-speech conversion */
  model_id?: string;
  
  /** Voice transformation settings */
  voice_settings?: VoiceSettings;
  
  /** Output audio format */
  output_format?: OutputFormat;
  
  /** Enable request logging */
  enable_logging?: boolean;
  
  /** Remove background noise from input */
  remove_background_noise?: boolean;
  
  /** Optimization for streaming latency */
  optimize_streaming_latency?: number;
}

// Clean up noisy audio recordings
const cleanedAudio = await client.audioIsolation.audioIsolation({
  audio: fs.createReadStream('noisy_recording.mp3'),
  file_format: "mp3" // Output format
});

// Save cleaned audio
await pipelineAsync(
  cleanedAudio,
  fs.createWriteStream('cleaned_audio.mp3')
);

console.log('Background noise removed successfully');

// Process multiple audio files
const audioFiles = [
  'interview_1.wav',
  'interview_2.wav', 
  'interview_3.wav'
];

for (let i = 0; i < audioFiles.length; i++) {
  const inputFile = audioFiles[i];
  const outputFile = `cleaned_${inputFile}`;
  
  try {
    console.log(`Processing ${inputFile}...`);
    
    const cleanedAudio = await client.audioIsolation.audioIsolation({
      audio: fs.createReadStream(inputFile),
      file_format: "wav"
    });
    
    await pipelineAsync(
      cleanedAudio,
      fs.createWriteStream(outputFile)
    );
    
    console.log(`✓ Cleaned ${inputFile} → ${outputFile}`);
    
  } catch (error) {
    console.error(`✗ Failed to process ${inputFile}:`, error);
  }
}

interface BodyAudioIsolationV1AudioIsolationPost {
  /** Input audio file with background noise */
  audio: File | fs.ReadStream | Blob;
  
  /** 
   * Output file format
   * Supports: mp3, wav, flac, opus, m4a, ogg, mpeg, webm
   */
  file_format?: string;
}

// Generate sound effects from text descriptions
const rainSound = await client.textToSoundEffects.convert({
  text: "Heavy rain falling on a metal roof with distant thunder",
  duration_seconds: 15,
  prompt_influence: 0.8 // How closely to follow the text prompt
});

await pipelineAsync(
  rainSound,
  fs.createWriteStream('rain_sound_effect.mp3')
);

// Generate multiple variations
const soundVariations = [
  "Birds chirping in a peaceful forest at dawn",
  "Busy city street with car horns and footsteps", 
  "Ocean waves crashing against rocky cliffs",
  "Crackling campfire with gentle wind in trees",
  "Vintage typewriter keys clicking rapidly"
];

for (let i = 0; i < soundVariations.length; i++) {
  const description = soundVariations[i];
  
  const soundEffect = await client.textToSoundEffects.convert({
    text: description,
    duration_seconds: 10,
    prompt_influence: 0.7,
    output_format: "mp3_44100_128"
  });
  
  const filename = `sound_effect_${i + 1}.mp3`;
  await pipelineAsync(soundEffect, fs.createWriteStream(filename));
  
  console.log(`Generated: ${filename} - "${description}"`);
}

// Create complex layered sound effects
const complexScene = await client.textToSoundEffects.convert({
  text: "Medieval tavern ambiance with crackling fireplace, distant lute music, muffled conversations, and the occasional clink of metal tankards",
  duration_seconds: 30,
  prompt_influence: 0.9, // High adherence to prompt
  output_format: "wav_48000" // High quality for post-production
});

// Generate game sound effects
const gameSounds = [
  {
    text: "Magical spell casting with ethereal energy buildup and mystical release",
    duration: 3,
    filename: "spell_cast.wav"
  },
  {
    text: "Heavy medieval sword striking metal armor with metallic ring",
    duration: 2,
    filename: "sword_strike.wav"  
  },
  {
    text: "Treasure chest opening with creaking hinges and gold coins jingling",
    duration: 4,
    filename: "treasure_open.wav"
  },
  {
    text: "Footsteps on creaky wooden floorboards in an old haunted house",
    duration: 8,
    filename: "creaky_footsteps.wav"
  }
];

for (const sound of gameSounds) {
  const soundEffect = await client.textToSoundEffects.convert({
    text: sound.text,
    duration_seconds: sound.duration,
    prompt_influence: 0.85,
    output_format: "wav_44100"
  });
  
  await pipelineAsync(soundEffect, fs.createWriteStream(sound.filename));
  console.log(`✓ Generated ${sound.filename}`);
}

interface CreateSoundEffectRequest {
  /** Text description of the desired sound effect */
  text: string;
  
  /** Duration of the sound effect in seconds (max varies by tier) */
  duration_seconds?: number;
  
  /** 
   * How closely to follow the text prompt (0.0-1.0)
   * Higher values = more faithful to prompt
   * Lower values = more creative interpretation
   */
  prompt_influence?: number;
  
  /** Output audio format */
  output_format?: OutputFormat;
}

// Dub a video into a different language
const dubbingProject = await client.dubbing.createDubbing({
  name: "Product Demo - Spanish Dub",
  source_url: "https://company.com/videos/product_demo.mp4", // Video URL
  target_lang: "es", // Spanish
  source_lang: "en", // English (auto-detect if not specified)
  num_speakers: 2,   // Number of speakers in the video
  watermark: false,  // Remove watermark (paid feature)
  high_quality: true // High quality processing
});

console.log('Dubbing project created:', dubbingProject.dubbing_id);
console.log('Processing status:', dubbingProject.status);

// Create dubbing with custom voice mapping
const advancedDubbing = await client.dubbing.createDubbing({
  name: "Corporate Training - Multi-Language",
  source_url: "https://training.company.com/leadership_course.mp4",
  target_lang: "fr", // French
  source_lang: "en",
  
  // Advanced configuration
  dubbing_studio: true,        // Enable studio features
  highest_resolution: true,    // Maximum quality
  exclude_background_audio: false, // Keep background music
  
  // Custom voice assignments per speaker
  voice_settings: {
    speaker_1: {
      voice_id: "french_professional_voice_id",
      stability: 0.7,
      similarity_boost: 0.8
    },
    speaker_2: {
      voice_id: "french_narrator_voice_id", 
      stability: 0.6,
      similarity_boost: 0.9
    }
  }
});

// Check dubbing status and progress
const dubbingStatus = await client.dubbing.getDubbing(dubbingProject.dubbing_id);

console.log('Dubbing progress:', {
  status: dubbingStatus.status,
  progress: dubbingStatus.progress_percentage,
  expectedDuration: dubbingStatus.expected_duration_seconds,
  timeRemaining: dubbingStatus.time_left_seconds
});

// Wait for completion with polling
async function waitForDubbingCompletion(dubbingId: string): Promise<DubbingMetadataResponse> {
  let status = 'processing';
  
  while (status === 'processing' || status === 'queued') {
    const dubbing = await client.dubbing.getDubbing(dubbingId);
    status = dubbing.status;
    
    console.log(`Status: ${status} (${dubbing.progress_percentage}%)`);
    
    if (status === 'completed') {
      return dubbing;
    } else if (status === 'failed') {
      throw new Error('Dubbing failed: ' + dubbing.error_message);
    }
    
    // Wait 30 seconds before next check
    await new Promise(resolve => setTimeout(resolve, 30000));
  }
  
  throw new Error('Unexpected dubbing status: ' + status);
}

// Usage
try {
  const completedDubbing = await waitForDubbingCompletion(dubbingProject.dubbing_id);
  console.log('Dubbing completed!');
  console.log('Download URL:', completedDubbing.dubbed_file_url);
} catch (error) {
  console.error('Dubbing failed:', error);
}

// Get all dubbing projects
const allDubbings = await client.dubbing.getDubbings({
  page_size: 50
});

console.log(`Found ${allDubbings.dubbings.length} dubbing projects`);

// Download completed dubbing
for (const dubbing of allDubbings.dubbings) {
  if (dubbing.status === 'completed' && dubbing.dubbed_file_url) {
    console.log(`Downloading: ${dubbing.name}`);
    
    // Download the dubbed video
    const response = await fetch(dubbing.dubbed_file_url);
    const videoBuffer = await response.arrayBuffer();
    
    const filename = `${dubbing.name.replace(/\s+/g, '_')}_dubbed.mp4`;
    fs.writeFileSync(filename, Buffer.from(videoBuffer));
    
    console.log(`✓ Downloaded: ${filename}`);
  }
}

// Delete old dubbing projects
const oldDubbings = allDubbings.dubbings.filter(
  dubbing => dubbing.status === 'completed' && 
             Date.now() - new Date(dubbing.created_at).getTime() > 30 * 24 * 60 * 60 * 1000 // 30 days old
);

for (const dubbing of oldDubbings) {
  try {
    await client.dubbing.deleteDubbing(dubbing.dubbing_id);
    console.log(`Deleted old dubbing: ${dubbing.name}`);
  } catch (error) {
    console.error(`Failed to delete ${dubbing.dubbing_id}:`, error);
  }
}

// Complete podcast processing workflow
async function processPodcastEpisode(inputFile: string, outputFile: string) {
  console.log('Starting podcast processing pipeline...');
  
  // Step 1: Remove background noise
  console.log('1. Removing background noise...');
  const cleanedAudio = await client.audioIsolation.audioIsolation({
    audio: fs.createReadStream(inputFile),
    file_format: "wav"
  });
  
  // Save cleaned audio temporarily
  const tempCleanFile = 'temp_cleaned.wav';
  await pipelineAsync(cleanedAudio, fs.createWriteStream(tempCleanFile));
  
  // Step 2: Convert to professional narrator voice
  console.log('2. Converting to professional voice...');
  const professionalAudio = await client.speechToSpeech.convert(
    "professional_narrator_voice_id",
    {
      audio: fs.createReadStream(tempCleanFile),
      model_id: "eleven_english_sts_v2",
      voice_settings: {
        stability: 0.8,
        similarity_boost: 0.9,
        style: 0.1,
        use_speaker_boost: true
      },
      output_format: "mp3_44100_128"
    }
  );
  
  // Step 3: Save final output
  await pipelineAsync(professionalAudio, fs.createWriteStream(outputFile));
  
  // Clean up temporary file
  fs.unlinkSync(tempCleanFile);
  
  console.log(`✓ Podcast processing complete: ${outputFile}`);
}

// Usage
await processPodcastEpisode('raw_recording.wav', 'professional_podcast.mp3');

// Generate complete game audio package
async function createGameAudioPackage() {
  const soundEffects = [
    { name: "sword_clash", text: "Two metal swords clashing in epic battle", duration: 2 },
    { name: "magic_heal", text: "Gentle magical healing spell with warm energy", duration: 3 },
    { name: "coin_pickup", text: "Single gold coin being picked up with satisfying clink", duration: 1 },
    { name: "door_creak", text: "Old wooden door slowly creaking open", duration: 4 },
    { name: "victory_fanfare", text: "Triumphant orchestral victory theme", duration: 5 },
    { name: "ambient_forest", text: "Peaceful forest with birds and rustling leaves", duration: 20 },
    { name: "dungeon_atmosphere", text: "Dark dungeon with dripping water and distant echoes", duration: 15 }
  ];
  
  console.log('Generating game audio package...');
  
  for (let i = 0; i < soundEffects.length; i++) {
    const effect = soundEffects[i];
    console.log(`${i + 1}/${soundEffects.length}: Generating ${effect.name}...`);
    
    try {
      const audio = await client.textToSoundEffects.convert({
        text: effect.text,
        duration_seconds: effect.duration,
        prompt_influence: 0.8,
        output_format: "wav_44100"
      });
      
      const filename = `game_audio/${effect.name}.wav`;
      
      // Ensure directory exists
      if (!fs.existsSync('game_audio')) {
        fs.mkdirSync('game_audio');
      }
      
      await pipelineAsync(audio, fs.createWriteStream(filename));
      console.log(`✓ Generated: ${filename}`);
      
    } catch (error) {
      console.error(`✗ Failed to generate ${effect.name}:`, error);
    }
  }
  
  console.log('Game audio package complete!');
}

// Usage
await createGameAudioPackage();

// Enhance voice acting recordings
async function enhanceVoiceActing(actorFiles: string[], characterVoiceId: string) {
  const enhancedFiles = [];
  
  for (let i = 0; i < actorFiles.length; i++) {
    const inputFile = actorFiles[i];
    const outputFile = `enhanced_${inputFile}`;
    
    console.log(`Enhancing ${inputFile}...`);
    
    try {
      // Step 1: Clean audio
      const cleanedAudio = await client.audioIsolation.audioIsolation({
        audio: fs.createReadStream(inputFile),
        file_format: "wav"
      });
      
      // Save to temporary file
      const tempFile = `temp_clean_${i}.wav`;
      await pipelineAsync(cleanedAudio, fs.createWriteStream(tempFile));
      
      // Step 2: Transform to character voice
      const characterAudio = await client.speechToSpeech.convert(
        characterVoiceId,
        {
          audio: fs.createReadStream(tempFile),
          model_id: "eleven_english_sts_v2",
          voice_settings: {
            stability: 0.4,  // Allow more expression variation
            similarity_boost: 0.7,
            style: 0.6,      // Higher style for character voices
            use_speaker_boost: true
          },
          output_format: "wav_48000", // High quality for post-production
          remove_background_noise: false // Already cleaned
        }
      );
      
      await pipelineAsync(characterAudio, fs.createWriteStream(outputFile));
      
      // Clean up temp file
      fs.unlinkSync(tempFile);
      
      enhancedFiles.push(outputFile);
      console.log(`✓ Enhanced: ${outputFile}`);
      
    } catch (error) {
      console.error(`✗ Failed to enhance ${inputFile}:`, error);
    }
  }
  
  return enhancedFiles;
}

// Efficient batch processing with concurrency control
class AudioProcessingManager {
  private maxConcurrent: number;
  private activeJobs = 0;
  private jobQueue: (() => Promise<void>)[] = [];
  
  constructor(maxConcurrent = 3) {
    this.maxConcurrent = maxConcurrent;
  }
  
  async addJob(job: () => Promise<void>) {
    return new Promise<void>((resolve, reject) => {
      this.jobQueue.push(async () => {
        try {
          await job();
          resolve();
        } catch (error) {
          reject(error);
        }
      });
      
      this.processQueue();
    });
  }
  
  private async processQueue() {
    if (this.activeJobs >= this.maxConcurrent || this.jobQueue.length === 0) {
      return;
    }
    
    const job = this.jobQueue.shift()!;
    this.activeJobs++;
    
    try {
      await job();
    } finally {
      this.activeJobs--;
      this.processQueue(); // Process next job
    }
  }
}

// Usage
const processor = new AudioProcessingManager(2); // Max 2 concurrent jobs

const files = ['audio1.mp3', 'audio2.mp3', 'audio3.mp3', 'audio4.mp3'];

await Promise.all(files.map(file => 
  processor.addJob(async () => {
    console.log(`Processing ${file}...`);
    const cleanedAudio = await client.audioIsolation.audioIsolation({
      audio: fs.createReadStream(file),
      file_format: "mp3"
    });
    await pipelineAsync(cleanedAudio, fs.createWriteStream(`cleaned_${file}`));
    console.log(`✓ Completed ${file}`);
  })
));

import { ElevenLabsError, ElevenLabsTimeoutError } from 'elevenlabs';

async function robustAudioProcessing(inputFile: string) {
  try {
    // Attempt audio processing
    const processedAudio = await client.speechToSpeech.convert(
      "target_voice_id",
      {
        audio: fs.createReadStream(inputFile),
        model_id: "eleven_english_sts_v2"
      }
    );
    
    return processedAudio;
    
  } catch (error) {
    if (error instanceof ElevenLabsTimeoutError) {
      console.error('Processing timed out for:', inputFile);
      // Implement retry with exponential backoff
      
    } else if (error instanceof ElevenLabsError) {
      console.error('API error:', error.statusCode);
      
      if (error.statusCode === 400) {
        console.error('Invalid audio file format or parameters');
      } else if (error.statusCode === 413) {
        console.error('Audio file too large');
      } else if (error.statusCode === 422) {
        console.error('Audio processing validation error:', error.body);
      } else if (error.statusCode === 429) {
        console.error('Rate limit exceeded - implement backoff');
      }
    }
    
    throw error;
  }
}