Audio Processing

import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js";

const client = new ElevenLabsClient({ apiKey: "your-api-key" });
// Access this API via: client.audioIsolation

/**
 * @param request - Audio file to process
 * @param requestOptions - Optional request configuration
 * @returns ReadableStream of cleaned audio
 * @throws UnprocessableEntityError if request fails
 */
client.audioIsolation.convert(
  request: BodyAudioIsolationV1AudioIsolationPost,
  requestOptions?: RequestOptions
): HttpResponsePromise<ReadableStream<Uint8Array>>;

interface BodyAudioIsolationV1AudioIsolationPost {
  /** Audio file to process */
  audio: File | Blob;
  /** Input file format */
  file_format?: AudioIsolationFileFormat;
}

enum AudioIsolationFileFormat {
  MP3 = "mp3",
  MP4 = "mp4",
  MPEG = "mpeg",
  MPGA = "mpga",
  M4A = "m4a",
  WAV = "wav",
  WEBM = "webm",
}

/**
 * @param request - Audio file for streaming isolation
 * @param requestOptions - Optional request configuration
 * @returns ReadableStream of cleaned audio chunks
 * @throws UnprocessableEntityError if request fails
 */
client.audioIsolation.stream(
  request: BodyAudioIsolationStreamV1AudioIsolationStreamPost,
  requestOptions?: RequestOptions
): HttpResponsePromise<ReadableStream<Uint8Array>>;

interface BodyAudioIsolationStreamV1AudioIsolationStreamPost {
  /** Audio file to process */
  audio: File | Blob;
  /** Input file format */
  file_format?: AudioIsolationFileFormat;
}

import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js";
import { readFile, writeFile } from "fs/promises";

const client = new ElevenLabsClient({ apiKey: "your-api-key" });

// Load noisy audio
const noisyAudio = await readFile("recording_with_noise.mp3");

// Remove background noise
const cleanAudio = await client.audioIsolation.convert({
  audio: new File([noisyAudio], "recording_with_noise.mp3"),
});

// Save cleaned audio
const chunks: Uint8Array[] = [];
for await (const chunk of cleanAudio) {
  chunks.push(chunk);
}
await writeFile("clean_recording.mp3", Buffer.concat(chunks));

// Process WAV file
const wavAudio = await readFile("recording.wav");

const cleaned = await client.audioIsolation.convert({
  audio: new File([wavAudio], "recording.wav"),
  file_format: "wav",
});

// Stream audio cleaning for lower latency
const audioFile = await readFile("podcast.mp3");

const cleanStream = await client.audioIsolation.stream({
  audio: new File([audioFile], "podcast.mp3"),
});

// Process chunks as they arrive
for await (const chunk of cleanStream) {
  console.log("Received cleaned chunk:", chunk.length, "bytes");
  // Can process or play chunk immediately
}

import { readdir } from "fs/promises";
import { join } from "path";

// Clean multiple audio files
async function cleanAudioDirectory(
  inputDir: string,
  outputDir: string
): Promise<void> {
  const files = await readdir(inputDir);

  for (const file of files) {
    if (!file.endsWith(".mp3")) continue;

    console.log(`Cleaning ${file}...`);

    const inputPath = join(inputDir, file);
    const outputPath = join(outputDir, `clean_${file}`);

    const audioData = await readFile(inputPath);
    const cleaned = await client.audioIsolation.convert({
      audio: new File([audioData], file),
    });

    const chunks: Uint8Array[] = [];
    for await (const chunk of cleaned) {
      chunks.push(chunk);
    }
    await writeFile(outputPath, Buffer.concat(chunks));

    console.log(`Saved ${outputPath}`);
  }
}

await cleanAudioDirectory("./raw_recordings", "./cleaned_recordings");

// Remove background noise from interview
async function cleanInterview(
  inputPath: string,
  outputPath: string
): Promise<void> {
  const audio = await readFile(inputPath);

  const cleaned = await client.audioIsolation.convert({
    audio: new File([audio], inputPath),
  });

  const chunks: Uint8Array[] = [];
  for await (const chunk of cleaned) {
    chunks.push(chunk);
  }
  await writeFile(outputPath, Buffer.concat(chunks));

  console.log("Interview cleaned successfully");
}

await cleanInterview("interview_raw.mp3", "interview_clean.mp3");

// Clean podcast episode before publishing
async function postProcessPodcast(
  episodePath: string
): Promise<Buffer> {
  const rawAudio = await readFile(episodePath);

  // Remove background noise
  const cleaned = await client.audioIsolation.convert({
    audio: new File([rawAudio], episodePath),
  });

  const chunks: Uint8Array[] = [];
  for await (const chunk of cleaned) {
    chunks.push(chunk);
  }

  return Buffer.concat(chunks);
}

const cleanPodcast = await postProcessPodcast("episode_01_raw.mp3");
await writeFile("episode_01_final.mp3", cleanPodcast);

// Clean audio samples before using for voice cloning
async function cleanVoiceSamples(
  samplePaths: string[]
): Promise<File[]> {
  const cleanedSamples: File[] = [];

  for (const path of samplePaths) {
    const sample = await readFile(path);

    const cleaned = await client.audioIsolation.convert({
      audio: new File([sample], path),
    });

    const chunks: Uint8Array[] = [];
    for await (const chunk of cleaned) {
      chunks.push(chunk);
    }

    cleanedSamples.push(
      new File([Buffer.concat(chunks)], path)
    );
  }

  return cleanedSamples;
}

const cleanSamples = await cleanVoiceSamples([
  "sample1.mp3",
  "sample2.mp3",
  "sample3.mp3",
]);

// Use cleaned samples for voice cloning
await client.voices.ivc.create({
  name: "Clean Custom Voice",
  files: cleanSamples,
});

// Extract and clean audio from video
async function cleanVideoAudio(
  videoPath: string,
  outputAudioPath: string
): Promise<void> {
  // Note: You'd need to extract audio from video first
  // This example assumes audio is already extracted

  const audioData = await readFile(videoPath);

  const cleaned = await client.audioIsolation.convert({
    audio: new File([audioData], videoPath),
    file_format: "mp4",
  });

  const chunks: Uint8Array[] = [];
  for await (const chunk of cleaned) {
    chunks.push(chunk);
  }
  await writeFile(outputAudioPath, Buffer.concat(chunks));
}

// Stream audio cleaning for real-time applications
async function streamAudioCleaning(
  audioSource: string
): Promise<void> {
  const audioData = await readFile(audioSource);

  const cleanStream = await client.audioIsolation.stream({
    audio: new File([audioData], audioSource),
  });

  // Process each chunk as it arrives
  for await (const chunk of cleanStream) {
    // Send to audio player, WebSocket, etc.
    sendToAudioOutput(chunk);
  }
}

function sendToAudioOutput(chunk: Uint8Array): void {
  // Send to audio output system
  console.log("Playing cleaned chunk");
}

// Clean various audio formats
const formats = [
  { path: "recording.mp3", format: "mp3" as const },
  { path: "recording.wav", format: "wav" as const },
  { path: "recording.m4a", format: "m4a" as const },
  { path: "recording.webm", format: "webm" as const },
];

for (const { path, format } of formats) {
  const audio = await readFile(path);

  const cleaned = await client.audioIsolation.convert({
    audio: new File([audio], path),
    file_format: format,
  });

  // Save cleaned version
  const chunks: Uint8Array[] = [];
  for await (const chunk of cleaned) {
    chunks.push(chunk);
  }
  await writeFile(`clean_${path}`, Buffer.concat(chunks));
}

// Clean audio and compare file sizes
async function compareAudioQuality(
  inputPath: string
): Promise<void> {
  const originalAudio = await readFile(inputPath);
  console.log("Original size:", originalAudio.length, "bytes");

  const cleaned = await client.audioIsolation.convert({
    audio: new File([originalAudio], inputPath),
  });

  const chunks: Uint8Array[] = [];
  for await (const chunk of cleaned) {
    chunks.push(chunk);
  }
  const cleanedBuffer = Buffer.concat(chunks);

  console.log("Cleaned size:", cleanedBuffer.length, "bytes");
  console.log("Size difference:", originalAudio.length - cleanedBuffer.length, "bytes");

  await writeFile("cleaned_" + inputPath, cleanedBuffer);
}

await compareAudioQuality("recording.mp3");

// Integrate audio cleaning into processing pipeline
async function processAudioPipeline(
  inputPath: string
): Promise<void> {
  // Step 1: Load audio
  const audio = await readFile(inputPath);

  // Step 2: Clean audio
  const cleaned = await client.audioIsolation.convert({
    audio: new File([audio], inputPath),
  });

  const cleanedChunks: Uint8Array[] = [];
  for await (const chunk of cleaned) {
    cleanedChunks.push(chunk);
  }
  const cleanedBuffer = Buffer.concat(cleanedChunks);

  // Step 3: Use cleaned audio for TTS or voice cloning
  // ... additional processing

  // Step 4: Save final result
  await writeFile("processed_" + inputPath, cleanedBuffer);

  console.log("Audio processing pipeline complete");
}