0
# Live API (Experimental)
1

2
The Live module provides real-time bidirectional communication with models via WebSocket, supporting multimodal streaming input and output including text, audio, and video.
3

4
## Capabilities
5

6
### connect
7

8
Establish a WebSocket connection to a model for real-time interaction.
9

10
```typescript { .api }
11
/**
12
 * Establish WebSocket connection to model
13
 * @param params - Live connection parameters
14
 * @returns Promise resolving to session
15
 */
16
function connect(
17
  params: LiveConnectParameters
18
): Promise<Session>;
19

20
interface LiveConnectParameters {
21
  /** Model name (e.g., 'gemini-2.0-flash-exp') */
22
  model: string;
23
  /** Connection configuration */
24
  config?: LiveConnectConfig;
25
  /** Event callbacks */
26
  callbacks?: LiveCallbacks;
27
}
28
```
29

30
**Usage Examples:**
31

32
```typescript
33
import { GoogleGenAI } from '@google/genai';
34

35
const client = new GoogleGenAI({ apiKey: 'YOUR_API_KEY' });
36

37
// Connect to model
38
const session = await client.live.connect({
39
  model: 'gemini-2.0-flash-exp',
40
  config: {
41
    responseModalities: ['TEXT', 'AUDIO']
42
  },
43
  callbacks: {
44
    onopen: () => {
45
      console.log('Session opened');
46
    },
47
    onmessage: (message) => {
48
      console.log('Message received:', message);
49
    },
50
    onerror: (error) => {
51
      console.error('Error:', error);
52
    },
53
    onclose: () => {
54
      console.log('Session closed');
55
    }
56
  }
57
});
58

59
// Send text input
60
session.send({
61
  clientContent: {
62
    turns: [{
63
      role: 'user',
64
      parts: [{ text: 'Hello, how are you?' }]
65
    }]
66
  }
67
});
68

69
// Close when done
70
session.close();
71
```
72

73
### Session Class
74

75
Represents an active WebSocket session for real-time communication.
76

77
```typescript { .api }
78
/**
79
 * Live WebSocket session
80
 */
81
class Session {
82
  /**
83
   * Send client message via WebSocket
84
   * @param message - Message to send
85
   */
86
  send(message: LiveClientMessage): void;
87

88
  /**
89
   * Send real-time input (audio/video)
90
   * @param params - Realtime input parameters
91
   */
92
  sendRealtimeInput(params: LiveSendRealtimeInputParameters): void;
93

94
  /**
95
   * Send tool/function response
96
   * @param params - Tool response parameters
97
   */
98
  sendToolResponse(params: LiveSendToolResponseParameters): void;
99

100
  /**
101
   * Close the WebSocket connection
102
   */
103
  close(): void;
104
}
105
```
106

107
### Live Music
108

109
Specialized sub-module for live music generation.
110

111
```typescript { .api }
112
/**
113
 * Live music generation
114
 */
115
class LiveMusic {
116
  /**
117
   * Connect to live music generation
118
   * @param params - Music connection parameters
119
   * @returns Promise resolving to music session
120
   */
121
  connect(params: LiveMusicConnectParameters): Promise<LiveMusicSession>;
122
}
123

124
interface LiveMusicConnectParameters {
125
  /** Model name */
126
  model: string;
127
  /** Music generation configuration */
128
  config?: LiveMusicConfig;
129
  /** Event callbacks */
130
  callbacks?: LiveCallbacks;
131
}
132
```
133

134
**Usage Examples:**
135

136
```typescript
137
// Access live music
138
const musicSession = await client.live.music.connect({
139
  model: 'music-generation-model',
140
  config: {
141
    tempo: 120,
142
    scale: Scale.C_MAJOR_A_MINOR,
143
    mode: MusicGenerationMode.QUALITY
144
  },
145
  callbacks: {
146
    onmessage: (message) => {
147
      // Handle music chunks
148
      console.log('Music chunk received');
149
    }
150
  }
151
});
152
```
153

154
## Types
155

156
### LiveConnectConfig
157

158
Configuration for live connection.
159

160
```typescript { .api }
161
interface LiveConnectConfig {
162
  /** Output modalities (TEXT, AUDIO, IMAGE) */
163
  responseModalities?: Modality[];
164
  /** System instruction */
165
  systemInstruction?: Content | string;
166
  /** Tools available */
167
  tools?: ToolListUnion;
168
  /** Tool configuration */
169
  toolConfig?: ToolConfig;
170
  /** Generation config */
171
  generationConfig?: GenerationConfig;
172
  /** Speech config */
173
  speechConfig?: SpeechConfig;
174
}
175

176
interface GenerationConfig {
177
  /** Temperature */
178
  temperature?: number;
179
  /** Top-P */
180
  topP?: number;
181
  /** Top-K */
182
  topK?: number;
183
  /** Max output tokens */
184
  maxOutputTokens?: number;
185
}
186

187
interface SpeechConfig {
188
  /** Voice preset */
189
  voiceConfig?: VoiceConfig;
190
}
191

192
interface VoiceConfig {
193
  /** Preset voice name */
194
  presetVoice?: string;
195
}
196
```
197

198
### LiveCallbacks
199

200
Event callbacks for live session.
201

202
```typescript { .api }
203
interface LiveCallbacks {
204
  /** Called when connection opens */
205
  onopen?: () => void;
206
  /** Called when message received */
207
  onmessage?: (message: LiveServerMessage) => void;
208
  /** Called on error */
209
  onerror?: (error: Error) => void;
210
  /** Called when connection closes */
211
  onclose?: () => void;
212
}
213
```
214

215
### LiveClientMessage
216

217
Messages sent from client to model.
218

219
```typescript { .api }
220
interface LiveClientMessage {
221
  /** Setup configuration */
222
  setup?: LiveClientSetup;
223
  /** Client content */
224
  clientContent?: LiveClientContent;
225
  /** Realtime input */
226
  realtimeInput?: RealtimeInput;
227
  /** Tool response */
228
  toolResponse?: ToolResponse;
229
}
230

231
interface LiveClientSetup {
232
  /** Generation config */
233
  generationConfig?: GenerationConfig;
234
  /** System instruction */
235
  systemInstruction?: Content;
236
  /** Tools */
237
  tools?: Tool[];
238
}
239

240
interface LiveClientContent {
241
  /** Turn information */
242
  turns?: Content[];
243
  /** Turn complete flag */
244
  turnComplete?: boolean;
245
}
246
```
247

248
### LiveServerMessage
249

250
Messages received from model.
251

252
```typescript { .api }
253
interface LiveServerMessage {
254
  /** Setup complete */
255
  setupComplete?: LiveSetupComplete;
256
  /** Server content */
257
  serverContent?: LiveServerContent;
258
  /** Tool call */
259
  toolCall?: ToolCall;
260
  /** Tool call cancellation */
261
  toolCallCancellation?: ToolCallCancellation;
262
}
263

264
interface LiveServerContent {
265
  /** Model turn */
266
  modelTurn?: Content;
267
  /** Turn complete flag */
268
  turnComplete?: boolean;
269
  /** Interrupted flag */
270
  interrupted?: boolean;
271
}
272
```
273

274
### RealtimeInput
275

276
Real-time audio/video input.
277

278
```typescript { .api }
279
interface RealtimeInput {
280
  /** Media chunks */
281
  mediaChunks?: MediaChunk[];
282
}
283

284
interface MediaChunk {
285
  /** MIME type */
286
  mimeType?: string;
287
  /** Base64-encoded data */
288
  data?: string;
289
}
290
```
291

292
### LiveSendRealtimeInputParameters
293

294
Parameters for sending realtime input.
295

296
```typescript { .api }
297
interface LiveSendRealtimeInputParameters {
298
  /** Realtime input */
299
  realtimeInput: RealtimeInput;
300
}
301
```
302

303
### LiveSendToolResponseParameters
304

305
Parameters for sending tool response.
306

307
```typescript { .api }
308
interface LiveSendToolResponseParameters {
309
  /** Tool response */
310
  toolResponse: ToolResponse;
311
}
312
```
313

314
### ToolResponse
315

316
Tool execution response.
317

318
```typescript { .api }
319
interface ToolResponse {
320
  /** Function responses */
321
  functionResponses?: FunctionResponse[];
322
}
323
```
324

325
### Music Generation Types
326

327
```typescript { .api }
328
interface LiveMusicConfig {
329
  /** Tempo (BPM) */
330
  tempo?: number;
331
  /** Musical scale */
332
  scale?: Scale;
333
  /** Generation mode */
334
  mode?: MusicGenerationMode;
335
}
336

337
enum Scale {
338
  C_MAJOR_A_MINOR = 'C_MAJOR_A_MINOR',
339
  C_SHARP_D_FLAT_MAJOR_A_SHARP_B_FLAT_MINOR = 'C_SHARP_D_FLAT_MAJOR_A_SHARP_B_FLAT_MINOR',
340
  // ... additional scales
341
}
342

343
enum MusicGenerationMode {
344
  QUALITY = 'QUALITY',
345
  DIVERSITY = 'DIVERSITY',
346
  VOCALIZATION = 'VOCALIZATION'
347
}
348
```
349

350
## Complete Examples
351

352
### Text-based Live Chat
353

354
```typescript
355
import { GoogleGenAI } from '@google/genai';
356

357
const client = new GoogleGenAI({ apiKey: 'YOUR_API_KEY' });
358

359
// Connect with text modality
360
const session = await client.live.connect({
361
  model: 'gemini-2.0-flash-exp',
362
  config: {
363
    responseModalities: ['TEXT']
364
  },
365
  callbacks: {
366
    onopen: () => {
367
      console.log('Connected');
368
    },
369
    onmessage: (message) => {
370
      if (message.serverContent?.modelTurn) {
371
        const text = message.serverContent.modelTurn.parts?.[0]?.text;
372
        if (text) {
373
          console.log('Model:', text);
374
        }
375
      }
376
    },
377
    onerror: (error) => {
378
      console.error('Error:', error);
379
    }
380
  }
381
});
382

383
// Send messages
384
session.send({
385
  clientContent: {
386
    turns: [{
387
      role: 'user',
388
      parts: [{ text: 'What is quantum computing?' }]
389
    }],
390
    turnComplete: true
391
  }
392
});
393

394
// Continue conversation
395
setTimeout(() => {
396
  session.send({
397
    clientContent: {
398
      turns: [{
399
        role: 'user',
400
        parts: [{ text: 'Can you explain it more simply?' }]
401
      }],
402
      turnComplete: true
403
    }
404
  });
405
}, 3000);
406

407
// Close after 10 seconds
408
setTimeout(() => {
409
  session.close();
410
}, 10000);
411
```
412

413
### Audio Input and Output
414

415
```typescript
416
// Connect with audio modalities
417
const session = await client.live.connect({
418
  model: 'gemini-2.0-flash-exp',
419
  config: {
420
    responseModalities: ['TEXT', 'AUDIO'],
421
    speechConfig: {
422
      voiceConfig: {
423
        presetVoice: 'en-US-Standard-A'
424
      }
425
    }
426
  },
427
  callbacks: {
428
    onmessage: (message) => {
429
      if (message.serverContent?.modelTurn) {
430
        message.serverContent.modelTurn.parts?.forEach(part => {
431
          if (part.text) {
432
            console.log('Text:', part.text);
433
          }
434
          if (part.inlineData?.mimeType?.startsWith('audio/')) {
435
            console.log('Received audio chunk');
436
            // Process audio data
437
            const audioData = part.inlineData.data;
438
            // Play or save audio
439
          }
440
        });
441
      }
442
    }
443
  }
444
});
445

446
// Send audio input (e.g., from microphone)
447
const audioChunk = getAudioFromMicrophone(); // Your audio capture logic
448

449
session.sendRealtimeInput({
450
  realtimeInput: {
451
    mediaChunks: [{
452
      mimeType: 'audio/pcm',
453
      data: audioChunk
454
    }]
455
  }
456
});
457
```
458

459
### Live Function Calling
460

461
```typescript
462
import { Tool, Type } from '@google/genai';
463

464
const weatherTool: Tool = {
465
  functionDeclarations: [{
466
    name: 'getWeather',
467
    description: 'Get current weather',
468
    parameters: {
469
      type: Type.OBJECT,
470
      properties: {
471
        location: { type: Type.STRING }
472
      },
473
      required: ['location']
474
    }
475
  }]
476
};
477

478
const session = await client.live.connect({
479
  model: 'gemini-2.0-flash-exp',
480
  config: {
481
    responseModalities: ['TEXT'],
482
    tools: [weatherTool]
483
  },
484
  callbacks: {
485
    onmessage: (message) => {
486
      // Handle regular content
487
      if (message.serverContent?.modelTurn) {
488
        console.log('Model:', message.serverContent.modelTurn.parts?.[0]?.text);
489
      }
490

491
      // Handle function calls
492
      if (message.toolCall) {
493
        console.log('Tool call requested:', message.toolCall);
494

495
        // Execute function
496
        const functionCall = message.toolCall.functionCalls?.[0];
497
        if (functionCall?.name === 'getWeather') {
498
          const location = functionCall.args?.location;
499
          const weather = { temperature: 22, condition: 'sunny' };
500

501
          // Send response
502
          session.sendToolResponse({
503
            toolResponse: {
504
              functionResponses: [{
505
                name: 'getWeather',
506
                response: weather,
507
                id: functionCall.id
508
              }]
509
            }
510
          });
511
        }
512
      }
513
    }
514
  }
515
});
516

517
// Ask weather question
518
session.send({
519
  clientContent: {
520
    turns: [{
521
      role: 'user',
522
      parts: [{ text: 'What is the weather in Tokyo?' }]
523
    }],
524
    turnComplete: true
525
  }
526
});
527
```
528

529
### Screen Sharing / Video Input
530

531
```typescript
532
// Connect for video processing
533
const session = await client.live.connect({
534
  model: 'gemini-2.0-flash-exp',
535
  config: {
536
    responseModalities: ['TEXT']
537
  },
538
  callbacks: {
539
    onmessage: (message) => {
540
      if (message.serverContent?.modelTurn) {
541
        console.log('Analysis:', message.serverContent.modelTurn.parts?.[0]?.text);
542
      }
543
    }
544
  }
545
});
546

547
// Send video frames (e.g., from screen capture)
548
function sendVideoFrame(frameData: string) {
549
  session.sendRealtimeInput({
550
    realtimeInput: {
551
      mediaChunks: [{
552
        mimeType: 'image/jpeg',
553
        data: frameData
554
      }]
555
    }
556
  });
557
}
558

559
// Send initial question
560
session.send({
561
  clientContent: {
562
    turns: [{
563
      role: 'user',
564
      parts: [{ text: 'Analyze what you see on the screen' }]
565
    }],
566
    turnComplete: true
567
  }
568
});
569

570
// Continuously send frames
571
const frameInterval = setInterval(() => {
572
  const frame = captureScreenFrame(); // Your screen capture logic
573
  sendVideoFrame(frame);
574
}, 1000); // Send 1 frame per second
575

576
// Stop after 30 seconds
577
setTimeout(() => {
578
  clearInterval(frameInterval);
579
  session.close();
580
}, 30000);
581
```
582

583
### Interruption Handling
584

585
```typescript
586
const session = await client.live.connect({
587
  model: 'gemini-2.0-flash-exp',
588
  config: {
589
    responseModalities: ['TEXT', 'AUDIO']
590
  },
591
  callbacks: {
592
    onmessage: (message) => {
593
      // Check if model was interrupted
594
      if (message.serverContent?.interrupted) {
595
        console.log('Model response was interrupted');
596
      }
597

598
      if (message.serverContent?.modelTurn && !message.serverContent.interrupted) {
599
        console.log('Model:', message.serverContent.modelTurn.parts?.[0]?.text);
600
      }
601
    }
602
  }
603
});
604

605
// Send first message
606
session.send({
607
  clientContent: {
608
    turns: [{
609
      role: 'user',
610
      parts: [{ text: 'Tell me a long story' }]
611
    }],
612
    turnComplete: true
613
  }
614
});
615

616
// Interrupt with new message
617
setTimeout(() => {
618
  session.send({
619
    clientContent: {
620
      turns: [{
621
        role: 'user',
622
        parts: [{ text: 'Actually, just tell me a joke instead' }]
623
      }],
624
      turnComplete: true
625
    }
626
  });
627
}, 2000);
628
```
629

630
### Multi-turn Conversation
631

632
```typescript
633
const conversationHistory: Content[] = [];
634

635
const session = await client.live.connect({
636
  model: 'gemini-2.0-flash-exp',
637
  config: {
638
    responseModalities: ['TEXT'],
639
    systemInstruction: 'You are a helpful coding assistant'
640
  },
641
  callbacks: {
642
    onmessage: (message) => {
643
      if (message.serverContent?.modelTurn && message.serverContent.turnComplete) {
644
        // Save to history
645
        conversationHistory.push(message.serverContent.modelTurn);
646

647
        const text = message.serverContent.modelTurn.parts?.[0]?.text;
648
        console.log('Assistant:', text);
649
      }
650
    }
651
  }
652
});
653

654
// Helper to send message
655
function sendMessage(text: string) {
656
  const userTurn: Content = {
657
    role: 'user',
658
    parts: [{ text }]
659
  };
660

661
  conversationHistory.push(userTurn);
662

663
  session.send({
664
    clientContent: {
665
      turns: [userTurn],
666
      turnComplete: true
667
    }
668
  });
669
}
670

671
// Conversation
672
sendMessage('How do I read a file in Python?');
673

674
setTimeout(() => {
675
  sendMessage('Can you show me an example with error handling?');
676
}, 3000);
677

678
setTimeout(() => {
679
  sendMessage('What about reading CSV files?');
680
}, 6000);
681
```
682

683
### Error Recovery
684

685
```typescript
686
let session: Session | null = null;
687
let reconnectAttempts = 0;
688
const MAX_RECONNECT_ATTEMPTS = 3;
689

690
async function connectWithRetry() {
691
  try {
692
    session = await client.live.connect({
693
      model: 'gemini-2.0-flash-exp',
694
      config: {
695
        responseModalities: ['TEXT']
696
      },
697
      callbacks: {
698
        onopen: () => {
699
          console.log('Connected');
700
          reconnectAttempts = 0;
701
        },
702
        onmessage: (message) => {
703
          // Handle messages
704
        },
705
        onerror: (error) => {
706
          console.error('Error:', error);
707
        },
708
        onclose: () => {
709
          console.log('Connection closed');
710

711
          // Attempt reconnect
712
          if (reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
713
            reconnectAttempts++;
714
            console.log(`Reconnecting... (attempt ${reconnectAttempts})`);
715
            setTimeout(connectWithRetry, 2000);
716
          }
717
        }
718
      }
719
    });
720
  } catch (error) {
721
    console.error('Connection failed:', error);
722

723
    if (reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
724
      reconnectAttempts++;
725
      setTimeout(connectWithRetry, 2000);
726
    }
727
  }
728
}
729

730
// Start connection
731
await connectWithRetry();
732
```
733

734
### Live Music Generation
735

736
```typescript
737
import { Scale, MusicGenerationMode } from '@google/genai';
738

739
const musicSession = await client.live.music.connect({
740
  model: 'music-generation-model',
741
  config: {
742
    tempo: 120,
743
    scale: Scale.C_MAJOR_A_MINOR,
744
    mode: MusicGenerationMode.QUALITY
745
  },
746
  callbacks: {
747
    onmessage: (message) => {
748
      if (message.serverContent?.modelTurn) {
749
        message.serverContent.modelTurn.parts?.forEach(part => {
750
          if (part.inlineData?.mimeType?.startsWith('audio/')) {
751
            console.log('Music chunk received');
752
            // Play or save audio chunk
753
            const audioData = part.inlineData.data;
754
            playAudioChunk(audioData);
755
          }
756
        });
757
      }
758
    }
759
  }
760
});
761

762
// Request music generation
763
musicSession.send({
764
  clientContent: {
765
    turns: [{
766
      role: 'user',
767
      parts: [{ text: 'Generate upbeat electronic music' }]
768
    }],
769
    turnComplete: true
770
  }
771
});
772
```
773

774
### Real-time Translation
775

776
```typescript
777
const session = await client.live.connect({
778
  model: 'gemini-2.0-flash-exp',
779
  config: {
780
    responseModalities: ['TEXT', 'AUDIO'],
781
    systemInstruction: 'Translate speech from English to Spanish in real-time'
782
  },
783
  callbacks: {
784
    onmessage: (message) => {
785
      if (message.serverContent?.modelTurn) {
786
        const translated = message.serverContent.modelTurn.parts?.[0]?.text;
787
        console.log('Translated:', translated);
788

789
        // Output audio translation if available
790
        const audiopart = message.serverContent.modelTurn.parts?.find(
791
          p => p.inlineData?.mimeType?.startsWith('audio/')
792
        );
793
        if (audiopart?.inlineData?.data) {
794
          playAudioChunk(audiopart.inlineData.data);
795
        }
796
      }
797
    }
798
  }
799
});
800

801
// Send audio in chunks as user speaks
802
microphoneStream.on('data', (audioChunk) => {
803
  session.sendRealtimeInput({
804
    realtimeInput: {
805
      mediaChunks: [{
806
        mimeType: 'audio/pcm',
807
        data: audioChunk.toString('base64')
808
      }]
809
    }
810
  });
811
});
812
```
813