Live API v1 (#9205)

* Live API v1

* update docs

* Docs and naming changes

* Skip problematic Google AI tests

* Fix mediaChunks errors

* Log warnings for 'close' event reasons

* Add note about support to getLiveGenerativeModel docs

* Review comments

* add license header

* docs review

* remove candidateCount

* Live Refactor WebSocket (#9212)

* refactor websocket

* remove test ws server

Author: dlarocque

common/api-review/ai.api.md

@@ -32,6 +32,7 @@ export const AIErrorCode: {
     readonly REQUEST_ERROR: "request-error";
     readonly RESPONSE_ERROR: "response-error";
     readonly FETCH_ERROR: "fetch-error";
+    readonly SESSION_CLOSED: "session-closed";
     readonly INVALID_CONTENT: "invalid-content";
     readonly API_NOT_ENABLED: "api-not-enabled";
     readonly INVALID_SCHEMA: "invalid-schema";
@@ -262,6 +263,7 @@ export type FinishReason = (typeof FinishReason)[keyof typeof FinishReason];
 export interface FunctionCall {
     // (undocumented)
     args: object;
+    id?: string;
     // (undocumented)
     name: string;
 }
@@ -310,6 +312,7 @@ export interface FunctionDeclarationsTool {
 
 // @public
 export interface FunctionResponse {
+    id?: string;
     // (undocumented)
     name: string;
     // (undocumented)
@@ -444,6 +447,9 @@ export function getGenerativeModel(ai: AI, modelParams: ModelParams, requestOpti
 // @beta
 export function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
 
+// @beta
+export function getLiveGenerativeModel(ai: AI, modelParams: LiveModelParams): LiveGenerativeModel;
+
 // @public
 export class GoogleAIBackend extends Backend {
     constructor();
@@ -699,6 +705,95 @@ export class IntegerSchema extends Schema {
     constructor(schemaParams?: SchemaParams);
 }
 
+// @beta
+export interface LiveGenerationConfig {
+    frequencyPenalty?: number;
+    maxOutputTokens?: number;
+    presencePenalty?: number;
+    responseModalities?: [ResponseModality];
+    speechConfig?: SpeechConfig;
+    temperature?: number;
+    topK?: number;
+    topP?: number;
+}
+
+// @beta
+export class LiveGenerativeModel extends AIModel {
+    // Warning: (ae-forgotten-export) The symbol "WebSocketHandler" needs to be exported by the entry point index.d.ts
+    //
+    // @internal
+    constructor(ai: AI, modelParams: LiveModelParams,
+    _webSocketHandler: WebSocketHandler);
+    connect(): Promise<LiveSession>;
+    // (undocumented)
+    generationConfig: LiveGenerationConfig;
+    // (undocumented)
+    systemInstruction?: Content;
+    // (undocumented)
+    toolConfig?: ToolConfig;
+    // (undocumented)
+    tools?: Tool[];
+    }
+
+// @beta
+export interface LiveModelParams {
+    // (undocumented)
+    generationConfig?: LiveGenerationConfig;
+    // (undocumented)
+    model: string;
+    // (undocumented)
+    systemInstruction?: string | Part | Content;
+    // (undocumented)
+    toolConfig?: ToolConfig;
+    // (undocumented)
+    tools?: Tool[];
+}
+
+// @beta
+export const LiveResponseType: {
+    SERVER_CONTENT: string;
+    TOOL_CALL: string;
+    TOOL_CALL_CANCELLATION: string;
+};
+
+// @beta
+export type LiveResponseType = (typeof LiveResponseType)[keyof typeof LiveResponseType];
+
+// @beta
+export interface LiveServerContent {
+    interrupted?: boolean;
+    modelTurn?: Content;
+    turnComplete?: boolean;
+    // (undocumented)
+    type: 'serverContent';
+}
+
+// @beta
+export interface LiveServerToolCall {
+    functionCalls: FunctionCall[];
+    // (undocumented)
+    type: 'toolCall';
+}
+
+// @beta
+export interface LiveServerToolCallCancellation {
+    functionIds: string[];
+    // (undocumented)
+    type: 'toolCallCancellation';
+}
+
+// @beta
+export class LiveSession {
+    // @internal
+    constructor(webSocketHandler: WebSocketHandler, serverMessages: AsyncGenerator<unknown>);
+    close(): Promise<void>;
+    isClosed: boolean;
+    receive(): AsyncGenerator<LiveServerContent | LiveServerToolCall | LiveServerToolCallCancellation>;
+    send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
+    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    }
+
 // @public
 export const Modality: {
     readonly MODALITY_UNSPECIFIED: "MODALITY_UNSPECIFIED";
@@ -763,6 +858,11 @@ export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionRespon
 // @public
 export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"];
 
+// @beta
+export interface PrebuiltVoiceConfig {
+    voiceConfig?: string;
+}
+
 // @public
 export interface PromptFeedback {
     // (undocumented)
@@ -926,6 +1026,11 @@ export interface Segment {
     text: string;
 }
 
+// @beta
+export interface SpeechConfig {
+    voiceConfig?: VoiceConfig;
+}
+
 // @public
 export interface StartChatParams extends BaseParams {
     // (undocumented)
@@ -1003,6 +1108,11 @@ export interface VideoMetadata {
     startOffset: string;
 }
 
+// @beta
+export interface VoiceConfig {
+    prebuiltVoiceConfig?: PrebuiltVoiceConfig;
+}
+
 // @public (undocumented)
 export interface WebAttribution {
     // (undocumented)

docs-devsite/_toc.yaml

@@ -108,6 +108,20 @@ toc:
     path: /docs/reference/js/ai.inlinedatapart.md
   - title: IntegerSchema
     path: /docs/reference/js/ai.integerschema.md
+  - title: LiveGenerationConfig
+    path: /docs/reference/js/ai.livegenerationconfig.md
+  - title: LiveGenerativeModel
+    path: /docs/reference/js/ai.livegenerativemodel.md
+  - title: LiveModelParams
+    path: /docs/reference/js/ai.livemodelparams.md
+  - title: LiveServerContent
+    path: /docs/reference/js/ai.liveservercontent.md
+  - title: LiveServerToolCall
+    path: /docs/reference/js/ai.liveservertoolcall.md
+  - title: LiveServerToolCallCancellation
+    path: /docs/reference/js/ai.liveservertoolcallcancellation.md
+  - title: LiveSession
+    path: /docs/reference/js/ai.livesession.md
   - title: ModalityTokenCount
     path: /docs/reference/js/ai.modalitytokencount.md
   - title: ModelParams
@@ -118,6 +132,8 @@ toc:
     path: /docs/reference/js/ai.objectschema.md
   - title: ObjectSchemaRequest
     path: /docs/reference/js/ai.objectschemarequest.md
+  - title: PrebuiltVoiceConfig
+    path: /docs/reference/js/ai.prebuiltvoiceconfig.md
   - title: PromptFeedback
     path: /docs/reference/js/ai.promptfeedback.md
   - title: RequestOptions
@@ -142,6 +158,8 @@ toc:
     path: /docs/reference/js/ai.searchentrypoint.md
   - title: Segment
     path: /docs/reference/js/ai.segment.md
+  - title: SpeechConfig
+    path: /docs/reference/js/ai.speechconfig.md
   - title: StartChatParams
     path: /docs/reference/js/ai.startchatparams.md
   - title: StringSchema
@@ -158,6 +176,8 @@ toc:
     path: /docs/reference/js/ai.vertexaibackend.md
   - title: VideoMetadata
     path: /docs/reference/js/ai.videometadata.md
+  - title: VoiceConfig
+    path: /docs/reference/js/ai.voiceconfig.md
   - title: WebAttribution
     path: /docs/reference/js/ai.webattribution.md
   - title: WebGroundingChunk

docs-devsite/ai.functioncall.md

@@ -23,6 +23,7 @@ export interface FunctionCall
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [args](./ai.functioncall.md#functioncallargs) | object |  |
+|  [id](./ai.functioncall.md#functioncallid) | string | The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->. |
 |  [name](./ai.functioncall.md#functioncallname) | string |  |
 
 ## FunctionCall.args
@@ -33,6 +34,18 @@ export interface FunctionCall
 args: object;
 ```
 
+## FunctionCall.id
+
+The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->.
+
+This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be `undefined`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+id?: string;
+```
+
 ## FunctionCall.name
 
 <b>Signature:</b>

docs-devsite/ai.functionresponse.md

@@ -22,9 +22,22 @@ export interface FunctionResponse
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [id](./ai.functionresponse.md#functionresponseid) | string | The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->. |
 |  [name](./ai.functionresponse.md#functionresponsename) | string |  |
 |  [response](./ai.functionresponse.md#functionresponseresponse) | object |  |
 
+## FunctionResponse.id
+
+The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->.
+
+This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be `undefined`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+id?: string;
+```
+
 ## FunctionResponse.name
 
 <b>Signature:</b>