Firebase AI Hybrid Inference Implementation (#9029)

Co-authored-by: Daniel La Rocque <[email protected]>
Co-authored-by: Erik Eldridge <[email protected]>
Co-authored-by: Christina Holland <[email protected]>

Author: 4 people

.changeset/purple-chairs-wait.md

@@ -0,0 +1,6 @@
+---
+'@firebase/ai': minor
+'firebase': minor
+---
+
+Add hybrid inference options to the Firebase AI SDK.

common/api-review/ai.api.md

@@ -125,7 +125,7 @@ export class BooleanSchema extends Schema {
 
 // @public
 export class ChatSession {
-    constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined);
+    constructor(apiSettings: ApiSettings, model: string, chromeAdapter?: ChromeAdapter | undefined, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined);
     getHistory(): Promise<Content[]>;
     // (undocumented)
     model: string;
@@ -137,6 +137,15 @@ export class ChatSession {
     sendMessageStream(request: string | Array<string | Part>): Promise<GenerateContentStreamResult>;
     }
 
+// @public
+export interface ChromeAdapter {
+    // @internal (undocumented)
+    countTokens(request: CountTokensRequest): Promise<Response>;
+    generateContent(request: GenerateContentRequest): Promise<Response>;
+    generateContentStream(request: GenerateContentRequest): Promise<Response>;
+    isAvailable(request: GenerateContentRequest): Promise<boolean>;
+}
+
 // @public
 export interface Citation {
     // (undocumented)
@@ -416,7 +425,7 @@ export interface GenerativeContentBlob {
 
 // @public
 export class GenerativeModel extends AIModel {
-    constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions);
+    constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions, chromeAdapter?: ChromeAdapter | undefined);
     countTokens(request: CountTokensRequest | string | Array<string | Part>): Promise<CountTokensResponse>;
     generateContent(request: GenerateContentRequest | string | Array<string | Part>): Promise<GenerateContentResult>;
     generateContentStream(request: GenerateContentRequest | string | Array<string | Part>): Promise<GenerateContentStreamResult>;
@@ -439,7 +448,7 @@ export class GenerativeModel extends AIModel {
 export function getAI(app?: FirebaseApp, options?: AIOptions): AI;
 
 // @public
-export function getGenerativeModel(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel;
+export function getGenerativeModel(ai: AI, modelParams: ModelParams | HybridParams, requestOptions?: RequestOptions): GenerativeModel;
 
 // @beta
 export function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
@@ -588,6 +597,13 @@ export const HarmSeverity: {
 // @public
 export type HarmSeverity = (typeof HarmSeverity)[keyof typeof HarmSeverity];
 
+// @public
+export interface HybridParams {
+    inCloudParams?: ModelParams;
+    mode: InferenceMode;
+    onDeviceParams?: OnDeviceParams;
+}
+
 // @beta
 export const ImagenAspectRatio: {
     readonly SQUARE: "1:1";
@@ -600,7 +616,7 @@ export const ImagenAspectRatio: {
 // @beta
 export type ImagenAspectRatio = (typeof ImagenAspectRatio)[keyof typeof ImagenAspectRatio];
 
-// @public
+// @beta
 export interface ImagenGCSImage {
     gcsURI: string;
     mimeType: string;
@@ -681,6 +697,16 @@ export interface ImagenSafetySettings {
     safetyFilterLevel?: ImagenSafetyFilterLevel;
 }
 
+// @public
+export const InferenceMode: {
+    readonly PREFER_ON_DEVICE: "prefer_on_device";
+    readonly ONLY_ON_DEVICE: "only_on_device";
+    readonly ONLY_IN_CLOUD: "only_in_cloud";
+};
+
+// @public
+export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode];
+
 // @public
 export interface InlineDataPart {
     // (undocumented)
@@ -699,6 +725,63 @@ export class IntegerSchema extends Schema {
     constructor(schemaParams?: SchemaParams);
 }
 
+// @public
+export interface LanguageModelCreateCoreOptions {
+    // (undocumented)
+    expectedInputs?: LanguageModelExpected[];
+    // (undocumented)
+    temperature?: number;
+    // (undocumented)
+    topK?: number;
+}
+
+// @public
+export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions {
+    // (undocumented)
+    initialPrompts?: LanguageModelMessage[];
+    // (undocumented)
+    signal?: AbortSignal;
+}
+
+// @public
+export interface LanguageModelExpected {
+    // (undocumented)
+    languages?: string[];
+    // (undocumented)
+    type: LanguageModelMessageType;
+}
+
+// @public
+export interface LanguageModelMessage {
+    // (undocumented)
+    content: LanguageModelMessageContent[];
+    // (undocumented)
+    role: LanguageModelMessageRole;
+}
+
+// @public
+export interface LanguageModelMessageContent {
+    // (undocumented)
+    type: LanguageModelMessageType;
+    // (undocumented)
+    value: LanguageModelMessageContentValue;
+}
+
+// @public
+export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | BufferSource | string;
+
+// @public
+export type LanguageModelMessageRole = 'system' | 'user' | 'assistant';
+
+// @public
+export type LanguageModelMessageType = 'text' | 'image' | 'audio';
+
+// @public
+export interface LanguageModelPromptOptions {
+    // (undocumented)
+    responseConstraint?: object;
+}
+
 // @public
 export const Modality: {
     readonly MODALITY_UNSPECIFIED: "MODALITY_UNSPECIFIED";
@@ -757,6 +840,14 @@ export interface ObjectSchemaRequest extends SchemaRequest {
     type: 'object';
 }
 
+// @public
+export interface OnDeviceParams {
+    // (undocumented)
+    createOptions?: LanguageModelCreateOptions;
+    // (undocumented)
+    promptOptions?: LanguageModelPromptOptions;
+}
+
 // @public
 export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart;
 

docs-devsite/_toc.yaml

@@ -24,6 +24,8 @@ toc:
     path: /docs/reference/js/ai.booleanschema.md
   - title: ChatSession
     path: /docs/reference/js/ai.chatsession.md
+  - title: ChromeAdapter
+    path: /docs/reference/js/ai.chromeadapter.md
   - title: Citation
     path: /docs/reference/js/ai.citation.md
   - title: CitationMetadata
@@ -88,6 +90,8 @@ toc:
     path: /docs/reference/js/ai.groundingmetadata.md
   - title: GroundingSupport
     path: /docs/reference/js/ai.groundingsupport.md
+  - title: HybridParams
+    path: /docs/reference/js/ai.hybridparams.md
   - title: ImagenGCSImage
     path: /docs/reference/js/ai.imagengcsimage.md
   - title: ImagenGenerationConfig
@@ -108,6 +112,18 @@ toc:
     path: /docs/reference/js/ai.inlinedatapart.md
   - title: IntegerSchema
     path: /docs/reference/js/ai.integerschema.md
+  - title: LanguageModelCreateCoreOptions
+    path: /docs/reference/js/ai.languagemodelcreatecoreoptions.md
+  - title: LanguageModelCreateOptions
+    path: /docs/reference/js/ai.languagemodelcreateoptions.md
+  - title: LanguageModelExpected
+    path: /docs/reference/js/ai.languagemodelexpected.md
+  - title: LanguageModelMessage
+    path: /docs/reference/js/ai.languagemodelmessage.md
+  - title: LanguageModelMessageContent
+    path: /docs/reference/js/ai.languagemodelmessagecontent.md
+  - title: LanguageModelPromptOptions
+    path: /docs/reference/js/ai.languagemodelpromptoptions.md
   - title: ModalityTokenCount
     path: /docs/reference/js/ai.modalitytokencount.md
   - title: ModelParams
@@ -118,6 +134,8 @@ toc:
     path: /docs/reference/js/ai.objectschema.md
   - title: ObjectSchemaRequest
     path: /docs/reference/js/ai.objectschemarequest.md
+  - title: OnDeviceParams
+    path: /docs/reference/js/ai.ondeviceparams.md
   - title: PromptFeedback
     path: /docs/reference/js/ai.promptfeedback.md
   - title: RequestOptions

docs-devsite/ai.chatsession.md

@@ -22,7 +22,7 @@ export declare class ChatSession
 
 |  Constructor | Modifiers | Description |
 |  --- | --- | --- |
-|  [(constructor)(apiSettings, model, params, requestOptions)](./ai.chatsession.md#chatsessionconstructor) |  | Constructs a new instance of the <code>ChatSession</code> class |
+|  [(constructor)(apiSettings, model, chromeAdapter, params, requestOptions)](./ai.chatsession.md#chatsessionconstructor) |  | Constructs a new instance of the <code>ChatSession</code> class |
 
 ## Properties
 
@@ -47,7 +47,7 @@ Constructs a new instance of the `ChatSession` class
 <b>Signature:</b>
 
 ```typescript
-constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined);
+constructor(apiSettings: ApiSettings, model: string, chromeAdapter?: ChromeAdapter | undefined, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined);
 ```
 
 #### Parameters
@@ -56,6 +56,7 @@ constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams |
 |  --- | --- | --- |
 |  apiSettings | ApiSettings |  |
 |  model | string |  |
+|  chromeAdapter | [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) \| undefined |  |
 |  params | [StartChatParams](./ai.startchatparams.md#startchatparams_interface) \| undefined |  |
 |  requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) \| undefined |  |
 

docs-devsite/ai.chromeadapter.md

@@ -0,0 +1,94 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# ChromeAdapter interface
+<b>(EXPERIMENTAL)</b> Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device inference is possible.
+
+These methods should not be called directly by the user.
+
+<b>Signature:</b>
+
+```typescript
+export interface ChromeAdapter 
+```
+
+## Methods
+
+|  Method | Description |
+|  --- | --- |
+|  [generateContent(request)](./ai.chromeadapter.md#chromeadaptergeneratecontent) | Generates content using on-device inference.<p>This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content using in-cloud inference.</p> |
+|  [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | Generates a content stream using on-device inference.<p>This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating a content stream using in-cloud inference.</p> |
+|  [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | Checks if the on-device model is capable of handling a given request. |
+
+## ChromeAdapter.generateContent()
+
+Generates content using on-device inference.
+
+<p>This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content using in-cloud inference.</p>
+
+<b>Signature:</b>
+
+```typescript
+generateContent(request: GenerateContentRequest): Promise<Response>;
+```
+
+#### Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | a standard Firebase AI [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) |
+
+<b>Returns:</b>
+
+Promise&lt;Response&gt;
+
+## ChromeAdapter.generateContentStream()
+
+Generates a content stream using on-device inference.
+
+<p>This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating a content stream using in-cloud inference.</p>
+
+<b>Signature:</b>
+
+```typescript
+generateContentStream(request: GenerateContentRequest): Promise<Response>;
+```
+
+#### Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | a standard Firebase AI [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) |
+
+<b>Returns:</b>
+
+Promise&lt;Response&gt;
+
+## ChromeAdapter.isAvailable()
+
+Checks if the on-device model is capable of handling a given request.
+
+<b>Signature:</b>
+
+```typescript
+isAvailable(request: GenerateContentRequest): Promise<boolean>;
+```
+
+#### Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | A potential request to be passed to the model. |
+
+<b>Returns:</b>
+
+Promise&lt;boolean&gt;
+

docs-devsite/ai.generativemodel.md

@@ -23,7 +23,7 @@ export declare class GenerativeModel extends AIModel
 
 |  Constructor | Modifiers | Description |
 |  --- | --- | --- |
-|  [(constructor)(ai, modelParams, requestOptions)](./ai.generativemodel.md#generativemodelconstructor) |  | Constructs a new instance of the <code>GenerativeModel</code> class |
+|  [(constructor)(ai, modelParams, requestOptions, chromeAdapter)](./ai.generativemodel.md#generativemodelconstructor) |  | Constructs a new instance of the <code>GenerativeModel</code> class |
 
 ## Properties
 
@@ -52,7 +52,7 @@ Constructs a new instance of the `GenerativeModel` class
 <b>Signature:</b>
 
 ```typescript
-constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions);
+constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions, chromeAdapter?: ChromeAdapter | undefined);
 ```
 
 #### Parameters
@@ -62,6 +62,7 @@ constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions);
 |  ai | [AI](./ai.ai.md#ai_interface) |  |
 |  modelParams | [ModelParams](./ai.modelparams.md#modelparams_interface) |  |
 |  requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) |  |
+|  chromeAdapter | [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) \| undefined |  |
 
 ## GenerativeModel.generationConfig