[Firebase AI] Add thought summary and signature support (#9192)

Author: andrewheard

.changeset/brave-llamas-impress.md

@@ -0,0 +1,6 @@
+---
+'@firebase/ai': minor
+'firebase': minor
+---
+
+Add `thoughtSummary()` convenience method to `EnhancedGenerateContentResponse`.

common/api-review/ai.api.md

@@ -223,10 +223,10 @@ export { Date_2 as Date }
 
 // @public
 export interface EnhancedGenerateContentResponse extends GenerateContentResponse {
-    // (undocumented)
     functionCalls: () => FunctionCall[] | undefined;
     inlineDataParts: () => InlineDataPart[] | undefined;
     text: () => string;
+    thoughtSummary: () => string | undefined;
 }
 
 // @public
@@ -264,6 +264,10 @@ export interface FileDataPart {
     inlineData?: never;
     // (undocumented)
     text?: never;
+    // (undocumented)
+    thought?: boolean;
+    // @internal (undocumented)
+    thoughtSignature?: never;
 }
 
 // @public
@@ -318,6 +322,10 @@ export interface FunctionCallPart {
     inlineData?: never;
     // (undocumented)
     text?: never;
+    // (undocumented)
+    thought?: boolean;
+    // @internal (undocumented)
+    thoughtSignature?: never;
 }
 
 // @public
@@ -350,6 +358,10 @@ export interface FunctionResponsePart {
     inlineData?: never;
     // (undocumented)
     text?: never;
+    // (undocumented)
+    thought?: boolean;
+    // @internal (undocumented)
+    thoughtSignature?: never;
 }
 
 // @public
@@ -732,6 +744,10 @@ export interface InlineDataPart {
     inlineData: GenerativeContentBlob;
     // (undocumented)
     text?: never;
+    // (undocumented)
+    thought?: boolean;
+    // @internal (undocumented)
+    thoughtSignature?: never;
     videoMetadata?: VideoMetadata;
 }
 
@@ -1063,10 +1079,15 @@ export interface TextPart {
     inlineData?: never;
     // (undocumented)
     text: string;
+    // (undocumented)
+    thought?: boolean;
+    // @internal (undocumented)
+    thoughtSignature?: string;
 }
 
 // @public
 export interface ThinkingConfig {
+    includeThoughts?: boolean;
     thinkingBudget?: number;
 }
 

docs-devsite/ai.enhancedgeneratecontentresponse.md

@@ -23,12 +23,15 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse
 
 |  Property | Type | Description |
 |  --- | --- | --- |
-|  [functionCalls](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsefunctioncalls) | () =&gt; [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->\[\] \| undefined |  |
-|  [inlineDataParts](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedataparts) | () =&gt; [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface)<!-- -->\[\] \| undefined | Aggregates and returns all [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface)<!-- -->s from the [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->'s first candidate. |
+|  [functionCalls](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsefunctioncalls) | () =&gt; [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->\[\] \| undefined | Aggregates and returns every [FunctionCall](./ai.functioncall.md#functioncall_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->. |
+|  [inlineDataParts](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponseinlinedataparts) | () =&gt; [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface)<!-- -->\[\] \| undefined | Aggregates and returns every [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->. |
 |  [text](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsetext) | () =&gt; string | Returns the text string from the response, if available. Throws if the prompt or candidate was blocked. |
+|  [thoughtSummary](./ai.enhancedgeneratecontentresponse.md#enhancedgeneratecontentresponsethoughtsummary) | () =&gt; string \| undefined | Aggregates and returns every [TextPart](./ai.textpart.md#textpart_interface) with their <code>thought</code> property set to <code>true</code> from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->. |
 
 ## EnhancedGenerateContentResponse.functionCalls
 
+Aggregates and returns every [FunctionCall](./ai.functioncall.md#functioncall_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->.
+
 <b>Signature:</b>
 
 ```typescript
@@ -37,7 +40,7 @@ functionCalls: () => FunctionCall[] | undefined;
 
 ## EnhancedGenerateContentResponse.inlineDataParts
 
-Aggregates and returns all [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface)<!-- -->s from the [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->'s first candidate.
+Aggregates and returns every [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->.
 
 <b>Signature:</b>
 
@@ -54,3 +57,17 @@ Returns the text string from the response, if available. Throws if the prompt or
 ```typescript
 text: () => string;
 ```
+
+## EnhancedGenerateContentResponse.thoughtSummary
+
+Aggregates and returns every [TextPart](./ai.textpart.md#textpart_interface) with their `thought` property set to `true` from the first candidate of [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->.
+
+Thought summaries provide a brief overview of the model's internal thinking process, offering insight into how it arrived at the final answer. This can be useful for debugging, understanding the model's reasoning, and verifying its accuracy.
+
+Thoughts will only be included if [ThinkingConfig.includeThoughts](./ai.thinkingconfig.md#thinkingconfigincludethoughts) is set to `true`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+thoughtSummary: () => string | undefined;
+```

docs-devsite/ai.filedatapart.md

@@ -27,6 +27,7 @@ export interface FileDataPart
 |  [functionResponse](./ai.filedatapart.md#filedatapartfunctionresponse) | never |  |
 |  [inlineData](./ai.filedatapart.md#filedatapartinlinedata) | never |  |
 |  [text](./ai.filedatapart.md#filedataparttext) | never |  |
+|  [thought](./ai.filedatapart.md#filedatapartthought) | boolean |  |
 
 ## FileDataPart.fileData
 
@@ -67,3 +68,11 @@ inlineData?: never;
 ```typescript
 text?: never;
 ```
+
+## FileDataPart.thought
+
+<b>Signature:</b>
+
+```typescript
+thought?: boolean;
+```

docs-devsite/ai.functioncallpart.md

@@ -26,6 +26,7 @@ export interface FunctionCallPart
 |  [functionResponse](./ai.functioncallpart.md#functioncallpartfunctionresponse) | never |  |
 |  [inlineData](./ai.functioncallpart.md#functioncallpartinlinedata) | never |  |
 |  [text](./ai.functioncallpart.md#functioncallparttext) | never |  |
+|  [thought](./ai.functioncallpart.md#functioncallpartthought) | boolean |  |
 
 ## FunctionCallPart.functionCall
 
@@ -58,3 +59,11 @@ inlineData?: never;
 ```typescript
 text?: never;
 ```
+
+## FunctionCallPart.thought
+
+<b>Signature:</b>
+
+```typescript
+thought?: boolean;
+```

docs-devsite/ai.functionresponsepart.md

@@ -26,6 +26,7 @@ export interface FunctionResponsePart
 |  [functionResponse](./ai.functionresponsepart.md#functionresponsepartfunctionresponse) | [FunctionResponse](./ai.functionresponse.md#functionresponse_interface) |  |
 |  [inlineData](./ai.functionresponsepart.md#functionresponsepartinlinedata) | never |  |
 |  [text](./ai.functionresponsepart.md#functionresponseparttext) | never |  |
+|  [thought](./ai.functionresponsepart.md#functionresponsepartthought) | boolean |  |
 
 ## FunctionResponsePart.functionCall
 
@@ -58,3 +59,11 @@ inlineData?: never;
 ```typescript
 text?: never;
 ```
+
+## FunctionResponsePart.thought
+
+<b>Signature:</b>
+
+```typescript
+thought?: boolean;
+```

docs-devsite/ai.inlinedatapart.md

@@ -26,6 +26,7 @@ export interface InlineDataPart
 |  [functionResponse](./ai.inlinedatapart.md#inlinedatapartfunctionresponse) | never |  |
 |  [inlineData](./ai.inlinedatapart.md#inlinedatapartinlinedata) | [GenerativeContentBlob](./ai.generativecontentblob.md#generativecontentblob_interface) |  |
 |  [text](./ai.inlinedatapart.md#inlinedataparttext) | never |  |
+|  [thought](./ai.inlinedatapart.md#inlinedatapartthought) | boolean |  |
 |  [videoMetadata](./ai.inlinedatapart.md#inlinedatapartvideometadata) | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Applicable if <code>inlineData</code> is a video. |
 
 ## InlineDataPart.functionCall
@@ -60,6 +61,14 @@ inlineData: GenerativeContentBlob;
 text?: never;
 ```
 
+## InlineDataPart.thought
+
+<b>Signature:</b>
+
+```typescript
+thought?: boolean;
+```
+
 ## InlineDataPart.videoMetadata
 
 Applicable if `inlineData` is a video.

docs-devsite/ai.textpart.md

@@ -26,6 +26,7 @@ export interface TextPart
 |  [functionResponse](./ai.textpart.md#textpartfunctionresponse) | never |  |
 |  [inlineData](./ai.textpart.md#textpartinlinedata) | never |  |
 |  [text](./ai.textpart.md#textparttext) | string |  |
+|  [thought](./ai.textpart.md#textpartthought) | boolean |  |
 
 ## TextPart.functionCall
 
@@ -58,3 +59,11 @@ inlineData?: never;
 ```typescript
 text: string;
 ```
+
+## TextPart.thought
+
+<b>Signature:</b>
+
+```typescript
+thought?: boolean;
+```

docs-devsite/ai.thinkingconfig.md

@@ -24,8 +24,21 @@ export interface ThinkingConfig
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [includeThoughts](./ai.thinkingconfig.md#thinkingconfigincludethoughts) | boolean | Whether to include "thought summaries" in the model's response. |
 |  [thinkingBudget](./ai.thinkingconfig.md#thinkingconfigthinkingbudget) | number | The thinking budget, in tokens.<!-- -->This parameter sets an upper limit on the number of tokens the model can use for its internal "thinking" process. A higher budget may result in higher quality responses for complex tasks but can also increase latency and cost.<!-- -->If you don't specify a budget, the model will determine the appropriate amount of thinking based on the complexity of the prompt.<!-- -->An error will be thrown if you set a thinking budget for a model that does not support this feature or if the specified budget is not within the model's supported range. |
 
+## ThinkingConfig.includeThoughts
+
+Whether to include "thought summaries" in the model's response.
+
+Thought summaries provide a brief overview of the model's internal thinking process, offering insight into how it arrived at the final answer. This can be useful for debugging, understanding the model's reasoning, and verifying its accuracy.
+
+<b>Signature:</b>
+
+```typescript
+includeThoughts?: boolean;
+```
+
 ## ThinkingConfig.thinkingBudget
 
 The thinking budget, in tokens.

packages/ai/src/methods/chat-session-helpers.test.ts

@@ -22,7 +22,11 @@ import { FirebaseError } from '@firebase/util';
 
 describe('chat-session-helpers', () => {
   describe('validateChatHistory', () => {
-    const TCS: Array<{ history: Content[]; isValid: boolean }> = [
+    const TCS: Array<{
+      history: Content[];
+      isValid: boolean;
+      errorShouldInclude?: string;
+    }> = [
       {
         history: [{ role: 'user', parts: [{ text: 'hi' }] }],
         isValid: true
@@ -99,19 +103,23 @@ describe('chat-session-helpers', () => {
       {
         //@ts-expect-error
         history: [{ role: 'user', parts: '' }],
+        errorShouldInclude: `array of Parts`,
         isValid: false
       },
       {
         //@ts-expect-error
         history: [{ role: 'user' }],
+        errorShouldInclude: `array of Parts`,
         isValid: false
       },
       {
         history: [{ role: 'user', parts: [] }],
+        errorShouldInclude: `at least one part`,
         isValid: false
       },
       {
         history: [{ role: 'model', parts: [{ text: 'hi' }] }],
+        errorShouldInclude: `model`,
         isValid: false
       },
       {
@@ -125,13 +133,15 @@ describe('chat-session-helpers', () => {
             ]
           }
         ],
+        errorShouldInclude: `function`,
         isValid: false
       },
       {
         history: [
           { role: 'user', parts: [{ text: 'hi' }] },
           { role: 'user', parts: [{ text: 'hi' }] }
         ],
+        errorShouldInclude: `can't follow 'user'`,
         isValid: false
       },
       {
@@ -140,6 +150,45 @@ describe('chat-session-helpers', () => {
           { role: 'model', parts: [{ text: 'hi' }] },
           { role: 'model', parts: [{ text: 'hi' }] }
         ],
+        errorShouldInclude: `can't follow 'model'`,
+        isValid: false
+      },
+      {
+        history: [
+          { role: 'user', parts: [{ text: 'hi' }] },
+          {
+            role: 'model',
+            parts: [
+              { text: 'hi' },
+              {
+                text: 'thought about hi',
+                thought: true,
+                thoughtSignature: 'thought signature'
+              }
+            ]
+          }
+        ],
+        isValid: true
+      },
+      {
+        history: [
+          {
+            role: 'user',
+            parts: [{ text: 'hi', thought: true, thoughtSignature: 'sig' }]
+          },
+          {
+            role: 'model',
+            parts: [
+              { text: 'hi' },
+              {
+                text: 'thought about hi',
+                thought: true,
+                thoughtSignature: 'thought signature'
+              }
+            ]
+          }
+        ],
+        errorShouldInclude: 'thought',
         isValid: false
       }
     ];
@@ -149,7 +198,14 @@ describe('chat-session-helpers', () => {
         if (tc.isValid) {
           expect(fn).to.not.throw();
         } else {
-          expect(fn).to.throw(FirebaseError);
+          try {
+            fn();
+          } catch (e) {
+            expect(e).to.be.instanceOf(FirebaseError);
+            if (e instanceof FirebaseError && tc.errorShouldInclude) {
+              expect(e.message).to.include(tc.errorShouldInclude);
+            }
+          }
         }
       });
     });