diff --git a/firebaseai/ChatExample/Assets.xcassets/AccentColor.colorset/Contents.json b/firebaseai/ChatExample/Assets.xcassets/AccentColor.colorset/Contents.json deleted file mode 100644 index eb8789700..000000000 --- a/firebaseai/ChatExample/Assets.xcassets/AccentColor.colorset/Contents.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "colors" : [ - { - "idiom" : "universal" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/ChatExample/Assets.xcassets/AppIcon.appiconset/Contents.json b/firebaseai/ChatExample/Assets.xcassets/AppIcon.appiconset/Contents.json deleted file mode 100644 index 13613e3ee..000000000 --- a/firebaseai/ChatExample/Assets.xcassets/AppIcon.appiconset/Contents.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "images" : [ - { - "idiom" : "universal", - "platform" : "ios", - "size" : "1024x1024" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/ChatExample/Models/ChatMessage.swift b/firebaseai/ChatExample/Models/ChatMessage.swift index f1171c748..4f2ab04c1 100644 --- a/firebaseai/ChatExample/Models/ChatMessage.swift +++ b/firebaseai/ChatExample/Models/ChatMessage.swift @@ -15,27 +15,34 @@ import FirebaseAI import Foundation import ConversationKit +import UIKit public struct ChatMessage: Message { public let id: UUID = .init() public var content: String? - public let imageURL: String? public let participant: Participant public let error: (any Error)? public var pending = false public var groundingMetadata: GroundingMetadata? + public var attachments: [MultimodalAttachment] = [] + public var image: UIImage? + // required by the Message protocol, but not used in this app + public var imageURL: String? public init(content: String? = nil, imageURL: String? = nil, participant: Participant, - error: (any Error)? = nil, pending: Bool = false) { + error: (any Error)? = nil, pending: Bool = false, + attachments: [MultimodalAttachment] = [], image: UIImage? = nil) { self.content = content self.imageURL = imageURL self.participant = participant self.error = error self.pending = pending + self.attachments = attachments + self.image = image } // Protocol-required initializer - public init(content: String?, imageURL: String?, participant: Participant) { + public init(content: String?, imageURL: String? = nil, participant: Participant) { self.content = content self.imageURL = imageURL self.participant = participant @@ -54,16 +61,18 @@ extension ChatMessage { public static func == (lhs: ChatMessage, rhs: ChatMessage) -> Bool { lhs.id == rhs.id && lhs.content == rhs.content && - lhs.imageURL == rhs.imageURL && - lhs.participant == rhs.participant + lhs.participant == rhs.participant && + lhs.image == rhs.image && + lhs.attachments == rhs.attachments // intentionally ignore `error` } public func hash(into hasher: inout Hasher) { hasher.combine(id) hasher.combine(content) - hasher.combine(imageURL) hasher.combine(participant) + hasher.combine(image) + hasher.combine(attachments) // intentionally ignore `error` } } diff --git a/firebaseai/ChatExample/Screens/ChatScreen.swift b/firebaseai/ChatExample/Screens/ChatScreen.swift index 880a97ce4..4f35ddb11 100644 --- a/firebaseai/ChatExample/Screens/ChatScreen.swift +++ b/firebaseai/ChatExample/Screens/ChatScreen.swift @@ -17,13 +17,13 @@ import SwiftUI import ConversationKit struct ChatScreen: View { - let firebaseService: FirebaseAI + let backendType: BackendOption @StateObject var viewModel: ChatViewModel - init(firebaseService: FirebaseAI, sample: Sample? = nil) { - self.firebaseService = firebaseService + init(backendType: BackendOption, sample: Sample? = nil) { + self.backendType = backendType _viewModel = - StateObject(wrappedValue: ChatViewModel(firebaseService: firebaseService, + StateObject(wrappedValue: ChatViewModel(backendType: backendType, sample: sample)) } @@ -35,9 +35,7 @@ struct ChatScreen: View { } .disableAttachments() .onSendMessage { message in - Task { - await viewModel.sendMessage(message.content ?? "", streaming: true) - } + await viewModel.sendMessage(message.content ?? "", streaming: true) } .onError { error in viewModel.presentErrorDetails = true @@ -65,5 +63,5 @@ struct ChatScreen: View { } #Preview { - ChatScreen(firebaseService: FirebaseAI.firebaseAI()) + ChatScreen(backendType: .googleAI) } diff --git a/firebaseai/ChatExample/ViewModels/ChatViewModel.swift b/firebaseai/ChatExample/ViewModels/ChatViewModel.swift index ad1077407..58f04f9ef 100644 --- a/firebaseai/ChatExample/ViewModels/ChatViewModel.swift +++ b/firebaseai/ChatExample/ViewModels/ChatViewModel.swift @@ -36,24 +36,27 @@ class ChatViewModel: ObservableObject { private var model: GenerativeModel private var chat: Chat - private var stopGenerating = false private var chatTask: Task? private var sample: Sample? + private var backendType: BackendOption - init(firebaseService: FirebaseAI, sample: Sample? = nil) { + init(backendType: BackendOption, sample: Sample? = nil) { self.sample = sample + self.backendType = backendType + + let firebaseService = backendType == .googleAI + ? FirebaseAI.firebaseAI(backend: .googleAI()) + : FirebaseAI.firebaseAI(backend: .vertexAI()) - // create a generative model with sample data model = firebaseService.generativeModel( - modelName: "gemini-2.0-flash-001", - tools: sample?.tools, + modelName: sample?.modelName ?? "gemini-2.5-flash", + generationConfig: sample?.generationConfig, systemInstruction: sample?.systemInstruction ) if let chatHistory = sample?.chatHistory, !chatHistory.isEmpty { - // Initialize with sample chat history if it's available messages = ChatMessage.from(chatHistory) chat = model.startChat(history: chatHistory) } else { @@ -112,13 +115,14 @@ class ChatViewModel: ObservableObject { .content = (messages[messages.count - 1].content ?? "") + text } - if let candidate = chunk.candidates.first { - if let groundingMetadata = candidate.groundingMetadata { - self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata + if let inlineDataPart = chunk.inlineDataParts.first { + if let uiImage = UIImage(data: inlineDataPart.data) { + messages[messages.count - 1].image = uiImage + } else { + print("Failed to convert inline data to UIImage") } } } - } catch { self.error = error print(error.localizedDescription) @@ -156,11 +160,13 @@ class ChatViewModel: ObservableObject { // replace pending message with backend response messages[messages.count - 1].content = responseText messages[messages.count - 1].pending = false + } - if let candidate = response?.candidates.first { - if let groundingMetadata = candidate.groundingMetadata { - self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata - } + if let inlineDataPart = response?.inlineDataParts.first { + if let uiImage = UIImage(data: inlineDataPart.data) { + messages[messages.count - 1].image = uiImage + } else { + print("Failed to convert inline data to UIImage") } } } catch { diff --git a/firebaseai/ChatExample/Views/MessageView.swift b/firebaseai/ChatExample/Views/MessageView.swift index 2242a02cd..5eb3ac6b0 100644 --- a/firebaseai/ChatExample/Views/MessageView.swift +++ b/firebaseai/ChatExample/Views/MessageView.swift @@ -53,16 +53,28 @@ struct MessageContentView: View { } .labelStyle(.iconOnly) } - } + } else { + VStack(alignment: .leading, spacing: 8) { + if message.participant == .user && !message.attachments.isEmpty { + AttachmentPreviewScrollView(attachments: message.attachments) + } - // Grounded Response - else if let groundingMetadata = message.groundingMetadata { - GroundedResponseView(message: message, groundingMetadata: groundingMetadata) - } + if let image = message.image { + Image(uiImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxWidth: 300, maxHeight: 300) + .clipShape(RoundedRectangle(cornerRadius: 8)) + } - // Non-grounded response - else { - ResponseTextView(message: message) + // Grounded Response + if let groundingMetadata = message.groundingMetadata { + GroundedResponseView(message: message, groundingMetadata: groundingMetadata) + } else { + // Non-grounded response + ResponseTextView(message: message) + } + } } } } diff --git a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj index 2217a26aa..b2c42e3f9 100644 --- a/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj +++ b/firebaseai/FirebaseAIExample.xcodeproj/project.pbxproj @@ -7,25 +7,27 @@ objects = { /* Begin PBXBuildFile section */ + 7210F4A22E52317E002FE9F2 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 7210F4972E52317E002FE9F2 /* Preview Assets.xcassets */; }; + 7210F4A32E52317E002FE9F2 /* MultimodalViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F49B2E52317E002FE9F2 /* MultimodalViewModel.swift */; }; + 7210F4A42E52317E002FE9F2 /* AttachmentPreviewCard.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F49D2E52317E002FE9F2 /* AttachmentPreviewCard.swift */; }; + 7210F4A52E52317E002FE9F2 /* MultimodalAttachment.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F4952E52317E002FE9F2 /* MultimodalAttachment.swift */; }; + 7210F4A62E52317E002FE9F2 /* MultimodalScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F4992E52317E002FE9F2 /* MultimodalScreen.swift */; }; + 7210F4B12E525A64002FE9F2 /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 7210F4B02E525A64002FE9F2 /* ConversationKit */; }; + 7210F4BA2E526AA1002FE9F2 /* GroundingScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F4B92E526A9B002FE9F2 /* GroundingScreen.swift */; }; + 7210F4BC2E526AB2002FE9F2 /* GroundingViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7210F4BB2E526AAA002FE9F2 /* GroundingViewModel.swift */; }; + 7210F4C82E527A39002FE9F2 /* GoogleService-Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = 7210F4C72E527A39002FE9F2 /* GoogleService-Info.plist */; }; 726490D92E3F39E000A92700 /* Sample.swift in Sources */ = {isa = PBXBuildFile; fileRef = 726490D22E3F39D200A92700 /* Sample.swift */; }; 726490DA2E3F39E000A92701 /* UseCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 726490D32E3F39D200A92700 /* UseCase.swift */; }; - 726490DC2E3F39E000A92703 /* InputField.swift in Sources */ = {isa = PBXBuildFile; fileRef = 726490D72E3F39D900A92700 /* InputField.swift */; }; - 726490DD2E3F39E000A92704 /* MultimodalInputField.swift in Sources */ = {isa = PBXBuildFile; fileRef = 726490D82E3F39DC00A92700 /* MultimodalInputField.swift */; }; 72DA044F2E385DF3004FED7D /* ChatMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 72DA044E2E385DF3004FED7D /* ChatMessage.swift */; }; 72E040752E448731003D4135 /* WeatherService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 72E040742E44872C003D4135 /* WeatherService.swift */; }; - 869200B32B879C4F00482873 /* GoogleService-Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = 869200B22B879C4F00482873 /* GoogleService-Info.plist */; }; 86C1F4832BC726150026816F /* FunctionCallingScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = 86C1F47E2BC726150026816F /* FunctionCallingScreen.swift */; }; 86C1F4842BC726150026816F /* FunctionCallingViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 86C1F4802BC726150026816F /* FunctionCallingViewModel.swift */; }; 88263BF12B239C11008AB09B /* ErrorDetailsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 889873842B208563005B4896 /* ErrorDetailsView.swift */; }; - 884298E12E4B8110005F535F /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 884298E02E4B8110005F535F /* ConversationKit */; }; 8848C8332B0D04BC007B434F /* FirebaseAIExampleApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8848C8322B0D04BC007B434F /* FirebaseAIExampleApp.swift */; }; 8848C8352B0D04BC007B434F /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8848C8342B0D04BC007B434F /* ContentView.swift */; }; 8848C8372B0D04BD007B434F /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8848C8362B0D04BD007B434F /* Assets.xcassets */; }; 8848C83A2B0D04BD007B434F /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8848C8392B0D04BD007B434F /* Preview Assets.xcassets */; }; - 885D0CA12E4CB7CD00A217A0 /* ConversationKit in Frameworks */ = {isa = PBXBuildFile; productRef = 885D0CA02E4CB7CD00A217A0 /* ConversationKit */; }; 886F95D82B17BA420036F07A /* MarkdownUI in Frameworks */ = {isa = PBXBuildFile; productRef = 886F95D72B17BA420036F07A /* MarkdownUI */; }; - 886F95DB2B17BAEF0036F07A /* PhotoReasoningViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8802666F2B0FC39000CF7CB6 /* PhotoReasoningViewModel.swift */; }; - 886F95DC2B17BAEF0036F07A /* PhotoReasoningScreen.swift in Sources */ = {isa = PBXBuildFile; fileRef = 880266752B0FC39000CF7CB6 /* PhotoReasoningScreen.swift */; }; 886F95DD2B17D5010036F07A /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88E10F5A2B11133E00C08E95 /* MessageView.swift */; }; 886F95DF2B17D5010036F07A /* BouncingDots.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88E10F5C2B11135000C08E95 /* BouncingDots.swift */; }; 886F95E02B17D5010036F07A /* ChatViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88E10F562B1112F600C08E95 /* ChatViewModel.swift */; }; @@ -40,26 +42,26 @@ /* End PBXBuildFile section */ /* Begin PBXFileReference section */ + 7210F4952E52317E002FE9F2 /* MultimodalAttachment.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultimodalAttachment.swift; sourceTree = ""; }; + 7210F4972E52317E002FE9F2 /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; + 7210F4992E52317E002FE9F2 /* MultimodalScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultimodalScreen.swift; sourceTree = ""; }; + 7210F49B2E52317E002FE9F2 /* MultimodalViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultimodalViewModel.swift; sourceTree = ""; }; + 7210F49D2E52317E002FE9F2 /* AttachmentPreviewCard.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AttachmentPreviewCard.swift; sourceTree = ""; }; + 7210F4B92E526A9B002FE9F2 /* GroundingScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GroundingScreen.swift; sourceTree = ""; }; + 7210F4BB2E526AAA002FE9F2 /* GroundingViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GroundingViewModel.swift; sourceTree = ""; }; + 7210F4C72E527A39002FE9F2 /* GoogleService-Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = "GoogleService-Info.plist"; sourceTree = ""; }; 726490D22E3F39D200A92700 /* Sample.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Sample.swift; sourceTree = ""; }; 726490D32E3F39D200A92700 /* UseCase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UseCase.swift; sourceTree = ""; }; - 726490D72E3F39D900A92700 /* InputField.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InputField.swift; sourceTree = ""; }; - 726490D82E3F39DC00A92700 /* MultimodalInputField.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MultimodalInputField.swift; sourceTree = ""; }; 72DA044E2E385DF3004FED7D /* ChatMessage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatMessage.swift; sourceTree = ""; }; 72E040742E44872C003D4135 /* WeatherService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WeatherService.swift; sourceTree = ""; }; - 869200B22B879C4F00482873 /* GoogleService-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "GoogleService-Info.plist"; sourceTree = ""; }; 86C1F47E2BC726150026816F /* FunctionCallingScreen.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FunctionCallingScreen.swift; sourceTree = ""; }; 86C1F4802BC726150026816F /* FunctionCallingViewModel.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FunctionCallingViewModel.swift; sourceTree = ""; }; - 8802666F2B0FC39000CF7CB6 /* PhotoReasoningViewModel.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PhotoReasoningViewModel.swift; sourceTree = ""; }; - 880266752B0FC39000CF7CB6 /* PhotoReasoningScreen.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PhotoReasoningScreen.swift; sourceTree = ""; }; 8848C82F2B0D04BC007B434F /* FirebaseAIExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = FirebaseAIExample.app; sourceTree = BUILT_PRODUCTS_DIR; }; 8848C8322B0D04BC007B434F /* FirebaseAIExampleApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FirebaseAIExampleApp.swift; sourceTree = ""; }; 8848C8342B0D04BC007B434F /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; 8848C8362B0D04BD007B434F /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 8848C8392B0D04BD007B434F /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; - 8848C85C2B0D056D007B434F /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; - 8848C85F2B0D056D007B434F /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; 889873842B208563005B4896 /* ErrorDetailsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ErrorDetailsView.swift; sourceTree = ""; }; - 88E10F482B110D5400C08E95 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 88E10F4B2B110D5400C08E95 /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; 88E10F542B1112CA00C08E95 /* ChatScreen.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatScreen.swift; sourceTree = ""; }; 88E10F562B1112F600C08E95 /* ChatViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatViewModel.swift; sourceTree = ""; }; @@ -78,9 +80,8 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - 884298E12E4B8110005F535F /* ConversationKit in Frameworks */, + 7210F4B12E525A64002FE9F2 /* ConversationKit in Frameworks */, DE26D95F2DBB3E9F007E6668 /* FirebaseAI in Frameworks */, - 885D0CA12E4CB7CD00A217A0 /* ConversationKit in Frameworks */, 886F95D82B17BA420036F07A /* MarkdownUI in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -88,6 +89,93 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 7210F4962E52317E002FE9F2 /* Models */ = { + isa = PBXGroup; + children = ( + 7210F4952E52317E002FE9F2 /* MultimodalAttachment.swift */, + ); + path = Models; + sourceTree = ""; + }; + 7210F4982E52317E002FE9F2 /* Preview Content */ = { + isa = PBXGroup; + children = ( + 7210F4972E52317E002FE9F2 /* Preview Assets.xcassets */, + ); + path = "Preview Content"; + sourceTree = ""; + }; + 7210F49A2E52317E002FE9F2 /* Screens */ = { + isa = PBXGroup; + children = ( + 7210F4992E52317E002FE9F2 /* MultimodalScreen.swift */, + ); + path = Screens; + sourceTree = ""; + }; + 7210F49C2E52317E002FE9F2 /* ViewModels */ = { + isa = PBXGroup; + children = ( + 7210F49B2E52317E002FE9F2 /* MultimodalViewModel.swift */, + ); + path = ViewModels; + sourceTree = ""; + }; + 7210F49E2E52317E002FE9F2 /* Views */ = { + isa = PBXGroup; + children = ( + 7210F49D2E52317E002FE9F2 /* AttachmentPreviewCard.swift */, + ); + path = Views; + sourceTree = ""; + }; + 7210F4A02E52317E002FE9F2 /* MultimodalExample */ = { + isa = PBXGroup; + children = ( + 7210F4962E52317E002FE9F2 /* Models */, + 7210F4982E52317E002FE9F2 /* Preview Content */, + 7210F49A2E52317E002FE9F2 /* Screens */, + 7210F49C2E52317E002FE9F2 /* ViewModels */, + 7210F49E2E52317E002FE9F2 /* Views */, + ); + path = MultimodalExample; + sourceTree = ""; + }; + 7210F4B42E526A5B002FE9F2 /* GroundingExample */ = { + isa = PBXGroup; + children = ( + 7210F4B82E526A82002FE9F2 /* Screens */, + 7210F4B62E526A69002FE9F2 /* ViewModels */, + 7210F4B52E526A64002FE9F2 /* Views */, + ); + path = GroundingExample; + sourceTree = ""; + }; + 7210F4B52E526A64002FE9F2 /* Views */ = { + isa = PBXGroup; + children = ( + AEE793DC2E256D3900708F02 /* GoogleSearchSuggestionView.swift */, + AEE793DD2E256D3900708F02 /* GroundedResponseView.swift */, + ); + path = Views; + sourceTree = ""; + }; + 7210F4B62E526A69002FE9F2 /* ViewModels */ = { + isa = PBXGroup; + children = ( + 7210F4BB2E526AAA002FE9F2 /* GroundingViewModel.swift */, + ); + path = ViewModels; + sourceTree = ""; + }; + 7210F4B82E526A82002FE9F2 /* Screens */ = { + isa = PBXGroup; + children = ( + 7210F4B92E526A9B002FE9F2 /* GroundingScreen.swift */, + ); + path = Screens; + sourceTree = ""; + }; 726490D12E3F39C900A92700 /* UIComponents */ = { isa = PBXGroup; children = ( @@ -109,8 +197,6 @@ 726490D62E3F39D600A92700 /* Views */ = { isa = PBXGroup; children = ( - 726490D72E3F39D900A92700 /* InputField.swift */, - 726490D82E3F39DC00A92700 /* MultimodalInputField.swift */, ); path = Views; sourceTree = ""; @@ -157,22 +243,6 @@ path = FunctionCallingExample; sourceTree = ""; }; - 8802666E2B0FC39000CF7CB6 /* ViewModels */ = { - isa = PBXGroup; - children = ( - 8802666F2B0FC39000CF7CB6 /* PhotoReasoningViewModel.swift */, - ); - path = ViewModels; - sourceTree = ""; - }; - 880266742B0FC39000CF7CB6 /* Screens */ = { - isa = PBXGroup; - children = ( - 880266752B0FC39000CF7CB6 /* PhotoReasoningScreen.swift */, - ); - path = Screens; - sourceTree = ""; - }; 88209C222B0FBE1700F64795 /* Frameworks */ = { isa = PBXGroup; children = ( @@ -183,15 +253,16 @@ 8848C8262B0D04BC007B434F = { isa = PBXGroup; children = ( + 7210F4C72E527A39002FE9F2 /* GoogleService-Info.plist */, + 7210F4A02E52317E002FE9F2 /* MultimodalExample */, 726490D12E3F39C900A92700 /* UIComponents */, - DEFECAA82D7B4CCD00EF9621 /* ImagenScreen */, - 869200B22B879C4F00482873 /* GoogleService-Info.plist */, + DEFECAA82D7B4CCD00EF9621 /* ImagenExample */, 8848C8312B0D04BC007B434F /* FirebaseAIExample */, - 8848C8572B0D056C007B434F /* GenerativeAIMultimodalExample */, 88E10F432B110D5300C08E95 /* ChatExample */, 86C1F4822BC726150026816F /* FunctionCallingExample */, 8848C8302B0D04BC007B434F /* Products */, 88209C222B0FBE1700F64795 /* Frameworks */, + 7210F4B42E526A5B002FE9F2 /* GroundingExample */, ); sourceTree = ""; }; @@ -223,25 +294,6 @@ path = "Preview Content"; sourceTree = ""; }; - 8848C8572B0D056C007B434F /* GenerativeAIMultimodalExample */ = { - isa = PBXGroup; - children = ( - 8802666E2B0FC39000CF7CB6 /* ViewModels */, - 880266742B0FC39000CF7CB6 /* Screens */, - 8848C85C2B0D056D007B434F /* Assets.xcassets */, - 8848C85E2B0D056D007B434F /* Preview Content */, - ); - path = GenerativeAIMultimodalExample; - sourceTree = ""; - }; - 8848C85E2B0D056D007B434F /* Preview Content */ = { - isa = PBXGroup; - children = ( - 8848C85F2B0D056D007B434F /* Preview Assets.xcassets */, - ); - path = "Preview Content"; - sourceTree = ""; - }; 88E10F432B110D5300C08E95 /* ChatExample */ = { isa = PBXGroup; children = ( @@ -249,7 +301,6 @@ 88E10F502B11123600C08E95 /* ViewModels */, 88E10F512B11124100C08E95 /* Views */, 88E10F532B1112B900C08E95 /* Screens */, - 88E10F482B110D5400C08E95 /* Assets.xcassets */, 88E10F4A2B110D5400C08E95 /* Preview Content */, ); path = ChatExample; @@ -274,7 +325,6 @@ 88E10F512B11124100C08E95 /* Views */ = { isa = PBXGroup; children = ( - AEE793DE2E256D3900708F02 /* Grounding */, 88E10F5A2B11133E00C08E95 /* MessageView.swift */, 88E10F5C2B11135000C08E95 /* BouncingDots.swift */, 889873842B208563005B4896 /* ErrorDetailsView.swift */, @@ -299,22 +349,13 @@ path = Views; sourceTree = ""; }; - AEE793DE2E256D3900708F02 /* Grounding */ = { - isa = PBXGroup; - children = ( - AEE793DC2E256D3900708F02 /* GoogleSearchSuggestionView.swift */, - AEE793DD2E256D3900708F02 /* GroundedResponseView.swift */, - ); - path = Grounding; - sourceTree = ""; - }; - DEFECAA82D7B4CCD00EF9621 /* ImagenScreen */ = { + DEFECAA82D7B4CCD00EF9621 /* ImagenExample */ = { isa = PBXGroup; children = ( DEFECAA62D7B4CCD00EF9621 /* ImagenScreen.swift */, DEFECAA72D7B4CCD00EF9621 /* ImagenViewModel.swift */, ); - path = ImagenScreen; + path = ImagenExample; sourceTree = ""; }; /* End PBXGroup section */ @@ -336,8 +377,7 @@ packageProductDependencies = ( 886F95D72B17BA420036F07A /* MarkdownUI */, DE26D95E2DBB3E9F007E6668 /* FirebaseAI */, - 884298E02E4B8110005F535F /* ConversationKit */, - 885D0CA02E4CB7CD00A217A0 /* ConversationKit */, + 7210F4B02E525A64002FE9F2 /* ConversationKit */, ); productName = GenerativeAIExample; productReference = 8848C82F2B0D04BC007B434F /* FirebaseAIExample.app */; @@ -371,7 +411,7 @@ 88209C212B0FBDF700F64795 /* XCRemoteSwiftPackageReference "swift-markdown-ui" */, DEA09AC32B1FCE22001962D9 /* XCRemoteSwiftPackageReference "NetworkImage" */, DEFECAAB2D7BB49700EF9621 /* XCRemoteSwiftPackageReference "firebase-ios-sdk" */, - 885D0C9F2E4CB7CD00A217A0 /* XCRemoteSwiftPackageReference "ConversationKit" */, + 7210F4AF2E525A64002FE9F2 /* XCRemoteSwiftPackageReference "ConversationKit" */, ); productRefGroup = 8848C8302B0D04BC007B434F /* Products */; projectDirPath = ""; @@ -388,8 +428,9 @@ buildActionMask = 2147483647; files = ( 8848C83A2B0D04BD007B434F /* Preview Assets.xcassets in Resources */, + 7210F4C82E527A39002FE9F2 /* GoogleService-Info.plist in Resources */, 8848C8372B0D04BD007B434F /* Assets.xcassets in Resources */, - 869200B32B879C4F00482873 /* GoogleService-Info.plist in Resources */, + 7210F4A22E52317E002FE9F2 /* Preview Assets.xcassets in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -408,10 +449,13 @@ 8848C8332B0D04BC007B434F /* FirebaseAIExampleApp.swift in Sources */, 886F95E02B17D5010036F07A /* ChatViewModel.swift in Sources */, 886F95DD2B17D5010036F07A /* MessageView.swift in Sources */, - 886F95DC2B17BAEF0036F07A /* PhotoReasoningScreen.swift in Sources */, + 7210F4BC2E526AB2002FE9F2 /* GroundingViewModel.swift in Sources */, DEFECAA92D7B4CCD00EF9621 /* ImagenViewModel.swift in Sources */, + 7210F4A32E52317E002FE9F2 /* MultimodalViewModel.swift in Sources */, + 7210F4A42E52317E002FE9F2 /* AttachmentPreviewCard.swift in Sources */, + 7210F4A52E52317E002FE9F2 /* MultimodalAttachment.swift in Sources */, + 7210F4A62E52317E002FE9F2 /* MultimodalScreen.swift in Sources */, DEFECAAA2D7B4CCD00EF9621 /* ImagenScreen.swift in Sources */, - 886F95DB2B17BAEF0036F07A /* PhotoReasoningViewModel.swift in Sources */, 72E040752E448731003D4135 /* WeatherService.swift in Sources */, 886F95E12B17D5010036F07A /* ChatScreen.swift in Sources */, 72DA044F2E385DF3004FED7D /* ChatMessage.swift in Sources */, @@ -419,10 +463,9 @@ A5E8E3CA2C3B4F388A7A4A1A /* SampleCardView.swift in Sources */, AEE793DF2E256D3900708F02 /* GoogleSearchSuggestionView.swift in Sources */, AEE793E02E256D3900708F02 /* GroundedResponseView.swift in Sources */, + 7210F4BA2E526AA1002FE9F2 /* GroundingScreen.swift in Sources */, 726490D92E3F39E000A92700 /* Sample.swift in Sources */, 726490DA2E3F39E000A92701 /* UseCase.swift in Sources */, - 726490DC2E3F39E000A92703 /* InputField.swift in Sources */, - 726490DD2E3F39E000A92704 /* MultimodalInputField.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -632,20 +675,20 @@ /* End XCConfigurationList section */ /* Begin XCRemoteSwiftPackageReference section */ - 88209C212B0FBDF700F64795 /* XCRemoteSwiftPackageReference "swift-markdown-ui" */ = { + 7210F4AF2E525A64002FE9F2 /* XCRemoteSwiftPackageReference "ConversationKit" */ = { isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/gonzalezreal/swift-markdown-ui"; + repositoryURL = "https://github.com/YoungHypo/ConversationKit"; requirement = { - kind = revision; - revision = 55441810c0f678c78ed7e2ebd46dde89228e02fc; + branch = main; + kind = branch; }; }; - 885D0C9F2E4CB7CD00A217A0 /* XCRemoteSwiftPackageReference "ConversationKit" */ = { + 88209C212B0FBDF700F64795 /* XCRemoteSwiftPackageReference "swift-markdown-ui" */ = { isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/peterfriese/ConversationKit"; + repositoryURL = "https://github.com/gonzalezreal/swift-markdown-ui"; requirement = { - kind = upToNextMajorVersion; - minimumVersion = 0.0.2; + kind = revision; + revision = 55441810c0f678c78ed7e2ebd46dde89228e02fc; }; }; DEA09AC32B1FCE22001962D9 /* XCRemoteSwiftPackageReference "NetworkImage" */ = { @@ -667,13 +710,9 @@ /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ - 884298E02E4B8110005F535F /* ConversationKit */ = { - isa = XCSwiftPackageProductDependency; - productName = ConversationKit; - }; - 885D0CA02E4CB7CD00A217A0 /* ConversationKit */ = { + 7210F4B02E525A64002FE9F2 /* ConversationKit */ = { isa = XCSwiftPackageProductDependency; - package = 885D0C9F2E4CB7CD00A217A0 /* XCRemoteSwiftPackageReference "ConversationKit" */; + package = 7210F4AF2E525A64002FE9F2 /* XCRemoteSwiftPackageReference "ConversationKit" */; productName = ConversationKit; }; 886F95D72B17BA420036F07A /* MarkdownUI */ = { diff --git a/firebaseai/FirebaseAIExample/ContentView.swift b/firebaseai/FirebaseAIExample/ContentView.swift index 5af66fad7..d3bcaa5ab 100644 --- a/firebaseai/FirebaseAIExample/ContentView.swift +++ b/firebaseai/FirebaseAIExample/ContentView.swift @@ -18,25 +18,20 @@ import FirebaseAI enum BackendOption: String, CaseIterable, Identifiable { case googleAI = "Gemini Developer API" case vertexAI = "Vertex AI Gemini API" - var id: String { rawValue } - var backendValue: FirebaseAI { - switch self { - case .googleAI: - return FirebaseAI.firebaseAI(backend: .googleAI()) - case .vertexAI: - return FirebaseAI.firebaseAI(backend: .vertexAI()) - } - } + var id: String { rawValue } } struct ContentView: View { @State private var selectedBackend: BackendOption = .googleAI - @State private var firebaseService: FirebaseAI = FirebaseAI.firebaseAI(backend: .googleAI()) - @State private var selectedUseCase: UseCase = .text + @State private var selectedUseCase: UseCase = .all var filteredSamples: [Sample] { - Sample.samples.filter { $0.useCases.contains(selectedUseCase) } + if selectedUseCase == .all { + return Sample.samples + } else { + return Sample.samples.filter { $0.useCases.contains(selectedUseCase) } + } } let columns = [ @@ -102,9 +97,6 @@ struct ContentView: View { } .background(Color(.systemGroupedBackground)) .navigationTitle("Firebase AI Logic") - .onChange(of: selectedBackend) { newBackend in - firebaseService = newBackend.backendValue - } } } @@ -112,13 +104,15 @@ struct ContentView: View { private func destinationView(for sample: Sample) -> some View { switch sample.navRoute { case "ChatScreen": - ChatScreen(firebaseService: firebaseService, sample: sample) + ChatScreen(backendType: selectedBackend, sample: sample) case "ImagenScreen": - ImagenScreen(firebaseService: firebaseService, sample: sample) - case "PhotoReasoningScreen": - PhotoReasoningScreen(firebaseService: firebaseService) + ImagenScreen(backendType: selectedBackend, sample: sample) + case "MultimodalScreen": + MultimodalScreen(backendType: selectedBackend, sample: sample) case "FunctionCallingScreen": - FunctionCallingScreen(firebaseService: firebaseService, sample: sample) + FunctionCallingScreen(backendType: selectedBackend, sample: sample) + case "GroundingScreen": + GroundingScreen(backendType: selectedBackend, sample: sample) default: EmptyView() } diff --git a/firebaseai/FirebaseAIExample/Views/SampleCardView.swift b/firebaseai/FirebaseAIExample/Views/SampleCardView.swift index af4c4680b..58034475a 100644 --- a/firebaseai/FirebaseAIExample/Views/SampleCardView.swift +++ b/firebaseai/FirebaseAIExample/Views/SampleCardView.swift @@ -39,6 +39,7 @@ struct SampleCardView: View { private func systemName(for useCase: UseCase) -> String { switch useCase { + case .all: "square.grid.2x2.fill" case .text: "text.bubble.fill" case .image: "photo.fill" case .video: "video.fill" @@ -50,6 +51,7 @@ struct SampleCardView: View { private func color(for useCase: UseCase) -> Color { switch useCase { + case .all:.primary case .text:.blue case .image:.purple case .video:.red diff --git a/firebaseai/FunctionCallingExample/Screens/FunctionCallingScreen.swift b/firebaseai/FunctionCallingExample/Screens/FunctionCallingScreen.swift index 732900985..7e814be3e 100644 --- a/firebaseai/FunctionCallingExample/Screens/FunctionCallingScreen.swift +++ b/firebaseai/FunctionCallingExample/Screens/FunctionCallingScreen.swift @@ -17,13 +17,13 @@ import SwiftUI import ConversationKit struct FunctionCallingScreen: View { - let firebaseService: FirebaseAI + let backendType: BackendOption @StateObject var viewModel: FunctionCallingViewModel - init(firebaseService: FirebaseAI, sample: Sample? = nil) { - self.firebaseService = firebaseService + init(backendType: BackendOption, sample: Sample? = nil) { + self.backendType = backendType _viewModel = - StateObject(wrappedValue: FunctionCallingViewModel(firebaseService: firebaseService, + StateObject(wrappedValue: FunctionCallingViewModel(backendType: backendType, sample: sample)) } @@ -35,9 +35,7 @@ struct FunctionCallingScreen: View { } .disableAttachments() .onSendMessage { message in - Task { - await viewModel.sendMessage(message.content ?? "", streaming: true) - } + await viewModel.sendMessage(message.content ?? "", streaming: true) } .onError { error in viewModel.presentErrorDetails = true @@ -65,5 +63,5 @@ struct FunctionCallingScreen: View { } #Preview { - FunctionCallingScreen(firebaseService: FirebaseAI.firebaseAI()) + FunctionCallingScreen(backendType: .googleAI) } diff --git a/firebaseai/FunctionCallingExample/ViewModels/FunctionCallingViewModel.swift b/firebaseai/FunctionCallingExample/ViewModels/FunctionCallingViewModel.swift index e288ef585..c4ba58aa5 100644 --- a/firebaseai/FunctionCallingExample/ViewModels/FunctionCallingViewModel.swift +++ b/firebaseai/FunctionCallingExample/ViewModels/FunctionCallingViewModel.swift @@ -40,13 +40,19 @@ class FunctionCallingViewModel: ObservableObject { private var chatTask: Task? private var sample: Sample? + private var backendType: BackendOption - init(firebaseService: FirebaseAI, sample: Sample? = nil) { + init(backendType: BackendOption, sample: Sample? = nil) { self.sample = sample + self.backendType = backendType + + let firebaseService = backendType == .googleAI + ? FirebaseAI.firebaseAI(backend: .googleAI()) + : FirebaseAI.firebaseAI(backend: .vertexAI()) // create a generative model with sample data model = firebaseService.generativeModel( - modelName: "gemini-2.0-flash-001", + modelName: sample?.modelName ?? "gemini-2.0-flash-001", tools: sample?.tools, systemInstruction: sample?.systemInstruction ) @@ -99,18 +105,24 @@ class FunctionCallingViewModel: ObservableObject { do { let responseStream = try chat.sendMessageStream(text) + var functionCalls = [FunctionCallPart]() + for try await chunk in responseStream { if !chunk.functionCalls.isEmpty { - try await handleFunctionCallsStreaming(chunk) - } else { - if let text = chunk.text { - messages[messages.count - 1] - .content = (messages[messages.count - 1].content ?? "") + text - messages[messages.count - 1].pending = false - } + functionCalls.append(contentsOf: chunk.functionCalls) + } + if let text = chunk.text { + messages[messages.count - 1] + .content = (messages[messages.count - 1].content ?? "") + text + messages[messages.count - 1].pending = false } } + // On functionCalls, never keep reading the old stream or call the second API inside the first for-loop. + // Start a NEW stream only after the function response turn is sent. + if !functionCalls.isEmpty { + try await handleFunctionCallsStreaming(functionCalls) + } } catch { self.error = error print(error.localizedDescription) @@ -164,10 +176,10 @@ class FunctionCallingViewModel: ObservableObject { } } - private func handleFunctionCallsStreaming(_ response: GenerateContentResponse) async throws { + private func handleFunctionCallsStreaming(_ functionCalls: [FunctionCallPart]) async throws { var functionResponses = [FunctionResponsePart]() - for functionCall in response.functionCalls { + for functionCall in functionCalls { switch functionCall.name { case "fetchWeather": guard case let .string(city) = functionCall.args["city"], @@ -194,7 +206,7 @@ class FunctionCallingViewModel: ObservableObject { } if !functionResponses.isEmpty { - let finalResponse = try await chat + let finalResponse = try chat .sendMessageStream([ModelContent(role: "function", parts: functionResponses)]) for try await chunk in finalResponse { diff --git a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AccentColor.colorset/Contents.json b/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AccentColor.colorset/Contents.json deleted file mode 100644 index eb8789700..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AccentColor.colorset/Contents.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "colors" : [ - { - "idiom" : "universal" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AppIcon.appiconset/Contents.json b/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AppIcon.appiconset/Contents.json deleted file mode 100644 index 13613e3ee..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/AppIcon.appiconset/Contents.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "images" : [ - { - "idiom" : "universal", - "platform" : "ios", - "size" : "1024x1024" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/Contents.json b/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/Contents.json deleted file mode 100644 index 73c00596a..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Assets.xcassets/Contents.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json b/firebaseai/GenerativeAIMultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json deleted file mode 100644 index 73c00596a..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/Screens/PhotoReasoningScreen.swift b/firebaseai/GenerativeAIMultimodalExample/Screens/PhotoReasoningScreen.swift deleted file mode 100644 index b1a992eae..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/Screens/PhotoReasoningScreen.swift +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import MarkdownUI -import PhotosUI -import SwiftUI -import FirebaseAI - -struct PhotoReasoningScreen: View { - let firebaseService: FirebaseAI - @StateObject var viewModel: PhotoReasoningViewModel - - init(firebaseService: FirebaseAI) { - self.firebaseService = firebaseService - _viewModel = - StateObject(wrappedValue: PhotoReasoningViewModel(firebaseService: firebaseService)) - } - - enum FocusedField: Hashable { - case message - } - - @FocusState - var focusedField: FocusedField? - - var body: some View { - VStack { - MultimodalInputField(text: $viewModel.userInput, selection: $viewModel.selectedItems) - .focused($focusedField, equals: .message) - .onSubmit { - onSendTapped() - } - - ScrollViewReader { scrollViewProxy in - List { - if let outputText = viewModel.outputText { - HStack(alignment: .top) { - if viewModel.inProgress { - ProgressView() - } else { - Image(systemName: "cloud.circle.fill") - .font(.title2) - } - - Markdown("\(outputText)") - } - .listRowSeparator(.hidden) - } - } - .listStyle(.plain) - } - } - .onTapGesture { - focusedField = nil - } - .navigationTitle("Multimodal example") - .onAppear { - focusedField = .message - } - } - - // MARK: - Actions - - private func onSendTapped() { - focusedField = nil - - Task { - await viewModel.reason() - } - } -} - -#Preview { - NavigationStack { - PhotoReasoningScreen(firebaseService: FirebaseAI.firebaseAI()) - } -} diff --git a/firebaseai/GenerativeAIMultimodalExample/ViewModels/PhotoReasoningViewModel.swift b/firebaseai/GenerativeAIMultimodalExample/ViewModels/PhotoReasoningViewModel.swift deleted file mode 100644 index 24a2e96e2..000000000 --- a/firebaseai/GenerativeAIMultimodalExample/ViewModels/PhotoReasoningViewModel.swift +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import FirebaseAI -import Foundation -import OSLog -import PhotosUI -import SwiftUI - -@MainActor -class PhotoReasoningViewModel: ObservableObject { - // Maximum value for the larger of the two image dimensions (height and width) in pixels. This is - // being used to reduce the image size in bytes. - private static let largestImageDimension = 768.0 - - private var logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "generative-ai") - - @Published - var userInput: String = "" - - @Published - var selectedItems = [PhotosPickerItem]() - - @Published - var outputText: String? = nil - - @Published - var errorMessage: String? - - @Published - var inProgress = false - - private var model: GenerativeModel? - - init(firebaseService: FirebaseAI) { - model = firebaseService.generativeModel(modelName: "gemini-2.0-flash-001") - } - - func reason() async { - defer { - inProgress = false - } - guard let model else { - return - } - - do { - inProgress = true - errorMessage = nil - outputText = "" - - let prompt = "Look at the image(s), and then answer the following question: \(userInput)" - - var images = [any PartsRepresentable]() - for item in selectedItems { - if let data = try? await item.loadTransferable(type: Data.self) { - guard let image = UIImage(data: data) else { - logger.error("Failed to parse data as an image, skipping.") - continue - } - if image.size.fits(largestDimension: PhotoReasoningViewModel.largestImageDimension) { - images.append(image) - } else { - guard let resizedImage = image - .preparingThumbnail(of: image.size - .aspectFit(largestDimension: PhotoReasoningViewModel.largestImageDimension)) else { - logger.error("Failed to resize image: \(image)") - continue - } - - images.append(resizedImage) - } - } - } - - let outputContentStream = try model.generateContentStream(prompt, images) - - // stream response - for try await outputContent in outputContentStream { - guard let line = outputContent.text else { - return - } - - outputText = (outputText ?? "") + line - } - } catch { - logger.error("\(error.localizedDescription)") - errorMessage = error.localizedDescription - } - } -} - -private extension CGSize { - func fits(largestDimension length: CGFloat) -> Bool { - return width <= length && height <= length - } - - func aspectFit(largestDimension length: CGFloat) -> CGSize { - let aspectRatio = width / height - if width > height { - let width = min(self.width, length) - return CGSize(width: width, height: round(width / aspectRatio)) - } else { - let height = min(self.height, length) - return CGSize(width: round(height * aspectRatio), height: height) - } - } -} diff --git a/firebaseai/GenerativeAITextExample/Assets.xcassets/AccentColor.colorset/Contents.json b/firebaseai/GenerativeAITextExample/Assets.xcassets/AccentColor.colorset/Contents.json deleted file mode 100644 index eb8789700..000000000 --- a/firebaseai/GenerativeAITextExample/Assets.xcassets/AccentColor.colorset/Contents.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "colors" : [ - { - "idiom" : "universal" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAITextExample/Assets.xcassets/AppIcon.appiconset/Contents.json b/firebaseai/GenerativeAITextExample/Assets.xcassets/AppIcon.appiconset/Contents.json deleted file mode 100644 index 13613e3ee..000000000 --- a/firebaseai/GenerativeAITextExample/Assets.xcassets/AppIcon.appiconset/Contents.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "images" : [ - { - "idiom" : "universal", - "platform" : "ios", - "size" : "1024x1024" - } - ], - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAITextExample/Assets.xcassets/Contents.json b/firebaseai/GenerativeAITextExample/Assets.xcassets/Contents.json deleted file mode 100644 index 73c00596a..000000000 --- a/firebaseai/GenerativeAITextExample/Assets.xcassets/Contents.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAITextExample/Preview Content/Preview Assets.xcassets/Contents.json b/firebaseai/GenerativeAITextExample/Preview Content/Preview Assets.xcassets/Contents.json deleted file mode 100644 index 73c00596a..000000000 --- a/firebaseai/GenerativeAITextExample/Preview Content/Preview Assets.xcassets/Contents.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "info" : { - "author" : "xcode", - "version" : 1 - } -} diff --git a/firebaseai/GenerativeAITextExample/Screens/GenerateContentScreen.swift b/firebaseai/GenerativeAITextExample/Screens/GenerateContentScreen.swift deleted file mode 100644 index 2d1648d5a..000000000 --- a/firebaseai/GenerativeAITextExample/Screens/GenerateContentScreen.swift +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import MarkdownUI -import SwiftUI -import FirebaseAI -import GenerativeAIUIComponents - -struct GenerateContentScreen: View { - let firebaseService: FirebaseAI - @StateObject var viewModel: GenerateContentViewModel - @State var userInput = "" - - init(firebaseService: FirebaseAI) { - self.firebaseService = firebaseService - _viewModel = - StateObject(wrappedValue: GenerateContentViewModel(firebaseService: firebaseService)) - } - - enum FocusedField: Hashable { - case message - } - - @FocusState - var focusedField: FocusedField? - - var body: some View { - VStack { - VStack(alignment: .leading) { - Text("Enter some text, then tap on _Go_ to run generateContent on it.") - .padding(.horizontal, 6) - InputField("Enter generate content input", text: $userInput) { - Text("Go") - } - .focused($focusedField, equals: .message) - .onSubmit { onGenerateContentTapped() } - } - .padding(.horizontal, 16) - - List { - HStack(alignment: .top) { - if viewModel.inProgress { - ProgressView() - } else { - Image(systemName: "cloud.circle.fill") - .font(.title2) - } - - Markdown("\(viewModel.outputText)") - } - .listRowSeparator(.hidden) - } - .listStyle(.plain) - } - .onTapGesture { - focusedField = nil - } - .navigationTitle("Text example") - } - - private func onGenerateContentTapped() { - focusedField = nil - - Task { - await viewModel.generateContent(inputText: userInput) - } - } -} - -#Preview { - NavigationStack { - GenerateContentScreen(firebaseService: FirebaseAI.firebaseAI()) - } -} diff --git a/firebaseai/GenerativeAITextExample/ViewModels/GenerateContentViewModel.swift b/firebaseai/GenerativeAITextExample/ViewModels/GenerateContentViewModel.swift deleted file mode 100644 index a9272ef5b..000000000 --- a/firebaseai/GenerativeAITextExample/ViewModels/GenerateContentViewModel.swift +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import FirebaseAI -import Foundation -import OSLog - -@MainActor -class GenerateContentViewModel: ObservableObject { - private var logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "generative-ai") - - @Published - var outputText = "" - - @Published - var errorMessage: String? - - @Published - var inProgress = false - - private var model: GenerativeModel? - - init(firebaseService: FirebaseAI) { - model = firebaseService.generativeModel(modelName: "gemini-2.0-flash-001") - } - - func generateContent(inputText: String) async { - defer { - inProgress = false - } - guard let model else { - return - } - - do { - inProgress = true - errorMessage = nil - outputText = "" - - let outputContentStream = try model.generateContentStream(inputText) - - // stream response - for try await outputContent in outputContentStream { - guard let line = outputContent.text else { - return - } - - outputText = outputText + line - } - } catch { - logger.error("\(error.localizedDescription)") - errorMessage = error.localizedDescription - } - } -} diff --git a/firebaseai/GroundingExample/Screens/GroundingScreen.swift b/firebaseai/GroundingExample/Screens/GroundingScreen.swift new file mode 100644 index 000000000..33c63a23b --- /dev/null +++ b/firebaseai/GroundingExample/Screens/GroundingScreen.swift @@ -0,0 +1,67 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseAI +import SwiftUI +import ConversationKit + +struct GroundingScreen: View { + let backendType: BackendOption + @StateObject var viewModel: GroundingViewModel + + init(backendType: BackendOption, sample: Sample? = nil) { + self.backendType = backendType + _viewModel = + StateObject(wrappedValue: GroundingViewModel(backendType: backendType, + sample: sample)) + } + + var body: some View { + NavigationStack { + ConversationView(messages: $viewModel.messages, + userPrompt: viewModel.initialPrompt) { message in + MessageView(message: message) + } + .disableAttachments() + .onSendMessage { message in + await viewModel.sendMessage(message.content ?? "", streaming: true) + } + .onError { error in + viewModel.presentErrorDetails = true + } + .sheet(isPresented: $viewModel.presentErrorDetails) { + if let error = viewModel.error { + ErrorDetailsView(error: error) + } + } + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button(action: newChat) { + Image(systemName: "square.and.pencil") + } + } + } + .navigationTitle(viewModel.title) + .navigationBarTitleDisplayMode(.inline) + } + } + + private func newChat() { + viewModel.startNewChat() + } +} + +#Preview { + GroundingScreen(backendType: .googleAI) +} diff --git a/firebaseai/GroundingExample/ViewModels/GroundingViewModel.swift b/firebaseai/GroundingExample/ViewModels/GroundingViewModel.swift new file mode 100644 index 000000000..2682f3ca9 --- /dev/null +++ b/firebaseai/GroundingExample/ViewModels/GroundingViewModel.swift @@ -0,0 +1,177 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseAI +import Foundation +import UIKit + +@MainActor +class GroundingViewModel: ObservableObject { + /// This array holds both the user's and the system's chat messages + @Published var messages = [ChatMessage]() + + /// Indicates we're waiting for the model to finish + @Published var busy = false + + @Published var error: Error? + var hasError: Bool { + return error != nil + } + + @Published var presentErrorDetails: Bool = false + + @Published var initialPrompt: String = "" + @Published var title: String = "" + + private var model: GenerativeModel + private var chat: Chat + + private var chatTask: Task? + + private var sample: Sample? + + private var backendType: BackendOption + + init(backendType: BackendOption, sample: Sample? = nil) { + self.sample = sample + self.backendType = backendType + + let firebaseService = backendType == .googleAI + ? FirebaseAI.firebaseAI(backend: .googleAI()) + : FirebaseAI.firebaseAI(backend: .vertexAI()) + + model = firebaseService.generativeModel( + modelName: sample?.modelName ?? "gemini-2.5-flash", + tools: sample?.tools, + systemInstruction: sample?.systemInstruction + ) + + chat = model.startChat() + + initialPrompt = sample?.initialPrompt ?? "" + title = sample?.title ?? "" + } + + func sendMessage(_ text: String, streaming: Bool = true) async { + error = nil + if streaming { + await internalSendMessageStreaming(text) + } else { + await internalSendMessage(text) + } + } + + func startNewChat() { + stop() + error = nil + chat = model.startChat() + messages.removeAll() + initialPrompt = "" + } + + func stop() { + chatTask?.cancel() + error = nil + } + + private func internalSendMessageStreaming(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(content: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .other) + messages.append(systemMessage) + + do { + let responseStream = try chat.sendMessageStream(text) + for try await chunk in responseStream { + messages[messages.count - 1].pending = false + if let text = chunk.text { + messages[messages.count - 1] + .content = (messages[messages.count - 1].content ?? "") + text + } + + if let candidate = chunk.candidates.first { + if let groundingMetadata = candidate.groundingMetadata { + self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata + } + } + } + + } catch { + self.error = error + print(error.localizedDescription) + let errorMessage = ChatMessage(content: "An error occurred. Please try again.", + participant: .other, + error: error, + pending: false) + messages[messages.count - 1] = errorMessage + } + } + } + + private func internalSendMessage(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(content: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .other) + messages.append(systemMessage) + + do { + var response: GenerateContentResponse? + response = try await chat.sendMessage(text) + + if let responseText = response?.text { + // replace pending message with backend response + messages[messages.count - 1].content = responseText + messages[messages.count - 1].pending = false + + if let candidate = response?.candidates.first { + if let groundingMetadata = candidate.groundingMetadata { + self.messages[self.messages.count - 1].groundingMetadata = groundingMetadata + } + } + } + + } catch { + self.error = error + print(error.localizedDescription) + let errorMessage = ChatMessage(content: "An error occurred. Please try again.", + participant: .other, + error: error, + pending: false) + messages[messages.count - 1] = errorMessage + } + } + } +} diff --git a/firebaseai/ChatExample/Views/Grounding/GoogleSearchSuggestionView.swift b/firebaseai/GroundingExample/Views/GoogleSearchSuggestionView.swift similarity index 100% rename from firebaseai/ChatExample/Views/Grounding/GoogleSearchSuggestionView.swift rename to firebaseai/GroundingExample/Views/GoogleSearchSuggestionView.swift diff --git a/firebaseai/ChatExample/Views/Grounding/GroundedResponseView.swift b/firebaseai/GroundingExample/Views/GroundedResponseView.swift similarity index 100% rename from firebaseai/ChatExample/Views/Grounding/GroundedResponseView.swift rename to firebaseai/GroundingExample/Views/GroundedResponseView.swift diff --git a/firebaseai/ImagenScreen/ImagenScreen.swift b/firebaseai/ImagenExample/ImagenScreen.swift similarity index 91% rename from firebaseai/ImagenScreen/ImagenScreen.swift rename to firebaseai/ImagenExample/ImagenScreen.swift index 4d546dc94..9c658bc52 100644 --- a/firebaseai/ImagenScreen/ImagenScreen.swift +++ b/firebaseai/ImagenExample/ImagenScreen.swift @@ -17,16 +17,16 @@ import FirebaseAI import ConversationKit struct ImagenScreen: View { - let firebaseService: FirebaseAI + let backendType: BackendOption @StateObject var viewModel: ImagenViewModel @State private var userPrompt = "" - init(firebaseService: FirebaseAI, sample: Sample? = nil) { - self.firebaseService = firebaseService + init(backendType: BackendOption, sample: Sample? = nil) { + self.backendType = backendType _viewModel = - StateObject(wrappedValue: ImagenViewModel(firebaseService: firebaseService, + StateObject(wrappedValue: ImagenViewModel(backendType: backendType, sample: sample)) } @@ -47,7 +47,7 @@ struct ImagenScreen: View { .disableAttachments() .onSubmitAction { sendOrStop() } - if let error = viewModel.error { + if viewModel.error != nil { HStack { Text("An error occurred.") Button("More information", systemImage: "info.circle") { @@ -137,5 +137,5 @@ struct ProgressOverlay: View { } #Preview { - ImagenScreen(firebaseService: FirebaseAI.firebaseAI()) + ImagenScreen(backendType: .googleAI) } diff --git a/firebaseai/ImagenScreen/ImagenViewModel.swift b/firebaseai/ImagenExample/ImagenViewModel.swift similarity index 83% rename from firebaseai/ImagenScreen/ImagenViewModel.swift rename to firebaseai/ImagenExample/ImagenViewModel.swift index 3b66dbfa8..1d79d8406 100644 --- a/firebaseai/ImagenScreen/ImagenViewModel.swift +++ b/firebaseai/ImagenExample/ImagenViewModel.swift @@ -40,13 +40,19 @@ class ImagenViewModel: ObservableObject { var inProgress = false private let model: ImagenModel + private var backendType: BackendOption private var generateImagesTask: Task? private var sample: Sample? - init(firebaseService: FirebaseAI, sample: Sample? = nil) { + init(backendType: BackendOption, sample: Sample? = nil) { self.sample = sample + self.backendType = backendType + + let firebaseService = backendType == .googleAI + ? FirebaseAI.firebaseAI(backend: .googleAI()) + : FirebaseAI.firebaseAI(backend: .vertexAI()) let modelName = "imagen-3.0-generate-002" let safetySettings = ImagenSafetySettings( @@ -75,16 +81,16 @@ class ImagenViewModel: ObservableObject { } do { - // 4. Call generateImages with the text prompt + // 1. Call generateImages with the text prompt let response = try await model.generateImages(prompt: prompt) - // 5. Print the reason images were filtered out, if any. + // 2. Print the reason images were filtered out, if any. if let filteredReason = response.filteredReason { print("Image(s) Blocked: \(filteredReason)") } if !Task.isCancelled { - // 6. Convert the image data to UIImage for display in the UI + // 3. Convert the image data to UIImage for display in the UI images = response.images.compactMap { UIImage(data: $0.data) } } } catch { diff --git a/firebaseai/MultimodalExample/Models/MultimodalAttachment.swift b/firebaseai/MultimodalExample/Models/MultimodalAttachment.swift new file mode 100644 index 000000000..f4f0bc86f --- /dev/null +++ b/firebaseai/MultimodalExample/Models/MultimodalAttachment.swift @@ -0,0 +1,267 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import SwiftUI +import PhotosUI +import FirebaseAI + +public enum MultimodalAttachmentError: LocalizedError { + case unsupportedFileType(extension: String) + case noDataAvailable + case loadingFailed(Error) + case mimeTypeMismatch(expected: String, provided: String, extension: String) + + public var errorDescription: String? { + switch self { + case let .unsupportedFileType(ext): + return "Unsupported file format: .\(ext). Please select a supported format file." + case .noDataAvailable: + return "File data is not available" + case let .loadingFailed(error): + return "File loading failed: \(error.localizedDescription)" + case let .mimeTypeMismatch(expected, provided, ext): + return "MIME type mismatch for .\(ext) file: expected '\(expected)', got '\(provided)'" + } + } +} + +// MultimodalAttachment is a struct used for transporting data between ViewModels and AttachmentPreviewCard +public struct MultimodalAttachment: Identifiable, Equatable, Hashable { + public let id = UUID() + public let mimeType: String + public let data: Data? + public let url: URL? + public var isCloudStorage: Bool = false + + public static func == (lhs: MultimodalAttachment, rhs: MultimodalAttachment) -> Bool { + return lhs.id == rhs.id + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(id) + } + + public init(mimeType: String, data: Data? = nil, url: URL? = nil) { + self.mimeType = mimeType + self.data = data + self.url = url + } + + public init(fileDataPart: FileDataPart) { + mimeType = fileDataPart.mimeType + data = nil + url = URL(string: fileDataPart.uri) + isCloudStorage = true + } +} + +// validate file type & mime type +extension MultimodalAttachment { + public static let supportedFileExtensions: Set = [ + // Documents / text + "pdf", "txt", "text", + // Images + "jpg", "jpeg", "png", "webp", + // Video + "flv", "mov", "qt", "mpeg", "mpg", "ps", "mp4", "webm", "wmv", "3gp", "3gpp", + // Audio + "aac", "flac", "mp3", "m4a", "mpga", "mp4a", "opus", "pcm", "raw", "wav", "weba", + ] + + public static func validateFileType(url: URL) throws { + let fileExtension = url.pathExtension.lowercased() + guard !fileExtension.isEmpty else { + throw MultimodalAttachmentError.unsupportedFileType(extension: "No extension") + } + + guard supportedFileExtensions.contains(fileExtension) else { + throw MultimodalAttachmentError.unsupportedFileType(extension: fileExtension) + } + } + + public static func validateMimeTypeMatch(url: URL, mimeType: String) throws { + let expectedMimeType = getMimeType(for: url) + + guard mimeType == expectedMimeType else { + throw MultimodalAttachmentError.mimeTypeMismatch( + expected: expectedMimeType, + provided: mimeType, + extension: url.pathExtension + ) + } + } + + public static func validatePhotoType(_ item: PhotosPickerItem) throws -> String { + guard let fileExtension = item.supportedContentTypes.first?.preferredFilenameExtension else { + throw MultimodalAttachmentError.unsupportedFileType(extension: "No extension") + } + + guard supportedFileExtensions.contains(fileExtension) else { + throw MultimodalAttachmentError.unsupportedFileType(extension: fileExtension) + } + + guard let fileMimeType = item.supportedContentTypes.first?.preferredMIMEType else { + throw MultimodalAttachmentError.unsupportedFileType(extension: "No MIME type") + } + + return fileMimeType + } +} + +// load data from picker item or url +extension MultimodalAttachment { + public static func fromPhotosPickerItem(_ item: PhotosPickerItem) async throws + -> MultimodalAttachment { + let fileMimeType = try validatePhotoType(item) + + do { + guard let data = try await item.loadTransferable(type: Data.self) else { + throw MultimodalAttachmentError.noDataAvailable + } + + return MultimodalAttachment( + mimeType: fileMimeType, + data: data + ) + } catch let error as MultimodalAttachmentError { + throw error + } catch { + throw MultimodalAttachmentError.loadingFailed(error) + } + } + + public static func fromFilePickerItem(from url: URL) async throws -> MultimodalAttachment { + try validateFileType(url: url) + + do { + let data = try await Task.detached(priority: .utility) { + try Data(contentsOf: url) + }.value + + let mimeType = Self.getMimeType(for: url) + + return MultimodalAttachment( + mimeType: mimeType, + data: data, + url: url + ) + } catch { + throw MultimodalAttachmentError.loadingFailed(error) + } + } + + public static func fromURL(_ url: URL, mimeType: String) async throws -> MultimodalAttachment { + try validateFileType(url: url) + try validateMimeTypeMatch(url: url, mimeType: mimeType) + + do { + let data = try await Task.detached(priority: .utility) { + try Data(contentsOf: url) + }.value + + return MultimodalAttachment( + mimeType: mimeType, + data: data, + url: url + ) + } catch { + throw MultimodalAttachmentError.loadingFailed(error) + } + } + + public func toInlineDataPart() async -> InlineDataPart? { + if let data = data, !data.isEmpty { + return InlineDataPart(data: data, mimeType: mimeType) + } + + // If the data is not available, try to read it from the url. + guard let url = url else { return nil } + do { + let data = try await Task.detached(priority: .utility) { + try Data(contentsOf: url) + }.value + + guard !data.isEmpty else { return nil } + return InlineDataPart(data: data, mimeType: mimeType) + } catch { + return nil + } + } + + private static func getMimeType(for url: URL) -> String { + let fileExtension = url.pathExtension.lowercased() + + switch fileExtension { + // Documents / text + case "pdf": + return "application/pdf" + case "txt", "text": + return "text/plain" + + // Images + case "jpg", "jpeg": + return "image/jpeg" + case "png": + return "image/png" + case "webp": + return "image/webp" + + // Video + case "flv": + return "video/x-flv" + case "mov", "qt": + return "video/quicktime" + case "mpeg": + return "video/mpeg" + case "mpg": + return "video/mpg" + case "ps": + return "video/mpegps" + case "mp4": + return "video/mp4" + case "webm": + return "video/webm" + case "wmv": + return "video/wmv" + case "3gp", "3gpp": + return "video/3gpp" + + // Audio + case "aac": + return "audio/aac" + case "flac": + return "audio/flac" + case "mp3": + return "audio/mpeg" + case "m4a": + return "audio/m4a" + case "mpga": + return "audio/mpga" + case "mp4a": + return "audio/mp4" + case "opus": + return "audio/opus" + case "pcm", "raw": + return "audio/pcm" + case "wav": + return "audio/wav" + case "weba": + return "audio/webm" + + default: + return "application/octet-stream" + } + } +} diff --git a/firebaseai/ChatExample/Assets.xcassets/Contents.json b/firebaseai/MultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json similarity index 100% rename from firebaseai/ChatExample/Assets.xcassets/Contents.json rename to firebaseai/MultimodalExample/Preview Content/Preview Assets.xcassets/Contents.json diff --git a/firebaseai/MultimodalExample/Screens/MultimodalScreen.swift b/firebaseai/MultimodalExample/Screens/MultimodalScreen.swift new file mode 100644 index 000000000..1f593f20c --- /dev/null +++ b/firebaseai/MultimodalExample/Screens/MultimodalScreen.swift @@ -0,0 +1,202 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseAI +import SwiftUI +import PhotosUI +import ConversationKit + +struct MultimodalScreen: View { + let backendType: BackendOption + @StateObject var viewModel: MultimodalViewModel + + @State private var showingPhotoPicker = false + @State private var showingFilePicker = false + @State private var showingLinkDialog = false + @State private var linkText = "" + @State private var linkMimeType = "" + @State private var selectedPhotoItems = [PhotosPickerItem]() + + init(backendType: BackendOption, sample: Sample? = nil) { + self.backendType = backendType + _viewModel = + StateObject(wrappedValue: MultimodalViewModel(backendType: backendType, + sample: sample)) + } + + private var attachmentPreviewScrollView: some View { + AttachmentPreviewScrollView( + attachments: viewModel.attachments, + onAttachmentRemove: viewModel.removeAttachment + ) + } + + var body: some View { + NavigationStack { + ConversationView(messages: $viewModel.messages, + userPrompt: viewModel.initialPrompt) { message in + MessageView(message: message) + } + .attachmentActions { + Button(action: showLinkDialog) { + Label("Link", systemImage: "link") + } + Button(action: showFilePicker) { + Label("File", systemImage: "doc.text") + } + Button(action: showPhotoPicker) { + Label("Photo", systemImage: "photo.on.rectangle.angled") + } + } + .attachmentPreview { attachmentPreviewScrollView } + .onSendMessage { message in + await viewModel.sendMessage(message.content ?? "", streaming: true) + } + .onError { error in + viewModel.presentErrorDetails = true + } + .sheet(isPresented: $viewModel.presentErrorDetails) { + if let error = viewModel.error { + ErrorDetailsView(error: error) + } + } + .photosPicker( + isPresented: $showingPhotoPicker, + selection: $selectedPhotoItems, + maxSelectionCount: 5, + matching: .any(of: [.images, .videos]) + ) + .fileImporter( + isPresented: $showingFilePicker, + allowedContentTypes: [.pdf, .audio], + allowsMultipleSelection: true + ) { result in + handleFileImport(result) + } + .alert("Add Web URL", isPresented: $showingLinkDialog) { + TextField("Enter URL", text: $linkText) + TextField("Enter mimeType", text: $linkMimeType) + Button("Add") { + handleLinkAttachment() + } + Button("Cancel", role: .cancel) { + linkText = "" + linkMimeType = "" + } + } + } + .onChange(of: selectedPhotoItems) { _, newItems in + handlePhotoSelection(newItems) + } + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button(action: newChat) { + Image(systemName: "square.and.pencil") + } + } + } + .navigationTitle(viewModel.title) + .navigationBarTitleDisplayMode(.inline) + } + + private func newChat() { + viewModel.startNewChat() + } + + private func showPhotoPicker() { + showingPhotoPicker = true + } + + private func showFilePicker() { + showingFilePicker = true + } + + private func showLinkDialog() { + showingLinkDialog = true + } + + private func handlePhotoSelection(_ items: [PhotosPickerItem]) { + Task { + for item in items { + do { + let attachment = try await MultimodalAttachment.fromPhotosPickerItem(item) + await MainActor.run { + viewModel.addAttachment(attachment) + } + } catch { + await MainActor.run { + viewModel.error = error + viewModel.presentErrorDetails = true + } + } + } + await MainActor.run { + selectedPhotoItems = [] + } + } + } + + private func handleFileImport(_ result: Result<[URL], Error>) { + switch result { + case let .success(urls): + Task { + for url in urls { + do { + let attachment = try await MultimodalAttachment.fromFilePickerItem(from: url) + await MainActor.run { + viewModel.addAttachment(attachment) + } + } catch { + await MainActor.run { + viewModel.error = error + viewModel.presentErrorDetails = true + } + } + } + } + case let .failure(error): + viewModel.error = error + viewModel.presentErrorDetails = true + } + } + + private func handleLinkAttachment() { + guard !linkText.isEmpty, let url = URL(string: linkText) else { + return + } + + let trimmedMime = linkMimeType.lowercased().trimmingCharacters(in: .whitespacesAndNewlines) + Task { + do { + let attachment = try await MultimodalAttachment.fromURL(url, mimeType: trimmedMime) + await MainActor.run { + viewModel.addAttachment(attachment) + } + } catch { + await MainActor.run { + viewModel.error = error + viewModel.presentErrorDetails = true + } + } + await MainActor.run { + linkText = "" + linkMimeType = "" + } + } + } +} + +#Preview { + MultimodalScreen(backendType: .googleAI) +} diff --git a/firebaseai/MultimodalExample/ViewModels/MultimodalViewModel.swift b/firebaseai/MultimodalExample/ViewModels/MultimodalViewModel.swift new file mode 100644 index 000000000..496cb21cb --- /dev/null +++ b/firebaseai/MultimodalExample/ViewModels/MultimodalViewModel.swift @@ -0,0 +1,217 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseAI +import Foundation +import OSLog +import PhotosUI +import SwiftUI +import AVFoundation + +@MainActor +class MultimodalViewModel: ObservableObject { + @Published var messages = [ChatMessage]() + @Published var initialPrompt: String = "" + @Published var title: String = "" + @Published var error: Error? + @Published var inProgress = false + + @Published var presentErrorDetails: Bool = false + + @Published var attachments = [MultimodalAttachment]() + + private var model: GenerativeModel + private var chat: Chat + private var chatTask: Task? + private let logger = Logger(subsystem: "com.example.firebaseai", category: "MultimodalViewModel") + + private var sample: Sample? + private var backendType: BackendOption + private var fileDataParts: [FileDataPart]? + + init(backendType: BackendOption, sample: Sample? = nil) { + self.sample = sample + self.backendType = backendType + + let firebaseService = backendType == .googleAI + ? FirebaseAI.firebaseAI(backend: .googleAI()) + : FirebaseAI.firebaseAI(backend: .vertexAI()) + + model = firebaseService.generativeModel( + modelName: sample?.modelName ?? "gemini-2.5-flash", + systemInstruction: sample?.systemInstruction + ) + + if let chatHistory = sample?.chatHistory, !chatHistory.isEmpty { + messages = ChatMessage.from(chatHistory) + chat = model.startChat(history: chatHistory) + } else { + chat = model.startChat() + } + + initialPrompt = sample?.initialPrompt ?? "" + title = sample?.title ?? "" + + fileDataParts = sample?.fileDataParts + if let fileDataParts = fileDataParts, !fileDataParts.isEmpty { + for fileDataPart in fileDataParts { + attachments.append(MultimodalAttachment(fileDataPart: fileDataPart)) + } + } + } + + func sendMessage(_ text: String, streaming: Bool = true) async { + error = nil + if streaming { + await internalSendMessageStreaming(text) + } else { + await internalSendMessage(text) + } + } + + func startNewChat() { + stop() + error = nil + chat = model.startChat() + messages.removeAll() + attachments.removeAll() + initialPrompt = "" + } + + func stop() { + chatTask?.cancel() + error = nil + } + + private func internalSendMessageStreaming(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + inProgress = true + defer { + inProgress = false + } + + let userMessage = ChatMessage(content: text, participant: .user, attachments: attachments) + messages.append(userMessage) + let systemMessage = ChatMessage.pending(participant: .other) + messages.append(systemMessage) + + do { + var parts: [any PartsRepresentable] = [text] + + if backendType == .vertexAI, let fileDataParts = fileDataParts { + // This is a patch for Cloud Storage support. Only available when using Vertex AI Gemini API. + // For non-text inputs (e.g., media files), you can attach files from Cloud Storage to the request. + // if you do not want to use Cloud Storage, you can remove this `if` statement. + // Reference: https://firebase.google.com/docs/ai-logic/solutions/cloud-storage + for fileDataPart in fileDataParts { + parts.append(fileDataPart) + } + } else { + for attachment in attachments { + if let inlineDataPart = await attachment.toInlineDataPart() { + parts.append(inlineDataPart) + } + } + } + + attachments.removeAll() + + let responseStream = try chat.sendMessageStream(parts) + for try await chunk in responseStream { + messages[messages.count - 1].pending = false + if let text = chunk.text { + messages[messages.count - 1] + .content = (messages[messages.count - 1].content ?? "") + text + } + } + } catch { + self.error = error + logger.error("\(error.localizedDescription)") + let errorMessage = ChatMessage(content: "An error occurred. Please try again.", + participant: .other, + error: error, + pending: false) + messages[messages.count - 1] = errorMessage + } + } + } + + private func internalSendMessage(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + inProgress = true + defer { + inProgress = false + } + let userMessage = ChatMessage(content: text, participant: .user, attachments: attachments) + messages.append(userMessage) + + let systemMessage = ChatMessage.pending(participant: .other) + messages.append(systemMessage) + + do { + var parts: [any PartsRepresentable] = [text] + + if backendType == .vertexAI, let fileDataParts = fileDataParts { + // This is a patch for Cloud Storage support. Only available when using Vertex AI Gemini API. + // For non-text inputs (e.g., media files), you can attach files from Cloud Storage to the request. + // if you do not want to use Cloud Storage, you can remove this `if` statement. + // Reference: https://firebase.google.com/docs/ai-logic/solutions/cloud-storage + for fileDataPart in fileDataParts { + parts.append(fileDataPart) + } + } else { + for attachment in attachments { + if let inlineDataPart = await attachment.toInlineDataPart() { + parts.append(inlineDataPart) + } + } + } + + attachments.removeAll() + + let response = try await chat.sendMessage(parts) + + if let responseText = response.text { + messages[messages.count - 1].content = responseText + messages[messages.count - 1].pending = false + } + } catch { + self.error = error + logger.error("\(error.localizedDescription)") + let errorMessage = ChatMessage(content: "An error occurred. Please try again.", + participant: .other, + error: error, + pending: false) + messages[messages.count - 1] = errorMessage + } + } + } + + func addAttachment(_ attachment: MultimodalAttachment) { + attachments.append(attachment) + } + + func removeAttachment(_ attachment: MultimodalAttachment) { + if attachment.isCloudStorage { + // Remove corresponding fileDataPart when attachment is deleted. + fileDataParts?.removeAll { $0.uri == attachment.url?.absoluteString } + } + + attachments.removeAll { $0.id == attachment.id } + } +} diff --git a/firebaseai/MultimodalExample/Views/AttachmentPreviewCard.swift b/firebaseai/MultimodalExample/Views/AttachmentPreviewCard.swift new file mode 100644 index 000000000..2bb37e25a --- /dev/null +++ b/firebaseai/MultimodalExample/Views/AttachmentPreviewCard.swift @@ -0,0 +1,188 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +private enum AttachmentType: String { + case image, video, audio, pdf, other + + init(mimeType: String) { + let mt = mimeType.lowercased() + if mt.hasPrefix("image/") { self = .image } + else if mt.hasPrefix("video/") { self = .video } + else if mt.hasPrefix("audio/") { self = .audio } + else if mt == "application/pdf" { self = .pdf } + else { self = .other } + } + + var systemImageName: String { + switch self { + case .image: return "photo" + case .video: return "video" + case .audio: return "waveform" + case .pdf: return "doc.text" + case .other: return "questionmark" + } + } + + var typeTagColor: Color { + switch self { + case .image: return .green + case .video: return .purple + case .audio: return .orange + case .pdf: return .red + case .other: return .blue + } + } + + var displayFileType: String { + switch self { + case .image: return "IMAGE" + case .video: return "VIDEO" + case .audio: return "AUDIO" + case .pdf: return "PDF" + case .other: return "UNKNOWN" + } + } +} + +struct AttachmentPreviewCard: View { + let attachment: MultimodalAttachment + let onRemove: (() -> Void)? + + private var attachmentType: AttachmentType { + AttachmentType(mimeType: attachment.mimeType) + } + + var body: some View { + HStack(spacing: 12) { + Image(systemName: attachmentType.systemImageName) + .font(.system(size: 20)) + .foregroundColor(.blue) + .frame(width: 40, height: 40) + .background(Color.blue.opacity(0.1)) + .clipShape(RoundedRectangle(cornerRadius: 6)) + + VStack(alignment: .leading, spacing: 4) { + Text(displayName) + .font(.system(size: 14, weight: .medium)) + .lineLimit(1) + .truncationMode(.middle) + .foregroundColor(.primary) + + HStack(spacing: 8) { + Text(attachmentType.displayFileType) + .font(.system(size: 10, weight: .semibold)) + .padding(.horizontal, 6) + .padding(.vertical, 2) + .background(attachmentType.typeTagColor) + .foregroundColor(.white) + .clipShape(Capsule()) + + Spacer() + } + } + + if let onRemove = onRemove { + Button(action: onRemove) { + Image(systemName: "xmark.circle.fill") + .font(.system(size: 16)) + .foregroundColor(.gray) + } + .buttonStyle(PlainButtonStyle()) + } + } + .padding(12) + .background(Color(.systemGray6)) + .clipShape(RoundedRectangle(cornerRadius: 12)) + .overlay( + RoundedRectangle(cornerRadius: 12) + .stroke(Color(.separator), lineWidth: 0.5) + ) + } + + private var displayName: String { + let fileName = attachment.url?.lastPathComponent ?? "Default" + let maxLength = 30 + if fileName.count <= maxLength { + return fileName + } + + let prefixName = fileName.prefix(15) + let suffixName = fileName.suffix(10) + return "\(prefixName)...\(suffixName)" + } +} + +struct AttachmentPreviewScrollView: View { + let attachments: [MultimodalAttachment] + var onAttachmentRemove: ((MultimodalAttachment) -> Void)? = nil + + var body: some View { + if !attachments.isEmpty { + ScrollView(.horizontal, showsIndicators: false) { + LazyHStack(spacing: 8) { + ForEach(attachments) { attachment in + AttachmentPreviewCard( + attachment: attachment, + onRemove: onAttachmentRemove == nil ? nil : { onAttachmentRemove?(attachment) } + ) + .frame(width: 180) + } + } + .padding(.horizontal, 16) + } + .frame(height: 80) + } else { + EmptyView() + } + } +} + +#Preview { + VStack(spacing: 20) { + AttachmentPreviewCard( + attachment: MultimodalAttachment( + mimeType: "image/jpeg", + data: Data() + ), + onRemove: { print("Image removed") } + ) + + AttachmentPreviewCard( + attachment: MultimodalAttachment( + mimeType: "application/pdf", + data: Data() + ), + onRemove: { print("PDF removed") } + ) + + AttachmentPreviewCard( + attachment: MultimodalAttachment( + mimeType: "video/mp4", + data: Data() + ), + onRemove: { print("Video removed") } + ) + + AttachmentPreviewCard( + attachment: MultimodalAttachment( + mimeType: "audio/mpeg", + data: Data() + ), + onRemove: { print("Audio removed") } + ) + } + .padding() +} diff --git a/firebaseai/UIComponents/Models/Sample.swift b/firebaseai/UIComponents/Models/Sample.swift index 65649235b..e81b2189a 100644 --- a/firebaseai/UIComponents/Models/Sample.swift +++ b/firebaseai/UIComponents/Models/Sample.swift @@ -21,27 +21,36 @@ public struct Sample: Identifiable { public let description: String public let useCases: [UseCase] public let navRoute: String + public let modelName: String public let chatHistory: [ModelContent]? public let initialPrompt: String? public let systemInstruction: ModelContent? public let tools: [Tool]? + public let generationConfig: GenerationConfig? + public let fileDataParts: [FileDataPart]? public init(title: String, description: String, useCases: [UseCase], navRoute: String, + modelName: String = "gemini-2.5-flash", chatHistory: [ModelContent]? = nil, initialPrompt: String? = nil, systemInstruction: ModelContent? = nil, - tools: [Tool]? = nil) { + tools: [Tool]? = nil, + generationConfig: GenerationConfig? = nil, + fileDataParts: [FileDataPart]? = nil) { self.title = title self.description = description self.useCases = useCases self.navRoute = navRoute + self.modelName = modelName self.chatHistory = chatHistory self.initialPrompt = initialPrompt self.systemInstruction = systemInstruction self.tools = tools + self.generationConfig = generationConfig + self.fileDataParts = fileDataParts } } @@ -90,110 +99,116 @@ extension Sample { title: "Blog post creator", description: "Create a blog post from an image file stored in Cloud Storage.", useCases: [.image], - navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you help me create a blog post about this image?"), - ModelContent( - role: "model", - parts: "I'd be happy to help you create a blog post! Please share the image you'd like me to analyze and write about." + navRoute: "MultimodalScreen", + initialPrompt: "Write a short, engaging blog post based on this picture." + + " It should include a description of the meal in the" + + " photo and talk about my journey meal prepping.", + fileDataParts: [ + FileDataPart( + uri: "https://storage.googleapis.com/cloud-samples-data/generative-ai/image/meal-prep.jpeg", + mimeType: "image/jpeg" ), - ], - initialPrompt: "Please analyze this image and create an engaging blog post" + ] ), Sample( - title: "Imagen 3 - image generation", + title: "Imagen - image generation", description: "Generate images using Imagen 3", useCases: [.image], navRoute: "ImagenScreen", initialPrompt: "A photo of a modern building with water in the background" ), Sample( - title: "Gemini 2.0 Flash - image generation", + title: "Gemini Flash - image generation", description: "Generate and/or edit images using Gemini 2.0 Flash", useCases: [.image], navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you edit this image to make it brighter?"), - ModelContent( - role: "model", - parts: "I can help you edit images using Gemini 2.0 Flash. Please share the image you'd like me to modify." - ), - ], - initialPrompt: "" + modelName: "gemini-2.0-flash-preview-image-generation", + initialPrompt: "Hi, can you create a 3d rendered image of a pig " + + "with wings and a top hat flying over a happy " + + "futuristic scifi city with lots of greenery?", + generationConfig: GenerationConfig(responseModalities: [.text, .image]), ), // Video Sample( title: "Hashtags for a video", description: "Generate hashtags for a video ad stored in Cloud Storage.", useCases: [.video], - navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you suggest hashtags for my product video?"), - ModelContent( - role: "model", - parts: "I'd be happy to help you generate relevant hashtags! Please share your video or describe what it's about so I can suggest appropriate hashtags." + navRoute: "MultimodalScreen", + initialPrompt: "Generate 5-10 hashtags that relate to the video content." + + " Try to use more popular and engaging terms," + + " e.g. #Viral. Do not add content not related to" + + " the video.\n Start the output with 'Tags:'", + fileDataParts: [ + FileDataPart( + uri: "https://storage.googleapis.com/cloud-samples-data/generative-ai/video/google_home_celebrity_ad.mp4", + mimeType: "video/mp4" ), - ], - initialPrompt: "" + ] ), Sample( title: "Summarize video", description: "Summarize a video and extract important dialogue.", useCases: [.video], - navRoute: "ChatScreen", + navRoute: "MultimodalScreen", chatHistory: [ - ModelContent(role: "user", parts: "Can you summarize this video for me?"), + ModelContent(role: "user", parts: "Can you help me with the description of a video file?"), ModelContent( role: "model", - parts: "I can help you summarize videos and extract key dialogue. Please share the video you'd like me to analyze." + parts: "Sure! Click on the attach button below and choose a video file for me to describe." ), ], - initialPrompt: "" + initialPrompt: "I have attached the video file. Provide a description of" + + " the video. The description should also contain" + + " anything important which people say in the video." ), // Audio Sample( title: "Audio Summarization", description: "Summarize an audio file", useCases: [.audio], - navRoute: "ChatScreen", + navRoute: "MultimodalScreen", chatHistory: [ - ModelContent(role: "user", parts: "Can you summarize this audio recording?"), + ModelContent(role: "user", parts: "Can you help me summarize an audio file?"), ModelContent( role: "model", - parts: "I can help you summarize audio files. Please share the audio recording you'd like me to analyze." + parts: "Of course! Click on the attach button below and choose an audio file for me to summarize." ), ], - initialPrompt: "" + initialPrompt: "I have attached the audio file. Please analyze it and summarize the contents" + + " of the audio as bullet points." ), Sample( title: "Translation from audio", description: "Translate an audio file stored in Cloud Storage", useCases: [.audio], - navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you translate this audio from Spanish to English?"), - ModelContent( - role: "model", - parts: "I can help you translate audio files. Please share the audio file you'd like me to translate." + navRoute: "MultimodalScreen", + initialPrompt: "Please translate the audio in Mandarin.", + fileDataParts: [ + FileDataPart( + uri: "https://storage.googleapis.com/cloud-samples-data/generative-ai/audio/How_to_create_a_My_Map_in_Google_Maps.mp3", + mimeType: "audio/mp3" ), - ], - initialPrompt: "" + ] ), // Document Sample( title: "Document comparison", description: "Compare the contents of 2 documents." + - " Only supported by the Vertex AI Gemini API because the documents are stored in Cloud Storage", + " Supported by the Vertex AI Gemini API because the documents are stored in Cloud Storage", useCases: [.document], - navRoute: "ChatScreen", - chatHistory: [ - ModelContent(role: "user", parts: "Can you compare these two documents for me?"), - ModelContent( - role: "model", - parts: "I can help you compare documents using the Vertex AI Gemini API. Please share the two documents you'd like me to compare." + navRoute: "MultimodalScreen", + initialPrompt: "The first document is from 2013, and the second document is" + + " from 2023. How did the standard deduction evolve?", + fileDataParts: [ + FileDataPart( + uri: "https://storage.googleapis.com/cloud-samples-data/generative-ai/pdf/form_1040_2013.pdf", + mimeType: "application/pdf" ), - ], - initialPrompt: "" + FileDataPart( + uri: "https://storage.googleapis.com/cloud-samples-data/generative-ai/pdf/form_1040_2023.pdf", + mimeType: "application/pdf" + ), + ] ), // Function Calling Sample( @@ -221,7 +236,7 @@ extension Sample { title: "Grounding with Google Search", description: "Use Grounding with Google Search to get responses based on up-to-date information from the web.", useCases: [.text], - navRoute: "ChatScreen", + navRoute: "GroundingScreen", initialPrompt: "What's the weather in Chicago this weekend?", tools: [.googleSearch()] ), diff --git a/firebaseai/UIComponents/Models/UseCase.swift b/firebaseai/UIComponents/Models/UseCase.swift index 5448dc01b..ee4e80f8a 100644 --- a/firebaseai/UIComponents/Models/UseCase.swift +++ b/firebaseai/UIComponents/Models/UseCase.swift @@ -15,6 +15,7 @@ import Foundation public enum UseCase: String, CaseIterable, Identifiable { + case all = "All" case text = "Text" case image = "Image" case video = "Video" diff --git a/firebaseai/UIComponents/Views/InputField.swift b/firebaseai/UIComponents/Views/InputField.swift deleted file mode 100644 index 67941c370..000000000 --- a/firebaseai/UIComponents/Views/InputField.swift +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2023 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import SwiftUI - -public struct InputField