From 5e8e2342bcb5b21db80035a9fee546fba6bd0b2a Mon Sep 17 00:00:00 2001 From: dantelmomsft Date: Thu, 10 Jul 2025 16:44:54 +0200 Subject: [PATCH 1/2] initial draft of adk-java + langchain4j + azure open ai --- .../business/mcp/server/UserMCPService.java | 3 - .../business/TransactionService.java | 2 +- app/copilot/copilot-backend/pom.xml | 17 +- .../config/BlobStorageProxyConfiguration.java | 7 - .../config/Langchain4JConfiguration.java | 4 +- .../config/MCPAgentsConfiguration.java | 56 ------ .../assistant/config/agent/ADKUtils.java | 25 +++ .../assistant/config/agent/AccountAgent.java | 45 +++++ .../config/agent}/InvoiceScanTool.java | 29 +-- .../assistant/config/agent/PaymentAgent.java | 166 ++++++++++++++++ .../config/agent/SupervisorAgent.java | 60 ++++++ .../config/agent/TransactionAgent.java | 82 ++++++++ .../isolated/CollaborationEvaluatorAgent.java | 55 ++++++ .../isolated/PaymentAgentNoDependencies.java | 158 ++++++++++++++++ .../TransactionAgentNoDependencies.java | 84 +++++++++ .../controller/ADKChatController.java | 120 ++++++++++++ .../assistant/controller/ChatAppRequest.java | 5 +- .../assistant/controller/ChatController.java | 79 -------- .../assistant/controller/ChatResponse.java | 19 +- .../plugin/InvoiceScanPlugin.java.sample | 87 --------- .../plugin/LoggedUserPlugin.java.sample | 23 --- .../plugin/PaymentMockPlugin.java.sample | 22 --- .../TransactionHistoryMockPlugin.java.sample | 41 ---- .../plugin/mock/PaymentTransaction.java | 13 -- .../plugin/mock/TransactionServiceMock.java | 56 ------ .../samples/assistant/proxy/OpenAIProxy.java | 106 ----------- .../AccountAgentIntegrationTest.java | 46 +++++ .../assistant/AzureOpenAILangchain4J.java | 41 ++++ .../EvaluationLoopIntegrationTest.java | 124 ++++++++++++ .../PaymentAgentIntegrationTest.java | 82 ++++++++ .../PaymentOrchestrationIntegrationTest.java | 104 ++++++++++ .../SupervisorAgentIntegrationTest.java | 113 +++++++++++ .../TransactionAgentIntegrationTest.java | 68 +++++++ app/copilot/langchain4j-agents/pom.xml | 65 ------- .../langchain4j/agent/AbstractReActAgent.java | 116 ------------ .../microsoft/langchain4j/agent/Agent.java | 12 -- .../agent/AgentExecutionException.java | 11 -- .../langchain4j/agent/AgentMetadata.java | 7 - .../agent/mcp/MCPProtocolType.java | 6 - .../agent/mcp/MCPServerMetadata.java | 5 - .../langchain4j/agent/mcp/MCPToolAgent.java | 112 ----------- .../langchain4j/agent/SupervisorAgent.java | 122 ------------ .../agent/mcp/AccountMCPAgent.java | 54 ------ .../agent/mcp/PaymentMCPAgent.java | 128 ------------- .../agent/mcp/TransactionHistoryMCPAgent.java | 64 ------- .../mcp/AccountMCPAgentIntegrationTest.java | 36 ---- .../mcp/PaymentMCPAgentIntegrationTest.java | 73 ------- ...ymentMCPAgentIntegrationWithImageTest.java | 77 -------- ...rAgentLongConversationIntegrationTest.java | 105 ----------- ...pervisorAgentNoRoutingIntegrationTest.java | 83 -------- ...SupervisorAgentRoutingIntegrationTest.java | 79 -------- ...sactionHistoryMCPAgentIntegrationTest.java | 37 ---- .../src/test/resources/account.yaml | 178 ------------------ .../src/test/resources/logback.xml | 16 -- .../src/test/resources/payments.yaml | 60 ------ .../test/resources/transaction-history.yaml | 94 --------- app/copilot/pom.xml | 3 +- app/frontend/src/pages/chat/Chat.tsx | 21 ++- infra/shared/host/container-apps.bicep | 4 +- 59 files changed, 1442 insertions(+), 2068 deletions(-) delete mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/MCPAgentsConfiguration.java create mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/ADKUtils.java create mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/AccountAgent.java rename app/copilot/{langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/tools => copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent}/InvoiceScanTool.java (50%) create mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/PaymentAgent.java create mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/SupervisorAgent.java create mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/TransactionAgent.java create mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/CollaborationEvaluatorAgent.java create mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/PaymentAgentNoDependencies.java create mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/TransactionAgentNoDependencies.java create mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ADKChatController.java delete mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatController.java delete mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/InvoiceScanPlugin.java.sample delete mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/LoggedUserPlugin.java.sample delete mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/PaymentMockPlugin.java.sample delete mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/TransactionHistoryMockPlugin.java.sample delete mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/mock/PaymentTransaction.java delete mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/mock/TransactionServiceMock.java delete mode 100644 app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/proxy/OpenAIProxy.java create mode 100644 app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AzureOpenAILangchain4J.java create mode 100644 app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/EvaluationLoopIntegrationTest.java create mode 100644 app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/PaymentAgentIntegrationTest.java create mode 100644 app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/PaymentOrchestrationIntegrationTest.java create mode 100644 app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/SupervisorAgentIntegrationTest.java create mode 100644 app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/TransactionAgentIntegrationTest.java delete mode 100644 app/copilot/langchain4j-agents/pom.xml delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AbstractReActAgent.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/Agent.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AgentExecutionException.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AgentMetadata.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPProtocolType.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPServerMetadata.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPToolAgent.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/SupervisorAgent.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/AccountMCPAgent.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/PaymentMCPAgent.java delete mode 100644 app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/TransactionHistoryMCPAgent.java delete mode 100644 app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/AccountMCPAgentIntegrationTest.java delete mode 100644 app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/PaymentMCPAgentIntegrationTest.java delete mode 100644 app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/PaymentMCPAgentIntegrationWithImageTest.java delete mode 100644 app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentLongConversationIntegrationTest.java delete mode 100644 app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentNoRoutingIntegrationTest.java delete mode 100644 app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentRoutingIntegrationTest.java delete mode 100644 app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/TransactionHistoryMCPAgentIntegrationTest.java delete mode 100644 app/copilot/langchain4j-agents/src/test/resources/account.yaml delete mode 100644 app/copilot/langchain4j-agents/src/test/resources/logback.xml delete mode 100644 app/copilot/langchain4j-agents/src/test/resources/payments.yaml delete mode 100644 app/copilot/langchain4j-agents/src/test/resources/transaction-history.yaml diff --git a/app/business-api/account/src/main/java/com/microsoft/openai/samples/assistant/business/mcp/server/UserMCPService.java b/app/business-api/account/src/main/java/com/microsoft/openai/samples/assistant/business/mcp/server/UserMCPService.java index c1ed7bc..c0eff86 100644 --- a/app/business-api/account/src/main/java/com/microsoft/openai/samples/assistant/business/mcp/server/UserMCPService.java +++ b/app/business-api/account/src/main/java/com/microsoft/openai/samples/assistant/business/mcp/server/UserMCPService.java @@ -6,10 +6,7 @@ import org.springframework.ai.tool.annotation.ToolParam; import org.springframework.stereotype.Service; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; @Service diff --git a/app/business-api/transactions-history/src/main/java/com/microsoft/openai/samples/assistant/business/TransactionService.java b/app/business-api/transactions-history/src/main/java/com/microsoft/openai/samples/assistant/business/TransactionService.java index 23d50cf..9d2313e 100644 --- a/app/business-api/transactions-history/src/main/java/com/microsoft/openai/samples/assistant/business/TransactionService.java +++ b/app/business-api/transactions-history/src/main/java/com/microsoft/openai/samples/assistant/business/TransactionService.java @@ -16,7 +16,7 @@ public TransactionService(){ lastTransactions.put("1010",new ArrayList<> (Arrays.asList( new Transaction("11", "Payment of the bill 334398", "outcome","acme", "0001", "1010", "BankTransfer", "100.00", "2024-4-01T12:00:00Z"), - new Transaction("22", "Payment of the bill 4613","outcome", "contoso", "0002", "1010", "CreditCard", "200.00", "2024-3-02T12:00:00Z"), + new Transaction("22", "Payment of theTr bill 4613","outcome", "contoso", "0002", "1010", "CreditCard", "200.00", "2024-3-02T12:00:00Z"), new Transaction("33", "Payment of the bill 724563","outcome", "duff", "0003", "1010", "BankTransfer", "300.00", "2023-10-03T12:00:00Z"), new Transaction("43", "Payment of the bill 8898943","outcome", "wayne enterprises", "0004", "1010", "DirectDebit", "400.00", "2023-8-04T12:00:00Z"), new Transaction("53", "Payment of the bill 19dee","outcome", "oscorp", "0005", "1010", "BankTransfer", "500.00", "2023-4-05T12:00:00Z")) diff --git a/app/copilot/copilot-backend/pom.xml b/app/copilot/copilot-backend/pom.xml index a976805..a90eeb4 100644 --- a/app/copilot/copilot-backend/pom.xml +++ b/app/copilot/copilot-backend/pom.xml @@ -20,7 +20,9 @@ 5.20.0 4.5.1 - 1.0.0-beta2 + 1.0.0 + 1.1.0-rc1 + 0.2.1-SNAPSHOT @@ -42,6 +44,17 @@ langchain4j-agents 1.0.0-SNAPSHOT + + com.google.adk + google-adk + ${google-adk.version} + + + com.google.adk + google-adk-contrib-langchain4j + ${google-adk.version} + + org.springframework.boot spring-boot-starter-web @@ -53,7 +66,7 @@ dev.langchain4j langchain4j-azure-open-ai - ${langchain4j.version} + ${langchain4j-azure-openai.version} com.azure diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/BlobStorageProxyConfiguration.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/BlobStorageProxyConfiguration.java index 71a7fe1..1e25350 100644 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/BlobStorageProxyConfiguration.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/BlobStorageProxyConfiguration.java @@ -1,15 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.openai.samples.assistant.config; -import com.azure.ai.documentintelligence.DocumentIntelligenceClient; import com.azure.core.credential.TokenCredential; -import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.AccountMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.PaymentMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.TransactionHistoryMCPAgent; import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; -import com.microsoft.openai.samples.assistant.security.LoggedUserService; -import dev.langchain4j.model.chat.ChatLanguageModel; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/Langchain4JConfiguration.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/Langchain4JConfiguration.java index 378e16f..fba498f 100644 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/Langchain4JConfiguration.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/Langchain4JConfiguration.java @@ -5,7 +5,7 @@ import com.azure.ai.openai.OpenAIClient; import dev.langchain4j.model.azure.AzureOpenAiChatModel; -import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.chat.ChatModel; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -17,7 +17,7 @@ public class Langchain4JConfiguration { private String gptChatDeploymentModelId; @Bean - public ChatLanguageModel chatLanguageModel(OpenAIClient azureOpenAICLient) { + public ChatModel chatLanguageModel(OpenAIClient azureOpenAICLient) { return AzureOpenAiChatModel.builder() .openAIClient(azureOpenAICLient) diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/MCPAgentsConfiguration.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/MCPAgentsConfiguration.java deleted file mode 100644 index ec6f72f..0000000 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/MCPAgentsConfiguration.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.openai.samples.assistant.config; - -import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import com.microsoft.openai.samples.assistant.langchain4j.agent.SupervisorAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.AccountMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.PaymentMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.TransactionHistoryMCPAgent; -import com.microsoft.openai.samples.assistant.security.LoggedUserService; -import dev.langchain4j.model.chat.ChatLanguageModel; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.List; - -@Configuration -public class MCPAgentsConfiguration { - @Value("${transactions.api.url}") String transactionsMCPServerUrl; - @Value("${accounts.api.url}") String accountsMCPServerUrl; - @Value("${payments.api.url}") String paymentsMCPServerUrl; - - private final ChatLanguageModel chatLanguageModel; - private final LoggedUserService loggedUserService; - private final DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper; - - public MCPAgentsConfiguration(ChatLanguageModel chatLanguageModel, LoggedUserService loggedUserService, DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper) { - this.chatLanguageModel = chatLanguageModel; - this.loggedUserService = loggedUserService; - this.documentIntelligenceInvoiceScanHelper = documentIntelligenceInvoiceScanHelper; - } - @Bean - public AccountMCPAgent accountMCPAgent() { - return new AccountMCPAgent(chatLanguageModel, loggedUserService.getLoggedUser().username(), accountsMCPServerUrl); - } - - @Bean - public TransactionHistoryMCPAgent transactionHistoryMCPAgent() { - return new TransactionHistoryMCPAgent(chatLanguageModel, loggedUserService.getLoggedUser().username(), transactionsMCPServerUrl,accountsMCPServerUrl); - } - - @Bean - public PaymentMCPAgent paymentMCPAgent() { - return new PaymentMCPAgent(chatLanguageModel,documentIntelligenceInvoiceScanHelper, loggedUserService.getLoggedUser().username(),transactionsMCPServerUrl,accountsMCPServerUrl, paymentsMCPServerUrl); - } - - @Bean - public SupervisorAgent supervisorAgent(ChatLanguageModel chatLanguageModel){ - return new SupervisorAgent(chatLanguageModel, - List.of(accountMCPAgent(), - transactionHistoryMCPAgent(), - paymentMCPAgent())); - - } - -} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/ADKUtils.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/ADKUtils.java new file mode 100644 index 0000000..98df938 --- /dev/null +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/ADKUtils.java @@ -0,0 +1,25 @@ +package com.microsoft.openai.samples.assistant.config.agent; + +import com.google.adk.tools.mcp.McpTool; +import com.google.adk.tools.mcp.McpToolset; +import com.google.adk.tools.mcp.SseServerParameters; + +import java.util.List; + +public class ADKUtils { + + public static List buildMCPTools(String mcpServerUrl) { + SseServerParameters sseParams = SseServerParameters.builder() + .url(mcpServerUrl) + .build(); + + McpToolset.McpToolsAndToolsetResult toolsAndToolsetResult; + try { + toolsAndToolsetResult = McpToolset.fromServer(sseParams).get(); + } catch (Exception e) { + throw new IllegalArgumentException( "Error while trying to setup MCP tools for server" + mcpServerUrl, e); + } + + return toolsAndToolsetResult.getTools(); + } +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/AccountAgent.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/AccountAgent.java new file mode 100644 index 0000000..c4f207f --- /dev/null +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/AccountAgent.java @@ -0,0 +1,45 @@ +package com.microsoft.openai.samples.assistant.config.agent; + + +import com.google.adk.agents.LlmAgent; +import com.google.adk.models.langchain4j.LangChain4j; +import com.google.adk.tools.mcp.McpTool; +import dev.langchain4j.model.chat.ChatModel; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + + +@Configuration +public class AccountAgent { + String accountsMCPServerUrl; + ChatModel chatModel; + + public AccountAgent(ChatModel chatModel, @Value("${accounts.api.url}") String accountsMCPServerUrl){ + this.chatModel = chatModel; + this.accountsMCPServerUrl = accountsMCPServerUrl; + } + + @Bean(name = "adkAccountAgent") + public LlmAgent getAgent() { + + List accountTools = ADKUtils.buildMCPTools(accountsMCPServerUrl); + + return LlmAgent.builder() + .name("AccountAgent") + .model(new LangChain4j(this.chatModel)) + .description("Agent to retrieve information about bank accounts, payment methods, and credit cards details and beneficiaries registered code. It requires logged username to retrieve the account information.") + .instruction(""" + you are a personal financial advisor who help the users to retrieve information about their bank accounts, payment methods, credit cards. + Use html list or table to display the account information. + Always use the below logged user details to retrieve account info: + '{{loggedUserName}}' + """) + .tools(accountTools) + .build(); + } + + +} diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/tools/InvoiceScanTool.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/InvoiceScanTool.java similarity index 50% rename from app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/tools/InvoiceScanTool.java rename to app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/InvoiceScanTool.java index 55c5273..ba42432 100644 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/tools/InvoiceScanTool.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/InvoiceScanTool.java @@ -1,11 +1,8 @@ -package com.microsoft.openai.samples.assistant.langchain4j.tools; +package com.microsoft.openai.samples.assistant.config.agent; +import com.google.adk.tools.Annotations.Schema; import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import dev.langchain4j.agent.tool.P; -import dev.langchain4j.agent.tool.Tool; -import dev.langchain4j.agent.tool.ToolExecutionRequest; -import dev.langchain4j.service.tool.ToolExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,28 +10,36 @@ import java.util.Map; -public class InvoiceScanTool { +public class InvoiceScanTool { private static final Logger LOGGER = LoggerFactory.getLogger(InvoiceScanTool.class); private final DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper; + + private static DocumentIntelligenceInvoiceScanHelper staticHelper; public InvoiceScanTool(DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper) { this.documentIntelligenceInvoiceScanHelper = documentIntelligenceInvoiceScanHelper; } - @Tool( "Extract the invoice or bill data scanning a photo or image") - public String scanInvoice( - @P("the path to the file containing the image or photo") String filePath) { + + + public void init() { + //adk-java require tools to be static, so we assign the injected value to a static field + staticHelper = this.documentIntelligenceInvoiceScanHelper; // Assigning to static field + } + + public Map scanInvoice( + @Schema(description ="the path to the file containing the image or photo") String filePath) { Map scanData = null; try{ - scanData = documentIntelligenceInvoiceScanHelper.scan(filePath); + scanData = this.documentIntelligenceInvoiceScanHelper.scan(filePath); } catch (Exception e) { LOGGER.warn("Error extracting data from invoice {}:", filePath,e); - scanData = new HashMap<>(); + return Map.of("status","error","report","Error extracting data from invoice: " + e.getMessage()); } LOGGER.info("scanInvoice tool: Data extracted {}:{}", filePath,scanData); - return scanData.toString(); + return scanData; } diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/PaymentAgent.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/PaymentAgent.java new file mode 100644 index 0000000..abfa106 --- /dev/null +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/PaymentAgent.java @@ -0,0 +1,166 @@ +package com.microsoft.openai.samples.assistant.config.agent; + + +import com.google.adk.agents.LlmAgent; +import com.google.adk.models.langchain4j.LangChain4j; +import com.google.adk.tools.AgentTool; +import com.google.adk.tools.BaseTool; +import com.google.adk.tools.FunctionTool; +import com.google.adk.tools.mcp.McpTool; +import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; +import dev.langchain4j.model.chat.ChatModel; +import io.reactivex.rxjava3.core.Maybe; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Configuration +public class PaymentAgent { + String paymentsMCPServerUrl; + ChatModel chatModel; + DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper; + LlmAgent accountAgent; + LlmAgent transactionAgent; + + public PaymentAgent(ChatModel chatModel, + @Value("${payments.api.url}") String paymentsMCPServerUrl, + DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper, + @Qualifier("adkAccountAgent") LlmAgent accountAgent, + @Qualifier("adkTransactionAgent") LlmAgent transactionAgent + ) { + this.chatModel = chatModel; + this.paymentsMCPServerUrl = paymentsMCPServerUrl; + this.documentIntelligenceInvoiceScanHelper = documentIntelligenceInvoiceScanHelper; + this.accountAgent = accountAgent; + this.transactionAgent = transactionAgent; + } + + @Bean(name = "adkPaymentAgent") + public LlmAgent getAgent() { + + var invoiceScanTool = new InvoiceScanTool( + this.documentIntelligenceInvoiceScanHelper); + + List paymentTools = ADKUtils.buildMCPTools(this.paymentsMCPServerUrl); + + //Build a BaseTool list out of MCP tools and add the InvoiceScanTool + List allTools = new ArrayList<>(paymentTools); + allTools.add(FunctionTool.create(invoiceScanTool, "scanInvoice")); + allTools.add(AgentTool.create(accountAgent)); + allTools.add(AgentTool.create(transactionAgent)); + + return LlmAgent.builder() + .name("PaymentAgent") + .model(new LangChain4j(this.chatModel)) + .description("you are an agent that helps users initiate payments using credit cards or bank transfer") + .instruction(""" + you are a personal financial advisor who help the users to initiate payments using credit or debit cards or with direct bank transfers. + + ## 1. Business Rules & Guardrails + - **Privacy First:** + Do NOT display, store, or process any full account numbers, Social Security Numbers, card CVV codes, or other sensitive personally identifiable information beyond the user's partial account name or masked number (e.g., ****4321). + - ** Generic Payment Rules:** + - If the user ask for a generic payment, you need to know the: recipient bank code,the total amount, a description and the payment method. + - Always check if the recipient bank code is in the registered beneficiaries list before proceeding with payment. + - Always check if a similar payment to recipient bank code has been executed in the past using payments history before proceeding with payment. + - Ask for the payment method to use based on the available methods on the user account. + - Check if the payment method selected by the user has enough funds to process the payment. + - Before submitting the payment to the system ask the user confirmation providing the payment details. + - ** Bill or Invoice Payment Rules:** + - If the user ask for bill or invoice payment you need to know the: bill id or invoice number, payee name, the total amount. + - The user can ask to pay a bill or invoice uploading photos or images.Always ask the user to confirm the extracted data from the photo or image. + - Ask for the payment method to use based on the available methods on the user account. + - Always check if the bill or invoice has been already paid based on user account payment history. + - Check if the payment method selected by the user has enough funds to process the payment. + - Before submitting the payment to the system ask the user confirmation providing the payment details. + - Include in the payment description the invoice id or bill id as following: payment for invoice 1527248. + - **Missing information and Error Handling:** + - Always retrieve the account id using the account agent. + - If input is incomplete, ambiguous, or not understood, politely request clarification without guessing about financial information. + - For payment methods details,transaction history or account details always ask help to other agents. + - If the payment succeeds provide the user with the payment confirmation. If not provide the user with the error message. + + ## 2. User Query Handling & Response Formatting + - **Acceptable Queries:** + - “I want to pay this bill mybill.png” + - “I want to transfer 1000€ to my daughter school for next trip in Italy ” + - “I need to pay my monthly electric system bill ” + + - **Formatting Rules:** + - Use HTML list or table to display bill extracted data, payments, account or transaction details. + - Always include a summary above the table (e.g., “Here are your 10 most recent payments to Amazon”) and a note reminding about privacy (e.g., “Note: Sensitive account details are masked for your security.”). \s + + - **Example Output:** + + - Example of showing Payment information: + + + + + + + + + + + + + + + + + + + + + +
Payee Namecontoso
Invoice ID9524011000817857
Amount€85.20
Payment MethodVisa (Card Number: ***477)
DescriptionPayment for invoice 9524011000817857
+ + - Example of showing Payment methods: +
    +
  1. Bank Transfer
    2. +
  2. Visa (Card Number: ***3667)
    3. +
+ + ## 3. Use below context information + - Current Timestamp: '{timestamp}' + + #Important + For payment methods details,transaction history or account details always ask help to other agents. Don't try to answer these questions by yourself or extracting data from the message history. + """) + .tools(allTools) + .beforeToolCallback((invocationContext, baseTool, input, toolContext) -> { + if (baseTool.name().equals("processPayment")) { + System.out.println("User approval required for payment processing."); + //Object escalateForPayment = invocationContext.session().state().get("paymentEscalate"); + Object escalateForPayment = toolContext.state().get("paymentEscalate"); + if(escalateForPayment != null && + escalateForPayment.equals(input)) + { + System.out.println("Payment already escalated, skipping escalation."); + return Maybe.empty(); + } else { + System.out.println("Escalating payment for user approval."); + //toolContext.actions().setEscalate(true); + //invocationContext.session().state().put("paymentEscalate", input); + toolContext.state().put("paymentEscalate", input); + return Maybe.just(Map.of("result", "User need to approve the payment with word 'Approve'. Details:" + input)); + + } + + } + return Maybe.empty(); + } + ) + .build(); + + + } + + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/SupervisorAgent.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/SupervisorAgent.java new file mode 100644 index 0000000..2e36748 --- /dev/null +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/SupervisorAgent.java @@ -0,0 +1,60 @@ +package com.microsoft.openai.samples.assistant.config.agent; + + +import com.google.adk.agents.LlmAgent; +import com.google.adk.models.langchain4j.LangChain4j; +import com.google.adk.tools.AgentTool; +import com.google.adk.tools.BaseTool; +import com.google.adk.tools.FunctionTool; +import com.google.adk.tools.mcp.McpTool; +import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; +import dev.langchain4j.model.chat.ChatModel; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.ArrayList; +import java.util.List; + +@Configuration +public class SupervisorAgent { + ChatModel chatModel; + LlmAgent accountAgent; + LlmAgent transactionAgent; + LlmAgent paymentAgent; + + public SupervisorAgent(ChatModel chatModel, + + @Qualifier("adkAccountAgent") LlmAgent accountAgent, + @Qualifier("adkTransactionAgent") LlmAgent transactionAgent, + @Qualifier("adkPaymentAgent") LlmAgent paymentAgent + ) { + this.chatModel = chatModel; + this.accountAgent = accountAgent; + this.transactionAgent = transactionAgent; + this.paymentAgent = paymentAgent; + } + + @Bean(name = "adkSupervisorAgent") + public LlmAgent getAgent() { + + return LlmAgent.builder() + .name("SupervisorAgent") + .model(new LangChain4j(this.chatModel)) + .instruction(""" + You are a banking customer support agent triaging conversation and coordinating agents work to solve the customer need. + #Coordination Rules + - If the task is about account details, and credit cards details and beneficiaries registered code select the AccountAgent. + - If the task is about banking movements, transactions or payments history select the TransactionAgent. + - If the task is about initiating and/or progress towards a payment task select the PaymentAgent. + + """) + .subAgents(accountAgent,transactionAgent,paymentAgent) + .build(); + + + } + + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/TransactionAgent.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/TransactionAgent.java new file mode 100644 index 0000000..58f0da4 --- /dev/null +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/TransactionAgent.java @@ -0,0 +1,82 @@ +package com.microsoft.openai.samples.assistant.config.agent; + + +import com.google.adk.agents.LlmAgent; +import com.google.adk.models.langchain4j.LangChain4j; +import com.google.adk.tools.AgentTool; +import com.google.adk.tools.BaseTool; +import com.google.adk.tools.FunctionTool; +import com.google.adk.tools.mcp.McpTool; +import dev.langchain4j.model.chat.ChatModel; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.ArrayList; +import java.util.List; + +@Configuration +public class TransactionAgent { + String transactionMCPServerUrl; + ChatModel chatModel; + LlmAgent accountAgent; + + public TransactionAgent(ChatModel chatModel, + @Value("${transactions.api.url}") String transactionMCPServerUrl, + @Qualifier("adkAccountAgent") LlmAgent accountAgent){ + this.chatModel = chatModel; + this.transactionMCPServerUrl = transactionMCPServerUrl; + this.accountAgent = accountAgent; + } + + @Bean(name = "adkTransactionAgent") + public LlmAgent getAgent() { + + List transactionHistoryTools = ADKUtils.buildMCPTools(this.transactionMCPServerUrl); + + //Build a BaseTool list out of MCP tools and add the InvoiceScanTool + List allTools = new ArrayList<>(transactionHistoryTools); + allTools.add(AgentTool.create(accountAgent)); + + return LlmAgent.builder() + .name("TransactionAgent") + .model(new LangChain4j(this.chatModel)) + .description("Agent to help users review their account transactions, banking movements and payments history.") + .instruction(""" + you are a personal financial advisor who help the users to review their account transactions, banking movements and payments history. + + ## 1. Business Rules & Guardrails + - **Privacy First:** + Do NOT display, store, or process any full account numbers, Social Security Numbers, card CVV codes, or other sensitive personally identifiable information beyond the user's partial account name or masked number (e.g., ****4321). + - **No Actionable Changes:** + Never access, alter, or initiate payments, transfers, or any other account changes. This assistant is strictly informational. + - **Data Scope:** + - If the user want to search last transactions for a specific payee, ask to provide the payee name + - Only reference transactions, balances, and payment history within the past 3 months. + - Only show the last 10 transactions ordered by date by default. + - **Error Handling:** + If input is incomplete, ambiguous, or not understood, politely request clarification without guessing about financial information. + ## 2. User Query Handling & Response Formatting + - **Acceptable Queries:** + - “Show me my last transactions.” + - “Can I see payments made to Amazon in May?” + - “List all direct debits from my checking account.” + - “What was my highest payment last month?” + - "When was last time I paid my electricity bill?" + + - **Formatting Rules:** + - List relevant transactions in a clear html tabular format with columns such as ID, Date, Description, Payment Type, Amount, Recipient Name. + - Always include a summary above the table (e.g., “Here are your 10 most recent payments to Amazon”) and a note reminding about privacy (e.g., “Note: Sensitive account details are masked for your security.”). \s + + + ## 3. Use below context information + - Current Timestamp: '{timestamp}' + - Logged User name: '{loggedUserName}' + """) + .tools(allTools) + .build(); + } + + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/CollaborationEvaluatorAgent.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/CollaborationEvaluatorAgent.java new file mode 100644 index 0000000..c451bf5 --- /dev/null +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/CollaborationEvaluatorAgent.java @@ -0,0 +1,55 @@ +package com.microsoft.openai.samples.assistant.config.agent.isolated; + + +import com.google.adk.agents.LlmAgent; +import com.google.adk.models.langchain4j.LangChain4j; +import com.google.adk.tools.*; +import dev.langchain4j.model.chat.ChatModel; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.Map; + +@Configuration +public class CollaborationEvaluatorAgent { + ChatModel chatModel; + + + public CollaborationEvaluatorAgent(ChatModel chatModel + ) { + this.chatModel = chatModel; + } + + @Bean(name = "adkCollaborationEvaluatorAgent") + public LlmAgent getAgent() { + + return LlmAgent.builder() + .name("CollaborationEvaluatorAgent") + .model(new LangChain4j(this.chatModel)) + .instruction(""" + You need to evaluate a conversation among agents collaborating to solve requests from home banking customers. + #Evaluation Rules + - if account ID is required let account agent to figure that out. + - If during transaction review or payment processing account info details are needed agents need to keep working together. + - If user input or review is required or an answer with full details about user ask is generated you MUST call the 'exitLoop' function. + - DO NOT CALL 'exitLoop' function if account ID is required. + """) + //.subAgents(accountAgent,transactionAgent,paymentAgent) + .tools(FunctionTool.create(CollaborationEvaluatorAgent.class, "exitLoop")) + .build(); + + } + + @Annotations.Schema( + description = + "Call this function ONLY when collaboration between agents need to stop or pause for user input or to show final response") + public static Map exitLoop(@Annotations.Schema(name = "toolContext") ToolContext toolContext) { + System.out.printf("[Tool Call] exitLoop triggered by %s \n", toolContext.agentName()); + toolContext.actions().setEscalate(true); + // Return empty dict as tools should typically return JSON-serializable output + return Map.of(); + } + + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/PaymentAgentNoDependencies.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/PaymentAgentNoDependencies.java new file mode 100644 index 0000000..834f2b2 --- /dev/null +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/PaymentAgentNoDependencies.java @@ -0,0 +1,158 @@ +package com.microsoft.openai.samples.assistant.config.agent.isolated; + + +import com.google.adk.agents.LlmAgent; +import com.google.adk.models.langchain4j.LangChain4j; +import com.google.adk.tools.BaseTool; +import com.google.adk.tools.FunctionTool; +import com.google.adk.tools.mcp.McpTool; +import com.microsoft.openai.samples.assistant.config.agent.ADKUtils; +import com.microsoft.openai.samples.assistant.config.agent.InvoiceScanTool; +import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; +import dev.langchain4j.model.chat.ChatModel; +import io.reactivex.rxjava3.core.Maybe; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Configuration +public class PaymentAgentNoDependencies { + String paymentsMCPServerUrl; + ChatModel chatModel; + DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper; + + public PaymentAgentNoDependencies(ChatModel chatModel, + @Value("${payments.api.url}") String paymentsMCPServerUrl, + DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper + ) { + this.chatModel = chatModel; + this.paymentsMCPServerUrl = paymentsMCPServerUrl; + this.documentIntelligenceInvoiceScanHelper = documentIntelligenceInvoiceScanHelper; + } + + @Bean(name = "adkPaymentAgentNoDependencies") + public LlmAgent getAgent() { + + var invoiceScanTool = new InvoiceScanTool( + this.documentIntelligenceInvoiceScanHelper); + + List paymentTools = ADKUtils.buildMCPTools(this.paymentsMCPServerUrl); + + //Build a BaseTool list out of MCP tools and add the InvoiceScanTool + List allTools = new ArrayList<>(paymentTools); + allTools.add(FunctionTool.create(invoiceScanTool, "scanInvoice")); + + return LlmAgent.builder() + .name("PaymentAgent") + .model(new LangChain4j(this.chatModel)) + .description("you are an agent that helps users initiate payments using credit cards or bank transfer") + .instruction(""" + you are a personal financial advisor who help the users to initiate payments using credit or debit cards or with direct bank transfers. + + ## 1. Business Rules & Guardrails + - **Privacy First:** + Do NOT display, store, or process any full account numbers, Social Security Numbers, card CVV codes, or other sensitive personally identifiable information beyond the user's partial account name or masked number (e.g., ****4321). + - ** Generic Payment Rules:** + - If the user ask for a generic payment, you need to know the: recipient bank code,the total amount, a description and the payment method. + - Always check if the recipient bank code is in the registered beneficiaries list before proceeding with payment. + - Always check if a similar payment to recipient bank code has been executed in the past using payments history before proceeding with payment. + - Ask for the payment method to use based on the available methods on the user account. + - Check if the payment method selected by the user has enough funds to process the payment. + - Before submitting the payment to the system ask the user confirmation providing the payment details. + - ** Bill or Invoice Payment Rules:** + - If the user ask for bill or invoice payment you need to know the: bill id or invoice number, payee name, the total amount. + - The user can ask to pay a bill or invoice uploading photos or images.Always ask the user to confirm the extracted data from the photo or image. + - Ask for the payment method to use based on the available methods on the user account. + - Always check if the bill or invoice has been already paid based on payment history before proceeding with payment. + - Check if the payment method selected by the user has enough funds to process the payment. + - Before submitting the payment to the system ask the user confirmation providing the payment details. + - Include in the payment description the invoice id or bill id as following: payment for invoice 1527248. + - **Missing information and Error Handling:** + - Always retrieve the account id using the account agent. + - If input is incomplete, ambiguous, or not understood, politely request clarification without guessing about financial information. + - For payment methods details,transaction history or account details always ask help to other agents. + - If the payment succeeds provide the user with the payment confirmation. If not provide the user with the error message. + + ## 2. User Query Handling & Response Formatting + - **Acceptable Queries:** + - “I want to pay this bill mybill.png” + - “I want to transfer 1000€ to my daughter school for next trip in Italy ” + - “I need to pay my monthly electric system bill ” + + - **Formatting Rules:** + - Use HTML list or table to display bill extracted data, payments, account or transaction details. + - Always include a summary above the table (e.g., “Here are your 10 most recent payments to Amazon”) and a note reminding about privacy (e.g., “Note: Sensitive account details are masked for your security.”). \s + + - **Example Output:** + + - Example of showing Payment information: + + + + + + + + + + + + + + + + + + + + + +
Payee Namecontoso
Invoice ID9524011000817857
Amount€85.20
Payment MethodVisa (Card Number: ***477)
DescriptionPayment for invoice 9524011000817857
+ + - Example of showing Payment methods: +
    +
  1. Bank Transfer
    2. +
  2. Visa (Card Number: ***3667)
    3. +
+ + ## 3. Use below context information + - Current Timestamp: '{timestamp}' + + #Important + For payment methods details,transaction history or account details always ask help to other agents. Don't try to answer these questions by yourself or extracting data from the message history. + """) + .tools(allTools) + .beforeToolCallback((invocationContext, baseTool, input, toolContext) -> { + if (baseTool.name().equals("processPayment")) { + System.out.println("User approval required for payment processing."); + //Object escalateForPayment = invocationContext.session().state().get("paymentEscalate"); + Object escalateForPayment = toolContext.state().get("paymentEscalate"); + if(escalateForPayment != null && + escalateForPayment.equals(input)) + { + System.out.println("Payment already escalated, skipping escalation."); + return Maybe.empty(); + } else { + System.out.println("Escalating payment for user approval."); + //toolContext.actions().setEscalate(true); + //invocationContext.session().state().put("paymentEscalate", input); + toolContext.state().put("paymentEscalate", input); + return Maybe.just(Map.of("result", "User approval required for payment processing. Details:" + input)); + + } + + } + return Maybe.empty(); + } + ) + .build(); + + + } + + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/TransactionAgentNoDependencies.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/TransactionAgentNoDependencies.java new file mode 100644 index 0000000..b29c5d0 --- /dev/null +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/agent/isolated/TransactionAgentNoDependencies.java @@ -0,0 +1,84 @@ +package com.microsoft.openai.samples.assistant.config.agent.isolated; + + +import com.google.adk.agents.LlmAgent; +import com.google.adk.models.langchain4j.LangChain4j; +import com.google.adk.tools.AgentTool; +import com.google.adk.tools.BaseTool; +import com.google.adk.tools.mcp.McpTool; +import com.microsoft.openai.samples.assistant.config.agent.ADKUtils; +import dev.langchain4j.model.chat.ChatModel; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.ArrayList; +import java.util.List; + +@Configuration +public class TransactionAgentNoDependencies { + String transactionMCPServerUrl; + ChatModel chatModel; + + public TransactionAgentNoDependencies(ChatModel chatModel, + @Value("${transactions.api.url}") String transactionMCPServerUrl){ + this.chatModel = chatModel; + this.transactionMCPServerUrl = transactionMCPServerUrl; + } + + @Bean(name = "adkTransactionAgentNoDependencies") + public LlmAgent getAgent() { + + List transactionHistoryTools = ADKUtils.buildMCPTools(this.transactionMCPServerUrl); + + return LlmAgent.builder() + .name("TransactionAgent") + .model(new LangChain4j(this.chatModel)) + .description("Agent to help users review their account transactions, banking movements and payments history.") + .instruction(""" + you are a personal financial advisor who help the users to review their account transactions, banking movements and payments history. + + ## 1. Business Rules & Guardrails + - **Privacy First:** + Do NOT display, store, or process any full account numbers, Social Security Numbers, card CVV codes, or other sensitive personally identifiable information beyond the user's partial account name or masked number (e.g., ****4321). + - **No Actionable Changes:** + Never access, alter, or initiate payments, transfers, or any other account changes. This assistant is strictly informational. + - **Data Scope:** + - To search about the payments history you need to know the payee name and the account id. + - If the user want to search last transactions for a specific payee, ask to provide the payee name + - Only reference transactions, balances, and payment history within the past 3 months. + - Only show the last 10 transactions ordered by date by default. + - **Error Handling:** + If input is incomplete, ambiguous, or not understood, politely request clarification without guessing about financial information. + ## 2. User Query Handling & Response Formatting + - **Acceptable Queries:** + - “Show me my last transactions.” + - “Can I see payments made to Amazon in May?” + - “List all direct debits from my checking account.” + - “What was my highest payment last month?” + - "When was last time I paid my electricity bill?" + + - **Formatting Rules:** + - List relevant transactions in a clear tabular format with columns such as ID, Date, Description, Payment Type, Amount, Recipient Name. + - Always include a summary above the table (e.g., “Here are your 10 most recent payments to Amazon”) and a note reminding about privacy (e.g., “Note: Sensitive account details are masked for your security.”). \s + + - **Example Output:** \s + + Here are your 10 most recent transactions in your Checking account (**data as of June 13, 2024**): + + | ID | Date | Description | Recipient |Type | Amount | + |-----------|------------|----------------------------|-----------|--------------|-----------| + |15526te53 | 06/12/2024 | order 123312 | Amazon |Bank Trasfer | $45.00 | + |shj467466 | 06/11/2024 | Payroll | contoso |Bank Transfer | €1.200,20 | + |1256265ww | 06/10/2024 | Payment of the bill 334398 | acme | Credit Card | $5.11 | + + ## 3. Use below context information + - Current Timestamp: '{timestamp}' + """) + .tools(transactionHistoryTools) + .build(); + } + + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ADKChatController.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ADKChatController.java new file mode 100644 index 0000000..30455a2 --- /dev/null +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ADKChatController.java @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.openai.samples.assistant.controller; + + +import com.google.adk.agents.LlmAgent; +import com.google.adk.events.Event; +import com.google.adk.runner.InMemoryRunner; +import com.google.adk.runner.Runner; +import com.google.adk.sessions.BaseSessionService; +import com.google.adk.sessions.GetSessionConfig; +import com.google.adk.sessions.InMemorySessionService; +import com.google.adk.sessions.Session; +import com.google.genai.types.Content; +import com.google.genai.types.Part; +import com.microsoft.openai.samples.assistant.config.agent.SupervisorAgent; +import com.microsoft.openai.samples.assistant.security.LoggedUser; +import com.microsoft.openai.samples.assistant.security.LoggedUserService; +import io.reactivex.rxjava3.core.Flowable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; + +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; + +@RestController +public class ADKChatController { + + private static final Logger LOGGER = LoggerFactory.getLogger(ADKChatController.class); + private static final String APPNAME = "Copilot"; + private final LlmAgent supervisorAgent; + private final InMemoryRunner runner; + private final LoggedUserService loggedUserService; + + public ADKChatController(@Qualifier("adkSupervisorAgent") LlmAgent supervisorAgent, LoggedUserService loggedUserService){ + this.supervisorAgent = supervisorAgent; + this.loggedUserService = loggedUserService; + this.runner = new InMemoryRunner(supervisorAgent,APPNAME); + } + + + @PostMapping(value = "/api/chat", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity openAIAsk(@RequestBody ChatAppRequest chatRequest) { + if (chatRequest.stream()) { + LOGGER.warn( + "Requested a content-type of application/json however also requested streaming." + + " Please use a content-type of application/ndjson"); + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, + "Requested a content-type of application/json however also requested streaming." + + " Please use a content-type of application/ndjson"); + } + + if (chatRequest.messages() == null || chatRequest.messages().isEmpty()) { + LOGGER.warn("history cannot be null in Chat request"); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(null); + } + + ResponseMessage userMessage = chatRequest.messages().get(chatRequest.messages().size()-1); + LOGGER.debug("Processing user message..", userMessage ); + + String inputText = userMessage.content(); + if(userMessage.attachments()!= null && !userMessage.attachments().isEmpty()) { + inputText += " Attachments: " + userMessage.attachments(); + } + String threadId = chatRequest.threadId(); + + //attach the session to the runner or create a new one + Session session = buildSession(threadId,loggedUserService.getLoggedUser(),runner); + + //Run the agent flow + Content userMsg = Content.fromParts(Part.fromText(inputText)); + Flowable events = runner.runAsync(loggedUserService.getLoggedUser().username(), session.id(), userMsg); + + AtomicReference agentResponse = new AtomicReference<>(""); + + events.blockingForEach(event ->{ + LOGGER.info(" {} > {} ",event.author()+(event.finalResponse()?"[Final]" :""),event.stringifyContent()); + if(event.finalResponse()) + agentResponse.set(event.stringifyContent()); + + } + ); + return ResponseEntity.ok( + ChatResponse.buildChatResponse(agentResponse.get(),session.id())); + } + + private Session buildSession(String threadId, LoggedUser loggedUser, Runner runner) { + + if (threadId != null && !threadId.isEmpty()) { + LOGGER.debug("Using existing threadId: {}", threadId); + return runner.sessionService().getSession(APPNAME,loggedUser.username(), threadId, Optional.of(GetSessionConfig.builder().build())).blockingGet(); + } else { + LOGGER.debug("Creating new threadId: {}", threadId); + + ConcurrentMap initialState = new ConcurrentHashMap<>(); + initialState.put("loggedUserName", loggedUser.username()); + var datetimeIso8601 = java.time.ZonedDateTime.now(java.time.ZoneId.of("UTC")).toInstant().toString(); + initialState.put("timestamp", datetimeIso8601); + + return runner.sessionService() + .createSession(APPNAME, loggedUser.username(),initialState,null) + .blockingGet(); + } + + } + + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java index 89a7feb..ade31d8 100644 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java @@ -3,10 +3,11 @@ import java.util.List; + public record ChatAppRequest( List messages, - List attachments, ChatAppRequestContext context, boolean stream, - String approach) {} + String approach, + String threadId) {} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatController.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatController.java deleted file mode 100644 index e766cea..0000000 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatController.java +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.openai.samples.assistant.controller; - - -import com.microsoft.openai.samples.assistant.langchain4j.agent.SupervisorAgent; -import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; - -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.server.ResponseStatusException; - -import java.util.ArrayList; -import java.util.List; - -@RestController -public class ChatController { - - private static final Logger LOGGER = LoggerFactory.getLogger(ChatController.class); - private final SupervisorAgent supervisorAgent; - - public ChatController(SupervisorAgent supervisorAgent){ - this.supervisorAgent = supervisorAgent; - } - - - @PostMapping(value = "/api/chat", produces = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity openAIAsk(@RequestBody ChatAppRequest chatRequest) { - if (chatRequest.stream()) { - LOGGER.warn( - "Requested a content-type of application/json however also requested streaming." - + " Please use a content-type of application/ndjson"); - throw new ResponseStatusException( - HttpStatus.BAD_REQUEST, - "Requested a content-type of application/json however also requested streaming." - + " Please use a content-type of application/ndjson"); - } - - if (chatRequest.messages() == null || chatRequest.messages().isEmpty()) { - LOGGER.warn("history cannot be null in Chat request"); - return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(null); - } - - List chatHistory = convertToLangchain4j(chatRequest); - - - LOGGER.debug("Processing chat conversation..", chatHistory.get(chatHistory.size()-1)); - - List agentsResponse = supervisorAgent.invoke(chatHistory); - - AiMessage generatedResponse = (AiMessage) agentsResponse.get(agentsResponse.size()-1); - return ResponseEntity.ok( - ChatResponse.buildChatResponse(generatedResponse)); - } - - private List convertToLangchain4j(ChatAppRequest chatAppRequest) { - List chatHistory = new ArrayList<>(); - chatAppRequest.messages().forEach( - historyChat -> { - if("user".equals(historyChat.role())) { - if(historyChat.attachments() == null || historyChat.attachments().isEmpty()) - chatHistory.add(UserMessage.from(historyChat.content())); - else - chatHistory.add(UserMessage.from(historyChat.content() + " " + historyChat.attachments().toString())); - } - if("assistant".equals(historyChat.role())) - chatHistory.add(AiMessage.from(historyChat.content())); - }); - return chatHistory; - - } -} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java index 064f246..17d2265 100644 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java @@ -9,9 +9,12 @@ import java.util.Collections; import java.util.List; -public record ChatResponse(List choices) { +public record ChatResponse( + List choices, + String threadId +) { - public static ChatResponse buildChatResponse(AiMessage aiMessage) { + public static ChatResponse buildChatResponse(String agentResponse, String threadId) { List dataPoints = Collections.emptyList(); String thoughts = ""; List attachments = Collections.emptyList(); @@ -21,15 +24,19 @@ public static ChatResponse buildChatResponse(AiMessage aiMessage) { new ResponseChoice( 0, new ResponseMessage( - aiMessage.text(), + agentResponse, ChatGPTMessage.ChatRole.ASSISTANT.toString(), attachments - ), + ), new ResponseContext(thoughts, dataPoints), new ResponseMessage( - aiMessage.text(), + agentResponse, ChatGPTMessage.ChatRole.ASSISTANT.toString(), - attachments)))); + attachments) + ) + ), + threadId + ); } } diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/InvoiceScanPlugin.java.sample b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/InvoiceScanPlugin.java.sample deleted file mode 100644 index 1cc9289..0000000 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/InvoiceScanPlugin.java.sample +++ /dev/null @@ -1,87 +0,0 @@ -package com.microsoft.openai.samples.assistant.plugin; - -import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; -import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Map; - - -/** - * reference: https://learn.microsoft.com/en-us/java/api/overview/azure/ai-openai-readme?view=azure-java-preview - * - * { - * "enhancements": { - * "ocr": { - * "enabled": true - * }, - * "grounding": { - * "enabled": true - * } - * }, - * "data_sources": [ - * { - * "type": "AzureComputerVision", - * "parameters": { - * "endpoint": "", - * "key": "" - * } - * }], - * "messages": [ - * { - * "role": "system", - * "content": "You are a helpful assistant." - * }, - * { - * "role": "user", - * "content": [ - * { - * "type": "text", - * "text": "Describe this picture:" - * }, - * { - * "type": "image_url", - * "image_url": { - * "url":"" - * } - * } - * ] - * } - * ], - * "max_tokens": 100, - * "stream": false - * } - */ -public class InvoiceScanPlugin { - - private static final Logger LOGGER = LoggerFactory.getLogger(InvoiceScanPlugin.class); - private final DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper; - public InvoiceScanPlugin(DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper) { - this.documentIntelligenceInvoiceScanHelper = documentIntelligenceInvoiceScanHelper; - } - @DefineKernelFunction(name = "scanInvoice", description = "Extract the invoice or bill data scanning a photo or image") - public String scanInvoice( - @KernelFunctionParameter(name = "filePath", description = "the path to the file containing the image or photo") String filePath) { - - Map scanData = null; - - try{ - scanData = documentIntelligenceInvoiceScanHelper.scan(filePath); - } catch (Exception e) { - LOGGER.warn("Error extracting data from invoice {}:", filePath,e); - scanData = new HashMap<>(); - } - - LOGGER.info("SK scanInvoice plugin: Data extracted {}:{}", filePath,scanData); - return scanData.toString(); - - } - - - - -} - diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/LoggedUserPlugin.java.sample b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/LoggedUserPlugin.java.sample deleted file mode 100644 index cac3b61..0000000 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/LoggedUserPlugin.java.sample +++ /dev/null @@ -1,23 +0,0 @@ -package com.microsoft.openai.samples.assistant.plugin; - -import com.microsoft.openai.samples.assistant.security.LoggedUser; -import com.microsoft.openai.samples.assistant.security.LoggedUserService; -import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; - -public class LoggedUserPlugin { - - private final LoggedUserService loggedUserService; - public LoggedUserPlugin(LoggedUserService loggedUserService) - { - this.loggedUserService = loggedUserService; - } - @DefineKernelFunction(name = "UserContext", description = "Gets the user details after login") - public String getUserContext() { - LoggedUser loggedUser = loggedUserService.getLoggedUser(); - return loggedUser.toString(); - - - } - -} - diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/PaymentMockPlugin.java.sample b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/PaymentMockPlugin.java.sample deleted file mode 100644 index adfb074..0000000 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/PaymentMockPlugin.java.sample +++ /dev/null @@ -1,22 +0,0 @@ -package com.microsoft.openai.samples.assistant.plugin; - -import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; -import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; - -public class PaymentMockPlugin { - - - @DefineKernelFunction(name = "payBill", description = "Gets the last payment transactions based on the payee, recipient name") - public String submitBillPayment( - @KernelFunctionParameter(name = "recipientName", description = "Name of the payee, recipient") String recipientName, - @KernelFunctionParameter(name = "documentId", description = " the bill id or invoice number") String documentID, - @KernelFunctionParameter(name = "amount", description = "the total amount to pay") String amount) { - - System.out.println("Bill payment executed for recipient: " + recipientName + " with documentId: " + documentID + " and amount: " + amount); - - return "Payment Successful"; - - } - -} - diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/TransactionHistoryMockPlugin.java.sample b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/TransactionHistoryMockPlugin.java.sample deleted file mode 100644 index ab142c2..0000000 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/TransactionHistoryMockPlugin.java.sample +++ /dev/null @@ -1,41 +0,0 @@ -package com.microsoft.openai.samples.assistant.plugin; - -import com.microsoft.openai.samples.assistant.plugin.mock.TransactionServiceMock; -import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; -import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TransactionHistoryMockPlugin { - private static final Logger LOGGER = LoggerFactory.getLogger(TransactionHistoryMockPlugin.class); - private final TransactionServiceMock transactionService; - public TransactionHistoryMockPlugin(){ - this.transactionService = new TransactionServiceMock(); - } - - @DefineKernelFunction(name = "getTransactionsByRecepient", description = "Gets the last payment transactions based on the payee, recipient name") - public String getTransactionsByRecepient( - @KernelFunctionParameter(name = "accountId", description = "The banking account id of the user") String accountId, - @KernelFunctionParameter(name = "recipientName", description = "Name of the payee, recipient") String recipientName) { - String transactionsByRecipient = transactionService.getTransactionsByRecipientName(accountId,recipientName).toString(); - LOGGER.info("Transactions for [{}]:{} ",recipientName,transactionsByRecipient); - return transactionsByRecipient; - - - } - - - @DefineKernelFunction(name = "getTransactions", description = "Gets the last payment transactions") - public String getTransactions( - @KernelFunctionParameter(name = "accountId", description = "The banking account id of the user") String accountId - ) { - String lastTransactions = transactionService.getlastTransactions(accountId).toString(); - LOGGER.info("Last transactions:{} ",lastTransactions); - return lastTransactions; - - - } - - -} - diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/mock/PaymentTransaction.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/mock/PaymentTransaction.java deleted file mode 100644 index ba15680..0000000 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/mock/PaymentTransaction.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.microsoft.openai.samples.assistant.plugin.mock; - -public record PaymentTransaction( - String id, - String description, - String recipientName, - String recipientBankReference, - String accountId, - String paymentType, - String amount, - String timestamp -) {} - diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/mock/TransactionServiceMock.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/mock/TransactionServiceMock.java deleted file mode 100644 index cf152a5..0000000 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/plugin/mock/TransactionServiceMock.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.microsoft.openai.samples.assistant.plugin.mock; - -import org.springframework.stereotype.Service; - -import java.util.*; -import java.util.stream.Collectors; - -@Service -public class TransactionServiceMock { - - private Map> lastTransactions= new HashMap<>(); - private Map> allTransactions= new HashMap<>(); - - public TransactionServiceMock(){ - - lastTransactions.put("1010", Arrays.asList( - new PaymentTransaction("11", "Payment of the bill 334398", "Mike ThePlumber", "0001", "1010", "BankTransfer", "100.00", "2024-4-01T12:00:00Z"), - new PaymentTransaction("22", "Payment of the bill 4613", "Jane TheElectrician", "0002", "1010", "CreditCard", "200.00", "2024-3-02T12:00:00Z"), - new PaymentTransaction("33", "Payment of the bill 724563", "Bob TheCarpenter", "0003", "1010", "BankTransfer", "300.00", "2023-10-03T12:00:00Z"), - new PaymentTransaction("43", "Payment of the bill 8898943", "Alice ThePainter", "0004", "1010", "DirectDebit", "400.00", "2023-8-04T12:00:00Z"), - new PaymentTransaction("53", "Payment of the bill 19dee", "Charlie TheMechanic", "0005", "1010", "BankTransfer", "500.00", "2023-4-05T12:00:00Z")) - ); - - - allTransactions.put("1010",Arrays.asList( - new PaymentTransaction("11", "payment of bill id with 0001", "Mike ThePlumber", "A012TABTYT156!", "1010", "BankTransfer", "100.00", "2024-4-01T12:00:00Z"), - new PaymentTransaction("21", "Payment of the bill 4200", "Mike ThePlumber", "0002", "1010", "BankTransfer", "200.00", "2024-1-02T12:00:00Z"), - new PaymentTransaction("31", "Payment of the bill 3743", "Mike ThePlumber", "0003", "1010", "DirectDebit", "300.00", "2023-10-03T12:00:00Z"), - new PaymentTransaction("41", "Payment of the bill 8921", "Mike ThePlumber", "0004", "1010", "Transfer", "400.00", "2023-8-04T12:00:00Z"), - new PaymentTransaction("51", "Payment of the bill 7666", "Mike ThePlumber", "0005", "1010", "CreditCard", "500.00", "2023-4-05T12:00:00Z"), - - new PaymentTransaction("12", "Payment of the bill 5517", "Jane TheElectrician", "0001", "1010", "CreditCard", "100.00", "2024-3-01T12:00:00Z"), - new PaymentTransaction("22", "Payment of the bill 682222", "Jane TheElectrician", "0002", "1010", "CreditCard", "200.00", "2023-1-02T12:00:00Z"), - new PaymentTransaction("32", "Payment of the bill 94112", "Jane TheElectrician", "0003", "1010", "Transfer", "300.00", "2022-10-03T12:00:00Z"), - new PaymentTransaction("42", "Payment of the bill 23122", "Jane TheElectrician", "0004", "1010", "Transfer", "400.00", "2022-8-04T12:00:00Z"), - new PaymentTransaction("52", "Payment of the bill 171443", "Jane TheElectrician", "0005", "1010", "Transfer", "500.00", "2020-4-05T12:00:00Z") - )); - - - - } - public List getTransactionsByRecipientName(String accountId, String name) { - - return allTransactions.get(accountId) - .stream() - .filter(transaction -> transaction.recipientName().contains(name)) - .collect(Collectors.toList()); - - } - - public List getlastTransactions(String accountId) { - - return lastTransactions.get(accountId); - } - -} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/proxy/OpenAIProxy.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/proxy/OpenAIProxy.java deleted file mode 100644 index 182cd9d..0000000 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/proxy/OpenAIProxy.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.openai.samples.assistant.proxy; - -import com.azure.ai.openai.OpenAIClient; -import com.azure.ai.openai.models.*; -import com.azure.core.exception.HttpResponseException; -import com.azure.core.util.IterableStream; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.web.server.ResponseStatusException; - -import java.util.List; - -/** - * This class is a proxy to the OpenAI API to simplify cross-cutting concerns management (security, - * load balancing, monitoring, resiliency). It is responsible for: - calling the OpenAI API - - * handling errors and retry strategy - load balance requests across open AI instances - add - * monitoring points - add circuit breaker with exponential backoff - * - *

It also makes unit testing easy using mockito to provide mock implementation for this bean. - */ -@Component -public class OpenAIProxy { - - private final OpenAIClient client; - - @Value("${openai.chatgpt.deployment}") - private String gptChatDeploymentModelId; - - @Value("${openai.embedding.deployment}") - private String embeddingDeploymentModelId; - - public OpenAIProxy(OpenAIClient client) { - this.client = client; - } - - public Completions getCompletions(CompletionsOptions completionsOptions) { - Completions completions; - try { - completions = client.getCompletions(this.gptChatDeploymentModelId, completionsOptions); - } catch (HttpResponseException e) { - throw new ResponseStatusException( - e.getResponse().getStatusCode(), "Error calling OpenAI API:" + e.getValue(), e); - } - return completions; - } - - public Completions getCompletions(String prompt) { - - Completions completions; - try { - completions = client.getCompletions(this.gptChatDeploymentModelId, prompt); - } catch (HttpResponseException e) { - throw new ResponseStatusException( - e.getResponse().getStatusCode(), - "Error calling OpenAI API:" + e.getMessage(), - e); - } - return completions; - } - - public ChatCompletions getChatCompletions(ChatCompletionsOptions chatCompletionsOptions) { - ChatCompletions chatCompletions; - try { - chatCompletions = - client.getChatCompletions( - this.gptChatDeploymentModelId, chatCompletionsOptions); - } catch (HttpResponseException e) { - throw new ResponseStatusException( - e.getResponse().getStatusCode(), - "Error calling OpenAI API:" + e.getMessage(), - e); - } - return chatCompletions; - } - - public IterableStream getChatCompletionsStream( - ChatCompletionsOptions chatCompletionsOptions) { - try { - return client.getChatCompletionsStream( - this.gptChatDeploymentModelId, chatCompletionsOptions); - } catch (HttpResponseException e) { - throw new ResponseStatusException( - e.getResponse().getStatusCode(), - "Error calling OpenAI API:" + e.getMessage(), - e); - } - } - - public Embeddings getEmbeddings(List texts) { - Embeddings embeddings; - try { - EmbeddingsOptions embeddingsOptions = new EmbeddingsOptions(texts); - embeddingsOptions.setUser("search-openai-demo-java"); - embeddingsOptions.setModel(this.embeddingDeploymentModelId); - embeddingsOptions.setInputType("query"); - embeddings = client.getEmbeddings(this.embeddingDeploymentModelId, embeddingsOptions); - } catch (HttpResponseException e) { - throw new ResponseStatusException( - e.getResponse().getStatusCode(), - "Error calling OpenAI API:" + e.getMessage(), - e); - } - return embeddings; - } -} diff --git a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AccountAgentIntegrationTest.java b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AccountAgentIntegrationTest.java index e86eff4..cf03a7e 100644 --- a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AccountAgentIntegrationTest.java +++ b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AccountAgentIntegrationTest.java @@ -1,8 +1,54 @@ package com.microsoft.openai.samples.assistant; +import com.google.adk.events.Event; +import com.google.adk.runner.InMemoryRunner; +import com.google.adk.sessions.Session; +import com.google.genai.types.Content; +import com.google.genai.types.Part; +import com.microsoft.openai.samples.assistant.config.agent.AccountAgent; +import dev.langchain4j.model.chat.ChatModel; +import io.reactivex.rxjava3.core.Flowable; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + public class AccountAgentIntegrationTest { + private static String APPNAME = "AccountAgentTest"; + private static String USER= "user1"; public static void main(String[] args) { + String accountMCPServerURL = System.getenv("ACCOUNT_MCP_SERVER"); + + ChatModel langchain4jModel = + AzureOpenAILangchain4J.buildChatModel(); + + var agent = new AccountAgent(langchain4jModel,accountMCPServerURL); + + var runner = new InMemoryRunner(agent.getAgent(), APPNAME); + + var initialState = new ConcurrentHashMap(); + initialState.put("loggedUserName", "bob.user@contoso.com"); + + Session session = + runner.sessionService() + .createSession(APPNAME, USER,initialState,null) + .blockingGet(); + + + Content userMsg = Content.fromParts(Part.fromText("What is my account balance?")); + System.out.print("\nYou > "+ userMsg.text()); + Flowable events = runner.runAsync(USER, session.id(), userMsg); + + System.out.print("\nAgent > "); + events.blockingForEach(event -> System.out.println(event.stringifyContent())); + + userMsg = Content.fromParts(Part.fromText("What about my visa ?")); + System.out.print("\nYou > "+userMsg.text()); + + events = runner.runAsync(USER, session.id(), userMsg); + + System.out.print("\nAgent > "); + events.blockingForEach(event -> System.out.println(event.stringifyContent())); } } diff --git a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AzureOpenAILangchain4J.java b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AzureOpenAILangchain4J.java new file mode 100644 index 0000000..ba58c0d --- /dev/null +++ b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AzureOpenAILangchain4J.java @@ -0,0 +1,41 @@ +package com.microsoft.openai.samples.assistant; + +import com.azure.ai.openai.OpenAIClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.TokenCredential; +import com.azure.core.http.policy.HttpLogDetailLevel; +import com.azure.core.http.policy.HttpLogOptions; +import com.azure.identity.AzureCliCredentialBuilder; +import dev.langchain4j.model.azure.AzureOpenAiChatModel; +import dev.langchain4j.model.chat.ChatModel; + +public class AzureOpenAILangchain4J { + + public static ChatModel buildChatModel(){ + String azureOpenAIName = System.getenv("AZURE_OPENAI_NAME"); + String chatDeploymentName = System.getenv("AZURE_OPENAI_DEPLOYMENT_ID"); + if (azureOpenAIName == null || azureOpenAIName.isEmpty()) + throw new IllegalArgumentException("AZURE_OPENAI_NAME environment variable is not set."); + + + TokenCredential tokenCredential = new AzureCliCredentialBuilder().build(); + + String endpoint = "https://%s.openai.azure.com".formatted(azureOpenAIName); + + var httpLogOptions = new HttpLogOptions(); + // httpLogOptions.setPrettyPrintBody(true); + httpLogOptions.setLogLevel(HttpLogDetailLevel.BODY_AND_HEADERS); + + OpenAIClient client = new OpenAIClientBuilder() + .endpoint(endpoint) + .credential(tokenCredential) + .httpLogOptions(httpLogOptions) + .buildClient(); + + return AzureOpenAiChatModel.builder() + .openAIClient(client) + .logRequestsAndResponses(true) + .deploymentName(chatDeploymentName) + .build(); + } +} diff --git a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/EvaluationLoopIntegrationTest.java b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/EvaluationLoopIntegrationTest.java new file mode 100644 index 0000000..1113900 --- /dev/null +++ b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/EvaluationLoopIntegrationTest.java @@ -0,0 +1,124 @@ +package com.microsoft.openai.samples.assistant; + +import com.azure.ai.documentintelligence.DocumentIntelligenceClient; +import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder; +import com.azure.identity.AzureCliCredentialBuilder; +import com.google.adk.agents.LlmAgent; +import com.google.adk.agents.LoopAgent; +import com.google.adk.events.Event; +import com.google.adk.runner.InMemoryRunner; +import com.google.adk.sessions.Session; +import com.google.genai.types.Content; +import com.google.genai.types.Part; +import com.microsoft.openai.samples.assistant.config.agent.*; +import com.microsoft.openai.samples.assistant.config.agent.isolated.CollaborationEvaluatorAgent; +import com.microsoft.openai.samples.assistant.config.agent.isolated.PaymentAgentNoDependencies; +import com.microsoft.openai.samples.assistant.config.agent.isolated.TransactionAgentNoDependencies; +import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; +import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; +import dev.langchain4j.model.chat.ChatModel; +import io.reactivex.rxjava3.core.Flowable; + +import java.nio.charset.StandardCharsets; +import java.util.Scanner; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class EvaluationLoopIntegrationTest { + private static String APPNAME = "PaymentAgentTest"; + private static String USER= "user1"; + + public static void main(String[] args) { + + ChatModel langchain4jModel = + AzureOpenAILangchain4J.buildChatModel(); + + LlmAgent accountAgent = buildAccountAgent(langchain4jModel); + LlmAgent transactionsAgent = buildTransactionsAgent(langchain4jModel); + LlmAgent paymentAgent = buildPaymentAgent(langchain4jModel); + LlmAgent evaluatorAgent = buildEvaluatorAgent(langchain4jModel); + + + var supervisorAgent = new SupervisorAgent(langchain4jModel,accountAgent,transactionsAgent,paymentAgent); + + LoopAgent loopAgent = LoopAgent.builder() + .name("AgentGroupChatLoop") + .description("Repeatedly evaluate agents collaboration") + .subAgents(supervisorAgent.getAgent(), evaluatorAgent) + .maxIterations(5) + .build(); + + InMemoryRunner runner = new InMemoryRunner(loopAgent, APPNAME); + + ConcurrentMap initialState = new ConcurrentHashMap<>(); + // initialState.put("accountId", "1010"); + initialState.put("loggedUserName", "bob.user@contoso.com"); + var datetimeIso8601 = java.time.ZonedDateTime.now(java.time.ZoneId.of("UTC")).toInstant().toString(); + initialState.put("timestamp", datetimeIso8601); + + Session session = + runner.sessionService() + .createSession(APPNAME, USER,initialState,null) + .blockingGet(); + + + try (Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8)) { + while (true) { + System.out.print("\nYou > "); + String userInput = scanner.nextLine(); + + if ("quit".equalsIgnoreCase(userInput)) { + break; + } + + Content userMsg = Content.fromParts(Part.fromText(userInput)); + Flowable events = runner.runAsync(USER, session.id(), userMsg); + + events.blockingForEach(event -> System.out.println("\n"+event.author()+(event.finalResponse()?"[Final]" :"") +" > " +event.stringifyContent())); + } + } + } + + private static LlmAgent buildEvaluatorAgent(ChatModel langchain4jModel) { + return new CollaborationEvaluatorAgent(langchain4jModel).getAgent(); + } + + private static LlmAgent buildPaymentAgent(ChatModel langchain4jModel) { + String paymentMCPServerURL = System.getenv("PAYMENT_MCP_SERVER"); + + var documentIntelligenceInvoiceScanHelper = new DocumentIntelligenceInvoiceScanHelper(getDocumentIntelligenceClient(),getBlobStorageProxyClient()); + + var paymentAgent = new PaymentAgentNoDependencies(langchain4jModel,paymentMCPServerURL,documentIntelligenceInvoiceScanHelper); + return paymentAgent.getAgent(); + } + + private static LlmAgent buildTransactionsAgent(ChatModel langchain4jModel) { + String transactionsMCPServerURL = System.getenv("TRANSACTION_MCP_SERVER"); + var transactionsAgent = new TransactionAgentNoDependencies(langchain4jModel,transactionsMCPServerURL); + return transactionsAgent.getAgent(); + } + + private static LlmAgent buildAccountAgent(ChatModel langchain4jModel) { + + String accountMCPServerURL = System.getenv("ACCOUNT_MCP_SERVER"); + var accountAgent = new AccountAgent(langchain4jModel,accountMCPServerURL); + return accountAgent.getAgent(); + + } + + + private static BlobStorageProxy getBlobStorageProxyClient() { + + String containerName = "content"; + return new BlobStorageProxy(System.getenv("AZURE_STORAGE_ACCOUNT"),containerName,new AzureCliCredentialBuilder().build()); + } + + private static DocumentIntelligenceClient getDocumentIntelligenceClient() { + String endpoint = "https://%s.cognitiveservices.azure.com".formatted(System.getenv("AZURE_DOCUMENT_INTELLIGENCE_SERVICE")); + + return new DocumentIntelligenceClientBuilder() + .credential(new AzureCliCredentialBuilder().build()) + .endpoint(endpoint) + .buildClient(); + } +} diff --git a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/PaymentAgentIntegrationTest.java b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/PaymentAgentIntegrationTest.java new file mode 100644 index 0000000..068e016 --- /dev/null +++ b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/PaymentAgentIntegrationTest.java @@ -0,0 +1,82 @@ +package com.microsoft.openai.samples.assistant; + +import com.azure.ai.documentintelligence.DocumentIntelligenceClient; +import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder; +import com.azure.identity.AzureCliCredentialBuilder; +import com.google.adk.events.Event; +import com.google.adk.runner.InMemoryRunner; +import com.google.adk.sessions.Session; +import com.google.genai.types.Content; +import com.google.genai.types.Part; +import com.microsoft.openai.samples.assistant.config.agent.isolated.PaymentAgentNoDependencies; +import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; +import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; +import dev.langchain4j.model.chat.ChatModel; +import io.reactivex.rxjava3.core.Flowable; + +import java.nio.charset.StandardCharsets; +import java.util.Scanner; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class PaymentAgentIntegrationTest { + private static String APPNAME = "PaymentAgentTest"; + private static String USER= "user1"; + + public static void main(String[] args) { + + String paymentMCPServerURL = System.getenv("PAYMENT_MCP_SERVER"); + + ChatModel langchain4jModel = + AzureOpenAILangchain4J.buildChatModel(); + + var documentIntelligenceInvoiceScanHelper = new DocumentIntelligenceInvoiceScanHelper(getDocumentIntelligenceClient(),getBlobStorageProxyClient()); + + var agent = new PaymentAgentNoDependencies(langchain4jModel,paymentMCPServerURL,documentIntelligenceInvoiceScanHelper); + + InMemoryRunner runner = new InMemoryRunner(agent.getAgent(), APPNAME); + + ConcurrentMap initialState = new ConcurrentHashMap<>(); + initialState.put("accountId", "1010"); + var datetimeIso8601 = java.time.ZonedDateTime.now(java.time.ZoneId.of("UTC")).toInstant().toString(); + initialState.put("timestamp", datetimeIso8601); + + Session session = + runner.sessionService() + .createSession(APPNAME, USER,initialState,null) + .blockingGet(); + + + try (Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8)) { + while (true) { + System.out.print("\nYou > "); + String userInput = scanner.nextLine(); + + if ("quit".equalsIgnoreCase(userInput)) { + break; + } + + Content userMsg = Content.fromParts(Part.fromText(userInput)); + Flowable events = runner.runAsync(USER, session.id(), userMsg); + + System.out.print("\nAgent > "); + events.blockingForEach(event -> System.out.println(event.stringifyContent())); + } + } + } + + private static BlobStorageProxy getBlobStorageProxyClient() { + + String containerName = "content"; + return new BlobStorageProxy(System.getenv("AZURE_STORAGE_ACCOUNT"),containerName,new AzureCliCredentialBuilder().build()); + } + + private static DocumentIntelligenceClient getDocumentIntelligenceClient() { + String endpoint = "https://%s.cognitiveservices.azure.com".formatted(System.getenv("AZURE_DOCUMENT_INTELLIGENCE_SERVICE")); + + return new DocumentIntelligenceClientBuilder() + .credential(new AzureCliCredentialBuilder().build()) + .endpoint(endpoint) + .buildClient(); + } +} diff --git a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/PaymentOrchestrationIntegrationTest.java b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/PaymentOrchestrationIntegrationTest.java new file mode 100644 index 0000000..4960b55 --- /dev/null +++ b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/PaymentOrchestrationIntegrationTest.java @@ -0,0 +1,104 @@ +package com.microsoft.openai.samples.assistant; + +import com.azure.ai.documentintelligence.DocumentIntelligenceClient; +import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder; +import com.azure.identity.AzureCliCredentialBuilder; +import com.google.adk.agents.LlmAgent; +import com.google.adk.events.Event; +import com.google.adk.runner.InMemoryRunner; +import com.google.adk.sessions.Session; +import com.google.genai.types.Content; +import com.google.genai.types.Part; +import com.microsoft.openai.samples.assistant.config.agent.AccountAgent; +import com.microsoft.openai.samples.assistant.config.agent.PaymentAgent; +import com.microsoft.openai.samples.assistant.config.agent.TransactionAgent; +import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; +import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; +import dev.langchain4j.model.chat.ChatModel; +import io.reactivex.rxjava3.core.Flowable; + +import java.nio.charset.StandardCharsets; +import java.util.Scanner; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class PaymentOrchestrationIntegrationTest { + private static String APPNAME = "PaymentAgentTest"; + private static String USER= "user1"; + + public static void main(String[] args) { + + ChatModel langchain4jModel = + AzureOpenAILangchain4J.buildChatModel(); + + LlmAgent accountAgent = buildAccountAgent(langchain4jModel); + LlmAgent transactionsAgent = buildTransactionsAgent(langchain4jModel,accountAgent); + + String paymentMCPServerURL = System.getenv("PAYMENT_MCP_SERVER"); + + var documentIntelligenceInvoiceScanHelper = new DocumentIntelligenceInvoiceScanHelper(getDocumentIntelligenceClient(),getBlobStorageProxyClient()); + + var agent = new PaymentAgent(langchain4jModel,paymentMCPServerURL,documentIntelligenceInvoiceScanHelper,accountAgent,transactionsAgent); + + InMemoryRunner runner = new InMemoryRunner(agent.getAgent(), APPNAME); + + ConcurrentMap initialState = new ConcurrentHashMap<>(); + // initialState.put("accountId", "1010"); + initialState.put("loggedUserName", "bob.user@contoso.com"); + var datetimeIso8601 = java.time.ZonedDateTime.now(java.time.ZoneId.of("UTC")).toInstant().toString(); + initialState.put("timestamp", datetimeIso8601); + + Session session = + runner.sessionService() + .createSession(APPNAME, USER,initialState,null) + .blockingGet(); + + + try (Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8)) { + while (true) { + System.out.print("\nYou > "); + String userInput = scanner.nextLine(); + + if ("quit".equalsIgnoreCase(userInput)) { + break; + } + + Content userMsg = Content.fromParts(Part.fromText(userInput)); + Flowable events = runner.runAsync(USER, session.id(), userMsg); + + System.out.print("\nAgent > "); + events.blockingForEach(event -> System.out.println(event.stringifyContent())); + } + } + } + + private static LlmAgent buildTransactionsAgent(ChatModel langchain4jModel,LlmAgent accountAgent) { + String transactionsMCPServerURL = System.getenv("TRANSACTION_MCP_SERVER"); + var transactionsAgent = new TransactionAgent(langchain4jModel,transactionsMCPServerURL,accountAgent); + return transactionsAgent.getAgent(); + } + + private static LlmAgent buildAccountAgent(ChatModel langchain4jModel) { + + String accountMCPServerURL = System.getenv("ACCOUNT_MCP_SERVER"); + var accountAgent = new AccountAgent(langchain4jModel,accountMCPServerURL); + return accountAgent.getAgent(); + + } + + + private static BlobStorageProxy getBlobStorageProxyClient() { + + String containerName = "content"; + return new BlobStorageProxy(System.getenv("AZURE_STORAGE_ACCOUNT"),containerName,new AzureCliCredentialBuilder().build()); + } + + private static DocumentIntelligenceClient getDocumentIntelligenceClient() { + String endpoint = "https://%s.cognitiveservices.azure.com".formatted(System.getenv("AZURE_DOCUMENT_INTELLIGENCE_SERVICE")); + + return new DocumentIntelligenceClientBuilder() + .credential(new AzureCliCredentialBuilder().build()) + .endpoint(endpoint) + .buildClient(); + } +} diff --git a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/SupervisorAgentIntegrationTest.java b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/SupervisorAgentIntegrationTest.java new file mode 100644 index 0000000..17820be --- /dev/null +++ b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/SupervisorAgentIntegrationTest.java @@ -0,0 +1,113 @@ +package com.microsoft.openai.samples.assistant; + +import com.azure.ai.documentintelligence.DocumentIntelligenceClient; +import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder; +import com.azure.identity.AzureCliCredentialBuilder; +import com.google.adk.agents.LlmAgent; +import com.google.adk.events.Event; +import com.google.adk.runner.InMemoryRunner; +import com.google.adk.sessions.Session; +import com.google.genai.types.Content; +import com.google.genai.types.Part; +import com.microsoft.openai.samples.assistant.config.agent.*; +import com.microsoft.openai.samples.assistant.config.agent.isolated.PaymentAgentNoDependencies; +import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; +import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; +import dev.langchain4j.model.chat.ChatModel; +import io.reactivex.rxjava3.core.Flowable; + +import java.nio.charset.StandardCharsets; +import java.util.Scanner; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class SupervisorAgentIntegrationTest { + private static String APPNAME = "SupervisorTest"; + private static String USER= "user1"; + + public static void main(String[] args) { + + ChatModel langchain4jModel = + AzureOpenAILangchain4J.buildChatModel(); + + LlmAgent accountAgent = buildAccountAgent(langchain4jModel); + LlmAgent transactionsAgent = buildTransactionsAgent(langchain4jModel,accountAgent); + LlmAgent paymentAgent = buildPaymentAgent(langchain4jModel,accountAgent,transactionsAgent); + + var supervisorAgent = new SupervisorAgent(langchain4jModel,accountAgent,transactionsAgent,paymentAgent); + + InMemoryRunner runner = new InMemoryRunner(supervisorAgent.getAgent(), APPNAME); + + ConcurrentMap initialState = new ConcurrentHashMap<>(); + // initialState.put("accountId", "1010"); + initialState.put("loggedUserName", "bob.user@contoso.com"); + var datetimeIso8601 = java.time.ZonedDateTime.now(java.time.ZoneId.of("UTC")).toInstant().toString(); + initialState.put("timestamp", datetimeIso8601); + + Session session = + runner.sessionService() + .createSession(APPNAME, USER,initialState,null) + .blockingGet(); + + + try (Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8)) { + while (true) { + System.out.print("\nYou > "); + String userInput = scanner.nextLine(); + + if ("quit".equalsIgnoreCase(userInput)) { + break; + } + + Content userMsg = Content.fromParts(Part.fromText(userInput)); + Flowable events = runner.runAsync(USER, session.id(), userMsg); + + //events.blockingForEach(event -> System.out.println("\n"+event.author()+" > " +event.stringifyContent())); + events.blockingIterable().forEach(event -> System.out.println("\n"+event.author()+(event.finalResponse()?"[Final]" :"") +" > " +event.stringifyContent())); + } + } + } + + private static LlmAgent buildPaymentAgent(ChatModel langchain4jModel,LlmAgent accountAgent,LlmAgent transactionsAgent) { + String paymentMCPServerURL = System.getenv("PAYMENT_MCP_SERVER"); + + var documentIntelligenceInvoiceScanHelper = new DocumentIntelligenceInvoiceScanHelper(getDocumentIntelligenceClient(),getBlobStorageProxyClient()); + + var paymentAgent = new PaymentAgent(langchain4jModel, + paymentMCPServerURL, + documentIntelligenceInvoiceScanHelper, + accountAgent, + transactionsAgent); + return paymentAgent.getAgent(); + } + + private static LlmAgent buildTransactionsAgent(ChatModel langchain4jModel,LlmAgent accountAgent) { + String transactionsMCPServerURL = System.getenv("TRANSACTION_MCP_SERVER"); + var transactionsAgent = new TransactionAgent(langchain4jModel,transactionsMCPServerURL,accountAgent); + return transactionsAgent.getAgent(); + } + + private static LlmAgent buildAccountAgent(ChatModel langchain4jModel) { + + String accountMCPServerURL = System.getenv("ACCOUNT_MCP_SERVER"); + var accountAgent = new AccountAgent(langchain4jModel,accountMCPServerURL); + return accountAgent.getAgent(); + + } + + + private static BlobStorageProxy getBlobStorageProxyClient() { + + String containerName = "content"; + return new BlobStorageProxy(System.getenv("AZURE_STORAGE_ACCOUNT"),containerName,new AzureCliCredentialBuilder().build()); + } + + private static DocumentIntelligenceClient getDocumentIntelligenceClient() { + String endpoint = "https://%s.cognitiveservices.azure.com".formatted(System.getenv("AZURE_DOCUMENT_INTELLIGENCE_SERVICE")); + + return new DocumentIntelligenceClientBuilder() + .credential(new AzureCliCredentialBuilder().build()) + .endpoint(endpoint) + .buildClient(); + } +} diff --git a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/TransactionAgentIntegrationTest.java b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/TransactionAgentIntegrationTest.java new file mode 100644 index 0000000..83337e8 --- /dev/null +++ b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/TransactionAgentIntegrationTest.java @@ -0,0 +1,68 @@ +package com.microsoft.openai.samples.assistant; + +import com.google.adk.agents.LlmAgent; +import com.google.adk.events.Event; +import com.google.adk.runner.InMemoryRunner; +import com.google.adk.sessions.Session; +import com.google.genai.types.Content; +import com.google.genai.types.Part; +import com.microsoft.openai.samples.assistant.config.agent.AccountAgent; +import com.microsoft.openai.samples.assistant.config.agent.TransactionAgent; +import dev.langchain4j.model.chat.ChatModel; +import io.reactivex.rxjava3.core.Flowable; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class TransactionAgentIntegrationTest { + private static String APPNAME = "TransactionAgentTest"; + private static String USER= "user1"; + + public static void main(String[] args) { + + String transactionMCPServerURL = System.getenv("TRANSACTION_MCP_SERVER"); + + ChatModel langchain4jModel = + AzureOpenAILangchain4J.buildChatModel(); + + LlmAgent accountAgent = buildAccountAgent(langchain4jModel); + + var agent = new TransactionAgent(langchain4jModel,transactionMCPServerURL,accountAgent); + + InMemoryRunner runner = new InMemoryRunner(agent.getAgent(), APPNAME); + + ConcurrentMap initialState = new ConcurrentHashMap<>(); + initialState.put("loggedUserName", "bob.user@contoso.com"); + var datetimeIso8601 = java.time.ZonedDateTime.now(java.time.ZoneId.of("UTC")).toInstant().toString(); + initialState.put("timestamp", datetimeIso8601); + + Session session = + runner.sessionService() + .createSession(APPNAME, USER,initialState,null) + .blockingGet(); + + + Content userMsg = Content.fromParts(Part.fromText("When was last time I've paid contoso?")); + System.out.print("\nYou > "+ userMsg.text()); + Flowable events = runner.runAsync(USER, session.id(), userMsg); + + System.out.print("\nAgent > "); + events.blockingForEach(event -> System.out.println(event.stringifyContent())); + + userMsg = Content.fromParts(Part.fromText("ok, what about my last transactions")); + System.out.print("\nYou > "+userMsg.text()); + + events = runner.runAsync(USER, session.id(), userMsg); + + System.out.print("\nAgent > "); + events.blockingForEach(event -> System.out.println(event.stringifyContent())); + } + + private static LlmAgent buildAccountAgent(ChatModel langchain4jModel) { + + String accountMCPServerURL = System.getenv("ACCOUNT_MCP_SERVER"); + var accountAgent = new AccountAgent(langchain4jModel,accountMCPServerURL); + return accountAgent.getAgent(); + + } +} diff --git a/app/copilot/langchain4j-agents/pom.xml b/app/copilot/langchain4j-agents/pom.xml deleted file mode 100644 index 0d84ef1..0000000 --- a/app/copilot/langchain4j-agents/pom.xml +++ /dev/null @@ -1,65 +0,0 @@ - - - 4.0.0 - - com.microsoft.openai.samples.assistant - copilot-parent - 1.0.0-SNAPSHOT - ../pom.xml - - - langchain4j-agents - agents implementation based on LangChain4j - - - - - com.microsoft.openai.samples.assistant - copilot-common - 1.0.0-SNAPSHOT - - - dev.langchain4j - langchain4j-mcp - - ${langchain4j.version} - - - dev.langchain4j - langchain4j-azure-open-ai - ${langchain4j.version} - test - - - com.azure - azure-identity - - test - - - org.junit.jupiter - junit-jupiter-api - 5.11.3 - test - - - ch.qos.logback - logback-classic - 1.5.8 - test - - - org.assertj - assertj-core - 3.27.3 - test - - - org.wiremock - wiremock - 3.12.1 - test - - - - \ No newline at end of file diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AbstractReActAgent.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AbstractReActAgent.java deleted file mode 100644 index 9fe57b8..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AbstractReActAgent.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.microsoft.langchain4j.agent; - -import dev.langchain4j.agent.tool.ToolSpecification; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.SystemMessage; -import dev.langchain4j.memory.ChatMemory; -import dev.langchain4j.memory.chat.MessageWindowChatMemory; -import dev.langchain4j.agent.tool.ToolExecutionRequest; -import dev.langchain4j.data.message.ToolExecutionResultMessage; -import dev.langchain4j.model.chat.ChatLanguageModel; -import dev.langchain4j.model.chat.request.ChatRequest; -import dev.langchain4j.model.chat.request.ChatRequestParameters; -import dev.langchain4j.service.tool.ToolExecutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -public abstract class AbstractReActAgent implements Agent { - - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractReActAgent.class); - - protected final ChatLanguageModel chatModel; - - protected AbstractReActAgent(ChatLanguageModel chatModel) { - if (chatModel == null) { - throw new IllegalArgumentException("chatModel cannot be null"); - } - this.chatModel = chatModel; - } - - @Override - public List invoke(List chatHistory) throws AgentExecutionException { - LOGGER.info("------------- {} -------------", this.getName()); - - try { - var internalChatMemory = buildInternalChat(chatHistory); - - ChatRequestParameters parameters = ChatRequestParameters.builder() - .toolSpecifications(getToolSpecifications()) - .build(); - - ChatRequest request = ChatRequest.builder() - .messages(internalChatMemory.messages()) - .parameters(parameters) - .build(); - - var aiMessage = chatModel.chat(request).aiMessage(); - - // ReAct planning with tools - while (aiMessage != null && aiMessage.hasToolExecutionRequests()) { - List toolExecutionResultMessages = executeToolRequests(aiMessage.toolExecutionRequests()); - - internalChatMemory.add(aiMessage); - toolExecutionResultMessages.forEach(internalChatMemory::add); - - ChatRequest toolExecutionResultResponseRequest = ChatRequest.builder() - .messages(internalChatMemory.messages()) - .parameters(parameters) - .build(); - - aiMessage = chatModel.chat(toolExecutionResultResponseRequest).aiMessage(); - } - - LOGGER.info("Agent response: {}", aiMessage.text()); - - // add last ai message to agent internal memory - internalChatMemory.add(aiMessage); - return buildResponse(chatHistory, internalChatMemory); - } catch (Exception e) { - throw new AgentExecutionException("Error during agent [%s] invocation".formatted(this.getName()), e); - } - } - - protected List buildResponse(List chatHistory, ChatMemory internalChatMemory) { - return internalChatMemory.messages() - .stream() - .filter(m -> !(m instanceof SystemMessage)) - .collect(Collectors.toList()); - } - - protected List executeToolRequests(List toolExecutionRequests) { - List toolExecutionResultMessages = new ArrayList<>(); - for (ToolExecutionRequest toolExecutionRequest : toolExecutionRequests) { - var toolExecutor = getToolExecutor(toolExecutionRequest.name()); - LOGGER.info("Executing {} with params {}", toolExecutionRequest.name(), toolExecutionRequest.arguments()); - String result = toolExecutor.execute(toolExecutionRequest, null); - LOGGER.info("Response from {}: {}", toolExecutionRequest.name(), result); - if (result == null || result.isEmpty()) { - LOGGER.warn("Tool {} returned empty result but successfully completed. Setting result=ok.", toolExecutionRequest.name()); - result = "ok"; - } - toolExecutionResultMessages.add(ToolExecutionResultMessage.from(toolExecutionRequest, result)); - } - return toolExecutionResultMessages; - } - - protected ChatMemory buildInternalChat(List chatHistory) { - var internalChatMemory = MessageWindowChatMemory.builder() - .id("default") - .maxMessages(20) - .build(); - - internalChatMemory.add(SystemMessage.from(getSystemMessage())); - chatHistory.forEach(internalChatMemory::add); - return internalChatMemory; - } - - protected abstract String getSystemMessage(); - - protected abstract List getToolSpecifications(); - - protected abstract ToolExecutor getToolExecutor(String toolName); -} \ No newline at end of file diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/Agent.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/Agent.java deleted file mode 100644 index 631d58f..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/Agent.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.microsoft.langchain4j.agent; - -import dev.langchain4j.data.message.ChatMessage; - -import java.util.List; - -public interface Agent { - - String getName(); - AgentMetadata getMetadata(); - List invoke(List chatHistory) throws AgentExecutionException; -} diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AgentExecutionException.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AgentExecutionException.java deleted file mode 100644 index 6b5cf5d..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AgentExecutionException.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.microsoft.langchain4j.agent; - -public class AgentExecutionException extends RuntimeException { - public AgentExecutionException(String message) { - super(message); - } - - public AgentExecutionException(String message, Throwable cause) { - super(message, cause); - } -} \ No newline at end of file diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AgentMetadata.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AgentMetadata.java deleted file mode 100644 index 26eb16f..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/AgentMetadata.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.microsoft.langchain4j.agent; - -import java.util.List; - -public record AgentMetadata(String description, List intents) { -} - diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPProtocolType.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPProtocolType.java deleted file mode 100644 index 5247ce1..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPProtocolType.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.microsoft.langchain4j.agent.mcp; - -public enum MCPProtocolType { - SSE, - STDIO -} diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPServerMetadata.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPServerMetadata.java deleted file mode 100644 index 9bbdb96..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPServerMetadata.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.microsoft.langchain4j.agent.mcp; - -public record MCPServerMetadata(String serverName, String url, MCPProtocolType protocolType) { -} - diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPToolAgent.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPToolAgent.java deleted file mode 100644 index 69a3cc7..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/langchain4j/agent/mcp/MCPToolAgent.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.microsoft.langchain4j.agent.mcp; - -import com.microsoft.langchain4j.agent.AbstractReActAgent; - -import com.microsoft.langchain4j.agent.AgentExecutionException; -import dev.langchain4j.agent.tool.ToolExecutionRequest; -import dev.langchain4j.agent.tool.ToolSpecification; -import dev.langchain4j.data.message.ToolExecutionResultMessage; - -import dev.langchain4j.mcp.client.DefaultMcpClient; -import dev.langchain4j.mcp.client.McpClient; -import dev.langchain4j.mcp.client.transport.McpTransport; -import dev.langchain4j.mcp.client.transport.http.HttpMcpTransport; -import dev.langchain4j.model.chat.ChatLanguageModel; - -import dev.langchain4j.service.tool.ToolExecutor; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public abstract class MCPToolAgent extends AbstractReActAgent { - private static final Logger LOGGER = LoggerFactory.getLogger(MCPToolAgent.class); - - protected List toolSpecifications; - protected Map extendedExecutorMap; - protected List mcpClients; - protected Map tool2ClientMap; - - protected MCPToolAgent(ChatLanguageModel chatModel, List mcpServerMetadata) { - super(chatModel); - this.mcpClients = new ArrayList<>(); - this.tool2ClientMap = new HashMap<>(); - this.toolSpecifications = new ArrayList<>(); - this.extendedExecutorMap = new HashMap<>(); - - mcpServerMetadata.forEach(metadata -> { - //only SSE is supported - if(metadata.protocolType().equals(MCPProtocolType.SSE)){ - McpTransport transport = new HttpMcpTransport.Builder() - .sseUrl(metadata.url()) - .logRequests(true) // if you want to see the traffic in the log - .logResponses(true) - .timeout(Duration.ofHours(3)) - .build(); - - McpClient mcpClient = new DefaultMcpClient.Builder() - .transport(transport) - .build(); - mcpClient - .listTools() - .forEach(toolSpecification -> { - this.tool2ClientMap.put(toolSpecification.name(),mcpClient); - this.toolSpecifications.add(toolSpecification); - } - ); - this.mcpClients.add(mcpClient); - - } - - }); - - } - - @Override - protected List getToolSpecifications() { - return this.toolSpecifications; - } - - - @Override - protected ToolExecutor getToolExecutor(String toolName) { - throw new AgentExecutionException("getToolExecutor not required when using MCP. if you landed here please review your agent code"); - } - - protected List executeToolRequests(List toolExecutionRequests) { - List toolExecutionResultMessages = new ArrayList<>(); - for (ToolExecutionRequest toolExecutionRequest : toolExecutionRequests) { - - String result = "ko"; - - // try first the extended executors - var toolExecutor = extendedExecutorMap.get(toolExecutionRequest.name()); - if( toolExecutor != null){ - LOGGER.info("Executing {} with params {}", toolExecutionRequest.name(), toolExecutionRequest.arguments()); - result = toolExecutor.execute(toolExecutionRequest,null); - LOGGER.info("Response from {}: {}", toolExecutionRequest.name(), result); - - }else{ - var mcpClient = tool2ClientMap.get(toolExecutionRequest.name()); - if (mcpClient == null) { - throw new IllegalArgumentException("No MCP executor found for tool name: " + toolExecutionRequest.name()); - } - LOGGER.info("Executing {} with params {}", toolExecutionRequest.name(), toolExecutionRequest.arguments()); - result = mcpClient.executeTool(toolExecutionRequest); - LOGGER.info("Response from {}: {}", toolExecutionRequest.name(), result); - } - - if (result == null || result.isEmpty()) { - LOGGER.warn("Tool {} returned empty result but successfully completed. Setting result=ok.", toolExecutionRequest.name()); - result = "ok"; - } - toolExecutionResultMessages.add(ToolExecutionResultMessage.from(toolExecutionRequest, result)); - } - return toolExecutionResultMessages; - } -} \ No newline at end of file diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/SupervisorAgent.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/SupervisorAgent.java deleted file mode 100644 index 1616899..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/SupervisorAgent.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.microsoft.openai.samples.assistant.langchain4j.agent; - - -import com.microsoft.langchain4j.agent.Agent; -import com.microsoft.langchain4j.agent.AgentExecutionException; -import com.microsoft.langchain4j.agent.AgentMetadata; -import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.ToolExecutionResultMessage; -import dev.langchain4j.memory.ChatMemory; -import dev.langchain4j.memory.chat.MessageWindowChatMemory; -import dev.langchain4j.model.chat.ChatLanguageModel; -import dev.langchain4j.model.chat.request.ChatRequest; -import dev.langchain4j.model.input.Prompt; -import dev.langchain4j.model.input.PromptTemplate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class SupervisorAgent { - - private final Logger LOGGER = LoggerFactory.getLogger(SupervisorAgent.class); - private final ChatLanguageModel chatLanguageModel; - private final List agents; - private final Map agentsMetadata; - private final Prompt agentPrompt; - //When false only detect the next agent but doesn't route to it. It will answer with the agent name. - private Boolean routing = true; - - private final String SUPERVISOR_AGENT_SINGLETURN_SYSTEM_MESSAGE = """ - You are a banking customer support agent triaging conversation and select the best agent name that can solve the customer need. - Use the below list of agents metadata to select the best one for the customer request: - {{agentsMetadata}} - Answer only with the agent name. - if you are not able to select an agent answer with none. - """; - - public SupervisorAgent(ChatLanguageModel chatLanguageModel, List agents, Boolean routing) { - this.chatLanguageModel = chatLanguageModel; - this.agents = agents; - this.routing = routing; - - this.agentsMetadata = agents.stream() - .collect(Collectors.toMap(Agent::getName, Agent::getMetadata)); - - PromptTemplate promptTemplate = PromptTemplate.from(SUPERVISOR_AGENT_SINGLETURN_SYSTEM_MESSAGE); - agentPrompt =promptTemplate.apply(Map.of("agentsMetadata", this.agentsMetadata)); - - } - public SupervisorAgent(ChatLanguageModel chatLanguageModel, List agents) { - this(chatLanguageModel, agents, true); - } - - - public List invoke(List chatHistory) { - LOGGER.info("------------- SupervisorAgent -------------"); - - var internalChatMemory = buildInternalChat(chatHistory); - - ChatRequest request = ChatRequest.builder() - .messages(internalChatMemory.messages()) - .build(); - - AiMessage aiMessage = chatLanguageModel.chat(request).aiMessage(); - String nextAgent = aiMessage.text(); - LOGGER.info("Supervisor Agent handoff to [{}]", nextAgent); - - if (routing) { - return singleTurnRouting(nextAgent, chatHistory); - } - - return new ArrayList<>(); - } - - - protected List singleTurnRouting(String nextAgent, List chatHistory) { - if("none".equalsIgnoreCase(nextAgent)){ - LOGGER.info("Gracefully handle clarification.. "); - AiMessage clarificationMessage = AiMessage.builder(). - text(" I'm not sure about your request. Can you please clarify?") - .build(); - chatHistory.add(clarificationMessage); - return chatHistory; - } - - Agent agent = agents.stream() - .filter(a -> a.getName().equals(nextAgent)) - .findFirst() - .orElseThrow(() -> new AgentExecutionException("Agent not found: " + nextAgent)); - - return agent.invoke(chatHistory); - } - - - - private ChatMemory buildInternalChat(List chatHistory) { - //build a new chat memory to preserve order of messages otherwise the model hallucinate. - var internalChatMemory = MessageWindowChatMemory.builder() - .id("default") - .maxMessages(20) - .build(); - - internalChatMemory.add(dev.langchain4j.data.message.SystemMessage.from(agentPrompt.text())); - // filter out tool requests and tool execution results - chatHistory.stream() - .filter(chatMessage -> { - if (chatMessage instanceof ToolExecutionResultMessage) { - return false; - } - if (chatMessage instanceof AiMessage) { - return !((AiMessage) chatMessage).hasToolExecutionRequests(); - } - return true; - }) - .forEach(internalChatMemory::add); - return internalChatMemory; - } -} diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/AccountMCPAgent.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/AccountMCPAgent.java deleted file mode 100644 index 729d853..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/AccountMCPAgent.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.microsoft.openai.samples.assistant.langchain4j.agent.mcp; - -import com.microsoft.langchain4j.agent.AgentMetadata; -import com.microsoft.langchain4j.agent.mcp.MCPProtocolType; -import com.microsoft.langchain4j.agent.mcp.MCPServerMetadata; -import com.microsoft.langchain4j.agent.mcp.MCPToolAgent; -import dev.langchain4j.model.chat.ChatLanguageModel; -import dev.langchain4j.model.input.Prompt; -import dev.langchain4j.model.input.PromptTemplate; - -import java.util.List; -import java.util.Map; - -public class AccountMCPAgent extends MCPToolAgent { - - private final Prompt agentPrompt; - - private static final String ACCOUNT_AGENT_SYSTEM_MESSAGE = """ - you are a personal financial advisor who help the user to retrieve information about their bank accounts. - Use html list or table to display the account information. - Always use the below logged user details to retrieve account info: - '{{loggedUserName}}' - """; - - public AccountMCPAgent(ChatLanguageModel chatModel, String loggedUserName, String accountMCPServerUrl) { - super(chatModel, List.of(new MCPServerMetadata("account", accountMCPServerUrl, MCPProtocolType.SSE))); - - if (loggedUserName == null || loggedUserName.isEmpty()) { - throw new IllegalArgumentException("loggedUserName cannot be null or empty"); - } - - PromptTemplate promptTemplate = PromptTemplate.from(ACCOUNT_AGENT_SYSTEM_MESSAGE); - this.agentPrompt = promptTemplate.apply(Map.of("loggedUserName", loggedUserName)); - } - - @Override - public String getName() { - return "AccountAgent"; - } - - @Override - public AgentMetadata getMetadata() { - return new AgentMetadata( - "Personal financial advisor for retrieving bank account information.", - List.of("RetrieveAccountInfo", "DisplayAccountDetails") - ); - } - - @Override - protected String getSystemMessage() { - return agentPrompt.text(); - } - -} diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/PaymentMCPAgent.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/PaymentMCPAgent.java deleted file mode 100644 index 35f0d1c..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/PaymentMCPAgent.java +++ /dev/null @@ -1,128 +0,0 @@ -package com.microsoft.openai.samples.assistant.langchain4j.agent.mcp; - -import com.microsoft.langchain4j.agent.AgentExecutionException; -import com.microsoft.langchain4j.agent.AgentMetadata; -import com.microsoft.langchain4j.agent.mcp.MCPProtocolType; -import com.microsoft.langchain4j.agent.mcp.MCPServerMetadata; -import com.microsoft.langchain4j.agent.mcp.MCPToolAgent; -import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import com.microsoft.openai.samples.assistant.langchain4j.tools.InvoiceScanTool; -import dev.langchain4j.agent.tool.ToolSpecifications; -import dev.langchain4j.model.chat.ChatLanguageModel; -import dev.langchain4j.model.input.Prompt; -import dev.langchain4j.model.input.PromptTemplate; -import dev.langchain4j.service.tool.DefaultToolExecutor; - -import java.lang.reflect.Method; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.List; -import java.util.Map; - -public class PaymentMCPAgent extends MCPToolAgent { - - private final Prompt agentPrompt; - - private static final String PAYMENT_AGENT_SYSTEM_MESSAGE = """ - you are a personal financial advisor who help the user with their recurrent bill payments. The user may want to pay the bill uploading a photo of the bill, or it may start the payment checking transactions history for a specific payee. - For the bill payment you need to know the: bill id or invoice number, payee name, the total amount. - If you don't have enough information to pay the bill ask the user to provide the missing information. - If the user submit a photo, always ask the user to confirm the extracted data from the photo. - Always check if the bill has been paid already based on payment history before asking to execute the bill payment. - Ask for the payment method to use based on the available methods on the user account. - if the user wants to pay using bank transfer, check if the payee is in account registered beneficiaries list. If not ask the user to provide the payee bank code. - Check if the payment method selected by the user has enough funds to pay the bill. Don't use the account balance to evaluate the funds. - Before submitting the payment to the system ask the user confirmation providing the payment details. - Include in the payment description the invoice id or bill id as following: payment for invoice 1527248. - When submitting payment always use the available functions to retrieve accountId, paymentMethodId. - If the payment succeeds provide the user with the payment confirmation. If not provide the user with the error message. - Use HTML list or table to display bill extracted data, payments, account or transaction details. - Always use the below logged user details to retrieve account info: - '{{loggedUserName}}' - Current timestamp: - '{{currentDateTime}}' - Don't try to guess accountId,paymentMethodId from the conversation.When submitting payment always use functions to retrieve accountId, paymentMethodId. - - ### Output format - - Example of showing Payment information: - - - - - - - - - - - - - - - - - - - - - -
Payee Namecontoso
Invoice ID9524011000817857
Amount€85.20
Payment MethodVisa (Card Number: ***477)
DescriptionPayment for invoice 9524011000817857
- - - Example of showing Payment methods: -

    -
  1. Bank Transfer
    2. -
  2. Visa (Card Number: ***3667)
    3. -
- - """; - - public PaymentMCPAgent(ChatLanguageModel chatModel, DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper, String loggedUserName, String transactionMCPServerURL, String accountMCPServerUrl, String paymentsMCPServerUrl) { - super(chatModel, List.of(new MCPServerMetadata("payment", paymentsMCPServerUrl, MCPProtocolType.SSE), - new MCPServerMetadata("transaction", transactionMCPServerURL, MCPProtocolType.SSE), - new MCPServerMetadata("account", accountMCPServerUrl, MCPProtocolType.SSE))); - - if (loggedUserName == null || loggedUserName.isEmpty()) { - throw new IllegalArgumentException("loggedUserName cannot be null or empty"); - } - - extendToolMap(documentIntelligenceInvoiceScanHelper); - - PromptTemplate promptTemplate = PromptTemplate.from(PAYMENT_AGENT_SYSTEM_MESSAGE); - var datetimeIso8601 = ZonedDateTime.now(ZoneId.of("UTC")).toInstant().toString(); - - this.agentPrompt = promptTemplate.apply(Map.of( - "loggedUserName", loggedUserName, - "currentDateTime", datetimeIso8601 - )); - } - - @Override - public String getName() { - return "PaymentAgent"; - } - - @Override - public AgentMetadata getMetadata() { - return new AgentMetadata( - "Personal financial advisor for submitting payment request.", - List.of("RetrievePaymentInfo", "DisplayPaymentDetails", "SubmitPayment") - ); - } - - @Override - protected String getSystemMessage() { - return agentPrompt.text(); - } - - protected void extendToolMap(DocumentIntelligenceInvoiceScanHelper documentIntelligenceInvoiceScanHelper) { - try { - Method scanInvoiceMethod = InvoiceScanTool.class.getMethod("scanInvoice", String.class); - InvoiceScanTool invoiceScanTool = new InvoiceScanTool(documentIntelligenceInvoiceScanHelper); - - this.toolSpecifications.addAll(ToolSpecifications.toolSpecificationsFrom(InvoiceScanTool.class)); - this.extendedExecutorMap.put("scanInvoice", new DefaultToolExecutor(invoiceScanTool, scanInvoiceMethod)); - } catch (NoSuchMethodException e) { - throw new AgentExecutionException("scanInvoice method not found in InvoiceScanTool class. Align class code to be used by Payment Agent", e); - } - } -} \ No newline at end of file diff --git a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/TransactionHistoryMCPAgent.java b/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/TransactionHistoryMCPAgent.java deleted file mode 100644 index 9f97d3c..0000000 --- a/app/copilot/langchain4j-agents/src/main/java/com/microsoft/openai/samples/assistant/langchain4j/agent/mcp/TransactionHistoryMCPAgent.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.microsoft.openai.samples.assistant.langchain4j.agent.mcp; - -import com.microsoft.langchain4j.agent.AgentMetadata; -import com.microsoft.langchain4j.agent.mcp.MCPProtocolType; -import com.microsoft.langchain4j.agent.mcp.MCPServerMetadata; -import com.microsoft.langchain4j.agent.mcp.MCPToolAgent; -import dev.langchain4j.model.chat.ChatLanguageModel; -import dev.langchain4j.model.input.Prompt; -import dev.langchain4j.model.input.PromptTemplate; - -import java.util.List; -import java.util.Map; - -public class TransactionHistoryMCPAgent extends MCPToolAgent { - - private final Prompt agentPrompt; - - private static final String TRANSACTION_HISTORY_AGENT_SYSTEM_MESSAGE = """ - you are a personal financial advisor who help the user with their recurrent bill payments. To search about the payments history you need to know the payee name and the account id. - If the user doesn't provide the payee name, search the last 10 transactions order by date. - If the user want to search last transactions for a specific payee, ask to provide the payee name. - Use html list or table to display the transaction information. - Always use the below logged user details to retrieve account info: - '{{loggedUserName}}' - Current timestamp: - '{{currentDateTime}}' - """; - - public TransactionHistoryMCPAgent(ChatLanguageModel chatModel, String loggedUserName, String transactionMCPServerUrl, String accountMCPServerUrl) { - super(chatModel, List.of(new MCPServerMetadata("transaction-history", transactionMCPServerUrl, MCPProtocolType.SSE), - new MCPServerMetadata("account", accountMCPServerUrl, MCPProtocolType.SSE))); - - if (loggedUserName == null || loggedUserName.isEmpty()) { - throw new IllegalArgumentException("loggedUserName cannot be null or empty"); - } - - PromptTemplate promptTemplate = PromptTemplate.from(TRANSACTION_HISTORY_AGENT_SYSTEM_MESSAGE); - var datetimeIso8601 = java.time.ZonedDateTime.now(java.time.ZoneId.of("UTC")).toInstant().toString(); - - this.agentPrompt = promptTemplate.apply(Map.of( - "loggedUserName", loggedUserName, - "currentDateTime", datetimeIso8601 - )); - } - - @Override - public String getName() { - return "TransactionHistoryAgent"; - } - - @Override - public AgentMetadata getMetadata() { - return new AgentMetadata( - "Personal financial advisor for retrieving transaction history information.", - List.of("RetrieveTransactionHistory", "DisplayTransactionDetails") - ); - } - - @Override - protected String getSystemMessage() { - return agentPrompt.text(); - } - -} \ No newline at end of file diff --git a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/AccountMCPAgentIntegrationTest.java b/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/AccountMCPAgentIntegrationTest.java deleted file mode 100644 index 25d9977..0000000 --- a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/AccountMCPAgentIntegrationTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package dev.langchain4j.openapi.mcp; - -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.AccountMCPAgent; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.model.azure.AzureOpenAiChatModel; - -import java.util.ArrayList; - -public class AccountMCPAgentIntegrationTest { - - public static void main(String[] args) throws Exception { - - //Azure Open AI Chat Model - var azureOpenAiChatModel = AzureOpenAiChatModel.builder() - .apiKey(System.getenv("AZURE_OPENAI_KEY")) - .endpoint(System.getenv("AZURE_OPENAI_ENDPOINT")) - .deploymentName(System.getenv("AZURE_OPENAI_DEPLOYMENT_NAME")) - .temperature(0.3) - .logRequestsAndResponses(true) - .build(); - - var accountAgent = new AccountMCPAgent(azureOpenAiChatModel,"bob.user@contoso.com","http://localhost:8070/sse"); - - var chatHistory = new ArrayList(); - chatHistory.add(UserMessage.from("How much money do I have in my account?")); - - accountAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - chatHistory.add(UserMessage.from("what about my visa")); - accountAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - } -} diff --git a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/PaymentMCPAgentIntegrationTest.java b/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/PaymentMCPAgentIntegrationTest.java deleted file mode 100644 index b18e4eb..0000000 --- a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/PaymentMCPAgentIntegrationTest.java +++ /dev/null @@ -1,73 +0,0 @@ -package dev.langchain4j.openapi.mcp; - -import com.azure.ai.documentintelligence.DocumentIntelligenceClient; -import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder; -import com.azure.identity.AzureCliCredentialBuilder; -import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.PaymentMCPAgent; -import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.model.azure.AzureOpenAiChatModel; - -import java.util.ArrayList; - -public class PaymentMCPAgentIntegrationTest { - - public static void main(String[] args) throws Exception { - - //Azure Open AI Chat Model - var azureOpenAiChatModel = AzureOpenAiChatModel.builder() - .apiKey(System.getenv("AZURE_OPENAI_KEY")) - .endpoint(System.getenv("AZURE_OPENAI_ENDPOINT")) - .deploymentName(System.getenv("AZURE_OPENAI_DEPLOYMENT_NAME")) - .temperature(0.3) - .logRequestsAndResponses(true) - .build(); - - var documentIntelligenceInvoiceScanHelper = new DocumentIntelligenceInvoiceScanHelper(getDocumentIntelligenceClient(),getBlobStorageProxyClient()); - - var paymentAgent = new PaymentMCPAgent(azureOpenAiChatModel, - documentIntelligenceInvoiceScanHelper, - "bob.user@contoso.com", - "http://localhost:8090/sse", - "http://localhost:8070/sse", - "http://localhost:8060/sse"); - - var chatHistory = new ArrayList(); - chatHistory.add(UserMessage.from("Please pay the bill: bill id 1234, payee name contoso, total amount 30.")); - - - paymentAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - - chatHistory.add(UserMessage.from("use my visa")); - paymentAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - - chatHistory.add(UserMessage.from("yes please proceed with payment")); - paymentAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - - - } - - private static BlobStorageProxy getBlobStorageProxyClient() { - - String storageAccountService = "https://%s.blob.core.windows.net".formatted(System.getenv("AZURE_STORAGE_ACCOUNT")); - String containerName = "content"; - return new BlobStorageProxy(storageAccountService,containerName,new AzureCliCredentialBuilder().build()); - } - - private static DocumentIntelligenceClient getDocumentIntelligenceClient() { - String endpoint = "https://%s.cognitiveservices.azure.com".formatted(System.getenv("AZURE_STORAGE_ACCOUNT")); - - return new DocumentIntelligenceClientBuilder() - .credential(new AzureCliCredentialBuilder().build()) - .endpoint(endpoint) - .buildClient(); - } -} diff --git a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/PaymentMCPAgentIntegrationWithImageTest.java b/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/PaymentMCPAgentIntegrationWithImageTest.java deleted file mode 100644 index 14cd6a4..0000000 --- a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/PaymentMCPAgentIntegrationWithImageTest.java +++ /dev/null @@ -1,77 +0,0 @@ -package dev.langchain4j.openapi.mcp; - -import com.azure.ai.documentintelligence.DocumentIntelligenceClient; -import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder; -import com.azure.identity.AzureCliCredentialBuilder; -import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.PaymentMCPAgent; -import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.model.azure.AzureOpenAiChatModel; - -import java.util.ArrayList; - -public class PaymentMCPAgentIntegrationWithImageTest { - - public static void main(String[] args) throws Exception { - - //Azure Open AI Chat Model - var azureOpenAiChatModel = AzureOpenAiChatModel.builder() - .apiKey(System.getenv("AZURE_OPENAI_KEY")) - .endpoint(System.getenv("AZURE_OPENAI_ENDPOINT")) - .deploymentName(System.getenv("AZURE_OPENAI_DEPLOYMENT_NAME")) - .temperature(0.3) - .logRequestsAndResponses(true) - .build(); - - var documentIntelligenceInvoiceScanHelper = new DocumentIntelligenceInvoiceScanHelper(getDocumentIntelligenceClient(),getBlobStorageProxyClient()); - - var paymentAgent = new PaymentMCPAgent(azureOpenAiChatModel, - documentIntelligenceInvoiceScanHelper, - "bob.user@contoso.com", - "http://localhost:8090", - "http://localhost:8070", - "http://localhost:8060"); - - var chatHistory = new ArrayList(); - chatHistory.add(UserMessage.from("Please pay this bill gori.png")); - - //this flow should activate the scanInvoice tool - - paymentAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - chatHistory.add(UserMessage.from("yep, they are correct")); - paymentAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - - chatHistory.add(UserMessage.from("use my visa")); - paymentAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - - chatHistory.add(UserMessage.from("yes please proceed with payment")); - paymentAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - - - } - - private static BlobStorageProxy getBlobStorageProxyClient() { - - String containerName = "content"; - return new BlobStorageProxy(System.getenv("AZURE_STORAGE_ACCOUNT"),containerName,new AzureCliCredentialBuilder().build()); - } - - private static DocumentIntelligenceClient getDocumentIntelligenceClient() { - String endpoint = "https://%s.cognitiveservices.azure.com".formatted(System.getenv("AZURE_DOCUMENT_INTELLIGENCE_SERVICE")); - - return new DocumentIntelligenceClientBuilder() - .credential(new AzureCliCredentialBuilder().build()) - .endpoint(endpoint) - .buildClient(); - } -} diff --git a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentLongConversationIntegrationTest.java b/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentLongConversationIntegrationTest.java deleted file mode 100644 index 7ccdc1a..0000000 --- a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentLongConversationIntegrationTest.java +++ /dev/null @@ -1,105 +0,0 @@ -package dev.langchain4j.openapi.mcp; - -import com.azure.ai.documentintelligence.DocumentIntelligenceClient; -import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder; -import com.azure.identity.AzureCliCredentialBuilder; -import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import com.microsoft.openai.samples.assistant.langchain4j.agent.SupervisorAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.AccountMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.PaymentMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.TransactionHistoryMCPAgent; -import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.model.azure.AzureOpenAiChatModel; - -import java.util.ArrayList; -import java.util.List; - -public class SupervisorAgentLongConversationIntegrationTest { - - public static void main(String[] args) throws Exception { - - //Azure Open AI Chat Model - var azureOpenAiChatModel = AzureOpenAiChatModel.builder() - .apiKey(System.getenv("AZURE_OPENAI_KEY")) - .endpoint(System.getenv("AZURE_OPENAI_ENDPOINT")) - .deploymentName(System.getenv("AZURE_OPENAI_DEPLOYMENT_NAME")) - .temperature(0.3) - .logRequestsAndResponses(true) - .build(); - - var documentIntelligenceInvoiceScanHelper = new DocumentIntelligenceInvoiceScanHelper(getDocumentIntelligenceClient(),getBlobStorageProxyClient()); - - var accountAgent = new AccountMCPAgent(azureOpenAiChatModel, - "bob.user@contoso.com", - "http://localhost:8070/sse"); - var transactionHistoryAgent = new TransactionHistoryMCPAgent(azureOpenAiChatModel, - "bob.user@contoso.com", - "http://localhost:8090/sse", - "http://localhost:8070/sse"); - var paymentAgent = new PaymentMCPAgent(azureOpenAiChatModel, - documentIntelligenceInvoiceScanHelper, - "bob.user@contoso.com", - "http://localhost:8090/sse", - "http://localhost:8070/sse", - "http://localhost:8060/sse"); - - var supervisorAgent = new SupervisorAgent(azureOpenAiChatModel, List.of(accountAgent,transactionHistoryAgent,paymentAgent)); - var chatHistory = new ArrayList(); - - - chatHistory.add(UserMessage.from("How much money do I have in my account?")); - System.out.println(chatHistory.get(chatHistory.size()-1)); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - chatHistory.add(UserMessage.from("what about my visa")); - System.out.println(chatHistory.get(chatHistory.size()-1)); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - chatHistory.add(UserMessage.from("When was last time I've paid contoso?")); - System.out.println(chatHistory.get(chatHistory.size()-1)); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - chatHistory.add(UserMessage.from("Please pay this bill gori.png")); - - //this flow should activate the scanInvoice tool - System.out.println(chatHistory.get(chatHistory.size()-1)); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - chatHistory.add(UserMessage.from("yep, they are correct")); - System.out.println(chatHistory.get(chatHistory.size()-1)); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - - chatHistory.add(UserMessage.from("use my visa")); - System.out.println(chatHistory.get(chatHistory.size()-1)); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - - chatHistory.add(UserMessage.from("yes please proceed with payment")); - System.out.println(chatHistory.get(chatHistory.size()-1)); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - } - private static BlobStorageProxy getBlobStorageProxyClient() { - - String containerName = "content"; - return new BlobStorageProxy(System.getenv("AZURE_STORAGE_ACCOUNT"),containerName,new AzureCliCredentialBuilder().build()); - } - - private static DocumentIntelligenceClient getDocumentIntelligenceClient() { - String endpoint = "https://%s.cognitiveservices.azure.com".formatted(System.getenv("AZURE_DOCUMENT_INTELLIGENCE_SERVICE")); - - return new DocumentIntelligenceClientBuilder() - .credential(new AzureCliCredentialBuilder().build()) - .endpoint(endpoint) - .buildClient(); - } -} diff --git a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentNoRoutingIntegrationTest.java b/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentNoRoutingIntegrationTest.java deleted file mode 100644 index db0cebc..0000000 --- a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentNoRoutingIntegrationTest.java +++ /dev/null @@ -1,83 +0,0 @@ -package dev.langchain4j.openapi.mcp; - -import com.azure.ai.documentintelligence.DocumentIntelligenceClient; -import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder; -import com.azure.identity.AzureCliCredentialBuilder; -import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import com.microsoft.openai.samples.assistant.langchain4j.agent.SupervisorAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.AccountMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.PaymentMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.TransactionHistoryMCPAgent; -import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.model.azure.AzureOpenAiChatModel; - -import java.util.ArrayList; -import java.util.List; - -public class SupervisorAgentNoRoutingIntegrationTest { - - public static void main(String[] args) throws Exception { - - //Azure Open AI Chat Model - var azureOpenAiChatModel = AzureOpenAiChatModel.builder() - .apiKey(System.getenv("AZURE_OPENAI_KEY")) - .endpoint(System.getenv("AZURE_OPENAI_ENDPOINT")) - .deploymentName(System.getenv("AZURE_OPENAI_DEPLOYMENT_NAME")) - .temperature(0.3) - .logRequestsAndResponses(true) - .build(); - - var documentIntelligenceInvoiceScanHelper = new DocumentIntelligenceInvoiceScanHelper(getDocumentIntelligenceClient(),getBlobStorageProxyClient()); - - var accountAgent = new AccountMCPAgent(azureOpenAiChatModel,"bob.user@contoso.com/sse","http://localhost:8070/sse"); - var transactionHistoryAgent = new TransactionHistoryMCPAgent(azureOpenAiChatModel, - "bob.user@contoso.com", - "http://localhost:8090/sse", - "http://localhost:8070/sse"); - var paymentAgent = new PaymentMCPAgent(azureOpenAiChatModel, - documentIntelligenceInvoiceScanHelper, - "bob.user@contoso.com", - "http://localhost:8090/sse", - "http://localhost:8070/sse", - "http://localhost:8060/sse"); - - var supervisorAgent = new SupervisorAgent(azureOpenAiChatModel, List.of(accountAgent,transactionHistoryAgent,paymentAgent), false); - var chatHistory = new ArrayList(); - - chatHistory.add(UserMessage.from("How much money do I have in my account?")); - supervisorAgent.invoke(chatHistory); - - chatHistory.add(UserMessage.from("you have 1000 on your account")); - - chatHistory.add(UserMessage.from("what about my visa")); - supervisorAgent.invoke(chatHistory); - chatHistory.add(UserMessage.from("these are the data for your visa card: id 1717171, expiration date 12/2023, cvv 123 balance 500")); - - chatHistory.add(UserMessage.from("When was last time I've paid contoso?")); - supervisorAgent.invoke(chatHistory); - - chatHistory.add(UserMessage.from("Can you help me plan an investement?")); - supervisorAgent.invoke(chatHistory); - - chatHistory.add(UserMessage.from("Ok so can you pay this bill for me?")); - supervisorAgent.invoke(chatHistory); - - } - - private static BlobStorageProxy getBlobStorageProxyClient() { - - String containerName = "content"; - return new BlobStorageProxy(System.getenv("AZURE_STORAGE_ACCOUNT"),containerName,new AzureCliCredentialBuilder().build()); - } - - private static DocumentIntelligenceClient getDocumentIntelligenceClient() { - String endpoint = "https://%s.cognitiveservices.azure.com".formatted(System.getenv("AZURE_DOCUMENT_INTELLIGENCE_SERVICE")); - - return new DocumentIntelligenceClientBuilder() - .credential(new AzureCliCredentialBuilder().build()) - .endpoint(endpoint) - .buildClient(); - } -} diff --git a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentRoutingIntegrationTest.java b/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentRoutingIntegrationTest.java deleted file mode 100644 index 3bef60d..0000000 --- a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/SupervisorAgentRoutingIntegrationTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package dev.langchain4j.openapi.mcp; - -import com.azure.ai.documentintelligence.DocumentIntelligenceClient; -import com.azure.ai.documentintelligence.DocumentIntelligenceClientBuilder; -import com.azure.identity.AzureCliCredentialBuilder; -import com.microsoft.openai.samples.assistant.invoice.DocumentIntelligenceInvoiceScanHelper; -import com.microsoft.openai.samples.assistant.langchain4j.agent.SupervisorAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.AccountMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.PaymentMCPAgent; -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.TransactionHistoryMCPAgent; -import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.model.azure.AzureOpenAiChatModel; - -import java.util.ArrayList; -import java.util.List; - -public class SupervisorAgentRoutingIntegrationTest { - - public static void main(String[] args) throws Exception { - - //Azure Open AI Chat Model - var azureOpenAiChatModel = AzureOpenAiChatModel.builder() - .apiKey(System.getenv("AZURE_OPENAI_KEY")) - .endpoint(System.getenv("AZURE_OPENAI_ENDPOINT")) - .deploymentName(System.getenv("AZURE_OPENAI_DEPLOYMENT_NAME")) - .temperature(0.3) - .logRequestsAndResponses(true) - .build(); - - var documentIntelligenceInvoiceScanHelper = new DocumentIntelligenceInvoiceScanHelper(getDocumentIntelligenceClient(),getBlobStorageProxyClient()); - - var accountAgent = new AccountMCPAgent(azureOpenAiChatModel,"bob.user@contoso.com", - "http://localhost:8070/sse"); - var transactionHistoryAgent = new TransactionHistoryMCPAgent(azureOpenAiChatModel, - "bob.user@contoso.com", - "http://localhost:8090/sse", - "http://localhost:8070/sse"); - var paymentAgent = new PaymentMCPAgent(azureOpenAiChatModel, - documentIntelligenceInvoiceScanHelper, - "bob.user@contoso.com", - "http://localhost:8090/sse", - "http://localhost:8070/sse", - "http://localhost:8060/sse"); - - var supervisorAgent = new SupervisorAgent(azureOpenAiChatModel, List.of(accountAgent,transactionHistoryAgent,paymentAgent)); - var chatHistory = new ArrayList(); - - - chatHistory.add(UserMessage.from("How much money do I have in my account?")); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - chatHistory.add(UserMessage.from("what about my visa")); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - chatHistory.add(UserMessage.from("When was las time I've paid contoso?")); - supervisorAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - } - - private static BlobStorageProxy getBlobStorageProxyClient() { - - String containerName = "content"; - return new BlobStorageProxy(System.getenv("AZURE_STORAGE_ACCOUNT"),containerName,new AzureCliCredentialBuilder().build()); - } - - private static DocumentIntelligenceClient getDocumentIntelligenceClient() { - String endpoint = "https://%s.cognitiveservices.azure.com".formatted(System.getenv("AZURE_DOCUMENT_INTELLIGENCE_SERVICE")); - - return new DocumentIntelligenceClientBuilder() - .credential(new AzureCliCredentialBuilder().build()) - .endpoint(endpoint) - .buildClient(); - } -} diff --git a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/TransactionHistoryMCPAgentIntegrationTest.java b/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/TransactionHistoryMCPAgentIntegrationTest.java deleted file mode 100644 index aa41b94..0000000 --- a/app/copilot/langchain4j-agents/src/test/java/dev/langchain4j/openapi/mcp/TransactionHistoryMCPAgentIntegrationTest.java +++ /dev/null @@ -1,37 +0,0 @@ -package dev.langchain4j.openapi.mcp; - -import com.microsoft.openai.samples.assistant.langchain4j.agent.mcp.TransactionHistoryMCPAgent; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.model.azure.AzureOpenAiChatModel; - -import java.util.ArrayList; - -public class TransactionHistoryMCPAgentIntegrationTest { - - public static void main(String[] args) throws Exception { - - //Azure Open AI Chat Model - var azureOpenAiChatModel = AzureOpenAiChatModel.builder() - .apiKey(System.getenv("AZURE_OPENAI_KEY")) - .endpoint(System.getenv("AZURE_OPENAI_ENDPOINT")) - .deploymentName(System.getenv("AZURE_OPENAI_DEPLOYMENT_NAME")) - .temperature(0.3) - .logRequestsAndResponses(true) - .build(); - - var transactionHistoryAgent = new TransactionHistoryMCPAgent(azureOpenAiChatModel, - "bob.user@contoso.com", - "http://localhost:8090/sse", - "http://localhost:8070/sse"); - - var chatHistory = new ArrayList(); - - - chatHistory.add(UserMessage.from("When was last time I've paid contoso?")); - transactionHistoryAgent.invoke(chatHistory); - System.out.println(chatHistory.get(chatHistory.size()-1)); - - - } -} diff --git a/app/copilot/langchain4j-agents/src/test/resources/account.yaml b/app/copilot/langchain4j-agents/src/test/resources/account.yaml deleted file mode 100644 index 0708a9c..0000000 --- a/app/copilot/langchain4j-agents/src/test/resources/account.yaml +++ /dev/null @@ -1,178 +0,0 @@ -openapi: 3.0.3 -info: - title: Account API - version: 1.0.0 -paths: - /users/{user_name}/accounts: - get: - summary: Get the list of all accounts for a specific user - description: Get the list of all accounts for a specific user - operationId: getAccountsByUserName - parameters: - - name: user_name - description: userName once the user has logged. - in: path - required: true - schema: - type: string - responses: - '200': - description: A list of accounts - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/Account' - /accounts/{accountid}: - get: - summary: Get account details and available payment methods - description: Get account details and available payment methods - operationId: getAccountDetails - parameters: - - name: accountid - description: id of specific account. - in: path - required: true - schema: - type: integer - example: 123456 - responses: - '200': - description: Account details - content: - application/json: - schema: - type: object - items: - $ref: '#/components/schemas/Account' - /accounts/{accountid}/paymentmethods/{methodid}: - get: - summary: Get payment method detail with available balance - description: Get payment method detail with available balance - operationId: getPaymentMethodDetails - parameters: - - name: accountid - description: id of specific account. - in: path - required: true - schema: - type: integer - example: 123456 - - name: methodid - description: id of specific payment method available for the account id. - in: path - required: true - schema: - type: integer - example: 74839113 - responses: - '200': - description: Payment method details - content: - application/json: - schema: - type: object - items: - $ref: '#/components/schemas/PaymentMethod' - /accounts/{accountid}/registeredBeneficiaries: - get: - summary: Get list of registered beneficiaries for a specific account - description: Get list of registered beneficiaries for a specific account - operationId: getBeneficiaryMethodDetails - parameters: - - name: accountid - description: id of specific account. - in: path - required: true - schema: - type: integer - example: 123456 - responses: - '200': - description: Payment method details - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/Beneficiary' -components: - schemas: - Account: - type: object - properties: - id: - type: string - description: The unique identifier for the account - userName: - type: string - description: The unique identifier for the user - accountHolderFullName: - type: string - description: The full name of the account holder - currency: - type: string - description: The currency of the account - activationDate: - type: string - description: The date when the account was activated - balance: - type: string - description: The current balance of the account - paymentMethods: - type: array - items: - $ref: '#/components/schemas/PaymentMethod' - description: The list of payment methods associated with the account - PaymentMethodSummary: - type: object - properties: - id: - type: string - description: The unique identifier for the payment method - type: - type: string - description: The type of the payment method - activationDate: - type: string - description: The date when the payment method was activated - expirationDate: - type: string - description: The date when the payment method will expire - PaymentMethod: - type: object - properties: - id: - type: string - description: The unique identifier for the payment method - type: - type: string - description: The type of the payment method - activationDate: - type: string - description: The date when the payment method was activated - expirationDate: - type: string - description: The date when the payment method will expire - availableBalance: - type: string - description: The available balance of the payment method - cardNumber: - type: string - description: The card number of the payment method - Beneficiary: - type: object - properties: - id: - type: string - description: The unique identifier for the beneficiary - fullName: - type: string - description: The full name of the beneficiary - bankCode: - type: string - description: The bank code of the beneficiary's bank - bankName: - type: string - description: The name of the beneficiary's bank \ No newline at end of file diff --git a/app/copilot/langchain4j-agents/src/test/resources/logback.xml b/app/copilot/langchain4j-agents/src/test/resources/logback.xml deleted file mode 100644 index 4447925..0000000 --- a/app/copilot/langchain4j-agents/src/test/resources/logback.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n - - - - - - - - - \ No newline at end of file diff --git a/app/copilot/langchain4j-agents/src/test/resources/payments.yaml b/app/copilot/langchain4j-agents/src/test/resources/payments.yaml deleted file mode 100644 index 49ab21b..0000000 --- a/app/copilot/langchain4j-agents/src/test/resources/payments.yaml +++ /dev/null @@ -1,60 +0,0 @@ -openapi: 3.0.3 -info: - title: Payment API - version: 1.0.0 -paths: - /payments: - post: - operationId: submitPayment - summary: Submit a payment request - description: Submit a payment request - requestBody: - required: true - description: Payment to submit - content: - application/json: - schema: - $ref: '#/components/schemas/Payment' - responses: - '200': - description: Payment request submitted successfully - '400': - description: Invalid request body - '500': - description: Internal server error -components: - schemas: - Payment: - type: object - properties: - description: - type: string - description: Description of the payment - recipientName: - type: string - description: Name of the recipient - recipientBankCode: - type: string - description: Bank code of the recipient - accountId: - type: string - description: ID of the account - paymentMethodId: - type: string - description: ID of the payment method - paymentType: - type: string - description: 'The type of payment: creditcard, banktransfer, directdebit, visa, mastercard, paypal, etc.' - amount: - type: string - description: Amount of the payment - timestamp: - type: string - description: Timestamp of the payment - requestBodies: - Payment: - content: - application/json: - schema: - $ref: '#/components/schemas/Payment' - description: Payment object to submit \ No newline at end of file diff --git a/app/copilot/langchain4j-agents/src/test/resources/transaction-history.yaml b/app/copilot/langchain4j-agents/src/test/resources/transaction-history.yaml deleted file mode 100644 index 200771f..0000000 --- a/app/copilot/langchain4j-agents/src/test/resources/transaction-history.yaml +++ /dev/null @@ -1,94 +0,0 @@ -openapi: 3.0.3 -info: - title: Transaction History and Reporting API - version: 1.0.0 -paths: - /transactions/{accountid}: - get: - summary: Get transactions list. - description: Gets the transactions lists. They can be filtered based on recipient name - operationId: getTransactionsByRecipientName - parameters: - - name: accountid - description: id of specific account. - in: path - required: true - schema: - type: integer - example: 123456 - - name: recipient_name - description: Name of the payee, recipient - in: query - required: false - schema: - type: string - example: contoso - - responses: - '200': - description: A list of transactions for a specific recipient - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/Transaction' - post: - operationId: notifyTransaction - summary: Notify the banking transaction so that it's being stored in the history - description: Notify the banking transaction so that it's being stored in the history - parameters: - - name: accountid - description: id of specific account. - in: path - required: true - schema: - type: integer - example: 123456 - requestBody: - required: true - description: transaction to notify - content: - application/json: - schema: - $ref: '#/components/schemas/Transaction' - responses: - '200': - description: Payment request submitted successfully - '400': - description: Invalid request body - '500': - description: Internal server error -components: - schemas: - Transaction: - type: object - properties: - id: - type: string - description: 'The unique identifier for the transaction' - description: - type: string - description: 'The description of the transaction which contains reason for the payment and other details' - type: - type: string - description: 'The transaction type expressed as income or outcome transaction' - recipientName: - type: string - description: 'The name of the recipient' - recipientBankReference: - type: string - description: 'The bank reference of the recipient' - accountId: - type: string - description: 'The account ID associated with the transaction' - paymentType: - type: string - description: 'The type of payment: creditcard, banktransfer, directdebit, visa, mastercard, paypal, etc.' - amount: - type: string - description: 'The amount of the transaction' - timestamp: - type: string - format: date-time - description: 'The timestamp of the transaction' \ No newline at end of file diff --git a/app/copilot/pom.xml b/app/copilot/pom.xml index 0279d82..45d48c3 100644 --- a/app/copilot/pom.xml +++ b/app/copilot/pom.xml @@ -13,7 +13,7 @@ 17 17 UTF-8 - 1.0.0-beta2 + 1.1.0 1.2.33 @@ -44,7 +44,6 @@ copilot-backend - langchain4j-agents copilot-common diff --git a/app/frontend/src/pages/chat/Chat.tsx b/app/frontend/src/pages/chat/Chat.tsx index e4abe90..2b48b8d 100644 --- a/app/frontend/src/pages/chat/Chat.tsx +++ b/app/frontend/src/pages/chat/Chat.tsx @@ -45,6 +45,7 @@ const Chat = () => { const [useSuggestFollowupQuestions, setUseSuggestFollowupQuestions] = useState(false); const [useOidSecurityFilter, setUseOidSecurityFilter] = useState(false); const [useGroupsSecurityFilter, setUseGroupsSecurityFilter] = useState(false); + const [threadId, setThreadId] = useState(undefined); // Updated type to match ChatAppRequest const lastQuestionRef = useRef(""); const lastAttachementsRef = useRef([]); @@ -113,15 +114,18 @@ const Chat = () => { const token = client ? await getToken(client) : undefined; + try { + /** const messages: ResponseMessage[] = answers.flatMap(a => [ { content: a[0], role: "user", attachments: a[1]}, { content: a[2].choices[0].message.content, role: "assistant" } ]); - const stream = streamAvailable && shouldStream; + */ + const stream = streamAvailable && shouldStream; const request: ChatAppRequest = { - messages: [...messages, { content: questionContext.question, role: "user", attachments: questionContext.attachments }], + messages: [{ content: questionContext.question, role: "user", attachments: questionContext.attachments }], stream: stream, context: { overrides: { @@ -138,8 +142,8 @@ const Chat = () => { } }, approach: approach, - // ChatAppProtocol: Client must pass on any session state received from the server - session_state: answers.length ? answers[answers.length - 1][2].choices[0].session_state : null + session_state: answers.length ? answers[answers.length - 1][2].choices[0].session_state : null, + threadId: threadId // Include threadId in the request }; const response = await chatApi(request, token?.accessToken); @@ -147,14 +151,16 @@ const Chat = () => { throw Error("No response body"); } if (stream) { - const parsedResponse: ChatAppResponse = await handleAsyncRequest(questionContext.question,questionContext.attachments || [], answers, setAnswers, response.body); - setAnswers([...answers, [questionContext.question,questionContext.attachments || [], parsedResponse]]); + const parsedResponse: ChatAppResponse = await handleAsyncRequest(questionContext.question, questionContext.attachments || [], answers, setAnswers, response.body); + setAnswers([...answers, [questionContext.question, questionContext.attachments || [], parsedResponse]]); + setThreadId(parsedResponse.threadId || undefined); // Update threadId from the response } else { const parsedResponse: ChatAppResponseOrError = await response.json(); if (response.status > 299 || !response.ok) { throw Error(parsedResponse.error || "Unknown error"); } - setAnswers([...answers, [questionContext.question,questionContext.attachments || [], parsedResponse as ChatAppResponse]]); + setAnswers([...answers, [questionContext.question, questionContext.attachments || [], parsedResponse as ChatAppResponse]]); + setThreadId((parsedResponse as ChatAppResponse).threadId || undefined); // Update threadId from the response } } catch (e) { setError(e); @@ -173,6 +179,7 @@ const Chat = () => { setStreamedAnswers([]); setIsLoading(false); setIsStreaming(false); + setThreadId(undefined); // Reset threadId }; useEffect(() => chatMessageStreamEnd.current?.scrollIntoView({ behavior: "smooth" }), [isLoading]); diff --git a/infra/shared/host/container-apps.bicep b/infra/shared/host/container-apps.bicep index 10691aa..8f30178 100644 --- a/infra/shared/host/container-apps.bicep +++ b/infra/shared/host/container-apps.bicep @@ -11,6 +11,8 @@ param logAnalyticsWorkspaceName string param applicationInsightsName string = '' param daprEnabled bool = false +var containerRegistryResourceGroupEvaluated = !empty(containerRegistryResourceGroupName) ? resourceGroup(containerRegistryResourceGroupName) : resourceGroup() + module containerAppsEnvironment 'container-apps-environment.bicep' = { name: '${name}-container-apps-environment' params: { @@ -25,7 +27,7 @@ module containerAppsEnvironment 'container-apps-environment.bicep' = { module containerRegistry 'container-registry.bicep' = { name: '${name}-container-registry' - scope: !empty(containerRegistryResourceGroupName) ? resourceGroup(containerRegistryResourceGroupName) : resourceGroup() + scope: containerRegistryResourceGroupEvaluated params: { name: containerRegistryName location: location From 0e6cada992b6c11c2fc2f9715621195b0fa5583f Mon Sep 17 00:00:00 2001 From: dantelmomsft Date: Mon, 8 Sep 2025 15:32:30 +0200 Subject: [PATCH 2/2] update to review. not sure it works --- app/copilot/copilot-backend/pom.xml | 169 ++- .../config/BlobStorageProxyConfiguration.java | 44 +- .../config/Langchain4JConfiguration.java | 62 +- .../controller/ADKChatController.java | 240 ++--- .../assistant/controller/ChatAppRequest.java | 26 +- .../assistant/controller/ChatResponse.java | 84 +- .../assistant/AzureOpenAILangchain4J.java | 82 +- app/frontend/src/pages/chat/Chat.tsx | 976 +++++++++--------- infra/shared/host/container-apps.bicep | 90 +- 9 files changed, 884 insertions(+), 889 deletions(-) diff --git a/app/copilot/copilot-backend/pom.xml b/app/copilot/copilot-backend/pom.xml index a90eeb4..d5166b0 100644 --- a/app/copilot/copilot-backend/pom.xml +++ b/app/copilot/copilot-backend/pom.xml @@ -1,87 +1,82 @@ - - - 4.0.0 - - org.springframework.boot - spring-boot-starter-parent - 3.3.6 - - - com.microsoft.openai.samples.assistant - personal-finance-assistant-copilot - 1.0.0-SNAPSHOT - personal-finance-assistant-copilot-langchain4j - This sample demonstrate how to create a generative ai multi-agent solution for a banking personal assistant - - - 17 - 17 - - 5.20.0 - 4.5.1 - 1.0.0 - 1.1.0-rc1 - 0.2.1-SNAPSHOT - - - - - - com.azure.spring - spring-cloud-azure-dependencies - ${spring-cloud-azure.version} - pom - import - - - - - - - - com.microsoft.openai.samples.assistant - langchain4j-agents - 1.0.0-SNAPSHOT - - - com.google.adk - google-adk - ${google-adk.version} - - - com.google.adk - google-adk-contrib-langchain4j - ${google-adk.version} - - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-security - - - dev.langchain4j - langchain4j-azure-open-ai - ${langchain4j-azure-openai.version} - - - com.azure - azure-identity - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - - - - - + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.3.6 + + + com.microsoft.openai.samples.assistant + personal-finance-assistant-copilot + 1.0.0-SNAPSHOT + personal-finance-assistant-copilot-langchain4j + This sample demonstrate how to create a generative ai multi-agent solution for a banking personal assistant + + + 17 + 17 + + 5.20.0 + 4.5.1 + 1.0.0 + 1.1.0-rc1 + 0.2.1-SNAPSHOT + + + + + + com.azure.spring + spring-cloud-azure-dependencies + ${spring-cloud-azure.version} + pom + import + + + + + + + + com.google.adk + google-adk + ${google-adk.version} + + + com.google.adk + google-adk-contrib-langchain4j + ${google-adk.version} + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-security + + + dev.langchain4j + langchain4j-azure-open-ai + ${langchain4j-azure-openai.version} + + + com.azure + azure-identity + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/BlobStorageProxyConfiguration.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/BlobStorageProxyConfiguration.java index 1e25350..08a0197 100644 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/BlobStorageProxyConfiguration.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/BlobStorageProxyConfiguration.java @@ -1,22 +1,22 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.openai.samples.assistant.config; - -import com.azure.core.credential.TokenCredential; -import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class BlobStorageProxyConfiguration { - @Value("${storage-account.service}") - String storageAccountServiceName; - @Value("${blob.container.name}") - String containerName; - - @Bean - public BlobStorageProxy blobStorageProxy(TokenCredential tokenCredential) { - return new BlobStorageProxy(storageAccountServiceName,containerName,tokenCredential); - } - -} +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.openai.samples.assistant.config; + +import com.azure.core.credential.TokenCredential; +import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class BlobStorageProxyConfiguration { + @Value("${storage-account.service}") + String storageAccountServiceName; + @Value("${blob.container.name}") + String containerName; + + @Bean + public BlobStorageProxy blobStorageProxy(TokenCredential tokenCredential) { + return new BlobStorageProxy(storageAccountServiceName,containerName,tokenCredential); + } + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/Langchain4JConfiguration.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/Langchain4JConfiguration.java index fba498f..749f857 100644 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/Langchain4JConfiguration.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/config/Langchain4JConfiguration.java @@ -1,31 +1,31 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.openai.samples.assistant.config; - - -import com.azure.ai.openai.OpenAIClient; - -import dev.langchain4j.model.azure.AzureOpenAiChatModel; -import dev.langchain4j.model.chat.ChatModel; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class Langchain4JConfiguration { - - @Value("${openai.chatgpt.deployment}") - private String gptChatDeploymentModelId; - - @Bean - public ChatModel chatLanguageModel(OpenAIClient azureOpenAICLient) { - - return AzureOpenAiChatModel.builder() - .openAIClient(azureOpenAICLient) - .deploymentName(gptChatDeploymentModelId) - .temperature(0.3) - .logRequestsAndResponses(true) - .build(); - } - - -} +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.openai.samples.assistant.config; + + +import com.azure.ai.openai.OpenAIClient; + +import dev.langchain4j.model.azure.AzureOpenAiChatModel; +import dev.langchain4j.model.chat.ChatModel; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class Langchain4JConfiguration { + + @Value("${openai.chatgpt.deployment}") + private String gptChatDeploymentModelId; + + @Bean + public ChatModel chatLanguageModel(OpenAIClient azureOpenAICLient) { + + return AzureOpenAiChatModel.builder() + .openAIClient(azureOpenAICLient) + .deploymentName(gptChatDeploymentModelId) + .temperature(0.3) + .logRequestsAndResponses(true) + .build(); + } + + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ADKChatController.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ADKChatController.java index 30455a2..c96a77d 100644 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ADKChatController.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ADKChatController.java @@ -1,120 +1,120 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.openai.samples.assistant.controller; - - -import com.google.adk.agents.LlmAgent; -import com.google.adk.events.Event; -import com.google.adk.runner.InMemoryRunner; -import com.google.adk.runner.Runner; -import com.google.adk.sessions.BaseSessionService; -import com.google.adk.sessions.GetSessionConfig; -import com.google.adk.sessions.InMemorySessionService; -import com.google.adk.sessions.Session; -import com.google.genai.types.Content; -import com.google.genai.types.Part; -import com.microsoft.openai.samples.assistant.config.agent.SupervisorAgent; -import com.microsoft.openai.samples.assistant.security.LoggedUser; -import com.microsoft.openai.samples.assistant.security.LoggedUserService; -import io.reactivex.rxjava3.core.Flowable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; - -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.server.ResponseStatusException; - -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicReference; - -@RestController -public class ADKChatController { - - private static final Logger LOGGER = LoggerFactory.getLogger(ADKChatController.class); - private static final String APPNAME = "Copilot"; - private final LlmAgent supervisorAgent; - private final InMemoryRunner runner; - private final LoggedUserService loggedUserService; - - public ADKChatController(@Qualifier("adkSupervisorAgent") LlmAgent supervisorAgent, LoggedUserService loggedUserService){ - this.supervisorAgent = supervisorAgent; - this.loggedUserService = loggedUserService; - this.runner = new InMemoryRunner(supervisorAgent,APPNAME); - } - - - @PostMapping(value = "/api/chat", produces = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity openAIAsk(@RequestBody ChatAppRequest chatRequest) { - if (chatRequest.stream()) { - LOGGER.warn( - "Requested a content-type of application/json however also requested streaming." - + " Please use a content-type of application/ndjson"); - throw new ResponseStatusException( - HttpStatus.BAD_REQUEST, - "Requested a content-type of application/json however also requested streaming." - + " Please use a content-type of application/ndjson"); - } - - if (chatRequest.messages() == null || chatRequest.messages().isEmpty()) { - LOGGER.warn("history cannot be null in Chat request"); - return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(null); - } - - ResponseMessage userMessage = chatRequest.messages().get(chatRequest.messages().size()-1); - LOGGER.debug("Processing user message..", userMessage ); - - String inputText = userMessage.content(); - if(userMessage.attachments()!= null && !userMessage.attachments().isEmpty()) { - inputText += " Attachments: " + userMessage.attachments(); - } - String threadId = chatRequest.threadId(); - - //attach the session to the runner or create a new one - Session session = buildSession(threadId,loggedUserService.getLoggedUser(),runner); - - //Run the agent flow - Content userMsg = Content.fromParts(Part.fromText(inputText)); - Flowable events = runner.runAsync(loggedUserService.getLoggedUser().username(), session.id(), userMsg); - - AtomicReference agentResponse = new AtomicReference<>(""); - - events.blockingForEach(event ->{ - LOGGER.info(" {} > {} ",event.author()+(event.finalResponse()?"[Final]" :""),event.stringifyContent()); - if(event.finalResponse()) - agentResponse.set(event.stringifyContent()); - - } - ); - return ResponseEntity.ok( - ChatResponse.buildChatResponse(agentResponse.get(),session.id())); - } - - private Session buildSession(String threadId, LoggedUser loggedUser, Runner runner) { - - if (threadId != null && !threadId.isEmpty()) { - LOGGER.debug("Using existing threadId: {}", threadId); - return runner.sessionService().getSession(APPNAME,loggedUser.username(), threadId, Optional.of(GetSessionConfig.builder().build())).blockingGet(); - } else { - LOGGER.debug("Creating new threadId: {}", threadId); - - ConcurrentMap initialState = new ConcurrentHashMap<>(); - initialState.put("loggedUserName", loggedUser.username()); - var datetimeIso8601 = java.time.ZonedDateTime.now(java.time.ZoneId.of("UTC")).toInstant().toString(); - initialState.put("timestamp", datetimeIso8601); - - return runner.sessionService() - .createSession(APPNAME, loggedUser.username(),initialState,null) - .blockingGet(); - } - - } - - -} +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.openai.samples.assistant.controller; + + +import com.google.adk.agents.LlmAgent; +import com.google.adk.events.Event; +import com.google.adk.runner.InMemoryRunner; +import com.google.adk.runner.Runner; +import com.google.adk.sessions.BaseSessionService; +import com.google.adk.sessions.GetSessionConfig; +import com.google.adk.sessions.InMemorySessionService; +import com.google.adk.sessions.Session; +import com.google.genai.types.Content; +import com.google.genai.types.Part; +import com.microsoft.openai.samples.assistant.config.agent.SupervisorAgent; +import com.microsoft.openai.samples.assistant.security.LoggedUser; +import com.microsoft.openai.samples.assistant.security.LoggedUserService; +import io.reactivex.rxjava3.core.Flowable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; + +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; + +@RestController +public class ADKChatController { + + private static final Logger LOGGER = LoggerFactory.getLogger(ADKChatController.class); + private static final String APPNAME = "Copilot"; + private final LlmAgent supervisorAgent; + private final InMemoryRunner runner; + private final LoggedUserService loggedUserService; + + public ADKChatController(@Qualifier("adkSupervisorAgent") LlmAgent supervisorAgent, LoggedUserService loggedUserService){ + this.supervisorAgent = supervisorAgent; + this.loggedUserService = loggedUserService; + this.runner = new InMemoryRunner(supervisorAgent,APPNAME); + } + + + @PostMapping(value = "/api/chat", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity openAIAsk(@RequestBody ChatAppRequest chatRequest) { + if (chatRequest.stream()) { + LOGGER.warn( + "Requested a content-type of application/json however also requested streaming." + + " Please use a content-type of application/ndjson"); + throw new ResponseStatusException( + HttpStatus.BAD_REQUEST, + "Requested a content-type of application/json however also requested streaming." + + " Please use a content-type of application/ndjson"); + } + + if (chatRequest.messages() == null || chatRequest.messages().isEmpty()) { + LOGGER.warn("history cannot be null in Chat request"); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(null); + } + + ResponseMessage userMessage = chatRequest.messages().get(chatRequest.messages().size()-1); + LOGGER.debug("Processing user message..", userMessage ); + + String inputText = userMessage.content(); + if(userMessage.attachments()!= null && !userMessage.attachments().isEmpty()) { + inputText += " Attachments: " + userMessage.attachments(); + } + String threadId = chatRequest.threadId(); + + //attach the session to the runner or create a new one + Session session = buildSession(threadId,loggedUserService.getLoggedUser(),runner); + + //Run the agent flow + Content userMsg = Content.fromParts(Part.fromText(inputText)); + Flowable events = runner.runAsync(loggedUserService.getLoggedUser().username(), session.id(), userMsg); + + AtomicReference agentResponse = new AtomicReference<>(""); + + events.blockingForEach(event ->{ + LOGGER.info(" {} > {} ",event.author()+(event.finalResponse()?"[Final]" :""),event.stringifyContent()); + if(event.finalResponse()) + agentResponse.set(event.stringifyContent()); + + } + ); + return ResponseEntity.ok( + ChatResponse.buildChatResponse(agentResponse.get(),session.id())); + } + + private Session buildSession(String threadId, LoggedUser loggedUser, Runner runner) { + + if (threadId != null && !threadId.isEmpty()) { + LOGGER.debug("Using existing threadId: {}", threadId); + return runner.sessionService().getSession(APPNAME,loggedUser.username(), threadId, Optional.of(GetSessionConfig.builder().build())).blockingGet(); + } else { + LOGGER.debug("Creating new threadId: {}", threadId); + + ConcurrentMap initialState = new ConcurrentHashMap<>(); + initialState.put("loggedUserName", loggedUser.username()); + var datetimeIso8601 = java.time.ZonedDateTime.now(java.time.ZoneId.of("UTC")).toInstant().toString(); + initialState.put("timestamp", datetimeIso8601); + + return runner.sessionService() + .createSession(APPNAME, loggedUser.username(),initialState,null) + .blockingGet(); + } + + } + + +} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java index ade31d8..c5c57b5 100644 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java @@ -1,13 +1,13 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.openai.samples.assistant.controller; - -import java.util.List; - - -public record ChatAppRequest( - List messages, - List attachments, - ChatAppRequestContext context, - boolean stream, - String approach, - String threadId) {} +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.openai.samples.assistant.controller; + +import java.util.List; + + +public record ChatAppRequest( + List messages, + List attachments, + ChatAppRequestContext context, + boolean stream, + String approach, + String threadId) {} diff --git a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java index 17d2265..b855904 100644 --- a/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java +++ b/app/copilot/copilot-backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java @@ -1,42 +1,42 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.openai.samples.assistant.controller; - - - -import com.microsoft.openai.samples.assistant.common.ChatGPTMessage; -import dev.langchain4j.data.message.AiMessage; - -import java.util.Collections; -import java.util.List; - -public record ChatResponse( - List choices, - String threadId -) { - - public static ChatResponse buildChatResponse(String agentResponse, String threadId) { - List dataPoints = Collections.emptyList(); - String thoughts = ""; - List attachments = Collections.emptyList(); - - return new ChatResponse( - List.of( - new ResponseChoice( - 0, - new ResponseMessage( - agentResponse, - ChatGPTMessage.ChatRole.ASSISTANT.toString(), - attachments - ), - new ResponseContext(thoughts, dataPoints), - new ResponseMessage( - agentResponse, - ChatGPTMessage.ChatRole.ASSISTANT.toString(), - attachments) - ) - ), - threadId - ); - } - -} +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.openai.samples.assistant.controller; + + + +import com.microsoft.openai.samples.assistant.common.ChatGPTMessage; +import dev.langchain4j.data.message.AiMessage; + +import java.util.Collections; +import java.util.List; + +public record ChatResponse( + List choices, + String threadId +) { + + public static ChatResponse buildChatResponse(String agentResponse, String threadId) { + List dataPoints = Collections.emptyList(); + String thoughts = ""; + List attachments = Collections.emptyList(); + + return new ChatResponse( + List.of( + new ResponseChoice( + 0, + new ResponseMessage( + agentResponse, + ChatGPTMessage.ChatRole.ASSISTANT.toString(), + attachments + ), + new ResponseContext(thoughts, dataPoints), + new ResponseMessage( + agentResponse, + ChatGPTMessage.ChatRole.ASSISTANT.toString(), + attachments) + ) + ), + threadId + ); + } + +} diff --git a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AzureOpenAILangchain4J.java b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AzureOpenAILangchain4J.java index ba58c0d..1be9f10 100644 --- a/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AzureOpenAILangchain4J.java +++ b/app/copilot/copilot-backend/src/test/java/com/microsoft/openai/samples/assistant/AzureOpenAILangchain4J.java @@ -1,41 +1,41 @@ -package com.microsoft.openai.samples.assistant; - -import com.azure.ai.openai.OpenAIClient; -import com.azure.ai.openai.OpenAIClientBuilder; -import com.azure.core.credential.TokenCredential; -import com.azure.core.http.policy.HttpLogDetailLevel; -import com.azure.core.http.policy.HttpLogOptions; -import com.azure.identity.AzureCliCredentialBuilder; -import dev.langchain4j.model.azure.AzureOpenAiChatModel; -import dev.langchain4j.model.chat.ChatModel; - -public class AzureOpenAILangchain4J { - - public static ChatModel buildChatModel(){ - String azureOpenAIName = System.getenv("AZURE_OPENAI_NAME"); - String chatDeploymentName = System.getenv("AZURE_OPENAI_DEPLOYMENT_ID"); - if (azureOpenAIName == null || azureOpenAIName.isEmpty()) - throw new IllegalArgumentException("AZURE_OPENAI_NAME environment variable is not set."); - - - TokenCredential tokenCredential = new AzureCliCredentialBuilder().build(); - - String endpoint = "https://%s.openai.azure.com".formatted(azureOpenAIName); - - var httpLogOptions = new HttpLogOptions(); - // httpLogOptions.setPrettyPrintBody(true); - httpLogOptions.setLogLevel(HttpLogDetailLevel.BODY_AND_HEADERS); - - OpenAIClient client = new OpenAIClientBuilder() - .endpoint(endpoint) - .credential(tokenCredential) - .httpLogOptions(httpLogOptions) - .buildClient(); - - return AzureOpenAiChatModel.builder() - .openAIClient(client) - .logRequestsAndResponses(true) - .deploymentName(chatDeploymentName) - .build(); - } -} +package com.microsoft.openai.samples.assistant; + +import com.azure.ai.openai.OpenAIClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.TokenCredential; +import com.azure.core.http.policy.HttpLogDetailLevel; +import com.azure.core.http.policy.HttpLogOptions; +import com.azure.identity.AzureCliCredentialBuilder; +import dev.langchain4j.model.azure.AzureOpenAiChatModel; +import dev.langchain4j.model.chat.ChatModel; + +public class AzureOpenAILangchain4J { + + public static ChatModel buildChatModel(){ + String azureOpenAIName = System.getenv("AZURE_OPENAI_NAME"); + String chatDeploymentName = System.getenv("AZURE_OPENAI_DEPLOYMENT_ID"); + if (azureOpenAIName == null || azureOpenAIName.isEmpty()) + throw new IllegalArgumentException("AZURE_OPENAI_NAME environment variable is not set."); + + + TokenCredential tokenCredential = new AzureCliCredentialBuilder().build(); + + String endpoint = "https://%s.openai.azure.com".formatted(azureOpenAIName); + + var httpLogOptions = new HttpLogOptions(); + // httpLogOptions.setPrettyPrintBody(true); + httpLogOptions.setLogLevel(HttpLogDetailLevel.BODY_AND_HEADERS); + + OpenAIClient client = new OpenAIClientBuilder() + .endpoint(endpoint) + .credential(tokenCredential) + .httpLogOptions(httpLogOptions) + .buildClient(); + + return AzureOpenAiChatModel.builder() + .openAIClient(client) + .logRequestsAndResponses(true) + .deploymentName(chatDeploymentName) + .build(); + } +} diff --git a/app/frontend/src/pages/chat/Chat.tsx b/app/frontend/src/pages/chat/Chat.tsx index 2b48b8d..44b89c1 100644 --- a/app/frontend/src/pages/chat/Chat.tsx +++ b/app/frontend/src/pages/chat/Chat.tsx @@ -1,488 +1,488 @@ -import { useRef, useState, useEffect } from "react"; -import { Checkbox, ChoiceGroup, Panel, DefaultButton, TextField, SpinButton, Dropdown, IDropdownOption, IChoiceGroupOption } from "@fluentui/react"; -import { SparkleFilled } from "@fluentui/react-icons"; -import readNDJSONStream from "ndjson-readablestream"; - - -import styles from "./Chat.module.css"; - -import { - chatApi, - RetrievalMode, - ChatAppResponse, - ChatAppResponseOrError, - ChatAppRequest, - ResponseMessage, - Approaches, - SKMode -} from "../../api"; -import { Answer, AnswerError, AnswerLoading } from "../../components/Answer"; -import { QuestionInput } from "../../components/QuestionInput"; -import { QuestionContextType } from "../../components/QuestionInput/QuestionContext"; -import { ExampleList } from "../../components/Example"; -import { UserChatMessage } from "../../components/UserChatMessage"; -import { AnalysisPanel, AnalysisPanelTabs } from "../../components/AnalysisPanel"; -import { SettingsButton } from "../../components/SettingsButton"; -import { ClearChatButton } from "../../components/ClearChatButton"; -import { useLogin, getToken } from "../../authConfig"; -import { useMsal } from "@azure/msal-react"; -import { TokenClaimsDisplay } from "../../components/TokenClaimsDisplay"; -import { AttachmentType } from "../../components/AttachmentType"; - - -const Chat = () => { - const [isConfigPanelOpen, setIsConfigPanelOpen] = useState(false); - const [approach, setApproach] = useState(Approaches.JAVA_OPENAI_SDK); - const [skMode, setSKMode] = useState(SKMode.Chains); - const [promptTemplate, setPromptTemplate] = useState(""); - const [retrieveCount, setRetrieveCount] = useState(3); - const [retrievalMode, setRetrievalMode] = useState(RetrievalMode.Hybrid); - const [useSemanticRanker, setUseSemanticRanker] = useState(true); - const [shouldStream, setShouldStream] = useState(false); - const [streamAvailable, setStreamAvailable] = useState(true); - const [useSemanticCaptions, setUseSemanticCaptions] = useState(false); - const [excludeCategory, setExcludeCategory] = useState(""); - const [useSuggestFollowupQuestions, setUseSuggestFollowupQuestions] = useState(false); - const [useOidSecurityFilter, setUseOidSecurityFilter] = useState(false); - const [useGroupsSecurityFilter, setUseGroupsSecurityFilter] = useState(false); - const [threadId, setThreadId] = useState(undefined); // Updated type to match ChatAppRequest - - const lastQuestionRef = useRef(""); - const lastAttachementsRef = useRef([]); - const chatMessageStreamEnd = useRef(null); - - const [isLoading, setIsLoading] = useState(false); - const [isStreaming, setIsStreaming] = useState(false); - const [error, setError] = useState(); - - const [activeCitation, setActiveCitation] = useState(); - const [activeAnalysisPanelTab, setActiveAnalysisPanelTab] = useState(undefined); - - const [selectedAnswer, setSelectedAnswer] = useState(0); - const [answers, setAnswers] = useState<[user: string, attachments: string[], response: ChatAppResponse][]>([]); - const [streamedAnswers, setStreamedAnswers] = useState<[user: string, attachments: string[], response: ChatAppResponse][]>([]); - - const handleAsyncRequest = async (question: string, attachments: string[], answers: [string, string[],ChatAppResponse][], setAnswers: Function, responseBody: ReadableStream) => { - let answer: string = ""; - let askResponse: ChatAppResponse = {} as ChatAppResponse; - - const updateState = (newContent: string) => { - return new Promise(resolve => { - setTimeout(() => { - answer += newContent; - const latestResponse: ChatAppResponse = { - ...askResponse, - choices: [{ ...askResponse.choices[0], message: { content: answer, role: askResponse.choices[0].message.role } }] - }; - setStreamedAnswers([...answers, [question,attachments, latestResponse]]); - resolve(null); - }, 33); - }); - }; - try { - setIsStreaming(true); - for await (const event of readNDJSONStream(responseBody)) { - if (event["choices"] && event["choices"][0]["context"] && event["choices"][0]["context"]["data_points"]) { - event["choices"][0]["message"] = event["choices"][0]["delta"]; - askResponse = event; - answer = askResponse["choices"][0]["message"]["content"]; - } else if (event["choices"] && event["choices"][0]["delta"]["content"]) { - setIsLoading(false); - await updateState(event["choices"][0]["delta"]["content"]); - } - } - } finally { - setIsStreaming(false); - } - const fullResponse: ChatAppResponse = { - ...askResponse, - choices: [{ ...askResponse.choices[0], message: { content: answer, role: askResponse.choices[0].message.role } }] - }; - return fullResponse; - }; - - const client = useLogin ? useMsal().instance : undefined; - - const makeApiRequest = async (questionContext: QuestionContextType) => { - lastQuestionRef.current = questionContext.question; - lastAttachementsRef.current = questionContext.attachments || []; - - error && setError(undefined); - setIsLoading(true); - setActiveCitation(undefined); - setActiveAnalysisPanelTab(undefined); - - const token = client ? await getToken(client) : undefined; - - - try { - /** - const messages: ResponseMessage[] = answers.flatMap(a => [ - { content: a[0], role: "user", attachments: a[1]}, - { content: a[2].choices[0].message.content, role: "assistant" } - ]); - - */ - const stream = streamAvailable && shouldStream; - const request: ChatAppRequest = { - messages: [{ content: questionContext.question, role: "user", attachments: questionContext.attachments }], - stream: stream, - context: { - overrides: { - prompt_template: promptTemplate.length === 0 ? undefined : promptTemplate, - exclude_category: excludeCategory.length === 0 ? undefined : excludeCategory, - top: retrieveCount, - retrieval_mode: retrievalMode, - semantic_ranker: useSemanticRanker, - semantic_captions: useSemanticCaptions, - suggest_followup_questions: useSuggestFollowupQuestions, - use_oid_security_filter: useOidSecurityFilter, - use_groups_security_filter: useGroupsSecurityFilter, - semantic_kernel_mode: skMode - } - }, - approach: approach, - session_state: answers.length ? answers[answers.length - 1][2].choices[0].session_state : null, - threadId: threadId // Include threadId in the request - }; - - const response = await chatApi(request, token?.accessToken); - if (!response.body) { - throw Error("No response body"); - } - if (stream) { - const parsedResponse: ChatAppResponse = await handleAsyncRequest(questionContext.question, questionContext.attachments || [], answers, setAnswers, response.body); - setAnswers([...answers, [questionContext.question, questionContext.attachments || [], parsedResponse]]); - setThreadId(parsedResponse.threadId || undefined); // Update threadId from the response - } else { - const parsedResponse: ChatAppResponseOrError = await response.json(); - if (response.status > 299 || !response.ok) { - throw Error(parsedResponse.error || "Unknown error"); - } - setAnswers([...answers, [questionContext.question, questionContext.attachments || [], parsedResponse as ChatAppResponse]]); - setThreadId((parsedResponse as ChatAppResponse).threadId || undefined); // Update threadId from the response - } - } catch (e) { - setError(e); - } finally { - setIsLoading(false); - } - }; - - const clearChat = () => { - lastQuestionRef.current = ""; - lastAttachementsRef.current = []; - error && setError(undefined); - setActiveCitation(undefined); - setActiveAnalysisPanelTab(undefined); - setAnswers([]); - setStreamedAnswers([]); - setIsLoading(false); - setIsStreaming(false); - setThreadId(undefined); // Reset threadId - }; - - useEffect(() => chatMessageStreamEnd.current?.scrollIntoView({ behavior: "smooth" }), [isLoading]); - useEffect(() => chatMessageStreamEnd.current?.scrollIntoView({ behavior: "auto" }), [streamedAnswers]); - - const onPromptTemplateChange = (_ev?: React.FormEvent, newValue?: string) => { - setPromptTemplate(newValue || ""); - }; - - const onRetrieveCountChange = (_ev?: React.SyntheticEvent, newValue?: string) => { - setRetrieveCount(parseInt(newValue || "3")); - }; - - const onRetrievalModeChange = (_ev: React.FormEvent, option?: IDropdownOption | undefined, index?: number | undefined) => { - setRetrievalMode(option?.data || RetrievalMode.Hybrid); - }; - - const onSKModeChange = (_ev: React.FormEvent, option?: IDropdownOption | undefined, index?: number | undefined) => { - setSKMode(option?.data || SKMode.Chains); - }; - - const onApproachChange = (_ev?: React.FormEvent, option?: IChoiceGroupOption) => { - const newApproach = (option?.key as Approaches); - setApproach(newApproach || Approaches.JAVA_OPENAI_SDK); - setStreamAvailable(newApproach === Approaches.JAVA_OPENAI_SDK); - }; - - const onUseSemanticRankerChange = (_ev?: React.FormEvent, checked?: boolean) => { - setUseSemanticRanker(!!checked); - }; - - const onUseSemanticCaptionsChange = (_ev?: React.FormEvent, checked?: boolean) => { - setUseSemanticCaptions(!!checked); - }; - - const onShouldStreamChange = (_ev?: React.FormEvent, checked?: boolean) => { - setShouldStream(!!checked); - }; - - const onExcludeCategoryChanged = (_ev?: React.FormEvent, newValue?: string) => { - setExcludeCategory(newValue || ""); - }; - - const onUseSuggestFollowupQuestionsChange = (_ev?: React.FormEvent, checked?: boolean) => { - setUseSuggestFollowupQuestions(!!checked); - }; - - const onUseOidSecurityFilterChange = (_ev?: React.FormEvent, checked?: boolean) => { - setUseOidSecurityFilter(!!checked); - }; - - const onUseGroupsSecurityFilterChange = (_ev?: React.FormEvent, checked?: boolean) => { - setUseGroupsSecurityFilter(!!checked); - }; - - const onExampleClicked = (example: string) => { - makeApiRequest({question:example}); - }; - - const onShowCitation = (citation: string, index: number) => { - if (activeCitation === citation && activeAnalysisPanelTab === AnalysisPanelTabs.CitationTab && selectedAnswer === index) { - setActiveAnalysisPanelTab(undefined); - } else { - setActiveCitation(citation); - setActiveAnalysisPanelTab(AnalysisPanelTabs.CitationTab); - } - - setSelectedAnswer(index); - }; - - const onToggleTab = (tab: AnalysisPanelTabs, index: number) => { - if (activeAnalysisPanelTab === tab && selectedAnswer === index) { - setActiveAnalysisPanelTab(undefined); - } else { - setActiveAnalysisPanelTab(tab); - } - - setSelectedAnswer(index); - }; - - const approaches: IChoiceGroupOption[] = [ - { - key: Approaches.JAVA_OPENAI_SDK, - text: "Java Azure Open AI SDK" - }, - /* Pending Semantic Kernel Memory implementation in V1.0.0 - { - key: Approaches.JAVA_SEMANTIC_KERNEL, - text: "Java Semantic Kernel - Memory" - },*/ - { - key: Approaches.JAVA_SEMANTIC_KERNEL_PLANNER, - text: "Java Semantic Kernel - Orchestration" - } - ]; - - return ( -
-
- - -
-
-
- {!lastQuestionRef.current ? ( -
-
- ) : ( -
- {isStreaming && - streamedAnswers.map((streamedAnswer, index) => ( -
- -
- onShowCitation(c, index)} - onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab, index)} - onSupportingContentClicked={() => onToggleTab(AnalysisPanelTabs.SupportingContentTab, index)} - onFollowupQuestionClicked={q => makeApiRequest({question:q})} - showFollowupQuestions={useSuggestFollowupQuestions && answers.length - 1 === index} - /> -
-
- ))} - {!isStreaming && - answers.map((answer, index) => ( -
- -
- onShowCitation(c, index)} - onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab, index)} - onSupportingContentClicked={() => onToggleTab(AnalysisPanelTabs.SupportingContentTab, index)} - onFollowupQuestionClicked={q => makeApiRequest({question:q})} - showFollowupQuestions={useSuggestFollowupQuestions && answers.length - 1 === index} - /> -
-
- ))} - {isLoading && ( - <> - -
- -
- - )} - {error ? ( - <> - -
- makeApiRequest({question:lastQuestionRef.current})} /> -
- - ) : null} -
-
- )} - -
- makeApiRequest(question)} - /> -
-
- - {answers.length > 0 && activeAnalysisPanelTab && ( - onToggleTab(x, selectedAnswer)} - citationHeight="810px" - answer={answers[selectedAnswer][2]} - activeTab={activeAnalysisPanelTab} - /> - )} - - setIsConfigPanelOpen(false)} - closeButtonAriaLabel="Close" - onRenderFooterContent={() => setIsConfigPanelOpen(false)}>Close} - isFooterAtBottom={true} - > - - - {(approach === Approaches.JAVA_OPENAI_SDK || approach === Approaches.JAVA_SEMANTIC_KERNEL) && ( - - )} - {(approach === Approaches.JAVA_SEMANTIC_KERNEL_PLANNER) && ( - - )} - - - - - - - {useLogin && ( - - )} - {useLogin && ( - - )} - - {streamAvailable && - - } - - {useLogin && } - -
-
- ); -}; - -export default Chat; +import { useRef, useState, useEffect } from "react"; +import { Checkbox, ChoiceGroup, Panel, DefaultButton, TextField, SpinButton, Dropdown, IDropdownOption, IChoiceGroupOption } from "@fluentui/react"; +import { SparkleFilled } from "@fluentui/react-icons"; +import readNDJSONStream from "ndjson-readablestream"; + + +import styles from "./Chat.module.css"; + +import { + chatApi, + RetrievalMode, + ChatAppResponse, + ChatAppResponseOrError, + ChatAppRequest, + ResponseMessage, + Approaches, + SKMode +} from "../../api"; +import { Answer, AnswerError, AnswerLoading } from "../../components/Answer"; +import { QuestionInput } from "../../components/QuestionInput"; +import { QuestionContextType } from "../../components/QuestionInput/QuestionContext"; +import { ExampleList } from "../../components/Example"; +import { UserChatMessage } from "../../components/UserChatMessage"; +import { AnalysisPanel, AnalysisPanelTabs } from "../../components/AnalysisPanel"; +import { SettingsButton } from "../../components/SettingsButton"; +import { ClearChatButton } from "../../components/ClearChatButton"; +import { useLogin, getToken } from "../../authConfig"; +import { useMsal } from "@azure/msal-react"; +import { TokenClaimsDisplay } from "../../components/TokenClaimsDisplay"; +import { AttachmentType } from "../../components/AttachmentType"; + + +const Chat = () => { + const [isConfigPanelOpen, setIsConfigPanelOpen] = useState(false); + const [approach, setApproach] = useState(Approaches.JAVA_OPENAI_SDK); + const [skMode, setSKMode] = useState(SKMode.Chains); + const [promptTemplate, setPromptTemplate] = useState(""); + const [retrieveCount, setRetrieveCount] = useState(3); + const [retrievalMode, setRetrievalMode] = useState(RetrievalMode.Hybrid); + const [useSemanticRanker, setUseSemanticRanker] = useState(true); + const [shouldStream, setShouldStream] = useState(false); + const [streamAvailable, setStreamAvailable] = useState(true); + const [useSemanticCaptions, setUseSemanticCaptions] = useState(false); + const [excludeCategory, setExcludeCategory] = useState(""); + const [useSuggestFollowupQuestions, setUseSuggestFollowupQuestions] = useState(false); + const [useOidSecurityFilter, setUseOidSecurityFilter] = useState(false); + const [useGroupsSecurityFilter, setUseGroupsSecurityFilter] = useState(false); + const [threadId, setThreadId] = useState(undefined); // Updated type to match ChatAppRequest + + const lastQuestionRef = useRef(""); + const lastAttachementsRef = useRef([]); + const chatMessageStreamEnd = useRef(null); + + const [isLoading, setIsLoading] = useState(false); + const [isStreaming, setIsStreaming] = useState(false); + const [error, setError] = useState(); + + const [activeCitation, setActiveCitation] = useState(); + const [activeAnalysisPanelTab, setActiveAnalysisPanelTab] = useState(undefined); + + const [selectedAnswer, setSelectedAnswer] = useState(0); + const [answers, setAnswers] = useState<[user: string, attachments: string[], response: ChatAppResponse][]>([]); + const [streamedAnswers, setStreamedAnswers] = useState<[user: string, attachments: string[], response: ChatAppResponse][]>([]); + + const handleAsyncRequest = async (question: string, attachments: string[], answers: [string, string[],ChatAppResponse][], setAnswers: Function, responseBody: ReadableStream) => { + let answer: string = ""; + let askResponse: ChatAppResponse = {} as ChatAppResponse; + + const updateState = (newContent: string) => { + return new Promise(resolve => { + setTimeout(() => { + answer += newContent; + const latestResponse: ChatAppResponse = { + ...askResponse, + choices: [{ ...askResponse.choices[0], message: { content: answer, role: askResponse.choices[0].message.role } }] + }; + setStreamedAnswers([...answers, [question,attachments, latestResponse]]); + resolve(null); + }, 33); + }); + }; + try { + setIsStreaming(true); + for await (const event of readNDJSONStream(responseBody)) { + if (event["choices"] && event["choices"][0]["context"] && event["choices"][0]["context"]["data_points"]) { + event["choices"][0]["message"] = event["choices"][0]["delta"]; + askResponse = event; + answer = askResponse["choices"][0]["message"]["content"]; + } else if (event["choices"] && event["choices"][0]["delta"]["content"]) { + setIsLoading(false); + await updateState(event["choices"][0]["delta"]["content"]); + } + } + } finally { + setIsStreaming(false); + } + const fullResponse: ChatAppResponse = { + ...askResponse, + choices: [{ ...askResponse.choices[0], message: { content: answer, role: askResponse.choices[0].message.role } }] + }; + return fullResponse; + }; + + const client = useLogin ? useMsal().instance : undefined; + + const makeApiRequest = async (questionContext: QuestionContextType) => { + lastQuestionRef.current = questionContext.question; + lastAttachementsRef.current = questionContext.attachments || []; + + error && setError(undefined); + setIsLoading(true); + setActiveCitation(undefined); + setActiveAnalysisPanelTab(undefined); + + const token = client ? await getToken(client) : undefined; + + + try { + /** + const messages: ResponseMessage[] = answers.flatMap(a => [ + { content: a[0], role: "user", attachments: a[1]}, + { content: a[2].choices[0].message.content, role: "assistant" } + ]); + + */ + const stream = streamAvailable && shouldStream; + const request: ChatAppRequest = { + messages: [{ content: questionContext.question, role: "user", attachments: questionContext.attachments }], + stream: stream, + context: { + overrides: { + prompt_template: promptTemplate.length === 0 ? undefined : promptTemplate, + exclude_category: excludeCategory.length === 0 ? undefined : excludeCategory, + top: retrieveCount, + retrieval_mode: retrievalMode, + semantic_ranker: useSemanticRanker, + semantic_captions: useSemanticCaptions, + suggest_followup_questions: useSuggestFollowupQuestions, + use_oid_security_filter: useOidSecurityFilter, + use_groups_security_filter: useGroupsSecurityFilter, + semantic_kernel_mode: skMode + } + }, + approach: approach, + session_state: answers.length ? answers[answers.length - 1][2].choices[0].session_state : null, + threadId: threadId // Include threadId in the request + }; + + const response = await chatApi(request, token?.accessToken); + if (!response.body) { + throw Error("No response body"); + } + if (stream) { + const parsedResponse: ChatAppResponse = await handleAsyncRequest(questionContext.question, questionContext.attachments || [], answers, setAnswers, response.body); + setAnswers([...answers, [questionContext.question, questionContext.attachments || [], parsedResponse]]); + setThreadId(parsedResponse.threadId || undefined); // Update threadId from the response + } else { + const parsedResponse: ChatAppResponseOrError = await response.json(); + if (response.status > 299 || !response.ok) { + throw Error(parsedResponse.error || "Unknown error"); + } + setAnswers([...answers, [questionContext.question, questionContext.attachments || [], parsedResponse as ChatAppResponse]]); + setThreadId((parsedResponse as ChatAppResponse).threadId || undefined); // Update threadId from the response + } + } catch (e) { + setError(e); + } finally { + setIsLoading(false); + } + }; + + const clearChat = () => { + lastQuestionRef.current = ""; + lastAttachementsRef.current = []; + error && setError(undefined); + setActiveCitation(undefined); + setActiveAnalysisPanelTab(undefined); + setAnswers([]); + setStreamedAnswers([]); + setIsLoading(false); + setIsStreaming(false); + setThreadId(undefined); // Reset threadId + }; + + useEffect(() => chatMessageStreamEnd.current?.scrollIntoView({ behavior: "smooth" }), [isLoading]); + useEffect(() => chatMessageStreamEnd.current?.scrollIntoView({ behavior: "auto" }), [streamedAnswers]); + + const onPromptTemplateChange = (_ev?: React.FormEvent, newValue?: string) => { + setPromptTemplate(newValue || ""); + }; + + const onRetrieveCountChange = (_ev?: React.SyntheticEvent, newValue?: string) => { + setRetrieveCount(parseInt(newValue || "3")); + }; + + const onRetrievalModeChange = (_ev: React.FormEvent, option?: IDropdownOption | undefined, index?: number | undefined) => { + setRetrievalMode(option?.data || RetrievalMode.Hybrid); + }; + + const onSKModeChange = (_ev: React.FormEvent, option?: IDropdownOption | undefined, index?: number | undefined) => { + setSKMode(option?.data || SKMode.Chains); + }; + + const onApproachChange = (_ev?: React.FormEvent, option?: IChoiceGroupOption) => { + const newApproach = (option?.key as Approaches); + setApproach(newApproach || Approaches.JAVA_OPENAI_SDK); + setStreamAvailable(newApproach === Approaches.JAVA_OPENAI_SDK); + }; + + const onUseSemanticRankerChange = (_ev?: React.FormEvent, checked?: boolean) => { + setUseSemanticRanker(!!checked); + }; + + const onUseSemanticCaptionsChange = (_ev?: React.FormEvent, checked?: boolean) => { + setUseSemanticCaptions(!!checked); + }; + + const onShouldStreamChange = (_ev?: React.FormEvent, checked?: boolean) => { + setShouldStream(!!checked); + }; + + const onExcludeCategoryChanged = (_ev?: React.FormEvent, newValue?: string) => { + setExcludeCategory(newValue || ""); + }; + + const onUseSuggestFollowupQuestionsChange = (_ev?: React.FormEvent, checked?: boolean) => { + setUseSuggestFollowupQuestions(!!checked); + }; + + const onUseOidSecurityFilterChange = (_ev?: React.FormEvent, checked?: boolean) => { + setUseOidSecurityFilter(!!checked); + }; + + const onUseGroupsSecurityFilterChange = (_ev?: React.FormEvent, checked?: boolean) => { + setUseGroupsSecurityFilter(!!checked); + }; + + const onExampleClicked = (example: string) => { + makeApiRequest({question:example}); + }; + + const onShowCitation = (citation: string, index: number) => { + if (activeCitation === citation && activeAnalysisPanelTab === AnalysisPanelTabs.CitationTab && selectedAnswer === index) { + setActiveAnalysisPanelTab(undefined); + } else { + setActiveCitation(citation); + setActiveAnalysisPanelTab(AnalysisPanelTabs.CitationTab); + } + + setSelectedAnswer(index); + }; + + const onToggleTab = (tab: AnalysisPanelTabs, index: number) => { + if (activeAnalysisPanelTab === tab && selectedAnswer === index) { + setActiveAnalysisPanelTab(undefined); + } else { + setActiveAnalysisPanelTab(tab); + } + + setSelectedAnswer(index); + }; + + const approaches: IChoiceGroupOption[] = [ + { + key: Approaches.JAVA_OPENAI_SDK, + text: "Java Azure Open AI SDK" + }, + /* Pending Semantic Kernel Memory implementation in V1.0.0 + { + key: Approaches.JAVA_SEMANTIC_KERNEL, + text: "Java Semantic Kernel - Memory" + },*/ + { + key: Approaches.JAVA_SEMANTIC_KERNEL_PLANNER, + text: "Java Semantic Kernel - Orchestration" + } + ]; + + return ( +
+
+ + +
+
+
+ {!lastQuestionRef.current ? ( +
+
+ ) : ( +
+ {isStreaming && + streamedAnswers.map((streamedAnswer, index) => ( +
+ +
+ onShowCitation(c, index)} + onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab, index)} + onSupportingContentClicked={() => onToggleTab(AnalysisPanelTabs.SupportingContentTab, index)} + onFollowupQuestionClicked={q => makeApiRequest({question:q})} + showFollowupQuestions={useSuggestFollowupQuestions && answers.length - 1 === index} + /> +
+
+ ))} + {!isStreaming && + answers.map((answer, index) => ( +
+ +
+ onShowCitation(c, index)} + onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab, index)} + onSupportingContentClicked={() => onToggleTab(AnalysisPanelTabs.SupportingContentTab, index)} + onFollowupQuestionClicked={q => makeApiRequest({question:q})} + showFollowupQuestions={useSuggestFollowupQuestions && answers.length - 1 === index} + /> +
+
+ ))} + {isLoading && ( + <> + +
+ +
+ + )} + {error ? ( + <> + +
+ makeApiRequest({question:lastQuestionRef.current})} /> +
+ + ) : null} +
+
+ )} + +
+ makeApiRequest(question)} + /> +
+
+ + {answers.length > 0 && activeAnalysisPanelTab && ( + onToggleTab(x, selectedAnswer)} + citationHeight="810px" + answer={answers[selectedAnswer][2]} + activeTab={activeAnalysisPanelTab} + /> + )} + + setIsConfigPanelOpen(false)} + closeButtonAriaLabel="Close" + onRenderFooterContent={() => setIsConfigPanelOpen(false)}>Close} + isFooterAtBottom={true} + > + + + {(approach === Approaches.JAVA_OPENAI_SDK || approach === Approaches.JAVA_SEMANTIC_KERNEL) && ( + + )} + {(approach === Approaches.JAVA_SEMANTIC_KERNEL_PLANNER) && ( + + )} + + + + + + + {useLogin && ( + + )} + {useLogin && ( + + )} + + {streamAvailable && + + } + + {useLogin && } + +
+
+ ); +}; + +export default Chat; diff --git a/infra/shared/host/container-apps.bicep b/infra/shared/host/container-apps.bicep index 8f30178..a886c16 100644 --- a/infra/shared/host/container-apps.bicep +++ b/infra/shared/host/container-apps.bicep @@ -1,45 +1,45 @@ -metadata description = 'Creates an Azure Container Registry and an Azure Container Apps environment.' -param name string -param location string = resourceGroup().location -param tags object = {} - -param containerAppsEnvironmentName string -param containerRegistryName string -param containerRegistryResourceGroupName string = '' -param containerRegistryAdminUserEnabled bool = false -param logAnalyticsWorkspaceName string -param applicationInsightsName string = '' -param daprEnabled bool = false - -var containerRegistryResourceGroupEvaluated = !empty(containerRegistryResourceGroupName) ? resourceGroup(containerRegistryResourceGroupName) : resourceGroup() - -module containerAppsEnvironment 'container-apps-environment.bicep' = { - name: '${name}-container-apps-environment' - params: { - name: containerAppsEnvironmentName - location: location - tags: tags - logAnalyticsWorkspaceName: logAnalyticsWorkspaceName - applicationInsightsName: applicationInsightsName - daprEnabled: daprEnabled - } -} - -module containerRegistry 'container-registry.bicep' = { - name: '${name}-container-registry' - scope: containerRegistryResourceGroupEvaluated - params: { - name: containerRegistryName - location: location - adminUserEnabled: containerRegistryAdminUserEnabled - tags: tags - } -} - -output defaultDomain string = containerAppsEnvironment.outputs.defaultDomain -output environmentName string = containerAppsEnvironment.outputs.name -output environmentId string = containerAppsEnvironment.outputs.id - -output registryLoginServer string = containerRegistry.outputs.loginServer -output registryName string = containerRegistry.outputs.name - +metadata description = 'Creates an Azure Container Registry and an Azure Container Apps environment.' +param name string +param location string = resourceGroup().location +param tags object = {} + +param containerAppsEnvironmentName string +param containerRegistryName string +param containerRegistryResourceGroupName string = '' +param containerRegistryAdminUserEnabled bool = false +param logAnalyticsWorkspaceName string +param applicationInsightsName string = '' +param daprEnabled bool = false + +var containerRegistryResourceGroupEvaluated = !empty(containerRegistryResourceGroupName) ? resourceGroup(containerRegistryResourceGroupName) : resourceGroup() + +module containerAppsEnvironment 'container-apps-environment.bicep' = { + name: '${name}-container-apps-environment' + params: { + name: containerAppsEnvironmentName + location: location + tags: tags + logAnalyticsWorkspaceName: logAnalyticsWorkspaceName + applicationInsightsName: applicationInsightsName + daprEnabled: daprEnabled + } +} + +module containerRegistry 'container-registry.bicep' = { + name: '${name}-container-registry' + scope: containerRegistryResourceGroupEvaluated + params: { + name: containerRegistryName + location: location + adminUserEnabled: containerRegistryAdminUserEnabled + tags: tags + } +} + +output defaultDomain string = containerAppsEnvironment.outputs.defaultDomain +output environmentName string = containerAppsEnvironment.outputs.name +output environmentId string = containerAppsEnvironment.outputs.id + +output registryLoginServer string = containerRegistry.outputs.loginServer +output registryName string = containerRegistry.outputs.name +