From a5662558248b6391615ed9f37f0cd0a3bd848490 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Mon, 11 Nov 2024 10:42:24 +0200 Subject: [PATCH 01/12] Added ModerationModel ChatMemoryProvider (but might need rework). --- examples/liberty-car-booking/pom.xml | 4 + .../io/jefrajames/booking/DocRagIngestor.java | 14 ++-- .../llm/aiservice/CommonAIServiceCreator.java | 76 ++++++++++++++++--- .../smallrye/llm/spi/RegisterAIService.java | 4 + 4 files changed, 81 insertions(+), 17 deletions(-) diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index cb0c9c4..2276eb5 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -25,6 +25,10 @@ 3.13.0 3.4.0 0.34.0 + + + + ${project.build.directory}/liberty/wlp/usr/shared/resources/lib/ diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java index 70d7ff1..2be0473 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java @@ -5,21 +5,21 @@ import java.io.File; import java.util.List; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.context.Initialized; -import jakarta.enterprise.event.Observes; -import jakarta.enterprise.inject.Produces; -import jakarta.inject.Inject; - import org.eclipse.microprofile.config.inject.ConfigProperty; import dev.langchain4j.data.document.Document; import dev.langchain4j.data.document.parser.TextDocumentParser; import dev.langchain4j.data.document.splitter.DocumentSplitters; +import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; import dev.langchain4j.store.embedding.EmbeddingStoreIngestor; import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.context.Initialized; +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Inject; import lombok.extern.java.Log; @Log @@ -32,7 +32,7 @@ public class DocRagIngestor { // Used by ContentRetriever @Produces - private InMemoryEmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); + private InMemoryEmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); // private File docs = new File(System.getProperty("docragdir")); diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java index 35546d9..60d0755 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java @@ -1,26 +1,33 @@ package io.smallrye.llm.aiservice; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Parameter; import java.util.ArrayList; import java.util.List; -import jakarta.enterprise.inject.Instance; -import jakarta.enterprise.inject.literal.NamedLiteral; - import org.jboss.logging.Logger; +import dev.langchain4j.memory.chat.ChatMemoryProvider; +import dev.langchain4j.memory.chat.MessageWindowChatMemory; import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.moderation.ModerationModel; import dev.langchain4j.rag.content.retriever.ContentRetriever; import dev.langchain4j.service.AiServices; +import dev.langchain4j.service.MemoryId; +import dev.langchain4j.service.Moderate; +import dev.langchain4j.store.memory.chat.ChatMemoryStore; import io.smallrye.llm.core.langchain4j.core.config.spi.ChatMemoryFactoryProvider; import io.smallrye.llm.spi.RegisterAIService; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.literal.NamedLiteral; public class CommonAIServiceCreator { - private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); + private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); @SuppressWarnings("unchecked") - public static Object create(Instance lookup, Class interfaceClass) { + public static X create(Instance lookup, Class interfaceClass) { RegisterAIService annotation = interfaceClass.getAnnotation(RegisterAIService.class); Instance chatLanguageModel = getInstance(lookup, ChatLanguageModel.class, annotation.chatLanguageModelName()); @@ -38,7 +45,7 @@ public static Object create(Instance lookup, Class interfaceClass) { } if (annotation.tools() != null && annotation.tools().length > 0) { List tools = new ArrayList<>(annotation.tools().length); - for (Class toolClass : annotation.tools()) { + for (Class toolClass : annotation.tools()) { try { tools.add(toolClass.getConstructor(null).newInstance(null)); } catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException @@ -47,20 +54,69 @@ public static Object create(Instance lookup, Class interfaceClass) { } aiServices.tools(tools); } - aiServices.chatMemory( + + ChatMemoryProvider chatMemoryProvider = createChatMemoryProvider(lookup, interfaceClass, annotation); + if (chatMemoryProvider != null) { + aiServices.chatMemoryProvider(chatMemoryProvider); + } else { + aiServices.chatMemory( ChatMemoryFactoryProvider.getChatMemoryFactory().getChatMemory(lookup, annotation.chatMemoryMaxMessages())); - return aiServices.build(); + } + + ModerationModel moderationModel = findModerationModel(lookup, interfaceClass, annotation); + if (moderationModel != null) { + aiServices.moderationModel(moderationModel); + } + return (X)aiServices.build(); } catch (Exception e) { throw new RuntimeException(e); } } - private static Instance getInstance(Instance lookup, Class type, String name) { + private static Instance getInstance(Instance lookup, Class type, String name) { LOGGER.info("Getinstance of '" + type + "' with name '" + name + "'"); if (name == null || name.isBlank()) { return lookup.select(type); } return lookup.select(type, NamedLiteral.of(name)); } - + + private static ModerationModel findModerationModel(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + Moderate moderate = method.getAnnotation(Moderate.class); + if (moderate != null) { + Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, registerAIService.moderationModelName()); + if (moderationModelInstance != null && moderationModelInstance.isResolvable()) return moderationModelInstance.get(); + } + } + + return null; + } + + + private static ChatMemoryProvider createChatMemoryProvider(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + for (Parameter parameter : method.getParameters()) { + MemoryId memoryIdAnnotation = parameter.getAnnotation(MemoryId.class); + if (memoryIdAnnotation != null) { + Instance chatMemoryStore = getInstance(lookup, ChatMemoryStore.class, + registerAIService.chatMemoryStoreName()); + if (chatMemoryStore == null || !chatMemoryStore.isResolvable()) { + throw new IllegalStateException("Unable to resolve a ChatMemoryStore for your ChatMemoryProvider."); + } + + ChatMemoryProvider chatMemoryProvider = memoryId -> MessageWindowChatMemory.builder() + .id(memoryId) + .maxMessages(registerAIService.chatMemoryMaxMessages()) + .chatMemoryStore(chatMemoryStore.get()) + .build(); + return chatMemoryProvider; + } + } + } + + return null; + } } diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java index b48605f..91dbbea 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java @@ -30,4 +30,8 @@ String embeddingStoreName() default ""; String contentRetrieverName() default ""; + + String moderationModelName() default ""; + + String chatMemoryStoreName() default ""; } From aef662b3b76a68966128790ae05bb2339a922785 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Sat, 23 Nov 2024 19:13:29 +0200 Subject: [PATCH 02/12] Enable the Fault Tolerance Interceptor. --- .../helidon-car-booking-portable-ext/pom.xml | 2 +- examples/helidon-car-booking/pom.xml | 2 +- examples/liberty-car-booking/README.md | 2 +- examples/liberty-car-booking/pom.xml | 40 +--- .../booking/CarBookingResource.java | 2 - .../io/jefrajames/booking/ChatAiService.java | 4 +- .../io/jefrajames/booking/DocRagIngestor.java | 21 +- .../io/jefrajames/booking/FraudAiService.java | 15 +- .../META-INF/microprofile-config.properties | 10 +- .../src/main/resources/application.properties | 2 +- pom.xml | 2 +- .../llm/aiservice/CommonAIServiceCreator.java | 99 +++++----- .../llm/plugin/CommonLLMPluginCreator.java | 33 +++- .../smallrye/llm/spi/RegisterAIService.java | 4 +- .../pom.xml | 7 +- .../LangChain4JAIServiceBean.java | 182 ++++++++++++++++++ ...LangChain4JAIServicePortableExtension.java | 11 +- .../Langchain4JFaultToleranceExtension.java | 44 +++++ .../jakarta.enterprise.inject.spi.Extension | 1 + 19 files changed, 354 insertions(+), 129 deletions(-) create mode 100644 smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java create mode 100644 smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java diff --git a/examples/helidon-car-booking-portable-ext/pom.xml b/examples/helidon-car-booking-portable-ext/pom.xml index b9a2307..4be96f8 100644 --- a/examples/helidon-car-booking-portable-ext/pom.xml +++ b/examples/helidon-car-booking-portable-ext/pom.xml @@ -12,7 +12,7 @@ io.helidon.Main - 0.34.0 + 0.36.0 diff --git a/examples/helidon-car-booking/pom.xml b/examples/helidon-car-booking/pom.xml index 47ee4b0..9c41ba5 100644 --- a/examples/helidon-car-booking/pom.xml +++ b/examples/helidon-car-booking/pom.xml @@ -12,7 +12,7 @@ io.helidon.Main - 0.34.0 + 0.36.0 diff --git a/examples/liberty-car-booking/README.md b/examples/liberty-car-booking/README.md index 92561c7..3d61654 100644 --- a/examples/liberty-car-booking/README.md +++ b/examples/liberty-car-booking/README.md @@ -39,7 +39,7 @@ To package the application in JVM mode run: `mvn package`. ## Configuration -All configuration is centralized in `microprofile-config.properties`(found is `resources\META-INF` folder) and can be redefined using environment variables. +All configuration is centralized in `microprofile-config.properties` (found is `resources\META-INF` folder) and can be redefined using environment variables. ## Running the application diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 2276eb5..32b6c21 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -24,7 +24,7 @@ 6.1 3.13.0 3.4.0 - 0.34.0 + 0.36.0 @@ -52,15 +52,6 @@ provided - - io.smallrye.llm smallrye-llm-langchain4j-config-mpconfig @@ -72,11 +63,12 @@ smallrye-llm-langchain4j-portable-extension 1.0.0-SNAPSHOT + @@ -129,15 +121,6 @@ - - - org.eclipse.microprofile microprofile @@ -154,20 +137,10 @@ smallrye-llm-langchain4j-portable-extension - - @@ -175,7 +148,6 @@ org.projectlombok lombok 1.18.32 - provided diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java index 8faf5f9..1922a3b 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java @@ -40,9 +40,7 @@ public String chatWithAssistant( @Path("/fraud") public FraudResponse detectFraudForCustomer( @QueryParam("name") String name, - @QueryParam("surname") String surname) { return fraudService.detectFraudForCustomer(name, surname); } - } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java index bc6892f..660e718 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java @@ -2,6 +2,8 @@ import java.time.temporal.ChronoUnit; +import jakarta.enterprise.context.ApplicationScoped; + import org.eclipse.microprofile.faulttolerance.Fallback; import org.eclipse.microprofile.faulttolerance.Retry; import org.eclipse.microprofile.faulttolerance.Timeout; @@ -10,7 +12,7 @@ import io.smallrye.llm.spi.RegisterAIService; //@SuppressWarnings("CdiManagedBeanInconsistencyInspection") -@RegisterAIService(tools = BookingService.class, chatMemoryMaxMessages = 10) +@RegisterAIService(scope = ApplicationScoped.class, tools = BookingService.class, chatMemoryMaxMessages = 10) public interface ChatAiService { @SystemMessage(""" diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java index 2be0473..f50a395 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java @@ -5,6 +5,12 @@ import java.io.File; import java.util.List; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.context.Initialized; +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Inject; + import org.eclipse.microprofile.config.inject.ConfigProperty; import dev.langchain4j.data.document.Document; @@ -13,13 +19,9 @@ import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; import dev.langchain4j.store.embedding.EmbeddingStoreIngestor; import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.context.Initialized; -import jakarta.enterprise.event.Observes; -import jakarta.enterprise.inject.Produces; -import jakarta.inject.Inject; import lombok.extern.java.Log; @Log @@ -32,9 +34,7 @@ public class DocRagIngestor { // Used by ContentRetriever @Produces - private InMemoryEmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); - - // private File docs = new File(System.getProperty("docragdir")); + private EmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); @Inject @ConfigProperty(name = "app.docs-for-rag.dir") @@ -60,9 +60,4 @@ public void ingest(@Observes @Initialized(ApplicationScoped.class) Object pointl log.info(String.format("DEMO %d documents ingested in %d msec", docs.size(), System.currentTimeMillis() - start)); } - - public static void main(String[] args) { - - System.out.println(InMemoryEmbeddingStore.class.getInterfaces()[0]); - } } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java index 68488de..77e5086 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java @@ -1,14 +1,18 @@ package io.jefrajames.booking; +import java.time.temporal.ChronoUnit; + +import org.eclipse.microprofile.faulttolerance.Fallback; +import org.eclipse.microprofile.faulttolerance.Retry; +import org.eclipse.microprofile.faulttolerance.Timeout; + import dev.langchain4j.service.SystemMessage; import dev.langchain4j.service.UserMessage; import dev.langchain4j.service.V; import io.smallrye.llm.spi.RegisterAIService; -@SuppressWarnings("CdiManagedBeanInconsistencyInspection") -@RegisterAIService(chatMemoryMaxMessages = 5, - - chatLanguageModelName = "chat-model") +//@SuppressWarnings("CdiManagedBeanInconsistencyInspection") +@RegisterAIService(chatMemoryMaxMessages = 5, chatLanguageModelName = "chat-model") public interface FraudAiService { @SystemMessage(""" @@ -44,6 +48,9 @@ A booking overlap (and hence a fraud) occurs when there are several bookings for You must not wrap JSON response in backticks, markdown, or in any other way, but return it as plain text. """) + @Timeout(unit = ChronoUnit.MINUTES, value = 5) + @Retry(maxRetries = 2) + @Fallback(fallbackMethod = "fraudFallback") FraudResponse detectFraudForCustomer(@V("name") String name, @V("surname") String surname); default FraudResponse fraudFallback(String name, String surname) { diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties index c793531..ef84021 100644 --- a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties +++ b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties @@ -2,15 +2,15 @@ server.port=9080 smallrye.llm.plugin.chat-model.class=dev.langchain4j.model.azure.AzureOpenAiChatModel -smallrye.llm.plugin.chat-model.config.api-key=${azure.openai.api.key} -smallrye.llm.plugin.chat-model.config.endpoint=${azure.openai.endpoint} -smallrye.llm.plugin.chat-model.config.service-version=2024-02-15-preview -smallrye.llm.plugin.chat-model.config.deployment-name=${azure.openai.deployment.name} +smallrye.llm.plugin.chat-model.config.api-key=2b4a74c3acd64272a374bfeb84925068 +smallrye.llm.plugin.chat-model.config.endpoint=https://sindi-openai.openai.azure.com/ +smallrye.llm.plugin.chat-model.config.service-version=2023-03-15-preview +smallrye.llm.plugin.chat-model.config.deployment-name=gpt-4 smallrye.llm.plugin.chat-model.config.temperature=0.1 smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s smallrye.llm.plugin.chat-model.config.max-retries=2 -#smallrye.llm.plugin.chat-model.config.logRequestsAndResponsess=false +#smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=true smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever diff --git a/examples/quarkus-car-booking/src/main/resources/application.properties b/examples/quarkus-car-booking/src/main/resources/application.properties index 08f08e3..d9ab556 100644 --- a/examples/quarkus-car-booking/src/main/resources/application.properties +++ b/examples/quarkus-car-booking/src/main/resources/application.properties @@ -11,7 +11,7 @@ smallrye.llm.plugin.chat-model.config.temperature=0.1 smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s smallrye.llm.plugin.chat-model.config.max-retries=2 -#smallrye.llm.plugin.chat-model.config.logRequestsAndResponsess=false +#smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=false smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever diff --git a/pom.xml b/pom.xml index 881e44b..ad4d5f1 100644 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,7 @@ 4.0.0.Final 5.1.3.Final 3.5.3.Final - 0.34.0 + 0.36.0 17 17 diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java index 60d0755..1db83a6 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java @@ -6,6 +6,9 @@ import java.util.ArrayList; import java.util.List; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.literal.NamedLiteral; + import org.jboss.logging.Logger; import dev.langchain4j.memory.chat.ChatMemoryProvider; @@ -19,12 +22,10 @@ import dev.langchain4j.store.memory.chat.ChatMemoryStore; import io.smallrye.llm.core.langchain4j.core.config.spi.ChatMemoryFactoryProvider; import io.smallrye.llm.spi.RegisterAIService; -import jakarta.enterprise.inject.Instance; -import jakarta.enterprise.inject.literal.NamedLiteral; public class CommonAIServiceCreator { - private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); + private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); @SuppressWarnings("unchecked") public static X create(Instance lookup, Class interfaceClass) { @@ -54,20 +55,21 @@ public static X create(Instance lookup, Class interfaceClass) { } aiServices.tools(tools); } - + ChatMemoryProvider chatMemoryProvider = createChatMemoryProvider(lookup, interfaceClass, annotation); if (chatMemoryProvider != null) { - aiServices.chatMemoryProvider(chatMemoryProvider); + aiServices.chatMemoryProvider(chatMemoryProvider); } else { - aiServices.chatMemory( - ChatMemoryFactoryProvider.getChatMemoryFactory().getChatMemory(lookup, annotation.chatMemoryMaxMessages())); + aiServices.chatMemory( + ChatMemoryFactoryProvider.getChatMemoryFactory().getChatMemory(lookup, + annotation.chatMemoryMaxMessages())); } - + ModerationModel moderationModel = findModerationModel(lookup, interfaceClass, annotation); if (moderationModel != null) { - aiServices.moderationModel(moderationModel); + aiServices.moderationModel(moderationModel); } - return (X)aiServices.build(); + return (X) aiServices.build(); } catch (Exception e) { throw new RuntimeException(e); } @@ -80,43 +82,46 @@ private static Instance getInstance(Instance lookup, Class typ } return lookup.select(type, NamedLiteral.of(name)); } - - private static ModerationModel findModerationModel(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { - //Get all methods. - for (Method method : interfaceClass.getMethods()) { - Moderate moderate = method.getAnnotation(Moderate.class); - if (moderate != null) { - Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, registerAIService.moderationModelName()); - if (moderationModelInstance != null && moderationModelInstance.isResolvable()) return moderationModelInstance.get(); - } - } - - return null; + + private static ModerationModel findModerationModel(Instance lookup, Class interfaceClass, + RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + Moderate moderate = method.getAnnotation(Moderate.class); + if (moderate != null) { + Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, + registerAIService.moderationModelName()); + if (moderationModelInstance != null && moderationModelInstance.isResolvable()) + return moderationModelInstance.get(); + } + } + + return null; } - - - private static ChatMemoryProvider createChatMemoryProvider(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { - //Get all methods. - for (Method method : interfaceClass.getMethods()) { - for (Parameter parameter : method.getParameters()) { - MemoryId memoryIdAnnotation = parameter.getAnnotation(MemoryId.class); - if (memoryIdAnnotation != null) { - Instance chatMemoryStore = getInstance(lookup, ChatMemoryStore.class, - registerAIService.chatMemoryStoreName()); - if (chatMemoryStore == null || !chatMemoryStore.isResolvable()) { - throw new IllegalStateException("Unable to resolve a ChatMemoryStore for your ChatMemoryProvider."); - } - - ChatMemoryProvider chatMemoryProvider = memoryId -> MessageWindowChatMemory.builder() - .id(memoryId) - .maxMessages(registerAIService.chatMemoryMaxMessages()) - .chatMemoryStore(chatMemoryStore.get()) - .build(); - return chatMemoryProvider; - } - } - } - - return null; + + private static ChatMemoryProvider createChatMemoryProvider(Instance lookup, Class interfaceClass, + RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + for (Parameter parameter : method.getParameters()) { + MemoryId memoryIdAnnotation = parameter.getAnnotation(MemoryId.class); + if (memoryIdAnnotation != null) { + Instance chatMemoryStore = getInstance(lookup, ChatMemoryStore.class, + registerAIService.chatMemoryStoreName()); + if (chatMemoryStore == null || !chatMemoryStore.isResolvable()) { + throw new IllegalStateException("Unable to resolve a ChatMemoryStore for your ChatMemoryProvider."); + } + + ChatMemoryProvider chatMemoryProvider = memoryId -> MessageWindowChatMemory.builder() + .id(memoryId) + .maxMessages(registerAIService.chatMemoryMaxMessages()) + .chatMemoryStore(chatMemoryStore.get()) + .build(); + return chatMemoryProvider; + } + } + } + + return null; } } diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java index 6d5525b..f46f912 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java @@ -6,7 +6,9 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -15,10 +17,12 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.literal.NamedLiteral; -import jakarta.enterprise.inject.spi.CDI; +import jakarta.enterprise.util.TypeLiteral; import org.jboss.logging.Logger; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfig; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfigProvider; @@ -34,6 +38,13 @@ public class CommonLLMPluginCreator { public static final Logger LOGGER = Logger.getLogger(CommonLLMPluginCreator.class); + private static final Map, TypeLiteral> TYPE_LITERALS = new HashMap<>(); + + static { + TYPE_LITERALS.put(EmbeddingStore.class, new TypeLiteral>() { + }); + } + @SuppressWarnings("unchecked") public static void createAllLLMBeans(LLMConfig llmConfig, Consumer beanBuilder) throws ClassNotFoundException { Set beanNameToCreate = llmConfig.getBeanNames(); @@ -135,12 +146,9 @@ public static Object create(Instance lookup, String beanName, Class t LOGGER.info("Lookup " + lookupableBean + " " + parameterType); Instance inst; if ("default".equals(lookupableBean)) { - inst = lookup.select(parameterType); - if (!inst.isResolvable()) { - inst = CDI.current().select(parameterType); - } + inst = getInstance(lookup, parameterType); } else { - inst = lookup.select(parameterType, NamedLiteral.of(lookupableBean)); + inst = getInstance(lookup, parameterType, lookupableBean); } methodToCall.invoke(builder, inst.get()); break; @@ -165,4 +173,17 @@ public static Object create(Instance lookup, String beanName, Class t private static Class loadClass(String scopeClassName) throws ClassNotFoundException { return Thread.currentThread().getContextClassLoader().loadClass(scopeClassName); } + + @SuppressWarnings("unchecked") + private static Instance getInstance(Instance lookup, Class clazz) { + if (TYPE_LITERALS.containsKey(clazz)) + return (Instance) lookup.select(TYPE_LITERALS.get(clazz)); + return lookup.select(clazz); + } + + private static Instance getInstance(Instance lookup, Class clazz, String lookupName) { + if (lookupName == null || lookupName.isBlank()) + return getInstance(lookup, clazz); + return lookup.select(clazz, NamedLiteral.of(lookupName)); + } } diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java index 91dbbea..733073a 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java @@ -30,8 +30,8 @@ String embeddingStoreName() default ""; String contentRetrieverName() default ""; - + String moderationModelName() default ""; - + String chatMemoryStoreName() default ""; } diff --git a/smallrye-llm-langchain4j-portable-extension/pom.xml b/smallrye-llm-langchain4j-portable-extension/pom.xml index 0bbf84d..e96eee9 100644 --- a/smallrye-llm-langchain4j-portable-extension/pom.xml +++ b/smallrye-llm-langchain4j-portable-extension/pom.xml @@ -30,7 +30,12 @@ org.jboss.logging jboss-logging - + + + io.smallrye + smallrye-fault-tolerance + 6.6.2 + diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java new file mode 100644 index 0000000..6a04751 --- /dev/null +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java @@ -0,0 +1,182 @@ +package io.smallrye.llm.core.langchain4j.portableextension; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import jakarta.enterprise.context.spi.CreationalContext; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Default; +import jakarta.enterprise.inject.spi.Bean; +import jakarta.enterprise.inject.spi.BeanManager; +import jakarta.enterprise.inject.spi.CDI; +import jakarta.enterprise.inject.spi.InjectionPoint; +import jakarta.enterprise.inject.spi.InterceptionFactory; +import jakarta.enterprise.inject.spi.PassivationCapable; +import jakarta.enterprise.util.AnnotationLiteral; + +import io.smallrye.faulttolerance.FaultToleranceBinding.Literal; +import io.smallrye.llm.aiservice.CommonAIServiceCreator; +import io.smallrye.llm.spi.RegisterAIService; + +/** + * @author Buhake Sindi + * @since 21 November 2024 + */ +public class LangChain4JAIServiceBean implements Bean, PassivationCapable { + + private final Class aiServiceInterfaceClass; + + private final BeanManager beanManager; + + private final Class scope; + + /** + * @param aiServiceInterfaceClass + * @param beanManager + */ + public LangChain4JAIServiceBean(Class aiServiceInterfaceClass, BeanManager beanManager) { + super(); + final RegisterAIService annotation = (this.aiServiceInterfaceClass = aiServiceInterfaceClass) + .getAnnotation(RegisterAIService.class); + this.scope = annotation.scope(); + this.beanManager = beanManager; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.PassivationCapable#getId() + */ + @Override + public String getId() { + // TODO Auto-generated method stub + return aiServiceInterfaceClass.getName(); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.context.spi.Contextual#create(jakarta.enterprise.context.spi.CreationalContext) + */ + @Override + public T create(CreationalContext creationalContext) { + // TODO Auto-generated method stub + InterceptionFactory factory = beanManager.createInterceptionFactory(creationalContext, aiServiceInterfaceClass); + factory.configure().add(new Literal()); + return factory.createInterceptedInstance(CommonAIServiceCreator.create(CDI.current(), aiServiceInterfaceClass)); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.context.spi.Contextual#destroy(java.lang.Object, + * jakarta.enterprise.context.spi.CreationalContext) + */ + @Override + public void destroy(T instance, CreationalContext creationalContext) { + // TODO Auto-generated method stub + + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getTypes() + */ + @Override + public Set getTypes() { + // TODO Auto-generated method stub + return Collections.singleton(aiServiceInterfaceClass); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getQualifiers() + */ + @Override + public Set getQualifiers() { + // TODO Auto-generated method stub + Set annotations = new HashSet<>(); + annotations.add(new AnnotationLiteral() { + }); + annotations.add(new AnnotationLiteral() { + }); + return Collections.unmodifiableSet(annotations); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getScope() + */ + @Override + public Class getScope() { + // TODO Auto-generated method stub + return scope; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getName() + */ + @Override + public String getName() { + // TODO Auto-generated method stub + return "registeredAIService-" + aiServiceInterfaceClass.getName(); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getStereotypes() + */ + @Override + public Set> getStereotypes() { + // TODO Auto-generated method stub + return Collections.singleton(RegisterAIService.class); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#isAlternative() + */ + @Override + public boolean isAlternative() { + // TODO Auto-generated method stub + return false; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.Bean#getBeanClass() + */ + @Override + public Class getBeanClass() { + // TODO Auto-generated method stub + return aiServiceInterfaceClass; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.Bean#getInjectionPoints() + */ + @Override + public Set getInjectionPoints() { + // TODO Auto-generated method stub + return Collections.emptySet(); + } + + @Override + public String toString() { + return "AiService [ interfaceType: " + aiServiceInterfaceClass.getSimpleName() + " ] with Qualifiers [" + + getQualifiers() + "]"; + } +} diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java index 304de6c..f5cf6f0 100644 --- a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java @@ -10,7 +10,6 @@ import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.spi.AfterBeanDiscovery; import jakarta.enterprise.inject.spi.BeanManager; -import jakarta.enterprise.inject.spi.CDI; import jakarta.enterprise.inject.spi.Extension; import jakarta.enterprise.inject.spi.InjectionPoint; import jakarta.enterprise.inject.spi.ProcessAnnotatedType; @@ -19,7 +18,6 @@ import org.jboss.logging.Logger; -import io.smallrye.llm.aiservice.CommonAIServiceCreator; import io.smallrye.llm.spi.RegisterAIService; public class LangChain4JAIServicePortableExtension implements Extension { @@ -63,13 +61,8 @@ void processInjectionPoints(@Observes ProcessInjectionPoint event) { void afterBeanDiscovery(@Observes AfterBeanDiscovery afterBeanDiscovery, BeanManager beanManager) throws ClassNotFoundException { for (Class aiServiceClass : detectedAIServicesDeclaredInterfaces) { - LOGGER.info("afterBeanDiscovery create synthetic : " + aiServiceClass.getName()); - final RegisterAIService annotation = aiServiceClass.getAnnotation(RegisterAIService.class); - afterBeanDiscovery.addBean() - .types(aiServiceClass) - .scope(annotation.scope()) - .name("registeredAIService-" + aiServiceClass.getName()) //Without this, the container won't create a CreationalContext - .createWith(creationalContext -> CommonAIServiceCreator.create(CDI.current(), aiServiceClass)); + LOGGER.info("afterBeanDiscovery create synthetic: " + aiServiceClass.getName()); + afterBeanDiscovery.addBean(new LangChain4JAIServiceBean<>(aiServiceClass, beanManager)); } } diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java new file mode 100644 index 0000000..fc1cd09 --- /dev/null +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java @@ -0,0 +1,44 @@ +package io.smallrye.llm.core.langchain4j.portableextension; + +import java.util.logging.Logger; + +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.spi.AnnotatedMethod; +import jakarta.enterprise.inject.spi.AnnotatedType; +import jakarta.enterprise.inject.spi.BeanManager; +import jakarta.enterprise.inject.spi.DeploymentException; +import jakarta.enterprise.inject.spi.Extension; +import jakarta.enterprise.inject.spi.ProcessSyntheticBean; + +import org.eclipse.microprofile.faulttolerance.exceptions.FaultToleranceDefinitionException; + +import io.smallrye.faulttolerance.autoconfig.FaultToleranceMethod; +import io.smallrye.faulttolerance.config.FaultToleranceMethods; +import io.smallrye.faulttolerance.config.FaultToleranceOperation; + +/** + * @author Buhake Sindi + * @since 29 November 2024 + */ +public class Langchain4JFaultToleranceExtension implements Extension { + + private static final Logger LOGGER = Logger.getLogger(Langchain4JFaultToleranceExtension.class.getName()); + + void validateFaultToleranceOperations(@Observes ProcessSyntheticBean event, BeanManager bm) { + LOGGER.info("validateFaultToleranceOperations: Synthetic Event -> " + event.getBean().getBeanClass()); + try { + AnnotatedType annotatedType = bm.createAnnotatedType(event.getBean().getBeanClass()); + for (AnnotatedMethod annotatedMethod : annotatedType.getMethods()) { + FaultToleranceMethod method = FaultToleranceMethods.create(annotatedType.getJavaClass(), annotatedMethod); + if (method.isLegitimate()) { + FaultToleranceOperation operation = FaultToleranceOperation.create(method); + operation.validate(); + LOGGER.info("Found: " + operation); + } + } + } catch (FaultToleranceDefinitionException e) { + // TODO Auto-generated catch block + throw new DeploymentException(e); + } + } +} diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension b/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension index 4f3c31b..decdbc2 100644 --- a/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension +++ b/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension @@ -1,2 +1,3 @@ io.smallrye.llm.core.langchain4j.portableextension.LangChain4JPluginsPortableExtension io.smallrye.llm.core.langchain4j.portableextension.LangChain4JAIServicePortableExtension +io.smallrye.llm.core.langchain4j.portableextension.Langchain4JFaultToleranceExtension From 2b4eecb4599ea22a0de4a0a0253799fc8bbc77e9 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Sat, 23 Nov 2024 19:18:36 +0200 Subject: [PATCH 03/12] Enable the Fault Tolerance Interceptor. --- examples/liberty-car-booking/pom.xml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 32b6c21..eeffcd3 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -64,14 +64,6 @@ 1.0.0-SNAPSHOT - - dev.langchain4j langchain4j @@ -137,13 +129,6 @@ smallrye-llm-langchain4j-portable-extension - - org.projectlombok lombok From 9413a6c3d27a803416dffc41ad0838525faaa8d5 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Wed, 27 Nov 2024 21:52:54 +0200 Subject: [PATCH 04/12] Added support for Metrics and Telemetry tracing using Microprofile Metrics & Microprofile Telemetry respectively. --- examples/liberty-car-booking/pom.xml | 22 +++ .../io/jefrajames/booking/ChatAiService.java | 4 +- .../src/main/liberty/config/server.xml | 4 + .../META-INF/microprofile-config.properties | 5 + pom.xml | 15 ++ .../pom.xml | 1 - .../llm/plugin/CommonLLMPluginCreator.java | 18 ++- smallrye-llm-langchain4j-metrics/pom.xml | 35 +++++ .../metrics/MetricsChatModelListener.java | 144 ++++++++++++++++++ smallrye-llm-langchain4j-telemetry/pom.xml | 36 +++++ .../telemetry/SpanChatModelListener.java | 106 +++++++++++++ 11 files changed, 385 insertions(+), 5 deletions(-) create mode 100644 smallrye-llm-langchain4j-metrics/pom.xml create mode 100644 smallrye-llm-langchain4j-metrics/src/main/java/io/smallrye/llm/langchain4j/metrics/MetricsChatModelListener.java create mode 100644 smallrye-llm-langchain4j-telemetry/pom.xml create mode 100644 smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index eeffcd3..229623c 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -64,6 +64,18 @@ 1.0.0-SNAPSHOT + + io.smallrye.llm + smallrye-llm-langchain4j-metrics + 1.0.0-SNAPSHOT + + + + io.smallrye.llm + smallrye-llm-langchain4j-telemetry + 1.0.0-SNAPSHOT + + dev.langchain4j langchain4j @@ -129,6 +141,16 @@ smallrye-llm-langchain4j-portable-extension + + io.smallrye.llm + smallrye-llm-langchain4j-metrics + + + + io.smallrye.llm + smallrye-llm-langchain4j-telemetry + + org.projectlombok lombok diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java index 660e718..bc6892f 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java @@ -2,8 +2,6 @@ import java.time.temporal.ChronoUnit; -import jakarta.enterprise.context.ApplicationScoped; - import org.eclipse.microprofile.faulttolerance.Fallback; import org.eclipse.microprofile.faulttolerance.Retry; import org.eclipse.microprofile.faulttolerance.Timeout; @@ -12,7 +10,7 @@ import io.smallrye.llm.spi.RegisterAIService; //@SuppressWarnings("CdiManagedBeanInconsistencyInspection") -@RegisterAIService(scope = ApplicationScoped.class, tools = BookingService.class, chatMemoryMaxMessages = 10) +@RegisterAIService(tools = BookingService.class, chatMemoryMaxMessages = 10) public interface ChatAiService { @SystemMessage(""" diff --git a/examples/liberty-car-booking/src/main/liberty/config/server.xml b/examples/liberty-car-booking/src/main/liberty/config/server.xml index 2653bc7..138c567 100644 --- a/examples/liberty-car-booking/src/main/liberty/config/server.xml +++ b/examples/liberty-car-booking/src/main/liberty/config/server.xml @@ -3,12 +3,16 @@ microProfile-6.1 + mpTelemetry-2.0 + + + diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties index ef84021..3482fb4 100644 --- a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties +++ b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties @@ -11,6 +11,7 @@ smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s smallrye.llm.plugin.chat-model.config.max-retries=2 #smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=true +smallrye.llm.plugin.chat-model.config.listeners=@all smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever @@ -31,3 +32,7 @@ fraud.memory.max.messages=20 # Location of documents to RAG app.docs-for-rag.dir=docs-for-rag + +otel.sdk.disabled=false +otel.service.name=api +otel.traces.exporter=otlp diff --git a/pom.xml b/pom.xml index ad4d5f1..ee638ab 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,8 @@ 3.1 + 5.1.2 + 2.0.1 2.4.0 9.7 2.3.0 @@ -79,6 +81,8 @@ smallrye-llm-langchain4j-buildcompatible-extension smallrye-llm-langchain4j-core smallrye-llm-langchain4j-config-mpconfig + smallrye-llm-langchain4j-metrics + smallrye-llm-langchain4j-telemetry @@ -88,6 +92,17 @@ microprofile-config-api ${version.eclipse.microprofile.config} + + org.eclipse.microprofile.metrics + microprofile-metrics-api + ${version.eclipse.microprofile.metrics} + + + org.eclipse.microprofile.telemetry + microprofile-telemetry-api + ${version.eclipse.microprofile.telemetry} + pom + io.smallrye.common smallrye-common-bom diff --git a/smallrye-llm-langchain4j-config-mpconfig/pom.xml b/smallrye-llm-langchain4j-config-mpconfig/pom.xml index d80a581..b3d7354 100644 --- a/smallrye-llm-langchain4j-config-mpconfig/pom.xml +++ b/smallrye-llm-langchain4j-config-mpconfig/pom.xml @@ -28,5 +28,4 @@ ${project.version} - diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java index f46f912..ea1c846 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java @@ -13,6 +13,7 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Instance; @@ -22,6 +23,8 @@ import org.jboss.logging.Logger; import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.chat.listener.ChatModelListener; import dev.langchain4j.store.embedding.EmbeddingStore; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfig; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfigProvider; @@ -141,7 +144,20 @@ public static Object create(Instance lookup, String beanName, Class t } else { for (Method methodToCall : methodsToCall) { Class parameterType = methodToCall.getParameterTypes()[0]; - if (stringValue.startsWith("lookup:")) { + if ("listeners".equals(property) && "@all".equals(stringValue)) { + Class typeParameterClass = ChatLanguageModel.class.isAssignableFrom(targetClass) + ? ChatModelListener.class + : parameterType.getTypeParameters()[0].getGenericDeclaration(); + Instance inst = getInstance(lookup, typeParameterClass); + if (inst != null) { + List listeners = StreamSupport.stream(inst.spliterator(), false) + .collect(Collectors.toList()); + if (listeners != null && !listeners.isEmpty()) { + listeners.stream().forEach(l -> LOGGER.info("Adding listener: " + l.getClass().getName())); + methodToCall.invoke(builder, listeners); + } + } + } else if (stringValue.startsWith("lookup:")) { String lookupableBean = stringValue.substring("lookup:".length()); LOGGER.info("Lookup " + lookupableBean + " " + parameterType); Instance inst; diff --git a/smallrye-llm-langchain4j-metrics/pom.xml b/smallrye-llm-langchain4j-metrics/pom.xml new file mode 100644 index 0000000..197a25e --- /dev/null +++ b/smallrye-llm-langchain4j-metrics/pom.xml @@ -0,0 +1,35 @@ + + 4.0.0 + + io.smallrye.llm + smallrye-llm-parent + 1.0.0-SNAPSHOT + + smallrye-llm-langchain4j-metrics + + + + jakarta.enterprise + jakarta.enterprise.cdi-api + provided + + + org.eclipse.microprofile.metrics + microprofile-metrics-api + provided + + + io.smallrye.llm + smallrye-llm-langchain4j-core + ${project.version} + + + + io.smallrye + smallrye-metrics + 5.1.0 + + + \ No newline at end of file diff --git a/smallrye-llm-langchain4j-metrics/src/main/java/io/smallrye/llm/langchain4j/metrics/MetricsChatModelListener.java b/smallrye-llm-langchain4j-metrics/src/main/java/io/smallrye/llm/langchain4j/metrics/MetricsChatModelListener.java new file mode 100644 index 0000000..c7597ca --- /dev/null +++ b/smallrye-llm-langchain4j-metrics/src/main/java/io/smallrye/llm/langchain4j/metrics/MetricsChatModelListener.java @@ -0,0 +1,144 @@ +package io.smallrye.llm.langchain4j.metrics; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import jakarta.enterprise.context.Dependent; + +import org.eclipse.microprofile.metrics.Metadata; +import org.eclipse.microprofile.metrics.MetadataBuilder; +import org.eclipse.microprofile.metrics.MetricRegistry; +import org.eclipse.microprofile.metrics.MetricUnits; +import org.eclipse.microprofile.metrics.Tag; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.model.chat.listener.ChatModelErrorContext; +import dev.langchain4j.model.chat.listener.ChatModelListener; +import dev.langchain4j.model.chat.listener.ChatModelRequest; +import dev.langchain4j.model.chat.listener.ChatModelRequestContext; +import dev.langchain4j.model.chat.listener.ChatModelResponse; +import dev.langchain4j.model.chat.listener.ChatModelResponseContext; +import io.smallrye.metrics.SharedMetricRegistries; + +/** + * Creates metrics that follow the + * Semantic Conventions + * for GenAI Metrics. + * + * @author Buhake Sindi + * @since 25 November 2024 + */ +@Dependent +public class MetricsChatModelListener implements ChatModelListener { + + private static final String MP_AI_METRIC_START_TIME_NAME = "MP_AI_METRIC_START_TIME"; + + private static final String METRIC_CLIENT_TOKEN_USAGE_NAME = "gen_ai.client.token.usage"; + private static final String METRIC_CLIENT_OPERATION_DURATION_NAME = "gen_ai.client.operation.duration"; + // private static final String METRIC_SERVER_REQUEST_DURATION_NAME = "gen_ai.server.request.duration"; + private static final Tag OPERATION_NAME_TAG = new Tag("gen_ai.operation.name", "chat"); + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onRequest(dev.langchain4j.model.chat.listener. + * ChatModelRequestContext) + */ + @Override + public void onRequest(ChatModelRequestContext requestContext) { + // TODO Auto-generated method stub + requestContext.attributes().put(MP_AI_METRIC_START_TIME_NAME, System.nanoTime()); + } + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onResponse(dev.langchain4j.model.chat.listener. + * ChatModelResponseContext) + */ + @Override + public void onResponse(ChatModelResponseContext responseContext) { + // TODO Auto-generated method stub + final long endTime = System.nanoTime(); + final long startTime = (Long) responseContext.attributes().get(MP_AI_METRIC_START_TIME_NAME); + + final ChatModelRequest request = responseContext.request(); + final ChatModelResponse response = responseContext.response(); + Tag requestModelTag = new Tag("gen_ai.request.model", request.model()); + Tag responseModelTag = new Tag("gen_ai.response.model", response.model()); + + List inputTokenCountTags = List.of(OPERATION_NAME_TAG, requestModelTag, responseModelTag, + new Tag("gen_ai.token.type", "input")); + List outputTokenCountTags = List.of(OPERATION_NAME_TAG, requestModelTag, responseModelTag, + new Tag("gen_ai.token.type", "output")); + List durationTags = List.of(OPERATION_NAME_TAG, requestModelTag, responseModelTag); + + recordInputTokenUsage(response.tokenUsage().inputTokenCount(), + inputTokenCountTags.toArray(new Tag[inputTokenCountTags.size()])); + recordOutputTokenUsage(response.tokenUsage().outputTokenCount(), + outputTokenCountTags.toArray(new Tag[inputTokenCountTags.size()])); + recordDuration(startTime, endTime, durationTags.toArray(new Tag[durationTags.size()])); + } + + /* + * (non-Javadoc) + * + * @see + * dev.langchain4j.model.chat.listener.ChatModelListener#onError(dev.langchain4j.model.chat.listener.ChatModelErrorContext) + */ + @Override + public void onError(ChatModelErrorContext errorContext) { + // TODO Auto-generated method stub + final long endTime = System.nanoTime(); + final long startTime = (Long) errorContext.attributes().get(MP_AI_METRIC_START_TIME_NAME); + final ChatModelRequest request = errorContext.request(); + final ChatModelResponse response = errorContext.partialResponse(); + Tag requestModelTag = new Tag("gen_ai.request.model", request.model()); + Tag responseModelTag = new Tag("gen_ai.response.model", response.model()); + + StringBuilder sb = new StringBuilder() + .append(errorContext.error().getClass().getName()); + + AiMessage aiMessage = errorContext.partialResponse().aiMessage(); + if (aiMessage != null) { + sb.append(";" + aiMessage.text()); + } + + Tag errorType = new Tag("error.type", sb.toString()); + List tags = List.of(OPERATION_NAME_TAG, requestModelTag, responseModelTag, errorType); + recordDuration(startTime, endTime, tags.toArray(new Tag[tags.size()])); + } + + private MetadataBuilder createMetadataBuilder(final String name, final String description) { + return Metadata.builder() + .withName(name) + .withDescription(description); + } + + private MetadataBuilder createMetadataBuilder(final String name, final String description, final String metricUnit) { + return createMetadataBuilder(name, description) + .withUnit(metricUnit); + } + + private void recordInputTokenUsage(final long tokenCount, final Tag[] tags) { + Metadata inputTokenUsageMetadata = createMetadataBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME, + "Measures number of input tokens used").build(); + MetricRegistry registry = SharedMetricRegistries.getOrCreate(MetricRegistry.BASE_SCOPE); + registry.histogram(inputTokenUsageMetadata, tags).update(tokenCount); + } + + private void recordOutputTokenUsage(final long tokenCount, final Tag[] tags) { + Metadata ouputTokenUsageMetadata = createMetadataBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME, + "Measures number of output tokens used").build(); + MetricRegistry registry = SharedMetricRegistries.getOrCreate(MetricRegistry.BASE_SCOPE); + registry.histogram(ouputTokenUsageMetadata, tags).update(tokenCount); + } + + private void recordDuration(final long startTime, long endTime, final Tag[] tags) { + Metadata durationMetadata = createMetadataBuilder(METRIC_CLIENT_OPERATION_DURATION_NAME, "GenAI operation duration", + MetricUnits.NANOSECONDS).build(); + MetricRegistry registry = SharedMetricRegistries.getOrCreate(MetricRegistry.BASE_SCOPE); + registry.timer(durationMetadata, tags).update(Duration.of(endTime - startTime, ChronoUnit.NANOS)); + } +} diff --git a/smallrye-llm-langchain4j-telemetry/pom.xml b/smallrye-llm-langchain4j-telemetry/pom.xml new file mode 100644 index 0000000..be4dc0f --- /dev/null +++ b/smallrye-llm-langchain4j-telemetry/pom.xml @@ -0,0 +1,36 @@ + + 4.0.0 + + io.smallrye.llm + smallrye-llm-parent + 1.0.0-SNAPSHOT + + smallrye-llm-langchain4j-telemetry + + + + jakarta.enterprise + jakarta.enterprise.cdi-api + provided + + + org.eclipse.microprofile.telemetry + microprofile-telemetry-api + pom + provided + + + io.smallrye.llm + smallrye-llm-langchain4j-core + ${project.version} + + + + io.smallrye + smallrye-metrics + 5.1.0 + + + \ No newline at end of file diff --git a/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java new file mode 100644 index 0000000..431c68a --- /dev/null +++ b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java @@ -0,0 +1,106 @@ +package io.smallrye.llm.langchain4j.telemetry; + +import jakarta.enterprise.context.Dependent; +import jakarta.inject.Inject; + +import dev.langchain4j.model.chat.listener.ChatModelErrorContext; +import dev.langchain4j.model.chat.listener.ChatModelListener; +import dev.langchain4j.model.chat.listener.ChatModelRequest; +import dev.langchain4j.model.chat.listener.ChatModelRequestContext; +import dev.langchain4j.model.chat.listener.ChatModelResponse; +import dev.langchain4j.model.chat.listener.ChatModelResponseContext; +import dev.langchain4j.model.output.TokenUsage; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; + +/** + * Creates metrics that follow the + * Semantic Conventions + * for GenAI spans. + * + * @author Buhake Sindi + * @since 25 November 2024 + */ +@Dependent +public class SpanChatModelListener implements ChatModelListener { + + private static final String OTEL_SCOPE_KEY_NAME = "OTelScope"; + private static final String OTEL_SPAN_KEY_NAME = "OTelSpan"; + + @Inject + private Tracer tracer; + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onRequest(dev.langchain4j.model.chat.listener. + * ChatModelRequestContext) + */ + @Override + public void onRequest(ChatModelRequestContext requestContext) { + // TODO Auto-generated method stub + final ChatModelRequest request = requestContext.request(); + Span span = tracer.spanBuilder("chat " + request.model()) + .setAttribute("gen_ai.operation.name", "chat") + .setAttribute("gen_ai.request.max_tokens", request.maxTokens()) + .setAttribute("gen_ai.request.temperature", request.temperature()) + .setAttribute("gen_ai.request.top_p", request.topP()) + .startSpan(); + Scope scope = span.makeCurrent(); + + requestContext.attributes().put(OTEL_SCOPE_KEY_NAME, scope); + requestContext.attributes().put(OTEL_SPAN_KEY_NAME, span); + } + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onResponse(dev.langchain4j.model.chat.listener. + * ChatModelResponseContext) + */ + @Override + public void onResponse(ChatModelResponseContext responseContext) { + // TODO Auto-generated method stub + Span span = (Span) responseContext.attributes().get(OTEL_SPAN_KEY_NAME); + if (span != null) { + ChatModelResponse response = responseContext.response(); + span.setAttribute("gen_ai.response.id", response.id()) + .setAttribute("gen_ai.response.model", response.model()); + if (response.finishReason() != null) { + span.setAttribute("gen_ai.response.finish_reasons", response.finishReason().toString()); + } + TokenUsage tokenUsage = response.tokenUsage(); + if (tokenUsage != null) { + span.setAttribute("gen_ai.usage.output_tokens", tokenUsage.outputTokenCount()) + .setAttribute("gen_ai.usage.input_tokens", tokenUsage.inputTokenCount()); + } + span.end(); + } + + closeScope((Scope) responseContext.attributes().get(OTEL_SCOPE_KEY_NAME)); + } + + /* + * (non-Javadoc) + * + * @see + * dev.langchain4j.model.chat.listener.ChatModelListener#onError(dev.langchain4j.model.chat.listener.ChatModelErrorContext) + */ + @Override + public void onError(ChatModelErrorContext errorContext) { + // TODO Auto-generated method stub + Span span = (Span) errorContext.attributes().get(OTEL_SPAN_KEY_NAME); + if (span != null) { + span.recordException(errorContext.error()); + } + + closeScope((Scope) errorContext.attributes().get(OTEL_SCOPE_KEY_NAME)); + } + + private void closeScope(Scope scope) { + if (scope != null) { + scope.close(); + } + } +} From 7c2d0b606e2dfe5ceabadb78b06650538b08c9ad Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Fri, 29 Nov 2024 01:23:39 +0200 Subject: [PATCH 05/12] Rewrote the Metrics using Microprofile Telemetry API instead of MP Metrics as previously. --- examples/liberty-car-booking/pom.xml | 11 -- .../java/io/jefrajames/booking/Booking.java | 162 +++++++++++++++++- .../io/jefrajames/booking/BookingService.java | 13 +- .../java/io/jefrajames/booking/Customer.java | 100 ++++++++++- .../io/jefrajames/booking/DocRagIngestor.java | 7 +- .../io/jefrajames/booking/DummyLLConfig.java | 3 +- pom.xml | 6 - smallrye-llm-langchain4j-metrics/pom.xml | 35 ---- .../metrics/MetricsChatModelListener.java | 144 ---------------- smallrye-llm-langchain4j-telemetry/pom.xml | 6 - .../telemetry/MetricsChatModelListener.java | 145 ++++++++++++++++ .../telemetry/SpanChatModelListener.java | 19 +- 12 files changed, 418 insertions(+), 233 deletions(-) delete mode 100644 smallrye-llm-langchain4j-metrics/pom.xml delete mode 100644 smallrye-llm-langchain4j-metrics/src/main/java/io/smallrye/llm/langchain4j/metrics/MetricsChatModelListener.java create mode 100644 smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 229623c..33a691e 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -64,12 +64,6 @@ 1.0.0-SNAPSHOT - - io.smallrye.llm - smallrye-llm-langchain4j-metrics - 1.0.0-SNAPSHOT - - io.smallrye.llm smallrye-llm-langchain4j-telemetry @@ -141,11 +135,6 @@ smallrye-llm-langchain4j-portable-extension - - io.smallrye.llm - smallrye-llm-langchain4j-metrics - - io.smallrye.llm smallrye-llm-langchain4j-telemetry diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Booking.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Booking.java index a8e03c1..bf1ea1e 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Booking.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Booking.java @@ -1,14 +1,11 @@ package io.jefrajames.booking; import java.time.LocalDate; +import java.util.Objects; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor +//@Data +//@NoArgsConstructor +//@AllArgsConstructor public class Booking { private String bookingNumber; @@ -18,4 +15,155 @@ public class Booking { private boolean canceled = false; private String carModel; + /** + * + */ + public Booking() { + super(); + //TODO Auto-generated constructor stub + } + + /** + * @param bookingNumber + * @param start + * @param end + * @param customer + * @param canceled + * @param carModel + */ + public Booking(String bookingNumber, LocalDate start, LocalDate end, Customer customer, boolean canceled, + String carModel) { + super(); + this.bookingNumber = bookingNumber; + this.start = start; + this.end = end; + this.customer = customer; + this.canceled = canceled; + this.carModel = carModel; + } + + /** + * @return the bookingNumber + */ + public String getBookingNumber() { + return bookingNumber; + } + + /** + * @param bookingNumber the bookingNumber to set + */ + public void setBookingNumber(String bookingNumber) { + this.bookingNumber = bookingNumber; + } + + /** + * @return the start + */ + public LocalDate getStart() { + return start; + } + + /** + * @param start the start to set + */ + public void setStart(LocalDate start) { + this.start = start; + } + + /** + * @return the end + */ + public LocalDate getEnd() { + return end; + } + + /** + * @param end the end to set + */ + public void setEnd(LocalDate end) { + this.end = end; + } + + /** + * @return the customer + */ + public Customer getCustomer() { + return customer; + } + + /** + * @param customer the customer to set + */ + public void setCustomer(Customer customer) { + this.customer = customer; + } + + /** + * @return the canceled + */ + public boolean isCanceled() { + return canceled; + } + + /** + * @param canceled the canceled to set + */ + public void setCanceled(boolean canceled) { + this.canceled = canceled; + } + + /** + * @return the carModel + */ + public String getCarModel() { + return carModel; + } + + /** + * @param carModel the carModel to set + */ + public void setCarModel(String carModel) { + this.carModel = carModel; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + return Objects.hash(bookingNumber, canceled, carModel, customer, end, start); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Booking other = (Booking) obj; + return Objects.equals(bookingNumber, other.bookingNumber) && canceled == other.canceled + && Objects.equals(carModel, other.carModel) && Objects.equals(customer, other.customer) + && Objects.equals(end, other.end) && Objects.equals(start, other.start); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "Booking [bookingNumber=" + bookingNumber + ", start=" + start + ", end=" + end + ", customer=" + + customer + ", canceled=" + canceled + ", carModel=" + carModel + "]"; + } } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/BookingService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/BookingService.java index 4ee5ff5..36fd025 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/BookingService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/BookingService.java @@ -8,13 +8,16 @@ import jakarta.enterprise.context.ApplicationScoped; +import org.jboss.logging.Logger; + import dev.langchain4j.agent.tool.Tool; -import lombok.extern.java.Log; @ApplicationScoped -@Log +//@Log public class BookingService { + private static final Logger LOGGER = Logger.getLogger(BookingService.class.getName()); + // Pseudo database private static final Map BOOKINGS = new HashMap<>(); static { @@ -44,14 +47,14 @@ private Booking checkBookingExists(String bookingNumber, String name, String sur @Tool("Get booking details given a booking number and customer name and surname") public Booking getBookingDetails(String bookingNumber, String name, String surname) { - log.info("DEMO: Calling Tool-getBookingDetails: " + bookingNumber + " and customer: " + LOGGER.info("DEMO: Calling Tool-getBookingDetails: " + bookingNumber + " and customer: " + name + " " + surname); return checkBookingExists(bookingNumber, name, surname); } @Tool("Get all booking ids for a customer given his name and surname") public List getBookingsForCustomer(String name, String surname) { - log.info("DEMO: Calling Tool-getBookingsForCustomer: " + name + " " + surname); + LOGGER.info("DEMO: Calling Tool-getBookingsForCustomer: " + name + " " + surname); Customer customer = new Customer(name, surname); return BOOKINGS.values() .stream() @@ -77,7 +80,7 @@ public void checkCancelPolicy(Booking booking) { @Tool("Cancel a booking given its booking number and customer name and surname") public Booking cancelBooking(String bookingNumber, String name, String surname) { - log.info("DEMO: Calling Tool-cancelBooking " + bookingNumber + " for customer: " + name + LOGGER.info("DEMO: Calling Tool-cancelBooking " + bookingNumber + " for customer: " + name + " " + surname); Booking booking = checkBookingExists(bookingNumber, name, surname); diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Customer.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Customer.java index f2488f0..34203f3 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Customer.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Customer.java @@ -1,15 +1,97 @@ package io.jefrajames.booking; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(of = { "name", "surname" }) +import java.util.Objects; + +//@Data +//@NoArgsConstructor +//@AllArgsConstructor +//@EqualsAndHashCode(of = { "name", "surname" }) public class Customer { private String name; private String surname; + + /** + * + */ + public Customer() { + super(); + //TODO Auto-generated constructor stub + } + + /** + * @param name + * @param surname + */ + public Customer(String name, String surname) { + super(); + this.name = name; + this.surname = surname; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the surname + */ + public String getSurname() { + return surname; + } + + /** + * @param surname the surname to set + */ + public void setSurname(String surname) { + this.surname = surname; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + return Objects.hash(name, surname); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Customer other = (Customer) obj; + return Objects.equals(name, other.name) && Objects.equals(surname, other.surname); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + + @Override + public String toString() { + return "Customer [name=" + name + ", surname=" + surname + "]"; + } } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java index f50a395..70e8118 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java @@ -12,6 +12,7 @@ import jakarta.inject.Inject; import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; import dev.langchain4j.data.document.Document; import dev.langchain4j.data.document.parser.TextDocumentParser; @@ -22,12 +23,12 @@ import dev.langchain4j.store.embedding.EmbeddingStore; import dev.langchain4j.store.embedding.EmbeddingStoreIngestor; import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore; -import lombok.extern.java.Log; -@Log @ApplicationScoped public class DocRagIngestor { + private static final Logger LOGGER = Logger.getLogger(DocRagIngestor.class.getName()); + // Used by ContentRetriever @Produces private EmbeddingModel embeddingModel = new AllMiniLmL6V2EmbeddingModel(); @@ -57,7 +58,7 @@ public void ingest(@Observes @Initialized(ApplicationScoped.class) Object pointl List docs = loadDocs(); ingestor.ingest(docs); - log.info(String.format("DEMO %d documents ingested in %d msec", docs.size(), + LOGGER.info(String.format("DEMO %d documents ingested in %d msec", docs.size(), System.currentTimeMillis() - start)); } } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java index e120c91..245762c 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java @@ -29,7 +29,8 @@ public Set getBeanNames() { .collect(Collectors.toSet()); } - @Override + @SuppressWarnings("unchecked") + @Override public T getBeanPropertyValue(String beanName, String propertyName, Class type) { String value = properties.getProperty(PREFIX + "." + beanName + "." + propertyName); if (value == null) diff --git a/pom.xml b/pom.xml index ee638ab..060b498 100644 --- a/pom.xml +++ b/pom.xml @@ -81,7 +81,6 @@ smallrye-llm-langchain4j-buildcompatible-extension smallrye-llm-langchain4j-core smallrye-llm-langchain4j-config-mpconfig - smallrye-llm-langchain4j-metrics smallrye-llm-langchain4j-telemetry @@ -91,11 +90,6 @@ org.eclipse.microprofile.config microprofile-config-api ${version.eclipse.microprofile.config} - - - org.eclipse.microprofile.metrics - microprofile-metrics-api - ${version.eclipse.microprofile.metrics} org.eclipse.microprofile.telemetry diff --git a/smallrye-llm-langchain4j-metrics/pom.xml b/smallrye-llm-langchain4j-metrics/pom.xml deleted file mode 100644 index 197a25e..0000000 --- a/smallrye-llm-langchain4j-metrics/pom.xml +++ /dev/null @@ -1,35 +0,0 @@ - - 4.0.0 - - io.smallrye.llm - smallrye-llm-parent - 1.0.0-SNAPSHOT - - smallrye-llm-langchain4j-metrics - - - - jakarta.enterprise - jakarta.enterprise.cdi-api - provided - - - org.eclipse.microprofile.metrics - microprofile-metrics-api - provided - - - io.smallrye.llm - smallrye-llm-langchain4j-core - ${project.version} - - - - io.smallrye - smallrye-metrics - 5.1.0 - - - \ No newline at end of file diff --git a/smallrye-llm-langchain4j-metrics/src/main/java/io/smallrye/llm/langchain4j/metrics/MetricsChatModelListener.java b/smallrye-llm-langchain4j-metrics/src/main/java/io/smallrye/llm/langchain4j/metrics/MetricsChatModelListener.java deleted file mode 100644 index c7597ca..0000000 --- a/smallrye-llm-langchain4j-metrics/src/main/java/io/smallrye/llm/langchain4j/metrics/MetricsChatModelListener.java +++ /dev/null @@ -1,144 +0,0 @@ -package io.smallrye.llm.langchain4j.metrics; - -import java.time.Duration; -import java.time.temporal.ChronoUnit; -import java.util.List; - -import jakarta.enterprise.context.Dependent; - -import org.eclipse.microprofile.metrics.Metadata; -import org.eclipse.microprofile.metrics.MetadataBuilder; -import org.eclipse.microprofile.metrics.MetricRegistry; -import org.eclipse.microprofile.metrics.MetricUnits; -import org.eclipse.microprofile.metrics.Tag; - -import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.model.chat.listener.ChatModelErrorContext; -import dev.langchain4j.model.chat.listener.ChatModelListener; -import dev.langchain4j.model.chat.listener.ChatModelRequest; -import dev.langchain4j.model.chat.listener.ChatModelRequestContext; -import dev.langchain4j.model.chat.listener.ChatModelResponse; -import dev.langchain4j.model.chat.listener.ChatModelResponseContext; -import io.smallrye.metrics.SharedMetricRegistries; - -/** - * Creates metrics that follow the - * Semantic Conventions - * for GenAI Metrics. - * - * @author Buhake Sindi - * @since 25 November 2024 - */ -@Dependent -public class MetricsChatModelListener implements ChatModelListener { - - private static final String MP_AI_METRIC_START_TIME_NAME = "MP_AI_METRIC_START_TIME"; - - private static final String METRIC_CLIENT_TOKEN_USAGE_NAME = "gen_ai.client.token.usage"; - private static final String METRIC_CLIENT_OPERATION_DURATION_NAME = "gen_ai.client.operation.duration"; - // private static final String METRIC_SERVER_REQUEST_DURATION_NAME = "gen_ai.server.request.duration"; - private static final Tag OPERATION_NAME_TAG = new Tag("gen_ai.operation.name", "chat"); - - /* - * (non-Javadoc) - * - * @see dev.langchain4j.model.chat.listener.ChatModelListener#onRequest(dev.langchain4j.model.chat.listener. - * ChatModelRequestContext) - */ - @Override - public void onRequest(ChatModelRequestContext requestContext) { - // TODO Auto-generated method stub - requestContext.attributes().put(MP_AI_METRIC_START_TIME_NAME, System.nanoTime()); - } - - /* - * (non-Javadoc) - * - * @see dev.langchain4j.model.chat.listener.ChatModelListener#onResponse(dev.langchain4j.model.chat.listener. - * ChatModelResponseContext) - */ - @Override - public void onResponse(ChatModelResponseContext responseContext) { - // TODO Auto-generated method stub - final long endTime = System.nanoTime(); - final long startTime = (Long) responseContext.attributes().get(MP_AI_METRIC_START_TIME_NAME); - - final ChatModelRequest request = responseContext.request(); - final ChatModelResponse response = responseContext.response(); - Tag requestModelTag = new Tag("gen_ai.request.model", request.model()); - Tag responseModelTag = new Tag("gen_ai.response.model", response.model()); - - List inputTokenCountTags = List.of(OPERATION_NAME_TAG, requestModelTag, responseModelTag, - new Tag("gen_ai.token.type", "input")); - List outputTokenCountTags = List.of(OPERATION_NAME_TAG, requestModelTag, responseModelTag, - new Tag("gen_ai.token.type", "output")); - List durationTags = List.of(OPERATION_NAME_TAG, requestModelTag, responseModelTag); - - recordInputTokenUsage(response.tokenUsage().inputTokenCount(), - inputTokenCountTags.toArray(new Tag[inputTokenCountTags.size()])); - recordOutputTokenUsage(response.tokenUsage().outputTokenCount(), - outputTokenCountTags.toArray(new Tag[inputTokenCountTags.size()])); - recordDuration(startTime, endTime, durationTags.toArray(new Tag[durationTags.size()])); - } - - /* - * (non-Javadoc) - * - * @see - * dev.langchain4j.model.chat.listener.ChatModelListener#onError(dev.langchain4j.model.chat.listener.ChatModelErrorContext) - */ - @Override - public void onError(ChatModelErrorContext errorContext) { - // TODO Auto-generated method stub - final long endTime = System.nanoTime(); - final long startTime = (Long) errorContext.attributes().get(MP_AI_METRIC_START_TIME_NAME); - final ChatModelRequest request = errorContext.request(); - final ChatModelResponse response = errorContext.partialResponse(); - Tag requestModelTag = new Tag("gen_ai.request.model", request.model()); - Tag responseModelTag = new Tag("gen_ai.response.model", response.model()); - - StringBuilder sb = new StringBuilder() - .append(errorContext.error().getClass().getName()); - - AiMessage aiMessage = errorContext.partialResponse().aiMessage(); - if (aiMessage != null) { - sb.append(";" + aiMessage.text()); - } - - Tag errorType = new Tag("error.type", sb.toString()); - List tags = List.of(OPERATION_NAME_TAG, requestModelTag, responseModelTag, errorType); - recordDuration(startTime, endTime, tags.toArray(new Tag[tags.size()])); - } - - private MetadataBuilder createMetadataBuilder(final String name, final String description) { - return Metadata.builder() - .withName(name) - .withDescription(description); - } - - private MetadataBuilder createMetadataBuilder(final String name, final String description, final String metricUnit) { - return createMetadataBuilder(name, description) - .withUnit(metricUnit); - } - - private void recordInputTokenUsage(final long tokenCount, final Tag[] tags) { - Metadata inputTokenUsageMetadata = createMetadataBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME, - "Measures number of input tokens used").build(); - MetricRegistry registry = SharedMetricRegistries.getOrCreate(MetricRegistry.BASE_SCOPE); - registry.histogram(inputTokenUsageMetadata, tags).update(tokenCount); - } - - private void recordOutputTokenUsage(final long tokenCount, final Tag[] tags) { - Metadata ouputTokenUsageMetadata = createMetadataBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME, - "Measures number of output tokens used").build(); - MetricRegistry registry = SharedMetricRegistries.getOrCreate(MetricRegistry.BASE_SCOPE); - registry.histogram(ouputTokenUsageMetadata, tags).update(tokenCount); - } - - private void recordDuration(final long startTime, long endTime, final Tag[] tags) { - Metadata durationMetadata = createMetadataBuilder(METRIC_CLIENT_OPERATION_DURATION_NAME, "GenAI operation duration", - MetricUnits.NANOSECONDS).build(); - MetricRegistry registry = SharedMetricRegistries.getOrCreate(MetricRegistry.BASE_SCOPE); - registry.timer(durationMetadata, tags).update(Duration.of(endTime - startTime, ChronoUnit.NANOS)); - } -} diff --git a/smallrye-llm-langchain4j-telemetry/pom.xml b/smallrye-llm-langchain4j-telemetry/pom.xml index be4dc0f..06cf5c9 100644 --- a/smallrye-llm-langchain4j-telemetry/pom.xml +++ b/smallrye-llm-langchain4j-telemetry/pom.xml @@ -26,11 +26,5 @@ smallrye-llm-langchain4j-core ${project.version} - - - io.smallrye - smallrye-metrics - 5.1.0 - \ No newline at end of file diff --git a/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java new file mode 100644 index 0000000..b016882 --- /dev/null +++ b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java @@ -0,0 +1,145 @@ +package io.smallrye.llm.langchain4j.telemetry; + +import java.util.concurrent.TimeUnit; + +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.Dependent; +import jakarta.inject.Inject; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.model.chat.listener.ChatModelErrorContext; +import dev.langchain4j.model.chat.listener.ChatModelListener; +import dev.langchain4j.model.chat.listener.ChatModelRequest; +import dev.langchain4j.model.chat.listener.ChatModelRequestContext; +import dev.langchain4j.model.chat.listener.ChatModelResponse; +import dev.langchain4j.model.chat.listener.ChatModelResponseContext; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.Meter; + +/** + * Creates metrics that follow the + * Semantic Conventions + * for GenAI Metrics. + * + * @author Buhake Sindi + * @since 25 November 2024 + */ +@Dependent +public class MetricsChatModelListener implements ChatModelListener { + + private static final String MP_AI_METRIC_START_TIME_NAME = "MP_AI_METRIC_START_TIME"; + + private static final String METRIC_CLIENT_TOKEN_USAGE_NAME = "gen_ai.client.token.usage"; + private static final String METRIC_CLIENT_OPERATION_DURATION_NAME = "gen_ai.client.operation.duration"; + + private LongHistogram inputTokenUsage; + private LongHistogram outputTokenUsage; + private DoubleHistogram duration; + + @Inject + private Meter meter; + + @PostConstruct + private void init() { + inputTokenUsage = meter.histogramBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME) + .ofLongs() + .setDescription("Measures number of input tokens used") + .build(); + + outputTokenUsage = meter.histogramBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME) + .ofLongs() + .setDescription("Measures number of output tokens used") + .build(); + + duration = meter.histogramBuilder(METRIC_CLIENT_OPERATION_DURATION_NAME) + .setDescription("GenAI operation duration") + .setUnit("s") + .build(); + } + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onRequest(dev.langchain4j.model.chat.listener. + * ChatModelRequestContext) + */ + @Override + public void onRequest(ChatModelRequestContext requestContext) { + // TODO Auto-generated method stub + requestContext.attributes().put(MP_AI_METRIC_START_TIME_NAME, System.nanoTime()); + } + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onResponse(dev.langchain4j.model.chat.listener. + * ChatModelResponseContext) + */ + @Override + public void onResponse(ChatModelResponseContext responseContext) { + // TODO Auto-generated method stub + final long endTime = System.nanoTime(); + final long startTime = (Long) responseContext.attributes().get(MP_AI_METRIC_START_TIME_NAME); + + final ChatModelRequest request = responseContext.request(); + final ChatModelResponse response = responseContext.response(); + + Attributes inputTokenCountAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", + AttributeKey.stringKey("gen_ai.request.model"), request.model(), + AttributeKey.stringKey("gen_ai.response.model"), response.model(), + AttributeKey.stringKey("gen_ai.token.type"), "input"); + //Record + inputTokenUsage.record(response.tokenUsage().inputTokenCount(), inputTokenCountAttributes); + + Attributes outputTokenCountAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", + AttributeKey.stringKey("gen_ai.request.model"), request.model(), + AttributeKey.stringKey("gen_ai.response.model"), response.model(), + AttributeKey.stringKey("gen_ai.token.type"), "output"); + + //Record + outputTokenUsage.record(response.tokenUsage().outputTokenCount(), outputTokenCountAttributes); + + //Record duration + Attributes durationAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", + AttributeKey.stringKey("gen_ai.request.model"), request.model(), + AttributeKey.stringKey("gen_ai.response.model"), response.model()); + recordDuration(startTime, endTime, durationAttributes); + } + + /* + * (non-Javadoc) + * + * @see + * dev.langchain4j.model.chat.listener.ChatModelListener#onError(dev.langchain4j.model.chat.listener.ChatModelErrorContext) + */ + @Override + public void onError(ChatModelErrorContext errorContext) { + // TODO Auto-generated method stub + final long endTime = System.nanoTime(); + final long startTime = (Long) errorContext.attributes().get(MP_AI_METRIC_START_TIME_NAME); + final ChatModelRequest request = errorContext.request(); + final ChatModelResponse response = errorContext.partialResponse(); + + StringBuilder sb = new StringBuilder() + .append(errorContext.error().getClass().getName()); + + AiMessage aiMessage = errorContext.partialResponse().aiMessage(); + if (aiMessage != null) { + sb.append(";" + aiMessage.text()); + } + + //Record duration + Attributes durationAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", + AttributeKey.stringKey("gen_ai.request.model"), request.model(), + AttributeKey.stringKey("gen_ai.response.model"), response.model(), + AttributeKey.stringKey("error.type"), sb.toString()); + recordDuration(startTime, endTime, durationAttributes); + } + + private void recordDuration(final long startTime, long endTime, final Attributes attributes) { + duration.record(TimeUnit.SECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS), attributes); + } +} diff --git a/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java index 431c68a..c3a1350 100644 --- a/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java +++ b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java @@ -11,6 +11,7 @@ import dev.langchain4j.model.chat.listener.ChatModelResponseContext; import dev.langchain4j.model.output.TokenUsage; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; @@ -41,12 +42,18 @@ public class SpanChatModelListener implements ChatModelListener { public void onRequest(ChatModelRequestContext requestContext) { // TODO Auto-generated method stub final ChatModelRequest request = requestContext.request(); - Span span = tracer.spanBuilder("chat " + request.model()) - .setAttribute("gen_ai.operation.name", "chat") - .setAttribute("gen_ai.request.max_tokens", request.maxTokens()) - .setAttribute("gen_ai.request.temperature", request.temperature()) - .setAttribute("gen_ai.request.top_p", request.topP()) - .startSpan(); + SpanBuilder spanBuilder = tracer.spanBuilder("chat " + request.model()) + .setAttribute("gen_ai.operation.name", "chat"); + if (request.maxTokens() != null) + spanBuilder.setAttribute("gen_ai.request.max_tokens", request.maxTokens()); + + if (request.temperature() != null) + spanBuilder.setAttribute("gen_ai.request.temperature", request.temperature()); + + if (request.topP() != null) + spanBuilder.setAttribute("gen_ai.request.top_p", request.topP()); + + Span span = spanBuilder.startSpan(); Scope scope = span.makeCurrent(); requestContext.attributes().put(OTEL_SCOPE_KEY_NAME, scope); From f20c2530f9857b83089a549fd2533db7a46f53f0 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Fri, 29 Nov 2024 01:32:08 +0200 Subject: [PATCH 06/12] Rewrote the Metrics using Microprofile Telemetry API instead of MP Metrics as previously. --- .../resources/META-INF/microprofile-config.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties index 3482fb4..1749054 100644 --- a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties +++ b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties @@ -2,10 +2,10 @@ server.port=9080 smallrye.llm.plugin.chat-model.class=dev.langchain4j.model.azure.AzureOpenAiChatModel -smallrye.llm.plugin.chat-model.config.api-key=2b4a74c3acd64272a374bfeb84925068 -smallrye.llm.plugin.chat-model.config.endpoint=https://sindi-openai.openai.azure.com/ -smallrye.llm.plugin.chat-model.config.service-version=2023-03-15-preview -smallrye.llm.plugin.chat-model.config.deployment-name=gpt-4 +smallrye.llm.plugin.chat-model.config.api-key=${azure.openai.api.key} +smallrye.llm.plugin.chat-model.config.endpoint=${azure.openai.endpoint} +smallrye.llm.plugin.chat-model.config.service-version=2024-02-15-preview +smallrye.llm.plugin.chat-model.config.deployment-name=${azure.openai.deployment.name} smallrye.llm.plugin.chat-model.config.temperature=0.1 smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s @@ -35,4 +35,4 @@ app.docs-for-rag.dir=docs-for-rag otel.sdk.disabled=false otel.service.name=api -otel.traces.exporter=otlp +#otel.traces.exporter=otlp From fa056590a27208f0f3b69281c4b6ca2568e247d1 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Fri, 29 Nov 2024 04:09:58 +0200 Subject: [PATCH 07/12] Bug fixes. --- .../io/jefrajames/booking/DummyLLConfig.java | 2 +- .../telemetry/MetricsChatModelListener.java | 33 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java index 245762c..e58ac4f 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java @@ -30,7 +30,7 @@ public Set getBeanNames() { } @SuppressWarnings("unchecked") - @Override + @Override public T getBeanPropertyValue(String beanName, String propertyName, Class type) { String value = properties.getProperty(PREFIX + "." + beanName + "." + propertyName); if (value == null) diff --git a/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java index b016882..c95ab3f 100644 --- a/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java +++ b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java @@ -1,5 +1,6 @@ package io.smallrye.llm.langchain4j.telemetry; +import java.util.List; import java.util.concurrent.TimeUnit; import jakarta.annotation.PostConstruct; @@ -35,27 +36,25 @@ public class MetricsChatModelListener implements ChatModelListener { private static final String METRIC_CLIENT_TOKEN_USAGE_NAME = "gen_ai.client.token.usage"; private static final String METRIC_CLIENT_OPERATION_DURATION_NAME = "gen_ai.client.operation.duration"; - private LongHistogram inputTokenUsage; - private LongHistogram outputTokenUsage; - private DoubleHistogram duration; + private LongHistogram clientTokenUsage; + private DoubleHistogram clientOperationDuration; @Inject private Meter meter; @PostConstruct private void init() { - inputTokenUsage = meter.histogramBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME) + clientTokenUsage = meter.histogramBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME) .ofLongs() - .setDescription("Measures number of input tokens used") + .setDescription("Measures number of input and output tokens used") + .setExplicitBucketBoundariesAdvice(List.of(1L, 4L, 16L, 64L, 256L, 1024L, 4096L, 16384L, 65536L, 262144L, + 1048576L, 4194304L, 16777216L, 67108864L)) .build(); - outputTokenUsage = meter.histogramBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME) - .ofLongs() - .setDescription("Measures number of output tokens used") - .build(); - - duration = meter.histogramBuilder(METRIC_CLIENT_OPERATION_DURATION_NAME) + clientOperationDuration = meter.histogramBuilder(METRIC_CLIENT_OPERATION_DURATION_NAME) .setDescription("GenAI operation duration") + .setExplicitBucketBoundariesAdvice( + List.of(0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92)) .setUnit("s") .build(); } @@ -92,7 +91,7 @@ public void onResponse(ChatModelResponseContext responseContext) { AttributeKey.stringKey("gen_ai.response.model"), response.model(), AttributeKey.stringKey("gen_ai.token.type"), "input"); //Record - inputTokenUsage.record(response.tokenUsage().inputTokenCount(), inputTokenCountAttributes); + clientTokenUsage.record(response.tokenUsage().inputTokenCount(), inputTokenCountAttributes); Attributes outputTokenCountAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", AttributeKey.stringKey("gen_ai.request.model"), request.model(), @@ -100,13 +99,13 @@ public void onResponse(ChatModelResponseContext responseContext) { AttributeKey.stringKey("gen_ai.token.type"), "output"); //Record - outputTokenUsage.record(response.tokenUsage().outputTokenCount(), outputTokenCountAttributes); + clientTokenUsage.record(response.tokenUsage().outputTokenCount(), outputTokenCountAttributes); //Record duration Attributes durationAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", AttributeKey.stringKey("gen_ai.request.model"), request.model(), AttributeKey.stringKey("gen_ai.response.model"), response.model()); - recordDuration(startTime, endTime, durationAttributes); + recordClientOperationDuration(startTime, endTime, durationAttributes); } /* @@ -136,10 +135,10 @@ public void onError(ChatModelErrorContext errorContext) { AttributeKey.stringKey("gen_ai.request.model"), request.model(), AttributeKey.stringKey("gen_ai.response.model"), response.model(), AttributeKey.stringKey("error.type"), sb.toString()); - recordDuration(startTime, endTime, durationAttributes); + recordClientOperationDuration(startTime, endTime, durationAttributes); } - private void recordDuration(final long startTime, long endTime, final Attributes attributes) { - duration.record(TimeUnit.SECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS), attributes); + private void recordClientOperationDuration(final long startTime, long endTime, final Attributes attributes) { + clientOperationDuration.record(TimeUnit.SECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS), attributes); } } From 4e0941a27b0a867f8bc6d0e722ed6894e1dd0691 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Tue, 3 Dec 2024 09:53:11 +0200 Subject: [PATCH 08/12] Enabled Telemetry. --- examples/liberty-car-booking/pom.xml | 26 +++++++++++++++++++ .../src/main/liberty/config/server.xml | 8 ++++-- .../META-INF/microprofile-config.properties | 7 +++-- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 33a691e..7f95878 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -52,6 +52,21 @@ provided + + + io.opentelemetry + opentelemetry-api + 1.44.1 + provided + + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-annotations + 2.10.0 + provided + + io.smallrye.llm smallrye-llm-langchain4j-config-mpconfig @@ -125,6 +140,17 @@ pom + + io.opentelemetry + opentelemetry-api + + + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-annotations + + io.smallrye.llm smallrye-llm-langchain4j-config-mpconfig diff --git a/examples/liberty-car-booking/src/main/liberty/config/server.xml b/examples/liberty-car-booking/src/main/liberty/config/server.xml index 138c567..a4cc2a0 100644 --- a/examples/liberty-car-booking/src/main/liberty/config/server.xml +++ b/examples/liberty-car-booking/src/main/liberty/config/server.xml @@ -8,7 +8,7 @@ - + @@ -18,5 +18,9 @@ + + + + diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties index 1749054..fb72dc3 100644 --- a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties +++ b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties @@ -33,6 +33,9 @@ fraud.memory.max.messages=20 # Location of documents to RAG app.docs-for-rag.dir=docs-for-rag +# Micrprofile Telemetry +otel.service.name=liberty-car-booking otel.sdk.disabled=false -otel.service.name=api -#otel.traces.exporter=otlp +otel.logs.exporter=otlp,console +otel.metrics.exporter=otlp,console +otel.traces.exporter=otlp,console From b943ad8ab501a13b84f14d9739e2421dc1cdef75 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Fri, 6 Dec 2024 21:31:33 +0200 Subject: [PATCH 09/12] Update to register listeners. --- .../META-INF/microprofile-config.properties | 2 +- .../llm/plugin/CommonLLMPluginCreator.java | 55 +++++++++++++------ 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties index fb72dc3..7c796d0 100644 --- a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties +++ b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties @@ -33,7 +33,7 @@ fraud.memory.max.messages=20 # Location of documents to RAG app.docs-for-rag.dir=docs-for-rag -# Micrprofile Telemetry +# Microprofile Telemetry otel.service.name=liberty-car-booking otel.sdk.disabled=false otel.logs.exporter=otlp,console diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java index ea1c846..67f3e3c 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java @@ -5,7 +5,9 @@ import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -18,6 +20,7 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.literal.NamedLiteral; +import jakarta.enterprise.inject.spi.DeploymentException; import jakarta.enterprise.util.TypeLiteral; import org.jboss.logging.Logger; @@ -76,11 +79,18 @@ public static void createAllLLMBeans(LLMConfig llmConfig, Consumer bea } beanBuilder.accept( new BeanData(targetClass, builderCLass, scopeClass, beanName, - (Instance creationalContext) -> CommonLLMPluginCreator.create( - creationalContext, - beanName, - targetClass, - builderCLass))); + (Instance creationalContext) -> { + try { + return CommonLLMPluginCreator.create( + creationalContext, + beanName, + targetClass, + builderCLass); + } catch (ClassNotFoundException e) { + // TODO Auto-generated catch block + throw new DeploymentException(e); + } + })); } } } @@ -123,7 +133,8 @@ public Function, Object> getCallback() { } } - public static Object create(Instance lookup, String beanName, Class targetClass, Class builderClass) { + @SuppressWarnings("unchecked") + public static Object create(Instance lookup, String beanName, Class targetClass, Class builderClass) throws ClassNotFoundException { LLMConfig llmConfig = LLMConfigProvider.getLlmConfig(); LOGGER.info( "Create instance config:" + beanName + ", target class : " + targetClass + ", builderClass : " + builderClass); @@ -144,18 +155,28 @@ public static Object create(Instance lookup, String beanName, Class t } else { for (Method methodToCall : methodsToCall) { Class parameterType = methodToCall.getParameterTypes()[0]; - if ("listeners".equals(property) && "@all".equals(stringValue)) { - Class typeParameterClass = ChatLanguageModel.class.isAssignableFrom(targetClass) + if ("listeners".equals(property)) { + Class typeParameterClass = ChatLanguageModel.class.isAssignableFrom(targetClass) ? ChatModelListener.class : parameterType.getTypeParameters()[0].getGenericDeclaration(); - Instance inst = getInstance(lookup, typeParameterClass); - if (inst != null) { - List listeners = StreamSupport.stream(inst.spliterator(), false) + List listeners = (List) Collections.checkedList(new ArrayList<>(), typeParameterClass); + if ("@all".equals(stringValue.trim())) { + Instance inst = (Instance) getInstance(lookup, typeParameterClass); + if (inst != null) { + inst.forEach(listeners::add); + listeners = StreamSupport.stream(inst.spliterator(), false) .collect(Collectors.toList()); - if (listeners != null && !listeners.isEmpty()) { - listeners.stream().forEach(l -> LOGGER.info("Adding listener: " + l.getClass().getName())); - methodToCall.invoke(builder, listeners); - } + } + } else { + for (String className : stringValue.split(",")) { + Instance inst = getInstance(lookup, loadClass(className.trim())); + listeners.add(inst.get()); + } + } + + if (listeners != null && !listeners.isEmpty()) { + listeners.stream().forEach(l -> LOGGER.info("Adding listener: " + l.getClass().getName())); + methodToCall.invoke(builder, listeners); } } else if (stringValue.startsWith("lookup:")) { String lookupableBean = stringValue.substring("lookup:".length()); @@ -186,8 +207,8 @@ public static Object create(Instance lookup, String beanName, Class t } } - private static Class loadClass(String scopeClassName) throws ClassNotFoundException { - return Thread.currentThread().getContextClassLoader().loadClass(scopeClassName); + private static Class loadClass(String className) throws ClassNotFoundException { + return Thread.currentThread().getContextClassLoader().loadClass(className); } @SuppressWarnings("unchecked") From cc1ae9fdccd4d3eea6ff08e634a017995004bf79 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Fri, 6 Dec 2024 22:13:29 +0200 Subject: [PATCH 10/12] Bug fix. --- .../llm/plugin/CommonLLMPluginCreator.java | 50 +++++++++---------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java index 67f3e3c..7f6c187 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java @@ -15,12 +15,10 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.literal.NamedLiteral; -import jakarta.enterprise.inject.spi.DeploymentException; import jakarta.enterprise.util.TypeLiteral; import org.jboss.logging.Logger; @@ -80,17 +78,12 @@ public static void createAllLLMBeans(LLMConfig llmConfig, Consumer bea beanBuilder.accept( new BeanData(targetClass, builderCLass, scopeClass, beanName, (Instance creationalContext) -> { - try { - return CommonLLMPluginCreator.create( - creationalContext, - beanName, - targetClass, - builderCLass); - } catch (ClassNotFoundException e) { - // TODO Auto-generated catch block - throw new DeploymentException(e); - } - })); + return CommonLLMPluginCreator.create( + creationalContext, + beanName, + targetClass, + builderCLass); + })); } } } @@ -134,7 +127,7 @@ public Function, Object> getCallback() { } @SuppressWarnings("unchecked") - public static Object create(Instance lookup, String beanName, Class targetClass, Class builderClass) throws ClassNotFoundException { + public static Object create(Instance lookup, String beanName, Class targetClass, Class builderClass) { LLMConfig llmConfig = LLMConfigProvider.getLlmConfig(); LOGGER.info( "Create instance config:" + beanName + ", target class : " + targetClass + ", builderClass : " + builderClass); @@ -156,24 +149,27 @@ public static Object create(Instance lookup, String beanName, Class t for (Method methodToCall : methodsToCall) { Class parameterType = methodToCall.getParameterTypes()[0]; if ("listeners".equals(property)) { - Class typeParameterClass = ChatLanguageModel.class.isAssignableFrom(targetClass) + Class typeParameterClass = ChatLanguageModel.class.isAssignableFrom(targetClass) ? ChatModelListener.class : parameterType.getTypeParameters()[0].getGenericDeclaration(); - List listeners = (List) Collections.checkedList(new ArrayList<>(), typeParameterClass); + List listeners = (List) Collections.checkedList(new ArrayList<>(), + typeParameterClass); if ("@all".equals(stringValue.trim())) { - Instance inst = (Instance) getInstance(lookup, typeParameterClass); - if (inst != null) { - inst.forEach(listeners::add); - listeners = StreamSupport.stream(inst.spliterator(), false) - .collect(Collectors.toList()); - } + Instance inst = (Instance) getInstance(lookup, typeParameterClass); + if (inst != null) { + inst.forEach(listeners::add); + } } else { - for (String className : stringValue.split(",")) { - Instance inst = getInstance(lookup, loadClass(className.trim())); - listeners.add(inst.get()); - } + try { + for (String className : stringValue.split(",")) { + Instance inst = getInstance(lookup, loadClass(className.trim())); + listeners.add(inst.get()); + } + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } } - + if (listeners != null && !listeners.isEmpty()) { listeners.stream().forEach(l -> LOGGER.info("Adding listener: " + l.getClass().getName())); methodToCall.invoke(builder, listeners); From 59a91524b5be67bd5cc3fcbeeb12dbb84419edcb Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Fri, 13 Dec 2024 10:16:37 +0200 Subject: [PATCH 11/12] Updated to support CDI creation for RAG support. --- .../llm/aiservice/CommonAIServiceCreator.java | 120 +++++++----------- .../llm/plugin/CommonLLMPluginCreator.java | 12 +- .../smallrye/llm/spi/RegisterAIService.java | 16 +-- 3 files changed, 62 insertions(+), 86 deletions(-) diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java index 1db83a6..d0817b6 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java @@ -1,127 +1,103 @@ package io.smallrye.llm.aiservice; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Parameter; import java.util.ArrayList; import java.util.List; -import jakarta.enterprise.inject.Instance; -import jakarta.enterprise.inject.literal.NamedLiteral; - import org.jboss.logging.Logger; +import dev.langchain4j.memory.ChatMemory; import dev.langchain4j.memory.chat.ChatMemoryProvider; -import dev.langchain4j.memory.chat.MessageWindowChatMemory; import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.chat.StreamingChatLanguageModel; import dev.langchain4j.model.moderation.ModerationModel; +import dev.langchain4j.rag.RetrievalAugmentor; import dev.langchain4j.rag.content.retriever.ContentRetriever; import dev.langchain4j.service.AiServices; -import dev.langchain4j.service.MemoryId; -import dev.langchain4j.service.Moderate; -import dev.langchain4j.store.memory.chat.ChatMemoryStore; -import io.smallrye.llm.core.langchain4j.core.config.spi.ChatMemoryFactoryProvider; import io.smallrye.llm.spi.RegisterAIService; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.literal.NamedLiteral; public class CommonAIServiceCreator { private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); - @SuppressWarnings("unchecked") public static X create(Instance lookup, Class interfaceClass) { RegisterAIService annotation = interfaceClass.getAnnotation(RegisterAIService.class); Instance chatLanguageModel = getInstance(lookup, ChatLanguageModel.class, annotation.chatLanguageModelName()); + Instance streamingChatLanguageModel = getInstance(lookup, StreamingChatLanguageModel.class, + annotation.streamingChatLanguageModelName()); Instance contentRetriever = getInstance(lookup, ContentRetriever.class, annotation.contentRetrieverName()); + Instance retrievalAugmentor = getInstance(lookup, RetrievalAugmentor.class, + annotation.retrievalAugmentorName()); try { - AiServices aiServices = AiServices.builder(interfaceClass); - if (chatLanguageModel.isResolvable()) { + AiServices aiServices = AiServices.builder(interfaceClass); + if (chatLanguageModel != null && chatLanguageModel.isResolvable()) { LOGGER.info("ChatLanguageModel " + chatLanguageModel.get()); aiServices.chatLanguageModel(chatLanguageModel.get()); } - if (contentRetriever.isResolvable()) { + if (streamingChatLanguageModel != null && streamingChatLanguageModel.isResolvable()) { + LOGGER.info("StreamingChatLanguageModel " + streamingChatLanguageModel.get()); + aiServices.streamingChatLanguageModel(streamingChatLanguageModel.get()); + } + if (contentRetriever != null && contentRetriever.isResolvable()) { LOGGER.info("ContentRetriever " + contentRetriever.get()); aiServices.contentRetriever(contentRetriever.get()); } + if (retrievalAugmentor != null && retrievalAugmentor.isResolvable()) { + LOGGER.info("RetrievalAugmentor " + retrievalAugmentor.get()); + aiServices.retrievalAugmentor(retrievalAugmentor.get()); + } if (annotation.tools() != null && annotation.tools().length > 0) { List tools = new ArrayList<>(annotation.tools().length); for (Class toolClass : annotation.tools()) { try { - tools.add(toolClass.getConstructor(null).newInstance(null)); + tools.add(toolClass.getConstructor((Class[])null).newInstance((Object[])null)); } catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) { } } aiServices.tools(tools); } + + Instance chatMemory = getInstance(lookup, ChatMemory.class, + annotation.chatMemoryName()); + if (chatMemory != null && chatMemory.isResolvable()) { + LOGGER.info("ChatMemory " + chatMemory.get()); + aiServices.chatMemory(chatMemory.get()); + } - ChatMemoryProvider chatMemoryProvider = createChatMemoryProvider(lookup, interfaceClass, annotation); - if (chatMemoryProvider != null) { - aiServices.chatMemoryProvider(chatMemoryProvider); - } else { - aiServices.chatMemory( - ChatMemoryFactoryProvider.getChatMemoryFactory().getChatMemory(lookup, - annotation.chatMemoryMaxMessages())); - } + Instance chatMemoryProvider = getInstance(lookup, ChatMemoryProvider.class, + annotation.chatMemoryProviderName()); + if (chatMemoryProvider != null && chatMemoryProvider.isResolvable()) { + LOGGER.info("ChatMemoryProvider " + chatMemoryProvider.get()); + aiServices.chatMemoryProvider(chatMemoryProvider.get()); + } - ModerationModel moderationModel = findModerationModel(lookup, interfaceClass, annotation); - if (moderationModel != null) { - aiServices.moderationModel(moderationModel); + Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, + annotation.moderationModelName()); + if (moderationModelInstance != null && moderationModelInstance.isResolvable()) { + LOGGER.info("ModerationModel " + moderationModelInstance.get()); + aiServices.moderationModel(moderationModelInstance.get()); } - return (X) aiServices.build(); + + return aiServices.build(); } catch (Exception e) { throw new RuntimeException(e); } } private static Instance getInstance(Instance lookup, Class type, String name) { - LOGGER.info("Getinstance of '" + type + "' with name '" + name + "'"); - if (name == null || name.isBlank()) { - return lookup.select(type); - } - return lookup.select(type, NamedLiteral.of(name)); - } - - private static ModerationModel findModerationModel(Instance lookup, Class interfaceClass, - RegisterAIService registerAIService) { - //Get all methods. - for (Method method : interfaceClass.getMethods()) { - Moderate moderate = method.getAnnotation(Moderate.class); - if (moderate != null) { - Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, - registerAIService.moderationModelName()); - if (moderationModelInstance != null && moderationModelInstance.isResolvable()) - return moderationModelInstance.get(); - } + LOGGER.info("CDI get instance of type '" + type + "' with name '" + name + "'"); + if (name != null && !name.isBlank()) { + if ("#default".equals(name)) + return lookup.select(type); + + return lookup.select(type, NamedLiteral.of(name)); } - - return null; - } - - private static ChatMemoryProvider createChatMemoryProvider(Instance lookup, Class interfaceClass, - RegisterAIService registerAIService) { - //Get all methods. - for (Method method : interfaceClass.getMethods()) { - for (Parameter parameter : method.getParameters()) { - MemoryId memoryIdAnnotation = parameter.getAnnotation(MemoryId.class); - if (memoryIdAnnotation != null) { - Instance chatMemoryStore = getInstance(lookup, ChatMemoryStore.class, - registerAIService.chatMemoryStoreName()); - if (chatMemoryStore == null || !chatMemoryStore.isResolvable()) { - throw new IllegalStateException("Unable to resolve a ChatMemoryStore for your ChatMemoryProvider."); - } - - ChatMemoryProvider chatMemoryProvider = memoryId -> MessageWindowChatMemory.builder() - .id(memoryId) - .maxMessages(registerAIService.chatMemoryMaxMessages()) - .chatMemoryStore(chatMemoryStore.get()) - .build(); - return chatMemoryProvider; - } - } - } - + return null; } } diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java index 7f6c187..0a45b96 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java @@ -16,19 +16,19 @@ import java.util.function.Function; import java.util.stream.Collectors; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.inject.Instance; -import jakarta.enterprise.inject.literal.NamedLiteral; -import jakarta.enterprise.util.TypeLiteral; - import org.jboss.logging.Logger; import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.chat.StreamingChatLanguageModel; import dev.langchain4j.model.chat.listener.ChatModelListener; import dev.langchain4j.store.embedding.EmbeddingStore; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfig; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfigProvider; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.literal.NamedLiteral; +import jakarta.enterprise.util.TypeLiteral; /* smallrye.llm.plugin.content-retriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever @@ -149,7 +149,7 @@ public static Object create(Instance lookup, String beanName, Class t for (Method methodToCall : methodsToCall) { Class parameterType = methodToCall.getParameterTypes()[0]; if ("listeners".equals(property)) { - Class typeParameterClass = ChatLanguageModel.class.isAssignableFrom(targetClass) + Class typeParameterClass = ChatLanguageModel.class.isAssignableFrom(targetClass) || StreamingChatLanguageModel.class.isAssignableFrom(targetClass) ? ChatModelListener.class : parameterType.getTypeParameters()[0].getGenericDeclaration(); List listeners = (List) Collections.checkedList(new ArrayList<>(), diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java index 733073a..729cc62 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java @@ -19,19 +19,19 @@ Class[] tools() default {}; - String chatLanguageModelName() default ""; + String chatLanguageModelName() default "#default"; + + String streamingChatLanguageModelName() default ""; String contentRetrieverModelName() default ""; - int chatMemoryMaxMessages() default 10; - - String embeddingModelName() default ""; - - String embeddingStoreName() default ""; - String contentRetrieverName() default ""; String moderationModelName() default ""; - String chatMemoryStoreName() default ""; + String chatMemoryName() default ""; + + String chatMemoryProviderName() default ""; + + String retrievalAugmentorName() default ""; } From 6c9e14ef38efca8f369e568b54a24274982a8f6b Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Fri, 13 Dec 2024 18:22:55 +0200 Subject: [PATCH 12/12] Git merge with main. --- examples/helidon-car-booking/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/helidon-car-booking/pom.xml b/examples/helidon-car-booking/pom.xml index b719960..d564ed2 100644 --- a/examples/helidon-car-booking/pom.xml +++ b/examples/helidon-car-booking/pom.xml @@ -12,7 +12,6 @@ io.helidon.Main - 0.36.0