From 13135c79b2d4f62dd2c88d78ca57d9b622f3286b Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Mon, 11 Nov 2024 10:42:24 +0200 Subject: [PATCH 1/6] Added ModerationModel ChatMemoryProvider (but might need rework). --- examples/liberty-car-booking/pom.xml | 26 ++++--- .../io/jefrajames/booking/DocRagIngestor.java | 14 ++-- .../llm/aiservice/CommonAIServiceCreator.java | 76 ++++++++++++++++--- .../smallrye/llm/spi/RegisterAIService.java | 4 + 4 files changed, 93 insertions(+), 27 deletions(-) diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 69ef05c..786d7fc 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -17,16 +17,22 @@ - - UTF-8 - UTF-8 - 10.0.0 - 6.1 - 3.13.0 - 3.4.0 - - ${project.build.directory}/liberty/wlp/usr/shared/resources/lib/ - + + UTF-8 + UTF-8 + 10.0.0 + 6.1 + 3.13.0 + 3.4.0 + 0.34.0 + + + + + + + ${project.build.directory}/liberty/wlp/usr/shared/resources/lib/ + diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java index 70d7ff1..2be0473 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java @@ -5,21 +5,21 @@ import java.io.File; import java.util.List; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.context.Initialized; -import jakarta.enterprise.event.Observes; -import jakarta.enterprise.inject.Produces; -import jakarta.inject.Inject; - import org.eclipse.microprofile.config.inject.ConfigProperty; import dev.langchain4j.data.document.Document; import dev.langchain4j.data.document.parser.TextDocumentParser; import dev.langchain4j.data.document.splitter.DocumentSplitters; +import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; import dev.langchain4j.store.embedding.EmbeddingStoreIngestor; import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.context.Initialized; +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Inject; import lombok.extern.java.Log; @Log @@ -32,7 +32,7 @@ public class DocRagIngestor { // Used by ContentRetriever @Produces - private InMemoryEmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); + private InMemoryEmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); // private File docs = new File(System.getProperty("docragdir")); diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java index 35546d9..60d0755 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java @@ -1,26 +1,33 @@ package io.smallrye.llm.aiservice; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Parameter; import java.util.ArrayList; import java.util.List; -import jakarta.enterprise.inject.Instance; -import jakarta.enterprise.inject.literal.NamedLiteral; - import org.jboss.logging.Logger; +import dev.langchain4j.memory.chat.ChatMemoryProvider; +import dev.langchain4j.memory.chat.MessageWindowChatMemory; import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.moderation.ModerationModel; import dev.langchain4j.rag.content.retriever.ContentRetriever; import dev.langchain4j.service.AiServices; +import dev.langchain4j.service.MemoryId; +import dev.langchain4j.service.Moderate; +import dev.langchain4j.store.memory.chat.ChatMemoryStore; import io.smallrye.llm.core.langchain4j.core.config.spi.ChatMemoryFactoryProvider; import io.smallrye.llm.spi.RegisterAIService; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.literal.NamedLiteral; public class CommonAIServiceCreator { - private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); + private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); @SuppressWarnings("unchecked") - public static Object create(Instance lookup, Class interfaceClass) { + public static X create(Instance lookup, Class interfaceClass) { RegisterAIService annotation = interfaceClass.getAnnotation(RegisterAIService.class); Instance chatLanguageModel = getInstance(lookup, ChatLanguageModel.class, annotation.chatLanguageModelName()); @@ -38,7 +45,7 @@ public static Object create(Instance lookup, Class interfaceClass) { } if (annotation.tools() != null && annotation.tools().length > 0) { List tools = new ArrayList<>(annotation.tools().length); - for (Class toolClass : annotation.tools()) { + for (Class toolClass : annotation.tools()) { try { tools.add(toolClass.getConstructor(null).newInstance(null)); } catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException @@ -47,20 +54,69 @@ public static Object create(Instance lookup, Class interfaceClass) { } aiServices.tools(tools); } - aiServices.chatMemory( + + ChatMemoryProvider chatMemoryProvider = createChatMemoryProvider(lookup, interfaceClass, annotation); + if (chatMemoryProvider != null) { + aiServices.chatMemoryProvider(chatMemoryProvider); + } else { + aiServices.chatMemory( ChatMemoryFactoryProvider.getChatMemoryFactory().getChatMemory(lookup, annotation.chatMemoryMaxMessages())); - return aiServices.build(); + } + + ModerationModel moderationModel = findModerationModel(lookup, interfaceClass, annotation); + if (moderationModel != null) { + aiServices.moderationModel(moderationModel); + } + return (X)aiServices.build(); } catch (Exception e) { throw new RuntimeException(e); } } - private static Instance getInstance(Instance lookup, Class type, String name) { + private static Instance getInstance(Instance lookup, Class type, String name) { LOGGER.info("Getinstance of '" + type + "' with name '" + name + "'"); if (name == null || name.isBlank()) { return lookup.select(type); } return lookup.select(type, NamedLiteral.of(name)); } - + + private static ModerationModel findModerationModel(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + Moderate moderate = method.getAnnotation(Moderate.class); + if (moderate != null) { + Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, registerAIService.moderationModelName()); + if (moderationModelInstance != null && moderationModelInstance.isResolvable()) return moderationModelInstance.get(); + } + } + + return null; + } + + + private static ChatMemoryProvider createChatMemoryProvider(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + for (Parameter parameter : method.getParameters()) { + MemoryId memoryIdAnnotation = parameter.getAnnotation(MemoryId.class); + if (memoryIdAnnotation != null) { + Instance chatMemoryStore = getInstance(lookup, ChatMemoryStore.class, + registerAIService.chatMemoryStoreName()); + if (chatMemoryStore == null || !chatMemoryStore.isResolvable()) { + throw new IllegalStateException("Unable to resolve a ChatMemoryStore for your ChatMemoryProvider."); + } + + ChatMemoryProvider chatMemoryProvider = memoryId -> MessageWindowChatMemory.builder() + .id(memoryId) + .maxMessages(registerAIService.chatMemoryMaxMessages()) + .chatMemoryStore(chatMemoryStore.get()) + .build(); + return chatMemoryProvider; + } + } + } + + return null; + } } diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java index b48605f..91dbbea 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java @@ -30,4 +30,8 @@ String embeddingStoreName() default ""; String contentRetrieverName() default ""; + + String moderationModelName() default ""; + + String chatMemoryStoreName() default ""; } From 494e646b638c5e69c25533e3bdd4f8e3f0340b1c Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Sat, 23 Nov 2024 19:13:29 +0200 Subject: [PATCH 2/6] Enable the Fault Tolerance Interceptor. --- examples/helidon-car-booking/pom.xml | 2 +- examples/liberty-car-booking/README.md | 2 +- examples/liberty-car-booking/pom.xml | 137 +++++++------ .../booking/CarBookingResource.java | 2 - .../io/jefrajames/booking/ChatAiService.java | 4 +- .../io/jefrajames/booking/DocRagIngestor.java | 21 +- .../io/jefrajames/booking/FraudAiService.java | 15 +- .../META-INF/microprofile-config.properties | 10 +- .../src/main/resources/application.properties | 2 +- .../llm/aiservice/CommonAIServiceCreator.java | 99 +++++----- .../llm/plugin/CommonLLMPluginCreator.java | 33 +++- .../smallrye/llm/spi/RegisterAIService.java | 4 +- .../pom.xml | 7 +- .../LangChain4JAIServiceBean.java | 182 ++++++++++++++++++ ...LangChain4JAIServicePortableExtension.java | 11 +- .../Langchain4JFaultToleranceExtension.java | 44 +++++ .../jakarta.enterprise.inject.spi.Extension | 1 + 17 files changed, 427 insertions(+), 149 deletions(-) create mode 100644 smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java create mode 100644 smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java diff --git a/examples/helidon-car-booking/pom.xml b/examples/helidon-car-booking/pom.xml index 3e1d666..b719960 100644 --- a/examples/helidon-car-booking/pom.xml +++ b/examples/helidon-car-booking/pom.xml @@ -12,7 +12,7 @@ io.helidon.Main - 0.34.0 + 0.36.0 diff --git a/examples/liberty-car-booking/README.md b/examples/liberty-car-booking/README.md index 92561c7..3d61654 100644 --- a/examples/liberty-car-booking/README.md +++ b/examples/liberty-car-booking/README.md @@ -39,7 +39,7 @@ To package the application in JVM mode run: `mvn package`. ## Configuration -All configuration is centralized in `microprofile-config.properties`(found is `resources\META-INF` folder) and can be redefined using environment variables. +All configuration is centralized in `microprofile-config.properties` (found is `resources\META-INF` folder) and can be redefined using environment variables. ## Running the application diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 786d7fc..0563933 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -24,7 +24,7 @@ 6.1 3.13.0 3.4.0 - 0.34.0 + 0.36.0 @@ -52,45 +52,81 @@ provided - - - - - ai.djl.huggingface - tokenizers - 0.30.0 - - - - - - - - - org.eclipse.microprofile - microprofile - pom - + + io.smallrye.llm + smallrye-llm-langchain4j-config-mpconfig + 1.0.0-SNAPSHOT + + + + io.smallrye.llm + smallrye-llm-langchain4j-portable-extension + 1.0.0-SNAPSHOT + + + + + + dev.langchain4j + langchain4j + ${dev.langchain4j.version} + + + + dev.langchain4j + langchain4j-hugging-face + ${dev.langchain4j.version} + + + + + dev.langchain4j + langchain4j-azure-open-ai + ${dev.langchain4j.version} + + + + dev.langchain4j + langchain4j-open-ai + ${dev.langchain4j.version} + + + + dev.langchain4j + langchain4j-embeddings-all-minilm-l6-v2 + ${dev.langchain4j.version} + + + + + ai.djl.huggingface + tokenizers + 0.30.0 + + + + + org.slf4j + slf4j-jdk14 + runtime + 2.0.9 + + + + + + + org.eclipse.microprofile + microprofile + pom + io.smallrye.llm @@ -102,27 +138,16 @@ smallrye-llm-langchain4j-portable-extension - - - + org.projectlombok lombok - provided diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java index 8faf5f9..1922a3b 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java @@ -40,9 +40,7 @@ public String chatWithAssistant( @Path("/fraud") public FraudResponse detectFraudForCustomer( @QueryParam("name") String name, - @QueryParam("surname") String surname) { return fraudService.detectFraudForCustomer(name, surname); } - } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java index bc6892f..660e718 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java @@ -2,6 +2,8 @@ import java.time.temporal.ChronoUnit; +import jakarta.enterprise.context.ApplicationScoped; + import org.eclipse.microprofile.faulttolerance.Fallback; import org.eclipse.microprofile.faulttolerance.Retry; import org.eclipse.microprofile.faulttolerance.Timeout; @@ -10,7 +12,7 @@ import io.smallrye.llm.spi.RegisterAIService; //@SuppressWarnings("CdiManagedBeanInconsistencyInspection") -@RegisterAIService(tools = BookingService.class, chatMemoryMaxMessages = 10) +@RegisterAIService(scope = ApplicationScoped.class, tools = BookingService.class, chatMemoryMaxMessages = 10) public interface ChatAiService { @SystemMessage(""" diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java index 2be0473..f50a395 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java @@ -5,6 +5,12 @@ import java.io.File; import java.util.List; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.context.Initialized; +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Inject; + import org.eclipse.microprofile.config.inject.ConfigProperty; import dev.langchain4j.data.document.Document; @@ -13,13 +19,9 @@ import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; import dev.langchain4j.store.embedding.EmbeddingStoreIngestor; import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.context.Initialized; -import jakarta.enterprise.event.Observes; -import jakarta.enterprise.inject.Produces; -import jakarta.inject.Inject; import lombok.extern.java.Log; @Log @@ -32,9 +34,7 @@ public class DocRagIngestor { // Used by ContentRetriever @Produces - private InMemoryEmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); - - // private File docs = new File(System.getProperty("docragdir")); + private EmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); @Inject @ConfigProperty(name = "app.docs-for-rag.dir") @@ -60,9 +60,4 @@ public void ingest(@Observes @Initialized(ApplicationScoped.class) Object pointl log.info(String.format("DEMO %d documents ingested in %d msec", docs.size(), System.currentTimeMillis() - start)); } - - public static void main(String[] args) { - - System.out.println(InMemoryEmbeddingStore.class.getInterfaces()[0]); - } } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java index 68488de..77e5086 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java @@ -1,14 +1,18 @@ package io.jefrajames.booking; +import java.time.temporal.ChronoUnit; + +import org.eclipse.microprofile.faulttolerance.Fallback; +import org.eclipse.microprofile.faulttolerance.Retry; +import org.eclipse.microprofile.faulttolerance.Timeout; + import dev.langchain4j.service.SystemMessage; import dev.langchain4j.service.UserMessage; import dev.langchain4j.service.V; import io.smallrye.llm.spi.RegisterAIService; -@SuppressWarnings("CdiManagedBeanInconsistencyInspection") -@RegisterAIService(chatMemoryMaxMessages = 5, - - chatLanguageModelName = "chat-model") +//@SuppressWarnings("CdiManagedBeanInconsistencyInspection") +@RegisterAIService(chatMemoryMaxMessages = 5, chatLanguageModelName = "chat-model") public interface FraudAiService { @SystemMessage(""" @@ -44,6 +48,9 @@ A booking overlap (and hence a fraud) occurs when there are several bookings for You must not wrap JSON response in backticks, markdown, or in any other way, but return it as plain text. """) + @Timeout(unit = ChronoUnit.MINUTES, value = 5) + @Retry(maxRetries = 2) + @Fallback(fallbackMethod = "fraudFallback") FraudResponse detectFraudForCustomer(@V("name") String name, @V("surname") String surname); default FraudResponse fraudFallback(String name, String surname) { diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties index c793531..ef84021 100644 --- a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties +++ b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties @@ -2,15 +2,15 @@ server.port=9080 smallrye.llm.plugin.chat-model.class=dev.langchain4j.model.azure.AzureOpenAiChatModel -smallrye.llm.plugin.chat-model.config.api-key=${azure.openai.api.key} -smallrye.llm.plugin.chat-model.config.endpoint=${azure.openai.endpoint} -smallrye.llm.plugin.chat-model.config.service-version=2024-02-15-preview -smallrye.llm.plugin.chat-model.config.deployment-name=${azure.openai.deployment.name} +smallrye.llm.plugin.chat-model.config.api-key=2b4a74c3acd64272a374bfeb84925068 +smallrye.llm.plugin.chat-model.config.endpoint=https://sindi-openai.openai.azure.com/ +smallrye.llm.plugin.chat-model.config.service-version=2023-03-15-preview +smallrye.llm.plugin.chat-model.config.deployment-name=gpt-4 smallrye.llm.plugin.chat-model.config.temperature=0.1 smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s smallrye.llm.plugin.chat-model.config.max-retries=2 -#smallrye.llm.plugin.chat-model.config.logRequestsAndResponsess=false +#smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=true smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever diff --git a/examples/quarkus-car-booking/src/main/resources/application.properties b/examples/quarkus-car-booking/src/main/resources/application.properties index 08f08e3..d9ab556 100644 --- a/examples/quarkus-car-booking/src/main/resources/application.properties +++ b/examples/quarkus-car-booking/src/main/resources/application.properties @@ -11,7 +11,7 @@ smallrye.llm.plugin.chat-model.config.temperature=0.1 smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s smallrye.llm.plugin.chat-model.config.max-retries=2 -#smallrye.llm.plugin.chat-model.config.logRequestsAndResponsess=false +#smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=false smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java index 60d0755..1db83a6 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java @@ -6,6 +6,9 @@ import java.util.ArrayList; import java.util.List; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.literal.NamedLiteral; + import org.jboss.logging.Logger; import dev.langchain4j.memory.chat.ChatMemoryProvider; @@ -19,12 +22,10 @@ import dev.langchain4j.store.memory.chat.ChatMemoryStore; import io.smallrye.llm.core.langchain4j.core.config.spi.ChatMemoryFactoryProvider; import io.smallrye.llm.spi.RegisterAIService; -import jakarta.enterprise.inject.Instance; -import jakarta.enterprise.inject.literal.NamedLiteral; public class CommonAIServiceCreator { - private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); + private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); @SuppressWarnings("unchecked") public static X create(Instance lookup, Class interfaceClass) { @@ -54,20 +55,21 @@ public static X create(Instance lookup, Class interfaceClass) { } aiServices.tools(tools); } - + ChatMemoryProvider chatMemoryProvider = createChatMemoryProvider(lookup, interfaceClass, annotation); if (chatMemoryProvider != null) { - aiServices.chatMemoryProvider(chatMemoryProvider); + aiServices.chatMemoryProvider(chatMemoryProvider); } else { - aiServices.chatMemory( - ChatMemoryFactoryProvider.getChatMemoryFactory().getChatMemory(lookup, annotation.chatMemoryMaxMessages())); + aiServices.chatMemory( + ChatMemoryFactoryProvider.getChatMemoryFactory().getChatMemory(lookup, + annotation.chatMemoryMaxMessages())); } - + ModerationModel moderationModel = findModerationModel(lookup, interfaceClass, annotation); if (moderationModel != null) { - aiServices.moderationModel(moderationModel); + aiServices.moderationModel(moderationModel); } - return (X)aiServices.build(); + return (X) aiServices.build(); } catch (Exception e) { throw new RuntimeException(e); } @@ -80,43 +82,46 @@ private static Instance getInstance(Instance lookup, Class typ } return lookup.select(type, NamedLiteral.of(name)); } - - private static ModerationModel findModerationModel(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { - //Get all methods. - for (Method method : interfaceClass.getMethods()) { - Moderate moderate = method.getAnnotation(Moderate.class); - if (moderate != null) { - Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, registerAIService.moderationModelName()); - if (moderationModelInstance != null && moderationModelInstance.isResolvable()) return moderationModelInstance.get(); - } - } - - return null; + + private static ModerationModel findModerationModel(Instance lookup, Class interfaceClass, + RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + Moderate moderate = method.getAnnotation(Moderate.class); + if (moderate != null) { + Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, + registerAIService.moderationModelName()); + if (moderationModelInstance != null && moderationModelInstance.isResolvable()) + return moderationModelInstance.get(); + } + } + + return null; } - - - private static ChatMemoryProvider createChatMemoryProvider(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { - //Get all methods. - for (Method method : interfaceClass.getMethods()) { - for (Parameter parameter : method.getParameters()) { - MemoryId memoryIdAnnotation = parameter.getAnnotation(MemoryId.class); - if (memoryIdAnnotation != null) { - Instance chatMemoryStore = getInstance(lookup, ChatMemoryStore.class, - registerAIService.chatMemoryStoreName()); - if (chatMemoryStore == null || !chatMemoryStore.isResolvable()) { - throw new IllegalStateException("Unable to resolve a ChatMemoryStore for your ChatMemoryProvider."); - } - - ChatMemoryProvider chatMemoryProvider = memoryId -> MessageWindowChatMemory.builder() - .id(memoryId) - .maxMessages(registerAIService.chatMemoryMaxMessages()) - .chatMemoryStore(chatMemoryStore.get()) - .build(); - return chatMemoryProvider; - } - } - } - - return null; + + private static ChatMemoryProvider createChatMemoryProvider(Instance lookup, Class interfaceClass, + RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + for (Parameter parameter : method.getParameters()) { + MemoryId memoryIdAnnotation = parameter.getAnnotation(MemoryId.class); + if (memoryIdAnnotation != null) { + Instance chatMemoryStore = getInstance(lookup, ChatMemoryStore.class, + registerAIService.chatMemoryStoreName()); + if (chatMemoryStore == null || !chatMemoryStore.isResolvable()) { + throw new IllegalStateException("Unable to resolve a ChatMemoryStore for your ChatMemoryProvider."); + } + + ChatMemoryProvider chatMemoryProvider = memoryId -> MessageWindowChatMemory.builder() + .id(memoryId) + .maxMessages(registerAIService.chatMemoryMaxMessages()) + .chatMemoryStore(chatMemoryStore.get()) + .build(); + return chatMemoryProvider; + } + } + } + + return null; } } diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java index 6d5525b..f46f912 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java @@ -6,7 +6,9 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -15,10 +17,12 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.literal.NamedLiteral; -import jakarta.enterprise.inject.spi.CDI; +import jakarta.enterprise.util.TypeLiteral; import org.jboss.logging.Logger; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfig; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfigProvider; @@ -34,6 +38,13 @@ public class CommonLLMPluginCreator { public static final Logger LOGGER = Logger.getLogger(CommonLLMPluginCreator.class); + private static final Map, TypeLiteral> TYPE_LITERALS = new HashMap<>(); + + static { + TYPE_LITERALS.put(EmbeddingStore.class, new TypeLiteral>() { + }); + } + @SuppressWarnings("unchecked") public static void createAllLLMBeans(LLMConfig llmConfig, Consumer beanBuilder) throws ClassNotFoundException { Set beanNameToCreate = llmConfig.getBeanNames(); @@ -135,12 +146,9 @@ public static Object create(Instance lookup, String beanName, Class t LOGGER.info("Lookup " + lookupableBean + " " + parameterType); Instance inst; if ("default".equals(lookupableBean)) { - inst = lookup.select(parameterType); - if (!inst.isResolvable()) { - inst = CDI.current().select(parameterType); - } + inst = getInstance(lookup, parameterType); } else { - inst = lookup.select(parameterType, NamedLiteral.of(lookupableBean)); + inst = getInstance(lookup, parameterType, lookupableBean); } methodToCall.invoke(builder, inst.get()); break; @@ -165,4 +173,17 @@ public static Object create(Instance lookup, String beanName, Class t private static Class loadClass(String scopeClassName) throws ClassNotFoundException { return Thread.currentThread().getContextClassLoader().loadClass(scopeClassName); } + + @SuppressWarnings("unchecked") + private static Instance getInstance(Instance lookup, Class clazz) { + if (TYPE_LITERALS.containsKey(clazz)) + return (Instance) lookup.select(TYPE_LITERALS.get(clazz)); + return lookup.select(clazz); + } + + private static Instance getInstance(Instance lookup, Class clazz, String lookupName) { + if (lookupName == null || lookupName.isBlank()) + return getInstance(lookup, clazz); + return lookup.select(clazz, NamedLiteral.of(lookupName)); + } } diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java index 91dbbea..733073a 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java @@ -30,8 +30,8 @@ String embeddingStoreName() default ""; String contentRetrieverName() default ""; - + String moderationModelName() default ""; - + String chatMemoryStoreName() default ""; } diff --git a/smallrye-llm-langchain4j-portable-extension/pom.xml b/smallrye-llm-langchain4j-portable-extension/pom.xml index 0bbf84d..e96eee9 100644 --- a/smallrye-llm-langchain4j-portable-extension/pom.xml +++ b/smallrye-llm-langchain4j-portable-extension/pom.xml @@ -30,7 +30,12 @@ org.jboss.logging jboss-logging - + + + io.smallrye + smallrye-fault-tolerance + 6.6.2 + diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java new file mode 100644 index 0000000..6a04751 --- /dev/null +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java @@ -0,0 +1,182 @@ +package io.smallrye.llm.core.langchain4j.portableextension; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import jakarta.enterprise.context.spi.CreationalContext; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Default; +import jakarta.enterprise.inject.spi.Bean; +import jakarta.enterprise.inject.spi.BeanManager; +import jakarta.enterprise.inject.spi.CDI; +import jakarta.enterprise.inject.spi.InjectionPoint; +import jakarta.enterprise.inject.spi.InterceptionFactory; +import jakarta.enterprise.inject.spi.PassivationCapable; +import jakarta.enterprise.util.AnnotationLiteral; + +import io.smallrye.faulttolerance.FaultToleranceBinding.Literal; +import io.smallrye.llm.aiservice.CommonAIServiceCreator; +import io.smallrye.llm.spi.RegisterAIService; + +/** + * @author Buhake Sindi + * @since 21 November 2024 + */ +public class LangChain4JAIServiceBean implements Bean, PassivationCapable { + + private final Class aiServiceInterfaceClass; + + private final BeanManager beanManager; + + private final Class scope; + + /** + * @param aiServiceInterfaceClass + * @param beanManager + */ + public LangChain4JAIServiceBean(Class aiServiceInterfaceClass, BeanManager beanManager) { + super(); + final RegisterAIService annotation = (this.aiServiceInterfaceClass = aiServiceInterfaceClass) + .getAnnotation(RegisterAIService.class); + this.scope = annotation.scope(); + this.beanManager = beanManager; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.PassivationCapable#getId() + */ + @Override + public String getId() { + // TODO Auto-generated method stub + return aiServiceInterfaceClass.getName(); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.context.spi.Contextual#create(jakarta.enterprise.context.spi.CreationalContext) + */ + @Override + public T create(CreationalContext creationalContext) { + // TODO Auto-generated method stub + InterceptionFactory factory = beanManager.createInterceptionFactory(creationalContext, aiServiceInterfaceClass); + factory.configure().add(new Literal()); + return factory.createInterceptedInstance(CommonAIServiceCreator.create(CDI.current(), aiServiceInterfaceClass)); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.context.spi.Contextual#destroy(java.lang.Object, + * jakarta.enterprise.context.spi.CreationalContext) + */ + @Override + public void destroy(T instance, CreationalContext creationalContext) { + // TODO Auto-generated method stub + + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getTypes() + */ + @Override + public Set getTypes() { + // TODO Auto-generated method stub + return Collections.singleton(aiServiceInterfaceClass); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getQualifiers() + */ + @Override + public Set getQualifiers() { + // TODO Auto-generated method stub + Set annotations = new HashSet<>(); + annotations.add(new AnnotationLiteral() { + }); + annotations.add(new AnnotationLiteral() { + }); + return Collections.unmodifiableSet(annotations); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getScope() + */ + @Override + public Class getScope() { + // TODO Auto-generated method stub + return scope; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getName() + */ + @Override + public String getName() { + // TODO Auto-generated method stub + return "registeredAIService-" + aiServiceInterfaceClass.getName(); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getStereotypes() + */ + @Override + public Set> getStereotypes() { + // TODO Auto-generated method stub + return Collections.singleton(RegisterAIService.class); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#isAlternative() + */ + @Override + public boolean isAlternative() { + // TODO Auto-generated method stub + return false; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.Bean#getBeanClass() + */ + @Override + public Class getBeanClass() { + // TODO Auto-generated method stub + return aiServiceInterfaceClass; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.Bean#getInjectionPoints() + */ + @Override + public Set getInjectionPoints() { + // TODO Auto-generated method stub + return Collections.emptySet(); + } + + @Override + public String toString() { + return "AiService [ interfaceType: " + aiServiceInterfaceClass.getSimpleName() + " ] with Qualifiers [" + + getQualifiers() + "]"; + } +} diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java index 304de6c..f5cf6f0 100644 --- a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java @@ -10,7 +10,6 @@ import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.spi.AfterBeanDiscovery; import jakarta.enterprise.inject.spi.BeanManager; -import jakarta.enterprise.inject.spi.CDI; import jakarta.enterprise.inject.spi.Extension; import jakarta.enterprise.inject.spi.InjectionPoint; import jakarta.enterprise.inject.spi.ProcessAnnotatedType; @@ -19,7 +18,6 @@ import org.jboss.logging.Logger; -import io.smallrye.llm.aiservice.CommonAIServiceCreator; import io.smallrye.llm.spi.RegisterAIService; public class LangChain4JAIServicePortableExtension implements Extension { @@ -63,13 +61,8 @@ void processInjectionPoints(@Observes ProcessInjectionPoint event) { void afterBeanDiscovery(@Observes AfterBeanDiscovery afterBeanDiscovery, BeanManager beanManager) throws ClassNotFoundException { for (Class aiServiceClass : detectedAIServicesDeclaredInterfaces) { - LOGGER.info("afterBeanDiscovery create synthetic : " + aiServiceClass.getName()); - final RegisterAIService annotation = aiServiceClass.getAnnotation(RegisterAIService.class); - afterBeanDiscovery.addBean() - .types(aiServiceClass) - .scope(annotation.scope()) - .name("registeredAIService-" + aiServiceClass.getName()) //Without this, the container won't create a CreationalContext - .createWith(creationalContext -> CommonAIServiceCreator.create(CDI.current(), aiServiceClass)); + LOGGER.info("afterBeanDiscovery create synthetic: " + aiServiceClass.getName()); + afterBeanDiscovery.addBean(new LangChain4JAIServiceBean<>(aiServiceClass, beanManager)); } } diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java new file mode 100644 index 0000000..fc1cd09 --- /dev/null +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java @@ -0,0 +1,44 @@ +package io.smallrye.llm.core.langchain4j.portableextension; + +import java.util.logging.Logger; + +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.spi.AnnotatedMethod; +import jakarta.enterprise.inject.spi.AnnotatedType; +import jakarta.enterprise.inject.spi.BeanManager; +import jakarta.enterprise.inject.spi.DeploymentException; +import jakarta.enterprise.inject.spi.Extension; +import jakarta.enterprise.inject.spi.ProcessSyntheticBean; + +import org.eclipse.microprofile.faulttolerance.exceptions.FaultToleranceDefinitionException; + +import io.smallrye.faulttolerance.autoconfig.FaultToleranceMethod; +import io.smallrye.faulttolerance.config.FaultToleranceMethods; +import io.smallrye.faulttolerance.config.FaultToleranceOperation; + +/** + * @author Buhake Sindi + * @since 29 November 2024 + */ +public class Langchain4JFaultToleranceExtension implements Extension { + + private static final Logger LOGGER = Logger.getLogger(Langchain4JFaultToleranceExtension.class.getName()); + + void validateFaultToleranceOperations(@Observes ProcessSyntheticBean event, BeanManager bm) { + LOGGER.info("validateFaultToleranceOperations: Synthetic Event -> " + event.getBean().getBeanClass()); + try { + AnnotatedType annotatedType = bm.createAnnotatedType(event.getBean().getBeanClass()); + for (AnnotatedMethod annotatedMethod : annotatedType.getMethods()) { + FaultToleranceMethod method = FaultToleranceMethods.create(annotatedType.getJavaClass(), annotatedMethod); + if (method.isLegitimate()) { + FaultToleranceOperation operation = FaultToleranceOperation.create(method); + operation.validate(); + LOGGER.info("Found: " + operation); + } + } + } catch (FaultToleranceDefinitionException e) { + // TODO Auto-generated catch block + throw new DeploymentException(e); + } + } +} diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension b/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension index 4f3c31b..decdbc2 100644 --- a/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension +++ b/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension @@ -1,2 +1,3 @@ io.smallrye.llm.core.langchain4j.portableextension.LangChain4JPluginsPortableExtension io.smallrye.llm.core.langchain4j.portableextension.LangChain4JAIServicePortableExtension +io.smallrye.llm.core.langchain4j.portableextension.Langchain4JFaultToleranceExtension From 0e7347936af35ecace74e722a9b73646f23b4e6d Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Sat, 23 Nov 2024 19:18:36 +0200 Subject: [PATCH 3/6] Enable the Fault Tolerance Interceptor. --- examples/liberty-car-booking/pom.xml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 0563933..ab098af 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -65,14 +65,6 @@ 1.0.0-SNAPSHOT - - dev.langchain4j langchain4j @@ -138,13 +130,6 @@ smallrye-llm-langchain4j-portable-extension - - org.projectlombok lombok From 6e7f3a81c4d76715974fa3a54ee28c4e801cc63d Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Sat, 23 Nov 2024 19:18:36 +0200 Subject: [PATCH 4/6] Enable the Fault Tolerance Interceptor. Fix issues on PR. --- examples/helidon-car-booking/pom.xml | 2 +- examples/liberty-car-booking/pom.xml | 19 ++----------------- .../META-INF/microprofile-config.properties | 8 ++++---- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/examples/helidon-car-booking/pom.xml b/examples/helidon-car-booking/pom.xml index b719960..592cd39 100644 --- a/examples/helidon-car-booking/pom.xml +++ b/examples/helidon-car-booking/pom.xml @@ -12,7 +12,7 @@ io.helidon.Main - 0.36.0 + 0.36.1 diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 0563933..e0325d9 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -56,23 +56,15 @@ io.smallrye.llm smallrye-llm-langchain4j-config-mpconfig - 1.0.0-SNAPSHOT + ${project.version} io.smallrye.llm smallrye-llm-langchain4j-portable-extension - 1.0.0-SNAPSHOT + ${project.version} - - dev.langchain4j langchain4j @@ -138,13 +130,6 @@ smallrye-llm-langchain4j-portable-extension - - org.projectlombok lombok diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties index ef84021..89d8303 100644 --- a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties +++ b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties @@ -2,10 +2,10 @@ server.port=9080 smallrye.llm.plugin.chat-model.class=dev.langchain4j.model.azure.AzureOpenAiChatModel -smallrye.llm.plugin.chat-model.config.api-key=2b4a74c3acd64272a374bfeb84925068 -smallrye.llm.plugin.chat-model.config.endpoint=https://sindi-openai.openai.azure.com/ -smallrye.llm.plugin.chat-model.config.service-version=2023-03-15-preview -smallrye.llm.plugin.chat-model.config.deployment-name=gpt-4 +smallrye.llm.plugin.chat-model.config.api-key=${azure.openai.api.key} +smallrye.llm.plugin.chat-model.config.endpoint=${azure.openai.endpoint} +smallrye.llm.plugin.chat-model.config.service-version=2024-02-15-preview +smallrye.llm.plugin.chat-model.config.deployment-name=${azure.openai.deployment.name} smallrye.llm.plugin.chat-model.config.temperature=0.1 smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s From dcf66ac13385c25e0e8bb09867572d3bb184c4ab Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Tue, 17 Dec 2024 13:23:14 +0200 Subject: [PATCH 5/6] Enable the Fault Tolerance Interceptor. --- examples/liberty-car-booking/pom.xml | 59 ------------------- .../io/jefrajames/booking/FraudAiService.java | 1 - .../LangChain4JAIServiceBean.java | 12 ---- 3 files changed, 72 deletions(-) diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index e0325d9..a76744c 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -24,7 +24,6 @@ 6.1 3.13.0 3.4.0 - 0.36.0 @@ -36,7 +35,6 @@ - jakarta.platform jakarta.jakartaee-api @@ -51,65 +49,12 @@ pom provided - - - - io.smallrye.llm - smallrye-llm-langchain4j-config-mpconfig - ${project.version} - - - - io.smallrye.llm - smallrye-llm-langchain4j-portable-extension - ${project.version} - - - - dev.langchain4j - langchain4j - ${dev.langchain4j.version} - - - - dev.langchain4j - langchain4j-hugging-face - ${dev.langchain4j.version} - - - - - dev.langchain4j - langchain4j-azure-open-ai - ${dev.langchain4j.version} - - - - dev.langchain4j - langchain4j-open-ai - ${dev.langchain4j.version} - - - - dev.langchain4j - langchain4j-embeddings-all-minilm-l6-v2 - ${dev.langchain4j.version} - - ai.djl.huggingface tokenizers 0.30.0 - - - - org.slf4j - slf4j-jdk14 - runtime - 2.0.9 - @@ -146,7 +91,6 @@ langchain4j-hugging-face - dev.langchain4j langchain4j-azure-open-ai @@ -162,7 +106,6 @@ langchain4j-embeddings-all-minilm-l6-v2 - ai.djl.huggingface tokenizers @@ -172,7 +115,6 @@ org.slf4j slf4j-jdk14 - runtime @@ -180,7 +122,6 @@ ${project.artifactId} - io.openliberty.tools liberty-maven-plugin diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java index 77e5086..bb33cd4 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java @@ -11,7 +11,6 @@ import dev.langchain4j.service.V; import io.smallrye.llm.spi.RegisterAIService; -//@SuppressWarnings("CdiManagedBeanInconsistencyInspection") @RegisterAIService(chatMemoryMaxMessages = 5, chatLanguageModelName = "chat-model") public interface FraudAiService { diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java index 6a04751..f7dc2b1 100644 --- a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java @@ -52,7 +52,6 @@ public LangChain4JAIServiceBean(Class aiServiceInterfaceClass, BeanManager be */ @Override public String getId() { - // TODO Auto-generated method stub return aiServiceInterfaceClass.getName(); } @@ -63,7 +62,6 @@ public String getId() { */ @Override public T create(CreationalContext creationalContext) { - // TODO Auto-generated method stub InterceptionFactory factory = beanManager.createInterceptionFactory(creationalContext, aiServiceInterfaceClass); factory.configure().add(new Literal()); return factory.createInterceptedInstance(CommonAIServiceCreator.create(CDI.current(), aiServiceInterfaceClass)); @@ -77,8 +75,6 @@ public T create(CreationalContext creationalContext) { */ @Override public void destroy(T instance, CreationalContext creationalContext) { - // TODO Auto-generated method stub - } /* @@ -88,7 +84,6 @@ public void destroy(T instance, CreationalContext creationalContext) { */ @Override public Set getTypes() { - // TODO Auto-generated method stub return Collections.singleton(aiServiceInterfaceClass); } @@ -99,7 +94,6 @@ public Set getTypes() { */ @Override public Set getQualifiers() { - // TODO Auto-generated method stub Set annotations = new HashSet<>(); annotations.add(new AnnotationLiteral() { }); @@ -115,7 +109,6 @@ public Set getQualifiers() { */ @Override public Class getScope() { - // TODO Auto-generated method stub return scope; } @@ -126,7 +119,6 @@ public Class getScope() { */ @Override public String getName() { - // TODO Auto-generated method stub return "registeredAIService-" + aiServiceInterfaceClass.getName(); } @@ -137,7 +129,6 @@ public String getName() { */ @Override public Set> getStereotypes() { - // TODO Auto-generated method stub return Collections.singleton(RegisterAIService.class); } @@ -148,7 +139,6 @@ public Set> getStereotypes() { */ @Override public boolean isAlternative() { - // TODO Auto-generated method stub return false; } @@ -159,7 +149,6 @@ public boolean isAlternative() { */ @Override public Class getBeanClass() { - // TODO Auto-generated method stub return aiServiceInterfaceClass; } @@ -170,7 +159,6 @@ public Class getBeanClass() { */ @Override public Set getInjectionPoints() { - // TODO Auto-generated method stub return Collections.emptySet(); } From 8a7bd8fa04fcd629eeb9be3c992363d42bb4563b Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Sat, 23 Nov 2024 19:13:29 +0200 Subject: [PATCH 6/6] Enable the Fault Tolerance Interceptor. Enable the Fault Tolerance Interceptor. Fix issues on PR. Enable the Fault Tolerance Interceptor. Enable the Fault Tolerance Interceptor. --- examples/helidon-car-booking/pom.xml | 2 +- examples/liberty-car-booking/README.md | 2 +- examples/liberty-car-booking/pom.xml | 77 ++------ .../booking/CarBookingResource.java | 2 - .../io/jefrajames/booking/ChatAiService.java | 4 +- .../io/jefrajames/booking/DocRagIngestor.java | 21 +-- .../io/jefrajames/booking/FraudAiService.java | 14 +- .../META-INF/microprofile-config.properties | 2 +- .../src/main/resources/application.properties | 2 +- .../llm/aiservice/CommonAIServiceCreator.java | 99 +++++----- .../llm/plugin/CommonLLMPluginCreator.java | 33 +++- .../smallrye/llm/spi/RegisterAIService.java | 4 +- .../pom.xml | 7 +- .../LangChain4JAIServiceBean.java | 170 ++++++++++++++++++ ...LangChain4JAIServicePortableExtension.java | 11 +- .../Langchain4JFaultToleranceExtension.java | 44 +++++ .../jakarta.enterprise.inject.spi.Extension | 1 + 17 files changed, 343 insertions(+), 152 deletions(-) create mode 100644 smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java create mode 100644 smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java diff --git a/examples/helidon-car-booking/pom.xml b/examples/helidon-car-booking/pom.xml index 3e1d666..592cd39 100644 --- a/examples/helidon-car-booking/pom.xml +++ b/examples/helidon-car-booking/pom.xml @@ -12,7 +12,7 @@ io.helidon.Main - 0.34.0 + 0.36.1 diff --git a/examples/liberty-car-booking/README.md b/examples/liberty-car-booking/README.md index 92561c7..3d61654 100644 --- a/examples/liberty-car-booking/README.md +++ b/examples/liberty-car-booking/README.md @@ -39,7 +39,7 @@ To package the application in JVM mode run: `mvn package`. ## Configuration -All configuration is centralized in `microprofile-config.properties`(found is `resources\META-INF` folder) and can be redefined using environment variables. +All configuration is centralized in `microprofile-config.properties` (found is `resources\META-INF` folder) and can be redefined using environment variables. ## Running the application diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 786d7fc..a76744c 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -24,7 +24,6 @@ 6.1 3.13.0 3.4.0 - 0.34.0 @@ -36,7 +35,6 @@ - jakarta.platform jakarta.jakartaee-api @@ -51,46 +49,21 @@ pom provided - - - - - - ai.djl.huggingface - tokenizers - 0.30.0 - - - + + + ai.djl.huggingface + tokenizers + 0.30.0 + + + - - - - - - org.eclipse.microprofile - microprofile - pom - + + + org.eclipse.microprofile + microprofile + pom + io.smallrye.llm @@ -102,27 +75,9 @@ smallrye-llm-langchain4j-portable-extension - - - - org.projectlombok lombok - provided @@ -136,7 +91,6 @@ langchain4j-hugging-face - dev.langchain4j langchain4j-azure-open-ai @@ -152,7 +106,6 @@ langchain4j-embeddings-all-minilm-l6-v2 - ai.djl.huggingface tokenizers @@ -162,7 +115,6 @@ org.slf4j slf4j-jdk14 - runtime @@ -170,7 +122,6 @@ ${project.artifactId} - io.openliberty.tools liberty-maven-plugin diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java index 8faf5f9..1922a3b 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/CarBookingResource.java @@ -40,9 +40,7 @@ public String chatWithAssistant( @Path("/fraud") public FraudResponse detectFraudForCustomer( @QueryParam("name") String name, - @QueryParam("surname") String surname) { return fraudService.detectFraudForCustomer(name, surname); } - } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java index bc6892f..660e718 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/ChatAiService.java @@ -2,6 +2,8 @@ import java.time.temporal.ChronoUnit; +import jakarta.enterprise.context.ApplicationScoped; + import org.eclipse.microprofile.faulttolerance.Fallback; import org.eclipse.microprofile.faulttolerance.Retry; import org.eclipse.microprofile.faulttolerance.Timeout; @@ -10,7 +12,7 @@ import io.smallrye.llm.spi.RegisterAIService; //@SuppressWarnings("CdiManagedBeanInconsistencyInspection") -@RegisterAIService(tools = BookingService.class, chatMemoryMaxMessages = 10) +@RegisterAIService(scope = ApplicationScoped.class, tools = BookingService.class, chatMemoryMaxMessages = 10) public interface ChatAiService { @SystemMessage(""" diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java index 2be0473..f50a395 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java @@ -5,6 +5,12 @@ import java.io.File; import java.util.List; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.context.Initialized; +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Inject; + import org.eclipse.microprofile.config.inject.ConfigProperty; import dev.langchain4j.data.document.Document; @@ -13,13 +19,9 @@ import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; import dev.langchain4j.store.embedding.EmbeddingStoreIngestor; import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.context.Initialized; -import jakarta.enterprise.event.Observes; -import jakarta.enterprise.inject.Produces; -import jakarta.inject.Inject; import lombok.extern.java.Log; @Log @@ -32,9 +34,7 @@ public class DocRagIngestor { // Used by ContentRetriever @Produces - private InMemoryEmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); - - // private File docs = new File(System.getProperty("docragdir")); + private EmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); @Inject @ConfigProperty(name = "app.docs-for-rag.dir") @@ -60,9 +60,4 @@ public void ingest(@Observes @Initialized(ApplicationScoped.class) Object pointl log.info(String.format("DEMO %d documents ingested in %d msec", docs.size(), System.currentTimeMillis() - start)); } - - public static void main(String[] args) { - - System.out.println(InMemoryEmbeddingStore.class.getInterfaces()[0]); - } } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java index 68488de..bb33cd4 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/FraudAiService.java @@ -1,14 +1,17 @@ package io.jefrajames.booking; +import java.time.temporal.ChronoUnit; + +import org.eclipse.microprofile.faulttolerance.Fallback; +import org.eclipse.microprofile.faulttolerance.Retry; +import org.eclipse.microprofile.faulttolerance.Timeout; + import dev.langchain4j.service.SystemMessage; import dev.langchain4j.service.UserMessage; import dev.langchain4j.service.V; import io.smallrye.llm.spi.RegisterAIService; -@SuppressWarnings("CdiManagedBeanInconsistencyInspection") -@RegisterAIService(chatMemoryMaxMessages = 5, - - chatLanguageModelName = "chat-model") +@RegisterAIService(chatMemoryMaxMessages = 5, chatLanguageModelName = "chat-model") public interface FraudAiService { @SystemMessage(""" @@ -44,6 +47,9 @@ A booking overlap (and hence a fraud) occurs when there are several bookings for You must not wrap JSON response in backticks, markdown, or in any other way, but return it as plain text. """) + @Timeout(unit = ChronoUnit.MINUTES, value = 5) + @Retry(maxRetries = 2) + @Fallback(fallbackMethod = "fraudFallback") FraudResponse detectFraudForCustomer(@V("name") String name, @V("surname") String surname); default FraudResponse fraudFallback(String name, String surname) { diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties index c793531..89d8303 100644 --- a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties +++ b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties @@ -10,7 +10,7 @@ smallrye.llm.plugin.chat-model.config.temperature=0.1 smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s smallrye.llm.plugin.chat-model.config.max-retries=2 -#smallrye.llm.plugin.chat-model.config.logRequestsAndResponsess=false +#smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=true smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever diff --git a/examples/quarkus-car-booking/src/main/resources/application.properties b/examples/quarkus-car-booking/src/main/resources/application.properties index 08f08e3..d9ab556 100644 --- a/examples/quarkus-car-booking/src/main/resources/application.properties +++ b/examples/quarkus-car-booking/src/main/resources/application.properties @@ -11,7 +11,7 @@ smallrye.llm.plugin.chat-model.config.temperature=0.1 smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s smallrye.llm.plugin.chat-model.config.max-retries=2 -#smallrye.llm.plugin.chat-model.config.logRequestsAndResponsess=false +#smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=false smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java index 60d0755..1db83a6 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/aiservice/CommonAIServiceCreator.java @@ -6,6 +6,9 @@ import java.util.ArrayList; import java.util.List; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.literal.NamedLiteral; + import org.jboss.logging.Logger; import dev.langchain4j.memory.chat.ChatMemoryProvider; @@ -19,12 +22,10 @@ import dev.langchain4j.store.memory.chat.ChatMemoryStore; import io.smallrye.llm.core.langchain4j.core.config.spi.ChatMemoryFactoryProvider; import io.smallrye.llm.spi.RegisterAIService; -import jakarta.enterprise.inject.Instance; -import jakarta.enterprise.inject.literal.NamedLiteral; public class CommonAIServiceCreator { - private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); + private static final Logger LOGGER = Logger.getLogger(CommonAIServiceCreator.class); @SuppressWarnings("unchecked") public static X create(Instance lookup, Class interfaceClass) { @@ -54,20 +55,21 @@ public static X create(Instance lookup, Class interfaceClass) { } aiServices.tools(tools); } - + ChatMemoryProvider chatMemoryProvider = createChatMemoryProvider(lookup, interfaceClass, annotation); if (chatMemoryProvider != null) { - aiServices.chatMemoryProvider(chatMemoryProvider); + aiServices.chatMemoryProvider(chatMemoryProvider); } else { - aiServices.chatMemory( - ChatMemoryFactoryProvider.getChatMemoryFactory().getChatMemory(lookup, annotation.chatMemoryMaxMessages())); + aiServices.chatMemory( + ChatMemoryFactoryProvider.getChatMemoryFactory().getChatMemory(lookup, + annotation.chatMemoryMaxMessages())); } - + ModerationModel moderationModel = findModerationModel(lookup, interfaceClass, annotation); if (moderationModel != null) { - aiServices.moderationModel(moderationModel); + aiServices.moderationModel(moderationModel); } - return (X)aiServices.build(); + return (X) aiServices.build(); } catch (Exception e) { throw new RuntimeException(e); } @@ -80,43 +82,46 @@ private static Instance getInstance(Instance lookup, Class typ } return lookup.select(type, NamedLiteral.of(name)); } - - private static ModerationModel findModerationModel(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { - //Get all methods. - for (Method method : interfaceClass.getMethods()) { - Moderate moderate = method.getAnnotation(Moderate.class); - if (moderate != null) { - Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, registerAIService.moderationModelName()); - if (moderationModelInstance != null && moderationModelInstance.isResolvable()) return moderationModelInstance.get(); - } - } - - return null; + + private static ModerationModel findModerationModel(Instance lookup, Class interfaceClass, + RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + Moderate moderate = method.getAnnotation(Moderate.class); + if (moderate != null) { + Instance moderationModelInstance = getInstance(lookup, ModerationModel.class, + registerAIService.moderationModelName()); + if (moderationModelInstance != null && moderationModelInstance.isResolvable()) + return moderationModelInstance.get(); + } + } + + return null; } - - - private static ChatMemoryProvider createChatMemoryProvider(Instance lookup, Class interfaceClass, RegisterAIService registerAIService) { - //Get all methods. - for (Method method : interfaceClass.getMethods()) { - for (Parameter parameter : method.getParameters()) { - MemoryId memoryIdAnnotation = parameter.getAnnotation(MemoryId.class); - if (memoryIdAnnotation != null) { - Instance chatMemoryStore = getInstance(lookup, ChatMemoryStore.class, - registerAIService.chatMemoryStoreName()); - if (chatMemoryStore == null || !chatMemoryStore.isResolvable()) { - throw new IllegalStateException("Unable to resolve a ChatMemoryStore for your ChatMemoryProvider."); - } - - ChatMemoryProvider chatMemoryProvider = memoryId -> MessageWindowChatMemory.builder() - .id(memoryId) - .maxMessages(registerAIService.chatMemoryMaxMessages()) - .chatMemoryStore(chatMemoryStore.get()) - .build(); - return chatMemoryProvider; - } - } - } - - return null; + + private static ChatMemoryProvider createChatMemoryProvider(Instance lookup, Class interfaceClass, + RegisterAIService registerAIService) { + //Get all methods. + for (Method method : interfaceClass.getMethods()) { + for (Parameter parameter : method.getParameters()) { + MemoryId memoryIdAnnotation = parameter.getAnnotation(MemoryId.class); + if (memoryIdAnnotation != null) { + Instance chatMemoryStore = getInstance(lookup, ChatMemoryStore.class, + registerAIService.chatMemoryStoreName()); + if (chatMemoryStore == null || !chatMemoryStore.isResolvable()) { + throw new IllegalStateException("Unable to resolve a ChatMemoryStore for your ChatMemoryProvider."); + } + + ChatMemoryProvider chatMemoryProvider = memoryId -> MessageWindowChatMemory.builder() + .id(memoryId) + .maxMessages(registerAIService.chatMemoryMaxMessages()) + .chatMemoryStore(chatMemoryStore.get()) + .build(); + return chatMemoryProvider; + } + } + } + + return null; } } diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java index 6d5525b..f46f912 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java @@ -6,7 +6,9 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -15,10 +17,12 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.literal.NamedLiteral; -import jakarta.enterprise.inject.spi.CDI; +import jakarta.enterprise.util.TypeLiteral; import org.jboss.logging.Logger; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfig; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfigProvider; @@ -34,6 +38,13 @@ public class CommonLLMPluginCreator { public static final Logger LOGGER = Logger.getLogger(CommonLLMPluginCreator.class); + private static final Map, TypeLiteral> TYPE_LITERALS = new HashMap<>(); + + static { + TYPE_LITERALS.put(EmbeddingStore.class, new TypeLiteral>() { + }); + } + @SuppressWarnings("unchecked") public static void createAllLLMBeans(LLMConfig llmConfig, Consumer beanBuilder) throws ClassNotFoundException { Set beanNameToCreate = llmConfig.getBeanNames(); @@ -135,12 +146,9 @@ public static Object create(Instance lookup, String beanName, Class t LOGGER.info("Lookup " + lookupableBean + " " + parameterType); Instance inst; if ("default".equals(lookupableBean)) { - inst = lookup.select(parameterType); - if (!inst.isResolvable()) { - inst = CDI.current().select(parameterType); - } + inst = getInstance(lookup, parameterType); } else { - inst = lookup.select(parameterType, NamedLiteral.of(lookupableBean)); + inst = getInstance(lookup, parameterType, lookupableBean); } methodToCall.invoke(builder, inst.get()); break; @@ -165,4 +173,17 @@ public static Object create(Instance lookup, String beanName, Class t private static Class loadClass(String scopeClassName) throws ClassNotFoundException { return Thread.currentThread().getContextClassLoader().loadClass(scopeClassName); } + + @SuppressWarnings("unchecked") + private static Instance getInstance(Instance lookup, Class clazz) { + if (TYPE_LITERALS.containsKey(clazz)) + return (Instance) lookup.select(TYPE_LITERALS.get(clazz)); + return lookup.select(clazz); + } + + private static Instance getInstance(Instance lookup, Class clazz, String lookupName) { + if (lookupName == null || lookupName.isBlank()) + return getInstance(lookup, clazz); + return lookup.select(clazz, NamedLiteral.of(lookupName)); + } } diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java index 91dbbea..733073a 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/spi/RegisterAIService.java @@ -30,8 +30,8 @@ String embeddingStoreName() default ""; String contentRetrieverName() default ""; - + String moderationModelName() default ""; - + String chatMemoryStoreName() default ""; } diff --git a/smallrye-llm-langchain4j-portable-extension/pom.xml b/smallrye-llm-langchain4j-portable-extension/pom.xml index 0bbf84d..e96eee9 100644 --- a/smallrye-llm-langchain4j-portable-extension/pom.xml +++ b/smallrye-llm-langchain4j-portable-extension/pom.xml @@ -30,7 +30,12 @@ org.jboss.logging jboss-logging - + + + io.smallrye + smallrye-fault-tolerance + 6.6.2 + diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java new file mode 100644 index 0000000..f7dc2b1 --- /dev/null +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServiceBean.java @@ -0,0 +1,170 @@ +package io.smallrye.llm.core.langchain4j.portableextension; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import jakarta.enterprise.context.spi.CreationalContext; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Default; +import jakarta.enterprise.inject.spi.Bean; +import jakarta.enterprise.inject.spi.BeanManager; +import jakarta.enterprise.inject.spi.CDI; +import jakarta.enterprise.inject.spi.InjectionPoint; +import jakarta.enterprise.inject.spi.InterceptionFactory; +import jakarta.enterprise.inject.spi.PassivationCapable; +import jakarta.enterprise.util.AnnotationLiteral; + +import io.smallrye.faulttolerance.FaultToleranceBinding.Literal; +import io.smallrye.llm.aiservice.CommonAIServiceCreator; +import io.smallrye.llm.spi.RegisterAIService; + +/** + * @author Buhake Sindi + * @since 21 November 2024 + */ +public class LangChain4JAIServiceBean implements Bean, PassivationCapable { + + private final Class aiServiceInterfaceClass; + + private final BeanManager beanManager; + + private final Class scope; + + /** + * @param aiServiceInterfaceClass + * @param beanManager + */ + public LangChain4JAIServiceBean(Class aiServiceInterfaceClass, BeanManager beanManager) { + super(); + final RegisterAIService annotation = (this.aiServiceInterfaceClass = aiServiceInterfaceClass) + .getAnnotation(RegisterAIService.class); + this.scope = annotation.scope(); + this.beanManager = beanManager; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.PassivationCapable#getId() + */ + @Override + public String getId() { + return aiServiceInterfaceClass.getName(); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.context.spi.Contextual#create(jakarta.enterprise.context.spi.CreationalContext) + */ + @Override + public T create(CreationalContext creationalContext) { + InterceptionFactory factory = beanManager.createInterceptionFactory(creationalContext, aiServiceInterfaceClass); + factory.configure().add(new Literal()); + return factory.createInterceptedInstance(CommonAIServiceCreator.create(CDI.current(), aiServiceInterfaceClass)); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.context.spi.Contextual#destroy(java.lang.Object, + * jakarta.enterprise.context.spi.CreationalContext) + */ + @Override + public void destroy(T instance, CreationalContext creationalContext) { + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getTypes() + */ + @Override + public Set getTypes() { + return Collections.singleton(aiServiceInterfaceClass); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getQualifiers() + */ + @Override + public Set getQualifiers() { + Set annotations = new HashSet<>(); + annotations.add(new AnnotationLiteral() { + }); + annotations.add(new AnnotationLiteral() { + }); + return Collections.unmodifiableSet(annotations); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getScope() + */ + @Override + public Class getScope() { + return scope; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getName() + */ + @Override + public String getName() { + return "registeredAIService-" + aiServiceInterfaceClass.getName(); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#getStereotypes() + */ + @Override + public Set> getStereotypes() { + return Collections.singleton(RegisterAIService.class); + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.BeanAttributes#isAlternative() + */ + @Override + public boolean isAlternative() { + return false; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.Bean#getBeanClass() + */ + @Override + public Class getBeanClass() { + return aiServiceInterfaceClass; + } + + /* + * (non-Javadoc) + * + * @see jakarta.enterprise.inject.spi.Bean#getInjectionPoints() + */ + @Override + public Set getInjectionPoints() { + return Collections.emptySet(); + } + + @Override + public String toString() { + return "AiService [ interfaceType: " + aiServiceInterfaceClass.getSimpleName() + " ] with Qualifiers [" + + getQualifiers() + "]"; + } +} diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java index 304de6c..f5cf6f0 100644 --- a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/LangChain4JAIServicePortableExtension.java @@ -10,7 +10,6 @@ import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.spi.AfterBeanDiscovery; import jakarta.enterprise.inject.spi.BeanManager; -import jakarta.enterprise.inject.spi.CDI; import jakarta.enterprise.inject.spi.Extension; import jakarta.enterprise.inject.spi.InjectionPoint; import jakarta.enterprise.inject.spi.ProcessAnnotatedType; @@ -19,7 +18,6 @@ import org.jboss.logging.Logger; -import io.smallrye.llm.aiservice.CommonAIServiceCreator; import io.smallrye.llm.spi.RegisterAIService; public class LangChain4JAIServicePortableExtension implements Extension { @@ -63,13 +61,8 @@ void processInjectionPoints(@Observes ProcessInjectionPoint event) { void afterBeanDiscovery(@Observes AfterBeanDiscovery afterBeanDiscovery, BeanManager beanManager) throws ClassNotFoundException { for (Class aiServiceClass : detectedAIServicesDeclaredInterfaces) { - LOGGER.info("afterBeanDiscovery create synthetic : " + aiServiceClass.getName()); - final RegisterAIService annotation = aiServiceClass.getAnnotation(RegisterAIService.class); - afterBeanDiscovery.addBean() - .types(aiServiceClass) - .scope(annotation.scope()) - .name("registeredAIService-" + aiServiceClass.getName()) //Without this, the container won't create a CreationalContext - .createWith(creationalContext -> CommonAIServiceCreator.create(CDI.current(), aiServiceClass)); + LOGGER.info("afterBeanDiscovery create synthetic: " + aiServiceClass.getName()); + afterBeanDiscovery.addBean(new LangChain4JAIServiceBean<>(aiServiceClass, beanManager)); } } diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java new file mode 100644 index 0000000..fc1cd09 --- /dev/null +++ b/smallrye-llm-langchain4j-portable-extension/src/main/java/io/smallrye/llm/core/langchain4j/portableextension/Langchain4JFaultToleranceExtension.java @@ -0,0 +1,44 @@ +package io.smallrye.llm.core.langchain4j.portableextension; + +import java.util.logging.Logger; + +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.spi.AnnotatedMethod; +import jakarta.enterprise.inject.spi.AnnotatedType; +import jakarta.enterprise.inject.spi.BeanManager; +import jakarta.enterprise.inject.spi.DeploymentException; +import jakarta.enterprise.inject.spi.Extension; +import jakarta.enterprise.inject.spi.ProcessSyntheticBean; + +import org.eclipse.microprofile.faulttolerance.exceptions.FaultToleranceDefinitionException; + +import io.smallrye.faulttolerance.autoconfig.FaultToleranceMethod; +import io.smallrye.faulttolerance.config.FaultToleranceMethods; +import io.smallrye.faulttolerance.config.FaultToleranceOperation; + +/** + * @author Buhake Sindi + * @since 29 November 2024 + */ +public class Langchain4JFaultToleranceExtension implements Extension { + + private static final Logger LOGGER = Logger.getLogger(Langchain4JFaultToleranceExtension.class.getName()); + + void validateFaultToleranceOperations(@Observes ProcessSyntheticBean event, BeanManager bm) { + LOGGER.info("validateFaultToleranceOperations: Synthetic Event -> " + event.getBean().getBeanClass()); + try { + AnnotatedType annotatedType = bm.createAnnotatedType(event.getBean().getBeanClass()); + for (AnnotatedMethod annotatedMethod : annotatedType.getMethods()) { + FaultToleranceMethod method = FaultToleranceMethods.create(annotatedType.getJavaClass(), annotatedMethod); + if (method.isLegitimate()) { + FaultToleranceOperation operation = FaultToleranceOperation.create(method); + operation.validate(); + LOGGER.info("Found: " + operation); + } + } + } catch (FaultToleranceDefinitionException e) { + // TODO Auto-generated catch block + throw new DeploymentException(e); + } + } +} diff --git a/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension b/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension index 4f3c31b..decdbc2 100644 --- a/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension +++ b/smallrye-llm-langchain4j-portable-extension/src/main/resources/META-INF/services/jakarta.enterprise.inject.spi.Extension @@ -1,2 +1,3 @@ io.smallrye.llm.core.langchain4j.portableextension.LangChain4JPluginsPortableExtension io.smallrye.llm.core.langchain4j.portableextension.LangChain4JAIServicePortableExtension +io.smallrye.llm.core.langchain4j.portableextension.Langchain4JFaultToleranceExtension