From 18d4262d8c9003082068a3383bb8fb8c92f09048 Mon Sep 17 00:00:00 2001 From: Buhake Sindi Date: Wed, 27 Nov 2024 21:52:54 +0200 Subject: [PATCH] Added support for Metrics and Telemetry tracing using Microprofile Metrics & Microprofile Telemetry respectively. --- examples/liberty-car-booking/pom.xml | 138 +++++++++++---- .../java/io/jefrajames/booking/Booking.java | 162 +++++++++++++++++- .../io/jefrajames/booking/BookingService.java | 13 +- .../java/io/jefrajames/booking/Customer.java | 100 ++++++++++- .../io/jefrajames/booking/DocRagIngestor.java | 7 +- .../io/jefrajames/booking/DummyLLConfig.java | 1 + .../src/main/liberty/config/server.xml | 12 +- .../META-INF/microprofile-config.properties | 10 +- pom.xml | 10 +- .../pom.xml | 1 - .../llm/plugin/CommonLLMPluginCreator.java | 82 +++++++-- smallrye-llm-langchain4j-telemetry/pom.xml | 30 ++++ .../telemetry/MetricsChatModelListener.java | 144 ++++++++++++++++ .../telemetry/SpanChatModelListener.java | 113 ++++++++++++ 14 files changed, 743 insertions(+), 80 deletions(-) create mode 100644 smallrye-llm-langchain4j-telemetry/pom.xml create mode 100644 smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java create mode 100644 smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java diff --git a/examples/liberty-car-booking/pom.xml b/examples/liberty-car-booking/pom.xml index 69ef05c..fdb876c 100644 --- a/examples/liberty-car-booking/pom.xml +++ b/examples/liberty-car-booking/pom.xml @@ -46,45 +46,104 @@ provided - - - - - ai.djl.huggingface - tokenizers - 0.30.0 - - - + + + io.opentelemetry + opentelemetry-api + 1.44.1 + provided + + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-annotations + 2.10.0 + provided + - - - + + io.smallrye.llm + smallrye-llm-langchain4j-config-mpconfig + 1.0.0-SNAPSHOT + + + + io.smallrye.llm + smallrye-llm-langchain4j-portable-extension + 1.0.0-SNAPSHOT + + + + io.smallrye.llm + smallrye-llm-langchain4j-telemetry + 1.0.0-SNAPSHOT + + + + dev.langchain4j + langchain4j + ${dev.langchain4j.version} + + + + dev.langchain4j + langchain4j-hugging-face + ${dev.langchain4j.version} + + + + + dev.langchain4j + langchain4j-azure-open-ai + ${dev.langchain4j.version} + + + + dev.langchain4j + langchain4j-open-ai + ${dev.langchain4j.version} + + + + dev.langchain4j + langchain4j-embeddings-all-minilm-l6-v2 + ${dev.langchain4j.version} + + + + + ai.djl.huggingface + tokenizers + 0.30.0 + + + + + org.slf4j + slf4j-jdk14 + runtime + 2.0.9 + + + + + + + org.eclipse.microprofile + microprofile + pom + - - org.eclipse.microprofile - microprofile - pom - + + io.opentelemetry + opentelemetry-api + + + + + io.opentelemetry.instrumentation + opentelemetry-instrumentation-annotations + io.smallrye.llm @@ -113,6 +172,11 @@ --> + + io.smallrye.llm + smallrye-llm-langchain4j-telemetry + + org.projectlombok lombok diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Booking.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Booking.java index a8e03c1..bf1ea1e 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Booking.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Booking.java @@ -1,14 +1,11 @@ package io.jefrajames.booking; import java.time.LocalDate; +import java.util.Objects; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor +//@Data +//@NoArgsConstructor +//@AllArgsConstructor public class Booking { private String bookingNumber; @@ -18,4 +15,155 @@ public class Booking { private boolean canceled = false; private String carModel; + /** + * + */ + public Booking() { + super(); + //TODO Auto-generated constructor stub + } + + /** + * @param bookingNumber + * @param start + * @param end + * @param customer + * @param canceled + * @param carModel + */ + public Booking(String bookingNumber, LocalDate start, LocalDate end, Customer customer, boolean canceled, + String carModel) { + super(); + this.bookingNumber = bookingNumber; + this.start = start; + this.end = end; + this.customer = customer; + this.canceled = canceled; + this.carModel = carModel; + } + + /** + * @return the bookingNumber + */ + public String getBookingNumber() { + return bookingNumber; + } + + /** + * @param bookingNumber the bookingNumber to set + */ + public void setBookingNumber(String bookingNumber) { + this.bookingNumber = bookingNumber; + } + + /** + * @return the start + */ + public LocalDate getStart() { + return start; + } + + /** + * @param start the start to set + */ + public void setStart(LocalDate start) { + this.start = start; + } + + /** + * @return the end + */ + public LocalDate getEnd() { + return end; + } + + /** + * @param end the end to set + */ + public void setEnd(LocalDate end) { + this.end = end; + } + + /** + * @return the customer + */ + public Customer getCustomer() { + return customer; + } + + /** + * @param customer the customer to set + */ + public void setCustomer(Customer customer) { + this.customer = customer; + } + + /** + * @return the canceled + */ + public boolean isCanceled() { + return canceled; + } + + /** + * @param canceled the canceled to set + */ + public void setCanceled(boolean canceled) { + this.canceled = canceled; + } + + /** + * @return the carModel + */ + public String getCarModel() { + return carModel; + } + + /** + * @param carModel the carModel to set + */ + public void setCarModel(String carModel) { + this.carModel = carModel; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + return Objects.hash(bookingNumber, canceled, carModel, customer, end, start); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Booking other = (Booking) obj; + return Objects.equals(bookingNumber, other.bookingNumber) && canceled == other.canceled + && Objects.equals(carModel, other.carModel) && Objects.equals(customer, other.customer) + && Objects.equals(end, other.end) && Objects.equals(start, other.start); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "Booking [bookingNumber=" + bookingNumber + ", start=" + start + ", end=" + end + ", customer=" + + customer + ", canceled=" + canceled + ", carModel=" + carModel + "]"; + } } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/BookingService.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/BookingService.java index 4ee5ff5..36fd025 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/BookingService.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/BookingService.java @@ -8,13 +8,16 @@ import jakarta.enterprise.context.ApplicationScoped; +import org.jboss.logging.Logger; + import dev.langchain4j.agent.tool.Tool; -import lombok.extern.java.Log; @ApplicationScoped -@Log +//@Log public class BookingService { + private static final Logger LOGGER = Logger.getLogger(BookingService.class.getName()); + // Pseudo database private static final Map BOOKINGS = new HashMap<>(); static { @@ -44,14 +47,14 @@ private Booking checkBookingExists(String bookingNumber, String name, String sur @Tool("Get booking details given a booking number and customer name and surname") public Booking getBookingDetails(String bookingNumber, String name, String surname) { - log.info("DEMO: Calling Tool-getBookingDetails: " + bookingNumber + " and customer: " + LOGGER.info("DEMO: Calling Tool-getBookingDetails: " + bookingNumber + " and customer: " + name + " " + surname); return checkBookingExists(bookingNumber, name, surname); } @Tool("Get all booking ids for a customer given his name and surname") public List getBookingsForCustomer(String name, String surname) { - log.info("DEMO: Calling Tool-getBookingsForCustomer: " + name + " " + surname); + LOGGER.info("DEMO: Calling Tool-getBookingsForCustomer: " + name + " " + surname); Customer customer = new Customer(name, surname); return BOOKINGS.values() .stream() @@ -77,7 +80,7 @@ public void checkCancelPolicy(Booking booking) { @Tool("Cancel a booking given its booking number and customer name and surname") public Booking cancelBooking(String bookingNumber, String name, String surname) { - log.info("DEMO: Calling Tool-cancelBooking " + bookingNumber + " for customer: " + name + LOGGER.info("DEMO: Calling Tool-cancelBooking " + bookingNumber + " for customer: " + name + " " + surname); Booking booking = checkBookingExists(bookingNumber, name, surname); diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Customer.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Customer.java index f2488f0..34203f3 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Customer.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/Customer.java @@ -1,15 +1,97 @@ package io.jefrajames.booking; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(of = { "name", "surname" }) +import java.util.Objects; + +//@Data +//@NoArgsConstructor +//@AllArgsConstructor +//@EqualsAndHashCode(of = { "name", "surname" }) public class Customer { private String name; private String surname; + + /** + * + */ + public Customer() { + super(); + //TODO Auto-generated constructor stub + } + + /** + * @param name + * @param surname + */ + public Customer(String name, String surname) { + super(); + this.name = name; + this.surname = surname; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the surname + */ + public String getSurname() { + return surname; + } + + /** + * @param surname the surname to set + */ + public void setSurname(String surname) { + this.surname = surname; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + return Objects.hash(name, surname); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Customer other = (Customer) obj; + return Objects.equals(name, other.name) && Objects.equals(surname, other.surname); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + + @Override + public String toString() { + return "Customer [name=" + name + ", surname=" + surname + "]"; + } } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java index 70d7ff1..cb1c3c9 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DocRagIngestor.java @@ -12,6 +12,7 @@ import jakarta.inject.Inject; import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; import dev.langchain4j.data.document.Document; import dev.langchain4j.data.document.parser.TextDocumentParser; @@ -20,12 +21,12 @@ import dev.langchain4j.model.embedding.onnx.allminilml6v2.AllMiniLmL6V2EmbeddingModel; import dev.langchain4j.store.embedding.EmbeddingStoreIngestor; import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore; -import lombok.extern.java.Log; -@Log @ApplicationScoped public class DocRagIngestor { + private static final Logger LOGGER = Logger.getLogger(DocRagIngestor.class.getName()); + // Used by ContentRetriever @Produces private EmbeddingModel embeddingModel = new AllMiniLmL6V2EmbeddingModel(); @@ -57,7 +58,7 @@ public void ingest(@Observes @Initialized(ApplicationScoped.class) Object pointl List docs = loadDocs(); ingestor.ingest(docs); - log.info(String.format("DEMO %d documents ingested in %d msec", docs.size(), + LOGGER.info(String.format("DEMO %d documents ingested in %d msec", docs.size(), System.currentTimeMillis() - start)); } diff --git a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java index e120c91..e58ac4f 100644 --- a/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java +++ b/examples/liberty-car-booking/src/main/java/io/jefrajames/booking/DummyLLConfig.java @@ -29,6 +29,7 @@ public Set getBeanNames() { .collect(Collectors.toSet()); } + @SuppressWarnings("unchecked") @Override public T getBeanPropertyValue(String beanName, String propertyName, Class type) { String value = properties.getProperty(PREFIX + "." + beanName + "." + propertyName); diff --git a/examples/liberty-car-booking/src/main/liberty/config/server.xml b/examples/liberty-car-booking/src/main/liberty/config/server.xml index 2653bc7..a4cc2a0 100644 --- a/examples/liberty-car-booking/src/main/liberty/config/server.xml +++ b/examples/liberty-car-booking/src/main/liberty/config/server.xml @@ -3,16 +3,24 @@ microProfile-6.1 + mpTelemetry-2.0 - + + + + + + + + diff --git a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties index c793531..7c796d0 100644 --- a/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties +++ b/examples/liberty-car-booking/src/main/resources/META-INF/microprofile-config.properties @@ -10,7 +10,8 @@ smallrye.llm.plugin.chat-model.config.temperature=0.1 smallrye.llm.plugin.chat-model.config.topP=0.1 smallrye.llm.plugin.chat-model.config.timeout=120s smallrye.llm.plugin.chat-model.config.max-retries=2 -#smallrye.llm.plugin.chat-model.config.logRequestsAndResponsess=false +#smallrye.llm.plugin.chat-model.config.logRequestsAndResponses=true +smallrye.llm.plugin.chat-model.config.listeners=@all smallrye.llm.plugin.docRagRetriever.class=dev.langchain4j.rag.content.retriever.EmbeddingStoreContentRetriever @@ -31,3 +32,10 @@ fraud.memory.max.messages=20 # Location of documents to RAG app.docs-for-rag.dir=docs-for-rag + +# Microprofile Telemetry +otel.service.name=liberty-car-booking +otel.sdk.disabled=false +otel.logs.exporter=otlp,console +otel.metrics.exporter=otlp,console +otel.traces.exporter=otlp,console diff --git a/pom.xml b/pom.xml index 33d1300..05011e6 100644 --- a/pom.xml +++ b/pom.xml @@ -35,8 +35,9 @@ 3.1 + 2.0.1 2.9.0 - 9.7 + 9.7 2.3.1 4.1.0 4.0.3.Final @@ -79,6 +80,7 @@ smallrye-llm-langchain4j-buildcompatible-extension smallrye-llm-langchain4j-core smallrye-llm-langchain4j-config-mpconfig + smallrye-llm-langchain4j-telemetry @@ -88,6 +90,12 @@ microprofile-config-api ${version.eclipse.microprofile.config} + + org.eclipse.microprofile.telemetry + microprofile-telemetry-api + ${version.eclipse.microprofile.telemetry} + pom + io.smallrye.common smallrye-common-bom diff --git a/smallrye-llm-langchain4j-config-mpconfig/pom.xml b/smallrye-llm-langchain4j-config-mpconfig/pom.xml index d80a581..b3d7354 100644 --- a/smallrye-llm-langchain4j-config-mpconfig/pom.xml +++ b/smallrye-llm-langchain4j-config-mpconfig/pom.xml @@ -28,5 +28,4 @@ ${project.version} - diff --git a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java index 6d5525b..9236b56 100644 --- a/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java +++ b/smallrye-llm-langchain4j-core/src/main/java/io/smallrye/llm/plugin/CommonLLMPluginCreator.java @@ -5,8 +5,12 @@ import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -15,10 +19,14 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.literal.NamedLiteral; -import jakarta.enterprise.inject.spi.CDI; +import jakarta.enterprise.util.TypeLiteral; import org.jboss.logging.Logger; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.chat.listener.ChatModelListener; +import dev.langchain4j.store.embedding.EmbeddingStore; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfig; import io.smallrye.llm.core.langchain4j.core.config.spi.LLMConfigProvider; @@ -33,6 +41,13 @@ public class CommonLLMPluginCreator { public static final Logger LOGGER = Logger.getLogger(CommonLLMPluginCreator.class); + + private static final Map, TypeLiteral> TYPE_LITERALS = new HashMap<>(); + + static { + TYPE_LITERALS.put(EmbeddingStore.class, new TypeLiteral>() { + }); + } @SuppressWarnings("unchecked") public static void createAllLLMBeans(LLMConfig llmConfig, Consumer beanBuilder) throws ClassNotFoundException { @@ -62,11 +77,13 @@ public static void createAllLLMBeans(LLMConfig llmConfig, Consumer bea } beanBuilder.accept( new BeanData(targetClass, builderCLass, scopeClass, beanName, - (Instance creationalContext) -> CommonLLMPluginCreator.create( - creationalContext, - beanName, - targetClass, - builderCLass))); + (Instance creationalContext) -> { + return CommonLLMPluginCreator.create( + creationalContext, + beanName, + targetClass, + builderCLass); + })); } } } @@ -109,6 +126,7 @@ public Function, Object> getCallback() { } } + @SuppressWarnings("unchecked") public static Object create(Instance lookup, String beanName, Class targetClass, Class builderClass) { LLMConfig llmConfig = LLMConfigProvider.getLlmConfig(); LOGGER.info( @@ -130,17 +148,40 @@ public static Object create(Instance lookup, String beanName, Class t } else { for (Method methodToCall : methodsToCall) { Class parameterType = methodToCall.getParameterTypes()[0]; - if (stringValue.startsWith("lookup:")) { + if ("listeners".equals(property)) { + Class typeParameterClass = ChatLanguageModel.class.isAssignableFrom(targetClass) + ? ChatModelListener.class + : parameterType.getTypeParameters()[0].getGenericDeclaration(); + List listeners = (List) Collections.checkedList(new ArrayList<>(), + typeParameterClass); + if ("@all".equals(stringValue.trim())) { + Instance inst = (Instance) getInstance(lookup, typeParameterClass); + if (inst != null) { + inst.forEach(listeners::add); + } + } else { + try { + for (String className : stringValue.split(",")) { + Instance inst = getInstance(lookup, loadClass(className.trim())); + listeners.add(inst.get()); + } + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + if (listeners != null && !listeners.isEmpty()) { + listeners.stream().forEach(l -> LOGGER.info("Adding listener: " + l.getClass().getName())); + methodToCall.invoke(builder, listeners); + } + } else if (stringValue.startsWith("lookup:")) { String lookupableBean = stringValue.substring("lookup:".length()); LOGGER.info("Lookup " + lookupableBean + " " + parameterType); Instance inst; if ("default".equals(lookupableBean)) { - inst = lookup.select(parameterType); - if (!inst.isResolvable()) { - inst = CDI.current().select(parameterType); - } + inst = getInstance(lookup, parameterType); } else { - inst = lookup.select(parameterType, NamedLiteral.of(lookupableBean)); + inst = getInstance(lookup, parameterType, lookupableBean); } methodToCall.invoke(builder, inst.get()); break; @@ -162,7 +203,20 @@ public static Object create(Instance lookup, String beanName, Class t } } - private static Class loadClass(String scopeClassName) throws ClassNotFoundException { - return Thread.currentThread().getContextClassLoader().loadClass(scopeClassName); + private static Class loadClass(String className) throws ClassNotFoundException { + return Thread.currentThread().getContextClassLoader().loadClass(className); + } + + @SuppressWarnings("unchecked") + private static Instance getInstance(Instance lookup, Class clazz) { + if (TYPE_LITERALS.containsKey(clazz)) + return (Instance) lookup.select(TYPE_LITERALS.get(clazz)); + return lookup.select(clazz); + } + + private static Instance getInstance(Instance lookup, Class clazz, String lookupName) { + if (lookupName == null || lookupName.isBlank()) + return getInstance(lookup, clazz); + return lookup.select(clazz, NamedLiteral.of(lookupName)); } } diff --git a/smallrye-llm-langchain4j-telemetry/pom.xml b/smallrye-llm-langchain4j-telemetry/pom.xml new file mode 100644 index 0000000..06cf5c9 --- /dev/null +++ b/smallrye-llm-langchain4j-telemetry/pom.xml @@ -0,0 +1,30 @@ + + 4.0.0 + + io.smallrye.llm + smallrye-llm-parent + 1.0.0-SNAPSHOT + + smallrye-llm-langchain4j-telemetry + + + + jakarta.enterprise + jakarta.enterprise.cdi-api + provided + + + org.eclipse.microprofile.telemetry + microprofile-telemetry-api + pom + provided + + + io.smallrye.llm + smallrye-llm-langchain4j-core + ${project.version} + + + \ No newline at end of file diff --git a/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java new file mode 100644 index 0000000..c95ab3f --- /dev/null +++ b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/MetricsChatModelListener.java @@ -0,0 +1,144 @@ +package io.smallrye.llm.langchain4j.telemetry; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.Dependent; +import jakarta.inject.Inject; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.model.chat.listener.ChatModelErrorContext; +import dev.langchain4j.model.chat.listener.ChatModelListener; +import dev.langchain4j.model.chat.listener.ChatModelRequest; +import dev.langchain4j.model.chat.listener.ChatModelRequestContext; +import dev.langchain4j.model.chat.listener.ChatModelResponse; +import dev.langchain4j.model.chat.listener.ChatModelResponseContext; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.Meter; + +/** + * Creates metrics that follow the + * Semantic Conventions + * for GenAI Metrics. + * + * @author Buhake Sindi + * @since 25 November 2024 + */ +@Dependent +public class MetricsChatModelListener implements ChatModelListener { + + private static final String MP_AI_METRIC_START_TIME_NAME = "MP_AI_METRIC_START_TIME"; + + private static final String METRIC_CLIENT_TOKEN_USAGE_NAME = "gen_ai.client.token.usage"; + private static final String METRIC_CLIENT_OPERATION_DURATION_NAME = "gen_ai.client.operation.duration"; + + private LongHistogram clientTokenUsage; + private DoubleHistogram clientOperationDuration; + + @Inject + private Meter meter; + + @PostConstruct + private void init() { + clientTokenUsage = meter.histogramBuilder(METRIC_CLIENT_TOKEN_USAGE_NAME) + .ofLongs() + .setDescription("Measures number of input and output tokens used") + .setExplicitBucketBoundariesAdvice(List.of(1L, 4L, 16L, 64L, 256L, 1024L, 4096L, 16384L, 65536L, 262144L, + 1048576L, 4194304L, 16777216L, 67108864L)) + .build(); + + clientOperationDuration = meter.histogramBuilder(METRIC_CLIENT_OPERATION_DURATION_NAME) + .setDescription("GenAI operation duration") + .setExplicitBucketBoundariesAdvice( + List.of(0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92)) + .setUnit("s") + .build(); + } + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onRequest(dev.langchain4j.model.chat.listener. + * ChatModelRequestContext) + */ + @Override + public void onRequest(ChatModelRequestContext requestContext) { + // TODO Auto-generated method stub + requestContext.attributes().put(MP_AI_METRIC_START_TIME_NAME, System.nanoTime()); + } + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onResponse(dev.langchain4j.model.chat.listener. + * ChatModelResponseContext) + */ + @Override + public void onResponse(ChatModelResponseContext responseContext) { + // TODO Auto-generated method stub + final long endTime = System.nanoTime(); + final long startTime = (Long) responseContext.attributes().get(MP_AI_METRIC_START_TIME_NAME); + + final ChatModelRequest request = responseContext.request(); + final ChatModelResponse response = responseContext.response(); + + Attributes inputTokenCountAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", + AttributeKey.stringKey("gen_ai.request.model"), request.model(), + AttributeKey.stringKey("gen_ai.response.model"), response.model(), + AttributeKey.stringKey("gen_ai.token.type"), "input"); + //Record + clientTokenUsage.record(response.tokenUsage().inputTokenCount(), inputTokenCountAttributes); + + Attributes outputTokenCountAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", + AttributeKey.stringKey("gen_ai.request.model"), request.model(), + AttributeKey.stringKey("gen_ai.response.model"), response.model(), + AttributeKey.stringKey("gen_ai.token.type"), "output"); + + //Record + clientTokenUsage.record(response.tokenUsage().outputTokenCount(), outputTokenCountAttributes); + + //Record duration + Attributes durationAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", + AttributeKey.stringKey("gen_ai.request.model"), request.model(), + AttributeKey.stringKey("gen_ai.response.model"), response.model()); + recordClientOperationDuration(startTime, endTime, durationAttributes); + } + + /* + * (non-Javadoc) + * + * @see + * dev.langchain4j.model.chat.listener.ChatModelListener#onError(dev.langchain4j.model.chat.listener.ChatModelErrorContext) + */ + @Override + public void onError(ChatModelErrorContext errorContext) { + // TODO Auto-generated method stub + final long endTime = System.nanoTime(); + final long startTime = (Long) errorContext.attributes().get(MP_AI_METRIC_START_TIME_NAME); + final ChatModelRequest request = errorContext.request(); + final ChatModelResponse response = errorContext.partialResponse(); + + StringBuilder sb = new StringBuilder() + .append(errorContext.error().getClass().getName()); + + AiMessage aiMessage = errorContext.partialResponse().aiMessage(); + if (aiMessage != null) { + sb.append(";" + aiMessage.text()); + } + + //Record duration + Attributes durationAttributes = Attributes.of(AttributeKey.stringKey("gen_ai.operation.name"), "chat", + AttributeKey.stringKey("gen_ai.request.model"), request.model(), + AttributeKey.stringKey("gen_ai.response.model"), response.model(), + AttributeKey.stringKey("error.type"), sb.toString()); + recordClientOperationDuration(startTime, endTime, durationAttributes); + } + + private void recordClientOperationDuration(final long startTime, long endTime, final Attributes attributes) { + clientOperationDuration.record(TimeUnit.SECONDS.convert(endTime - startTime, TimeUnit.NANOSECONDS), attributes); + } +} diff --git a/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java new file mode 100644 index 0000000..c3a1350 --- /dev/null +++ b/smallrye-llm-langchain4j-telemetry/src/main/java/io/smallrye/llm/langchain4j/telemetry/SpanChatModelListener.java @@ -0,0 +1,113 @@ +package io.smallrye.llm.langchain4j.telemetry; + +import jakarta.enterprise.context.Dependent; +import jakarta.inject.Inject; + +import dev.langchain4j.model.chat.listener.ChatModelErrorContext; +import dev.langchain4j.model.chat.listener.ChatModelListener; +import dev.langchain4j.model.chat.listener.ChatModelRequest; +import dev.langchain4j.model.chat.listener.ChatModelRequestContext; +import dev.langchain4j.model.chat.listener.ChatModelResponse; +import dev.langchain4j.model.chat.listener.ChatModelResponseContext; +import dev.langchain4j.model.output.TokenUsage; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; + +/** + * Creates metrics that follow the + * Semantic Conventions + * for GenAI spans. + * + * @author Buhake Sindi + * @since 25 November 2024 + */ +@Dependent +public class SpanChatModelListener implements ChatModelListener { + + private static final String OTEL_SCOPE_KEY_NAME = "OTelScope"; + private static final String OTEL_SPAN_KEY_NAME = "OTelSpan"; + + @Inject + private Tracer tracer; + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onRequest(dev.langchain4j.model.chat.listener. + * ChatModelRequestContext) + */ + @Override + public void onRequest(ChatModelRequestContext requestContext) { + // TODO Auto-generated method stub + final ChatModelRequest request = requestContext.request(); + SpanBuilder spanBuilder = tracer.spanBuilder("chat " + request.model()) + .setAttribute("gen_ai.operation.name", "chat"); + if (request.maxTokens() != null) + spanBuilder.setAttribute("gen_ai.request.max_tokens", request.maxTokens()); + + if (request.temperature() != null) + spanBuilder.setAttribute("gen_ai.request.temperature", request.temperature()); + + if (request.topP() != null) + spanBuilder.setAttribute("gen_ai.request.top_p", request.topP()); + + Span span = spanBuilder.startSpan(); + Scope scope = span.makeCurrent(); + + requestContext.attributes().put(OTEL_SCOPE_KEY_NAME, scope); + requestContext.attributes().put(OTEL_SPAN_KEY_NAME, span); + } + + /* + * (non-Javadoc) + * + * @see dev.langchain4j.model.chat.listener.ChatModelListener#onResponse(dev.langchain4j.model.chat.listener. + * ChatModelResponseContext) + */ + @Override + public void onResponse(ChatModelResponseContext responseContext) { + // TODO Auto-generated method stub + Span span = (Span) responseContext.attributes().get(OTEL_SPAN_KEY_NAME); + if (span != null) { + ChatModelResponse response = responseContext.response(); + span.setAttribute("gen_ai.response.id", response.id()) + .setAttribute("gen_ai.response.model", response.model()); + if (response.finishReason() != null) { + span.setAttribute("gen_ai.response.finish_reasons", response.finishReason().toString()); + } + TokenUsage tokenUsage = response.tokenUsage(); + if (tokenUsage != null) { + span.setAttribute("gen_ai.usage.output_tokens", tokenUsage.outputTokenCount()) + .setAttribute("gen_ai.usage.input_tokens", tokenUsage.inputTokenCount()); + } + span.end(); + } + + closeScope((Scope) responseContext.attributes().get(OTEL_SCOPE_KEY_NAME)); + } + + /* + * (non-Javadoc) + * + * @see + * dev.langchain4j.model.chat.listener.ChatModelListener#onError(dev.langchain4j.model.chat.listener.ChatModelErrorContext) + */ + @Override + public void onError(ChatModelErrorContext errorContext) { + // TODO Auto-generated method stub + Span span = (Span) errorContext.attributes().get(OTEL_SPAN_KEY_NAME); + if (span != null) { + span.recordException(errorContext.error()); + } + + closeScope((Scope) errorContext.attributes().get(OTEL_SCOPE_KEY_NAME)); + } + + private void closeScope(Scope scope) { + if (scope != null) { + scope.close(); + } + } +}