From d319aa96b2ddab69f6642523e2bef655606c2463 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 16 Aug 2024 15:17:50 +0200
Subject: [PATCH 01/55] OpenAI streaming

---
 e2e-test-app/pom.xml                          |  4 ++
 .../sdk/app/controllers/OpenAiController.java | 51 +++++++++++++++++
 .../ai/sdk/app/controllers/OpenAiTest.java    |  6 ++
 .../foundationmodels/openai/OpenAiClient.java | 39 ++++++++++++-
 .../openai/OpenAiStreamingHandler.java        | 55 +++++++++++++++++++
 .../model/OpenAiChatCompletionChoice.java     |  3 +
 .../model/OpenAiChatCompletionOutput.java     | 23 ++++++++
 .../model/OpenAiChatCompletionParameters.java |  6 ++
 .../model/OpenAiChatCompletionStream.java     | 18 ++++++
 .../openai/model/OpenAiChatMessage.java       |  6 +-
 10 files changed, 209 insertions(+), 2 deletions(-)
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java

diff --git a/e2e-test-app/pom.xml b/e2e-test-app/pom.xml
index dee74a7d..a5c9add3 100644
--- a/e2e-test-app/pom.xml
+++ b/e2e-test-app/pom.xml
@@ -95,6 +95,10 @@
       <groupId>org.springframework</groupId>
       <artifactId>spring-web</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.springframework</groupId>
+      <artifactId>spring-webmvc</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 08a02b5a..8122c30e 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -14,11 +14,15 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage.ImageDetailLevel;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
+import com.sap.cloud.sdk.cloudplatform.thread.ThreadContextExecutors;
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter;
 
 /** Endpoints for OpenAI operations */
 @RestController
@@ -38,6 +42,53 @@ public static OpenAiChatCompletionOutput chatCompletion() {
     return OpenAiClient.forModel(GPT_35_TURBO).chatCompletion(request);
   }
 
+  /**
+   * Stream chat request to OpenAI
+   *
+   * @return the emitter that streams the assistant message response
+   */
+  @GetMapping("/chatCompletion")
+  @Nonnull
+  public static ResponseBodyEmitter stream() {
+    final var request =
+        new OpenAiChatCompletionParameters()
+            .setMessages(
+                List.of(
+                    new OpenAiChatUserMessage()
+                        .addText(
+                            "Can you give me the first 100 number of the Fibonacci sequence?")));
+
+    Stream<OpenAiChatCompletionOutput> delta =
+        OpenAiClient.forModel(GPT_35_TURBO).stream(request).getDelta();
+
+    ResponseBodyEmitter emitter = new ResponseBodyEmitter();
+    // Start streaming the content asynchronously
+    ThreadContextExecutors.getExecutor()
+        .submit(
+            () -> {
+              try {
+                delta.forEach(
+                    line -> {
+                      try {
+                        if (!line.getChoices().isEmpty()
+                            && line.getChoices().get(0).getMessage() != null
+                            && line.getChoices().get(0).getMessage().getContent() != null) {
+                          emitter.send(line.getChoices().get(0).getMessage().getContent());
+                        }
+                      } catch (IOException e) {
+                        e.printStackTrace();
+                        emitter.completeWithError(e);
+                      }
+                    });
+                // Once all the data is sent, complete the emitter
+                emitter.complete();
+              } catch (Exception e) {
+                emitter.completeWithError(e);
+              }
+            });
+    return emitter;
+  }
+
   /**
    * Chat request to OpenAI with an image
    *
diff --git a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
index ea826aa7..1b79a19a 100644
--- a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
+++ b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
@@ -23,6 +23,12 @@ void chatCompletionImage() {
     assertThat(message.getContent()).isNotEmpty();
   }
 
+  @Test
+  void stream() {
+    final var emitter = OpenAiController.stream();
+    // TODO: assert on the emitter
+  }
+
   @Test
   void chatCompletionTools() {
     final var completion = OpenAiController.chatCompletionTools();
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index af85738a..e5f9d6df 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -9,6 +9,7 @@
 import com.sap.ai.sdk.core.Core;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionStream;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor;
@@ -30,7 +31,7 @@
 @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
 public final class OpenAiClient {
   private static final String DEFAULT_API_VERSION = "2024-02-01";
-  private static final ObjectMapper JACKSON;
+  static final ObjectMapper JACKSON;
 
   static {
     JACKSON =
@@ -105,6 +106,20 @@ public OpenAiChatCompletionOutput chatCompletion(
     return execute("/chat/completions", parameters, OpenAiChatCompletionOutput.class);
   }
 
+  /**
+   * Generate a completion for the given prompt.
+   *
+   * @param parameters the prompt, including messages and other parameters.
+   * @return the completion output
+   * @throws OpenAiClientException if the request fails
+   */
+  @Nonnull
+  public OpenAiChatCompletionStream stream(@Nonnull final OpenAiChatCompletionParameters parameters)
+      throws OpenAiClientException {
+    parameters.setStream(true);
+    return stream("/chat/completions", parameters, OpenAiChatCompletionStream.class);
+  }
+
   /**
    * Get a vector representation of a given input that can be easily consumed by machine learning
    * models and algorithms.
@@ -129,6 +144,16 @@ private <T> T execute(
     return executeRequest(request, responseType);
   }
 
+  @Nonnull
+  private <T> T stream(
+      @Nonnull final String path,
+      @Nonnull final Object payload,
+      @Nonnull final Class<T> responseType) {
+    final var request = new HttpPost(path);
+    serializeAndSetHttpEntity(request, payload);
+    return streamRequest(request, responseType);
+  }
+
   private static void serializeAndSetHttpEntity(
       @Nonnull final BasicClassicHttpRequest request, @Nonnull final Object payload) {
     try {
@@ -150,4 +175,16 @@ private <T> T executeRequest(
       throw new OpenAiClientException("Request to OpenAI model failed.", e);
     }
   }
+
+  @Nonnull
+  private <T> T streamRequest(
+      final BasicClassicHttpRequest request, @Nonnull final Class<T> responseType) {
+    try {
+      @SuppressWarnings("UnstableApiUsage")
+      final var client = ApacheHttpClient5Accessor.getHttpClient(destination);
+      return (T) OpenAiStreamingHandler.handleResponse(client.executeOpen(null, request, null));
+    } catch (final IOException e) {
+      throw new OpenAiClientException("Request to OpenAI model failed.", e);
+    }
+  }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
new file mode 100644
index 00000000..eebed425
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -0,0 +1,55 @@
+package com.sap.ai.sdk.foundationmodels.openai;
+
+import static com.sap.ai.sdk.foundationmodels.openai.OpenAiClient.JACKSON;
+
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionStream;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.stream.Stream;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.hc.core5.http.ClassicHttpResponse;
+
+@Slf4j
+@RequiredArgsConstructor
+class OpenAiStreamingHandler {
+
+  public static OpenAiChatCompletionStream handleResponse(ClassicHttpResponse response)
+      throws IOException {
+    return getContent(response.getEntity().getContent());
+  }
+
+  /**
+   * @param inputStream
+   * @return
+   * @author stippi
+   */
+  public static OpenAiChatCompletionStream getContent(InputStream inputStream) {
+
+    OpenAiChatCompletionOutput total = null;
+    BufferedReader br =
+        new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+
+    Stream<OpenAiChatCompletionOutput> delta =
+        br.lines()
+            .filter(
+                responseLine ->
+                    !responseLine.isEmpty() && !"data: [DONE]".equals(responseLine.trim()))
+            .map(
+                responseLine -> {
+                  String data = responseLine.substring(5).replace("delta", "message");
+                  try {
+                    return JACKSON.readValue(data, OpenAiChatCompletionOutput.class);
+                  } catch (IOException e) {
+                    throw new RuntimeException(e);
+                  }
+                });
+    var outputStream = new OpenAiChatCompletionStream();
+    outputStream.setDelta(delta);
+    return outputStream;
+  }
+}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java
index 3b18ff92..970a98e3 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java
@@ -3,8 +3,10 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatAssistantMessage;
 import javax.annotation.Nonnull;
+import lombok.AccessLevel;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.Setter;
 import lombok.ToString;
 import lombok.experimental.Accessors;
 
@@ -16,5 +18,6 @@ public class OpenAiChatCompletionChoice extends OpenAiCompletionChoice {
   /** Completion chat message. */
   @JsonProperty("message")
   @Getter(onMethod_ = @Nonnull)
+  @Setter(onMethod_ = @Nonnull, value = AccessLevel.PACKAGE)
   private OpenAiChatAssistantMessage message;
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
index 0254aee2..2cec01c6 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
@@ -1,6 +1,7 @@
 package com.sap.ai.sdk.foundationmodels.openai.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatAssistantMessage;
 import java.util.List;
 import javax.annotation.Nonnull;
 import lombok.EqualsAndHashCode;
@@ -25,4 +26,26 @@ public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput {
   @JsonProperty("system_fingerprint")
   @Getter(onMethod_ = @Nonnull)
   private String systemFingerprint;
+
+  public void addDelta(OpenAiChatCompletionOutput delta) {
+    // TODO: Assign every field if not null on all parent and children classes.
+    //       Right now we only assign content message.
+    var deltaMessage = delta.getChoices().get(0).getMessage();
+    if (deltaMessage == null) {
+      return;
+    }
+    String deltaContent = deltaMessage.getContent();
+    if (deltaContent == null) {
+      return;
+    }
+    if (choices.isEmpty()) {
+      var choice =
+          new OpenAiChatCompletionChoice()
+              .setMessage(new OpenAiChatAssistantMessage().setContent(deltaContent));
+      choices.add(choice);
+      return;
+    }
+    var message = choices.get(0).getMessage();
+    message.setContent(message.getContent() + deltaContent);
+  }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
index 8dccfe55..d8b498b0 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
@@ -23,6 +23,12 @@
 @EqualsAndHashCode(callSuper = true)
 @ToString
 public class OpenAiChatCompletionParameters extends OpenAiCompletionParameters {
+
+  /** Whether to stream the response. */
+  @JsonProperty("stream")
+  @Setter(AccessLevel.PUBLIC)
+  private Boolean stream;
+
   /** A list of messages comprising the conversation so far. */
   @JsonProperty("messages")
   @Setter(onParam_ = @Nonnull)
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
new file mode 100644
index 00000000..6ce03b5b
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
@@ -0,0 +1,18 @@
+package com.sap.ai.sdk.foundationmodels.openai.model;
+
+import java.util.concurrent.Future;
+import java.util.stream.Stream;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+public class OpenAiChatCompletionStream implements AutoCloseable {
+  @Setter private Stream<OpenAiChatCompletionOutput> delta;
+  private Future<OpenAiChatCompletionOutput> total;
+
+  @Override
+  public void close() {
+    delta.close();
+    // TODO: fill out total or cancel it
+  }
+}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
index 181938c5..c87abcfa 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
@@ -18,6 +18,7 @@
 import java.util.List;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -28,7 +29,9 @@
 import lombok.experimental.Accessors;
 
 /** OpenAI chat message types. */
-@JsonTypeInfo(use = Id.NAME, property = "role") // This is the field that determines the class type
+@JsonTypeInfo(use = Id.NAME, property = "role", defaultImpl = OpenAiChatAssistantMessage.class)
+// role is the field that determines the class type
+// if role is missing we default to OpenAiChatAssistantMessage
 @JsonSubTypes({
   @Type(value = OpenAiChatSystemMessage.class, name = "system"),
   @Type(value = OpenAiChatUserMessage.class, name = "user"),
@@ -241,6 +244,7 @@ class OpenAiChatAssistantMessage implements OpenAiChatMessage {
     /** Message content. */
     @JsonProperty("content")
     @Getter(onMethod_ = @Nullable)
+    @Setter(onParam_ = @Nonnull, value = AccessLevel.PACKAGE)
     private String content; // must be String or null
 
     /** The tool calls generated by the model, such as function calls. */

From 69ae7eb220f19dd973330d3b6ce0e47c569fd3b7 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 16 Aug 2024 15:20:58 +0200
Subject: [PATCH 02/55] Added homepage and error handling todo

---
 .../com/sap/ai/sdk/app/controllers/OpenAiController.java   | 2 +-
 e2e-test-app/src/main/resources/static/index.html          | 1 +
 .../foundationmodels/openai/OpenAiStreamingHandler.java    | 7 +++++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 8122c30e..f9931e57 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -47,7 +47,7 @@ public static OpenAiChatCompletionOutput chatCompletion() {
    *
    * @return the emitter that streams the assistant message response
    */
-  @GetMapping("/chatCompletion")
+  @GetMapping("/stream")
   @Nonnull
   public static ResponseBodyEmitter stream() {
     final var request =
diff --git a/e2e-test-app/src/main/resources/static/index.html b/e2e-test-app/src/main/resources/static/index.html
index f601884e..9ee4f111 100644
--- a/e2e-test-app/src/main/resources/static/index.html
+++ b/e2e-test-app/src/main/resources/static/index.html
@@ -71,6 +71,7 @@ <h2>Endpoints</h2>
             <li><h4>OpenAI</h4></li>
             <ul>
                 <li><a href="/chatCompletion">/chatCompletion</a></li>
+                <li><a href="/stream">/stream</a></li>
                 <li><a href="/chatCompletionTool">/chatCompletionTool</a></li>
                 <li><a href="/chatCompletionImage">/chatCompletionImage</a></li>
                 <li><a href="/embedding">/embedding</a></li>
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index eebed425..790c0695 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -41,6 +41,13 @@ public static OpenAiChatCompletionStream getContent(InputStream inputStream) {
                     !responseLine.isEmpty() && !"data: [DONE]".equals(responseLine.trim()))
             .map(
                 responseLine -> {
+                  // TODO: handle errors
+                  //       {
+                  //         "error": {
+                  //           "code": "429",
+                  //           "message": "exceeded token rate limit"
+                  //         }
+                  //      }
                   String data = responseLine.substring(5).replace("delta", "message");
                   try {
                     return JACKSON.readValue(data, OpenAiChatCompletionOutput.class);

From 7870e6d08828c50a4433cb601a13671ec56cf55d Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Mon, 19 Aug 2024 08:19:59 +0200
Subject: [PATCH 03/55] Renamed vars

---
 .../ai/sdk/app/controllers/OpenAiController.java   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index f9931e57..1f99bc21 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -58,7 +58,7 @@ public static ResponseBodyEmitter stream() {
                         .addText(
                             "Can you give me the first 100 number of the Fibonacci sequence?")));
 
-    Stream<OpenAiChatCompletionOutput> delta =
+    Stream<OpenAiChatCompletionOutput> stream =
         OpenAiClient.forModel(GPT_35_TURBO).stream(request).getDelta();
 
     ResponseBodyEmitter emitter = new ResponseBodyEmitter();
@@ -67,13 +67,13 @@ public static ResponseBodyEmitter stream() {
         .submit(
             () -> {
               try {
-                delta.forEach(
-                    line -> {
+                stream.forEach(
+                    delta -> {
                       try {
-                        if (!line.getChoices().isEmpty()
-                            && line.getChoices().get(0).getMessage() != null
-                            && line.getChoices().get(0).getMessage().getContent() != null) {
-                          emitter.send(line.getChoices().get(0).getMessage().getContent());
+                        if (!delta.getChoices().isEmpty()
+                            && delta.getChoices().get(0).getMessage() != null
+                            && delta.getChoices().get(0).getMessage().getContent() != null) {
+                          emitter.send(delta.getChoices().get(0).getMessage().getContent());
                         }
                       } catch (IOException e) {
                         e.printStackTrace();

From 652ec1e9b6a5464f18af86d3c1ca7487686a6ee0 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Mon, 19 Aug 2024 08:39:26 +0200
Subject: [PATCH 04/55] Added todos

---
 .../java/com/sap/ai/sdk/app/controllers/OpenAiController.java | 3 +++
 .../com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java  | 1 +
 .../sdk/foundationmodels/openai/OpenAiStreamingHandler.java   | 2 +-
 .../openai/model/OpenAiChatCompletionParameters.java          | 2 +-
 .../openai/model/OpenAiChatCompletionStream.java              | 4 ++--
 5 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 1f99bc21..6232fedc 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -58,6 +58,7 @@ public static ResponseBodyEmitter stream() {
                         .addText(
                             "Can you give me the first 100 number of the Fibonacci sequence?")));
 
+    // TODO: close AutoCloseable or make it automatic
     Stream<OpenAiChatCompletionOutput> stream =
         OpenAiClient.forModel(GPT_35_TURBO).stream(request).getDelta();
 
@@ -70,6 +71,8 @@ public static ResponseBodyEmitter stream() {
                 stream.forEach(
                     delta -> {
                       try {
+                        // TODO: Change the types to nullable? Maybe create a class
+                        //   OpenAiChatCompletionDelta...
                         if (!delta.getChoices().isEmpty()
                             && delta.getChoices().get(0).getMessage() != null
                             && delta.getChoices().get(0).getMessage().getContent() != null) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index e5f9d6df..891d1528 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -182,6 +182,7 @@ private <T> T streamRequest(
     try {
       @SuppressWarnings("UnstableApiUsage")
       final var client = ApacheHttpClient5Accessor.getHttpClient(destination);
+      // TODO: OpenAiStreamingHandler should return generic T instead of OpenAiChatCompletionStream
       return (T) OpenAiStreamingHandler.handleResponse(client.executeOpen(null, request, null));
     } catch (final IOException e) {
       throw new OpenAiClientException("Request to OpenAI model failed.", e);
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 790c0695..8862faba 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -47,7 +47,7 @@ public static OpenAiChatCompletionStream getContent(InputStream inputStream) {
                   //           "code": "429",
                   //           "message": "exceeded token rate limit"
                   //         }
-                  //      }
+                  //       }
                   String data = responseLine.substring(5).replace("delta", "message");
                   try {
                     return JACKSON.readValue(data, OpenAiChatCompletionOutput.class);
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
index d8b498b0..70bd31d0 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
@@ -26,7 +26,7 @@ public class OpenAiChatCompletionParameters extends OpenAiCompletionParameters {
 
   /** Whether to stream the response. */
   @JsonProperty("stream")
-  @Setter(AccessLevel.PUBLIC)
+  @Setter(AccessLevel.PUBLIC) // TODO: Change AccessLevel to not be public
   private Boolean stream;
 
   /** A list of messages comprising the conversation so far. */
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
index 6ce03b5b..23b27db2 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
@@ -8,11 +8,11 @@
 @Getter
 public class OpenAiChatCompletionStream implements AutoCloseable {
   @Setter private Stream<OpenAiChatCompletionOutput> delta;
-  private Future<OpenAiChatCompletionOutput> total;
+  private Future<OpenAiChatCompletionOutput> total; // TODO: fill out "total" delta by delta
 
   @Override
   public void close() {
     delta.close();
-    // TODO: fill out total or cancel it
+    // TODO: fill out "total" or cancel it
   }
 }

From 727b3d436d556aa2853c0ecec3a6f2d7a0faed4d Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Wed, 21 Aug 2024 09:37:31 +0200
Subject: [PATCH 05/55] Made stream generic, try-with resources,
 TEXT_EVENT_STREAM, exception refactored

---
 .../sdk/app/controllers/OpenAiController.java | 19 +++---
 .../foundationmodels/openai/OpenAiClient.java | 20 +++---
 .../openai/OpenAiClientException.java         | 15 +++++
 .../openai/OpenAiResponseHandler.java         | 44 +++++++++----
 .../openai/OpenAiStreamingHandler.java        | 62 ++++++++++++-------
 .../model/OpenAiChatCompletionParameters.java |  6 --
 .../model/OpenAiChatCompletionStream.java     | 18 ++++--
 .../model/OpenAiCompletionParameters.java     | 11 +---
 8 files changed, 124 insertions(+), 71 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 6232fedc..0746ebe2 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -16,15 +16,21 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.cloud.sdk.cloudplatform.thread.ThreadContextExecutors;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter;
 
 /** Endpoints for OpenAI operations */
+@Slf4j
 @RestController
 class OpenAiController {
   /**
@@ -49,7 +55,7 @@ public static OpenAiChatCompletionOutput chatCompletion() {
    */
   @GetMapping("/stream")
   @Nonnull
-  public static ResponseBodyEmitter stream() {
+  public static ResponseEntity<ResponseBodyEmitter> stream() {
     final var request =
         new OpenAiChatCompletionParameters()
             .setMessages(
@@ -58,16 +64,13 @@ public static ResponseBodyEmitter stream() {
                         .addText(
                             "Can you give me the first 100 number of the Fibonacci sequence?")));
 
-    // TODO: close AutoCloseable or make it automatic
-    Stream<OpenAiChatCompletionOutput> stream =
-        OpenAiClient.forModel(GPT_35_TURBO).stream(request).getDelta();
-
     ResponseBodyEmitter emitter = new ResponseBodyEmitter();
     // Start streaming the content asynchronously
     ThreadContextExecutors.getExecutor()
         .submit(
             () -> {
-              try {
+              try (Stream<OpenAiChatCompletionOutput> stream =
+                  OpenAiClient.forModel(GPT_35_TURBO).stream(request).getDeltaStream()) {
                 stream.forEach(
                     delta -> {
                       try {
@@ -79,7 +82,7 @@ public static ResponseBodyEmitter stream() {
                           emitter.send(delta.getChoices().get(0).getMessage().getContent());
                         }
                       } catch (IOException e) {
-                        e.printStackTrace();
+                        log.error(Arrays.toString(e.getStackTrace()));
                         emitter.completeWithError(e);
                       }
                     });
@@ -89,7 +92,7 @@ public static ResponseBodyEmitter stream() {
                 emitter.completeWithError(e);
               }
             });
-    return emitter;
+    return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
   /**
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 891d1528..536fce51 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -114,10 +114,10 @@ public OpenAiChatCompletionOutput chatCompletion(
    * @throws OpenAiClientException if the request fails
    */
   @Nonnull
-  public OpenAiChatCompletionStream stream(@Nonnull final OpenAiChatCompletionParameters parameters)
-      throws OpenAiClientException {
+  public OpenAiChatCompletionStream<OpenAiChatCompletionOutput> stream(
+      @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
     parameters.setStream(true);
-    return stream("/chat/completions", parameters, OpenAiChatCompletionStream.class);
+    return stream("/chat/completions", parameters, OpenAiChatCompletionOutput.class);
   }
 
   /**
@@ -145,7 +145,7 @@ private <T> T execute(
   }
 
   @Nonnull
-  private <T> T stream(
+  private <T> OpenAiChatCompletionStream<T> stream(
       @Nonnull final String path,
       @Nonnull final Object payload,
       @Nonnull final Class<T> responseType) {
@@ -170,22 +170,22 @@ private <T> T executeRequest(
     try {
       @SuppressWarnings("UnstableApiUsage")
       final var client = ApacheHttpClient5Accessor.getHttpClient(destination);
-      return client.execute(request, new OpenAiResponseHandler<>(responseType, JACKSON));
+      return client.execute(request, new OpenAiResponseHandler<>(responseType));
     } catch (final IOException e) {
-      throw new OpenAiClientException("Request to OpenAI model failed.", e);
+      throw new OpenAiClientException(e);
     }
   }
 
   @Nonnull
-  private <T> T streamRequest(
+  private <T> OpenAiChatCompletionStream<T> streamRequest(
       final BasicClassicHttpRequest request, @Nonnull final Class<T> responseType) {
     try {
       @SuppressWarnings("UnstableApiUsage")
       final var client = ApacheHttpClient5Accessor.getHttpClient(destination);
-      // TODO: OpenAiStreamingHandler should return generic T instead of OpenAiChatCompletionStream
-      return (T) OpenAiStreamingHandler.handleResponse(client.executeOpen(null, request, null));
+      return new OpenAiStreamingHandler<>(responseType)
+          .handleResponse(client.executeOpen(null, request, null));
     } catch (final IOException e) {
-      throw new OpenAiClientException("Request to OpenAI model failed.", e);
+      throw new OpenAiClientException(e);
     }
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java
index d61bfa08..8f174e84 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java
@@ -5,8 +5,23 @@
 
 /** Generic exception for errors occurring when using OpenAI foundation models. */
 public class OpenAiClientException extends RuntimeException {
+  static final String BASE_ERROR_MESSAGE = "Request to OpenAI model failed";
   @Serial private static final long serialVersionUID = -7345541120979974432L;
 
+  /** Create a new exception with the base message: {@code Request to OpenAI model failed} */
+  public OpenAiClientException() {
+    super(BASE_ERROR_MESSAGE);
+  }
+
+  /**
+   * Create a new exception with the base message: {@code Request to OpenAI model failed}
+   *
+   * @param e the cause
+   */
+  public OpenAiClientException(@Nonnull final Exception e) {
+    super(BASE_ERROR_MESSAGE, e);
+  }
+
   /**
    * Create a new exception with the given message.
    *
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
index f91eed79..6ceb5859 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
@@ -1,7 +1,8 @@
 package com.sap.ai.sdk.foundationmodels.openai;
 
+import static com.sap.ai.sdk.foundationmodels.openai.OpenAiClient.JACKSON;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiError;
 import io.vavr.control.Try;
 import java.io.IOException;
@@ -21,8 +22,14 @@
 class OpenAiResponseHandler<T> implements HttpClientResponseHandler<T> {
 
   @Nonnull private final Class<T> responseType;
-  @Nonnull private final ObjectMapper jackson;
 
+  /**
+   * Processes a {@link ClassicHttpResponse} and returns some value corresponding to that response.
+   *
+   * @param response The response to process
+   * @return A model class instantiated from the response
+   * @throws OpenAiClientException in case of a problem or the connection was aborted
+   */
   @Override
   public T handleResponse(@Nonnull final ClassicHttpResponse response)
       throws OpenAiClientException {
@@ -35,14 +42,15 @@ public T handleResponse(@Nonnull final ClassicHttpResponse response)
   // The InputStream of the HTTP entity is closed by EntityUtils.toString
   @SuppressWarnings("PMD.CloseResource")
   @Nonnull
-  private T parseResponse(@Nonnull final ClassicHttpResponse response) {
+  private T parseResponse(@Nonnull final ClassicHttpResponse response)
+      throws OpenAiClientException {
     final HttpEntity responseEntity = response.getEntity();
     if (responseEntity == null) {
       throw new OpenAiClientException("Response from OpenAI model was empty.");
     }
     final var content = getContent(responseEntity);
     try {
-      return jackson.readValue(content, responseType);
+      return JACKSON.readValue(content, responseType);
     } catch (final JsonProcessingException e) {
       log.error("Failed to parse the following response from OpenAI model: {}", content);
       throw new OpenAiClientException("Failed to parse response from OpenAI model", e);
@@ -60,7 +68,8 @@ private static String getContent(@Nonnull final HttpEntity entity) {
 
   // The InputStream of the HTTP entity is closed by EntityUtils.toString
   @SuppressWarnings("PMD.CloseResource")
-  private void buildExceptionAndThrow(@Nonnull final ClassicHttpResponse response) {
+  static void buildExceptionAndThrow(@Nonnull final ClassicHttpResponse response)
+      throws OpenAiClientException {
     final var exception =
         new OpenAiClientException(
             "Request to OpenAI model failed with status %s %s "
@@ -85,19 +94,28 @@ private void buildExceptionAndThrow(@Nonnull final ClassicHttpResponse response)
       throw exception;
     }
 
-    final var maybeError = Try.of(() -> jackson.readValue(content, OpenAiError.class));
+    parseErrorAndThrow(content, exception);
+  }
+
+  /**
+   * Parse the error response and throw an exception.
+   *
+   * @param errorResponse the error response, most likely a JSON of {@link OpenAiError}.
+   * @param baseException a base exception to add the error message to.
+   */
+  static void parseErrorAndThrow(String errorResponse, OpenAiClientException baseException)
+      throws OpenAiClientException {
+    final var maybeError = Try.of(() -> JACKSON.readValue(errorResponse, OpenAiError.class));
     if (maybeError.isFailure()) {
-      exception.addSuppressed(maybeError.getCause());
-      throw exception;
+      baseException.addSuppressed(maybeError.getCause());
+      throw baseException;
     }
 
     final var error = maybeError.get().getError();
     if (error == null) {
-      throw exception;
+      throw baseException;
     }
-    final var message =
-        "Request to OpenAI model failed with %s %s and error message: '%s'"
-            .formatted(response.getCode(), response.getReasonPhrase(), error.getMessage());
-    throw new OpenAiClientException(message);
+    throw new OpenAiClientException(
+        baseException.getMessage() + "and error message: '%s'".formatted(error.getMessage()));
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 8862faba..059f8ded 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -1,8 +1,9 @@
 package com.sap.ai.sdk.foundationmodels.openai;
 
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiClient.JACKSON;
+import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.buildExceptionAndThrow;
+import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.parseErrorAndThrow;
 
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionStream;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -10,53 +11,72 @@
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.util.stream.Stream;
+import javax.annotation.Nonnull;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hc.core5.http.ClassicHttpResponse;
 
 @Slf4j
 @RequiredArgsConstructor
-class OpenAiStreamingHandler {
+class OpenAiStreamingHandler<T> {
 
-  public static OpenAiChatCompletionStream handleResponse(ClassicHttpResponse response)
-      throws IOException {
-    return getContent(response.getEntity().getContent());
+  @Nonnull private final Class<T> responseType;
+
+  /**
+   * Processes a {@link ClassicHttpResponse} and returns some value corresponding to that response.
+   *
+   * @param response The response to process
+   * @return A {@link OpenAiChatCompletionStream} of a model class instantiated from the response
+   * @throws OpenAiClientException in case of a problem or the connection was aborted
+   */
+  @Nonnull
+  public OpenAiChatCompletionStream<T> handleResponse(@Nonnull final ClassicHttpResponse response)
+      throws OpenAiClientException {
+    if (response.getCode() >= 300) {
+      buildExceptionAndThrow(response);
+    }
+    return parseResponse(response);
   }
 
   /**
-   * @param inputStream
-   * @return
+   * @param response The response to process
+   * @return A {@link OpenAiChatCompletionStream} of a model class instantiated from the response
    * @author stippi
    */
-  public static OpenAiChatCompletionStream getContent(InputStream inputStream) {
+  private OpenAiChatCompletionStream<T> parseResponse(@Nonnull final ClassicHttpResponse response)
+      throws OpenAiClientException {
 
-    OpenAiChatCompletionOutput total = null;
+    InputStream inputStream;
+    try {
+      inputStream = response.getEntity().getContent();
+    } catch (IOException e) {
+      throw new OpenAiClientException("Failed to read response content.", e);
+    }
+    var output = new OpenAiChatCompletionStream<T>();
     BufferedReader br =
         new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
 
-    Stream<OpenAiChatCompletionOutput> delta =
+    // TODO: set total
+    Stream<T> deltaStream =
         br.lines()
             .filter(
                 responseLine ->
                     !responseLine.isEmpty() && !"data: [DONE]".equals(responseLine.trim()))
+            .peek(
+                responseLine -> {
+                  if (!responseLine.startsWith("data: ")) {
+                    parseErrorAndThrow(responseLine, new OpenAiClientException());
+                  }
+                })
             .map(
                 responseLine -> {
-                  // TODO: handle errors
-                  //       {
-                  //         "error": {
-                  //           "code": "429",
-                  //           "message": "exceeded token rate limit"
-                  //         }
-                  //       }
                   String data = responseLine.substring(5).replace("delta", "message");
                   try {
-                    return JACKSON.readValue(data, OpenAiChatCompletionOutput.class);
+                    return JACKSON.readValue(data, responseType);
                   } catch (IOException e) {
                     throw new RuntimeException(e);
                   }
                 });
-    var outputStream = new OpenAiChatCompletionStream();
-    outputStream.setDelta(delta);
-    return outputStream;
+    return output.setDeltaStream(deltaStream);
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
index 70bd31d0..8dccfe55 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
@@ -23,12 +23,6 @@
 @EqualsAndHashCode(callSuper = true)
 @ToString
 public class OpenAiChatCompletionParameters extends OpenAiCompletionParameters {
-
-  /** Whether to stream the response. */
-  @JsonProperty("stream")
-  @Setter(AccessLevel.PUBLIC) // TODO: Change AccessLevel to not be public
-  private Boolean stream;
-
   /** A list of messages comprising the conversation so far. */
   @JsonProperty("messages")
   @Setter(onParam_ = @Nonnull)
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
index 23b27db2..92fbb5d7 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
@@ -2,17 +2,27 @@
 
 import java.util.concurrent.Future;
 import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import lombok.AccessLevel;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;
 import lombok.Setter;
+import lombok.experimental.Accessors;
 
 @Getter
-public class OpenAiChatCompletionStream implements AutoCloseable {
-  @Setter private Stream<OpenAiChatCompletionOutput> delta;
-  private Future<OpenAiChatCompletionOutput> total; // TODO: fill out "total" delta by delta
+@RequiredArgsConstructor(access = AccessLevel.NONE)
+@NoArgsConstructor
+@Setter
+@Accessors(chain = true)
+public class OpenAiChatCompletionStream<T> implements AutoCloseable {
+
+  @Nonnull private Stream<T> deltaStream;
+  @Nonnull private Future<T> total; // TODO: fill out "total" delta by delta
 
   @Override
   public void close() {
-    delta.close();
+    deltaStream.close();
     // TODO: fill out "total" or cancel it
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
index 76bf6417..3e989fca 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
@@ -6,7 +6,6 @@
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.AccessLevel;
 import lombok.EqualsAndHashCode;
 import lombok.Setter;
 import lombok.ToString;
@@ -80,7 +79,6 @@ public class OpenAiCompletionParameters {
    * contain the stop sequence.
    */
   @JsonProperty("stop")
-  @Setter(value = AccessLevel.NONE)
   @JsonFormat(with = JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
   @Nullable
   private List<String> stop;
@@ -103,18 +101,13 @@ public class OpenAiCompletionParameters {
   private Double frequencyPenalty;
 
   /**
-   * <b>NOTE:</b> This method is currently not supported. Therefore, it stays protected.<br>
-   * <br>
    * If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only
    * server-sent events as they become available, with the stream terminated by a `data: [DONE]`
    * message. Default: false.
    */
   @JsonProperty("stream")
-  @Setter(value = AccessLevel.NONE)
-  @Nullable
-  private Boolean stream; // TODO for implementation details, please find
-
-  // https://github.com/Azure/azure-rest-api-specs/blob/3cb1b51638616435470fc10ea00de92512186ece/specification/cognitiveservices/data-plane/AzureOpenAI/inference/stable/2024-02-01/inference.json#L1149
+  @Setter(onParam_ = @Nullable)
+  private Boolean stream;
 
   /**
    * Up to four sequences where the API will stop generating further tokens. The returned text won't

From b3190a5ab1e725ecfbe26928337bd49e6e0c8292 Mon Sep 17 00:00:00 2001
From: SAP Cloud SDK Bot <cloudsdk@sap.com>
Date: Wed, 21 Aug 2024 07:38:02 +0000
Subject: [PATCH 06/55] Formatting

---
 .../java/com/sap/ai/sdk/app/controllers/OpenAiController.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 0746ebe2..d54ed2ec 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -21,7 +21,6 @@
 import java.util.Map;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
-
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;

From f0fa3e6654b159063ca20945e3f876da94dcbad9 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Wed, 21 Aug 2024 12:46:52 +0200
Subject: [PATCH 07/55] close stream correctly

---
 .../sdk/app/controllers/OpenAiController.java | 36 ++++++++++---------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index d54ed2ec..f54419a2 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -9,6 +9,7 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionFunction;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionStream;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionTool;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage.ImageDetailLevel;
@@ -68,23 +69,24 @@ public static ResponseEntity<ResponseBodyEmitter> stream() {
     ThreadContextExecutors.getExecutor()
         .submit(
             () -> {
-              try (Stream<OpenAiChatCompletionOutput> stream =
-                  OpenAiClient.forModel(GPT_35_TURBO).stream(request).getDeltaStream()) {
-                stream.forEach(
-                    delta -> {
-                      try {
-                        // TODO: Change the types to nullable? Maybe create a class
-                        //   OpenAiChatCompletionDelta...
-                        if (!delta.getChoices().isEmpty()
-                            && delta.getChoices().get(0).getMessage() != null
-                            && delta.getChoices().get(0).getMessage().getContent() != null) {
-                          emitter.send(delta.getChoices().get(0).getMessage().getContent());
-                        }
-                      } catch (IOException e) {
-                        log.error(Arrays.toString(e.getStackTrace()));
-                        emitter.completeWithError(e);
-                      }
-                    });
+              try (var stream = OpenAiClient.forModel(GPT_35_TURBO).stream(request)) {
+                stream
+                    .getDeltaStream()
+                    .forEach(
+                        delta -> {
+                          try {
+                            // TODO: Change the types to nullable? Maybe create a class
+                            //   OpenAiChatCompletionDelta...
+                            if (!delta.getChoices().isEmpty()
+                                && delta.getChoices().get(0).getMessage() != null
+                                && delta.getChoices().get(0).getMessage().getContent() != null) {
+                              emitter.send(delta.getChoices().get(0).getMessage().getContent());
+                            }
+                          } catch (IOException e) {
+                            log.error(Arrays.toString(e.getStackTrace()));
+                            emitter.completeWithError(e);
+                          }
+                        });
                 // Once all the data is sent, complete the emitter
                 emitter.complete();
               } catch (Exception e) {

From 09ca6ea55e76756632ae958bd015fb61bdda16c7 Mon Sep 17 00:00:00 2001
From: SAP Cloud SDK Bot <cloudsdk@sap.com>
Date: Wed, 21 Aug 2024 10:47:27 +0000
Subject: [PATCH 08/55] Formatting

---
 .../java/com/sap/ai/sdk/app/controllers/OpenAiController.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index f54419a2..2b760655 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -9,7 +9,6 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionFunction;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionStream;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionTool;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage.ImageDetailLevel;
@@ -20,7 +19,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.MediaType;

From d86243a1097b54dd7ae53fc81ba00608f2897a09 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Wed, 21 Aug 2024 16:45:03 +0200
Subject: [PATCH 09/55] Created OpenAiStreamOutput

---
 .../sdk/app/controllers/OpenAiController.java | 16 ++++---------
 .../foundationmodels/openai/OpenAiClient.java |  5 ++--
 .../openai/model/OpenAiStreamOutput.java      | 24 +++++++++++++++++++
 3 files changed, 31 insertions(+), 14 deletions(-)
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index f54419a2..d4132b2b 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -9,18 +9,17 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionFunction;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionStream;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionTool;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage.ImageDetailLevel;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.cloud.sdk.cloudplatform.thread.ThreadContextExecutors;
+
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.MediaType;
@@ -72,25 +71,18 @@ public static ResponseEntity<ResponseBodyEmitter> stream() {
               try (var stream = OpenAiClient.forModel(GPT_35_TURBO).stream(request)) {
                 stream
                     .getDeltaStream()
+                    .filter(delta -> delta.getDeltaContent() != null)
                     .forEach(
                         delta -> {
                           try {
-                            // TODO: Change the types to nullable? Maybe create a class
-                            //   OpenAiChatCompletionDelta...
-                            if (!delta.getChoices().isEmpty()
-                                && delta.getChoices().get(0).getMessage() != null
-                                && delta.getChoices().get(0).getMessage().getContent() != null) {
-                              emitter.send(delta.getChoices().get(0).getMessage().getContent());
-                            }
+                            emitter.send(delta.getDeltaContent());
                           } catch (IOException e) {
                             log.error(Arrays.toString(e.getStackTrace()));
                             emitter.completeWithError(e);
                           }
                         });
-                // Once all the data is sent, complete the emitter
+              } finally{
                 emitter.complete();
-              } catch (Exception e) {
-                emitter.completeWithError(e);
               }
             });
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 536fce51..e72ac6ba 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -12,6 +12,7 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionStream;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiStreamOutput;
 import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor;
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import com.sap.cloud.sdk.cloudplatform.connectivity.Destination;
@@ -114,10 +115,10 @@ public OpenAiChatCompletionOutput chatCompletion(
    * @throws OpenAiClientException if the request fails
    */
   @Nonnull
-  public OpenAiChatCompletionStream<OpenAiChatCompletionOutput> stream(
+  public OpenAiChatCompletionStream<OpenAiStreamOutput> stream(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
     parameters.setStream(true);
-    return stream("/chat/completions", parameters, OpenAiChatCompletionOutput.class);
+    return stream("/chat/completions", parameters, OpenAiStreamOutput.class);
   }
 
   /**
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java
new file mode 100644
index 00000000..1b964a6b
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java
@@ -0,0 +1,24 @@
+package com.sap.ai.sdk.foundationmodels.openai.model;
+
+import javax.annotation.Nullable;
+
+/** OpenAI chat completion output for streaming. */
+public class OpenAiStreamOutput extends OpenAiChatCompletionOutput {
+  /**
+   * Get the message content from the delta.
+   * @return the message content from the delta or null.
+   */
+  @Nullable
+  public String getDeltaContent() {
+    // Check if choices: []
+    if (!getChoices().isEmpty()
+        // Check if "choices":[{"delta":{"content":"","role":"assistant"}}]
+        && getChoices().get(0).getMessage() != null
+        // Check if "choices":[{"delta":{}}]
+        && getChoices().get(0).getMessage().getContent() != null) {
+
+      return getChoices().get(0).getMessage().getContent();
+    }
+    return null;
+  }
+}

From cf6ec464e28051c130a6193ea5ea7203c146086d Mon Sep 17 00:00:00 2001
From: SAP Cloud SDK Bot <cloudsdk@sap.com>
Date: Wed, 21 Aug 2024 14:47:23 +0000
Subject: [PATCH 10/55] Formatting

---
 .../java/com/sap/ai/sdk/app/controllers/OpenAiController.java  | 3 +--
 .../sdk/foundationmodels/openai/model/OpenAiStreamOutput.java  | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index d4132b2b..fa41205e 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -15,7 +15,6 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.cloud.sdk.cloudplatform.thread.ThreadContextExecutors;
-
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
@@ -81,7 +80,7 @@ public static ResponseEntity<ResponseBodyEmitter> stream() {
                             emitter.completeWithError(e);
                           }
                         });
-              } finally{
+              } finally {
                 emitter.complete();
               }
             });
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java
index 1b964a6b..bdd15735 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java
@@ -6,6 +6,7 @@
 public class OpenAiStreamOutput extends OpenAiChatCompletionOutput {
   /**
    * Get the message content from the delta.
+   *
    * @return the message content from the delta or null.
    */
   @Nullable

From a73f037f868efbf7101e2c6245c1055d309f5e8c Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Thu, 22 Aug 2024 10:41:40 +0200
Subject: [PATCH 11/55] Renamed stream to streamChatCompletion, Added comments

---
 .../ai/sdk/app/controllers/OpenAiController.java    | 13 ++++++++-----
 e2e-test-app/src/main/resources/static/index.html   |  2 +-
 .../com/sap/ai/sdk/app/controllers/OpenAiTest.java  |  4 ++--
 .../sdk/foundationmodels/openai/OpenAiClient.java   |  6 +++---
 .../openai/OpenAiStreamingHandler.java              | 13 ++++++++++---
 5 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index fa41205e..6f4db0d0 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -47,13 +47,13 @@ public static OpenAiChatCompletionOutput chatCompletion() {
   }
 
   /**
-   * Stream chat request to OpenAI
+   * Asynchronous stream of an OpenAI chat request
    *
    * @return the emitter that streams the assistant message response
    */
-  @GetMapping("/stream")
+  @GetMapping("/streamChatCompletion")
   @Nonnull
-  public static ResponseEntity<ResponseBodyEmitter> stream() {
+  public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
     final var request =
         new OpenAiChatCompletionParameters()
             .setMessages(
@@ -63,13 +63,15 @@ public static ResponseEntity<ResponseBodyEmitter> stream() {
                             "Can you give me the first 100 number of the Fibonacci sequence?")));
 
     ResponseBodyEmitter emitter = new ResponseBodyEmitter();
-    // Start streaming the content asynchronously
+    // Cloud SDK's ThreadContext is vital for the request to successfully execute.
     ThreadContextExecutors.getExecutor()
         .submit(
             () -> {
-              try (var stream = OpenAiClient.forModel(GPT_35_TURBO).stream(request)) {
+              // try-with-resources to ensure the OpenAiChatCompletionStream is closed
+              try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
                 stream
                     .getDeltaStream()
+                    // The first two and the last delta do not contain any content
                     .filter(delta -> delta.getDeltaContent() != null)
                     .forEach(
                         delta -> {
@@ -84,6 +86,7 @@ public static ResponseEntity<ResponseBodyEmitter> stream() {
                 emitter.complete();
               }
             });
+    // MediaType.TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
diff --git a/e2e-test-app/src/main/resources/static/index.html b/e2e-test-app/src/main/resources/static/index.html
index 9ee4f111..1cb4146a 100644
--- a/e2e-test-app/src/main/resources/static/index.html
+++ b/e2e-test-app/src/main/resources/static/index.html
@@ -71,7 +71,7 @@ <h2>Endpoints</h2>
             <li><h4>OpenAI</h4></li>
             <ul>
                 <li><a href="/chatCompletion">/chatCompletion</a></li>
-                <li><a href="/stream">/stream</a></li>
+                <li><a href="/streamChatCompletion">/streamChatCompletion</a></li>
                 <li><a href="/chatCompletionTool">/chatCompletionTool</a></li>
                 <li><a href="/chatCompletionImage">/chatCompletionImage</a></li>
                 <li><a href="/embedding">/embedding</a></li>
diff --git a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
index 1b79a19a..30a10e89 100644
--- a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
+++ b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
@@ -24,8 +24,8 @@ void chatCompletionImage() {
   }
 
   @Test
-  void stream() {
-    final var emitter = OpenAiController.stream();
+  void streamChatCompletion() {
+    final var emitter = OpenAiController.streamChatCompletion();
     // TODO: assert on the emitter
   }
 
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index e72ac6ba..07ac28fd 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -115,10 +115,10 @@ public OpenAiChatCompletionOutput chatCompletion(
    * @throws OpenAiClientException if the request fails
    */
   @Nonnull
-  public OpenAiChatCompletionStream<OpenAiStreamOutput> stream(
+  public OpenAiChatCompletionStream<OpenAiStreamOutput> streamChatCompletion(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
     parameters.setStream(true);
-    return stream("/chat/completions", parameters, OpenAiStreamOutput.class);
+    return streamChatCompletion("/chat/completions", parameters, OpenAiStreamOutput.class);
   }
 
   /**
@@ -146,7 +146,7 @@ private <T> T execute(
   }
 
   @Nonnull
-  private <T> OpenAiChatCompletionStream<T> stream(
+  private <T> OpenAiChatCompletionStream<T> streamChatCompletion(
       @Nonnull final String path,
       @Nonnull final Object payload,
       @Nonnull final Class<T> responseType) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 059f8ded..ea49e111 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -15,6 +15,7 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hc.core5.http.ClassicHttpResponse;
+import org.apache.hc.core5.http.HttpEntity;
 
 @Slf4j
 @RequiredArgsConstructor
@@ -46,21 +47,27 @@ public OpenAiChatCompletionStream<T> handleResponse(@Nonnull final ClassicHttpRe
   private OpenAiChatCompletionStream<T> parseResponse(@Nonnull final ClassicHttpResponse response)
       throws OpenAiClientException {
 
+    final HttpEntity responseEntity = response.getEntity();
+    if (responseEntity == null) {
+      throw new OpenAiClientException("Response from OpenAI model was empty.");
+    }
+
     InputStream inputStream;
     try {
-      inputStream = response.getEntity().getContent();
+      inputStream = responseEntity.getContent();
     } catch (IOException e) {
       throw new OpenAiClientException("Failed to read response content.", e);
     }
     var output = new OpenAiChatCompletionStream<T>();
-    BufferedReader br =
-        new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+    final var br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
 
+    // https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
     // TODO: set total
     Stream<T> deltaStream =
         br.lines()
             .filter(
                 responseLine ->
+                    // half of the lines are empty newlines, the last line is "data: [DONE]"
                     !responseLine.isEmpty() && !"data: [DONE]".equals(responseLine.trim()))
             .peek(
                 responseLine -> {

From eb3f24a945403d23cd98b96c30bb40ac1b487bab Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 23 Aug 2024 11:29:38 +0200
Subject: [PATCH 12/55] Added total output

---
 .../sdk/app/controllers/OpenAiController.java |  8 +--
 .../ai/sdk/foundationmodels/openai/Delta.java | 14 +++++
 .../foundationmodels/openai/OpenAiClient.java | 28 ++++++----
 .../openai/OpenAiClientException.java         |  4 +-
 .../openai/OpenAiStreamingHandler.java        | 37 ++++++++-----
 .../model/OpenAiChatCompletionDelta.java      | 52 +++++++++++++++++++
 .../model/OpenAiChatCompletionOutput.java     | 16 ++++--
 .../model/OpenAiChatCompletionStream.java     | 28 ----------
 .../openai/model/OpenAiChatMessage.java       |  2 +-
 .../openai/model/OpenAiCompletionChoice.java  |  2 +-
 .../openai/model/OpenAiStream.java            | 45 ++++++++++++++++
 .../openai/model/OpenAiStreamOutput.java      | 25 ---------
 .../openai/model/Streamable.java              |  6 +++
 13 files changed, 180 insertions(+), 87 deletions(-)
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/Delta.java
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
 delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStream.java
 delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/Streamable.java

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 6f4db0d0..d352ad2d 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -60,7 +60,7 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                 List.of(
                     new OpenAiChatUserMessage()
                         .addText(
-                            "Can you give me the first 100 number of the Fibonacci sequence?")));
+                            "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
     ResponseBodyEmitter emitter = new ResponseBodyEmitter();
     // Cloud SDK's ThreadContext is vital for the request to successfully execute.
@@ -68,10 +68,11 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
         .submit(
             () -> {
               // try-with-resources to ensure the OpenAiChatCompletionStream is closed
-              try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
+              // TODO: mode to outside the thread to throw
+              try (var stream = OpenAiClient.forModel(GPT_35_TURBO).stream(request)) {
                 stream
                     .getDeltaStream()
-                    // The first two and the last delta do not contain any content
+                    // The first two and the last delta do not contain any message content
                     .filter(delta -> delta.getDeltaContent() != null)
                     .forEach(
                         delta -> {
@@ -82,6 +83,7 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                             emitter.completeWithError(e);
                           }
                         });
+                // TODO: send totalOutput
               } finally {
                 emitter.complete();
               }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/Delta.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/Delta.java
new file mode 100644
index 00000000..13759427
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/Delta.java
@@ -0,0 +1,14 @@
+package com.sap.ai.sdk.foundationmodels.openai;
+
+import javax.annotation.Nullable;
+
+public interface Delta {
+
+  /**
+   * Get the content from the delta.
+   *
+   * @return the content from the delta or null.
+   */
+  @Nullable
+  public String getDeltaContent();
+}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 07ac28fd..13d69db8 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -7,12 +7,13 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.sap.ai.sdk.core.Core;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionDelta;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionStream;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiStreamOutput;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiStream;
+import com.sap.ai.sdk.foundationmodels.openai.model.Streamable;
 import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor;
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import com.sap.cloud.sdk.cloudplatform.connectivity.Destination;
@@ -115,10 +116,14 @@ public OpenAiChatCompletionOutput chatCompletion(
    * @throws OpenAiClientException if the request fails
    */
   @Nonnull
-  public OpenAiChatCompletionStream<OpenAiStreamOutput> streamChatCompletion(
+  public OpenAiStream<OpenAiChatCompletionDelta, OpenAiChatCompletionOutput> stream(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
     parameters.setStream(true);
-    return streamChatCompletion("/chat/completions", parameters, OpenAiStreamOutput.class);
+    return stream(
+        "/chat/completions",
+        parameters,
+        OpenAiChatCompletionDelta.class,
+        OpenAiChatCompletionOutput.class);
   }
 
   /**
@@ -146,13 +151,14 @@ private <T> T execute(
   }
 
   @Nonnull
-  private <T> OpenAiChatCompletionStream<T> streamChatCompletion(
+  private <D extends Delta, T extends Streamable<D>> OpenAiStream<D, T> stream(
       @Nonnull final String path,
       @Nonnull final Object payload,
-      @Nonnull final Class<T> responseType) {
+      @Nonnull final Class<D> deltaType,
+      @Nonnull final Class<T> totalType) {
     final var request = new HttpPost(path);
     serializeAndSetHttpEntity(request, payload);
-    return streamRequest(request, responseType);
+    return streamRequest(request, deltaType, totalType);
   }
 
   private static void serializeAndSetHttpEntity(
@@ -178,12 +184,14 @@ private <T> T executeRequest(
   }
 
   @Nonnull
-  private <T> OpenAiChatCompletionStream<T> streamRequest(
-      final BasicClassicHttpRequest request, @Nonnull final Class<T> responseType) {
+  private <D extends Delta, T extends Streamable<D>> OpenAiStream<D, T> streamRequest(
+      final BasicClassicHttpRequest request,
+      @Nonnull final Class<D> deltaType,
+      @Nonnull final Class<T> totalType) {
     try {
       @SuppressWarnings("UnstableApiUsage")
       final var client = ApacheHttpClient5Accessor.getHttpClient(destination);
-      return new OpenAiStreamingHandler<>(responseType)
+      return new OpenAiStreamingHandler<>(deltaType, totalType)
           .handleResponse(client.executeOpen(null, request, null));
     } catch (final IOException e) {
       throw new OpenAiClientException(e);
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java
index 8f174e84..18a2aaeb 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java
@@ -10,7 +10,7 @@ public class OpenAiClientException extends RuntimeException {
 
   /** Create a new exception with the base message: {@code Request to OpenAI model failed} */
   public OpenAiClientException() {
-    super(BASE_ERROR_MESSAGE);
+    this(BASE_ERROR_MESSAGE);
   }
 
   /**
@@ -19,7 +19,7 @@ public OpenAiClientException() {
    * @param e the cause
    */
   public OpenAiClientException(@Nonnull final Exception e) {
-    super(BASE_ERROR_MESSAGE, e);
+    this(BASE_ERROR_MESSAGE, e);
   }
 
   /**
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index ea49e111..bb3b6949 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -4,11 +4,13 @@
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.buildExceptionAndThrow;
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.parseErrorAndThrow;
 
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionStream;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiStream;
+import com.sap.ai.sdk.foundationmodels.openai.model.Streamable;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.charset.StandardCharsets;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
@@ -19,19 +21,20 @@
 
 @Slf4j
 @RequiredArgsConstructor
-class OpenAiStreamingHandler<T> {
+class OpenAiStreamingHandler<D extends Delta, T extends Streamable<D>> {
 
-  @Nonnull private final Class<T> responseType;
+  @Nonnull private final Class<D> deltaType;
+  @Nonnull private final Class<T> totalType;
 
   /**
    * Processes a {@link ClassicHttpResponse} and returns some value corresponding to that response.
    *
    * @param response The response to process
-   * @return A {@link OpenAiChatCompletionStream} of a model class instantiated from the response
+   * @return A {@link OpenAiStream} of a model class instantiated from the response
    * @throws OpenAiClientException in case of a problem or the connection was aborted
    */
   @Nonnull
-  public OpenAiChatCompletionStream<T> handleResponse(@Nonnull final ClassicHttpResponse response)
+  public OpenAiStream<D, T> handleResponse(@Nonnull final ClassicHttpResponse response)
       throws OpenAiClientException {
     if (response.getCode() >= 300) {
       buildExceptionAndThrow(response);
@@ -41,29 +44,35 @@ public OpenAiChatCompletionStream<T> handleResponse(@Nonnull final ClassicHttpRe
 
   /**
    * @param response The response to process
-   * @return A {@link OpenAiChatCompletionStream} of a model class instantiated from the response
+   * @return A {@link OpenAiStream} of a model class instantiated from the response
    * @author stippi
    */
-  private OpenAiChatCompletionStream<T> parseResponse(@Nonnull final ClassicHttpResponse response)
+  private OpenAiStream<D, T> parseResponse(@Nonnull final ClassicHttpResponse response)
       throws OpenAiClientException {
-
     final HttpEntity responseEntity = response.getEntity();
     if (responseEntity == null) {
       throw new OpenAiClientException("Response from OpenAI model was empty.");
     }
-
     InputStream inputStream;
     try {
       inputStream = responseEntity.getContent();
     } catch (IOException e) {
       throw new OpenAiClientException("Failed to read response content.", e);
     }
-    var output = new OpenAiChatCompletionStream<T>();
     final var br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
 
+    OpenAiStream<D, T> output = new OpenAiStream<D, T>();
+    try {
+      output.setTotal(totalType.getDeclaredConstructor().newInstance());
+    } catch (InstantiationException
+        | IllegalAccessException
+        | NoSuchMethodException
+        | InvocationTargetException e) {
+      throw new OpenAiClientException(e);
+    }
+
     // https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
-    // TODO: set total
-    Stream<T> deltaStream =
+    Stream<D> deltaStream =
         br.lines()
             .filter(
                 responseLine ->
@@ -79,7 +88,9 @@ private OpenAiChatCompletionStream<T> parseResponse(@Nonnull final ClassicHttpRe
                 responseLine -> {
                   String data = responseLine.substring(5).replace("delta", "message");
                   try {
-                    return JACKSON.readValue(data, responseType);
+                    D delta = JACKSON.readValue(data, deltaType);
+                    output.addDelta(delta);
+                    return delta;
                   } catch (IOException e) {
                     throw new RuntimeException(e);
                   }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
new file mode 100644
index 00000000..cca587cd
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
@@ -0,0 +1,52 @@
+package com.sap.ai.sdk.foundationmodels.openai.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.sap.ai.sdk.foundationmodels.openai.Delta;
+import java.util.List;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+
+/** OpenAI chat completion output delta for streaming. */
+@Accessors(chain = true)
+@EqualsAndHashCode(callSuper = true)
+@ToString
+public class OpenAiChatCompletionDelta extends OpenAiCompletionOutput implements Delta {
+  /** List of result candidates. */
+  @JsonProperty("choices")
+  @Getter(onMethod_ = @Nonnull)
+  private List<OpenAiChatCompletionChoice> choices;
+
+  /**
+   * Can be used in conjunction with the seed request parameter to understand when backend changes
+   * have been made that might impact determinism.
+   */
+  @JsonProperty("system_fingerprint")
+  @Getter(onMethod_ = @Nonnull)
+  private String systemFingerprint;
+
+  /**
+   * Get the message content from the delta.
+   *
+   * @return the message content from the delta or null.
+   */
+  @Nullable
+  public String getDeltaContent() {
+    // Avoid the first delta: "choices":[]
+    if (!getChoices().isEmpty()
+        // Multiple choices are spread out on multiple deltas
+        // A delta only contains one choice with a variable index
+        && getChoices().get(0).getIndex() == 0
+        // Avoid the second to last delta: "choices":[{"delta":{"content":"","role":"assistant"}}]
+        && getChoices().get(0).getMessage() != null
+        // Avoid the last delta "choices":[{"delta":{}}]
+        && getChoices().get(0).getMessage().getContent() != null) {
+
+      return getChoices().get(0).getMessage().getContent();
+    }
+    return null;
+  }
+}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
index 2cec01c6..af12f17c 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
@@ -13,7 +13,8 @@
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
 @ToString
-public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput {
+public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
+    implements Streamable<OpenAiChatCompletionDelta> {
   /** List of result candidates. */
   @JsonProperty("choices")
   @Getter(onMethod_ = @Nonnull)
@@ -27,9 +28,16 @@ public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput {
   @Getter(onMethod_ = @Nonnull)
   private String systemFingerprint;
 
-  public void addDelta(OpenAiChatCompletionOutput delta) {
+  public void addDelta(OpenAiChatCompletionDelta delta) {
     // TODO: Assign every field if not null on all parent and children classes.
     //       Right now we only assign content message.
+
+    if (delta.getChoices().isEmpty()
+        // Multiple choices are spread out on multiple deltas
+        // A delta only contains one choice with a variable index
+        || delta.getChoices().get(0).getIndex() != 0) {
+      return;
+    }
     var deltaMessage = delta.getChoices().get(0).getMessage();
     if (deltaMessage == null) {
       return;
@@ -38,11 +46,11 @@ public void addDelta(OpenAiChatCompletionOutput delta) {
     if (deltaContent == null) {
       return;
     }
-    if (choices.isEmpty()) {
+    if (choices == null || choices.isEmpty()) {
       var choice =
           new OpenAiChatCompletionChoice()
               .setMessage(new OpenAiChatAssistantMessage().setContent(deltaContent));
-      choices.add(choice);
+      choices = List.of(choice);
       return;
     }
     var message = choices.get(0).getMessage();
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
deleted file mode 100644
index 92fbb5d7..00000000
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionStream.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package com.sap.ai.sdk.foundationmodels.openai.model;
-
-import java.util.concurrent.Future;
-import java.util.stream.Stream;
-import javax.annotation.Nonnull;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.RequiredArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-
-@Getter
-@RequiredArgsConstructor(access = AccessLevel.NONE)
-@NoArgsConstructor
-@Setter
-@Accessors(chain = true)
-public class OpenAiChatCompletionStream<T> implements AutoCloseable {
-
-  @Nonnull private Stream<T> deltaStream;
-  @Nonnull private Future<T> total; // TODO: fill out "total" delta by delta
-
-  @Override
-  public void close() {
-    deltaStream.close();
-    // TODO: fill out "total" or cancel it
-  }
-}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
index c87abcfa..e090990a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
@@ -244,7 +244,7 @@ class OpenAiChatAssistantMessage implements OpenAiChatMessage {
     /** Message content. */
     @JsonProperty("content")
     @Getter(onMethod_ = @Nullable)
-    @Setter(onParam_ = @Nonnull, value = AccessLevel.PACKAGE)
+    @Setter(onParam_ = @Nullable, value = AccessLevel.PACKAGE)
     private String content; // must be String or null
 
     /** The tool calls generated by the model, such as function calls. */
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
index 701f86d8..3c43e732 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
@@ -19,7 +19,7 @@ public class OpenAiCompletionChoice {
 
   /** Index of choice. */
   @JsonProperty("index")
-  @Getter(onMethod_ = @Nullable)
+  @Getter // Nullable
   private Integer index;
 
   /**
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStream.java
new file mode 100644
index 00000000..564660d7
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStream.java
@@ -0,0 +1,45 @@
+package com.sap.ai.sdk.foundationmodels.openai.model;
+
+import com.sap.ai.sdk.foundationmodels.openai.Delta;
+import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+/** OpenAI chat completion stream output. */
+@RequiredArgsConstructor(access = AccessLevel.NONE)
+@NoArgsConstructor
+@Setter // TODO: package private
+@Accessors(chain = true)
+// T implements Streamable but Java generics, oddly enough, use extends for interfaces 😣
+public class OpenAiStream<D extends Delta, T extends Streamable<D>> implements AutoCloseable {
+
+  @Getter @Nonnull private Stream<D> deltaStream;
+  @Nonnull private T total;
+
+  // TODO: package private
+  public void addDelta(D delta) {
+    total.addDelta(delta);
+  }
+
+  public Future<T> getTotalOutput() {
+    // wait for deltaStream to be closed
+    return new FutureTask<>(
+        () -> {
+          // wait for deltaStream to be closed
+          deltaStream.onClose(() -> {});
+          return total;
+        });
+  }
+
+  @Override
+  public void close() {
+    deltaStream.close();
+  }
+}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java
deleted file mode 100644
index bdd15735..00000000
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStreamOutput.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package com.sap.ai.sdk.foundationmodels.openai.model;
-
-import javax.annotation.Nullable;
-
-/** OpenAI chat completion output for streaming. */
-public class OpenAiStreamOutput extends OpenAiChatCompletionOutput {
-  /**
-   * Get the message content from the delta.
-   *
-   * @return the message content from the delta or null.
-   */
-  @Nullable
-  public String getDeltaContent() {
-    // Check if choices: []
-    if (!getChoices().isEmpty()
-        // Check if "choices":[{"delta":{"content":"","role":"assistant"}}]
-        && getChoices().get(0).getMessage() != null
-        // Check if "choices":[{"delta":{}}]
-        && getChoices().get(0).getMessage().getContent() != null) {
-
-      return getChoices().get(0).getMessage().getContent();
-    }
-    return null;
-  }
-}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/Streamable.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/Streamable.java
new file mode 100644
index 00000000..eaafc875
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/Streamable.java
@@ -0,0 +1,6 @@
+package com.sap.ai.sdk.foundationmodels.openai.model;
+
+public interface Streamable<D> {
+
+  void addDelta(D delta);
+}

From fb2cdafeb67289082f18fc068f19a067b0456c1a Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 23 Aug 2024 14:26:47 +0200
Subject: [PATCH 13/55] Total output is printed

---
 .../sdk/app/controllers/OpenAiController.java | 45 ++++++++++---------
 .../ai/sdk/foundationmodels/openai/Delta.java | 14 ------
 .../foundationmodels/openai/OpenAiClient.java |  8 ++--
 .../foundationmodels/openai/OpenAiStream.java | 42 +++++++++++++++++
 .../openai/OpenAiStreamingHandler.java        |  9 ++--
 .../openai/model/DeltaAggregatable.java       | 13 ++++++
 .../model/OpenAiChatCompletionDelta.java      |  9 ++--
 .../model/OpenAiChatCompletionOutput.java     |  2 +-
 .../openai/model/OpenAiStream.java            | 45 -------------------
 .../openai/model/Streamable.java              |  6 ---
 .../openai/model/StreamedDelta.java           | 21 +++++++++
 11 files changed, 115 insertions(+), 99 deletions(-)
 delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/Delta.java
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
 delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStream.java
 delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/Streamable.java
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index d352ad2d..5ffda61c 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -6,6 +6,7 @@
 import static com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionTool.ToolType.FUNCTION;
 
 import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionDelta;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionFunction;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
@@ -19,6 +20,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.MediaType;
@@ -62,36 +64,37 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                         .addText(
                             "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-    ResponseBodyEmitter emitter = new ResponseBodyEmitter();
+    var emitter = new ResponseBodyEmitter();
+    var stream = OpenAiClient.forModel(GPT_35_TURBO).stream(request);
+
     // Cloud SDK's ThreadContext is vital for the request to successfully execute.
     ThreadContextExecutors.getExecutor()
         .submit(
             () -> {
-              // try-with-resources to ensure the OpenAiChatCompletionStream is closed
-              // TODO: mode to outside the thread to throw
-              try (var stream = OpenAiClient.forModel(GPT_35_TURBO).stream(request)) {
-                stream
-                    .getDeltaStream()
-                    // The first two and the last delta do not contain any message content
-                    .filter(delta -> delta.getDeltaContent() != null)
-                    .forEach(
-                        delta -> {
-                          try {
-                            emitter.send(delta.getDeltaContent());
-                          } catch (IOException e) {
-                            log.error(Arrays.toString(e.getStackTrace()));
-                            emitter.completeWithError(e);
-                          }
-                        });
-                // TODO: send totalOutput
-              } finally {
-                emitter.complete();
-              }
+              stream
+                  .getDeltaStream()
+                  .map(OpenAiChatCompletionDelta::getDeltaContent)
+                  // The first two and the last delta do not contain any message content
+                  .filter(Objects::nonNull)
+                  .forEach(content -> send(emitter, content));
+
+              send(emitter, "\n\n-----Total Output-----\n\n" + stream.getTotalOutput());
+              emitter.complete();
+              stream.close();
             });
     // MediaType.TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
+  private static void send(ResponseBodyEmitter emitter, String chunk) {
+    try {
+      emitter.send(chunk);
+    } catch (IOException e) {
+      log.error(Arrays.toString(e.getStackTrace()));
+      emitter.completeWithError(e);
+    }
+  }
+
   /**
    * Chat request to OpenAI with an image
    *
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/Delta.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/Delta.java
deleted file mode 100644
index 13759427..00000000
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/Delta.java
+++ /dev/null
@@ -1,14 +0,0 @@
-package com.sap.ai.sdk.foundationmodels.openai;
-
-import javax.annotation.Nullable;
-
-public interface Delta {
-
-  /**
-   * Get the content from the delta.
-   *
-   * @return the content from the delta or null.
-   */
-  @Nullable
-  public String getDeltaContent();
-}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 13d69db8..8c729258 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -7,13 +7,13 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.sap.ai.sdk.core.Core;
+import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionDelta;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiStream;
-import com.sap.ai.sdk.foundationmodels.openai.model.Streamable;
+import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
 import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor;
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import com.sap.cloud.sdk.cloudplatform.connectivity.Destination;
@@ -151,7 +151,7 @@ private <T> T execute(
   }
 
   @Nonnull
-  private <D extends Delta, T extends Streamable<D>> OpenAiStream<D, T> stream(
+  private <D extends StreamedDelta, T extends DeltaAggregatable<D>> OpenAiStream<D, T> stream(
       @Nonnull final String path,
       @Nonnull final Object payload,
       @Nonnull final Class<D> deltaType,
@@ -184,7 +184,7 @@ private <T> T executeRequest(
   }
 
   @Nonnull
-  private <D extends Delta, T extends Streamable<D>> OpenAiStream<D, T> streamRequest(
+  private <D extends StreamedDelta, T extends DeltaAggregatable<D>> OpenAiStream<D, T> streamRequest(
       final BasicClassicHttpRequest request,
       @Nonnull final Class<D> deltaType,
       @Nonnull final Class<T> totalType) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
new file mode 100644
index 00000000..5d1699b9
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
@@ -0,0 +1,42 @@
+package com.sap.ai.sdk.foundationmodels.openai;
+
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+
+import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
+import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+/** Generic OpenAI stream output. */
+@RequiredArgsConstructor(access = AccessLevel.NONE)
+@NoArgsConstructor
+@Setter // TODO: package private
+@Accessors(chain = true)
+// D extends StreamedDelta but Java generics, oddly enough, use extends for interfaces
+public class OpenAiStream<D extends StreamedDelta, T extends DeltaAggregatable<D>>
+    implements AutoCloseable {
+
+  @Getter @Nonnull private Stream<D> deltaStream;
+  @Nonnull private T totalOutput;
+
+  void addDelta(D delta) {
+    totalOutput.addDelta(delta);
+  }
+
+  /** Get the total aggregated output. */
+  public T getTotalOutput() {
+    deltaStream.close();
+    return totalOutput;
+  }
+
+  /** Close the delta stream. */
+  @Override
+  public void close() {
+    deltaStream.close();
+  }
+}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index bb3b6949..38de8f0a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -4,8 +4,6 @@
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.buildExceptionAndThrow;
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.parseErrorAndThrow;
 
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiStream;
-import com.sap.ai.sdk.foundationmodels.openai.model.Streamable;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -14,6 +12,9 @@
 import java.nio.charset.StandardCharsets;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
+
+import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
+import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hc.core5.http.ClassicHttpResponse;
@@ -21,7 +22,7 @@
 
 @Slf4j
 @RequiredArgsConstructor
-class OpenAiStreamingHandler<D extends Delta, T extends Streamable<D>> {
+class OpenAiStreamingHandler<D extends StreamedDelta, T extends DeltaAggregatable<D>> {
 
   @Nonnull private final Class<D> deltaType;
   @Nonnull private final Class<T> totalType;
@@ -63,7 +64,7 @@ private OpenAiStream<D, T> parseResponse(@Nonnull final ClassicHttpResponse resp
 
     OpenAiStream<D, T> output = new OpenAiStream<D, T>();
     try {
-      output.setTotal(totalType.getDeclaredConstructor().newInstance());
+      output.setTotalOutput(totalType.getDeclaredConstructor().newInstance());
     } catch (InstantiationException
         | IllegalAccessException
         | NoSuchMethodException
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
new file mode 100644
index 00000000..643072a0
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
@@ -0,0 +1,13 @@
+package com.sap.ai.sdk.foundationmodels.openai.model;
+
+/**
+ * Interface for model classes that can be created from aggregated streamed deltas.
+ *
+ * <p>For example aggregating chat completions deltas into a single chat completion output.
+ *
+ * @param <D> the delta type.
+ */
+public interface DeltaAggregatable<D> {
+
+  void addDelta( D delta );
+}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
index cca587cd..aaad7177 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
@@ -1,7 +1,7 @@
 package com.sap.ai.sdk.foundationmodels.openai.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.sap.ai.sdk.foundationmodels.openai.Delta;
+
 import java.util.List;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -14,7 +14,8 @@
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
 @ToString
-public class OpenAiChatCompletionDelta extends OpenAiCompletionOutput implements Delta {
+public class OpenAiChatCompletionDelta extends OpenAiCompletionOutput implements StreamedDelta
+{
   /** List of result candidates. */
   @JsonProperty("choices")
   @Getter(onMethod_ = @Nonnull)
@@ -31,7 +32,7 @@ public class OpenAiChatCompletionDelta extends OpenAiCompletionOutput implements
   /**
    * Get the message content from the delta.
    *
-   * @return the message content from the delta or null.
+   * @return the message content or null.
    */
   @Nullable
   public String getDeltaContent() {
@@ -40,7 +41,7 @@ public String getDeltaContent() {
         // Multiple choices are spread out on multiple deltas
         // A delta only contains one choice with a variable index
         && getChoices().get(0).getIndex() == 0
-        // Avoid the second to last delta: "choices":[{"delta":{"content":"","role":"assistant"}}]
+        // Avoid the second delta: "choices":[{"delta":{"content":"","role":"assistant"}}]
         && getChoices().get(0).getMessage() != null
         // Avoid the last delta "choices":[{"delta":{}}]
         && getChoices().get(0).getMessage().getContent() != null) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
index af12f17c..f6da1fc9 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
@@ -14,7 +14,7 @@
 @EqualsAndHashCode(callSuper = true)
 @ToString
 public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
-    implements Streamable<OpenAiChatCompletionDelta> {
+    implements DeltaAggregatable<OpenAiChatCompletionDelta> {
   /** List of result candidates. */
   @JsonProperty("choices")
   @Getter(onMethod_ = @Nonnull)
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStream.java
deleted file mode 100644
index 564660d7..00000000
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiStream.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package com.sap.ai.sdk.foundationmodels.openai.model;
-
-import com.sap.ai.sdk.foundationmodels.openai.Delta;
-import java.util.concurrent.Future;
-import java.util.concurrent.FutureTask;
-import java.util.stream.Stream;
-import javax.annotation.Nonnull;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.RequiredArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-
-/** OpenAI chat completion stream output. */
-@RequiredArgsConstructor(access = AccessLevel.NONE)
-@NoArgsConstructor
-@Setter // TODO: package private
-@Accessors(chain = true)
-// T implements Streamable but Java generics, oddly enough, use extends for interfaces 😣
-public class OpenAiStream<D extends Delta, T extends Streamable<D>> implements AutoCloseable {
-
-  @Getter @Nonnull private Stream<D> deltaStream;
-  @Nonnull private T total;
-
-  // TODO: package private
-  public void addDelta(D delta) {
-    total.addDelta(delta);
-  }
-
-  public Future<T> getTotalOutput() {
-    // wait for deltaStream to be closed
-    return new FutureTask<>(
-        () -> {
-          // wait for deltaStream to be closed
-          deltaStream.onClose(() -> {});
-          return total;
-        });
-  }
-
-  @Override
-  public void close() {
-    deltaStream.close();
-  }
-}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/Streamable.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/Streamable.java
deleted file mode 100644
index eaafc875..00000000
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/Streamable.java
+++ /dev/null
@@ -1,6 +0,0 @@
-package com.sap.ai.sdk.foundationmodels.openai.model;
-
-public interface Streamable<D> {
-
-  void addDelta(D delta);
-}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java
new file mode 100644
index 00000000..f7756b09
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java
@@ -0,0 +1,21 @@
+package com.sap.ai.sdk.foundationmodels.openai.model;
+
+import javax.annotation.Nullable;
+
+/**
+ * Interface for streamed delta classes.
+ *
+ * <p>This interface defines a method to retrieve the content from a delta, which is a chunk in a
+ * stream of data. Implementations of this interface should provide the logic to extract the
+ * relevant content from the delta.
+ */
+public interface StreamedDelta {
+
+  /**
+   * Get the content from the delta.
+   *
+   * @return the content from the delta or null if no content is available.
+   */
+  @Nullable
+  String getDeltaContent();
+}

From fe078c7bbbed2bb3c6a700203031a9d25206954c Mon Sep 17 00:00:00 2001
From: SAP Cloud SDK Bot <cloudsdk@sap.com>
Date: Fri, 23 Aug 2024 12:27:18 +0000
Subject: [PATCH 14/55] Formatting

---
 .../sap/ai/sdk/foundationmodels/openai/OpenAiClient.java | 9 +++++----
 .../sap/ai/sdk/foundationmodels/openai/OpenAiStream.java | 5 ++---
 .../foundationmodels/openai/OpenAiStreamingHandler.java  | 5 ++---
 .../foundationmodels/openai/model/DeltaAggregatable.java | 2 +-
 .../openai/model/OpenAiChatCompletionDelta.java          | 4 +---
 5 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 8c729258..db518e4d 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -184,10 +184,11 @@ private <T> T executeRequest(
   }
 
   @Nonnull
-  private <D extends StreamedDelta, T extends DeltaAggregatable<D>> OpenAiStream<D, T> streamRequest(
-      final BasicClassicHttpRequest request,
-      @Nonnull final Class<D> deltaType,
-      @Nonnull final Class<T> totalType) {
+  private <D extends StreamedDelta, T extends DeltaAggregatable<D>>
+      OpenAiStream<D, T> streamRequest(
+          final BasicClassicHttpRequest request,
+          @Nonnull final Class<D> deltaType,
+          @Nonnull final Class<T> totalType) {
     try {
       @SuppressWarnings("UnstableApiUsage")
       final var client = ApacheHttpClient5Accessor.getHttpClient(destination);
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
index 5d1699b9..124bbe8d 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
@@ -1,10 +1,9 @@
 package com.sap.ai.sdk.foundationmodels.openai;
 
-import java.util.stream.Stream;
-import javax.annotation.Nonnull;
-
 import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
 import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 38de8f0a..117401a3 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -4,6 +4,8 @@
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.buildExceptionAndThrow;
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.parseErrorAndThrow;
 
+import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
+import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -12,9 +14,6 @@
 import java.nio.charset.StandardCharsets;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
-
-import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
-import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hc.core5.http.ClassicHttpResponse;
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
index 643072a0..b049fec0 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
@@ -9,5 +9,5 @@
  */
 public interface DeltaAggregatable<D> {
 
-  void addDelta( D delta );
+  void addDelta(D delta);
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
index aaad7177..7704a16a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
@@ -1,7 +1,6 @@
 package com.sap.ai.sdk.foundationmodels.openai.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-
 import java.util.List;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -14,8 +13,7 @@
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
 @ToString
-public class OpenAiChatCompletionDelta extends OpenAiCompletionOutput implements StreamedDelta
-{
+public class OpenAiChatCompletionDelta extends OpenAiCompletionOutput implements StreamedDelta {
   /** List of result candidates. */
   @JsonProperty("choices")
   @Getter(onMethod_ = @Nonnull)

From 09e1be0431875e2eb607bd6d7eae53bb598bc175 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 23 Aug 2024 16:10:16 +0200
Subject: [PATCH 15/55] addDelta is propagated everywhere

---
 .../sdk/app/controllers/OpenAiController.java | 17 +++++--
 .../foundationmodels/openai/OpenAiClient.java |  6 +--
 .../openai/model/DeltaAggregatable.java       |  5 +++
 .../model/OpenAiChatCompletionChoice.java     | 13 +++++-
 .../model/OpenAiChatCompletionFunction.java   |  2 +-
 .../model/OpenAiChatCompletionOutput.java     | 45 ++++++++-----------
 .../model/OpenAiChatCompletionParameters.java |  4 +-
 .../model/OpenAiChatCompletionTool.java       |  2 +-
 .../openai/model/OpenAiChatFunctionCall.java  |  2 +-
 .../openai/model/OpenAiChatMessage.java       | 30 ++++++++++---
 .../openai/model/OpenAiChatToolCall.java      |  2 +-
 .../openai/model/OpenAiCompletionChoice.java  | 20 ++++++++-
 .../openai/model/OpenAiCompletionOutput.java  |  2 +-
 .../model/OpenAiCompletionParameters.java     |  2 +-
 .../OpenAiContentFilterDetectedResult.java    |  2 +-
 .../OpenAiContentFilterPromptResults.java     | 10 ++++-
 .../model/OpenAiContentFilterResultBase.java  |  2 +-
 .../model/OpenAiContentFilterResultsBase.java | 27 ++++++++++-
 .../OpenAiContentFilterSeverityResult.java    |  2 +-
 ...ta.java => OpenAiDeltaChatCompletion.java} | 12 +++--
 .../OpenAiDeltaChatCompletionChoice.java      | 23 ++++++++++
 .../openai/model/OpenAiEmbeddingData.java     |  2 +-
 .../openai/model/OpenAiEmbeddingOutput.java   |  2 +-
 .../model/OpenAiEmbeddingParameters.java      |  2 +-
 .../openai/model/OpenAiError.java             |  2 +-
 .../openai/model/OpenAiErrorBase.java         |  2 +-
 .../model/OpenAiPromptFilterResult.java       |  2 +-
 .../openai/model/OpenAiUsage.java             |  2 +-
 28 files changed, 176 insertions(+), 68 deletions(-)
 rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/{OpenAiChatCompletionDelta.java => OpenAiDeltaChatCompletion.java} (80%)
 create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletionChoice.java

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 5ffda61c..cb97d700 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -5,14 +5,16 @@
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiModel.TEXT_EMBEDDING_ADA_002;
 import static com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionTool.ToolType.FUNCTION;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionDelta;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionFunction;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionTool;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage.ImageDetailLevel;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiDeltaChatCompletion;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.cloud.sdk.cloudplatform.thread.ThreadContextExecutors;
@@ -73,12 +75,13 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
             () -> {
               stream
                   .getDeltaStream()
-                  .map(OpenAiChatCompletionDelta::getDeltaContent)
+                  .map(OpenAiDeltaChatCompletion::getDeltaContent)
                   // The first two and the last delta do not contain any message content
                   .filter(Objects::nonNull)
                   .forEach(content -> send(emitter, content));
 
-              send(emitter, "\n\n-----Total Output-----\n\n" + stream.getTotalOutput());
+              String indentedJson = objectToJson(stream.getTotalOutput());
+              send(emitter, "\n\n-----Total Output-----\n\n" + indentedJson);
               emitter.complete();
               stream.close();
             });
@@ -95,6 +98,14 @@ private static void send(ResponseBodyEmitter emitter, String chunk) {
     }
   }
 
+  private static String objectToJson(Object obj) {
+    try {
+      return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(obj);
+    } catch (JsonProcessingException ignored) {
+      return "Could not parse object to JSON";
+    }
+  }
+
   /**
    * Chat request to OpenAI with an image
    *
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index db518e4d..2856b163 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -8,9 +8,9 @@
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.sap.ai.sdk.core.Core;
 import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionDelta;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiDeltaChatCompletion;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
@@ -116,13 +116,13 @@ public OpenAiChatCompletionOutput chatCompletion(
    * @throws OpenAiClientException if the request fails
    */
   @Nonnull
-  public OpenAiStream<OpenAiChatCompletionDelta, OpenAiChatCompletionOutput> stream(
+  public OpenAiStream<OpenAiDeltaChatCompletion, OpenAiChatCompletionOutput> stream(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
     parameters.setStream(true);
     return stream(
         "/chat/completions",
         parameters,
-        OpenAiChatCompletionDelta.class,
+        OpenAiDeltaChatCompletion.class,
         OpenAiChatCompletionOutput.class);
   }
 
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
index b049fec0..8159bd9c 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
@@ -9,5 +9,10 @@
  */
 public interface DeltaAggregatable<D> {
 
+  /**
+   * Add a streamed delta to the total output.
+   *
+   * @param delta the delta to add.
+   */
   void addDelta(D delta);
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java
index 970a98e3..2c1f125a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java
@@ -13,11 +13,22 @@
 /** Result candidates for OpenAI chat completion output. */
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
-@ToString
+@ToString(callSuper = true)
 public class OpenAiChatCompletionChoice extends OpenAiCompletionChoice {
   /** Completion chat message. */
   @JsonProperty("message")
   @Getter(onMethod_ = @Nonnull)
   @Setter(onMethod_ = @Nonnull, value = AccessLevel.PACKAGE)
   private OpenAiChatAssistantMessage message;
+
+  void addDelta(OpenAiDeltaChatCompletionChoice delta) {
+    super.addDelta(delta);
+
+    if (delta.getMessage() != null) {
+      if (message == null) {
+        message = new OpenAiChatAssistantMessage();
+      }
+      message.addDelta(delta.getMessage());
+    }
+  }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionFunction.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionFunction.java
index b3ff6cd2..c8ea342b 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionFunction.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionFunction.java
@@ -11,7 +11,7 @@
 /** OpenAI function signature. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiChatCompletionFunction {
   /**
    * Name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes,
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
index f6da1fc9..a43bf802 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
@@ -1,7 +1,7 @@
 package com.sap.ai.sdk.foundationmodels.openai.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatAssistantMessage;
+import java.util.ArrayList;
 import java.util.List;
 import javax.annotation.Nonnull;
 import lombok.EqualsAndHashCode;
@@ -12,9 +12,9 @@
 /** OpenAI chat completion output. */
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
-@ToString
+@ToString(callSuper = true)
 public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
-    implements DeltaAggregatable<OpenAiChatCompletionDelta> {
+    implements DeltaAggregatable<OpenAiDeltaChatCompletion> {
   /** List of result candidates. */
   @JsonProperty("choices")
   @Getter(onMethod_ = @Nonnull)
@@ -28,32 +28,23 @@ public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
   @Getter(onMethod_ = @Nonnull)
   private String systemFingerprint;
 
-  public void addDelta(OpenAiChatCompletionDelta delta) {
-    // TODO: Assign every field if not null on all parent and children classes.
-    //       Right now we only assign content message.
+  void addDelta(OpenAiDeltaChatCompletion delta) {
 
-    if (delta.getChoices().isEmpty()
-        // Multiple choices are spread out on multiple deltas
-        // A delta only contains one choice with a variable index
-        || delta.getChoices().get(0).getIndex() != 0) {
-      return;
+    if (delta.getSystemFingerprint() != null) {
+      systemFingerprint = delta.getSystemFingerprint();
     }
-    var deltaMessage = delta.getChoices().get(0).getMessage();
-    if (deltaMessage == null) {
-      return;
-    }
-    String deltaContent = deltaMessage.getContent();
-    if (deltaContent == null) {
-      return;
-    }
-    if (choices == null || choices.isEmpty()) {
-      var choice =
-          new OpenAiChatCompletionChoice()
-              .setMessage(new OpenAiChatAssistantMessage().setContent(deltaContent));
-      choices = List.of(choice);
-      return;
+
+    if (!delta.getChoices().isEmpty()) {
+      if (choices == null) {
+        choices = new ArrayList<>();
+      }
+      // Multiple choices are spread out on multiple deltas
+      // A delta only contains one choice with a variable index
+      int index = delta.getChoices().get(0).getIndex();
+      for (int i = choices.size(); i < index + 1; i++) {
+        choices.add(new OpenAiChatCompletionChoice());
+      }
+      choices.get(index).addDelta(delta.getChoices().get(0));
     }
-    var message = choices.get(0).getMessage();
-    message.setContent(message.getContent() + deltaContent);
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
index 8dccfe55..6f28134a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
@@ -21,7 +21,7 @@
 /** OpenAI chat completion input parameters. */
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
-@ToString
+@ToString(callSuper = true)
 public class OpenAiChatCompletionParameters extends OpenAiCompletionParameters {
   /** A list of messages comprising the conversation so far. */
   @JsonProperty("messages")
@@ -177,7 +177,7 @@ private enum ToolChoiceType implements ToolChoice {
   }
 
   @EqualsAndHashCode
-  @ToString
+  @ToString(callSuper = true)
   private static class FunctionToolChoice implements ToolChoice {
     @JsonProperty("function")
     @Setter(onParam_ = @Nonnull)
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionTool.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionTool.java
index 6f6cd8ec..91de5424 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionTool.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionTool.java
@@ -12,7 +12,7 @@
 /** OpenAI tool signature. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiChatCompletionTool {
   /** Specifies a tool the model should use. Use to force the model to call a specific function. */
   @JsonProperty("type")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatFunctionCall.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatFunctionCall.java
index 7a59c85c..b228fa4c 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatFunctionCall.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatFunctionCall.java
@@ -11,7 +11,7 @@
 /** The name of the function to call, or, the function that the model called. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiChatFunctionCall {
   /** Name of the function call. */
   @JsonProperty("name")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
index e090990a..2cb6028b 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
@@ -59,7 +59,7 @@ public interface OpenAiChatMessage {
   /** OpenAI system message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString
+  @ToString(callSuper = true)
   class OpenAiChatSystemMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
@@ -75,7 +75,7 @@ class OpenAiChatSystemMessage implements OpenAiChatMessage {
   /** OpenAI user message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString
+  @ToString(callSuper = true)
   class OpenAiChatUserMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
@@ -235,7 +235,7 @@ public enum ImageDetailLevel {
   /** OpenAI assistant message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString
+  @ToString(callSuper = true)
   class OpenAiChatAssistantMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
@@ -245,7 +245,7 @@ class OpenAiChatAssistantMessage implements OpenAiChatMessage {
     @JsonProperty("content")
     @Getter(onMethod_ = @Nullable)
     @Setter(onParam_ = @Nullable, value = AccessLevel.PACKAGE)
-    private String content; // must be String or null
+    private String content;
 
     /** The tool calls generated by the model, such as function calls. */
     @JsonProperty("tool_calls")
@@ -254,12 +254,30 @@ class OpenAiChatAssistantMessage implements OpenAiChatMessage {
 
     // TODO: add context
     // https://github.com/Azure/azure-rest-api-specs/blob/07d286359f828bbc7901e86288a5d62b48ae2052/specification/cognitiveservices/data-plane/AzureOpenAI/inference/stable/2024-02-01/inference.json#L1599
+
+    void addDelta(OpenAiChatAssistantMessage delta) {
+
+      if (delta.getContent() != null) {
+        if (content == null) {
+          content = "";
+        }
+        content += delta.getContent();
+      }
+
+      if (delta.getTool_calls() != null) {
+        if (tool_calls == null) {
+          tool_calls = new ArrayList<>();
+        }
+        // TODO: camel case
+        tool_calls.addAll(delta.getTool_calls());
+      }
+    }
   }
 
   /** OpenAI tool message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString
+  @ToString(callSuper = true)
   class OpenAiChatToolMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
@@ -280,7 +298,7 @@ class OpenAiChatToolMessage implements OpenAiChatMessage {
   /** OpenAI function message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString
+  @ToString(callSuper = true)
   class OpenAiChatFunctionMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatToolCall.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatToolCall.java
index 65a5d0f9..624a98d1 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatToolCall.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatToolCall.java
@@ -10,7 +10,7 @@
 /** OpenAI tool call by AI. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiChatToolCall {
   /** The ID of the tool call. */
   @JsonProperty("id")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
index 3c43e732..e0ca972e 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
@@ -10,7 +10,7 @@
 /** Result for OpenAI chat completion output. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiCompletionChoice {
   /** Reason for finish. */
   @JsonProperty("finish_reason")
@@ -29,4 +29,22 @@ public class OpenAiCompletionChoice {
   @JsonProperty("content_filter_results")
   @Getter(onMethod_ = @Nullable)
   private OpenAiContentFilterPromptResults contentFilterResults;
+
+  void addDelta(OpenAiCompletionChoice delta) {
+
+    if (delta.getFinishReason() != null) {
+      finishReason = delta.getFinishReason();
+    }
+
+    if (delta.getIndex() != null) {
+      index = delta.getIndex();
+    }
+
+    if (delta.getContentFilterResults() != null) {
+      if (contentFilterResults == null) {
+        contentFilterResults = new OpenAiContentFilterPromptResults();
+      }
+      contentFilterResults.addDelta(delta.getContentFilterResults());
+    }
+  }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
index ee70e23a..bfc372f6 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
@@ -12,7 +12,7 @@
 /** OpenAI completion output . */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiCompletionOutput {
   /** Creation date Unix timestamp. */
   @JsonProperty("created")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
index 3e989fca..b690bcff 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
@@ -14,7 +14,7 @@
 /** OpenAI completion input parameters. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiCompletionParameters {
   /**
    * The maximum number of [tokens](/tokenizer) that can be generated in the completion. The token
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterDetectedResult.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterDetectedResult.java
index 16bee568..c8a0b1d5 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterDetectedResult.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterDetectedResult.java
@@ -10,7 +10,7 @@
 /** OpenAI content filter detected result. */
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
-@ToString
+@ToString(callSuper = true)
 public class OpenAiContentFilterDetectedResult extends OpenAiContentFilterResultBase {
   /** Whether the content was detected. */
   @JsonProperty("detected")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterPromptResults.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterPromptResults.java
index a7c07dc7..57e5a79a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterPromptResults.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterPromptResults.java
@@ -10,9 +10,17 @@
 /** Content filtering results for a prompt in the request. */
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
-@ToString
+@ToString(callSuper = true)
 public class OpenAiContentFilterPromptResults extends OpenAiContentFilterResultsBase {
   @JsonProperty("jailbreak")
   @Getter(onMethod_ = @Nullable)
   private OpenAiContentFilterDetectedResult jailbreak;
+
+  void addDelta(OpenAiContentFilterPromptResults delta) {
+    super.addDelta(delta);
+
+    if (delta.getJailbreak() != null) {
+      jailbreak = delta.getJailbreak();
+    }
+  }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultBase.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultBase.java
index 167627b6..a7c1033e 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultBase.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultBase.java
@@ -10,7 +10,7 @@
 /** OpenAI content filter result. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiContentFilterResultBase {
   /** Whether the content was filtered. */
   @JsonProperty("filtered")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java
index f2cab6fd..99b1e70a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java
@@ -10,7 +10,7 @@
 /** Information about the content filtering results. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiContentFilterResultsBase {
   /** Sexual content filter result. */
   @JsonProperty("sexual")
@@ -40,4 +40,29 @@ public class OpenAiContentFilterResultsBase {
   @JsonProperty("error")
   @Getter(onMethod_ = @Nullable)
   private OpenAiErrorBase error;
+
+  void addDelta(OpenAiContentFilterPromptResults delta) {
+    if (delta.getSexual() != null) {
+      sexual = delta.getSexual();
+      System.out.println(sexual.getSeverity());
+      System.out.println(sexual.isFiltered());
+    } else {
+      System.out.println("Sexual is null");
+    }
+    if (delta.getViolence() != null) {
+      violence = delta.getViolence();
+    }
+    if (delta.getHate() != null) {
+      hate = delta.getHate();
+    }
+    if (delta.getSelfHarm() != null) {
+      selfHarm = delta.getSelfHarm();
+    }
+    if (delta.getProfanity() != null) {
+      profanity = delta.getProfanity();
+    }
+    if (delta.getError() != null) {
+      error = delta.getError();
+    }
+  }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterSeverityResult.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterSeverityResult.java
index 0d008e29..fd158aad 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterSeverityResult.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterSeverityResult.java
@@ -12,7 +12,7 @@
 /** Information about the content filtering results. */
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
-@ToString
+@ToString(callSuper = true)
 public class OpenAiContentFilterSeverityResult extends OpenAiContentFilterResultBase {
   /** Severity of the content. */
   @JsonProperty("severity")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
similarity index 80%
rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
index 7704a16a..6fce1768 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
@@ -12,19 +12,19 @@
 /** OpenAI chat completion output delta for streaming. */
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
-@ToString
-public class OpenAiChatCompletionDelta extends OpenAiCompletionOutput implements StreamedDelta {
+@ToString(callSuper = true)
+public class OpenAiDeltaChatCompletion extends OpenAiCompletionOutput implements StreamedDelta {
   /** List of result candidates. */
   @JsonProperty("choices")
   @Getter(onMethod_ = @Nonnull)
-  private List<OpenAiChatCompletionChoice> choices;
+  private List<OpenAiDeltaChatCompletionChoice> choices;
 
   /**
    * Can be used in conjunction with the seed request parameter to understand when backend changes
    * have been made that might impact determinism.
    */
   @JsonProperty("system_fingerprint")
-  @Getter(onMethod_ = @Nonnull)
+  @Getter(onMethod_ = @Nullable)
   private String systemFingerprint;
 
   /**
@@ -40,9 +40,7 @@ public String getDeltaContent() {
         // A delta only contains one choice with a variable index
         && getChoices().get(0).getIndex() == 0
         // Avoid the second delta: "choices":[{"delta":{"content":"","role":"assistant"}}]
-        && getChoices().get(0).getMessage() != null
-        // Avoid the last delta "choices":[{"delta":{}}]
-        && getChoices().get(0).getMessage().getContent() != null) {
+        && getChoices().get(0).getMessage() != null) {
 
       return getChoices().get(0).getMessage().getContent();
     }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletionChoice.java
new file mode 100644
index 00000000..af6cdb58
--- /dev/null
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletionChoice.java
@@ -0,0 +1,23 @@
+package com.sap.ai.sdk.foundationmodels.openai.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatAssistantMessage;
+import javax.annotation.Nullable;
+import lombok.AccessLevel;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+
+/** Result candidates for OpenAI chat completion output. */
+@Accessors(chain = true)
+@EqualsAndHashCode(callSuper = true)
+@ToString(callSuper = true)
+public class OpenAiDeltaChatCompletionChoice extends OpenAiCompletionChoice {
+  /** Completion chat message. */
+  @JsonProperty("message")
+  @Getter(onMethod_ = @Nullable)
+  @Setter(onMethod_ = @Nullable, value = AccessLevel.PACKAGE)
+  private OpenAiChatAssistantMessage message;
+}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingData.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingData.java
index 181e3b22..86f3be83 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingData.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingData.java
@@ -11,7 +11,7 @@
 /** Result candidates for OpenAI embedding output. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiEmbeddingData {
   /** Embedding object. */
   @JsonProperty("object")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingOutput.java
index ba93fcf6..d28974dd 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingOutput.java
@@ -11,7 +11,7 @@
 /** OpenAI embedding output. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiEmbeddingOutput {
   /** List object. */
   @JsonProperty("object")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingParameters.java
index a932674f..6026483d 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingParameters.java
@@ -14,7 +14,7 @@
 /** OpenAI embedding input parameters. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiEmbeddingParameters {
   /**
    * Input text to get embeddings for, encoded as a string. The number of input tokens varies
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiError.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiError.java
index 1d373b63..fbfbb21b 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiError.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiError.java
@@ -10,7 +10,7 @@
 /** OpenAI error. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiError {
   /** The error object. */
   @JsonProperty("error")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiErrorBase.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiErrorBase.java
index d71ecffc..2967d586 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiErrorBase.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiErrorBase.java
@@ -10,7 +10,7 @@
 /** OpenAI error. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiErrorBase {
   /** The error code. */
   @JsonProperty("code")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiPromptFilterResult.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiPromptFilterResult.java
index 85ddfb11..22f0e7ec 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiPromptFilterResult.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiPromptFilterResult.java
@@ -10,7 +10,7 @@
 /** Content filtering results for a single prompt in the request. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiPromptFilterResult {
   /** Index of the prompt in the request. */
   @JsonProperty("prompt_index")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
index 132f3071..d7192d4a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
@@ -11,7 +11,7 @@
 /** Usage statistics for the completion request. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString
+@ToString(callSuper = true)
 public class OpenAiUsage {
   /** Tokens consumed for output text completion. */
   @JsonProperty("completion_tokens")

From 42ae946ae2a957a6979257ef33bafe25fc6d19b1 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 23 Aug 2024 16:10:42 +0200
Subject: [PATCH 16/55] addDelta is propagated everywhere

---
 .../openai/model/OpenAiChatCompletionOutput.java           | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
index a43bf802..86861d62 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
@@ -28,7 +28,12 @@ public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
   @Getter(onMethod_ = @Nonnull)
   private String systemFingerprint;
 
-  void addDelta(OpenAiDeltaChatCompletion delta) {
+  /**
+   * Add a streamed delta to the total output.
+   *
+   * @param delta the delta to add.
+   */
+  public void addDelta(OpenAiDeltaChatCompletion delta) {
 
     if (delta.getSystemFingerprint() != null) {
       systemFingerprint = delta.getSystemFingerprint();

From e6e009acb2575f537733ae74501876b988868d8b Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 23 Aug 2024 16:20:47 +0200
Subject: [PATCH 17/55] forgotten addDeltas

---
 .../model/OpenAiChatCompletionOutput.java     |  1 +
 .../openai/model/OpenAiCompletionOutput.java  | 21 +++++++++++++++++++
 .../openai/model/OpenAiUsage.java             |  8 +++++++
 3 files changed, 30 insertions(+)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
index 86861d62..165f3434 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
@@ -34,6 +34,7 @@ public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
    * @param delta the delta to add.
    */
   public void addDelta(OpenAiDeltaChatCompletion delta) {
+    super.addDelta(delta);
 
     if (delta.getSystemFingerprint() != null) {
       systemFingerprint = delta.getSystemFingerprint();
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
index bfc372f6..5cca1543 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
@@ -43,4 +43,25 @@ public class OpenAiCompletionOutput {
   @JsonProperty("prompt_filter_results")
   @Getter(onMethod_ = @Nullable)
   private List<OpenAiPromptFilterResult> promptFilterResults;
+
+  void addDelta(OpenAiDeltaChatCompletion delta) {
+    created = delta.getCreated();
+    id = delta.getId();
+    model = delta.getModel();
+    object = delta.getObject();
+
+    if (delta.getUsage() != null) {
+      if (usage == null) {
+        usage = new OpenAiUsage();
+      }
+      usage.addDelta(delta.getUsage());
+    }
+
+    if (delta.getPromptFilterResults() != null) {
+      if (promptFilterResults == null) {
+        promptFilterResults = delta.getPromptFilterResults();
+      }
+      // prompt_filter_results is only present once in the first delta
+    }
+  }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
index d7192d4a..65bf22eb 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
@@ -27,4 +27,12 @@ public class OpenAiUsage {
   @JsonProperty("total_tokens")
   @Getter(onMethod_ = @Nonnull)
   private Integer totalTokens;
+
+  void addDelta(OpenAiUsage delta) {
+    if (delta.getCompletionTokens() != null) {
+      completionTokens = delta.getCompletionTokens();
+    }
+    promptTokens = delta.getPromptTokens();
+    totalTokens = delta.getTotalTokens();
+  }
 }

From bee8fdccd784432e141c382534e1c96f6c74283a Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 23 Aug 2024 16:21:53 +0200
Subject: [PATCH 18/55] Added jackson dependencies

---
 e2e-test-app/pom.xml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/e2e-test-app/pom.xml b/e2e-test-app/pom.xml
index a5c9add3..3af0f62c 100644
--- a/e2e-test-app/pom.xml
+++ b/e2e-test-app/pom.xml
@@ -107,6 +107,14 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+    </dependency>
     <!-- scope "runtime" -->
     <dependency>
       <groupId>ch.qos.logback</groupId>

From 5f03c6fb2c617a466644b3870292061f4079d7e6 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 23 Aug 2024 16:27:33 +0200
Subject: [PATCH 19/55] Added Javadoc

---
 .../sap/ai/sdk/foundationmodels/openai/OpenAiStream.java   | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
index 124bbe8d..a35c56ff 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
@@ -11,7 +11,12 @@
 import lombok.Setter;
 import lombok.experimental.Accessors;
 
-/** Generic OpenAI stream output. */
+/**
+ * Generic OpenAI stream output.
+ *
+ * @param <D> the type of the streamed delta
+ * @param <T> the type of the total output
+ */
 @RequiredArgsConstructor(access = AccessLevel.NONE)
 @NoArgsConstructor
 @Setter // TODO: package private

From e79ca8e49029fe771c983001ee433fcf169eb1d4 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 23 Aug 2024 16:29:15 +0200
Subject: [PATCH 20/55] Removed 1 TODO

---
 .../com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
index a35c56ff..1b43f4c2 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
@@ -19,7 +19,7 @@
  */
 @RequiredArgsConstructor(access = AccessLevel.NONE)
 @NoArgsConstructor
-@Setter // TODO: package private
+@Setter(AccessLevel.PACKAGE)
 @Accessors(chain = true)
 // D extends StreamedDelta but Java generics, oddly enough, use extends for interfaces
 public class OpenAiStream<D extends StreamedDelta, T extends DeltaAggregatable<D>>

From ba2c5e069d4c51651234b9c0a46a2ea7602f3911 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 27 Aug 2024 10:07:01 +0200
Subject: [PATCH 21/55] PMD

---
 .../sdk/app/controllers/OpenAiController.java | 36 ++++++++++---------
 .../openai/OpenAiResponseHandler.java         |  3 +-
 .../foundationmodels/openai/OpenAiStream.java | 11 ++++--
 .../openai/OpenAiStreamingHandler.java        | 14 ++++----
 .../openai/model/DeltaAggregatable.java       |  4 ++-
 .../model/OpenAiChatCompletionChoice.java     |  2 +-
 .../model/OpenAiChatCompletionOutput.java     |  4 +--
 .../openai/model/OpenAiCompletionChoice.java  |  3 +-
 .../openai/model/OpenAiCompletionOutput.java  | 11 +++---
 .../OpenAiContentFilterPromptResults.java     |  3 +-
 .../model/OpenAiContentFilterResultsBase.java |  7 ++--
 .../openai/model/OpenAiUsage.java             |  2 +-
 12 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index cb97d700..88e82cd0 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -66,42 +66,44 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                         .addText(
                             "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-    var emitter = new ResponseBodyEmitter();
-    var stream = OpenAiClient.forModel(GPT_35_TURBO).stream(request);
+    final var emitter = new ResponseBodyEmitter();
 
     // Cloud SDK's ThreadContext is vital for the request to successfully execute.
     ThreadContextExecutors.getExecutor()
         .submit(
             () -> {
-              stream
-                  .getDeltaStream()
-                  .map(OpenAiDeltaChatCompletion::getDeltaContent)
-                  // The first two and the last delta do not contain any message content
-                  .filter(Objects::nonNull)
-                  .forEach(content -> send(emitter, content));
-
-              String indentedJson = objectToJson(stream.getTotalOutput());
-              send(emitter, "\n\n-----Total Output-----\n\n" + indentedJson);
-              emitter.complete();
-              stream.close();
+              // try-with-resources ensures that the stream is closed after the response is sent.
+              try (final var result = OpenAiClient.forModel(GPT_35_TURBO).stream(request)) {
+                result
+                    .getDeltaStream()
+                    .map(OpenAiDeltaChatCompletion::getDeltaContent)
+                    // The first two and the last deltaStream do not contain any message content
+                    .filter(Objects::nonNull)
+                    .forEach(content -> send(emitter, content));
+
+                final String indentedJson = objectToJson(result.getTotalOutput());
+                send(emitter, "\n\n-----Total Output-----\n\n" + indentedJson);
+                emitter.complete();
+              }
             });
     // MediaType.TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
-  private static void send(ResponseBodyEmitter emitter, String chunk) {
+  private static void send(
+      @Nonnull final ResponseBodyEmitter emitter, @Nonnull final String chunk) {
     try {
       emitter.send(chunk);
-    } catch (IOException e) {
+    } catch (final IOException e) {
       log.error(Arrays.toString(e.getStackTrace()));
       emitter.completeWithError(e);
     }
   }
 
-  private static String objectToJson(Object obj) {
+  private static String objectToJson(@Nonnull final Object obj) {
     try {
       return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(obj);
-    } catch (JsonProcessingException ignored) {
+    } catch (final JsonProcessingException ignored) {
       return "Could not parse object to JSON";
     }
   }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
index 6ceb5859..0946c405 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
@@ -103,7 +103,8 @@ static void buildExceptionAndThrow(@Nonnull final ClassicHttpResponse response)
    * @param errorResponse the error response, most likely a JSON of {@link OpenAiError}.
    * @param baseException a base exception to add the error message to.
    */
-  static void parseErrorAndThrow(String errorResponse, OpenAiClientException baseException)
+  static void parseErrorAndThrow(
+      @Nonnull final String errorResponse, @Nonnull final OpenAiClientException baseException)
       throws OpenAiClientException {
     final var maybeError = Try.of(() -> JACKSON.readValue(errorResponse, OpenAiError.class));
     if (maybeError.isFailure()) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
index 1b43f4c2..aa684f14 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
@@ -25,14 +25,21 @@
 public class OpenAiStream<D extends StreamedDelta, T extends DeltaAggregatable<D>>
     implements AutoCloseable {
 
-  @Getter @Nonnull private Stream<D> deltaStream;
+  @Getter(onMethod_ = @Nonnull)
+  private Stream<D> deltaStream;
+
   @Nonnull private T totalOutput;
 
   void addDelta(D delta) {
     totalOutput.addDelta(delta);
   }
 
-  /** Get the total aggregated output. */
+  /**
+   * Get the total aggregated output from all deltas. Closes the delta stream.
+   *
+   * @return the total output until now.
+   */
+  @Nonnull
   public T getTotalOutput() {
     deltaStream.close();
     return totalOutput;
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 117401a3..3b227f5a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -47,13 +47,15 @@ public OpenAiStream<D, T> handleResponse(@Nonnull final ClassicHttpResponse resp
    * @return A {@link OpenAiStream} of a model class instantiated from the response
    * @author stippi
    */
+  // The stream is closed by the user of the OpenAiStream
+  @SuppressWarnings("PMD.CloseResource")
   private OpenAiStream<D, T> parseResponse(@Nonnull final ClassicHttpResponse response)
       throws OpenAiClientException {
     final HttpEntity responseEntity = response.getEntity();
     if (responseEntity == null) {
       throw new OpenAiClientException("Response from OpenAI model was empty.");
     }
-    InputStream inputStream;
+    final InputStream inputStream;
     try {
       inputStream = responseEntity.getContent();
     } catch (IOException e) {
@@ -61,7 +63,7 @@ private OpenAiStream<D, T> parseResponse(@Nonnull final ClassicHttpResponse resp
     }
     final var br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
 
-    OpenAiStream<D, T> output = new OpenAiStream<D, T>();
+    final OpenAiStream<D, T> output = new OpenAiStream<>();
     try {
       output.setTotalOutput(totalType.getDeclaredConstructor().newInstance());
     } catch (InstantiationException
@@ -72,7 +74,7 @@ private OpenAiStream<D, T> parseResponse(@Nonnull final ClassicHttpResponse resp
     }
 
     // https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
-    Stream<D> deltaStream =
+    final Stream<D> deltaStream =
         br.lines()
             .filter(
                 responseLine ->
@@ -88,11 +90,11 @@ private OpenAiStream<D, T> parseResponse(@Nonnull final ClassicHttpResponse resp
                 responseLine -> {
                   String data = responseLine.substring(5).replace("delta", "message");
                   try {
-                    D delta = JACKSON.readValue(data, deltaType);
+                    final D delta = JACKSON.readValue(data, deltaType);
                     output.addDelta(delta);
                     return delta;
-                  } catch (IOException e) {
-                    throw new RuntimeException(e);
+                  } catch (final IOException e) {
+                    throw new OpenAiClientException("Failed to parse delta message: " + data, e);
                   }
                 });
     return output.setDeltaStream(deltaStream);
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
index 8159bd9c..58ac00ea 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/DeltaAggregatable.java
@@ -1,5 +1,7 @@
 package com.sap.ai.sdk.foundationmodels.openai.model;
 
+import javax.annotation.Nonnull;
+
 /**
  * Interface for model classes that can be created from aggregated streamed deltas.
  *
@@ -14,5 +16,5 @@ public interface DeltaAggregatable<D> {
    *
    * @param delta the delta to add.
    */
-  void addDelta(D delta);
+  void addDelta(@Nonnull final D delta);
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java
index 2c1f125a..91a6ff65 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionChoice.java
@@ -21,7 +21,7 @@ public class OpenAiChatCompletionChoice extends OpenAiCompletionChoice {
   @Setter(onMethod_ = @Nonnull, value = AccessLevel.PACKAGE)
   private OpenAiChatAssistantMessage message;
 
-  void addDelta(OpenAiDeltaChatCompletionChoice delta) {
+  void addDelta(@Nonnull final OpenAiDeltaChatCompletionChoice delta) {
     super.addDelta(delta);
 
     if (delta.getMessage() != null) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
index 165f3434..2469d0cf 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
@@ -33,7 +33,7 @@ public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
    *
    * @param delta the delta to add.
    */
-  public void addDelta(OpenAiDeltaChatCompletion delta) {
+  public void addDelta(@Nonnull final OpenAiDeltaChatCompletion delta) {
     super.addDelta(delta);
 
     if (delta.getSystemFingerprint() != null) {
@@ -46,7 +46,7 @@ public void addDelta(OpenAiDeltaChatCompletion delta) {
       }
       // Multiple choices are spread out on multiple deltas
       // A delta only contains one choice with a variable index
-      int index = delta.getChoices().get(0).getIndex();
+      final int index = delta.getChoices().get(0).getIndex();
       for (int i = choices.size(); i < index + 1; i++) {
         choices.add(new OpenAiChatCompletionChoice());
       }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
index e0ca972e..2b2b4f3b 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
@@ -1,6 +1,7 @@
 package com.sap.ai.sdk.foundationmodels.openai.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -30,7 +31,7 @@ public class OpenAiCompletionChoice {
   @Getter(onMethod_ = @Nullable)
   private OpenAiContentFilterPromptResults contentFilterResults;
 
-  void addDelta(OpenAiCompletionChoice delta) {
+  void addDelta(@Nonnull final OpenAiCompletionChoice delta) {
 
     if (delta.getFinishReason() != null) {
       finishReason = delta.getFinishReason();
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
index 5cca1543..35e401d9 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
@@ -44,7 +44,7 @@ public class OpenAiCompletionOutput {
   @Getter(onMethod_ = @Nullable)
   private List<OpenAiPromptFilterResult> promptFilterResults;
 
-  void addDelta(OpenAiDeltaChatCompletion delta) {
+  void addDelta(@Nonnull final OpenAiDeltaChatCompletion delta) {
     created = delta.getCreated();
     id = delta.getId();
     model = delta.getModel();
@@ -57,11 +57,10 @@ void addDelta(OpenAiDeltaChatCompletion delta) {
       usage.addDelta(delta.getUsage());
     }
 
-    if (delta.getPromptFilterResults() != null) {
-      if (promptFilterResults == null) {
-        promptFilterResults = delta.getPromptFilterResults();
-      }
-      // prompt_filter_results is only present once in the first delta
+    if (delta.getPromptFilterResults() != null && promptFilterResults == null) {
+      promptFilterResults = delta.getPromptFilterResults();
+      // prompt_filter_results is overriden instead of updated because it is only present once in
+      // the first delta
     }
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterPromptResults.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterPromptResults.java
index 57e5a79a..342914cb 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterPromptResults.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterPromptResults.java
@@ -1,6 +1,7 @@
 package com.sap.ai.sdk.foundationmodels.openai.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -16,7 +17,7 @@ public class OpenAiContentFilterPromptResults extends OpenAiContentFilterResults
   @Getter(onMethod_ = @Nullable)
   private OpenAiContentFilterDetectedResult jailbreak;
 
-  void addDelta(OpenAiContentFilterPromptResults delta) {
+  void addDelta(@Nonnull final OpenAiContentFilterPromptResults delta) {
     super.addDelta(delta);
 
     if (delta.getJailbreak() != null) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java
index 99b1e70a..2c454db3 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java
@@ -1,6 +1,7 @@
 package com.sap.ai.sdk.foundationmodels.openai.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -41,13 +42,9 @@ public class OpenAiContentFilterResultsBase {
   @Getter(onMethod_ = @Nullable)
   private OpenAiErrorBase error;
 
-  void addDelta(OpenAiContentFilterPromptResults delta) {
+  void addDelta(@Nonnull final OpenAiContentFilterPromptResults delta) {
     if (delta.getSexual() != null) {
       sexual = delta.getSexual();
-      System.out.println(sexual.getSeverity());
-      System.out.println(sexual.isFiltered());
-    } else {
-      System.out.println("Sexual is null");
     }
     if (delta.getViolence() != null) {
       violence = delta.getViolence();
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
index 65bf22eb..7691caed 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
@@ -28,7 +28,7 @@ public class OpenAiUsage {
   @Getter(onMethod_ = @Nonnull)
   private Integer totalTokens;
 
-  void addDelta(OpenAiUsage delta) {
+  void addDelta(@Nonnull final OpenAiUsage delta) {
     if (delta.getCompletionTokens() != null) {
       completionTokens = delta.getCompletionTokens();
     }

From c10eecb77789f212c7f578356ad75c12983d8529 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 27 Aug 2024 10:39:05 +0200
Subject: [PATCH 22/55] PMD again

---
 .../java/com/sap/ai/sdk/app/controllers/OpenAiController.java   | 2 +-
 .../com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java    | 2 +-
 .../ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java  | 2 +-
 .../ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 88e82cd0..b4d6c4c6 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -73,7 +73,7 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
         .submit(
             () -> {
               // try-with-resources ensures that the stream is closed after the response is sent.
-              try (final var result = OpenAiClient.forModel(GPT_35_TURBO).stream(request)) {
+              try (var result = OpenAiClient.forModel(GPT_35_TURBO).stream(request)) {
                 result
                     .getDeltaStream()
                     .map(OpenAiDeltaChatCompletion::getDeltaContent)
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
index aa684f14..c4d1cadf 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
@@ -30,7 +30,7 @@ public class OpenAiStream<D extends StreamedDelta, T extends DeltaAggregatable<D
 
   @Nonnull private T totalOutput;
 
-  void addDelta(D delta) {
+  void addDelta(@Nonnull final D delta) {
     totalOutput.addDelta(delta);
   }
 
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 3b227f5a..024768b9 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -88,7 +88,7 @@ private OpenAiStream<D, T> parseResponse(@Nonnull final ClassicHttpResponse resp
                 })
             .map(
                 responseLine -> {
-                  String data = responseLine.substring(5).replace("delta", "message");
+                  final String data = responseLine.substring(5).replace("delta", "message");
                   try {
                     final D delta = JACKSON.readValue(data, deltaType);
                     output.addDelta(delta);
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
index 2cb6028b..2289d72a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
@@ -255,7 +255,7 @@ class OpenAiChatAssistantMessage implements OpenAiChatMessage {
     // TODO: add context
     // https://github.com/Azure/azure-rest-api-specs/blob/07d286359f828bbc7901e86288a5d62b48ae2052/specification/cognitiveservices/data-plane/AzureOpenAI/inference/stable/2024-02-01/inference.json#L1599
 
-    void addDelta(OpenAiChatAssistantMessage delta) {
+    void addDelta(@Nonnull final OpenAiChatAssistantMessage delta) {
 
       if (delta.getContent() != null) {
         if (content == null) {

From 0e1a1679a8693c123f942ca7748f930f1bac64cc Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Wed, 28 Aug 2024 09:15:25 +0200
Subject: [PATCH 23/55] Added OpenAiClientTest.streamChatCompletion()

---
 .../openai/OpenAiClientTest.java              | 371 ++++++++++++------
 .../resources/chatCompletionResponse.json     |   4 +-
 .../test/resources/streamChatCompletion.txt   |   7 +
 3 files changed, 264 insertions(+), 118 deletions(-)
 create mode 100644 foundation-models/openai/src/test/resources/streamChatCompletion.txt

diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 35beb7b9..13616431 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -9,12 +9,13 @@
 import com.github.tomakehurst.wiremock.junit5.WireMockTest;
 import com.github.tomakehurst.wiremock.stubbing.Scenario;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionChoice;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiContentFilterPromptResults;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiDeltaChatCompletion;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiPromptFilterResult;
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import io.vavr.control.Try;
 import java.io.IOException;
@@ -35,7 +36,7 @@ void setup(WireMockRuntimeInfo server) {
   }
 
   @Test
-  void testApiVersion() {
+  void apiVersion() {
     stubFor(post(anyUrl()).willReturn(okJson("{}")));
     Try.of(() -> client.chatCompletion(new OpenAiChatCompletionParameters()));
 
@@ -54,7 +55,7 @@ void testApiVersion() {
   }
 
   @Test
-  void testErrorHandling() {
+  void errorHandling() {
     final var errorJson =
         """
             { "error": { "code": null, "message": "foo", "type": "invalid stuff" } }
@@ -128,122 +129,260 @@ void testErrorHandling() {
   }
 
   @Test
-  void testChatCompletion() throws IOException {
-    final String response =
-        new String(
-            getClass()
-                .getClassLoader()
-                .getResourceAsStream("chatCompletionResponse.json")
-                .readAllBytes());
-    stubFor(post(anyUrl()).willReturn(okJson(response)));
-
-    final var systemMessage =
-        new OpenAiChatMessage.OpenAiChatSystemMessage()
-            .setContent("You are a helpful, friendly and sometimes slightly snarky AI assistant!");
-    final var userMessage =
-        new OpenAiChatUserMessage().addText("Hello World! Why is this phrase so famous?");
-    final var request =
-        new OpenAiChatCompletionParameters().setMessages(List.of(systemMessage, userMessage));
-
-    final var result = client.chatCompletion(request);
-
-    assertThat(result).isNotNull();
-    assertThat(result.getCreated()).isEqualTo(1719300073);
-    assertThat(result.getId()).isEqualTo("chatcmpl-9dumHtDEyysGFnknk17n4Lt37tg7T");
-    assertThat(result.getModel()).isEqualTo("gpt-4-32k");
-    assertThat(result.getObject()).isEqualTo("chat.completion");
-    assertThat(result.getSystemFingerprint()).isNull();
-
-    assertThat(result.getUsage().getCompletionTokens()).isEqualTo(54);
-    assertThat(result.getUsage().getPromptTokens()).isEqualTo(13);
-    assertThat(result.getUsage().getTotalTokens()).isEqualTo(67);
-
-    assertThat(result.getPromptFilterResults()).hasSize(1);
-    OpenAiPromptFilterResult promptFilterResults = result.getPromptFilterResults().get(0);
-    assertThat(promptFilterResults.getPromptIndex()).isEqualTo(0);
-    assertThat(promptFilterResults.getContentFilterResults().getSexual().isFiltered()).isFalse();
-    assertThat(promptFilterResults.getContentFilterResults().getSexual().getSeverity())
-        .isEqualTo(SAFE);
-    assertThat(promptFilterResults.getContentFilterResults().getViolence().isFiltered()).isFalse();
-    assertThat(promptFilterResults.getContentFilterResults().getViolence().getSeverity())
-        .isEqualTo(SAFE);
-    assertThat(promptFilterResults.getContentFilterResults().getHate().isFiltered()).isFalse();
-    assertThat(promptFilterResults.getContentFilterResults().getHate().getSeverity())
-        .isEqualTo(SAFE);
-    assertThat(promptFilterResults.getContentFilterResults().getSelfHarm()).isNull();
-    assertThat(promptFilterResults.getContentFilterResults().getProfanity()).isNull();
-    assertThat(promptFilterResults.getContentFilterResults().getError()).isNull();
-    assertThat(promptFilterResults.getContentFilterResults().getJailbreak().isFiltered()).isFalse();
-    assertThat(promptFilterResults.getContentFilterResults().getJailbreak().isDetected()).isFalse();
-
-    assertThat(result.getChoices()).hasSize(1);
-    OpenAiChatCompletionChoice choice = result.getChoices().get(0);
-    assertThat(choice.getFinishReason()).isEqualTo("stop");
-    assertThat(choice.getIndex()).isEqualTo(0);
-    assertThat(choice.getMessage().getContent())
-        .isEqualTo(
-            """
+  void chatCompletion() throws IOException {
+    try (var inputStream =
+        getClass().getClassLoader().getResourceAsStream("chatCompletionResponse.json")) {
+
+      assert inputStream != null;
+      final String response = new String(inputStream.readAllBytes());
+      stubFor(post(anyUrl()).willReturn(okJson(response)));
+
+      final var systemMessage =
+          new OpenAiChatMessage.OpenAiChatSystemMessage()
+              .setContent(
+                  "You are a helpful, friendly and sometimes slightly snarky AI assistant!");
+      final var userMessage =
+          new OpenAiChatUserMessage().addText("Hello World! Why is this phrase so famous?");
+      final var request =
+          new OpenAiChatCompletionParameters().setMessages(List.of(systemMessage, userMessage));
+
+      final var result = client.chatCompletion(request);
+
+      assertThat(result).isNotNull();
+      assertThat(result.getCreated()).isEqualTo(1719300073);
+      assertThat(result.getId()).isEqualTo("chatcmpl-9dumHtDEyysGFnknk17n4Lt37tg7T");
+      assertThat(result.getModel()).isEqualTo("gpt-4-32k");
+      assertThat(result.getObject()).isEqualTo("chat.completion");
+      assertThat(result.getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
+
+      assertThat(result.getUsage()).isNotNull();
+      assertThat(result.getUsage().getCompletionTokens()).isEqualTo(54);
+      assertThat(result.getUsage().getPromptTokens()).isEqualTo(13);
+      assertThat(result.getUsage().getTotalTokens()).isEqualTo(67);
+
+      assertThat(result.getPromptFilterResults()).hasSize(1);
+      assertThat(result.getPromptFilterResults().get(0).getPromptIndex()).isEqualTo(0);
+      OpenAiContentFilterPromptResults promptFilterResults =
+          result.getPromptFilterResults().get(0).getContentFilterResults();
+      assertThat(promptFilterResults).isNotNull();
+      assertThat(promptFilterResults.getSexual()).isNotNull();
+      assertThat(promptFilterResults.getSexual().isFiltered()).isFalse();
+      assertThat(promptFilterResults.getSexual().getSeverity()).isEqualTo(SAFE);
+      assertThat(promptFilterResults.getViolence()).isNotNull();
+      assertThat(promptFilterResults.getViolence().isFiltered()).isFalse();
+      assertThat(promptFilterResults.getViolence().getSeverity()).isEqualTo(SAFE);
+      assertThat(promptFilterResults.getHate()).isNotNull();
+      assertThat(promptFilterResults.getHate().isFiltered()).isFalse();
+      assertThat(promptFilterResults.getHate().getSeverity()).isEqualTo(SAFE);
+      // TODO: update the JSON response and those assertions
+      assertThat(promptFilterResults.getSelfHarm()).isNull();
+      assertThat(promptFilterResults.getProfanity()).isNull();
+      assertThat(promptFilterResults.getError()).isNull();
+      assertThat(promptFilterResults.getJailbreak()).isNotNull();
+      assertThat(promptFilterResults.getJailbreak().isFiltered()).isFalse();
+      assertThat(promptFilterResults.getJailbreak().isDetected()).isFalse();
+
+      assertThat(result.getChoices()).hasSize(1);
+      OpenAiChatCompletionChoice choice = result.getChoices().get(0);
+      assertThat(choice.getFinishReason()).isEqualTo("stop");
+      assertThat(choice.getIndex()).isEqualTo(0);
+      assertThat(choice.getMessage().getContent())
+          .isEqualTo(
+              """
                 This is a highly subjective question as the concept of beauty differs from one person to another. It's based on personal preferences and cultural standards. There are attractive people in all walks of life and industries, making it impossible to universally determine who is the "prettiest".""");
-    assertThat(choice.getMessage().getRole()).isEqualTo("assistant");
-
-    OpenAiContentFilterPromptResults contentFilterResults = choice.getContentFilterResults();
-    assertThat(contentFilterResults.getSexual().isFiltered()).isFalse();
-    assertThat(contentFilterResults.getSexual().getSeverity()).isEqualTo(SAFE);
-    assertThat(contentFilterResults.getViolence().isFiltered()).isFalse();
-    assertThat(contentFilterResults.getViolence().getSeverity()).isEqualTo(SAFE);
-    assertThat(contentFilterResults.getHate().isFiltered()).isFalse();
-    assertThat(contentFilterResults.getHate().getSeverity()).isEqualTo(SAFE);
-    assertThat(contentFilterResults.getSelfHarm()).isNull();
-    assertThat(contentFilterResults.getProfanity()).isNull();
-    assertThat(contentFilterResults.getError()).isNull();
-    assertThat(contentFilterResults.getJailbreak()).isNull();
-
-    verify(
-        postRequestedFor(urlPathEqualTo("/chat/completions"))
-            .withRequestBody(
-                equalToJson(
-                    """
-                                {"messages":[{"role":"system","content":"You are a helpful, friendly and sometimes slightly snarky AI assistant!"},{"role":"user","content":[{"type":"text","text":"Hello World! Why is this phrase so famous?"}]}]}""")));
+      assertThat(choice.getMessage().getRole()).isEqualTo("assistant");
+
+      OpenAiContentFilterPromptResults contentFilterResults = choice.getContentFilterResults();
+      assertThat(contentFilterResults).isNotNull();
+      assertThat(contentFilterResults.getSexual()).isNotNull();
+      assertThat(contentFilterResults.getSexual().isFiltered()).isFalse();
+      assertThat(contentFilterResults.getSexual().getSeverity()).isEqualTo(SAFE);
+      assertThat(contentFilterResults.getViolence()).isNotNull();
+      assertThat(contentFilterResults.getViolence().isFiltered()).isFalse();
+      assertThat(contentFilterResults.getViolence().getSeverity()).isEqualTo(SAFE);
+      assertThat(contentFilterResults.getHate()).isNotNull();
+      assertThat(contentFilterResults.getHate().isFiltered()).isFalse();
+      assertThat(contentFilterResults.getHate().getSeverity()).isEqualTo(SAFE);
+      assertThat(contentFilterResults.getSelfHarm()).isNull();
+      assertThat(contentFilterResults.getProfanity()).isNull();
+      assertThat(contentFilterResults.getError()).isNull();
+      assertThat(contentFilterResults.getJailbreak()).isNull();
+
+      verify(
+          postRequestedFor(urlPathEqualTo("/chat/completions"))
+              .withRequestBody(
+                  equalToJson(
+                      """
+                          {"messages":[{"role":"system","content":"You are a helpful, friendly and sometimes slightly snarky AI assistant!"},{"role":"user","content":[{"type":"text","text":"Hello World! Why is this phrase so famous?"}]}]}""")));
+    }
+  }
+
+  @Test
+  void embedding() throws IOException {
+    try (var inputStream =
+        getClass().getClassLoader().getResourceAsStream("embeddingResponse.json")) {
+
+      assert inputStream != null;
+      final String response = new String(inputStream.readAllBytes());
+      stubFor(post(anyUrl()).willReturn(okJson(response)));
+
+      final var request = new OpenAiEmbeddingParameters().setInput("Hello World");
+      final var result = client.embedding(request);
+
+      assertThat(result).isNotNull();
+      assertThat(result.getModel()).isEqualTo("ada");
+      assertThat(result.getObject()).isEqualTo("list");
+
+      assertThat(result.getUsage()).isNotNull();
+      assertThat(result.getUsage().getCompletionTokens()).isNull();
+      assertThat(result.getUsage().getPromptTokens()).isEqualTo(2);
+      assertThat(result.getUsage().getTotalTokens()).isEqualTo(2);
+
+      assertThat(result.getData()).isNotNull().hasSize(1);
+      var embeddingData = result.getData().get(0);
+      assertThat(embeddingData).isNotNull();
+      assertThat(embeddingData.getObject()).isEqualTo("embedding");
+      assertThat(embeddingData.getIndex()).isEqualTo(0);
+      assertThat(embeddingData.getEmbedding())
+          .isNotNull()
+          .isNotEmpty()
+          .containsExactly(-0.0000000070958645d, 2.123e-300d, -0.0069813123d, -3.385849e-05d)
+          // ensure double precision
+          .hasToString("[-7.0958645E-9, 2.123E-300, -0.0069813123, -3.385849E-5]");
+
+      verify(
+          postRequestedFor(urlPathEqualTo("/embeddings"))
+              .withRequestBody(
+                  equalToJson("""
+                          {"input":["Hello World"]}""")));
+    }
   }
 
   @Test
-  void testEmbedding() throws IOException {
-    final String response =
-        new String(
-            getClass()
-                .getClassLoader()
-                .getResourceAsStream("embeddingResponse.json")
-                .readAllBytes());
-    stubFor(post(anyUrl()).willReturn(okJson(response)));
-
-    final var request = new OpenAiEmbeddingParameters().setInput("Hello World");
-    final var result = client.embedding(request);
-
-    assertThat(result).isNotNull();
-    assertThat(result.getModel()).isEqualTo("ada");
-    assertThat(result.getObject()).isEqualTo("list");
-
-    assertThat(result.getUsage()).isNotNull();
-    assertThat(result.getUsage().getCompletionTokens()).isNull();
-    assertThat(result.getUsage().getPromptTokens()).isEqualTo(2);
-    assertThat(result.getUsage().getTotalTokens()).isEqualTo(2);
-
-    assertThat(result.getData()).isNotNull().hasSize(1);
-    var embeddingData = result.getData().get(0);
-    assertThat(embeddingData).isNotNull();
-    assertThat(embeddingData.getObject()).isEqualTo("embedding");
-    assertThat(embeddingData.getIndex()).isEqualTo(0);
-    assertThat(embeddingData.getEmbedding())
-        .isNotNull()
-        .isNotEmpty()
-        .containsExactly(-0.0000000070958645d, 2.123e-300d, -0.0069813123d, -3.385849e-05d)
-        .hasToString(
-            "[-7.0958645E-9, 2.123E-300, -0.0069813123, -3.385849E-5]"); // ensure double precision
-
-    verify(
-        postRequestedFor(urlPathEqualTo("/embeddings"))
-            .withRequestBody(
-                equalToJson("""
-                                {"input":["Hello World"]}""")));
+  void streamChatCompletion() throws IOException {
+    try (var inputStream =
+        getClass().getClassLoader().getResourceAsStream("streamChatCompletion.txt")) {
+
+      assert inputStream != null;
+      final String response = new String(inputStream.readAllBytes());
+      stubFor(
+          post(anyUrl())
+              .willReturn(ok().withBody(response).withHeader("Content-Type", "text/event-stream")));
+
+      final var request =
+          new OpenAiChatCompletionParameters()
+              .setMessages(
+                  List.of(
+                      new OpenAiChatUserMessage()
+                          .addText(
+                              "Can you give me the first 100 numbers of the Fibonacci sequence?")));
+
+      try (OpenAiStream<OpenAiDeltaChatCompletion, OpenAiChatCompletionOutput> result =
+          client.stream(request)) {
+
+        final List<OpenAiDeltaChatCompletion> deltaList = result.getDeltaStream().toList();
+
+        assertThat(deltaList).hasSize(5);
+        // the first two and the last delta don't have any content
+        assertThat(deltaList.get(0).getDeltaContent()).isNull();
+        assertThat(deltaList.get(1).getDeltaContent()).isEqualTo("");
+        assertThat(deltaList.get(2).getDeltaContent()).isEqualTo("Sure");
+        assertThat(deltaList.get(3).getDeltaContent()).isEqualTo("!");
+        assertThat(deltaList.get(4).getDeltaContent()).isNull();
+
+        assertThat(deltaList.get(0).getSystemFingerprint()).isNull();
+        assertThat(deltaList.get(1).getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
+        assertThat(deltaList.get(2).getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
+        assertThat(deltaList.get(3).getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
+        assertThat(deltaList.get(4).getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
+
+        assertThat(deltaList.get(0).getChoices()).isEmpty();
+        assertThat(deltaList.get(1).getChoices()).hasSize(1);
+        assertThat(deltaList.get(2).getChoices()).hasSize(1);
+        assertThat(deltaList.get(3).getChoices()).hasSize(1);
+        assertThat(deltaList.get(4).getChoices()).hasSize(1);
+
+        final var delta0 = deltaList.get(0);
+        assertThat(delta0.getId()).isEqualTo("");
+        assertThat(delta0.getCreated()).isEqualTo(0);
+        assertThat(delta0.getModel()).isEqualTo("");
+        assertThat(delta0.getObject()).isEqualTo("");
+        assertThat(delta0.getUsage()).isNull();
+        assertThat(delta0.getChoices()).isEmpty();
+        // prompt filter results are only present in delta 0
+        assertThat(delta0.getPromptFilterResults()).isNotNull();
+        assertThat(delta0.getPromptFilterResults().get(0).getPromptIndex()).isEqualTo(0);
+        final var promptFilter0 = delta0.getPromptFilterResults().get(0).getContentFilterResults();
+        assertThat(promptFilter0).isNotNull();
+        assertFilter(promptFilter0);
+
+        final var delta2 = deltaList.get(2);
+        assertThat(delta2.getId()).isEqualTo("chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1");
+        assertThat(delta2.getCreated()).isEqualTo(1724825677);
+        assertThat(delta2.getModel()).isEqualTo("gpt-35-turbo");
+        assertThat(delta2.getObject()).isEqualTo("chat.completion.chunk");
+        assertThat(delta2.getUsage()).isNull();
+        assertThat(delta2.getPromptFilterResults()).isNull();
+        final var choices2 = delta2.getChoices().get(0);
+        assertThat(choices2.getIndex()).isEqualTo(0);
+        assertThat(choices2.getFinishReason()).isNull();
+        assertThat(choices2.getMessage()).isNotNull();
+        // the role is only defined in delta 1, but it defaults to "assistant" for all deltas
+        assertThat(choices2.getMessage().getRole()).isEqualTo("assistant");
+        assertThat(choices2.getMessage().getContent()).isEqualTo("Sure");
+        assertThat(choices2.getMessage().getTool_calls()).isNull();
+        final var filter2 = choices2.getContentFilterResults();
+        assertFilter(filter2);
+
+        final var delta3 = deltaList.get(3);
+        assertThat(delta3.getDeltaContent()).isEqualTo("!");
+
+        final var delta4Choice = deltaList.get(4).getChoices().get(0);
+        assertThat(delta4Choice.getFinishReason()).isEqualTo("stop");
+        assertThat(delta4Choice.getMessage()).isNotNull();
+        // the role is only defined in delta 1, but it defaults to "assistant" for all deltas
+        assertThat(delta4Choice.getMessage().getRole()).isEqualTo("assistant");
+        assertThat(delta4Choice.getMessage().getContent()).isNull();
+        assertThat(delta4Choice.getMessage().getTool_calls()).isNull();
+
+        final var totalOutput = result.getTotalOutput();
+        assertThat(totalOutput.getChoices()).hasSize(1);
+        final var choice = totalOutput.getChoices().get(0);
+        assertThat(choice.getFinishReason()).isEqualTo("stop");
+        assertFilter(choice.getContentFilterResults());
+        assertThat(choice.getFinishReason()).isEqualTo("stop");
+        assertThat(choice.getMessage()).isNotNull();
+        assertThat(choice.getMessage().getRole()).isEqualTo("assistant");
+        assertThat(choice.getMessage().getContent()).isEqualTo("Sure!");
+        assertThat(choice.getMessage().getTool_calls()).isNull();
+        assertThat(totalOutput.getId()).isEqualTo("chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1");
+        assertThat(totalOutput.getCreated()).isEqualTo(1724825677);
+        assertThat(totalOutput.getModel()).isEqualTo("gpt-35-turbo");
+        assertThat(totalOutput.getObject()).isEqualTo("chat.completion.chunk");
+        assertThat(totalOutput.getUsage()).isNull();
+        assertThat(totalOutput.getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
+        assertThat(totalOutput.getPromptFilterResults()).isNotNull();
+        assertFilter(totalOutput.getPromptFilterResults().get(0).getContentFilterResults());
+      }
+    }
+  }
+
+  void assertFilter(OpenAiContentFilterPromptResults filter) {
+    assertThat(filter).isNotNull();
+    assertThat(filter.getHate()).isNotNull();
+    assertThat(filter.getHate().isFiltered()).isFalse();
+    assertThat(filter.getHate().getSeverity()).isEqualTo(SAFE);
+    assertThat(filter.getSelfHarm()).isNotNull();
+    assertThat(filter.getSelfHarm().isFiltered()).isFalse();
+    assertThat(filter.getSelfHarm().getSeverity()).isEqualTo(SAFE);
+    assertThat(filter.getSexual()).isNotNull();
+    assertThat(filter.getSexual().isFiltered()).isFalse();
+    assertThat(filter.getSexual().getSeverity()).isEqualTo(SAFE);
+    assertThat(filter.getViolence()).isNotNull();
+    assertThat(filter.getViolence().isFiltered()).isFalse();
+    assertThat(filter.getViolence().getSeverity()).isEqualTo(SAFE);
+    assertThat(filter.getJailbreak()).isNull();
+    assertThat(filter.getProfanity()).isNull();
+    assertThat(filter.getError()).isNull();
   }
 }
diff --git a/foundation-models/openai/src/test/resources/chatCompletionResponse.json b/foundation-models/openai/src/test/resources/chatCompletionResponse.json
index 2bbeb89c..056576d4 100644
--- a/foundation-models/openai/src/test/resources/chatCompletionResponse.json
+++ b/foundation-models/openai/src/test/resources/chatCompletionResponse.json
@@ -63,5 +63,5 @@
       }
     }
   ],
-  "system_fingerprint": null
-}
\ No newline at end of file
+  "system_fingerprint": "fp_e49e4201a9"
+}
diff --git a/foundation-models/openai/src/test/resources/streamChatCompletion.txt b/foundation-models/openai/src/test/resources/streamChatCompletion.txt
new file mode 100644
index 00000000..d6aca514
--- /dev/null
+++ b/foundation-models/openai/src/test/resources/streamChatCompletion.txt
@@ -0,0 +1,7 @@
+
+data: {"choices":[],"created":0,"id":"","model":"","object":"","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}]}
+data: {"choices":[{"content_filter_results":{},"delta":{"content":"","role":"assistant"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}
+data: {"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"Sure"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}
+data: {"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"!"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}
+data: {"choices":[{"content_filter_results":{},"delta":{},"finish_reason":"stop","index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}
+data: [DONE]

From 31dbd52c3275958e17eb53a514783c112e812ec5 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Thu, 29 Aug 2024 10:02:12 +0200
Subject: [PATCH 24/55] Change return type of stream, added e2e test

---
 .../sdk/app/controllers/OpenAiController.java | 81 ++++++++++++++++---
 .../ai/sdk/app/controllers/OpenAiTest.java    | 40 ++++++++-
 .../foundationmodels/openai/OpenAiClient.java | 26 +++---
 .../foundationmodels/openai/OpenAiStream.java | 53 ------------
 .../openai/OpenAiStreamingHandler.java        | 69 ++++++----------
 .../model/OpenAiDeltaChatCompletion.java      | 20 +++--
 .../openai/OpenAiClientTest.java              | 16 ++--
 7 files changed, 162 insertions(+), 143 deletions(-)
 delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index b4d6c4c6..8b8b4d4d 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -22,7 +22,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.function.Consumer;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.MediaType;
@@ -55,9 +55,10 @@ public static OpenAiChatCompletionOutput chatCompletion() {
    *
    * @return the emitter that streams the assistant message response
    */
-  @GetMapping("/streamChatCompletion")
+  // This is a code sample, works like #streamChatCompletion but with a readable code
+  @SuppressWarnings("unused")
   @Nonnull
-  public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
+  public static ResponseEntity<ResponseBodyEmitter> streamChatCompletionExample() {
     final var request =
         new OpenAiChatCompletionParameters()
             .setMessages(
@@ -73,20 +74,19 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
         .submit(
             () -> {
               // try-with-resources ensures that the stream is closed after the response is sent.
-              try (var result = OpenAiClient.forModel(GPT_35_TURBO).stream(request)) {
-                result
-                    .getDeltaStream()
-                    .map(OpenAiDeltaChatCompletion::getDeltaContent)
-                    // The first two and the last deltaStream do not contain any message content
-                    .filter(Objects::nonNull)
-                    .forEach(content -> send(emitter, content));
-
-                final String indentedJson = objectToJson(result.getTotalOutput());
+              try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
+
+                final var totalOutput = new OpenAiChatCompletionOutput();
+                stream
+                    .peek(totalOutput::addDelta)
+                    .forEach(delta -> send(emitter, delta.getDeltaContent()));
+
+                final String indentedJson = objectToJson(totalOutput);
                 send(emitter, "\n\n-----Total Output-----\n\n" + indentedJson);
                 emitter.complete();
               }
             });
-    // MediaType.TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
+    // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
@@ -100,6 +100,61 @@ private static void send(
     }
   }
 
+  /**
+   * Asynchronous stream of an OpenAI chat request
+   *
+   * @return the emitter that streams the assistant message response
+   */
+  @SuppressWarnings("unused") // #streamToConsumer is the method that is tested
+  @GetMapping("/streamChatCompletion")
+  @Nonnull
+  public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
+    final var request =
+        new OpenAiChatCompletionParameters()
+            .setMessages(
+                List.of(
+                    new OpenAiChatUserMessage()
+                        .addText(
+                            "Can you give me the first 100 numbers of the Fibonacci sequence?")));
+
+    final var emitter = new ResponseBodyEmitter();
+    final var totalOutput = new OpenAiChatCompletionOutput();
+    final var consumer =
+        new Consumer<OpenAiDeltaChatCompletion>() {
+          @Override
+          public void accept(@Nonnull final OpenAiDeltaChatCompletion delta) {
+            totalOutput.addDelta(delta);
+            send(emitter, delta.getDeltaContent());
+          }
+        };
+
+    // Cloud SDK's ThreadContext is vital for the request to successfully execute.
+    ThreadContextExecutors.getExecutor()
+        .submit(
+            () -> {
+              streamToConsumer(request, consumer);
+              send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
+              emitter.complete();
+            });
+    // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
+    return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
+  }
+
+  /**
+   * Streams the OpenAI chat completion into the accept method of the consumer.
+   *
+   * @param request the chat completion request
+   * @param consumer the consumer that asynchronously accepts the chat completion
+   */
+  static void streamToConsumer(
+      @Nonnull final OpenAiChatCompletionParameters request,
+      @Nonnull final Consumer<OpenAiDeltaChatCompletion> consumer) {
+    // try-with-resources ensures that the stream is closed after the response is sent.
+    try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
+      stream.forEach(consumer);
+    }
+  }
+
   private static String objectToJson(@Nonnull final Object obj) {
     try {
       return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(obj);
diff --git a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
index 30a10e89..c828edd9 100644
--- a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
+++ b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
@@ -2,8 +2,17 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiDeltaChatCompletion;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
 
+@Slf4j
 class OpenAiTest {
   @Test
   void chatCompletion() {
@@ -25,8 +34,34 @@ void chatCompletionImage() {
 
   @Test
   void streamChatCompletion() {
-    final var emitter = OpenAiController.streamChatCompletion();
-    // TODO: assert on the emitter
+    final var request =
+        new OpenAiChatCompletionParameters()
+            .setMessages(List.of(new OpenAiChatUserMessage().addText("Who is the prettiest?")));
+
+    final var totalOutput = new OpenAiChatCompletionOutput();
+    final var emptyDeltaCount = new AtomicInteger(0);
+    var consumer =
+        new Consumer<OpenAiDeltaChatCompletion>() {
+          @Override
+          public void accept(OpenAiDeltaChatCompletion delta) {
+            totalOutput.addDelta(delta);
+            final String deltaContent = delta.getDeltaContent();
+            log.info("deltaContent: {}", deltaContent);
+            if (deltaContent.isEmpty()) {
+              emptyDeltaCount.incrementAndGet();
+            }
+          }
+        };
+    OpenAiController.streamToConsumer(request, consumer);
+
+    // the first two and the last delta don't have any content
+    // see OpenAiDeltaChatCompletion#getDeltaContent
+    assertThat(emptyDeltaCount.get()).isLessThanOrEqualTo(3);
+
+    assertThat(totalOutput.getChoices()).isNotEmpty();
+    assertThat(totalOutput.getChoices().get(0).getMessage().getContent()).isNotEmpty();
+    assertThat(totalOutput.getPromptFilterResults()).isNotNull();
+    assertThat(totalOutput.getChoices().get(0).getContentFilterResults()).isNotNull();
   }
 
   @Test
@@ -35,6 +70,7 @@ void chatCompletionTools() {
 
     final var message = completion.getChoices().get(0).getMessage();
     assertThat(message.getRole()).isEqualTo("assistant");
+    assertThat(message.getTool_calls()).isNotNull();
     assertThat(message.getTool_calls().get(0).getFunction().getName()).isEqualTo("fibonacci");
   }
 
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 2856b163..39f37bf3 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -7,7 +7,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.sap.ai.sdk.core.Core;
-import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiDeltaChatCompletion;
@@ -18,6 +17,7 @@
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import com.sap.cloud.sdk.cloudplatform.connectivity.Destination;
 import java.io.IOException;
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -116,14 +116,10 @@ public OpenAiChatCompletionOutput chatCompletion(
    * @throws OpenAiClientException if the request fails
    */
   @Nonnull
-  public OpenAiStream<OpenAiDeltaChatCompletion, OpenAiChatCompletionOutput> stream(
+  public Stream<OpenAiDeltaChatCompletion> streamChatCompletion(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
     parameters.setStream(true);
-    return stream(
-        "/chat/completions",
-        parameters,
-        OpenAiDeltaChatCompletion.class,
-        OpenAiChatCompletionOutput.class);
+    return streamChatCompletion("/chat/completions", parameters, OpenAiDeltaChatCompletion.class);
   }
 
   /**
@@ -151,14 +147,13 @@ private <T> T execute(
   }
 
   @Nonnull
-  private <D extends StreamedDelta, T extends DeltaAggregatable<D>> OpenAiStream<D, T> stream(
+  private <D extends StreamedDelta> Stream<D> streamChatCompletion(
       @Nonnull final String path,
       @Nonnull final Object payload,
-      @Nonnull final Class<D> deltaType,
-      @Nonnull final Class<T> totalType) {
+      @Nonnull final Class<D> deltaType) {
     final var request = new HttpPost(path);
     serializeAndSetHttpEntity(request, payload);
-    return streamRequest(request, deltaType, totalType);
+    return streamRequest(request, deltaType);
   }
 
   private static void serializeAndSetHttpEntity(
@@ -184,15 +179,12 @@ private <T> T executeRequest(
   }
 
   @Nonnull
-  private <D extends StreamedDelta, T extends DeltaAggregatable<D>>
-      OpenAiStream<D, T> streamRequest(
-          final BasicClassicHttpRequest request,
-          @Nonnull final Class<D> deltaType,
-          @Nonnull final Class<T> totalType) {
+  private <D extends StreamedDelta> Stream<D> streamRequest(
+      final BasicClassicHttpRequest request, @Nonnull final Class<D> deltaType) {
     try {
       @SuppressWarnings("UnstableApiUsage")
       final var client = ApacheHttpClient5Accessor.getHttpClient(destination);
-      return new OpenAiStreamingHandler<>(deltaType, totalType)
+      return new OpenAiStreamingHandler<>(deltaType)
           .handleResponse(client.executeOpen(null, request, null));
     } catch (final IOException e) {
       throw new OpenAiClientException(e);
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
deleted file mode 100644
index c4d1cadf..00000000
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStream.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package com.sap.ai.sdk.foundationmodels.openai;
-
-import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
-import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
-import java.util.stream.Stream;
-import javax.annotation.Nonnull;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.RequiredArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-
-/**
- * Generic OpenAI stream output.
- *
- * @param <D> the type of the streamed delta
- * @param <T> the type of the total output
- */
-@RequiredArgsConstructor(access = AccessLevel.NONE)
-@NoArgsConstructor
-@Setter(AccessLevel.PACKAGE)
-@Accessors(chain = true)
-// D extends StreamedDelta but Java generics, oddly enough, use extends for interfaces
-public class OpenAiStream<D extends StreamedDelta, T extends DeltaAggregatable<D>>
-    implements AutoCloseable {
-
-  @Getter(onMethod_ = @Nonnull)
-  private Stream<D> deltaStream;
-
-  @Nonnull private T totalOutput;
-
-  void addDelta(@Nonnull final D delta) {
-    totalOutput.addDelta(delta);
-  }
-
-  /**
-   * Get the total aggregated output from all deltas. Closes the delta stream.
-   *
-   * @return the total output until now.
-   */
-  @Nonnull
-  public T getTotalOutput() {
-    deltaStream.close();
-    return totalOutput;
-  }
-
-  /** Close the delta stream. */
-  @Override
-  public void close() {
-    deltaStream.close();
-  }
-}
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 024768b9..719804b2 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -4,13 +4,11 @@
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.buildExceptionAndThrow;
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.parseErrorAndThrow;
 
-import com.sap.ai.sdk.foundationmodels.openai.model.DeltaAggregatable;
 import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.lang.reflect.InvocationTargetException;
 import java.nio.charset.StandardCharsets;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
@@ -21,20 +19,19 @@
 
 @Slf4j
 @RequiredArgsConstructor
-class OpenAiStreamingHandler<D extends StreamedDelta, T extends DeltaAggregatable<D>> {
+class OpenAiStreamingHandler<D extends StreamedDelta> {
 
   @Nonnull private final Class<D> deltaType;
-  @Nonnull private final Class<T> totalType;
 
   /**
    * Processes a {@link ClassicHttpResponse} and returns some value corresponding to that response.
    *
    * @param response The response to process
-   * @return A {@link OpenAiStream} of a model class instantiated from the response
+   * @return A {@link Stream} of a model class instantiated from the response
    * @throws OpenAiClientException in case of a problem or the connection was aborted
    */
   @Nonnull
-  public OpenAiStream<D, T> handleResponse(@Nonnull final ClassicHttpResponse response)
+  public Stream<D> handleResponse(@Nonnull final ClassicHttpResponse response)
       throws OpenAiClientException {
     if (response.getCode() >= 300) {
       buildExceptionAndThrow(response);
@@ -44,12 +41,12 @@ public OpenAiStream<D, T> handleResponse(@Nonnull final ClassicHttpResponse resp
 
   /**
    * @param response The response to process
-   * @return A {@link OpenAiStream} of a model class instantiated from the response
+   * @return A {@link Stream} of a model class instantiated from the response
    * @author stippi
    */
-  // The stream is closed by the user of the OpenAiStream
+  // The stream is closed by the user of the Stream
   @SuppressWarnings("PMD.CloseResource")
-  private OpenAiStream<D, T> parseResponse(@Nonnull final ClassicHttpResponse response)
+  private Stream<D> parseResponse(@Nonnull final ClassicHttpResponse response)
       throws OpenAiClientException {
     final HttpEntity responseEntity = response.getEntity();
     if (responseEntity == null) {
@@ -63,40 +60,26 @@ private OpenAiStream<D, T> parseResponse(@Nonnull final ClassicHttpResponse resp
     }
     final var br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
 
-    final OpenAiStream<D, T> output = new OpenAiStream<>();
-    try {
-      output.setTotalOutput(totalType.getDeclaredConstructor().newInstance());
-    } catch (InstantiationException
-        | IllegalAccessException
-        | NoSuchMethodException
-        | InvocationTargetException e) {
-      throw new OpenAiClientException(e);
-    }
-
     // https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
-    final Stream<D> deltaStream =
-        br.lines()
-            .filter(
-                responseLine ->
-                    // half of the lines are empty newlines, the last line is "data: [DONE]"
-                    !responseLine.isEmpty() && !"data: [DONE]".equals(responseLine.trim()))
-            .peek(
-                responseLine -> {
-                  if (!responseLine.startsWith("data: ")) {
-                    parseErrorAndThrow(responseLine, new OpenAiClientException());
-                  }
-                })
-            .map(
-                responseLine -> {
-                  final String data = responseLine.substring(5).replace("delta", "message");
-                  try {
-                    final D delta = JACKSON.readValue(data, deltaType);
-                    output.addDelta(delta);
-                    return delta;
-                  } catch (final IOException e) {
-                    throw new OpenAiClientException("Failed to parse delta message: " + data, e);
-                  }
-                });
-    return output.setDeltaStream(deltaStream);
+    return br.lines()
+        .filter(
+            responseLine ->
+                // half of the lines are empty newlines, the last line is "data: [DONE]"
+                !responseLine.isEmpty() && !"data: [DONE]".equals(responseLine.trim()))
+        .peek(
+            responseLine -> {
+              if (!responseLine.startsWith("data: ")) {
+                parseErrorAndThrow(responseLine, new OpenAiClientException());
+              }
+            })
+        .map(
+            responseLine -> {
+              final String data = responseLine.substring(5).replace("delta", "message");
+              try {
+                return JACKSON.readValue(data, deltaType);
+              } catch (final IOException e) {
+                throw new OpenAiClientException("Failed to parse delta message: " + data, e);
+              }
+            });
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
index 6fce1768..65cd6859 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
@@ -30,20 +30,26 @@ public class OpenAiDeltaChatCompletion extends OpenAiCompletionOutput implements
   /**
    * Get the message content from the delta.
    *
-   * @return the message content or null.
+   * <p>Note: If there are multiple choices only the first one is returned
+   *
+   * <p>Note: The first two and the last delta do not contain any content
+   *
+   * @return the message content or empty string.
    */
-  @Nullable
+  @Nonnull
   public String getDeltaContent() {
     // Avoid the first delta: "choices":[]
     if (!getChoices().isEmpty()
         // Multiple choices are spread out on multiple deltas
         // A delta only contains one choice with a variable index
-        && getChoices().get(0).getIndex() == 0
-        // Avoid the second delta: "choices":[{"delta":{"content":"","role":"assistant"}}]
-        && getChoices().get(0).getMessage() != null) {
+        && getChoices().get(0).getIndex() == 0) {
 
-      return getChoices().get(0).getMessage().getContent();
+      final var message = getChoices().get(0).getMessage();
+      // Avoid the second delta: "choices":[{"delta":{"content":"","role":"assistant"}}]
+      if (message != null && message.getContent() != null) {
+        return message.getContent();
+      }
     }
-    return null;
+    return "";
   }
 }
diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 13616431..6e968d8f 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -20,6 +20,7 @@
 import io.vavr.control.Try;
 import java.io.IOException;
 import java.util.List;
+import java.util.stream.Stream;
 import org.assertj.core.api.SoftAssertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -259,7 +260,7 @@ void embedding() throws IOException {
   }
 
   @Test
-  void streamChatCompletion() throws IOException {
+  void streamChatCompletionChatCompletion() throws IOException {
     try (var inputStream =
         getClass().getClassLoader().getResourceAsStream("streamChatCompletion.txt")) {
 
@@ -277,18 +278,19 @@ void streamChatCompletion() throws IOException {
                           .addText(
                               "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-      try (OpenAiStream<OpenAiDeltaChatCompletion, OpenAiChatCompletionOutput> result =
-          client.stream(request)) {
+      try (Stream<OpenAiDeltaChatCompletion> stream = client.streamChatCompletion(request)) {
 
-        final List<OpenAiDeltaChatCompletion> deltaList = result.getDeltaStream().toList();
+        OpenAiChatCompletionOutput totalOutput = new OpenAiChatCompletionOutput();
+        final List<OpenAiDeltaChatCompletion> deltaList =
+            stream.peek(totalOutput::addDelta).toList();
 
         assertThat(deltaList).hasSize(5);
         // the first two and the last delta don't have any content
-        assertThat(deltaList.get(0).getDeltaContent()).isNull();
+        assertThat(deltaList.get(0).getDeltaContent()).isEqualTo("");
         assertThat(deltaList.get(1).getDeltaContent()).isEqualTo("");
         assertThat(deltaList.get(2).getDeltaContent()).isEqualTo("Sure");
         assertThat(deltaList.get(3).getDeltaContent()).isEqualTo("!");
-        assertThat(deltaList.get(4).getDeltaContent()).isNull();
+        assertThat(deltaList.get(4).getDeltaContent()).isEqualTo("");
 
         assertThat(deltaList.get(0).getSystemFingerprint()).isNull();
         assertThat(deltaList.get(1).getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
@@ -344,8 +346,6 @@ void streamChatCompletion() throws IOException {
         assertThat(delta4Choice.getMessage().getRole()).isEqualTo("assistant");
         assertThat(delta4Choice.getMessage().getContent()).isNull();
         assertThat(delta4Choice.getMessage().getTool_calls()).isNull();
-
-        final var totalOutput = result.getTotalOutput();
         assertThat(totalOutput.getChoices()).hasSize(1);
         final var choice = totalOutput.getChoices().get(0);
         assertThat(choice.getFinishReason()).isEqualTo("stop");

From de7e7f0d6054f102f2484d2dd3d767bbf4549c3f Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Thu, 29 Aug 2024 10:12:46 +0200
Subject: [PATCH 25/55] Added documentation

---
 README.md                                     | 46 +++++++++++++-
 .../sdk/app/controllers/OpenAiController.java | 60 ++++---------------
 2 files changed, 55 insertions(+), 51 deletions(-)

diff --git a/README.md b/README.md
index 9197c791..32c174c4 100644
--- a/README.md
+++ b/README.md
@@ -197,13 +197,57 @@ final String resultMessage = result.getChoices().get(0).getMessage().getContent(
 
 See [an example in our Spring Boot application](e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java)
 
-### Chat completion with a model not in defined `OpenAiModel`
+### Chat completion with a model not defined in `OpenAiModel`
 
 ```java
 final OpenAiChatCompletionOutput result =
     OpenAiClient.forModel(new OpenAiModel("model")).chatCompletion(request);
 ```
 
+### Stream chat completion
+
+#### Spring Boot example
+```java
+public ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
+  final var request =
+      new OpenAiChatCompletionParameters()
+          .setMessages(
+              List.of(
+                  new OpenAiChatUserMessage()
+                      .addText(
+                          "Can you give me the first 100 numbers of the Fibonacci sequence?")));
+
+  final var emitter = new ResponseBodyEmitter();
+
+  // Cloud SDK's ThreadContext is vital for the request to successfully execute.
+  ThreadContextExecutors.getExecutor()
+      .submit(
+          () -> {
+            final var totalOutput = new OpenAiChatCompletionOutput();
+
+            // try-with-resources ensures that the stream is closed after the response is sent.
+            try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
+              stream
+                  .peek(totalOutput::addDelta) // optional: collect all deltas
+                  .forEach(delta -> {
+                    try {
+                      emitter.send(delta.getDeltaContent());
+                    } catch (final IOException e) {
+                      log.error(Arrays.toString(e.getStackTrace()));
+                      emitter.completeWithError(e);
+                    }
+                  });
+            }
+            // optional: totalOutput can be looked at here
+            emitter.complete();
+          });
+  // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
+  return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
+}
+```
+
+See [an example in our Spring Boot application](e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java)
+
 ## Orchestration chat completion
 
 ### Prerequisites
diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 8b8b4d4d..c098fbc7 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -50,56 +50,6 @@ public static OpenAiChatCompletionOutput chatCompletion() {
     return OpenAiClient.forModel(GPT_35_TURBO).chatCompletion(request);
   }
 
-  /**
-   * Asynchronous stream of an OpenAI chat request
-   *
-   * @return the emitter that streams the assistant message response
-   */
-  // This is a code sample, works like #streamChatCompletion but with a readable code
-  @SuppressWarnings("unused")
-  @Nonnull
-  public static ResponseEntity<ResponseBodyEmitter> streamChatCompletionExample() {
-    final var request =
-        new OpenAiChatCompletionParameters()
-            .setMessages(
-                List.of(
-                    new OpenAiChatUserMessage()
-                        .addText(
-                            "Can you give me the first 100 numbers of the Fibonacci sequence?")));
-
-    final var emitter = new ResponseBodyEmitter();
-
-    // Cloud SDK's ThreadContext is vital for the request to successfully execute.
-    ThreadContextExecutors.getExecutor()
-        .submit(
-            () -> {
-              // try-with-resources ensures that the stream is closed after the response is sent.
-              try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
-
-                final var totalOutput = new OpenAiChatCompletionOutput();
-                stream
-                    .peek(totalOutput::addDelta)
-                    .forEach(delta -> send(emitter, delta.getDeltaContent()));
-
-                final String indentedJson = objectToJson(totalOutput);
-                send(emitter, "\n\n-----Total Output-----\n\n" + indentedJson);
-                emitter.complete();
-              }
-            });
-    // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
-    return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
-  }
-
-  private static void send(
-      @Nonnull final ResponseBodyEmitter emitter, @Nonnull final String chunk) {
-    try {
-      emitter.send(chunk);
-    } catch (final IOException e) {
-      log.error(Arrays.toString(e.getStackTrace()));
-      emitter.completeWithError(e);
-    }
-  }
-
   /**
    * Asynchronous stream of an OpenAI chat request
    *
@@ -140,6 +90,16 @@ public void accept(@Nonnull final OpenAiDeltaChatCompletion delta) {
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
+  private static void send(
+      @Nonnull final ResponseBodyEmitter emitter, @Nonnull final String chunk) {
+    try {
+      emitter.send(chunk);
+    } catch (final IOException e) {
+      log.error(Arrays.toString(e.getStackTrace()));
+      emitter.completeWithError(e);
+    }
+  }
+
   /**
    * Streams the OpenAI chat completion into the accept method of the consumer.
    *

From 349936f72afc7eeb3b064ceeb957c0dde52c68b4 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Thu, 29 Aug 2024 11:55:52 +0200
Subject: [PATCH 26/55] Added documentation framework-agnostic + throw if
 finish reason is invalid

---
 README.md                                     | 31 +++++++++++++++++++
 .../foundationmodels/openai/OpenAiClient.java |  5 +--
 .../openai/OpenAiStreamingHandler.java        | 12 +++++++
 .../openai/model/OpenAiCompletionChoice.java  | 15 ++++++++-
 .../model/OpenAiDeltaChatCompletion.java      | 23 ++++++++------
 .../openai/model/StreamedDelta.java           | 25 +++++++++++++--
 .../test/resources/streamChatCompletion.txt   |  1 -
 7 files changed, 97 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 32c174c4..90a02c10 100644
--- a/README.md
+++ b/README.md
@@ -206,6 +206,37 @@ final OpenAiChatCompletionOutput result =
 
 ### Stream chat completion
 
+#### Framework-agnostic example
+```java
+public ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
+  final var request =
+      new OpenAiChatCompletionParameters()
+          .setMessages(
+              List.of(
+                  new OpenAiChatUserMessage()
+                      .addText(
+                          "Can you give me the first 100 numbers of the Fibonacci sequence?")));
+
+  // Cloud SDK's ThreadContext is vital for the request to successfully execute.
+  ThreadContextExecutors.getExecutor()
+      .submit(
+          () -> {
+            final var totalOutput = new OpenAiChatCompletionOutput();
+
+            // try-with-resources ensures that the stream is closed after the response is sent.
+            try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
+              stream
+                  // optional: collect all deltas
+                  .peek(totalOutput::addDelta)
+                  // send is defined by your framework
+                  .forEach(delta -> send(delta.getDeltaContent()));
+            }
+            // optional: totalOutput can be looked at here
+          });
+  // Note: set the header content-type to text/event-stream on the sent response
+}
+```
+
 #### Spring Boot example
 ```java
 public ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 39f37bf3..d43deaff 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -112,8 +112,9 @@ public OpenAiChatCompletionOutput chatCompletion(
    * Generate a completion for the given prompt.
    *
    * @param parameters the prompt, including messages and other parameters.
-   * @return the completion output
-   * @throws OpenAiClientException if the request fails
+   * @return A stream of chat completions deltas
+   * @throws OpenAiClientException if the request fails or if the finish reason is content_filter or
+   *     length (token limit).
    */
   @Nonnull
   public Stream<OpenAiDeltaChatCompletion> streamChatCompletion(
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 719804b2..89cf969a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -80,6 +80,18 @@ private Stream<D> parseResponse(@Nonnull final ClassicHttpResponse response)
               } catch (final IOException e) {
                 throw new OpenAiClientException("Failed to parse delta message: " + data, e);
               }
+            })
+        .peek(
+            delta -> {
+              final String finishReason = delta.getFinishReason();
+              if (finishReason != null) {
+                if (finishReason.equals("content_filter")) {
+                  throw new OpenAiClientException("Content filter filtered the output.");
+                } else if (finishReason.equals("length")) {
+                  throw new OpenAiClientException(
+                      "Incomplete output due to max_tokens parameter or token limit.");
+                }
+              }
             });
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
index 2b2b4f3b..edddf457 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
@@ -13,7 +13,20 @@
 @EqualsAndHashCode
 @ToString(callSuper = true)
 public class OpenAiCompletionChoice {
-  /** Reason for finish. */
+  /**
+   * Reason for finish. The possible values are:
+   *
+   * <p>{@code stop}: API returned complete message, or a message terminated by one of the stop
+   * sequences provided via the stop parameter
+   *
+   * <p>{@code length}: Incomplete model output due to max_tokens parameter or token limit
+   *
+   * <p>{@code function_call}: The model decided to call a function
+   *
+   * <p>{@code content_filter}: Omitted content due to a flag from our content filters
+   *
+   * <p>{@code null}: API response still in progress or incomplete
+   */
   @JsonProperty("finish_reason")
   @Getter(onMethod_ = @Nullable)
   private String finishReason;
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
index 65cd6859..f8049c82 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
@@ -27,16 +27,8 @@ public class OpenAiDeltaChatCompletion extends OpenAiCompletionOutput implements
   @Getter(onMethod_ = @Nullable)
   private String systemFingerprint;
 
-  /**
-   * Get the message content from the delta.
-   *
-   * <p>Note: If there are multiple choices only the first one is returned
-   *
-   * <p>Note: The first two and the last delta do not contain any content
-   *
-   * @return the message content or empty string.
-   */
   @Nonnull
+  @Override
   public String getDeltaContent() {
     // Avoid the first delta: "choices":[]
     if (!getChoices().isEmpty()
@@ -52,4 +44,17 @@ && getChoices().get(0).getIndex() == 0) {
     }
     return "";
   }
+
+  @Nullable
+  @Override
+  public String getFinishReason() {
+    // Avoid the first delta: "choices":[]
+    if (!getChoices().isEmpty()
+        // Multiple choices are spread out on multiple deltas
+        // A delta only contains one choice with a variable index
+        && getChoices().get(0).getIndex() == 0) {
+      return getChoices().get(0).getFinishReason();
+    }
+    return null;
+  }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java
index f7756b09..17a01367 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java
@@ -12,10 +12,31 @@
 public interface StreamedDelta {
 
   /**
-   * Get the content from the delta.
+   * Get the message content from the delta.
    *
-   * @return the content from the delta or null if no content is available.
+   * <p>Note: If there are multiple choices only the first one is returned
+   *
+   * <p>Note: The first two and the last delta do not contain any content
+   *
+   * @return the message content or empty string.
    */
   @Nullable
   String getDeltaContent();
+
+  /**
+   * Reason for finish. The possible values are:
+   *
+   * <p>{@code stop}: API returned complete message, or a message terminated by one of the stop
+   * sequences provided via the stop parameter
+   *
+   * <p>{@code length}: Incomplete model output due to max_tokens parameter or token limit
+   *
+   * <p>{@code function_call}: The model decided to call a function
+   *
+   * <p>{@code content_filter}: Omitted content due to a flag from our content filters
+   *
+   * <p>{@code null}: API response still in progress or incomplete
+   */
+  @Nullable
+  String getFinishReason();
 }
diff --git a/foundation-models/openai/src/test/resources/streamChatCompletion.txt b/foundation-models/openai/src/test/resources/streamChatCompletion.txt
index d6aca514..14d23729 100644
--- a/foundation-models/openai/src/test/resources/streamChatCompletion.txt
+++ b/foundation-models/openai/src/test/resources/streamChatCompletion.txt
@@ -1,4 +1,3 @@
-
 data: {"choices":[],"created":0,"id":"","model":"","object":"","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}]}
 data: {"choices":[{"content_filter_results":{},"delta":{"content":"","role":"assistant"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}
 data: {"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"Sure"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}

From 3366c2ea48de0725901de9e1bda178c969e2d3dc Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Fri, 30 Aug 2024 08:34:09 +0200
Subject: [PATCH 27/55] Added error handling test

---
 e2e-test-app/pom.xml                          |  1 +
 foundation-models/openai/pom.xml              |  5 ++++
 .../openai/OpenAiStreamingHandler.java        |  9 +++++--
 .../openai/OpenAiClientTest.java              | 24 +++++++++++++++----
 pom.xml                                       |  5 ++++
 5 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/e2e-test-app/pom.xml b/e2e-test-app/pom.xml
index aed6b77f..1a0a56b8 100644
--- a/e2e-test-app/pom.xml
+++ b/e2e-test-app/pom.xml
@@ -90,6 +90,7 @@
     <dependency>
       <groupId>org.springframework</groupId>
       <artifactId>spring-webmvc</artifactId>
+      <version>${springframework.version}</version>
     </dependency>
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml
index b1319ac0..a9289772 100644
--- a/foundation-models/openai/pom.xml
+++ b/foundation-models/openai/pom.xml
@@ -97,6 +97,11 @@
       <artifactId>junit-jupiter-api</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.wiremock</groupId>
       <artifactId>wiremock</artifactId>
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 89cf969a..ef3b5c19 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -69,7 +69,9 @@ private Stream<D> parseResponse(@Nonnull final ClassicHttpResponse response)
         .peek(
             responseLine -> {
               if (!responseLine.startsWith("data: ")) {
-                parseErrorAndThrow(responseLine, new OpenAiClientException());
+                parseErrorAndThrow(
+                    responseLine,
+                    new OpenAiClientException("Failed to parse response from OpenAI model"));
               }
             })
         .map(
@@ -78,7 +80,10 @@ private Stream<D> parseResponse(@Nonnull final ClassicHttpResponse response)
               try {
                 return JACKSON.readValue(data, deltaType);
               } catch (final IOException e) {
-                throw new OpenAiClientException("Failed to parse delta message: " + data, e);
+                log.error(
+                    "Failed to parse the following response from OpenAI model: {}", responseLine);
+                throw new OpenAiClientException(
+                    "Failed to parse delta message: " + responseLine, e);
               }
             })
         .peek(
diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 6e968d8f..1c8b0d2f 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -21,10 +21,16 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.stream.Stream;
+import javax.annotation.Nonnull;
 import org.assertj.core.api.SoftAssertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestInstance.Lifecycle;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
+@TestInstance(Lifecycle.PER_CLASS)
 @WireMockTest
 class OpenAiClientTest {
   private OpenAiClient client;
@@ -55,8 +61,19 @@ void apiVersion() {
     verify(exactly(2), postRequestedFor(anyUrl()).withoutQueryParam("api-version"));
   }
 
-  @Test
-  void errorHandling() {
+  private Stream<Runnable> chatCompletionCalls() {
+    return Stream.of(
+        () -> client.chatCompletion(new OpenAiChatCompletionParameters()),
+        () ->
+            client
+                .streamChatCompletion(new OpenAiChatCompletionParameters())
+                // the stream needs to be consumed to parse the response
+                .forEach(System.out::println));
+  }
+
+  @ParameterizedTest
+  @MethodSource("chatCompletionCalls")
+  void chatCompletionErrorHandling(@Nonnull final Runnable request) {
     final var errorJson =
         """
             { "error": { "code": null, "message": "foo", "type": "invalid stuff" } }
@@ -91,7 +108,6 @@ void errorHandling() {
             .willSetStateTo("4"));
     stubFor(post(anyUrl()).inScenario("Errors").whenScenarioStateIs("4").willReturn(noContent()));
 
-    final Runnable request = () -> client.chatCompletion(new OpenAiChatCompletionParameters());
     final var softly = new SoftAssertions();
 
     softly
@@ -255,7 +271,7 @@ void embedding() throws IOException {
           postRequestedFor(urlPathEqualTo("/embeddings"))
               .withRequestBody(
                   equalToJson("""
-                          {"input":["Hello World"]}""")));
+                      {"input":["Hello World"]}""")));
     }
   }
 
diff --git a/pom.xml b/pom.xml
index c9dd00da..8840341b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -91,6 +91,11 @@
         <version>${junit-jupiter.version}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.junit.jupiter</groupId>
+        <artifactId>junit-jupiter-params</artifactId>
+        <version>${junit-jupiter.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.wiremock</groupId>
         <artifactId>wiremock</artifactId>

From c709d31d31dedca48c2ac37fca4c0e01e2652275 Mon Sep 17 00:00:00 2001
From: Matthias Kuhr <matthias.kuhr@sap.com>
Date: Fri, 30 Aug 2024 15:29:36 +0200
Subject: [PATCH 28/55] Updates from pair review / discussion

---
 .../sdk/app/controllers/OpenAiController.java | 64 +++++++++----------
 .../ai/sdk/app/controllers/OpenAiTest.java    | 64 +++++++++----------
 .../foundationmodels/openai/OpenAiClient.java |  8 +++
 3 files changed, 69 insertions(+), 67 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index c098fbc7..7ef4f702 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -22,7 +22,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Consumer;
+import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.MediaType;
@@ -67,51 +67,51 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                         .addText(
                             "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
+    var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request);
+
     final var emitter = new ResponseBodyEmitter();
-    final var totalOutput = new OpenAiChatCompletionOutput();
-    final var consumer =
-        new Consumer<OpenAiDeltaChatCompletion>() {
-          @Override
-          public void accept(@Nonnull final OpenAiDeltaChatCompletion delta) {
-            totalOutput.addDelta(delta);
-            send(emitter, delta.getDeltaContent());
+
+    Runnable r =
+        () -> {
+          final var totalOutput = new OpenAiChatCompletionOutput();
+
+          try {
+            stream
+                .peek(totalOutput::addDelta)
+                // foreach consumes all elements, closing the stream at the end
+                .forEach(delta -> send(emitter, delta.getDeltaContent()));
+            send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
+            emitter.complete();
+          } catch (RuntimeException e) {
+            emitter.completeWithError(e);
+          } finally {
+            stream.close();
           }
         };
 
-    // Cloud SDK's ThreadContext is vital for the request to successfully execute.
-    ThreadContextExecutors.getExecutor()
-        .submit(
-            () -> {
-              streamToConsumer(request, consumer);
-              send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
-              emitter.complete();
-            });
+    ThreadContextExecutors.getExecutor().submit(r);
+
     // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
+  private static void consume(
+      Stream<OpenAiDeltaChatCompletion> stream, ResponseBodyEmitter emitter) {
+    final var totalOutput = new OpenAiChatCompletionOutput();
+
+    stream.peek(totalOutput::addDelta).forEach(delta -> send(emitter, delta.getDeltaContent()));
+
+    send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
+    emitter.complete();
+  }
+
   private static void send(
       @Nonnull final ResponseBodyEmitter emitter, @Nonnull final String chunk) {
     try {
       emitter.send(chunk);
     } catch (final IOException e) {
       log.error(Arrays.toString(e.getStackTrace()));
-      emitter.completeWithError(e);
-    }
-  }
-
-  /**
-   * Streams the OpenAI chat completion into the accept method of the consumer.
-   *
-   * @param request the chat completion request
-   * @param consumer the consumer that asynchronously accepts the chat completion
-   */
-  static void streamToConsumer(
-      @Nonnull final OpenAiChatCompletionParameters request,
-      @Nonnull final Consumer<OpenAiDeltaChatCompletion> consumer) {
-    // try-with-resources ensures that the stream is closed after the response is sent.
-    try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
-      stream.forEach(consumer);
+      throw new RuntimeException(e);
     }
   }
 
diff --git a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
index c828edd9..f945c319 100644
--- a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
+++ b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
@@ -2,13 +2,6 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiDeltaChatCompletion;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Consumer;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
 
@@ -34,34 +27,35 @@ void chatCompletionImage() {
 
   @Test
   void streamChatCompletion() {
-    final var request =
-        new OpenAiChatCompletionParameters()
-            .setMessages(List.of(new OpenAiChatUserMessage().addText("Who is the prettiest?")));
-
-    final var totalOutput = new OpenAiChatCompletionOutput();
-    final var emptyDeltaCount = new AtomicInteger(0);
-    var consumer =
-        new Consumer<OpenAiDeltaChatCompletion>() {
-          @Override
-          public void accept(OpenAiDeltaChatCompletion delta) {
-            totalOutput.addDelta(delta);
-            final String deltaContent = delta.getDeltaContent();
-            log.info("deltaContent: {}", deltaContent);
-            if (deltaContent.isEmpty()) {
-              emptyDeltaCount.incrementAndGet();
-            }
-          }
-        };
-    OpenAiController.streamToConsumer(request, consumer);
-
-    // the first two and the last delta don't have any content
-    // see OpenAiDeltaChatCompletion#getDeltaContent
-    assertThat(emptyDeltaCount.get()).isLessThanOrEqualTo(3);
-
-    assertThat(totalOutput.getChoices()).isNotEmpty();
-    assertThat(totalOutput.getChoices().get(0).getMessage().getContent()).isNotEmpty();
-    assertThat(totalOutput.getPromptFilterResults()).isNotNull();
-    assertThat(totalOutput.getChoices().get(0).getContentFilterResults()).isNotNull();
+    //    final var request =
+    //        new OpenAiChatCompletionParameters()
+    //            .setMessages(List.of(new OpenAiChatUserMessage().addText("Who is the
+    // prettiest?")));
+    //
+    //    final var totalOutput = new OpenAiChatCompletionOutput();
+    //    final var emptyDeltaCount = new AtomicInteger(0);
+    //    var consumer =
+    //        new Consumer<OpenAiDeltaChatCompletion>() {
+    //          @Override
+    //          public void accept(OpenAiDeltaChatCompletion delta) {
+    //            totalOutput.addDelta(delta);
+    //            final String deltaContent = delta.getDeltaContent();
+    //            log.info("deltaContent: {}", deltaContent);
+    //            if (deltaContent.isEmpty()) {
+    //              emptyDeltaCount.incrementAndGet();
+    //            }
+    //          }
+    //        };
+    //    OpenAiController.streamToConsumer(request, consumer);
+    //
+    //    // the first two and the last delta don't have any content
+    //    // see OpenAiDeltaChatCompletion#getDeltaContent
+    //    assertThat(emptyDeltaCount.get()).isLessThanOrEqualTo(3);
+    //
+    //    assertThat(totalOutput.getChoices()).isNotEmpty();
+    //    assertThat(totalOutput.getChoices().get(0).getMessage().getContent()).isNotEmpty();
+    //    assertThat(totalOutput.getPromptFilterResults()).isNotNull();
+    //    assertThat(totalOutput.getChoices().get(0).getContentFilterResults()).isNotNull();
   }
 
   @Test
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index d43deaff..73acb05d 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -123,6 +123,14 @@ public Stream<OpenAiDeltaChatCompletion> streamChatCompletion(
     return streamChatCompletion("/chat/completions", parameters, OpenAiDeltaChatCompletion.class);
   }
 
+  @Nonnull
+  public Stream<String> streamChatCompletionSimpleEasyMode(
+      @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
+    return streamChatCompletion(parameters)
+        .filter(it -> !"content_filter".equalsIgnoreCase(it.getFinishReason()))
+        .map(OpenAiDeltaChatCompletion::getDeltaContent);
+  }
+
   /**
    * Get a vector representation of a given input that can be easily consumed by machine learning
    * models and algorithms.

From 73031d1ce7aad7bc91f7b3b72e6f3612c67ec3c1 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Mon, 2 Sep 2024 10:24:19 +0200
Subject: [PATCH 29/55] Cleanup + streamChatCompletion doesn't throw

---
 .../sdk/app/controllers/OpenAiController.java | 55 +++++++++++-----
 .../ai/sdk/app/controllers/OpenAiTest.java    | 63 ++++++++++---------
 .../foundationmodels/openai/OpenAiClient.java | 28 ++++++---
 .../openai/OpenAiStreamingHandler.java        | 12 ----
 ...on.java => OpenAiChatCompletionDelta.java} |  2 +-
 .../model/OpenAiChatCompletionOutput.java     |  4 +-
 .../openai/model/OpenAiCompletionOutput.java  |  2 +-
 .../openai/OpenAiClientTest.java              |  6 +-
 8 files changed, 102 insertions(+), 70 deletions(-)
 rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/{OpenAiDeltaChatCompletion.java => OpenAiChatCompletionDelta.java} (96%)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 7ef4f702..b9d5386a 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -14,7 +14,6 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionTool;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage.ImageDetailLevel;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiDeltaChatCompletion;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.cloud.sdk.cloudplatform.thread.ThreadContextExecutors;
@@ -22,7 +21,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.MediaType;
@@ -55,7 +53,7 @@ public static OpenAiChatCompletionOutput chatCompletion() {
    *
    * @return the emitter that streams the assistant message response
    */
-  @SuppressWarnings("unused") // #streamToConsumer is the method that is tested
+  @SuppressWarnings("unused") // The e2e test doesn't use this method
   @GetMapping("/streamChatCompletion")
   @Nonnull
   public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
@@ -67,42 +65,70 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                         .addText(
                             "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-    var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request);
+    final var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request);
 
     final var emitter = new ResponseBodyEmitter();
 
-    Runnable r =
+    final Runnable consumeStream =
         () -> {
           final var totalOutput = new OpenAiChatCompletionOutput();
 
           try {
             stream
                 .peek(totalOutput::addDelta)
-                // foreach consumes all elements, closing the stream at the end
                 .forEach(delta -> send(emitter, delta.getDeltaContent()));
             send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
             emitter.complete();
           } catch (RuntimeException e) {
-            emitter.completeWithError(e);
+            emitter.completeWithError(e.getCause());
           } finally {
             stream.close();
           }
         };
 
-    ThreadContextExecutors.getExecutor().submit(r);
+    ThreadContextExecutors.getExecutor().submit(consumeStream);
 
     // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
-  private static void consume(
-      Stream<OpenAiDeltaChatCompletion> stream, ResponseBodyEmitter emitter) {
-    final var totalOutput = new OpenAiChatCompletionOutput();
+  /**
+   * Asynchronous stream of an OpenAI chat request
+   *
+   * @return the emitter that streams the assistant message response
+   */
+  @SuppressWarnings("unused")
+  @GetMapping("/simpleStreamChatCompletion")
+  @Nonnull
+  public static ResponseEntity<ResponseBodyEmitter> simpleStreamChatCompletion() {
+    final var request =
+        new OpenAiChatCompletionParameters()
+            .setMessages(
+                List.of(
+                    new OpenAiChatUserMessage()
+                        .addText(
+                            "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-    stream.peek(totalOutput::addDelta).forEach(delta -> send(emitter, delta.getDeltaContent()));
+    final var stream = OpenAiClient.forModel(GPT_35_TURBO).simpleStreamChatCompletion(request);
 
-    send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
-    emitter.complete();
+    final var emitter = new ResponseBodyEmitter();
+
+    final Runnable consumeStream =
+        () -> {
+          try {
+            stream.forEach(deltaMessage -> send(emitter, deltaMessage));
+            emitter.complete();
+          } catch (RuntimeException e) {
+            emitter.completeWithError(e.getCause());
+          } finally {
+            stream.close();
+          }
+        };
+
+    ThreadContextExecutors.getExecutor().submit(consumeStream);
+
+    // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
+    return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
   private static void send(
@@ -111,6 +137,7 @@ private static void send(
       emitter.send(chunk);
     } catch (final IOException e) {
       log.error(Arrays.toString(e.getStackTrace()));
+      // only RuntimeExceptions can stop a stream.forEach()
       throw new RuntimeException(e);
     }
   }
diff --git a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
index f945c319..273b4101 100644
--- a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
+++ b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
@@ -1,7 +1,14 @@
 package com.sap.ai.sdk.app.controllers;
 
+import static com.sap.ai.sdk.foundationmodels.openai.OpenAiModel.GPT_35_TURBO;
 import static org.assertj.core.api.Assertions.assertThat;
 
+import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.Test;
 
@@ -27,35 +34,33 @@ void chatCompletionImage() {
 
   @Test
   void streamChatCompletion() {
-    //    final var request =
-    //        new OpenAiChatCompletionParameters()
-    //            .setMessages(List.of(new OpenAiChatUserMessage().addText("Who is the
-    // prettiest?")));
-    //
-    //    final var totalOutput = new OpenAiChatCompletionOutput();
-    //    final var emptyDeltaCount = new AtomicInteger(0);
-    //    var consumer =
-    //        new Consumer<OpenAiDeltaChatCompletion>() {
-    //          @Override
-    //          public void accept(OpenAiDeltaChatCompletion delta) {
-    //            totalOutput.addDelta(delta);
-    //            final String deltaContent = delta.getDeltaContent();
-    //            log.info("deltaContent: {}", deltaContent);
-    //            if (deltaContent.isEmpty()) {
-    //              emptyDeltaCount.incrementAndGet();
-    //            }
-    //          }
-    //        };
-    //    OpenAiController.streamToConsumer(request, consumer);
-    //
-    //    // the first two and the last delta don't have any content
-    //    // see OpenAiDeltaChatCompletion#getDeltaContent
-    //    assertThat(emptyDeltaCount.get()).isLessThanOrEqualTo(3);
-    //
-    //    assertThat(totalOutput.getChoices()).isNotEmpty();
-    //    assertThat(totalOutput.getChoices().get(0).getMessage().getContent()).isNotEmpty();
-    //    assertThat(totalOutput.getPromptFilterResults()).isNotNull();
-    //    assertThat(totalOutput.getChoices().get(0).getContentFilterResults()).isNotNull();
+    final var request =
+        new OpenAiChatCompletionParameters()
+            .setMessages(List.of(new OpenAiChatUserMessage().addText("Who is the prettiest?")));
+
+    final var totalOutput = new OpenAiChatCompletionOutput();
+    final var emptyDeltaCount = new AtomicInteger(0);
+    OpenAiClient.forModel(GPT_35_TURBO)
+        .streamChatCompletion(request)
+        .peek(totalOutput::addDelta)
+        // foreach consumes all elements, closing the stream at the end
+        .forEach(
+            delta -> {
+              final String deltaContent = delta.getDeltaContent();
+              log.info("deltaContent: {}", deltaContent);
+              if (deltaContent.isEmpty()) {
+                emptyDeltaCount.incrementAndGet();
+              }
+            });
+
+    // the first two and the last delta don't have any content
+    // see OpenAiChatCompletionDelta#getDeltaContent
+    assertThat(emptyDeltaCount.get()).isLessThanOrEqualTo(3);
+
+    assertThat(totalOutput.getChoices()).isNotEmpty();
+    assertThat(totalOutput.getChoices().get(0).getMessage().getContent()).isNotEmpty();
+    assertThat(totalOutput.getPromptFilterResults()).isNotNull();
+    assertThat(totalOutput.getChoices().get(0).getContentFilterResults()).isNotNull();
   }
 
   @Test
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 73acb05d..7f2b4af4 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -7,9 +7,9 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.sap.ai.sdk.core.Core;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionDelta;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiDeltaChatCompletion;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
@@ -113,22 +113,34 @@ public OpenAiChatCompletionOutput chatCompletion(
    *
    * @param parameters the prompt, including messages and other parameters.
    * @return A stream of chat completions deltas
-   * @throws OpenAiClientException if the request fails or if the finish reason is content_filter or
-   *     length (token limit).
+   * @throws OpenAiClientException if the request fails
    */
   @Nonnull
-  public Stream<OpenAiDeltaChatCompletion> streamChatCompletion(
+  public Stream<OpenAiChatCompletionDelta> streamChatCompletion(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
     parameters.setStream(true);
-    return streamChatCompletion("/chat/completions", parameters, OpenAiDeltaChatCompletion.class);
+    return streamChatCompletion("/chat/completions", parameters, OpenAiChatCompletionDelta.class);
   }
 
+  /**
+   * Generate a completion for the given prompt.
+   *
+   * @param parameters the prompt, including messages and other parameters.
+   * @return A stream of message deltas
+   * @throws OpenAiClientException if the request fails or if the finish reason is content_filter
+   */
   @Nonnull
-  public Stream<String> streamChatCompletionSimpleEasyMode(
+  public Stream<String> simpleStreamChatCompletion(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
     return streamChatCompletion(parameters)
-        .filter(it -> !"content_filter".equalsIgnoreCase(it.getFinishReason()))
-        .map(OpenAiDeltaChatCompletion::getDeltaContent);
+        .peek(
+            delta -> {
+              final String finishReason = delta.getFinishReason();
+              if (finishReason != null && finishReason.equals("content_filter")) {
+                throw new OpenAiClientException("Content filter filtered the output.");
+              }
+            })
+        .map(OpenAiChatCompletionDelta::getDeltaContent);
   }
 
   /**
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index ef3b5c19..0af6f6ed 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -85,18 +85,6 @@ private Stream<D> parseResponse(@Nonnull final ClassicHttpResponse response)
                 throw new OpenAiClientException(
                     "Failed to parse delta message: " + responseLine, e);
               }
-            })
-        .peek(
-            delta -> {
-              final String finishReason = delta.getFinishReason();
-              if (finishReason != null) {
-                if (finishReason.equals("content_filter")) {
-                  throw new OpenAiClientException("Content filter filtered the output.");
-                } else if (finishReason.equals("length")) {
-                  throw new OpenAiClientException(
-                      "Incomplete output due to max_tokens parameter or token limit.");
-                }
-              }
             });
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
similarity index 96%
rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
index f8049c82..1136e066 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletion.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionDelta.java
@@ -13,7 +13,7 @@
 @Accessors(chain = true)
 @EqualsAndHashCode(callSuper = true)
 @ToString(callSuper = true)
-public class OpenAiDeltaChatCompletion extends OpenAiCompletionOutput implements StreamedDelta {
+public class OpenAiChatCompletionDelta extends OpenAiCompletionOutput implements StreamedDelta {
   /** List of result candidates. */
   @JsonProperty("choices")
   @Getter(onMethod_ = @Nonnull)
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
index 2469d0cf..449f3e8f 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionOutput.java
@@ -14,7 +14,7 @@
 @EqualsAndHashCode(callSuper = true)
 @ToString(callSuper = true)
 public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
-    implements DeltaAggregatable<OpenAiDeltaChatCompletion> {
+    implements DeltaAggregatable<OpenAiChatCompletionDelta> {
   /** List of result candidates. */
   @JsonProperty("choices")
   @Getter(onMethod_ = @Nonnull)
@@ -33,7 +33,7 @@ public class OpenAiChatCompletionOutput extends OpenAiCompletionOutput
    *
    * @param delta the delta to add.
    */
-  public void addDelta(@Nonnull final OpenAiDeltaChatCompletion delta) {
+  public void addDelta(@Nonnull final OpenAiChatCompletionDelta delta) {
     super.addDelta(delta);
 
     if (delta.getSystemFingerprint() != null) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
index 35e401d9..8b856910 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
@@ -44,7 +44,7 @@ public class OpenAiCompletionOutput {
   @Getter(onMethod_ = @Nullable)
   private List<OpenAiPromptFilterResult> promptFilterResults;
 
-  void addDelta(@Nonnull final OpenAiDeltaChatCompletion delta) {
+  void addDelta(@Nonnull final OpenAiChatCompletionDelta delta) {
     created = delta.getCreated();
     id = delta.getId();
     model = delta.getModel();
diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 1c8b0d2f..320b7331 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -9,12 +9,12 @@
 import com.github.tomakehurst.wiremock.junit5.WireMockTest;
 import com.github.tomakehurst.wiremock.stubbing.Scenario;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionChoice;
+import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionDelta;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiContentFilterPromptResults;
-import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiDeltaChatCompletion;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import io.vavr.control.Try;
@@ -294,10 +294,10 @@ void streamChatCompletionChatCompletion() throws IOException {
                           .addText(
                               "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-      try (Stream<OpenAiDeltaChatCompletion> stream = client.streamChatCompletion(request)) {
+      try (Stream<OpenAiChatCompletionDelta> stream = client.streamChatCompletion(request)) {
 
         OpenAiChatCompletionOutput totalOutput = new OpenAiChatCompletionOutput();
-        final List<OpenAiDeltaChatCompletion> deltaList =
+        final List<OpenAiChatCompletionDelta> deltaList =
             stream.peek(totalOutput::addDelta).toList();
 
         assertThat(deltaList).hasSize(5);

From 6b1bfd07fb04b297cfc013ae33135050dfd7b864 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Mon, 2 Sep 2024 10:27:44 +0200
Subject: [PATCH 30/55] PMD

---
 .../ai/sdk/app/controllers/OpenAiController.java | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index b9d5386a..47fd43ab 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -8,6 +8,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient;
+import com.sap.ai.sdk.foundationmodels.openai.OpenAiClientException;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionFunction;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
@@ -72,17 +73,14 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
     final Runnable consumeStream =
         () -> {
           final var totalOutput = new OpenAiChatCompletionOutput();
-
-          try {
+          try (stream) {
             stream
                 .peek(totalOutput::addDelta)
                 .forEach(delta -> send(emitter, delta.getDeltaContent()));
             send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
             emitter.complete();
-          } catch (RuntimeException e) {
+          } catch (OpenAiClientException e) {
             emitter.completeWithError(e.getCause());
-          } finally {
-            stream.close();
           }
         };
 
@@ -115,13 +113,11 @@ public static ResponseEntity<ResponseBodyEmitter> simpleStreamChatCompletion() {
 
     final Runnable consumeStream =
         () -> {
-          try {
+          try (stream) {
             stream.forEach(deltaMessage -> send(emitter, deltaMessage));
             emitter.complete();
-          } catch (RuntimeException e) {
+          } catch (OpenAiClientException e) {
             emitter.completeWithError(e.getCause());
-          } finally {
-            stream.close();
           }
         };
 
@@ -138,7 +134,7 @@ private static void send(
     } catch (final IOException e) {
       log.error(Arrays.toString(e.getStackTrace()));
       // only RuntimeExceptions can stop a stream.forEach()
-      throw new RuntimeException(e);
+      throw new OpenAiClientException(e);
     }
   }
 

From 23474baadd5e0cc9284d6bc6041d098c01bf0ed2 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Mon, 2 Sep 2024 16:22:24 +0200
Subject: [PATCH 31/55] Added errorHandling test

---
 README.md                                     | 73 ++++++++++--------
 .../sdk/app/controllers/OpenAiController.java | 33 ++++----
 foundation-models/openai/pom.xml              |  5 ++
 .../openai/OpenAiResponseHandler.java         |  4 +-
 .../openai/OpenAiStreamingHandler.java        |  8 ++
 .../openai/OpenAiClientTest.java              | 77 +++++++++++++++++--
 .../resources/streamChatCompletionError.txt   |  2 +
 pom.xml                                       |  7 ++
 8 files changed, 147 insertions(+), 62 deletions(-)
 create mode 100644 foundation-models/openai/src/test/resources/streamChatCompletionError.txt

diff --git a/README.md b/README.md
index 90a02c10..714865af 100644
--- a/README.md
+++ b/README.md
@@ -208,7 +208,7 @@ final OpenAiChatCompletionOutput result =
 
 #### Framework-agnostic example
 ```java
-public ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
+public static ReturnType streamChatCompletion() {
   final var request =
       new OpenAiChatCompletionParameters()
           .setMessages(
@@ -217,29 +217,31 @@ public ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                       .addText(
                           "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-  // Cloud SDK's ThreadContext is vital for the request to successfully execute.
-  ThreadContextExecutors.getExecutor()
-      .submit(
-          () -> {
-            final var totalOutput = new OpenAiChatCompletionOutput();
-
-            // try-with-resources ensures that the stream is closed after the response is sent.
-            try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
-              stream
-                  // optional: collect all deltas
-                  .peek(totalOutput::addDelta)
-                  // send is defined by your framework
-                  .forEach(delta -> send(delta.getDeltaContent()));
-            }
-            // optional: totalOutput can be looked at here
-          });
-  // Note: set the header content-type to text/event-stream on the sent response
+  final var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request);
+
+  final Runnable consumeStream =
+      () -> {
+        final var totalOutput = new OpenAiChatCompletionOutput();
+        // try-with-resources ensures the stream is closed
+        try (stream) {
+          stream
+              // optional: collect all deltas
+              .peek(totalOutput::addDelta)
+              // send is defined by your framework
+              .forEach(delta -> send(delta.getDeltaContent()));
+        }
+        // optional: totalOutput can be looked at here
+      };
+
+  ThreadContextExecutors.getExecutor().submit(consumeStream);
+
+  // Note: set the header content-type:text/event-stream on the sent response
 }
 ```
 
 #### Spring Boot example
 ```java
-public ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
+public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
   final var request =
       new OpenAiChatCompletionParameters()
           .setMessages(
@@ -248,19 +250,19 @@ public ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                       .addText(
                           "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
+  final var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request);
+
   final var emitter = new ResponseBodyEmitter();
 
-  // Cloud SDK's ThreadContext is vital for the request to successfully execute.
-  ThreadContextExecutors.getExecutor()
-      .submit(
-          () -> {
-            final var totalOutput = new OpenAiChatCompletionOutput();
-
-            // try-with-resources ensures that the stream is closed after the response is sent.
-            try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
-              stream
-                  .peek(totalOutput::addDelta) // optional: collect all deltas
-                  .forEach(delta -> {
+  final Runnable consumeStream =
+      () -> {
+        final var totalOutput = new OpenAiChatCompletionOutput();
+        // try-with-resources ensures the stream is closed
+        try (stream) {
+          stream
+              .peek(totalOutput::addDelta) // optional: collect all deltas
+              .forEach(
+                  delta -> {
                     try {
                       emitter.send(delta.getDeltaContent());
                     } catch (final IOException e) {
@@ -268,10 +270,13 @@ public ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                       emitter.completeWithError(e);
                     }
                   });
-            }
-            // optional: totalOutput can be looked at here
-            emitter.complete();
-          });
+        }
+        // optional: totalOutput can be looked at here
+        emitter.complete();
+      };
+
+  ThreadContextExecutors.getExecutor().submit(consumeStream);
+
   // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
   return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
 }
diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 47fd43ab..43549d3f 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -8,7 +8,6 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient;
-import com.sap.ai.sdk.foundationmodels.openai.OpenAiClientException;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionFunction;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatCompletionParameters;
@@ -54,7 +53,7 @@ public static OpenAiChatCompletionOutput chatCompletion() {
    *
    * @return the emitter that streams the assistant message response
    */
-  @SuppressWarnings("unused") // The e2e test doesn't use this method
+  @SuppressWarnings("unused") // The end-to-end test doesn't use this method
   @GetMapping("/streamChatCompletion")
   @Nonnull
   public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
@@ -73,15 +72,14 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
     final Runnable consumeStream =
         () -> {
           final var totalOutput = new OpenAiChatCompletionOutput();
+          // try-with-resources ensures the stream is closed
           try (stream) {
             stream
                 .peek(totalOutput::addDelta)
                 .forEach(delta -> send(emitter, delta.getDeltaContent()));
-            send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
-            emitter.complete();
-          } catch (OpenAiClientException e) {
-            emitter.completeWithError(e.getCause());
           }
+          send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
+          emitter.complete();
         };
 
     ThreadContextExecutors.getExecutor().submit(consumeStream);
@@ -90,12 +88,20 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
   }
 
+  private static String objectToJson(@Nonnull final Object obj) {
+    try {
+      return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(obj);
+    } catch (final JsonProcessingException ignored) {
+      return "Could not parse object to JSON";
+    }
+  }
+
   /**
    * Asynchronous stream of an OpenAI chat request
    *
    * @return the emitter that streams the assistant message response
    */
-  @SuppressWarnings("unused")
+  @SuppressWarnings("unused") // The end-to-end test doesn't use this method
   @GetMapping("/simpleStreamChatCompletion")
   @Nonnull
   public static ResponseEntity<ResponseBodyEmitter> simpleStreamChatCompletion() {
@@ -116,8 +122,6 @@ public static ResponseEntity<ResponseBodyEmitter> simpleStreamChatCompletion() {
           try (stream) {
             stream.forEach(deltaMessage -> send(emitter, deltaMessage));
             emitter.complete();
-          } catch (OpenAiClientException e) {
-            emitter.completeWithError(e.getCause());
           }
         };
 
@@ -133,16 +137,7 @@ private static void send(
       emitter.send(chunk);
     } catch (final IOException e) {
       log.error(Arrays.toString(e.getStackTrace()));
-      // only RuntimeExceptions can stop a stream.forEach()
-      throw new OpenAiClientException(e);
-    }
-  }
-
-  private static String objectToJson(@Nonnull final Object obj) {
-    try {
-      return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(obj);
-    } catch (final JsonProcessingException ignored) {
-      return "Could not parse object to JSON";
+      emitter.completeWithError(e);
     }
   }
 
diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml
index a9289772..adaf0052 100644
--- a/foundation-models/openai/pom.xml
+++ b/foundation-models/openai/pom.xml
@@ -112,5 +112,10 @@
       <artifactId>assertj-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-junit-jupiter</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
index 0946c405..4e6ed09b 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
@@ -72,7 +72,7 @@ static void buildExceptionAndThrow(@Nonnull final ClassicHttpResponse response)
       throws OpenAiClientException {
     final var exception =
         new OpenAiClientException(
-            "Request to OpenAI model failed with status %s %s "
+            "Request to OpenAI model failed with status %s %s"
                 .formatted(response.getCode(), response.getReasonPhrase()));
     final var entity = response.getEntity();
     if (entity == null) {
@@ -117,6 +117,6 @@ static void parseErrorAndThrow(
       throw baseException;
     }
     throw new OpenAiClientException(
-        baseException.getMessage() + "and error message: '%s'".formatted(error.getMessage()));
+        baseException.getMessage() + " and error message: '%s'".formatted(error.getMessage()));
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 0af6f6ed..c067518f 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -85,6 +85,14 @@ private Stream<D> parseResponse(@Nonnull final ClassicHttpResponse response)
                 throw new OpenAiClientException(
                     "Failed to parse delta message: " + responseLine, e);
               }
+            })
+        .onClose(
+            () -> {
+              try {
+                inputStream.close();
+              } catch (IOException e) {
+                log.error("Could not close HTTP input stream", e);
+              }
             });
   }
 }
diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 320b7331..753c5cb6 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -3,6 +3,12 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.*;
 import static com.sap.ai.sdk.foundationmodels.openai.model.OpenAiContentFilterSeverityResult.Severity.SAFE;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 
 import com.fasterxml.jackson.core.JsonParseException;
 import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
@@ -16,12 +22,18 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiContentFilterPromptResults;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
+import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor;
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import io.vavr.control.Try;
 import java.io.IOException;
 import java.util.List;
+import java.util.Objects;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.io.entity.InputStreamEntity;
+import org.apache.hc.core5.http.message.BasicClassicHttpResponse;
 import org.assertj.core.api.SoftAssertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -29,6 +41,7 @@
 import org.junit.jupiter.api.TestInstance.Lifecycle;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
 
 @TestInstance(Lifecycle.PER_CLASS)
 @WireMockTest
@@ -276,15 +289,63 @@ void embedding() throws IOException {
   }
 
   @Test
-  void streamChatCompletionChatCompletion() throws IOException {
+  void streamChatCompletionErrorHandling() throws IOException {
     try (var inputStream =
-        getClass().getClassLoader().getResourceAsStream("streamChatCompletion.txt")) {
+        spy(
+            Objects.requireNonNull(
+                getClass()
+                    .getClassLoader()
+                    .getResourceAsStream("streamChatCompletionError.txt")))) {
 
-      assert inputStream != null;
-      final String response = new String(inputStream.readAllBytes());
-      stubFor(
-          post(anyUrl())
-              .willReturn(ok().withBody(response).withHeader("Content-Type", "text/event-stream")));
+      final var httpClient = mock(HttpClient.class);
+      ApacheHttpClient5Accessor.setHttpClientFactory(destination -> httpClient);
+
+      // Create a mock response
+      final var mockResponse = new BasicClassicHttpResponse(200, "OK");
+      final var inputStreamEntity = new InputStreamEntity(inputStream, ContentType.TEXT_PLAIN);
+      mockResponse.setEntity(inputStreamEntity);
+      mockResponse.setHeader("Content-Type", "text/event-stream");
+
+      // Configure the HttpClient mock to return the mock response
+      doReturn(mockResponse).when(httpClient).executeOpen(any(), any(), any());
+
+      final var request =
+          new OpenAiChatCompletionParameters()
+              .setMessages(
+                  List.of(
+                      new OpenAiChatUserMessage()
+                          .addText(
+                              "Can you give me the first 100 numbers of the Fibonacci sequence?")));
+
+      try (Stream<OpenAiChatCompletionDelta> stream = client.streamChatCompletion(request)) {
+        assertThatThrownBy(() -> stream.forEach(System.out::println))
+            .isInstanceOf(OpenAiClientException.class)
+            .hasMessage(
+                "Failed to parse response from OpenAI model and error message: 'exceeded token rate limit'");
+      }
+
+      Mockito.verify(inputStream, atLeastOnce()).close();
+    }
+  }
+
+  @Test
+  void streamChatCompletion() throws IOException {
+    try (var inputStream =
+        spy(
+            Objects.requireNonNull(
+                getClass().getClassLoader().getResourceAsStream("streamChatCompletion.txt")))) {
+
+      final var httpClient = mock(HttpClient.class);
+      ApacheHttpClient5Accessor.setHttpClientFactory(destination -> httpClient);
+
+      // Create a mock response
+      final var mockResponse = new BasicClassicHttpResponse(200, "OK");
+      final var inputStreamEntity = new InputStreamEntity(inputStream, ContentType.TEXT_PLAIN);
+      mockResponse.setEntity(inputStreamEntity);
+      mockResponse.setHeader("Content-Type", "text/event-stream");
+
+      // Configure the HttpClient mock to return the mock response
+      doReturn(mockResponse).when(httpClient).executeOpen(any(), any(), any());
 
       final var request =
           new OpenAiChatCompletionParameters()
@@ -380,6 +441,8 @@ void streamChatCompletionChatCompletion() throws IOException {
         assertThat(totalOutput.getPromptFilterResults()).isNotNull();
         assertFilter(totalOutput.getPromptFilterResults().get(0).getContentFilterResults());
       }
+
+      Mockito.verify(inputStream, atLeastOnce()).close();
     }
   }
 
diff --git a/foundation-models/openai/src/test/resources/streamChatCompletionError.txt b/foundation-models/openai/src/test/resources/streamChatCompletionError.txt
new file mode 100644
index 00000000..174d47c0
--- /dev/null
+++ b/foundation-models/openai/src/test/resources/streamChatCompletionError.txt
@@ -0,0 +1,2 @@
+data: {"choices":[],"created":0,"id":"","model":"","object":"","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}]}
+{"error":{"code":"429","message":"exceeded token rate limit"}}
diff --git a/pom.xml b/pom.xml
index 8840341b..cc435d12 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,6 +57,7 @@
     <surefire.version>3.4.0</surefire.version>
     <system-stubs.version>2.1.3</system-stubs.version>
     <springframework.version>6.1.12</springframework.version>
+    <mockito.version>5.12.0</mockito.version>
     <!--  Quality assurance  -->
     <enforcer.skipBanLoggingFrameworks>false</enforcer.skipBanLoggingFrameworks>
     <enforcer.skipEnforceScopeLoggerBridges>false</enforcer.skipEnforceScopeLoggerBridges>
@@ -108,6 +109,12 @@
         <version>${assertj-core.version}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.mockito</groupId>
+        <artifactId>mockito-junit-jupiter</artifactId>
+        <version>${mockito.version}</version>
+        <scope>test</scope>
+      </dependency>
       <!-- Modules BoM -->
       <dependency>
         <groupId>com.sap.ai.sdk</groupId>

From 769cd7da92954426c79db0841e3c3b82daadada6 Mon Sep 17 00:00:00 2001
From: Charles Dubois <103174266+CharlesDuboisSAP@users.noreply.github.com>
Date: Tue, 3 Sep 2024 08:28:25 +0200
Subject: [PATCH 32/55] Apply suggestions from code review

Co-authored-by: Matthias Kuhr <52661546+MatKuhr@users.noreply.github.com>
---
 .../sap/ai/sdk/foundationmodels/openai/OpenAiClient.java | 2 +-
 .../foundationmodels/openai/OpenAiStreamingHandler.java  | 9 +--------
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 7f2b4af4..ce6fb809 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -112,7 +112,7 @@ public OpenAiChatCompletionOutput chatCompletion(
    * Generate a completion for the given prompt.
    *
    * @param parameters the prompt, including messages and other parameters.
-   * @return A stream of chat completions deltas
+   * @return A stream of chat completion delta elements.
    * @throws OpenAiClientException if the request fails
    */
   @Nonnull
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index c067518f..19e6959f 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -23,15 +23,8 @@ class OpenAiStreamingHandler<D extends StreamedDelta> {
 
   @Nonnull private final Class<D> deltaType;
 
-  /**
-   * Processes a {@link ClassicHttpResponse} and returns some value corresponding to that response.
-   *
-   * @param response The response to process
-   * @return A {@link Stream} of a model class instantiated from the response
-   * @throws OpenAiClientException in case of a problem or the connection was aborted
-   */
   @Nonnull
-  public Stream<D> handleResponse(@Nonnull final ClassicHttpResponse response)
+  Stream<D> handleResponse(@Nonnull final ClassicHttpResponse response)
       throws OpenAiClientException {
     if (response.getCode() >= 300) {
       buildExceptionAndThrow(response);

From 118dc6914e29724be2746b752e33dad52eb73133 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 3 Sep 2024 08:29:01 +0200
Subject: [PATCH 33/55] Dependency analyze

---
 foundation-models/openai/pom.xml | 2 +-
 pom.xml                          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml
index adaf0052..37721f18 100644
--- a/foundation-models/openai/pom.xml
+++ b/foundation-models/openai/pom.xml
@@ -114,7 +114,7 @@
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
-      <artifactId>mockito-junit-jupiter</artifactId>
+      <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
   </dependencies>
diff --git a/pom.xml b/pom.xml
index cc435d12..0e97658e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -111,7 +111,7 @@
       </dependency>
       <dependency>
         <groupId>org.mockito</groupId>
-        <artifactId>mockito-junit-jupiter</artifactId>
+        <artifactId>mockito-core</artifactId>
         <version>${mockito.version}</version>
         <scope>test</scope>
       </dependency>

From acd21c09c926f84288b9093f56df8bbb5c855ea0 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 3 Sep 2024 10:18:04 +0200
Subject: [PATCH 34/55] Review comments

---
 README.md                                     | 80 ++++++++++++++-----
 .../sdk/app/controllers/OpenAiController.java | 18 +++--
 .../src/main/resources/static/index.html      |  1 +
 .../ai/sdk/app/controllers/OpenAiTest.java    |  2 +-
 .../foundationmodels/openai/OpenAiClient.java | 37 ++++-----
 .../openai/OpenAiStreamingHandler.java        |  2 +-
 .../OpenAiDeltaChatCompletionChoice.java      |  2 +-
 .../openai/OpenAiClientTest.java              | 10 +--
 8 files changed, 97 insertions(+), 55 deletions(-)

diff --git a/README.md b/README.md
index 714865af..b4edd4e4 100644
--- a/README.md
+++ b/README.md
@@ -206,6 +206,52 @@ final OpenAiChatCompletionOutput result =
 
 ### Stream chat completion
 
+How to pass a stream of chat completion delta elements from the backend to the frontend in real-time.
+
+#### Print the stream to the console
+This is a simple example to get started:
+```java
+void printStream() {
+  final var request =
+      new OpenAiChatCompletionParameters()
+          .setMessages(
+              List.of(
+                  new OpenAiChatUserMessage()
+                      .addText(
+                          "Can you give me the first 100 numbers of the Fibonacci sequence?")));
+
+  // try-with-resources ensures the stream is closed
+  try(var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
+    stream.forEach(deltaMessage -> System.out.println(deltaMessage));
+  }
+}
+```
+
+<details>
+<summary>It's also possible to collect the total output</summary>
+<pre>
+
+```java
+void printStream() {
+  final var request =
+      new OpenAiChatCompletionParameters()
+          .setMessages(
+              List.of(
+                  new OpenAiChatUserMessage()
+                      .addText(
+                          "Can you give me the first 100 numbers of the Fibonacci sequence?")));
+
+  final var totalOutput = new OpenAiChatCompletionOutput();
+  // try-with-resources ensures the stream is closed
+  try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletionDeltas(request)) {
+    stream.peek(totalOutput::addDelta).forEach(delta -> System.out.println(delta));
+  }
+  // totalOutput can be looked at here
+}
+```
+</pre>
+</details>
+
 #### Framework-agnostic example
 ```java
 public static ReturnType streamChatCompletion() {
@@ -221,16 +267,11 @@ public static ReturnType streamChatCompletion() {
 
   final Runnable consumeStream =
       () -> {
-        final var totalOutput = new OpenAiChatCompletionOutput();
         // try-with-resources ensures the stream is closed
         try (stream) {
-          stream
-              // optional: collect all deltas
-              .peek(totalOutput::addDelta)
-              // send is defined by your framework
-              .forEach(delta -> send(delta.getDeltaContent()));
+          // send is defined by your framework
+          stream.forEach(delta -> send(delta));
         }
-        // optional: totalOutput can be looked at here
       };
 
   ThreadContextExecutors.getExecutor().submit(consumeStream);
@@ -256,23 +297,20 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
 
   final Runnable consumeStream =
       () -> {
-        final var totalOutput = new OpenAiChatCompletionOutput();
         // try-with-resources ensures the stream is closed
         try (stream) {
-          stream
-              .peek(totalOutput::addDelta) // optional: collect all deltas
-              .forEach(
-                  delta -> {
-                    try {
-                      emitter.send(delta.getDeltaContent());
-                    } catch (final IOException e) {
-                      log.error(Arrays.toString(e.getStackTrace()));
-                      emitter.completeWithError(e);
-                    }
-                  });
+          stream.forEach(
+              delta -> {
+                try {
+                  emitter.send(delta);
+                } catch (final IOException e) {
+                  log.error(Arrays.toString(e.getStackTrace()));
+                  emitter.completeWithError(e);
+                }
+              });
+        } finally {
+          emitter.complete();
         }
-        // optional: totalOutput can be looked at here
-        emitter.complete();
       };
 
   ThreadContextExecutors.getExecutor().submit(consumeStream);
diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 43549d3f..84610071 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -54,9 +54,9 @@ public static OpenAiChatCompletionOutput chatCompletion() {
    * @return the emitter that streams the assistant message response
    */
   @SuppressWarnings("unused") // The end-to-end test doesn't use this method
-  @GetMapping("/streamChatCompletion")
+  @GetMapping("/streamChatCompletionDeltas")
   @Nonnull
-  public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
+  public static ResponseEntity<ResponseBodyEmitter> streamChatCompletionDeltas() {
     final var request =
         new OpenAiChatCompletionParameters()
             .setMessages(
@@ -65,7 +65,7 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
                         .addText(
                             "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-    final var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request);
+    final var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletionDeltas(request);
 
     final var emitter = new ResponseBodyEmitter();
 
@@ -77,9 +77,10 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
             stream
                 .peek(totalOutput::addDelta)
                 .forEach(delta -> send(emitter, delta.getDeltaContent()));
+          } finally {
+            send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
+            emitter.complete();
           }
-          send(emitter, "\n\n-----Total Output-----\n\n" + objectToJson(totalOutput));
-          emitter.complete();
         };
 
     ThreadContextExecutors.getExecutor().submit(consumeStream);
@@ -102,9 +103,9 @@ private static String objectToJson(@Nonnull final Object obj) {
    * @return the emitter that streams the assistant message response
    */
   @SuppressWarnings("unused") // The end-to-end test doesn't use this method
-  @GetMapping("/simpleStreamChatCompletion")
+  @GetMapping("/streamChatCompletion")
   @Nonnull
-  public static ResponseEntity<ResponseBodyEmitter> simpleStreamChatCompletion() {
+  public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
     final var request =
         new OpenAiChatCompletionParameters()
             .setMessages(
@@ -113,7 +114,7 @@ public static ResponseEntity<ResponseBodyEmitter> simpleStreamChatCompletion() {
                         .addText(
                             "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-    final var stream = OpenAiClient.forModel(GPT_35_TURBO).simpleStreamChatCompletion(request);
+    final var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request);
 
     final var emitter = new ResponseBodyEmitter();
 
@@ -121,6 +122,7 @@ public static ResponseEntity<ResponseBodyEmitter> simpleStreamChatCompletion() {
         () -> {
           try (stream) {
             stream.forEach(deltaMessage -> send(emitter, deltaMessage));
+          } finally {
             emitter.complete();
           }
         };
diff --git a/e2e-test-app/src/main/resources/static/index.html b/e2e-test-app/src/main/resources/static/index.html
index 1cb4146a..b7a2d904 100644
--- a/e2e-test-app/src/main/resources/static/index.html
+++ b/e2e-test-app/src/main/resources/static/index.html
@@ -72,6 +72,7 @@ <h2>Endpoints</h2>
             <ul>
                 <li><a href="/chatCompletion">/chatCompletion</a></li>
                 <li><a href="/streamChatCompletion">/streamChatCompletion</a></li>
+                <li><a href="/streamChatCompletionDeltas">/streamChatCompletionDeltas</a></li>
                 <li><a href="/chatCompletionTool">/chatCompletionTool</a></li>
                 <li><a href="/chatCompletionImage">/chatCompletionImage</a></li>
                 <li><a href="/embedding">/embedding</a></li>
diff --git a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
index 273b4101..c37aff53 100644
--- a/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
+++ b/e2e-test-app/src/test/java/com/sap/ai/sdk/app/controllers/OpenAiTest.java
@@ -41,7 +41,7 @@ void streamChatCompletion() {
     final var totalOutput = new OpenAiChatCompletionOutput();
     final var emptyDeltaCount = new AtomicInteger(0);
     OpenAiClient.forModel(GPT_35_TURBO)
-        .streamChatCompletion(request)
+        .streamChatCompletionDeltas(request)
         .peek(totalOutput::addDelta)
         // foreach consumes all elements, closing the stream at the end
         .forEach(
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index ce6fb809..b12f502e 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -112,35 +112,36 @@ public OpenAiChatCompletionOutput chatCompletion(
    * Generate a completion for the given prompt.
    *
    * @param parameters the prompt, including messages and other parameters.
-   * @return A stream of chat completion delta elements.
-   * @throws OpenAiClientException if the request fails
+   * @return A stream of message deltas
+   * @throws OpenAiClientException if the request fails or if the finish reason is content_filter
    */
   @Nonnull
-  public Stream<OpenAiChatCompletionDelta> streamChatCompletion(
+  public Stream<String> streamChatCompletion(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
-    parameters.setStream(true);
-    return streamChatCompletion("/chat/completions", parameters, OpenAiChatCompletionDelta.class);
+    return streamChatCompletionDeltas(parameters)
+        .peek(OpenAiClient::throwOnContentFilter)
+        .map(OpenAiChatCompletionDelta::getDeltaContent);
+  }
+
+  private static void throwOnContentFilter(OpenAiChatCompletionDelta delta) {
+    final String finishReason = delta.getFinishReason();
+    if (finishReason != null && finishReason.equals("content_filter")) {
+      throw new OpenAiClientException("Content filter filtered the output.");
+    }
   }
 
   /**
    * Generate a completion for the given prompt.
    *
    * @param parameters the prompt, including messages and other parameters.
-   * @return A stream of message deltas
-   * @throws OpenAiClientException if the request fails or if the finish reason is content_filter
+   * @return A stream of chat completion delta elements.
+   * @throws OpenAiClientException if the request fails
    */
   @Nonnull
-  public Stream<String> simpleStreamChatCompletion(
+  public Stream<OpenAiChatCompletionDelta> streamChatCompletionDeltas(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
-    return streamChatCompletion(parameters)
-        .peek(
-            delta -> {
-              final String finishReason = delta.getFinishReason();
-              if (finishReason != null && finishReason.equals("content_filter")) {
-                throw new OpenAiClientException("Content filter filtered the output.");
-              }
-            })
-        .map(OpenAiChatCompletionDelta::getDeltaContent);
+    parameters.setStream(true);
+    return executeStream("/chat/completions", parameters, OpenAiChatCompletionDelta.class);
   }
 
   /**
@@ -168,7 +169,7 @@ private <T> T execute(
   }
 
   @Nonnull
-  private <D extends StreamedDelta> Stream<D> streamChatCompletion(
+  private <D extends StreamedDelta> Stream<D> executeStream(
       @Nonnull final String path,
       @Nonnull final Object payload,
       @Nonnull final Class<D> deltaType) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 19e6959f..33180a80 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -69,7 +69,7 @@ private Stream<D> parseResponse(@Nonnull final ClassicHttpResponse response)
             })
         .map(
             responseLine -> {
-              final String data = responseLine.substring(5).replace("delta", "message");
+              final String data = responseLine.substring(5); // remove "data: "
               try {
                 return JACKSON.readValue(data, deltaType);
               } catch (final IOException e) {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletionChoice.java
index af6cdb58..dc9ed2e9 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletionChoice.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiDeltaChatCompletionChoice.java
@@ -16,7 +16,7 @@
 @ToString(callSuper = true)
 public class OpenAiDeltaChatCompletionChoice extends OpenAiCompletionChoice {
   /** Completion chat message. */
-  @JsonProperty("message")
+  @JsonProperty("delta")
   @Getter(onMethod_ = @Nullable)
   @Setter(onMethod_ = @Nullable, value = AccessLevel.PACKAGE)
   private OpenAiChatAssistantMessage message;
diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 753c5cb6..e0befdfb 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -79,7 +79,7 @@ private Stream<Runnable> chatCompletionCalls() {
         () -> client.chatCompletion(new OpenAiChatCompletionParameters()),
         () ->
             client
-                .streamChatCompletion(new OpenAiChatCompletionParameters())
+                .streamChatCompletionDeltas(new OpenAiChatCompletionParameters())
                 // the stream needs to be consumed to parse the response
                 .forEach(System.out::println));
   }
@@ -289,7 +289,7 @@ void embedding() throws IOException {
   }
 
   @Test
-  void streamChatCompletionErrorHandling() throws IOException {
+  void streamChatCompletionDeltasErrorHandling() throws IOException {
     try (var inputStream =
         spy(
             Objects.requireNonNull(
@@ -317,7 +317,7 @@ void streamChatCompletionErrorHandling() throws IOException {
                           .addText(
                               "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-      try (Stream<OpenAiChatCompletionDelta> stream = client.streamChatCompletion(request)) {
+      try (Stream<OpenAiChatCompletionDelta> stream = client.streamChatCompletionDeltas(request)) {
         assertThatThrownBy(() -> stream.forEach(System.out::println))
             .isInstanceOf(OpenAiClientException.class)
             .hasMessage(
@@ -329,7 +329,7 @@ void streamChatCompletionErrorHandling() throws IOException {
   }
 
   @Test
-  void streamChatCompletion() throws IOException {
+  void streamChatCompletionDeltas() throws IOException {
     try (var inputStream =
         spy(
             Objects.requireNonNull(
@@ -355,7 +355,7 @@ void streamChatCompletion() throws IOException {
                           .addText(
                               "Can you give me the first 100 numbers of the Fibonacci sequence?")));
 
-      try (Stream<OpenAiChatCompletionDelta> stream = client.streamChatCompletion(request)) {
+      try (Stream<OpenAiChatCompletionDelta> stream = client.streamChatCompletionDeltas(request)) {
 
         OpenAiChatCompletionOutput totalOutput = new OpenAiChatCompletionOutput();
         final List<OpenAiChatCompletionDelta> deltaList =

From 28268b2f497f0532df28e020b62ff2923e5877d8 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 3 Sep 2024 10:22:12 +0200
Subject: [PATCH 35/55] Make client static

---
 .../sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index e0befdfb..727fe8d4 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -43,10 +43,9 @@
 import org.junit.jupiter.params.provider.MethodSource;
 import org.mockito.Mockito;
 
-@TestInstance(Lifecycle.PER_CLASS)
 @WireMockTest
 class OpenAiClientTest {
-  private OpenAiClient client;
+  private static OpenAiClient client;
 
   @BeforeEach
   void setup(WireMockRuntimeInfo server) {
@@ -74,7 +73,7 @@ void apiVersion() {
     verify(exactly(2), postRequestedFor(anyUrl()).withoutQueryParam("api-version"));
   }
 
-  private Stream<Runnable> chatCompletionCalls() {
+  private static Stream<Runnable> chatCompletionCalls() {
     return Stream.of(
         () -> client.chatCompletion(new OpenAiChatCompletionParameters()),
         () ->

From 9a9a44bbbc2daceff7a5cd137ff02cb3ce5290c3 Mon Sep 17 00:00:00 2001
From: SAP Cloud SDK Bot <cloudsdk@sap.com>
Date: Tue, 3 Sep 2024 08:22:47 +0000
Subject: [PATCH 36/55] Formatting

---
 .../sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java    | 2 --
 1 file changed, 2 deletions(-)

diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 727fe8d4..4e55ab11 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -37,8 +37,6 @@
 import org.assertj.core.api.SoftAssertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-import org.junit.jupiter.api.TestInstance.Lifecycle;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.mockito.Mockito;

From 788db038d0ff5c2c952d8857fab4d79972485e97 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 3 Sep 2024 10:27:04 +0200
Subject: [PATCH 37/55] PMD

---
 .../com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index b12f502e..e072ef50 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -123,7 +123,7 @@ public Stream<String> streamChatCompletion(
         .map(OpenAiChatCompletionDelta::getDeltaContent);
   }
 
-  private static void throwOnContentFilter(OpenAiChatCompletionDelta delta) {
+  private static void throwOnContentFilter(@Nonnull final OpenAiChatCompletionDelta delta) {
     final String finishReason = delta.getFinishReason();
     if (finishReason != null && finishReason.equals("content_filter")) {
       throw new OpenAiClientException("Content filter filtered the output.");

From 0616f55bcfff4b2e6326e72531cf4368df7fc331 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 3 Sep 2024 15:26:44 +0200
Subject: [PATCH 38/55] Fix tests

---
 .../foundationmodels/openai/OpenAiStreamingHandler.java    | 2 +-
 .../ai/sdk/foundationmodels/openai/OpenAiClientTest.java   | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 33180a80..893e8ea1 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -82,7 +82,7 @@ private Stream<D> parseResponse(@Nonnull final ClassicHttpResponse response)
         .onClose(
             () -> {
               try {
-                inputStream.close();
+                br.close();
               } catch (IOException e) {
                 log.error("Could not close HTTP input stream", e);
               }
diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 4e55ab11..72ea6513 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -23,6 +23,7 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiContentFilterPromptResults;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
 import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor;
+import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Cache;
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import io.vavr.control.Try;
 import java.io.IOException;
@@ -50,6 +51,8 @@ void setup(WireMockRuntimeInfo server) {
     final DefaultHttpDestination destination =
         DefaultHttpDestination.builder(server.getHttpBaseUrl()).build();
     client = OpenAiClient.withCustomDestination(destination);
+    ApacheHttpClient5Accessor.setHttpClientCache(ApacheHttpClient5Cache.DISABLED);
+    ApacheHttpClient5Accessor.setHttpClientFactory(null);
   }
 
   @Test
@@ -162,7 +165,7 @@ void chatCompletion() throws IOException {
 
       assert inputStream != null;
       final String response = new String(inputStream.readAllBytes());
-      stubFor(post(anyUrl()).willReturn(okJson(response)));
+      stubFor(post("/chat/completions").willReturn(okJson(response)));
 
       final var systemMessage =
           new OpenAiChatMessage.OpenAiChatSystemMessage()
@@ -251,7 +254,7 @@ void embedding() throws IOException {
 
       assert inputStream != null;
       final String response = new String(inputStream.readAllBytes());
-      stubFor(post(anyUrl()).willReturn(okJson(response)));
+      stubFor(post("/embeddings").willReturn(okJson(response)));
 
       final var request = new OpenAiEmbeddingParameters().setInput("Hello World");
       final var result = client.embedding(request);

From 3446bf03e888c09cab0e14d6ea3e8055bcfd11b2 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 3 Sep 2024 15:41:06 +0200
Subject: [PATCH 39/55] Removed exception constructors no args

---
 .../sdk/foundationmodels/openai/OpenAiClient.java |  4 ++--
 .../openai/OpenAiClientException.java             | 15 ---------------
 2 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index e072ef50..257b55dd 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -196,7 +196,7 @@ private <T> T executeRequest(
       final var client = ApacheHttpClient5Accessor.getHttpClient(destination);
       return client.execute(request, new OpenAiResponseHandler<>(responseType));
     } catch (final IOException e) {
-      throw new OpenAiClientException(e);
+      throw new OpenAiClientException("Request to OpenAI model failed", e);
     }
   }
 
@@ -209,7 +209,7 @@ private <D extends StreamedDelta> Stream<D> streamRequest(
       return new OpenAiStreamingHandler<>(deltaType)
           .handleResponse(client.executeOpen(null, request, null));
     } catch (final IOException e) {
-      throw new OpenAiClientException(e);
+      throw new OpenAiClientException("Request to OpenAI model failed", e);
     }
   }
 }
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java
index 18a2aaeb..d61bfa08 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientException.java
@@ -5,23 +5,8 @@
 
 /** Generic exception for errors occurring when using OpenAI foundation models. */
 public class OpenAiClientException extends RuntimeException {
-  static final String BASE_ERROR_MESSAGE = "Request to OpenAI model failed";
   @Serial private static final long serialVersionUID = -7345541120979974432L;
 
-  /** Create a new exception with the base message: {@code Request to OpenAI model failed} */
-  public OpenAiClientException() {
-    this(BASE_ERROR_MESSAGE);
-  }
-
-  /**
-   * Create a new exception with the base message: {@code Request to OpenAI model failed}
-   *
-   * @param e the cause
-   */
-  public OpenAiClientException(@Nonnull final Exception e) {
-    this(BASE_ERROR_MESSAGE, e);
-  }
-
   /**
    * Create a new exception with the given message.
    *

From 45a20c63efd1929fc7de6f4bd805db10d4412480 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 3 Sep 2024 15:42:39 +0200
Subject: [PATCH 40/55] Refactor exception message

---
 .../ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
index 4e6ed09b..9d7dd0bd 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiResponseHandler.java
@@ -117,6 +117,6 @@ static void parseErrorAndThrow(
       throw baseException;
     }
     throw new OpenAiClientException(
-        baseException.getMessage() + " and error message: '%s'".formatted(error.getMessage()));
+        "%s and error message: '%s'".formatted(baseException.getMessage(), error.getMessage()));
   }
 }

From f843061324d10df33138ff9c814d98780eb45e81 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 3 Sep 2024 15:45:18 +0200
Subject: [PATCH 41/55] Readme sentences

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index b4edd4e4..67e1d90f 100644
--- a/README.md
+++ b/README.md
@@ -253,6 +253,7 @@ void printStream() {
 </details>
 
 #### Framework-agnostic example
+This example passes the stream of chat completion delta messages to a frontend in real-time.
 ```java
 public static ReturnType streamChatCompletion() {
   final var request =
@@ -281,6 +282,7 @@ public static ReturnType streamChatCompletion() {
 ```
 
 #### Spring Boot example
+This example uses Spring Boot's `ResponseBodyEmitter` to stream the chat completion delta messages to the frontend in real-time.
 ```java
 public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
   final var request =

From 5edcf711da69af614d88143e09782303b57c7a59 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Tue, 3 Sep 2024 15:54:00 +0200
Subject: [PATCH 42/55] Remove superfluous call super

---
 .../openai/model/OpenAiChatCompletionFunction.java     |  2 +-
 .../openai/model/OpenAiChatCompletionParameters.java   |  2 +-
 .../openai/model/OpenAiChatCompletionTool.java         |  2 +-
 .../openai/model/OpenAiChatFunctionCall.java           |  2 +-
 .../openai/model/OpenAiChatMessage.java                | 10 +++++-----
 .../openai/model/OpenAiChatToolCall.java               |  2 +-
 .../openai/model/OpenAiCompletionChoice.java           |  4 ++--
 .../openai/model/OpenAiCompletionOutput.java           |  2 +-
 .../openai/model/OpenAiCompletionParameters.java       |  2 +-
 .../openai/model/OpenAiContentFilterResultBase.java    |  2 +-
 .../openai/model/OpenAiContentFilterResultsBase.java   |  2 +-
 .../openai/model/OpenAiEmbeddingData.java              |  2 +-
 .../openai/model/OpenAiEmbeddingOutput.java            |  2 +-
 .../openai/model/OpenAiEmbeddingParameters.java        |  2 +-
 .../sdk/foundationmodels/openai/model/OpenAiError.java |  2 +-
 .../foundationmodels/openai/model/OpenAiErrorBase.java |  2 +-
 .../openai/model/OpenAiPromptFilterResult.java         |  2 +-
 .../sdk/foundationmodels/openai/model/OpenAiUsage.java |  2 +-
 18 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionFunction.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionFunction.java
index c8ea342b..b3ff6cd2 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionFunction.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionFunction.java
@@ -11,7 +11,7 @@
 /** OpenAI function signature. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiChatCompletionFunction {
   /**
    * Name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes,
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
index 6f28134a..19a3c425 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionParameters.java
@@ -177,7 +177,7 @@ private enum ToolChoiceType implements ToolChoice {
   }
 
   @EqualsAndHashCode
-  @ToString(callSuper = true)
+  @ToString
   private static class FunctionToolChoice implements ToolChoice {
     @JsonProperty("function")
     @Setter(onParam_ = @Nonnull)
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionTool.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionTool.java
index 91de5424..6f6cd8ec 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionTool.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatCompletionTool.java
@@ -12,7 +12,7 @@
 /** OpenAI tool signature. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiChatCompletionTool {
   /** Specifies a tool the model should use. Use to force the model to call a specific function. */
   @JsonProperty("type")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatFunctionCall.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatFunctionCall.java
index b228fa4c..7a59c85c 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatFunctionCall.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatFunctionCall.java
@@ -11,7 +11,7 @@
 /** The name of the function to call, or, the function that the model called. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiChatFunctionCall {
   /** Name of the function call. */
   @JsonProperty("name")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
index 2289d72a..a22e5db4 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatMessage.java
@@ -59,7 +59,7 @@ public interface OpenAiChatMessage {
   /** OpenAI system message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString(callSuper = true)
+  @ToString
   class OpenAiChatSystemMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
@@ -75,7 +75,7 @@ class OpenAiChatSystemMessage implements OpenAiChatMessage {
   /** OpenAI user message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString(callSuper = true)
+  @ToString
   class OpenAiChatUserMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
@@ -235,7 +235,7 @@ public enum ImageDetailLevel {
   /** OpenAI assistant message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString(callSuper = true)
+  @ToString
   class OpenAiChatAssistantMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
@@ -277,7 +277,7 @@ void addDelta(@Nonnull final OpenAiChatAssistantMessage delta) {
   /** OpenAI tool message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString(callSuper = true)
+  @ToString
   class OpenAiChatToolMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
@@ -298,7 +298,7 @@ class OpenAiChatToolMessage implements OpenAiChatMessage {
   /** OpenAI function message. */
   @Accessors(chain = true)
   @EqualsAndHashCode
-  @ToString(callSuper = true)
+  @ToString
   class OpenAiChatFunctionMessage implements OpenAiChatMessage {
     /** The role of the messages author. */
     @Getter(onMethod_ = @Nonnull)
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatToolCall.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatToolCall.java
index 624a98d1..65a5d0f9 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatToolCall.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiChatToolCall.java
@@ -10,7 +10,7 @@
 /** OpenAI tool call by AI. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiChatToolCall {
   /** The ID of the tool call. */
   @JsonProperty("id")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
index edddf457..80c5baa3 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionChoice.java
@@ -11,7 +11,7 @@
 /** Result for OpenAI chat completion output. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiCompletionChoice {
   /**
    * Reason for finish. The possible values are:
@@ -33,7 +33,7 @@ public class OpenAiCompletionChoice {
 
   /** Index of choice. */
   @JsonProperty("index")
-  @Getter // Nullable
+  @Getter(onMethod_ = @Nullable)
   private Integer index;
 
   /**
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
index 8b856910..366e4137 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionOutput.java
@@ -12,7 +12,7 @@
 /** OpenAI completion output . */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiCompletionOutput {
   /** Creation date Unix timestamp. */
   @JsonProperty("created")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
index b690bcff..3e989fca 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
@@ -14,7 +14,7 @@
 /** OpenAI completion input parameters. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiCompletionParameters {
   /**
    * The maximum number of [tokens](/tokenizer) that can be generated in the completion. The token
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultBase.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultBase.java
index a7c1033e..167627b6 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultBase.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultBase.java
@@ -10,7 +10,7 @@
 /** OpenAI content filter result. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiContentFilterResultBase {
   /** Whether the content was filtered. */
   @JsonProperty("filtered")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java
index 2c454db3..78670475 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiContentFilterResultsBase.java
@@ -11,7 +11,7 @@
 /** Information about the content filtering results. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiContentFilterResultsBase {
   /** Sexual content filter result. */
   @JsonProperty("sexual")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingData.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingData.java
index 86f3be83..181e3b22 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingData.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingData.java
@@ -11,7 +11,7 @@
 /** Result candidates for OpenAI embedding output. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiEmbeddingData {
   /** Embedding object. */
   @JsonProperty("object")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingOutput.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingOutput.java
index d28974dd..ba93fcf6 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingOutput.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingOutput.java
@@ -11,7 +11,7 @@
 /** OpenAI embedding output. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiEmbeddingOutput {
   /** List object. */
   @JsonProperty("object")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingParameters.java
index 6026483d..a932674f 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiEmbeddingParameters.java
@@ -14,7 +14,7 @@
 /** OpenAI embedding input parameters. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiEmbeddingParameters {
   /**
    * Input text to get embeddings for, encoded as a string. The number of input tokens varies
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiError.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiError.java
index fbfbb21b..1d373b63 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiError.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiError.java
@@ -10,7 +10,7 @@
 /** OpenAI error. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiError {
   /** The error object. */
   @JsonProperty("error")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiErrorBase.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiErrorBase.java
index 2967d586..d71ecffc 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiErrorBase.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiErrorBase.java
@@ -10,7 +10,7 @@
 /** OpenAI error. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiErrorBase {
   /** The error code. */
   @JsonProperty("code")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiPromptFilterResult.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiPromptFilterResult.java
index 22f0e7ec..85ddfb11 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiPromptFilterResult.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiPromptFilterResult.java
@@ -10,7 +10,7 @@
 /** Content filtering results for a single prompt in the request. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiPromptFilterResult {
   /** Index of the prompt in the request. */
   @JsonProperty("prompt_index")
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
index 7691caed..7884b8c2 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiUsage.java
@@ -11,7 +11,7 @@
 /** Usage statistics for the completion request. */
 @Accessors(chain = true)
 @EqualsAndHashCode
-@ToString(callSuper = true)
+@ToString
 public class OpenAiUsage {
   /** Tokens consumed for output text completion. */
   @JsonProperty("completion_tokens")

From 7474fb1fcc899d3117ce117e11c7876b82b5ade0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20D=C3=BCmont?= <alexander.duemont@sap.com>
Date: Tue, 3 Sep 2024 16:36:18 +0200
Subject: [PATCH 43/55] reset httpclient-cache and -factory after each test
 case

---
 .../ai/sdk/foundationmodels/openai/OpenAiClientTest.java    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 72ea6513..1b7f3a88 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -36,6 +36,7 @@
 import org.apache.hc.core5.http.io.entity.InputStreamEntity;
 import org.apache.hc.core5.http.message.BasicClassicHttpResponse;
 import org.assertj.core.api.SoftAssertions;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -52,6 +53,11 @@ void setup(WireMockRuntimeInfo server) {
         DefaultHttpDestination.builder(server.getHttpBaseUrl()).build();
     client = OpenAiClient.withCustomDestination(destination);
     ApacheHttpClient5Accessor.setHttpClientCache(ApacheHttpClient5Cache.DISABLED);
+  }
+
+  @AfterEach
+  void reset() {
+    ApacheHttpClient5Accessor.setHttpClientCache(null);
     ApacheHttpClient5Accessor.setHttpClientFactory(null);
   }
 

From ac6f36c7de0c5c5428c6bd7e3d3d59425f973eca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20D=C3=BCmont?= <alexander.duemont@sap.com>
Date: Tue, 3 Sep 2024 17:06:50 +0200
Subject: [PATCH 44/55] Very minor code-style improvements in test

---
 .../openai/OpenAiClientTest.java              | 47 ++++++++-----------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index 1b7f3a88..f2d45371 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -5,10 +5,10 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
 
 import com.fasterxml.jackson.core.JsonParseException;
 import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
@@ -27,8 +27,10 @@
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import io.vavr.control.Try;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Function;
 import java.util.stream.Stream;
 import javax.annotation.Nonnull;
 import org.apache.hc.client5.http.classic.HttpClient;
@@ -46,6 +48,8 @@
 @WireMockTest
 class OpenAiClientTest {
   private static OpenAiClient client;
+  private final Function<String, InputStream> TEST_FILE_LOADER =
+      filename -> Objects.requireNonNull(getClass().getClassLoader().getResourceAsStream(filename));
 
   @BeforeEach
   void setup(WireMockRuntimeInfo server) {
@@ -80,14 +84,15 @@ void apiVersion() {
     verify(exactly(2), postRequestedFor(anyUrl()).withoutQueryParam("api-version"));
   }
 
-  private static Stream<Runnable> chatCompletionCalls() {
-    return Stream.of(
-        () -> client.chatCompletion(new OpenAiChatCompletionParameters()),
-        () ->
-            client
-                .streamChatCompletionDeltas(new OpenAiChatCompletionParameters())
-                // the stream needs to be consumed to parse the response
-                .forEach(System.out::println));
+  private static Runnable[] chatCompletionCalls() {
+    return new Runnable[] {
+      () -> client.chatCompletion(new OpenAiChatCompletionParameters()),
+      () ->
+          client
+              .streamChatCompletionDeltas(new OpenAiChatCompletionParameters())
+              // the stream needs to be consumed to parse the response
+              .forEach(System.out::println)
+    };
   }
 
   @ParameterizedTest
@@ -166,10 +171,8 @@ void chatCompletionErrorHandling(@Nonnull final Runnable request) {
 
   @Test
   void chatCompletion() throws IOException {
-    try (var inputStream =
-        getClass().getClassLoader().getResourceAsStream("chatCompletionResponse.json")) {
+    try (var inputStream = TEST_FILE_LOADER.apply("chatCompletionResponse.json")) {
 
-      assert inputStream != null;
       final String response = new String(inputStream.readAllBytes());
       stubFor(post("/chat/completions").willReturn(okJson(response)));
 
@@ -255,10 +258,8 @@ void chatCompletion() throws IOException {
 
   @Test
   void embedding() throws IOException {
-    try (var inputStream =
-        getClass().getClassLoader().getResourceAsStream("embeddingResponse.json")) {
+    try (var inputStream = TEST_FILE_LOADER.apply("embeddingResponse.json")) {
 
-      assert inputStream != null;
       final String response = new String(inputStream.readAllBytes());
       stubFor(post("/embeddings").willReturn(okJson(response)));
 
@@ -296,12 +297,7 @@ void embedding() throws IOException {
 
   @Test
   void streamChatCompletionDeltasErrorHandling() throws IOException {
-    try (var inputStream =
-        spy(
-            Objects.requireNonNull(
-                getClass()
-                    .getClassLoader()
-                    .getResourceAsStream("streamChatCompletionError.txt")))) {
+    try (var inputStream = spy(TEST_FILE_LOADER.apply("streamChatCompletionError.txt"))) {
 
       final var httpClient = mock(HttpClient.class);
       ApacheHttpClient5Accessor.setHttpClientFactory(destination -> httpClient);
@@ -330,16 +326,13 @@ void streamChatCompletionDeltasErrorHandling() throws IOException {
                 "Failed to parse response from OpenAI model and error message: 'exceeded token rate limit'");
       }
 
-      Mockito.verify(inputStream, atLeastOnce()).close();
+      Mockito.verify(inputStream, times(1)).close();
     }
   }
 
   @Test
   void streamChatCompletionDeltas() throws IOException {
-    try (var inputStream =
-        spy(
-            Objects.requireNonNull(
-                getClass().getClassLoader().getResourceAsStream("streamChatCompletion.txt")))) {
+    try (var inputStream = spy(TEST_FILE_LOADER.apply("streamChatCompletion.txt"))) {
 
       final var httpClient = mock(HttpClient.class);
       ApacheHttpClient5Accessor.setHttpClientFactory(destination -> httpClient);
@@ -448,7 +441,7 @@ void streamChatCompletionDeltas() throws IOException {
         assertFilter(totalOutput.getPromptFilterResults().get(0).getContentFilterResults());
       }
 
-      Mockito.verify(inputStream, atLeastOnce()).close();
+      Mockito.verify(inputStream, times(1)).close();
     }
   }
 

From ffa369a6df98d6cbe1234f0da828d2c9cfefed01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20D=C3=BCmont?= <alexander.duemont@sap.com>
Date: Tue, 3 Sep 2024 17:59:54 +0200
Subject: [PATCH 45/55] Minor code-style in OpenAIController

---
 .../com/sap/ai/sdk/app/controllers/OpenAiController.java   | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 84610071..01630d61 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -57,13 +57,10 @@ public static OpenAiChatCompletionOutput chatCompletion() {
   @GetMapping("/streamChatCompletionDeltas")
   @Nonnull
   public static ResponseEntity<ResponseBodyEmitter> streamChatCompletionDeltas() {
+    final var msg = "Can you give me the first 100 numbers of the Fibonacci sequence?";
     final var request =
         new OpenAiChatCompletionParameters()
-            .setMessages(
-                List.of(
-                    new OpenAiChatUserMessage()
-                        .addText(
-                            "Can you give me the first 100 numbers of the Fibonacci sequence?")));
+            .setMessages(List.of(new OpenAiChatUserMessage().addText(msg)));
 
     final var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletionDeltas(request);
 

From 6cfeee931851bba4d47f3044384b2c1208b92df8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20D=C3=BCmont?= <alexander.duemont@sap.com>
Date: Tue, 3 Sep 2024 18:01:25 +0200
Subject: [PATCH 46/55] Reduce README sample code

---
 README.md | 138 ++++++++++++++++++------------------------------------
 1 file changed, 46 insertions(+), 92 deletions(-)

diff --git a/README.md b/README.md
index 67e1d90f..1bc44d5f 100644
--- a/README.md
+++ b/README.md
@@ -206,24 +206,22 @@ final OpenAiChatCompletionOutput result =
 
 ### Stream chat completion
 
-How to pass a stream of chat completion delta elements from the backend to the frontend in real-time.
+It's possible to pass a stream of chat completion delta elements, e.g. from the application backend to the frontend in real-time.
 
 #### Print the stream to the console
 This is a simple example to get started:
 ```java
-void printStream() {
-  final var request =
-      new OpenAiChatCompletionParameters()
-          .setMessages(
-              List.of(
-                  new OpenAiChatUserMessage()
-                      .addText(
-                          "Can you give me the first 100 numbers of the Fibonacci sequence?")));
-
-  // try-with-resources ensures the stream is closed
-  try(var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request)) {
-    stream.forEach(deltaMessage -> System.out.println(deltaMessage));
-  }
+String msg = "Can you give me the first 100 numbers of the Fibonacci sequence?";
+
+OpenAiChatCompletionParameters request =
+    new OpenAiChatCompletionParameters()
+        .setMessages(List.of(new OpenAiChatUserMessage().addText(msg)));
+
+OpenAiClient client = OpenAiClient.forModel(GPT_35_TURBO);
+
+// try-with-resources on stream ensures the connection will be closed
+try( Stream<String> stream = client.streamChatCompletion(request)) {
+  stream.forEach(deltaString -> System.out.println(deltaString));
 }
 ```
 
@@ -232,97 +230,53 @@ void printStream() {
 <pre>
 
 ```java
-void printStream() {
-  final var request =
-      new OpenAiChatCompletionParameters()
-          .setMessages(
-              List.of(
-                  new OpenAiChatUserMessage()
-                      .addText(
-                          "Can you give me the first 100 numbers of the Fibonacci sequence?")));
-
-  final var totalOutput = new OpenAiChatCompletionOutput();
-  // try-with-resources ensures the stream is closed
-  try (var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletionDeltas(request)) {
-    stream.peek(totalOutput::addDelta).forEach(delta -> System.out.println(delta));
-  }
-  // totalOutput can be looked at here
+String msg = "Can you give me the first 100 numbers of the Fibonacci sequence?";
+
+OpenAiChatCompletionParameters request =
+    new OpenAiChatCompletionParameters()
+        .setMessages(List.of(new OpenAiChatUserMessage().addText(msg)));
+
+OpenAiChatCompletionOutput totalOutput = new OpenAiChatCompletionOutput();
+OpenAiClient client = OpenAiClient.forModel(GPT_35_TURBO);
+
+// try-with-resources on stream ensures the connection will be closed
+try( Stream<OpenAiChatCompletionDelta> stream = client.streamChatCompletionDeltas(request)) {
+    stream.peek(totalOutput::addDelta).forEach(deltaObject -> System.out.println(deltaObject));
 }
+
+// access aggregated information from total output, e.g.
+System.out.println("Tokens: " + totalOutput.getUsage().getCompletionTokens());
 ```
 </pre>
 </details>
 
 #### Framework-agnostic example
-This example passes the stream of chat completion delta messages to a frontend in real-time.
+This example passes the stream of chat completion delta messages to a generic `Consumer<String>`.
 ```java
-public static ReturnType streamChatCompletion() {
-  final var request =
-      new OpenAiChatCompletionParameters()
-          .setMessages(
-              List.of(
-                  new OpenAiChatUserMessage()
-                      .addText(
-                          "Can you give me the first 100 numbers of the Fibonacci sequence?")));
-
-  final var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request);
-
-  final Runnable consumeStream =
-      () -> {
-        // try-with-resources ensures the stream is closed
-        try (stream) {
-          // send is defined by your framework
-          stream.forEach(delta -> send(delta));
-        }
-      };
+Consumer<String> emit;
 
-  ThreadContextExecutors.getExecutor().submit(consumeStream);
+String msg = "Can you give me the first 100 numbers of the Fibonacci sequence?";
 
-  // Note: set the header content-type:text/event-stream on the sent response
-}
-```
-
-#### Spring Boot example
-This example uses Spring Boot's `ResponseBodyEmitter` to stream the chat completion delta messages to the frontend in real-time.
-```java
-public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
-  final var request =
-      new OpenAiChatCompletionParameters()
-          .setMessages(
-              List.of(
-                  new OpenAiChatUserMessage()
-                      .addText(
-                          "Can you give me the first 100 numbers of the Fibonacci sequence?")));
-
-  final var stream = OpenAiClient.forModel(GPT_35_TURBO).streamChatCompletion(request);
-
-  final var emitter = new ResponseBodyEmitter();
-
-  final Runnable consumeStream =
-      () -> {
-        // try-with-resources ensures the stream is closed
-        try (stream) {
-          stream.forEach(
-              delta -> {
-                try {
-                  emitter.send(delta);
-                } catch (final IOException e) {
-                  log.error(Arrays.toString(e.getStackTrace()));
-                  emitter.completeWithError(e);
-                }
-              });
-        } finally {
-          emitter.complete();
-        }
-      };
+OpenAiChatCompletionParameters request =
+    new OpenAiChatCompletionParameters()
+        .setMessages(List.of(new OpenAiChatUserMessage().addText(msg)));
 
-  ThreadContextExecutors.getExecutor().submit(consumeStream);
+OpenAiClient client = OpenAiClient.forModel(GPT_35_TURBO);
+Stream<String> stream = client.streamChatCompletion(request);
 
-  // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
-  return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
-}
+ThreadContextExecutors.getExecutor().submit(() -> {
+    // try-with-resources ensures the stream is closed
+    try (stream) {
+        // send is defined by your framework
+        stream.forEach(delta -> emit.accept(delta));
+    }
+});
 ```
 
-See [an example in our Spring Boot application](e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java)
+#### Spring Boot example
+
+Please find [an example in our Spring Boot application](e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java).
+It shows the usage of Spring Boot's `ResponseBodyEmitter` to stream the chat completion delta messages to the frontend in real-time.
 
 ## Orchestration chat completion
 

From 6d4fd2f5ceb3b438762740387dbc08bbafb5cfe3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20D=C3=BCmont?=
 <22489773+newtork@users.noreply.github.com>
Date: Tue, 3 Sep 2024 18:51:00 +0200
Subject: [PATCH 47/55] Update OpenAiStreamingHandler.java (#43)

---
 .../openai/OpenAiStreamingHandler.java        | 35 +++++++------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 893e8ea1..65c21b77 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -55,37 +55,28 @@ private Stream<D> parseResponse(@Nonnull final ClassicHttpResponse response)
 
     // https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format
     return br.lines()
-        .filter(
-            responseLine ->
-                // half of the lines are empty newlines, the last line is "data: [DONE]"
-                !responseLine.isEmpty() && !"data: [DONE]".equals(responseLine.trim()))
+        // half of the lines are empty newlines, the last line is "data: [DONE]"
+        .filter(line -> !line.isEmpty() && !"data: [DONE]".equals(line.trim()))
         .peek(
-            responseLine -> {
-              if (!responseLine.startsWith("data: ")) {
-                parseErrorAndThrow(
-                    responseLine,
-                    new OpenAiClientException("Failed to parse response from OpenAI model"));
+            line -> {
+              if (!line.startsWith("data: ")) {
+                final String msg = "Failed to parse response from OpenAI model";
+                parseErrorAndThrow(line, new OpenAiClientException(msg));
               }
             })
         .map(
-            responseLine -> {
-              final String data = responseLine.substring(5); // remove "data: "
+            line -> {
+              final String data = line.substring(5); // remove "data: "
               try {
                 return JACKSON.readValue(data, deltaType);
               } catch (final IOException e) {
-                log.error(
-                    "Failed to parse the following response from OpenAI model: {}", responseLine);
-                throw new OpenAiClientException(
-                    "Failed to parse delta message: " + responseLine, e);
+                log.error("Failed to parse the following response from OpenAI model: {}", line);
+                throw new OpenAiClientException("Failed to parse delta message: " + line, e);
               }
             })
         .onClose(
-            () -> {
-              try {
-                br.close();
-              } catch (IOException e) {
-                log.error("Could not close HTTP input stream", e);
-              }
-            });
+            () ->
+                Try.run(inputStream::close)
+                    .onFailure(e -> log.error("Could not close HTTP input stream", e)));
   }
 }

From a6c566abc4c389b3a747c9cd01065f488564c10a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20D=C3=BCmont?= <alexander.duemont@sap.com>
Date: Tue, 3 Sep 2024 18:51:54 +0200
Subject: [PATCH 48/55] Fix import

---
 .../ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
index 65c21b77..3a264705 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiStreamingHandler.java
@@ -5,6 +5,7 @@
 import static com.sap.ai.sdk.foundationmodels.openai.OpenAiResponseHandler.parseErrorAndThrow;
 
 import com.sap.ai.sdk.foundationmodels.openai.model.StreamedDelta;
+import io.vavr.control.Try;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;

From f6a4fe656f36d3a6eb14e2915aa0fceb367a8c58 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Wed, 4 Sep 2024 10:55:43 +0200
Subject: [PATCH 49/55] Added stream_options to model

---
 README.md                                     |  1 +
 .../model/OpenAiCompletionParameters.java     | 41 ++++++++++++++++++-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1bc44d5f..eb1c3677 100644
--- a/README.md
+++ b/README.md
@@ -262,6 +262,7 @@ OpenAiChatCompletionParameters request =
         .setMessages(List.of(new OpenAiChatUserMessage().addText(msg)));
 
 OpenAiClient client = OpenAiClient.forModel(GPT_35_TURBO);
+// Do the request before the thread to easily return errors
 Stream<String> stream = client.streamChatCompletion(request);
 
 ThreadContextExecutors.getExecutor().submit(() -> {
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
index 3e989fca..fc3f7d1a 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
@@ -7,6 +7,7 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 import lombok.ToString;
 import lombok.experimental.Accessors;
@@ -106,9 +107,47 @@ public class OpenAiCompletionParameters {
    * message. Default: false.
    */
   @JsonProperty("stream")
-  @Setter(onParam_ = @Nullable)
   private Boolean stream;
 
+  /**
+   * If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only
+   * <a
+   * href="https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format">server-sent
+   * events</a> as they become available, with the stream terminated by a {@code data: [DONE]}
+   * message. Only set this when you set {@code stream: true}.
+   */
+  @JsonProperty("stream_options")
+  private OpenAiStreamOptions streamOptions;
+
+  /** "stream_options": { "include_usage": "true" } */
+  @RequiredArgsConstructor
+  @Setter
+  @JsonFormat(shape = JsonFormat.Shape.OBJECT)
+  @EqualsAndHashCode
+  @ToString
+  public static class OpenAiStreamOptions {
+    /**
+     * If set, an additional chunk will be streamed before the {@code data: [DONE]} message. The
+     * usage field on this chunk shows the token usage statistics for the entire request, and the
+     * choices field will always be an empty array. All other chunks will also include a {@code
+     * usage} field, but with a null value.
+     */
+    @JsonProperty("include_usage")
+    private Boolean include_usage;
+  }
+
+  @Nonnull
+  public OpenAiCompletionParameters setStream(final boolean stream) {
+    if (stream) {
+      this.stream = true;
+      this.streamOptions = new OpenAiStreamOptions().setInclude_usage(true);
+    } else {
+      this.stream = null;
+      this.streamOptions = null;
+    }
+    return this;
+  }
+
   /**
    * Up to four sequences where the API will stop generating further tokens. The returned text won't
    * contain the stop sequence.

From 05dedf9227dd91081e50f146c34034b6d9c7eb34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20D=C3=BCmont?= <alexander.duemont@sap.com>
Date: Wed, 4 Sep 2024 13:15:59 +0200
Subject: [PATCH 50/55] Change Executor#submit() to #execute()

---
 README.md                                                     | 2 +-
 .../java/com/sap/ai/sdk/app/controllers/OpenAiController.java | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 1bc44d5f..41016f50 100644
--- a/README.md
+++ b/README.md
@@ -264,7 +264,7 @@ OpenAiChatCompletionParameters request =
 OpenAiClient client = OpenAiClient.forModel(GPT_35_TURBO);
 Stream<String> stream = client.streamChatCompletion(request);
 
-ThreadContextExecutors.getExecutor().submit(() -> {
+ThreadContextExecutors.getExecutor().execute(() -> {
     // try-with-resources ensures the stream is closed
     try (stream) {
         // send is defined by your framework
diff --git a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
index 01630d61..2e563a50 100644
--- a/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
+++ b/e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java
@@ -80,7 +80,7 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletionDeltas() {
           }
         };
 
-    ThreadContextExecutors.getExecutor().submit(consumeStream);
+    ThreadContextExecutors.getExecutor().execute(consumeStream);
 
     // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);
@@ -124,7 +124,7 @@ public static ResponseEntity<ResponseBodyEmitter> streamChatCompletion() {
           }
         };
 
-    ThreadContextExecutors.getExecutor().submit(consumeStream);
+    ThreadContextExecutors.getExecutor().execute(consumeStream);
 
     // TEXT_EVENT_STREAM allows the browser to display the content as it is streamed
     return ResponseEntity.ok().contentType(MediaType.TEXT_EVENT_STREAM).body(emitter);

From a0ae7793b2ac10302e057861a89b18ecc1f230b0 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Wed, 4 Sep 2024 13:23:55 +0200
Subject: [PATCH 51/55] Added usage testing

---
 .../openai/OpenAiClientTest.java                 | 16 +++++++++++++++-
 .../src/test/resources/streamChatCompletion.txt  |  8 ++++----
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
index f2d45371..aebc3718 100644
--- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
+++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClientTest.java
@@ -374,6 +374,16 @@ void streamChatCompletionDeltas() throws IOException {
         assertThat(deltaList.get(3).getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
         assertThat(deltaList.get(4).getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
 
+        assertThat(deltaList.get(0).getUsage()).isNull();
+        assertThat(deltaList.get(1).getUsage()).isNull();
+        assertThat(deltaList.get(2).getUsage()).isNull();
+        assertThat(deltaList.get(3).getUsage()).isNull();
+        final var usage = deltaList.get(4).getUsage();
+        assertThat(usage).isNotNull();
+        assertThat(usage.getCompletionTokens()).isEqualTo(607);
+        assertThat(usage.getPromptTokens()).isEqualTo(21);
+        assertThat(usage.getTotalTokens()).isEqualTo(628);
+
         assertThat(deltaList.get(0).getChoices()).isEmpty();
         assertThat(deltaList.get(1).getChoices()).hasSize(1);
         assertThat(deltaList.get(2).getChoices()).hasSize(1);
@@ -435,7 +445,11 @@ void streamChatCompletionDeltas() throws IOException {
         assertThat(totalOutput.getCreated()).isEqualTo(1724825677);
         assertThat(totalOutput.getModel()).isEqualTo("gpt-35-turbo");
         assertThat(totalOutput.getObject()).isEqualTo("chat.completion.chunk");
-        assertThat(totalOutput.getUsage()).isNull();
+        final var totalUsage = totalOutput.getUsage();
+        assertThat(totalUsage).isNotNull();
+        assertThat(totalUsage.getCompletionTokens()).isEqualTo(607);
+        assertThat(totalUsage.getPromptTokens()).isEqualTo(21);
+        assertThat(totalUsage.getTotalTokens()).isEqualTo(628);
         assertThat(totalOutput.getSystemFingerprint()).isEqualTo("fp_e49e4201a9");
         assertThat(totalOutput.getPromptFilterResults()).isNotNull();
         assertFilter(totalOutput.getPromptFilterResults().get(0).getContentFilterResults());
diff --git a/foundation-models/openai/src/test/resources/streamChatCompletion.txt b/foundation-models/openai/src/test/resources/streamChatCompletion.txt
index 14d23729..6a4f849c 100644
--- a/foundation-models/openai/src/test/resources/streamChatCompletion.txt
+++ b/foundation-models/openai/src/test/resources/streamChatCompletion.txt
@@ -1,6 +1,6 @@
 data: {"choices":[],"created":0,"id":"","model":"","object":"","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}]}
-data: {"choices":[{"content_filter_results":{},"delta":{"content":"","role":"assistant"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}
-data: {"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"Sure"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}
-data: {"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"!"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}
-data: {"choices":[{"content_filter_results":{},"delta":{},"finish_reason":"stop","index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9"}
+data: {"choices":[{"content_filter_results":{},"delta":{"content":"","role":"assistant"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9","usage":null}
+data: {"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"Sure"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9","usage":null}
+data: {"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"delta":{"content":"!"},"finish_reason":null,"index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9","usage":null}
+data: {"choices":[{"content_filter_results":{},"delta":{},"finish_reason":"stop","index":0}],"created":1724825677,"id":"chatcmpl-A16EvnkgEm6AdxY0NoOmGPjsJucQ1","model":"gpt-35-turbo","object":"chat.completion.chunk","system_fingerprint":"fp_e49e4201a9","usage":{"completion_tokens":607,"prompt_tokens":21,"total_tokens":628}}
 data: [DONE]

From 2c934f7f58127ed4a57e8a95fe03adddf6018467 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Wed, 4 Sep 2024 13:31:24 +0200
Subject: [PATCH 52/55] Added beautiful Javadoc to enableStreaming

---
 .../foundationmodels/openai/OpenAiClient.java |  2 +-
 .../model/OpenAiCompletionParameters.java     | 20 +++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
index 257b55dd..333ffbed 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java
@@ -140,7 +140,7 @@ private static void throwOnContentFilter(@Nonnull final OpenAiChatCompletionDelt
   @Nonnull
   public Stream<OpenAiChatCompletionDelta> streamChatCompletionDeltas(
       @Nonnull final OpenAiChatCompletionParameters parameters) throws OpenAiClientException {
-    parameters.setStream(true);
+    parameters.enableStreaming();
     return executeStream("/chat/completions", parameters, OpenAiChatCompletionDelta.class);
   }
 
diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
index fc3f7d1a..c2bb9373 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/OpenAiCompletionParameters.java
@@ -136,16 +136,16 @@ public static class OpenAiStreamOptions {
     private Boolean include_usage;
   }
 
-  @Nonnull
-  public OpenAiCompletionParameters setStream(final boolean stream) {
-    if (stream) {
-      this.stream = true;
-      this.streamOptions = new OpenAiStreamOptions().setInclude_usage(true);
-    } else {
-      this.stream = null;
-      this.streamOptions = null;
-    }
-    return this;
+  /**
+   * Please use {@link
+   * com.sap.ai.sdk.foundationmodels.openai.OpenAiClient#streamChatCompletion(OpenAiChatCompletionParameters)}
+   * instead.
+   *
+   * <p>Enable streaming of the completion. If enabled, partial message deltas will be sent.
+   */
+  public void enableStreaming() {
+    this.stream = true;
+    this.streamOptions = new OpenAiStreamOptions().setInclude_usage(true);
   }
 
   /**

From 77eb4643c268c5387558326bfcb314d3bc672abf Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Wed, 4 Sep 2024 13:59:12 +0200
Subject: [PATCH 53/55] typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a434d101..4c2abca5 100644
--- a/README.md
+++ b/README.md
@@ -262,7 +262,7 @@ OpenAiChatCompletionParameters request =
         .setMessages(List.of(new OpenAiChatUserMessage().addText(msg)));
 
 OpenAiClient client = OpenAiClient.forModel(GPT_35_TURBO);
-// Do the request before the thread to easily return errors
+// Do the request before the thread to easily throw errors
 Stream<String> stream = client.streamChatCompletion(request);
 
 ThreadContextExecutors.getExecutor().execute(() -> {

From 488f060c9a868112a5780fc9b5f4cc972e2398c8 Mon Sep 17 00:00:00 2001
From: I538344 <charles.dubois@sap.com>
Date: Wed, 4 Sep 2024 15:57:25 +0200
Subject: [PATCH 54/55] Fix mistake

---
 .../ai/sdk/foundationmodels/openai/model/StreamedDelta.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java
index 17a01367..ac790901 100644
--- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java
+++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/model/StreamedDelta.java
@@ -1,5 +1,6 @@
 package com.sap.ai.sdk.foundationmodels.openai.model;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 /**
@@ -20,7 +21,7 @@ public interface StreamedDelta {
    *
    * @return the message content or empty string.
    */
-  @Nullable
+  @Nonnull
   String getDeltaContent();
 
   /**

From 59e1382e69b0c194d4fcb216e90725989c93d2d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexander=20D=C3=BCmont?=
 <22489773+newtork@users.noreply.github.com>
Date: Wed, 4 Sep 2024 16:42:47 +0200
Subject: [PATCH 55/55] streaming readme (#48)

* Reduce sample code

* Format
---
 README.md | 56 +++++++++++++++++++++++--------------------------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/README.md b/README.md
index 4c2abca5..9121fa2b 100644
--- a/README.md
+++ b/README.md
@@ -208,8 +208,8 @@ final OpenAiChatCompletionOutput result =
 
 It's possible to pass a stream of chat completion delta elements, e.g. from the application backend to the frontend in real-time.
 
-#### Print the stream to the console
-This is a simple example to get started:
+#### Stream the chat completion asynchronously
+This is a blocking example for streaming and printing directly to the console:
 ```java
 String msg = "Can you give me the first 100 numbers of the Fibonacci sequence?";
 
@@ -221,13 +221,18 @@ OpenAiClient client = OpenAiClient.forModel(GPT_35_TURBO);
 
 // try-with-resources on stream ensures the connection will be closed
 try( Stream<String> stream = client.streamChatCompletion(request)) {
-  stream.forEach(deltaString -> System.out.println(deltaString));
+    stream.forEach(deltaString -> {
+        System.out.print(deltaString);
+        System.out.flush();
+    });
 }
 ```
 
 <details>
-<summary>It's also possible to collect the total output</summary>
-<pre>
+<summary>It's also possible to aggregate the total output.</summary>
+
+The following example is non-blocking.
+Any asynchronous library can be used, e.g. classic Thread API.
 
 ```java
 String msg = "Can you give me the first 100 numbers of the Fibonacci sequence?";
@@ -239,41 +244,26 @@ OpenAiChatCompletionParameters request =
 OpenAiChatCompletionOutput totalOutput = new OpenAiChatCompletionOutput();
 OpenAiClient client = OpenAiClient.forModel(GPT_35_TURBO);
 
-// try-with-resources on stream ensures the connection will be closed
-try( Stream<OpenAiChatCompletionDelta> stream = client.streamChatCompletionDeltas(request)) {
-    stream.peek(totalOutput::addDelta).forEach(deltaObject -> System.out.println(deltaObject));
-}
-
-// access aggregated information from total output, e.g.
-System.out.println("Tokens: " + totalOutput.getUsage().getCompletionTokens());
-```
-</pre>
-</details>
-
-#### Framework-agnostic example
-This example passes the stream of chat completion delta messages to a generic `Consumer<String>`.
-```java
-Consumer<String> emit;
-
-String msg = "Can you give me the first 100 numbers of the Fibonacci sequence?";
-
-OpenAiChatCompletionParameters request =
-    new OpenAiChatCompletionParameters()
-        .setMessages(List.of(new OpenAiChatUserMessage().addText(msg)));
-
-OpenAiClient client = OpenAiClient.forModel(GPT_35_TURBO);
-// Do the request before the thread to easily throw errors
-Stream<String> stream = client.streamChatCompletion(request);
+// Do the request before the thread starts to handle exceptions during request initialization 
+Stream<OpenAiChatCompletionDelta> stream = client.streamChatCompletionDeltas(request);
 
-ThreadContextExecutors.getExecutor().execute(() -> {
+Thread thread = new Thread(() -> {
     // try-with-resources ensures the stream is closed
     try (stream) {
-        // send is defined by your framework
-        stream.forEach(delta -> emit.accept(delta));
+        stream.peek(totalOutput::addDelta).forEach(delta -> System.out.println(delta));
     }
 });
+thread.start(); // non-blocking
+
+thread.join(); // blocking
+
+// access aggregated information from total output, e.g.
+Integer tokens = totalOutput.getUsage().getCompletionTokens();
+System.out.println("Tokens: " + tokens);
 ```
 
+</details>
+
 #### Spring Boot example
 
 Please find [an example in our Spring Boot application](e2e-test-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java).