Merge branch 'langchain4j:main' into main

Suhas-Koheda · Nov 29, 2024 · 548d979 · 548d979
2 parents c155d3a + d9c2f72
commit 548d979
Show file tree

Hide file tree

Showing 134 changed files with 2,030 additions and 4,512 deletions.
diff --git a/.editorconfig b/.editorconfig
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -21,7 +21,8 @@ Closes #
 ## General checklist
 <!-- Please double-check the following points and mark them like this: [X] -->
 - [ ] There are no breaking changes
-- [ ] I have added unit and integration tests for my change
+- [ ] I have added unit and/or integration tests for my change
+- [ ] The tests cover both positive and negative cases
 - [ ] I have manually run all the unit and integration tests in the module I have added/changed, and they are all green
 - [ ] I have manually run all the unit and integration tests in the [core](https://github.com/langchain4j/langchain4j/tree/main/langchain4j-core) and [main](https://github.com/langchain4j/langchain4j/tree/main/langchain4j) modules, and they are all green
 <!-- Before adding documentation and example(s) (below), please wait until the PR is reviewed and approved. -->

diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -102,9 +102,6 @@ jobs:
           MILVUS_API_KEY: ${{ secrets.MILVUS_API_KEY }}
           MILVUS_URI: ${{ secrets.MILVUS_URI }}
           MISTRAL_AI_API_KEY: ${{ secrets.MISTRAL_AI_API_KEY }}
-          MONGODB_ATLAS_USERNAME: ${{ secrets.MONGODB_ATLAS_USERNAME }}
-          MONGODB_ATLAS_PASSWORD: ${{ secrets.MONGODB_ATLAS_PASSWORD }}
-          MONGODB_ATLAS_HOST: ${{ secrets.MONGODB_ATLAS_HOST }}
           NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}

diff --git a/.github/workflows/nightly_jdk17.yaml b/.github/workflows/nightly_jdk17.yaml
@@ -70,9 +70,6 @@ jobs:
           MILVUS_API_KEY: ${{ secrets.MILVUS_API_KEY }}
           MILVUS_URI: ${{ secrets.MILVUS_URI }}
           MISTRAL_AI_API_KEY: ${{ secrets.MISTRAL_AI_API_KEY }}
-          MONGODB_ATLAS_USERNAME: ${{ secrets.MONGODB_ATLAS_USERNAME }}
-          MONGODB_ATLAS_PASSWORD: ${{ secrets.MONGODB_ATLAS_PASSWORD }}
-          MONGODB_ATLAS_HOST: ${{ secrets.MONGODB_ATLAS_HOST }}
           NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
@@ -83,11 +80,12 @@ jobs:
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           WEAVIATE_API_KEY: ${{ secrets.WEAVIATE_API_KEY }}
           WEAVIATE_HOST: ${{ secrets.WEAVIATE_HOST }}
+          CI_DELAY_SECONDS_AZURE_AI_SEARCH: 3
           CI_DELAY_SECONDS_AZURE_OPENAI: 1
-          CI_DELAY_SECONDS_BEDROCK: 3
-          CI_DELAY_SECONDS_GITHUB_MODELS: 3
+          CI_DELAY_SECONDS_BEDROCK: 5
+          CI_DELAY_SECONDS_GITHUB_MODELS: 5
           CI_DELAY_SECONDS_GOOGLE_AI_GEMINI: 5
-          CI_DELAY_SECONDS_VERTEX_AI_GEMINI: 7
+          CI_DELAY_SECONDS_VERTEX_AI_GEMINI: 10
           CI_DELAY_SECONDS_VOYAGE_AI: 22
 
       - name: Upload Test Reports

diff --git a/.github/workflows/nightly_jdk21.yaml b/.github/workflows/nightly_jdk21.yaml
@@ -70,9 +70,6 @@ jobs:
           MILVUS_API_KEY: ${{ secrets.MILVUS_API_KEY }}
           MILVUS_URI: ${{ secrets.MILVUS_URI }}
           MISTRAL_AI_API_KEY: ${{ secrets.MISTRAL_AI_API_KEY }}
-          MONGODB_ATLAS_USERNAME: ${{ secrets.MONGODB_ATLAS_USERNAME }}
-          MONGODB_ATLAS_PASSWORD: ${{ secrets.MONGODB_ATLAS_PASSWORD }}
-          MONGODB_ATLAS_HOST: ${{ secrets.MONGODB_ATLAS_HOST }}
           NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
@@ -83,11 +80,12 @@ jobs:
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           WEAVIATE_API_KEY: ${{ secrets.WEAVIATE_API_KEY }}
           WEAVIATE_HOST: ${{ secrets.WEAVIATE_HOST }}
+          CI_DELAY_SECONDS_AZURE_AI_SEARCH: 3
           CI_DELAY_SECONDS_AZURE_OPENAI: 1
-          CI_DELAY_SECONDS_BEDROCK: 3
-          CI_DELAY_SECONDS_GITHUB_MODELS: 3
+          CI_DELAY_SECONDS_BEDROCK: 5
+          CI_DELAY_SECONDS_GITHUB_MODELS: 5
           CI_DELAY_SECONDS_GOOGLE_AI_GEMINI: 5
-          CI_DELAY_SECONDS_VERTEX_AI_GEMINI: 7
+          CI_DELAY_SECONDS_VERTEX_AI_GEMINI: 10
           CI_DELAY_SECONDS_VOYAGE_AI: 22
 
       - name: Upload Test Reports

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -65,9 +65,6 @@ jobs:
           MILVUS_API_KEY: ${{ secrets.MILVUS_API_KEY }}
           MILVUS_URI: ${{ secrets.MILVUS_URI }}
           MISTRAL_AI_API_KEY: ${{ secrets.MISTRAL_AI_API_KEY }}
-          MONGODB_ATLAS_USERNAME: ${{ secrets.MONGODB_ATLAS_USERNAME }}
-          MONGODB_ATLAS_PASSWORD: ${{ secrets.MONGODB_ATLAS_PASSWORD }}
-          MONGODB_ATLAS_HOST: ${{ secrets.MONGODB_ATLAS_HOST }}
           NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}

diff --git a/.mvn/jvm.config b/.mvn/jvm.config
@@ -1 +1,3 @@
+--add-opens=java.base/java.lang=ALL-UNNAMED
+--add-opens=java.base/java.io=ALL-UNNAMED
 -Djava.awt.headless=true
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -7,6 +7,7 @@ Thank you for investing your time and effort in contributing to our project, we
 - Keep the code compatible with Java 17.
 - Avoid adding new dependencies as much as possible (new dependencies with test scope are OK). If absolutely necessary, try to use the same libraries which are already used in the project. Make sure you run `mvn dependency:analyze` to identify unnecessary dependencies.
 - Write unit and/or integration tests for your code. This is critical: no tests, no review!
+- The tests should cover both positive and negative cases.
 - Make sure you run all unit tests on all modules with `mvn clean test`
 - Avoid making breaking changes. Always keep backward compatibility in mind. For example, instead of removing fields/methods/etc, mark them `@Deprecated` and make sure they still work as before.
 - Follow existing naming conventions.
@@ -44,6 +45,7 @@ Please note that we do not have the capacity to review PRs immediately. We ask f
 
 # Guidelines on adding a new model integration
 
+- Please open PRs with new model integrations in the [langchain4j-community](https://github.com/langchain4j/langchain4j-community) repository
 - [Integration with Anthropic](https://github.com/langchain4j/langchain4j/tree/main/langchain4j-anthropic) is a good example.
 - Use the official SDK if available.
 - If the official SDK is not available, use Java 11 HTTP Client and Jackson to implement the client.
@@ -54,6 +56,7 @@ Please note that we do not have the capacity to review PRs immediately. We ask f
 
 # Guidelines on adding a new embedding store integration
 
+- Please open PRs with new embedding store integrations in the [langchain4j-community](https://github.com/langchain4j/langchain4j-community) repository
 - [Integration with Chroma](https://github.com/langchain4j/langchain4j/tree/main/langchain4j-chroma) is a good example.
 - Use the official SDK if available.
 - If the official SDK is not available, use Java 11 HTTP Client and Jackson to implement the client.

diff --git a/README.md b/README.md
@@ -5,6 +5,7 @@
 [![CODACY](https://img.shields.io/badge/Codacy-Dashboard-blue?style=for-the-badge&logo=codacy)](https://app.codacy.com/gh/langchain4j/langchain4j/dashboard)
 
 [![Discord](https://dcbadge.vercel.app/api/server/JzTFvyjG6R?style=for-the-badge)](https://discord.gg/JzTFvyjG6R)
+[![BlueSky](https://img.shields.io/badge/@langchain4j-follow-blue?logo=bluesky&style=for-the-badge)](https://bsky.app/profile/langchain4j.dev)
 [![X](https://img.shields.io/badge/@langchain4j-follow-blue?logo=x&style=for-the-badge)](https://x.com/langchain4j)
 [![Maven Version](https://img.shields.io/maven-central/v/dev.langchain4j/langchain4j?logo=apachemaven&style=for-the-badge)](https://search.maven.org/#search|gav|1|g:"dev.langchain4j"%20AND%20a:"langchain4j")
 

diff --git a/code-execution-engines/langchain4j-code-execution-engine-judge0/pom.xml b/code-execution-engines/langchain4j-code-execution-engine-judge0/pom.xml
@@ -32,11 +32,6 @@
             <groupId>com.squareup.okhttp3</groupId>
             <artifactId>okhttp</artifactId>
         </dependency>
-
-        <dependency>
-            <groupId>org.jetbrains.kotlin</groupId>
-            <artifactId>kotlin-stdlib-jdk8</artifactId>
-        </dependency>
         <!-- DEPENDENCY CONFLICT RESOLUTION FOR OKHTTP (END)  -->
 
         <dependency>

diff --git a/detekt.yml b/detekt.yml
@@ -0,0 +1,8 @@
+style:
+  active: true
+  MethodName:
+    active: true
+    ignoreOverriddenFunctions: true
+    excludeAnnotatedMethodsOrClasses:
+      - "org.junit.jupiter.api.Test"
+    ignoreTestFiles: true
diff --git a/docs/docs/integrations/embedding-models/dashscope.md b/docs/docs/integrations/embedding-models/dashscope.md
@@ -9,6 +9,12 @@ https://dashscope.aliyun.com/
 
 ## Maven Dependency
 
+:::note
+Since `0.37.0`, `langchain4j-dashscope` has migrated to `langchain4j-community` and is renamed to `langchain4j-community-dashscope`.
+:::
+
+`0.36.2` and previous:
+
 ```xml
 <dependency>
     <groupId>dev.langchain4j</groupId>
@@ -17,6 +23,30 @@ https://dashscope.aliyun.com/
 </dependency>
 ```
 
+`0.37.0` and later:
+
+```xml
+<dependency>
+    <groupId>dev.langchain4j</groupId>
+    <artifactId>langchain4j-community-dashscope</artifactId>
+    <version>0.37.0</version>
+</dependency>
+```
+
+Or, you can use BOM to manage dependencies consistently:
+
+```xml
+<dependencyManagement>
+    <dependency>
+        <groupId>dev.langchain4j</groupId>
+        <artifactId>langchain4j-community-bom</artifactId>
+        <version>0.37.0</version>
+        <typ>pom</typ>
+        <scope>import</scope>
+    </dependency>
+</dependencyManagement>
+```
+
 ## APIs
 
 - `QwenEmbeddingModel`

diff --git a/docs/docs/integrations/embedding-stores/duckdb.md b/docs/docs/integrations/embedding-stores/duckdb.md
@@ -0,0 +1,54 @@
+---
+sidebar_position: 11
+---
+
+# DuckDB
+
+https://duckdb.org/
+
+## Maven Dependency
+
+```xml
+<dependency>
+    <groupId>dev.langchain4j</groupId>
+    <artifactId>langchain4j-community-duckdb</artifactId>
+    <version>0.37.0</version>
+</dependency>
+```
+
+## APIs
+
+- `DuckDBEmbeddingStore`
+
+## Examples
+
+```java
+// Init Model and Store
+var embeddingStore = DuckDBEmbeddingStore.inMemory();
+var embeddingModel = new AllMiniLmL6V2QuantizedEmbeddingModel();
+
+//Create embeddings
+Stream.of(
+            "DuckDB is an amazing database engine!",
+            "Python really lack of typing :D")
+    .forEach(text -> {
+        var segment = TextSegment.from(text);
+        var embedding = embeddingModel.embed(segment).content();
+        embeddingStore.add(embedding, segment);
+    });
+
+
+// Search request
+var queryEmbedding = embeddingModel.embed("What is the best database engine").content();
+var request = EmbeddingSearchRequest.builder()
+               .queryEmbedding(queryEmbedding)
+               .maxResults(1)
+               .build();
+
+var relevant = embeddingStore.search(request);
+EmbeddingMatch<TextSegment> embeddingMatch = relevant.matches().get(0);
+
+// Show results
+System.out.println(embeddingMatch.score()); // 0.8416415629618381
+System.out.println(embeddingMatch.embedded().text()); //DuckDB is an amazing database engine!
+```
diff --git a/docs/docs/integrations/embedding-stores/elasticsearch.md b/docs/docs/integrations/embedding-stores/elasticsearch.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 11
+sidebar_position: 12
 ---
 
 # Elasticsearch

diff --git a/docs/docs/integrations/embedding-stores/index.md b/docs/docs/integrations/embedding-stores/index.md
@@ -16,16 +16,17 @@ sidebar_position: 0
 | [ClickHouse](/integrations/embedding-stores/clickhouse)                               | ✅                | ✅                          | ✅                   |
 | [Coherence](/integrations/embedding-stores/coherence)                                 | ✅                | ✅                          | ✅                   |
 | [Couchbase](/integrations/embedding-stores/couchbase)                                 | ✅                |                            | ✅                   |
+| [DuckDB](/integrations/embedding-stores/duckdb)                                       | ✅                | ✅                          | ✅                   |
 | [Elasticsearch](/integrations/embedding-stores/elasticsearch)                         | ✅                | ✅                          | ✅                   |
 | [Infinispan](/integrations/embedding-stores/infinispan)                               | ✅                |                            |                     |
 | [Milvus](/integrations/embedding-stores/milvus)                                       | ✅                | ✅                          | ✅                   |
-| [MongoDB Atlas](/integrations/embedding-stores/mongodb-atlas)                         | ✅                | Only native filter support |                     |
+| [MongoDB Atlas](/integrations/embedding-stores/mongodb-atlas)                         | ✅                | ✅                          | ✅                   |
 | [Neo4j](/integrations/embedding-stores/neo4j)                                         | ✅                |                            |                     |
 | [OpenSearch](/integrations/embedding-stores/opensearch)                               | ✅                |                            |                     |
 | [Oracle](/integrations/embedding-stores/oracle)                                       | ✅                | ✅                          | ✅                   |
 | [PGVector](/integrations/embedding-stores/pgvector)                                   | ✅                | ✅                          | ✅                   |
 | [Pinecone](/integrations/embedding-stores/pinecone)                                   | ✅                | ✅                          | ✅                   |
-| [Qdrant](/integrations/embedding-stores/qdrant)                                       | ✅                | ✅                          |                     |
+| [Qdrant](/integrations/embedding-stores/qdrant)                                       | ✅                | ✅                          | ✅                   |
 | [Redis](/integrations/embedding-stores/redis)                                         | ✅                |                            |                     |
 | [Tablestore](/integrations/embedding-stores/tablestore)                               | ✅                | ✅                          | ✅                   |
 | [Vearch](/integrations/embedding-stores/vearch)                                       | ✅                |                            |                     |

diff --git a/docs/docs/integrations/embedding-stores/infinispan.md b/docs/docs/integrations/embedding-stores/infinispan.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 12
+sidebar_position: 13
 ---
 
 # Infinispan

diff --git a/docs/docs/integrations/embedding-stores/milvus.md b/docs/docs/integrations/embedding-stores/milvus.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 13
+sidebar_position: 14
 ---
 
 # Milvus

diff --git a/docs/docs/integrations/embedding-stores/mongodb-atlas.md b/docs/docs/integrations/embedding-stores/mongodb-atlas.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 14
+sidebar_position: 15
 ---
 
 # MongoDB Atlas and Vector Search
@@ -10,13 +10,16 @@ sidebar_position: 14
 
 ## Prerequisites
 
-You must have an Atlas cluster that runs one of the following MongoDB versions:
+Vector Search requires the following or later MongoDB versions:
 
 - 6.0.11
 - 7.0.2
-- Later versions (including Release Candidates).
 
-To use Atlas Vector search, you need to have an Atlas deployment. MongoDB offers a free forever cluster that you can use for testing. See the [Get Started with Atlas](https://www.mongodb.com/docs/atlas/getting-started/) tutorial to learn more. Once you deploy a cluster, you can create a vector search index by using the index JSON editor.
+Vector Search is available on Atlas, and as of 2024 is available in early access for Community Edition.
+
+MongoDB offers a free forever cluster. See the [Get Started with Atlas](https://www.mongodb.com/docs/atlas/getting-started/) tutorial to learn more. Once you deploy a cluster, you can create a vector search index by using the index JSON editor. Alternatively, the LangChain4j `MongoDbEmbeddingStore` builder offers a `createIndex` option to automate index creation.
+
+MongoDB also offers [Atlas CLI](https://www.mongodb.com/docs/atlas/cli/current/) for local development.
 
 ## Maven Dependency
 
@@ -34,6 +37,5 @@ To use Atlas Vector search, you need to have an Atlas deployment. MongoDB offers
 
 ## Examples
 
-- [MongoDbEmbeddingStoreCloudIT](https://github.com/langchain4j/langchain4j/blob/main/langchain4j-mongodb-atlas/src/test/java/dev/langchain4j/store/embedding/mongodb/MongoDbEmbeddingStoreCloudIT.java)
-- [MongoDbEmbeddingStoreLocalIT](https://github.com/langchain4j/langchain4j/blob/main/langchain4j-mongodb-atlas/src/test/java/dev/langchain4j/store/embedding/mongodb/MongoDbEmbeddingStoreLocalIT.java)
+- [MongoDbEmbeddingStoreIT](https://github.com/langchain4j/langchain4j/blob/main/langchain4j-mongodb-atlas/src/test/java/dev/langchain4j/store/embedding/mongodb/MongoDbEmbeddingStoreIT.java)
 
diff --git a/docs/docs/integrations/embedding-stores/neo4j.md b/docs/docs/integrations/embedding-stores/neo4j.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 15
+sidebar_position: 16
 ---
 
 # Neo4j

diff --git a/docs/docs/integrations/embedding-stores/opensearch.md b/docs/docs/integrations/embedding-stores/opensearch.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 16
+sidebar_position: 17
 ---
 
 # OpenSearch

diff --git a/docs/docs/integrations/embedding-stores/oracle.md b/docs/docs/integrations/embedding-stores/oracle.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 17
+sidebar_position: 18
 ---
 
 # Oracle

diff --git a/docs/docs/integrations/embedding-stores/pgvector.md b/docs/docs/integrations/embedding-stores/pgvector.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 18
+sidebar_position: 19
 ---
 
 # PGVector

diff --git a/docs/docs/integrations/embedding-stores/pinecone.md b/docs/docs/integrations/embedding-stores/pinecone.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 19
+sidebar_position: 20
 ---
 
 # Pinecone

diff --git a/docs/docs/integrations/embedding-stores/qdrant.md b/docs/docs/integrations/embedding-stores/qdrant.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 20
+sidebar_position: 21
 ---
 
 # Qdrant

diff --git a/docs/docs/integrations/embedding-stores/redis.md b/docs/docs/integrations/embedding-stores/redis.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 21
+sidebar_position: 22
 ---
 
 # Redis

diff --git a/docs/docs/integrations/embedding-stores/tablestore.md b/docs/docs/integrations/embedding-stores/tablestore.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 22
+sidebar_position: 23
 ---
 
 # Tablestore

diff --git a/docs/docs/integrations/embedding-stores/vearch.md b/docs/docs/integrations/embedding-stores/vearch.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 23
+sidebar_position: 24
 ---
 
 # Vearch
@@ -10,7 +10,7 @@ https://github.com/vearch/vearch
 ## Maven Dependency
 
 :::note
-Since `0.37.0`, `langchain4j-vearch` has migrated to `langchain4j-community` and is renamed to `langchain4j-community-vearch
+Since `0.37.0`, `langchain4j-vearch` has migrated to `langchain4j-community` and is renamed to `langchain4j-community-vearch`.
 :::
 
 `0.36.2` and previous:

diff --git a/docs/docs/integrations/embedding-stores/vespa.md b/docs/docs/integrations/embedding-stores/vespa.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 24
+sidebar_position: 25
 ---
 
 # Vespa

diff --git a/docs/docs/integrations/embedding-stores/weaviate.md b/docs/docs/integrations/embedding-stores/weaviate.md
@@ -1,5 +1,5 @@
 ---
-sidebar_position: 25
+sidebar_position: 26
 ---
 
 # Weaviate