diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 07d2a65984f4a..711826c38900b 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -2,6 +2,8 @@ ### API Changes +1. Infra: Removes support for jdbcURL prefix of `jdbc:tc:shardingsphere0clickhouse:` - [#33707](https://github.com/apache/shardingsphere/pull/33707) + ### New Features ### Enhancements diff --git a/docs/document/content/user-manual/common-config/builtin-algorithm/expr.cn.md b/docs/document/content/user-manual/common-config/builtin-algorithm/expr.cn.md index f615696ecb4af..eaeba840b3101 100644 --- a/docs/document/content/user-manual/common-config/builtin-algorithm/expr.cn.md +++ b/docs/document/content/user-manual/common-config/builtin-algorithm/expr.cn.md @@ -122,6 +122,7 @@ Truffle 与 JDK 的向后兼容性矩阵位于 https://medium.com/graalvm/40027a 考虑一个简单的 SPI 实现类， ```java +package org.example; import org.apache.shardingsphere.infra.expr.spi.InlineExpressionParser; import java.util.Arrays; import java.util.List; @@ -146,6 +147,12 @@ public final class CustomInlineExpressionParserFixture implements InlineExpressi } ``` +并在项目的 classpath 添加 `META-INF/services/org.apache.shardingsphere.infra.expr.spi.InlineExpressionParser`文件， + +``` +org.example.CustomInlineExpressionParserFixture +``` + 此时对于 ShardingSphere 配置文件中的 `actualDataNodes`， 1. 若配置为 `spring`，将被转化为 `t_order_2024_01, t_order_2024_02`。 2. 若配置为 `summer`，将被转化为 `t_order_2024_03, t_order_2024_04`。 diff --git a/docs/document/content/user-manual/common-config/builtin-algorithm/expr.en.md b/docs/document/content/user-manual/common-config/builtin-algorithm/expr.en.md index 40914d27955b2..5863beb412086 100644 --- a/docs/document/content/user-manual/common-config/builtin-algorithm/expr.en.md +++ b/docs/document/content/user-manual/common-config/builtin-algorithm/expr.en.md @@ -159,6 +159,12 @@ public final class CustomInlineExpressionParserFixture implements InlineExpressi } ``` +And add the file `META-INF/services/org.apache.shardingsphere.infra.expr.spi.InlineExpressionParser` to the project classpath, + +``` +org.example.CustomInlineExpressionParserFixture +``` + At this time, for `actualDataNodes` in the ShardingSphere configuration file, 1. If configured as `spring`, it will be converted to `t_order_2024_01, t_order_2024_02`. 2. If configured as `summer`, it will be converted to `t_order_2024_03, t_order_2024_04`. diff --git a/docs/document/content/user-manual/common-config/builtin-algorithm/sharding.cn.md b/docs/document/content/user-manual/common-config/builtin-algorithm/sharding.cn.md index 0cd29976d13fe..f71f6800df3bd 100644 --- a/docs/document/content/user-manual/common-config/builtin-algorithm/sharding.cn.md +++ b/docs/document/content/user-manual/common-config/builtin-algorithm/sharding.cn.md @@ -96,14 +96,14 @@ Apache ShardingSphere 内置的标准分片算法实现类包括： 可配置属性： -| *属性名称* | *数据类型* | *说明* | *默认值* | -|------------------------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------|-------| -| datetime-pattern | String | 分片键的时间戳格式，必须遵循 Java DateTimeFormatter 的格式。例如：yyyy-MM-dd HH:mm:ss，yyyy-MM-dd 或 HH:mm:ss 等。但不支持与 `java.time.chrono.JapaneseDate` 相关的 Gy-MM 等 | | -| datetime-lower | String | 时间分片下界值，格式与 `datetime-pattern` 定义的时间戳格式一致 | | -| datetime-upper (?) | String | 时间分片上界值，格式与 `datetime-pattern` 定义的时间戳格式一致 | 当前时间 | -| sharding-suffix-pattern | String | 分片数据源或真实表的后缀格式，必须遵循 Java DateTimeFormatter 的格式，必须和 `datetime-interval-unit` 保持一致。例如：yyyyMM | | -| datetime-interval-amount (?) | int | 分片键时间间隔，超过该时间间隔将进入下一分片 | 1 | -| datetime-interval-unit (?) | String | 分片键时间间隔单位，必须遵循 Java ChronoUnit 的枚举值。例如：MONTHS | DAYS | +| *属性名称* | *数据类型* | *说明* | *默认值* | +|------------------------------|--------|-----------------------------------------------------------------------------------------------------------------------------------------------|-------| +| datetime-pattern | String | 分片键的时间戳格式，必须遵循 Java DateTimeFormatter 的格式。例如：yyyy-MM-dd HH:mm:ss，yyyy-MM-dd 或 HH:mm:ss 等。但不支持与 `java.time.chrono.JapaneseDate` 相关的 GGGGy-MM 等 | | +| datetime-lower | String | 时间分片下界值，格式与 `datetime-pattern` 定义的时间戳格式一致 | | +| datetime-upper (?) | String | 时间分片上界值，格式与 `datetime-pattern` 定义的时间戳格式一致 | 当前时间 | +| sharding-suffix-pattern | String | 分片数据源或真实表的后缀格式，必须遵循 Java DateTimeFormatter 的格式，必须和 `datetime-interval-unit` 保持一致。例如：yyyyMM | | +| datetime-interval-amount (?) | int | 分片键时间间隔，超过该时间间隔将进入下一分片 | 1 | +| datetime-interval-unit (?) | String | 分片键时间间隔单位，必须遵循 Java ChronoUnit 的枚举值。例如：MONTHS | DAYS | ### 复合分片算法 diff --git a/docs/document/content/user-manual/common-config/builtin-algorithm/sharding.en.md b/docs/document/content/user-manual/common-config/builtin-algorithm/sharding.en.md index 04c0ce3844ec2..c841936185695 100644 --- a/docs/document/content/user-manual/common-config/builtin-algorithm/sharding.en.md +++ b/docs/document/content/user-manual/common-config/builtin-algorithm/sharding.en.md @@ -100,14 +100,14 @@ Type: INTERVAL Attributes: -| *Name* | *DataType* | *Description* | *Default Value* | -|------------------------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------| -| datetime-pattern | String | Timestamp pattern of sharding value, must can be transformed to Java LocalDateTime. For example: yyyy-MM-dd HH:mm:ss, yyyy-MM-dd or HH:mm:ss etc. But Gy-MM etc. related to `java.time.chrono.JapaneseDate` are not supported | - | -| datetime-lower | String | Datetime sharding lower boundary, pattern is defined `datetime-pattern` | - | -| datetime-upper (?) | String | Datetime sharding upper boundary, pattern is defined `datetime-pattern` | Now | -| sharding-suffix-pattern | String | Suffix pattern of sharding data sources or tables, must can be transformed to Java LocalDateTime, must be consistent with `datetime-interval-unit`. For example: yyyyMM | - | -| datetime-interval-amount (?) | int | Interval of sharding value, after which the next shard will be entered | 1 | -| datetime-interval-unit (?) | String | Unit of sharding value interval, must can be transformed to Java ChronoUnit's Enum value. For example: MONTHS | DAYS | +| *Name* | *DataType* | *Description* | *Default Value* | +|------------------------------|------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------| +| datetime-pattern | String | Timestamp pattern of sharding value, must can be transformed to Java LocalDateTime. For example: yyyy-MM-dd HH:mm:ss, yyyy-MM-dd or HH:mm:ss etc. But GGGGy-MM etc. related to `java.time.chrono.JapaneseDate` are not supported | - | +| datetime-lower | String | Datetime sharding lower boundary, pattern is defined `datetime-pattern` | - | +| datetime-upper (?) | String | Datetime sharding upper boundary, pattern is defined `datetime-pattern` | Now | +| sharding-suffix-pattern | String | Suffix pattern of sharding data sources or tables, must can be transformed to Java LocalDateTime, must be consistent with `datetime-interval-unit`. For example: yyyyMM | - | +| datetime-interval-amount (?) | int | Interval of sharding value, after which the next shard will be entered | 1 | +| datetime-interval-unit (?) | String | Unit of sharding value interval, must can be transformed to Java ChronoUnit's Enum value. For example: MONTHS | DAYS | ### Complex Sharding Algorithm diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.cn.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.cn.md new file mode 100644 index 0000000000000..1ff66f3ce460b --- /dev/null +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.cn.md @@ -0,0 +1,40 @@ ++++ +title = "Testcontainers" +weight = 6 ++++ + +ShardingSphere 默认情况下不提供对 `org.testcontainers.jdbc.ContainerDatabaseDriver` 的 `driverClassName` 的支持。 +要在 ShardingSphere 的配置文件为数据节点使用类似 `jdbc:tc:postgresql:17.1-bookworm://test-native-databases-postgres/demo_ds_0` 的 `jdbcUrl`， +可能的 Maven 依赖关系如下， + +```xml + + + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-infra-database-testcontainers + ${shardingsphere.version} + + + org.testcontainers + postgresql + 1.20.3 + test + + +``` + +`org.apache.shardingsphere:shardingsphere-infra-database-testcontainers` 为 testcontainers-java 分格的 jdbcURL 提供支持， +包括但不限于， + +1. 为 `jdbc:tc:clickhouse:` 的 jdbcURL 前缀提供支持的 Maven 模块 `org.testcontainers:clickhouse:1.20.3` +2. 为 `jdbc:tc:postgresql:` 的 jdbcURL 前缀提供支持的 Maven 模块 `org.testcontainers:postgresql:1.20.3` +3. 为 `jdbc:tc:sqlserver:` 的 jdbcURL 前缀提供支持的 Maven 模块 `org.testcontainers:mssqlserver:1.20.3` +4. 为 `jdbc:tc:mariadb:` 的 jdbcURL 前缀提供支持的 Maven 模块 `org.testcontainers:mariadb:1.20.3` +5. 为 `jdbc:tc:mysql:` 的 jdbcURL 前缀提供支持的 Maven 模块 `org.testcontainers:mysql:1.20.3` +6. 为 `jdbc:tc:oracle:` 的 jdbcURL 前缀提供支持的 Maven 模块 `org.testcontainers:oracle-xe:1.20.3` 和 `org.testcontainers:oracle-free:1.20.3` +7. 为 `jdbc:tc:tidb:` 的 jdbcURL 前缀提供支持的 Maven 模块 `org.testcontainers:tidb:1.20.3` diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.en.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.en.md new file mode 100644 index 0000000000000..0593e098d52ea --- /dev/null +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/testcontainers/_index.en.md @@ -0,0 +1,40 @@ ++++ +title = "Testcontainers" +weight = 6 ++++ + +ShardingSphere does not provide support for `driverClassName` of `org.testcontainers.jdbc.ContainerDatabaseDriver` by default. +To use `jdbcUrl` like `jdbc:tc:postgresql:17.1-bookworm://test-native-databases-postgres/demo_ds_0` for data nodes in ShardingSphere's configuration file, +the possible Maven dependencies are as follows, + +```xml + + + org.apache.shardingsphere + shardingsphere-jdbc + ${shardingsphere.version} + + + org.apache.shardingsphere + shardingsphere-infra-database-testcontainers + ${shardingsphere.version} + + + org.testcontainers + postgresql + 1.20.3 + test + + +``` + +`org.apache.shardingsphere:shardingsphere-infra-database-testcontainers` provides support for jdbcURL in the testcontainers-java partition, +including but not limited to, + +1. Maven module `org.testcontainers:clickhouse:1.20.3` that provides support for jdbcURL prefixes for `jdbc:tc:clickhouse:` +2. Maven module `org.testcontainers:postgresql:1.20.3` that provides support for jdbcURL prefixes for `jdbc:tc:postgresql:` +3. Maven module `org.testcontainers:mssqlserver:1.20.3` that provides support for jdbcURL prefixes for `jdbc:tc:sqlserver:` +4. Maven module `org.testcontainers:mariadb:1.20.3` that provides support for jdbcURL prefixes for `jdbc:tc:mariadb:` +5. Maven module `org.testcontainers:mysql:1.20.3` that provides support for jdbcURL prefixes of `jdbc:tc:mysql:` +6. Maven modules `org.testcontainers:oracle-xe:1.20.3` and `org.testcontainers:oracle-free:1.20.3` that provide support for jdbcURL prefixes of `jdbc:tc:oracle:` +7. Maven module `org.testcontainers:tidb:1.20.3` that provides support for jdbcURL prefixes of `jdbc:tc:tidb:` diff --git a/examples/pom.xml b/examples/pom.xml index 85e522eeb64d3..64085056a1fc9 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -338,7 +338,7 @@ - jdk11-22 + jdk11+ [11,) diff --git a/infra/database/type/testcontainers/src/main/java/org/apache/shardingsphere/infra/database/testcontainers/type/TcClickHouseDatabaseType.java b/infra/database/type/testcontainers/src/main/java/org/apache/shardingsphere/infra/database/testcontainers/type/TcClickHouseDatabaseType.java index 0f96b8da17706..c213c7aa71c56 100644 --- a/infra/database/type/testcontainers/src/main/java/org/apache/shardingsphere/infra/database/testcontainers/type/TcClickHouseDatabaseType.java +++ b/infra/database/type/testcontainers/src/main/java/org/apache/shardingsphere/infra/database/testcontainers/type/TcClickHouseDatabaseType.java @@ -20,8 +20,8 @@ import org.apache.shardingsphere.infra.database.core.type.DatabaseType; import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Optional; /** @@ -29,14 +29,9 @@ */ public final class TcClickHouseDatabaseType implements TestcontainersDatabaseType { - /** - * TODO See the JavaDoc for `org.apache.shardingsphere.test.natived.jdbc.commons.testcontainers.ClickHouseProvider`. - * - * @return prefixes of JDBC URL - */ @Override public Collection getJdbcUrlPrefixes() { - return Arrays.asList("jdbc:tc:clickhouse:", "jdbc:tc:shardingsphere0clickhouse:"); + return Collections.singletonList("jdbc:tc:clickhouse:"); } @Override diff --git a/infra/database/type/testcontainers/src/test/java/org/apache/shardingsphere/infra/database/testcontainers/type/TestcontainersDatabaseTypeTest.java b/infra/database/type/testcontainers/src/test/java/org/apache/shardingsphere/infra/database/testcontainers/type/TestcontainersDatabaseTypeTest.java index 4fb78478e2780..c81750acbe1e2 100644 --- a/infra/database/type/testcontainers/src/test/java/org/apache/shardingsphere/infra/database/testcontainers/type/TestcontainersDatabaseTypeTest.java +++ b/infra/database/type/testcontainers/src/test/java/org/apache/shardingsphere/infra/database/testcontainers/type/TestcontainersDatabaseTypeTest.java @@ -21,7 +21,6 @@ import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collections; import static org.hamcrest.CoreMatchers.is; @@ -31,7 +30,7 @@ class TestcontainersDatabaseTypeTest { @Test void assertGetJdbcUrlPrefixes() { - assertThat(TypedSPILoader.getService(DatabaseType.class, "TC-ClickHouse").getJdbcUrlPrefixes(), is(Arrays.asList("jdbc:tc:clickhouse:", "jdbc:tc:shardingsphere0clickhouse:"))); + assertThat(TypedSPILoader.getService(DatabaseType.class, "TC-ClickHouse").getJdbcUrlPrefixes(), is(Collections.singletonList("jdbc:tc:clickhouse:"))); assertThat(TypedSPILoader.getService(DatabaseType.class, "TC-MariaDB").getJdbcUrlPrefixes(), is(Collections.singleton("jdbc:tc:mariadb:"))); assertThat(TypedSPILoader.getService(DatabaseType.class, "TC-MySQL").getJdbcUrlPrefixes(), is(Collections.singleton("jdbc:tc:mysql:"))); assertThat(TypedSPILoader.getService(DatabaseType.class, "TC-Oracle").getJdbcUrlPrefixes(), is(Collections.singleton("jdbc:tc:oracle:"))); diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/0.6.3/resource-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/0.6.3/resource-config.json index 621f0ad33a9bb..4e119f34a54d0 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/0.6.3/resource-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/0.6.3/resource-config.json @@ -3,6 +3,18 @@ "includes":[{ "condition":{"typeReachable":"com.clickhouse.logging.LoggerFactory"}, "pattern":"\\QMETA-INF/services/com.clickhouse.logging.LoggerFactory\\E" + }, { + "condition":{"typeReachable":"com.clickhouse.client.ClickHouseRequestManager"}, + "pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseRequestManager\\E" + }, { + "condition":{"typeReachable":"com.clickhouse.data.ClickHouseDataStreamFactory"}, + "pattern":"\\QMETA-INF/services/com.clickhouse.data.ClickHouseDataStreamFactory\\E" + }, { + "condition":{"typeReachable":"com.clickhouse.jdbc.JdbcTypeMapping"}, + "pattern":"\\QMETA-INF/services/com.clickhouse.jdbc.JdbcTypeMapping\\E" + }, { + "condition":{"typeReachable":"com.clickhouse.client.internal.jpountz.lz4.LZ4JNI"}, + "pattern":"\\Qcom/clickhouse/client/internal/jpountz/util/linux/amd64/liblz4-java.so\\E" }]}, "bundles":[] } diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json index 326a51669d3fa..57bbb3722e155 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json @@ -60,7 +60,7 @@ "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.standalone.persist.StandaloneMetaDataManagerPersistService"}, + "condition":{"typeReachable":"org.apache.shardingsphere.metadata.persist.service.version.MetaDataVersionPersistService"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, { @@ -176,6 +176,11 @@ "queryAllDeclaredMethods":true, "methods":[{"name":"close","parameterTypes":[] }, {"name":"read","parameterTypes":["java.nio.CharBuffer"] }] }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, + "name":"java.io.Serializable", + "queryAllDeclaredMethods":true +}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"}, "name":"java.io.Serializable", @@ -468,11 +473,6 @@ "condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"}, "name":"java.net.Socket" }, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, - "name":"java.net.SocketException", - "methods":[{"name":"","parameterTypes":["java.lang.String"] }] -}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"}, "name":"java.net.URI" @@ -1103,7 +1103,7 @@ "methods":[{"name":"","parameterTypes":[] }] }, { - "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.DatabaseConnector"}, + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.StandardDatabaseConnector"}, "name":"org.apache.shardingsphere.encrypt.checker.sql.EncryptSupportedSQLCheckersBuilder", "methods":[{"name":"","parameterTypes":[] }] }, @@ -2070,7 +2070,7 @@ "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f490fb226e0"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007ffa47b22d00"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.MetaDataChangedSubscriber" }, { @@ -2417,10 +2417,6 @@ "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader"}, "name":"org.apache.shardingsphere.readwritesplitting.checker.ReadwriteSplittingRuleConfigurationChecker" }, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.infra.datanode.DataNodes"}, - "name":"org.apache.shardingsphere.readwritesplitting.datanode.ReadwriteSplittingDataNodeBuilder" -}, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.mysql.command.query.text.query.MySQLComQueryPacketExecutor"}, "name":"org.apache.shardingsphere.readwritesplitting.distsql.handler.update.AlterReadwriteSplittingRuleExecutor" @@ -2548,10 +2544,6 @@ "condition":{"typeReachable":"org.apache.shardingsphere.infra.spi.type.ordered.OrderedSPILoader"}, "name":"org.apache.shardingsphere.shadow.checker.ShadowRuleConfigurationChecker" }, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.infra.datanode.DataNodes"}, - "name":"org.apache.shardingsphere.shadow.datanode.ShadowDataNodeBuilder" -}, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.mysql.command.query.text.query.MySQLComQueryPacketExecutor"}, "name":"org.apache.shardingsphere.shadow.distsql.handler.update.AlterDefaultShadowAlgorithmExecutor" @@ -3602,5 +3594,10 @@ { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.initializer.BootstrapInitializer"}, "name":"org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfigurationCustomizer" +}, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.kernel.ExecutorEngine"}, + "name":"sun.security.provider.SecureRandom", + "methods":[{"name":"","parameterTypes":[] }, {"name":"","parameterTypes":["java.security.SecureRandomParameters"] }] } ] \ No newline at end of file diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json index 5a97d455368ad..fe5faec8d6aa4 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json @@ -78,15 +78,6 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, "pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E" - }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, - "pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseRequestManager\\E" - }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, - "pattern":"\\QMETA-INF/services/com.clickhouse.data.ClickHouseDataStreamFactory\\E" - }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, - "pattern":"\\QMETA-INF/services/com.clickhouse.jdbc.JdbcTypeMapping\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.cluster.etcd.EtcdRepository"}, "pattern":"\\QMETA-INF/services/io.grpc.LoadBalancerProvider\\E" @@ -222,9 +213,6 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.props.TypedPropertyValue"}, "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.database.core.type.DatabaseType\\E" - }, { - "condition":{"typeReachable":"org.apache.shardingsphere.infra.datanode.DataNodes"}, - "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.datanode.DataNodeBuilder\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.datasource.pool.destroyer.DataSourcePoolDestroyer"}, "pattern":"\\QMETA-INF/services/org.apache.shardingsphere.infra.datasource.pool.destroyer.DataSourcePoolActiveDetector\\E" @@ -393,9 +381,6 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.transaction.xa.atomikos.manager.AtomikosTransactionManagerProvider"}, "pattern":"\\Qcom/atomikos/icatch/provider/imp/transactions.properties\\E" - }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, - "pattern":"\\Qcom/clickhouse/client/internal/jpountz/util/linux/amd64/liblz4-java.so\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, "pattern":"\\Qcontainer-license-acceptance.txt\\E" @@ -1971,9 +1956,6 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, "pattern":"\\Qtest-native/sql/seata-script-client-at-postgresql.sql\\E" - }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, - "pattern":"\\Qtest-native/sql/test-native-databases-clickhouse.sql\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.url.classpath.ClassPathURLLoader"}, "pattern":"\\Qtest-native/yaml/jdbc/databases/clickhouse.yaml\\E" diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json index 44550b54ae641..fbbf20840062c 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json @@ -188,6 +188,11 @@ "name":"java.lang.management.MemoryUsage", "allPublicMethods": true }, +{ + "condition":{"typeReachable":"java.net.SocketException"}, + "name":"java.net.SocketException", + "methods":[{"name":"","parameterTypes":["java.lang.String"] }] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.util.yaml.YamlEngine"}, "name":"org.apache.shardingsphere.encrypt.yaml.config.rule.YamlEncryptColumnItemRuleConfiguration", diff --git a/pom.xml b/pom.xml index 6fc13b629b5fa..0964f7d6b070f 100644 --- a/pom.xml +++ b/pom.xml @@ -142,7 +142,7 @@ 3.0 4.11.0 4.2.2 - 1.20.1 + 1.20.3 1.9.0 24.1.0 @@ -1001,7 +1001,7 @@ - jdk11-22 + jdk11+ [11,) diff --git a/test/native/native-image-filter/extra-filter.json b/test/native/native-image-filter/extra-filter.json index 23de743f07c54..c4df37dbd7733 100644 --- a/test/native/native-image-filter/extra-filter.json +++ b/test/native/native-image-filter/extra-filter.json @@ -4,6 +4,8 @@ {"excludeClasses": "android.app.**"}, {"excludeClasses": "com.**"}, + {"includeClasses": "com.oracle.svm.core.**"}, + {"includeClasses": "com.sun.management.**"}, {"excludeClasses": "ch.qos.logback.**"}, {"excludeClasses": "groovy.**"}, {"excludeClasses": "io.**"}, diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/testcontainers/ClickHouseProvider.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/testcontainers/ClickHouseProvider.java deleted file mode 100644 index b65fe9ae6b665..0000000000000 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/testcontainers/ClickHouseProvider.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.shardingsphere.test.natived.commons.testcontainers; - -import org.testcontainers.clickhouse.ClickHouseContainer; -import org.testcontainers.containers.JdbcDatabaseContainer; -import org.testcontainers.containers.JdbcDatabaseContainerProvider; -import org.testcontainers.utility.DockerImageName; - -/** - * TODO Initialize a new JdbcDatabaseContainerProvider implementation due to - * testcontainers/testcontainers-java#8736 not closed. - */ -public final class ClickHouseProvider extends JdbcDatabaseContainerProvider { - - /** - * The reason for using `0` as the separator is that the URL_MATCHING_PATTERN of the {@link org.testcontainers.jdbc.ConnectionUrl.Patterns} - * sets {@code "(?[a-z0-9]+)"} for the `databaseType` part. - * - * @param databaseType {@link String} - * @return true when provider can handle this database type, else false. - */ - @Override - public boolean supports(final String databaseType) { - return "shardingsphere0clickhouse".equals(databaseType); - } - - @Override - public JdbcDatabaseContainer newInstance(final String tag) { - return new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server").withTag(tag)); - } -} diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java index 70100f992b13e..7bbbfd506052c 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java @@ -20,30 +20,69 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import org.apache.shardingsphere.test.natived.commons.TestShardingService; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledInNativeImage; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.Properties; +import java.util.stream.Stream; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +/** + * Cannot use testcontainers-java style jdbcURL for Clickhouse Server due to unresolved + * testcontainers/testcontainers-java#8736. + */ +@SuppressWarnings("SqlNoDataSourceInspection") @EnabledInNativeImage +@Testcontainers class ClickHouseTest { + @Container + public static final ClickHouseContainer CONTAINER = new ClickHouseContainer("clickhouse/clickhouse-server:24.6.2.17"); + + private static final String SYSTEM_PROP_KEY_PREFIX = "fixture.test-native.yaml.database.clickhouse."; + + private String jdbcUrlPrefix; + private TestShardingService testShardingService; + @BeforeAll + static void beforeAll() { + assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url"), is(nullValue())); + assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url"), is(nullValue())); + assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url"), is(nullValue())); + } + + @AfterAll + static void afterAll() { + System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url"); + System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url"); + System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url"); + } + /** * TODO Need to fix `shardingsphere-parser-sql-clickhouse` module to use {@link TestShardingService#cleanEnvironment()} * after {@link TestShardingService#processSuccessInClickHouse()}. - * */ @Test - void assertShardingInLocalTransactions() { - HikariConfig config = new HikariConfig(); - config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); - config.setJdbcUrl("jdbc:shardingsphere:classpath:test-native/yaml/jdbc/databases/clickhouse.yaml"); - DataSource dataSource = new HikariDataSource(config); + void assertShardingInLocalTransactions() throws SQLException { + jdbcUrlPrefix = "jdbc:ch://localhost:" + CONTAINER.getMappedPort(8123) + "/"; + DataSource dataSource = createDataSource(); testShardingService = new TestShardingService(dataSource); assertDoesNotThrow(() -> testShardingService.processSuccessInClickHouse()); } @@ -63,4 +102,71 @@ private void initEnvironment() throws SQLException { testShardingService.getOrderItemRepository().truncateTable(); testShardingService.getAddressRepository().truncateTable(); } + + private Connection openConnection(final String databaseName) throws SQLException { + Properties props = new Properties(); + props.setProperty("user", CONTAINER.getUsername()); + props.setProperty("password", CONTAINER.getPassword()); + return DriverManager.getConnection(jdbcUrlPrefix + databaseName, props); + } + + private DataSource createDataSource() throws SQLException { + Awaitility.await().atMost(Duration.ofMinutes(1L)).ignoreExceptions().until(() -> { + openConnection(CONTAINER.getDatabaseName()).close(); + return true; + }); + try ( + Connection connection = openConnection(CONTAINER.getDatabaseName()); + Statement statement = connection.createStatement()) { + statement.executeUpdate("CREATE DATABASE demo_ds_0"); + statement.executeUpdate("CREATE DATABASE demo_ds_1"); + statement.executeUpdate("CREATE DATABASE demo_ds_2"); + } + Stream.of("demo_ds_0", "demo_ds_1", "demo_ds_2").parallel().forEach(this::initTable); + HikariConfig config = new HikariConfig(); + config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); + config.setJdbcUrl("jdbc:shardingsphere:classpath:test-native/yaml/jdbc/databases/clickhouse.yaml?placeholder-type=system_props"); + System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url", jdbcUrlPrefix + "demo_ds_0"); + System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url", jdbcUrlPrefix + "demo_ds_1"); + System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url", jdbcUrlPrefix + "demo_ds_2"); + return new HikariDataSource(config); + } + + private void initTable(final String databaseName) { + try ( + Connection connection = openConnection(databaseName); + Statement statement = connection.createStatement()) { + statement.executeUpdate("create table IF NOT EXISTS t_order\n" + + "(\n" + + " order_id Int64 NOT NULL DEFAULT rand(),\n" + + " order_type Int32,\n" + + " user_id Int32 NOT NULL,\n" + + " address_id Int64 NOT NULL,\n" + + " status String\n" + + ") engine = MergeTree\n" + + " primary key (order_id)\n" + + " order by (order_id)"); + statement.executeUpdate("create table IF NOT EXISTS t_order_item\n" + + "(\n" + + " order_item_id Int64 NOT NULL DEFAULT rand(),\n" + + " order_id Int64 NOT NULL,\n" + + " user_id Int32 NOT NULL,\n" + + " phone String,\n" + + " status String\n" + + ") engine = MergeTree\n" + + " primary key (order_item_id)\n" + + " order by (order_item_id)"); + statement.executeUpdate("CREATE TABLE IF NOT EXISTS t_address\n" + + "(\n" + + " address_id BIGINT NOT NULL,\n" + + " address_name VARCHAR(100) NOT NULL,\n" + + " PRIMARY KEY (address_id)\n" + + ")"); + statement.executeUpdate("TRUNCATE TABLE t_order"); + statement.executeUpdate("TRUNCATE TABLE t_order_item"); + statement.executeUpdate("TRUNCATE TABLE t_address"); + } catch (final SQLException exception) { + throw new RuntimeException(exception); + } + } } diff --git a/test/native/src/test/resources/META-INF/services/org.testcontainers.containers.JdbcDatabaseContainerProvider b/test/native/src/test/resources/META-INF/services/org.testcontainers.containers.JdbcDatabaseContainerProvider deleted file mode 100644 index ff304ca3b3e4a..0000000000000 --- a/test/native/src/test/resources/META-INF/services/org.testcontainers.containers.JdbcDatabaseContainerProvider +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -org.apache.shardingsphere.test.natived.commons.testcontainers.ClickHouseProvider diff --git a/test/native/src/test/resources/test-native/sql/test-native-databases-clickhouse.sql b/test/native/src/test/resources/test-native/sql/test-native-databases-clickhouse.sql deleted file mode 100644 index 09c5647a73983..0000000000000 --- a/test/native/src/test/resources/test-native/sql/test-native-databases-clickhouse.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -create table IF NOT EXISTS t_order -( - order_id Int64 NOT NULL DEFAULT rand(), - order_type Int32, - user_id Int32 NOT NULL, - address_id Int64 NOT NULL, - status String -) engine = MergeTree - primary key (order_id) - order by (order_id); - -create table IF NOT EXISTS t_order_item -( - order_item_id Int64 NOT NULL DEFAULT rand(), - order_id Int64 NOT NULL, - user_id Int32 NOT NULL, - phone String, - status String -) engine = MergeTree - primary key (order_item_id) - order by (order_item_id); - -CREATE TABLE IF NOT EXISTS t_address -( - address_id BIGINT NOT NULL, - address_name VARCHAR(100) NOT NULL, - PRIMARY KEY (address_id) -); - -TRUNCATE TABLE t_order; -TRUNCATE TABLE t_order_item; -TRUNCATE TABLE t_address; diff --git a/test/native/src/test/resources/test-native/yaml/jdbc/databases/clickhouse.yaml b/test/native/src/test/resources/test-native/yaml/jdbc/databases/clickhouse.yaml index bd2aa6413513e..5d62805858aef 100644 --- a/test/native/src/test/resources/test-native/yaml/jdbc/databases/clickhouse.yaml +++ b/test/native/src/test/resources/test-native/yaml/jdbc/databases/clickhouse.yaml @@ -20,20 +20,25 @@ mode: repository: type: JDBC -# TODO See the JavaDoc for `org.apache.shardingsphere.test.natived.jdbc.commons.testcontainers.ClickHouseProvider`. dataSources: ds_0: dataSourceClassName: com.zaxxer.hikari.HikariDataSource - driverClassName: org.testcontainers.jdbc.ContainerDatabaseDriver - jdbcUrl: jdbc:tc:shardingsphere0clickhouse:24.6.2.17://test-native-databases-clickhouse/demo_ds_0?TC_INITSCRIPT=test-native/sql/test-native-databases-clickhouse.sql + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: $${fixture.test-native.yaml.database.clickhouse.ds0.jdbc-url::} + username: default + password: ds_1: dataSourceClassName: com.zaxxer.hikari.HikariDataSource - driverClassName: org.testcontainers.jdbc.ContainerDatabaseDriver - jdbcUrl: jdbc:tc:shardingsphere0clickhouse:24.6.2.17://test-native-databases-clickhouse/demo_ds_1?TC_INITSCRIPT=test-native/sql/test-native-databases-clickhouse.sql + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: $${fixture.test-native.yaml.database.clickhouse.ds1.jdbc-url::} + username: default + password: ds_2: dataSourceClassName: com.zaxxer.hikari.HikariDataSource - driverClassName: org.testcontainers.jdbc.ContainerDatabaseDriver - jdbcUrl: jdbc:tc:shardingsphere0clickhouse:24.6.2.17://test-native-databases-clickhouse/demo_ds_2?TC_INITSCRIPT=test-native/sql/test-native-databases-clickhouse.sql + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: $${fixture.test-native.yaml.database.clickhouse.ds2.jdbc-url::} + username: default + password: rules: - !SHARDING