From c50437b0833cb2ef1e4a5b4ffc4b95fca6253d85 Mon Sep 17 00:00:00 2001 From: linghengqian Date: Mon, 25 Nov 2024 20:08:44 +0800 Subject: [PATCH] Replaces Zookeeper with ClickHouse Keeper --- RELEASE-NOTES.md | 1 + .../optional-plugins/clickhouse/_index.cn.md | 175 ++++++++++++++++- .../optional-plugins/clickhouse/_index.en.md | 178 +++++++++++++++++- .../optional-plugins/hiveserver2/_index.cn.md | 3 +- .../optional-plugins/hiveserver2/_index.en.md | 5 +- .../reflect-config.json | 15 +- .../resource-config.json | 2 +- .../reflect-config.json | 15 -- .../natived/commons/TestShardingService.java | 4 +- .../commons/repository/AddressRepository.java | 19 -- .../repository/OrderItemRepository.java | 71 ++----- .../commons/repository/OrderRepository.java | 46 ----- .../jdbc/databases/ClickHouseTest.java | 50 +++-- .../jdbc/databases/hive/HiveSimpleTest.java | 27 +-- .../HiveZookeeperServiceDiscoveryTest.java | 24 +-- .../natived/proxy/databases/PostgresTest.java | 2 +- .../natived/proxy/features/ShardingTest.java | 2 +- .../resource-config.json | 3 + .../sql/test-native-databases-hive.sql | 3 +- .../test-native/xml/keeper_config.xml | 31 +++ ...ouse-transactions.xml => transactions.xml} | 4 +- .../yaml/jdbc/features/sharding.yaml | 3 + 22 files changed, 448 insertions(+), 235 deletions(-) create mode 100644 test/native/src/test/resources/test-native/xml/keeper_config.xml rename test/native/src/test/resources/test-native/xml/{clickhouse-transactions.xml => transactions.xml} (92%) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 01833a33ee55ee..4cd92365e0220a 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -27,6 +27,7 @@ 1. Infra: Support connecting to HiveServer2 through database connection pools other than HikariCP - [#33762](https://github.com/apache/shardingsphere/pull/33762) 1. Infra: Partial support for connecting to embedded ClickHouse `chDB` - [#33786](https://github.com/apache/shardingsphere/pull/33786) 1. Proxy Native: Support connecting to HiveServer2 with ZooKeeper Service Discovery enabled in GraalVM Native Image - [#33768](https://github.com/apache/shardingsphere/pull/33768) +1. Proxy Native: Support local transactions of ClickHouse under GraalVM Native Image - [#33801](https://github.com/apache/shardingsphere/pull/33801) 1. Doc: Adds documentation for ClickHouse support - [#33779](https://github.com/apache/shardingsphere/pull/33779) ### Bug Fixes diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.cn.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.cn.md index 039eccab1792b8..3ee296c14933b9 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.cn.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.cn.md @@ -80,7 +80,7 @@ create table IF NOT EXISTS t_order ( order_type Int32, user_id Int32 NOT NULL, address_id Int64 NOT NULL, - status String + status VARCHAR(50) ) engine = MergeTree primary key (order_id) order by (order_id); @@ -158,7 +158,7 @@ public class ExampleUtils { ### SQL 限制 -ShardingSphere JDBC DataSource 尚不支持执行 ClickHouse 的 `CREATE TABLE` 语句和 `TRUNCATE TABLE` 语句。 +ShardingSphere JDBC DataSource 尚不支持执行 ClickHouse 的 `create table`，`truncate table` 和 `drop table` 语句。 用户应考虑为 ShardingSphere 提交包含单元测试的 PR。 ### 分布式序列限制 @@ -207,9 +207,178 @@ public class ExampleTest { ### 事务限制 -ClickHouse 不支持 ShardingSphere 集成级别的本地事务，XA 事务和 Seata AT 模式事务， +ClickHouse 不支持 ShardingSphere 集成级别的 XA 事务， +因为 https://github.com/ClickHouse/clickhouse-java 未实现 `javax.sql.XADataSource` 的相关 Java 接口。 + +ClickHouse 不支持 ShardingSphere 集成级别的 Seata AT 模式事务， +因为 https://github.com/apache/incubator-seata 未实现 ClickHouse 的 SQL 方言解析。 + +ClickHouse 支持 ShardingSphere 集成级别的本地事务，但需要对 ClickHouse 进行额外配置， 更多讨论位于 https://github.com/ClickHouse/clickhouse-docs/issues/2300 。 +引入讨论，编写 Docker Compose 文件来启动 ClickHouse 和 ClickHouse Keeper。 + +```yaml +services: + clickhouse-keeper-01: + image: clickhouse/clickhouse-keeper:24.10.2.80 + volumes: + - ./keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml + clickhouse-server: + image: clickhouse/clickhouse-server:24.10.2.80 + depends_on: + - clickhouse-keeper-01 + ports: + - "8123:8123" + volumes: + - ./transactions.xml:/etc/clickhouse-server/config.d/transactions.xml +``` + +`./keeper_config.xml` 的内容如下， + +```xml + + 0.0.0.0 + + 9181 + 1 + /var/lib/clickhouse/coordination/snapshots + + + 1 + clickhouse-keeper-01 + 9234 + + + + +``` + +`./transactions.xml` 的内容如下， + +```xml + + 1 + + + clickhouse-keeper-01 + 9181 + + + +``` + +在 DBeaver Community 内，使用 `jdbc:ch://localhost:8123/default` 的 `jdbcUrl`，`default` 的`username` 连接至 ClickHouse， +`password` 留空。 +执行如下 SQL， + +```sql +-- noinspection SqlNoDataSourceInspectionForFile +CREATE DATABASE demo_ds_0; +CREATE DATABASE demo_ds_1; +CREATE DATABASE demo_ds_2; +``` + +分别使用 `jdbc:ch://localhost:8123/demo_ds_0` ， +`jdbc:ch://localhost:8123/demo_ds_1` 和 `jdbc:ch://localhost:8123/demo_ds_2` 的 `jdbcUrl` 连接至 ClickHouse 来执行如下 SQL， + +```sql +-- noinspection SqlNoDataSourceInspectionForFile +create table IF NOT EXISTS t_order ( + order_id Int64 NOT NULL, + order_type Int32, + user_id Int32 NOT NULL, + address_id Int64 NOT NULL, + status VARCHAR(50) +) engine = MergeTree + primary key (order_id) + order by (order_id); + +TRUNCATE TABLE t_order; +``` + +在业务项目引入`前提条件`涉及的依赖后，在业务项目的 classpath 上编写 ShardingSphere 数据源的配置文件`demo.yaml`， + +```yaml +dataSources: + ds_0: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_0?transactionSupport=true + ds_1: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_1?transactionSupport=true + ds_2: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_2?transactionSupport=true +rules: +- !SHARDING + tables: + t_order: + actualDataNodes: + keyGenerateStrategy: + column: order_id + keyGeneratorName: snowflake + defaultDatabaseStrategy: + standard: + shardingColumn: user_id + shardingAlgorithmName: inline + shardingAlgorithms: + inline: + type: INLINE + props: + algorithm-expression: ds_${user_id % 2} + keyGenerators: + snowflake: + type: SNOWFLAKE +``` + +创建 ShardingSphere 的数据源后可正常使用本地事务， + +```java +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.sql.Connection; +import java.sql.SQLException; +@SuppressWarnings({"SqlNoDataSourceInspection", "AssertWithSideEffects"}) +public class ExampleUtils { + void test() throws SQLException { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:shardingsphere:classpath:demo.yaml"); + config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); + try (HikariDataSource dataSource = new HikariDataSource(config); Connection connection = dataSource.getConnection()) { + try { + connection.setAutoCommit(false); + connection.createStatement().executeUpdate("INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (2024, 0, 2024, 'INSERT_TEST')"); + connection.createStatement().executeUpdate("INSERT INTO t_order_does_not_exist (test_id_does_not_exist) VALUES (2024)"); + connection.commit(); + } catch (final SQLException ignored) { + connection.rollback(); + } finally { + connection.setAutoCommit(true); + } + try (Connection conn = dataSource.getConnection()) { + assert !conn.createStatement().executeQuery("SELECT * FROM t_order WHERE user_id = 2024").next(); + } + } + } +} +``` + +一旦在 ShardingSphere 的配置文件为 ClickHouse JDBC Driver 的 jdbcUrl 设置 `transactionSupport=true`， +**ClickHouse 的 DML SQL 执行速度将大幅下降**。此时，用户在执行 `insert` 和 `alter table` 等 DML 语句的前后可能需要进行等待， +以避免如下 Error 的发生， + +```shell +java.sql.BatchUpdateException: Code: 341. DB::Exception: Exception happened during execution of mutation 'mutation_6.txt' with part 'all_1_1_0' reason: 'Serialization error: part all_1_1_0 is locked by transaction 5672402456378293316'. This error maybe retryable or not. In case of unretryable error, mutation can be killed with KILL MUTATION query. (UNFINISHED) (version 24.10.2.80 (official build)) + at com.clickhouse.jdbc.SqlExceptionUtils.batchUpdateError(SqlExceptionUtils.java:107) + at com.clickhouse.jdbc.internal.SqlBasedPreparedStatement.executeAny(SqlBasedPreparedStatement.java:223) + at com.clickhouse.jdbc.internal.SqlBasedPreparedStatement.executeLargeUpdate(SqlBasedPreparedStatement.java:302) + at com.clickhouse.jdbc.internal.AbstractPreparedStatement.executeUpdate(AbstractPreparedStatement.java:135) +``` + ### 嵌入式 ClickHouse 限制 嵌入式 ClickHouse `chDB` 尚未发布 Java 客户端， diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.en.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.en.md index 43205c1ebdc768..2bcad649a16bb7 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.en.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/clickhouse/_index.en.md @@ -81,7 +81,7 @@ create table IF NOT EXISTS t_order ( order_type Int32, user_id Int32 NOT NULL, address_id Int64 NOT NULL, - status String + status VARCHAR(50) ) engine = MergeTree primary key (order_id) order by (order_id); @@ -160,7 +160,8 @@ public class ExampleUtils { ### SQL Limitations -ShardingSphere JDBC DataSource does not yet support the execution of ClickHouse's `CREATE TABLE` statement and `TRUNCATE TABLE` statement. +ShardingSphere JDBC DataSource does not yet support executing ClickHouse's `create table`, `truncate table`, +and `drop table` statements. Users should consider submitting a PR containing unit tests for ShardingSphere. ### Key Generate restrictions @@ -212,8 +213,177 @@ public class ExampleTest { ### Transaction Limitations -ClickHouse does not support ShardingSphere integration-level local transactions, XA transactions, and Seata AT mode transactions. -More discussions are at https://github.com/ClickHouse/clickhouse-docs/issues/2300 . +ClickHouse does not support XA transactions at the ShardingSphere integration level, +because https://github.com/ClickHouse/clickhouse-java does not implement the relevant Java interface of `javax.sql.XADataSource`. + +ClickHouse does not support Seata AT mode transactions at the ShardingSphere integration level, +because https://github.com/apache/incubator-seata does not implement ClickHouse's SQL dialect parsing. + +ClickHouse supports local transactions at the ShardingSphere integration level, but additional configuration of ClickHouse is required, +For more discussion, please visit https://github.com/ClickHouse/clickhouse-docs/issues/2300 . + +Introduce the discussion of writing a Docker Compose file to start ClickHouse and ClickHouse Keeper. + +```yaml +services: + clickhouse-keeper-01: + image: clickhouse/clickhouse-keeper:24.10.2.80 + volumes: + - ./keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml + clickhouse-server: + image: clickhouse/clickhouse-server:24.10.2.80 + depends_on: + - clickhouse-keeper-01 + ports: + - "8123:8123" + volumes: + - ./transactions.xml:/etc/clickhouse-server/config.d/transactions.xml +``` + +The content of `./keeper_config.xml` is as follows, + +```xml + + 0.0.0.0 + + 9181 + 1 + /var/lib/clickhouse/coordination/snapshots + + + 1 + clickhouse-keeper-01 + 9234 + + + + +``` + +The content of `./transactions.xml` is as follows, + +```xml + + 1 + + + clickhouse-keeper-01 + 9181 + + + +``` + +In DBeaver Community, use `jdbcUrl` of `jdbc:ch://localhost:8123/default`, `username` of `default` to connect to ClickHouse, +and leave `password` blank. Execute the following SQL, + +```sql +-- noinspection SqlNoDataSourceInspectionForFile +CREATE DATABASE demo_ds_0; +CREATE DATABASE demo_ds_1; +CREATE DATABASE demo_ds_2; +``` + +Use `jdbcUrl` of `jdbc:ch://localhost:8123/demo_ds_0`, `jdbc:ch://localhost:8123/demo_ds_1` +and `jdbc:ch://localhost:8123/demo_ds_2` to connect to ClickHouse and execute the following SQL. + +```sql +-- noinspection SqlNoDataSourceInspectionForFile +create table IF NOT EXISTS t_order ( + order_id Int64 NOT NULL, + order_type Int32, + user_id Int32 NOT NULL, + address_id Int64 NOT NULL, + status VARCHAR(50) +) engine = MergeTree + primary key (order_id) + order by (order_id); + +TRUNCATE TABLE t_order; +``` + +After the business project introduces the dependencies involved in the `prerequisites`, +write the ShardingSphere data source configuration file `demo.yaml` on the classpath of the business project. + +```yaml +dataSources: + ds_0: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_0?transactionSupport=true + ds_1: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_1?transactionSupport=true + ds_2: + dataSourceClassName: com.zaxxer.hikari.HikariDataSource + driverClassName: com.clickhouse.jdbc.ClickHouseDriver + jdbcUrl: jdbc:ch://localhost:8123/demo_ds_2?transactionSupport=true +rules: +- !SHARDING + tables: + t_order: + actualDataNodes: + keyGenerateStrategy: + column: order_id + keyGeneratorName: snowflake + defaultDatabaseStrategy: + standard: + shardingColumn: user_id + shardingAlgorithmName: inline + shardingAlgorithms: + inline: + type: INLINE + props: + algorithm-expression: ds_${user_id % 2} + keyGenerators: + snowflake: + type: SNOWFLAKE +``` + +After creating the ShardingSphere data source, local transactions can be used normally. + +```java +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.sql.Connection; +import java.sql.SQLException; +@SuppressWarnings({"SqlNoDataSourceInspection", "AssertWithSideEffects"}) +public class ExampleUtils { + void test() throws SQLException { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:shardingsphere:classpath:demo.yaml"); + config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver"); + try (HikariDataSource dataSource = new HikariDataSource(config); Connection connection = dataSource.getConnection()) { + try { + connection.setAutoCommit(false); + connection.createStatement().executeUpdate("INSERT INTO t_order (user_id, order_type, address_id, status) VALUES (2024, 0, 2024, 'INSERT_TEST')"); + connection.createStatement().executeUpdate("INSERT INTO t_order_does_not_exist (test_id_does_not_exist) VALUES (2024)"); + connection.commit(); + } catch (final SQLException ignored) { + connection.rollback(); + } finally { + connection.setAutoCommit(true); + } + try (Connection conn = dataSource.getConnection()) { + assert !conn.createStatement().executeQuery("SELECT * FROM t_order WHERE user_id = 2024").next(); + } + } + } +} +``` + +Once `transactionSupport=true` is set for the jdbcUrl of ClickHouse JDBC Driver in the ShardingSphere configuration file, +**ClickHouse's DML SQL execution speed will drop significantly**. At this time, +users may need to wait before and after executing DML statements such as `insert` and `alter table` to avoid the following Error: + +```shell +java.sql.BatchUpdateException: Code: 341. DB::Exception: Exception happened during execution of mutation 'mutation_6.txt' with part 'all_1_1_0' reason: 'Serialization error: part all_1_1_0 is locked by transaction 5672402456378293316'. This error maybe retryable or not. In case of unretryable error, mutation can be killed with KILL MUTATION query. (UNFINISHED) (version 24.10.2.80 (official build)) + at com.clickhouse.jdbc.SqlExceptionUtils.batchUpdateError(SqlExceptionUtils.java:107) + at com.clickhouse.jdbc.internal.SqlBasedPreparedStatement.executeAny(SqlBasedPreparedStatement.java:223) + at com.clickhouse.jdbc.internal.SqlBasedPreparedStatement.executeLargeUpdate(SqlBasedPreparedStatement.java:302) + at com.clickhouse.jdbc.internal.AbstractPreparedStatement.executeUpdate(AbstractPreparedStatement.java:135) +``` ### Embedded ClickHouse Limitations diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md index bb11ed7def68a2..1facfebdf4c13e 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.cn.md @@ -438,8 +438,7 @@ HiveServer2 JDBC Driver `4.0.1` 不支持 Hadoop `3.4.1`， ### SQL 限制 -ShardingSphere JDBC DataSource 尚不支持执行 HiveServer2 的 `SET` 语句，`CREATE TABLE` 语句和 `TRUNCATE TABLE` 语句。 - +ShardingSphere JDBC DataSource 尚不支持执行 HiveServer2 的 `set`，`create table`，`truncate table` 和 `drop table` 语句。 用户应考虑为 ShardingSphere 提交包含单元测试的 PR。 ### 在 ShardingSphere 数据源上使用 DML SQL 语句的前提条件 diff --git a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md index 38c6781a568e28..a5791e0d3efbfa 100644 --- a/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md +++ b/docs/document/content/user-manual/shardingsphere-jdbc/optional-plugins/hiveserver2/_index.en.md @@ -444,9 +444,8 @@ Reference https://github.com/apache/hive/pull/5500. ### SQL Limitations -ShardingSphere JDBC DataSource does not yet support executing HiveServer2's `SET` statement, -`CREATE TABLE` statement, and `TRUNCATE TABLE` statement. - +ShardingSphere JDBC DataSource does not yet support executing HiveServer2's `set`, `create table`, `truncate table`, +and `drop table` statements. Users should consider submitting a PR containing unit tests for ShardingSphere. ### Prerequisites for using DML SQL statements on ShardingSphere data sources diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json index fe75b38e7ab6df..66fe43711c2685 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/reflect-config.json @@ -16,7 +16,7 @@ "name":"[Lcom.github.dockerjava.api.model.VolumesFrom;" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f4b07ce0c30"}, + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007fb723cac500"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, { @@ -75,6 +75,10 @@ "condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.JDBCRepository"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.jdbc.datasource.JDBCBackendDataSource"}, + "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" +}, { "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.postgresql.command.query.extended.Portal"}, "name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;" @@ -696,6 +700,11 @@ "queryAllPublicConstructors":true, "methods":[{"name":"","parameterTypes":[] }, {"name":"add","parameterTypes":["long"] }, {"name":"sum","parameterTypes":[] }] }, +{ + "condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.command.CommandExecutorTask"}, + "name":"java.util.concurrent.atomic.Striped64$Cell", + "fields":[{"name":"value"}] +}, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.expr.groovy.GroovyInlineExpressionParser"}, "name":"java.util.function.DoubleFunction", @@ -1067,7 +1076,7 @@ "name":"org.apache.shardingsphere.driver.yaml.YamlJDBCConfiguration", "allDeclaredFields":true, "queryAllPublicMethods":true, - "methods":[{"name":"","parameterTypes":[] }, {"name":"setDataSources","parameterTypes":["java.util.Map"] }, {"name":"setMode","parameterTypes":["org.apache.shardingsphere.infra.yaml.config.pojo.mode.YamlModeConfiguration"] }, {"name":"setRules","parameterTypes":["java.util.Collection"] }, {"name":"setSqlParser","parameterTypes":["org.apache.shardingsphere.parser.yaml.config.YamlSQLParserRuleConfiguration"] }, {"name":"setTransaction","parameterTypes":["org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfiguration"] }] + "methods":[{"name":"","parameterTypes":[] }, {"name":"setDataSources","parameterTypes":["java.util.Map"] }, {"name":"setMode","parameterTypes":["org.apache.shardingsphere.infra.yaml.config.pojo.mode.YamlModeConfiguration"] }, {"name":"setProps","parameterTypes":["java.util.Properties"] }, {"name":"setRules","parameterTypes":["java.util.Collection"] }, {"name":"setSqlParser","parameterTypes":["org.apache.shardingsphere.parser.yaml.config.YamlSQLParserRuleConfiguration"] }, {"name":"setTransaction","parameterTypes":["org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfiguration"] }] }, { "condition":{"typeReachable":"org.apache.shardingsphere.infra.util.yaml.YamlEngine"}, @@ -2074,7 +2083,7 @@ "queryAllDeclaredMethods":true }, { - "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f4b07b277d8"}, + "condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007fb723b26e30"}, "name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.MetaDataChangedSubscriber" }, { diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json index 294fcfe11ad1e1..7bd0965927b2a6 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/generated-reachability-metadata/resource-config.json @@ -79,7 +79,7 @@ "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager"}, "pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E" }, { - "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f4b07b34b38"}, + "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007fb723b369d0"}, "pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E" }, { "condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"}, diff --git a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json index 46d056e7c417fa..4597bf4a5967cb 100644 --- a/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json +++ b/infra/reachability-metadata/src/main/resources/META-INF/native-image/org.apache.shardingsphere/shardingsphere-infra-reachability-metadata/reflect-config.json @@ -304,21 +304,6 @@ "name":"org.apache.shardingsphere.sql.parser.postgresql.visitor.statement.type.PostgreSQLTCLStatementVisitor", "methods":[{"name":"","parameterTypes":[] }] }, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.clickhouse.parser.ClickHouseLexer"}, - "name":"org.apache.shardingsphere.sql.parser.clickhouse.parser.ClickHouseLexer", - "methods":[{"name":"","parameterTypes":["org.antlr.v4.runtime.CharStream"] }] -}, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.clickhouse.parser.ClickHouseParser"}, - "name":"org.apache.shardingsphere.sql.parser.clickhouse.parser.ClickHouseParser", - "methods":[{"name":"","parameterTypes":["org.antlr.v4.runtime.TokenStream"] }] -}, -{ - "condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.clickhouse.visitor.statement.type.ClickHouseDMLStatementVisitor"}, - "name":"org.apache.shardingsphere.sql.parser.clickhouse.visitor.statement.type.ClickHouseDMLStatementVisitor", - "methods":[{"name":"","parameterTypes":[] }] -}, { "condition":{"typeReachable":"org.apache.shardingsphere.sharding.yaml.config.strategy.sharding.YamlBaseShardingStrategyConfiguration"}, "name":"org.apache.shardingsphere.sharding.yaml.config.strategy.sharding.YamlBaseShardingStrategyConfiguration", diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java index b2f6ac9259099c..37a78afc0370ba 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/TestShardingService.java @@ -176,14 +176,14 @@ public void deleteData(final Collection orderIds) throws SQLException { /** * Delete data in ClickHouse. - * TODO It is necessary to avoid the use of {@code Awaitility.await().pollDelay(Duration.ofSeconds(9L)).until(()->true)}. + * TODO It is necessary to avoid the use of {@code Awaitility.await().pollDelay(Duration.ofSeconds(5L)).until(()->true)}. * After ClickHouse enables experimental transactions, performance drops significantly. * * @param orderIds orderId of the insert statement. * @throws SQLException An exception that provides information on a database access error or other errors. */ public void deleteDataInClickHouse(final Collection orderIds) throws SQLException { - Awaitility.await().pollDelay(Duration.ofSeconds(9L)).until(() -> true); + Awaitility.await().pollDelay(Duration.ofSeconds(5L)).until(() -> true); long count = 1L; for (Long each : orderIds) { orderRepository.deleteInClickHouse(each); diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/AddressRepository.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/AddressRepository.java index 2797433e0a61f9..1774289c577ce7 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/AddressRepository.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/AddressRepository.java @@ -70,25 +70,6 @@ public void createTableInSQLServer() throws SQLException { } } - /** - * create table t_address if not exists in Hive. - * - * @throws SQLException SQL exception - */ - public void createTableIfNotExistsInHive() throws SQLException { - String sql = "CREATE TABLE IF NOT EXISTS t_address\n" - + "(\n" - + " address_id BIGINT NOT NULL,\n" - + " address_name VARCHAR(100) NOT NULL,\n" - + " PRIMARY KEY (address_id) disable novalidate\n" - + ") STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2')"; - try ( - Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement()) { - statement.executeUpdate(sql); - } - } - /** * drop table t_address. * diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderItemRepository.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderItemRepository.java index 180bc6ef0e7f03..bcd26777467dd8 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderItemRepository.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderItemRepository.java @@ -102,52 +102,6 @@ public void createTableInSQLServer() throws SQLException { } } - /** - * create table if not exists in ClickHouse. - * ClickHouse does not support `AUTO_INCREMENT`, refer to ClickHouse/ClickHouse#56228 . - * - * @throws SQLException SQL exception - */ - public void createTableIfNotExistsInClickHouse() throws SQLException { - String sql = "create table IF NOT EXISTS t_order_item( \n" - + "order_item_id Int64 NOT NULL, \n" - + "order_id Int64 NOT NULL, \n" - + "user_id Int32 NOT NULL, \n" - + "phone String,\n" - + "status String\n" - + ") engine = MergeTree \n" - + "primary key (order_item_id)\n" - + "order by(order_item_id); "; - try ( - Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement()) { - statement.executeUpdate(sql); - } - } - - /** - * create table if not exists in Hive. - * Hive does not support `AUTO_INCREMENT`, refer to HIVE-6905 . - * - * @throws SQLException SQL exception - */ - public void createTableIfNotExistsInHive() throws SQLException { - String sql = "CREATE TABLE IF NOT EXISTS t_order_item\n" - + "(\n" - + " order_item_id BIGINT,\n" - + " order_id BIGINT NOT NULL,\n" - + " user_id INT NOT NULL,\n" - + " phone VARCHAR(50),\n" - + " status VARCHAR(50),\n" - + " PRIMARY KEY (order_item_id) disable novalidate\n" - + ") STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2')"; - try ( - Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement()) { - statement.executeUpdate(sql); - } - } - /** * drop table. * @@ -323,22 +277,23 @@ public List selectAll() throws SQLException { /** * Assert rollback with transactions. - * This is currently just a simple test against a non-existent table and does not involve the competition scenario of distributed transactions. + * This is currently just a simple test against a non-existent table {@code t_order_item_does_not_exist} + * and does not involve the competition scenario of distributed transactions. * * @throws SQLException An exception that provides information on a database access error or other errors. */ public void assertRollbackWithTransactions() throws SQLException { - Connection connection = dataSource.getConnection(); - try { - connection.setAutoCommit(false); - connection.createStatement().executeUpdate("INSERT INTO t_order_item (order_id, user_id, phone, status) VALUES (2024, 2024, '13800000001', 'INSERT_TEST')"); - connection.createStatement().executeUpdate("INSERT INTO t_order_item_does_not_exist (test_id_does_not_exist) VALUES (2024)"); - connection.commit(); - } catch (final SQLException ignored) { - connection.rollback(); - } finally { - connection.setAutoCommit(true); - connection.close(); + try (Connection connection = dataSource.getConnection()) { + try { + connection.setAutoCommit(false); + connection.createStatement().executeUpdate("INSERT INTO t_order_item (order_id, user_id, phone, status) VALUES (2024, 2024, '13800000001', 'INSERT_TEST')"); + connection.createStatement().executeUpdate("INSERT INTO t_order_item_does_not_exist (test_id_does_not_exist) VALUES (2024)"); + connection.commit(); + } catch (final SQLException ignored) { + connection.rollback(); + } finally { + connection.setAutoCommit(true); + } } try ( Connection conn = dataSource.getConnection(); diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderRepository.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderRepository.java index 73e2679472322f..8e9683b5290449 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderRepository.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/commons/repository/OrderRepository.java @@ -99,52 +99,6 @@ public void createTableInSQLServer() throws SQLException { } } - /** - * create table in ClickHouse. - * ClickHouse does not support `AUTO_INCREMENT`, refer to ClickHouse/ClickHouse#56228 . - * - * @throws SQLException SQL exception - */ - public void createTableIfNotExistsInClickHouse() throws SQLException { - String sql = "create table IF NOT EXISTS t_order( \n" - + "order_id Int64 NOT NULL, \n" - + "order_type Int32, \n" - + "user_id Int32 NOT NULL, \n" - + "address_id Int64 NOT NULL,\n" - + "status String\n" - + ") engine = MergeTree \n" - + "primary key (order_id)\n" - + "order by(order_id)"; - try ( - Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement()) { - statement.executeUpdate(sql); - } - } - - /** - * create table in Hive. - * Hive does not support `AUTO_INCREMENT`, refer to HIVE-6905 . - * - * @throws SQLException SQL exception - */ - public void createTableIfNotExistsInHive() throws SQLException { - String sql = "CREATE TABLE IF NOT EXISTS t_order\n" - + "(\n" - + " order_id BIGINT,\n" - + " order_type INT,\n" - + " user_id INT NOT NULL,\n" - + " address_id BIGINT NOT NULL,\n" - + " status VARCHAR(50),\n" - + " PRIMARY KEY (order_id) disable novalidate\n" - + ") STORED BY ICEBERG STORED AS ORC TBLPROPERTIES ('format-version' = '2')"; - try ( - Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement()) { - statement.executeUpdate(sql); - } - } - /** * drop table. * TODO There is a bug in this function in shadow's unit test and requires additional fixes. diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java index db2ad31090b52d..b24a06dac8fc20 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/ClickHouseTest.java @@ -33,6 +33,7 @@ import org.testcontainers.utility.MountableFile; import javax.sql.DataSource; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -57,23 +58,25 @@ class ClickHouseTest { private static final Network NETWORK = Network.newNetwork(); @Container - private static final GenericContainer ZOOKEEPER_CONTAINER = new GenericContainer<>("zookeeper:3.9.3-jre-17") + private static final GenericContainer CLICKHOUSE_KEEPER_CONTAINER = new GenericContainer<>("clickhouse/clickhouse-keeper:24.10.2.80") + .withCopyFileToContainer( + MountableFile.forHostPath(Paths.get("src/test/resources/test-native/xml/keeper_config.xml").toAbsolutePath()), + "/etc/clickhouse-keeper/keeper_config.xml") .withNetwork(NETWORK) - .withNetworkAliases("foo"); + .withNetworkAliases("clickhouse-keeper-01"); @Container public static final ClickHouseContainer CONTAINER = new ClickHouseContainer("clickhouse/clickhouse-server:24.10.2.80") - .withCopyFileToContainer(MountableFile.forClasspathResource("test-native/xml/clickhouse-transactions.xml"), + .withCopyFileToContainer( + MountableFile.forHostPath(Paths.get("src/test/resources/test-native/xml/transactions.xml").toAbsolutePath()), "/etc/clickhouse-server/config.d/transactions.xml") .withNetwork(NETWORK) - .dependsOn(ZOOKEEPER_CONTAINER); + .dependsOn(CLICKHOUSE_KEEPER_CONTAINER); private static final String SYSTEM_PROP_KEY_PREFIX = "fixture.test-native.yaml.database.clickhouse."; private String jdbcUrlPrefix; - private TestShardingService testShardingService; - @BeforeAll static void beforeAll() { assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url"), is(nullValue())); @@ -90,33 +93,17 @@ static void afterAll() { } /** - * TODO Need to fix `shardingsphere-parser-sql-clickhouse` module to use {@link TestShardingService#cleanEnvironment()} - * after {@link TestShardingService#processSuccessInClickHouse()}. + * TODO The {@code shardingsphere-parser-sql-clickhouse} module needs to be fixed to use SQL like `create table`, + * `truncate table` and `drop table`. */ @Test void assertShardingInLocalTransactions() throws SQLException { jdbcUrlPrefix = "jdbc:ch://localhost:" + CONTAINER.getMappedPort(8123) + "/"; DataSource dataSource = createDataSource(); - testShardingService = new TestShardingService(dataSource); + TestShardingService testShardingService = new TestShardingService(dataSource); testShardingService.processSuccessInClickHouse(); } - /** - * TODO Need to fix `shardingsphere-parser-sql-clickhouse` module to use `initEnvironment()` - * before {@link TestShardingService#processSuccessInClickHouse()}. - * - * @throws SQLException An exception that provides information on a database access error or other errors. - */ - @SuppressWarnings("unused") - private void initEnvironment() throws SQLException { - testShardingService.getOrderRepository().createTableIfNotExistsInClickHouse(); - testShardingService.getOrderItemRepository().createTableIfNotExistsInClickHouse(); - testShardingService.getAddressRepository().createTableIfNotExistsInMySQL(); - testShardingService.getOrderRepository().truncateTable(); - testShardingService.getOrderItemRepository().truncateTable(); - testShardingService.getAddressRepository().truncateTable(); - } - private Connection openConnection(final String databaseName) throws SQLException { Properties props = new Properties(); props.setProperty("user", CONTAINER.getUsername()); @@ -146,6 +133,13 @@ private DataSource createDataSource() throws SQLException { return new HikariDataSource(config); } + /** + * ClickHouse does not support `AUTO_INCREMENT`, + * refer to ClickHouse/ClickHouse#56228 . + * + * @param databaseName database name + * @throws RuntimeException SQL exception + */ private void initTable(final String databaseName) { try ( Connection connection = openConnection(databaseName); @@ -156,7 +150,7 @@ private void initTable(final String databaseName) { + " order_type Int32,\n" + " user_id Int32 NOT NULL,\n" + " address_id Int64 NOT NULL,\n" - + " status String\n" + + " status VARCHAR(50)\n" + ") engine = MergeTree\n" + " primary key (order_id)\n" + " order by (order_id)"); @@ -165,8 +159,8 @@ private void initTable(final String databaseName) { + " order_item_id Int64 NOT NULL,\n" + " order_id Int64 NOT NULL,\n" + " user_id Int32 NOT NULL,\n" - + " phone String,\n" - + " status String\n" + + " phone VARCHAR(50),\n" + + " status VARCHAR(50)\n" + ") engine = MergeTree\n" + " primary key (order_item_id)\n" + " order by (order_item_id)"); diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/HiveSimpleTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/HiveSimpleTest.java index adaa894756bc4b..72e0e781489b6b 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/HiveSimpleTest.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/HiveSimpleTest.java @@ -56,12 +56,10 @@ class HiveSimpleTest { private static final String SYSTEM_PROP_KEY_PREFIX = "fixture.test-native.yaml.database.hive.simple."; // Due to https://issues.apache.org/jira/browse/HIVE-28317 , the `initFile` parameter of HiveServer2 JDBC Driver must be an absolute path. - private static final String ABSOLUTE_PATH = Paths.get("src/test/resources/test-native/sql/test-native-databases-hive.sql").toAbsolutePath().normalize().toString(); + private static final String ABSOLUTE_PATH = Paths.get("src/test/resources/test-native/sql/test-native-databases-hive.sql").toAbsolutePath().toString(); private String jdbcUrlPrefix; - private TestShardingService testShardingService; - @BeforeAll static void beforeAll() { assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url"), is(nullValue())); @@ -76,35 +74,14 @@ static void afterAll() { System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url"); } - /** - * TODO Need to fix `shardingsphere-parser-sql-hive` module to use {@link TestShardingService#cleanEnvironment()} - * after {@link TestShardingService#processSuccessInHive()}. - * - * @throws SQLException An exception that provides information on a database access error or other errors. - */ @Test void assertShardingInLocalTransactions() throws SQLException { jdbcUrlPrefix = "jdbc:hive2://localhost:" + CONTAINER.getMappedPort(10000) + "/"; DataSource dataSource = createDataSource(); - testShardingService = new TestShardingService(dataSource); + TestShardingService testShardingService = new TestShardingService(dataSource); testShardingService.processSuccessInHive(); } - /** - * TODO Need to fix `shardingsphere-parser-sql-hive` module to use `initEnvironment()` before {@link TestShardingService#processSuccessInHive()}. - * - * @throws SQLException An exception that provides information on a database access error or other errors. - */ - @SuppressWarnings("unused") - private void initEnvironment() throws SQLException { - testShardingService.getOrderRepository().createTableIfNotExistsInHive(); - testShardingService.getOrderItemRepository().createTableIfNotExistsInHive(); - testShardingService.getAddressRepository().createTableIfNotExistsInHive(); - testShardingService.getOrderRepository().truncateTable(); - testShardingService.getOrderItemRepository().truncateTable(); - testShardingService.getAddressRepository().truncateTable(); - } - private Connection openConnection() throws SQLException { Properties props = new Properties(); return DriverManager.getConnection(jdbcUrlPrefix, props); diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/HiveZookeeperServiceDiscoveryTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/HiveZookeeperServiceDiscoveryTest.java index 1debb4aa849bfe..5c6a9e7c35da7e 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/HiveZookeeperServiceDiscoveryTest.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/jdbc/databases/hive/HiveZookeeperServiceDiscoveryTest.java @@ -67,6 +67,7 @@ class HiveZookeeperServiceDiscoveryTest { /** * TODO Maybe we should be able to find a better solution than {@link InstanceSpec#getRandomPort()} to use a random available port on the host. * It is not a good practice to use {@link FixedHostPortGenericContainer}. + * See testcontainers/testcontainers-java#9553 . */ @SuppressWarnings("unused") @Container @@ -83,14 +84,12 @@ class HiveZookeeperServiceDiscoveryTest { private static final String SYSTEM_PROP_KEY_PREFIX = "fixture.test-native.yaml.database.hive.zookeeper.sde."; // Due to https://issues.apache.org/jira/browse/HIVE-28317 , the `initFile` parameter of HiveServer2 JDBC Driver must be an absolute path. - private static final String ABSOLUTE_PATH = Paths.get("src/test/resources/test-native/sql/test-native-databases-hive.sql").toAbsolutePath().normalize().toString(); + private static final String ABSOLUTE_PATH = Paths.get("src/test/resources/test-native/sql/test-native-databases-hive.sql").toAbsolutePath().toString(); private final String jdbcUrlSuffix = ";serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2"; private String jdbcUrlPrefix; - private TestShardingService testShardingService; - @BeforeAll static void beforeAll() { assertThat(System.getProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url"), is(nullValue())); @@ -107,8 +106,6 @@ static void afterAll() { } /** - * TODO Need to fix `shardingsphere-parser-sql-hive` module to use {@link TestShardingService#cleanEnvironment()} - * after {@link TestShardingService#processSuccessInHive()}. * TODO Same problem {@link InstanceSpec#getRandomPort()} as {@code HIVE_SERVER2_1_CONTAINER}. * * @throws SQLException An exception that provides information on a database access error or other errors. @@ -117,7 +114,7 @@ static void afterAll() { void assertShardingInLocalTransactions() throws SQLException { jdbcUrlPrefix = "jdbc:hive2://" + ZOOKEEPER_CONTAINER.getHost() + ":" + ZOOKEEPER_CONTAINER.getMappedPort(2181) + "/"; DataSource dataSource = createDataSource(); - testShardingService = new TestShardingService(dataSource); + TestShardingService testShardingService = new TestShardingService(dataSource); testShardingService.processSuccessInHive(); HIVE_SERVER2_1_CONTAINER.stop(); int randomPortSecond = InstanceSpec.getRandomPort(); @@ -137,21 +134,6 @@ void assertShardingInLocalTransactions() throws SQLException { } } - /** - * TODO Need to fix `shardingsphere-parser-sql-hive` module to use `initEnvironment()` before {@link TestShardingService#processSuccessInHive()}. - * - * @throws SQLException An exception that provides information on a database access error or other errors. - */ - @SuppressWarnings("unused") - private void initEnvironment() throws SQLException { - testShardingService.getOrderRepository().createTableIfNotExistsInHive(); - testShardingService.getOrderItemRepository().createTableIfNotExistsInHive(); - testShardingService.getAddressRepository().createTableIfNotExistsInHive(); - testShardingService.getOrderRepository().truncateTable(); - testShardingService.getOrderItemRepository().truncateTable(); - testShardingService.getAddressRepository().truncateTable(); - } - private Connection openConnection() throws SQLException { Properties props = new Properties(); return DriverManager.getConnection(jdbcUrlPrefix + jdbcUrlSuffix, props); diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/proxy/databases/PostgresTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/proxy/databases/PostgresTest.java index cc1a1625612c1d..4e5413aba1a953 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/proxy/databases/PostgresTest.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/proxy/databases/PostgresTest.java @@ -67,7 +67,7 @@ static void beforeAll() throws SQLException { statement.executeUpdate("CREATE DATABASE demo_ds_1"); statement.executeUpdate("CREATE DATABASE demo_ds_2"); } - String absolutePath = Paths.get("src/test/resources/test-native/yaml/proxy/databases/postgresql").toAbsolutePath().normalize().toString(); + String absolutePath = Paths.get("src/test/resources/test-native/yaml/proxy/databases/postgresql").toAbsolutePath().toString(); proxyTestingServer = new ProxyTestingServer(absolutePath); Awaitility.await().atMost(Duration.ofSeconds(30L)).ignoreExceptions().until(() -> { openConnection("root", "root", "jdbc:postgresql://127.0.0.1:" + proxyTestingServer.getProxyPort() + "/postgres").close(); diff --git a/test/native/src/test/java/org/apache/shardingsphere/test/natived/proxy/features/ShardingTest.java b/test/native/src/test/java/org/apache/shardingsphere/test/natived/proxy/features/ShardingTest.java index 8c993bfb917524..3b318ea4f55019 100644 --- a/test/native/src/test/java/org/apache/shardingsphere/test/natived/proxy/features/ShardingTest.java +++ b/test/native/src/test/java/org/apache/shardingsphere/test/natived/proxy/features/ShardingTest.java @@ -68,7 +68,7 @@ static void beforeAll() throws SQLException { statement.executeUpdate("CREATE DATABASE demo_ds_1"); statement.executeUpdate("CREATE DATABASE demo_ds_2"); } - String absolutePath = Paths.get("src/test/resources/test-native/yaml/proxy/features/sharding").toAbsolutePath().normalize().toString(); + String absolutePath = Paths.get("src/test/resources/test-native/yaml/proxy/features/sharding").toAbsolutePath().toString(); proxyTestingServer = new ProxyTestingServer(absolutePath); Awaitility.await().atMost(Duration.ofSeconds(30L)).ignoreExceptionsMatching(e -> e instanceof CommunicationsException).until(() -> { openConnection("root", "root", "jdbc:mysql://127.0.0.1:" + proxyTestingServer.getProxyPort()).close(); diff --git a/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/resource-config.json b/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/resource-config.json index 5a22d077ea78e5..0f6f4683a75bf4 100644 --- a/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/resource-config.json +++ b/test/native/src/test/resources/META-INF/native-image/shardingsphere-test-native-test-metadata/resource-config.json @@ -6,6 +6,9 @@ }, { "condition":{"typeReachable":"org.apache.shardingsphere.test.natived.jdbc.transactions.base.SeataTest"}, "pattern":".*test-native/sql/.+\\.sql$" + }, { + "condition":{"typeReachable":"org.apache.shardingsphere.test.natived.jdbc.databases.ClickHouseTest"}, + "pattern":".*test-native/xml/.+\\.xml$" }, { "condition":{"typeReachable":"org.apache.shardingsphere.test.natived.jdbc.databases.SQLServerTest"}, "pattern":"\\Qcontainer-license-acceptance.txt\\E" diff --git a/test/native/src/test/resources/test-native/sql/test-native-databases-hive.sql b/test/native/src/test/resources/test-native/sql/test-native-databases-hive.sql index 43bc516e458e27..e9353ae7ca53eb 100644 --- a/test/native/src/test/resources/test-native/sql/test-native-databases-hive.sql +++ b/test/native/src/test/resources/test-native/sql/test-native-databases-hive.sql @@ -15,8 +15,9 @@ -- limitations under the License. -- --- TODO `shardingsphere-parser-sql-hive` module does not support `CREATE`, `SET`, `TRUNCATE` statements yet, +-- TODO `shardingsphere-parser-sql-hive` module does not support `create table`, `truncate table` and `drop table` statements yet, -- we always need to execute the following Hive Session-level SQL in the current `javax.sql.DataSource`. +-- Hive does not support `AUTO_INCREMENT`, refer to HIVE-6905 . -- Fixes `Unsupported Hive type VARCHAR, use string instead or enable automatic type conversion, set 'iceberg.mr.schema.auto.conversion' to true` set iceberg.mr.schema.auto.conversion=true; diff --git a/test/native/src/test/resources/test-native/xml/keeper_config.xml b/test/native/src/test/resources/test-native/xml/keeper_config.xml new file mode 100644 index 00000000000000..2a7d254adfa744 --- /dev/null +++ b/test/native/src/test/resources/test-native/xml/keeper_config.xml @@ -0,0 +1,31 @@ + + + 0.0.0.0 + + 9181 + 1 + /var/lib/clickhouse/coordination/snapshots + + + 1 + clickhouse-keeper-01 + 9234 + + + + diff --git a/test/native/src/test/resources/test-native/xml/clickhouse-transactions.xml b/test/native/src/test/resources/test-native/xml/transactions.xml similarity index 92% rename from test/native/src/test/resources/test-native/xml/clickhouse-transactions.xml rename to test/native/src/test/resources/test-native/xml/transactions.xml index 4ded8989a10049..1c54dd5f003d54 100644 --- a/test/native/src/test/resources/test-native/xml/clickhouse-transactions.xml +++ b/test/native/src/test/resources/test-native/xml/transactions.xml @@ -18,8 +18,8 @@ 1 - foo - 2181 + clickhouse-keeper-01 + 9181 diff --git a/test/native/src/test/resources/test-native/yaml/jdbc/features/sharding.yaml b/test/native/src/test/resources/test-native/yaml/jdbc/features/sharding.yaml index c59e15721247aa..7c572d08b61aac 100644 --- a/test/native/src/test/resources/test-native/yaml/jdbc/features/sharding.yaml +++ b/test/native/src/test/resources/test-native/yaml/jdbc/features/sharding.yaml @@ -71,3 +71,6 @@ rules: - !BROADCAST tables: - t_address + +props: + sql-show: false