Skip to content

Commit

Permalink
[test](migrate) move 2 cases from p2 to p0 (apache#36935)
Browse files Browse the repository at this point in the history
1. Extend the exploration time of hive docker
2. move case
- test_hive_remove_partition
- test_hive_statistics_from_hms
- test_hive_partition_column_analyze
- test_hive_parquet_alter_column
- test_hive_analyze_db
  • Loading branch information
zddr authored Jul 2, 2024
1 parent 95cb544 commit b3b035e
Show file tree
Hide file tree
Showing 89 changed files with 2,740 additions and 298 deletions.
2 changes: 1 addition & 1 deletion docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ services:
- hive-metastore-postgresql
healthcheck:
test: ["CMD", "sh", "-c", "/mnt/scripts/healthy_check.sh"]
interval: 10s
interval: 20s
timeout: 60s
retries: 120
network_mode: "host"
Expand Down
2 changes: 1 addition & 1 deletion docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ services:
- hive-metastore-postgresql
healthcheck:
test: ["CMD", "sh", "-c", "/mnt/scripts/healthy_check.sh"]
interval: 10s
interval: 20s
timeout: 60s
retries: 120
network_mode: "host"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
CREATE DATABASE IF NOT EXISTS multi_catalog;
USE multi_catalog;

CREATE TABLE `multi_catalog.parquet_alter_column_to_bigint`(
`col_int` bigint,
`col_smallint` bigint,
`col_tinyint` bigint,
`col_bigint` bigint,
`col_float` float,
`col_double` double,
`col_boolean` boolean,
`col_string` string,
`col_char` char(10),
`col_varchar` varchar(255),
`col_date` date,
`col_timestamp` timestamp,
`col_decimal` decimal(10,2))
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION '/user/doris/suites/multi_catalog/parquet_alter_column_to_bigint'
TBLPROPERTIES (
'last_modified_by'='hadoop',
'last_modified_time'='1697217352',
'transient_lastDdlTime'='1697217352');

msck repair table parquet_alter_column_to_bigint;
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
set -x

CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

## mkdir and put data to hdfs
cd "${CUR_DIR}" && rm -rf data/ && tar xzf data.tar.gz
hadoop fs -mkdir -p /user/doris/suites/multi_catalog/
hadoop fs -put "${CUR_DIR}"/data/* /user/doris/suites/multi_catalog/

# create table
hive -f "${CUR_DIR}/create_table.hql"
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
CREATE DATABASE IF NOT EXISTS multi_catalog;
USE multi_catalog;

CREATE TABLE `multi_catalog.parquet_alter_column_to_boolean`(
`col_int` int,
`col_smallint` smallint,
`col_tinyint` tinyint,
`col_bigint` bigint,
`col_float` float,
`col_double` double,
`col_boolean` boolean,
`col_string` string,
`col_char` char(10),
`col_varchar` varchar(255),
`col_date` date,
`col_timestamp` timestamp,
`col_decimal` decimal(10,2))
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION '/user/doris/suites/multi_catalog/parquet_alter_column_to_boolean'
TBLPROPERTIES (
'last_modified_by'='hadoop',
'last_modified_time'='1697217386',
'transient_lastDdlTime'='1697217386');

msck repair table parquet_alter_column_to_boolean;
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
set -x

CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

## mkdir and put data to hdfs
cd "${CUR_DIR}" && rm -rf data/ && tar xzf data.tar.gz
hadoop fs -mkdir -p /user/doris/suites/multi_catalog/
hadoop fs -put "${CUR_DIR}"/data/* /user/doris/suites/multi_catalog/

# create table
hive -f "${CUR_DIR}/create_table.hql"
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
CREATE DATABASE IF NOT EXISTS multi_catalog;
USE multi_catalog;

CREATE TABLE `multi_catalog.parquet_alter_column_to_char`(
`col_int` char(10),
`col_smallint` char(10),
`col_tinyint` char(10),
`col_bigint` char(10),
`col_float` char(10),
`col_double` char(10),
`col_boolean` boolean,
`col_string` char(10),
`col_char` char(10),
`col_varchar` char(10),
`col_date` char(10),
`col_timestamp` char(10),
`col_decimal` char(10))
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION '/user/doris/suites/multi_catalog/parquet_alter_column_to_char'
TBLPROPERTIES (
'last_modified_by'='hadoop',
'last_modified_time'='1697275142',
'transient_lastDdlTime'='1697275142');

msck repair table parquet_alter_column_to_char;
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
set -x

CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

## mkdir and put data to hdfs
cd "${CUR_DIR}" && rm -rf data/ && tar xzf data.tar.gz
hadoop fs -mkdir -p /user/doris/suites/multi_catalog/
hadoop fs -put "${CUR_DIR}"/data/* /user/doris/suites/multi_catalog/

# create table
hive -f "${CUR_DIR}/create_table.hql"
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
CREATE DATABASE IF NOT EXISTS multi_catalog;
USE multi_catalog;

CREATE TABLE `multi_catalog.parquet_alter_column_to_date`(
`col_int` int,
`col_smallint` smallint,
`col_tinyint` tinyint,
`col_bigint` bigint,
`col_float` float,
`col_double` double,
`col_boolean` boolean,
`col_string` string,
`col_char` char(10),
`col_varchar` varchar(255),
`col_date` date,
`col_timestamp` timestamp,
`col_decimal` decimal(10,2))
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION '/user/doris/suites/multi_catalog/parquet_alter_column_to_date'
TBLPROPERTIES (
'last_modified_by'='hadoop',
'last_modified_time'='1697217393',
'transient_lastDdlTime'='1697217393');

msck repair table parquet_alter_column_to_date;
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
set -x

CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

## mkdir and put data to hdfs
cd "${CUR_DIR}" && rm -rf data/ && tar xzf data.tar.gz
hadoop fs -mkdir -p /user/doris/suites/multi_catalog/
hadoop fs -put "${CUR_DIR}"/data/* /user/doris/suites/multi_catalog/

# create table
hive -f "${CUR_DIR}/create_table.hql"
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
CREATE DATABASE IF NOT EXISTS multi_catalog;
USE multi_catalog;

CREATE TABLE `multi_catalog.parquet_alter_column_to_decimal`(
`col_int` decimal(5,1),
`col_smallint` decimal(5,1),
`col_tinyint` decimal(5,1),
`col_bigint` decimal(5,1),
`col_float` float,
`col_double` double,
`col_boolean` boolean,
`col_string` string,
`col_char` char(10),
`col_varchar` varchar(255),
`col_date` date,
`col_timestamp` timestamp,
`col_decimal` decimal(5,1))
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION '/user/doris/suites/multi_catalog/parquet_alter_column_to_decimal'
TBLPROPERTIES (
'last_modified_by'='hadoop',
'last_modified_time'='1697217403',
'transient_lastDdlTime'='1697217403');

msck repair table parquet_alter_column_to_decimal;
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
set -x

CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

## mkdir and put data to hdfs
cd "${CUR_DIR}" && rm -rf data/ && tar xzf data.tar.gz
hadoop fs -mkdir -p /user/doris/suites/multi_catalog/
hadoop fs -put "${CUR_DIR}"/data/* /user/doris/suites/multi_catalog/

# create table
hive -f "${CUR_DIR}/create_table.hql"
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
CREATE DATABASE IF NOT EXISTS multi_catalog;
USE multi_catalog;

CREATE TABLE `multi_catalog.parquet_alter_column_to_double`(
`col_int` double,
`col_smallint` double,
`col_tinyint` double,
`col_bigint` double,
`col_float` double,
`col_double` double,
`col_boolean` boolean,
`col_string` string,
`col_char` char(10),
`col_varchar` varchar(255),
`col_date` date,
`col_timestamp` timestamp,
`col_decimal` decimal(10,2))
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION '/user/doris/suites/multi_catalog/parquet_alter_column_to_double'
TBLPROPERTIES (
'last_modified_by'='hadoop',
'last_modified_time'='1697270364',
'transient_lastDdlTime'='1697270364');

msck repair table parquet_alter_column_to_double;
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
set -x

CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

## mkdir and put data to hdfs
cd "${CUR_DIR}" && rm -rf data/ && tar xzf data.tar.gz
hadoop fs -mkdir -p /user/doris/suites/multi_catalog/
hadoop fs -put "${CUR_DIR}"/data/* /user/doris/suites/multi_catalog/

# create table
hive -f "${CUR_DIR}/create_table.hql"
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
CREATE DATABASE IF NOT EXISTS multi_catalog;
USE multi_catalog;

CREATE TABLE `multi_catalog.parquet_alter_column_to_float`(
`col_int` float,
`col_smallint` float,
`col_tinyint` float,
`col_bigint` float,
`col_float` float,
`col_double` double,
`col_boolean` boolean,
`col_string` string,
`col_char` char(10),
`col_varchar` varchar(255),
`col_date` date,
`col_timestamp` timestamp,
`col_decimal` decimal(10,2))
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION '/user/doris/suites/multi_catalog/parquet_alter_column_to_float'
TBLPROPERTIES (
'last_modified_by'='hadoop',
'last_modified_time'='1697270277',
'transient_lastDdlTime'='1697270277');

msck repair table parquet_alter_column_to_float;
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
set -x

CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

## mkdir and put data to hdfs
cd "${CUR_DIR}" && rm -rf data/ && tar xzf data.tar.gz
hadoop fs -mkdir -p /user/doris/suites/multi_catalog/
hadoop fs -put "${CUR_DIR}"/data/* /user/doris/suites/multi_catalog/

# create table
hive -f "${CUR_DIR}/create_table.hql"
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
CREATE DATABASE IF NOT EXISTS multi_catalog;
USE multi_catalog;

CREATE TABLE `multi_catalog.parquet_alter_column_to_int`(
`col_int` int,
`col_smallint` int,
`col_tinyint` int,
`col_bigint` bigint,
`col_float` float,
`col_double` double,
`col_boolean` boolean,
`col_string` string,
`col_char` char(10),
`col_varchar` varchar(255),
`col_date` date,
`col_timestamp` timestamp,
`col_decimal` decimal(10,2))
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION '/user/doris/suites/multi_catalog/parquet_alter_column_to_int'
TBLPROPERTIES (
'last_modified_by'='hadoop',
'last_modified_time'='1697216968',
'transient_lastDdlTime'='1697216968');

msck repair table parquet_alter_column_to_int;
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
set -x

CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

## mkdir and put data to hdfs
cd "${CUR_DIR}" && rm -rf data/ && tar xzf data.tar.gz
hadoop fs -mkdir -p /user/doris/suites/multi_catalog/
hadoop fs -put "${CUR_DIR}"/data/* /user/doris/suites/multi_catalog/

# create table
hive -f "${CUR_DIR}/create_table.hql"
Loading

0 comments on commit b3b035e

Please sign in to comment.