From 755e751f30e1f07ca0917274d72443b4e0ee82dd Mon Sep 17 00:00:00 2001 From: Emilio Date: Mon, 11 Nov 2024 08:58:19 -0500 Subject: [PATCH] Upgrade to Hadoop 3.4.1, HBase 2.6.1 (#3226) --- build/cqs.tsv | 68 ++++++++++++++++++++----------------------- docs/user/upgrade.rst | 2 ++ pom.xml | 56 +++++++++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 39 deletions(-) diff --git a/build/cqs.tsv b/build/cqs.tsv index 81dfb47d01d8..dac1e98e89f5 100644 --- a/build/cqs.tsv +++ b/build/cqs.tsv @@ -20,7 +20,6 @@ com.github.pureconfig:pureconfig-generic-base_2.12 0.17.4 compile com.github.pureconfig:pureconfig-generic_2.12 0.17.4 compile com.github.pureconfig:pureconfig_2.12 0.17.4 compile com.google.code.gson:gson 2.10.1 compile -com.google.errorprone:error_prone_annotations 2.21.1 compile com.google.flatbuffers:flatbuffers-java 24.3.25 compile com.google.guava:failureaccess 1.0 compile com.google.guava:failureaccess 1.0.2 compile @@ -170,24 +169,24 @@ org.apache.curator:curator-framework 5.6.0 compile org.apache.curator:curator-recipes 5.6.0 compile org.apache.datasketches:datasketches-java 4.1.0 compile org.apache.datasketches:datasketches-memory 2.2.0 compile -org.apache.hbase.thirdparty:hbase-shaded-gson 4.1.5 compile -org.apache.hbase.thirdparty:hbase-shaded-miscellaneous 4.1.5 compile -org.apache.hbase.thirdparty:hbase-shaded-netty 4.1.5 compile -org.apache.hbase.thirdparty:hbase-shaded-protobuf 4.1.5 compile -org.apache.hbase.thirdparty:hbase-unsafe 4.1.5 compile -org.apache.hbase:hbase-asyncfs 2.5.8-hadoop3 compile -org.apache.hbase:hbase-client 2.5.8-hadoop3 compile -org.apache.hbase:hbase-common 2.5.8-hadoop3 compile -org.apache.hbase:hbase-hadoop-compat 2.5.8-hadoop3 compile -org.apache.hbase:hbase-hadoop2-compat 2.5.8-hadoop3 compile -org.apache.hbase:hbase-logging 2.5.8-hadoop3 compile -org.apache.hbase:hbase-mapreduce 2.5.8-hadoop3 compile -org.apache.hbase:hbase-metrics 2.5.8-hadoop3 compile -org.apache.hbase:hbase-metrics-api 2.5.8-hadoop3 compile -org.apache.hbase:hbase-protocol 2.5.8-hadoop3 compile -org.apache.hbase:hbase-protocol-shaded 2.5.8-hadoop3 compile -org.apache.hbase:hbase-replication 2.5.8-hadoop3 compile -org.apache.hbase:hbase-zookeeper 2.5.8-hadoop3 compile +org.apache.hbase.thirdparty:hbase-shaded-gson 4.1.9 compile +org.apache.hbase.thirdparty:hbase-shaded-miscellaneous 4.1.9 compile +org.apache.hbase.thirdparty:hbase-shaded-netty 4.1.9 compile +org.apache.hbase.thirdparty:hbase-shaded-protobuf 4.1.9 compile +org.apache.hbase.thirdparty:hbase-unsafe 4.1.9 compile +org.apache.hbase:hbase-asyncfs 2.6.1-hadoop3 compile +org.apache.hbase:hbase-client 2.6.1-hadoop3 compile +org.apache.hbase:hbase-common 2.6.1-hadoop3 compile +org.apache.hbase:hbase-hadoop-compat 2.6.1-hadoop3 compile +org.apache.hbase:hbase-hadoop2-compat 2.6.1-hadoop3 compile +org.apache.hbase:hbase-logging 2.6.1-hadoop3 compile +org.apache.hbase:hbase-mapreduce 2.6.1-hadoop3 compile +org.apache.hbase:hbase-metrics 2.6.1-hadoop3 compile +org.apache.hbase:hbase-metrics-api 2.6.1-hadoop3 compile +org.apache.hbase:hbase-protocol 2.6.1-hadoop3 compile +org.apache.hbase:hbase-protocol-shaded 2.6.1-hadoop3 compile +org.apache.hbase:hbase-replication 2.6.1-hadoop3 compile +org.apache.hbase:hbase-zookeeper 2.6.1-hadoop3 compile org.apache.htrace:htrace-core 3.2.0-incubating compile org.apache.httpcomponents:httpclient 4.5.14 compile org.apache.httpcomponents:httpcore 4.4.16 compile @@ -208,8 +207,6 @@ org.apache.zookeeper:zookeeper 3.9.2 compile org.apache.zookeeper:zookeeper-jute 3.9.2 compile org.apiguardian:apiguardian-api 1.1.2 compile org.calrissian.mango:mango-core 3.0.0 compile -org.checkerframework:checker-qual 3.37.0 compile -org.checkerframework:checker-qual 3.42.0 compile org.eclipse.emf:org.eclipse.emf.common 2.15.0 compile org.eclipse.emf:org.eclipse.emf.ecore 2.15.0 compile org.eclipse.emf:org.eclipse.emf.ecore.xmi 2.15.0 compile @@ -249,8 +246,8 @@ org.geotools:gt-transform 32.0 compile org.geotools:gt-xml 32.0 compile org.huldra.math:bigint 0.7.1 compile org.jetbrains:annotations 17.0.0 compile -org.jruby.jcodings:jcodings 1.0.55 compile -org.jruby.joni:joni 2.1.31 compile +org.jruby.jcodings:jcodings 1.0.58 compile +org.jruby.joni:joni 2.2.1 compile org.json4s:json4s-ast_2.12 3.6.12 compile org.json4s:json4s-core_2.12 3.6.12 compile org.json4s:json4s-native_2.12 3.6.12 compile @@ -295,18 +292,17 @@ io.prometheus:prometheus-metrics-exporter-httpserver 1.3.1 provided io.prometheus:prometheus-metrics-exporter-pushgateway 1.3.1 provided javax.media:jai_core 1.1.3 provided org.apache.accumulo:accumulo-start 2.1.3 provided -org.apache.hadoop:hadoop-annotations 3.4.0 provided -org.apache.hadoop:hadoop-aws 3.4.0 provided -org.apache.hadoop:hadoop-client 3.4.0 provided -org.apache.hadoop:hadoop-common 3.4.0 provided -org.apache.hadoop:hadoop-distcp 3.4.0 provided -org.apache.hadoop:hadoop-hdfs 3.4.0 provided -org.apache.hadoop:hadoop-mapreduce-client-core 3.3.6 provided -org.apache.hadoop:hadoop-mapreduce-client-core 3.4.0 provided -org.apache.hadoop:hadoop-mapreduce-client-jobclient 3.4.0 provided -org.apache.hadoop:hadoop-yarn-api 3.4.0 provided -org.apache.hadoop:hadoop-yarn-common 3.4.0 provided -org.apache.hbase:hbase-server 2.5.8-hadoop3 provided +org.apache.hadoop:hadoop-annotations 3.4.1 provided +org.apache.hadoop:hadoop-aws 3.4.1 provided +org.apache.hadoop:hadoop-client 3.4.1 provided +org.apache.hadoop:hadoop-common 3.4.1 provided +org.apache.hadoop:hadoop-distcp 3.4.1 provided +org.apache.hadoop:hadoop-hdfs 3.4.1 provided +org.apache.hadoop:hadoop-mapreduce-client-core 3.4.1 provided +org.apache.hadoop:hadoop-mapreduce-client-jobclient 3.4.1 provided +org.apache.hadoop:hadoop-yarn-api 3.4.1 provided +org.apache.hadoop:hadoop-yarn-common 3.4.1 provided +org.apache.hbase:hbase-server 2.6.1-hadoop3 provided org.apache.htrace:htrace-core4 4.1.0-incubating provided org.apache.kafka:kafka-clients 3.7.0 provided org.apache.kafka:kafka-streams-scala_2.12 3.7.0 provided @@ -332,7 +328,7 @@ org.apache.arrow:arrow-vector tests:17.0.0 test org.apache.cassandra:cassandra-all 3.11.14 test org.apache.cassandra:cassandra-thrift 3.11.14 test org.apache.curator:curator-test 5.6.0 test -org.apache.hbase:hbase-testing-util 2.5.8-hadoop3 test +org.apache.hbase:hbase-testing-util 2.6.1-hadoop3 test org.apache.kafka:kafka-clients test:3.7.0 test org.apache.kafka:kafka-streams-test-utils 3.7.0 test org.apache.logging.log4j:log4j-api 2.22.1 test diff --git a/docs/user/upgrade.rst b/docs/user/upgrade.rst index 91d735f16738..1ff86e0718b6 100644 --- a/docs/user/upgrade.rst +++ b/docs/user/upgrade.rst @@ -98,6 +98,8 @@ Dependency Version Upgrades The following dependencies have been upgraded: +* hbase ``2.5.8-hadoop3`` -> ``2.6.1-hadoop3`` +* hadoop ``3.4.0`` -> ``3.4.1`` * spring-security ``5.8.14`` -> ``5.8.15`` Version 5.1.0 Upgrade Guide diff --git a/pom.xml b/pom.xml index 5ec085ea36b9..99d84f49ee99 100644 --- a/pom.xml +++ b/pom.xml @@ -134,10 +134,10 @@ 0.17.0 0.12.0 0.12.0 - 3.4.0 + 3.4.1 3.5.0 - 2.5.8-hadoop3 - 3.3.6 + 2.6.1-hadoop3 + 3.4.1 4.1.0-incubating 2.5.0 3.25.3 @@ -1015,6 +1015,13 @@ org.postgresql postgresql ${postgres.version} + + + + org.checkerframework + checker-qual + + org.geotools @@ -1337,11 +1344,33 @@ org.calrissian.mango mango-core ${mango.version} + + + + org.checkerframework + checker-qual + + + com.google.errorprone + error_prone_annotations + + com.github.ben-manes.caffeine caffeine ${caffeine.version} + + + + org.checkerframework + checker-qual + + + com.google.errorprone + error_prone_annotations + + com.google.code.gson @@ -1914,6 +1943,11 @@ org.apache.logging.log4j * + + + org.checkerframework + checker-qual + @@ -2329,6 +2363,10 @@ log4j * + + com.google.errorprone + error_prone_annotations + @@ -2365,6 +2403,10 @@ log4j * + + com.google.errorprone + error_prone_annotations + @@ -2381,6 +2423,10 @@ org.slf4j slf4j-log4j12 + + com.google.errorprone + error_prone_annotations + org.mortbay.jetty @@ -2398,6 +2444,10 @@ log4j * + + com.google.errorprone + error_prone_annotations +