From b04886069c7cecf632642f601aa77aa56c515b94 Mon Sep 17 00:00:00 2001 From: Emilio Date: Mon, 16 Dec 2024 14:31:10 -0500 Subject: [PATCH] GEOMESA-3425 CLI - Fix older Hadoop versions jar downloads (#3247) --- .../conf-filtered/dependencies.sh | 5 +++++ .../geomesa-fs-tools/conf-filtered/dependencies.sh | 5 +++++ .../geomesa-gt-tools/conf-filtered/dependencies.sh | 13 ++++++++++--- .../conf-filtered/dependencies.sh | 13 ++++++++++--- .../conf-filtered/dependencies.sh | 9 ++++++--- geomesa-tools/conf-filtered/parquet-dependencies.sh | 5 +++++ 6 files changed, 41 insertions(+), 9 deletions(-) diff --git a/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh b/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh index 6a80bbb9dc11..218c44a3ad17 100755 --- a/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh +++ b/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh @@ -101,6 +101,11 @@ function dependencies() { "commons-configuration:commons-configuration:1.6:jar" ) fi + if ! version_ge "${hadoop_version}" 3.4.0; then + gavs+=( + "commons-collections:commons-collections:3.2.2:jar" + ) + fi # compare the version of zookeeper to determine if we need zookeeper-jute (version >= 3.5.5) JUTE_FROM_VERSION="3.5.5" diff --git a/geomesa-fs/geomesa-fs-tools/conf-filtered/dependencies.sh b/geomesa-fs/geomesa-fs-tools/conf-filtered/dependencies.sh index d9696e586f5b..0616baeceecc 100755 --- a/geomesa-fs/geomesa-fs-tools/conf-filtered/dependencies.sh +++ b/geomesa-fs/geomesa-fs-tools/conf-filtered/dependencies.sh @@ -74,6 +74,11 @@ function dependencies() { "com.google.guava:guava:11.0.2:jar" ) fi + if ! version_ge "${hadoop_version}" 3.4.0; then + gavs+=( + "commons-collections:commons-collections:3.2.2:jar" + ) + fi # aws sdk if version_ge "${hadoop_version}" 3.4.0; then diff --git a/geomesa-gt/geomesa-gt-tools/conf-filtered/dependencies.sh b/geomesa-gt/geomesa-gt-tools/conf-filtered/dependencies.sh index 3cb8204803c5..952edd94a8f7 100755 --- a/geomesa-gt/geomesa-gt-tools/conf-filtered/dependencies.sh +++ b/geomesa-gt/geomesa-gt-tools/conf-filtered/dependencies.sh @@ -43,13 +43,20 @@ function dependencies() { ) # add hadoop 3+ jars if needed - local hadoop_maj_ver - hadoop_maj_ver="$([[ "$hadoop_version" =~ ([0-9][0-9]*)\. ]] && echo "${BASH_REMATCH[1]}")" - if [[ "$hadoop_maj_ver" -ge 3 ]]; then + if version_ge "${hadoop_version}" 3.0.0; then gavs+=( "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" ) + else + gavs+=( + "commons-configuration:commons-configuration:1.6:jar" + ) + fi + if ! version_ge "${hadoop_version}" 3.4.0; then + gavs+=( + "commons-collections:commons-collections:3.2.2:jar" + ) fi echo "${gavs[@]}" | tr ' ' '\n' | sort | tr '\n' ' ' diff --git a/geomesa-hbase/geomesa-hbase-tools/conf-filtered/dependencies.sh b/geomesa-hbase/geomesa-hbase-tools/conf-filtered/dependencies.sh index e04512c1fbdc..7b5bce61aac5 100755 --- a/geomesa-hbase/geomesa-hbase-tools/conf-filtered/dependencies.sh +++ b/geomesa-hbase/geomesa-hbase-tools/conf-filtered/dependencies.sh @@ -100,13 +100,20 @@ function dependencies() { fi # add hadoop 3+ jars if needed - local hadoop_maj_ver - hadoop_maj_ver="$([[ "$hadoop_version" =~ ([0-9][0-9]*)\. ]] && echo "${BASH_REMATCH[1]}")" - if [[ "$hadoop_maj_ver" -ge 3 ]]; then + if version_ge "${hadoop_version}" 3.0.0; then gavs+=( "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" ) + else + gavs+=( + "commons-configuration:commons-configuration:1.6:jar" + ) + fi + if ! version_ge "${hadoop_version}" 3.4.0; then + gavs+=( + "commons-collections:commons-collections:3.2.2:jar" + ) fi # compare the version of zookeeper to determine if we need zookeeper-jute (version >= 3.5.5) diff --git a/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh b/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh index c607a986a191..eb4744333417 100755 --- a/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh +++ b/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh @@ -85,9 +85,7 @@ function dependencies() { fi # add hadoop 3+ jars if needed - local hadoop_maj_ver - hadoop_maj_ver="$([[ "$hadoop_version" =~ ([0-9][0-9]*)\. ]] && echo "${BASH_REMATCH[1]}")" - if [[ "$hadoop_maj_ver" -ge 3 ]]; then + if version_ge "${hadoop_version}" 3.0.0; then gavs+=( "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" @@ -97,6 +95,11 @@ function dependencies() { "commons-configuration:commons-configuration:1.6:jar" ) fi + if ! version_ge "${hadoop_version}" 3.4.0; then + gavs+=( + "commons-collections:commons-collections:3.2.2:jar" + ) + fi # compare the version of zookeeper to determine if we need zookeeper-jute (version >= 3.5.5) JUTE_FROM_VERSION="3.5.5" diff --git a/geomesa-tools/conf-filtered/parquet-dependencies.sh b/geomesa-tools/conf-filtered/parquet-dependencies.sh index 228d13a74dda..71cb58a4ec94 100755 --- a/geomesa-tools/conf-filtered/parquet-dependencies.sh +++ b/geomesa-tools/conf-filtered/parquet-dependencies.sh @@ -44,6 +44,11 @@ function dependencies() { "commons-configuration:commons-configuration:1.6:jar" ) fi + if ! version_ge "${hadoop_version}" 3.4.0; then + gavs+=( + "commons-collections:commons-collections:3.2.2:jar" + ) + fi echo "${gavs[@]}" | tr ' ' '\n' | sort | tr '\n' ' ' }