[DO NOT MERGE] GRD-72478

.travis.yml

@@ -1,10 +1,13 @@
-language: java
+language: shell
 
 os:
   - linux
 
 before_script:
-  - echo "Running before script..."
+  - chmod -R 777 .
+  - sudo apt-get update && sudo apt-get install -y parallel  # Install GNU parallel if needed
 
 script:
-  - echo "Running tests..."
+  - chmod -R 755 ./build
+  - travis_wait 80 ./build/buildUCDefaultOfflinePackage.sh  # Increase timeout to 80 minutes
+  - ./build/test.sh

build/Dockerfile

@@ -0,0 +1,54 @@
+# Arguments for the Docker repository and image tag
+ARG DOCKER_REPO
+ARG IMAGE_TAG
+
+# Base image
+FROM ${DOCKER_REPO}/${IMAGE_TAG}
+
+# Environment variables for Logstash
+ARG LOGSTASH_VERSION=8.3.3
+
+# Metadata as described in the image
+LABEL name="Universal-Connector plugins" \
+      vendor="International Business Machines Corporation" \
+      summary="Universal-Connector plugins build image based on ubi-minimal" \
+      description="For more information on this image please see https://github.com/IBM/universal-connectors/blob/main/README.md"
+
+# Set environment variables for Logstash installation
+ENV LOGSTASH_VERSION=${LOGSTASH_VERSION} \
+    LOGSTASH_OSS_FULL_VERSION=logstash-oss-${LOGSTASH_VERSION}-linux-x86_64 \
+    LOGSTASH_GIT_TAG=v${LOGSTASH_VERSION} \
+    LOGSTASH_SRC_DIR=/usr/share/logstashSRC \
+    LOGSTASH_DIR=/usr/share/logstash
+
+# Install necessary packages and clean up
+RUN microdnf install -y tzdata openssl curl wget ca-certificates fontconfig \
+                       glibc-langpack-en zip gzip tar git vim findutils && \
+    microdnf update -y && \
+    microdnf clean all
+
+# Download and extract Logstash
+RUN mkdir -p ${LOGSTASH_DIR} && \
+    curl -L https://artifacts.elastic.co/downloads/logstash/${LOGSTASH_OSS_FULL_VERSION}.tar.gz \
+         -o ${LOGSTASH_DIR}/${LOGSTASH_OSS_FULL_VERSION}.tar.gz && \
+    tar -xzf ${LOGSTASH_DIR}/${LOGSTASH_OSS_FULL_VERSION}.tar.gz -C ${LOGSTASH_DIR} --strip-components=1 && \
+    rm ${LOGSTASH_DIR}/${LOGSTASH_OSS_FULL_VERSION}.tar.gz
+
+# Set environment variables for Logstash JDK and JRuby after installation
+ENV JAVA_HOME="${LOGSTASH_DIR}/jdk" \
+    JRUBY_HOME="${LOGSTASH_DIR}/vendor/jruby"
+
+# Set up directories and permissions
+RUN mkdir -p ${LOGSTASH_SRC_DIR} && \
+    chown -R 1000:1000 ${LOGSTASH_DIR} ${LOGSTASH_SRC_DIR}
+
+USER 1000
+
+# Clone Logstash source code and build
+RUN git clone -b "${LOGSTASH_GIT_TAG}" https://github.com/elastic/logstash.git ${LOGSTASH_SRC_DIR}/logstash && \
+    cd ${LOGSTASH_SRC_DIR}/logstash && \
+    ./gradlew clean -qq --warning-mode all && ./gradlew assemble -qq
+
+ENV PATH="${PATH}:${JAVA_HOME}/bin:${JRUBY_HOME}/bin"
+
+WORKDIR ${LOGSTASH_DIR}

build/buildUCDefaultOfflinePackage.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+# Constants
+LOGSTASH_DIR="/usr/share/logstash"
+PLUGIN_FILE="build/defaultOfflinePackagePlugins.txt"
+DOCKER_CONTAINER_NAME="Alan"
+DOCKER_IMAGE_NAME="guc_dit:latest"
+
+# Function to update the "defaultOfflinePackagePlugins.txt" with the latest version for each plugin
+verify_plugins_version() {
+  if [ ! -f "$PLUGIN_FILE" ]; then
+    echo "Plugin file not found: $PLUGIN_FILE"
+    return 1
+  fi
+
+  echo "Updating plugin file names with the latest version in the $PLUGIN_FILE file"
+
+  # Read each line in the file, ignore comments and process
+  grep -v '^#' "$PLUGIN_FILE" | while read -r line; do
+    plugin_name=$(basename "$line")
+    plugin_location=$(dirname "$line")
+    version_file_path="$plugin_location/VERSION"
+
+    # Read the current version from the VERSION file
+    if [ -f "$version_file_path" ]; then
+      version=$(<"$version_file_path")
+      new_line="$plugin_location/$plugin_name-$version.gem"
+
+      # Update the plugin file with the new version using `sed`
+      if [[ $OSTYPE == 'darwin'* ]]; then
+        sed -i '' "s|$line|$new_line|g" "$PLUGIN_FILE"
+      else
+        sed -i "s|$line|$new_line|g" "$PLUGIN_FILE"
+      fi
+    else
+      echo "Version file not found for plugin: $plugin_name"
+    fi
+  done
+}
+
+# Function to build Docker image
+build_docker_image() {
+  echo "Building Docker image..."
+  (cd build && docker build --build-arg DOCKER_REPO="${DOCKER_REPO}" --build-arg IMAGE_TAG="${IMAGE_TAG}" -qt "${DOCKER_IMAGE_NAME}" .)
+}
+
+# Function to run Docker container
+run_docker_container() {
+  echo "Running Docker container..."
+  docker run --name="${DOCKER_CONTAINER_NAME}" -v "$(pwd)/:${LOGSTASH_DIR}/universal-connectors/" -dit "${DOCKER_IMAGE_NAME}" bash
+}
+
+# Function to build plugin gems inside the Docker container
+build_plugin_gems() {
+  echo "Building plugin gems in Docker container..."
+  if docker exec "${DOCKER_CONTAINER_NAME}" bash -c "cd universal-connectors && ./build/buildUCPluginGems.sh"; then
+    echo "Successfully tested and built plugins."
+  else
+    echo "Failed to test and build plugins."
+    exit 1
+  fi
+}
+
+# Function to prepare the default offline package
+prepare_offline_package() {
+  echo "Preparing default offline package..."
+  grep -v '^#' "$PLUGIN_FILE" | while read -r line; do
+    docker cp "$line" "${DOCKER_CONTAINER_NAME}:${LOGSTASH_DIR}/."
+  done
+  docker exec "${DOCKER_CONTAINER_NAME}" bash -c "cd universal-connectors && ./build/prepareUCDefaultOfflinePackage.sh"
+}
+
+# Function to list files in the 'dist' directory
+list_dist_files() {
+  echo "Listing files in the 'dist' directory:"
+  find dist -type f
+}
+
+# Main Script Execution
+echo "================ Starting Build Process ================="
+build_docker_image
+verify_plugins_version
+
+echo "Final list in $PLUGIN_FILE:"
+cat "$PLUGIN_FILE"
+
+run_docker_container
+build_plugin_gems
+prepare_offline_package
+list_dist_files
+
+echo "================ Build Process Complete ================="
+exit 0

build/buildUCPluginGems.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+BASE_DIR=$(pwd)
+
+function buildUCCommons() {
+  cd "${BASE_DIR}/common" || exit
+  ./gradlew test >/dev/null 2>&1
+  if [ $? -eq 0 ]; then
+    echo "Successfully tested uc-commons"
+  else
+    echo "Failed to test uc-commons"
+    exit 1
+  fi
+  ./gradlew jar >/dev/null 2>&1
+  if [ $? -eq 0 ]; then
+    echo "Successfully built jar uc-commons"
+  else
+    echo "Failed to build jar uc-commons"
+    exit 2
+  fi
+  cp ./build/libs/common-1.0.0.jar ./build/libs/guardium-universalconnector-commons-1.0.0.jar
+  cd "${BASE_DIR}" || exit
+}
+
+buildUCCommons
+
+# Build Java plugins in parallel
+grep -v '^#' "${BASE_DIR}/build/javaPluginsToBuild.txt" | xargs -P 2 -I {} bash -c '
+BASE_DIR="'${BASE_DIR}'"
+
+function adjustToLogstash8() {
+  sed -i "s/logstash-core-*.*.*.jar/logstash-core.jar/" build.gradle
+}
+
+function buildUCPluginGem() {
+  echo "================ Building $1 gem file ================="
+  cd "${BASE_DIR}/$1" || exit
+  adjustToLogstash8
+  cp "${BASE_DIR}/build/gradle.properties" .
+  ./gradlew --no-daemon test >/dev/null 2>&1
+  if [ $? -eq 0 ]; then
+    echo "Successfully tested $1"
+    ./gradlew --no-daemon gem >/dev/null 2>&1
+    if [ $? -eq 0 ]; then
+      echo "Successfully built gem $1"
+    else
+      echo "Failed to build gem $1"
+    fi
+  else
+    echo "Failed to test $1"
+  fi
+}
+
+buildUCPluginGem "{}"
+'
+
+# Build Ruby plugins in parallel
+grep -v '^#' "${BASE_DIR}/build/rubyPluginsToBuild.txt" | xargs -P 2 -I {} bash -c '
+BASE_DIR="'${BASE_DIR}'"
+
+function buildRubyPlugin() {
+  cd "${BASE_DIR}/$1" || exit
+  bundle install >/dev/null 2>&1
+  gem build "$2"
+}
+
+buildRubyPlugin "{}" "${}/*.gemspec"
+'
+
+exit 0

build/gradle.properties

@@ -0,0 +1,3 @@
+# Env vars for the build scripts
+LOGSTASH_CORE_PATH=/usr/share/logstashSRC/logstash/logstash-core
+GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH=../../common/build/libs

build/packagePluginsForGuardiumInsights.sh

@@ -3,80 +3,75 @@
 # Copyright 2020-2021 IBM Inc. All rights reserved
 # SPDX-License-Identifier: Apache2.0
 #---------------------------------------------------------------------------------
-#---------------------------------------------------------------------------------
 
+# Constants
 VERIFIED_UC_GDP_PLUGINS_FILE="verifiedUCPlugins_gdp.txt"
 VERIFIED_UC_GI_PLUGINS_FILE="verifiedUCPlugins_gi.txt"
-
-#The full list with the offline packages names
 VERIFIED_FULL_PLUGINS_LIST="verified_UC_plugins_full_list.txt"
 
-PACKAGED_PLUGINS_DIR=dist/plugins
-GI_PLUGINS_DIR=dist/temp/GI
-GDP_PLUGINS_DIR=dist/temp/GDP
-
-PLUGINS_LISTS_DIR=build/
-
+PACKAGED_PLUGINS_DIR="dist/plugins"
+GI_PLUGINS_DIR="dist/temp/GI"
+GDP_PLUGINS_DIR="dist/temp/GDP"
+PLUGINS_LISTS_DIR="build"
 
 GI_PLUGINS_TEMPLATES="gi_plugins_templates.zip"
 GDP_PLUGINS_TEMPLATES="gdp_plugins_templates.zip"
 originalPath=$(pwd)
 
-function zipPackage {
+# Function to zip a package along with optional offline plugins
+zip_package() {
   IFS=';' read -r package_folder offline_plugin <<< "$1"
 
-  echo "package_folder is: $package_folder"
-  if [ -n "$offline_plugin" ]; then
-    echo "offline_plugin is: $offline_plugin"
-  fi
-
+  echo "Packaging folder: $package_folder"
+  [ -n "$offline_plugin" ] && echo "Including offline plugin: $offline_plugin"
 
-  zipName="$(echo $package_folder | awk -F'/' '{print $NF}')"
-  packageFolder=$(dirname $package_folder)
-  targetDirectory=$2
+  zip_name=$(basename "$package_folder")
+  package_dir=$(dirname "$package_folder")
+  target_directory="$2"
 
-  cd "${originalPath}/${packageFolder}"
-  echo "location of folder to package: $(pwd)"
-  echo "folder to package: ${zipName}"
-  zip -r "${originalPath}/${targetDirectory}/${zipName}.zip" $zipName -x "*.zip"
+  cd "${originalPath}/${package_dir}" || { echo "Failed to enter directory: ${package_dir}"; exit 1; }
+  echo "Current directory: $(pwd)"
+  echo "Zipping folder: ${zip_name}"
 
-  # if there is also offline plugins, enter it to the zip:
-  if [ -e "$originalPath/build/offline_packages/zips/$offline_plugin" ]; then
-      echo "The file $offline_plugin exists."
+  zip -r "${originalPath}/${target_directory}/${zip_name}.zip" "$zip_name" -x "*.zip"
 
-      zip -uj "${originalPath}/${targetDirectory}/${zipName}.zip" $originalPath/build/offline_packages/zips/$offline_plugin
+  # If there is an offline plugin, include it in the zip
+  offline_plugin_path="$originalPath/build/offline_packages/zips/$offline_plugin"
+  if [ -e "$offline_plugin_path" ]; then
+    echo "Adding offline plugin: $offline_plugin to ${zip_name}.zip"
+    zip -uj "${originalPath}/${target_directory}/${zip_name}.zip" "$offline_plugin_path"
   fi
 
-  cd "$originalPath"
+  cd "$originalPath" || { echo "Failed to return to original directory"; exit 1; }
 }
 
-mkdir -p ${PACKAGED_PLUGINS_DIR};
-mkdir -p ${GI_PLUGINS_DIR}
-mkdir -p ${GDP_PLUGINS_DIR}
-
-
-# go to the build direction and zip all the plug-ins packages includes the offline-package plugin (if exist)
-cd ${PLUGINS_LISTS_DIR}
-grep -v '^#' ${VERIFIED_FULL_PLUGINS_LIST} | while read -r line ; do zipPackage "$line" "${PACKAGED_PLUGINS_DIR}" ; done
+# Create necessary directories
+mkdir -p "${PACKAGED_PLUGINS_DIR}" "${GI_PLUGINS_DIR}" "${GDP_PLUGINS_DIR}"
 
+# Zip all plugin packages, including offline-package plugins if they exist
+cd "${PLUGINS_LISTS_DIR}" || { echo "Failed to enter directory: ${PLUGINS_LISTS_DIR}"; exit 1; }
+grep -v '^#' "${VERIFIED_FULL_PLUGINS_LIST}" | while read -r line; do
+  zip_package "$line" "${PACKAGED_PLUGINS_DIR}"
+done
 
-#zip the GDP plugins templates to a temp location
-grep -v '^#' ${VERIFIED_UC_GDP_PLUGINS_FILE} | while read -r line ; do zipPackage "$line" "${GDP_PLUGINS_DIR}" ; done
+# Zip GDP plugins templates to a temporary location
+grep -v '^#' "${VERIFIED_UC_GDP_PLUGINS_FILE}" | while read -r line; do
+  zip_package "$line" "${GDP_PLUGINS_DIR}"
+done
 
-#zip the GI plugins templates to a temp location
-grep -v '^#' ${VERIFIED_UC_GI_PLUGINS_FILE} | while read -r line ; do zipPackage "$line" "${GI_PLUGINS_DIR}" ; done
+# Zip GI plugins templates to a temporary location
+grep -v '^#' "${VERIFIED_UC_GI_PLUGINS_FILE}" | while read -r line; do
+  zip_package "$line" "${GI_PLUGINS_DIR}"
+done
 
-# list all GI plugins into a file
-cd ${originalPath}/${GI_PLUGINS_DIR}
+# List all GI plugins into a file
+cd "${originalPath}/${GI_PLUGINS_DIR}" || { echo "Failed to enter directory: ${GI_PLUGINS_DIR}"; exit 1; }
 ls | grep -v '/$' | grep -v 'plugins_list.txt' > "plugins_list.txt"
 
-
-#zip all plugins zips (the templates only, not the offline plugin zip) and the names list into one zip
-cd ${originalPath}
+# Zip all plugin zips (the templates only, not the offline plugin zip) and the names list into one zip
+cd "${originalPath}" || { echo "Failed to return to original directory"; exit 1; }
 zip -j -r "${GDP_PLUGINS_TEMPLATES}" "${GDP_PLUGINS_DIR}"
 zip -j -r "${GI_PLUGINS_TEMPLATES}" "${GI_PLUGINS_DIR}"
 
-
-# remove the temporary directions
-rm -r ${originalPath}/${GI_PLUGINS_DIR}
-rm -r ${originalPath}/${GDP_PLUGINS_DIR}
+# Remove the temporary directories
+rm -r "${GI_PLUGINS_DIR}" "${GDP_PLUGINS_DIR}"

build/pluginsToBuild_GI.txt

@@ -1,29 +1,3 @@
 filter-plugin/logstash-filter-mongodb-guardium
-filter-plugin/logstash-filter-cosmos-azure-guardium
-filter-plugin/logstash-filter-mysql-guardium
-filter-plugin/logstash-filter-s3-guardium
-filter-plugin/logstash-filter-hdfs-guardium
-filter-plugin/logstash-filter-mysql-percona-guardium
-filter-plugin/logstash-filter-generic-guardium
-filter-plugin/logstash-filter-saphana-guardium
-filter-plugin/logstash-filter-cassandra-guardium
-filter-plugin/logstash-filter-aurora-mysql-guardium
 filter-plugin/logstash-filter-dynamodb-guardium
-filter-plugin/logstash-filter-neptune-aws-guardium
-filter-plugin/logstash-filter-couchbasedb-guardium
-filter-plugin/logstash-filter-couchdb-guardium
-filter-plugin/logstash-filter-mariadb-guardium
-filter-plugin/logstash-filter-azure-apachesolr-guardium
-filter-plugin/logstash-filter-onPremGreenplumdb-guardium
-filter-plugin/logstash-filter-documentdb-aws-guardium
-filter-plugin/logstash-filter-azure-postgresql-guardium
-filter-plugin/logstash-filter-azure-sql-guardium
-filter-plugin/logstash-filter-neo4j-guardium
-filter-plugin/logstash-filter-snowflake-guardium
-filter-plugin/logstash-filter-yugabyte-guardium
-filter-plugin/logstash-filter-teradatadb-guardium
-filter-plugin/logstash-filter-intersystems-iris-guardium
-filter-plugin/logstash-filter-postgres-ibmcloud-guardium
-filter-plugin/logstash-filter-mysql-azure-guardium
-filter-plugin/logstash-filter-scylldb-guardium
-input-plugin/logstash-input-mongo-atlas
+filter-plugin/logstash-filter-pubsub-firestore-guardium