NTNX: Changes for building envoy on Centos UBVM

Signed-off-by: Basundhara Chakrabarty <[email protected]>

.bazelrc

@@ -54,6 +54,9 @@ build:docs-ci --action_env=DOCS_RST_CHECK=1 --host_action_env=DOCS_RST_CHECK=1
 build --incompatible_config_setting_private_default_visibility
 build --incompatible_enforce_config_setting_visibility
 
+# Nutanix: We always want to build a fips compliant version.
+build --define boringssl=fips
+
 test --test_verbose_timeout_warnings
 test --experimental_ui_max_stdouterr_bytes=11712829 #default 1048576
 
@@ -125,6 +128,10 @@ build:asan --copt -D__SANITIZE_ADDRESS__
 build:asan --test_env=ASAN_OPTIONS=handle_abort=1:allow_addr2line=true:check_initialization_order=true:strict_init_order=true:detect_odr_violation=1
 build:asan --test_env=UBSAN_OPTIONS=halt_on_error=true:print_stacktrace=1
 build:asan --test_env=ASAN_SYMBOLIZER_PATH
+# Nutanix specific: Allow alloc_dealloc_mismatch error without failing the test. This error is seen only on Nutanix circle ci setups.
+build:asan --test_env=ASAN_OPTIONS=handle_abort=1:allow_addr2line=true:check_initialization_order=true:strict_init_order=true:detect_odr_violation=1:alloc_dealloc_mismatch=0
+build:asan --test_env=UBSAN_OPTIONS=halt_on_error=true:print_stacktrace=1
+build:asan --test_env=ASAN_SYMBOLIZER_PATH
 # ASAN needs -O1 to get reasonable performance.
 build:asan --copt -O1
 build:asan --copt -fno-optimize-sibling-calls
@@ -143,6 +150,10 @@ build:clang-asan --action_env=ENVOY_UBSAN_VPTR=1
 build:clang-asan --copt=-fsanitize=vptr,function
 build:clang-asan --linkopt=-fsanitize=vptr,function
 
+# Nutanix specific: Configure clang linker to link shared asan libraries.
+build:clang-asan --linkopt=-shared-libasan
+build:clang-asan --linkopt=-shared-libsan
+
 # macOS
 build:macos --cxxopt=-std=c++20 --host_cxxopt=-std=c++20
 build:macos --action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/bin:/usr/bin:/bin

.dockerignore

@@ -5,6 +5,8 @@
 !/configs/google-vrp
 !/configs/*yaml
 !/linux/amd64/release.tar.zst
+!/linux/amd64/build_envoy_release*
+!/linux/amd64/build_envoy_debug*
 !/linux/amd64/schema_validator_tool
 !/linux/amd64/router_check_tool
 !/linux/arm64/release.tar.zst

bazel/external/boringssl_fips.BUILD

@@ -1,25 +1,67 @@
 licenses(["notice"])  # Apache 2
 
+# Nutanix specific: Create separate targets for building static and
+# dynamic libraries. Static library is used to statically linked to
+# envoy binary, while the dynamic library is linked to the test
+# binaries. This saves disk space when multiple unit tests are
+# run together for asan and tsan jobs.
+
 cc_library(
-    name = "crypto",
+    name = "crypto_static",
     srcs = [
         "crypto/libcrypto.a",
     ],
     hdrs = glob(["boringssl/include/openssl/*.h"]),
     defines = ["BORINGSSL_FIPS"],
     includes = ["boringssl/include"],
-    visibility = ["//visibility:public"],
 )
 
 cc_library(
-    name = "ssl",
+    name = "ssl_static",
     srcs = [
         "ssl/libssl.a",
     ],
     hdrs = glob(["boringssl/include/openssl/*.h"]),
     includes = ["boringssl/include"],
+    deps = [":crypto_static"],
+)
+
+cc_library(
+    name = "crypto_dynamic",
+    srcs = [
+        "crypto/libcrypto.so",
+    ],
+    hdrs = glob(["boringssl/include/openssl/*.h"]),
+    defines = ["BORINGSSL_FIPS"],
+    includes = ["boringssl/include"],
+)
+
+cc_library(
+    name = "ssl_dynamic",
+    srcs = [
+        "ssl/libssl.so",
+    ],
+    hdrs = glob(["boringssl/include/openssl/*.h"]),
+    includes = ["boringssl/include"],
+    deps = [":crypto_dynamic"],
+)
+
+alias(
+    name = "ssl",
+    actual = select({
+        "@envoy//bazel:dynamic_link_tests": "@boringssl_fips//:ssl_dynamic",
+        "//conditions:default": "@boringssl_fips//:ssl_static",
+    }),
+    visibility = ["//visibility:public"],
+)
+
+alias(
+    name = "crypto",
+    actual = select({
+        "@envoy//bazel:dynamic_link_tests": "@boringssl_fips//:crypto_dynamic",
+        "//conditions:default": "@boringssl_fips//:crypto_static",
+    }),
     visibility = ["//visibility:public"],
-    deps = [":crypto"],
 )
 
 genrule(

bazel/external/boringssl_fips.genrule_cmd

@@ -33,14 +33,18 @@ export HOME="$PWD"
 export PATH="/usr/bin:/bin"
 
 # Clang
-VERSION=12.0.0
-if [[ "$ARCH" == "x86_64" ]]; then
-  PLATFORM="x86_64-linux-gnu-ubuntu-20.04"
-  SHA256=a9ff205eb0b73ca7c86afc6432eed1c2d49133bd0d49e47b15be59bbf0dd292e
-else
-  PLATFORM="aarch64-linux-gnu"
-  SHA256=d05f0b04fb248ce1e7a61fcd2087e6be8bc4b06b2cc348792f383abf414dec48
-fi
+VERSION=14.0.0
+
+# Nutanix specific: Use Clang for CentOS.
+PLATFORM="x86_64-linux-sles12.4"
+SHA256=78f70cc94c3b6f562455b15cebb63e75571d50c3d488d53d9aa4cd9dded30627
+# if [[ "$ARCH" == "x86_64" ]]; then
+#   PLATFORM="x86_64-linux-gnu-ubuntu-18.04"
+#   SHA256=61582215dafafb7b576ea30cc136be92c877ba1f1c31ddbbd372d6d65622fef5
+# else
+#   PLATFORM="aarch64-linux-gnu"
+#   SHA256=1792badcd44066c79148ffeb1746058422cc9d838462be07e3cb19a4b724a1ee
+# fi
 
 curl -sLO https://github.com/llvm/llvm-project/releases/download/llvmorg-"$VERSION"/clang+llvm-"$VERSION"-"$PLATFORM".tar.xz
 echo "$SHA256" clang+llvm-"$VERSION"-"$PLATFORM".tar.xz | sha256sum --check
@@ -117,19 +121,45 @@ fi
 
 # Clean after previous build.
 rm -rf boringssl/build
+rm -rf boringssl/build_dynamic
+
+# Nutanix specific: Build BoringSSL static library when building the envoy-static release binary and
+# when other times when the flag dynamic_link_tests is not set.
 
 # Build BoringSSL.
 cd boringssl
 # Setting -fPIC only affects the compilation of the non-module code in libcrypto.a,
 # because the FIPS module itself is already built with -fPIC.
 mkdir build && cd build && cmake -GNinja -DCMAKE_TOOLCHAIN_FILE=${HOME}/toolchain -DFIPS=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS="-fPIC" ..
 ninja
+
+# The `HostMatching` test contains hard-coded expired certificates and always fails due to the expiration.
+# This should be removed during the next FIPS version upgrade, as the test code is fixed in later versions.
+export GTEST_FILTER="-SSLTest.HostMatching"
+
+# Nutanix specific: 'URandomTest' requires ptrace which is failing due to
+# PTRACE_TRACEME: Operation not permitted.
+export GTEST_FILTER="$GTEST_FILTER:URandomTest.Test"
+ninja run_tests
+./crypto/crypto_test
+
+# Verify correctness of the FIPS build.
+if [[ `tool/bssl isfips` != "1" ]]; then
+  echo "ERROR: BoringSSL tool didn't report FIPS static build."
+  exit 1
+fi
+
+
+# Nutanix specific: Build dynamic library to be linked when the build flag dynamic_link_tests is set.
+cd ..
+mkdir build_dynamic && cd build_dynamic && cmake -GNinja -DBUILD_SHARED_LIBS=1 -DCMAKE_TOOLCHAIN_FILE=${HOME}/toolchain -DFIPS=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS="-fPIC" ..
+ninja
 ninja run_tests
 ./crypto/crypto_test
 
 # Verify correctness of the FIPS build.
 if [[ `tool/bssl isfips` != "1" ]]; then
-  echo "ERROR: BoringSSL tool didn't report FIPS build."
+  echo "ERROR: BoringSSL tool didn't report FIPS dynamic build."
   exit 1
 fi
 

bazel/repository_locations.bzl

@@ -423,17 +423,17 @@ REPOSITORY_LOCATIONS_SPEC = dict(
     com_github_intel_ipp_crypto_crypto_mb = dict(
         project_name = "libipp-crypto",
         project_desc = "Intel® Integrated Performance Primitives Cryptography",
-        project_url = "https://github.com/intel/ipp-crypto",
+        project_url = "https://github.com/intel/cryptography-primitives",
         version = "2021.11.1",
-        sha256 = "d785fd8d5245ada79068588e5cc4d721d35c50e7d26fc268306f4aaae28ec6d6",
-        strip_prefix = "ipp-crypto-ippcp_{version}",
-        urls = ["https://github.com/intel/ipp-crypto/archive/ippcp_{version}.tar.gz"],
+        sha256 = "76779724378fb3ec8b59047be02f6ca609c1ee70372b0205d937d451987bdb6c",
+        strip_prefix = "cryptography-primitives-ippcp_{version}",
+        urls = ["https://github.com/intel/cryptography-primitives/archive/ippcp_{version}.tar.gz"],
         release_date = "2024-02-28",
         use_category = ["dataplane_ext"],
         extensions = ["envoy.tls.key_providers.cryptomb"],
         cpe = "cpe:2.3:a:intel:cryptography_for_intel_integrated_performance_primitives:*",
         license = "Apache-2.0",
-        license_url = "https://github.com/intel/ipp-crypto/blob/ippcp_{version}/LICENSE",
+        license_url = "https://github.com/intel/cryptography-primitives/blob/ippcp_{version}/LICENSE",
     ),
     com_github_intel_qatlib = dict(
         project_name = "qatlib",

ci/Dockerfile-envoy

@@ -1,107 +1,89 @@
 ARG BUILD_OS=ubuntu
-ARG BUILD_TAG=22.04@sha256:a6d2b38300ce017add71440577d5b0a90460d0e57fd7aec21dd0d1b0761bbfb2
-ARG ENVOY_VRP_BASE_IMAGE=envoy-base
+ARG BUILD_TAG=20.04
+ARG ENVOY_VRP_BASE_IMAGE=envoy
 
 
 FROM scratch AS binary
-COPY ci/docker-entrypoint.sh /
-ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml
-# See https://github.com/docker/buildx/issues/510 for why this _must_ be this way
+
 ARG TARGETPLATFORM
-ENV TARGETPLATFORM="${TARGETPLATFORM:-linux/amd64}"
-ADD "${TARGETPLATFORM}/release.tar.zst" /usr/local/bin/
+ENV TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64}
+ARG ENVOY_BINARY=envoy
+ARG ENVOY_BINARY_SUFFIX=
+ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_debug${ENVOY_BINARY_SUFFIX}/envoy* /usr/local/bin/
+ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml
+COPY ${TARGETPLATFORM}/build_${ENVOY_BINARY}_debug/schema_validator_tool /usr/local/bin/schema_validator_tool
+COPY ci/docker-entrypoint.sh /
+
 
+# STAGE: envoy
+FROM ${BUILD_OS}:${BUILD_TAG} AS envoy
 
-# STAGE: envoy-base
-FROM ${BUILD_OS}:${BUILD_TAG} AS envoy-base
 ENV DEBIAN_FRONTEND=noninteractive
-EXPOSE 10000
-CMD ["envoy", "-c", "/etc/envoy/envoy.yaml"]
-RUN mkdir -p /etc/envoy \
-    && adduser --group --system envoy
-ENTRYPOINT ["/docker-entrypoint.sh"]
-# NB: Adding this here means that following steps, for example updating the system packages, are run
-#   when the version file changes. This should mean that a release version will always update.
-#   In PRs this will just use cached layers unless either this file changes or the version has changed.
-ADD VERSION.txt /etc/envoy
-RUN --mount=type=tmpfs,target=/var/cache/apt \
-    --mount=type=tmpfs,target=/var/lib/apt/lists \
-    apt-get -qq update \
-    && apt-get -qq upgrade -y \
-    && apt-get -qq install --no-install-recommends -y ca-certificates \
-    && apt-get -qq autoremove -y
 
+RUN apt-get update && apt-get upgrade -qq -y \
+    && apt-get install -qq --no-install-recommends -y ca-certificates iproute2 iputils-ping curl wget \
+    && apt-get autoremove -y -qq && apt-get clean \
+    && rm -rf /tmp/* /var/tmp/* \
+    && rm -rf /var/lib/apt/lists/*
 
-# STAGE: envoy
-FROM envoy-base AS envoy
-COPY --from=binary --chown=0:0 --chmod=644 \
-    /etc/envoy/envoy.yaml /etc/envoy/envoy.yaml
-COPY --from=binary --chown=0:0 --chmod=755 \
-    /docker-entrypoint.sh /
-COPY --from=binary --chown=0:0 --chmod=755 \
-    /usr/local/bin/utils/su-exec /usr/local/bin/
-ARG ENVOY_BINARY=envoy
-ARG ENVOY_BINARY_PREFIX=
-COPY --from=binary --chown=0:0 --chmod=755 \
-    "/usr/local/bin/${ENVOY_BINARY_PREFIX}${ENVOY_BINARY}" /usr/local/bin/envoy
-COPY --from=binary --chown=0:0 --chmod=755 \
-    /usr/local/bin/${ENVOY_BINARY_PREFIX}${ENVOY_BINARY}\.* /usr/local/bin/
+RUN mkdir -p /etc/envoy
 
+COPY --from=binary /usr/local/bin/envoy* /usr/local/bin/
+COPY --from=binary /etc/envoy/envoy.yaml /etc/envoy/envoy.yaml
+COPY --from=binary /docker-entrypoint.sh /
 
-# STAGE: envoy-tools
-FROM envoy AS envoy-tools
-# See https://github.com/docker/buildx/issues/510 for why this _must_ be this way
-ARG TARGETPLATFORM
-ENV TARGETPLATFORM="${TARGETPLATFORM:-linux/amd64}"
-COPY --chown=0:0 --chmod=755 \
-    "${TARGETPLATFORM}/schema_validator_tool" "${TARGETPLATFORM}/router_check_tool" /usr/local/bin/
+# RUN adduser --group --system envoy
+
+EXPOSE 10000
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["envoy", "-c", "/etc/envoy/envoy.yaml"]
 
 
 # STAGE: envoy-distroless
-FROM gcr.io/distroless/base-nossl-debian12:nonroot@sha256:8a09e5752fb3ab9c9534fcc627eb1f451cd9bcfe66a6b149df62dcb84fb841a6 AS envoy-distroless
+# gcr.io/distroless/base-nossl-debian11:nonroot
+FROM gcr.io/distroless/base-nossl-debian11:nonroot@sha256:f10e1fbf558c630a4b74a987e6c754d45bf59f9ddcefce090f6b111925996767 AS envoy-distroless
+
+COPY --from=binary /usr/local/bin/envoy* /usr/local/bin/
+COPY --from=binary /etc/envoy/envoy.yaml /etc/envoy/envoy.yaml
+
 EXPOSE 10000
+
 ENTRYPOINT ["/usr/local/bin/envoy"]
 CMD ["-c", "/etc/envoy/envoy.yaml"]
-COPY --from=binary --chown=0:0 --chmod=644 \
-    /etc/envoy/envoy.yaml /etc/envoy/envoy.yaml
-COPY --from=binary --chown=0:0 --chmod=755 \
-    /usr/local/bin/envoy /usr/local/bin/
 
 
-# STAGE: envoy-google-vrp-base
-FROM ${ENVOY_VRP_BASE_IMAGE} AS envoy-google-vrp-base
-EXPOSE 10000
-EXPOSE 10001
-CMD ["supervisord", "-c", "/etc/supervisor.conf"]
-ENTRYPOINT []
-ADD --chown=0:0 --chmod=644 \
-    configs/google-vrp/*.yaml /etc/envoy/
-ADD --chown=0:0 --chmod=755 \
-    configs/google-vrp/launch_envoy.sh /usr/local/bin/launch_envoy.sh
-ADD --chown=0:0 --chmod=644 \
-    test/config/integration/certs/serverkey.pem /etc/envoy/certs/serverkey.pem
-ADD --chown=0:0 --chmod=644 \
-    test/config/integration/certs/servercert.pem /etc/envoy/certs/servercert.pem
-RUN --mount=type=tmpfs,target=/var/cache/apt \
-    --mount=type=tmpfs,target=/var/lib/apt/lists \
-    apt-get -qq update \
-    && apt-get -qq upgrade -y \
-    && apt-get -qq install -y libc++1 supervisor gdb strace tshark \
+# STAGE: envoy-google-vrp
+FROM ${ENVOY_VRP_BASE_IMAGE} AS envoy-google-vrp
+
+RUN apt-get update \
+    && apt-get upgrade -y -qq \
+    && apt-get install -y -qq libc++1 supervisor gdb strace tshark \
     && apt-get autoremove -y \
-    && chmod 777 /var/log/supervisor
-ADD --chown=0:0 --chmod=755 configs/google-vrp/supervisor.conf /etc/supervisor.conf
+    && apt-get clean \
+    && rm -rf /tmp/* /var/tmp/* \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD configs/google-vrp/envoy-edge.yaml /etc/envoy/envoy-edge.yaml
+ADD configs/google-vrp/envoy-origin.yaml /etc/envoy/envoy-origin.yaml
+ADD configs/google-vrp/launch_envoy.sh /usr/local/bin/launch_envoy.sh
+ADD configs/google-vrp/supervisor.conf /etc/supervisor.conf
+ADD test/config/integration/certs/serverkey.pem /etc/envoy/certs/serverkey.pem
+ADD test/config/integration/certs/servercert.pem /etc/envoy/certs/servercert.pem
+# ADD %local envoy bin% /usr/local/bin/envoy
+RUN chmod 777 /var/log/supervisor
+RUN chmod a+r /etc/supervisor.conf /etc/envoy/* /etc/envoy/certs/*
+RUN chmod a+rx /usr/local/bin/launch_envoy.sh
 
+EXPOSE 10000
+EXPOSE 10001
 
-# STAGE: envoy-google-vrp
-FROM envoy-google-vrp-base as envoy-google-vrp
-COPY  --from=binary --chown=0:0 --chmod=755 \
-    /usr/local/bin/envoy /usr/local/bin/envoy
+CMD ["supervisord", "-c", "/etc/supervisor.conf"]
 
+# STAGE: envoy-tools
+FROM ${BUILD_OS}:${BUILD_TAG} AS envoy-tools
 
-# STAGE: envoy-google-vrp-custom
-FROM envoy-google-vrp-base as envoy-google-vrp-custom
-ARG ENVOY_CTX_BINARY_PATH
-ADD "${ENVOY_CTX_BINARY_PATH}" /usr/local/bin/envoy
+COPY --from=binary /usr/local/bin/schema_validator_tool /usr/local/bin/
 
 
 # Make envoy image as last stage so it is built by default