Make Fedora/UBI build working (#1)

* Fix MLMD build on Fedora

Signed-off-by: Andrea Lamparelli <[email protected]>

* Revert zetasql upgrade and switch to ubi8 (#2)

---------

Signed-off-by: Andrea Lamparelli <[email protected]>

.bazelrc

@@ -14,7 +14,7 @@
 
 # This macro is needed in order for mlmd to build with ZetaSQL which can only
 # be compiled upon c++17 or higher.
-build --cxxopt="-std=c++17"
+build --cxxopt="-std=c++17" --host_cxxopt="-std=c++17"
 
 # Needed to avoid zetasql proto error.
 build --protocopt=--experimental_allow_proto3_optional

.dockerignore

@@ -0,0 +1 @@
+.git

.gitignore

@@ -0,0 +1,2 @@
+# Ignore bazel target dirs
+bazel-*

WORKSPACE

@@ -43,17 +43,19 @@ rules_foreign_cc_dependencies()
 
 http_archive(
     name = "com_google_absl",
-    urls = ["https://github.com/abseil/abseil-cpp/archive/940c06c25d2953f44310b68eb8aab6114dba11fb.zip"],
-    strip_prefix = "abseil-cpp-940c06c25d2953f44310b68eb8aab6114dba11fb",
-    sha256 = "0e800799aa64d0b4d354f3ff317bbd5fbf42f3a522ab0456bb749fc8d3b67415",
+    sha256 = "59d2976af9d6ecf001a81a35749a6e551a335b949d34918cfade07737b9d93c5",
+    strip_prefix = "abseil-cpp-20230802.0",
+    urls = [
+        "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz"
+    ],
 )
 
 http_archive(
     name = "boringssl",
-    sha256 = "1188e29000013ed6517168600fc35a010d58c5d321846d6a6dfee74e4c788b45",
-    strip_prefix = "boringssl-7f634429a04abc48e2eb041c81c5235816c96514",
+    sha256 = "f69738ca17f1dd30ae3ddb1fa7519245044737d27c8a3defa7a94718d9dfd724",
+    strip_prefix = "boringssl-68dcc7f7b816e199c8f373ea0a2d6a4e1f526e2d",
     urls = [
-        "https://github.com/google/boringssl/archive/7f634429a04abc48e2eb041c81c5235816c96514.tar.gz",
+        "https://github.com/google/boringssl/archive/68dcc7f7b816e199c8f373ea0a2d6a4e1f526e2d.tar.gz",
     ],
 )
 
@@ -122,9 +124,9 @@ protobuf_deps()
 http_archive(
     name = "zlib",
     build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
-    sha256 = "d8688496ea40fb61787500e863cc63c9afcbc524468cedeb478068924eb54932",
-    strip_prefix = "zlib-1.2.12",
-    urls = ["https://github.com/madler/zlib/archive/v1.2.12.tar.gz"],
+    sha256 = "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e",
+    strip_prefix = "zlib-1.3",
+    urls = ["https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz"],
 )
 
 http_archive(
@@ -248,6 +250,8 @@ http_archive(
     url = "https://github.com/gflags/gflags/archive/a738fdf9338412f83ab3f26f31ac11ed3f3ec4bd.zip",
 )
 
+# TODO: can we import the proper zetasql version based on some config provided in the build command?
+# Required for Fedora39 and ubi8
 ZETASQL_COMMIT = "ac37cf5c0d80b5605176fc0f29e87b12f00be693" # 08/10/2022
 http_archive(
     name = "com_google_zetasql",
@@ -257,6 +261,16 @@ http_archive(
     sha256 = '651a768cd51627f58aa6de7039aba9ddab22f4b0450521169800555269447840'
 )
 
+# Required for Fedora:38 and ubi9
+# ZETASQL_COMMIT = "f764f4e986ac1516ab5ae95e6d6ce2f4416cc6ff" # 02/03/2023
+# http_archive(
+#     name = "com_google_zetasql",
+#     urls = ["https://github.com/google/zetasql/archive/%s.zip" % ZETASQL_COMMIT],
+#     strip_prefix = "zetasql-%s" % ZETASQL_COMMIT,
+#     #patches = ["//ml_metadata/third_party:zetasql.patch"],
+#     sha256 = '27e3d8bfd1f76918fc4d7a8f29646b8a0cdca567f921e0bff4a07f79448e92c0'
+# )
+
 load("@com_google_zetasql//bazel:zetasql_deps_step_1.bzl", "zetasql_deps_step_1")
 zetasql_deps_step_1()
 load("@com_google_zetasql//bazel:zetasql_deps_step_2.bzl", "zetasql_deps_step_2")

ml_metadata/tools/docker_server/Dockerfile.fedora

@@ -0,0 +1,61 @@
+FROM fedora:38 as builder
+
+RUN dnf update -y && \
+  dnf install -y \
+  which \
+  patch \
+  gcc \
+  clang \
+  cmake \
+  make \
+  musl-devel \
+  openssl \
+  curl \
+  ca-certificates \
+  unzip \
+  git \
+  python-is-python3 \
+  python3-devel
+
+# Set up Bazel 5.3.0
+ENV BAZEL_VERSION 5.3.0
+WORKDIR /
+RUN mkdir /bazel && \
+    cd /bazel && \
+    curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
+    curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \
+    chmod +x bazel-*.sh && \
+    ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
+    cd / && \
+    rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
+
+ADD . /mlmd-src
+WORKDIR /mlmd-src
+
+# "-std=c++17" is needed in order to build with ZetaSQL.
+RUN bazel build -c opt --action_env=PATH \
+  --define=grpc_no_ares=true \
+  //ml_metadata/metadata_store:metadata_store_server \
+  --cxxopt="-std=c++17" --host_cxxopt="-std=c++17"
+
+# copying libmysqlclient source onto THIRD_PARTY folder.
+RUN mkdir -p /mlmd-src/third_party
+RUN cp -RL /mlmd-src/bazel-mlmd-src/external/libmysqlclient /mlmd-src/third_party/mariadb-connector-c
+
+FROM fedora:38
+
+COPY --from=builder /mlmd-src/bazel-bin/ml_metadata/metadata_store/metadata_store_server /bin/metadata_store_server
+COPY --from=builder /mlmd-src/third_party /mlmd-src/third_party
+
+ENV GRPC_PORT "8080"
+ENV METADATA_STORE_SERVER_CONFIG_FILE ""
+
+# Introduces tzdata package here to avoid LoadTimeZone check failed error in the metadata store server.
+RUN dnf update -y && \
+  dnf install -y \
+  tzdata
+
+ENTRYPOINT \
+  "/bin/metadata_store_server" \
+  "--grpc_port=${GRPC_PORT}" \
+  "--metadata_store_server_config_file=${METADATA_STORE_SERVER_CONFIG_FILE}"

ml_metadata/tools/docker_server/Dockerfile.redhat

@@ -0,0 +1,64 @@
+FROM registry.access.redhat.com/ubi8/ubi:8.9 as builder
+
+USER root
+
+RUN dnf update -y -q && \
+  dnf install -y -q \
+  which \
+  patch \
+  gcc \
+  clang \
+  cmake \
+  make \
+  openssl \
+  ca-certificates \
+  unzip \
+  git \
+  findutils \
+  python3
+
+# Set up Bazel 5.3.0
+ENV BAZEL_VERSION 5.3.0
+WORKDIR /
+RUN mkdir /bazel && \
+    cd /bazel && \
+    curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
+    curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \
+    chmod +x bazel-*.sh && \
+    ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
+    cd / && \
+    rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
+
+
+COPY . /mlmd-src
+WORKDIR /mlmd-src
+
+# Running in offline mode with --nofetch arg, cache and deps must be cloned 
+# into the local root bazel cache
+# "-std=c++17" is needed in order to build with ZetaSQL.
+RUN bazel build -c opt --action_env=PATH \
+  --define=grpc_no_ares=true \
+  //ml_metadata/metadata_store:metadata_store_server \
+  --cxxopt="-std=c++17" --host_cxxopt="-std=c++17"
+
+# copying libmysqlclient source onto THIRD_PARTY folder.
+RUN mkdir -p /mlmd-src/third_party
+RUN cp -RL /mlmd-src/bazel-mlmd-src/external/libmysqlclient /mlmd-src/third_party/mariadb-connector-c
+
+FROM registry.redhat.io/ubi8/ubi-minimal:8.9
+
+COPY --from=builder /mlmd-src/bazel-bin/ml_metadata/metadata_store/metadata_store_server /bin/metadata_store_server
+COPY --from=builder /mlmd-src/third_party /mlmd-src/third_party
+
+ENV GRPC_PORT "8080"
+ENV METADATA_STORE_SERVER_CONFIG_FILE ""
+
+# Introduces tzdata package here to avoid LoadTimeZone check failed error in the metadata store server.
+RUN microdnf update -y && \
+  microdnf install -y \
+  tzdata
+
+ENTRYPOINT \
+  "/bin/metadata_store_server" \
+  "--grpc_port=${GRPC_PORT}" \
+  "--metadata_store_server_config_file=${METADATA_STORE_SERVER_CONFIG_FILE}"

ml_metadata/tools/docker_server/README.md

@@ -1,6 +1,22 @@
 Files for building the [Docker](http://www.docker.com) image for running the
 MLMD gRPC server.
 
-To build a docker image, run
+### Ubuntu
+
+To build a docker image from Ubuntu, run
 `./ml_metadata/tools/docker_server/build_docker_image.sh` under root directory
 of github checkout.
+
+### Fedora 38 and UBI9
+
+To build from Fedora image, run
+`DOCKER_FILE=Dockerfile.fedora ./ml_metadata/tools/docker_server/build_docker_image.sh` under root directory
+
+Assure the `ZETASQL_COMMIT` == `f764f4e986ac1516ab5ae95e6d6ce2f4416cc6ff` in WORKSPACE file
+
+### Fedora 39 and UBI8
+
+To build from RedHat UBI image, run
+`DOCKER_FILE=Dockerfile.redhat ./ml_metadata/tools/docker_server/build_docker_image.sh` under root directory
+
+Assure the `ZETASQL_COMMIT` == `ac37cf5c0d80b5605176fc0f29e87b12f00be693` in WORKSPACE file