diff --git a/.bazelrc b/.bazelrc index acfc7bdda..f1fcb34fe 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,7 +14,7 @@ # This macro is needed in order for mlmd to build with ZetaSQL which can only # be compiled upon c++17 or higher. -build --cxxopt="-std=c++17" +build --cxxopt="-std=c++17" --host_cxxopt="-std=c++17" # Needed to avoid zetasql proto error. build --protocopt=--experimental_allow_proto3_optional diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..6b8710a71 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.git diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..065916c4f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# Ignore bazel target dirs +bazel-* \ No newline at end of file diff --git a/WORKSPACE b/WORKSPACE index dbb9951bd..1f99e3de7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -43,17 +43,19 @@ rules_foreign_cc_dependencies() http_archive( name = "com_google_absl", - urls = ["https://github.com/abseil/abseil-cpp/archive/940c06c25d2953f44310b68eb8aab6114dba11fb.zip"], - strip_prefix = "abseil-cpp-940c06c25d2953f44310b68eb8aab6114dba11fb", - sha256 = "0e800799aa64d0b4d354f3ff317bbd5fbf42f3a522ab0456bb749fc8d3b67415", + sha256 = "59d2976af9d6ecf001a81a35749a6e551a335b949d34918cfade07737b9d93c5", + strip_prefix = "abseil-cpp-20230802.0", + urls = [ + "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz" + ], ) http_archive( name = "boringssl", - sha256 = "1188e29000013ed6517168600fc35a010d58c5d321846d6a6dfee74e4c788b45", - strip_prefix = "boringssl-7f634429a04abc48e2eb041c81c5235816c96514", + sha256 = "f69738ca17f1dd30ae3ddb1fa7519245044737d27c8a3defa7a94718d9dfd724", + strip_prefix = "boringssl-68dcc7f7b816e199c8f373ea0a2d6a4e1f526e2d", urls = [ - "https://github.com/google/boringssl/archive/7f634429a04abc48e2eb041c81c5235816c96514.tar.gz", + "https://github.com/google/boringssl/archive/68dcc7f7b816e199c8f373ea0a2d6a4e1f526e2d.tar.gz", ], ) @@ -122,9 +124,9 @@ protobuf_deps() http_archive( name = "zlib", build_file = "@com_google_protobuf//:third_party/zlib.BUILD", - sha256 = "d8688496ea40fb61787500e863cc63c9afcbc524468cedeb478068924eb54932", - strip_prefix = "zlib-1.2.12", - urls = ["https://github.com/madler/zlib/archive/v1.2.12.tar.gz"], + sha256 = "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e", + strip_prefix = "zlib-1.3", + urls = ["https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz"], ) http_archive( @@ -248,6 +250,8 @@ http_archive( url = "https://github.com/gflags/gflags/archive/a738fdf9338412f83ab3f26f31ac11ed3f3ec4bd.zip", ) +# TODO: can we import the proper zetasql version based on some config provided in the build command? +# Required for Fedora39 and ubi8 ZETASQL_COMMIT = "ac37cf5c0d80b5605176fc0f29e87b12f00be693" # 08/10/2022 http_archive( name = "com_google_zetasql", @@ -257,6 +261,16 @@ http_archive( sha256 = '651a768cd51627f58aa6de7039aba9ddab22f4b0450521169800555269447840' ) +# Required for Fedora:38 and ubi9 +# ZETASQL_COMMIT = "f764f4e986ac1516ab5ae95e6d6ce2f4416cc6ff" # 02/03/2023 +# http_archive( +# name = "com_google_zetasql", +# urls = ["https://github.com/google/zetasql/archive/%s.zip" % ZETASQL_COMMIT], +# strip_prefix = "zetasql-%s" % ZETASQL_COMMIT, +# #patches = ["//ml_metadata/third_party:zetasql.patch"], +# sha256 = '27e3d8bfd1f76918fc4d7a8f29646b8a0cdca567f921e0bff4a07f79448e92c0' +# ) + load("@com_google_zetasql//bazel:zetasql_deps_step_1.bzl", "zetasql_deps_step_1") zetasql_deps_step_1() load("@com_google_zetasql//bazel:zetasql_deps_step_2.bzl", "zetasql_deps_step_2") diff --git a/ml_metadata/tools/docker_server/Dockerfile.fedora b/ml_metadata/tools/docker_server/Dockerfile.fedora new file mode 100644 index 000000000..98dc588a8 --- /dev/null +++ b/ml_metadata/tools/docker_server/Dockerfile.fedora @@ -0,0 +1,61 @@ +FROM fedora:38 as builder + +RUN dnf update -y && \ + dnf install -y \ + which \ + patch \ + gcc \ + clang \ + cmake \ + make \ + musl-devel \ + openssl \ + curl \ + ca-certificates \ + unzip \ + git \ + python-is-python3 \ + python3-devel + +# Set up Bazel 5.3.0 +ENV BAZEL_VERSION 5.3.0 +WORKDIR / +RUN mkdir /bazel && \ + cd /bazel && \ + curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ + curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ + chmod +x bazel-*.sh && \ + ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ + cd / && \ + rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh + +ADD . /mlmd-src +WORKDIR /mlmd-src + +# "-std=c++17" is needed in order to build with ZetaSQL. +RUN bazel build -c opt --action_env=PATH \ + --define=grpc_no_ares=true \ + //ml_metadata/metadata_store:metadata_store_server \ + --cxxopt="-std=c++17" --host_cxxopt="-std=c++17" + +# copying libmysqlclient source onto THIRD_PARTY folder. +RUN mkdir -p /mlmd-src/third_party +RUN cp -RL /mlmd-src/bazel-mlmd-src/external/libmysqlclient /mlmd-src/third_party/mariadb-connector-c + +FROM fedora:38 + +COPY --from=builder /mlmd-src/bazel-bin/ml_metadata/metadata_store/metadata_store_server /bin/metadata_store_server +COPY --from=builder /mlmd-src/third_party /mlmd-src/third_party + +ENV GRPC_PORT "8080" +ENV METADATA_STORE_SERVER_CONFIG_FILE "" + +# Introduces tzdata package here to avoid LoadTimeZone check failed error in the metadata store server. +RUN dnf update -y && \ + dnf install -y \ + tzdata + +ENTRYPOINT \ + "/bin/metadata_store_server" \ + "--grpc_port=${GRPC_PORT}" \ + "--metadata_store_server_config_file=${METADATA_STORE_SERVER_CONFIG_FILE}" diff --git a/ml_metadata/tools/docker_server/Dockerfile.redhat b/ml_metadata/tools/docker_server/Dockerfile.redhat new file mode 100644 index 000000000..3b2298c4c --- /dev/null +++ b/ml_metadata/tools/docker_server/Dockerfile.redhat @@ -0,0 +1,64 @@ +FROM registry.access.redhat.com/ubi8/ubi:8.9 as builder + +USER root + +RUN dnf update -y -q && \ + dnf install -y -q \ + which \ + patch \ + gcc \ + clang \ + cmake \ + make \ + openssl \ + ca-certificates \ + unzip \ + git \ + findutils \ + python3 + +# Set up Bazel 5.3.0 +ENV BAZEL_VERSION 5.3.0 +WORKDIR / +RUN mkdir /bazel && \ + cd /bazel && \ + curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ + curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ + chmod +x bazel-*.sh && \ + ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ + cd / && \ + rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh + + +COPY . /mlmd-src +WORKDIR /mlmd-src + +# Running in offline mode with --nofetch arg, cache and deps must be cloned +# into the local root bazel cache +# "-std=c++17" is needed in order to build with ZetaSQL. +RUN bazel build -c opt --action_env=PATH \ + --define=grpc_no_ares=true \ + //ml_metadata/metadata_store:metadata_store_server \ + --cxxopt="-std=c++17" --host_cxxopt="-std=c++17" + +# copying libmysqlclient source onto THIRD_PARTY folder. +RUN mkdir -p /mlmd-src/third_party +RUN cp -RL /mlmd-src/bazel-mlmd-src/external/libmysqlclient /mlmd-src/third_party/mariadb-connector-c + +FROM registry.redhat.io/ubi8/ubi-minimal:8.9 + +COPY --from=builder /mlmd-src/bazel-bin/ml_metadata/metadata_store/metadata_store_server /bin/metadata_store_server +COPY --from=builder /mlmd-src/third_party /mlmd-src/third_party + +ENV GRPC_PORT "8080" +ENV METADATA_STORE_SERVER_CONFIG_FILE "" + +# Introduces tzdata package here to avoid LoadTimeZone check failed error in the metadata store server. +RUN microdnf update -y && \ + microdnf install -y \ + tzdata + +ENTRYPOINT \ + "/bin/metadata_store_server" \ + "--grpc_port=${GRPC_PORT}" \ + "--metadata_store_server_config_file=${METADATA_STORE_SERVER_CONFIG_FILE}" diff --git a/ml_metadata/tools/docker_server/README.md b/ml_metadata/tools/docker_server/README.md index 2f465434f..3872a07e7 100644 --- a/ml_metadata/tools/docker_server/README.md +++ b/ml_metadata/tools/docker_server/README.md @@ -1,6 +1,22 @@ Files for building the [Docker](http://www.docker.com) image for running the MLMD gRPC server. -To build a docker image, run +### Ubuntu + +To build a docker image from Ubuntu, run `./ml_metadata/tools/docker_server/build_docker_image.sh` under root directory of github checkout. + +### Fedora 38 and UBI9 + +To build from Fedora image, run +`DOCKER_FILE=Dockerfile.fedora ./ml_metadata/tools/docker_server/build_docker_image.sh` under root directory + +Assure the `ZETASQL_COMMIT` == `f764f4e986ac1516ab5ae95e6d6ce2f4416cc6ff` in WORKSPACE file + +### Fedora 39 and UBI8 + +To build from RedHat UBI image, run +`DOCKER_FILE=Dockerfile.redhat ./ml_metadata/tools/docker_server/build_docker_image.sh` under root directory + +Assure the `ZETASQL_COMMIT` == `ac37cf5c0d80b5605176fc0f29e87b12f00be693` in WORKSPACE file