repo-sync-2024-10-22T19:34:07+0800 (#121)

* repo-sync-2024-10-22T19:34:07+0800 * fix build * limit bazel jobs * fix build * fix build
secretflow · Oct 23, 2024 · 6a8af51 · 6a8af51
1 parent f2a6b02
commit 6a8af51
Show file tree

Hide file tree

Showing 36 changed files with 648 additions and 253 deletions.
diff --git a/.ci/inferencer_test.py b/.ci/inferencer_test.py
@@ -16,27 +16,12 @@
 from importlib import resources
 import asyncio
 import os
+import pandas as pd
+import numpy as np
 
 current_file_path = os.path.abspath(__file__)
 code_dir = os.path.dirname(os.path.dirname(current_file_path))
 
-alice_serving_config_file_path = os.path.join(
-    code_dir,
-    "secretflow_serving/tools/inferencer/example/alice/serving.config",
-)
-alice_inference_config_file_path = os.path.join(
-    code_dir,
-    "secretflow_serving/tools/inferencer/example/alice/inference.config",
-)
-bob_serving_config_file_path = os.path.join(
-    code_dir,
-    "secretflow_serving/tools/inferencer/example/bob/serving.config",
-)
-bob_inference_config_file_path = os.path.join(
-    code_dir,
-    "secretflow_serving/tools/inferencer/example/bob/inference.config",
-)
-
 
 async def run_process(command):
     process = await asyncio.create_subprocess_exec(
@@ -53,26 +38,55 @@ async def run_process(command):
         )
 
 
-async def main():
+async def run_inferencer_example(exmaple_dir: str, result_path: str, target_path: str):
+    print(f"====begin example: {exmaple_dir}=====")
+
     with resources.path(
         'secretflow_serving.tools.inferencer', 'inferencer'
     ) as tool_path:
         alice_command = [
             str(tool_path),
-            f'--serving_config_file={alice_serving_config_file_path}',
-            f'--inference_config_file={alice_inference_config_file_path}',
+            f'--serving_config_file={exmaple_dir}/alice/serving.config',
+            f'--inference_config_file={exmaple_dir}/alice/inference.config',
         ]
         bob_command = [
             str(tool_path),
-            f'--serving_config_file={bob_serving_config_file_path}',
-            f'--inference_config_file={bob_inference_config_file_path}',
+            f'--serving_config_file={exmaple_dir}/bob/serving.config',
+            f'--inference_config_file={exmaple_dir}/bob/inference.config',
         ]
         commands = [alice_command, bob_command]
 
     tasks = [run_process(command) for command in commands]
 
     await asyncio.gather(*tasks)
 
+    result_df = pd.read_csv(result_path)
+    target_df = pd.read_csv(target_path)
+
+    score_col = result_df['score']
+    pred_col = target_df['pred']
+
+    assert len(score_col) == len(pred_col)
+
+    are_close = np.isclose(score_col, pred_col, atol=0.0001)
+
+    for i, match in enumerate(are_close):
+        assert match, f"row {i} mismatch: {score_col[i]} != {pred_col[i]}"
+
 
 if __name__ == '__main__':
-    asyncio.run(main())
+    asyncio.run(
+        run_inferencer_example(
+            "secretflow_serving/tools/inferencer/example/normal",
+            "tmp/alice/score.csv",
+            ".ci/test_data/bin_onehot_glm/predict.csv",
+        )
+    )
+
+    asyncio.run(
+        run_inferencer_example(
+            "secretflow_serving/tools/inferencer/example/one_party_no_feature",
+            "tmp/bob/score.csv",
+            ".ci/test_data/bin_onehot_glm_alice_no_feature/predict.csv",
+        )
+    )
diff --git a/.circleci/continue-config.yml b/.circleci/continue-config.yml
@@ -50,6 +50,8 @@ commands:
       - run:
           name: "build"
           command: |
+            export BAZEL_MAX_JOBS=16
+
             sh ./build_wheel_entrypoint.sh
             bazel build <<parameters.bazel_targets>> -c opt --jobs 16
       - run:

diff --git a/.circleci/release-config.yml b/.circleci/release-config.yml
@@ -55,6 +55,7 @@ commands:
             conda create -n build python=<< parameters.python_ver >> -y
             conda activate build
 
+            export BAZEL_MAX_JOBS=16
             sh ./build_wheel_entrypoint.sh
             python3 -m pip install twine
             ls python_lib/dist/*.whl

diff --git a/bazel/ipp.BUILD b/bazel/ipp.BUILD
@@ -0,0 +1,41 @@
+# Copyright 2024 Ant Group Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@yacl//bazel:yacl.bzl", "yacl_cmake_external")
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "all_srcs",
+    srcs = glob(["**"]),
+)
+
+yacl_cmake_external(
+    name = "ipp",
+    cache_entries = {
+        "ARCH": "intel64",
+        "OPENSSL_INCLUDE_DIR": "$EXT_BUILD_DEPS/openssl/include",
+        "OPENSSL_LIBRARIES": "$EXT_BUILD_DEPS/openssl/lib",
+        "OPENSSL_ROOT_DIR": "$EXT_BUILD_DEPS/openssl",
+        "CMAKE_BUILD_TYPE": "Release",
+    },
+    lib_source = ":all_srcs",
+    out_static_libs = [
+        "intel64/libippcp.a",
+        "intel64/libcrypto_mb.a",
+    ],
+    deps = [
+        "@com_github_openssl_openssl//:openssl",
+    ],
+)
diff --git a/bazel/patches/ippcp.patch b/bazel/patches/ippcp.patch
@@ -0,0 +1,172 @@
+diff --git a/sources/cmake/linux/GNU8.2.0.cmake b/sources/cmake/linux/GNU8.2.0.cmake
+index 24d7e0f..15dd433 100644
+--- a/sources/cmake/linux/GNU8.2.0.cmake
++++ b/sources/cmake/linux/GNU8.2.0.cmake
+@@ -32,7 +32,7 @@ set(LINK_FLAG_DYNAMIC_LINUX "${LINK_FLAG_SECURITY} -nostdlib")
+ # Dynamically link lib c (libdl is for old apps)
+ set(LINK_FLAG_DYNAMIC_LINUX "${LINK_FLAG_DYNAMIC_LINUX} -Wl,-call_shared,-lc")
+ # Create a shared library
+-set(LINK_FLAG_DYNAMIC_LINUX "-Wl,-shared")
++set(LINK_FLAG_DYNAMIC_LINUX "-Wl,-shared,-fuse-ld=bfd")
+ if(${ARCH} MATCHES "ia32")
+   # Tells the compiler to generate code for a specific architecture (32)
+   set(LINK_FLAG_DYNAMIC_LINUX "${LINK_FLAG_DYNAMIC_LINUX} -m32")
+@@ -74,7 +74,7 @@ if ((${ARCH} MATCHES "ia32") OR (NOT NONPIC_LIB))
+ endif()
+
+ # Security flag that adds compile-time and run-time checks
+-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FORTIFY_SOURCE=2")
++set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2")
+
+ if(NOT NONPIC_LIB)
+   # Position Independent Execution (PIE)
+@@ -95,6 +95,8 @@ if(${ARCH} MATCHES "ia32")
+   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
+ endif(${ARCH} MATCHES "ia32")
+
++set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unknown-pragmas -Wno-missing-braces -Wno-comment -Wno-strict-aliasing -Wno-parentheses -Wno-array-parameter")
++
+ # Optimization level = 3, no-debug definition (turns off asserts), warnings=errors
+ set (CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG -Werror")
+
+diff --git a/sources/cmake/macosx/AppleClang11.0.0.cmake b/sources/cmake/macosx/AppleClang11.0.0.cmake
+index 5b92877..ccb963e 100644
+--- a/sources/cmake/macosx/AppleClang11.0.0.cmake
++++ b/sources/cmake/macosx/AppleClang11.0.0.cmake
+@@ -20,12 +20,6 @@
+
+ # Security Linker flags
+ set(LINK_FLAG_SECURITY "")
+-# Disallows undefined symbols in object files. Undefined symbols in shared libraries are still allowed
+-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,defs")
+-# Stack execution protection
+-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,noexecstack")
+-# Data relocation and protection (RELRO)
+-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,relro -Wl,-z,now")
+ # Prevents the compiler from using standard libraries and startup files when linking.
+ set(LINK_FLAG_DYNAMIC_MACOSX "${LINK_FLAG_SECURITY} -nostdlib")
+ # Dynamically link lib c (libdl is for old apps)
+@@ -79,7 +73,7 @@ if ((${ARCH} MATCHES "ia32") OR (NOT NONPIC_LIB))
+ endif()
+
+ # Security flag that adds compile-time and run-time checks
+-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FORTIFY_SOURCE=2")
++set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2")
+
+ if(NOT NONPIC_LIB)
+   # Position Independent Execution (PIE)
+@@ -98,6 +92,8 @@ if(${ARCH} MATCHES "ia32")
+   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
+ endif(${ARCH} MATCHES "ia32")
+
++set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unknown-pragmas -Wno-missing-braces -Wno-unused-command-line-argument -Wno-unused-but-set-variable -Wno-unknown-warning-option")
++
+ # Optimization level = 3, no-debug definition (turns off asserts), warnings=errors
+ set (CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG -Werror")
+
+@@ -115,3 +111,5 @@ set(l9_opt "${l9_opt} -march=haswell -mavx2 -maes -mpclmul -msha -mrdrnd -mrdsee
+ set(n0_opt "${n0_opt} -march=knl -mavx2 -maes -mavx512f -mavx512cd -mavx512pf -mavx512er -mpclmul -msha -mrdrnd -mrdseed")
+ set(k0_opt "${k0_opt} -march=skylake-avx512")
+ set(k0_opt "${k0_opt} -maes -mavx512f -mavx512cd -mavx512vl -mavx512bw -mavx512dq -mavx512ifma -mpclmul -msha -mrdrnd -mrdseed -madx -mgfni -mvaes -mvpclmulqdq -mavx512vbmi -mavx512vbmi2")
++set(k1_opt "${k1_opt} -march=skylake-avx512")
++set(k1_opt "${k1_opt} -maes -mavx512f -mavx512cd -mavx512vl -mavx512bw -mavx512dq -mavx512ifma -mpclmul -msha -mrdrnd -mrdseed -madx -mgfni -mvaes -mvpclmulqdq -mavx512vbmi -mavx512vbmi2")
+diff --git a/sources/cmake/macosx/common.cmake b/sources/cmake/macosx/common.cmake
+index 85ec3ad..67bb9f9 100644
+--- a/sources/cmake/macosx/common.cmake
++++ b/sources/cmake/macosx/common.cmake
+@@ -18,7 +18,7 @@
+ # Intel® Integrated Performance Primitives Cryptography (Intel® IPP Cryptography)
+ #
+
+-set(OS_DEFAULT_COMPILER Intel19.0.0)
++set(OS_DEFAULT_COMPILER AppleClang11.0.0)
+
+ set(LIBRARY_DEFINES "${LIBRARY_DEFINES} -DIPP_PIC -DOSXEM64T -DLINUX32E -D_ARCH_EM64T")
+ #set(LIBRARY_DEFINES "${LIBRARY_DEFINES} -DBN_OPENSSL_DISABLE")
+\ No newline at end of file
+diff --git a/sources/ippcp/crypto_mb/src/cmake/linux/GNU.cmake b/sources/ippcp/crypto_mb/src/cmake/linux/GNU.cmake
+index a2abeeb..67aca8b 100644
+--- a/sources/ippcp/crypto_mb/src/cmake/linux/GNU.cmake
++++ b/sources/ippcp/crypto_mb/src/cmake/linux/GNU.cmake
+@@ -31,7 +31,7 @@ set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -Wformat -Wformat-security
+ if(${CMAKE_BUILD_TYPE} STREQUAL "Release")
+     if(NOT DEFINED NO_FORTIFY_SOURCE)
+         # Security flag that adds compile-time and run-time checks. 
+-        set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -D_FORTIFY_SOURCE=2")
++        set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2")
+     endif()
+ endif()
+
+@@ -51,7 +51,7 @@ set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -Werror")
+ # Linker flags
+
+ # Create shared library
+-set(LINK_FLAGS_DYNAMIC " -Wl,-shared")
++set(LINK_FLAGS_DYNAMIC " -Wl,-shared,-fuse-ld=bfd")
+ # Add export files
+ set(LINK_FLAGS_DYNAMIC "${LINK_FLAGS_DYNAMIC} ${CRYPTO_MB_SOURCES_DIR}/cmake/dll_export/crypto_mb.linux.lib-export")
+
+@@ -69,6 +69,7 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+
+ # Suppress warnings from casts from a pointer to an integer type of a different size
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-pointer-to-int-cast")
++set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unknown-pragmas -Wno-missing-braces -Wno-comment -Wno-strict-aliasing -Wno-parentheses")
+
+ # Optimization level = 3, no-debug definition (turns off asserts)
+ set(CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG")
+diff --git a/sources/ippcp/crypto_mb/src/cmake/macosx/AppleClang.cmake b/sources/ippcp/crypto_mb/src/cmake/macosx/AppleClang.cmake
+index ea1641d..f98fc2d 100644
+--- a/sources/ippcp/crypto_mb/src/cmake/macosx/AppleClang.cmake
++++ b/sources/ippcp/crypto_mb/src/cmake/macosx/AppleClang.cmake
+@@ -17,10 +17,6 @@
+ # Security Linker flags
+
+ set(LINK_FLAG_SECURITY "") 
+-# Data relocation and protection (RELRO)
+-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,relro -Wl,-z,now")
+-# Stack execution protection
+-set(LINK_FLAG_SECURITY "${LINK_FLAG_SECURITY} -Wl,-z,noexecstack")
+
+ # Security Compiler flags
+
+@@ -30,7 +26,7 @@ set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -Wformat -Wformat-security
+
+ if(${CMAKE_BUILD_TYPE} STREQUAL "Release")
+     # Security flag that adds compile-time and run-time checks. 
+-    set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -D_FORTIFY_SOURCE=2")
++    set(CMAKE_C_FLAGS_SECURITY "${CMAKE_C_FLAGS_SECURITY} -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2")
+ endif()
+
+ # Stack-based Buffer Overrun Detection
+@@ -65,6 +61,8 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+ # Suppress warnings from casts from a pointer to an integer type of a different size
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-pointer-to-int-cast")
+
++set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unknown-pragmas -Wno-missing-braces -Wno-unknown-warning-option")
++
+ # Optimization level = 3, no-debug definition (turns off asserts)
+ set(CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
+
+diff --git a/sources/cmake/linux/Clang9.0.0.cmake b/sources/cmake/linux/Clang9.0.0.cmake
+index 0015431..f93411c 100644
+--- a/sources/cmake/linux/Clang9.0.0.cmake
++++ b/sources/cmake/linux/Clang9.0.0.cmake
+@@ -79,7 +79,7 @@ endif()
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fcf-protection=full")
+
+ # Security flag that adds compile-time and run-time checks
+-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FORTIFY_SOURCE=2")
++set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+
+ if(NOT NONPIC_LIB)
+   # Position Independent Execution (PIE)
+@@ -107,7 +107,7 @@ if(SANITIZERS)
+ endif(SANITIZERS)
+
+ # Optimization level = 3, no-debug definition (turns off asserts), warnings=errors
+-set (CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG -Werror")
++set (CMAKE_C_FLAGS_RELEASE " -O3 -DNDEBUG -Werror -Wno-unused-function -Wno-missing-braces -Wno-unused-but-set-variable -Wno-unknown-pragmas")
+
+ # DEBUG flags - optimization level = 0, generation GDB information (-g)
+ set (CMAKE_C_FLAGS_DEBUG " -O0 -g")
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -33,6 +33,8 @@ def sf_serving_deps():
     _com_aws_checksums()
     _com_aws_sdk()
 
+    _com_github_intel_ipp()
+
     _heu()
     _dataproxy()
 
@@ -209,3 +211,19 @@ def _com_github_brpc_brpc():
             "https://github.com/apache/brpc/archive/refs/tags/1.9.0.tar.gz",
         ],
     )
+
+def _com_github_intel_ipp():
+    maybe(
+        http_archive,
+        name = "com_github_intel_ipp",
+        sha256 = "d70f42832337775edb022ca8ac1ac418f272e791ec147778ef7942aede414cdc",
+        strip_prefix = "cryptography-primitives-ippcp_2021.8",
+        build_file = "@sf_serving//bazel:ipp.BUILD",
+        patch_args = ["-p1"],
+        patches = [
+            "@sf_serving//bazel:patches/ippcp.patch",
+        ],
+        urls = [
+            "https://github.com/intel/ipp-crypto/archive/refs/tags/ippcp_2021.8.tar.gz",
+        ],
+    )