build.sh

#!/bin/sh

# Periodically check the valgrind results on the build server by going to:
# https://dev.azure.com/ms/interpret/_build?definitionId=293&_a=summary
# By clicking on the build of interest, then "Test ebm_native Linux", then "View raw log"
# We normally have 2 "still reachable" blocks from the Testing executable.
#
# We run the clang-tidy and Visual Studio static analysis tools on the build server.  Warnings do not stop the build, 
# so these need to be inspected to catch static analysis issues.  The results can be viewed in the build logs here:
# https://dev.azure.com/ms/interpret/_build?definitionId=293&_a=summary
# By clicking on the build of interest, then "Build ebm_native Windows", then "View raw log"
#
# Ideally we'd prefer to have static analysis running on all OSes, but we can probably just rely on our
# build system to handle this aspect since adding it in multiple places adds complexity to this build script.
# Also, I ran into issues when I first tried it, which to me suggests these might introduce periodic issues:
#   - on Mac, clang-tidy doesn't seem to come by default in the OS.  You are suposed to 
#     "brew reinstall llvm", but I got a message that llvm was part of the OS and it suggested 
#     that upgrading was a very bad idea.  You could also compile it from scratch, but this seems
#     to me like it would complicate this build script too much for the benefit
#   - on Linux, I was able to get clang-tidy to work by using "sudo apt-get -y install clang clang-tidy"
#     but this requires installing clang and clang-tidy.  I have a better solution using Visual Studio
#   - on Ubuntu, "sudo apt-get -y install cppcheck" seems to hang my build machines, so that sucks.
#   - Visual Studio now includes support for both it's own static analysis tool and clang-tidy.  This seems to
#     be the easiest ways to access these tools for us since they require no additional installation.
#   - by adding "/p:EnableClangTidyCodeAnalysis=True /p:RunCodeAnalysis=True" to MSBuild I can get the static
#     analysis tools to run on the build system, but they won't run in typical builds in Visual Studio, which
#     would slow down our builds.
#   - If you want to enable these static checks on build in Visual Studio, go to:
#     "Solution Explorer" -> right click the project "ebm_native" -> "Properties" -> "Code Analysis"
#     From there you can enable "Clang-Tidy" and "Enable Code Analysis on Build"
#   - You also for free see the Visual Studio static analysis in the "Error List" window if you have
#     "Build + IntelliSense" selected in the drop down window with that option.
#   - any static analysis warnings don't kill the build it seems.  That's good since static analysis tool warnings
#     constantly change, so we probably don't want to turn them into errors otherwise it'll constantly be breaking.
#   - https://include-what-you-use.org/ is alpha, and it looks like it changes a lot.  Doesn't seem worth the benefit.
#   - NOTE: scan-build and clang-tidy are really the same thing, but with different interfaces


# TODO also build our html resources here, and also in the .bat file for Windows

clang_pp_bin=clang++
g_pp_bin=g++
os_type=`uname`
root_path=`dirname "$0"`
src_path="$root_path/shared/ebm_native"
python_lib="$root_path/python/interpret-core/interpret/lib"
staging_path="$root_path/staging"

build_32_bit=0
for arg in "$@"; do
   if [ "$arg" = "-32bit" ]; then
      build_32_bit=1
   fi
done

# re-enable these warnings when they are better supported by g++ or clang: -Wduplicated-cond -Wduplicated-branches -Wrestrict
compile_all=""
compile_all="$compile_all \"$src_path/ApplyModelUpdate.cpp\""
compile_all="$compile_all \"$src_path/ApplyModelUpdateTraining.cpp\""
compile_all="$compile_all \"$src_path/ApplyModelUpdateValidation.cpp\""
compile_all="$compile_all \"$src_path/BinBoosting.cpp\""
compile_all="$compile_all \"$src_path/BinInteraction.cpp\""
compile_all="$compile_all \"$src_path/BinningQuantile.cpp\""
compile_all="$compile_all \"$src_path/BinningUniform.cpp\""
compile_all="$compile_all \"$src_path/BinningWinsorized.cpp\""
compile_all="$compile_all \"$src_path/Booster.cpp\""
compile_all="$compile_all \"$src_path/CalculateInteractionScore.cpp\""
compile_all="$compile_all \"$src_path/CachedThreadResourcesBoosting.cpp\""
compile_all="$compile_all \"$src_path/CachedThreadResourcesInteraction.cpp\""
compile_all="$compile_all \"$src_path/CutRandom.cpp\""
compile_all="$compile_all \"$src_path/DataSetBoosting.cpp\""
compile_all="$compile_all \"$src_path/DataSetInteraction.cpp\""
compile_all="$compile_all \"$src_path/DebugEbm.cpp\""
compile_all="$compile_all \"$src_path/Discretization.cpp\""
compile_all="$compile_all \"$src_path/FeatureGroup.cpp\""
compile_all="$compile_all \"$src_path/FindBestBoostingSplitsPairs.cpp\""
compile_all="$compile_all \"$src_path/FindBestInteractionGainPairs.cpp\""
compile_all="$compile_all \"$src_path/GenerateModelFeatureGroupUpdate.cpp\""
compile_all="$compile_all \"$src_path/GrowDecisionTree.cpp\""
compile_all="$compile_all \"$src_path/InitializeResiduals.cpp\""
compile_all="$compile_all \"$src_path/InteractionDetector.cpp\""
compile_all="$compile_all \"$src_path/InterpretableNumerics.cpp\""
compile_all="$compile_all \"$src_path/Logging.cpp\""
compile_all="$compile_all \"$src_path/RandomExternal.cpp\""
compile_all="$compile_all \"$src_path/RandomStream.cpp\""
compile_all="$compile_all \"$src_path/SamplingSet.cpp\""
compile_all="$compile_all \"$src_path/SegmentedTensor.cpp\""
compile_all="$compile_all \"$src_path/SumHistogramBuckets.cpp\""
compile_all="$compile_all \"$src_path/TensorTotalsBuild.cpp\""
compile_all="$compile_all -I\"$src_path\""
compile_all="$compile_all -I\"$src_path/inc\""
compile_all="$compile_all -Wall -Wextra"
compile_all="$compile_all -Wunused-result"
compile_all="$compile_all -Wno-parentheses"
compile_all="$compile_all -Wold-style-cast"
compile_all="$compile_all -Wdouble-promotion"
compile_all="$compile_all -Wshadow"
compile_all="$compile_all -Wformat=2"
compile_all="$compile_all -std=c++11"
compile_all="$compile_all -fvisibility=hidden -fvisibility-inlines-hidden"
compile_all="$compile_all -fno-math-errno -fno-trapping-math"
compile_all="$compile_all -march=core2"
compile_all="$compile_all -fpic"
compile_all="$compile_all -DEBM_NATIVE_EXPORTS"

if [ "$os_type" = "Darwin" ]; then
   # reference on rpath & install_name: https://www.mikeash.com/pyblog/friday-qa-2009-11-06-linking-and-install-names.html

   # try moving some of these clang specific warnings into compile_all if g++ eventually supports them
   compile_mac="$compile_all -Wnull-dereference -Wgnu-zero-variadic-macro-arguments -dynamiclib"

   printf "%s\n" "Creating initial directories"
   [ -d "$staging_path" ] || mkdir -p "$staging_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   [ -d "$python_lib" ] || mkdir -p "$python_lib"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi


   ########################## macOS release|x64

   printf "%s\n" "Compiling ebm_native with $clang_pp_bin for macOS release|x64"
   intermediate_path="$root_path/tmp/clang/intermediate/release/mac/x64/ebm_native"
   bin_path="$root_path/tmp/clang/bin/release/mac/x64/ebm_native"
   bin_file="lib_ebm_native_mac_x64.dylib"
   log_file="$intermediate_path/ebm_native_release_mac_x64_build_log.txt"
   compile_command="$clang_pp_bin $compile_mac -m64 -DNDEBUG -O3 -install_name @rpath/$bin_file -o \"$bin_path/$bin_file\" 2>&1"
   
   [ -d "$intermediate_path" ] || mkdir -p "$intermediate_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   [ -d "$bin_path" ] || mkdir -p "$bin_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   compile_out=`eval $compile_command`
   ret_code=$?
   printf "%s\n" "$compile_out"
   printf "%s\n" "$compile_out" > "$log_file"
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   cp "$bin_path/$bin_file" "$python_lib/"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   cp "$bin_path/$bin_file" "$staging_path/"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi


   ########################## macOS debug|x64

   printf "%s\n" "Compiling ebm_native with $clang_pp_bin for macOS debug|x64"
   intermediate_path="$root_path/tmp/clang/intermediate/debug/mac/x64/ebm_native"
   bin_path="$root_path/tmp/clang/bin/debug/mac/x64/ebm_native"
   bin_file="lib_ebm_native_mac_x64_debug.dylib"
   log_file="$intermediate_path/ebm_native_debug_mac_x64_build_log.txt"
   compile_command="$clang_pp_bin $compile_mac -m64 -O1 -fsanitize=address,undefined -fno-sanitize-recover=address,undefined -fno-optimize-sibling-calls -fno-omit-frame-pointer -install_name @rpath/$bin_file -o \"$bin_path/$bin_file\" 2>&1"
   
   [ -d "$intermediate_path" ] || mkdir -p "$intermediate_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   [ -d "$bin_path" ] || mkdir -p "$bin_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   compile_out=`eval $compile_command`
   ret_code=$?
   printf "%s\n" "$compile_out"
   printf "%s\n" "$compile_out" > "$log_file"
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   cp "$bin_path/$bin_file" "$python_lib/"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   cp "$bin_path/$bin_file" "$staging_path/"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
elif [ "$os_type" = "Linux" ]; then

   # try moving some of these g++ specific warnings into compile_all if clang eventually supports them
   compile_linux="$compile_all"
   compile_linux="$compile_linux -Wlogical-op -Wl,--version-script=\"$src_path/ebm_native_exports.txt\" -Wl,--exclude-libs,ALL -Wl,-z,relro,-z,now"
   compile_linux="$compile_linux -Wl,--wrap=memcpy \"$src_path/wrap_func.cpp\" -static-libgcc -static-libstdc++ -shared"

   printf "%s\n" "Creating initial directories"
   [ -d "$staging_path" ] || mkdir -p "$staging_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   [ -d "$python_lib" ] || mkdir -p "$python_lib"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi


   ########################## Linux release|x64

   printf "%s\n" "Compiling ebm_native with $g_pp_bin for Linux release|x64"
   intermediate_path="$root_path/tmp/gcc/intermediate/release/linux/x64/ebm_native"
   bin_path="$root_path/tmp/gcc/bin/release/linux/x64/ebm_native"
   bin_file="lib_ebm_native_linux_x64.so"
   log_file="$intermediate_path/ebm_native_release_linux_x64_build_log.txt"
   compile_command="$g_pp_bin $compile_linux -m64 -DNDEBUG -O3 -o \"$bin_path/$bin_file\" 2>&1"
   
   [ -d "$intermediate_path" ] || mkdir -p "$intermediate_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   [ -d "$bin_path" ] || mkdir -p "$bin_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   compile_out=`eval $compile_command`
   ret_code=$?
   printf "%s\n" "$compile_out"
   printf "%s\n" "$compile_out" > "$log_file"
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   cp "$bin_path/$bin_file" "$python_lib/"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   cp "$bin_path/$bin_file" "$staging_path/"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi


   ########################## Linux debug|x64

   printf "%s\n" "Compiling ebm_native with $g_pp_bin for Linux debug|x64"
   intermediate_path="$root_path/tmp/gcc/intermediate/debug/linux/x64/ebm_native"
   bin_path="$root_path/tmp/gcc/bin/debug/linux/x64/ebm_native"
   bin_file="lib_ebm_native_linux_x64_debug.so"
   log_file="$intermediate_path/ebm_native_debug_linux_x64_build_log.txt"
   compile_command="$g_pp_bin $compile_linux -m64 -O1 -o \"$bin_path/$bin_file\" 2>&1"
   
   [ -d "$intermediate_path" ] || mkdir -p "$intermediate_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   [ -d "$bin_path" ] || mkdir -p "$bin_path"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   compile_out=`eval $compile_command`
   ret_code=$?
   printf "%s\n" "$compile_out"
   printf "%s\n" "$compile_out" > "$log_file"
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   cp "$bin_path/$bin_file" "$python_lib/"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi
   cp "$bin_path/$bin_file" "$staging_path/"
   ret_code=$?
   if [ $ret_code -ne 0 ]; then 
      exit $ret_code
   fi

   if [ $build_32_bit -eq 1 ]; then

      ########################## Linux release|x86

      printf "%s\n" "Compiling ebm_native with $g_pp_bin for Linux release|x86"
      intermediate_path="$root_path/tmp/gcc/intermediate/release/linux/x86/ebm_native"
      bin_path="$root_path/tmp/gcc/bin/release/linux/x86/ebm_native"
      bin_file="lib_ebm_native_linux_x86.so"
      log_file="$intermediate_path/ebm_native_release_linux_x86_build_log.txt"
      compile_command="$g_pp_bin $compile_linux -msse2 -mfpmath=sse -m32 -DNDEBUG -O3 -o \"$bin_path/$bin_file\" 2>&1"
      
      if [ ! -d "$intermediate_path" ]; then
         printf "%s\n" "Doing first time installation of x86"

         # this is the first time we're being compiled x86 on this machine, so install other required items

         # TODO consider NOT running sudo inside this script and move that requirement to the caller
         #      per https://askubuntu.com/questions/425754/how-do-i-run-a-sudo-command-inside-a-script

         sudo apt-get -y update
         ret_code=$?
         if [ $ret_code -ne 0 ]; then 
            exit $ret_code
         fi

         sudo apt-get -y install g++-multilib
         ret_code=$?
         if [ $ret_code -ne 0 ]; then 
            exit $ret_code
         fi

         mkdir -p "$intermediate_path"
      fi
      ret_code=$?
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi
      [ -d "$bin_path" ] || mkdir -p "$bin_path"
      ret_code=$?
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi
      compile_out=`eval $compile_command`
      ret_code=$?
      printf "%s\n" "$compile_out"
      printf "%s\n" "$compile_out" > "$log_file"
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi
      cp "$bin_path/$bin_file" "$python_lib/"
      ret_code=$?
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi
      cp "$bin_path/$bin_file" "$staging_path/"
      ret_code=$?
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi


      ########################## Linux debug|x86

      printf "%s\n" "Compiling ebm_native with $g_pp_bin for Linux debug|x86"
      intermediate_path="$root_path/tmp/gcc/intermediate/debug/linux/x86/ebm_native"
      bin_path="$root_path/tmp/gcc/bin/debug/linux/x86/ebm_native"
      bin_file="lib_ebm_native_linux_x86_debug.so"
      log_file="$intermediate_path/ebm_native_debug_linux_x86_build_log.txt"
      compile_command="$g_pp_bin $compile_linux -msse2 -mfpmath=sse -m32 -O1 -o \"$bin_path/$bin_file\" 2>&1"
      
      [ -d "$intermediate_path" ] || mkdir -p "$intermediate_path"
      ret_code=$?
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi
      [ -d "$bin_path" ] || mkdir -p "$bin_path"
      ret_code=$?
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi
      compile_out=`eval $compile_command`
      ret_code=$?
      printf "%s\n" "$compile_out"
      printf "%s\n" "$compile_out" > "$log_file"
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi
      cp "$bin_path/$bin_file" "$python_lib/"
      ret_code=$?
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi
      cp "$bin_path/$bin_file" "$staging_path/"
      ret_code=$?
      if [ $ret_code -ne 0 ]; then 
         exit $ret_code
      fi
   fi
else
   printf "%s\n" "OS $os_type not recognized.  We support $clang_pp_bin on macOS and $g_pp_bin on Linux"
   exit 1
fi