From 3650de7071e187fc1b1805135be97db92289acf2 Mon Sep 17 00:00:00 2001 From: leiyanfei Date: Tue, 12 Mar 2024 14:53:37 +0800 Subject: [PATCH 1/3] [build] Add pr-jobs-linux check --- .github/actions/install-gflags/action.yml | 7 ++ .github/actions/post-steps/action.yml | 38 ++++++++ .github/actions/pre-steps/action.yml | 18 ++++ .github/actions/setup-upstream/action.yml | 20 ++++ .github/workflows/pr-jobs-linux.yml | 107 ++++++++++++++++++++++ 5 files changed, 190 insertions(+) create mode 100644 .github/actions/install-gflags/action.yml create mode 100644 .github/actions/post-steps/action.yml create mode 100644 .github/actions/pre-steps/action.yml create mode 100644 .github/actions/setup-upstream/action.yml create mode 100644 .github/workflows/pr-jobs-linux.yml diff --git a/.github/actions/install-gflags/action.yml b/.github/actions/install-gflags/action.yml new file mode 100644 index 000000000..d47619722 --- /dev/null +++ b/.github/actions/install-gflags/action.yml @@ -0,0 +1,7 @@ +name: install-gflags +runs: + using: composite + steps: + - name: Install gflags + run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev + shell: bash \ No newline at end of file diff --git a/.github/actions/post-steps/action.yml b/.github/actions/post-steps/action.yml new file mode 100644 index 000000000..5bb7502ec --- /dev/null +++ b/.github/actions/post-steps/action.yml @@ -0,0 +1,38 @@ +name: post-steps +description: Steps that are taken after a RocksDB job +inputs: + artifact-prefix: + description: Prefix to append to the name of artifacts that are uploaded + required: true + default: "${{ github.job }}" +runs: + using: composite + steps: + - name: Upload Test Results artifact + uses: actions/upload-artifact@v4.0.0 + with: + name: "${{ inputs.artifact-prefix }}-test-results" + path: "${{ runner.temp }}/test-results/**" + - name: Upload DB LOG file artifact + uses: actions/upload-artifact@v4.0.0 + with: + name: "${{ inputs.artifact-prefix }}-db-log-file" + path: LOG + - name: Copy Test Logs (on Failure) + if: ${{ failure() }} + run: | + mkdir -p ${{ runner.temp }}/failure-test-logs + cp -r t/* ${{ runner.temp }}/failure-test-logs + shell: bash + - name: Upload Test Logs (on Failure) artifact + uses: actions/upload-artifact@v4.0.0 + with: + name: "${{ inputs.artifact-prefix }}-failure-test-logs" + path: ${{ runner.temp }}/failure-test-logs/** + if-no-files-found: ignore + - name: Upload Core Dumps artifact + uses: actions/upload-artifact@v4.0.0 + with: + name: "${{ inputs.artifact-prefix }}-core-dumps" + path: "core.*" + if-no-files-found: ignore \ No newline at end of file diff --git a/.github/actions/pre-steps/action.yml b/.github/actions/pre-steps/action.yml new file mode 100644 index 000000000..d40254610 --- /dev/null +++ b/.github/actions/pre-steps/action.yml @@ -0,0 +1,18 @@ +name: pre-steps +runs: + using: composite + steps: + - name: Setup Environment Variables + run: |- + echo "GTEST_THROW_ON_FAILURE=0" >> "$GITHUB_ENV" + echo "GTEST_OUTPUT=\"xml:${{ runner.temp }}/test-results/\"" >> "$GITHUB_ENV" + echo "SKIP_FORMAT_BUCK_CHECKS=1" >> "$GITHUB_ENV" + echo "GTEST_COLOR=1" >> "$GITHUB_ENV" + echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV" + echo "CTEST_TEST_TIMEOUT=300" >> "$GITHUB_ENV" + echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> "$GITHUB_ENV" + echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> "$GITHUB_ENV" + echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> "$GITHUB_ENV" + echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> "$GITHUB_ENV" + echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> "$GITHUB_ENV" + shell: bash \ No newline at end of file diff --git a/.github/actions/setup-upstream/action.yml b/.github/actions/setup-upstream/action.yml new file mode 100644 index 000000000..6cbe22771 --- /dev/null +++ b/.github/actions/setup-upstream/action.yml @@ -0,0 +1,20 @@ +name: build-folly +runs: + using: composite + steps: + - name: Fix repo ownership + # Needed in some cases, as safe.directory setting doesn't take effect + # under env -i + run: chown `whoami` . || true + shell: bash + - name: Set upstream + run: git remote add upstream https://github.com/facebook/rocksdb.git + shell: bash + - name: Fetch upstream + run: git fetch upstream + shell: bash + - name: Git status + # NOTE: some old branch builds under check_format_compatible.sh invoke + # git under env -i + run: git status && git remote -v && env -i git branch + shell: bash \ No newline at end of file diff --git a/.github/workflows/pr-jobs-linux.yml b/.github/workflows/pr-jobs-linux.yml new file mode 100644 index 000000000..8141165d3 --- /dev/null +++ b/.github/workflows/pr-jobs-linux.yml @@ -0,0 +1,107 @@ +name: ververica/forst/pr-jobs-linux +on: [push, pull_request] +jobs: + # ======================== Fast Initial Checks ====================== # + check-format-and-targets: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.0 + with: + fetch-depth: 0 # Need full checkout to determine merge base + fetch-tags: true + - uses: "./.github/actions/setup-upstream" + - name: Setup Python + uses: actions/setup-python@v5 + - name: Install Dependencies + run: python -m pip install --upgrade pip + - name: Install argparse + run: pip install argparse + - name: Download clang-format-diff.py + run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py + - name: Check format + run: VERBOSE_CHECK=1 make check-format + - name: Simple source code checks + run: make check-sources + # ========================= Linux With Tests ======================== # + build-linux: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/pre-steps" + - uses: "./.github/actions/install-gflags" + - run: make V=1 J=8 -j8 check + - uses: "./.github/actions/post-steps" + # ======================== Linux No Test Runs ======================= # + build-linux-release: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4.1.0 + - uses: "./.github/actions/install-gflags" + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: "./db_stress --version" + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: "./db_stress --version" + - run: make clean + - run: sudo apt-get remove -y libgflags-dev + - run: make V=1 -j32 LIB_MODE=shared release + - run: ls librocksdb.so + - run: if ./db_stress --version; then false; else true; fi + - run: make clean + - run: make V=1 -j32 release + - run: ls librocksdb.a + - run: if ./db_stress --version; then false; else true; fi + - uses: "./.github/actions/post-steps" + # ============================ Java Jobs ============================ # + build-linux-java: + runs-on: ubuntu-latest + container: evolvedbinary/rocksjava:centos6_x64-be + steps: + # The docker image is intentionally based on an OS that has an older GLIBC version. + # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step. + - name: Checkout + env: + GH_TOKEN: ${{ github.token }} + run: | + chown `whoami` . || true + git clone --no-checkout https://oath2:$GH_TOKEN@github.com/${{ github.repository }}.git . + git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }} + git checkout --progress --force ${{ github.ref }} + git log -1 --format='%H' + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest' + # NOTE: post-steps skipped because of compatibility issues with docker image + build-linux-java-static: + runs-on: ubuntu-latest + container: evolvedbinary/rocksjava:centos6_x64-be + steps: + # The docker image is intentionally based on an OS that has an older GLIBC version. + # That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step. + - name: Checkout + env: + GH_TOKEN: ${{ github.token }} + run: | + chown `whoami` . || true + git clone --no-checkout https://oath2:$GH_TOKEN@github.com/${{ github.repository }}.git . + git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }} + git checkout --progress --force ${{ github.ref }} + git log -1 --format='%H' + - uses: "./.github/actions/pre-steps" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava Static Library + run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' + # NOTE: post-steps skipped because of compatibility issues with docker image \ No newline at end of file From 24cb1ab3c8722ae83a9eadb1e7866b9c7ca9dae7 Mon Sep 17 00:00:00 2001 From: leiyanfei Date: Tue, 12 Mar 2024 16:21:14 +0800 Subject: [PATCH 2/3] Add macOS ci --- .github/actions/build-folly/action.yml | 7 ++ .../action.yml | 10 +++ .../install-gflags-on-macos/action.yml | 7 ++ .../actions/install-jdk8-on-macos/action.yml | 9 ++ .github/actions/pre-steps-macos/action.yml | 5 ++ .github/actions/setup-folly/action.yml | 7 ++ .../{pr-jobs-linux.yml => pr-jobs.yml} | 88 ++++++++++++++++--- Makefile | 6 ++ java/Makefile | 2 - 9 files changed, 128 insertions(+), 13 deletions(-) create mode 100644 .github/actions/build-folly/action.yml create mode 100644 .github/actions/increase-max-open-files-on-macos/action.yml create mode 100644 .github/actions/install-gflags-on-macos/action.yml create mode 100644 .github/actions/install-jdk8-on-macos/action.yml create mode 100644 .github/actions/pre-steps-macos/action.yml create mode 100644 .github/actions/setup-folly/action.yml rename .github/workflows/{pr-jobs-linux.yml => pr-jobs.yml} (55%) diff --git a/.github/actions/build-folly/action.yml b/.github/actions/build-folly/action.yml new file mode 100644 index 000000000..cd6cdfc06 --- /dev/null +++ b/.github/actions/build-folly/action.yml @@ -0,0 +1,7 @@ +name: build-folly +runs: + using: composite + steps: + - name: Build folly and dependencies + run: make build_folly + shell: bash \ No newline at end of file diff --git a/.github/actions/increase-max-open-files-on-macos/action.yml b/.github/actions/increase-max-open-files-on-macos/action.yml new file mode 100644 index 000000000..869cd14ed --- /dev/null +++ b/.github/actions/increase-max-open-files-on-macos/action.yml @@ -0,0 +1,10 @@ +name: increase-max-open-files-on-macos +runs: + using: composite + steps: + - name: Increase max open files + run: |- + sudo sysctl -w kern.maxfiles=1048576 + sudo sysctl -w kern.maxfilesperproc=1048576 + sudo launchctl limit maxfiles 1048576 + shell: bash \ No newline at end of file diff --git a/.github/actions/install-gflags-on-macos/action.yml b/.github/actions/install-gflags-on-macos/action.yml new file mode 100644 index 000000000..3de06f614 --- /dev/null +++ b/.github/actions/install-gflags-on-macos/action.yml @@ -0,0 +1,7 @@ +name: install-gflags-on-macos +runs: + using: composite + steps: + - name: Install gflags on macos + run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags + shell: bash \ No newline at end of file diff --git a/.github/actions/install-jdk8-on-macos/action.yml b/.github/actions/install-jdk8-on-macos/action.yml new file mode 100644 index 000000000..80c56da09 --- /dev/null +++ b/.github/actions/install-jdk8-on-macos/action.yml @@ -0,0 +1,9 @@ +name: install-jdk8-on-macos +runs: + using: composite + steps: + - name: Install JDK 8 on macos + run: |- + HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica + HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8 + shell: bash \ No newline at end of file diff --git a/.github/actions/pre-steps-macos/action.yml b/.github/actions/pre-steps-macos/action.yml new file mode 100644 index 000000000..86c83b3b4 --- /dev/null +++ b/.github/actions/pre-steps-macos/action.yml @@ -0,0 +1,5 @@ +name: pre-steps-macos +runs: + using: composite + steps: + - uses: "./.github/actions/pre-steps" \ No newline at end of file diff --git a/.github/actions/setup-folly/action.yml b/.github/actions/setup-folly/action.yml new file mode 100644 index 000000000..cf2b2900b --- /dev/null +++ b/.github/actions/setup-folly/action.yml @@ -0,0 +1,7 @@ +name: setup-folly +runs: + using: composite + steps: + - name: Checkout folly sources + run: make checkout_folly + shell: bash \ No newline at end of file diff --git a/.github/workflows/pr-jobs-linux.yml b/.github/workflows/pr-jobs.yml similarity index 55% rename from .github/workflows/pr-jobs-linux.yml rename to .github/workflows/pr-jobs.yml index 8141165d3..a73f5bacc 100644 --- a/.github/workflows/pr-jobs-linux.yml +++ b/.github/workflows/pr-jobs.yml @@ -30,7 +30,8 @@ jobs: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/pre-steps" - uses: "./.github/actions/install-gflags" - - run: make V=1 J=8 -j8 check + - run: echo "JAVA_HOME=${JAVA_HOME}" + - run: DISABLE_WARNING_AS_ERROR=1 make V=1 J=8 -j8 check - uses: "./.github/actions/post-steps" # ======================== Linux No Test Runs ======================= # build-linux-release: @@ -39,20 +40,22 @@ jobs: steps: - uses: actions/checkout@v4.1.0 - uses: "./.github/actions/install-gflags" - - run: make V=1 -j32 LIB_MODE=shared release + - run: echo "JAVA_HOME=${JAVA_HOME}" + - run: echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_PATH + - run: DISABLE_WARNING_AS_ERROR=1 make V=1 -j32 LIB_MODE=shared release - run: ls librocksdb.so - run: "./db_stress --version" - - run: make clean - - run: make V=1 -j32 release + - run: DISABLE_WARNING_AS_ERROR=1 make clean + - run: DISABLE_WARNING_AS_ERROR=1 make V=1 -j32 release - run: ls librocksdb.a - run: "./db_stress --version" - - run: make clean + - run: DISABLE_WARNING_AS_ERROR=1 make clean - run: sudo apt-get remove -y libgflags-dev - - run: make V=1 -j32 LIB_MODE=shared release + - run: DISABLE_WARNING_AS_ERROR=1 make V=1 -j32 LIB_MODE=shared release - run: ls librocksdb.so - run: if ./db_stress --version; then false; else true; fi - - run: make clean - - run: make V=1 -j32 release + - run: DISABLE_WARNING_AS_ERROR=1 make clean + - run: DISABLE_WARNING_AS_ERROR=1 make V=1 -j32 release - run: ls librocksdb.a - run: if ./db_stress --version; then false; else true; fi - uses: "./.github/actions/post-steps" @@ -76,10 +79,11 @@ jobs: - name: Set Java Environment run: |- echo "JAVA_HOME=${JAVA_HOME}" + echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $GITHUB_PATH which java && java -version which javac && javac -version - name: Test RocksDBJava - run: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest' + run: scl enable devtoolset-7 'DISABLE_WARNING_AS_ERROR=1 make V=1 J=8 -j8 jtest' # NOTE: post-steps skipped because of compatibility issues with docker image build-linux-java-static: runs-on: ubuntu-latest @@ -103,5 +107,67 @@ jobs: which java && java -version which javac && javac -version - name: Build RocksDBJava Static Library - run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic' - # NOTE: post-steps skipped because of compatibility issues with docker image \ No newline at end of file + run: scl enable devtoolset-7 'DISABLE_WARNING_AS_ERROR=1 make V=1 J=8 -j8 rocksdbjavastatic' + # NOTE: post-steps skipped because of compatibility issues with docker image + + # ========================= MacOS build only ======================== # + build-macos: + runs-on: macos-13 + timeout-minutes: 120 + env: + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Build + run: ulimit -S -n `ulimit -H -n` && DISABLE_WARNING_AS_ERROR=1 make V=1 J=16 -j16 all + - uses: "./.github/actions/post-steps" + # ========================= MacOS with java ======================== # + build-macos-java: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + ROCKSDB_DISABLE_JEMALLOC: 1 + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Test RocksDBJava + run: DISABLE_WARNING_AS_ERROR=1 make V=1 J=16 -j16 jtest + - uses: "./.github/actions/post-steps" + build-macos-java-static: + runs-on: macos-13 + env: + JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home" + steps: + - uses: actions/checkout@v4.1.0 + - uses: maxim-lobanov/setup-xcode@v1.6.0 + with: + xcode-version: 14.3.1 + - uses: "./.github/actions/increase-max-open-files-on-macos" + - uses: "./.github/actions/install-gflags-on-macos" + - uses: "./.github/actions/install-jdk8-on-macos" + - uses: "./.github/actions/pre-steps-macos" + - name: Set Java Environment + run: |- + echo "JAVA_HOME=${JAVA_HOME}" + which java && java -version + which javac && javac -version + - name: Build RocksDBJava x86 and ARM Static Libraries + run: DISABLE_WARNING_AS_ERROR=1 make V=1 J=16 -j16 rocksdbjavastaticosx + - uses: "./.github/actions/post-steps" \ No newline at end of file diff --git a/Makefile b/Makefile index 0c111485c..1658f24c1 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,15 @@ BASH_EXISTS := $(shell which bash) SHELL := $(shell which bash) include common.mk +MY_JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux +ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","") + MY_JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin +endif + CLEAN_FILES = # deliberately empty, so we can append below. CFLAGS += ${EXTRA_CFLAGS} CXXFLAGS += ${EXTRA_CXXFLAGS} +CXXFLAGS += ${MY_JAVA_INCLUDE} LDFLAGS += $(EXTRA_LDFLAGS) MACHINE ?= $(shell uname -m) ARFLAGS = ${EXTRA_ARFLAGS} rs diff --git a/java/Makefile b/java/Makefile index 5f32dc7e5..ea8ca7eb6 100644 --- a/java/Makefile +++ b/java/Makefile @@ -338,8 +338,6 @@ javalib: java java_test javadocs java: java-version $(AM_V_GEN)mkdir -p $(MAIN_CLASSES) $(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -h $(NATIVE_INCLUDE) -d $(MAIN_CLASSES) $(SOURCES) - $(AM_V_at)@cp ../HISTORY.md ./HISTORY-CPP.md - $(AM_V_at)@rm -f ./HISTORY-CPP.md sample: java $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) From 807e71d9b34b9dcffe7230c4795c0b1aaab69336 Mon Sep 17 00:00:00 2001 From: leiyanfei Date: Wed, 13 Mar 2024 11:24:01 +0800 Subject: [PATCH 3/3] rename job --- .github/workflows/pr-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-jobs.yml b/.github/workflows/pr-jobs.yml index a73f5bacc..385bd2dde 100644 --- a/.github/workflows/pr-jobs.yml +++ b/.github/workflows/pr-jobs.yml @@ -1,4 +1,4 @@ -name: ververica/forst/pr-jobs-linux +name: ververica/forst/pr-jobs on: [push, pull_request] jobs: # ======================== Fast Initial Checks ====================== #