diff --git a/.github/workflows/prestissimo.yml b/.github/workflows/prestissimo.yml new file mode 100644 index 0000000000000..3bfc59eb20513 --- /dev/null +++ b/.github/workflows/prestissimo.yml @@ -0,0 +1,134 @@ +name: Prestissimo + +on: + push: + branches: + - 'master' + pull_request: + +permissions: + contents: read + pull-requests: read + +concurrency: + group: prestissimo-${{ github.head_ref || github.sha }} + cancel-in-progress: true + +jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + # do not run on draft PRs + if: ${{ github.repository == 'prestodb/presto' && github.event.pull_request.draft != true }} + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.ACTION_RUNNER_AWS_KEY_ID }} + aws-secret-access-key: ${{ secrets.ACTION_RUNNER_AWS_SECRET }} + aws-region: us-east-1 + + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2 + with: + mode: start + github-token: ${{ secrets.ACTION_RUNNER_GITHUB_TOKEN }} + ec2-image-id: ami-01d900dce06687dfe + ec2-instance-type: c5.2xlarge + subnet-id: subnet-0c2653c918291180b + security-group-id: sg-07ec7cddcb0e1276f + + detect-file-changes: + name: Detect file changes + runs-on: ubuntu-latest + # do not run on draft PRs + if: ${{ github.repository == 'prestodb/presto' && github.event.pull_request.draft != true }} + outputs: + run_linux_tests: ${{ steps.filter.outputs.run_linux_tests }} + check_and_build: ${{ steps.filter.outputs.check_and_build }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + run_linux_tests: + - '!presto-docs/**' + - '!**/*.md' + check_and_build: + - '.github/workflows/prestissimo.yml' + - 'presto-native-execution/**' + + check-and-build: + name: Check and build all + needs: [start-runner, detect-file-changes] + if: ${{ needs.detect-file-changes.outputs.check_and_build == 'true' }} + runs-on: ${{ needs.start-runner.outputs.label }} + container: prestodb/presto-native-dependency:latest + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Check formatting + run: | + cd presto-native-execution && make format-check + + - name: Check license headers + run: | + cd presto-native-execution && make header-check + + - name: Install all adapter dependencies + run: | + mkdir -p ${HOME}/adapter-deps/install + source /opt/rh/gcc-toolset-12/enable + set -xu + cd presto-native-execution + DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh jwt + + - name: Get Ccache Stash + uses: assignUser/stash/restore@v1 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-ubuntu-release + + - name: Build all + run: | + mkdir -p '${{ env.CCACHE_DIR }}' + ccache -sz -M 8Gi + source /opt/rh/gcc-toolset-12/enable + cd presto-native-execution + make velox-submodule + cmake \ + -B _build/release \ + -GNinja \ + -DAWSSDK_ROOT_DIR=${HOME}/adapter-deps/install \ + -DTREAT_WARNINGS_AS_ERRORS=1 \ + -DENABLE_ALL_WARNINGS=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_S3=ON \ + -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ + -DPRESTO_ENABLE_JWT=ON \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DMAX_LINK_JOBS=2 + ninja -C _build/release -j 8 + ccache -vs + + - uses: assignUser/stash/save@v1 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-ubuntu-release diff --git a/presto-native-execution/scripts/dockerfiles/centos-dependency.dockerfile b/presto-native-execution/scripts/dockerfiles/centos-dependency.dockerfile index ca93f48030e16..187670f2716d8 100644 --- a/presto-native-execution/scripts/dockerfiles/centos-dependency.dockerfile +++ b/presto-native-execution/scripts/dockerfiles/centos-dependency.dockerfile @@ -16,11 +16,14 @@ ENV PROMPT_ALWAYS_RESPOND=n ENV CC=/opt/rh/gcc-toolset-12/root/bin/gcc ENV CXX=/opt/rh/gcc-toolset-12/root/bin/g++ -RUN mkdir -p /scripts /velox/scripts COPY scripts /scripts COPY velox/scripts /velox/scripts RUN mkdir build && \ (cd build && ../scripts/setup-centos.sh && \ ../velox/scripts/setup-adapters.sh && \ ../scripts/setup-adapters.sh ) && \ - rm -rf build + rm -rf build && \ + dnf install -y -q 'dnf-command(config-manager)' && \ + dnf config-manager --add-repo 'https://cli.github.com/packages/rpm/gh-cli.repo' && \ + dnf install -y -q gh jq && \ + dnf clean all