storage controller: make scheduling more stable wrt optimization #21106
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check neon with extra platform builds | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| defaults: | |
| run: | |
| shell: bash -euxo pipefail {0} | |
| concurrency: | |
| # Allow only one workflow per any non-`main` branch. | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} | |
| cancel-in-progress: true | |
| env: | |
| RUST_BACKTRACE: 1 | |
| COPT: '-Werror' | |
| jobs: | |
| check-permissions: | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }} | |
| uses: ./.github/workflows/check-permissions.yml | |
| with: | |
| github-event-name: ${{ github.event_name}} | |
| check-build-tools-image: | |
| needs: [ check-permissions ] | |
| uses: ./.github/workflows/check-build-tools-image.yml | |
| build-build-tools-image: | |
| needs: [ check-build-tools-image ] | |
| uses: ./.github/workflows/build-build-tools-image.yml | |
| with: | |
| image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }} | |
| secrets: inherit | |
| check-macos-build: | |
| needs: [ check-permissions ] | |
| if: | | |
| contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || | |
| contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || | |
| github.ref_name == 'main' | |
| timeout-minutes: 90 | |
| runs-on: macos-14 | |
| env: | |
| # Use release build only, to have less debug info around | |
| # Hence keeping target/ (and general cache size) smaller | |
| BUILD_TYPE: release | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Install macOS postgres dependencies | |
| run: brew install flex bison openssl protobuf icu4c pkg-config | |
| - name: Set pg 14 revision for caching | |
| id: pg_v14_rev | |
| run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT | |
| - name: Set pg 15 revision for caching | |
| id: pg_v15_rev | |
| run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT | |
| - name: Set pg 16 revision for caching | |
| id: pg_v16_rev | |
| run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) >> $GITHUB_OUTPUT | |
| - name: Set pg 17 revision for caching | |
| id: pg_v17_rev | |
| run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v17) >> $GITHUB_OUTPUT | |
| - name: Cache postgres v14 build | |
| id: cache_pg_14 | |
| uses: actions/cache@v4 | |
| with: | |
| path: pg_install/v14 | |
| key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} | |
| - name: Cache postgres v15 build | |
| id: cache_pg_15 | |
| uses: actions/cache@v4 | |
| with: | |
| path: pg_install/v15 | |
| key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} | |
| - name: Cache postgres v16 build | |
| id: cache_pg_16 | |
| uses: actions/cache@v4 | |
| with: | |
| path: pg_install/v16 | |
| key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} | |
| - name: Cache postgres v17 build | |
| id: cache_pg_17 | |
| uses: actions/cache@v4 | |
| with: | |
| path: pg_install/v17 | |
| key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} | |
| - name: Set extra env for macOS | |
| run: | | |
| echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV | |
| echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV | |
| - name: Cache cargo deps | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| !~/.cargo/registry/src | |
| ~/.cargo/git | |
| target | |
| key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust | |
| - name: Build postgres v14 | |
| if: steps.cache_pg_14.outputs.cache-hit != 'true' | |
| run: make postgres-v14 -j$(sysctl -n hw.ncpu) | |
| - name: Build postgres v15 | |
| if: steps.cache_pg_15.outputs.cache-hit != 'true' | |
| run: make postgres-v15 -j$(sysctl -n hw.ncpu) | |
| - name: Build postgres v16 | |
| if: steps.cache_pg_16.outputs.cache-hit != 'true' | |
| run: make postgres-v16 -j$(sysctl -n hw.ncpu) | |
| - name: Build postgres v17 | |
| if: steps.cache_pg_17.outputs.cache-hit != 'true' | |
| run: make postgres-v17 -j$(sysctl -n hw.ncpu) | |
| - name: Build neon extensions | |
| run: make neon-pg-ext -j$(sysctl -n hw.ncpu) | |
| - name: Build walproposer-lib | |
| run: make walproposer-lib -j$(sysctl -n hw.ncpu) | |
| - name: Run cargo build | |
| run: PQ_LIB_DIR=$(pwd)/pg_install/v16/lib cargo build --all --release | |
| - name: Check that no warnings are produced | |
| run: ./run_clippy.sh | |
| gather-rust-build-stats: | |
| needs: [ check-permissions, build-build-tools-image ] | |
| if: | | |
| contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') || | |
| contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || | |
| github.ref_name == 'main' | |
| runs-on: [ self-hosted, large ] | |
| container: | |
| image: ${{ needs.build-build-tools-image.outputs.image }} | |
| credentials: | |
| username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
| options: --init | |
| env: | |
| BUILD_TYPE: release | |
| # build with incremental compilation produce partial results | |
| # so do not attempt to cache this build, also disable the incremental compilation | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| # Some of our rust modules use FFI and need those to be checked | |
| - name: Get postgres headers | |
| run: make postgres-headers -j$(nproc) | |
| - name: Build walproposer-lib | |
| run: make walproposer-lib -j$(nproc) | |
| - name: Produce the build stats | |
| run: PQ_LIB_DIR=$(pwd)/pg_install/v17/lib cargo build --all --release --timings -j$(nproc) | |
| - name: Upload the build stats | |
| id: upload-stats | |
| env: | |
| BUCKET: neon-github-public-dev | |
| SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} | |
| run: | | |
| REPORT_URL=https://${BUCKET}.s3.amazonaws.com/build-stats/${SHA}/${GITHUB_RUN_ID}/cargo-timing.html | |
| aws s3 cp --only-show-errors ./target/cargo-timings/cargo-timing.html "s3://${BUCKET}/build-stats/${SHA}/${GITHUB_RUN_ID}/" | |
| echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT | |
| - name: Publish build stats report | |
| uses: actions/github-script@v7 | |
| env: | |
| REPORT_URL: ${{ steps.upload-stats.outputs.report-url }} | |
| SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| with: | |
| script: | | |
| const { REPORT_URL, SHA } = process.env | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha: `${SHA}`, | |
| state: 'success', | |
| target_url: `${REPORT_URL}`, | |
| context: `Build stats (release)`, | |
| }) |