Add concurrency to the find-large-objects scrubber subcommand #17262

Workflow file for this run

.github/workflows/neon_extra_builds.yml at 9c5e2f3

	name: Check neon with extra platform builds

	on:
	push:
	branches:
	- main
	pull_request:

	defaults:
	run:
	shell: bash -euxo pipefail {0}

	concurrency:
	# Allow only one workflow per any non-`main` branch.
	group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha \|\| 'anysha' }}
	cancel-in-progress: true

	env:
	RUST_BACKTRACE: 1
	COPT: '-Werror'

	jobs:
	check-permissions:
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }}
	uses: ./.github/workflows/check-permissions.yml
	with:
	github-event-name: ${{ github.event_name}}

	check-build-tools-image:
	needs: [ check-permissions ]
	uses: ./.github/workflows/check-build-tools-image.yml

	build-build-tools-image:
	needs: [ check-build-tools-image ]
	uses: ./.github/workflows/build-build-tools-image.yml
	with:
	image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }}
	secrets: inherit

	check-macos-build:
	needs: [ check-permissions ]
	if: \|
	contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') \|\|
	contains(github.event.pull_request.labels..name, 'run-extra-build-') \|\|
	github.ref_name == 'main'
	timeout-minutes: 90
	runs-on: macos-14

	env:
	# Use release build only, to have less debug info around
	# Hence keeping target/ (and general cache size) smaller
	BUILD_TYPE: release

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 1

	- name: Install macOS postgres dependencies
	run: brew install flex bison openssl protobuf icu4c pkg-config

	- name: Set pg 14 revision for caching
	id: pg_v14_rev
	run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT

	- name: Set pg 15 revision for caching
	id: pg_v15_rev
	run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT

	- name: Set pg 16 revision for caching
	id: pg_v16_rev
	run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) >> $GITHUB_OUTPUT

	- name: Cache postgres v14 build
	id: cache_pg_14
	uses: actions/cache@v4
	with:
	path: pg_install/v14
	key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}

	- name: Cache postgres v15 build
	id: cache_pg_15
	uses: actions/cache@v4
	with:
	path: pg_install/v15
	key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}

	- name: Cache postgres v16 build
	id: cache_pg_16
	uses: actions/cache@v4
	with:
	path: pg_install/v16
	key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}

	- name: Set extra env for macOS
	run: \|
	echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
	echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV

	- name: Cache cargo deps
	uses: actions/cache@v4
	with:
	path: \|
	~/.cargo/registry
	!~/.cargo/registry/src
	~/.cargo/git
	target
	key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust

	- name: Build postgres v14
	if: steps.cache_pg_14.outputs.cache-hit != 'true'
	run: make postgres-v14 -j$(sysctl -n hw.ncpu)

	- name: Build postgres v15
	if: steps.cache_pg_15.outputs.cache-hit != 'true'
	run: make postgres-v15 -j$(sysctl -n hw.ncpu)

	- name: Build postgres v16
	if: steps.cache_pg_16.outputs.cache-hit != 'true'
	run: make postgres-v16 -j$(sysctl -n hw.ncpu)

	- name: Build neon extensions
	run: make neon-pg-ext -j$(sysctl -n hw.ncpu)

	- name: Build walproposer-lib
	run: make walproposer-lib -j$(sysctl -n hw.ncpu)

	- name: Run cargo build
	run: PQ_LIB_DIR=$(pwd)/pg_install/v16/lib cargo build --all --release

	- name: Check that no warnings are produced
	run: ./run_clippy.sh

	check-linux-arm-build:
	needs: [ check-permissions, build-build-tools-image ]
	timeout-minutes: 90
	runs-on: [ self-hosted, small-arm64 ]

	env:
	# Use release build only, to have less debug info around
	# Hence keeping target/ (and general cache size) smaller
	BUILD_TYPE: release
	CARGO_FEATURES: --features testing
	CARGO_FLAGS: --release
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }}

	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	options: --init

	steps:
	- name: Fix git ownership
	run: \|
	# Workaround for `fatal: detected dubious ownership in repository at ...`
	#
	# Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers
	# Ref https://github.com/actions/checkout/issues/785
	#
	git config --global --add safe.directory ${{ github.workspace }}
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	for r in 14 15 16; do
	git config --global --add safe.directory "${{ github.workspace }}/vendor/postgres-v$r"
	git config --global --add safe.directory "${GITHUB_WORKSPACE}/vendor/postgres-v$r"
	done

	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 1

	- name: Set pg 14 revision for caching
	id: pg_v14_rev
	run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT

	- name: Set pg 15 revision for caching
	id: pg_v15_rev
	run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT

	- name: Set pg 16 revision for caching
	id: pg_v16_rev
	run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) >> $GITHUB_OUTPUT

	- name: Set env variables
	run: \|
	echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo" >> $GITHUB_ENV

	- name: Cache postgres v14 build
	id: cache_pg_14
	uses: actions/cache@v4
	with:
	path: pg_install/v14
	key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}

	- name: Cache postgres v15 build
	id: cache_pg_15
	uses: actions/cache@v4
	with:
	path: pg_install/v15
	key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}

	- name: Cache postgres v16 build
	id: cache_pg_16
	uses: actions/cache@v4
	with:
	path: pg_install/v16
	key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}

	- name: Build postgres v14
	if: steps.cache_pg_14.outputs.cache-hit != 'true'
	run: mold -run make postgres-v14 -j$(nproc)

	- name: Build postgres v15
	if: steps.cache_pg_15.outputs.cache-hit != 'true'
	run: mold -run make postgres-v15 -j$(nproc)

	- name: Build postgres v16
	if: steps.cache_pg_16.outputs.cache-hit != 'true'
	run: mold -run make postgres-v16 -j$(nproc)

	- name: Build neon extensions
	run: mold -run make neon-pg-ext -j$(nproc)

	- name: Build walproposer-lib
	run: mold -run make walproposer-lib -j$(nproc)

	- name: Run cargo build
	run: \|
	PQ_LIB_DIR=$(pwd)/pg_install/v16/lib
	export PQ_LIB_DIR
	mold -run cargo build --locked $CARGO_FLAGS $CARGO_FEATURES --bins --tests -j$(nproc)

	- name: Run cargo test
	env:
	NEXTEST_RETRIES: 3
	run: \|
	PQ_LIB_DIR=$(pwd)/pg_install/v16/lib
	export PQ_LIB_DIR
	LD_LIBRARY_PATH=$(pwd)/pg_install/v16/lib
	export LD_LIBRARY_PATH

	cargo nextest run $CARGO_FEATURES -j$(nproc)

	# Run separate tests for real S3
	export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
	export REMOTE_STORAGE_S3_BUCKET=neon-github-ci-tests
	export REMOTE_STORAGE_S3_REGION=eu-central-1
	# Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now
	cargo nextest run --package remote_storage --test test_real_s3 -j$(nproc)

	# Run separate tests for real Azure Blob Storage
	# XXX: replace region with `eu-central-1`-like region
	export ENABLE_REAL_AZURE_REMOTE_STORAGE=y
	export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}"
	export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}"
	export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}"
	export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}"
	# Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now
	cargo nextest run --package remote_storage --test test_real_azure -j$(nproc)

	check-codestyle-rust-arm:
	needs: [ check-permissions, build-build-tools-image ]
	timeout-minutes: 90
	runs-on: [ self-hosted, small-arm64 ]

	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	options: --init

	strategy:
	fail-fast: false
	matrix:
	build_type: [ debug, release ]

	steps:
	- name: Fix git ownership
	run: \|
	# Workaround for `fatal: detected dubious ownership in repository at ...`
	#
	# Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers
	# Ref https://github.com/actions/checkout/issues/785
	#
	git config --global --add safe.directory ${{ github.workspace }}
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	for r in 14 15 16; do
	git config --global --add safe.directory "${{ github.workspace }}/vendor/postgres-v$r"
	git config --global --add safe.directory "${GITHUB_WORKSPACE}/vendor/postgres-v$r"
	done

	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 1

	# Some of our rust modules use FFI and need those to be checked
	- name: Get postgres headers
	run: make postgres-headers -j$(nproc)

	# cargo hack runs the given cargo subcommand (clippy in this case) for all feature combinations.
	# This will catch compiler & clippy warnings in all feature combinations.
	# TODO: use cargo hack for build and test as well, but, that's quite expensive.
	# NB: keep clippy args in sync with ./run_clippy.sh
	- run: \|
	CLIPPY_COMMON_ARGS="$( source .neon_clippy_args; echo "$CLIPPY_COMMON_ARGS")"
	if [ "$CLIPPY_COMMON_ARGS" = "" ]; then
	echo "No clippy args found in .neon_clippy_args"
	exit 1
	fi
	echo "CLIPPY_COMMON_ARGS=${CLIPPY_COMMON_ARGS}" >> $GITHUB_ENV

	- name: Run cargo clippy (debug)
	if: matrix.build_type == 'debug'
	run: cargo hack --feature-powerset clippy $CLIPPY_COMMON_ARGS
	- name: Run cargo clippy (release)
	if: matrix.build_type == 'release'
	run: cargo hack --feature-powerset clippy --release $CLIPPY_COMMON_ARGS

	- name: Check documentation generation
	if: matrix.build_type == 'release'
	run: cargo doc --workspace --no-deps --document-private-items -j$(nproc)
	env:
	RUSTDOCFLAGS: "-Dwarnings -Arustdoc::private_intra_doc_links"

	# Use `${{ !cancelled() }}` to run quck tests after the longer clippy run
	- name: Check formatting
	if: ${{ !cancelled() && matrix.build_type == 'release' }}
	run: cargo fmt --all -- --check

	# https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci
	- name: Check rust dependencies
	if: ${{ !cancelled() && matrix.build_type == 'release' }}
	run: \|
	cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date
	cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack

	# https://github.com/EmbarkStudios/cargo-deny
	- name: Check rust licenses/bans/advisories/sources
	if: ${{ !cancelled() && matrix.build_type == 'release' }}
	run: cargo deny check

	gather-rust-build-stats:
	needs: [ check-permissions, build-build-tools-image ]
	if: \|
	contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') \|\|
	contains(github.event.pull_request.labels..name, 'run-extra-build-') \|\|
	github.ref_name == 'main'
	runs-on: [ self-hosted, large ]
	container:
	image: ${{ needs.build-build-tools-image.outputs.image }}
	credentials:
	username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
	password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
	options: --init

	env:
	BUILD_TYPE: release
	# remove the cachepot wrapper and build without crate caches
	RUSTC_WRAPPER: ""
	# build with incremental compilation produce partial results
	# so do not attempt to cache this build, also disable the incremental compilation
	CARGO_INCREMENTAL: 0

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 1

	# Some of our rust modules use FFI and need those to be checked
	- name: Get postgres headers
	run: make postgres-headers -j$(nproc)

	- name: Build walproposer-lib
	run: make walproposer-lib -j$(nproc)

	- name: Produce the build stats
	run: PQ_LIB_DIR=$(pwd)/pg_install/v16/lib cargo build --all --release --timings -j$(nproc)

	- name: Upload the build stats
	id: upload-stats
	env:
	BUCKET: neon-github-public-dev
	SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }}
	run: \|
	REPORT_URL=https://${BUCKET}.s3.amazonaws.com/build-stats/${SHA}/${GITHUB_RUN_ID}/cargo-timing.html
	aws s3 cp --only-show-errors ./target/cargo-timings/cargo-timing.html "s3://${BUCKET}/build-stats/${SHA}/${GITHUB_RUN_ID}/"
	echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT

	- name: Publish build stats report
	uses: actions/github-script@v7
	env:
	REPORT_URL: ${{ steps.upload-stats.outputs.report-url }}
	SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	with:
	script: \|
	const { REPORT_URL, SHA } = process.env

	await github.rest.repos.createCommitStatus({
	owner: context.repo.owner,
	repo: context.repo.repo,
	sha: `${SHA}`,
	state: 'success',
	target_url: `${REPORT_URL}`,
	context: `Build stats (release)`,
	})

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add concurrency to the find-large-objects scrubber subcommand #17262

Workflow file

Add concurrency to the find-large-objects scrubber subcommand #17262

Jobs

Run details

Workflow file for this run