Send content length in S3 put_stream implementation (#1143) #4001

Workflow file for this run

	name: CI
	on:
	push:
	tags:
	- "v..*"
	branches:
	- main
	pull_request:

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	permissions:
	checks: write
	pull-requests: write

	env:
	BUILD_TARGET: "x86_64-unknown-linux-gnu"

	jobs:
	docker-targets-build:
	runs-on: ubuntu-latest
	if: github.event_name == 'push' && github.ref_type == 'tag'
	strategy:
	fail-fast: false
	matrix:
	platform:
	- platform: linux/amd64
	name: linux/amd64
	target: x86_64-unknown-linux-gnu
	- platform: linux/arm64
	name: linux/arm64
	target: aarch64-unknown-linux-gnu
	cross: true
	name: docker-targets-build (${{ matrix.platform.platform }})
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Prepare
	run: \|
	platform=${{ matrix.platform.platform }}
	echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
	- name: Setup Rust
	run: rustup update stable --no-self-update && rustup default stable && rustup target add ${{ matrix.platform.target }}
	- uses: davidB/rust-cargo-make@v1
	- name: Install Rust Target
	run: rustup target add ${{ matrix.platform.target }}
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}

	- run: cargo install cross
	if: ${{ matrix.platform.cross }}

	- name: Build executables
	run: cargo make --profile ci build-release
	env:
	PLATFORM_OVERRIDE: ${{ matrix.platform.name }}

	- name: Prepare Targets
	env:
	PLATFORM_OVERRIDE: ${{ matrix.platform.name }}
	run: cargo make --profile ci package-release

	- uses: actions/upload-artifact@v4
	name: Upload Targets
	with:
	name: docker-targets-build-${{ env.PLATFORM_PAIR }}
	path: target/${{ matrix.platform.target }}.tar
	docker-publish:
	runs-on: ubuntu-latest
	needs: [ docker-targets-build ]
	if: github.event_name == 'push' && github.ref_type == 'tag'
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Prepare
	run: \|
	echo "PLATFORMS=linux/amd64,linux/arm64" >> $GITHUB_ENV
	- uses: actions/download-artifact@v4
	name: Download Targets
	with:
	pattern: docker-targets-build-*
	path: target
	merge-multiple: true
	- name: Extract Targets
	run: \|
	ls -R target
	cd target
	for f in *.tar; do tar xvf "$f"; done
	- name: Set up QEMU
	uses: docker/setup-qemu-action@v3
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	- name: Set Docker version
	id: set-version
	run: \|
	if [ "${{ github.event_name }}" == 'push' ] && [ "${{ github.ref_type }}" == 'tag' ]; then
	DOCKER_VERSION=$(echo "${{ github.ref }}" \| sed 's\|^refs/tags/v\|\|')
	echo "DOCKER_VERSION=${DOCKER_VERSION}" >> $GITHUB_ENV
	else
	COMMIT_SHORT_HASH=$(git rev-parse --short=7 HEAD)
	echo "DOCKER_VERSION=${COMMIT_SHORT_HASH}" >> $GITHUB_ENV
	fi
	- uses: jpribyl/[email protected]
	continue-on-error: true
	- name: Log in to Docker Hub
	uses: docker/login-action@v3
	with:
	username: ${{ secrets.DOCKERHUB_USERNAME }}
	password: ${{ secrets.DOCKERHUB_PASSWORD }}
	- name: Extract metadata (tags, labels) for worker executor
	id: meta-worker-executor
	uses: docker/metadata-action@v5
	with:
	images: golemservices/golem-worker-executor
	- name: Build and push worker executor image
	uses: docker/build-push-action@v5
	with:
	context: .
	file: ./golem-worker-executor/docker/Dockerfile
	push: true
	platforms: ${{ env.PLATFORMS }}
	tags: ${{ steps.meta-worker-executor.outputs.tags }}
	labels: ${{ steps.meta-worker-executor.outputs.labels }}
	- name: Extract metadata (tags, labels) for shard manager
	id: meta-shard-manager
	uses: docker/metadata-action@v5
	with:
	images: golemservices/golem-shard-manager
	- name: Build and push shard manager image
	uses: docker/build-push-action@v5
	with:
	context: .
	file: ./golem-shard-manager/docker/Dockerfile
	push: true
	platforms: ${{ env.PLATFORMS }}
	tags: ${{ steps.meta-shard-manager.outputs.tags }}
	labels: ${{ steps.meta-shard-manager.outputs.labels }}
	- name: Extract metadata (tags, labels) for golem component service
	id: meta-golem-component-service
	uses: docker/metadata-action@v5
	with:
	images: golemservices/golem-component-service
	- name: Build and push golem component service image
	uses: docker/build-push-action@v5
	with:
	context: .
	file: ./golem-component-service/docker/Dockerfile
	push: true
	platforms: ${{ env.PLATFORMS }}
	tags: ${{ steps.meta-golem-component-service.outputs.tags }}
	labels: ${{ steps.meta-golem-component-service.outputs.labels }}
	- name: Extract metadata (tags, labels) for golem worker service
	id: meta-golem-worker-service
	uses: docker/metadata-action@v5
	with:
	images: golemservices/golem-worker-service
	- name: Build and push golem worker service image
	uses: docker/build-push-action@v5
	with:
	context: .
	file: ./golem-worker-service/docker/Dockerfile
	push: true
	platforms: ${{ env.PLATFORMS }}
	tags: ${{ steps.meta-golem-worker-service.outputs.tags }}
	labels: ${{ steps.meta-golem-worker-service.outputs.labels }}
	- name: Extract metadata (tags, labels) for golem component compilation service
	id: meta-golem-component-compilation-service
	uses: docker/metadata-action@v5
	with:
	images: golemservices/golem-component-compilation-service
	- name: Build and push golem component compilation service image
	uses: docker/build-push-action@v5
	with:
	context: .
	file: ./golem-component-compilation-service/docker/Dockerfile
	push: true
	platforms: ${{ env.PLATFORMS }}
	tags: ${{ steps.meta-golem-component-compilation-service.outputs.tags }}
	labels: ${{ steps.meta-golem-component-compilation-service.outputs.labels }}
	- name: Extract metadata (tags, labels) for golem router
	id: meta-golem-router
	uses: docker/metadata-action@v5
	with:
	images: golemservices/golem-router
	- name: Build and push golem router
	uses: docker/build-push-action@v5
	with:
	context: .
	file: ./golem-router/docker/Dockerfile
	push: true
	platforms: ${{ env.PLATFORMS }}
	tags: ${{ steps.meta-golem-router.outputs.tags }}
	labels: ${{ steps.meta-golem-router.outputs.labels }}

	build-and-test:
	runs-on: ubuntu-latest-xlarge
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Setup Rust
	run: rustup update stable --no-self-update && rustup default stable
	- uses: Swatinem/rust-cache@v2
	with:
	shared-key: debug
	cache-all-crates: true
	save-if: true
	- uses: davidB/rust-cargo-make@v1
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Build all targets
	run: cargo make --profile ci build
	- name: Check that OpenAPI specs are up-to-date
	run: cargo make --profile ci check-openapi
	- name: Check that configs are up-to-date
	run: cargo make --profile ci check-configs
	- name: Unit tests
	run: cargo make --profile ci unit-tests
	- name: Check formatting and clippy rules
	run: cargo make --profile ci check
	- name: Publish Test Report
	uses: mikepenz/action-junit-report@v4
	if: success() \|\| failure() # always run even if the previous step fails
	with:
	report_paths: '*/target/report-.xml'
	detailed_summary: true
	include_passed: true
	worker-tests-group1:
	runs-on: ubuntu-latest-xlarge
	name: worker-tests-group1
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Setup Rust
	run: rustup update stable --no-self-update && rustup default stable
	- uses: Swatinem/rust-cache@v2
	with:
	shared-key: debug-workertests
	cache-all-crates: true
	- uses: davidB/rust-cargo-make@v1
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Setup Redis
	uses: shogo82148/[email protected]
	with:
	redis-version: latest
	auto-start: false
	- name: Worker Executor integration tests
	run: cargo make --profile ci worker-executor-tests-group1
	timeout-minutes: 20
	- name: Publish Test Report
	uses: mikepenz/action-junit-report@v4
	if: success() \|\| failure() # always run even if the previous step fails
	with:
	report_paths: '*/target/report-.xml'
	detailed_summary: true
	include_passed: true
	worker-tests:
	needs: [ worker-tests-group1 ]
	runs-on: ubuntu-latest-xlarge
	strategy:
	fail-fast: true
	matrix:
	group:
	- name: group2
	- name: group3
	- name: group4
	- name: group5
	- name: group6
	- name: group7
	- name: group8
	name: worker-tests-${{ matrix.group.name }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Setup Rust
	run: rustup update stable --no-self-update && rustup default stable
	- uses: Swatinem/rust-cache@v2
	with:
	shared-key: debug-workertests
	cache-all-crates: true
	save-if: false
	- uses: davidB/rust-cargo-make@v1
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Setup Redis
	uses: shogo82148/[email protected]
	with:
	redis-version: latest
	auto-start: false
	- name: Worker Executor integration tests
	run: cargo make --profile ci worker-executor-tests-${{ matrix.group.name }}
	timeout-minutes: 40
	- name: Publish Test Report
	uses: mikepenz/action-junit-report@v4
	if: success() \|\| failure() # always run even if the previous step fails
	with:
	report_paths: '*/target/report-.xml'
	detailed_summary: true
	include_passed: true
	integration-tests:
	runs-on: ubuntu-latest-xlarge
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Setup Rust
	run: rustup update stable --no-self-update && rustup default stable
	- uses: Swatinem/rust-cache@v2
	with:
	shared-key: debug
	cache-all-crates: true
	save-if: false
	- uses: davidB/rust-cargo-make@v1
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Setup Redis
	uses: shogo82148/[email protected]
	with:
	redis-version: latest
	auto-start: false
	- name: Integration tests
	env:
	QUIET: true
	run: cargo make --profile ci integration-tests
	timeout-minutes: 30
	- name: Publish Test Report
	uses: mikepenz/action-junit-report@v4
	if: success() \|\| failure() # always run even if the previous step fails
	with:
	report_paths: '*/target/report-.xml'
	detailed_summary: true
	include_passed: true
	cli-tests:
	runs-on: ubuntu-latest-xlarge
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Setup Rust
	run: rustup update stable --no-self-update && rustup default stable
	- uses: Swatinem/rust-cache@v2
	with:
	shared-key: debug
	cache-all-crates: true
	save-if: false
	- uses: davidB/rust-cargo-make@v1
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Setup Redis
	uses: shogo82148/[email protected]
	with:
	redis-version: latest
	auto-start: false
	- name: CLI tests
	env:
	QUIET: true
	run: cargo make --profile ci cli-tests
	timeout-minutes: 35
	- name: Publish Test Report
	uses: mikepenz/action-junit-report@v4
	if: success() \|\| failure() # always run even if the previous step fails
	with:
	report_paths: '*/target/report-.xml'
	detailed_summary: true
	include_passed: true
	sharding-tests:
	runs-on: ubuntu-latest-xlarge
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Setup Rust
	run: rustup update stable --no-self-update && rustup default stable
	- uses: Swatinem/rust-cache@v2
	with:
	shared-key: debug
	cache-all-crates: true
	save-if: false
	- uses: davidB/rust-cargo-make@v1
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: Setup Redis
	uses: shogo82148/[email protected]
	with:
	redis-version: latest
	auto-start: false
	- name: Sharding tests
	env:
	QUIET: true
	run: cargo make --profile ci sharding-tests
	timeout-minutes: 40
	- name: Publish Test Report
	uses: mikepenz/action-junit-report@v4
	if: success() \|\| failure() # always run even if the previous step fails
	with:
	report_paths: '*/target/report-.xml'
	detailed_summary: true
	include_passed: true
	publish:
	needs:
	[
	build-and-test,
	worker-tests,
	integration-tests,
	cli-tests,
	sharding-tests,
	]
	if: "startsWith(github.ref, 'refs/tags/v')"
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Setup Rust
	run: rustup update stable --no-self-update && rustup default stable
	- uses: Swatinem/rust-cache@v2
	with:
	shared-key: release
	cache-all-crates: true
	- uses: davidB/rust-cargo-make@v1
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- id: get_version
	uses: battila7/get-version-action@v2
	- name: Publish all packages
	env:
	CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
	VERSION: ${{ steps.get_version.outputs.version-without-v }}
	run: cargo make --profile ci publish

	publish_cli_binaries:
	if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'golemcloud/golem'
	strategy:
	fail-fast: false
	matrix:
	project:
	- golem-cli
	- golem
	cfg:
	- rust-target: x86_64-unknown-linux-gnu
	os: ubuntu-latest-xlarge
	- rust-target: aarch64-unknown-linux-gnu
	os: ubuntu-latest-xlarge
	cross: true
	- rust-target: x86_64-apple-darwin
	os: macos-latest
	- rust-target: aarch64-apple-darwin
	os: macos-latest
	- rust-target: x86_64-pc-windows-gnu
	os: windows-latest
	exclude:
	- project: golem
	cfg:
	rust-target: x86_64-pc-windows-gnu
	os: windows-latest # excluded for now because sozu-proxy is not compilable on windows
	name: Publish binaries of ${{ matrix.project }}
	runs-on: ${{ matrix.cfg.os }}
	permissions:
	contents: write
	steps:
	- uses: actions/checkout@v4
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive
	- name: Fetch tag
	run: git fetch origin --deepen=1
	- name: Setup Rust
	run: rustup update stable --no-self-update && rustup default stable && rustup target add ${{ matrix.cfg.rust-target }}
	- uses: davidB/rust-cargo-make@v1
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- id: get_version
	uses: battila7/get-version-action@v2
	- name: Set version for all packages
	env:
	VERSION: ${{ steps.get_version.outputs.version-without-v }}
	run: cargo make --profile ci set-version

	- run: cargo install cross
	if: ${{ matrix.cfg.cross }}
	- uses: ilammy/setup-nasm@v1

	- run: cargo build -p ${{ matrix.project }} --release --target ${{ matrix.cfg.rust-target }}
	if: ${{ ! matrix.cfg.cross }}
	- run: cross build -p ${{ matrix.project }} --release --target ${{ matrix.cfg.rust-target }}
	if: ${{ matrix.cfg.cross }}

	- run: mv ./target/${{ matrix.cfg.rust-target }}/release/${{ matrix.project }}.exe ./target/${{ matrix.cfg.rust-target }}/release/${{ matrix.project }}-${{ matrix.cfg.rust-target }}.exe
	if: matrix.cfg.os == 'windows-latest'
	- run: mv ./target/${{ matrix.cfg.rust-target }}/release/${{ matrix.project }} ./target/${{ matrix.cfg.rust-target }}/release/${{ matrix.project }}-${{ matrix.cfg.rust-target }}
	if: matrix.cfg.os != 'windows-latest'
	- name: Login GH CLI
	shell: bash
	run: gh auth login --with-token < <(echo ${{ secrets.GITHUB_TOKEN }})
	- run: gh release upload -R golemcloud/golem --clobber ${{ github.ref_name }} target/${{ matrix.cfg.rust-target }}/release/${{ matrix.project }}-${{ matrix.cfg.rust-target }}.exe
	if: matrix.cfg.os == 'windows-latest'
	- run: gh release upload -R golemcloud/golem --clobber ${{ github.ref_name }} target/${{ matrix.cfg.rust-target }}/release/${{ matrix.project }}-${{ matrix.cfg.rust-target }}
	if: matrix.cfg.os != 'windows-latest'

	publish-slack-notification:
	if: ${{ always() && startsWith(github.ref, 'refs/tags/v') }}
	needs: [ publish, docker-publish, publish_cli_binaries ]
	runs-on: ubuntu-latest
	steps:
	- name: Publish Slack Notification
	uses: rtCamp/action-slack-notify@v2
	env:
	SLACK_COLOR: ${{ needs.publish.result }}
	SLACK_ICON: https://uploads-ssl.webflow.com/64721eeec7cd7ef4f6f1683e/64831138b73a996d0e831773_32.png
	SLACK_TITLE: "Release Notification"
	SLACK_MESSAGE: "Publish - cargo result: `${{ needs.publish.result }}`, cli bin result: `${{ needs.publish_cli_binaries.result }}`, docker result: `${{ needs.docker-publish.result }}`"
	SLACK_USERNAME: CI
	SLACK_WEBHOOK: ${{ secrets.SLACK_ALERT_URL }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Send content length in S3 put_stream implementation (#1143) #4001

Workflow file

Send content length in S3 put_stream implementation (#1143) #4001

Jobs

Run details

Workflow file for this run