datadog-agent/7.57.1 package update (#28672) #17300

Workflow file for this run

	name: Build Wolfi OS

	on:
	push:
	branches: ['main']
	paths-ignore:
	- '**.md'
	- '**.txt'

	workflow_dispatch:

	# Only run one build at a time to prevent out of sync signatures.
	concurrency: build

	jobs:
	build:
	name: Build packages
	if: github.repository == 'wolfi-dev/os'

	strategy:
	matrix:
	arch: [ "x86_64", "aarch64" ]
	fail-fast: false

	runs-on:
	group: wolfi-os-builder-${{ matrix.arch }}

	permissions:
	contents: read

	container:
	image: ghcr.io/wolfi-dev/sdk:latest@sha256:e8c9680e3262d27b28c38e84f51f8a8587c84dc192b0f198b96b11de27aafc34
	# TODO: Deprivilege
	options: \|
	--cap-add NET_ADMIN --cap-add SYS_ADMIN --device /dev/fuse --security-opt seccomp=unconfined --security-opt apparmor:unconfined

	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

	- name: 'Trust the github workspace'
	run: \|
	# This is to avoid fatal errors about "dubious ownership" because we are
	# running inside of a container action with the workspace mounted in.
	git config --global --add safe.directory "$(pwd)"

	# Build with a local key, we'll resign this with the real key later
	- name: 'Generate local signing key'
	run: \|
	make local-melange.rsa

	# Touch it with the epoch date to convince `make` that we don't need to
	# rebuild the targets that depend on this (all)
	touch -d @0 local-melange.rsa

	- name: 'Prepare package repository'
	run: \|
	# yay wolfi!
	apk add gcsfuse google-cloud-sdk

	# Set up a gcsfuse RO mount to the public bucket. This is a cheap and
	# cheerful way to recreate the make targets (class A HEADs) locally
	# without syncing the whole bucket (class A+B).
	mkdir -p /gcsfuse/wolfi-registry
	gcsfuse -o ro --implicit-dirs --only-dir os wolfi-production-registry-destination /gcsfuse/wolfi-registry

	mkdir -p ./packages/${{ matrix.arch }}
	# Symlink the gcsfuse mount to ./packages/ to workaround the Makefile CWD assumptions
	for f in /gcsfuse/wolfi-registry/${{ matrix.arch }}/*.apk; do
	ln -s "$f" ./packages/${{ matrix.arch }}/
	done

	# Make a copy of the APKINDEX.* since we'll need to write to it on package builds
	cp /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.* ./packages/${{ matrix.arch }}/

	# TODO: Remove the previous and next steps by folding the logic into wolfictl.
	- name: 'Build Wolfi'
	run: \|
	wolfictl build \
	--runner bubblewrap \
	--keyring-append /gcsfuse/wolfi-registry/wolfi-signing.rsa.pub \
	--repository-append ./packages \
	--keyring-append local-melange.rsa.pub \
	--repository-append https://packages.wolfi.dev/os \
	--keyring-append https://packages.wolfi.dev/os/wolfi-signing.rsa.pub \
	--signing-key local-melange.rsa \
	--arch ${{ matrix.arch }} \
	--namespace wolfi \
	--pipeline-dir ./pipelines/ \
	--destination-repository https://packages.wolfi.dev/os \
	--trace ./packages/${{ matrix.arch }}/trace.json

	# Always run this step for https://github.com/wolfi-dev/os/issues/8698
	- if: ${{ always() }}
	name: 'Create artifacts tarball'
	run: \|
	set -x
	set -e
	set -o pipefail

	# Overwrite the APKINDEX.tar.gz we just generated with the original one.
	# Since we use a temporary key during the build, we need to re-sign the APKs later.
	# We don't want to keep the newly indexed stuff because the size field is probably wrong.
	cp /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.* ./packages/${{ matrix.arch }}/

	# Pick up any stragglers that didn't get uploaded in previous builds.
	cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz \| tar -Oxz APKINDEX \| awk -F':' '$1 == "P" {printf "%s-", $2} $1 == "V" {printf "%s.apk\n", $2}' \| sort > indexed.txt
	# TODO: Figure out why ls through gcsfuse is so slow.
	gcloud storage ls gs://wolfi-production-registry-destination/os/${{ matrix.arch }} \| grep ".apk$" \| xargs -n1 basename \| sort > uploaded.txt

	# Lines that are only in uploaded.txt and not indexed.txt.
	comm -13 indexed.txt uploaded.txt > missing.txt

	# Clean up the symlinks to keep only packages we built.
	find ./packages/${{ matrix.arch }} -type l -exec rm -f {} \;

	# Merge any missing APKs into our new index.
	for missed in $(cat missing.txt); do
	# We could do this in one command instead of a loop, but it takes things on argv, which is a bit annoying.
	melange index --merge \
	--source ./packages/${{ matrix.arch }}/APKINDEX.tar.gz \
	--output new.tar.gz \
	/gcsfuse/wolfi-registry/${{ matrix.arch }}/${missed}

	# Overwrite what we're going to upload (and for the next loop).
	mv new.tar.gz ./packages/${{ matrix.arch }}/APKINDEX.tar.gz
	done

	diff \
	<(cat /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.tar.gz \| tar -Oxz APKINDEX) \
	<(cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz \| tar -Oxz APKINDEX) \|\| true

	# Move logs so we can upload them separately.
	mv ./packages/${{ matrix.arch }}/buildlogs /tmp/buildlogs

	# Move trace so we can upload it separately.
	mv ./packages/${{ matrix.arch }}/trace.json /tmp/trace.json

	# Create an archive for uploading
	tar -cvzf /tmp/packages-${{ matrix.arch }}.tar.gz ./packages/${{ matrix.arch }}

	# Always run these steps for https://github.com/wolfi-dev/os/issues/8698
	- if: ${{ always() }}
	name: 'Upload logs archive to GitHub Artifacts'
	uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
	with:
	name: logs-${{ matrix.arch }}
	path: /tmp/buildlogs/
	if-no-files-found: warn
	- if: ${{ always() }}
	name: 'Upload trace to GitHub Artifacts'
	uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
	with:
	name: trace-${{ matrix.arch }}
	path: /tmp/trace.json
	if-no-files-found: warn
	- if: ${{ always() }}
	name: 'Upload built packages archive to GitHub Artifacts'
	uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
	with:
	name: packages-${{ matrix.arch }}
	path: /tmp/packages-${{ matrix.arch }}.tar.gz
	retention-days: 1 # Low ttl since this is just an intermediary used once
	if-no-files-found: warn

	upload-packages:
	runs-on: ubuntu-latest-16-cores
	needs: build

	# Always run this job for https://github.com/wolfi-dev/os/issues/8698
	if: ${{ always() }}

	permissions:
	id-token: write
	contents: read

	container:
	# NOTE: This step only signs and uploads, so it doesn't need any privileges
	image: ghcr.io/wolfi-dev/sdk:latest@sha256:e8c9680e3262d27b28c38e84f51f8a8587c84dc192b0f198b96b11de27aafc34

	steps:
	- name: Harden Runner
	uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
	with:
	egress-policy: audit

	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

	- name: Free up runner disk space
	run: \|
	set -x
	rm -rf /usr/share/dotnet
	rm -rf "$AGENT_TOOLSDIRECTORY"

	- name: 'Trust the github workspace'
	run: \|
	# This is to avoid fatal errors about "dubious ownership" because we are
	# running inside of a container action with the workspace mounted in.
	git config --global --add safe.directory "$(pwd)"

	- name: 'Download x86_64 package archives'
	uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
	with:
	path: /tmp/artifacts/
	name: packages-x86_64

	- name: 'Download aarch64 package archives'
	uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
	with:
	path: /tmp/artifacts/
	name: packages-aarch64

	# This is managed here: https://github.com/chainguard-dev/secrets/blob/main/wolfi-dev.tf
	- uses: google-github-actions/auth@62cf5bd3e4211a0a0b51f2c6d6a37129d828611d # v2.1.5
	id: auth
	with:
	workload_identity_provider: "projects/12758742386/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
	service_account: "[email protected]"
	- uses: google-github-actions/setup-gcloud@f0990588f1e5b5af6827153b93673613abdc6ec7 # v2.1.1
	with:
	project_id: "chainguard-github-secrets"
	- uses: 'google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713' # v2.1.4
	id: secrets
	with:
	secrets: \|-
	token:chainguard-github-secrets/wolfi-dev-signing-key

	- run: echo "${{ steps.secrets.outputs.token }}" > ./wolfi-signing.rsa
	- run: \|
	mkdir -p /etc/apk/keys
	cp ./wolfi-signing.rsa.pub /etc/apk/keys/wolfi-signing.rsa.pub

	- name: 'Update the APKINDEX'
	run: \|
	for arch in "aarch64" "x86_64"; do
	mkdir -p ./packages/${arch}

	# Consolidate with the built artifacts
	tar xvf /tmp/artifacts/packages-${arch}.tar.gz

	# Only attempt to sign when *.apk's exist.
	apks=$(ls ./packages/${arch}/*.apk 2>/dev/null \|\| true)
	if [ -n "$apks" ]; then
	melange sign --signing-key ./wolfi-signing.rsa ./packages/${arch}/*.apk

	melange index --merge \
	--source ./packages/${arch}/APKINDEX.tar.gz \
	--output ./packages/${arch}/APKINDEX.tar.gz \
	./packages/${arch}/*.apk
	fi

	# Sign the APK index
	melange sign-index -f --signing-key ./wolfi-signing.rsa packages/${arch}/APKINDEX.tar.gz
	done

	# Clean up the signing key before uploading to storage out
	# of an abundance of caution.
	- run: rm ./wolfi-signing.rsa

	# We use a different GSA for our interaction with GCS.
	- uses: google-github-actions/auth@62cf5bd3e4211a0a0b51f2c6d6a37129d828611d # v2.1.5
	with:
	workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha"
	service_account: "[email protected]"
	- uses: google-github-actions/setup-gcloud@f0990588f1e5b5af6827153b93673613abdc6ec7 # v2.1.1
	with:
	project_id: "prod-images-c6e5"

	- name: 'Upload packages to GCS'
	run: \|
	for arch in "aarch64" "x86_64"; do
	# Only attempt to upload when *.apk's exist
	apks=$(ls ./packages/${arch}/*.apk 2>/dev/null \|\| true)
	if [ -n "$apks" ]; then
	# apks will be cached in CDN for an hour by default.
	# Don't upload the object if it already exists.
	gcloud --quiet storage cp \
	--no-clobber \
	"./packages/${arch}/*.apk" "gs://wolfi-production-registry-destination/os/${arch}/"
	fi
	done

	- name: 'Create APKINDEX tarball'
	run: \|
	# Tar up any 'APKINDEX.*' files {aarch64,x86_64} x {tar.gz,json}
	find ./packages/ -name 'APKINDEX.*' > to-include
	tar -cvzf /tmp/indexes.tar.gz --files-from to-include

	- name: 'Upload APKINDEX archive to GitHub Artifacts'
	uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
	with:
	name: indexes
	path: /tmp/indexes.tar.gz
	retention-days: 1 # Low ttl since this is just an intermediary used once
	if-no-files-found: warn

	upload-index:
	runs-on: ubuntu-latest-16-cores
	needs: upload-packages

	permissions:
	id-token: write
	contents: read

	container:
	# NOTE: This step only signs and uploads, so it doesn't need any privileges
	image: ghcr.io/wolfi-dev/sdk:latest@sha256:e8c9680e3262d27b28c38e84f51f8a8587c84dc192b0f198b96b11de27aafc34

	steps:
	- name: Harden Runner
	uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
	with:
	egress-policy: audit

	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

	- name: 'Trust the github workspace'
	run: \|
	# This is to avoid fatal errors about "dubious ownership" because we are
	# running inside of a container action with the workspace mounted in.
	git config --global --add safe.directory "$(pwd)"

	- id: auth
	name: 'Authenticate to Google Cloud'
	uses: google-github-actions/auth@62cf5bd3e4211a0a0b51f2c6d6a37129d828611d # v2.1.5
	with:
	workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha"
	service_account: "[email protected]"

	- uses: google-github-actions/setup-gcloud@f0990588f1e5b5af6827153b93673613abdc6ec7 # v2.1.1
	with:
	project_id: prod-images-c6e5

	- name: 'Download index archive'
	uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
	with:
	path: /tmp/artifacts/
	name: indexes

	- name: 'Upload indexes to GCS'
	run: \|
	tar xvf /tmp/artifacts/indexes.tar.gz

	for arch in "aarch64" "x86_64"; do
	# Don't cache the APKINDEX.
	gcloud --quiet storage cp \
	--cache-control=no-store \
	"./packages/${arch}/APKINDEX.tar.gz" "gs://wolfi-production-registry-destination/os/${arch}/"

	gcloud --quiet storage cp \
	--cache-control=no-store \
	"./packages/${arch}/APKINDEX.json" "gs://wolfi-production-registry-destination/os/${arch}/"
	done

	upload-packages-to-cgr:
	runs-on: ubuntu-latest
	needs: build

	# Always run this job for https://github.com/wolfi-dev/os/issues/8698
	if: ${{ always() }}

	permissions:
	id-token: write
	contents: read

	steps:
	- uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
	with:
	egress-policy: audit

	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

	- name: 'Trust the github workspace'
	run: \|
	# This is to avoid fatal errors about "dubious ownership" because we are
	# running inside of a container action with the workspace mounted in.
	git config --global --add safe.directory "$(pwd)"

	- name: 'Download x86_64 package archives'
	uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
	with:
	path: /tmp/artifacts/
	name: packages-x86_64

	- name: 'Download aarch64 package archives'
	uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
	with:
	path: /tmp/artifacts/
	name: packages-aarch64

	- name: 'Unpack the package archives'
	run: \|
	for arch in "aarch64" "x86_64"; do
	mkdir -p ./packages/${arch}

	# Consolidate with the built artifacts
	tar xvf /tmp/artifacts/packages-${arch}.tar.gz
	done

	# use public chainguard provider.
	- uses: chainguard-dev/setup-chainctl@f52718d822dc73d21a04ef2082822c4a203163b3 # v0.2.2
	with:
	# Managed here:
	# https://github.com/chainguard-dev/mono/blob/main/env/chainguard-images/iac/wolfi-os-pusher.tf
	identity: "720909c9f5279097d847ad02a2f24ba8f59de36a/6a26f2970f880c31"

	- name: 'Upload packages to apk.cgr.dev'
	run: \|
	set -ex
	# Populate the token here, since chainctl auth token
	# doesn't support all of the options we need.
	chainctl auth login --audience apk.cgr.dev \
	--identity "720909c9f5279097d847ad02a2f24ba8f59de36a/6a26f2970f880c31"
	tok=$(chainctl auth token --audience apk.cgr.dev)
	echo "::add-mask::${tok}"

	for arch in "aarch64" "x86_64"; do
	# Only attempt to upload when *.apk's exist
	apks=$(ls ./packages/${arch}/*.apk 2>/dev/null \|\| true)
	if [ -n "$apks" ]; then
	for apk in ${apks}; do
	package="$(basename ${apk})"

	# Check if package already exists in apk.cgr.dev
	code=$(curl -s -o /dev/null --head -w "%{http_code}" --user "user:${tok}" "https://apk.cgr.dev/chainguard/${arch}/${package}")
	if [ $code == "303" ]; then
	echo "Package already exists: ${package}"
	continue
	elif [ $code != "404" ]; then
	echo "Unexpected response code: $code"
	exit 1
	fi

	curl --fail -X POST \
	--user "user:${tok}" \
	--data-binary "@${apk}" \
	"https://apk.cgr.dev/chainguard/${arch}/${package}"
	done
	fi
	done

	postrun:
	name: Notify Slack
	runs-on: ubuntu-latest
	if: failure()
	needs: [build, upload-packages, upload-packages-to-cgr, upload-index]
	steps:
	- name: Harden Runner
	uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
	with:
	egress-policy: audit

	- uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 # v2.2.1
	env:
	SLACK_ICON: http://github.com/chainguard-dev.png?size=48
	SLACK_USERNAME: guardian
	SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
	SLACK_CHANNEL: C047DK5BUNP
	SLACK_MSG_AUTHOR: wolfi-bot
	SLACK_COLOR: '#8E1600'
	MSG_MINIMAL: 'true'
	SLACK_TITLE: '[build-wolfi-os] failure: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
	SLACK_MESSAGE: \|
	https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

datadog-agent/7.57.1 package update (#28672) #17300

Workflow file

datadog-agent/7.57.1 package update (#28672) #17300

Jobs

Run details

Workflow file for this run