diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..b96119e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,125 @@ +name: build +run-name: ${{ github.ref_type == 'tag' && github.ref_name || 'experimental' }} +on: + push: + tags: + - '*' + branches: + - main + workflow_dispatch: + inputs: + today: + type: boolean + default: false +jobs: + build: + runs-on: ubuntu-latest + environment: aws + permissions: + id-token: write + steps: + - name: setup binfmt + run: sudo podman run --privileged ghcr.io/gardenlinux/binfmt_container + - uses: actions/checkout@v4 + - name: resolve container digest + if: github.ref_type != 'tag' + run: | + set -o noclobber + if [ ! -e .container ]; then + image="ghcr.io/gardenlinux/repo-debian-snapshot" + podman pull "$image" + digest="$(podman image inspect --format '{{ .Digest }}' "$image")" + echo "$image@$digest" > .container + fi + - name: fetch package repo releases + if: github.ref_type != 'tag' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + podman build -t build --build-arg base="$(cat .container)" . + podman run --rm -e GH_TOKEN build /fetch_releases > package-releases + - name: download amd64 packages + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + container="$(cat .container)" + podman pull --arch amd64 "$container" + podman build -t build --build-arg base="$container" . + mkdir repo + podman run --rm -v "$PWD/repo:/repo" -v "$PWD/package-releases:/package-releases" -v "$PWD/package-imports:/package-imports" -e GH_TOKEN build /download_pkgs /repo /package-releases /package-imports + - name: download arm64 packages + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + container="$(cat .container)" + podman pull --arch arm64 "$container" + podman build -t build --build-arg base="$container" . + mkdir repo_arm64 + podman run --rm -v "$PWD/repo_arm64:/repo" -v "$PWD/package-releases:/package-releases" -v "$PWD/package-imports:/package-imports" -e GH_TOKEN build /download_pkgs /repo /package-releases /package-imports + cp -r --no-clobber repo_arm64/. repo/. + rm -rf repo_arm64 + - name: build kms signing container + run: | + podman build -t kms kms + podman build -t build --build-arg base=kms . + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_OIDC_ROLE }} + aws-region: ${{ vars.AWS_REGION }} + - run: aws sts get-caller-identity + - name: sync pool to S3 + run: | + find repo/pool -type f -printf '%P\n' | sort > local_objects + aws s3api list-objects --bucket '${{ vars.S3_BUCKET }}' --prefix pool/ | jq -r '.Contents // [] | .[].Key' | sed 's#^pool/##' | sort > aws_objects + join -v 1 local_objects aws_objects > new_objects + rm local_objects aws_objects + num_objects="$(wc -l new_objects | awk '{ print $1 }')" + cntr=0 + while read -r obj; do + aws s3 cp --quiet "repo/pool/$obj" "s3://${{ vars.S3_BUCKET }}/pool/$obj" + cntr="$(( cntr + 1 ))" + echo "[$cntr/$num_objects] $obj" + done < new_objects + rm new_objects + - name: check dist ${{ github.ref_name }} + if: github.ref_type == 'tag' + id: check + run: | + if ! aws s3api head-object --bucket '${{ vars.S3_BUCKET }}' --key 'gardenlinux/dists/${{ github.ref_name }}/InRelease' > /dev/null 2>&1; then + echo new_dist=true >> "$GITHUB_OUTPUT" + fi + - name: create dist ${{ github.ref_name }} + if: steps.check.outputs.new_dist == 'true' + run: | + podman run --rm \ + -e 'AWS_*' \ + -e 'KMS_KEY_ID=${{ secrets.KMS_KEY_ID }}' \ + -e 'KMS_KEY_CERT=${{ secrets.KMS_KEY_CERT }}' \ + -e 'KMS_KEY_GPG=${{ secrets.KMS_KEY_GPG }}' \ + -v "$PWD/repo:/repo" \ + build /create_dist /repo ${{ github.ref_name }} 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + aws s3 cp --recursive 'repo/dists/${{ github.ref_name }}' 's3://${{ vars.S3_BUCKET }}/gardenlinux/dists/${{ github.ref_name }}' + - name: create dist experimental + if: github.ref_type != 'tag' + run: | + podman run --rm \ + -e 'AWS_*' \ + -e 'KMS_KEY_ID=${{ secrets.KMS_KEY_ID }}' \ + -e 'KMS_KEY_CERT=${{ secrets.KMS_KEY_CERT }}' \ + -e 'KMS_KEY_GPG=${{ secrets.KMS_KEY_GPG }}' \ + -v "$PWD/repo:/repo" \ + build /create_dist /repo experimental 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + aws s3 cp --recursive 'repo/dists/experimental' 's3://${{ vars.S3_BUCKET }}/gardenlinux/dists/experimental' + aws cloudfront create-invalidation --distribution-id '${{ secrets.CLOUDFRONT_DISTRIBUTION }}' --paths '/gardenlinux/dists/experimental/*' + - name: create dist today + if: inputs.today + run: | + podman run --rm \ + -e 'AWS_*' \ + -e 'KMS_KEY_ID=${{ secrets.KMS_KEY_ID }}' \ + -e 'KMS_KEY_CERT=${{ secrets.KMS_KEY_CERT }}' \ + -e 'KMS_KEY_GPG=${{ secrets.KMS_KEY_GPG }}' \ + -v "$PWD/repo:/repo" \ + build /create_dist /repo today 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + aws s3 cp --recursive 'repo/dists/today' 's3://${{ vars.S3_BUCKET }}/gardenlinux/dists/today' + aws cloudfront create-invalidation --distribution-id '${{ secrets.CLOUDFRONT_DISTRIBUTION }}' --paths '/gardenlinux/dists/today/*' diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml new file mode 100644 index 0000000..73060a4 --- /dev/null +++ b/.github/workflows/update.yml @@ -0,0 +1,45 @@ +name: update +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' +jobs: + update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: resolve GL version + id: version + run: | + gl_start=2020-03-31 + currentdate="$(date -u '+%Y-%m-%d')" + datediff=$(( "$(date -u -d "$currentdate" '+%s')" - "$(date -u -d "$gl_start" '+%s')" )) + days_since_start=$(( datediff / 86400 )) + echo "version=$days_since_start.0" >> "$GITHUB_OUTPUT" + - name: resolve container digest + run: | + set -o noclobber + image="ghcr.io/gardenlinux/repo-debian-snapshot" + podman pull "$image" + digest="$(podman image inspect --format '{{ .Digest }}' "$image")" + echo "$image@$digest" > .container + - name: fetch package repo releases + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + podman build -t build --build-arg base="$(cat .container)" . + podman run --rm -e GH_TOKEN build /fetch_releases > package-releases + - name: commit + run: | + git checkout --detach HEAD + git add . + git config user.email "actions@github.com" + git config user.name "GitHub Actions" + git commit -m "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + git tag '${{ steps.version.outputs.version }}' + git push origin '${{ steps.version.outputs.version }}' + - name: dispatch build + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh workflow run --ref '${{ steps.version.outputs.version }}' -F today=true build.yml diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..94b8351 --- /dev/null +++ b/Containerfile @@ -0,0 +1,4 @@ +ARG base +FROM $base +RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y aptitude ca-certificates dpkg-dev gh jq +COPY fetch_releases download_pkgs create_dist / diff --git a/README.md b/README.md new file mode 100644 index 0000000..b5ea1c9 --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +# Gardenlinux Repo Infrastructure + +```mermaid +flowchart TD + repo[gardenlinux/repo] + snapshot[gardenlinux/repo-debian-snapshot] + pkg_build[gardenlinux/package-build] + pkg[gardenlinux/package-*] + ghcr_snapshot[ghcr.io/gardenlinux/repo-debian-snapshot] + s3[s3://gardenlinux-repo/gardenlinux] + s3_snapshot[s3://gardenlinux-repo/debian-snapshot] + deb[apt://deb.debian.org/debian] + + deb -- mirror --> snapshot + snapshot -- publish --> s3_snapshot + s3_snapshot -- ref --> ghcr_snapshot + snapshot -- publish --> ghcr_snapshot + + pkg_build -- use workflow / tooling --> pkg + ghcr_snapshot -- runs in --> pkg + + pkg -- get release artifacts --> repo + s3_snapshot -- get dependencies and imports --> repo + repo -- publish --> s3 +``` + +## GitHub + +- `gardenlinux/repo` + - collect packages from `package-*` repos, fetch all dependecies from debian snapshot and publish into a repo dist +- `gardenlinux/repo-debian-snapshot` + - regularly snapshot debian testing (needed for reproducible package and repo builds) +- `gardenlinux/package-build` + - tooling used by `package-*` repos to build binary artifacts +- `gardenlinux/package-*` + - repos for custom build packages + +## AWS + +- bucket: `gardenlinux-repo` + - `/pool` for all package files + - `/gardenlinux` for gardenlinux release dists + - `/debian-snapshot` for time stamp indexed debian testing snapshot dists +- cloudfront: `E2RAO851VDQ2KX` + - proxies bucket `gardenlinux-repo` using lambda `repoPathRewrite` to fix problem with aws S3 http endpoint handling `+` in filenames incorrectly and redirects all requests for `/*/pool` to `/pool` => allowing to use a shared pool directory for gardenlinux repo and debian-snapshot +- role: `github-repo-oidc-role` + - allows all actions running in an environment 'aws' from repos matching 'gardenlinux/repo-*' to access +- policy: `github-repo-policy` + - gives read/write access to S3 bucket `gardenlinux-repo` + - gives access to gardenlinux repo signing key on KMS diff --git a/create_dist b/create_dist new file mode 100755 index 0000000..f8d7606 --- /dev/null +++ b/create_dist @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -exufo pipefail + +target="$(realpath "$1")" +dist="$2" +description="$3" + +index_files="$(mktemp)" + +for arch in all amd64 arm64; do + dir="$target/dists/$dist/main/binary-$arch" + mkdir -p "$dir" + (cd "$target" && dpkg-scanpackages --arch "$arch" pool) > "$dir/Packages" + + size="$(wc -c "$dir/Packages" | awk '{ print $1 }')" + hash="$(sha256sum "$dir/Packages" | head -c 64)" + echo " $hash $size main/binary-$arch/Packages" >> "$index_files" + gzip < "$dir/Packages" > "$dir/Packages.gz" + rm "$dir/Packages" + size="$(wc -c "$dir/Packages.gz" | awk '{ print $1 }')" + hash="$(sha256sum "$dir/Packages.gz" | head -c 64)" + echo " $hash $size main/binary-$arch/Packages.gz" >> "$index_files" +done + +date="$(date -R -u)" + +cat << EOF | gpg --clearsign > "$target/dists/$dist/InRelease" +Codename: $dist +Description: $description +Components: main +Architectures: all amd64 arm64 +Date: $(date -R -u -d "$date") +Valid-Until: $(date -R -u -d "$date + 100 years") +SHA256: +$(cat "$index_files") + +EOF + +rm "$index_files" diff --git a/download_pkgs b/download_pkgs new file mode 100755 index 0000000..86c543e --- /dev/null +++ b/download_pkgs @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -exufo pipefail + +target="$(realpath "$1")" +releases="$(realpath "$2")" +import_list="$(realpath "$3")" + +dir="$(mktemp -d)" +trap 'cd / && rm -rf "$dir"' EXIT +cd "$dir" + +arch="$(dpkg --print-architecture)" + +mkdir download +while read -r repo tag; do + gh release download --dir download --repo "$repo" "$tag" +done < "$releases" + +truncate -s 0 pkgs depends + +find download -name "*_all.deb" -or -name "*_$arch.deb" | sort | while read -r pkg; do + control="$(dpkg-deb -I "$pkg" control)" + awk -F ': ' '$1 == "Package" { print $2 }' <<< "$control" >> pkgs + awk -F ': ' '$1 == "Depends" || $1 == "Pre-Depends" { print $2 }' <<< "$control" | tr ',' '\n' | awk '{ print $1 }' >> depends + + hash="$(sha256sum < "$pkg" | head -c 64)" + mkdir -p "$target/pool/$hash" + cp --update=none "$pkg" "$target/pool/$hash/" +done + +rm -rf download +sort -o pkgs -u pkgs + +cat "$import_list" >> depends +aptitude search '?priority(required)|?priority(important)' -F '%p' -q | cut -d : -f 1 >> depends + +sort -o depends -u depends +join -v 1 depends pkgs > needed + +xargs apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances < needed | grep '^\w' | cut -d : -f 1 | sort | uniq > recursive_depends +join -v 1 recursive_depends pkgs > recursive_needed + +mkdir apt_download +(cd apt_download && xargs apt-get download) < recursive_needed + +find apt_download -name "*.deb" | while read -r pkg; do + hash="$(sha256sum < "$pkg" | head -c 64)" + mkdir -p "$target/pool/$hash" + cp --update=none "$pkg" "$target/pool/$hash/" +done + +rm -rf apt_download diff --git a/fetch_releases b/fetch_releases new file mode 100755 index 0000000..71f00ed --- /dev/null +++ b/fetch_releases @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -eufo pipefail + +org=gardenlinux +match_pattern='^package-.*' +exclude_pattern='package-build' + +gh api --paginate "/orgs/$org/repos" | jq -r '.[] | .name' | grep -xE "$match_pattern" | grep -vxE "$exclude_pattern" | while read -r repo; do + tag="$(gh api "/repos/$org/$repo/releases/latest" 2> /dev/null | jq -r '.tag_name // ""' || true)" + [ -z "$tag" ] || echo "$org/$repo $tag" +done diff --git a/kms/Containerfile b/kms/Containerfile new file mode 100644 index 0000000..62976e2 --- /dev/null +++ b/kms/Containerfile @@ -0,0 +1,7 @@ +FROM debian:stable +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl gawk gnupg gnupg-pkcs11-scd libjson-c5 libcurl4 +RUN curl -sSLf "https://github.com/gardenlinux/aws-kms-pkcs11/releases/download/latest/aws_kms_pkcs11-$(dpkg --print-architecture).so" > /aws_kms_pkcs11.so +RUN mkdir -m 700 /root/.gnupg +COPY gpg-agent.conf gnupg-pkcs11-scd.conf /root/.gnupg/ +COPY init / +ENTRYPOINT [ "/init" ] diff --git a/kms/gnupg-pkcs11-scd.conf b/kms/gnupg-pkcs11-scd.conf new file mode 100644 index 0000000..f7e77d1 --- /dev/null +++ b/kms/gnupg-pkcs11-scd.conf @@ -0,0 +1,2 @@ +providers kms +provider-kms-library /aws_kms_pkcs11.so diff --git a/kms/gpg-agent.conf b/kms/gpg-agent.conf new file mode 100644 index 0000000..2553022 --- /dev/null +++ b/kms/gpg-agent.conf @@ -0,0 +1 @@ +scdaemon-program /usr/bin/gnupg-pkcs11-scd diff --git a/kms/init b/kms/init new file mode 100755 index 0000000..bb313b0 --- /dev/null +++ b/kms/init @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -eufo pipefail + +mkdir -p "$HOME/.config/aws-kms-pkcs11" +cat > "$HOME/.config/aws-kms-pkcs11/config.json" << EOF +{ + "slots": [ + { + "kms_key_id": "$KMS_KEY_ID", + "certificate": "$KMS_KEY_CERT" + } + ] +} +EOF + +gpg --card-status +base64 -d <<< "$KMS_KEY_GPG" | gpg --import +gpg --list-secret-keys + +"$@" diff --git a/package-imports b/package-imports new file mode 100644 index 0000000..b677212 --- /dev/null +++ b/package-imports @@ -0,0 +1,155 @@ +acl +aide +aide-common +amazon-ec2-utils +apparmor +arptables +auditd +awscli +azure-cli +binutils +bird2 +bsdextrautils +btrfs-progs +bubblewrap +build-essential +ca-certificates +ceph +chrony +cifs-utils +cloud-guest-utils +cloud-init +conntrack +containerd +containerd +containernetworking-plugins +cryptsetup +curl +deborphan +devscripts +dkms +dnsmasq +dnsutils +docker.io +docker.io +dosfstools +dracut +dracut-network +dwarves +ebtables +efibootmgr +efitools +ethtool +fail2ban +flex +gcc +gdisk +gettext +git +gnupg +google-compute-engine-oslogin +google-guest-agent +haveged +hdparm +ipmitool +iptables +iputils-arping +ipvsadm +jq +kernel-wedge +kexec-tools +keyutils +libengine-pkcs11-openssl +libpam-passwdqc +libpam-pwquality +libvirt-clients +libvirt-daemon-system +libxi6 +libxtst6 +libyang2 +linux-base +lsb-release +lsof +lvm2 +lz4 +lzip +lzop +make +man-db +mdadm +microsoft-authentication-library-for-python +multipath-tools +neofetch +net-tools +nfs-common +nodejs +npm +nullmailer +numactl +nvme-cli +open-vm-tools +openssh-server +openssl +openssl +ostree +ovmf +patchelf +pciutils +podman +podman-toolbox +policykit-1 +pristine-lfs +python3-apt +python3-boto +python3-cffi-backend +python3-click +python3-debian +python3-distutils +python3-networkx +python3-novaclient +python3-pefile +python3-pip +python3-pip-whl +python3-setuptools-whl +python3-systemd +python3.11 +qemu-block-extra +qemu-efi-aarch64 +qemu-guest-agent +qemu-system +qemu-utils +quilt +quota +rng-tools5 +rootlesskit +rsync +rsyslog-relp +runc +sbsigntool +selinux-basics +selinux-policy-default +sg3-utils +sgml-base +smartmontools +socat +sosreport +strace +sudo +swtpm +syslog-ng +syslog-ng-core +sysstat +systemd-cron +tcpd +tcpdump +tpm2-tools +traceroute +usbutils +usr-is-merged +vim +virtinst +waagent +wget +xfsprogs +yq +zstd