Skip to content

Commit

Permalink
ci/repo: Add apt publishing
Browse files Browse the repository at this point in the history
Signed-off-by: Ryan Northey <[email protected]>
  • Loading branch information
phlax committed Aug 19, 2024
1 parent 1c91de8 commit 3062a9b
Show file tree
Hide file tree
Showing 6 changed files with 174 additions and 7 deletions.
61 changes: 60 additions & 1 deletion .github/workflows/envoy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,72 @@ concurrency:
jobs:
envoy:
permissions:
actions: write
runs-on: ubuntu-24.04
if: github.repository_owner == 'envoyproxy'
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- run: |
bazel run --config=ci //tools/fetch -- -h
- run: |
bazel build --config=ci //:envoy_versions
cat bazel-bin/envoy_versions.json | jq '.'
- run: |
bazel run --config=ci //tools/fetch -- -h
gpg --import envoy-maintainers-public.key
- id: versions
run: |
bazel build --config=ci //debs:minor_versions
VERSION_HASH="$(cat bazel-bin/debs/minor_versions.txt | sha256sum | cut -d' ' -f1)"
echo "hash=${VERSION_HASH}" >> "$GITHUB_OUTPUT"
- uses: actions/cache/restore@v4
id: cache
with:
key: ${{ steps.versions.outputs.hash }}
path: /tmp/repository
- id: repository
run: |
mkdir -p /tmp/repository
rm -rf /tmp/repository/v1.28.3
ls /tmp/repository | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' | sort -u > debs/custom-excludes.txt
DIR_SHA=$(cat debs/custom-excludes.txt | sha256sum | cut -d' ' -f1)
echo "cached=${DIR_SHA}" >> $GITHUB_OUTPUT
- run: |
echo $GITHUB_TOKEN > debs/token.txt
bazel run \
--config=debs-ci \
//tools/tarball:unpack \
/tmp/repository
# rm -rf /tmp/repository/v1.28.3
- run: |
find /tmp/repository
- run: |
UPDATED=$(ls /tmp/repository | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' | sort -u | sha256sum | cut -d' ' -f1)
if [[ "${{ steps.repository.outputs.cached }}" != "$UPDATED" ]]; then
echo "changed=1" >> $GITHUB_OUTPUT
fi
id: update
if: ${{ github.event_name == 'push' && steps.cache.outputs.cache-hit && steps.update.outputs.changed }}
- uses: actions/upload-artifact@v4
if: false
with:
path: /tmp/repository
name: debs

# if we are on main and the files have changed, then cache - PRs always just use the main cache
- if: ${{ github.event_name == 'push' && steps.cache.outputs.cache-hit && steps.update.outputs.changed }}
run: |
gh extension install actions/gh-actions-cache
gh actions-cache delete --confirm ${{ steps.versions.outputs.hash }}
env:
GH_TOKEN: ${{ github.token }}

- uses: actions/cache/save@v4
if: ${{ github.event_name == 'push' && steps.update.outputs.changed }}
with:
key: ${{ steps.versions.outputs.hash }}
path: /tmp/repository
24 changes: 24 additions & 0 deletions build-repository.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,30 @@ create_excludes () {
fi
}

generate_private_key () {
echo -e "$(underline $(bold "Import maintainers private key (apt): repository signing"))"
gpg --batch --pinentry-mode loopback --passphrase "" --gen-key <<EOF
%echo Generating a basic OpenPGP key
Key-Type: 1
Key-Length: 4096
Subkey-Type: 1
Subkey-Length: 4096
Name-Real: Envoy CI
Name-Email: [email protected]
Expire-Date: 0
%commit
%echo done
EOF
}

if [[ -e /opt/build/cache/repository ]]; then
rm -rf /opt/build/cache/repository/v1.28.3
ls /opt/build/cache/repository | (grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' || echo '') | sort -u > debs/custom-excludes.txt
else
touch debs/custom-excludes.txt
fi

import_public_key
generate_private_key
create_excludes
bazel run --config=debs-ci //debs:publish
72 changes: 72 additions & 0 deletions debs/publish.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ publish_root () {
| jq -r '.FileSystemPublishEndpoints.public.rootDir'
}

REPOS=(focal jammy bullseye bookworm)
declare -A REPO_PACKAGES

create_dirs () {
PUBLIC_DIR="$(publish_root)"
mkdir -p "${PUBLIC_DIR}"
Expand All @@ -23,17 +26,86 @@ unpack_debs () {
tar xf "$DEBS" -C "$DEBS_ROOT"
}

create_repos () {
"$APTLY_BIN" -config=${PWD}/.aptly config show
existing_repos=$("${APTLY[@]}" repo list -json | jq -r '.[] | .Name')

for repo in "${REPOS[@]}"; do
if ! echo "$existing_repos" | tr ' ' '\n' | grep -q "^${repo}$"; then
echo "Create repo: ${repo}"
"${APTLY[@]}" repo create "$repo"
fi
done
}

IMPORTED=()

list_packages () {
for repo in "${REPOS[@]}"; do
# echo "Packages: ${repo}"
packages=$("${APTLY[@]}" repo search "$repo" || echo "" | tr '\n' ' ')
REPO_PACKAGES[$repo]=$packages
done

for key in "${!REPO_PACKAGES[@]}"; do
# Get the serialized string from the associative array
serialized_values="${REPO_PACKAGES[$key]}"

# Convert the serialized string back into an array
IFS=' ' read -r -a array <<< "$serialized_values"

# Print the key
# echo "Key: $key"

# Iterate over the elements of the array
for element in "${array[@]}"; do
filename="${element}.${key}.changes"
IMPORTED+=("$filename")
# echo "EXISTING: $filename"
done
done
IMPORTED=$(printf "%s\n" "${IMPORTED[@]}")
# echo $IMPORTED
}

FAILED=()
SUCCESS=()

include_debs () {
mapfile -t changes_files < <(find /opt/build/cache/repository -name "*.changes")
for file in "${changes_files[@]}"; do
filename=$(basename $file)
if echo $IMPORTED | tr ' ' '\n' | grep -q "^${filename}$"; then
# echo "SKIP: ${filename}"
continue
fi
"${APTLY[@]}" -no-remove-files repo include $file
done
}

publish_repository () {
PUBLIC_DIR="$(publish_root)"
KEY_URL="${DEPLOY_PRIME_URL}/envoy-maintainer-public.key"
cat "$MAINTAINER_KEY" > "${PUBLIC_DIR}/envoy-maintainer-public.key"
echo "<h1>COMING SOON: ${DEPLOY_PRIME_URL}</h1>" > "${PUBLIC_DIR}/index.html"
echo "<div>Signing key: <a href=\"${KEY_URL}\">${KEY_URL}</div>" >> "${PUBLIC_DIR}/index.html"
KEY_ID=$(gpg --list-secret-keys --keyid-format LONG "[email protected]" | grep 'sec' | awk '{print $2}' | cut -d'/' -f2)
echo "COMING SOON: ${DEPLOY_PRIME_URL}" > "${PUBLIC_DIR}/index.html"
for repo in "${REPOS[@]}"; do
"${APTLY[@]}" publish drop "${repo}" "filesystem:public:"
"${APTLY[@]}" publish repo -gpg-key="${KEY_ID}" -distribution "${repo}" "${repo}" "filesystem:public:"
done
}

publish () {
create_dirs
unpack_debs
create_repos
list_packages
rm -rf /opt/build/cache/repository/v1.28.3
unpack_debs
include_debs
# list_packages
publish_repository
}

Expand Down
5 changes: 5 additions & 0 deletions setup.bzl
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
load("@rules_proto//proto:setup.bzl", "rules_proto_setup")
load("@rules_python//python:repositories.bzl", "py_repositories")
# load("@envoy//bazel:api_binding.bzl", "envoy_api_binding")
# load("@envoy//bazel:repositories.bzl", "envoy_build_config")


def setup():
py_repositories()
rules_proto_setup()
# envoy_api_binding()
# envoy_build_config(name = "envoy_build_config")
7 changes: 7 additions & 0 deletions tools/tarball/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
load("@envoy_toolshed//tarball:macros.bzl", "unpacker")

licenses(["notice"]) # Apache 2

unpacker(
name = "unpack",
)
12 changes: 6 additions & 6 deletions versions.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,18 @@ VERSIONS = {
"envoy": {
"type": "github_archive",
"repo": "envoyproxy/envoy",
"version": "fea66c359069991e88bdfa4e0f2883c90cc39aef",
"sha256": "96294f4b491c676b650ddeb07c7986ec7e48b5ae5a75c4adebbb9a58741a2fb1",
"version": "4108a96e215897d47cfeb3578486f0578333c1bc",
"sha256": "62606a0dbe73d32edfde176faaf1325f5f7a1531d1d40e12f4d6d85b75bc6b2b",
"urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
"strip_prefix": "envoy-{version}",
},
"envoy_toolshed": {
"type": "github_archive",
"repo": "envoyproxy/toolshed",
"version": "0.1.4",
"sha256": "7ddfd251a89518b97c4eb8064a7d37454bbd998bf29e4cd3ad8f44227b5ca7b3",
"urls": ["https://github.com/{repo}/archive/bazel-v{version}.tar.gz"],
"strip_prefix": "toolshed-bazel-v{version}/bazel",
"version": "20c6067d87f1eb2e5a4d80f815d2aa3a10ed37ad",
"sha256": "9578a70d01585cdea680b1fddc86813f8282500565b1b9ceeb600940b122de43",
"urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
"strip_prefix": "toolshed-{version}/bazel",
},
"io_bazel_rules_go": {
"type": "github_archive",
Expand Down

0 comments on commit 3062a9b

Please sign in to comment.