ci/repo: Add apt publishing

Signed-off-by: Ryan Northey <[email protected]>

build-repository.sh

@@ -34,6 +34,30 @@ create_excludes () {
     fi
 }
 
+generate_private_key () {
+    echo -e "$(underline $(bold "Import maintainers private key (apt): repository signing"))"
+    gpg --batch --pinentry-mode loopback --passphrase "" --gen-key <<EOF
+%echo Generating a basic OpenPGP key
+Key-Type: 1
+Key-Length: 4096
+Subkey-Type: 1
+Subkey-Length: 4096
+Name-Real: Envoy CI
+Name-Email: [email protected]
+Expire-Date: 0
+%commit
+%echo done
+EOF
+}
+
+if [[ -e /opt/build/cache/repository ]]; then
+    rm -rf /opt/build/cache/repository/v1.28.3
+    ls /opt/build/cache/repository | (grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' || echo '') | sort -u > debs/custom-excludes.txt
+else
+    touch debs/custom-excludes.txt
+fi
+
 import_public_key
+generate_private_key
 create_excludes
 bazel run --config=debs-ci //debs:publish

debs/publish.sh

@@ -8,6 +8,16 @@ APTLY=("$APTLY_BIN" -config="${APTLY_CONF}")
 
 DEBS_ROOT="${DEBS_ROOT:-${DEBS_ROOT_DEFAULT}}"
 
+REPOS=(focal jammy bullseye bookworm)
+declare -A REPO_PACKAGES
+
+
+uid_generate() {
+    local length=${1:-7}
+    < /dev/urandom tr -dc 'A-Za-z0-9' | head -c "${length}"
+    echo
+}
+
 publish_dir () {
     "${APTLY[@]}" config show \
         | jq -r '.FileSystemPublishEndpoints.public.rootDir'
@@ -25,18 +35,84 @@ unpack_debs () {
     fi
 }
 
+create_repos () {
+    "$APTLY_BIN" -config=${PWD}/.aptly config show
+    existing_repos=$("${APTLY[@]}" repo list -json | jq -r '.[] | .Name')
+
+    for repo in "${REPOS[@]}"; do
+        if ! echo "$existing_repos" | tr ' ' '\n' | grep -q "^${repo}$"; then
+            "${APTLY[@]}" repo create "$repo"
+        fi
+    done
+}
+
+list_packages () {
+    local imported=()
+    for repo in "${REPOS[@]}"; do
+        "${APTLY[@]}" repo search "$repo" 2>/dev/null | while IFS= read -r package; do
+            echo "${package}.${repo}.changes"
+        done
+    done
+}
+
+include_debs () {
+    mapfile -t changes_files < <(find /opt/build/cache/repository -name "*.changes")
+    imported=$(list_packages)
+
+    for file in "${changes_files[@]}"; do
+        filename=$(basename $file)
+        if echo $imported | grep -q "^${filename}$"; then
+            continue
+        fi
+        "${APTLY[@]}" -no-remove-files repo include $file
+    done
+}
+
 publish_repository () {
+    local repo uid skip snapshot current result key
+    key=$(gpg --list-secret-keys --keyid-format LONG "[email protected]" | grep 'sec' | awk '{print $2}' | cut -d'/' -f2)
+
+    for repo in "${REPOS[@]}"; do
+        uid=$(uid_generate)
+        skip=
+        snapshot="${repo}-${uid}"
+        "${APTLY[@]}" snapshot create "$snapshot" from repo "$repo"
+        current=$("${APTLY[@]}" publish list filesystem:public: -- -json \
+            | jq --arg dist "$repo" \
+                 '.[] | select(.Distribution == $dist) | .Sources[] | select(.Component == "main") | .Name')
+        if [[ -n "$current" ]]; then
+            result=$("${APTLY[@]}" snapshot diff "$current" "${snapshot}")
+            if [[ "$result" == "Snapshots are identical." ]]; then
+                skip=1
+            else
+                "${APTLY[@]}" publish drop "${repo}" "filesystem:public:"
+            fi
+        fi
+        if [[ -z "$skip" ]]; then
+            "${APTLY[@]}" publish snapshot -gpg-key="${key}" -distribution "${repo}" "${snapshot}" "filesystem:public:"
+        fi
+    done
+}
+
+publish_index () {
     PUBLIC_DIR="$(publish_dir)"
     KEY_URL="${DEPLOY_PRIME_URL}/envoy-maintainer-public.key"
     cat "$MAINTAINER_KEY" > "${PUBLIC_DIR}/envoy-maintainer-public.key"
     echo "<h1>COMING SOON: ${DEPLOY_PRIME_URL}</h1>" > "${PUBLIC_DIR}/index.html"
     echo "<div>Signing key: <a href=\"${KEY_URL}\">${KEY_URL}</div>" >> "${PUBLIC_DIR}/index.html"
+    echo "COMING SOON: ${DEPLOY_PRIME_URL}" > "${PUBLIC_DIR}/index.html"
 }
 
-publish () {
+main () {
     create_dirs
     unpack_debs
+    create_repos
+    unpack_debs
+    include_debs
     publish_repository
+    publish_index
 }
 
-publish
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    main
+fi

setup.bzl

@@ -1,6 +1,11 @@
 load("@rules_proto//proto:setup.bzl", "rules_proto_setup")
 load("@rules_python//python:repositories.bzl", "py_repositories")
+# load("@envoy//bazel:api_binding.bzl", "envoy_api_binding")
+# load("@envoy//bazel:repositories.bzl", "envoy_build_config")
+
 
 def setup():
     py_repositories()
     rules_proto_setup()
+    # envoy_api_binding()
+    # envoy_build_config(name = "envoy_build_config")

tools/tarball/BUILD

@@ -0,0 +1,7 @@
+load("@envoy_toolshed//tarball:macros.bzl", "unpacker")
+
+licenses(["notice"])  # Apache 2
+
+unpacker(
+    name = "unpack",
+)

versions.bzl

@@ -40,18 +40,18 @@ VERSIONS = {
     "envoy": {
         "type": "github_archive",
         "repo": "envoyproxy/envoy",
-        "version": "fea66c359069991e88bdfa4e0f2883c90cc39aef",
-        "sha256": "96294f4b491c676b650ddeb07c7986ec7e48b5ae5a75c4adebbb9a58741a2fb1",
+        "version": "4108a96e215897d47cfeb3578486f0578333c1bc",
+        "sha256": "62606a0dbe73d32edfde176faaf1325f5f7a1531d1d40e12f4d6d85b75bc6b2b",
         "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
         "strip_prefix": "envoy-{version}",
     },
     "envoy_toolshed": {
         "type": "github_archive",
         "repo": "envoyproxy/toolshed",
-        "version": "0.1.4",
-        "sha256": "7ddfd251a89518b97c4eb8064a7d37454bbd998bf29e4cd3ad8f44227b5ca7b3",
-        "urls": ["https://github.com/{repo}/archive/bazel-v{version}.tar.gz"],
-        "strip_prefix": "toolshed-bazel-v{version}/bazel",
+        "version": "20c6067d87f1eb2e5a4d80f815d2aa3a10ed37ad",
+        "sha256": "9578a70d01585cdea680b1fddc86813f8282500565b1b9ceeb600940b122de43",
+        "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
+        "strip_prefix": "toolshed-{version}/bazel",
     },
     "io_bazel_rules_go": {
         "type": "github_archive",