ci/repo: Add apt publishing

Signed-off-by: Ryan Northey <[email protected]>

.aptly-ci.yaml

@@ -1,5 +1,5 @@
 rootDir: /opt/build/cache/aptly
 FileSystemPublishEndpoints:
   public:
-    rootDir: /opt/build/repo/repository
+    rootDir: /opt/build/cache/html
     linkMethod: symlink

.bazelrc

@@ -8,3 +8,7 @@ common:ci --//:aptly-custom=//:.aptly-ci-override
 common:debs-ci --config=ci
 common:debs-ci --//debs:excludes=//debs:custom-excludes.txt
 # common:debs-ci --//debs:token=//debs:token.txt
+
+common:publish-ci --config=debs-ci
+common:publish-ci --//tools/tarball:target=//:html
+common:publish-ci --//tools/tarball:overwrite=//tools/tarball:true

BUILD

@@ -91,3 +91,25 @@ jq(
     .[0] * .[1]
     """,
 )
+
+genrule(
+    name = "html",
+    outs = ["tar.gz"],
+    cmd = """
+    export APTLY_BIN="$(location @aptly)"
+    export MAINTAINER_KEY="$(location //:envoy-maintainers-public.key)"
+    export APTLY_CONF="$(location //:aptly-config)"
+    export DEBS="$(location //debs)"
+    export DEBS_ROOT_DEFAULT="/opt/build/cache/repository"
+    export SIGNING_KEY_DEFAULT="[email protected]"
+    $(location //debs:publish)
+    tar cf $@ -C /opt/build/cache/html .
+    """,
+    tools = [
+        "@aptly",
+        "//:aptly-config",
+        "//:envoy-maintainers-public.key",
+        "//debs",
+        "//debs:publish",
+    ]
+)

build-repository.sh

@@ -34,6 +34,29 @@ create_excludes () {
     fi
 }
 
+generate_private_key () {
+    echo -e "$(underline $(bold "Import maintainers private key (apt): repository signing"))"
+    gpg --batch --pinentry-mode loopback --passphrase "" --gen-key <<EOF
+%echo Generating a basic OpenPGP key
+Key-Type: 1
+Key-Length: 4096
+Subkey-Type: 1
+Subkey-Length: 4096
+Name-Real: Envoy CI
+Name-Email: [email protected]
+Expire-Date: 0
+%commit
+%echo done
+EOF
+}
+
+if [[ -e "${DEBS_ROOT}" ]]; then
+    ls "${DEBS_ROOT}" | (grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' || echo '') | sort -u > debs/custom-excludes.txt
+else
+    touch debs/custom-excludes.txt
+fi
+
 import_public_key
+generate_private_key
 create_excludes
-bazel run --config=debs-ci //debs:publish
+bazel run --config=publish-ci //tools/tarball:unpack /opt/build/repo/html

debs/BUILD

@@ -141,6 +141,7 @@ genrule(
         ":excludes",
         ":token",
     ],
+    visibility = ["//visibility:public"],
 )
 
 sh_binary(
@@ -152,11 +153,13 @@ sh_binary(
         "APTLY_CONF": "$(location //:aptly-config)",
         "DEBS": "$(location :debs)",
         "DEBS_ROOT_DEFAULT": "/opt/build/cache/repository",
+        "SIGNING_KEY_DEFAULT": "[email protected]",
     },
     data = [
         "@aptly",
         "//:aptly-config",
         "//:envoy-maintainers-public.key",
         ":debs"
     ],
+    visibility = ["//visibility:public"],
 )

debs/publish.sh

@@ -7,9 +7,22 @@ APTLY_CONF="${APTLY_CONF:-${APTLY_CONF}}"
 APTLY=("$APTLY_BIN" -config="${APTLY_CONF}")
 
 DEBS_ROOT="${DEBS_ROOT:-${DEBS_ROOT_DEFAULT}}"
+REPOS=(focal jammy bullseye bookworm)
+SIGNING_KEY="${SIGNING_KEY:-${SIGNING_KEY_DEFAULT}}"
+
+
+_aptly () {
+    "${APTLY[@]}" -- "${@}"
+}
+
+uid_generate() {
+    local length=${1:-7}
+    < /dev/urandom tr -dc 'A-Za-z0-9' | head -c "${length}"
+    echo
+}
 
 publish_dir () {
-    "${APTLY[@]}" config show \
+    _aptly config show \
         | jq -r '.FileSystemPublishEndpoints.public.rootDir'
 }
 
@@ -25,18 +38,75 @@ unpack_debs () {
     fi
 }
 
+create_repos () {
+    existing_repos=$(_aptly repo list -json | jq -r '.[] | .Name')
+
+    for repo in "${REPOS[@]}"; do
+        if ! echo "$existing_repos" | tr ' ' '\n' | grep -q "^${repo}$"; then
+            _aptly repo create "$repo"
+        fi
+    done
+}
+
+list_current_changes () {
+    for repo in "${REPOS[@]}"; do
+        while read -r package; do
+            echo "${package}.${repo}.changes"
+        done < <(_aptly repo show -with-packages -json "${repo}"  | jq -r '.Packages[]')
+    done
+}
+
+include_debs () {
+    declare -A imported
+    while read -r package; do
+        imported["$package"]=1
+    done < <(list_current_changes)
+    echo ${imported[@]}
+    while read -r file; do
+        filename="$(basename "$file")"
+        if [[ "${imported[$filename]}" ]]; then
+            continue
+        fi
+        _aptly repo include -no-remove-files "$file"
+    done < <(find "${DEBS_ROOT}" -name "*.changes")
+}
+
 publish_repository () {
-    PUBLIC_DIR="$(publish_dir)"
-    KEY_URL="${DEPLOY_PRIME_URL}/envoy-maintainer-public.key"
-    cat "$MAINTAINER_KEY" > "${PUBLIC_DIR}/envoy-maintainer-public.key"
-    echo "<h1>COMING SOON: ${DEPLOY_PRIME_URL}</h1>" > "${PUBLIC_DIR}/index.html"
-    echo "<div>Signing key: <a href=\"${KEY_URL}\">${KEY_URL}</div>" >> "${PUBLIC_DIR}/index.html"
+    local repo uid skip snapshot current result key
+    key=$(gpg --list-secret-keys --keyid-format LONG "$SIGNING_KEY" \
+              | grep 'sec' \
+              | awk '{print $2}' \
+              | cut -d'/' -f2)
+    for repo in "${REPOS[@]}"; do
+        uid=$(uid_generate)
+        skip=
+        snapshot="${repo}-${uid}"
+        _aptly snapshot create "$snapshot" from repo "$repo"
+        current=$(_aptly publish list -json \
+            | jq --arg dist "$repo" \
+                 '.[] | select(.Distribution == $dist) | .Sources[] | select(.Component == "main") | .Name')
+        if [[ -n "$current" ]]; then
+            result=$(_aptly snapshot diff "$current" "${snapshot}")
+            if [[ "$result" == "Snapshots are identical." ]]; then
+                skip=1
+            else
+                _aptly publish drop "${repo}" "filesystem:public:"
+            fi
+        fi
+        if [[ -z "$skip" ]]; then
+            _aptly publish snapshot -gpg-key="${key}" -distribution "${repo}" "${snapshot}" "filesystem:public:"
+        fi
+    done
 }
 
-publish () {
+main () {
     create_dirs
+    create_repos
     unpack_debs
+    include_debs
     publish_repository
 }
 
-publish
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    main
+fi

netlify.toml

@@ -1,5 +1,5 @@
 [build]
-publish = "repository"
+publish = "html"
 command = "./build-repository.sh"
 
 [build.environment]

setup.bzl

@@ -1,6 +1,11 @@
 load("@rules_proto//proto:setup.bzl", "rules_proto_setup")
 load("@rules_python//python:repositories.bzl", "py_repositories")
+# load("@envoy//bazel:api_binding.bzl", "envoy_api_binding")
+# load("@envoy//bazel:repositories.bzl", "envoy_build_config")
+
 
 def setup():
     py_repositories()
     rules_proto_setup()
+    # envoy_api_binding()
+    # envoy_build_config(name = "envoy_build_config")

tools/tarball/BUILD

@@ -0,0 +1,7 @@
+load("@envoy_toolshed//tarball:macros.bzl", "unpacker")
+
+licenses(["notice"])  # Apache 2
+
+unpacker(
+    name = "unpack",
+)

versions.bzl

@@ -40,18 +40,18 @@ VERSIONS = {
     "envoy": {
         "type": "github_archive",
         "repo": "envoyproxy/envoy",
-        "version": "fea66c359069991e88bdfa4e0f2883c90cc39aef",
-        "sha256": "96294f4b491c676b650ddeb07c7986ec7e48b5ae5a75c4adebbb9a58741a2fb1",
+        "version": "4108a96e215897d47cfeb3578486f0578333c1bc",
+        "sha256": "62606a0dbe73d32edfde176faaf1325f5f7a1531d1d40e12f4d6d85b75bc6b2b",
         "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
         "strip_prefix": "envoy-{version}",
     },
     "envoy_toolshed": {
         "type": "github_archive",
         "repo": "envoyproxy/toolshed",
-        "version": "0.1.4",
-        "sha256": "7ddfd251a89518b97c4eb8064a7d37454bbd998bf29e4cd3ad8f44227b5ca7b3",
-        "urls": ["https://github.com/{repo}/archive/bazel-v{version}.tar.gz"],
-        "strip_prefix": "toolshed-bazel-v{version}/bazel",
+        "version": "20c6067d87f1eb2e5a4d80f815d2aa3a10ed37ad",
+        "sha256": "9578a70d01585cdea680b1fddc86813f8282500565b1b9ceeb600940b122de43",
+        "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
+        "strip_prefix": "toolshed-{version}/bazel",
     },
     "io_bazel_rules_go": {
         "type": "github_archive",