ci/repo: Add apt publishing

Signed-off-by: Ryan Northey <[email protected]>

.aptly-ci.yaml

@@ -1,5 +1,5 @@
 rootDir: /opt/build/cache/aptly
 FileSystemPublishEndpoints:
   public:
-    rootDir: /opt/build/repo/repository
+    rootDir: /opt/build/cache/html
     linkMethod: symlink

.bazelrc

@@ -1,10 +1,20 @@
 
 common --color=yes
+common --action_env=APT_ROOT=apt
+common --host_action_env=APT_ROOT=apt
+
 common:ci --noshow_progress
 common:ci --noshow_loading_progress
 common:ci --test_output=errors
 common:ci --//:aptly-custom=//:.aptly-ci-override
+common:ci --action_env=APT_ROOT=/opt/build/cache
+common:ci --host_action_env=APT_ROOT=/opt/build/cache
 
 common:debs-ci --config=ci
 common:debs-ci --//debs:excludes=//debs:custom-excludes.txt
 # common:debs-ci --//debs:token=//debs:token.txt
+
+common:publish-ci --config=debs-ci
+common:publish-ci --//tools/tarball:target=//:html
+common:publish-ci --//tools/tarball:overwrite=//tools/tarball:true
+common:publish-ci --//debs:signing-token=//debs:signing-token.txt

.github/workflows/aptly.yml

@@ -24,4 +24,4 @@ jobs:
         bazel build --config=ci //:aptly-config
         bazel run --config=ci @aptly config show -- -config "${PWD}/bazel-bin/.aptly" \
           | jq -r '.FileSystemPublishEndpoints.public.rootDir' \
-          | grep /opt/build/repo/repository
+          | grep /opt/build/cache/html

BUILD

@@ -7,6 +7,27 @@ exports_files([
     "envoy-maintainers-public.key",
 ])
 
+filegroup(
+    name = "true",
+    srcs = [],
+)
+
+filegroup(
+    name = "false",
+    srcs = [],
+)
+
+label_flag(
+    name = "production",
+    build_setting_default = ":false",
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "production_build",
+    flag_values = {":production": ":true"},
+)
+
 # gazelle:prefix github.com/aptly-dev/aptly
 gazelle(name = "gazelle")
 
@@ -91,3 +112,33 @@ jq(
     .[0] * .[1]
     """,
 )
+
+HTML_BUILD_CMD = """
+    export APTLY_BIN="$(location @aptly)"
+    export MAINTAINER_KEY="$(location //:envoy-maintainers-public.key)"
+    export APTLY_CONF="$(location //:aptly-config)"
+    export DEBS="$(location //debs)"
+    export DEBS_ROOT="$${APT_ROOT}/repository"
+    export SIGNING_TOKEN="$(location //debs:signing-token)"
+    $(location //debs:publish)
+    tar xf $(location //site) -C $${APT_ROOT}/html
+    tar hcf $@ -C $${APT_ROOT}/html .
+"""
+
+genrule(
+    name = "html",
+    outs = ["html.tar.gz"],
+    cmd = select({
+        ":production_build": "export SIGNING_KEY=\"Envoy maintainers <[email protected]>\"",
+        "//conditions:default": "export SIGNING_KEY=\"[email protected]\"",
+    }) + HTML_BUILD_CMD,
+    tools = [
+        "@aptly",
+        "//:aptly-config",
+        "//:envoy-maintainers-public.key",
+        "//debs",
+        "//debs:publish",
+        "//debs:signing-token",
+        "//site",
+    ]
+)

build-repository.sh

@@ -8,6 +8,15 @@ NORMAL="\e[0m"
 
 EXCLUDE_FILE=debs/custom-excludes.txt
 DEBS_ROOT=/opt/build/cache/repository
+SIGNING_KEY_PASSPHRASE="${SIGNING_KEY_PASSPHRASE:-Hackme}"
+
+
+finally () {
+    rm -rf signing.key
+    rm -rf debs/signing-token.txt
+}
+
+trap finally EXIT
 
 bold () {
     echo -n "${BOLD}${*}${NORMAL}"
@@ -34,6 +43,57 @@ create_excludes () {
     fi
 }
 
-import_public_key
-create_excludes
-bazel run --config=debs-ci //debs:publish
+generate_private_key () {
+    echo -e "$(underline $(bold "Generate snakeoil private key: repository signing"))"
+    gpg --batch --pinentry-mode loopback --passphrase "" --gen-key <<EOF
+%echo Generating a basic OpenPGP key
+Key-Type: 1
+Key-Length: 4096
+Subkey-Type: 1
+Subkey-Length: 4096
+Name-Real: Envoy CI
+Name-Email: [email protected]
+Expire-Date: 0
+Passphrase: ${SIGNING_KEY_PASSPHRASE}
+%commit
+%echo done
+EOF
+}
+
+import_private_key () {
+    echo -e "$(underline $(bold "Import maintainers private signing key: repository signing"))"
+    echo "${SIGNING_KEY_0}${SIGNING_KEY_1}${SIGNING_KEY_2}${SIGNING_KEY_3}" | base64 -d > signing.key
+    gpg --batch --pinentry-mode loopback --import signing.key
+}
+
+create_excludes () {
+    if [[ -e "${DEBS_ROOT}" ]]; then
+        ls "${DEBS_ROOT}" \
+            | (grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' || echo '') \
+            | sort -u > debs/custom-excludes.txt
+    else
+        touch debs/custom-excludes.txt
+    fi
+}
+
+main () {
+    local bazel_args=(--config=publish-ci)
+    import_public_key
+    create_excludes
+    echo "${SIGNING_KEY_PASSPHRASE}" > debs/signing-token.txt
+    if [[ "$CONTEXT" == "deploy-preview" ]]; then
+        import_private_key
+        # generate_private_key
+    else
+        import_private_key
+        bazel_args+=(--//:production=//:true)
+    fi
+    bazel run \
+          "${bazel_args[@]}" \
+          //tools/tarball:unpack \
+          /opt/build/repo/html
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    main
+fi

debs/BUILD

@@ -5,8 +5,35 @@ MAINTAINER = "Envoy maintainers <[email protected]>"
 exports_files([
     "custom-excludes.txt",
     "token.txt",
+    "signing-token.txt",
 ])
 
+genrule(
+    name = "empty",
+    outs = ["empty.txt"],
+    cmd = """
+    touch $@
+    """,
+)
+
+label_flag(
+    name = "excludes",
+    build_setting_default = ":empty",
+    visibility = ["//visibility:public"],
+)
+
+label_flag(
+    name = "token",
+    build_setting_default = ":empty",
+    visibility = ["//visibility:public"],
+)
+
+label_flag(
+    name = "signing-token",
+    build_setting_default = ":empty",
+    visibility = ["//visibility:public"],
+)
+
 jq(
     name = "envoy_versions",
     srcs = ["@envoy_repo//:project"],
@@ -103,24 +130,6 @@ jq(
     visibility = ["//visibility:public"],
 )
 
-genrule(
-    name = "empty",
-    outs = ["empty.txt"],
-    cmd = """
-    touch $@
-    """,
-)
-
-label_flag(
-    name = "excludes",
-    build_setting_default = ":empty",
-)
-
-label_flag(
-    name = "token",
-    build_setting_default = ":empty",
-)
-
 genrule(
     name = "debs",
     outs = ["debs.tar.gz"],
@@ -141,22 +150,31 @@ genrule(
         ":excludes",
         ":token",
     ],
+    visibility = ["//visibility:public"],
 )
 
+PUBLISH_ENV = {
+    "APTLY_BIN": "$(location @aptly)",
+    "MAINTAINER_KEY": "$(location //:envoy-maintainers-public.key)",
+    "APTLY_CONF": "$(location //:aptly-config)",
+    "DEBS": "$(location :debs)",
+    "DEBS_ROOT": "$${APT_ROOT}/repository",
+    "SIGNING_TOKEN": "$(location :signing-token)",
+}
+
 sh_binary(
     name = "publish",
     srcs = ["publish.sh"],
-    env = {
-        "APTLY_BIN": "$(location @aptly)",
-        "MAINTAINER_KEY": "$(location //:envoy-maintainers-public.key)",
-        "APTLY_CONF": "$(location //:aptly-config)",
-        "DEBS": "$(location :debs)",
-        "DEBS_ROOT_DEFAULT": "/opt/build/cache/repository",
-    },
+    env = PUBLISH_ENV | select({
+        "//:production_build": {"SIGNING_KEY": "Envoy maintainers <[email protected]>"},
+        "//conditions:default": {"SIGNING_KEY": "Envoy CI <[email protected]>"},
+    }),
     data = [
         "@aptly",
         "//:aptly-config",
         "//:envoy-maintainers-public.key",
-        ":debs"
+        ":debs",
+        ":signing-token",
     ],
+    visibility = ["//visibility:public"],
 )

debs/publish.sh

@@ -2,14 +2,34 @@
 
 set -e -o pipefail
 
+
 APTLY_BIN="$APTLY_BIN"
 APTLY_CONF="${APTLY_CONF:-${APTLY_CONF}}"
 APTLY=("$APTLY_BIN" -config="${APTLY_CONF}")
 
 DEBS_ROOT="${DEBS_ROOT:-${DEBS_ROOT_DEFAULT}}"
+REPOS=(focal jammy bullseye bookworm)
+SIGNING_KEY="${SIGNING_KEY:-}"
+SIGNING_TOKEN="${SIGNING_TOKEN:-${SIGNING_TOKEN}}"
+
+if [[ -z "$SIGNING_KEY" ]]; then
+    echo "SIGNING_KEY must be set and exist as a GPG key" >&2
+    exit 1
+fi
+
+
+_aptly () {
+    "${APTLY[@]}" -- "${@}"
+}
+
+uid_generate() {
+    local length=${1:-7}
+    < /dev/urandom tr -dc 'A-Za-z0-9' | head -c "${length}"
+    echo
+}
 
 publish_dir () {
-    "${APTLY[@]}" config show \
+    _aptly config show \
         | jq -r '.FileSystemPublishEndpoints.public.rootDir'
 }
 
@@ -25,18 +45,78 @@ unpack_debs () {
     fi
 }
 
+create_repos () {
+    existing_repos=$(_aptly repo list -json | jq -r '.[] | .Name')
+
+    for repo in "${REPOS[@]}"; do
+        if ! echo "$existing_repos" | tr ' ' '\n' | grep -q "^${repo}$"; then
+            _aptly repo create "$repo"
+        fi
+    done
+}
+
+list_current_changes () {
+    for repo in "${REPOS[@]}"; do
+        while read -r package; do
+            echo "${package}.${repo}.changes"
+        done < <(_aptly repo show -with-packages -json "${repo}"  | jq -r '.Packages[]')
+    done
+}
+
+include_debs () {
+    declare -A imported
+    while read -r package; do
+        imported["$package"]=1
+    done < <(list_current_changes)
+    while read -r file; do
+        filename="$(basename "$file")"
+        if [[ "${imported[$filename]}" ]]; then
+            continue
+        fi
+        _aptly repo include -no-remove-files "$file"
+    done < <(find "${DEBS_ROOT}" -name "*.changes")
+}
+
 publish_repository () {
-    PUBLIC_DIR="$(publish_dir)"
-    KEY_URL="${DEPLOY_PRIME_URL}/envoy-maintainer-public.key"
-    cat "$MAINTAINER_KEY" > "${PUBLIC_DIR}/envoy-maintainer-public.key"
-    echo "<h1>COMING SOON: ${DEPLOY_PRIME_URL}</h1>" > "${PUBLIC_DIR}/index.html"
-    echo "<div>Signing key: <a href=\"${KEY_URL}\">${KEY_URL}</div>" >> "${PUBLIC_DIR}/index.html"
+    local repo uid skip snapshot current result key
+    key=$(gpg --list-secret-keys --keyid-format LONG "$SIGNING_KEY" \
+              | grep 'sec' \
+              | awk '{print $2}' \
+              | cut -d'/' -f2)
+    if [[ -e "$SIGNING_TOKEN" ]]; then
+        SIGNING_KEY_PASSPHRASE="$(cat "$SIGNING_TOKEN")"
+    fi
+    for repo in "${REPOS[@]}"; do
+        uid=$(uid_generate)
+        skip=
+        snapshot="${repo}-${uid}"
+        _aptly snapshot create "$snapshot" from repo "$repo"
+        current=$(_aptly publish list -json \
+            | jq -r --arg dist "$repo" \
+                 '.[] | select(.Distribution == $dist) | .Sources[] | select(.Component == "main") | .Name')
+        if [[ -n "$current" ]]; then
+            result=$(_aptly snapshot diff "$current" "${snapshot}")
+            if [[ "$result" == "Snapshots are identical." ]]; then
+                _aptly publish drop "${repo}" "filesystem:public:"
+                # skip=1
+            else
+                _aptly publish drop "${repo}" "filesystem:public:"
+            fi
+        fi
+        if [[ -z "$skip" ]]; then
+            _aptly publish snapshot -batch -passphrase "${SIGNING_KEY_PASSPHRASE}" -gpg-key="${key}" -distribution "${repo}" "${snapshot}" "filesystem:public:"
+        fi
+    done
 }
 
-publish () {
+main () {
     create_dirs
+    create_repos
     unpack_debs
+    include_debs
     publish_repository
 }
 
-publish
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    main
+fi