diff --git a/.bazelrc b/.bazelrc index 4e8363b..e6c915b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,3 +4,7 @@ common:ci --noshow_progress common:ci --noshow_loading_progress common:ci --test_output=errors common:ci --//:aptly-custom=//:.aptly-ci-override + +common:debs-ci --config=ci +common:debs-ci --//debs:excludes=//debs:custom-excludes.txt +# common:debs-ci --//debs:token=//debs:token.txt diff --git a/build-repository.sh b/build-repository.sh index 9751cc9..8f6f65b 100755 --- a/build-repository.sh +++ b/build-repository.sh @@ -6,6 +6,8 @@ BOLD="\e[1m" UNDERLINE="\e[4m" NORMAL="\e[0m" +EXCLUDE_FILE=debs/custom-excludes.txt +DEBS_ROOT=/opt/build/cache/repository bold () { echo -n "${BOLD}${*}${NORMAL}" @@ -23,5 +25,15 @@ import_public_key () { gpg --no-default-keyring --keyring trustedkeys.gpg --import envoy-maintainers-public.key } +create_excludes () { + # Prevent re-downloading cached files + if [[ -e "${DEBS_ROOT}" ]]; then + ls "${DEBS_ROOT}" | (grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' || echo '') | sort -u > "$EXCLUDE_FILE" + else + touch "$EXCLUDE_FILE" + fi +} + import_public_key -bazel run --config=ci //debs:publish +create_excludes +bazel run --config=debs-ci //debs:publish diff --git a/debs/BUILD b/debs/BUILD index cad4923..d996dbe 100644 --- a/debs/BUILD +++ b/debs/BUILD @@ -1,5 +1,148 @@ load("@aspect_bazel_lib//lib:jq.bzl", "jq") +MAINTAINER = "Envoy maintainers " + +exports_files([ + "custom-excludes.txt", + "token.txt", +]) + +jq( + name = "envoy_versions", + srcs = ["@envoy_repo//:project"], + out = "envoy_versions.json", + filter = """ + .releases as $releases + | { + latest_releases: [ + .stable_versions[] + | . as $minor + | { + version: $minor, + releases: ( + $releases + | map(select(startswith("v" + $minor + "."))) + ) + } + ] + } + """, + visibility = ["//visibility:public"], +) + +jq( + name = "minor_versions", + srcs = [":envoy_versions"], + out = "minor_versions.txt", + filter = """ + .latest_releases[] | .version + """, + args = ["-r"], + visibility = ["//visibility:public"], +) + +jq( + name = "patch_versions", + srcs = [":envoy_versions"], + out = "patch_versions.txt", + filter = """ + .latest_releases[] | .releases[] + """, + args = ["-r"], + visibility = ["//visibility:public"], +) + +jq( + name = "deb_checksum_downloads", + srcs = [":envoy_versions"], + out = "deb_checksum_downloads.txt", + filter = """ + reduce .latest_releases[].releases[] as $item ({}; + .[$item] = {"signature": "%s"}) + | with_entries( + {"key": "https://github.com/envoyproxy/envoy/releases/download/\\(.key)/checksums.txt.asc", + "value": .value}) + """ % MAINTAINER, + args = ["-r"], + visibility = ["//visibility:public"], +) + +genrule( + name = "published_checksums", + outs = ["published_checksums.txt"], + cmd = """ + $(location //tools/fetch) $(location :deb_checksum_downloads) --output=json > $@ + """, + tools = [ + "//tools/fetch", + ":deb_checksum_downloads", + ], +) + +jq( + name = "debs_downloads", + srcs = [":published_checksums"], + out = "debs_downloads.json", + filter = """ + with_entries( + .key as $key + | .value as $value + | ($key | capture("v(?[0-9.]+)") | .version) as $version + | {key: ("https://github.com/envoyproxy/envoy/releases/download/v\\($version)/debs.tar.gz"), + value: { + "path": "v\\($version)", + "checksum": ( + $value + | split("\n") + | map(select(endswith("debs.tar.gz"))) + | first + | split(" ") + | .[0] + )}}) + """, + visibility = ["//visibility:public"], +) + +genrule( + name = "empty", + outs = ["empty.txt"], + cmd = """ + touch $@ + """, +) + +label_flag( + name = "excludes", + build_setting_default = ":empty", +) + +label_flag( + name = "token", + build_setting_default = ":empty", +) + +genrule( + name = "debs", + outs = ["debs.tar.gz"], + cmd = """ + $(location //tools/fetch) $(location :debs_downloads) \ + --concurrency 4 \ + --excludes=$(location :excludes) \ + --token-path=$(location :token) \ + --extract-downloads \ + --output-path=$@ + if [[ ! -e $@ ]]; then + touch $@ + fi + """, + tools = [ + "//tools/fetch", + ":debs_downloads", + ":excludes", + ":token", + ], +) + sh_binary( name = "publish", srcs = ["publish.sh"], @@ -7,10 +150,13 @@ sh_binary( "APTLY_BIN": "$(location @aptly)", "MAINTAINER_KEY": "$(location //:envoy-maintainers-public.key)", "APTLY_CONF": "$(location //:aptly-config)", + "DEBS": "$(location :debs)", + "DEBS_ROOT_DEFAULT": "/opt/build/cache/repository", }, data = [ "@aptly", "//:aptly-config", "//:envoy-maintainers-public.key", + ":debs" ], ) diff --git a/debs/publish.sh b/debs/publish.sh index a08822f..56c5a45 100755 --- a/debs/publish.sh +++ b/debs/publish.sh @@ -6,18 +6,37 @@ APTLY_BIN="$APTLY_BIN" APTLY_CONF="${APTLY_CONF:-${APTLY_CONF}}" APTLY=("$APTLY_BIN" -config="${APTLY_CONF}") -publish_root () { +DEBS_ROOT="${DEBS_ROOT:-${DEBS_ROOT_DEFAULT}}" + +publish_dir () { "${APTLY[@]}" config show \ | jq -r '.FileSystemPublishEndpoints.public.rootDir' } -publish_repository () { - PUBLIC_DIR="$(publish_root)" +create_dirs () { + PUBLIC_DIR="$(publish_dir)" mkdir -p "${PUBLIC_DIR}" + mkdir -p "${DEBS_ROOT}" +} + +unpack_debs () { + if [[ -s "$DEBS" ]]; then + tar xf "$DEBS" -C "$DEBS_ROOT" + fi +} + +publish_repository () { + PUBLIC_DIR="$(publish_dir)" KEY_URL="${DEPLOY_PRIME_URL}/envoy-maintainer-public.key" cat "$MAINTAINER_KEY" > "${PUBLIC_DIR}/envoy-maintainer-public.key" echo "

COMING SOON: ${DEPLOY_PRIME_URL}

" > "${PUBLIC_DIR}/index.html" echo "
Signing key: ${KEY_URL}
" >> "${PUBLIC_DIR}/index.html" } -publish_repository +publish () { + create_dirs + unpack_debs + publish_repository +} + +publish diff --git a/tools/requirements.in b/tools/requirements.in index fa2ef4c..ffb8698 100644 --- a/tools/requirements.in +++ b/tools/requirements.in @@ -1 +1 @@ -envoy.base.utils>=0.5.4 +envoy.base.utils>=0.5.5 diff --git a/tools/requirements.txt b/tools/requirements.txt index 2e9df28..2329ef5 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -220,9 +220,9 @@ cryptography==43.0.0 \ --hash=sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5 \ --hash=sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0 # via pyjwt -envoy-base-utils==0.5.4 \ - --hash=sha256:90882337c2ce509b19fc21f5d66395dad44a706e4d616baabd1ffaea256ee358 \ - --hash=sha256:c1167f567994596a3071027d57ca895ac4a8bd8969d4de846b0f2cda499fa5ad +envoy-base-utils==0.5.5 \ + --hash=sha256:2341b7618c92969e3ac5fc43dd16fcc39d132981ffff3f0ddbabf08a336a6fa1 \ + --hash=sha256:a72af1df5680d4fcdeb91f06d19b7495d740f8b9a8a1549c012272e10fdd59ea # via -r requirements.in frozendict==2.4.4 \ --hash=sha256:07c3a5dee8bbb84cba770e273cdbf2c87c8e035903af8f781292d72583416801 \