ci/repo: Add deb fetcher

Signed-off-by: Ryan Northey <[email protected]>

.bazelrc

@@ -3,3 +3,9 @@ common --color=yes
 common:ci --noshow_progress
 common:ci --noshow_loading_progress
 common:ci --test_output=errors
+
+run:debs-ci --config=ci
+run:debs-ci --//debs:excludes=//debs:custom-excludes.txt
+run:debs-ci --//debs:token=//debs:token.txt
+run:debs-ci --//tools/tarball:target=//debs
+run:debs-ci --//tools/tarball:overwrite=//tools/tarball:true

.github/workflows/envoy.yml

@@ -20,6 +20,55 @@ jobs:
     if: github.repository_owner == 'envoyproxy'
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.1.7
+    - id: versions
+      run: |
+        bazel build --config=ci //debs:minor_versions
+        VERSION_HASH="$(cat bazel-bin/debs/minor_versions.txt | sha256sum | cut -d' ' -f1)"
+        echo "hash=${VERSION_HASH}" >> "$GITHUB_OUTPUT"
+    - uses: actions/cache/restore@v4
+      id: cache
+      with:
+        key: ${{ steps.versions.outputs.hash }}
+        path: /tmp/repository
+    - id: repository
+      run: |
+        mkdir -p /tmp/repository
+        rm -rf /tmp/repository/v1.28.3
+        ls /tmp/repository | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' | sort -u > debs/custom-excludes.txt
+        DIR_SHA=$(cat debs/custom-excludes.txt | sha256sum | cut -d' ' -f1)
+        echo "cached=${DIR_SHA}" >> $GITHUB_OUTPUT
     - run: |
-        bazel build --config=ci //:envoy_versions
-        cat bazel-bin/envoy_versions.json | jq '.'
+        echo $GITHUB_TOKEN > debs/token.txt
+        bazel run \
+            --config=debs-ci \
+            //tools/tarball:unpack \
+            /tmp/repository
+        rm -rf /tmp/repository/v1.28.3
+    - run: |
+        find /tmp/repository
+    - run: |
+        UPDATED=$(ls /tmp/repository | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' | sort -u | sha256sum | cut -d' ' -f1)
+        if [[ "${{ steps.repository.outputs.cached }}" != "$UPDATED" ]]; then
+           echo "changed=1" >> $GITHUB_OUTPUT
+        fi
+      id: update
+      if: ${{ github.event_name == 'push' && steps.cache.outputs.cache-hit && steps.update.outputs.changed }}
+
+    - uses: actions/upload-artifact@v4
+      with:
+        path: /tmp/repository
+        name: debs
+
+    # if we are on main and the files have changed, then cache - PRs always just use the main cache
+    - if: ${{ github.event_name == 'push' && steps.cache.outputs.cache-hit && steps.update.outputs.changed }}
+      run: |
+         gh extension install actions/gh-actions-cache
+         gh actions-cache delete --confirm ${{ steps.versions.outputs.hash }}
+      env:
+        GH_TOKEN: ${{ github.token }}
+
+    - uses: actions/cache/save@v4
+      if: ${{ github.event_name == 'push' && steps.update.outputs.changed }}
+      with:
+        key: ${{ steps.versions.outputs.hash }}
+        path: /tmp/repository

BUILD

@@ -1,6 +1,4 @@
 load("@bazel_gazelle//:def.bzl", "gazelle")
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
-load("@aspect_bazel_lib//lib:jq.bzl", "jq")
 
 # gazelle:prefix github.com/aptly-dev/aptly
 gazelle(name = "gazelle")
@@ -22,26 +20,3 @@ gazelle(
     command = "update-repos",
     data = [":go.mod"],
 )
-
-jq(
-    name = "envoy_versions",
-    srcs = ["@envoy_repo//:project"],
-    out = "envoy_versions.json",
-    filter = """
-    .releases as $releases
-    | {
-      latest_releases: [
-        .stable_versions[]
-        | . as $minor
-        | {
-            version: $minor,
-            releases: (
-              $releases
-              | map(select(startswith("v" + $minor + ".")))
-            )
-          }
-      ]
-    }
-    """,
-    visibility = ["//visibility:public"],
-)

debs/BUILD

@@ -0,0 +1,139 @@
+load("@aspect_bazel_lib//lib:jq.bzl", "jq")
+
+exports_files([
+    "custom-excludes.txt",
+    "token.txt",
+])
+
+jq(
+    name = "envoy_versions",
+    srcs = ["@envoy_repo//:project"],
+    out = "envoy_versions.json",
+    filter = """
+    .releases as $releases
+    | {
+      latest_releases: [
+        .stable_versions[]
+        | . as $minor
+        | {
+            version: $minor,
+            releases: (
+              $releases
+              | map(select(startswith("v" + $minor + ".")))
+            )
+          }
+      ]
+    }
+    """,
+    visibility = ["//visibility:public"],
+)
+
+jq(
+    name = "minor_versions",
+    srcs = [":envoy_versions"],
+    out = "minor_versions.txt",
+    filter = """
+    .latest_releases[] | .version
+    """,
+    args = ["-r"],
+    visibility = ["//visibility:public"],
+)
+
+jq(
+    name = "patch_versions",
+    srcs = [":envoy_versions"],
+    out = "patch_versions.txt",
+    filter = """
+    .latest_releases[] | .releases[]
+    """,
+    args = ["-r"],
+    visibility = ["//visibility:public"],
+)
+
+jq(
+    name = "deb_checksum_downloads",
+    srcs = [":envoy_versions"],
+    out = "deb_checksum_downloads.txt",
+    filter = """
+    reduce .latest_releases[].releases[] as $item ({};
+      .[$item] = {"signature": "[email protected]"})
+    | with_entries(
+        {"key": "https://github.com/envoyproxy/envoy/releases/download/\\(.key)/checksums.txt.asc",
+         "value": .value})
+    """,
+    args = ["-r"],
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "published_checksums",
+    outs = ["published_checksums.txt"],
+    cmd = """
+    $(location //tools/fetch) $(location :deb_checksum_downloads) --output=json > $@
+    """,
+    tools = [
+        "//tools/fetch",
+        ":deb_checksum_downloads",
+    ],
+)
+
+jq(
+    name = "debs_downloads",
+    srcs = [":published_checksums"],
+    out = "debs_downloads.json",
+    filter = """
+    with_entries(
+      .key as $key
+        | .value as $value
+        | ($key | capture("v(?<version>[0-9.]+)") | .version) as $version
+        | {key: ("https://github.com/envoyproxy/envoy/releases/download/v\\($version)/debs.tar.gz"),
+           value: {
+             "path": "v\\($version)",
+             "checksum": (
+               $value
+               | split("\n")
+               | map(select(endswith("debs.tar.gz")))
+               | first
+               | split(" ")
+               | .[0]
+             )}})
+    """,
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "empty",
+    outs = ["empty.txt"],
+    cmd = """
+    touch $@
+    """,
+)
+
+label_flag(
+    name = "excludes",
+    build_setting_default = ":empty",
+)
+
+label_flag(
+    name = "token",
+    build_setting_default = ":empty",
+)
+
+genrule(
+    name = "debs",
+    outs = ["debs.tar.gz"],
+    cmd = """
+    $(location //tools/fetch) $(location :debs_downloads) \
+        --concurrency 4 \
+        --excludes=$(location :excludes) \
+        --token-path=$(location :token) \
+        --extract-downloads \
+        --output-path=$@
+    """,
+    tools = [
+        "//tools/fetch",
+        ":debs_downloads",
+        ":excludes",
+        ":token",
+    ],
+)

setup.bzl

@@ -1,6 +1,11 @@
 load("@rules_proto//proto:setup.bzl", "rules_proto_setup")
 load("@rules_python//python:repositories.bzl", "py_repositories")
+# load("@envoy//bazel:api_binding.bzl", "envoy_api_binding")
+# load("@envoy//bazel:repositories.bzl", "envoy_build_config")
+
 
 def setup():
     py_repositories()
     rules_proto_setup()
+    # envoy_api_binding()
+    # envoy_build_config(name = "envoy_build_config")

tools/fetch/BUILD

@@ -0,0 +1,10 @@
+load("@base_pip3//:requirements.bzl", base_entry_point = "entry_point")
+load("@envoy_toolshed//py:macros.bzl", "entry_point")
+
+entry_point(
+    name = "fetch",
+    pkg = "envoy.base.utils",
+    script = "envoy.fetch",
+    entry_point_alias = base_entry_point,
+    visibility = ["//visibility:public"],
+)

tools/tarball/BUILD

@@ -0,0 +1,7 @@
+load("@envoy_toolshed//tarball:macros.bzl", "unpacker")
+
+licenses(["notice"])  # Apache 2
+
+unpacker(
+    name = "unpack",
+)

versions.bzl

@@ -40,18 +40,18 @@ VERSIONS = {
     "envoy": {
         "type": "github_archive",
         "repo": "envoyproxy/envoy",
-        "version": "fea66c359069991e88bdfa4e0f2883c90cc39aef",
-        "sha256": "96294f4b491c676b650ddeb07c7986ec7e48b5ae5a75c4adebbb9a58741a2fb1",
+        "version": "4108a96e215897d47cfeb3578486f0578333c1bc",
+        "sha256": "62606a0dbe73d32edfde176faaf1325f5f7a1531d1d40e12f4d6d85b75bc6b2b",
         "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
         "strip_prefix": "envoy-{version}",
     },
     "envoy_toolshed": {
         "type": "github_archive",
         "repo": "envoyproxy/toolshed",
-        "version": "0.1.4",
-        "sha256": "7ddfd251a89518b97c4eb8064a7d37454bbd998bf29e4cd3ad8f44227b5ca7b3",
-        "urls": ["https://github.com/{repo}/archive/bazel-v{version}.tar.gz"],
-        "strip_prefix": "toolshed-bazel-v{version}/bazel",
+        "version": "20c6067d87f1eb2e5a4d80f815d2aa3a10ed37ad",
+        "sha256": "9578a70d01585cdea680b1fddc86813f8282500565b1b9ceeb600940b122de43",
+        "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
+        "strip_prefix": "toolshed-{version}/bazel",
     },
     "io_bazel_rules_go": {
         "type": "github_archive",