Add infrastructure for generating the golang-debian image using upstream binaries

projects/golang/go/Makefile

@@ -38,6 +38,11 @@ IMAGE_NAME?=golang-debian
 IMAGE_TAG?=$(GIT_TAG)-$(BUILD_ID)-$(IMAGE_BUILD_ID)
 LATEST_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME):$(GIT_TAG)
 IMAGE?=$(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG),$(LATEST_IMAGE)
+# Variables used in the using upstream binarys in the golang-debian image
+BUILDER_BASE_VERSIONS_YAML=$(BASE_DIRECTORY)/builder-base/versions.yaml
+BUILDER_BASE_GO_VERSION=$(shell grep -E "^GOLANG_VERSION_$(subst .,,$(GO_SOURCE_VERSION))" $(BUILDER_BASE_VERSIONS_YAML))
+GO_BIN_VERSION_WITH_RELEASE=$(subst GOLANG_VERSION_$(subst .,,$(GO_SOURCE_VERSION)): ,,$(BUILDER_BASE_GO_VERSION))
+GO_BIN_VERSION_WITHOUT_RELEASE=$(shell [[ $(GO_BIN_VERSION_WITH_RELEASE) =~ [0-9]+.[0-9]+.[0-9]+ ]] && echo $${BASH_REMATCH[0]})
 
 PUSH_IMAGES?=true
 BUILDKIT_OUTPUT=type=image,oci-mediatypes=true,\"name=$(IMAGE)\",push=$(PUSH_IMAGES)
@@ -98,6 +103,10 @@ validate-go-archive-checksum:
 		fi ; \
 	done
 
+.PHONY: prod-release-images-upstream-bins
+prod-release-images-upstream-bins: export AWS_PROFILE=ecr-public-push
+prod-release-images-upstream-bins: images-upstream-bins
+
 .PHONY: fetch-golang-source-archive
 fetch-golang-source-archive:
 	curl -L -o $(HOME)/rpmbuild/SOURCES/$(GIT_TAG).src.tar.gz https://github.com/golang/go/archive/refs/tags/$(GIT_TAG).tar.gz --create-dirs 
@@ -144,6 +153,33 @@ images:
 		--progress plain \
 		--output $(BUILDKIT_OUTPUT)
 
+.PHONY: fetch-golang-upstream-bins
+fetch-golang-upstream-bins:
+	$(PROJECT_DIRECTORY)/scripts/get_upstream_golang.sh $(VERSION_DIRECTORY)/archives/ $(GO_BIN_VERSION_WITHOUT_RELEASE)
+
+.PHONY: local-images-upstream-bins
+local-images-upstream-bins: PUSH_IMAGES=false
+local-images-upstream-bins: export BUILDKIT_HOST=docker-container://buildkitd
+local-images-upstream-bins: images-upstream-bins
+
+.PHONY: images-upstream-bins
+images-upstream-bins: IMAGE_REPO=$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com
+images-upstream-bins: IMAGE_NAME=golang-debian
+images-upstream-bins: IMAGE_TAG=$(GO_BIN_VERSION_WITHOUT_RELEASE)-$(BUILD_ID)-$(IMAGE_BUILD_ID)
+images-upstream-bins: LATEST_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME):$(GO_BIN_VERSION_WITHOUT_RELEASE)
+images-upstream-bins: IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG),$(LATEST_IMAGE)
+images-upstream-bins: fetch-golang-upstream-bins buildkit-check
+images-upstream-bins:
+	$(BASE_DIRECTORY)/scripts/buildkit.sh \
+		build \
+		--frontend dockerfile.v0 \
+		--opt platform=$(GOOS)/$(ARCH_LOWER) \
+		--opt build-arg:GOLANG_ARCHIVE_PATH=$(GOOS)/$(ARCH_LOWER)/go$(GO_BIN_VERSION_WITHOUT_RELEASE).$(GOOS)-$(ARCH_LOWER).tar.gz \
+		--local dockerfile=$(PROJECT_DIRECTORY)/docker/debianBase \
+		--local context=$(VERSION_DIRECTORY)/archives \
+		--progress plain \
+		--output $(BUILDKIT_OUTPUT)
+
 .PHONY: generate-golang-archive
 generate-golang-archive: extract-golang-rpm tar-golang-sources
 

projects/golang/go/docker/debianBase/Dockerfile

@@ -11,7 +11,7 @@ COPY --from=go-untar /usr/local/go/ /usr/local/go/
 
 
 ENV GOPATH /go
-ENV PATH /usr/local/go/bin:$GOPATH/bin:$PATH
+ENV PATH /usr/local/go/go/bin:$GOPATH/bin:$PATH
 
 RUN set -eux; \
 	apt-get update; \
@@ -28,4 +28,4 @@ RUN set -eux; \
 	; \
 	go version
 
-WORKDIR $GOPATH
+WORKDIR $GOPATH

projects/golang/go/scripts/get_upstream_golang.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+set -o pipefail
+
+SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+BASE_DIRECTORY="$(git rev-parse --show-toplevel)"
+GO_PREFIX="go"
+ARCHITECTURE="linux/amd64" # Currently only build golang-debian image for amd64, if other arches are needed add
+
+OUTPUT_DIR="$1"
+GO_BIN_VERSION="$2"
+
+source ${BASE_DIRECTORY}/builder-base/scripts/common_vars.sh
+
+# Download from upstream and validate CHECKSUMs
+function build::go::download {
+  # Set up specific go version by using go get, additional versions apart from default can be installed by calling
+  # the function again with the specific parameter.
+  local version=${1}
+  local outputDir=${2}
+  local archs=${3}
+
+  for arch in ${archs/,/ }; do
+    local filename="$outputDir/${arch}/go$version.${arch/\//-}.tar.gz"
+    if [ ! -f $filename ]; then
+      curl -sSLf --retry 5 "https://go.dev/dl/go$version.${arch/\//-}.tar.gz" -o $filename --create-dirs
+      sha256sum=$(curl -sSLf --retry 5 "https://go.dev/dl/?mode=json" | jq -r --arg tar "go$version.${arch/\//-}.tar.gz" '.[].files[] | if .filename == $tar then .sha256 else "" end' | xargs)
+
+      if [[ $(sha256sum ${filename} | cut -d ' ' -f1) != "${sha256sum}" ]]; then
+        echo "CHECKSUMs don't match"
+        exit 1
+      fi
+    fi
+  done
+}
+
+# strip the release version off the end of
+build::go::download "${GO_BIN_VERSION}" "$OUTPUT_DIR" "$ARCHITECTURE"

projects/golang/go/scripts/prow_release_images.sh

@@ -14,24 +14,24 @@
 # limitations under the License.
 
 if [ "$ARCHITECTURE" == "ARM64" ]; then
-    echo "Won't perform image release for ARM64 arch"
-    exit 0
+  echo "Won't perform image release for ARM64 arch"
+  exit 0
 fi
 
 if [ "$AWS_ROLE_ARN" == "" ]; then
-    echo "Empty AWS_ROLE_ARN"
-    exit 1
+  echo "Empty AWS_ROLE_ARN"
+  exit 1
 fi
 
 if [ "$ECR_PUBLIC_PUSH_ROLE_ARN" == "" ]; then
-    echo "Empty ECR_PUBLIC_PUSH_ROLE_ARN"
-    exit 1
+  echo "Empty ECR_PUBLIC_PUSH_ROLE_ARN"
+  exit 1
 fi
 
 BASE_DIRECTORY=$(git rev-parse --show-toplevel)
 cd ${BASE_DIRECTORY} || exit
 
-cat << EOF > awscliconfig
+cat <<EOF >awscliconfig
 [default]
 output=json
 region=${AWS_REGION:-${AWS_DEFAULT_REGION:-us-west-2}}
@@ -47,4 +47,4 @@ export AWS_CONFIG_FILE=$(pwd)/awscliconfig
 export AWS_PROFILE=ecr-public-push
 unset AWS_ROLE_ARN AWS_WEB_IDENTITY_TOKEN_FILE
 
-make -C ${BASE_DIRECTORY}/projects/golang/go prod-release-images
+make -C ${BASE_DIRECTORY}/projects/golang/go "prod-release-images-upstream-bins"