transparently use docker targets when building (#2519)

Makefile

@@ -1,43 +1,139 @@
-MAKEFLAGS+=--no-builtin-rules --warn-undefined-variables
+MAKEFLAGS+=--no-builtin-rules --warn-undefined-variables --no-print-directory
 .SUFFIXES:
 
 BASE_DIRECTORY:=$(abspath .)
 BUILD_LIB=${BASE_DIRECTORY}/build/lib
 SHELL_TRACE?=false
-SHELL:=$(if $(filter true,$(SHELL_TRACE)),$(BUILD_LIB)/make_shell_trace.sh,bash)
-.SHELLFLAGS:=$(if $(filter true,$(SHELL_TRACE)),-c,-eu -o pipefail -c)
+DEFAULT_SHELL:=$(if $(filter true,$(SHELL_TRACE)),$(BUILD_LIB)/make_shell.sh trace,bash)
+SHELL:=$(DEFAULT_SHELL)
+.SHELLFLAGS:=-eu -o pipefail -c
 
 AWS_ACCOUNT_ID?=$(shell aws sts get-caller-identity --query Account --output text)
 AWS_REGION?=us-west-2
 IMAGE_REPO?=$(if $(AWS_ACCOUNT_ID),$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com,localhost:5000)
 ECR_PUBLIC_URI?=$(shell aws ecr-public describe-registries --region us-east-1 --query 'registries[0].registryUri' --output text)
 JOB_TYPE?=
 
-RELEASE_BRANCH?=
+RELEASE_BRANCH?=$(LATEST_EKSD_RELEASE)
 GIT_HASH=$(shell git -C $(BASE_DIRECTORY) rev-parse HEAD)
 ALL_PROJECTS=$(shell $(BUILD_LIB)/all_projects.sh $(BASE_DIRECTORY))
 
+LATEST_EKSD_RELEASE=$(shell source $(BUILD_LIB)/common.sh && build::eksd_releases::get_release_branch)
+
 # $1 - project name using _ as separator, ex: rancher_local-path-provisoner
 PROJECT_PATH_MAP=projects/$(patsubst $(firstword $(subst _, ,$(1)))_%,$(firstword $(subst _, ,$(1)))/%,$(1))
 
+BUILDER_PLATFORM_ARCH=$(if $(filter x86_64,$(shell uname -m)),amd64,arm64)
+
 # $1 - variable name to resolve and cache
 CACHE_RESULT = $(if $(filter undefined,$(origin _cached-$1)),$(eval _cached-$1 := 1)$(eval _cache-$1 := $($1)),)$(_cache-$1)
 
 # $1 - variable name
 CACHE_VARIABLE=$(eval _old-$(1)=$(value $(1)))$(eval $(1)=$$(call CACHE_RESULT,_old-$(1)))
 
-CACHE_VARS=ALL_PROJECTS AWS_ACCOUNT_ID GIT_HASH
+CACHE_VARS=ALL_PROJECTS AWS_ACCOUNT_ID BUILDER_PLATFORM_ARCH GIT_HASH LATEST_EKSD_RELEASE
 $(foreach v,$(CACHE_VARS),$(call CACHE_VARIABLE,$(v)))
 
 .PHONY: clean-project-%
+clean-project-%: PROJECT_PATH=$(call PROJECT_PATH_MAP,$*)
+clean-project-%: export RELEASE_BRANCH=$(LATEST_EKSD_RELEASE)
 clean-project-%:
-	$(eval PROJECT_PATH=$(call PROJECT_PATH_MAP,$*))
 	$(MAKE) clean -C $(PROJECT_PATH)
 
 .PHONY: clean
 clean: $(addprefix clean-project-, $(ALL_PROJECTS))
 	rm -rf _output
 
+
+############################## BUILD ALL ###################################
+
+.PHONY: build-all
+build-all: UPLOAD_ARTIFACTS_TO_S3?=false
+build-all: build-all-warning
+# Build projects with dependecies first to try and validate if there are any missing
+	@set -eu -o pipefail; \
+	export RELEASE_BRANCH=$(RELEASE_BRANCH); \
+	PROJECTS="$(foreach project,$(ALL_PROJECTS),$(call PROJECT_PATH_MAP,$(project)))"; \
+	PROJS=($${PROJECTS// / }); \
+  	for proj in "$${PROJS[@]}"; do \
+		if  [ -n "$$($(MAKE) -C $$proj var-value-PROJECT_DEPENDENCIES)" ] && [ ! -f $$proj/eks-anywhere-full-build-complete ]; then \
+			$(MAKE) $$proj/eks-anywhere-full-build-complete; \
+		fi; \
+	done; \
+	for proj in "$${PROJS[@]}"; do \
+		if  [ ! -f $$proj/eks-anywhere-full-build-complete ]; then \
+			$(MAKE) $$proj/eks-anywhere-full-build-complete; \
+		fi; \
+	done
+
+# Specific overrides
+projects/kubernetes-sigs/kind/eks-anywhere-full-build-complete: override IMAGE_PLATFORMS=linux/amd64,linux/arm64
+
+# tinkerbell/hook needs to be built with a public ecr repo so docker container can pull
+projects/tinkerbell/hook/eks-anywhere-full-build-complete: IMAGE_REPO=$(ECR_PUBLIC_URI)
+projects/tinkerbell/hook/eks-anywhere-full-build-complete: override IMAGE_PLATFORMS=linux/amd64,linux/arm64
+
+projects/kubernetes-sigs/image-builder/eks-anywhere-full-build-complete: MAIN_TARGET=build
+projects/kubernetes-sigs/image-builder/eks-anywhere-full-build-complete: export SKIP_METAL_INSTANCE_TEST=true
+
+projects/aws/eks-a-admin-image/eks-anywhere-full-build-complete: MAIN_TARGET=build
+
+projects/torvalds/linux/eks-anywhere-full-build-complete: export BINARY_PLATFORMS=linux/amd64 linux/arm64
+# Skips
+projects/aws/cluster-api-provider-aws-snow/eks-anywhere-full-build-complete:
+	@echo "Skipping aws/cluster-api-provider-aws-snow: container images are pulled cross account"
+	@touch $@
+
+projects/goharbor/harbor/eks-anywhere-full-build-complete:
+	@echo "Skipping /goharbor/harbor: we patch vendor directory so we skip go mod download, which can cause slight checksum differences"
+	@echo "run the 'clean-go-cache' target before running harbor if you want to build with matching checksums"
+	@touch $@
+
+# Actual target
+%/eks-anywhere-full-build-complete: IMAGE_PLATFORMS=linux/$(BUILDER_PLATFORM_ARCH)
+# override this on the command line to true if you want to push to your own s3 bucket
+%/eks-anywhere-full-build-complete: MAIN_TARGET=release
+%/eks-anywhere-full-build-complete:
+	@set -eu -o pipefail; \
+	export RELEASE_BRANCH=$(RELEASE_BRANCH); \
+	if  [ -n "$$($(MAKE) -C $(@D) var-value-PROJECT_DEPENDENCIES)" ]; then \
+		PROJECT_DEPS=$$($(MAKE) -C $(@D) var-value-PROJECT_DEPENDENCIES); \
+		DEPS=($${PROJECT_DEPS// / }); \
+  		for dep in "$${DEPS[@]}"; do \
+			if [[ "$${dep}" = *"eksa"* ]]; then \
+				OVERRIDES="IMAGE_REPO=$(IMAGE_REPO) IMAGE_PLATFORMS=$(IMAGE_PLATFORMS)"; \
+				DEP_RELEASE_BRANCH="$$(cut -d/ -f4 <<< $$dep)"; \
+				if [ -n "$${DEP_RELEASE_BRANCH}" ]; then \
+					dep="$$(dirname $$dep)"; \
+					OVERRIDES+=" RELEASE_BRANCH=$$DEP_RELEASE_BRANCH"; \
+				fi; \
+				echo "Running make $${dep#eksa/} as dependency for $(@D)"; \
+				$(MAKE) projects/$${dep#"eksa/"}/eks-anywhere-full-build-complete $$OVERRIDES; \
+				if [ -n "$${DEP_RELEASE_BRANCH}" ]; then \
+					rm projects/$${dep#"eksa/"}/eks-anywhere-full-build-complete; \
+				fi; \
+			fi; \
+		done; \
+	fi; \
+	TARGETS="attribution $(MAIN_TARGET)"; \
+	if  [ -n "$$($(MAKE) -C $(@D) var-value-IMAGE_NAMES)" ] || [ "$$($(MAKE) -C $(@D) var-value-HAS_HELM_CHART)" = "true" ]; then \
+		TARGETS="create-ecr-repos $${TARGETS}"; \
+	fi; \
+	if [ "$(UPLOAD_ARTIFACTS_TO_S3)" = "true" ]; then \
+		TARGETS+=" UPLOAD_DRY_RUN=false UPLOAD_CREATE_PUBLIC_ACL=false"; \
+	fi; \
+	TARGETS+=" IMAGE_REPO=$(IMAGE_REPO) IMAGE_PLATFORMS=$(IMAGE_PLATFORMS) RELEASE_BRANCH=$(RELEASE_BRANCH)"; \
+	echo "Running 'make -C $(@D) $${TARGETS}'"; \
+	make -C $(@D) $${TARGETS}; \
+	touch $@
+
+.PHONY: build-all-warning
+build-all-warning:
+	@echo "*** Warning: this target is not meant to used except for specific testing situations ***"
+	@echo "*** this will likely fail and either way run for a really long time ***"
+
+#########################################################################
+
 .PHONY: add-generated-help-block-project-%
 add-generated-help-block-project-%:
 	$(eval PROJECT_PATH=$(call PROJECT_PATH_MAP,$*))
@@ -70,14 +166,18 @@ update-checksum-files: $(addprefix checksum-files-project-, $(ALL_PROJECTS))
 update-attribution-files: add-generated-help-block attribution-files
 	build/update-attribution-files/create_pr.sh
 
+.PHONY: start-docker-builder
+start-docker-builder: # Start long lived builder base docker container
+	@$(MAKE) -C projects/aws/eks-anywhere-build-tooling start-docker-builder
+
 .PHONY: stop-docker-builder
 stop-docker-builder:
 	docker rm -f -v eks-a-builder
 
 .PHONY: run-buildkit-and-registry
 run-buildkit-and-registry:
-	docker run -d --name buildkitd --net host --privileged moby/buildkit:v0.10.6-rootless
-	docker run -d --name registry  --net host registry:2
+	docker run --rm -d --name buildkitd --net host --privileged moby/buildkit:v0.12.2-rootless
+	docker run --rm -d --name registry  --net host registry:2
 
 .PHONY: stop-buildkit-and-registry
 stop-buildkit-and-registry:

build/lib/buildkit.sh

@@ -23,7 +23,11 @@ if [ "${USE_BUILDX:-}" == "true" ]; then
     printf "\nBuilding with docker buildx\n" >&2
 
     CMD="docker buildx"
-    ARGS=""
+    # for the hook build pushing the provenance manifest borks linuxkit
+    # this is not pushed by buildctl which is what we use in prod
+    ARGS="build --provenance=false "
+    # shift build off the args list
+    shift
     while test $# -gt 0; do
         case "$1" in
             --frontend)

build/lib/common.sh

@@ -125,7 +125,7 @@ function build::common::upload_artifacts() {
 }
 
 function build::gather_licenses() {
-  local -r outputdir=$1
+  local -r outputdir="${1%/}" # remove trailing slash if it exists
   local -r patterns=$2
   local -r golang_version=$3
   local -r threshold=$4
@@ -266,7 +266,11 @@ function build::common::get_go_path() {
 }
 
 function build::common::use_go_version() {
-  local -r version=$1
+  local -r version=${1:-}
+
+  if [ -z "$version" ]; then
+    return
+  fi
 
   if (( "${version#*.}" < 16 )); then
     echo "Building with GO version $version is no longer supported!  Please update the build to use a newer version."
@@ -278,7 +282,8 @@ function build::common::use_go_version() {
   # Adding to the beginning of PATH to allow for builds on specific version if it exists
   export PATH=${gobinarypath}:$PATH
   export GOCACHE=$(go env GOCACHE)/$version
-  }
+  echo "$(go version)"
+}
 
 # Use a seperate build cache for each project/version to ensure there are no
 # shared bits which can mess up the final checksum calculation

build/lib/generate_help_body.sh

@@ -153,7 +153,7 @@ fi
 TARGETS=(${IN_DOCKER_TARGETS// / })
 DOCKER_TARGETS="${NL}${NL}##@ Run in Docker Targets"
 for target in "${TARGETS[@]}"; do
-    DOCKER_TARGETS+="${NL}run-${target}-in-docker: ## Run \`${target}\` in docker builder container"
+    DOCKER_TARGETS+="${NL}run-in-docker/${target}: ## Run \`${target}\` in docker builder container"
 done
 
 cat >> $HELPFILE << EOF

build/lib/make_shell.sh

@@ -48,7 +48,7 @@ DATE_NANO=$(if [ "$(uname -s)" = "Linux" ] || [ "$DATE" = "gdate" ]; then echo %
 
 START_TIME=$($DATE +%s.$DATE_NANO)
 
-echo -e "\n------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") Starting target=$TARGET -------------------"
+echo -e "\n------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") $([ -n "${DOCKER_RUN_BASE_DIRECTORY:-}" ] && echo "(In Docker) ")Starting target=$TARGET -------------------"
 echo "($(pwd)) \$ $@"
 eval "$@"
-echo -e "------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") Finished target=$TARGET duration=$(echo $($DATE +%s.$DATE_NANO) - $START_TIME | bc) seconds -------------------\n"
+echo -e "------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") $([ -n "${DOCKER_RUN_BASE_DIRECTORY:-}" ] && echo "(In Docker) ")Finished target=$TARGET duration=$(echo $($DATE +%s.$DATE_NANO) - $START_TIME | bc) seconds -------------------\n"

build/lib/prepare_build_container_user.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright 2020 Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+GROUP_ID="$1"
+USER_ID="$2"
+
+# when running the build container on linux, the user in the container needs 
+# to match the host user id and group id
+# otherwise there will be perms issues due to go mods being downloaded in the
+# container as root even though the host user is not
+
+sed -i 's/^CREATE_MAIL_SPOOL=yes/CREATE_MAIL_SPOOL=no/' /etc/default/useradd
+groupadd -g 100 users
+groupadd --gid "$GROUP_ID" matchinguser
+useradd  --no-create-home --uid "$USER_ID" --gid "$GROUP_ID" matchinguser
+mkdir -p /home/matchinguser
+chown -R matchinguser:matchinguser /home/matchinguser