diff --git a/Common.mk b/Common.mk index e52224b4b2..d877756aec 100644 --- a/Common.mk +++ b/Common.mk @@ -13,8 +13,11 @@ BUILD_LIB=$(BASE_DIRECTORY)/build/lib OUTPUT_BIN_DIR?=$(OUTPUT_DIR)/bin/$(REPO) SHELL_TRACE?=false -DEFAULT_SHELL:=$(if $(filter true,$(SHELL_TRACE)),$(BUILD_LIB)/make_shell.sh trace,bash) -SHELL:=$(DEFAULT_SHELL) +TRACE_SHELL=$(BUILD_LIB)/make_shell.sh trace +LOGGING_SHELL=$(BUILD_LIB)/make_shell.sh log +NOOP_SHELL=true +DEFAULT_SHELL=$(if $(filter true,$(SHELL_TRACE)),$(TRACE_SHELL),bash) +SHELL=$(DEFAULT_SHELL) .SHELLFLAGS:=-eu -o pipefail -c #################### AWS ########################### AWS_REGION?=us-west-2 @@ -96,9 +99,9 @@ SKIPPED_K8S_VERSIONS?= BINARIES_ARE_RELEASE_BRANCHED?=true IS_RELEASE_BRANCH_BUILD=$(filter true,$(HAS_RELEASE_BRANCHES)) UNRELEASE_BRANCH_BINARY_TARGETS=binaries attribution checksums -IS_UNRELEASE_BRANCH_TARGET=$(and $(filter false,$(BINARIES_ARE_RELEASE_BRANCHED)),$(filter $(UNRELEASE_BRANCH_BINARY_TARGETS) $(foreach target,$(UNRELEASE_BRANCH_BINARY_TARGETS),run-$(target)-in-docker),$(MAKECMDGOALS))) +IS_UNRELEASE_BRANCH_TARGET=$(and $(filter false,$(BINARIES_ARE_RELEASE_BRANCHED)),$(filter $(UNRELEASE_BRANCH_BINARY_TARGETS) $(foreach target,$(UNRELEASE_BRANCH_BINARY_TARGETS),run-$(target)-in-docker run-in-docker/$(target)),$(MAKECMDGOALS))) TARGETS_ALLOWED_WITH_NO_RELEASE_BRANCH?= -TARGETS_ALLOWED_WITH_NO_RELEASE_BRANCH+=build release clean clean-extra clean-go-cache help stop-docker-builder create-ecr-repos all-attributions all-checksums all-attributions-checksums update-patch-numbers check-for-release-branch-skip +TARGETS_ALLOWED_WITH_NO_RELEASE_BRANCH+=build release clean clean-extra clean-go-cache help start-docker-builder stop-docker-builder create-ecr-repos all-attributions all-checksums all-attributions-checksums update-patch-numbers check-for-release-branch-skip run-buildkit-and-registry MAKECMDGOALS_WITHOUT_VAR_VALUE=$(foreach t,$(MAKECMDGOALS),$(if $(findstring var-value-,$(t)),,$(t))) ifneq ($(and $(IS_RELEASE_BRANCH_BUILD),$(or $(RELEASE_BRANCH),$(IS_UNRELEASE_BRANCH_TARGET))),) RELEASE_BRANCH_SUFFIX=$(if $(filter true,$(BINARIES_ARE_RELEASE_BRANCHED)),/$(RELEASE_BRANCH),) @@ -173,17 +176,24 @@ IMAGE_IMPORT_CACHE?=type=registry,ref=$(LATEST_IMAGE) type=registry,ref=$(subst BUILD_OCI_TARS?=false -LOCAL_IMAGE_TARGETS=$(foreach image,$(IMAGE_NAMES),$(image)/images/amd64) +LOCAL_IMAGE_TARGETS=$(foreach image,$(IMAGE_NAMES),$(image)/images/$(BUILDER_PLATFORM_ARCH)) IMAGE_TARGETS=$(foreach image,$(IMAGE_NAMES),$(if $(filter true,$(BUILD_OCI_TARS)),$(call IMAGE_TARGETS_FOR_NAME,$(image)),$(image)/images/push)) +# intentionally not setting a default verison, we do not want projects depending on a default +GOLANG_VERSION?= # If running in the builder base on prow or codebuild, grab the current tag to be used when building with cgo -CURRENT_BUILDER_BASE_TAG=$(or $(and $(wildcard /config/BUILDER_BASE_TAG_FILE),$(shell cat /config/BUILDER_BASE_TAG_FILE)),latest) +BUILDER_BASE_TAG=${BUILDER_BASE_TAG:-$(curl -s https://raw.githubusercontent.com/aws/eks-anywhere-prow-jobs/main/BUILDER_BASE_TAG_FILE)} + +CURRENT_BUILDER_BASE_TAG=$(or \ + $(and $(wildcard /config/BUILDER_BASE_TAG_FILE),$(shell cat /config/BUILDER_BASE_TAG_FILE))\ + ,$(shell curl -s https://raw.githubusercontent.com/aws/eks-anywhere-prow-jobs/main/BUILDER_BASE_TAG_FILE)) CURRENT_BUILDER_BASE_IMAGE=$(if $(CODEBUILD_BUILD_IMAGE),$(CODEBUILD_BUILD_IMAGE),$(BASE_IMAGE_REPO)/builder-base:$(CURRENT_BUILDER_BASE_TAG)) GOLANG_GCC_BUILDER_IMAGE=$(BASE_IMAGE_REPO)/golang:$(shell cat $(BASE_DIRECTORY)/EKS_DISTRO_MINIMAL_BASE_GOLANG_COMPILER_$(GOLANG_VERSION)_GCC_TAG_FILE) # in CODEBUILD always use buildctl BUILDCTL_AVAILABLE=$(or $(filter true,$(CODEBUILD_CI)),$(shell command -v buildctl &> /dev/null && buildctl debug workers &> /dev/null && echo "true" || echo "false")) BUILDX_AVAILABLE=$(shell docker buildx inspect &> /dev/null && echo "true" || echo "false") +DOCKER_AVAILABLE=$(shell command -v docker &> /dev/null && docker info &> /dev/null && echo "true" || echo "false") #################################################### #################### HELM ########################## @@ -200,6 +210,7 @@ HELM_DESTINATION_REPOSITORY?=$(IMAGE_COMPONENT) HELM_IMAGE_LIST?= HELM_GIT_CHECKOUT_TARGET?=$(HELM_SOURCE_REPOSITORY)/eks-anywhere-checkout-$(HELM_GIT_TAG) HELM_GIT_PATCH_TARGET?=$(HELM_SOURCE_REPOSITORY)/eks-anywhere-helm-patched +PACKAGE_DEPENDENCIES?= #################################################### #### HELPERS ######## @@ -365,8 +376,8 @@ NEEDS_CGO_BUILDER=$(and $(if $(filter true,$(CGO_CREATE_BINARIES)),true,),$(if $ USE_DOCKER_FOR_CGO_BUILD?=false GO_MOD_CACHE=$(shell if source $(BUILD_LIB)/common.sh && build::common::use_go_version $(GOLANG_VERSION) > /dev/null 2>&1 && command -v go &> /dev/null; then go env GOMODCACHE; else echo $${HOME}/.cache/go/pkg/mod; fi) GO_BUILD_CACHE=$(shell if source $(BUILD_LIB)/common.sh && build::common::use_go_version $(GOLANG_VERSION) > /dev/null 2>&1 && command -v go &> /dev/null; then go env GOCACHE; else echo $${HOME}/.cache/go-build; fi) -CGO_TARGET?= GO_MODS_VENDORED?=false +DOCKER_PLATFORM?= ###################### #### BUILD FLAGS #### @@ -431,12 +442,52 @@ UPLOAD_DO_NOT_DELETE?=false KUSTOMIZE_VERSION=4.5.7 KUSTOMIZE_TARGET=$(OUTPUT_DIR)/kustomize GIT_DEPS_DIR?=$(OUTPUT_DIR)/gitdependencies -SPECIAL_TARGET_SECONDARY=$(strip $(PROJECT_DEPENDENCIES_TARGETS) $(GO_MOD_DOWNLOAD_TARGETS)) +SPECIAL_TARGET_SECONDARY+=$(strip $(PROJECT_DEPENDENCIES_TARGETS) $(GO_MOD_DOWNLOAD_TARGETS)) SKIP_CHECKSUM_VALIDATION?=false -IN_DOCKER_TARGETS=all-attributions all-attributions-checksums all-checksums attribution attribution-checksums binaries checksums clean clean-go-cache +IN_DOCKER_TARGETS=all-attributions all-attributions-checksums all-checksums attribution attribution-checksums binaries checksums clean clean-go-cache validate-checksums $(GO_MOD_DOWNLOAD_TARGETS) $(BINARY_TARGETS) $(GATHER_LICENSES_TARGETS) PRUNE_BUILDCTL?=false GITHUB_TOKEN?= + +# if this is set we are running in the context of a run-<>-in-docker target +DOCKER_RUN_BASE_DIRECTORY?= DEPENDENCY_TOOLS=buildctl helm jq lz4 skopeo tuftool yq + +# set to true if do not care about checksum match, default to false to limit confusion +# around checksum mismatch +FORCE_BUILD_ON_HOST?=false + +# $1 - target(s) +# $2 - should run on host +MAYBE_RUN_IN_DOCKER?=$(if \ + $(or \ + $(filter true,$(IS_ON_BUILDER_BASE)), \ + $(filter true,$(FORCE_BUILD_ON_HOST)), \ + $(filter true,$2), \ + $(filter false,$(EVAL_DOCKER_AVAILABLE)) \ + ),,$(foreach target,$1,run-in-docker/$(target))) + +# this can be used as a normal macro, $(ENABLE_DOCKER), or as a func with 1 param, $(call ENABLE_DOCKER) +# $1 - should run on host (optional) +ENABLE_DOCKER=$(ENABLE_LOGGING)$(eval $(call ENABLE_DOCKER_BODY,$@,$(if $(filter undefined,$(origin 1)),false,$(value 1)),))$(DOCKER_TARGET) +# $1 - container platform to use +ENABLE_DOCKER_PLATFORM=$(ENABLE_LOGGING)$(eval $(call ENABLE_DOCKER_BODY,$@,false,$(if $(filter undefined,$(origin 1)),,$(value 1))))$(DOCKER_TARGET) + +# $1 - target name +# $2 - should run on host +# SHELL is overriden to the cli `true` so that when the actual target ends up running on the host it is an noop +# since we are manipulating the SHELL here we have to make sure any variables which are going to be used in the +# context of MAYBE_RUN_IN_DOCKER cannot themselves call out to a shell, we force eval to avoid +define ENABLE_DOCKER_BODY +$(eval _TARGET:=$1) +$(eval _RUN_ON_HOST:=$2) +$(eval _DOCKER_PLATFORM:=$3) +$(_TARGET): EVAL_DOCKER_AVAILABLE:=$$(DOCKER_AVAILABLE) +$(_TARGET): EVAL_RUN_ON_HOST:=$(_RUN_ON_HOST) +$(_TARGET): DOCKER_PLATFORM:=$(_DOCKER_PLATFORM) +$(_TARGET): DOCKER_TARGET=$$(call MAYBE_RUN_IN_DOCKER,$$@,$$(EVAL_RUN_ON_HOST)) +$(_TARGET): SHELL=$$(if $$(DOCKER_TARGET),$$(NOOP_SHELL),$$(LOGGING_SHELL)) +endef + #################################################### #################### LOGGING ####################### @@ -453,7 +504,7 @@ ENABLE_LOGGING=$(eval $(call ENABLE_LOGGING_BODY,$@)) define ENABLE_LOGGING_BODY $(eval _TARGET:=$1) $(_TARGET): export LOGGING_TARGET=$(_TARGET) -$(_TARGET): SHELL=$(BUILD_LIB)/make_shell.sh log +$(_TARGET): SHELL=$$(LOGGING_SHELL) endef #################################################### @@ -465,7 +516,7 @@ RELEASE_TARGETS?=validate-checksums $(if $(IMAGE_NAMES),images,) $(if $(filter t # convert commonly used, usually shell call, variables to lazily resolved cached variables CACHE_VARS=AWS_ACCOUNT_ID BUILD_IDENTIFIER BUILDER_PLATFORM_ARCH BUILDER_PLATFORM_OS DATE_CMD DATE_NANO \ GIT_HASH GIT_TAG GO_BUILD_CACHE GO_MOD_CACHE GOLANG_VERSION HELM_GIT_TAG IS_ON_BUILDER_BASE \ - PROJECT_DEPENDENCIES_TARGETS SUPPORTED_K8S_VERSIONS BUILDCTL_AVAILABLE BUILDX_AVAILABLE + PROJECT_DEPENDENCIES_TARGETS SUPPORTED_K8S_VERSIONS BUILDCTL_AVAILABLE BUILDX_AVAILABLE DOCKER_AVAILABLE $(foreach v,$(strip $(CACHE_VARS)),$(call CACHE_VARIABLE,$(v))) define BUILDCTL @@ -486,23 +537,6 @@ define BUILDCTL --output type=$(IMAGE_OUTPUT_TYPE),oci-mediatypes=true,\"name=$(IMAGE),$(LATEST_IMAGE)\",$(IMAGE_OUTPUT) endef -# This will occansionally stall out in codebuild for an unknown reason -# retry after a configurable timeout -define CGO_DOCKER - $(BUILD_LIB)/run_target_docker.sh $(COMPONENT) $(CGO_TARGET) $(IMAGE_REPO) "$(RELEASE_BRANCH)" "$(ARTIFACTS_BUCKET)" "$(BASE_DIRECTORY)" "$(GO_MOD_CACHE)" true "$(IMAGE_PLATFORMS)" -endef - -define SIMPLE_CREATE_BINARIES_SHELL - @$(BASE_DIRECTORY)/build/lib/simple_create_binaries.sh $(MAKE_ROOT) $(MAKE_ROOT)/$(OUTPUT_PATH) $(REPO) $(GOLANG_VERSION) $(PLATFORM) "$(SOURCE_PATTERN)" \ - "$(GOBUILD_COMMAND)" "$(EXTRA_GOBUILD_FLAGS)" "$(GO_LDFLAGS)" $(CGO_ENABLED) "$(CGO_LDFLAGS)" "$(GO_MOD_PATH)" "$(BINARY_TARGET_FILES_BUILD_TOGETHER)" -endef - -# $1 - make target -# $2 - target directory -define CGO_CREATE_BINARIES_SHELL - @$(MAKE) binary-builder/cgo/$(PLATFORM:linux/%=%) IMAGE_OUTPUT=dest=$(OUTPUT_BIN_DIR)/$(2) CGO_TARGET=$(1) IMAGE_BUILD_ARGS="GOPROXY COMPONENT CGO_TARGET" -endef - define WRITE_LOCAL_IMAGE_TAG echo $(IMAGE_TAG) > $(IMAGE_OUTPUT_DIR)/$(IMAGE_OUTPUT_NAME).docker_tag echo $(IMAGE) > $(IMAGE_OUTPUT_DIR)/$(IMAGE_OUTPUT_NAME).docker_image_name @@ -549,11 +583,9 @@ endif ## GO mod download targets $(REPO)/%ks-anywhere-go-mod-download: REPO_SUBPATH=$(if $(filter e,$*),,$(*:%/e=%)) -$(REPO)/%ks-anywhere-go-mod-download: $(if $(PATCHES_DIR),$(GIT_PATCH_TARGET),$(GIT_CHECKOUT_TARGET)) - @echo -e $(call TARGET_START_LOG) - if [[ "$(GO_MODS_VENDORED)" == "false" ]]; then $(BASE_DIRECTORY)/build/lib/go_mod_download.sh $(MAKE_ROOT) $(REPO) $(GIT_TAG) $(GOLANG_VERSION) "$(REPO_SUBPATH)"; fi - @touch $@ - @echo -e $(call TARGET_END_LOG) +$(REPO)/%ks-anywhere-go-mod-download: $(if $(PATCHES_DIR),$(GIT_PATCH_TARGET),$(GIT_CHECKOUT_TARGET)) | $$(ENABLE_DOCKER) + @if [[ "$(GO_MODS_VENDORED)" == "false" ]]; then $(BASE_DIRECTORY)/build/lib/go_mod_download.sh $(MAKE_ROOT) $(REPO) $(GIT_TAG) $(GOLANG_VERSION) "$(REPO_SUBPATH)"; fi; \ + touch $@ ifneq ($(REPO),$(HELM_SOURCE_REPOSITORY)) $(HELM_SOURCE_REPOSITORY): @@ -588,8 +620,11 @@ $(OUTPUT_BIN_DIR)/%: BINARY_TARGET=$(@F:%.exe=%) $(OUTPUT_BIN_DIR)/%: SOURCE_PATTERN=$(if $(filter $(BINARY_TARGET),$(BINARY_TARGET_FILES_BUILD_TOGETHER)),$(SOURCE_PATTERNS_BUILD_TOGETHER),$(word $(call pos,$(BINARY_TARGET),$(BINARY_TARGET_FILES)),$(SOURCE_PATTERNS))) $(OUTPUT_BIN_DIR)/%: OUTPUT_PATH=$(if $(and $(if $(filter false,$(call IS_ONE_WORD,$(BINARY_TARGET_FILES_BUILD_TOGETHER))),$(filter $(BINARY_TARGET),$(BINARY_TARGET_FILES_BUILD_TOGETHER)))),$(@D)/,$@) $(OUTPUT_BIN_DIR)/%: GO_MOD_PATH=$($(call GO_MOD_TARGET_FOR_BINARY_VAR_NAME,$(BINARY_TARGET))) -$(OUTPUT_BIN_DIR)/%: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) | $$(ENABLE_LOGGING) - $(if $(NEEDS_CGO_BUILDER),$(call CGO_CREATE_BINARIES_SHELL,$@,$(*D)),$(call SIMPLE_CREATE_BINARIES_SHELL)) +$(OUTPUT_BIN_DIR)/%: GO_MOD_DOWNLOAD_TARGET=$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$(GO_MOD_PATH)) +# if cgo build, set platform when running the docker container +$(OUTPUT_BIN_DIR)/%: SET_PLATFORM=$(if $(filter true,$(NEEDS_CGO_BUILDER)),$(PLATFORM),) +$(OUTPUT_BIN_DIR)/%: $$(GO_MOD_DOWNLOAD_TARGET) | $$(call ENABLE_DOCKER_PLATFORM,$$(SET_PLATFORM)) + @$(BASE_DIRECTORY)/build/lib/simple_create_binaries.sh $(MAKE_ROOT) $(MAKE_ROOT)/$(OUTPUT_PATH) $(REPO) $(GOLANG_VERSION) $(PLATFORM) "$(SOURCE_PATTERN)" "$(GOBUILD_COMMAND)" "$(EXTRA_GOBUILD_FLAGS)" "$(GO_LDFLAGS)" $(CGO_ENABLED) "$(CGO_LDFLAGS)" "$(GO_MOD_PATH)" "$(BINARY_TARGET_FILES_BUILD_TOGETHER)" endif .PHONY: binaries @@ -633,7 +668,8 @@ $(OUTPUT_DIR)/%TTRIBUTION.txt: $(OUTPUT_DIR)/%ttribution/go-license.csv: BINARY_TARGET=$(if $(filter .,$(*D)),,$(*D)) $(OUTPUT_DIR)/%ttribution/go-license.csv: GO_MOD_PATH=$(if $(BINARY_TARGET),$(GO_MOD_TARGET_FOR_BINARY_$(call TO_UPPER,$(BINARY_TARGET))),$(word 1,$(UNIQ_GO_MOD_PATHS))) $(OUTPUT_DIR)/%ttribution/go-license.csv: LICENSE_PACKAGE_FILTER=$(GO_MOD_$(subst /,_,$(GO_MOD_PATH))_LICENSE_PACKAGE_FILTER) -$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) | ensure-jq $$(ENABLE_LOGGING) +$(OUTPUT_DIR)/%ttribution/go-license.csv: GO_MOD_DOWNLOAD_TARGET=$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$(GO_MOD_PATH)) +$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(GO_MOD_DOWNLOAD_TARGET) | ensure-jq $$(ENABLE_DOCKER) @$(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION) $(LICENSE_THRESHOLD) $(CGO_ENABLED) .PHONY: gather-licenses @@ -643,7 +679,9 @@ gather-licenses: $(GATHER_LICENSES_TARGETS) # if there is only one go mod path so only one attribution is created, the file will be named ATTRIBUTION.txt and licenses will be stored in _output, `%` will equal `A` # if multiple attributions are being generated, the file will be _ATTRIBUTION.txt and licenses will be stored in _output/, `%` will equal `_A` %TTRIBUTION.txt: LICENSE_OUTPUT_PATH=$(OUTPUT_DIR)$(if $(filter A,$(*F)),,/$(call TO_LOWER,$(*F:%_A=%))) -%TTRIBUTION.txt: $$(LICENSE_OUTPUT_PATH)/attribution/go-license.csv | $$(ENABLE_LOGGING) +%TTRIBUTION.txt: GENERATE_ATTR_AVAIL=$(shell command -v generate-attribution &> /dev/null && echo "true" || echo "false") +%TTRIBUTION.txt: LICENSE_TARGET=$(LICENSE_OUTPUT_PATH)/attribution/go-license.csv +%TTRIBUTION.txt: $$(LICENSE_TARGET) | $$(call ENABLE_DOCKER,$$(GENERATE_ATTR_AVAIL)) @$(BASE_DIRECTORY)/build/lib/create_attribution.sh $(MAKE_ROOT) $(GOLANG_VERSION) $(MAKE_ROOT)/$(LICENSE_OUTPUT_PATH) $(@F) $(RELEASE_BRANCH) .PHONY: attribution @@ -727,8 +765,8 @@ clean-job-caches: $(and $(findstring presubmit,$(JOB_TYPE)),$(filter true,$(PRUN %/images/push %/images/amd64 %/images/arm64: IMAGE_CONTEXT_DIR?=. %/images/push %/images/amd64 %/images/arm64: IMAGE_BUILD_ARGS?= # if there is neither buildctl or buildx, use ensure-buildctl to show the user that error -%/images/push %/images/amd64 %/images/arm64 %/cgo/amd64 %/cgo/arm64 %-useradd/images/export: ENSURE_PREREQ=$(if $(or $(filter true,$(BUILDCTL_AVAILABLE)),$(filter false,$(BUILDX_AVAILABLE))),ensure-buildkitd-host,) -%/images/push %/images/amd64 %/images/arm64 %/cgo/amd64 %/cgo/arm64 %-useradd/images/export: export USE_BUILDX=$(if $(filter ensure-buildkitd-host,$(ENSURE_PREREQ)),false,true) +%/images/push %/images/amd64 %/images/arm64 %-useradd/images/export: ENSURE_PREREQ=$(if $(or $(filter true,$(BUILDCTL_AVAILABLE)),$(filter false,$(BUILDX_AVAILABLE))),ensure-buildkitd-host,) +%/images/push %/images/amd64 %/images/arm64 %-useradd/images/export: export USE_BUILDX=$(if $(filter ensure-buildkitd-host,$(ENSURE_PREREQ)),false,true) # Build image using buildkit for all platforms, by default pushes to registry defined in IMAGE_REPO. %/images/push: IMAGE_PLATFORMS?=linux/amd64,linux/arm64 @@ -761,38 +799,15 @@ clean-job-caches: $(and $(findstring presubmit,$(JOB_TYPE)),$(filter true,$(PRUN $(WRITE_LOCAL_IMAGE_TAG) @echo -e $(call TARGET_END_LOG) -## CGO Targets -.PHONY: %/cgo/amd64 %/cgo/arm64 - -%/cgo/amd64 %/cgo/arm64: IMAGE_OUTPUT_TYPE?=local -%/cgo/amd64 %/cgo/arm64: DOCKERFILE_FOLDER?=$(BUILD_LIB)/docker/linux/cgo -%/cgo/amd64 %/cgo/arm64: IMAGE_NAME=binary-builder -%/cgo/amd64 %/cgo/arm64: IMAGE_BUILD_ARGS?=GOPROXY COMPONENT -%/cgo/amd64 %/cgo/arm64: IMAGE_CONTEXT_DIR?=. -%/cgo/amd64 %/cgo/arm64: BUILDER_IMAGE=$(CURRENT_BUILDER_BASE_IMAGE) - -%/cgo/amd64: IMAGE_PLATFORMS=linux/amd64 -%/cgo/arm64: IMAGE_PLATFORMS=linux/arm64 - -%/cgo/amd64: | ensure-jq - $(if $(filter true, $(USE_DOCKER_FOR_CGO_BUILD)),$(CGO_DOCKER),$(BUILDCTL)) - -%/cgo/arm64: | ensure-jq - $(if $(filter true, $(USE_DOCKER_FOR_CGO_BUILD)),$(CGO_DOCKER),$(BUILDCTL)) - -# As an attempt to see if using docker is more stable for cgo builds in Codebuild -binary-builder/cgo/%: USE_DOCKER_FOR_CGO_BUILD=$(shell command -v docker &> /dev/null && docker info > /dev/null 2>&1 && echo "true") - ## Useradd targets %-useradd/images/export: IMAGE_OUTPUT_TYPE=local %-useradd/images/export: IMAGE_OUTPUT_DIR=$(OUTPUT_DIR)/files/$* %-useradd/images/export: IMAGE_OUTPUT?=dest=$(IMAGE_OUTPUT_DIR) %-useradd/images/export: IMAGE_BUILD_ARGS=IMAGE_USERADD_USER_ID IMAGE_USERADD_USER_NAME %-useradd/images/export: DOCKERFILE_FOLDER=$(BUILD_LIB)/docker/linux/useradd -%-useradd/images/export: IMAGE_PLATFORMS=linux/amd64 -%-useradd/images/export: | $$(ENSURE_PREREQ) - @mkdir -p $(IMAGE_OUTPUT_DIR) - $(BUILDCTL) +%-useradd/images/export: IMAGE_PLATFORMS=linux/$(BUILDER_PLATFORM_ARCH) +%-useradd/images/export: | $$(ENSURE_PREREQ) $$(ENABLE_LOGGING) + @mkdir -p $(IMAGE_OUTPUT_DIR) && $(BUILDCTL) ## Helm Targets .PHONY: helm/pull @@ -821,11 +836,10 @@ helm/push: helm/build | ensure-helm $$(ENABLE_LOGGING) handle-dependencies: $(call PROJECT_DEPENDENCIES_TARGETS) $(BINARY_DEPS_DIR)/linux-%: | $$(ENABLE_LOGGING) - $(BUILD_LIB)/fetch_binaries.sh $(BINARY_DEPS_DIR) $* $(ARTIFACTS_BUCKET) $(LATEST) $(RELEASE_BRANCH) + @$(BUILD_LIB)/fetch_binaries.sh $(BINARY_DEPS_DIR) $* $(ARTIFACTS_BUCKET) $(LATEST) $(RELEASE_BRANCH) ## Build Targets .PHONY: build - build: $(BUILD_TARGETS) .PHONY: release @@ -865,9 +879,14 @@ clean-go-cache: clean-repo: @rm -rf $(REPO) $(HELM_SOURCE_REPOSITORY) +# intentionally not using OUTPUT_DIR variable to ensure we +# delete the entire output folder for release branch'd projects .PHONY: clean-output clean-output: - $(if $(wildcard _output),du -hs _output && rm -rf _output,) + @if [ -d _output ]; then \ + du -hs _output; \ + rm -rf _output; \ + fi .PHONY: clean clean: $(if $(filter true,$(REPO_NO_CLONE)),,clean-repo) clean-output @@ -897,13 +916,21 @@ add-generated-help-block: ## -------------------------------------- #@ Update Helpers -.PHONY: run-target-in-docker -run-target-in-docker: | ensure-docker # Run `MAKE_TARGET` using builder base docker container - $(BUILD_LIB)/run_target_docker.sh $(COMPONENT) $(MAKE_TARGET) $(IMAGE_REPO) "$(RELEASE_BRANCH)" "$(ARTIFACTS_BUCKET)" "$(BASE_DIRECTORY)" "$(GO_MOD_CACHE)" +.PHONY: start-docker-builder +start-docker-builder: # Start long lived builder base docker container + $(BUILD_LIB)/run_target_docker.sh $(COMPONENT) var-value-PULL_BASE_REF $(IMAGE_REPO) "$(RELEASE_BRANCH)" "$(ARTIFACTS_BUCKET)" "$(BASE_DIRECTORY)" "$(GO_MOD_CACHE)" "$(BUILDER_PLATFORM_ARCH)" false "$(CURRENT_BUILDER_BASE_TAG)" .PHONY: stop-docker-builder stop-docker-builder: # Clean up builder base docker container - docker rm -f -v eks-a-builder + $(MAKE) -C $(BASE_DIRECTORY) stop-docker-builder + +.PHONY: run-buildkit-and-registry +run-buildkit-and-registry: # Run buildkitd and a local docker registry as containers + $(MAKE) -C $(BASE_DIRECTORY) run-buildkit-and-registry + +.PHONY: stop-buildkit-and-registry +stop-buildkit-and-registry: # Stop the buildkitd and a local docker registry containers + $(MAKE) -C $(BASE_DIRECTORY) stop-buildkit-and-registry .PHONY: generate generate: | ensure-locale # Update UPSTREAM_PROJECTS.yaml @@ -930,8 +957,8 @@ patch-for-dep-update: checkout-repo .PHONY: %/create-ecr-repo %/create-ecr-repo: IMAGE_NAME=$* -%/create-ecr-repo: - cmd=( ecr ); \ +%/create-ecr-repo: | $$(ENABLE_LOGGING) + @cmd=( ecr ); \ if [[ "${IMAGE_REPO}" =~ ^public\.ecr\.aws/ ]]; then \ cmd=( ecr-public --region us-east-1 ); \ fi; \ @@ -1049,14 +1076,21 @@ ensure-bash-version: ## -------------------------------------- ## Docker Helpers ## -------------------------------------- -# $1 - target -define RUN_IN_DOCKER_TARGET -.PHONY: run-$(1)-in-docker -run-$(1)-in-docker: MAKE_TARGET=$(1) -run-$(1)-in-docker: run-target-in-docker -endef - -$(foreach target,$(IN_DOCKER_TARGETS),$(eval $(call RUN_IN_DOCKER_TARGET,$(target)))) +# since these targets will likely be file paths it has to be run-in-docker/ vs run-in-docker- otherwise make +# will not properly match the stem. this requires a change to how we used to name these targets +.PHONY: run-in-docker/% +run-in-docker/%: MAKE_TARGET=$* +# on AL2/AL23 the $$(ENABLE_LOGGING) approach does not appear to work, but it does on Mac +# on Mac the explicit vars here do not work. doing both seems to work in both cases +run-in-docker/%: export LOGGING_TARGET=$@ +run-in-docker/%: SHELL=$(LOGGING_SHELL) +run-in-docker/%: | ensure-docker $$(ENABLE_LOGGING) + @$(BUILD_LIB)/run_target_docker.sh $(COMPONENT) $(MAKE_TARGET) $(IMAGE_REPO) "$(RELEASE_BRANCH)" "$(ARTIFACTS_BUCKET)" "$(BASE_DIRECTORY)" "$(GO_MOD_CACHE)" "$(BUILDER_PLATFORM_ARCH)" true "$(CURRENT_BUILDER_BASE_TAG)" "$(DOCKER_PLATFORM)" + +# backcompat old style run-<>-in-docker style targets which work for anything that does not have a / in the target +.PHONY: run-%-in-docker +run-%-in-docker: run-in-docker/% + @echo "Please switch to 'run-in-docker/$*' style targets" # make sure by default all targets use the # if we do not have this as a catch all target then the first target diff --git a/Makefile b/Makefile index 46506c63fc..6230fd60d9 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,12 @@ -MAKEFLAGS+=--no-builtin-rules --warn-undefined-variables +MAKEFLAGS+=--no-builtin-rules --warn-undefined-variables --no-print-directory .SUFFIXES: BASE_DIRECTORY:=$(abspath .) BUILD_LIB=${BASE_DIRECTORY}/build/lib SHELL_TRACE?=false -SHELL:=$(if $(filter true,$(SHELL_TRACE)),$(BUILD_LIB)/make_shell_trace.sh,bash) -.SHELLFLAGS:=$(if $(filter true,$(SHELL_TRACE)),-c,-eu -o pipefail -c) +DEFAULT_SHELL:=$(if $(filter true,$(SHELL_TRACE)),$(BUILD_LIB)/make_shell.sh trace,bash) +SHELL:=$(DEFAULT_SHELL) +.SHELLFLAGS:=-eu -o pipefail -c AWS_ACCOUNT_ID?=$(shell aws sts get-caller-identity --query Account --output text) AWS_REGION?=us-west-2 @@ -13,31 +14,126 @@ IMAGE_REPO?=$(if $(AWS_ACCOUNT_ID),$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazo ECR_PUBLIC_URI?=$(shell aws ecr-public describe-registries --region us-east-1 --query 'registries[0].registryUri' --output text) JOB_TYPE?= -RELEASE_BRANCH?= +RELEASE_BRANCH?=$(LATEST_EKSD_RELEASE) GIT_HASH=$(shell git -C $(BASE_DIRECTORY) rev-parse HEAD) ALL_PROJECTS=$(shell $(BUILD_LIB)/all_projects.sh $(BASE_DIRECTORY)) +LATEST_EKSD_RELEASE=$(shell source $(BUILD_LIB)/common.sh && build::eksd_releases::get_release_branch) + # $1 - project name using _ as separator, ex: rancher_local-path-provisoner PROJECT_PATH_MAP=projects/$(patsubst $(firstword $(subst _, ,$(1)))_%,$(firstword $(subst _, ,$(1)))/%,$(1)) +BUILDER_PLATFORM_ARCH=$(if $(filter x86_64,$(shell uname -m)),amd64,arm64) + # $1 - variable name to resolve and cache CACHE_RESULT = $(if $(filter undefined,$(origin _cached-$1)),$(eval _cached-$1 := 1)$(eval _cache-$1 := $($1)),)$(_cache-$1) # $1 - variable name CACHE_VARIABLE=$(eval _old-$(1)=$(value $(1)))$(eval $(1)=$$(call CACHE_RESULT,_old-$(1))) -CACHE_VARS=ALL_PROJECTS AWS_ACCOUNT_ID GIT_HASH +CACHE_VARS=ALL_PROJECTS AWS_ACCOUNT_ID BUILDER_PLATFORM_ARCH GIT_HASH LATEST_EKSD_RELEASE $(foreach v,$(CACHE_VARS),$(call CACHE_VARIABLE,$(v))) .PHONY: clean-project-% +clean-project-%: PROJECT_PATH=$(call PROJECT_PATH_MAP,$*) +clean-project-%: export RELEASE_BRANCH=$(LATEST_EKSD_RELEASE) clean-project-%: - $(eval PROJECT_PATH=$(call PROJECT_PATH_MAP,$*)) $(MAKE) clean -C $(PROJECT_PATH) .PHONY: clean clean: $(addprefix clean-project-, $(ALL_PROJECTS)) rm -rf _output + +############################## BUILD ALL ################################### + +.PHONY: build-all +build-all: UPLOAD_ARTIFACTS_TO_S3?=false +build-all: build-all-warning +# Build projects with dependecies first to try and validate if there are any missing + @set -eu -o pipefail; \ + export RELEASE_BRANCH=$(RELEASE_BRANCH); \ + PROJECTS="$(foreach project,$(ALL_PROJECTS),$(call PROJECT_PATH_MAP,$(project)))"; \ + PROJS=($${PROJECTS// / }); \ + for proj in "$${PROJS[@]}"; do \ + if [ -n "$$($(MAKE) -C $$proj var-value-PROJECT_DEPENDENCIES)" ] && [ ! -f $$proj/eks-anywhere-full-build-complete ]; then \ + $(MAKE) $$proj/eks-anywhere-full-build-complete; \ + fi; \ + done; \ + for proj in "$${PROJS[@]}"; do \ + if [ ! -f $$proj/eks-anywhere-full-build-complete ]; then \ + $(MAKE) $$proj/eks-anywhere-full-build-complete; \ + fi; \ + done + +# Specific overrides +projects/kubernetes-sigs/kind/eks-anywhere-full-build-complete: override IMAGE_PLATFORMS=linux/amd64,linux/arm64 + +# tinkerbell/hook needs to be built with a public ecr repo so docker container can pull +projects/tinkerbell/hook/eks-anywhere-full-build-complete: IMAGE_REPO=$(ECR_PUBLIC_URI) +projects/tinkerbell/hook/eks-anywhere-full-build-complete: override IMAGE_PLATFORMS=linux/amd64,linux/arm64 + +projects/kubernetes-sigs/image-builder/eks-anywhere-full-build-complete: MAIN_TARGET=build +projects/kubernetes-sigs/image-builder/eks-anywhere-full-build-complete: export SKIP_METAL_INSTANCE_TEST=true + +projects/aws/eks-a-admin-image/eks-anywhere-full-build-complete: MAIN_TARGET=build + +projects/torvalds/linux/eks-anywhere-full-build-complete: export BINARY_PLATFORMS=linux/amd64 linux/arm64 +# Skips +projects/aws/cluster-api-provider-aws-snow/eks-anywhere-full-build-complete: + @echo "Skipping aws/cluster-api-provider-aws-snow: container images are pulled cross account" + @touch $@ + +projects/goharbor/harbor/eks-anywhere-full-build-complete: + @echo "Skipping /goharbor/harbor: we patch vendor directory so we skip go mod download, which can cause slight checksum differences" + @echo "run the 'clean-go-cache' target before running harbor if you want to build with matching checksums" + @touch $@ + +# Actual target +%/eks-anywhere-full-build-complete: IMAGE_PLATFORMS=linux/$(BUILDER_PLATFORM_ARCH) +# override this on the command line to true if you want to push to your own s3 bucket +%/eks-anywhere-full-build-complete: MAIN_TARGET=release +%/eks-anywhere-full-build-complete: + @set -eu -o pipefail; \ + export RELEASE_BRANCH=$(RELEASE_BRANCH); \ + if [ -n "$$($(MAKE) -C $(@D) var-value-PROJECT_DEPENDENCIES)" ]; then \ + PROJECT_DEPS=$$($(MAKE) -C $(@D) var-value-PROJECT_DEPENDENCIES); \ + DEPS=($${PROJECT_DEPS// / }); \ + for dep in "$${DEPS[@]}"; do \ + if [[ "$${dep}" = *"eksa"* ]]; then \ + OVERRIDES="IMAGE_REPO=$(IMAGE_REPO) IMAGE_PLATFORMS=$(IMAGE_PLATFORMS)"; \ + DEP_RELEASE_BRANCH="$$(cut -d/ -f4 <<< $$dep)"; \ + if [ -n "$${DEP_RELEASE_BRANCH}" ]; then \ + dep="$$(dirname $$dep)"; \ + OVERRIDES+=" RELEASE_BRANCH=$$DEP_RELEASE_BRANCH"; \ + fi; \ + echo "Running make $${dep#eksa/} as dependency for $(@D)"; \ + $(MAKE) projects/$${dep#"eksa/"}/eks-anywhere-full-build-complete $$OVERRIDES; \ + if [ -n "$${DEP_RELEASE_BRANCH}" ]; then \ + rm projects/$${dep#"eksa/"}/eks-anywhere-full-build-complete; \ + fi; \ + fi; \ + done; \ + fi; \ + TARGETS="attribution $(MAIN_TARGET)"; \ + if [ -n "$$($(MAKE) -C $(@D) var-value-IMAGE_NAMES)" ] || [ "$$($(MAKE) -C $(@D) var-value-HAS_HELM_CHART)" = "true" ]; then \ + TARGETS="create-ecr-repos $${TARGETS}"; \ + fi; \ + if [ "$(UPLOAD_ARTIFACTS_TO_S3)" = "true" ]; then \ + TARGETS+=" UPLOAD_DRY_RUN=false UPLOAD_CREATE_PUBLIC_ACL=false"; \ + fi; \ + TARGETS+=" IMAGE_REPO=$(IMAGE_REPO) IMAGE_PLATFORMS=$(IMAGE_PLATFORMS) RELEASE_BRANCH=$(RELEASE_BRANCH)"; \ + echo "Running 'make -C $(@D) $${TARGETS}'"; \ + make -C $(@D) $${TARGETS}; \ + touch $@ + +.PHONY: build-all-warning +build-all-warning: + @echo "*** Warning: this target is not meant to used except for specific testing situations ***" + @echo "*** this will likely fail and either way run for a really long time ***" + +######################################################################### + .PHONY: add-generated-help-block-project-% add-generated-help-block-project-%: $(eval PROJECT_PATH=$(call PROJECT_PATH_MAP,$*)) @@ -70,14 +166,18 @@ update-checksum-files: $(addprefix checksum-files-project-, $(ALL_PROJECTS)) update-attribution-files: add-generated-help-block attribution-files build/update-attribution-files/create_pr.sh +.PHONY: start-docker-builder +start-docker-builder: # Start long lived builder base docker container + @$(MAKE) -C projects/aws/eks-anywhere-build-tooling start-docker-builder + .PHONY: stop-docker-builder stop-docker-builder: docker rm -f -v eks-a-builder .PHONY: run-buildkit-and-registry run-buildkit-and-registry: - docker run -d --name buildkitd --net host --privileged moby/buildkit:v0.10.6-rootless - docker run -d --name registry --net host registry:2 + docker run --rm -d --name buildkitd --net host --privileged moby/buildkit:v0.12.2-rootless + docker run --rm -d --name registry --net host registry:2 .PHONY: stop-buildkit-and-registry stop-buildkit-and-registry: diff --git a/build/lib/buildkit.sh b/build/lib/buildkit.sh index 71ea50542d..2db43f9152 100755 --- a/build/lib/buildkit.sh +++ b/build/lib/buildkit.sh @@ -23,7 +23,11 @@ if [ "${USE_BUILDX:-}" == "true" ]; then printf "\nBuilding with docker buildx\n" >&2 CMD="docker buildx" - ARGS="" + # for the hook build pushing the provenance manifest borks linuxkit + # this is not pushed by buildctl which is what we use in prod + ARGS="build --provenance=false " + # shift build off the args list + shift while test $# -gt 0; do case "$1" in --frontend) diff --git a/build/lib/common.sh b/build/lib/common.sh index 769cbf7fc7..21bc82d45d 100755 --- a/build/lib/common.sh +++ b/build/lib/common.sh @@ -125,7 +125,7 @@ function build::common::upload_artifacts() { } function build::gather_licenses() { - local -r outputdir=$1 + local -r outputdir="${1%/}" # remove trailing slash if it exists local -r patterns=$2 local -r golang_version=$3 local -r threshold=$4 @@ -266,7 +266,11 @@ function build::common::get_go_path() { } function build::common::use_go_version() { - local -r version=$1 + local -r version=${1:-} + + if [ -z "$version" ]; then + return + fi if (( "${version#*.}" < 16 )); then echo "Building with GO version $version is no longer supported! Please update the build to use a newer version." @@ -278,7 +282,8 @@ function build::common::use_go_version() { # Adding to the beginning of PATH to allow for builds on specific version if it exists export PATH=${gobinarypath}:$PATH export GOCACHE=$(go env GOCACHE)/$version - } + echo "$(go version)" +} # Use a seperate build cache for each project/version to ensure there are no # shared bits which can mess up the final checksum calculation diff --git a/build/lib/docker/linux/cgo/Dockerfile b/build/lib/docker/linux/cgo/Dockerfile deleted file mode 100644 index d160a8d1f2..0000000000 --- a/build/lib/docker/linux/cgo/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# syntax=docker/dockerfile:experimental - -# To around the lack of a overlayfs in presubmit jobs since they run on fargate -# we have to keep the layers to a minimum. -# All the source files and librarys are gatered into once folder on the base and then -# copied in the builder -ARG BUILDER_IMAGE -FROM ${BUILDER_IMAGE} as builder - -ARG TARGETARCH -ARG TARGETOS -ARG COMPONENT -ARG CGO_TARGET - -ARG GOPROXY -ENV GOPROXY=$GOPROXY - -COPY ./eks-anywhere-build-tooling /eks-anywhere-build-tooling -COPY ./$TARGETOS-$TARGETARCH /eks-anywhere-build-tooling/projects/$COMPONENT/_output/source/$TARGETOS-$TARGETARCH - -WORKDIR /eks-anywhere-build-tooling -RUN make -C projects/$COMPONENT $CGO_TARGET BINARY_PLATFORMS=$TARGETOS/$TARGETARCH - -FROM scratch - -ARG TARGETARCH -ARG TARGETOS -ARG COMPONENT -ARG CGO_TARGET - -COPY --from=builder /eks-anywhere-build-tooling/projects/$COMPONENT/$CGO_TARGET . diff --git a/build/lib/generate_help_body.sh b/build/lib/generate_help_body.sh index 90a613962e..ff37383e59 100755 --- a/build/lib/generate_help_body.sh +++ b/build/lib/generate_help_body.sh @@ -153,7 +153,7 @@ fi TARGETS=(${IN_DOCKER_TARGETS// / }) DOCKER_TARGETS="${NL}${NL}##@ Run in Docker Targets" for target in "${TARGETS[@]}"; do - DOCKER_TARGETS+="${NL}run-${target}-in-docker: ## Run \`${target}\` in docker builder container" + DOCKER_TARGETS+="${NL}run-in-docker/${target}: ## Run \`${target}\` in docker builder container" done cat >> $HELPFILE << EOF diff --git a/build/lib/make_shell.sh b/build/lib/make_shell.sh index 60a042c22d..6174e466f6 100755 --- a/build/lib/make_shell.sh +++ b/build/lib/make_shell.sh @@ -48,7 +48,7 @@ DATE_NANO=$(if [ "$(uname -s)" = "Linux" ] || [ "$DATE" = "gdate" ]; then echo % START_TIME=$($DATE +%s.$DATE_NANO) -echo -e "\n------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") Starting target=$TARGET -------------------" +echo -e "\n------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") $([ -n "${DOCKER_RUN_BASE_DIRECTORY:-}" ] && echo "(In Docker) ")Starting target=$TARGET -------------------" echo "($(pwd)) \$ $@" eval "$@" -echo -e "------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") Finished target=$TARGET duration=$(echo $($DATE +%s.$DATE_NANO) - $START_TIME | bc) seconds -------------------\n" +echo -e "------------------- $($DATE +"%Y-%m-%dT%H:%M:%S.$DATE_NANO%z") $([ -n "${DOCKER_RUN_BASE_DIRECTORY:-}" ] && echo "(In Docker) ")Finished target=$TARGET duration=$(echo $($DATE +%s.$DATE_NANO) - $START_TIME | bc) seconds -------------------\n" diff --git a/build/lib/prepare_build_container_user.sh b/build/lib/prepare_build_container_user.sh new file mode 100755 index 0000000000..f931c8985a --- /dev/null +++ b/build/lib/prepare_build_container_user.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Copyright 2020 Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +GROUP_ID="$1" +USER_ID="$2" + +# when running the build container on linux, the user in the container needs +# to match the host user id and group id +# otherwise there will be perms issues due to go mods being downloaded in the +# container as root even though the host user is not + +sed -i 's/^CREATE_MAIL_SPOOL=yes/CREATE_MAIL_SPOOL=no/' /etc/default/useradd +groupadd -g 100 users +groupadd --gid "$GROUP_ID" matchinguser +useradd --no-create-home --uid "$USER_ID" --gid "$GROUP_ID" matchinguser +mkdir -p /home/matchinguser +chown -R matchinguser:matchinguser /home/matchinguser diff --git a/build/lib/run_target_docker.sh b/build/lib/run_target_docker.sh index 23ef05c701..83f68ad895 100755 --- a/build/lib/run_target_docker.sh +++ b/build/lib/run_target_docker.sh @@ -24,34 +24,25 @@ RELEASE_BRANCH="${4:-}" ARTIFACTS_BUCKET="${5:-$ARTIFACTS_BUCKET}" BASE_DIRECTORY="${6:-}" GO_MOD_CACHE="${7:-}" -REMOVE="${8:-false}" -PLATFORM="${9:-}" +BUILDER_PLATFORM_ARCH="${8:-amd64}" +REMOVE="${9:-false}" +BUILDER_BASE_TAG="${10:-latest}" +PLATFORM="${11:-}" SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" source "${SCRIPT_ROOT}/common.sh" -# Since we may actually be running docker in docker here for cgo builds -# we need to have the host path base_dir and go_mod_cache and netrc -DOCKER_RUN_BASE_DIRECTORY="${DOCKER_RUN_BASE_DIRECTORY:-$BASE_DIRECTORY}" -DOCKER_RUN_GO_MOD_CACHE="${DOCKER_RUN_GO_MOD_CACHE:-$GO_MOD_CACHE}" - MAKE_VARS="IMAGE_REPO=$IMAGE_REPO ARTIFACTS_BUCKET=$ARTIFACTS_BUCKET" function remove_container() { - docker rm -vf $CONTAINER_ID + docker rm -vf $CONTAINER_ID > /dev/null 2>&1 } SKIP_RUN="false" NAME="" if [[ "$REMOVE" == "false" ]]; then - echo "****************************************************************" - echo "A docker container with the name eks-a-builder will be launched." - echo "It will be left running to support running consecutive runs." - echo "Run 'make stop-docker-builder' when you are done to stop it." - echo "****************************************************************" - NAME="--name eks-a-builder" if docker ps -f name=eks-a-builder | grep -w eks-a-builder; then @@ -62,14 +53,16 @@ else trap "remove_container" EXIT fi -IMAGE="public.ecr.aws/eks-distro-build-tooling/builder-base:latest" -PLATFORM_ARG="" +IMAGE="public.ecr.aws/eks-distro-build-tooling/builder-base:$BUILDER_BASE_TAG" +# since if building cgo we will specifically set the arch to something other than the host +# ensure we always explictly ask for the host platform, unless override for cgo +PLATFORM_ARG="--platform linux/$BUILDER_PLATFORM_ARCH" if [[ -n "$PLATFORM" ]]; then - DIGEST=$(docker buildx imagetools inspect --raw public.ecr.aws/eks-distro-build-tooling/builder-base:latest | jq -r ".manifests[] | select(.platform.architecture == \"${PLATFORM#linux/}\") | .digest") + DIGEST=$(docker buildx imagetools inspect --raw public.ecr.aws/eks-distro-build-tooling/builder-base:$BUILDER_BASE_TAG | jq -r ".manifests[] | select(.platform.architecture == \"${PLATFORM#linux/}\") | .digest") IMAGE="public.ecr.aws/eks-distro-build-tooling/builder-base@$DIGEST" PLATFORM_ARG="--platform $PLATFORM" - MAKE_VARS=" BINARY_PLATFORMS=$PLATFORM" + MAKE_VARS+=" BINARY_PLATFORMS=$PLATFORM" fi DOCKER_USER_FLAG="" @@ -85,7 +78,11 @@ fi if [[ "$SKIP_RUN" == "false" ]]; then - build::docker::retry_pull $IMAGE + echo "Pulling $IMAGE...." + if ! build::docker::retry_pull $IMAGE > /dev/null 2>&1; then + # try one more time to show the error to the user + docker pull $IMAGE + fi NETRC="" if [ -f $HOME/.netrc ]; then @@ -95,24 +92,27 @@ if [[ "$SKIP_RUN" == "false" ]]; then DOCKER_RUN_NETRC="" fi - mkdir -p $DOCKER_RUN_GO_MOD_CACHE + mkdir -p $GO_MOD_CACHE CONTAINER_ID=$(build::common::echo_and_run docker run -d $NAME --privileged $NETRC $PLATFORM_ARG \ - --mount type=bind,source=$DOCKER_RUN_BASE_DIRECTORY,target=/eks-anywhere-build-tooling \ - --mount type=bind,source=$DOCKER_RUN_GO_MOD_CACHE,target=/mod-cache \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e GOPROXY=${GOPROXY:-} -e GOMODCACHE=/mod-cache -e DOCKER_RUN_BASE_DIRECTORY=$DOCKER_RUN_BASE_DIRECTORY -e DOCKER_RUN_GO_MOD_CACHE=$DOCKER_RUN_GO_MOD_CACHE -e DOCKER_RUN_NETRC=$DOCKER_RUN_NETRC \ + --mount type=bind,source=$BASE_DIRECTORY,target=/eks-anywhere-build-tooling \ + --mount type=bind,source=$GO_MOD_CACHE,target=/mod-cache \ + -e GOPROXY=${GOPROXY:-} -e GOMODCACHE=/mod-cache -e DOCKER_RUN_BASE_DIRECTORY=$BASE_DIRECTORY \ --entrypoint sleep $IMAGE infinity) if [ -n "$DOCKER_USER_FLAG" ]; then - docker exec -it $CONTAINER_ID groupadd -g 100 users - docker exec -it $CONTAINER_ID groupadd --gid "$USER_GROUP_ID" matchinguser - docker exec -it $CONTAINER_ID useradd --no-create-home --uid "$USER_ID" --gid "$USER_GROUP_ID" matchinguser - docker exec -it $CONTAINER_ID mkdir -p /home/matchinguser - docker exec -it $CONTAINER_ID chown -R matchinguser:matchinguser /home/matchinguser + build::common::echo_and_run docker exec -t $CONTAINER_ID /eks-anywhere-build-tooling/build/lib/prepare_build_container_user.sh "$USER_GROUP_ID" "$USER_GROUP_ID" + fi + + if [[ "$REMOVE" == "false" ]]; then + echo "****************************************************************" + echo "A docker container with the name eks-a-builder will be launched." + echo "It will be left running to support running consecutive runs." + echo "Run 'make stop-docker-builder' when you are done to stop it." + echo "****************************************************************" fi fi build::common::echo_and_run docker exec -e RELEASE_BRANCH=$RELEASE_BRANCH $DOCKER_USER_FLAG \ - -it $CONTAINER_ID \ - make $TARGET -C /eks-anywhere-build-tooling/projects/$PROJECT $MAKE_VARS + -t $CONTAINER_ID \ + make --no-print-directory $TARGET -C /eks-anywhere-build-tooling/projects/$PROJECT $MAKE_VARS diff --git a/docs/development/building-locally.md b/docs/development/building-locally.md index 249059b528..c52b4959eb 100644 --- a/docs/development/building-locally.md +++ b/docs/development/building-locally.md @@ -1,22 +1,51 @@ # Building locally All projects in this repo follow a common building (and updating) process. Make targets are standardized across the repo. -Projects can be built on a Mac or Linux host, as well as using the builder-base docker image used in prow for building. +Projects can be built on a Mac or Linux host. By default when running `go build`, and some of the other key targerts +such as `gather_licenses` and `attribution`, the builder-base docker image used in prow will be used. +This is to ensure checksums and attributions match what is generated in our CI and release systems. +To force running targets on the host: `export FORCE_GO_BUILD_ON_HOST=true`. -## Pre-requisites for building on host -* Multiple versions of golang are required to build the various projects. There is a helper -[script](../../build/lib/install_go_versions.sh) modeled from the builder-base to install all golang versions locally. -* If intending on generating checksums or attribution files, patch versions of golang must match. Validate versions in above script match those in [builder-base](https://github.com/aws/eks-distro-build-tooling/blob/main/builder-base/install.sh#L172). If not, -please send a PR updating script in this repo. -* Gathering licenses and generating attribution files requires additional setup outlined [here](attribution-files.md). -* For building container images, buildctl is used instead of docker to match our prow builds. Running `local-images` targets -will export images to a tar, but if running `images` targets which push images, a registry is required. By default, -an ECR repo in the currently logged in AWS account will be used. A common alternative is to run docker registry locally and override -this default behavior. This can be done with `IMAGE_REPO=localhost:5000 make images`. To run buildkit and the docker registry run from the repo root: - * `make run-buildkit-and-registry` - * `export BUILDKIT_HOST=docker-container://buildkitd` - * `make stop-buildkit-and-registry` to cleanup +## Pre-requisites for building on MacOS host +* Docker Desktop - https://docs.docker.com/desktop/install/mac-install/ +* Common utils required - `brew install jq yq` +* Bash 4.2+ is required - `breq install bash` +* A number of targets which run on the host require the gnu variants of common tools + such as `date` `find` `sed` `stat` `tar` - `brew install coreutils findutils gnu-sed gnu-tar` +* Building container images requires `buildctl` - `brew buildkit` +* Building helm charts requires `helm` and `skopeo` + * https://helm.sh/docs/intro/install/ - `curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash` + * https://github.com/containers/skopeo - `brew install skopeo` +* To make pushing to ECR easier: `brew install docker-credential-helper-ecr` + +## Pre-requisites for building on Linux (AL2/AL23) host +* Docker + * `sudo yum install -y docker && sudo usermod -a -G docker ec2-user && id ec2-user && newgrp docker` + * `sudo systemctl enable docker.service && sudo systemctl start docker.service` +* Common utils required + * `sudo yum -y install jq git-core make lz4` +* `yq` is required by a number of targets + * `sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_$([ "x86_64" = "$(uname -m)" ] && echo amd64 || echo arm64) && sudo chmod +x /usr/local/bin/yq` +* Building helm charts requires `helm` and `skopeo` + * https://helm.sh/docs/intro/install/ - `curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash` + * https://github.com/containers/skopeo - There is no binary release of skopeo for linux see [installing](https://github.com/containers/skopeo/blob/main/install.md) or + copy from builder-base image to host: `CONTAINER_ID=$(docker run -d -i public.ecr.aws/eks-distro-build-tooling/builder-base:standard-latest.al2 bash) && sudo docker cp $CONTAINER_ID:/usr/bin/skopeo /usr/local/bin && docker rm -vf $CONTAINER_ID` +* Building container images for different architectures will require `qemu` + * `curl -Ls https://github.com/tonistiigi/binfmt/releases/download/deploy%2Fv6.2.0-26/qemu_v6.2.0_linux-amd64.tar.gz | sudo tar xzvf - -C /usr/bin` + * `docker run --privileged --rm public.ecr.aws/eks-distro-build-tooling/binfmt-misc:qemu-v6.1.0 --install aarch64,amd64` +* To ensure string sorting matches our build pipelines - `export LANG=C.UTF-8` +* To make pushing to ECR easier: `sudo yum install amazon-ecr-credential-helper -y` +* (Skip for AL23) Upgrade to aws cli v2: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html +* Building image-builder + * `tuftool` to download Bottlerocket images - https://github.com/awslabs/tough - There is no binary release of tuftool for linux see [Readme](https://github.com/awslabs/tough/blob/develop/tuftool/README.md) or + copy from builder-base image to host: `CONTAINER_ID=$(docker run -d -i public.ecr.aws/eks-distro-build-tooling/builder-base:standard-latest.al2 bash) && sudo docker cp $CONTAINER_ID:/usr/bin/tuftool /usr/local/bin && docker rm -vf $CONTAINER_ID` + * (Skip for AL23) python 3.9 is required for the version of ansible required. AL2 does not ship a python 3.9 package so the easiest way to install it is building it with pyenv: + * https://github.com/pyenv/pyenv + * `curl https://pyenv.run | bash` + * `sudo yum install gcc make patch zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl11-devel tk-devel libffi-devel xz-devel -y` + * `pyenv install 3.9 && pyenv global 3.9` + ## Typical build targets @@ -24,7 +53,7 @@ Each project folder contains a Makefile with the following build targets. * `binaries` - clones repo, checks out git tag and builds binaries. * `local-images` - builds container images and writes to `/tmp/{project}.tar`. -* `images` - builds container images and pushes to configured registry. For a few projects (coredns, etcd, kubernetes) local oci tars are also built +* `images` - builds container images apnd pushes to configured registry. For a few projects (coredns, etcd, kubernetes) local oci tars are also built to be uploaded to s3. * `gather-licenses` - copies licenses from all project dependencies and puts in `_output/LICENSES` to be included in tarballs and container images. * `attribution` - regenerate the ATTRIBUTION.txt file, golang versions are important here so make sure you have the same versions as the builder-base. @@ -37,24 +66,55 @@ Each project folder contains a Makefile with the following build targets. * `release` - run via prow postsubmit, same as `build` except runs `images` to push to a remote registry and to push tars/binaries to s3. -## Running in builder-base -It may be easier to ensure building matches that in prow to use the builder-base locally. There are make target helpers in the root Makefile for this workflow. +## Pre-requisites for building container images + +* There are two options for building container images + * `buildctl` can be used instead of docker to match our prow builds. Running `local-images` targets + will export images to a tar, but if running `images` targets which push images, a registry is required. By default, + an ECR repo in the currently logged in AWS account will be used. A common alternative is to run docker registry locally and override + this default behavior. This can be done with `IMAGE_REPO=localhost:5000 make images`. To run buildkit and the docker registry run from the repo root (or a specific project folder): + * `curl -L https://github.com/moby/buildkit/releases/download/v0.12.2/buildkit-v0.12.2.linux-$([ "x86_64" = "$(uname -m)" ] && echo amd64 || echo arm64).tar.gz -o /tmp/buildkit.tgz && sudo tar -xf /tmp/buildkit.tgz -C /usr/local bin/buildctl` + * `make run-buildkit-and-registry` + * `export BUILDKIT_HOST=docker-container://buildkitd` + * `make stop-buildkit-and-registry` to cleanup + * (Experimental) `docker buildx` can be used which may be easier to setup since docker now ships with the buildx plugin. + docker buildx also uses buildkit behind the scenes so it should result in creating an image which is the same as using `buildctl` directly as prow does. + * You need to create a builder using either the `docker-container` or `remote` driver. For example: `docker buildx create --name multiarch --driver docker-container --use` +* See [docker-auth.md](./docker-auth.md) for helpful tips for configuring, securing, and using your AWS credentials with the targets in this project. +* By default, `IMAGE_REPO` will be set to your AWS account's private ECR registry based on your AWS_REGION. If you want to create the neccessary registeries +for the current project: + * `make create-ecr-repos` + * If you would like to create these repos in your ECR public registry - `IMAGE_REPO=public.ecr.aws/ create-ecr-repos` +* Some projects download built artifacts from our dev pipelines. To pull these from the EKS-Anywere dev bucket + * `export ARTIFACTS_BUCKET=s3://projectbuildpipeline-857-pipelineoutputartifactsb-10ajmk30khe3f` + +## (Not recommended) Running go builds on the host + +* Multiple versions of golang are required to build the various projects. There is a helper +[script](../../build/lib/install_go_versions.sh) modeled from the builder-base to install all golang versions locally. +**Note** Our build and release pipelines use [EKS Go](https://github.com/aws/eks-distro-build-tooling/blob/main/projects/golang/go/README.md) which +the helper script does not install from. If building on the host, checksums will almost certainly mismatch with what is checked into this repo. +* If testing gathering licenses and generating attribution files on the host, please follow these [instructions](attribution-files.md) to setup. + + +## Create a long running in builder-base container + +By default targets are run in a short-lived container using the builder-base image. If you would like to create a long lived container for testing: -* To run a target within a specific project: - * `make run-target-in-docker MAKE_TARGET= PROJECT=` - * any of the above targets can be used for `` - * `` will be in the format `org/repo`, ex: `coredns/coredns` - * A container is launched using the `builder-base:latest` image and the current source tree is copied into it. - If also building locally, you may want to run `make clean` to avoid copying unnecessary files into the container. - * The container will be named `eks-a-builder` and can be accessed to manually run targets or cleanup: - * `docker exec -it eks-a-builder bash` - * Subsequent calls to `make run-target-in-docker` will use the same running container. To stop the container: - * `make stop-docker-builder` -* Attribution and Checksum generation may be easier to run in docker to avoid needing specific golang versions locally. A specific make target exists for these: - * `make -C projects/ run-attribution-checksums-in-docker` - * This will build, generate attribution and checksums and then copy the resulting files out of the container to host project location. +* `make start-docker-builder` +* The container will be named `eks-a-builder` and can be accessed to manually run targets or cleanup: + * `docker exec -it eks-a-builder bash` +* To stop the container: + * `make stop-docker-builder` -## Docker and AWS credentials +## Other considerations -See [docker-auth.md](./docker-auth.md) for helpful tips for configuring, securing, and using your AWS credentials with the targets in this project. +* If you use a private `GOPROXY`, add `export GOPROXY=` to your ~/.bashrc or ~/.zshrc + * If your private proxy requires authentication, consider creating `~/.netrc`, ex: + ``` + machine + login + password + ``` + * `chmod og-rw ~/.netrc` diff --git a/projects/aws/bottlerocket-bootstrap/Makefile b/projects/aws/bottlerocket-bootstrap/Makefile index c9d2dcb234..f65484d9e6 100644 --- a/projects/aws/bottlerocket-bootstrap/Makefile +++ b/projects/aws/bottlerocket-bootstrap/Makefile @@ -89,8 +89,8 @@ clean-extra: clean: clean-extra -unit-test: - go test ./... +unit-test: | $$(ENABLE_DOCKER) + @go test ./... .PHONY: mocks mocks: $(MOCKGEN) diff --git a/projects/aws/eks-a-admin-image/Makefile b/projects/aws/eks-a-admin-image/Makefile index 07f82f79a8..e1f48725b4 100644 --- a/projects/aws/eks-a-admin-image/Makefile +++ b/projects/aws/eks-a-admin-image/Makefile @@ -58,10 +58,9 @@ build-%: KUBECTL_URL?=$(shell ./build/get_kubectl_url.sh) build-%: export PKR_VAR_kubectl-url=$(KUBECTL_URL) build-%: export PKR_VAR_yq-url=$(YQ_URL) -$(PACKER): OS=$(shell uname -s | tr '[:upper:]' '[:lower:]') $(PACKER): mkdir -p $(@D) - curl --silent -L https://releases.hashicorp.com/packer/$(PACKER_VERSION)/packer_$(PACKER_VERSION)_$(OS)_amd64.zip > packer.zip + curl --silent -L https://releases.hashicorp.com/packer/$(PACKER_VERSION)/packer_$(PACKER_VERSION)_$(BUILDER_PLATFORM_OS)_$(BUILDER_PLATFORM_ARCH).zip > packer.zip unzip packer.zip -d $(@D) rm packer.zip diff --git a/projects/emissary-ingress/emissary/Makefile b/projects/emissary-ingress/emissary/Makefile index d349e8b719..8b5cd74b10 100644 --- a/projects/emissary-ingress/emissary/Makefile +++ b/projects/emissary-ingress/emissary/Makefile @@ -26,6 +26,7 @@ HAS_HELM_CHART=true HELM_DIRECTORY=charts/emissary-ingress HELM_DESTINATION_REPOSITORY=emissary-ingress/emissary HELM_IMAGE_LIST=emissary-ingress/emissary +CHART_VERSION?= AMBASSADOR_VERSION_TARGET=$(REPO)/python/ambassador.version @@ -35,8 +36,11 @@ $(GO_MOD_DOWNLOAD_TARGETS): $(AMBASSADOR_VERSION_TARGET) # do not pass our makeflags down to emissary make since there will be unncessary warnings $(AMBASSADOR_VERSION_TARGET): MAKEFLAGS= -$(AMBASSADOR_VERSION_TARGET): - @export VERSION="v3.6.0" && export CHART_VERSION=$(CHART_VERSION) && cd emissary && GOPATH=/var && make python/ambassador.version +$(AMBASSADOR_VERSION_TARGET): export VERSION=v3.6.0 +$(AMBASSADOR_VERSION_TARGET): export CHART_VERSION:=$(CHART_VERSION) +$(AMBASSADOR_VERSION_TARGET): export GOPATH=/var +$(AMBASSADOR_VERSION_TARGET): | $$(ENABLE_DOCKER) + @$(MAKE) -C $(REPO) python/ambassador.version # Build helm chart .PHONY: helm/build diff --git a/projects/kubernetes-sigs/cluster-api-provider-cloudstack/Makefile b/projects/kubernetes-sigs/cluster-api-provider-cloudstack/Makefile index cdeb1f6536..56815d0b79 100644 --- a/projects/kubernetes-sigs/cluster-api-provider-cloudstack/Makefile +++ b/projects/kubernetes-sigs/cluster-api-provider-cloudstack/Makefile @@ -18,16 +18,15 @@ include $(BASE_DIRECTORY)/Common.mk $(REPO)/eks-anywhere-go-mod-download: $(CAPC_MANIFESTS_TARGET) -$(CAPC_MANIFESTS_TARGET): $(GIT_CHECKOUT_TARGET) - echo "Running generate-mocks for capc" - source $(BUILD_LIB)/common.sh && build::common::use_go_version $(GOLANG_VERSION) && make -C $(REPO) generate-mocks +$(CAPC_MANIFESTS_TARGET): $(GIT_CHECKOUT_TARGET) | $$(ENABLE_DOCKER) + @source $(BUILD_LIB)/common.sh && build::common::use_go_version $(GOLANG_VERSION) && $(MAKE) -C $(REPO) generate-mocks s3-artifacts: create-manifests .PHONY: create-manifests create-manifests: export PATH:=$(MAKE_ROOT)/$(OUTPUT_DIR):$(PATH) -create-manifests: $(KUSTOMIZE_TARGET) tarballs - build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION) +create-manifests: $(KUSTOMIZE_TARGET) tarballs | $$(call ENABLE_DOCKER) + @build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION) ########### DO NOT EDIT ############################# diff --git a/projects/kubernetes-sigs/cluster-api-provider-vsphere/Makefile b/projects/kubernetes-sigs/cluster-api-provider-vsphere/Makefile index 12354a1849..6177f7b254 100644 --- a/projects/kubernetes-sigs/cluster-api-provider-vsphere/Makefile +++ b/projects/kubernetes-sigs/cluster-api-provider-vsphere/Makefile @@ -23,8 +23,8 @@ s3-artifacts: create-manifests $(GATHER_LICENSES_TARGETS): | $(FIX_LICENSES_VM_OPERATOR_TARGET) .PHONY: create-manifests -create-manifests: tarballs | ensure-yq - build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION) +create-manifests: tarballs | ensure-yq $$(ENABLE_DOCKER) + @build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION) $(FIX_LICENSES_VM_OPERATOR_TARGET): | $(GO_MOD_DOWNLOAD_TARGETS) # The vmware-tanzu/vm-operator dependency github repo has a license however tanzu-topology and ncp are subfolders diff --git a/projects/kubernetes-sigs/cluster-api/Makefile b/projects/kubernetes-sigs/cluster-api/Makefile index fdfb0a781a..ab58362024 100644 --- a/projects/kubernetes-sigs/cluster-api/Makefile +++ b/projects/kubernetes-sigs/cluster-api/Makefile @@ -43,9 +43,10 @@ $(GATHER_LICENSES_TARGETS): | $(FIX_LICENSES_GO_JSON_TARGET) $(FIX_LICENSES_TEST cluster-api-docker-controller/images/%: BASE_IMAGE_NAME=eks-distro-minimal-base .PHONY: create-manifests -create-manifests: tarballs | ensure-yq - build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION) - +create-manifests: MAKEFLAGS= +create-manifests: tarballs | ensure-yq $$(ENABLE_DOCKER) + @build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION) + $(FIX_LICENSES_GO_JSON_TARGET): | $(GO_MOD_DOWNLOAD_TARGETS) # The ajeddeloh/go-json dependency github repo does not have a license, however it redirects to coreos/go-json repo # that does have its own license. Hence we need to manually download license from coreos/go-json repo diff --git a/projects/kubernetes-sigs/image-builder/Makefile b/projects/kubernetes-sigs/image-builder/Makefile index 81da70758c..59fc43c850 100644 --- a/projects/kubernetes-sigs/image-builder/Makefile +++ b/projects/kubernetes-sigs/image-builder/Makefile @@ -5,6 +5,9 @@ GIT_TAG=$(shell cat GIT_TAG) REPO=image-builder REPO_OWNER=kubernetes-sigs +# no container images for this project +IMAGE_NAMES= + IMAGE_BUILDER_DIR=$(REPO)/images/capi RAW_IMAGE_BUILD_AMI?=ami-095413544ce52437d RAW_IMAGE_BUILD_INSTANCE_TYPE?=t2.micro @@ -436,6 +439,11 @@ local-build-qemu-rhel-8: local-build-cloudstack-redhat-8 local-build-raw-ubuntu-2004-efi: local-build-raw-ubuntu-2004 @echo Done building $@ +.PHONY: clean-extra +clean-extra: + @rm -rf fake-* + +clean: clean-extra ############################################################################ ## -------------------------------------- ## Document dynamic targets diff --git a/projects/kubernetes-sigs/kind/Makefile b/projects/kubernetes-sigs/kind/Makefile index dd61e032d7..174783d879 100644 --- a/projects/kubernetes-sigs/kind/Makefile +++ b/projects/kubernetes-sigs/kind/Makefile @@ -149,10 +149,10 @@ $(KIND_BASE_KUBEADM_OVERRIDE): $(KIND_NODE_BUILD_AMD64_TARGET): KIND_CLI=./$(OUTPUT_BIN_DIR)/$(subst /,-,$(BUILDER_PLATFORM))/kind $(KIND_NODE_BUILD_AMD64_TARGET): $(KIND_NODE_IMAGE_BUILD_ARGS) $(ORGANIZE_BINARIES_AMD64_TARGET) | ensure-jq $(MAKE) $(KIND_CLI) BINARY_PLATFORMS=$(BUILDER_PLATFORM) - build/build-kind-node-image.sh $(RELEASE_BRANCH) $(VERSIONED_BASE_IMAGE) amd64 + build/build-kind-node-image.sh $(RELEASE_BRANCH) $(VERSIONED_BASE_IMAGE) amd64 $(BUILDER_PLATFORM_ARCH) $(KIND_NODE_BUILD_ARM64_TARGET): $(KIND_NODE_IMAGE_BUILD_ARGS) $(ORGANIZE_BINARIES_ARM64_TARGET) | ensure-jq - build/build-kind-node-image.sh $(RELEASE_BRANCH) $(VERSIONED_BASE_IMAGE) arm64 + build/build-kind-node-image.sh $(RELEASE_BRANCH) $(VERSIONED_BASE_IMAGE) arm64 $(BUILDER_PLATFORM_ARCH) $(ORGANIZE_BINARIES_AMD64_TARGET): $(HANDLE_DEPENDENCIES_TARGET) build/organize_binaries.sh $(RELEASE_BRANCH) amd64 diff --git a/projects/kubernetes-sigs/kind/build/build-kind-node-image.sh b/projects/kubernetes-sigs/kind/build/build-kind-node-image.sh index aec3704881..f0c2c290ff 100755 --- a/projects/kubernetes-sigs/kind/build/build-kind-node-image.sh +++ b/projects/kubernetes-sigs/kind/build/build-kind-node-image.sh @@ -19,6 +19,7 @@ set -o pipefail EKSD_RELEASE_BRANCH="${1?Specify first argument - release branch}" INTERMEDIATE_BASE_IMAGE="${2?Specify second argument - kind base tag}" ARCH="${3?Specify third argument - Targetarch}" +BUILDER_PLATFORM_ARCH="${4?Specify fourth argument - Hostarch}" MAKE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" source "${MAKE_ROOT}/../../../build/lib/common.sh" @@ -39,7 +40,7 @@ function build::kind::build_node_image(){ export EKSD_ASSET_URL=$EKSD_ASSET_URL export KUBE_ARCH=$ARCH - KIND_PATH="$MAKE_ROOT/_output/bin/kind/$(uname | tr '[:upper:]' '[:lower:]')-$(go env GOHOSTARCH)/kind" + KIND_PATH="$MAKE_ROOT/_output/bin/kind/$(uname | tr '[:upper:]' '[:lower:]')-$BUILDER_PLATFORM_ARCH/kind" $KIND_PATH build node-image $MAKE_ROOT/images/k8s.io/kubernetes \ --base-image $INTERMEDIATE_BASE_IMAGE --image $INTERMEDIATE_NODE_IMAGE --arch $ARCH } diff --git a/projects/kubernetes/autoscaler/Makefile b/projects/kubernetes/autoscaler/Makefile index 6f426a54cf..2c20cf541f 100644 --- a/projects/kubernetes/autoscaler/Makefile +++ b/projects/kubernetes/autoscaler/Makefile @@ -31,16 +31,17 @@ BUILDSPEC_VARS_KEYS=RELEASE_BRANCH BUILDSPEC_VARS_VALUES=SUPPORTED_K8S_VERSIONS BUILDSPEC_COMPUTE_TYPE=BUILD_GENERAL1_LARGE +REMOVE_CLOUD_PROVIDERS_TARGET=$(REPO)/cluster-autoscaler/cloudprovider/eks-anywhere-cloud-providers-removed-$(GIT_TAG) include $(BASE_DIRECTORY)/Common.mk -$(GO_MOD_DOWNLOAD_TARGETS): remove-cloud-providers -attributions: remove-cloud-providers +$(GO_MOD_DOWNLOAD_TARGETS): $(REMOVE_CLOUD_PROVIDERS_TARGET) +attributions: $(REMOVE_CLOUD_PROVIDERS_TARGET) -.PHONY: remove-cloud-providers -remove-cloud-providers: $(GIT_PATCH_TARGET) +$(REMOVE_CLOUD_PROVIDERS_TARGET): $(GIT_PATCH_TARGET) cd ./autoscaler/cluster-autoscaler/cloudprovider && ls . | grep -v -e builder -e clusterapi -e mocks -e test -e .go | xargs rm -rf + touch $@ $(GATHER_LICENSES_TARGETS): | $(FIX_LICENSES) diff --git a/projects/nutanix-cloud-native/cluster-api-provider-nutanix/Makefile b/projects/nutanix-cloud-native/cluster-api-provider-nutanix/Makefile index 2c32d0b004..d517b462ce 100644 --- a/projects/nutanix-cloud-native/cluster-api-provider-nutanix/Makefile +++ b/projects/nutanix-cloud-native/cluster-api-provider-nutanix/Makefile @@ -17,8 +17,8 @@ include $(BASE_DIRECTORY)/Common.mk s3-artifacts: create-manifests .PHONY: create-manifests -create-manifests: tarballs | ensure-yq - build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION) +create-manifests: tarballs | ensure-yq $$(ENABLE_DOCKER) + @build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION) ########### DO NOT EDIT ############################# diff --git a/projects/prometheus/prometheus/Makefile b/projects/prometheus/prometheus/Makefile index f4bfd8b4ce..08297091af 100644 --- a/projects/prometheus/prometheus/Makefile +++ b/projects/prometheus/prometheus/Makefile @@ -54,12 +54,13 @@ $(FIX_LICENSES_CRONEXPR_TARGET): | $(GO_MOD_DOWNLOAD_TARGETS) # the standard license file name it is not found by go-licenses cp $(@D)/APLv2 $@ -$(WEB_UI_TARGETS): - @echo '>> npm install and npm build' - cd prometheus/web/ui && npm install && CI="" npm run build - @echo ">> bundling npm licenses" - build/cp_npm_licenses.sh - @echo '>> compressing assets' +$(WEB_UI_TARGETS): NPM_AVAIL=$(shell command -v npm &> /dev/null && echo "true" || echo "false") +$(WEB_UI_TARGETS): | $$(call ENABLE_DOCKER,$$(NPM_AVAIL)) + @echo '>> npm install and npm build'; \ + npm -C prometheus/web/ui install && CI="" npm -C prometheus/web/ui run build; \ + echo ">> bundling npm licenses"; \ + build/cp_npm_licenses.sh; \ + echo '>> compressing assets'; \ build/compress_assets.sh $(EXTRA_DOCKER_FILE_TARGETS): diff --git a/projects/prometheus/prometheus/build/cp_npm_licenses.sh b/projects/prometheus/prometheus/build/cp_npm_licenses.sh index 85674a9bd7..0b8b53d470 100755 --- a/projects/prometheus/prometheus/build/cp_npm_licenses.sh +++ b/projects/prometheus/prometheus/build/cp_npm_licenses.sh @@ -17,10 +17,10 @@ set -o errexit set -o nounset set -o pipefail -TAR=tar -if [ "$(uname -s)" = "Darwin" ]; then - TAR=gtar -fi +MAKE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +source "${MAKE_ROOT}/../../../build/lib/common.sh" + +TAR=$(build::find::gnu_variant_on_mac tar) rm -f "prometheus/npm_licenses.tar.bz2" find prometheus/web/ui/node_modules -iname "license*" | $TAR cfj "prometheus/npm_licenses.tar.bz2" --transform 's/^/npm_licenses\//' --files-from=- diff --git a/projects/tinkerbell/cluster-api-provider-tinkerbell/Makefile b/projects/tinkerbell/cluster-api-provider-tinkerbell/Makefile index 42d0625d09..d27cbe2613 100644 --- a/projects/tinkerbell/cluster-api-provider-tinkerbell/Makefile +++ b/projects/tinkerbell/cluster-api-provider-tinkerbell/Makefile @@ -18,8 +18,8 @@ include $(BASE_DIRECTORY)/Common.mk s3-artifacts: create-manifests .PHONY: create-manifests -create-manifests: tarballs | ensure-yq - build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) +create-manifests: tarballs | ensure-yq $$(ENABLE_DOCKER) + @build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) ########### DO NOT EDIT ############################# diff --git a/projects/tinkerbell/hook/Makefile b/projects/tinkerbell/hook/Makefile index 52fa44643f..c567eb7faf 100644 --- a/projects/tinkerbell/hook/Makefile +++ b/projects/tinkerbell/hook/Makefile @@ -35,8 +35,8 @@ include $(BASE_DIRECTORY)/Common.mk $(GATHER_LICENSES_TARGETS): | $(FIX_LICENSES_HOOK_BOOTKIT_TARGET) $(FIX_LICENSES_HOOK_DOCKER_TARGET) # For local image builds we override to push to local registry. -%/images/amd64: IMAGE_OUTPUT=push=true -%/images/amd64: IMAGE_OUTPUT_TYPE=image +%/images/amd64 %/images/arm64: IMAGE_OUTPUT=push=true +%/images/amd64 %/images/arm64: IMAGE_OUTPUT_TYPE=image # hook-docker image required docker runtime. # We are using eks-distro-minimal-base-glibc as the base and builder to install docker. @@ -48,14 +48,15 @@ kernel/images/%: BASE_IMAGE=quay.io/tinkerbell/hook-kernel:$(LINUX_KERNEL_VERSIO s3-artifacts: $(CREATE_HOOK_FILES) -$(CREATE_HOOK_FILES): tarballs +$(CREATE_HOOK_FILES): MAKEFLAGS= +$(CREATE_HOOK_FILES): tarballs | $$(ENABLE_DOCKER) # Modify the linuxkit config file hook/hook.yaml to point to IMAGE_REPO. # Upstream make target `dist` is triggered to perform linuxkit build and generate OSIE files (). - source $(BUILD_LIB)/common.sh && build::common::use_go_version "1.19" && make dist IMAGE_REPO=$(IMAGE_REPO) LATEST_TAG=$(LATEST_TAG) -C $(REPO) - mkdir -p $(OUTPUT_DIR)/hook/$(GIT_TAG) - cp $(REPO)/out/sha-*/rel/vmlinuz-* $(OUTPUT_DIR)/hook/$(GIT_TAG)/ - cp $(REPO)/out/sha-*/rel/initramfs-* $(OUTPUT_DIR)/hook/$(GIT_TAG)/ - mkdir -p $(ARTIFACTS_PATH) + @source $(BUILD_LIB)/common.sh && build::common::use_go_version "1.19" && make dist IMAGE_REPO=$(IMAGE_REPO) LATEST_TAG=$(LATEST_TAG) -C $(REPO); \ + mkdir -p $(OUTPUT_DIR)/hook/$(GIT_TAG); \ + cp $(REPO)/out/sha-*/rel/vmlinuz-* $(OUTPUT_DIR)/hook/$(GIT_TAG)/; \ + cp $(REPO)/out/sha-*/rel/initramfs-* $(OUTPUT_DIR)/hook/$(GIT_TAG)/; \ + mkdir -p $(ARTIFACTS_PATH); \ cp -rf $(OUTPUT_DIR)/hook/* $(ARTIFACTS_PATH) $(REPO)/%/LICENSE: | $(GO_MOD_DOWNLOAD_TARGETS) diff --git a/projects/tinkerbell/rufio/Makefile b/projects/tinkerbell/rufio/Makefile index bcd65048a5..183a7e7d0e 100644 --- a/projects/tinkerbell/rufio/Makefile +++ b/projects/tinkerbell/rufio/Makefile @@ -16,8 +16,8 @@ include $(BASE_DIRECTORY)/Common.mk s3-artifacts: create-manifests .PHONY: create-manifests -create-manifests: tarballs | ensure-yq - build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(GOLANG_VERSION) +create-manifests: tarballs | ensure-yq $$(ENABLE_DOCKER) + @build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(GOLANG_VERSION) FIX_LICENSE_TARGET=$(REPO)/vendor/github.com/bmc-toolbox/common/LICENSE diff --git a/projects/tinkerbell/tink/Makefile b/projects/tinkerbell/tink/Makefile index 6c86d4c6a6..b26eca4582 100644 --- a/projects/tinkerbell/tink/Makefile +++ b/projects/tinkerbell/tink/Makefile @@ -28,8 +28,8 @@ include $(BASE_DIRECTORY)/Common.mk s3-artifacts: create-manifests .PHONY: create-manifests -create-manifests: tarballs - build/create_manifests.sh $(REPO) $(ARTIFACTS_PATH) $(IMAGE_REPO) $(TINK_SERVER_IMAGE_COMPONENT) ${TINK_CONTROLLER_IMAGE_COMPONENT} $(IMAGE_TAG) $(GOLANG_VERSION) +create-manifests: tarballs | $$(ENABLE_DOCKER) + @build/create_manifests.sh $(REPO) $(ARTIFACTS_PATH) $(IMAGE_REPO) $(TINK_SERVER_IMAGE_COMPONENT) ${TINK_CONTROLLER_IMAGE_COMPONENT} $(IMAGE_TAG) $(GOLANG_VERSION) ########### DO NOT EDIT ############################# diff --git a/projects/tinkerbell/tink/build/create_manifests.sh b/projects/tinkerbell/tink/build/create_manifests.sh index c7ae336c45..b59cbaa050 100755 --- a/projects/tinkerbell/tink/build/create_manifests.sh +++ b/projects/tinkerbell/tink/build/create_manifests.sh @@ -33,9 +33,9 @@ build::common::use_go_version ${GOLANG_VERSION} cd $REPO -npm install prettier --global +npm install prettier -make release-manifests \ +PATH=$(pwd)/node_modules/.bin/:$PATH make release-manifests \ TINK_SERVER_IMAGE=${IMAGE_REPO}/${TINK_SERVER_IMAGE_COMPONENT} \ TINK_CONTROLLER_IMAGE=${IMAGE_REPO}/${TINK_CONTROLLER_IMAGE_COMPONENT} \ TINK_SERVER_TAG=${IMAGE_TAG} \ diff --git a/projects/tinkerbell/tinkerbell-chart/Makefile b/projects/tinkerbell/tinkerbell-chart/Makefile index 8f6930df29..465d2b0854 100644 --- a/projects/tinkerbell/tinkerbell-chart/Makefile +++ b/projects/tinkerbell/tinkerbell-chart/Makefile @@ -4,6 +4,7 @@ GIT_TAG=$(shell cat GIT_TAG) REPO_OWNER=tinkerbell REPO=tinkerbell-chart +IMAGE_NAMES= REPO_NO_CLONE=true BUILD_TARGETS=helm/build RELEASE_TARGETS=helm/push diff --git a/projects/torvalds/linux/Makefile b/projects/torvalds/linux/Makefile index 3a76938d20..09d363cfa5 100644 --- a/projects/torvalds/linux/Makefile +++ b/projects/torvalds/linux/Makefile @@ -11,8 +11,8 @@ SIMPLE_CREATE_BINARIES=false # ensure we are building to a directory based on the current # host platform to avoid ever creating a different arch'd # binary in the wrong folder -BINARY_PLATFORMS?=$(BUILDER_PLATFORM) -BINARY_TARGETS=$(OUTPUT_BIN_DIR)/$(subst /,-,$(BINARY_PLATFORMS))/tools/bootconfig +BINARY_PLATFORMS?=linux/$(BUILDER_PLATFORM_ARCH) +BINARY_TARGETS=$(foreach platform,$(BINARY_PLATFORMS),$(OUTPUT_BIN_DIR)/$(subst /,-,$(platform))/tools/bootconfig) HAS_LICENSES=false @@ -39,9 +39,10 @@ include $(BASE_DIRECTORY)/Common.mk tarballs: gather-non-golang-licenses $(OUTPUT_DIR)/ATTRIBUTION.txt -$(BINARY_TARGETS): MAKEFLAGS= -$(BINARY_TARGETS): $(GIT_CHECKOUT_TARGET) - mkdir -p $(MAKE_ROOT)/$(@D) +$(OUTPUT_BIN_DIR)/%: MAKEFLAGS= +$(OUTPUT_BIN_DIR)/%: PLATFORM=$(subst -,/,$(*D:/tools=)) +$(OUTPUT_BIN_DIR)/%: $(GIT_CHECKOUT_TARGET) | $$(call ENABLE_DOCKER_PLATFORM,$$(PLATFORM)) + @mkdir -p $(MAKE_ROOT)/$(@D); \ $(MAKE) -C $(REPO)/tools/bootconfig OUTPUT=$(MAKE_ROOT)/$(@D)/ .PHONY: gather-non-golang-licenses