From 1ea3ca6468b7888dde071a98b77cdeb6d9032053 Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Tue, 10 Dec 2024 16:23:54 +0000 Subject: [PATCH 1/5] Validate provider version --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index edbbcc5e31b1..2f4f55111823 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,11 @@ PROVIDER_VERSION ?= 2.0.0-alpha.0+dev # These variables are lazy (no `:`) so they're not calculated until after the dependency is installed VERSION_GENERIC = $(shell bin/pulumictl convert-version -l generic -v "$(PROVIDER_VERSION)") VERSION_FLAGS = -ldflags "-X github.com/pulumi/pulumi-azure-native/v2/provider/pkg/version.Version=${VERSION_GENERIC}" +# Check version doesn't start with a "v" - this is a common mistake +ifeq ($(shell echo $(PROVIDER_VERSION) | cut -c1),v) +$(error PROVIDER_VERSION should not start with a "v") +endif + MAJOR_VERSION = $(shell echo $(PROVIDER_VERSION) | cut -d. -f1) PREVIOUS_MAJOR_VERSION = $(shell echo $(MAJOR_VERSION)-1 | bc) NEXT_MAJOR_VERSION = $(shell echo $(MAJOR_VERSION)+1 | bc) From 6c3e534f5a7c97f3870f4db5e470e75b73dde2b2 Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Tue, 10 Dec 2024 16:24:38 +0000 Subject: [PATCH 2/5] Use PROVIDER_VERSION instead of VERSION_GENERIC --- Makefile | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 2f4f55111823..dacc2feeec10 100644 --- a/Makefile +++ b/Makefile @@ -31,15 +31,12 @@ endif # Input during CI using `make [TARGET] PROVIDER_VERSION=""` or by setting a PROVIDER_VERSION environment variable # Local builds will just used this fixed default version unless specified PROVIDER_VERSION ?= 2.0.0-alpha.0+dev -# Ensure the leading "v" is removed - use this normalised version everywhere rather than the raw input to ensure consistency. -# These variables are lazy (no `:`) so they're not calculated until after the dependency is installed -VERSION_GENERIC = $(shell bin/pulumictl convert-version -l generic -v "$(PROVIDER_VERSION)") -VERSION_FLAGS = -ldflags "-X github.com/pulumi/pulumi-azure-native/v2/provider/pkg/version.Version=${VERSION_GENERIC}" # Check version doesn't start with a "v" - this is a common mistake ifeq ($(shell echo $(PROVIDER_VERSION) | cut -c1),v) $(error PROVIDER_VERSION should not start with a "v") endif +VERSION_FLAGS = -ldflags "-X github.com/pulumi/pulumi-azure-native/v2/provider/pkg/version.Version=${PROVIDER_VERSION}" MAJOR_VERSION = $(shell echo $(PROVIDER_VERSION) | cut -d. -f1) PREVIOUS_MAJOR_VERSION = $(shell echo $(MAJOR_VERSION)-1 | bc) NEXT_MAJOR_VERSION = $(shell echo $(MAJOR_VERSION)+1 | bc) @@ -53,7 +50,7 @@ _ := $(shell mkdir -p .make) .PHONY: default ensure dist default: provider build_sdks ensure: bin/pulumictl .make/provider_mod_download -dist: dist/pulumi-azure-native_$(VERSION_GENERIC)_checksums.txt dist/docs-schema.json +dist: dist/pulumi-azure-native_$(PROVIDER_VERSION)_checksums.txt dist/docs-schema.json # Binaries .PHONY: codegen provider @@ -173,7 +170,7 @@ test_nodejs: provider install_nodejs_sdk .PHONY: schema_squeeze schema_squeeze: bin/$(CODEGEN) - bin/$(CODEGEN) squeeze $(VERSION_GENERIC) + bin/$(CODEGEN) squeeze $(PROVIDER_VERSION) .PHONY: explode_schema explode_schema: dist/docs-schema.json @@ -229,12 +226,12 @@ bin/$(CODEGEN): bin/pulumictl .make/prebuild .make/provider_mod_download provide # Writes schema-full.json and metadata-compact.json to bin/ # Also re-calculates files in versions/ at same time bin/schema-full.json bin/metadata-compact.json &: bin/$(CODEGEN) $(SPECS) versions/az-provider-list.json versions/v${PREVIOUS_MAJOR_VERSION}-lock.json versions/v${MAJOR_VERSION}-config.yaml versions/v${MAJOR_VERSION}-spec.yaml versions/v${MAJOR_VERSION}-removed-resources.json versions/v${NEXT_MAJOR_VERSION}-removed-resources.json - bin/$(CODEGEN) schema $(VERSION_GENERIC) + bin/$(CODEGEN) schema $(PROVIDER_VERSION) # Docs schema - treat as phony becasuse it's committed so we always need to rebuild it. .PHONY: provider/cmd/pulumi-resource-azure-native/schema.json provider/cmd/pulumi-resource-azure-native/schema.json: bin/$(CODEGEN) $(SPECS) versions/v${PREVIOUS_MAJOR_VERSION}-lock.json versions/v${MAJOR_VERSION}-config.yaml versions/v${MAJOR_VERSION}-removed-resources.json - bin/$(CODEGEN) docs $(VERSION_GENERIC) + bin/$(CODEGEN) docs $(PROVIDER_VERSION) bin/$(LOCAL_PROVIDER_FILENAME): bin/pulumictl .make/prebuild .make/provider_mod_download provider/cmd/$(PROVIDER)/*.go .make/provider_prebuild $(PROVIDER_PKG) cd provider && \ @@ -263,12 +260,12 @@ dist/$(PROVIDER)-v$(PROVIDER_VERSION)-%.tar.gz: @# $< is the last dependency (the binary path from above) tar --gzip -cf $@ README.md LICENSE -C $$(dirname $<) . -dist/pulumi-azure-native_$(VERSION_GENERIC)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-linux-amd64.tar.gz -dist/pulumi-azure-native_$(VERSION_GENERIC)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-linux-arm64.tar.gz -dist/pulumi-azure-native_$(VERSION_GENERIC)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-darwin-amd64.tar.gz -dist/pulumi-azure-native_$(VERSION_GENERIC)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-darwin-arm64.tar.gz -dist/pulumi-azure-native_$(VERSION_GENERIC)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-windows-amd64.tar.gz - cd dist && shasum *.tar.gz > pulumi-azure-native_$(VERSION_GENERIC)_checksums.txt +dist/pulumi-azure-native_$(PROVIDER_VERSION)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-linux-amd64.tar.gz +dist/pulumi-azure-native_$(PROVIDER_VERSION)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-linux-arm64.tar.gz +dist/pulumi-azure-native_$(PROVIDER_VERSION)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-darwin-amd64.tar.gz +dist/pulumi-azure-native_$(PROVIDER_VERSION)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-darwin-arm64.tar.gz +dist/pulumi-azure-native_$(PROVIDER_VERSION)_checksums.txt: dist/$(PROVIDER)-v$(PROVIDER_VERSION)-windows-amd64.tar.gz + cd dist && shasum *.tar.gz > pulumi-azure-native_$(PROVIDER_VERSION)_checksums.txt # --------- Sentinel targets --------- # @@ -332,7 +329,7 @@ export FAKE_MODULE @# Unmark this is as an up-to-date local build rm -f .make/prepublish_go rm -rf $$(find sdk/pulumi-azure-native-sdk -mindepth 1 -maxdepth 1 ! -name ".git") - bin/$(CODEGEN) go $(VERSION_GENERIC) + bin/$(CODEGEN) go $(PROVIDER_VERSION) @# Tidy up all go.mod files find sdk/pulumi-azure-native-sdk -type d -maxdepth 1 -exec sh -c "cd \"{}\" && go mod tidy" \; @touch $@ @@ -357,7 +354,7 @@ export FAKE_MODULE yarn install --cwd sdk/nodejs @touch $@ -.make/build_nodejs: VERSION_JS = $(shell bin/pulumictl convert-version -l javascript -v "$(VERSION_GENERIC)") +.make/build_nodejs: VERSION_JS = $(shell bin/pulumictl convert-version -l javascript -v "$(PROVIDER_VERSION)") .make/build_nodejs: bin/pulumictl .make/nodejs_yarn_install cd sdk/nodejs/ && \ NODE_OPTIONS=--max-old-space-size=12288 yarn run tsc --diagnostics --incremental && \ @@ -366,7 +363,7 @@ export FAKE_MODULE sed -i.bak -e "s/\$${VERSION}/$(VERSION_JS)/g" ./bin/package.json @touch $@ -.make/build_python: VERSION_PYTHON = $(shell bin/pulumictl convert-version -l python -v "$(VERSION_GENERIC)") +.make/build_python: VERSION_PYTHON = $(shell bin/pulumictl convert-version -l python -v "$(PROVIDER_VERSION)") .make/build_python: bin/pulumictl .make/generate_python cd sdk/python && \ git clean -fxd && \ @@ -389,7 +386,7 @@ export FAKE_MODULE .make/build_java: bin/pulumictl .make/generate_java cd sdk/java/ && \ - gradle --console=plain -Pversion=$(VERSION_GENERIC) build + gradle --console=plain -Pversion=$(PROVIDER_VERSION) build @touch $@ .make/build_go: .make/generate_go_local From 5ce1baf2c7a66cf40575b97c7a83bda4081347a8 Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Tue, 10 Dec 2024 16:26:35 +0000 Subject: [PATCH 3/5] Remove unused pulumictl dependencies --- Makefile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index dacc2feeec10..b6fb89d36ccf 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,7 @@ dist/docs-schema.json: bin/schema-full.json mkdir -p dist yarn schema implode --cwd bin/schema --outFile dist/docs-schema.json -bin/$(CODEGEN): bin/pulumictl .make/prebuild .make/provider_mod_download provider/cmd/$(CODEGEN)/* $(PROVIDER_PKG) +bin/$(CODEGEN): .make/prebuild .make/provider_mod_download provider/cmd/$(CODEGEN)/* $(PROVIDER_PKG) cd provider && go build -o $(WORKING_DIR)/bin/$(CODEGEN) $(VERSION_FLAGS) $(PROJECT)/v2/provider/cmd/$(CODEGEN) # Writes schema-full.json and metadata-compact.json to bin/ @@ -233,7 +233,7 @@ bin/schema-full.json bin/metadata-compact.json &: bin/$(CODEGEN) $(SPECS) versio provider/cmd/pulumi-resource-azure-native/schema.json: bin/$(CODEGEN) $(SPECS) versions/v${PREVIOUS_MAJOR_VERSION}-lock.json versions/v${MAJOR_VERSION}-config.yaml versions/v${MAJOR_VERSION}-removed-resources.json bin/$(CODEGEN) docs $(PROVIDER_VERSION) -bin/$(LOCAL_PROVIDER_FILENAME): bin/pulumictl .make/prebuild .make/provider_mod_download provider/cmd/$(PROVIDER)/*.go .make/provider_prebuild $(PROVIDER_PKG) +bin/$(LOCAL_PROVIDER_FILENAME): .make/prebuild .make/provider_mod_download provider/cmd/$(PROVIDER)/*.go .make/provider_prebuild $(PROVIDER_PKG) cd provider && \ CGO_ENABLED=0 go build -o $(WORKING_DIR)/bin/$(LOCAL_PROVIDER_FILENAME) $(VERSION_FLAGS) $(PROJECT)/v2/provider/cmd/$(PROVIDER) @@ -242,7 +242,7 @@ bin/linux-arm64/$(PROVIDER): TARGET := linux-arm64 bin/darwin-amd64/$(PROVIDER): TARGET := darwin-amd64 bin/darwin-arm64/$(PROVIDER): TARGET := darwin-arm64 bin/windows-amd64/$(PROVIDER).exe: TARGET := windows-amd64 -bin/%/$(PROVIDER) bin/%/$(PROVIDER).exe: bin/pulumictl .make/provider_mod_download .make/prebuild provider/cmd/$(PROVIDER)/*.go .make/provider_prebuild $(PROVIDER_PKG) +bin/%/$(PROVIDER) bin/%/$(PROVIDER).exe: .make/provider_mod_download .make/prebuild provider/cmd/$(PROVIDER)/*.go .make/provider_prebuild $(PROVIDER_PKG) @# check the TARGET is set test $(TARGET) cd provider && \ @@ -291,14 +291,14 @@ endef export FAKE_MODULE # We use the docs schema for java but don't depend on it because it changes on every generation -.make/generate_java: bin/pulumictl bin/pulumi-java-gen +.make/generate_java: bin/pulumi-java-gen @mkdir -p sdk/java rm -rf $$(find sdk/java -mindepth 1 -maxdepth 1) bin/$(JAVA_GEN) generate --schema provider/cmd/$(PROVIDER)/schema.json --out sdk/java --build gradle-nexus echo "$$FAKE_MODULE" | sed 's/fake_module/fake_java_module/g' > sdk/java/go.mod @touch $@ -.make/generate_nodejs: bin/pulumictl .pulumi/bin/pulumi bin/schema-full.json +.make/generate_nodejs: .pulumi/bin/pulumi bin/schema-full.json mkdir -p sdk/nodejs rm -rf $$(find sdk/nodejs -mindepth 1 -maxdepth 1 ! -name "go.mod") .pulumi/bin/pulumi package gen-sdk bin/schema-full.json --language nodejs @@ -307,7 +307,7 @@ export FAKE_MODULE rm sdk/nodejs/tsconfig.json.bak @touch $@ -.make/generate_python: bin/pulumictl .pulumi/bin/pulumi bin/schema-full.json +.make/generate_python: .pulumi/bin/pulumi bin/schema-full.json mkdir -p sdk/python rm -rf $$(find sdk/python -mindepth 1 -maxdepth 1 ! -name "go.mod") .pulumi/bin/pulumi package gen-sdk bin/schema-full.json --language python @@ -315,7 +315,7 @@ export FAKE_MODULE cp README.md sdk/python @touch $@ -.make/generate_dotnet: bin/pulumictl .pulumi/bin/pulumi bin/schema-full.json +.make/generate_dotnet: .pulumi/bin/pulumi bin/schema-full.json mkdir -p sdk/dotnet rm -rf $$(find sdk/dotnet -mindepth 1 -maxdepth 1 ! -name "go.mod") .pulumi/bin/pulumi package gen-sdk bin/schema-full.json --language dotnet @@ -324,7 +324,7 @@ export FAKE_MODULE rm sdk/dotnet/Pulumi.AzureNative.csproj.bak @touch $@ -.make/generate_go_local: bin/pulumictl bin/$(CODEGEN) bin/schema-full.json +.make/generate_go_local: bin/$(CODEGEN) bin/schema-full.json @mkdir -p sdk/pulumi-azure-native-sdk @# Unmark this is as an up-to-date local build rm -f .make/prepublish_go @@ -384,7 +384,7 @@ export FAKE_MODULE dotnet build /p:Version=$(VERSION_DOTNET) @touch $@ -.make/build_java: bin/pulumictl .make/generate_java +.make/build_java: .make/generate_java cd sdk/java/ && \ gradle --console=plain -Pversion=$(PROVIDER_VERSION) build @touch $@ @@ -407,6 +407,6 @@ export FAKE_MODULE ; fi @touch $@ -.make/install_provider: bin/pulumictl .make/provider_mod_download provider/cmd/$(PROVIDER)/* $(PROVIDER_PKG) +.make/install_provider: .make/provider_mod_download provider/cmd/$(PROVIDER)/* $(PROVIDER_PKG) cd provider && go install $(VERSION_FLAGS) $(PROJECT)/provider/cmd/$(PROVIDER) @touch $@ From 69d125cd51a5814329d262ba40cd59e67f63015c Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Tue, 10 Dec 2024 17:13:03 +0000 Subject: [PATCH 4/5] Track changes from the specs via the submodule HEAD This saves ~3.5s per makefile run: Before: make schema 3.90s user 1.09s system 94% cpu 5.296 total After: make schema 0.12s user 0.18s system 80% cpu 0.369 total --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b6fb89d36ccf..9a1087d1b5d5 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ CODEGEN := pulumi-gen-azure-native WORKING_DIR := $(shell pwd) PROVIDER_PKG := $(shell find provider/pkg -type f) -SPECS := $(shell find azure-rest-api-specs/specification/*/resource-manager -type f -name "*.json" ! -path "**/examples/**") +SPECS := .git/modules/azure-rest-api-specs/HEAD # Fail fast if the specs submodule doesn't exist ifeq (,$(SPECS)) From f321a417d91e233642383fecdfba97163b58746a Mon Sep 17 00:00:00 2001 From: Daniel Bradley Date: Wed, 11 Dec 2024 09:31:40 +0000 Subject: [PATCH 5/5] Remove SDK version injection This is no longer required since adding respectSchemaVersion in the schema which creates these SDK-specific version numbers for us. - Remove version injection from dotnet, python and nodejs build steps. - Remove the unused SDK-specific version variables. - Create the dotnet version.txt as part of the generate step instead of the build. - Remove dependencies on pulumictl --- Makefile | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 9a1087d1b5d5..a56632875f35 100644 --- a/Makefile +++ b/Makefile @@ -322,6 +322,7 @@ export FAKE_MODULE echo "$$FAKE_MODULE" | sed 's/fake_module/fake_dotnet_module/g' > sdk/dotnet/go.mod sed -i.bak -e "s/<\/Nullable>/<\/Nullable>\n false<\/UseSharedCompilation>/g" sdk/dotnet/Pulumi.AzureNative.csproj rm sdk/dotnet/Pulumi.AzureNative.csproj.bak + echo "azure-native\n$(PROVIDER_VERSION)" > sdk/dotnet/version.txt @touch $@ .make/generate_go_local: bin/$(CODEGEN) bin/schema-full.json @@ -354,22 +355,16 @@ export FAKE_MODULE yarn install --cwd sdk/nodejs @touch $@ -.make/build_nodejs: VERSION_JS = $(shell bin/pulumictl convert-version -l javascript -v "$(PROVIDER_VERSION)") -.make/build_nodejs: bin/pulumictl .make/nodejs_yarn_install +.make/build_nodejs: .make/nodejs_yarn_install cd sdk/nodejs/ && \ NODE_OPTIONS=--max-old-space-size=12288 yarn run tsc --diagnostics --incremental && \ - cp ../../README.md ../../LICENSE package.json yarn.lock ./bin/ && \ - mkdir -p bin/scripts && \ - sed -i.bak -e "s/\$${VERSION}/$(VERSION_JS)/g" ./bin/package.json + cp ../../README.md ../../LICENSE package.json yarn.lock ./bin/ @touch $@ -.make/build_python: VERSION_PYTHON = $(shell bin/pulumictl convert-version -l python -v "$(PROVIDER_VERSION)") -.make/build_python: bin/pulumictl .make/generate_python +.make/build_python: .make/generate_python cd sdk/python && \ git clean -fxd && \ rm -rf ./bin/ ../python.bin/ && cp -R . ../python.bin && mv ../python.bin ./bin && \ - sed -i.bak -e 's/^ version = .*/ version = "$(VERSION_PYTHON)"/g' ./bin/pyproject.toml && \ - rm ./bin/pyproject.toml.bak && \ rm ./bin/go.mod && \ python3 -m venv venv && \ ./venv/bin/python -m pip install build && \ @@ -377,11 +372,8 @@ export FAKE_MODULE ../venv/bin/python -m build . @touch $@ -.make/build_dotnet: VERSION_DOTNET = $(shell bin/pulumictl convert-version -l dotnet -v "$(PROVIDER_VERSION)") -.make/build_dotnet: bin/pulumictl .make/generate_dotnet - cd sdk/dotnet && \ - echo "azure-native\n$(VERSION_DOTNET)" >version.txt && \ - dotnet build /p:Version=$(VERSION_DOTNET) +.make/build_dotnet: .make/generate_dotnet + cd sdk/dotnet && dotnet build @touch $@ .make/build_java: .make/generate_java