add validate

aws · Jun 3, 2022 · 3f2d8f6 · 3f2d8f6
1 parent 7b16d17
commit 3f2d8f6
Show file tree

Hide file tree

Showing 5 changed files with 48 additions and 36 deletions.
diff --git a/Makefile b/Makefile
@@ -88,8 +88,16 @@ generate-project-list:
 generate-staging-buildspec:
 	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)"
 
-.PHONY: generate generate-staging-buildspec
-generate: generate-project-list
+.PHONY: generate
+generate: generate-project-list generate-staging-buildspec
+
+.PHONY: validate-generated
+validate-generated: generate
+	@if [ "$$(git status --porcelain -- UPSTREAM_PROJECTS.yaml release/staging-build.yml | wc -l)" -gt 0 ]; then \
+		echo "Error: Generated files, UPSTREAM_PROJECTS.yaml release/staging-build.yml, do not match expected. Please run `make generate` to update"; \
+		git diff -- UPSTREAM_PROJECTS.yaml release/staging-build.yml; \
+		exit 1; \
+	fi
 
 .PHONY: check-project-path-exists
 check-project-path-exists:

diff --git a/UPSTREAM_PROJECTS.yaml b/UPSTREAM_PROJECTS.yaml
@@ -20,6 +20,14 @@ projects:
       - name: cloudstack-cloudmonkey
         versions:
           - tag: 6.2.0
+  - org: aquasecurity
+    repos:
+      - name: harbor-scanner-trivy
+        versions:
+          - tag: v0.26.0
+      - name: trivy
+        versions:
+          - tag: v0.24.2
   - org: aws
     repos:
       - name: etcdadm-bootstrap-provider
@@ -89,10 +97,10 @@ projects:
     repos:
       - name: cloud-provider-vsphere
         versions:
-          - tag: v1.23.0
-          - tag: v1.22.5
           - tag: v1.20.0
           - tag: v1.21.0
+          - tag: v1.22.5
+          - tag: v1.23.0
   - org: kubernetes-sigs
     repos:
       - name: cluster-api
@@ -146,10 +154,10 @@ projects:
     repos:
       - name: boots
         versions:
-          - commit: ab716aac37cfff2869952ea2e44a67cb6d1fe1c1
+          - commit: 94e4b4899b383e28b6002750b14e254cfbbdd81f
       - name: cluster-api-provider-tinkerbell
         versions:
-          - commit: ee045e095565f7943179a98f8d6567f56c3d3bcd
+          - commit: d3667631daf7c2b4e64c3bda801487e5c14c24c9
       - name: hegel
         versions:
           - commit: 7b286fdc8e8fa91a6e9a179a5494b6ee29fce17b
@@ -164,7 +172,7 @@ projects:
           - commit: 9a09ef8e6fd38d1e54359743a4c6a64dc598748f
       - name: rufio
         versions:
-          - commit: 7ab74dd6173a8f3ddf27cd1da0f624af9677adee
+          - commit: 4dc2085adc8eec03b73e51515515fa8966beba04
       - name: sandbox
         versions: []
       - name: tink

diff --git a/build/lib/generate_projects_list.sh b/build/lib/generate_projects_list.sh
@@ -56,7 +56,7 @@ for org_path in projects/*; do
         yq eval -i -P ".projects += [{\"org\": \"$org\", \"repos\": []}]" $UPSTREAM_PROJECTS_FILE # Add an entry for this org in the projects array
         for repo in "${repos[@]}"; do
             yq eval -i -P ".projects[$org_count].repos += [{\"name\": \"$repo\", \"versions\": []}]" $UPSTREAM_PROJECTS_FILE # Add each repo to the repos array
-            git_tags=$(find projects/$org/$repo -type f -name "GIT_TAG")
+            git_tags=$(find projects/$org/$repo -type f -name "GIT_TAG" | sort)
             for file in $git_tags; do
                 tag=$(cat $file)
                 if [ $repo = "cilium" ]; then
@@ -77,6 +77,3 @@ done
 
 HEAD_COMMENT=$(cat $BASE_DIRECTORY/hack/boilerplate.yq.txt)
 yq eval -i ". headComment=\"$HEAD_COMMENT\"" $UPSTREAM_PROJECTS_FILE # Add a header comment with license verbiage and no-edit warning
-yq eval $UPSTREAM_PROJECTS_FILE # Print generated YAML
-
-echo "Contents written to $UPSTREAM_PROJECTS_FILE"
diff --git a/build/lib/generate_staging_buildspec.sh b/build/lib/generate_staging_buildspec.sh
@@ -47,43 +47,45 @@ for project in "${PROJECTS[@]}"; do
     PROJECT_PATH=$MAKE_ROOT/projects/$org/$repo
 
     # TODO: refactor use of release_branch to get git_tag and golang_version in makefile, we should be able to push this to common.mk and avoid needing to pass it here
-    if [[ "true" == "$(make -C $PROJECT_PATH var-value-EXCLUDE_FROM_STAGING_BUILDSPEC RELEASE_BRANCH=1-20)" ]]; then
+    if [[ "true" == "$(make --no-print-directory -C $PROJECT_PATH var-value-EXCLUDE_FROM_STAGING_BUILDSPEC RELEASE_BRANCH=1-20)" ]]; then
         continue
     fi
 
     IDENTIFIER="${org//-/_}_${repo//-/_}"
 
     echo "Adding: $IDENTIFIER"
 
-    PROJECT_DEPENDENCIES=$(make -C $PROJECT_PATH var-value-PROJECT_DEPENDENCIES RELEASE_BRANCH=1-20)
     DEPEND_ON=""
-    DEPS=(${PROJECT_DEPENDENCIES// / })
-    for dep in "${DEPS[@]}"; do
-        DEP_PRODUCT="$(cut -d/ -f1 <<< $dep)"
-        DEP_ORG="$(cut -d/ -f2 <<< $dep)"
-        DEP_REPO="$(cut -d/ -f3 <<< $dep)"
-        if [[ "$DEP_PRODUCT" == "eksd" ]]; then
-            continue
-        fi
-        DEPEND_ON+="\"${DEP_ORG//-/_}_${DEP_REPO//-/_}\","
-
-        if [ ! -d $MAKE_ROOT/projects/$DEP_ORG/$DEP_REPO ]; then
-            echo "Non-existent project dependency: $dep!!!"
-            exit 1
-        fi
-    done
+    PROJECT_DEPENDENCIES=$(make --no-print-directory -C $PROJECT_PATH var-value-PROJECT_DEPENDENCIES RELEASE_BRANCH=1-20)
+    if [ -n "$PROJECT_DEPENDENCIES" ]; then
+        DEPS=(${PROJECT_DEPENDENCIES// / })
+        for dep in "${DEPS[@]}"; do
+            DEP_PRODUCT="$(cut -d/ -f1 <<< $dep)"
+            DEP_ORG="$(cut -d/ -f2 <<< $dep)"
+            DEP_REPO="$(cut -d/ -f3 <<< $dep)"
+            if [[ "$DEP_PRODUCT" == "eksd" ]]; then
+                continue
+            fi
+            DEPEND_ON+="\"${DEP_ORG//-/_}_${DEP_REPO//-/_}\","
+
+            if [ ! -d $MAKE_ROOT/projects/$DEP_ORG/$DEP_REPO ]; then
+                echo "Non-existent project dependency: $dep!!!"
+                exit 1
+            fi
+        done
+    fi
 
     if [ -n "$DEPEND_ON" ]; then
         DEPEND_ON="\"depend-on\":[${DEPEND_ON%?}],"
     fi
 
     CLONE_URL=""
-    if [[ "true" != "$(make -C $PROJECT_PATH var-value-REPO_NO_CLONE RELEASE_BRANCH=1-20)" ]]; then
-        REPO=$(make -C $PROJECT_PATH var-value-CLONE_URL AWS_REGION=us-west-2 CODEBUILD_CI=true RELEASE_BRANCH=1-20)
+    if [[ "true" != "$(make --no-print-directory -C $PROJECT_PATH var-value-REPO_NO_CLONE RELEASE_BRANCH=1-20)" ]]; then
+        REPO=$(make --no-print-directory -C $PROJECT_PATH var-value-CLONE_URL AWS_REGION=us-west-2 CODEBUILD_CI=true RELEASE_BRANCH=1-20)
         CLONE_URL=",\"CLONE_URL\":\"$REPO\""
     fi
 
-    BUILDSPECS=$(make -C $PROJECT_PATH var-value-BUILDSPECS RELEASE_BRANCH=1-20)
+    BUILDSPECS=$(make --no-print-directory -C $PROJECT_PATH var-value-BUILDSPECS RELEASE_BRANCH=1-20)
     SPECS=(${BUILDSPECS// / })
     for buildspec in "${SPECS[@]}"; do
         if [[ "${#SPECS[@]}" != "1" ]]; then
@@ -98,6 +100,3 @@ done
 
 HEAD_COMMENT=$(cat $BASE_DIRECTORY/hack/boilerplate.yq.txt)
 yq eval -i ". headComment=\"$HEAD_COMMENT\"" $STAGING_BUILDSPEC_FILE # Add a header comment with license verbiage and no-edit warning
-yq eval $STAGING_BUILDSPEC_FILE # Print generated YAML
-
-echo "Contents written to $STAGING_BUILDSPEC_FILE"
diff --git a/hack/boilerplate.yq.txt b/hack/boilerplate.yq.txt
@@ -1,13 +1,13 @@
 Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 
-Licensed under the Apache License, Version 2.0 (the \"License\");
+Licensed under the Apache License, Version 2.0 (the License);
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
      http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an \"AS IS\" BASIS,
+distributed under the License is distributed on an AS IS BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.