Enable remote tasks to be run in cluster

By default, we should run builds matching the local architecture in-cluster to reduce the overhead of provisioning platforms. This will enable a fully matrixed build for all images using only the remote builds. This change will require the multi-platform controller to set the /ssh/host to localhost in order for the builds to run in-cluster. Signed-off-by: arewm <[email protected]>
konflux-ci · Aug 5, 2024 · 71e0059 · 71e0059
1 parent 555aa46
commit 71e0059
Show file tree

Hide file tree

Showing 6 changed files with 359 additions and 284 deletions.
diff --git a/task-generator/remote/main.go b/task-generator/remote/main.go
@@ -140,55 +140,62 @@ if [ -e "/ssh/error" ]; then
   #no server could be provisioned
   cat /ssh/error
   exit 1
+fi
+export SSH_HOST=$(cat /ssh/host)
+IS_LOCALHOST=$([ "$SSH_HOST" == "localhost" ] && echo true || echo false)
+export IS_LOCALHOST
+
+if [[ $IS_LOCALHOST == true ]]; then
+  echo "Localhost detected; running build in cluster"
 elif [ -e "/ssh/otp" ]; then
- curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
- echo "" >> ~/.ssh/id_rsa
+  curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
+  echo "" >> ~/.ssh/id_rsa
 else
   cp /ssh/id_rsa ~/.ssh
 fi
-chmod 0400 ~/.ssh/id_rsa
-export SSH_HOST=$(cat /ssh/host)
-export BUILD_DIR=$(cat /ssh/user-dir)
-export SSH_ARGS="-o StrictHostKeyChecking=no"
+
 mkdir -p scripts
-echo "$BUILD_DIR"
-ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
-
-PORT_FORWARD=""
-PODMAN_PORT_FORWARD=""
-if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
-PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
-PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
-fi
+
+if [[ $IS_LOCALHOST != true ]]; then
+  chmod 0400 ~/.ssh/id_rsa
+  export BUILD_DIR=$(cat /ssh/user-dir)
+  export SSH_ARGS="-o StrictHostKeyChecking=no"
+  echo "$BUILD_DIR"
+  ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
+
+  PORT_FORWARD=""
+  PODMAN_PORT_FORWARD=""
+  if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
+    PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
+    PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
+  fi
 `
-		if taskVersion == "0.2" {
-			ret += adjustRemoteImage
-		}
 		env := "$PODMAN_PORT_FORWARD \\\n"
 
 		// disable podman subscription-manager integration
-		env += " --tmpfs /run/secrets \\\n"
+		env += "    --tmpfs /run/secrets \\\n"
 
 		// Before the build we sync the contents of the workspace to the remote host
 		for _, workspace := range task.Spec.Workspaces {
-			ret += "\nrsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
-			podmanArgs += " -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
+			ret += "\n  rsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
+			podmanArgs += "    -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
 		}
 		// Also sync the volume mounts from the template
 		for _, volume := range task.Spec.StepTemplate.VolumeMounts {
-			ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
-			podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
+			ret += "\n  rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
+			podmanArgs += "    -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
 		}
 		for _, volume := range step.VolumeMounts {
 			if syncVolumes[volume.Name] {
-				ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
-				podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
+				ret += "\n  rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
+				podmanArgs += "    -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
 			}
 		}
-		ret += "\nrsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
-		podmanArgs += " -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
-		ret += "\nrsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/tekton-results/\""
-		podmanArgs += " -v \"$BUILD_DIR/tekton-results/:/tekton/results:Z\" \\\n"
+		ret += "\n  rsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
+		podmanArgs += "    -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
+		ret += "\n  rsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/results/\""
+		podmanArgs += "    -v \"$BUILD_DIR/results/:/tekton/results:Z\" \\\n"
+		ret += "\nfi"
 
 		script := "scripts/script-" + step.Name + ".sh"
 
@@ -212,33 +219,40 @@ fi
 		ret += step.Script
 		ret += "\nbuildah push \"$IMAGE\" oci:rhtap-final-image"
 		ret += "\nREMOTESSHEOF"
-		ret += "\nchmod +x " + script
+		ret += "\nchmod +x " + script + "\n"
+
+		if taskVersion == "0.2" {
+			ret += adjustRemoteImage
+		}
 
 		if task.Spec.StepTemplate != nil {
 			for _, e := range task.Spec.StepTemplate.Env {
-				env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
+				env += "    -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
 			}
 		}
-		ret += "\nrsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
+		ret += "\nif [[ $IS_LOCALHOST != true ]]; then"
+		ret += "\n  rsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
 		containerScript := "/script/script-" + step.Name + ".sh"
 		for _, e := range step.Env {
-			env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
+			env += "    -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
 		}
-		podmanArgs += " -v $BUILD_DIR/scripts:/script:Z \\\n"
-		ret += "\nssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman  run " + env + "" + podmanArgs + "--user=0  --rm  \"$BUILDER_IMAGE\" " + containerScript
+		podmanArgs += "    -v $BUILD_DIR/scripts:/script:Z \\\n"
+		ret += "\n  ssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman  run " + env + "" + podmanArgs + "    --user=0  --rm  \"$BUILDER_IMAGE\" " + containerScript
 
 		// Sync the contents of the workspaces back so subsequent tasks can use them
 		for _, workspace := range task.Spec.Workspaces {
-			ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
+			ret += "\n  rsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
 		}
 
 		for _, volume := range task.Spec.StepTemplate.VolumeMounts {
-			ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
+			ret += "\n  rsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
 		}
 		//sync back results
-		ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/tekton-results/\" \"/tekton/results/\""
+		ret += "\n  rsync -ra \"$SSH_HOST:$BUILD_DIR/results/\" \"/tekton/results/\""
 
-		ret += "\nbuildah pull oci:rhtap-final-image"
+		ret += "\n  buildah pull oci:rhtap-final-image"
+		ret += "\nelse\n  bash " + containerScript
+		ret += "\nfi"
 		ret += "\nbuildah images"
 		ret += "\nbuildah tag localhost/rhtap-final-image \"$IMAGE\""
 		ret += "\ncontainer=$(buildah from --pull-never \"$IMAGE\")\nbuildah mount \"$container\" | tee /shared/container_path\necho $container > /shared/container_name"

diff --git a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml
@@ -537,6 +537,7 @@ spec:
       workingDir: /var/workdir
       volumeMounts:
         - mountPath: /var/lib/containers
+          name: varlibcontainers
         - mountPath: /mnt/trusted-ca
           name: trusted-ca
           readOnly: true

diff --git a/task/buildah-remote-oci-ta/0.1/buildah-remote-oci-ta.yaml b/task/buildah-remote-oci-ta/0.1/buildah-remote-oci-ta.yaml
@@ -222,35 +222,45 @@ spec:
         #no server could be provisioned
         cat /ssh/error
         exit 1
+      fi
+      export SSH_HOST=$(cat /ssh/host)
+      IS_LOCALHOST=$([ "$SSH_HOST" == "localhost" ] && echo true || echo false)
+      export IS_LOCALHOST
+
+      if [[ $IS_LOCALHOST == true ]]; then
+        echo "Localhost detected; running build in cluster"
       elif [ -e "/ssh/otp" ]; then
-       curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
-       echo "" >> ~/.ssh/id_rsa
+        curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
+        echo "" >> ~/.ssh/id_rsa
       else
         cp /ssh/id_rsa ~/.ssh
       fi
-      chmod 0400 ~/.ssh/id_rsa
-      export SSH_HOST=$(cat /ssh/host)
-      export BUILD_DIR=$(cat /ssh/user-dir)
-      export SSH_ARGS="-o StrictHostKeyChecking=no"
+
       mkdir -p scripts
-      echo "$BUILD_DIR"
-      ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
-
-      PORT_FORWARD=""
-      PODMAN_PORT_FORWARD=""
-      if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
-      PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
-      PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
-      fi
 
-      rsync -ra /shared/ "$SSH_HOST:$BUILD_DIR/volumes/shared/"
-      rsync -ra /var/workdir/ "$SSH_HOST:$BUILD_DIR/volumes/workdir/"
-      rsync -ra /entitlement/ "$SSH_HOST:$BUILD_DIR/volumes/etc-pki-entitlement/"
-      rsync -ra /activation-key/ "$SSH_HOST:$BUILD_DIR/volumes/activation-key/"
-      rsync -ra /additional-secret/ "$SSH_HOST:$BUILD_DIR/volumes/additional-secret/"
-      rsync -ra /mnt/trusted-ca/ "$SSH_HOST:$BUILD_DIR/volumes/trusted-ca/"
-      rsync -ra "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/"
-      rsync -ra "/tekton/results/" "$SSH_HOST:$BUILD_DIR/tekton-results/"
+      if [[ $IS_LOCALHOST != true ]]; then
+        chmod 0400 ~/.ssh/id_rsa
+        export BUILD_DIR=$(cat /ssh/user-dir)
+        export SSH_ARGS="-o StrictHostKeyChecking=no"
+        echo "$BUILD_DIR"
+        ssh $SSH_ARGS "$SSH_HOST"  mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
+
+        PORT_FORWARD=""
+        PODMAN_PORT_FORWARD=""
+        if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
+          PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
+          PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
+        fi
+
+        rsync -ra /shared/ "$SSH_HOST:$BUILD_DIR/volumes/shared/"
+        rsync -ra /var/workdir/ "$SSH_HOST:$BUILD_DIR/volumes/workdir/"
+        rsync -ra /entitlement/ "$SSH_HOST:$BUILD_DIR/volumes/etc-pki-entitlement/"
+        rsync -ra /activation-key/ "$SSH_HOST:$BUILD_DIR/volumes/activation-key/"
+        rsync -ra /additional-secret/ "$SSH_HOST:$BUILD_DIR/volumes/additional-secret/"
+        rsync -ra /mnt/trusted-ca/ "$SSH_HOST:$BUILD_DIR/volumes/trusted-ca/"
+        rsync -ra "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/"
+        rsync -ra "/tekton/results/" "$SSH_HOST:$BUILD_DIR/results/"
+      fi
       cat >scripts/script-build.sh <<'REMOTESSHEOF'
       #!/bin/bash
       set -o verbose
@@ -435,43 +445,48 @@ spec:
       buildah push "$IMAGE" oci:rhtap-final-image
       REMOTESSHEOF
       chmod +x scripts/script-build.sh
-      rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
-      ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
-       --tmpfs /run/secrets \
-       -e ACTIVATION_KEY="$ACTIVATION_KEY" \
-       -e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
-       -e ADD_CAPABILITIES="$ADD_CAPABILITIES" \
-       -e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
-       -e BUILD_ARGS_FILE="$BUILD_ARGS_FILE" \
-       -e CONTEXT="$CONTEXT" \
-       -e DOCKERFILE="$DOCKERFILE" \
-       -e ENTITLEMENT_SECRET="$ENTITLEMENT_SECRET" \
-       -e HERMETIC="$HERMETIC" \
-       -e IMAGE="$IMAGE" \
-       -e IMAGE_EXPIRES_AFTER="$IMAGE_EXPIRES_AFTER" \
-       -e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
-       -e SQUASH="$SQUASH" \
-       -e STORAGE_DRIVER="$STORAGE_DRIVER" \
-       -e TARGET_STAGE="$TARGET_STAGE" \
-       -e TLSVERIFY="$TLSVERIFY" \
-       -e YUM_REPOS_D_FETCHED="$YUM_REPOS_D_FETCHED" \
-       -e YUM_REPOS_D_SRC="$YUM_REPOS_D_SRC" \
-       -e YUM_REPOS_D_TARGET="$YUM_REPOS_D_TARGET" \
-       -e COMMIT_SHA="$COMMIT_SHA" \
-       -v "$BUILD_DIR/volumes/shared:/shared:Z" \
-       -v "$BUILD_DIR/volumes/workdir:/var/workdir:Z" \
-       -v "$BUILD_DIR/volumes/etc-pki-entitlement:/entitlement:Z" \
-       -v "$BUILD_DIR/volumes/activation-key:/activation-key:Z" \
-       -v "$BUILD_DIR/volumes/additional-secret:/additional-secret:Z" \
-       -v "$BUILD_DIR/volumes/trusted-ca:/mnt/trusted-ca:Z" \
-       -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
-       -v "$BUILD_DIR/tekton-results/:/tekton/results:Z" \
-       -v $BUILD_DIR/scripts:/script:Z \
-      --user=0  --rm  "$BUILDER_IMAGE" /script/script-build.sh
-      rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
-      rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/
-      rsync -ra "$SSH_HOST:$BUILD_DIR/tekton-results/" "/tekton/results/"
-      buildah pull oci:rhtap-final-image
+
+      if [[ $IS_LOCALHOST != true ]]; then
+        rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
+        ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman  run $PODMAN_PORT_FORWARD \
+          --tmpfs /run/secrets \
+          -e ACTIVATION_KEY="$ACTIVATION_KEY" \
+          -e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
+          -e ADD_CAPABILITIES="$ADD_CAPABILITIES" \
+          -e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
+          -e BUILD_ARGS_FILE="$BUILD_ARGS_FILE" \
+          -e CONTEXT="$CONTEXT" \
+          -e DOCKERFILE="$DOCKERFILE" \
+          -e ENTITLEMENT_SECRET="$ENTITLEMENT_SECRET" \
+          -e HERMETIC="$HERMETIC" \
+          -e IMAGE="$IMAGE" \
+          -e IMAGE_EXPIRES_AFTER="$IMAGE_EXPIRES_AFTER" \
+          -e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
+          -e SQUASH="$SQUASH" \
+          -e STORAGE_DRIVER="$STORAGE_DRIVER" \
+          -e TARGET_STAGE="$TARGET_STAGE" \
+          -e TLSVERIFY="$TLSVERIFY" \
+          -e YUM_REPOS_D_FETCHED="$YUM_REPOS_D_FETCHED" \
+          -e YUM_REPOS_D_SRC="$YUM_REPOS_D_SRC" \
+          -e YUM_REPOS_D_TARGET="$YUM_REPOS_D_TARGET" \
+          -e COMMIT_SHA="$COMMIT_SHA" \
+          -v "$BUILD_DIR/volumes/shared:/shared:Z" \
+          -v "$BUILD_DIR/volumes/workdir:/var/workdir:Z" \
+          -v "$BUILD_DIR/volumes/etc-pki-entitlement:/entitlement:Z" \
+          -v "$BUILD_DIR/volumes/activation-key:/activation-key:Z" \
+          -v "$BUILD_DIR/volumes/additional-secret:/additional-secret:Z" \
+          -v "$BUILD_DIR/volumes/trusted-ca:/mnt/trusted-ca:Z" \
+          -v "$BUILD_DIR/.docker/:/root/.docker:Z" \
+          -v "$BUILD_DIR/results/:/tekton/results:Z" \
+          -v $BUILD_DIR/scripts:/script:Z \
+          --user=0  --rm  "$BUILDER_IMAGE" /script/script-build.sh
+        rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
+        rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/
+        rsync -ra "$SSH_HOST:$BUILD_DIR/results/" "/tekton/results/"
+        buildah pull oci:rhtap-final-image
+      else
+        bash /script/script-build.sh
+      fi
       buildah images
       buildah tag localhost/rhtap-final-image "$IMAGE"
       container=$(buildah from --pull-never "$IMAGE")