Skip to content

Commit

Permalink
Enable remote tasks to be run in cluster
Browse files Browse the repository at this point in the history 
By default, we should run builds matching the local architecture
in-cluster to reduce the overhead of provisioning platforms. This will
enable a fully matrixed build for all images using only the remote
builds. This change will require the multi-platform controller to set
the /ssh/host to localhost in order for the builds to run in-cluster.

Signed-off-by: arewm <[email protected]>
  • Loading branch information
@arewm
arewm committed Aug 13, 2024
1 parent 3984e5f commit 2234cd8
Show file tree
Hide file tree
Showing 5 changed files with 345 additions and 272 deletions.
91 changes: 52 additions & 39 deletions task-generator/remote/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,55 +142,65 @@ if [ -e "/ssh/error" ]; then
#no server could be provisioned
cat /ssh/error
exit 1
fi
export SSH_HOST=$(cat /ssh/host)
[ "$SSH_HOST" == "localhost" ] && IS_LOCALHOST=true
if [[ $IS_LOCALHOST ]]; then
echo "Localhost detected; running build in cluster"
elif [ -e "/ssh/otp" ]; then
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
else
cp /ssh/id_rsa ~/.ssh
fi
chmod 0400 ~/.ssh/id_rsa
export SSH_HOST=$(cat /ssh/host)
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
mkdir -p scripts
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
if ! [[ $IS_LOCALHOST ]]; then
chmod 0400 ~/.ssh/id_rsa
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
`
if taskVersion != "0.1" {
ret += adjustRemoteImage
}
env := "$PODMAN_PORT_FORWARD \\\n"

// disable podman subscription-manager integration
env += " --tmpfs /run/secrets \\\n"
env += " --tmpfs /run/secrets \\\n"

// Before the build we sync the contents of the workspace to the remote host
for _, workspace := range task.Spec.Workspaces {
ret += "\nrsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
ret += "\n rsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
}
// Also sync the volume mounts from the template
for _, volume := range task.Spec.StepTemplate.VolumeMounts {
ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
ret += "\n rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
}
for _, volume := range step.VolumeMounts {
if syncVolumes[volume.Name] {
ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
ret += "\n rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
}
}
ret += "\nrsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
podmanArgs += " -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
ret += "\nrsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/tekton-results/\""
podmanArgs += " -v \"$BUILD_DIR/tekton-results/:/tekton/results:Z\" \\\n"
ret += "\n rsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
podmanArgs += " -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
ret += "\n rsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/results/\""
podmanArgs += " -v \"$BUILD_DIR/results/:/tekton/results:Z\" \\\n"
ret += "\nfi\n"

if taskVersion != "0.1" {
ret += adjustRemoteImage
}

script := "scripts/script-" + step.Name + ".sh"

Expand All @@ -214,33 +224,36 @@ fi
ret += step.Script
ret += "\nbuildah push \"$IMAGE\" oci:rhtap-final-image"
ret += "\nREMOTESSHEOF"
ret += "\nchmod +x " + script
ret += "\nchmod +x " + script + "\n"

if task.Spec.StepTemplate != nil {
for _, e := range task.Spec.StepTemplate.Env {
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
}
}
ret += "\nrsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
ret += "\nif ! [[ $IS_LOCALHOST ]]; then"
ret += "\n rsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
containerScript := "/script/script-" + step.Name + ".sh"
for _, e := range step.Env {
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
}
podmanArgs += " -v $BUILD_DIR/scripts:/script:Z \\\n"
ret += "\nssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman run " + env + "" + podmanArgs + "--user=0 --rm \"$BUILDER_IMAGE\" " + containerScript
podmanArgs += " -v $BUILD_DIR/scripts:/script:Z \\\n"
ret += "\n ssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman run " + env + "" + podmanArgs + " --user=0 --rm \"$BUILDER_IMAGE\" " + containerScript

// Sync the contents of the workspaces back so subsequent tasks can use them
for _, workspace := range task.Spec.Workspaces {
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
}

for _, volume := range task.Spec.StepTemplate.VolumeMounts {
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
}
//sync back results
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/tekton-results/\" \"/tekton/results/\""
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/results/\" \"/tekton/results/\""

ret += "\nbuildah pull oci:rhtap-final-image"
ret += "\n buildah pull oci:rhtap-final-image"
ret += "\nelse\n bash " + containerScript
ret += "\nfi"
ret += "\nbuildah images"
ret += "\nbuildah tag localhost/rhtap-final-image \"$IMAGE\""
ret += "\ncontainer=$(buildah from --pull-never \"$IMAGE\")\nbuildah mount \"$container\" | tee /shared/container_path\necho $container > /shared/container_name"
Expand Down
131 changes: 73 additions & 58 deletions task/buildah-remote-oci-ta/0.1/buildah-remote-oci-ta.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,35 +220,45 @@ spec:
#no server could be provisioned
cat /ssh/error
exit 1
fi
export SSH_HOST=$(cat /ssh/host)
[ "$SSH_HOST" == "localhost" ] && IS_LOCALHOST=true
if [[ $IS_LOCALHOST ]]; then
echo "Localhost detected; running build in cluster"
elif [ -e "/ssh/otp" ]; then
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
else
cp /ssh/id_rsa ~/.ssh
fi
chmod 0400 ~/.ssh/id_rsa
export SSH_HOST=$(cat /ssh/host)
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
mkdir -p scripts
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
if ! [[ $IS_LOCALHOST ]]; then
chmod 0400 ~/.ssh/id_rsa
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
rsync -ra /shared/ "$SSH_HOST:$BUILD_DIR/volumes/shared/"
rsync -ra /var/workdir/ "$SSH_HOST:$BUILD_DIR/volumes/workdir/"
rsync -ra /entitlement/ "$SSH_HOST:$BUILD_DIR/volumes/etc-pki-entitlement/"
rsync -ra /activation-key/ "$SSH_HOST:$BUILD_DIR/volumes/activation-key/"
rsync -ra /additional-secret/ "$SSH_HOST:$BUILD_DIR/volumes/additional-secret/"
rsync -ra /mnt/trusted-ca/ "$SSH_HOST:$BUILD_DIR/volumes/trusted-ca/"
rsync -ra "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/"
rsync -ra "/tekton/results/" "$SSH_HOST:$BUILD_DIR/results/"
fi
rsync -ra /shared/ "$SSH_HOST:$BUILD_DIR/volumes/shared/"
rsync -ra /var/workdir/ "$SSH_HOST:$BUILD_DIR/volumes/workdir/"
rsync -ra /entitlement/ "$SSH_HOST:$BUILD_DIR/volumes/etc-pki-entitlement/"
rsync -ra /activation-key/ "$SSH_HOST:$BUILD_DIR/volumes/activation-key/"
rsync -ra /additional-secret/ "$SSH_HOST:$BUILD_DIR/volumes/additional-secret/"
rsync -ra /mnt/trusted-ca/ "$SSH_HOST:$BUILD_DIR/volumes/trusted-ca/"
rsync -ra "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/"
rsync -ra "/tekton/results/" "$SSH_HOST:$BUILD_DIR/tekton-results/"
cat >scripts/script-build.sh <<'REMOTESSHEOF'
#!/bin/bash
set -o verbose
Expand Down Expand Up @@ -431,42 +441,47 @@ spec:
buildah push "$IMAGE" oci:rhtap-final-image
REMOTESSHEOF
chmod +x scripts/script-build.sh
rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman run $PODMAN_PORT_FORWARD \
--tmpfs /run/secrets \
-e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
-e ADD_CAPABILITIES="$ADD_CAPABILITIES" \
-e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
-e BUILD_ARGS_FILE="$BUILD_ARGS_FILE" \
-e CONTEXT="$CONTEXT" \
-e DOCKERFILE="$DOCKERFILE" \
-e ENTITLEMENT_SECRET="$ENTITLEMENT_SECRET" \
-e HERMETIC="$HERMETIC" \
-e IMAGE="$IMAGE" \
-e IMAGE_EXPIRES_AFTER="$IMAGE_EXPIRES_AFTER" \
-e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
-e SQUASH="$SQUASH" \
-e STORAGE_DRIVER="$STORAGE_DRIVER" \
-e TARGET_STAGE="$TARGET_STAGE" \
-e TLSVERIFY="$TLSVERIFY" \
-e YUM_REPOS_D_FETCHED="$YUM_REPOS_D_FETCHED" \
-e YUM_REPOS_D_SRC="$YUM_REPOS_D_SRC" \
-e YUM_REPOS_D_TARGET="$YUM_REPOS_D_TARGET" \
-e COMMIT_SHA="$COMMIT_SHA" \
-v "$BUILD_DIR/volumes/shared:/shared:Z" \
-v "$BUILD_DIR/volumes/workdir:/var/workdir:Z" \
-v "$BUILD_DIR/volumes/etc-pki-entitlement:/entitlement:Z" \
-v "$BUILD_DIR/volumes/activation-key:/activation-key:Z" \
-v "$BUILD_DIR/volumes/additional-secret:/additional-secret:Z" \
-v "$BUILD_DIR/volumes/trusted-ca:/mnt/trusted-ca:Z" \
-v "$BUILD_DIR/.docker/:/root/.docker:Z" \
-v "$BUILD_DIR/tekton-results/:/tekton/results:Z" \
-v $BUILD_DIR/scripts:/script:Z \
--user=0 --rm "$BUILDER_IMAGE" /script/script-build.sh
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/
rsync -ra "$SSH_HOST:$BUILD_DIR/tekton-results/" "/tekton/results/"
buildah pull oci:rhtap-final-image
if ! [[ $IS_LOCALHOST ]]; then
rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman run $PODMAN_PORT_FORWARD \
--tmpfs /run/secrets \
-e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
-e ADD_CAPABILITIES="$ADD_CAPABILITIES" \
-e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
-e BUILD_ARGS_FILE="$BUILD_ARGS_FILE" \
-e CONTEXT="$CONTEXT" \
-e DOCKERFILE="$DOCKERFILE" \
-e ENTITLEMENT_SECRET="$ENTITLEMENT_SECRET" \
-e HERMETIC="$HERMETIC" \
-e IMAGE="$IMAGE" \
-e IMAGE_EXPIRES_AFTER="$IMAGE_EXPIRES_AFTER" \
-e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
-e SQUASH="$SQUASH" \
-e STORAGE_DRIVER="$STORAGE_DRIVER" \
-e TARGET_STAGE="$TARGET_STAGE" \
-e TLSVERIFY="$TLSVERIFY" \
-e YUM_REPOS_D_FETCHED="$YUM_REPOS_D_FETCHED" \
-e YUM_REPOS_D_SRC="$YUM_REPOS_D_SRC" \
-e YUM_REPOS_D_TARGET="$YUM_REPOS_D_TARGET" \
-e COMMIT_SHA="$COMMIT_SHA" \
-v "$BUILD_DIR/volumes/shared:/shared:Z" \
-v "$BUILD_DIR/volumes/workdir:/var/workdir:Z" \
-v "$BUILD_DIR/volumes/etc-pki-entitlement:/entitlement:Z" \
-v "$BUILD_DIR/volumes/activation-key:/activation-key:Z" \
-v "$BUILD_DIR/volumes/additional-secret:/additional-secret:Z" \
-v "$BUILD_DIR/volumes/trusted-ca:/mnt/trusted-ca:Z" \
-v "$BUILD_DIR/.docker/:/root/.docker:Z" \
-v "$BUILD_DIR/results/:/tekton/results:Z" \
-v $BUILD_DIR/scripts:/script:Z \
--user=0 --rm "$BUILDER_IMAGE" /script/script-build.sh
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/
rsync -ra "$SSH_HOST:$BUILD_DIR/results/" "/tekton/results/"
buildah pull oci:rhtap-final-image
else
bash /script/script-build.sh
fi
buildah images
buildah tag localhost/rhtap-final-image "$IMAGE"
container=$(buildah from --pull-never "$IMAGE")
Expand Down
Loading

0 comments on commit 2234cd8

Please sign in to comment.