run maestro with rosa

Signed-off-by: Wei Liu <[email protected]>
openshift-online · Nov 5, 2024 · 61cd18e · 61cd18e
1 parent 42899b6
commit 61cd18e
Show file tree

Hide file tree

Showing 9 changed files with 563 additions and 2 deletions.
diff --git a/.gitignore b/.gitignore
@@ -52,12 +52,12 @@ hack/mosquitto-passwd.txt
 vendor/
 
 # Ignore test data
+_output
 test/e2e/.kubeconfig
 test/e2e/.consumer_id
 test/e2e/.consumer_name
 test/e2e/.external_host_ip
 test/e2e/report/*
 unit-test-results.json
 integration-test-results.json
-
-test/e2e/setup/aro/aro-hcp
+test/e2e/setup/aro/aro-hcp
diff --git a/test/e2e/setup/rosa/Makefile b/test/e2e/setup/rosa/Makefile
@@ -0,0 +1,30 @@
+SHELL:=/bin/bash
+
+e2e_dir=$(shell cd ${PWD}/../.. && pwd -P)
+
+rosa/setup-maestro:
+	./setup/maestro.sh
+.PHONY: rosa/setup-maestro
+
+rosa/setup-agent:
+	./setup/agent.sh
+.PHONY: rosa/setup-agent
+
+rosa/setup-e2e:
+	./setup/e2e.sh
+.PHONY: rosa/setup-e2e
+
+rosa/e2e-test: rosa/setup-e2e
+	ginkgo -v --fail-fast --label-filter="!(e2e-tests-spec-resync-reconnect||e2e-tests-status-resync-reconnect)" \
+	--output-dir="$(e2e_dir)/report" --json-report=report.json --junit-report=report.xml \
+	${e2e_dir}/pkg -- \
+	-api-server="http://127.0.0.1:8000" \
+	-grpc-server="127.0.0.1:8090" \
+	-server-kubeconfig=$(KUBECONFIG) \
+	-agent-kubeconfig=$(KUBECONFIG) \
+	-consumer-name=${PWD}/_output/consumer_id
+.PHONY: rosa/e2e-test
+
+rosa/teardown:
+	./setup/teardown.sh
+.PHONY: rosa/teardown
diff --git a/test/e2e/setup/rosa/README.md b/test/e2e/setup/rosa/README.md
@@ -0,0 +1,91 @@
+# Setup
+
+## Setup Maestro server and agent
+
+This setup demonstrate how to deploy the Maestro with AWS IoT and AWS RDS PostgreSQL.
+
+1. Run following command to deploy a Maestro server on a given cluster
+
+```sh
+export AWS_IOT_REGION="<your_aws_iot_region>"
+export CLUSTER_REGION="<your_cluster_region>"
+export CLUSTER_VPC="<your_cluster_vpc>"
+export KUBECONFIG="<your_cluster_kubeconfig>"
+
+make rosa/setup-maestro
+```
+
+This will
+- Create AWS IoT client certs and policy for Maestro server in the given AWS IoT region
+- Create AWS RDS PostgreSQL for Maestro server in the given cluster region and vpc
+- Deploy the Maestro server on the given cluster
+
+After the Maestro server is deployed, you can run following commands to access the Maestro RESTful service and GRPC service
+
+```sh
+oc port-forward svc/maestro 8000 -n maestro
+oc port-forward svc/maestro-grpc 8090 -n maestro
+```
+
+Then you can create a consumer in the Maestro, e.g.
+
+```sh
+curl -s -X POST -H "Content-Type: application/json" http://127.0.0.1:8000/api/maestro/v1/consumers -d '{"name":"cluster1"}'
+```
+
+2. Run following command to deploy a Maestro agent on a given cluster
+
+```sh
+export AWS_IOT_REGION="<your_aws_iot_region>" # Note: this region should be same with the above aws iot region
+export CONSUMER_ID="<your_maestro_consumer_id_or_name>"
+export KUBECONFIG="<your_cluster_kubeconfig>"
+
+make rosa/setup-agent
+```
+
+This will
+- Create AWS IoT client certs and policy for Maestro agent in the given AWS IoT region
+- Deploy the Maestro agent on the given cluster
+
+## Run a Maestro e2e on a ROSA cluster
+
+### Prepare
+
+1. Install the following CLIs
+	- `oc`
+	- `rosa`
+	- `aws`
+	- `jq`
+	- the [`krelay` plugin](https://github.com/knight42/krelay)
+
+2. Create a rosa cluster, for example
+
+```sh
+rosa create cluster --cluster-name=maestro-rosa-e2e --region=us-west-2 --sts --mode=auto
+```
+
+### Run Maestro e2e
+
+```sh
+export KUBECONFIG="<your_rosa_cluster_kubeconfig>"
+export REGION="<your_rosa_cluster_region>"
+export CLUSTER_ID="<your_rosa_cluster_name_or_id>"
+
+make e2e-test/rosa
+```
+
+### Cleanup
+
+1. Run following commands to cleanup the AWS IoT and RDS resources
+
+```sh
+export REGION="<your_rosa_cluster_region>"
+
+make rosa/teardown
+```
+
+2. Delete the ROSA cluster, for example
+
+```sh
+rosa delete cluster --cluster=maestro-rosa-e2e 
+```
diff --git a/test/e2e/setup/rosa/setup/agent.sh b/test/e2e/setup/rosa/setup/agent.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+#####################
+# Setup Maestro agent
+#####################
+
+PWD="$(cd "$(dirname ${BASH_SOURCE[0]})" ; pwd -P)"
+ROOT_DIR="$(cd ${PWD}/.. && pwd -P)"
+
+region=${AWS_IOT_REGION:-""}
+consumer_id=${CONSUMER_ID:-""}
+
+if [ -z "$region" ]; then
+    echo "aws iot region is required"
+    exit 1
+fi
+
+if [ -z "$consumer_id" ]; then
+    echo "consumer id is required"
+    exit 1
+fi
+
+IMAGE_REGISTRY=${IMAGE_REGISTRY:="quay.io/redhat-user-workloads/maestro-rhtap-tenant/maestro"}
+IMAGE_REPOSITORY="maestro"
+IMAGE_TAG=${IMAGE_TAG:-"1de63c6075f2c95c9661d790d164019f60d789f3"}
+
+output_dir=${ROOT_DIR}/_output
+certs_dir=${output_dir}/aws-certs
+consumer_cert_dir=${certs_dir}/iot/consumers
+policies_dir=${output_dir}/aws-policies
+
+mkdir -p ${consumer_cert_dir}
+mkdir -p ${policies_dir}
+
+# Download AWS IoT broker severing CA
+echo "Download AWS IoT broker severing CA ...."
+curl -s -o ${certs_dir}/iot-ca.pem https://www.amazontrust.com/repository/AmazonRootCA1.pem
+
+# Generated client certs for AWS IoT clients
+echo "Generate AWS IoT client certs for Maestro agent ...."
+consumer_cert_arn=$(aws iot create-keys-and-certificate \
+    --region ${region} \
+    --set-as-active \
+    --certificate-pem-outfile "${consumer_cert_dir}/${consumer_id}.crt" \
+    --public-key-outfile "${consumer_cert_dir}/${consumer_id}.public.key" \
+    --private-key-outfile "${consumer_cert_dir}/${consumer_id}.private.key" | jq -r '.certificateArn')
+echo "Maestro agent AWS IoT client certs are generated ($consumer_cert_arn)"
+
+# Attach policies for AWS IoT clients
+aws_account=$(aws sts get-caller-identity --region ${region} --output json | jq -r '.Account')
+
+echo "Generate AWS IoT policy for Maestro agent ...."
+cat $PWD/aws-iot-policies/consumer.template.json | sed "s/{region}/${region}/g" | sed "s/{aws_account}/${aws_account}/g" | sed "s/{consumer_id}/${consumer_id}/g" > $policies_dir/${consumer_id}.json
+policy_name=$(aws iot create-policy \
+    --region ${region} \
+    --policy-name maestro-${consumer_id} \
+    --policy-document "file://${policies_dir}/${consumer_id}.json" | jq -r '.policyName')
+aws iot attach-policy --region ${region} --policy-name maestro-${consumer_id} --target ${consumer_cert_arn}
+echo "Maestro agent AWS IoT policy $policy_name is generated"
+
+# Get AWS IoT broker endpoint
+mqtt_host=$(aws iot describe-endpoint --region ${region} --endpoint-type iot:Data-ATS | jq -r '.endpointAddress')
+echo "AWS IoT broke: ${mqtt_host}:8883"
+
+sleep 30
+
+# Deploy Maestro agent
+oc create namespace maestro-agent || true
+oc -n maestro-agent delete secrets maestro-agent-mqtt-creds --ignore-not-found
+oc -n maestro-agent create secret generic maestro-agent-mqtt-creds \
+    --from-file=ca.crt="${certs_dir}/iot-ca.pem" \
+    --from-file=client.crt="${consumer_cert_dir}/${consumer_id}.crt" \
+    --from-file=client.key="${consumer_cert_dir}/${consumer_id}.private.key"
+
+oc process --filename="https://raw.githubusercontent.com/openshift-online/maestro/refs/heads/main/templates/agent-template-rosa.yml" \
+    --local="true" \
+    --param="AGENT_NAMESPACE=maestro-agent" \
+    --param="CONSUMER_NAME=${consumer_id}" \
+    --param="IMAGE_REGISTRY=${IMAGE_REGISTRY}" \
+    --param="IMAGE_REPOSITORY=${IMAGE_REPOSITORY}" \
+    --param="IMAGE_TAG=${IMAGE_TAG}" \
+    --param="MQTT_HOST=${mqtt_host}" > ${output_dir}/maestro-${consumer_id}-rosa.json
+
+oc apply -f ${output_dir}/maestro-${consumer_id}-rosa.json
+oc -n maestro-agent wait deploy/maestro-agent --for condition=Available=True --timeout=300s
diff --git a/test/e2e/setup/rosa/setup/aws-iot-policies/consumer.template.json b/test/e2e/setup/rosa/setup/aws-iot-policies/consumer.template.json
@@ -0,0 +1,41 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Connect"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:client/{consumer_id}-client"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Publish"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/{consumer_id}/agentevents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Subscribe"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topicfilter/sources/maestro/consumers/{consumer_id}/sourceevents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Receive"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/{consumer_id}/sourceevents"
+            ]
+        }
+    ]
+}
diff --git a/test/e2e/setup/rosa/setup/aws-iot-policies/source.template.json b/test/e2e/setup/rosa/setup/aws-iot-policies/source.template.json
@@ -0,0 +1,43 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Connect"
+            ],
+            "Resource": [
+                "*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Publish"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/*/sourceevents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Subscribe"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topicfilter/sources/maestro/consumers/+/agentevents",
+                "arn:aws:iot:{region}:{aws_account}:topicfilter/$share/statussubscribers/sources/maestro/consumers/+/agentevents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iot:Receive"
+            ],
+            "Resource": [
+                "arn:aws:iot:{region}:{aws_account}:topic/sources/maestro/consumers/*/agentevents",
+                "arn:aws:iot:{region}:{aws_account}:topic/$share/statussubscribers/sources/maestro/consumers/*/agentevents"
+            ]
+        }
+    ]
+}
diff --git a/test/e2e/setup/rosa/setup/e2e.sh b/test/e2e/setup/rosa/setup/e2e.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+#####################
+# Setup Maestro e2e
+#####################
+
+PWD="$(cd "$(dirname ${BASH_SOURCE[0]})" ; pwd -P)"
+ROSA_DIR="$(cd ${PWD}/.. && pwd -P)"
+
+output_dir=${ROSA_DIR}/_output
+
+mkdir -p $output_dir
+
+echo "$output_dir"
+
+# Find Maestro server vpc
+rosa_infra_id=$(rosa describe cluster --region=${REGION} --cluster=${CLUSTER_ID} -ojson | jq -r '.infra_id')
+vpc=$(aws ec2 describe-vpcs --region=${REGION} \
+    --filters Name=tag:Name,Values=${rosa_infra_id}-vpc | jq -r '.Vpcs[0].VpcId')
+
+# Setup Maestro server
+AWS_IOT_REGION=$REGION CLUSTER_REGION=$REGION CLUSTER_VPC=tt ${PWD}/maestro.sh
+sleep 90 # wait the maestro service ready
+
+# Start Maestro servers
+exec oc relay service/maestro 8000:8000 -n maestro > ${output_dir}/maestro.svc.log 2>&1 &
+maestro_server_pid=$!
+echo "Maestro server started: $maestro_server_pid"
+echo "$maestro_server_pid" > ${output_dir}/maestro_server.pid
+exec oc relay service/maestro-grpc 8090:8090 -n maestro > ${output_dir}/maestro-grpc.svc.log 2>&1 &
+maestro_grpc_server_pid=$!
+echo "Maestro GRPC server started: $maestro_grpc_server_pid"
+echo "$maestro_grpc_server_pid" > ${output_dir}/maestro_grpc_server.pid
+
+# need to wait the relay build the connection before we get the consumer id
+sleep 15
+
+# Prepare a consumer
+consumer_id=$(curl -s -X POST -H "Content-Type: application/json" http://127.0.0.1:8000/api/maestro/v1/consumers -d '{}' | jq -r '.id')
+echo $consumer_id > ${output_dir}/consumer_id
+echo "Consumer $consumer_id is created"
+
+# Setup Maestro agent
+oc apply -f https://raw.githubusercontent.com/open-cluster-management-io/api/release-0.14/work/v1/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml
+AWS_IOT_REGION=$REGION CONSUMER_ID=t ${PWD}/agent.sh