test(e2e): add robot server setup

This commit adds setup code to initialize & use Robot servers in e2e
tests.

It uses installimage with autosetup to prepare the server and then uses k3sup similar to our existing setup to join the server into the existing kubernetes cluster.

Right now it only supports joining Robot servers as worker nodes, not for the control-plane.

Also, it only supports non-Private Network setups.

.github/workflows/test_e2e.yml

@@ -1,11 +1,11 @@
-name: Run e2e tests
+name: e2e tests
 on:
   pull_request: {}
   push:
     branches: [main]
 jobs:
-  k3s:
-    name: k3s ${{ matrix.k3s }}
+  cloud:
+    name: Cloud ${{ matrix.k3s }}
     permissions:
       id-token: write
     runs-on: ubuntu-latest
@@ -49,4 +49,64 @@ jobs:
           skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}"
           tag=$(skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}" --quiet --output="{{ (index .Builds 0).Tag }}")
           skaffold deploy --images=hetznercloud/hcloud-cloud-controller-manager=$tag
+          
+          go test ./tests/e2e -tags e2e -v -timeout 60m
+
+  robot:
+    name: Robot
+    permissions:
+      id-token: write
+
+    # Make sure that only one Job is using the server at a time
+    concurrency: robot-test-server
+    environment: e2e-robot
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+      - uses: actions/checkout@master
+      - uses: hetznercloud/tps-action@main
+        with:
+          token: ${{ secrets.HCLOUD_TOKEN  }}
+      - uses: 3bit/setup-hcloud@v2
+      - uses: yokawasa/[email protected]
+        with:
+          setup-tools: |
+            helm
+            kubectl
+            skaffold
+          helm: v3.11.2
+          kubectl: v1.28.1
+          skaffold: v2.3.0
+
+      - name: Run tests
+        env:
+          K3S_CHANNEL: v1.28
+          SCOPE: gha-${{ github.run_id }}-${{ github.run_attempt }}-robot
+
+          # Disable routing in dev-env, not supported for Robot.
+          ROUTING_ENABLED: "false"
+
+          ROBOT_USER_NAME: ${{ secrets.ROBOT_USER_NAME }}
+          ROBOT_PASSWORD: ${{ secrets.ROBOT_PASSWORD }}
+          SERVER_NUMBER: ${{ vars.SERVER_NUMBER }}
+        run: |
+          curl -sLS https://get.k3sup.dev | sh
+
+          trap "hack/dev-down.sh" EXIT
+          source <(hack/dev-up.sh)
+
+          skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}"
+          tag=$(skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}" --quiet --output="{{ (index .Builds 0).Tag }}")
+          skaffold deploy \
+            --profile=robot \
+            --images=hetznercloud/hcloud-cloud-controller-manager=$tag
+          
+          pushd hack/robot-e2e
+          ansible-galaxy install -r requirements.yml
+          ansible-playbook e2e-setup-robot-server.yml -e scope=$SCOPE -e server_number=$SERVER_NUMBER -vvv
+          popd
+          
           go test ./tests/e2e -tags e2e -v -timeout 60m

README.md

@@ -254,6 +254,15 @@ alias kgp="kubectl get pods"
 alias kgs="kubectl get services"
 ```
 
+The test suite is split in three parts:
+
+- **General Part**: Sets up the test env & checks if the HCCM Pod is properly running
+   - Build Tag: `e2e`
+- **Cloud Part**: Tests regular functionality against a Cloud-only environment
+   - Build Tag: `e2e && !robot`
+- **Robot Part**: Tests Robot functionality against a Cloud+Robot environment
+   - Build Tag: `e2e && robot`
+
 ## Local test setup
 This repository provides [skaffold](https://skaffold.dev/) to easily deploy / debug this controller on demand
 

hack/dev-up.sh

@@ -182,8 +182,10 @@ if [[ -n "${DEBUG:-}" ]]; then set -x; fi
   echo "Success - cluster fully initialized and ready, why not see for yourself?"
   echo '$ kubectl get nodes'
   kubectl get nodes
+  export CONTROL_IP=$(hcloud server ip "$scope_name-1")
 } >&2
 
 echo "export KUBECONFIG=$KUBECONFIG"
 $SCRIPT_DIR/registry-port-forward.sh
 echo "export SKAFFOLD_DEFAULT_REPO=localhost:30666"
+echo "export CONTROL_IP=$CONTROL_IP"

hack/robot-e2e/ansible.cfg

@@ -0,0 +1,7 @@
+[defaults]
+inventory = ${PWD}/inventory.yml
+forks = 10
+host_key_checking = False
+
+[ssh_connection]
+# pipelining = True

hack/robot-e2e/autosetup.j2

@@ -0,0 +1,6 @@
+HOSTNAME {{ server_name }}
+
+DRIVE1 /dev/sda
+PART / ext4 all
+
+IMAGE /root/.oldroot/nfs/images/Ubuntu-2204-jammy-amd64-base.tar.gz

hack/robot-e2e/e2e-setup-robot-server.yml

@@ -0,0 +1,96 @@
+---
+- name: Prepare Reinstall
+  hosts: localhost
+  connection: local
+  gather_facts: false
+
+  vars:
+    scope: dev
+    # Additional SSH keys to add to the server for debugging. Must already exist in Robot.
+    authorized_keys: []
+
+  module_defaults:
+    group/community.hrobot.robot:
+      hetzner_user: "{{ lookup('ansible.builtin.env', 'ROBOT_USER_NAME') }}"
+      hetzner_password: "{{ lookup('ansible.builtin.env', 'ROBOT_PASSWORD') }}"
+
+  tasks:
+    - name: Get Server Info
+      community.hrobot.server_info:
+        server_number: "{{ server_number }}"
+      register: server_info
+
+    - name: Set Server Facts
+      ansible.builtin.set_fact: server_ip="{{ server_info.servers[0].server_ip }}" server_name="{{ server_info.servers[0].server_name }}"
+
+    - name: Create SSH Key
+      community.hrobot.ssh_key:
+        name: "hccm-{{ scope }}"
+        public_key: "{{ lookup('file', '../.ssh-{{ scope }}.pub') }}"
+        state: present
+      register: ssh_key
+
+    - name: Enable Rescue System
+      community.hrobot.boot:
+        server_number: "{{ server_number }}"
+        rescue:
+          authorized_keys: "{{ authorized_keys + [ ssh_key.fingerprint ] }}"
+          os: linux
+
+    - name: Reset Server (to get to Rescue System)
+      community.hrobot.reset:
+        server_number: "{{ server_number }}"
+        reset_type: hardware # only type that does not require a separate reset for starting again
+
+    - name: Wait for SSH
+      ansible.builtin.wait_for:
+        host: "{{ server_ip }}"
+        port: "{{ 22 }}"
+        search_regex: SSH
+
+- name: Install OS to Server
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Write autosetup
+      ansible.builtin.template:
+        src: autosetup.j2
+        dest: /autosetup
+      vars:
+        server_name: "{{ hostvars['localhost']['server_name'] }}"
+
+    - name: installimage
+      # -t => Take over rescue system SSH public keys
+      ansible.builtin.command: /root/.oldroot/nfs/install/installimage -t yes
+
+    - name: Reboot
+      ansible.builtin.reboot:
+
+    - name: Create k3s directory
+      ansible.builtin.file:
+        path: /etc/rancher/k3s
+        state: directory
+
+    - name: Prepare Local Registry
+      ansible.builtin.copy:
+        src: ../k3s-registries.yaml
+        dest: /etc/rancher/k3s/registries.yaml
+
+- name: Join Kubernetes Cluster
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    control_ip: "{{ lookup('ansible.builtin.env', 'CONTROL_IP') }}"
+    k3s_channel: stable
+    scope: dev
+
+  tasks:
+    - name: k3sup
+      ansible.builtin.command: >-
+        k3sup join
+          --server-ip={{ control_ip | ansible.builtin.mandatory }}
+          --ip={{ server_ip }}
+          --k3s-channel={{ k3s_channel }}
+          --k3s-extra-args="--kubelet-arg cloud-provider=external --node-label instance.hetzner.cloud/is-root-server=true"
+          --ssh-key ../.ssh-{{ scope }}

hack/robot-e2e/inventory.yml

@@ -0,0 +1,8 @@
+all:
+  hosts:
+    # TODO: Dynamic inventory
+    hccm-test:
+      ansible_host: 142.132.203.104
+      ansible_user: root
+      ansible_ssh_private_key_file: ../.ssh-{{ scope }}
+      ansible_ssh_common_args: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'

hack/robot-e2e/requirements.yml

@@ -0,0 +1,2 @@
+collections:
+  - name: community.hrobot

tests/e2e/cloud_test.go

@@ -1,4 +1,4 @@
-//go:build e2e
+//go:build e2e && !robot
 
 package e2e
 

tests/e2e/robot_test.go

@@ -0,0 +1,89 @@
+//go:build e2e && robot
+
+package e2e
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/annotation"
+	"github.com/hetznercloud/hcloud-cloud-controller-manager/internal/providerid"
+)
+
+func TestNodeSetCorrectNodeLabelsAndIPAddressesRobot(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+
+	// Get a random Robot server from all Nodes in the cluster
+	nodes, err := testCluster.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{
+		LabelSelector: "instance.hetzner.cloud/is-root-server=true",
+	})
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(nodes.Items), 1)
+	node := nodes.Items[0]
+
+	// Parse the server number from the ProviderID
+	id, isCloudServer, err := providerid.ToServerID(node.Spec.ProviderID)
+	assert.NoError(t, err)
+	assert.False(t, isCloudServer)
+
+	// Get the server from the Robot API to cross-check Labels
+	server, err := testCluster.hrobot.ServerGet(int(id))
+	assert.NoError(t, err)
+
+	labels := node.Labels
+	expectedLabels := map[string]string{
+		"kubernetes.io/hostname": server.Name,
+		"kubernetes.io/os":       "linux",
+		"kubernetes.io/arch":     "amd64",
+	}
+	for expectedLabel, expectedValue := range expectedLabels {
+		assert.Equal(t, expectedValue, labels[expectedLabel], "node does not have expected label %s", expectedLabel)
+	}
+
+	expectedLabelsSet := []string{
+		"node.kubernetes.io/instance-type",
+		"topology.kubernetes.io/region",
+		"topology.kubernetes.io/zone",
+	}
+	for _, expectedLabel := range expectedLabelsSet {
+		_, ok := labels[expectedLabel]
+		assert.True(t, ok, "node is missing expected label %s", expectedLabel)
+	}
+
+	for _, address := range node.Status.Addresses {
+		if address.Type == corev1.NodeExternalIP {
+			expectedIP := server.ServerIP
+			assert.Equal(t, expectedIP, address.Address, "node has unexpected external ip")
+		}
+	}
+}
+
+func TestServiceLoadBalancersRobot(t *testing.T) {
+	t.Parallel()
+
+	lbTest := lbTestHelper{
+		t:         t,
+		K8sClient: testCluster.k8sClient,
+		podName:   "loadbalancer-robot-only",
+	}
+
+	pod := lbTest.DeployTestPod()
+
+	lbSvc := lbTest.ServiceDefinition(pod, map[string]string{
+		string(annotation.LBLocation): "nbg1",
+		// Only add the Robot server as a Load Balancer target
+		string(annotation.LBNodeSelector): "instance.hetzner.cloud/is-root-server=true",
+	})
+
+	lbSvc, err := lbTest.CreateService(lbSvc)
+	assert.NoError(t, err)
+
+	WaitForHTTPAvailable(t, lbSvc.Status.LoadBalancer.Ingress[0].IP, false)
+
+	lbTest.TearDown()
+}

tests/e2e/testing.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	hrobot "github.com/syself/hrobot-go"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,6 +36,7 @@ func init() {
 
 type TestCluster struct {
 	hcloud       *hcloud.Client
+	hrobot       hrobot.RobotClient
 	k8sClient    *kubernetes.Clientset
 	certificates []*hcloud.Certificate
 	scope        string
@@ -48,6 +50,7 @@ func (tc *TestCluster) Start() error {
 	}
 	tc.scope = scopeButcher.ReplaceAllString(tc.scope, "-")
 
+	// Hetzner Cloud Client
 	token := os.Getenv("HCLOUD_TOKEN")
 	if token == "" {
 		buf, err := os.ReadFile(fmt.Sprintf("../../hack/.token-%s", tc.scope))
@@ -68,6 +71,12 @@ func (tc *TestCluster) Start() error {
 	hcloudClient := hcloud.NewClient(opts...)
 	tc.hcloud = hcloudClient
 
+	// Hetzner Robot Client
+	if robotUser, set := os.LookupEnv("ROBOT_USER_NAME"); set {
+		robotPassword := os.Getenv("ROBOT_PASSWORD")
+		tc.hrobot = hrobot.NewBasicAuthClient(robotUser, robotPassword)
+	}
+
 	err := os.Setenv("KUBECONFIG", "../../hack/.kubeconfig-"+tc.scope)
 	if err != nil {
 		return err