add functional test for instascale machineset test case in accordance…

… with using envtest control plane and external-cluster

.github/workflows/functional-test.yml

@@ -0,0 +1,61 @@
+# This workflow will build a golang project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
+
+name: Go
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: v1.21.5
+
+    - name: init directories
+      run: mkdir -p "$(pwd)/bin"
+
+    - name: Download kubebuilder 
+      run: |
+        version=1.0.8
+        arch=amd64
+        curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v${version}/kubebuilder_${version}_linux_${arch}.tar.gz"
+        tar -zxvf kubebuilder_${version}_linux_${arch}.tar.gz
+        mv kubebuilder_${version}_linux_${arch} kubebuilder
+        echo "kubebuilder_${version}_linux_${arch} renamed to kubebuilder"
+        sudo mv kubebuilder /usr/local/
+        echo "kubebuilder moved to /usr/local/ path"
+        DIRECTORY=/usr/local/kubebuilder
+        if [ -d "$DIRECTORY" ]; then
+          echo "$DIRECTORY does exist."
+        else
+          echo "$DIRECTORY does not exist."
+        fi
+        echo "PATH=$PATH:$DIRECTORY/bin" >> $GITHUB_ENV
+        export PATH=$PATH:$DIRECTORY/bin
+
+    - name: Set up Kubebuilder and setup-envtest
+      run: |
+        make envtest
+        echo "Kubebuilder-envtest : $(./bin/setup-envtest use 1.23 -p path)"
+        echo "KUBEBUILDER_ASSETS=$(./bin/setup-envtest use 1.23 -p path)" >> $GITHUB_ENV
+        echo "USE_EXISTING_CLUSTER=false" >> $GITHUB_ENV
+        export KUBEBUILDER_ASSETS="$(./bin/setup-envtest use 1.23 -p path)"
+        export USE_EXISTING_CLUSTER=false
+        
+    - name: Test
+      run: |
+        pwd
+        echo "KUBEBUILDER_ASSETS : $KUBEBUILDER_ASSETS"
+        echo "USE_EXISTING_CLUSTER : $USE_EXISTING_CLUSTER"
+
+        go test -v ./functional-tests

.github/workflows/go.yml

@@ -25,4 +25,4 @@ jobs:
       run: go build -o bin/manager main.go
 
     - name: Test
-      run: go test -v ./...
+      run: go test -v ./controllers

functional-tests/appwrapper_reconciler_test.go

@@ -0,0 +1,222 @@
+package functional_tests
+
+import (
+	"context"
+	"go/build"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	. "github.com/onsi/gomega"
+	gstruct "github.com/onsi/gomega/gstruct"
+	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
+	machinev1 "github.com/openshift/client-go/machine/clientset/versioned"
+	. "github.com/project-codeflare/codeflare-common/support"
+	"github.com/project-codeflare/instascale/controllers"
+	"github.com/project-codeflare/instascale/pkg/config"
+	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+	mc "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned"
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+)
+
+var (
+	cfg       *rest.Config
+	k8sClient client.Client // You'll be using this client in your tests.
+	testEnv   *envtest.Environment
+	ctx       context.Context
+	cancel    context.CancelFunc
+	err       error
+)
+
+func startEnvTest(t *testing.T) *envtest.Environment {
+	test := With(t)
+	//specify testEnv configuration
+	testEnv = &envtest.Environment{
+		CRDDirectoryPaths: []string{
+			filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "project-codeflare", "[email protected]", "config", "crd", "bases"),
+			filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "openshift", "[email protected]", "machine", "v1beta1"),
+		},
+		ErrorIfCRDPathMissing: true,
+	}
+	cfg, err = testEnv.Start()
+	test.Expect(err).NotTo(HaveOccurred())
+
+	return testEnv
+}
+
+func establishClient(t *testing.T) {
+	test := With(t)
+	err = mcadv1beta1.AddToScheme(scheme.Scheme)
+	err = machinev1beta1.AddToScheme(scheme.Scheme)
+	test.Expect(err).NotTo(HaveOccurred())
+
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	test.Expect(err).NotTo(HaveOccurred())
+	test.Expect(k8sClient).NotTo(BeNil())
+
+	k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{
+		Scheme: scheme.Scheme,
+	})
+	test.Expect(err).ToNot(HaveOccurred())
+
+	instaScaleController := &controllers.AppWrapperReconciler{
+		Client: k8sManager.GetClient(),
+		Scheme: k8sManager.GetScheme(),
+		Config: config.InstaScaleConfiguration{
+			MachineSetsStrategy: "reuse",
+			MaxScaleoutAllowed:  5,
+		},
+	}
+	err = instaScaleController.SetupWithManager(context.Background(), k8sManager)
+	test.Expect(err).ToNot(HaveOccurred())
+
+	go func() {
+		err = k8sManager.Start(ctrl.SetupSignalHandler())
+		test.Expect(err).ToNot(HaveOccurred())
+	}()
+
+}
+
+func teardownTestEnv(testEnv *envtest.Environment) {
+	logger := ctrl.LoggerFrom(ctx)
+	if err := testEnv.Stop(); err != nil {
+		logger.Error(err, "Error stopping test Environment\n")
+	}
+}
+
+func instascaleAppwrapper(namespace string) *mcadv1beta1.AppWrapper {
+	aw := &mcadv1beta1.AppWrapper{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-instascale",
+			Namespace: namespace,
+			Labels: map[string]string{
+				"orderedinstance": "test.instance1",
+			},
+			Finalizers: []string{"instascale.codeflare.dev/finalizer"},
+		},
+		Spec: mcadv1beta1.AppWrapperSpec{
+			AggrResources: mcadv1beta1.AppWrapperResourceList{
+				GenericItems: []mcadv1beta1.AppWrapperGenericResource{
+					{
+						DesiredAvailable: 1,
+						CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{
+							{
+								Replicas: 1,
+								Requests: apiv1.ResourceList{
+									apiv1.ResourceCPU:    resource.MustParse("250m"),
+									apiv1.ResourceMemory: resource.MustParse("512Mi"),
+								},
+								Limits: apiv1.ResourceList{
+									apiv1.ResourceCPU:    resource.MustParse("1"),
+									apiv1.ResourceMemory: resource.MustParse("1G"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	return aw
+}
+
+func TestReconciler(t *testing.T) {
+	testEnv = startEnvTest(t)      // initiates setting up the EnvTest environment
+	defer teardownTestEnv(testEnv) //defer teardown of test environement until function exists
+
+	establishClient(t) // setup client required for test
+
+	test := With(t)
+
+	//create new test namespace
+	namespace := test.NewTestNamespace()
+
+	// initializes an appwrapper for the created namespace
+	aw := instascaleAppwrapper(namespace.Name)
+
+	// create namespace using envtest client in accordance with created kubeconfig's namespace name
+	client, _ := kubernetes.NewForConfig(testEnv.Config)
+	client.CoreV1().Namespaces().Create(context.TODO(), &apiv1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: namespace.Name,
+		},
+		Spec: apiv1.NamespaceSpec{Finalizers: []apiv1.FinalizerName{apiv1.FinalizerName(aw.GetFinalizers()[0])}},
+	}, metav1.CreateOptions{})
+
+	//create client to interact with kubernetes machine API
+	machineClient, err := machinev1.NewForConfig(cfg)
+	test.Expect(err).ToNot(HaveOccurred())
+
+	//read machineset yaml from file `test_instascale_machineset.yml`
+	b, err := os.ReadFile("test_instascale_machineset.yml")
+
+	//deserialize kubernetes object
+	decode := scheme.Codecs.UniversalDeserializer().Decode
+	ms, _, err := decode(b, nil, nil) //decode machineset content of YAML file into kubernetes object
+	test.Expect(err).ToNot(HaveOccurred())
+	msa := ms.(*machinev1beta1.MachineSet) //asserts that decoded object is of type `*machinev1beta1.MachineSet`
+
+	//create machineset in default namespace
+	ms, err = machineClient.MachineV1beta1().MachineSets("default").Create(test.Ctx(), msa, metav1.CreateOptions{})
+	test.Expect(err).ToNot(HaveOccurred())
+
+	//assert that the replicas in Machineset specification is 0, before appwrapper creation
+	machineset, err := machineClient.MachineV1beta1().MachineSets("default").Get(test.Ctx(), "test-instascale", metav1.GetOptions{})
+	test.Expect(machineset.Spec.Replicas).To(gstruct.PointTo(Equal(int32(0))))
+
+	// create MCADclient for interacting with cluster API using cfg provided
+	mcadClient, err := mc.NewForConfig(cfg)
+	test.Expect(err).ToNot(HaveOccurred())
+
+	// create appwrapper resource using mcadClient
+	aw, err = mcadClient.WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
+	test.Expect(err).ToNot(HaveOccurred())
+	time.Sleep(10 * time.Second)
+
+	//update appwrapper status to avoid appwrapper dispatch in case of Insufficient resources
+	aw.Status = mcadv1beta1.AppWrapperStatus{
+		Pending: 1,
+		State:   mcadv1beta1.AppWrapperStateEnqueued,
+		Conditions: []mcadv1beta1.AppWrapperCondition{
+			{
+				Type:    mcadv1beta1.AppWrapperCondBackoff,
+				Status:  apiv1.ConditionTrue,
+				Reason:  "AppWrapperNotRunnable",
+				Message: "Insufficient resources to dispatch AppWrapper.",
+			},
+		},
+	}
+	_, err = mcadClient.WorkloadV1beta1().AppWrappers(namespace.Name).UpdateStatus(test.Ctx(), aw, metav1.UpdateOptions{})
+	test.Expect(err).ToNot(HaveOccurred())
+	time.Sleep(10 * time.Second)
+
+	// assert for machine replicas belonging to the machine set after appwrapper creation- there should be 1
+	machinesetafter, err := machineClient.MachineV1beta1().MachineSets("default").Get(test.Ctx(), "test-instascale", metav1.GetOptions{})
+	test.Expect(err).ToNot(HaveOccurred())
+	test.Expect(machinesetafter.Spec.Replicas).To(gstruct.PointTo(Equal(int32(1))))
+
+	// delete appwrapper
+	err = mcadClient.WorkloadV1beta1().AppWrappers(namespace.Name).Delete(test.Ctx(), aw.Name, metav1.DeleteOptions{})
+	test.Expect(err).ToNot(HaveOccurred())
+	time.Sleep(10 * time.Second)
+
+	// assert for machine belonging to the machine set after appwrapper deletion- there should be none
+	test.Expect(machineset.Spec.Replicas).To(gstruct.PointTo(Equal(int32(0))))
+
+	// delete machineset
+	err = machineClient.MachineV1beta1().MachineSets("default").Delete(test.Ctx(), "test-instascale", metav1.DeleteOptions{})
+	test.Expect(err).ToNot(HaveOccurred())
+
+	// delete test namespace
+	err = client.CoreV1().Namespaces().Delete(context.TODO(), namespace.Name, metav1.DeleteOptions{})
+	test.Expect(err).ToNot(HaveOccurred())
+}

functional-tests/test_instascale_machineset.yml

@@ -0,0 +1,63 @@
+apiVersion: machine.openshift.io/v1beta1
+kind: MachineSet
+metadata:
+  name: test-instascale
+  labels:
+    machine.openshift.io/cluster-api-cluster: fionawaters-v2pm4
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      machine.openshift.io/cluster-api-cluster: fionawaters-v2pm4
+      machine.openshift.io/cluster-api-machineset: test-instascale
+  template:
+    metadata:
+      labels:
+        machine.openshift.io/cluster-api-cluster: fionawaters-v2pm4
+        machine.openshift.io/cluster-api-machine-role: worker
+        machine.openshift.io/cluster-api-machine-type: worker
+        machine.openshift.io/cluster-api-machineset: test-instascale
+    spec:
+      lifecycleHooks: {}
+      metadata:
+        labels:
+          node-role.kubernetes.io/<role>: ''
+      providerSpec:
+        value:
+          userDataSecret:
+            name: worker-user-data
+          placement:
+            availabilityZone: us-east-1a
+            region: us-east-1
+          credentialsSecret:
+            name: aws-cloud-credentials
+          instanceType: test.instance1
+          metadata:
+            creationTimestamp: null
+          blockDevices:
+            - ebs:
+                iops: 0
+                kmsKey: {}
+                volumeSize: 120
+                volumeType: gp2
+          securityGroups:
+            - filters:
+                - name: 'tag:Name'
+                  values:
+                    - fionawaters-v2pm4-worker-sg
+          kind: AWSMachineProviderConfig
+          metadataServiceOptions: {}
+          tags:
+            - name: kubernetes.io/cluster/fionawaters-v2pm4
+              value: owned
+          deviceIndex: 0
+          ami:
+            id: ami-0624891c612b5eaa0
+          subnet:
+            filters:
+              - name: 'tag:Name'
+                values:
+                  - fionawaters-v2pm4-private-us-east-1a
+          apiVersion: awsproviderconfig.openshift.io/v1beta1
+          iamInstanceProfile:
+            id: fionawaters-v2pm4-worker-profile