feat: dataplane konnect extension reconciler

CHANGELOG.md

@@ -101,6 +101,9 @@
   [#669](https://github.com/Kong/gateway-operator/pull/669)
 - Allow setting `KonnectGatewayControlPlane`s group membership
   [#697](https://github.com/Kong/gateway-operator/pull/697)
+- Apply Konnect-related customizations to `DataPlane`s that properly reference `DataPlaneKonnectExtension`
+  resources.
+  [#714](https://github.com/Kong/gateway-operator/pull/714)
 
 ### Fixed
 

api/v1alpha1/dataplane_konnect_extension_types.go

@@ -23,6 +23,11 @@ func init() {
 	SchemeBuilder.Register(&DataPlaneKonnectExtension{}, &DataPlaneKonnectExtensionList{})
 }
 
+const (
+	// DataPlaneKonnectExtensionKind holds the kind for the DataPlaneKonnectExtension.
+	DataPlaneKonnectExtensionKind = "DataPlaneKonnectExtension"
+)
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:object:root=true
@@ -60,6 +65,7 @@ type DataPlaneKonnectExtensionList struct {
 type DataPlaneKonnectExtensionSpec struct {
 	// ControlPlaneRef is a reference to a ControlPlane this DataPlaneKonnectExtension is associated with.
 	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:XValidation:rule="self.type == 'konnectID'", message="Only konnectID type currently supported as controlPlaneRef."
 	ControlPlaneRef configurationv1alpha1.ControlPlaneRef `json:"controlPlaneRef"`
 
 	// ControlPlaneRegion is the region of the Konnect Control Plane.
@@ -94,9 +100,9 @@ type DataPlaneKonnectExtensionSpec struct {
 // KonnectControlPlaneAPIAuthConfiguration contains the configuration to authenticate with Konnect API ControlPlane.
 // +apireference:kgo:include
 type KonnectControlPlaneAPIAuthConfiguration struct {
-	// ClusterCertificateSecretName is a name of the Secret containing the Konnect Control Plane's cluster certificate.
+	// ClusterCertificateSecretRef is the reference to the Secret containing the Konnect Control Plane's cluster certificate.
 	// +kubebuilder:validation:Required
-	ClusterCertificateSecretName ClusterCertificateSecretRef `json:"clusterCertificateSecretRef"`
+	ClusterCertificateSecretRef ClusterCertificateSecretRef `json:"clusterCertificateSecretRef"`
 }
 
 // ClusterCertificateSecretRef contains the reference to the Secret containing the Konnect Control Plane's cluster certificate.

config/crd/bases/gateway-operator.konghq.com_dataplanekonnectextensions.yaml

@@ -94,6 +94,8 @@ spec:
                 - type
                 type: object
                 x-kubernetes-validations:
+                - message: Only konnectID type currently supported as controlPlaneRef.
+                  rule: self.type == 'konnectID'
                 - message: when type is konnectNamespacedRef, konnectNamespacedRef
                     must be set
                   rule: 'self.type == ''konnectNamespacedRef'' ? has(self.konnectNamespacedRef)
@@ -123,8 +125,8 @@ spec:
                   API authentication.
                 properties:
                   clusterCertificateSecretRef:
-                    description: ClusterCertificateSecretName is a name of the Secret
-                      containing the Konnect Control Plane's cluster certificate.
+                    description: ClusterCertificateSecretRef is the reference to the
+                      Secret containing the Konnect Control Plane's cluster certificate.
                     properties:
                       name:
                         description: Name is the name of the Secret containing the

config/rbac/role/role.yaml

@@ -240,6 +240,7 @@ rules:
   resources:
   - aigateways/status
   - controlplanes/status
+  - dataplanekonnectextensions/status
   - dataplanes/status
   - kongplugininstallations/status
   verbs:
@@ -249,20 +250,28 @@ rules:
 - apiGroups:
   - gateway-operator.konghq.com
   resources:
-  - gatewayconfigurations
+  - dataplanekonnectextensions
+  - kongplugininstallations
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - gateway-operator.konghq.com
   resources:
-  - kongplugininstallations
+  - dataplanekonnectextensions/finalizers
   verbs:
-  - get
-  - list
   - patch
   - update
+- apiGroups:
+  - gateway-operator.konghq.com
+  resources:
+  - gatewayconfigurations
+  verbs:
+  - get
+  - list
   - watch
 - apiGroups:
   - gateway.networking.k8s.io

controller/dataplane/controller.go

@@ -90,6 +90,19 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, markErr
 	}
 
+	log.Trace(logger, "applying extensions", dataplane)
+	patched, requeue, err := applyExtensions(ctx, r.Client, logger, dataplane)
+	if err != nil {
+		if !requeue {
+			log.Debug(logger, "failed to apply extensions", dataplane, "error:", err)
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, err
+	}
+	if patched {
+		return ctrl.Result{}, nil
+	}
+
 	log.Trace(logger, "exposing DataPlane deployment admin API via headless service", dataplane)
 	res, dataplaneAdminService, err := ensureAdminServiceForDataPlane(ctx, r.Client, dataplane,
 		client.MatchingLabels{

controller/dataplane/controller_rbac.go

@@ -7,6 +7,8 @@ package dataplane
 // +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanes,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanes/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanes/finalizers,verbs=update
+// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanekonnectextensions,verbs=get;list;watch;update;patch
+// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanekonnectextensions/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;get;list;watch;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get
 // +kubebuilder:rbac:groups=core,resources=services,verbs=create;get;list;watch;update;patch;delete

controller/dataplane/controller_utils.go

@@ -3,8 +3,10 @@ package dataplane
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/go-logr/logr"
 	appsv1 "k8s.io/api/apps/v1"
@@ -308,3 +310,47 @@ func isDeploymentReady(deploymentStatus appsv1.DeploymentStatus) (metav1.Conditi
 		return metav1.ConditionFalse, false
 	}
 }
+
+// -----------------------------------------------------------------------------
+// DataPlane - Private Functions - extensions
+// -----------------------------------------------------------------------------
+
+// applyExtensions patches the dataplane spec by taking into account customizations from the referenced extensions.
+// In case any extension is referenced, it adds a resolvedRefs condition to the dataplane, indicating the status of the
+// extension reference. it returns 3 values:
+//   - patched: a boolean indicating if the dataplane was patched. If the dataplane was patched, a reconciliation loop will be automatically re-triggered.
+//   - requeue: a boolean indicating if the dataplane should be requeued. If the error was unexpected (e.g., because of API server error), the dataplane should be requeued.
+//     In case the error is related to a misconfiguration, the dataplane does not need to be requeued, and feedback is provided into the dataplane status.
+//   - err: an error in case of failure.
+func applyExtensions(ctx context.Context, cl client.Client, logger logr.Logger, dataplane *operatorv1beta1.DataPlane) (patched bool, requeue bool, err error) {
+	if len(dataplane.Spec.Extensions) == 0 {
+		return false, false, nil
+	}
+	condition := k8sutils.NewConditionWithGeneration(consts.ResolvedRefsType, metav1.ConditionTrue, consts.ResolvedRefsReason, "", dataplane.GetGeneration())
+	err = applyDataPlaneKonnectExtension(ctx, cl, dataplane)
+	if err != nil {
+		switch {
+		case errors.Is(err, ErrCrossNamespaceReference):
+			condition.Status = metav1.ConditionFalse
+			condition.Reason = string(consts.RefNotPermittedReason)
+			condition.Message = strings.ReplaceAll(err.Error(), "\n", " - ")
+		case errors.Is(err, ErrKonnectExtensionNotFound):
+			condition.Status = metav1.ConditionFalse
+			condition.Reason = string(consts.InvalidExtensionRefReason)
+			condition.Message = strings.ReplaceAll(err.Error(), "\n", " - ")
+		case errors.Is(err, ErrClusterCertificateNotFound):
+			condition.Status = metav1.ConditionFalse
+			condition.Reason = string(consts.InvalidSecretRefReason)
+			condition.Message = strings.ReplaceAll(err.Error(), "\n", " - ")
+		default:
+			return patched, true, err
+		}
+	}
+	newDataPlane := dataplane.DeepCopy()
+	k8sutils.SetCondition(condition, newDataPlane)
+	patched, patchErr := patchDataPlaneStatus(ctx, cl, logger, newDataPlane)
+	if patchErr != nil {
+		return false, true, fmt.Errorf("failed patching status for DataPlane %s/%s: %w", dataplane.Namespace, dataplane.Name, patchErr)
+	}
+	return patched, false, err
+}

controller/dataplane/dataplanekonnectextension_controller.go

@@ -0,0 +1,119 @@
+package dataplane
+
+import (
+	"context"
+	"reflect"
+
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	operatorv1alpha1 "github.com/kong/gateway-operator/api/v1alpha1"
+	operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1"
+	"github.com/kong/gateway-operator/controller/pkg/ctxinjector"
+	"github.com/kong/gateway-operator/controller/pkg/log"
+	operatorerrors "github.com/kong/gateway-operator/internal/errors"
+	"github.com/kong/gateway-operator/internal/utils/index"
+	"github.com/kong/gateway-operator/pkg/consts"
+)
+
+// -----------------------------------------------------------------------------
+// DataKonnectExtensionReconciler
+// -----------------------------------------------------------------------------
+
+// DataPlaneKonnectExtensionReconciler reconciles a DataPlaneKonnectExtension object.
+type DataPlaneKonnectExtensionReconciler struct {
+	client.Client
+	ContextInjector ctxinjector.CtxInjector
+	// DevelopmentMode indicates if the controller should run in development mode,
+	// which causes it to e.g. perform less validations.
+	DevelopmentMode bool
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *DataPlaneKonnectExtensionReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&operatorv1alpha1.DataPlaneKonnectExtension{}).
+		Watches(&operatorv1beta1.DataPlane{}, handler.EnqueueRequestsFromMapFunc(r.listDataPlaneExtensionsReferenced)).
+		Complete(r)
+}
+
+// listDataPlaneExtensionsReferenced returns a list of all the DataPlaneKonnectExtensions referenced by the DataPlane object.
+// Maximum one reference is expected.
+func (r *DataPlaneKonnectExtensionReconciler) listDataPlaneExtensionsReferenced(ctx context.Context, obj client.Object) []reconcile.Request {
+	logger := ctrllog.FromContext(ctx)
+	dataPlane, ok := obj.(*operatorv1beta1.DataPlane)
+	if !ok {
+		logger.Error(
+			operatorerrors.ErrUnexpectedObject,
+			"failed to run map funcs",
+			"expected", "DataPlane", "found", reflect.TypeOf(obj),
+		)
+		return nil
+	}
+
+	if len(dataPlane.Spec.Extensions) == 0 {
+		return nil
+	}
+
+	recs := []reconcile.Request{}
+
+	for _, ext := range dataPlane.Spec.Extensions {
+		namespace := dataPlane.Namespace
+		if ext.Group != operatorv1alpha1.SchemeGroupVersion.Group ||
+			ext.Kind != operatorv1alpha1.DataPlaneKonnectExtensionKind {
+			continue
+		}
+		if ext.Namespace != nil && *ext.Namespace != namespace {
+			continue
+		}
+		recs = append(recs, reconcile.Request{
+			NamespacedName: client.ObjectKey{
+				Namespace: namespace,
+				Name:      ext.Name,
+			},
+		})
+	}
+	return recs
+}
+
+// Reconcile reconciles a DataPlaneKonnectExtension object.
+func (r *DataPlaneKonnectExtensionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	ctx = r.ContextInjector.InjectKeyValues(ctx)
+	var konnectExtension operatorv1alpha1.DataPlaneKonnectExtension
+	if err := r.Client.Get(ctx, req.NamespacedName, &konnectExtension); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	logger := log.GetLogger(ctx, operatorv1alpha1.DataPlaneKonnectExtensionKind, r.DevelopmentMode)
+	var dataPlaneList operatorv1beta1.DataPlaneList
+	if err := r.List(ctx, &dataPlaneList, client.MatchingFields{
+		index.DataPlaneKonnectExtensionIndex: konnectExtension.Namespace + "/" + konnectExtension.Name,
+	}); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	var updated bool
+	switch len(dataPlaneList.Items) {
+	case 0:
+		updated = controllerutil.RemoveFinalizer(&konnectExtension, consts.DataPlaneExtensionFinalizer)
+	default:
+		updated = controllerutil.AddFinalizer(&konnectExtension, consts.DataPlaneExtensionFinalizer)
+	}
+	if updated {
+		if err := r.Client.Update(ctx, &konnectExtension); err != nil {
+			if k8serrors.IsConflict(err) {
+				return ctrl.Result{Requeue: true}, nil
+			}
+			return ctrl.Result{}, err
+		}
+
+		log.Info(logger, "DataPlaneKonnectExtension finalizer updated", konnectExtension)
+	}
+
+	return ctrl.Result{}, nil
+}

controller/dataplane/dataplanekonnectextension_rbac.go

@@ -0,0 +1,9 @@
+package dataplane
+
+// -----------------------------------------------------------------------------
+// DataPlaneKonnectExtensionReconciler - RBAC
+// -----------------------------------------------------------------------------
+
+// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanes,verbs=get;list;watch
+// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanekonnectextensions,verbs=get;list;watch
+// +kubebuilder:rbac:groups=gateway-operator.konghq.com,resources=dataplanekonnectextensions/finalizers,verbs=update;patch