From b24046279c04f3a8d52430cbfb3c6a5c7f81816b Mon Sep 17 00:00:00 2001 From: Guilherme Cassolato Date: Fri, 19 Jul 2024 17:33:01 +0200 Subject: [PATCH] example: kuadrant custom controller with support to multiple gateway api providers --- examples/kuadrant/Makefile | 12 +- examples/kuadrant/envoy-gateway.md | 2 +- examples/kuadrant/envoy_gateway.go | 176 +++++++++++ examples/kuadrant/istio.go | 287 ++++++++++++++++++ examples/kuadrant/main.go | 124 +++----- .../kuadrant/multiple-gateway-providers.md | 266 ++++++++++++++++ examples/kuadrant/reconciler.go | 138 +-------- go.mod | 3 + go.sum | 6 + 9 files changed, 815 insertions(+), 199 deletions(-) create mode 100644 examples/kuadrant/envoy_gateway.go create mode 100644 examples/kuadrant/istio.go create mode 100644 examples/kuadrant/multiple-gateway-providers.md diff --git a/examples/kuadrant/Makefile b/examples/kuadrant/Makefile index b02f943..185d937 100644 --- a/examples/kuadrant/Makefile +++ b/examples/kuadrant/Makefile @@ -83,8 +83,8 @@ install: manifests ## Install CRDs into a cluster. .PHONY: run run: generate ## Run the controller. -ifneq ($(PROVIDER),) - go run *.go --gateway-provider $(PROVIDER) +ifneq ($(PROVIDERS),) + go run *.go --gateway-providers $(PROVIDERS) else go run *.go endif @@ -96,6 +96,14 @@ install-envoy-gateway: helm ## Install Envoy Gateway. $(HELM) install eg oci://docker.io/envoyproxy/gateway-helm --version v1.0.2 -n envoy-gateway-system --create-namespace kubectl wait --timeout=5m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available +.PHONY: install-istio +install-istio: helm ## Install Istio. + $(HELM) repo add istio https://istio-release.storage.googleapis.com/charts + $(HELM) repo update + kubectl create namespace istio-system + $(HELM) install istio-base istio/base -n istio-system --set defaultRevision=default + $(HELM) install istiod istio/istiod -n istio-system --wait + .PHONY: install-kuadrant install-kuadrant: ## Install Kuadrant CRDs. kubectl apply -f config/crds diff --git a/examples/kuadrant/envoy-gateway.md b/examples/kuadrant/envoy-gateway.md index 31738f4..be0571e 100644 --- a/examples/kuadrant/envoy-gateway.md +++ b/examples/kuadrant/envoy-gateway.md @@ -43,7 +43,7 @@ make install-kuadrant Run the controller (holds the shell): ```sh -make run PROVIDER=envoygateway +make run PROVIDERS=envoygateway ``` ### Create the resources diff --git a/examples/kuadrant/envoy_gateway.go b/examples/kuadrant/envoy_gateway.go new file mode 100644 index 0000000..13ad08d --- /dev/null +++ b/examples/kuadrant/envoy_gateway.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "log" + + egv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/utils/ptr" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/kuadrant/policy-machinery/controller" + "github.com/kuadrant/policy-machinery/machinery" +) + +const envoyGatewayProvider = "envoygateway" + +var ( + _ GatewayProvider = &EnvoyGatewayProvider{} + + envoyGatewaySecurityPolicyKind = schema.GroupKind{Group: egv1alpha1.GroupName, Kind: "SecurityPolicy"} + envoyGatewaySecurityPoliciesResource = egv1alpha1.SchemeBuilder.GroupVersion.WithResource("securitypolicies") +) + +type EnvoyGatewayProvider struct { + *dynamic.DynamicClient +} + +func (p *EnvoyGatewayProvider) ReconcileGatewayCapabilities(topology *machinery.Topology, gateway machinery.Targetable, capabilities map[string][][]machinery.Targetable) { + // check if the gateway is managed by the envoy gateway controller + if !lo.ContainsBy(topology.Targetables().Parents(gateway), func(p machinery.Targetable) bool { + gc, ok := p.(*machinery.GatewayClass) + return ok && gc.Spec.ControllerName == "gateway.envoyproxy.io/gatewayclass-controller" + }) { + return + } + + // reconcile envoy gateway securitypolicy resources + if lo.ContainsBy(capabilities["auth"], func(path []machinery.Targetable) bool { + return lo.Contains(lo.Map(path, machinery.MapTargetableToURLFunc), gateway.GetURL()) + }) { + p.createSecurityPolicy(topology, gateway) + return + } + p.deleteSecurityPolicy(topology, gateway) +} + +func (p *EnvoyGatewayProvider) createSecurityPolicy(topology *machinery.Topology, gateway machinery.Targetable) { + desiredSecurityPolicy := &egv1alpha1.SecurityPolicy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: egv1alpha1.GroupVersion.String(), + Kind: envoyGatewaySecurityPolicyKind.Kind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: gateway.GetName(), + Namespace: gateway.GetNamespace(), + }, + Spec: egv1alpha1.SecurityPolicySpec{ + PolicyTargetReferences: egv1alpha1.PolicyTargetReferences{ + TargetRef: &gwapiv1alpha2.LocalPolicyTargetReferenceWithSectionName{ + LocalPolicyTargetReference: gwapiv1alpha2.LocalPolicyTargetReference{ + Group: gwapiv1alpha2.GroupName, + Kind: gwapiv1alpha2.Kind("Gateway"), + Name: gwapiv1.ObjectName(gateway.GetName()), + }, + }, + }, + ExtAuth: &egv1alpha1.ExtAuth{ + GRPC: &egv1alpha1.GRPCExtAuthService{ + BackendRef: &gwapiv1.BackendObjectReference{ + Name: gwapiv1.ObjectName("authorino-authorino-authorization"), + Namespace: ptr.To(gwapiv1.Namespace("kuadrant-system")), + Port: ptr.To(gwapiv1.PortNumber(50051)), + }, + }, + }, + }, + } + + resource := p.Resource(envoyGatewaySecurityPoliciesResource).Namespace(gateway.GetNamespace()) + + obj, found := lo.Find(topology.Objects().Children(gateway), func(o machinery.Object) bool { + return o.GroupVersionKind().GroupKind() == envoyGatewaySecurityPolicyKind && o.GetNamespace() == gateway.GetNamespace() && o.GetName() == gateway.GetName() + }) + + if !found { + o, _ := controller.Destruct(desiredSecurityPolicy) + _, err := resource.Create(context.TODO(), o, metav1.CreateOptions{}) + if err != nil { + log.Println("failed to create SecurityPolicy", err) + } + return + } + + securityPolicy := obj.(*controller.Object).RuntimeObject.(*egv1alpha1.SecurityPolicy) + + if securityPolicy.Spec.ExtAuth != nil && + securityPolicy.Spec.ExtAuth.GRPC != nil && + securityPolicy.Spec.ExtAuth.GRPC.BackendRef != nil && + securityPolicy.Spec.ExtAuth.GRPC.BackendRef.Namespace != nil && + *securityPolicy.Spec.ExtAuth.GRPC.BackendRef.Namespace == *desiredSecurityPolicy.Spec.ExtAuth.GRPC.BackendRef.Namespace && + securityPolicy.Spec.ExtAuth.GRPC.BackendRef.Name == desiredSecurityPolicy.Spec.ExtAuth.GRPC.BackendRef.Name && + securityPolicy.Spec.ExtAuth.GRPC.BackendRef.Port != nil && + *securityPolicy.Spec.ExtAuth.GRPC.BackendRef.Port == *desiredSecurityPolicy.Spec.ExtAuth.GRPC.BackendRef.Port { + return + } + + securityPolicy.Spec = desiredSecurityPolicy.Spec + o, _ := controller.Destruct(securityPolicy) + _, err := resource.Update(context.TODO(), o, metav1.UpdateOptions{}) + if err != nil { + log.Println("failed to update SecurityPolicy", err) + } +} + +func (p *EnvoyGatewayProvider) deleteSecurityPolicy(topology *machinery.Topology, gateway machinery.Targetable) { + _, found := lo.Find(topology.Objects().Children(gateway), func(o machinery.Object) bool { + return o.GroupVersionKind().GroupKind() == envoyGatewaySecurityPolicyKind && o.GetNamespace() == gateway.GetNamespace() && o.GetName() == gateway.GetName() + }) + + if !found { + return + } + + resource := p.Resource(envoyGatewaySecurityPoliciesResource).Namespace(gateway.GetNamespace()) + err := resource.Delete(context.TODO(), gateway.GetName(), metav1.DeleteOptions{}) + if err != nil { + log.Println("failed to delete SecurityPolicy", err) + } +} + +func linkGatewayToEnvoyGatewaySecurityPolicyFunc(objs controller.Store) machinery.LinkFunc { + gatewayKind := schema.GroupKind{Group: gwapiv1.GroupName, Kind: "Gateway"} + gateways := lo.FilterMap(lo.Values(objs[gatewayKind]), func(obj controller.RuntimeObject, _ int) (*gwapiv1.Gateway, bool) { + g, ok := obj.(*gwapiv1.Gateway) + if !ok { + return nil, false + } + return g, true + }) + + return machinery.LinkFunc{ + From: gatewayKind, + To: envoyGatewaySecurityPolicyKind, + Func: func(child machinery.Object) []machinery.Object { + o := child.(*controller.Object) + sp := o.RuntimeObject.(*egv1alpha1.SecurityPolicy) + refs := sp.Spec.PolicyTargetReferences.TargetRefs + if ref := sp.Spec.PolicyTargetReferences.TargetRef; ref != nil { + refs = append(refs, *ref) + } + refs = lo.Filter(refs, func(ref gwapiv1alpha2.LocalPolicyTargetReferenceWithSectionName, _ int) bool { + return ref.Group == gwapiv1.GroupName && ref.Kind == gwapiv1.Kind(gatewayKind.Kind) + }) + if len(refs) == 0 { + return nil + } + gateway, ok := lo.Find(gateways, func(g *gwapiv1.Gateway) bool { + if g.GetNamespace() != sp.GetNamespace() { + return false + } + return lo.ContainsBy(refs, func(ref gwapiv1alpha2.LocalPolicyTargetReferenceWithSectionName) bool { + return ref.Name == gwapiv1.ObjectName(g.GetName()) + }) + }) + if ok { + return []machinery.Object{&machinery.Gateway{Gateway: gateway}} + } + return nil + }, + } +} diff --git a/examples/kuadrant/istio.go b/examples/kuadrant/istio.go new file mode 100644 index 0000000..dbca58a --- /dev/null +++ b/examples/kuadrant/istio.go @@ -0,0 +1,287 @@ +package main + +import ( + "context" + "fmt" + "log" + "strings" + + "github.com/samber/lo" + istioapiv1 "istio.io/api/security/v1" + istiov1beta1 "istio.io/api/type/v1beta1" + istiov1 "istio.io/client-go/pkg/apis/security/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/utils/ptr" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/kuadrant/policy-machinery/controller" + "github.com/kuadrant/policy-machinery/machinery" +) + +const istioGatewayProvider = "istio" + +var ( + _ GatewayProvider = &IstioGatewayProvider{} + + istioAuthorizationPolicyKind = schema.GroupKind{Group: istiov1.GroupName, Kind: "AuthorizationPolicy"} + istioAuthorizationPoliciesResource = istiov1.SchemeGroupVersion.WithResource("authorizationpolicies") +) + +type IstioGatewayProvider struct { + *dynamic.DynamicClient +} + +func (p *IstioGatewayProvider) ReconcileGatewayCapabilities(topology *machinery.Topology, gateway machinery.Targetable, capabilities map[string][][]machinery.Targetable) { + // check if the gateway is managed by the istio gateway controller + if !lo.ContainsBy(topology.Targetables().Parents(gateway), func(p machinery.Targetable) bool { + gc, ok := p.(*machinery.GatewayClass) + return ok && gc.Spec.ControllerName == "istio.io/gateway-controller" + }) { + return + } + + // reconcile istio authorizationpolicy resources + paths := lo.Filter(capabilities["auth"], func(path []machinery.Targetable, _ int) bool { + return lo.Contains(lo.Map(path, machinery.MapTargetableToURLFunc), gateway.GetURL()) + }) + if len(paths) > 0 { + p.createAuthorizationPolicy(topology, gateway, paths) + return + } + p.deleteAuthorizationPolicy(topology, gateway) +} + +func (p *IstioGatewayProvider) createAuthorizationPolicy(topology *machinery.Topology, gateway machinery.Targetable, paths [][]machinery.Targetable) { + desiredAuthorizationPolicy := &istiov1.AuthorizationPolicy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: istiov1.SchemeGroupVersion.String(), + Kind: istioAuthorizationPolicyKind.Kind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: gateway.GetName(), + Namespace: gateway.GetNamespace(), + }, + Spec: istioapiv1.AuthorizationPolicy{ + TargetRef: &istiov1beta1.PolicyTargetReference{ + Group: gwapiv1alpha2.GroupName, + Kind: "Gateway", + Name: gateway.GetName(), + }, + Action: istioapiv1.AuthorizationPolicy_CUSTOM, + ActionDetail: &istioapiv1.AuthorizationPolicy_Provider{ + Provider: &istioapiv1.AuthorizationPolicy_ExtensionProvider{ + Name: "kuadrant-external-authorization", + }, + }, + }, + } + + for _, path := range paths { + if len(path) < 4 { + log.Printf("Unexpected topology path length to build Istio AuthorizationPolicy: %s\n", strings.Join(lo.Map(path, machinery.MapTargetableToURLFunc), " → ")) + continue + } + listener := path[1].(*machinery.Listener) + routeRule := path[3].(*machinery.HTTPRouteRule) + hostname := ptr.Deref(listener.Hostname, gwapiv1.Hostname("*")) + rules := istioAuthorizationPolicyRulesFromHTTPRouteRule(routeRule.HTTPRouteRule, []gwapiv1.Hostname{hostname}) + desiredAuthorizationPolicy.Spec.Rules = append(desiredAuthorizationPolicy.Spec.Rules, rules...) + } + + resource := p.Resource(istioAuthorizationPoliciesResource).Namespace(gateway.GetNamespace()) + + obj, found := lo.Find(topology.Objects().Children(gateway), func(o machinery.Object) bool { + return o.GroupVersionKind().GroupKind() == istioAuthorizationPolicyKind && o.GetNamespace() == gateway.GetNamespace() && o.GetName() == gateway.GetName() + }) + + if !found { + o, _ := controller.Destruct(desiredAuthorizationPolicy) + _, err := resource.Create(context.TODO(), o, metav1.CreateOptions{}) + if err != nil { + log.Println("failed to create AuthorizationPolicy", err) + } + return + } + + authorizationPolicy := obj.(*controller.Object).RuntimeObject.(*istiov1.AuthorizationPolicy) + + if authorizationPolicy.Spec.Action == desiredAuthorizationPolicy.Spec.Action && + authorizationPolicy.Spec.GetProvider() != nil && + authorizationPolicy.Spec.GetProvider().Name == desiredAuthorizationPolicy.Spec.GetProvider().Name && + len(authorizationPolicy.Spec.Rules) == len(desiredAuthorizationPolicy.Spec.Rules) && + lo.Every(authorizationPolicy.Spec.Rules, desiredAuthorizationPolicy.Spec.Rules) { + return + } + + authorizationPolicy.Spec.Action = desiredAuthorizationPolicy.Spec.Action + authorizationPolicy.Spec.ActionDetail = desiredAuthorizationPolicy.Spec.ActionDetail + authorizationPolicy.Spec.Rules = desiredAuthorizationPolicy.Spec.Rules + o, _ := controller.Destruct(authorizationPolicy) + _, err := resource.Update(context.TODO(), o, metav1.UpdateOptions{}) + if err != nil { + log.Println("failed to update AuthorizationPolicy", err) + } +} + +func (p *IstioGatewayProvider) deleteAuthorizationPolicy(topology *machinery.Topology, gateway machinery.Targetable) { + _, found := lo.Find(topology.Objects().Children(gateway), func(o machinery.Object) bool { + return o.GroupVersionKind().GroupKind() == istioAuthorizationPolicyKind && o.GetNamespace() == gateway.GetNamespace() && o.GetName() == gateway.GetName() + }) + + if !found { + return + } + + resource := p.Resource(istioAuthorizationPoliciesResource).Namespace(gateway.GetNamespace()) + err := resource.Delete(context.TODO(), gateway.GetName(), metav1.DeleteOptions{}) + if err != nil { + log.Println("failed to delete AuthorizationPolicy", err) + } +} + +func istioAuthorizationPolicyRulesFromHTTPRouteRule(rule *gwapiv1.HTTPRouteRule, hostnames []gwapiv1.Hostname) (istioRules []*istioapiv1.Rule) { + hosts := []string{} + for _, hostname := range hostnames { + if hostname == "*" { + continue + } + hosts = append(hosts, string(hostname)) + } + + // no http route matches → we only need one simple istio rule or even no rule at all + if len(rule.Matches) == 0 { + if len(hosts) == 0 { + return + } + istioRule := &istioapiv1.Rule{ + To: []*istioapiv1.Rule_To{ + { + Operation: &istioapiv1.Operation{ + Hosts: hosts, + }, + }, + }, + } + istioRules = append(istioRules, istioRule) + return + } + + // http route matches and possibly hostnames → we need one istio rule per http route match + for _, match := range rule.Matches { + istioRule := &istioapiv1.Rule{} + + var operation *istioapiv1.Operation + method := match.Method + path := match.Path + + if len(hosts) > 0 || method != nil || path != nil { + operation = &istioapiv1.Operation{} + } + + // hosts + if len(hosts) > 0 { + operation.Hosts = hosts + } + + // method + if method != nil { + operation.Methods = []string{string(*method)} + } + + // path + if path != nil { + operator := "*" // gateway api defaults to PathMatchPathPrefix + skip := false + if path.Type != nil { + switch *path.Type { + case gwapiv1.PathMatchExact: + operator = "" + case gwapiv1.PathMatchRegularExpression: + // ignore this rule as it is not supported by Istio - Authorino will check it anyway + skip = true + } + } + if !skip { + value := "/" + if path.Value != nil { + value = *path.Value + } + operation.Paths = []string{fmt.Sprintf("%s%s", value, operator)} + } + } + + if operation != nil { + istioRule.To = []*istioapiv1.Rule_To{ + {Operation: operation}, + } + } + + // headers + if len(match.Headers) > 0 { + istioRule.When = []*istioapiv1.Condition{} + + for idx := range match.Headers { + header := match.Headers[idx] + if header.Type != nil && *header.Type == gwapiv1.HeaderMatchRegularExpression { + // skip this rule as it is not supported by Istio - Authorino will check it anyway + continue + } + headerCondition := &istioapiv1.Condition{ + Key: fmt.Sprintf("request.headers[%s]", header.Name), + Values: []string{header.Value}, + } + istioRule.When = append(istioRule.When, headerCondition) + } + } + + // query params: istio does not support query params in authorization policies, so we build them in the authconfig instead + + istioRules = append(istioRules, istioRule) + } + return +} + +func linkGatewayToIstioAuthorizationPolicyFunc(objs controller.Store) machinery.LinkFunc { + gatewayKind := schema.GroupKind{Group: gwapiv1.GroupName, Kind: "Gateway"} + gateways := lo.FilterMap(lo.Values(objs[gatewayKind]), func(obj controller.RuntimeObject, _ int) (*gwapiv1.Gateway, bool) { + g, ok := obj.(*gwapiv1.Gateway) + if !ok { + return nil, false + } + return g, true + }) + + return machinery.LinkFunc{ + From: gatewayKind, + To: istioAuthorizationPolicyKind, + Func: func(child machinery.Object) []machinery.Object { + o := child.(*controller.Object) + ap := o.RuntimeObject.(*istiov1.AuthorizationPolicy) + refs := ap.Spec.TargetRefs + if ref := ap.Spec.TargetRef; ref != nil { + refs = append(refs, ref) + } + refs = lo.Filter(refs, func(ref *istiov1beta1.PolicyTargetReference, _ int) bool { + return ref.Group == gwapiv1.GroupName && ref.Kind == gatewayKind.Kind + }) + if len(refs) == 0 { + return nil + } + gateway, ok := lo.Find(gateways, func(g *gwapiv1.Gateway) bool { + if g.GetNamespace() != ap.GetNamespace() { + return false + } + return lo.ContainsBy(refs, func(ref *istiov1beta1.PolicyTargetReference) bool { + return ref.Name == g.GetName() + }) + }) + if ok { + return []machinery.Object{&machinery.Gateway{Gateway: gateway}} + } + return nil + }, + } +} diff --git a/examples/kuadrant/main.go b/examples/kuadrant/main.go index 3ddf404..9172365 100644 --- a/examples/kuadrant/main.go +++ b/examples/kuadrant/main.go @@ -7,37 +7,39 @@ import ( egv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/samber/lo" + istiov1 "istio.io/client-go/pkg/apis/security/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/clientcmd" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" - gwapiv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/kuadrant/policy-machinery/controller" - "github.com/kuadrant/policy-machinery/machinery" kuadrantv1alpha2 "github.com/kuadrant/policy-machinery/examples/kuadrant/apis/v1alpha2" kuadrantv1beta3 "github.com/kuadrant/policy-machinery/examples/kuadrant/apis/v1beta3" ) -const envoyGatewayProvider = "envoygateway" - -var ( - supportedGatewayProviders = []string{envoyGatewayProvider} - - securityPolicyKind = schema.GroupKind{Group: egv1alpha1.GroupName, Kind: "SecurityPolicy"} -) +var supportedGatewayProviders = []string{envoyGatewayProvider, istioGatewayProvider} func main() { - var gatewayProvider string + var gatewayProviders []string for i := range os.Args { switch os.Args[i] { - case "--gateway-provider": - if i == len(os.Args)-1 || !lo.Contains(supportedGatewayProviders, os.Args[i+1]) { - log.Fatalf("Invalid gateway provider. Use one of: %s\n", strings.Join(supportedGatewayProviders, ",")) + case "--gateway-providers": + { + defer func() { + if recover() != nil { + log.Fatalf("Invalid gateway provider. Supported: %s\n", strings.Join(supportedGatewayProviders, ",")) + } + }() + gatewayProviders = lo.Map(strings.Split(os.Args[i+1], ","), func(gp string, _ int) string { + return strings.TrimSpace(gp) + }) + if !lo.Every(supportedGatewayProviders, gatewayProviders) { + panic("") + } } - gatewayProvider = os.Args[i+1] } } @@ -68,80 +70,54 @@ func main() { schema.GroupKind{Group: kuadrantv1beta3.SchemeGroupVersion.Group, Kind: "AuthPolicy"}, schema.GroupKind{Group: kuadrantv1beta3.SchemeGroupVersion.Group, Kind: "RateLimitPolicy"}, ), - controller.WithCallback(buildReconcilerFor(gatewayProvider, client).Reconcile), + controller.WithCallback(buildReconciler(gatewayProviders, client).Reconcile), } - controllerOpts = append(controllerOpts, controllerOptionsFor(gatewayProvider)...) + controllerOpts = append(controllerOpts, controllerOptionsFor(gatewayProviders)...) controller.NewController(controllerOpts...).Start() } -func buildReconcilerFor(gatewayProvider string, client *dynamic.DynamicClient) *Reconciler { - var provider GatewayProvider +func buildReconciler(gatewayProviders []string, client *dynamic.DynamicClient) *Reconciler { + var providers []GatewayProvider - switch gatewayProvider { - case envoyGatewayProvider: - provider = &EnvoyGatewayProvider{client} - default: - provider = &DefaultGatewayProvider{} + for _, gatewayProvider := range gatewayProviders { + switch gatewayProvider { + case envoyGatewayProvider: + providers = append(providers, &EnvoyGatewayProvider{client}) + case istioGatewayProvider: + providers = append(providers, &IstioGatewayProvider{client}) + } + } + + if len(providers) == 0 { + providers = append(providers, &DefaultGatewayProvider{}) } return &Reconciler{ - GatewayProvider: provider, + GatewayProviders: providers, } } -func controllerOptionsFor(gatewayProvider string) []controller.ControllerOptionFunc { - switch gatewayProvider { - case envoyGatewayProvider: - return []controller.ControllerOptionFunc{ - controller.WithInformer("gatewayclass", controller.For[*gwapiv1.GatewayClass](gwapiv1.SchemeGroupVersion.WithResource("gatewayclasses"), metav1.NamespaceNone)), - controller.WithInformer("securitypolicy", controller.For[*egv1alpha1.SecurityPolicy](egv1alpha1.SchemeBuilder.GroupVersion.WithResource("securitypolicies"), metav1.NamespaceAll)), - controller.WithObjectKinds(securityPolicyKind), - controller.WithObjectLinks(linkGatewayToSecurityPolicyFunc), - } - default: - return nil +func controllerOptionsFor(gatewayProviders []string) []controller.ControllerOptionFunc { + var opts []controller.ControllerOptionFunc + + // if we care about specificities of gateway controllers, then let's add gateway classes to the topology too + if len(gatewayProviders) > 0 { + opts = append(opts, controller.WithInformer("gatewayclass", controller.For[*gwapiv1.GatewayClass](gwapiv1.SchemeGroupVersion.WithResource("gatewayclasses"), metav1.NamespaceNone))) } -} -func linkGatewayToSecurityPolicyFunc(objs controller.Store) machinery.LinkFunc { - gatewayKind := schema.GroupKind{Group: gwapiv1.GroupName, Kind: "Gateway"} - gateways := lo.FilterMap(lo.Values(objs[gatewayKind]), func(obj controller.RuntimeObject, _ int) (*gwapiv1.Gateway, bool) { - g, ok := obj.(*gwapiv1.Gateway) - if !ok { - return nil, false + for _, gatewayProvider := range gatewayProviders { + switch gatewayProvider { + case envoyGatewayProvider: + opts = append(opts, controller.WithInformer("envoygateway/securitypolicy", controller.For[*egv1alpha1.SecurityPolicy](envoyGatewaySecurityPoliciesResource, metav1.NamespaceAll))) + opts = append(opts, controller.WithObjectKinds(envoyGatewaySecurityPolicyKind)) + opts = append(opts, controller.WithObjectLinks(linkGatewayToEnvoyGatewaySecurityPolicyFunc)) + case istioGatewayProvider: + opts = append(opts, controller.WithInformer("istio/authorizationpolicy", controller.For[*istiov1.AuthorizationPolicy](istioAuthorizationPoliciesResource, metav1.NamespaceAll))) + opts = append(opts, controller.WithObjectKinds(istioAuthorizationPolicyKind)) + opts = append(opts, controller.WithObjectLinks(linkGatewayToIstioAuthorizationPolicyFunc)) } - return g, true - }) - - return machinery.LinkFunc{ - From: gatewayKind, - To: securityPolicyKind, - Func: func(child machinery.Object) []machinery.Object { - o := child.(*controller.Object) - sp := o.RuntimeObject.(*egv1alpha1.SecurityPolicy) - refs := sp.Spec.PolicyTargetReferences.TargetRefs - if ref := sp.Spec.PolicyTargetReferences.TargetRef; ref != nil { - refs = append(refs, *ref) - } - refs = lo.Filter(refs, func(ref gwapiv1alpha2.LocalPolicyTargetReferenceWithSectionName, _ int) bool { - return ref.Group == gwapiv1.GroupName && ref.Kind == gwapiv1.Kind(gatewayKind.Kind) - }) - if len(refs) == 0 { - return nil - } - gateway, ok := lo.Find(gateways, func(g *gwapiv1.Gateway) bool { - if g.GetNamespace() != sp.GetNamespace() { - return false - } - return lo.ContainsBy(refs, func(ref gwapiv1alpha2.LocalPolicyTargetReferenceWithSectionName) bool { - return ref.Name == gwapiv1.ObjectName(g.GetName()) - }) - }) - if ok { - return []machinery.Object{&machinery.Gateway{Gateway: gateway}} - } - return nil - }, } + + return opts } diff --git a/examples/kuadrant/multiple-gateway-providers.md b/examples/kuadrant/multiple-gateway-providers.md new file mode 100644 index 0000000..7779f85 --- /dev/null +++ b/examples/kuadrant/multiple-gateway-providers.md @@ -0,0 +1,266 @@ +# Kuadrant Controller with multiple gateway providers + +The example [custom controller](./README.md) working alongside with [Envoy Gateway](https://gateway.envoyproxy.io/) and [Istio](https://istio.io) gateway controllers. + +This example demonstrates how a controller can use the topology for reconciling other generic objects as well, along with targetables and policies. + +
+ +The controller watches for events related to: +- the 4 kinds of custom policies: DNSPolicy, TLSPolicy, AuthPolicy, and RateLimitPolicy; +- Gateway API resources: GatewayClass, Gateway, and HTTPRoute; +- Envoy Gateway resources: SecurityPolicy. +- Istio resources: AuthorizationPolicy. + +Apart from computing effective policies, the callback reconcile function also manages Envoy Gateway SecurityPolicy and Istio AuthorizationPolicy custom resources (create/update/delete) (used internally to implement the AuthPolicies.) + +## Demo + +### Requirements + +- [kubectl](https://kubernetes.io/docs/reference/kubectl/introduction/) +- [Kind](https://kind.sigs.k8s.io/) + +### Setup + +Create the cluster: + +```sh +kind create cluster +``` + +Install Envoy Gateway (installs Gateway API CRDs as well): + +```sh +make install-envoy-gateway +``` + +Install Istio: + +```sh +make install-istio +``` + +Install the CRDs: + +```sh +make install-kuadrant +``` + +Run the controller (holds the shell): + +```sh +make run PROVIDERS=envoygateway,istio +``` + +### Create the resources + +> **Note:** After each step below, check out the state of the topology (`topology.dot`). + +1. Create a Gateway managed by the Envoy Gateway gateway controller: + +```sh +kubectl apply -f -<= 8; h <= 17 } + strategy: merge +EOF +``` + +5. Try to delete the Envoy Gateway SecurityPolicy: + +```sh +kubectl delete securitypolicy/eg-gateway +``` + +6. Create a HTTPRoute-wide AuthPolicy to enforce API key authentication and affiliation to the 'admin' group: + +```sh +kubectl apply -f - < Listener policies @@ -76,10 +72,10 @@ func (r *Reconciler) Reconcile(eventType controller.EventType, oldObj, newObj co paths := targetables.Paths(gateway, listener) for i := range paths { if p := effectivePolicyForPath[*kuadrantv1alpha2.DNSPolicy](paths[i]); p != nil { - effectivePolicies["dns"] = append(effectivePolicies["dns"], *p) + // TODO: reconcile dns effective policy (i.e. create the DNSRecords for it) } if p := effectivePolicyForPath[*kuadrantv1alpha2.TLSPolicy](paths[i]); p != nil { - effectivePolicies["tls"] = append(effectivePolicies["tls"], *p) + // TODO: reconcile tls effective policy (i.e. create the certificate request for it) } } } @@ -89,15 +85,19 @@ func (r *Reconciler) Reconcile(eventType controller.EventType, oldObj, newObj co paths := targetables.Paths(gateway, httpRouteRule) for i := range paths { if p := effectivePolicyForPath[*kuadrantv1beta3.AuthPolicy](paths[i]); p != nil { - effectivePolicies["auth"] = append(effectivePolicies["auth"], *p) + capabilities["auth"] = append(capabilities["auth"], paths[i]) + // TODO: reconcile auth effective policy (i.e. create the Authorino AuthConfig) } if p := effectivePolicyForPath[*kuadrantv1beta3.RateLimitPolicy](paths[i]); p != nil { - effectivePolicies["ratelimit"] = append(effectivePolicies["ratelimit"], *p) + capabilities["ratelimit"] = append(capabilities["ratelimit"], paths[i]) + // TODO: reconcile rate-limit effective policy (i.e. create the Authorino AuthConfig) } } } - r.GatewayProvider.ReconcileGateway(topology, gateway, effectivePolicies) + for _, gatewayProvider := range r.GatewayProviders { + gatewayProvider.ReconcileGatewayCapabilities(topology, gateway, capabilities) + } } } @@ -150,111 +150,5 @@ var _ GatewayProvider = &DefaultGatewayProvider{} type DefaultGatewayProvider struct{} -func (p *DefaultGatewayProvider) ReconcileGateway(_ *machinery.Topology, _ machinery.Targetable, _ map[string][]machinery.Policy) { -} - -var _ GatewayProvider = &EnvoyGatewayProvider{} - -type EnvoyGatewayProvider struct { - *dynamic.DynamicClient -} - -func (p *EnvoyGatewayProvider) ReconcileGateway(topology *machinery.Topology, gateway machinery.Targetable, effectivePolicies map[string][]machinery.Policy) { - // check if the gateway is managed by the envoy gateway controller - if !lo.ContainsBy(topology.Targetables().Parents(gateway), func(p machinery.Targetable) bool { - gc, ok := p.(*machinery.GatewayClass) - return ok && gc.Spec.ControllerName == "gateway.envoyproxy.io/gatewayclass-controller" - }) { - return - } - - // reconcile envoy gateway securitypolicy resources - if len(effectivePolicies["auth"]) > 0 { - p.createSecurityPolicy(topology, gateway) - return - } - p.deleteSecurityPolicy(topology, gateway) -} - -func (p *EnvoyGatewayProvider) createSecurityPolicy(topology *machinery.Topology, gateway machinery.Targetable) { - resource := p.Resource(egv1alpha1.SchemeBuilder.GroupVersion.WithResource("securitypolicies")).Namespace(gateway.GetNamespace()) - - obj, found := lo.Find(topology.Objects().Children(gateway), func(o machinery.Object) bool { - return o.GroupVersionKind().GroupKind() == securityPolicyKind && o.GetNamespace() == gateway.GetNamespace() && o.GetName() == gateway.GetName() - }) - - desiredSecurityPolicy := &egv1alpha1.SecurityPolicy{ - TypeMeta: metav1.TypeMeta{ - APIVersion: egv1alpha1.GroupVersion.String(), - Kind: securityPolicyKind.Kind, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: gateway.GetName(), - Namespace: gateway.GetNamespace(), - }, - Spec: egv1alpha1.SecurityPolicySpec{ - PolicyTargetReferences: egv1alpha1.PolicyTargetReferences{ - TargetRef: &gwapiv1alpha2.LocalPolicyTargetReferenceWithSectionName{ - LocalPolicyTargetReference: gwapiv1alpha2.LocalPolicyTargetReference{ - Group: gwapiv1alpha2.GroupName, - Kind: gwapiv1alpha2.Kind("Gateway"), - Name: gwapiv1.ObjectName(gateway.GetName()), - }, - }, - }, - ExtAuth: &egv1alpha1.ExtAuth{ - GRPC: &egv1alpha1.GRPCExtAuthService{ - BackendRef: &gwapiv1.BackendObjectReference{ - Name: gwapiv1.ObjectName("authorino-authorino-authorization"), - Namespace: ptr.To(gwapiv1.Namespace("kuadrant-system")), - Port: ptr.To(gwapiv1.PortNumber(50051)), - }, - }, - }, - }, - } - - if !found { - o, _ := controller.Destruct(desiredSecurityPolicy) - _, err := resource.Create(context.TODO(), o, metav1.CreateOptions{}) - if err != nil { - log.Println("failed to create SecurityPolicy", err) - } - return - } - - securityPolicy := obj.(*controller.Object).RuntimeObject.(*egv1alpha1.SecurityPolicy) - - if securityPolicy.Spec.ExtAuth == nil || - securityPolicy.Spec.ExtAuth.GRPC == nil || - securityPolicy.Spec.ExtAuth.GRPC.BackendRef == nil || - securityPolicy.Spec.ExtAuth.GRPC.BackendRef.Namespace != desiredSecurityPolicy.Spec.ExtAuth.GRPC.BackendRef.Namespace || - securityPolicy.Spec.ExtAuth.GRPC.BackendRef.Name != desiredSecurityPolicy.Spec.ExtAuth.GRPC.BackendRef.Name || - securityPolicy.Spec.ExtAuth.GRPC.BackendRef.Port != desiredSecurityPolicy.Spec.ExtAuth.GRPC.BackendRef.Port { - return - } - - securityPolicy.Spec = desiredSecurityPolicy.Spec - o, _ := controller.Destruct(securityPolicy) - _, err := resource.Update(context.TODO(), o, metav1.UpdateOptions{}) - if err != nil { - log.Println("failed to update SecurityPolicy", err) - } -} - -func (p *EnvoyGatewayProvider) deleteSecurityPolicy(topology *machinery.Topology, gateway machinery.Targetable) { - resource := p.Resource(egv1alpha1.SchemeBuilder.GroupVersion.WithResource("securitypolicies")).Namespace(gateway.GetNamespace()) - - _, found := lo.Find(topology.Objects().Children(gateway), func(o machinery.Object) bool { - return o.GroupVersionKind().GroupKind() == securityPolicyKind && o.GetNamespace() == gateway.GetNamespace() && o.GetName() == gateway.GetName() - }) - - if !found { - return - } - - err := resource.Delete(context.TODO(), gateway.GetName(), metav1.DeleteOptions{}) - if err != nil { - log.Println("failed to delete SecurityPolicy", err) - } +func (p *DefaultGatewayProvider) ReconcileGatewayCapabilities(_ *machinery.Topology, _ machinery.Targetable, _ map[string][][]machinery.Targetable) { } diff --git a/go.mod b/go.mod index b53331f..9e9bd9a 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( github.com/google/go-cmp v0.6.0 github.com/kuadrant/authorino v0.17.2 github.com/samber/lo v1.39.0 + istio.io/api v1.22.3 + istio.io/client-go v1.22.3 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 @@ -61,6 +63,7 @@ require ( golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index e9dcba4..e12d3b6 100644 --- a/go.sum +++ b/go.sum @@ -168,6 +168,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -182,6 +184,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +istio.io/api v1.22.3 h1:V59wgcCm2fK2r137QBsddCDHNg0efg/DauIWEB9DFz8= +istio.io/api v1.22.3/go.mod h1:S3l8LWqNYS9yT+d4bH+jqzH2lMencPkW7SKM1Cu9EyM= +istio.io/client-go v1.22.3 h1:4WocGQYVTASpfn7tj1yGE8f0sgxzbxOkg56HX1LJQ5U= +istio.io/client-go v1.22.3/go.mod h1:D/vNne1n5586423NgGXMnPgshE/99mQgnjnxK/Vw2yM= k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= k8s.io/apiextensions-apiserver v0.30.2 h1:l7Eue2t6QiLHErfn2vwK4KgF4NeDgjQkCXtEbOocKIE=