From 467840acbf00d4bfcc87c0c8d37979e68355f6cf Mon Sep 17 00:00:00 2001 From: Afra Abnar <122131693+ti-afra@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:29:17 -0800 Subject: [PATCH] Add oidc certificate and configs for queryserver --- .../apiserver/apiserver_controller.go | 43 ++++++++++++++++++- .../apiserver/apiserver_controller_test.go | 7 +++ pkg/render/apiserver.go | 25 ++++++++--- .../common/networkpolicy/networkpolicy.go | 22 ++++++++++ pkg/render/compliance.go | 3 ++ 5 files changed, 94 insertions(+), 6 deletions(-) diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index a795b4ca87..fb39028dce 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -18,6 +18,7 @@ import ( "context" "fmt" + "github.com/tigera/operator/pkg/render/common/authentication" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -292,8 +293,10 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re var applicationLayer *operatorv1.ApplicationLayer var managementCluster *operatorv1.ManagementCluster var managementClusterConnection *operatorv1.ManagementClusterConnection + var keyValidatorConfig authentication.KeyValidatorConfig includeV3NetworkPolicy := false if installationSpec.Variant == operatorv1.TigeraSecureEnterprise { + trustedBundle = certificateManager.CreateTrustedBundle() applicationLayer, err = utils.GetApplicationLayer(ctx, r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ApplicationLayer", err, reqLogger) @@ -360,7 +363,44 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate", err, reqLogger) return reconcile.Result{}, err } else if prometheusCertificate != nil { - trustedBundle = certificateManager.CreateTrustedBundle(prometheusCertificate) + trustedBundle.AddCertificates(prometheusCertificate) + } + + var authenticationCR *operatorv1.Authentication + // Fetch the Authentication spec. If present, we use it to configure user authentication. + authenticationCR, err = utils.GetAuthentication(ctx, r.client) + if err != nil && !errors.IsNotFound(err) { + r.status.SetDegraded(operatorv1.ResourceReadError, "Error while fetching Authentication", err, reqLogger) + return reconcile.Result{}, err + } + + if authenticationCR != nil && authenticationCR.Status.State != operatorv1.TigeraStatusReady { + r.status.SetDegraded(operatorv1.ResourceNotReady, + fmt.Sprintf("Authentication is not ready authenticationCR status: %s", authenticationCR.Status.State), + nil, reqLogger) + return reconcile.Result{}, nil + } else if authenticationCR != nil && !utils.IsDexDisabled(authenticationCR) { + // Do not include DEX TLS Secret Name if authentication CR does not have type Dex + secret := render.DexTLSSecretName + certificate, err := certificateManager.GetCertificate(r.client, secret, common.OperatorNamespace()) + if err != nil { + r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve %s", secret), + err, reqLogger) + return reconcile.Result{}, err + } else if certificate == nil { + reqLogger.Info(fmt.Sprintf("Waiting for secret '%s' to become available", secret)) + r.status.SetDegraded(operatorv1.ResourceNotReady, + fmt.Sprintf("Waiting for secret '%s' to become available", secret), + nil, reqLogger) + return reconcile.Result{}, nil + } + trustedBundle.AddCertificates(certificate) + } + + keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.clusterDomain) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get KeyValidator Config", err, reqLogger) + return reconcile.Result{}, err } } @@ -395,6 +435,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re OpenShift: r.provider.IsOpenShift(), TrustedBundle: trustedBundle, MultiTenant: r.multiTenant, + KeyValidatorConfig: keyValidatorConfig, } component, err := render.APIServer(&apiServerCfg) diff --git a/pkg/controller/apiserver/apiserver_controller_test.go b/pkg/controller/apiserver/apiserver_controller_test.go index 79fe9b29bb..6c6a18d999 100644 --- a/pkg/controller/apiserver/apiserver_controller_test.go +++ b/pkg/controller/apiserver/apiserver_controller_test.go @@ -123,7 +123,11 @@ var _ = Describe("apiserver controller tests", func() { IssuerURL: "https://localhost:9443/dex", }, }, + Status: operatorv1.AuthenticationStatus{ + State: "Ready", + }, })).ToNot(HaveOccurred()) + dexSecret, err := secret.CreateTLSSecret(cryptoCA, render.DexTLSSecretName, common.OperatorNamespace(), corev1.TLSPrivateKeyKey, corev1.TLSCertKey, time.Hour, nil, dns.GetServiceDNSNames(render.DexTLSSecretName, render.DexNamespace, dns.DefaultClusterDomain)...) Expect(err).NotTo(HaveOccurred()) Expect(cli.Create(ctx, dexSecret)).ToNot(HaveOccurred()) @@ -149,6 +153,9 @@ var _ = Describe("apiserver controller tests", func() { mockStatus.On("RemoveCertificateSigningRequests", mock.Anything) mockStatus.On("ReadyToMonitor") mockStatus.On("SetMetaData", mock.Anything).Return() + mockStatus.On("SetDegraded", operatorv1.ResourceReadError, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + mockStatus.On("SetDegraded", operatorv1.ResourceNotReady, mock.Anything, mock.Anything, mock.Anything).Return().Maybe() + }) Context("verify reconciliation", func() { diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go index 12a1666b44..08c5ce0c01 100644 --- a/pkg/render/apiserver.go +++ b/pkg/render/apiserver.go @@ -18,6 +18,9 @@ import ( "fmt" "strings" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + "github.com/tigera/operator/pkg/render/common/authentication" admregv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -29,9 +32,6 @@ import ( apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "sigs.k8s.io/controller-runtime/pkg/client" - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" @@ -126,6 +126,7 @@ type APIServerConfiguration struct { OpenShift bool TrustedBundle certificatemanagement.TrustedBundle MultiTenant bool + KeyValidatorConfig authentication.KeyValidatorConfig } type apiServerComponent struct { @@ -497,11 +498,15 @@ func allowTigeraAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy { Destination: networkpolicy.PrometheusEntityRule, }, { - // Pass to subsequent tiers for further enforcement - Action: v3.Pass, + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: DexEntityRule, }, }...) + oidcEgressRule := networkpolicy.GetOIDCEgressRule(cfg.KeyValidatorConfig.Issuer()) + egressRules = append(egressRules, oidcEgressRule) + if r, err := cfg.K8SServiceEndpoint.DestinationEntityRule(); r != nil && err == nil { egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, @@ -510,6 +515,12 @@ func allowTigeraAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy { }) } + // add pass after all egress rules + egressRules = append(egressRules, v3.Rule{ + // Pass to subsequent tiers for further enforcement + Action: v3.Pass, + }) + // The ports Calico Enterprise API Server and Calico Enterprise Query Server are configured to listen on. ingressPorts := networkpolicy.Ports(443, APIServerPort, QueryServerPort, 10443) if cfg.IsSidecarInjectionEnabled() { @@ -1254,6 +1265,10 @@ func (c *apiServerComponent) queryServerContainer() corev1.Container { env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) } + if c.cfg.KeyValidatorConfig != nil { + env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) + } + volumeMounts := []corev1.VolumeMount{ c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), } diff --git a/pkg/render/common/networkpolicy/networkpolicy.go b/pkg/render/common/networkpolicy/networkpolicy.go index dd18a56245..5b89978aa6 100644 --- a/pkg/render/common/networkpolicy/networkpolicy.go +++ b/pkg/render/common/networkpolicy/networkpolicy.go @@ -16,6 +16,7 @@ package networkpolicy import ( "fmt" + "net/url" "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -92,6 +93,27 @@ func CreateSourceEntityRule(namespace string, deploymentName string) v3.EntityRu } } +// GetOIDCEgressRule creates egress rule for oidc connection. +// the result will include an egress rules with the urlString passed in: +// 1. egress rule: egress rule assuming the oidc is external to the cluster +func GetOIDCEgressRule(urlString string) v3.Rule { + parsedURL, err := url.Parse(urlString) + if err != nil { + panic(err) + } + + hostname := parsedURL.Hostname() + OIDCEntityRuleExternal := v3.EntityRule{ + Domains: []string{hostname}, + } + + return v3.Rule{ + Action: v3.Allow, + Protocol: &TCPProtocol, + Destination: OIDCEntityRuleExternal, + } +} + // AppendServiceSelectorDNSEgressRules is equivalent to AppendDNSEgressRules, utilizing service selector instead of label selector and ports. func AppendServiceSelectorDNSEgressRules(egressRules []v3.Rule, openShift bool) []v3.Rule { if openShift { diff --git a/pkg/render/compliance.go b/pkg/render/compliance.go index 2f7e38a4b9..97e40176d0 100644 --- a/pkg/render/compliance.go +++ b/pkg/render/compliance.go @@ -1680,6 +1680,9 @@ func (c *complianceComponent) complianceServerAllowTigeraNetworkPolicy() *v3.Net egressRules = networkpolicy.AppendDNSEgressRules(egressRules, c.cfg.OpenShift) + // add oidc egress rule + egressRules = append(egressRules, networkpolicy.GetOIDCEgressRule(c.cfg.KeyValidatorConfig.Issuer())) + egressRules = append(egressRules, []v3.Rule{ { Action: v3.Allow,