Add auto discovery of trusted proxies in Kubernetes:

They will eliminate issues from users having to
determine the trusted proxies. This functionality can
be disabled if desired. By default we run auto discovery.

This will help simplify the deployment of Smee in the Helm
Chart.

Signed-off-by: Jacob Weinstock <[email protected]>

cmd/smee/backend.go

@@ -2,11 +2,15 @@ package main
 
 import (
 	"context"
+	"strings"
 
 	"github.com/go-logr/logr"
 	"github.com/tinkerbell/dhcp/backend/file"
 	"github.com/tinkerbell/dhcp/backend/kube"
 	"github.com/tinkerbell/dhcp/handler"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 )
@@ -27,7 +31,7 @@ type File struct {
 	Enabled  bool
 }
 
-func (k *Kube) Backend(ctx context.Context) (handler.BackendReader, error) {
+func (k *Kube) getClient() (*rest.Config, error) {
 	ccfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
 		&clientcmd.ClientConfigLoadingRules{
 			ExplicitPath: k.ConfigFilePath,
@@ -47,6 +51,15 @@ func (k *Kube) Backend(ctx context.Context) (handler.BackendReader, error) {
 		return nil, err
 	}
 
+	return config, nil
+}
+
+func (k *Kube) backend(ctx context.Context) (handler.BackendReader, error) {
+	config, err := k.getClient()
+	if err != nil {
+		return nil, err
+	}
+
 	kb, err := kube.NewBackend(config)
 	if err != nil {
 		return nil, err
@@ -62,7 +75,7 @@ func (k *Kube) Backend(ctx context.Context) (handler.BackendReader, error) {
 	return kb, nil
 }
 
-func (s *File) Backend(ctx context.Context, logger logr.Logger) (handler.BackendReader, error) {
+func (s *File) backend(ctx context.Context, logger logr.Logger) (handler.BackendReader, error) {
 	f, err := file.NewWatcher(logger, s.FilePath)
 	if err != nil {
 		return nil, err
@@ -72,3 +85,82 @@ func (s *File) Backend(ctx context.Context, logger logr.Logger) (handler.Backend
 
 	return f, nil
 }
+
+// discoverTrustedProxies will use the Kubernetes client to discover the CIDR Ranges for Pods in cluster.
+func (k *Kube) discoverTrustedProxies(ctx context.Context, l logr.Logger, trustedProxies []string) []string {
+	config, err := k.getClient()
+	if err != nil {
+		l.Error(err, "failed to get Kubernetes client config")
+		return nil
+	}
+	c, err := corev1client.NewForConfig(config)
+	if err != nil {
+		l.Error(err, "failed to create Kubernetes client")
+		return nil
+	}
+
+	return combinedCIDRs(ctx, l, c, trustedProxies)
+}
+
+// combinedCIDRs returns the CIDR Ranges for Pods in cluster. Not all Kubernetes distributions provide a way to discover the entire podCIDR.
+// Some distributions just provide the podCIDRs assigned to each node. combinedCIDRs tries all known locations where pod CIDRs might exist.
+// For example, if a cluster has 3 nodes, each with a /24 podCIDR, and the cluster has a /16 podCIDR, combinedCIDRs will return 4 CIDR ranges.
+func combinedCIDRs(ctx context.Context, l logr.Logger, c *corev1client.CoreV1Client, trustedProxies []string) []string {
+	if podCIDRS, err := perNodePodCIDRs(ctx, c); err == nil {
+		trustedProxies = append(trustedProxies, podCIDRS...)
+	} else {
+		l.V(1).Info("failed to get per node podCIDRs", "err", err)
+	}
+
+	if clusterCIDR, err := clusterPodCIDR(ctx, c); err == nil {
+		trustedProxies = append(trustedProxies, clusterCIDR...)
+	} else {
+		l.V(1).Info("failed to get cluster wide podCIDR", "err", err)
+	}
+
+	return trustedProxies
+}
+
+// perNodePodCIDRs returns the CIDR Range for Pods on each node. This is the per node podCIDR as compared to the total podCIDR.
+// This will get the podCIDR from each node in the cluster, not the entire cluster podCIDR. If a cluster grows after this is run,
+// the new nodes will not be included until this func is run again.
+// This should be used in conjunction with ClusterPodCIDR to be as complete and cross distribution compatible as possible.
+func perNodePodCIDRs(ctx context.Context, c *corev1client.CoreV1Client) ([]string, error) {
+	ns, err := c.Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	var trustedProxies []string
+	for _, n := range ns.Items {
+		trustedProxies = append(trustedProxies, n.Spec.PodCIDRs...)
+	}
+
+	return trustedProxies, nil
+}
+
+// clusterPodCIDR returns the CIDR Range for Pods in cluster. This is the total podCIDR as compared to the per node podCIDR.
+// Some Kubernetes distributions do not run a kube-controller-manager pod, so this func should be used in conjunction with PerNodePodCIDRs
+// to be as complete and cross distribution compatible as possible.
+func clusterPodCIDR(ctx context.Context, c *corev1client.CoreV1Client) ([]string, error) {
+	// https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/
+	pods, err := c.Pods("kube-system").List(ctx, metav1.ListOptions{
+		LabelSelector: "component=kube-controller-manager",
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var trustedProxies []string
+	for _, p := range pods.Items {
+		for _, c := range p.Spec.Containers {
+			for _, e := range c.Command {
+				if strings.HasPrefix(e, "--cluster-cidr") {
+					trustedProxies = append(trustedProxies, strings.Split(e, "=")[1])
+				}
+			}
+		}
+	}
+
+	return trustedProxies, nil
+}

cmd/smee/flag.go

@@ -102,6 +102,7 @@ func ipxeHTTPScriptFlags(c *config, fs *flag.FlagSet) {
 	fs.StringVar(&c.ipxeHTTPScript.bindAddr, "http-addr", detectPublicIPv4(":80"), "[http] local IP:Port to listen on for iPXE HTTP script requests")
 	fs.StringVar(&c.ipxeHTTPScript.extraKernelArgs, "extra-kernel-args", "", "[http] extra set of kernel args (k=v k=v) that are appended to the kernel cmdline iPXE script")
 	fs.StringVar(&c.ipxeHTTPScript.trustedProxies, "trusted-proxies", "", "[http] comma separated list of trusted proxies in CIDR notation")
+	fs.BoolVar(&c.ipxeHTTPScript.disableDiscoverTrustedProxies, "disable-discover-trusted-proxies", false, "[http] disable discovery of trusted proxies from Kubernetes, only available for the Kubernetes backend")
 	fs.StringVar(&c.ipxeHTTPScript.hookURL, "osie-url", "", "[http] URL where OSIE (HookOS) images are located")
 	fs.StringVar(&c.ipxeHTTPScript.tinkServer, "tink-server", "", "[http] IP:Port for the Tink server")
 	fs.BoolVar(&c.ipxeHTTPScript.tinkServerUseTLS, "tink-server-tls", false, "[http] use TLS for Tink server")

cmd/smee/flag_test.go

@@ -100,6 +100,7 @@ FLAGS
   -dhcp-ip-for-packet                 [dhcp] IP address to use in DHCP packets (opt 54, etc) (default "%[1]v")
   -dhcp-syslog-ip                     [dhcp] Syslog server IP address to use in DHCP packets (opt 7) (default "%[1]v")
   -dhcp-tftp-ip                       [dhcp] TFTP server IP address to use in DHCP packets (opt 66, etc) (default "%[1]v:69")
+  -disable-discover-trusted-proxies   [http] disable discovery of trusted proxies from Kubernetes, only available for the Kubernetes backend (default "false")
   -extra-kernel-args                  [http] extra set of kernel args (k=v k=v) that are appended to the kernel cmdline iPXE script
   -http-addr                          [http] local IP:Port to listen on for iPXE HTTP script requests (default "%[1]v:80")
   -http-ipxe-binary-enabled           [http] enable iPXE HTTP binary server (default "true")

cmd/smee/main.go

@@ -73,13 +73,14 @@ type ipxeHTTPBinary struct {
 }
 
 type ipxeHTTPScript struct {
-	enabled          bool
-	bindAddr         string
-	extraKernelArgs  string
-	hookURL          string
-	tinkServer       string
-	tinkServerUseTLS bool
-	trustedProxies   string
+	enabled                       bool
+	bindAddr                      string
+	extraKernelArgs               string
+	hookURL                       string
+	tinkServer                    string
+	tinkServerUseTLS              bool
+	trustedProxies                string
+	disableDiscoverTrustedProxies bool
 }
 
 type dhcpConfig struct {
@@ -179,13 +180,13 @@ func main() {
 		case cfg.backends.file.Enabled && cfg.backends.kubernetes.Enabled:
 			panic("only one backend can be enabled at a time")
 		case cfg.backends.file.Enabled:
-			b, err := cfg.backends.file.Backend(ctx, log)
+			b, err := cfg.backends.file.backend(ctx, log)
 			if err != nil {
 				panic(fmt.Errorf("failed to run file backend: %w", err))
 			}
 			br = b
 		default: // default backend is kubernetes
-			b, err := cfg.backends.kubernetes.Backend(ctx)
+			b, err := cfg.backends.kubernetes.backend(ctx)
 			if err != nil {
 				panic(fmt.Errorf("failed to run kubernetes backend: %w", err))
 			}
@@ -207,13 +208,17 @@ func main() {
 
 	if len(handlers) > 0 {
 		// start the http server for ipxe binaries and scripts
+		tp := parseTrustedProxies(cfg.ipxeHTTPScript.trustedProxies)
+		if cfg.backends.kubernetes.Enabled {
+			tp = cfg.backends.kubernetes.discoverTrustedProxies(ctx, log, tp)
+		}
 		httpServer := &http.Config{
 			GitRev:         GitRev,
 			StartTime:      startTime,
 			Logger:         log,
-			TrustedProxies: parseTrustedProxies(cfg.ipxeHTTPScript.trustedProxies),
+			TrustedProxies: tp,
 		}
-		log.Info("serving http", "addr", cfg.ipxeHTTPScript.bindAddr)
+		log.Info("serving http", "addr", cfg.ipxeHTTPScript.bindAddr, "trusted_proxies", tp)
 		g.Go(func() error {
 			return httpServer.ServeHTTP(ctx, cfg.ipxeHTTPScript.bindAddr, handlers)
 		})
@@ -302,13 +307,13 @@ func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*reservation
 	case c.backends.file.Enabled && c.backends.kubernetes.Enabled:
 		panic("only one backend can be enabled at a time")
 	case c.backends.file.Enabled:
-		b, err := c.backends.file.Backend(ctx, log)
+		b, err := c.backends.file.backend(ctx, log)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create file backend: %w", err)
 		}
 		dh.Backend = b
 	default: // default backend is kubernetes
-		b, err := c.backends.kubernetes.Backend(ctx)
+		b, err := c.backends.kubernetes.backend(ctx)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create kubernetes backend: %w", err)
 		}

go.mod

@@ -18,6 +18,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.19.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/sync v0.5.0
+	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4
 )
 
@@ -93,7 +94,6 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/api v0.28.4 // indirect
-	k8s.io/apimachinery v0.28.4 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230928205116-a78145627833 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect