Merge pull request loft-sh#983 from ishankhare07/fix-prom-kubelets

reimplement fake kubelet services for node InternalIP addresses
infracloudio · Apr 3, 2023 · 74e1195 · 74e1195
2 parents d932d20 + 474e9e8
commit 74e1195
Show file tree

Hide file tree

Showing 22 changed files with 394 additions and 27 deletions.
diff --git a/charts/eks/templates/syncer-deployment.yaml b/charts/eks/templates/syncer-deployment.yaml
@@ -150,6 +150,9 @@ spec:
           {{- if .Values.sync.secrets.all }}
           - --sync-all-secrets=true
           {{- end }}
+          {{- if not .Values.sync.nodes.fakeKubeletIPs }}
+          - --fake-kubelet-ips=false
+          {{- end }}
           {{- range $f := .Values.syncer.extraArgs }}
           - {{ $f | quote }}
           {{- end }}

diff --git a/charts/eks/values.yaml b/charts/eks/values.yaml
@@ -54,6 +54,7 @@ sync:
   fake-persistentvolumes:
     enabled: true # will be ignored if persistentvolumes.enabled = true
   nodes:
+    fakeKubeletIPs: true
     enabled: false
     # If nodes sync is enabled, and syncAllNodes = true, the virtual cluster 
     # will sync all nodes instead of only the ones where some pods are running.

diff --git a/charts/k0s/templates/statefulset.yaml b/charts/k0s/templates/statefulset.yaml
@@ -203,6 +203,9 @@ spec:
           {{- if .Values.sync.secrets.all }}
           - --sync-all-secrets=true
           {{- end }}
+          {{- if not .Values.sync.nodes.fakeKubeletIPs }}
+          - --fake-kubelet-ips=false
+          {{- end }}
           {{- range $f := .Values.syncer.extraArgs }}
           - {{ $f | quote }}
           {{- end }}

diff --git a/charts/k0s/values.yaml b/charts/k0s/values.yaml
@@ -50,6 +50,7 @@ sync:
   fake-persistentvolumes:
     enabled: true # will be ignored if persistentvolumes.enabled = true
   nodes:
+    fakeKubeletIPs: true
     enabled: false
     # If nodes sync is enabled, and syncAllNodes = true, the virtual cluster 
     # will sync all nodes instead of only the ones where some pods are running.

diff --git a/charts/k3s/templates/statefulset.yaml b/charts/k3s/templates/statefulset.yaml
@@ -267,6 +267,9 @@ spec:
           {{- if .Values.sync.secrets.all }}
           - --sync-all-secrets=true
           {{- end }}
+          {{- if not .Values.sync.nodes.fakeKubeletIPs }}
+          - --fake-kubelet-ips=false
+          {{- end }}
           {{- range $f := .Values.syncer.extraArgs }}
           - {{ $f | quote }}
           {{- end }}

diff --git a/charts/k3s/values.yaml b/charts/k3s/values.yaml
@@ -55,6 +55,7 @@ sync:
   fake-persistentvolumes:
     enabled: true # will be ignored if persistentvolumes.enabled = true
   nodes:
+    fakeKubeletIPs: true
     enabled: false
     # If nodes sync is enabled, and syncAllNodes = true, the virtual cluster 
     # will sync all nodes instead of only the ones where some pods are running.

diff --git a/charts/k8s/templates/syncer-deployment.yaml b/charts/k8s/templates/syncer-deployment.yaml
@@ -185,6 +185,9 @@ spec:
           {{- if .Values.sync.secrets.all }}
           - --sync-all-secrets=true
           {{- end }}
+          {{- if not .Values.sync.nodes.fakeKubeletIPs }}
+          - --fake-kubelet-ips=false
+          {{- end }}
           {{- range $f := .Values.syncer.extraArgs }}
           - {{ $f | quote }}
           {{- end }}

diff --git a/charts/k8s/values.yaml b/charts/k8s/values.yaml
@@ -54,6 +54,7 @@ sync:
   fake-persistentvolumes:
     enabled: true # will be ignored if persistentvolumes.enabled = true
   nodes:
+    fakeKubeletIPs: true
     enabled: false
     # If nodes sync is enabled, and syncAllNodes = true, the virtual cluster 
     # will sync all nodes instead of only the ones where some pods are running.

diff --git a/cmd/vcluster/context/flag_options.go b/cmd/vcluster/context/flag_options.go
@@ -39,6 +39,7 @@ type VirtualClusterOptions struct {
 	SyncAllNodes        bool     `json:"syncAllNodes,omitempty"`
 	EnableScheduler     bool     `json:"enableScheduler,omitempty"`
 	DisableFakeKubelets bool     `json:"disableFakeKubelets,omitempty"`
+	FakeKubeletIPs      bool     `json:"fakeKubeletIPs,omitempty"`
 	ClearNodeImages     bool     `json:"clearNodeImages,omitempty"`
 	TranslateImages     []string `json:"translateImages,omitempty"`
 
@@ -121,6 +122,7 @@ func AddFlags(flags *pflag.FlagSet, options *VirtualClusterOptions) {
 	flags.BoolVar(&options.SyncAllNodes, "sync-all-nodes", false, "If enabled and --fake-nodes is false, the virtual cluster will sync all nodes instead of only the needed ones")
 	flags.BoolVar(&options.EnableScheduler, "enable-scheduler", false, "If enabled, will expect a scheduler running in the virtual cluster")
 	flags.BoolVar(&options.DisableFakeKubelets, "disable-fake-kubelets", false, "If disabled, the virtual cluster will not create fake kubelet endpoints to support metrics-servers")
+	flags.BoolVar(&options.FakeKubeletIPs, "fake-kubelet-ips", true, "If enabled, virtual cluster will assign fake ips of type NodeInternalIP to fake the kubelets")
 	flags.BoolVar(&options.ClearNodeImages, "node-clear-image-status", false, "If enabled, when syncing real nodes, the status.images data will be removed from the vcluster nodes")
 
 	flags.StringSliceVar(&options.TranslateImages, "translate-image", []string{}, "Translates image names from the virtual pod to the physical pod (e.g. coredns/coredns=mirror.io/coredns/coredns)")

diff --git a/pkg/constants/indices.go b/pkg/constants/indices.go
@@ -10,4 +10,6 @@ const (
 	IndexByConfigMap     = "IndexByConfigMap"
 	// IndexByHostName is used to map rewritten hostnames(advertised as node addresses) to nodenames
 	IndexByHostName = "IndexByHostName"
+
+	IndexByClusterIP = "IndexByClusterIP"
 )
diff --git a/pkg/controllers/resources/nodes/fake_syncer.go b/pkg/controllers/resources/nodes/fake_syncer.go
@@ -9,6 +9,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/api/equality"
 
+	"github.com/loft-sh/vcluster/pkg/controllers/resources/nodes/nodeservice"
 	podtranslate "github.com/loft-sh/vcluster/pkg/controllers/resources/pods/translate"
 	"github.com/loft-sh/vcluster/pkg/controllers/syncer"
 	synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context"
@@ -30,14 +31,18 @@ var (
 	FakeNodesVersion = "v1.19.1"
 )
 
-func NewFakeSyncer(ctx *synccontext.RegisterContext) (syncer.Object, error) {
+func NewFakeSyncer(ctx *synccontext.RegisterContext, nodeService nodeservice.NodeServiceProvider) (syncer.Object, error) {
 	return &fakeNodeSyncer{
-		currentNamespace: ctx.CurrentNamespace,
+		nodeServiceProvider: nodeService,
+		currentNamespace:    ctx.CurrentNamespace,
+		fakeKubeletIPs:      ctx.Options.FakeKubeletIPs,
 	}, nil
 }
 
 type fakeNodeSyncer struct {
-	currentNamespace string
+	nodeServiceProvider nodeservice.NodeServiceProvider
+	currentNamespace    string
+	fakeKubeletIPs      bool
 }
 
 func (r *fakeNodeSyncer) Resource() client.Object {
@@ -57,7 +62,7 @@ func (r *fakeNodeSyncer) RegisterIndices(ctx *synccontext.RegisterContext) error
 var _ syncer.ControllerModifier = &fakeNodeSyncer{}
 
 func (r *fakeNodeSyncer) ModifyController(ctx *synccontext.RegisterContext, builder *builder.Builder) (*builder.Builder, error) {
-	return modifyController(ctx, builder)
+	return modifyController(ctx, r.nodeServiceProvider, builder)
 }
 
 var _ syncer.FakeSyncer = &fakeNodeSyncer{}
@@ -71,7 +76,7 @@ func (r *fakeNodeSyncer) FakeSyncUp(ctx *synccontext.SyncContext, name types.Nam
 	}
 
 	ctx.Log.Infof("Create fake node %s", name.Name)
-	return ctrl.Result{}, CreateFakeNode(ctx.Context, r.currentNamespace, ctx.VirtualClient, name)
+	return ctrl.Result{}, CreateFakeNode(ctx.Context, r.fakeKubeletIPs, r.nodeServiceProvider, r.currentNamespace, ctx.VirtualClient, name)
 }
 
 func (r *fakeNodeSyncer) FakeSync(ctx *synccontext.SyncContext, vObj client.Object) (ctrl.Result, error) {
@@ -89,7 +94,7 @@ func (r *fakeNodeSyncer) FakeSync(ctx *synccontext.SyncContext, vObj client.Obje
 	}
 
 	// check if we need to update node ips
-	updated := r.updateIfNeeded(node)
+	updated := r.updateIfNeeded(ctx, node, types.NamespacedName{Name: node.Name, Namespace: r.currentNamespace})
 	if updated != nil {
 		ctx.Log.Infof("Update fake node %s", node.Name)
 		err := ctx.VirtualClient.Status().Update(ctx.Context, updated)
@@ -101,7 +106,7 @@ func (r *fakeNodeSyncer) FakeSync(ctx *synccontext.SyncContext, vObj client.Obje
 	return ctrl.Result{}, nil
 }
 
-func (r *fakeNodeSyncer) updateIfNeeded(node *corev1.Node) *corev1.Node {
+func (r *fakeNodeSyncer) updateIfNeeded(ctx *synccontext.SyncContext, node *corev1.Node, name types.NamespacedName) *corev1.Node {
 	var updated *corev1.Node
 
 	newAddresses := []corev1.NodeAddress{
@@ -110,6 +115,19 @@ func (r *fakeNodeSyncer) updateIfNeeded(node *corev1.Node) *corev1.Node {
 			Type:    corev1.NodeHostName,
 		},
 	}
+
+	if r.fakeKubeletIPs {
+		nodeIP, err := r.nodeServiceProvider.GetNodeIP(ctx.Context, name)
+		if err != nil {
+			ctx.Log.Errorf("error getting fake node ip: %v", err)
+		}
+
+		newAddresses = append(newAddresses, corev1.NodeAddress{
+			Address: nodeIP,
+			Type:    corev1.NodeInternalIP,
+		})
+	}
+
 	if !equality.Semantic.DeepEqual(node.Status.Addresses, newAddresses) {
 		updated = translator.NewIfNil(updated, node)
 		updated.Status.Addresses = newAddresses
@@ -127,7 +145,16 @@ func newGUID() string {
 	return random.RandomString(8) + "-" + random.RandomString(4) + "-" + random.RandomString(4) + "-" + random.RandomString(4) + "-" + random.RandomString(12)
 }
 
-func CreateFakeNode(ctx context.Context, currentNamespace string, virtualClient client.Client, name types.NamespacedName) error {
+func CreateFakeNode(ctx context.Context,
+	fakeKubeletIPs bool,
+	nodeServiceProvider nodeservice.NodeServiceProvider,
+	currentNamespace string,
+	virtualClient client.Client,
+	name types.NamespacedName) error {
+
+	nodeServiceProvider.Lock()
+	defer nodeServiceProvider.Unlock()
+
 	node := &corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name.Name,
@@ -228,6 +255,19 @@ func CreateFakeNode(ctx context.Context, currentNamespace string, virtualClient
 		},
 		Images: []corev1.ContainerImage{},
 	}
+
+	if fakeKubeletIPs {
+		nodeIP, err := nodeServiceProvider.GetNodeIP(ctx, name)
+		if err != nil {
+			return errors.Wrap(err, "create fake node ip")
+		}
+
+		node.Status.Addresses = append(node.Status.Addresses, corev1.NodeAddress{
+			Address: nodeIP,
+			Type:    corev1.NodeInternalIP,
+		})
+	}
+
 	err = virtualClient.Status().Patch(ctx, node, client.MergeFrom(orig))
 	if err != nil {
 		return err

diff --git a/pkg/controllers/resources/nodes/fake_syncer_test.go b/pkg/controllers/resources/nodes/fake_syncer_test.go
@@ -1,6 +1,7 @@
 package nodes
 
 import (
+	"context"
 	"testing"
 
 	"github.com/loft-sh/vcluster/pkg/controllers/syncer"
@@ -29,11 +30,20 @@ func newFakeFakeSyncer(t *testing.T, ctx *synccontext.RegisterContext) (*synccon
 	assert.NilError(t, err)
 
 	syncContext, object := generictesting.FakeStartSyncer(t, ctx, func(ctx *synccontext.RegisterContext) (syncer.Object, error) {
-		return NewFakeSyncer(ctx)
+		return NewFakeSyncer(ctx, &fakeNodeServiceProvider{})
 	})
 	return syncContext, object.(*fakeNodeSyncer)
 }
 
+type fakeNodeServiceProvider struct{}
+
+func (f *fakeNodeServiceProvider) Start(ctx context.Context) {}
+func (f *fakeNodeServiceProvider) Lock()                     {}
+func (f *fakeNodeServiceProvider) Unlock()                   {}
+func (f *fakeNodeServiceProvider) GetNodeIP(ctx context.Context, name types.NamespacedName) (string, error) {
+	return "127.0.0.1", nil
+}
+
 func TestFakeSync(t *testing.T) {
 	fakeGUID := newGUID()
 	now := metav1.Now()