From 10ab62d9cabd0782ba2a8f0c8009de0a22c76ddf Mon Sep 17 00:00:00 2001 From: Alexander Hebel Date: Fri, 27 Dec 2024 13:16:40 +0100 Subject: [PATCH] Fix reconciliation of user managed public IPs in flow --- pkg/azure/client/interface.go | 5 +++ pkg/azure/client/mock/mocks.go | 14 ++++++++ pkg/azure/client/publicip.go | 9 +++++ pkg/azure/client/types.go | 1 + .../infrastructure/infraflow/const.go | 5 +++ .../infrastructure/infraflow/ensurer.go | 35 ++++++++++++++++++- 6 files changed, 68 insertions(+), 1 deletion(-) diff --git a/pkg/azure/client/interface.go b/pkg/azure/client/interface.go index b76557b5c..e8ce45e69 100644 --- a/pkg/azure/client/interface.go +++ b/pkg/azure/client/interface.go @@ -82,3 +82,8 @@ type ContainerDeleteFunc[T any] interface { type ContainerCheckExistenceFunc[T any] interface { CheckExistence(ctx context.Context, container string) (bool, error) } + +// UpdateTags updates the tags of a resource. +type UpdateTags interface { + UpdateTags(ctx context.Context, name, resourceGroupName string, tags map[string]*string) error +} diff --git a/pkg/azure/client/mock/mocks.go b/pkg/azure/client/mock/mocks.go index 8cdf3fcd9..e3b1a6304 100644 --- a/pkg/azure/client/mock/mocks.go +++ b/pkg/azure/client/mock/mocks.go @@ -889,6 +889,20 @@ func (mr *MockPublicIPMockRecorder) List(ctx, resourceGroupName any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockPublicIP)(nil).List), ctx, resourceGroupName) } +// UpdateTags mocks base method. +func (m *MockPublicIP) UpdateTags(ctx context.Context, name, resourceGroupName string, tags map[string]*string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateTags", ctx, name, resourceGroupName, tags) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateTags indicates an expected call of UpdateTags. +func (mr *MockPublicIPMockRecorder) UpdateTags(ctx, name, resourceGroupName, tags any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTags", reflect.TypeOf((*MockPublicIP)(nil).UpdateTags), ctx, name, resourceGroupName, tags) +} + // MockAvailabilitySet is a mock of AvailabilitySet interface. type MockAvailabilitySet struct { ctrl *gomock.Controller diff --git a/pkg/azure/client/publicip.go b/pkg/azure/client/publicip.go index 16fd36504..d65e31e65 100644 --- a/pkg/azure/client/publicip.go +++ b/pkg/azure/client/publicip.go @@ -70,6 +70,15 @@ func (c *PublicIPClient) List(ctx context.Context, resourceGroupName string) ([] return ips, nil } +// UpdateTags will add tags to a network Public IP Address. +func (c *PublicIPClient) UpdateTags(ctx context.Context, name, resourceGroupName string, tags map[string]*string) error { + _, err := c.client.UpdateTags(ctx, resourceGroupName, name, armnetwork.TagsObject{Tags: tags}, nil) + if err != nil { + return FilterNotFoundError(err) + } + return nil +} + // Delete will delete a network Public IP Address. func (c *PublicIPClient) Delete(ctx context.Context, resourceGroupName, name string) error { future, err := c.client.BeginDelete(ctx, resourceGroupName, name, nil) diff --git a/pkg/azure/client/types.go b/pkg/azure/client/types.go index 7786b8709..c3377502a 100644 --- a/pkg/azure/client/types.go +++ b/pkg/azure/client/types.go @@ -99,6 +99,7 @@ type PublicIP interface { CreateOrUpdateFunc[armnetwork.PublicIPAddress] DeleteFunc[armnetwork.PublicIPAddress] ListFunc[armnetwork.PublicIPAddress] + UpdateTags } // NetworkInterface represents an Azure Network Interface k8sClient. diff --git a/pkg/controller/infrastructure/infraflow/const.go b/pkg/controller/infrastructure/infraflow/const.go index 8e0cc12b0..043af53c4 100644 --- a/pkg/controller/infrastructure/infraflow/const.go +++ b/pkg/controller/infrastructure/infraflow/const.go @@ -21,4 +21,9 @@ const ( ChildKeyMigration = "migration" // ChildKeyComplete is a key to indicate whether a task is complete. ChildKeyComplete = "complete" + + // IgnoredByGardenerTag is the tag used to mark resources managed by Gardener. + // It's used for an edge case where public IPs that are customer managed + migrated + // from terraform + reside in our RG + have the shoot prefix name. + IgnoredByGardenerTag = "managedByGardener" ) diff --git a/pkg/controller/infrastructure/infraflow/ensurer.go b/pkg/controller/infrastructure/infraflow/ensurer.go index e9ffb3eb6..9cd9fdcb2 100644 --- a/pkg/controller/infrastructure/infraflow/ensurer.go +++ b/pkg/controller/infrastructure/infraflow/ensurer.go @@ -365,9 +365,40 @@ func (fctx *FlowContext) ensureSecurityGroup(ctx context.Context) (*armnetwork.S // EnsurePublicIps reconciles the public IPs for the shoot. func (fctx *FlowContext) EnsurePublicIps(ctx context.Context) error { + // as a workaround for problems encountered during the tf migration public user IPs are annotated + // with the IgnoredByGardenerTag because they are otherwise deleted by the flow reconciler. + err := fctx.tagUserIp(ctx) + if err != nil { + return err + } return errors.Join(fctx.ensurePublicIps(ctx), fctx.ensureUserPublicIps(ctx)) } +// tagUserIp tags public IPs under the following conditions: +// - the IP is customer managed +// - the IP got migrated from terraform +// - the IP resides in our RG and +// - the IPs name has the shoot prefix. +func (fctx *FlowContext) tagUserIp(ctx context.Context) error { + natGatewayCfg := fctx.cfg.Networks.NatGateway + if natGatewayCfg != nil && natGatewayCfg.Enabled { + for _, ipRef := range natGatewayCfg.IPAddresses { + if fctx.adapter.HasShootPrefix(&ipRef.Name) && fctx.adapter.ResourceGroupName() == ipRef.ResourceGroup { + c, err := fctx.factory.PublicIP() + if err != nil { + return err + } + updateTags := map[string]*string{IgnoredByGardenerTag: to.Ptr("true")} + err = c.UpdateTags(ctx, ipRef.ResourceGroup, ipRef.Name, updateTags) + if err != nil { + return err + } + } + } + } + return nil +} + func (fctx *FlowContext) ensureUserPublicIps(ctx context.Context) error { c, err := fctx.factory.PublicIP() if err != nil { @@ -415,7 +446,9 @@ func (fctx *FlowContext) ensurePublicIps(ctx context.Context) error { currentIPs = Filter(currentIPs, func(address *armnetwork.PublicIPAddress) bool { // filter only these IpConfigs prefixed by the cluster name and that do not contain the CCM tags. return fctx.adapter.HasShootPrefix(address.Name) && - (address.Tags[azure.CCMServiceTagKey] == nil && address.Tags[azure.CCMLegacyServiceTagKey] == nil) + address.Tags[azure.CCMServiceTagKey] == nil && + address.Tags[azure.CCMLegacyServiceTagKey] == nil && + address.Tags[IgnoredByGardenerTag] == nil }) // obtain an indexed list of current IPs nameToCurrentIps := ToMap(currentIPs, func(t *armnetwork.PublicIPAddress) string {