From 2ac357b68cd2793e2faaad0003c4f8879cbb4726 Mon Sep 17 00:00:00 2001 From: Brett Lentz Date: Thu, 24 Jun 2021 09:26:14 -0400 Subject: [PATCH 1/5] allow gcp app default credentials to pass through --- gcp/gcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcp/gcp.go b/gcp/gcp.go index c504461c..5cacc76a 100644 --- a/gcp/gcp.go +++ b/gcp/gcp.go @@ -42,7 +42,7 @@ func init() { applicationCredentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") if len(applicationCredentials) == 0 { log.Warn("[GCP] GOOGLE_APPLICATION_CREDENTIALS environment variable is missing") - return + log.Warn("[GCP] Assuming Application Default Credentials are configured") } ctx.CloudProviders[types.GCP] = func() types.CloudProvider { if len(provider.projectID) == 0 { From ee399e75b469d585afd5906925f763b1a4559cea Mon Sep 17 00:00:00 2001 From: Brett Lentz Date: Mon, 28 Jun 2021 13:35:17 -0400 Subject: [PATCH 2/5] bugfix disk region detection --- gcp/gcp.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/gcp/gcp.go b/gcp/gcp.go index c504461c..f0fdacd8 100644 --- a/gcp/gcp.go +++ b/gcp/gcp.go @@ -994,7 +994,10 @@ func getZone(url string) string { func getRegionFromZoneURL(zoneURL *string) string { zoneURLParts := strings.Split(*zoneURL, "/") zone := zoneURLParts[len(zoneURLParts)-1] - return zone[:len(zone)-2] + if zone != "" { + return zone[:len(zone)-2] + } + return "unknown" } func (p gcpProvider) getDisks() ([]*types.Disk, error) { @@ -1014,11 +1017,27 @@ func (p gcpProvider) getDisks() ([]*types.Disk, error) { return nil, err } tags := convertTags(gDisk.Labels) + + var region string + switch { + case len(gDisk.Region) > 0: + region = gDisk.Region + break + case len(gDisk.Zone) > 0: + region = getRegionFromZoneURL(&gDisk.Zone) + break + case len(gDisk.ReplicaZones) > 0: + region = getRegionFromZoneURL(&gDisk.ReplicaZones[0]) + break + default: + region = "unknown" + } + aDisk := &types.Disk{ CloudType: types.GCP, ID: strconv.Itoa(int(gDisk.Id)), Name: gDisk.Name, - Region: getRegionFromZoneURL(&gDisk.Zone), + Region: region, Created: creationTimeStamp, Size: gDisk.SizeGb, Type: gDisk.Type, From 27c366fcfd84610c6f1f4fb932d1a533078f0539 Mon Sep 17 00:00:00 2001 From: Peter Kedvessy <9048479+pkedvessy@users.noreply.github.com> Date: Thu, 19 Aug 2021 09:45:40 +0200 Subject: [PATCH 3/5] CB-13769 Disk delete tries to detach the in-use disk before delete it. --- aws/aws.go | 55 +++++++++++++++++++++++++++++++++++++++++++-- aws/aws_test.go | 4 ++++ filter/ownerless.go | 5 +++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/aws/aws.go b/aws/aws.go index b7cd14e5..0b583685 100644 --- a/aws/aws.go +++ b/aws/aws.go @@ -5,14 +5,15 @@ import ( "encoding/json" "errors" "fmt" - "github.com/aws/aws-sdk-go/service/cloudformation" - "github.com/hortonworks/cloud-haunter/utils" "net/http" "os" "strings" "sync" "time" + "github.com/aws/aws-sdk-go/service/cloudformation" + "github.com/hortonworks/cloud-haunter/utils" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/autoscaling" @@ -454,6 +455,19 @@ func deleteVolumes(ec2Clients map[string]ec2Client, volumes []*types.Disk) []err if ctx.DryRun { log.Infof("[AWS] Dry-run set, volume is not deleted: %s:%s, region: %s", vol.Name, vol.ID, region) } else { + log.Infof("[AWS] Initiate delete volume: %s:%s", vol.Name, vol.ID) + var detachError error + if vol.State == types.InUse { + log.Infof("[AWS] Volume %s:%s is in-use, trying to detach", vol.Name, vol.ID) + if _, detachError = ec2Client.DetachVolume(&ec2.DetachVolumeInput{VolumeId: &vol.ID}); detachError == nil { + detachError = waitForVolumeUnusedState(ec2Client, vol) + } + } + + if detachError != nil { + log.Infof("[AWS] Skip volume %s:%s as it can not be detached by [%s].", vol.Name, vol.ID, detachError) + continue + } log.Infof("[AWS] Delete volume: %s:%s", vol.Name, vol.ID) if _, err := ec2Client.DeleteVolume(&ec2.DeleteVolumeInput{VolumeId: &vol.ID}); err != nil { errChan <- err @@ -475,6 +489,26 @@ func deleteVolumes(ec2Clients map[string]ec2Client, volumes []*types.Disk) []err return errs } +func waitForVolumeUnusedState(ec2Client ec2Client, vol *types.Disk) error { + log.Infof("[AWS] Waiting for Volume %s:%s 'available' state...", vol.Name, vol.ID) + //Polling state max 10 times with 1 sec interval + var counter int = 0 + d, e := getDisk(ec2Client, vol.ID) + for e == nil && d.State != types.Unused && counter < 10 { + time.Sleep(1 * time.Second) + d, e = getDisk(ec2Client, vol.ID) + counter++ + } + if e != nil { + return errors.New(fmt.Sprintf("Detach verification failed: %s", e)) + } else if d.State != types.Unused { + return errors.New(fmt.Sprintf("Detach verification failed, disk state is: %s", d.State)) + } else { + log.Infof("[AWS] Volume %s:%s is detached so it can be deleted.", vol.Name, vol.ID) + } + return nil +} + func deleteImages(ec2Clients map[string]ec2Client, images []*types.Image) []error { regionImages := map[string][]*types.Image{} for _, image := range images { @@ -535,6 +569,7 @@ type ec2Client interface { DeleteVolume(input *ec2.DeleteVolumeInput) (*ec2.DeleteVolumeOutput, error) DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) DeregisterImage(input *ec2.DeregisterImageInput) (*ec2.DeregisterImageOutput, error) + DetachVolume(input *ec2.DetachVolumeInput) (*ec2.VolumeAttachment, error) } type cfClient interface { @@ -672,6 +707,21 @@ func getImages(ec2Clients map[string]ec2Client) ([]*types.Image, error) { return images, nil } +func getDisk(ec2Client ec2Client, volumeId string) (*types.Disk, error) { + result, err := ec2Client.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{&volumeId}}) + if err != nil { + log.Errorf("[AWS] Failed to fetch the volume, err: %s", err) + return nil, err + } + log.Debugf("[AWS] Processing volumes (%d): [%s]", len(result.Volumes), result.Volumes) + + if len(result.Volumes) == 0 { + return nil, errors.New(fmt.Sprintf("Volume not found with id '%s'", volumeId)) + } + return newDisk(result.Volumes[0]), nil + +} + func getDisks(ec2Clients map[string]ec2Client) ([]*types.Disk, error) { diskChan := make(chan *types.Disk) wg := sync.WaitGroup{} @@ -1029,6 +1079,7 @@ func newDisk(volume *ec2.Volume) *types.Disk { Created: getCreated(volume.CreateTime), Size: *volume.Size, Type: *volume.VolumeType, + Owner: tags[ctx.OwnerLabel], } } diff --git a/aws/aws_test.go b/aws/aws_test.go index 9ba58169..30f4a2ca 100644 --- a/aws/aws_test.go +++ b/aws/aws_test.go @@ -174,6 +174,10 @@ func (t mockEc2Client) DeleteVolume(input *ec2.DeleteVolumeInput) (*ec2.DeleteVo return nil, nil } +func (t mockEc2Client) DetachVolume(input *ec2.DetachVolumeInput) (*ec2.VolumeAttachment, error) { + return nil, nil +} + func (t mockEc2Client) DeregisterImage(input *ec2.DeregisterImageInput) (*ec2.DeregisterImageOutput, error) { t.deregisterImagesChannel <- *input.ImageId return nil, nil diff --git a/filter/ownerless.go b/filter/ownerless.go index ff79b09b..d11185d6 100644 --- a/filter/ownerless.go +++ b/filter/ownerless.go @@ -28,6 +28,11 @@ func (o ownerless) Execute(items []types.CloudItem) []types.CloudItem { match := !utils.IsAnyMatch(stack.Tags, ctx.OwnerLabel) log.Debugf("[OWNERLESS] Stack: %s match: %v", stack.Name, match) return match + case types.Disk: + disk := item.(*types.Disk) + match := len(disk.Owner) == 0 + log.Debugf("[OWNERLESS] Disk: %s match: %v", disk.Name, match) + return match default: log.Fatalf("[OWNERLESS] Filter does not apply for cloud item: %s", item.GetName()) } From 5f8a0a6d3a47c5cb27c38acb3c4d707a471380e0 Mon Sep 17 00:00:00 2001 From: David Bajzath Date: Wed, 18 Aug 2021 18:24:49 +0200 Subject: [PATCH 4/5] CB-13780 Introduce V2 filter config format Includes refactor of filtering to enable v2 filter config --- Makefile | 2 +- aws/aws.go | 3 + azure/azure.go | 1 + context/context.go | 4 +- filter/common.go | 175 ++++++++++-------------------- gcp/gcp.go | 4 + main.go | 6 +- types/access.go | 5 + types/database.go | 4 + types/disk.go | 5 + types/filter_config.go | 44 ++++++++ types/filter_config_v2.go | 40 +++++++ types/image.go | 5 + types/instance.go | 4 + types/stack.go | 4 + types/types.go | 1 + utils/testdata/filterConfigV2.yml | 30 +++++ utils/utils.go | 16 ++- utils/utils_test.go | 53 ++++++--- 19 files changed, 267 insertions(+), 139 deletions(-) create mode 100644 types/filter_config_v2.go create mode 100644 utils/testdata/filterConfigV2.yml diff --git a/Makefile b/Makefile index 23d1d9e1..f8e56e7c 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ vet: GO111MODULE=on go vet -mod=vendor ./... test: - GO111MODULE=on go test -mod=vendor -timeout 30s -coverprofile coverage -race + GO111MODULE=on go test -mod=vendor -timeout 30s -coverprofile coverage -race ./... _build: build-darwin build-linux diff --git a/aws/aws.go b/aws/aws.go index 0b583685..fc21d206 100644 --- a/aws/aws.go +++ b/aws/aws.go @@ -818,6 +818,7 @@ func getAccesses(iamClient iamClient) ([]*types.Access, error) { Name: name, Owner: *akm.UserName, Created: getCreated(akm.CreateDate), + Tags: types.Tags{}, }) } } @@ -1080,6 +1081,7 @@ func newDisk(volume *ec2.Volume) *types.Disk { Size: *volume.Size, Type: *volume.VolumeType, Owner: tags[ctx.OwnerLabel], + Tags: tags, } } @@ -1096,6 +1098,7 @@ func newImage(image *ec2.Image, region string) *types.Image { CloudType: types.AWS, Region: region, Created: createdAt, + Tags: getEc2Tags(image.Tags), } } diff --git a/azure/azure.go b/azure/azure.go index bb514145..9709ce37 100644 --- a/azure/azure.go +++ b/azure/azure.go @@ -467,6 +467,7 @@ func newImage(image compute.Image) *types.Image { Name: *image.Name, Region: *image.Location, CloudType: types.AZURE, + Tags: utils.ConvertTags(image.Tags), } } diff --git a/context/context.go b/context/context.go index 7445b83a..a78851e9 100644 --- a/context/context.go +++ b/context/context.go @@ -33,7 +33,7 @@ var DryRun = false var Verbose = false // IgnoreLabelDisabled is a global flag for enabling/disabling ignore label usage -var IgnoreLabelDisabled = true +var IgnoreLabelDisabled = false // Operations contains all the available operations var Operations = make(map[types.OpType]types.Operation) @@ -51,4 +51,4 @@ var Dispatchers = make(map[string]types.Dispatcher) var Actions = make(map[types.ActionType]types.Action) // FilterConfig contains the include/exclude configurations from config file -var FilterConfig *types.FilterConfig +var FilterConfig types.IFilterConfig diff --git a/filter/common.go b/filter/common.go index 79e8be08..ea7e35ca 100644 --- a/filter/common.go +++ b/filter/common.go @@ -5,6 +5,7 @@ import ( "github.com/hortonworks/cloud-haunter/types" "github.com/hortonworks/cloud-haunter/utils" log "github.com/sirupsen/logrus" + "reflect" ) func filter(filterName string, items []types.CloudItem, filterType types.FilterConfigType, isNeeded func(types.CloudItem) bool) []types.CloudItem { @@ -28,138 +29,74 @@ func filter(filterName string, items []types.CloudItem, filterType types.FilterC return filtered } -func isFilterMatch(filterName string, item types.CloudItem, filterType types.FilterConfigType, filterConfig *types.FilterConfig) bool { - switch item.GetItem().(type) { - case types.Instance: - inst := item.GetItem().(types.Instance) - name := item.GetName() - ignoreLabelFound := utils.IsAnyMatch(inst.Tags, ctx.IgnoreLabel) - if ignoreLabelFound { - log.Debugf("[%s] Found ignore label on item: %s, label: %s", filterName, name, ctx.IgnoreLabel) - if ctx.IgnoreLabelDisabled { - log.Debugf("[%s] Ignore label usage is disabled, continuing to apply filter on item: %s", filterName, name) - } else { - if filterType.IsInclusive() { - log.Debugf("[%s] inclusive filter applied on item: %s", filterName, name) - return false - } - log.Debugf("[%s] exclusive filter applied on item: %s", filterName, name) - return true - } - } - filtered, applied := applyFilterConfig(filterConfig, filterType, item, filterName, inst.Tags) - if applied { - return filtered - } - case types.Stack: - stack := item.GetItem().(types.Stack) - name := item.GetName() - ignoreLabelFound := utils.IsAnyMatch(stack.Tags, ctx.IgnoreLabel) - if ignoreLabelFound { - log.Debugf("[%s] Found ignore label on item: %s, label: %s", filterName, name, ctx.IgnoreLabel) - if ctx.IgnoreLabelDisabled { - log.Debugf("[%s] Ignore label usage is disabled, continuing to apply filter on item: %s", filterName, name) - } else { - if filterType.IsInclusive() { - log.Debugf("[%s] inclusive filter applied on item: %s", filterName, name) - return false - } - log.Debugf("[%s] exclusive filter applied on item: %s", filterName, name) - return true +func isFilterMatch(filterName string, item types.CloudItem, filterType types.FilterConfigType, filterConfig types.IFilterConfig) bool { + name := item.GetName() + ignoreLabelFound := utils.IsAnyMatch(item.GetTags(), ctx.IgnoreLabel) + if ignoreLabelFound { + log.Debugf("[%s] Found ignore label on item: %s, label: %s", filterName, name, ctx.IgnoreLabel) + if ctx.IgnoreLabelDisabled { + log.Debugf("[%s] Ignore label usage is disabled, continuing to apply filter on item: %s", filterName, name) + } else { + if filterType.IsInclusive() { + log.Debugf("[%s] inclusive filter applied on item: %s", filterName, name) + return false } + log.Debugf("[%s] exclusive filter applied on item: %s", filterName, name) + return true } - filtered, applied := applyFilterConfig(filterConfig, filterType, item, filterName, stack.Tags) - if applied { - return filtered - } + } + + if filterConfig == nil { + return false + } + + var filterEntityType types.FilterEntityType + + switch item.GetItem().(type) { case types.Access: - accessFilter, _ := getFilterConfigs(filterConfig, filterType) - if accessFilter != nil { - switch item.GetCloudType() { - case types.AWS: - return isNameOrOwnerMatch(filterName, item, accessFilter.Aws.Names, accessFilter.Aws.Owners) - case types.AZURE: - return isNameOrOwnerMatch(filterName, item, accessFilter.Azure.Names, accessFilter.Azure.Owners) - case types.GCP: - return isNameOrOwnerMatch(filterName, item, accessFilter.Gcp.Names, accessFilter.Gcp.Owners) - default: - log.Warnf("[%s] Cloud type not supported: %s", filterName, item.GetCloudType()) - } - } - case types.Database: - database := item.GetItem().(types.Database) - name := item.GetName() - ignoreLabelFound := utils.IsAnyMatch(database.Tags, ctx.IgnoreLabel) - if ignoreLabelFound { - log.Debugf("[%s] Found ignore label on item: %s, label: %s", filterName, name, ctx.IgnoreLabel) - if ctx.IgnoreLabelDisabled { - log.Debugf("[%s] Ignore label usage is disabled, continuing to apply filter on item: %s", filterName, name) - } else { - if filterType.IsInclusive() { - log.Debugf("[%s] inclusive filter applied on item: %s", filterName, name) - return false - } - log.Debugf("[%s] exclusive filter applied on item: %s", filterName, name) - return true - } - } - filtered, applied := applyFilterConfig(filterConfig, filterType, item, filterName, database.Tags) - if applied { - return filtered + if filterType.IsInclusive() { + filterEntityType = types.IncludeAccess + } else { + filterEntityType = types.ExcludeAccess } - case types.Disk: - filtered, applied := applyFilterConfig(filterConfig, filterType, item, filterName, types.Tags{}) - if applied { - return filtered + case types.Instance, types.Stack, types.Database, types.Disk: + if filterType.IsInclusive() { + filterEntityType = types.IncludeInstance + } else { + filterEntityType = types.ExcludeInstance } + default: + log.Warnf("Filtering is not implemented for type %s", reflect.TypeOf(item)) + return false } - return false -} -func applyFilterConfig(filterConfig *types.FilterConfig, filterType types.FilterConfigType, item types.CloudItem, filterName string, tags types.Tags) (applied, filtered bool) { - _, instanceFilter := getFilterConfigs(filterConfig, filterType) - if instanceFilter != nil { - switch item.GetCloudType() { - case types.AWS: - return isMatchWithIgnores(filterName, item, tags, - instanceFilter.Aws.Names, instanceFilter.Aws.Owners, instanceFilter.Aws.Labels), true - case types.AZURE: - return isMatchWithIgnores(filterName, item, tags, - instanceFilter.Azure.Names, instanceFilter.Azure.Owners, instanceFilter.Azure.Labels), true - case types.GCP: - return isMatchWithIgnores(filterName, item, tags, - instanceFilter.Gcp.Names, instanceFilter.Gcp.Owners, instanceFilter.Gcp.Labels), true - default: - log.Warnf("[%s] Cloud type not supported: %s", filterName, item.GetCloudType()) - } + filtered, applied := false, false + + if names := filterConfig.GetFilterValues(filterEntityType, item.GetCloudType(), types.Name); names != nil { + log.Debugf("[%s] filtering item %s to names [%s]", filterName, item.GetName(), names) + filtered, applied = filtered || utils.IsStartsWith(item.GetName(), names...), true } - return false, false -} -func getFilterConfigs(filterConfig *types.FilterConfig, filterType types.FilterConfigType) (accessConfig *types.FilterAccessConfig, instanceConfig *types.FilterInstanceConfig) { - if filterConfig != nil { - if filterType.IsInclusive() { - return filterConfig.IncludeAccess, filterConfig.IncludeInstance - } - return filterConfig.ExcludeAccess, filterConfig.ExcludeInstance + if owners := filterConfig.GetFilterValues(filterEntityType, item.GetCloudType(), types.Owner); owners != nil { + log.Debugf("[%s] filtering item %s to owners [%s]", filterName, item.GetName(), owners) + filtered, applied = filtered || utils.IsStartsWith(item.GetOwner(), owners...), true } - return nil, nil -} -func isMatchWithIgnores(filterName string, item types.CloudItem, tags map[string]string, names, owners []string, labels []string) bool { - if isNameOrOwnerMatch(filterName, item, names, owners) || utils.IsAnyStartsWith(tags, labels...) { - log.Debugf("[%s] item %s match with name/owner or tag %s", filterName, item.GetName(), labels) - return true + if labels := filterConfig.GetFilterValues(filterEntityType, item.GetCloudType(), types.Label); labels != nil { + log.Debugf("[%s] filtering item %s to labels [%s]", filterName, item.GetName(), labels) + filtered, applied = filtered || utils.IsAnyStartsWith(item.GetTags(), labels...), true } - log.Debugf("[%s] item %s does not match with name/owner or tag %s", filterName, item.GetName(), labels) - return false -} -func isNameOrOwnerMatch(filterName string, item types.CloudItem, names, owners []string) bool { - if utils.IsStartsWith(item.GetName(), names...) || utils.IsStartsWith(item.GetOwner(), owners...) { - log.Debugf("[%s] item %s match with filter config name %s or owner %s", filterName, item.GetName(), names, owners) - return true + if applied { + if filtered { + log.Debugf("[%s] item %s matches filter", filterName, item.GetName()) + } else { + log.Debugf("[%s] item %s does not match filter", filterName, item.GetName()) + } + return filtered + } else { + log.Debugf("[%s] item %s could not be filtered", filterName, item.GetName()) } - log.Debugf("[%s] item %s does not match with filter config name %s or owner %s", filterName, item.GetName(), names, owners) + return false } diff --git a/gcp/gcp.go b/gcp/gcp.go index e4c2b2a7..c443abd7 100644 --- a/gcp/gcp.go +++ b/gcp/gcp.go @@ -255,6 +255,7 @@ func (p gcpProvider) GetStacks() ([]*types.Stack, error) { "database": gcpStack.DatabaseName, "zone": gcpStack.Zone, }, + Tags: types.Tags{}, } stacks = append(stacks, aStack) } @@ -901,6 +902,7 @@ func getAccesses(serviceAccountAggregator serviceAccountsListAggregator, getKeys Name: key.Name, Owner: account.Email, Created: validAfter, + Tags: types.Tags{}, }) } } @@ -918,6 +920,7 @@ func newImage(image *compute.Image) *types.Image { Created: created, CloudType: types.GCP, Region: "", + Tags: convertTags(image.Labels), } } @@ -1044,6 +1047,7 @@ func (p gcpProvider) getDisks() ([]*types.Disk, error) { State: getDiskStatus(gDisk), Owner: tags[ctx.OwnerLabel], Metadata: map[string]string{"zone": getZone(gDisk.Zone)}, + Tags: tags, } disks = append(disks, aDisk) } diff --git a/main.go b/main.go index 42415138..ae5d62a4 100644 --- a/main.go +++ b/main.go @@ -55,7 +55,11 @@ func main() { var err error ctx.FilterConfig, err = utils.LoadFilterConfig(*filterConfigLoc) if err != nil { - panic("Unable to parse filter configuration: " + err.Error()) + log.Warnf("[UTIL] Failed to load %s as V1 filter config, trying as V2. Error: %s", *filterConfigLoc, err.Error()) + ctx.FilterConfig, err = utils.LoadFilterConfigV2(*filterConfigLoc) + if err != nil { + panic("Unable to parse filter configuration: " + err.Error()) + } } } diff --git a/types/access.go b/types/access.go index 1ab3f729..c55558bd 100644 --- a/types/access.go +++ b/types/access.go @@ -8,6 +8,7 @@ type Access struct { Owner string `json:"Owner"` Created time.Time `json:"Created"` CloudType CloudType `json:"CloudType"` + Tags Tags `json:"Tags"` } // GetName returns the name of the access cloud object @@ -42,3 +43,7 @@ func (a Access) GetItem() interface{} { func (a Access) GetType() string { return "access" } + +func (a Access) GetTags() Tags { + return a.Tags +} diff --git a/types/database.go b/types/database.go index 759cc9b3..3076a81d 100644 --- a/types/database.go +++ b/types/database.go @@ -63,3 +63,7 @@ func (d Database) GetItem() interface{} { func (d Database) GetType() string { return "database" } + +func (d Database) GetTags() Tags { + return d.Tags +} diff --git a/types/disk.go b/types/disk.go index 38f26392..0e58070d 100644 --- a/types/disk.go +++ b/types/disk.go @@ -32,6 +32,7 @@ type Disk struct { Size int64 `json:"Size"` Type string `json:"Type"` Metadata map[string]string `json:"Metadata"` + Tags Tags `json:"Tags"` } // GetName returns the name of the disk @@ -63,3 +64,7 @@ func (d Disk) GetItem() interface{} { func (d Disk) GetType() string { return "disk" } + +func (d Disk) GetTags() Tags { + return d.Tags +} diff --git a/types/filter_config.go b/types/filter_config.go index dc3cd979..e65de047 100644 --- a/types/filter_config.go +++ b/types/filter_config.go @@ -1,5 +1,32 @@ package types +import ( + log "github.com/sirupsen/logrus" + "reflect" + "strings" +) + +type IFilterConfig interface { + GetFilterValues(fType FilterEntityType, cloud CloudType, property FilterConfigProperty) []string +} + +type FilterEntityType string + +const ( + ExcludeAccess = FilterEntityType("excludeAccess") + IncludeAccess = FilterEntityType("includeAccess") + ExcludeInstance = FilterEntityType("excludeInstance") + IncludeInstance = FilterEntityType("includeInstance") +) + +type FilterConfigProperty string + +const ( + Name = FilterConfigProperty("name") + Owner = FilterConfigProperty("owner") + Label = FilterConfigProperty("label") +) + // FilterConfig structure that stores the information provided by the exclude/include flag type FilterConfig struct { ExcludeAccess *FilterAccessConfig `yaml:"excludeAccess"` @@ -42,3 +69,20 @@ type FilterInstanceConfig struct { Owners []string `yaml:"owners"` } `yaml:"gcp"` } + +func (filterConfig FilterConfig) GetFilterValues(fType FilterEntityType, cloud CloudType, property FilterConfigProperty) []string { + log.Debugf("fType: %s, cloud: %s, property :%s", fType, cloud, property) + typeProperty := strings.ToUpper(string(string(fType)[0])) + string(fType)[1:] + cloudProperty := string(string(cloud)[0]) + strings.ToLower(string(cloud)[1:]) + propertyProperty := strings.ToUpper(string(string(property)[0])) + string(property)[1:] + "s" + log.Debugf("FilterEntityType: %s, CloudProperty: %s, FilterConfigProperty: %s", typeProperty, cloudProperty, propertyProperty) + + if typeField := reflect.ValueOf(filterConfig).FieldByName(typeProperty); typeField.IsValid() && !typeField.IsNil() { + if cloudField := reflect.Indirect(typeField).FieldByName(cloudProperty); cloudField.IsValid() { + if propertyField := reflect.Indirect(cloudField).FieldByName(propertyProperty); propertyField.IsValid() { + return propertyField.Interface().([]string) + } + } + } + return nil +} diff --git a/types/filter_config_v2.go b/types/filter_config_v2.go new file mode 100644 index 00000000..00d5153c --- /dev/null +++ b/types/filter_config_v2.go @@ -0,0 +1,40 @@ +package types + +import ( + log "github.com/sirupsen/logrus" + "strings" +) + +// FilterConfigV2 structure that stores the information provided by the exclude/include flag +type FilterConfigV2 struct { + Filters []FilterConfigV2Filter `yaml:"filters"` +} + +type FilterConfigV2Filter struct { + Types []FilterEntityType `yaml:"filterTypes"` + CloudTypes []CloudType `yaml:"cloudTypes"` + Properties []FilterConfigProperty `yaml:"filterProperties"` + Values []string `yaml:"filterValues"` +} + +func (filterConfig FilterConfigV2) GetFilterValues(fType FilterEntityType, cloud CloudType, property FilterConfigProperty) []string { + cloudProperty := strings.ToLower(string(cloud)) + log.Debugf("FilterEntityType: %s, CloudProperty: %s, FilterConfigProperty: %s", fType, cloudProperty, property) + + for _, filter := range filterConfig.Filters { + for _, fcType := range filter.Types { + if fcType == fType { + for _, fcCloud := range filter.CloudTypes { + if string(fcCloud) == cloudProperty { + for _, fcProperty := range filter.Properties { + if fcProperty == property { + return filter.Values + } + } + } + } + } + } + } + return nil +} diff --git a/types/image.go b/types/image.go index 3ff0901d..8e167487 100644 --- a/types/image.go +++ b/types/image.go @@ -27,6 +27,7 @@ type Image struct { Created time.Time `json:"Created"` CloudType CloudType `json:"CloudType"` Region string `json:"Region"` + Tags Tags `json:"Tags"` } // GetName returns the name of the image @@ -58,3 +59,7 @@ func (img Image) GetItem() interface{} { func (img Image) GetType() string { return "image" } + +func (img Image) GetTags() Tags { + return img.Tags +} diff --git a/types/instance.go b/types/instance.go index 6a6098d4..53c05085 100644 --- a/types/instance.go +++ b/types/instance.go @@ -70,3 +70,7 @@ func (i Instance) GetItem() interface{} { func (i Instance) GetType() string { return "instance" } + +func (i Instance) GetTags() Tags { + return i.Tags +} diff --git a/types/stack.go b/types/stack.go index d332beeb..872ec18e 100644 --- a/types/stack.go +++ b/types/stack.go @@ -64,3 +64,7 @@ func (s Stack) GetItem() interface{} { func (s Stack) GetType() string { return "stack" } + +func (s Stack) GetTags() Tags { + return s.Tags +} diff --git a/types/types.go b/types/types.go index 07c1fd71..9aa065e1 100644 --- a/types/types.go +++ b/types/types.go @@ -45,6 +45,7 @@ type CloudItem interface { GetCreated() time.Time GetItem() interface{} GetType() string + GetTags() Tags } // Dispatcher interface used to send the messages with diff --git a/utils/testdata/filterConfigV2.yml b/utils/testdata/filterConfigV2.yml new file mode 100644 index 00000000..ae8f4687 --- /dev/null +++ b/utils/testdata/filterConfigV2.yml @@ -0,0 +1,30 @@ +--- +filters: + - + filterTypes: + - includeAccess + - includeInstance + cloudTypes: + - aws + - azure + - gcp + filterProperties: + - name + - owner + - label + filterValues: + - includeThisValue + - + filterTypes: + - excludeAccess + - excludeInstance + cloudTypes: + - aws + - azure + - gcp + filterProperties: + - name + - owner + - label + filterValues: + - excludeThisValue diff --git a/utils/utils.go b/utils/utils.go index c32af4b5..0d477dbf 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -87,7 +87,7 @@ func LoadFilterConfig(location string) (*types.FilterConfig, error) { return nil, err } config := &types.FilterConfig{} - err = yaml.Unmarshal(raw, config) + err = yaml.UnmarshalStrict(raw, config) if err != nil { return nil, err } @@ -95,6 +95,20 @@ func LoadFilterConfig(location string) (*types.FilterConfig, error) { return config, nil } +func LoadFilterConfigV2(location string) (*types.FilterConfigV2, error) { + raw, err := ioutil.ReadFile(location) + if err != nil { + return nil, err + } + configV2 := &types.FilterConfigV2{} + err = yaml.UnmarshalStrict(raw, configV2) + if err != nil { + return nil, err + } + log.Debugf("[UTIL] Filter config V2 loaded:\n%s", raw) + return configV2, nil +} + // GetCloudAccountNames returns the name of the configured cloud accounts func GetCloudAccountNames() map[types.CloudType]string { var accounts = make(map[types.CloudType]string) diff --git a/utils/utils_test.go b/utils/utils_test.go index d7674cbd..eefe3c6a 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -55,25 +55,48 @@ func TestConvertTags(t *testing.T) { func TestLoadIgnores(t *testing.T) { filterConfig, _ := LoadFilterConfig("testdata/filterConfig.yml") - assert.Equal(t, []string{"awsName"}, filterConfig.ExcludeAccess.Aws.Names) - assert.Equal(t, []string{"azureName"}, filterConfig.ExcludeAccess.Azure.Names) - assert.Equal(t, []string{"gcpName"}, filterConfig.ExcludeAccess.Gcp.Names) + assert.Equal(t, []string{"awsName"}, filterConfig.GetFilterValues(types.ExcludeAccess, types.AWS, types.Name)) + assert.Equal(t, []string{"azureName"}, filterConfig.GetFilterValues(types.ExcludeAccess, types.AZURE, types.Name)) + assert.Equal(t, []string{"gcpName"}, filterConfig.GetFilterValues(types.ExcludeAccess, types.GCP, types.Name)) - assert.Equal(t, []string{"awsOwner"}, filterConfig.ExcludeAccess.Aws.Owners) - assert.Equal(t, []string{"azureOwner"}, filterConfig.ExcludeAccess.Azure.Owners) - assert.Equal(t, []string{"gcpOwner"}, filterConfig.ExcludeAccess.Gcp.Owners) + assert.Equal(t, []string{"awsOwner"}, filterConfig.GetFilterValues(types.ExcludeAccess, types.AWS, types.Owner)) + assert.Equal(t, []string{"azureOwner"}, filterConfig.GetFilterValues(types.ExcludeAccess, types.AZURE, types.Owner)) + assert.Equal(t, []string{"gcpOwner"}, filterConfig.GetFilterValues(types.ExcludeAccess, types.GCP, types.Owner)) - assert.Equal(t, []string{"awsLabel"}, filterConfig.ExcludeInstance.Aws.Labels) - assert.Equal(t, []string{"azureLabel"}, filterConfig.ExcludeInstance.Azure.Labels) - assert.Equal(t, []string{"gcpLabel"}, filterConfig.ExcludeInstance.Gcp.Labels) + assert.Equal(t, []string{"awsLabel"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.AWS, types.Label)) + assert.Equal(t, []string{"azureLabel"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.AZURE, types.Label)) + assert.Equal(t, []string{"gcpLabel"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.GCP, types.Label)) - assert.Equal(t, []string{"awsName"}, filterConfig.ExcludeInstance.Aws.Names) - assert.Equal(t, []string{"azureName"}, filterConfig.ExcludeInstance.Azure.Names) - assert.Equal(t, []string{"gcpName"}, filterConfig.ExcludeInstance.Gcp.Names) + assert.Equal(t, []string{"awsName"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.AWS, types.Name)) + assert.Equal(t, []string{"azureName"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.AZURE, types.Name)) + assert.Equal(t, []string{"gcpName"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.GCP, types.Name)) - assert.Equal(t, []string{"awsOwner"}, filterConfig.ExcludeInstance.Aws.Owners) - assert.Equal(t, []string{"azureOwner"}, filterConfig.ExcludeInstance.Azure.Owners) - assert.Equal(t, []string{"gcpOwner"}, filterConfig.ExcludeInstance.Gcp.Owners) + assert.Equal(t, []string{"awsOwner"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.AWS, types.Owner)) + assert.Equal(t, []string{"azureOwner"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.AZURE, types.Owner)) + assert.Equal(t, []string{"gcpOwner"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.GCP, types.Owner)) + + assert.Nil(t, filterConfig.GetFilterValues(types.IncludeInstance, types.GCP, types.Owner)) + assert.Nil(t, filterConfig.GetFilterValues(types.IncludeAccess, types.AWS, types.Name)) +} + +func TestLoadIgnoresV2(t *testing.T) { + filterConfig, _ := LoadFilterConfigV2("testdata/filterConfigV2.yml") + + assert.Equal(t, []string{"excludeThisValue"}, filterConfig.GetFilterValues(types.ExcludeAccess, types.AWS, types.Name)) + assert.Equal(t, []string{"excludeThisValue"}, filterConfig.GetFilterValues(types.ExcludeAccess, types.AZURE, types.Name)) + assert.Equal(t, []string{"excludeThisValue"}, filterConfig.GetFilterValues(types.ExcludeAccess, types.GCP, types.Name)) + + assert.Equal(t, []string{"excludeThisValue"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.AWS, types.Label)) + assert.Equal(t, []string{"excludeThisValue"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.AZURE, types.Label)) + assert.Equal(t, []string{"excludeThisValue"}, filterConfig.GetFilterValues(types.ExcludeInstance, types.GCP, types.Label)) + + assert.Equal(t, []string{"includeThisValue"}, filterConfig.GetFilterValues(types.IncludeAccess, types.AWS, types.Owner)) + assert.Equal(t, []string{"includeThisValue"}, filterConfig.GetFilterValues(types.IncludeAccess, types.AZURE, types.Owner)) + assert.Equal(t, []string{"includeThisValue"}, filterConfig.GetFilterValues(types.IncludeAccess, types.GCP, types.Owner)) + + assert.Equal(t, []string{"includeThisValue"}, filterConfig.GetFilterValues(types.IncludeInstance, types.AWS, types.Name)) + assert.Equal(t, []string{"includeThisValue"}, filterConfig.GetFilterValues(types.IncludeInstance, types.AZURE, types.Name)) + assert.Equal(t, []string{"includeThisValue"}, filterConfig.GetFilterValues(types.IncludeInstance, types.GCP, types.Name)) } func TestSplitListToMap(t *testing.T) { From 2d080a560f5f3365c578779b5276eec9520a3f55 Mon Sep 17 00:00:00 2001 From: David Bajzath Date: Wed, 25 Aug 2021 11:35:49 +0200 Subject: [PATCH 5/5] Fix formatting --- gcp/gcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcp/gcp.go b/gcp/gcp.go index c443abd7..0d0435d9 100644 --- a/gcp/gcp.go +++ b/gcp/gcp.go @@ -42,7 +42,7 @@ func init() { applicationCredentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") if len(applicationCredentials) == 0 { log.Warn("[GCP] GOOGLE_APPLICATION_CREDENTIALS environment variable is missing") - log.Warn("[GCP] Assuming Application Default Credentials are configured") + log.Warn("[GCP] Assuming Application Default Credentials are configured") } ctx.CloudProviders[types.GCP] = func() types.CloudProvider { if len(provider.projectID) == 0 {