From 27686eeb4549a6d56eb963c86cba2287cbd85058 Mon Sep 17 00:00:00 2001 From: Kai Parry Date: Tue, 5 Sep 2023 15:08:42 +0200 Subject: [PATCH] feat: expose metrics for all AWS API calls Signed-off-by: Kai Parry --- cmd/provider/main.go | 2 ++ go.mod | 4 ++-- go.sum | 4 ++-- pkg/clients/config.go | 43 +++++++++++++++++++++++++------------- pkg/utils/metrics/setup.go | 24 +++++++++++++++++++++ 5 files changed, 58 insertions(+), 19 deletions(-) create mode 100644 pkg/utils/metrics/setup.go diff --git a/cmd/provider/main.go b/cmd/provider/main.go index e1be815a3f..f63ee2b809 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -41,6 +41,7 @@ import ( "github.com/crossplane-contrib/provider-aws/apis/v1alpha1" "github.com/crossplane-contrib/provider-aws/pkg/controller" "github.com/crossplane-contrib/provider-aws/pkg/features" + "github.com/crossplane-contrib/provider-aws/pkg/utils/metrics" ) func main() { @@ -123,6 +124,7 @@ func main() { log.Info("Alpha feature enabled", "flag", features.EnableAlphaManagementPolicies) } + kingpin.FatalIfError(metrics.SetupMetrics(), "Cannot setup AWS metrics hook") kingpin.FatalIfError(controller.Setup(mgr, o), "Cannot setup AWS controllers") kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager") diff --git a/go.mod b/go.mod index 8e446c8b62..eb865de76b 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/mitchellh/copystructure v1.0.0 github.com/onsi/gomega v1.27.7 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.15.1 go.uber.org/zap v1.24.0 golang.org/x/net v0.12.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 @@ -121,10 +122,9 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/operator-framework/api v0.6.0 // indirect - github.com/prometheus/client_golang v1.15.1 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/samber/lo v1.37.0 // indirect github.com/sergi/go-diff v1.0.0 // indirect diff --git a/go.sum b/go.sum index 9d9fd3ab76..b1590aa4e7 100644 --- a/go.sum +++ b/go.sum @@ -683,8 +683,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.10.0 h1:UkG7GPYkO4UZyLnyXjaWYcgOSONqwdBqFUT95ugmt6I= -github.com/prometheus/procfs v0.10.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= diff --git a/pkg/clients/config.go b/pkg/clients/config.go index 82bf012040..542aae5258 100644 --- a/pkg/clients/config.go +++ b/pkg/clients/config.go @@ -45,6 +45,7 @@ import ( "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/crossplane-contrib/provider-aws/apis/v1beta1" + "github.com/crossplane-contrib/provider-aws/pkg/utils/metrics" "github.com/crossplane-contrib/provider-aws/pkg/version" ) @@ -74,9 +75,18 @@ const ( FieldRequired FieldOption = iota ) -// userAgentV2 constructs the Crossplane user agent for AWS v2 clients -var userAgentV2 = config.WithAPIOptions([]func(*middleware.Stack) error{ +// middlewareV2 constructs the AWS SDK v2 middleware +var middlewareV2 = config.WithAPIOptions([]func(*middleware.Stack) error{ awsmiddleware.AddUserAgentKeyValue("crossplane-provider-aws", version.Version), + func(s *middleware.Stack) error { + return s.Finalize.Add(recordRequestMetrics, middleware.After) + }, +}) + +// recordRequestMetrics records Prometheus metrics for requests to the AWS APIs +var recordRequestMetrics = middleware.FinalizeMiddlewareFunc("recordRequestMetrics", func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) { + metrics.IncAWSAPICall(awsmiddleware.GetServiceID(ctx), awsmiddleware.GetOperationName(ctx), "2") + return next.HandleFinalize(ctx, in) }) // userAgentV1 constructs the Crossplane user agent for AWS v1 clients @@ -256,7 +266,7 @@ func UseProviderSecret(ctx context.Context, data []byte, profile, region string) config, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, config.WithRegion(region), config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ Value: creds, @@ -275,7 +285,7 @@ func UseProviderSecretAssumeRole(ctx context.Context, data []byte, profile, regi config, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, config.WithRegion(region), config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ Value: creds, @@ -319,7 +329,7 @@ func UsePodServiceAccountAssumeRole(ctx context.Context, _ []byte, _, region str stsAssumeRoleOptions := SetAssumeRoleOptions(pc) cnf, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, config.WithRegion(cfg.Region), config.WithCredentialsProvider(aws.NewCredentialsCache( stscreds.NewAssumeRoleProvider( @@ -339,7 +349,7 @@ func UsePodServiceAccountAssumeRole(ctx context.Context, _ []byte, _, region str // configured via a ServiceAccount assume Cross account IAM roles // https://aws.amazon.com/blogs/containers/cross-account-iam-roles-for-kubernetes-service-accounts/ func UsePodServiceAccountAssumeRoleWithWebIdentity(ctx context.Context, _ []byte, _, region string, pc *v1beta1.ProviderConfig) (*aws.Config, error) { - cfg, err := config.LoadDefaultConfig(ctx, userAgentV2) + cfg, err := config.LoadDefaultConfig(ctx, middlewareV2) if err != nil { return nil, errors.Wrap(err, "failed to load default AWS config") } @@ -354,7 +364,7 @@ func UsePodServiceAccountAssumeRoleWithWebIdentity(ctx context.Context, _ []byte cnf, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, config.WithRegion(region), config.WithCredentialsProvider(aws.NewCredentialsCache( stscreds.NewWebIdentityRoleProvider( @@ -386,13 +396,13 @@ func UsePodServiceAccount(ctx context.Context, _ []byte, _, region string) (*aws if region == GlobalRegion { cfg, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, ) return &cfg, errors.Wrap(err, "failed to load default AWS config") } cfg, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, config.WithRegion(region), ) if err != nil { @@ -468,6 +478,9 @@ func GetSessionV1(cfg *awsv1.Config) (*session.Session, error) { return nil, err } session.Handlers.Build.PushBackNamed(userAgentV1) + session.Handlers.Send.PushFront(func(r *requestv1.Request) { + metrics.IncAWSAPICall(r.ClientInfo.ServiceName, r.Operation.Name, "1") + }) return session, nil } @@ -481,7 +494,7 @@ func UseProviderSecretV1AssumeRole(ctx context.Context, data []byte, pc *v1beta1 config, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, config.WithRegion(region), config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ Value: creds, @@ -554,7 +567,7 @@ func UseProviderSecretV1(_ context.Context, data []byte, pc *v1beta1.ProviderCon // assume Cross account IAM role // https://aws.amazon.com/blogs/containers/cross-account-iam-roles-for-kubernetes-service-accounts/ func UsePodServiceAccountV1AssumeRole(ctx context.Context, _ []byte, pc *v1beta1.ProviderConfig, _, region string) (*awsv1.Config, error) { - cfg, err := config.LoadDefaultConfig(ctx, userAgentV2) + cfg, err := config.LoadDefaultConfig(ctx, middlewareV2) if err != nil { return nil, errors.Wrap(err, "failed to load default AWS config") } @@ -570,7 +583,7 @@ func UsePodServiceAccountV1AssumeRole(ctx context.Context, _ []byte, pc *v1beta1 } cnf, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, config.WithRegion(region), config.WithCredentialsProvider(aws.NewCredentialsCache( stscreds.NewAssumeRoleProvider( @@ -598,7 +611,7 @@ func UsePodServiceAccountV1AssumeRole(ctx context.Context, _ []byte, pc *v1beta1 // assume Cross account IAM role // https://aws.amazon.com/blogs/containers/cross-account-iam-roles-for-kubernetes-service-accounts/ func UsePodServiceAccountV1AssumeRoleWithWebIdentity(ctx context.Context, _ []byte, pc *v1beta1.ProviderConfig, _, region string) (*awsv1.Config, error) { - cfg, err := config.LoadDefaultConfig(ctx, userAgentV2) + cfg, err := config.LoadDefaultConfig(ctx, middlewareV2) if err != nil { return nil, errors.Wrap(err, "failed to load default AWS config") } @@ -613,7 +626,7 @@ func UsePodServiceAccountV1AssumeRoleWithWebIdentity(ctx context.Context, _ []by cnf, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, config.WithRegion(region), config.WithCredentialsProvider(aws.NewCredentialsCache( stscreds.NewWebIdentityRoleProvider( @@ -643,7 +656,7 @@ func UsePodServiceAccountV1AssumeRoleWithWebIdentity(ctx context.Context, _ []by func UsePodServiceAccountV1(ctx context.Context, _ []byte, pc *v1beta1.ProviderConfig, _, region string) (*awsv1.Config, error) { cfg, err := config.LoadDefaultConfig( ctx, - userAgentV2, + middlewareV2, ) if err != nil { return nil, errors.Wrap(err, "failed to load default AWS config") diff --git a/pkg/utils/metrics/setup.go b/pkg/utils/metrics/setup.go new file mode 100644 index 0000000000..66ec5ba566 --- /dev/null +++ b/pkg/utils/metrics/setup.go @@ -0,0 +1,24 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + + k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + metricAWSAPICalls = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "aws_api_calls_total", + Help: "Number of API calls to the AWS API", + }, []string{"service", "operation", "api_version"}) +) + +// SetupMetrics will register the known Prometheus metrics with controller-runtime's metrics registry +func SetupMetrics() error { + return k8smetrics.Registry.Register(metricAWSAPICalls) +} + +// IncAWSAPICall will increment the aws_api_calls_total metric for the specified service, operation, and apiVersion tuple +func IncAWSAPICall(service, operation, apiVersion string) { + metricAWSAPICalls.WithLabelValues(service, operation, apiVersion).Inc() +}