Merge pull request #1862 from swisscom/feature/aws-api-call-metrics

cmd/provider/main.go

@@ -41,6 +41,7 @@ import (
 	"github.com/crossplane-contrib/provider-aws/apis/v1alpha1"
 	"github.com/crossplane-contrib/provider-aws/pkg/controller"
 	"github.com/crossplane-contrib/provider-aws/pkg/features"
+	"github.com/crossplane-contrib/provider-aws/pkg/utils/metrics"
 )
 
 func main() {
@@ -123,6 +124,7 @@ func main() {
 		log.Info("Alpha feature enabled", "flag", features.EnableAlphaManagementPolicies)
 	}
 
+	kingpin.FatalIfError(metrics.SetupMetrics(), "Cannot setup AWS metrics hook")
 	kingpin.FatalIfError(controller.Setup(mgr, o), "Cannot setup AWS controllers")
 	kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
 

go.mod

@@ -39,6 +39,7 @@ require (
 	github.com/mitchellh/copystructure v1.0.0
 	github.com/onsi/gomega v1.27.7
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.15.1
 	go.uber.org/zap v1.24.0
 	golang.org/x/net v0.12.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
@@ -121,10 +122,9 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/operator-framework/api v0.6.0 // indirect
-	github.com/prometheus/client_golang v1.15.1 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.10.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/samber/lo v1.37.0 // indirect
 	github.com/sergi/go-diff v1.0.0 // indirect

go.sum

@@ -683,8 +683,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.10.0 h1:UkG7GPYkO4UZyLnyXjaWYcgOSONqwdBqFUT95ugmt6I=
-github.com/prometheus/procfs v0.10.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=

pkg/clients/config.go

@@ -45,6 +45,7 @@ import (
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
 
 	"github.com/crossplane-contrib/provider-aws/apis/v1beta1"
+	"github.com/crossplane-contrib/provider-aws/pkg/utils/metrics"
 	"github.com/crossplane-contrib/provider-aws/pkg/version"
 )
 
@@ -74,9 +75,18 @@ const (
 	FieldRequired FieldOption = iota
 )
 
-// userAgentV2 constructs the Crossplane user agent for AWS v2 clients
-var userAgentV2 = config.WithAPIOptions([]func(*middleware.Stack) error{
+// middlewareV2 constructs the AWS SDK v2 middleware
+var middlewareV2 = config.WithAPIOptions([]func(*middleware.Stack) error{
 	awsmiddleware.AddUserAgentKeyValue("crossplane-provider-aws", version.Version),
+	func(s *middleware.Stack) error {
+		return s.Finalize.Add(recordRequestMetrics, middleware.After)
+	},
+})
+
+// recordRequestMetrics records Prometheus metrics for requests to the AWS APIs
+var recordRequestMetrics = middleware.FinalizeMiddlewareFunc("recordRequestMetrics", func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) {
+	metrics.IncAWSAPICall(awsmiddleware.GetServiceID(ctx), awsmiddleware.GetOperationName(ctx), "2")
+	return next.HandleFinalize(ctx, in)
 })
 
 // userAgentV1 constructs the Crossplane user agent for AWS v1 clients
@@ -256,7 +266,7 @@ func UseProviderSecret(ctx context.Context, data []byte, profile, region string)
 
 	config, err := config.LoadDefaultConfig(
 		ctx,
-		userAgentV2,
+		middlewareV2,
 		config.WithRegion(region),
 		config.WithCredentialsProvider(credentials.StaticCredentialsProvider{
 			Value: creds,
@@ -275,7 +285,7 @@ func UseProviderSecretAssumeRole(ctx context.Context, data []byte, profile, regi
 
 	config, err := config.LoadDefaultConfig(
 		ctx,
-		userAgentV2,
+		middlewareV2,
 		config.WithRegion(region),
 		config.WithCredentialsProvider(credentials.StaticCredentialsProvider{
 			Value: creds,
@@ -319,7 +329,7 @@ func UsePodServiceAccountAssumeRole(ctx context.Context, _ []byte, _, region str
 	stsAssumeRoleOptions := SetAssumeRoleOptions(pc)
 	cnf, err := config.LoadDefaultConfig(
 		ctx,
-		userAgentV2,
+		middlewareV2,
 		config.WithRegion(cfg.Region),
 		config.WithCredentialsProvider(aws.NewCredentialsCache(
 			stscreds.NewAssumeRoleProvider(
@@ -339,7 +349,7 @@ func UsePodServiceAccountAssumeRole(ctx context.Context, _ []byte, _, region str
 // configured via a ServiceAccount assume Cross account IAM roles
 // https://aws.amazon.com/blogs/containers/cross-account-iam-roles-for-kubernetes-service-accounts/
 func UsePodServiceAccountAssumeRoleWithWebIdentity(ctx context.Context, _ []byte, _, region string, pc *v1beta1.ProviderConfig) (*aws.Config, error) {
-	cfg, err := config.LoadDefaultConfig(ctx, userAgentV2)
+	cfg, err := config.LoadDefaultConfig(ctx, middlewareV2)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load default AWS config")
 	}
@@ -354,7 +364,7 @@ func UsePodServiceAccountAssumeRoleWithWebIdentity(ctx context.Context, _ []byte
 
 	cnf, err := config.LoadDefaultConfig(
 		ctx,
-		userAgentV2,
+		middlewareV2,
 		config.WithRegion(region),
 		config.WithCredentialsProvider(aws.NewCredentialsCache(
 			stscreds.NewWebIdentityRoleProvider(
@@ -386,13 +396,13 @@ func UsePodServiceAccount(ctx context.Context, _ []byte, _, region string) (*aws
 	if region == GlobalRegion {
 		cfg, err := config.LoadDefaultConfig(
 			ctx,
-			userAgentV2,
+			middlewareV2,
 		)
 		return &cfg, errors.Wrap(err, "failed to load default AWS config")
 	}
 	cfg, err := config.LoadDefaultConfig(
 		ctx,
-		userAgentV2,
+		middlewareV2,
 		config.WithRegion(region),
 	)
 	if err != nil {
@@ -468,6 +478,9 @@ func GetSessionV1(cfg *awsv1.Config) (*session.Session, error) {
 		return nil, err
 	}
 	session.Handlers.Build.PushBackNamed(userAgentV1)
+	session.Handlers.Send.PushFront(func(r *requestv1.Request) {
+		metrics.IncAWSAPICall(r.ClientInfo.ServiceName, r.Operation.Name, "1")
+	})
 	return session, nil
 }
 
@@ -481,7 +494,7 @@ func UseProviderSecretV1AssumeRole(ctx context.Context, data []byte, pc *v1beta1
 
 	config, err := config.LoadDefaultConfig(
 		ctx,
-		userAgentV2,
+		middlewareV2,
 		config.WithRegion(region),
 		config.WithCredentialsProvider(credentials.StaticCredentialsProvider{
 			Value: creds,
@@ -554,7 +567,7 @@ func UseProviderSecretV1(_ context.Context, data []byte, pc *v1beta1.ProviderCon
 // assume Cross account IAM role
 // https://aws.amazon.com/blogs/containers/cross-account-iam-roles-for-kubernetes-service-accounts/
 func UsePodServiceAccountV1AssumeRole(ctx context.Context, _ []byte, pc *v1beta1.ProviderConfig, _, region string) (*awsv1.Config, error) {
-	cfg, err := config.LoadDefaultConfig(ctx, userAgentV2)
+	cfg, err := config.LoadDefaultConfig(ctx, middlewareV2)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load default AWS config")
 	}
@@ -570,7 +583,7 @@ func UsePodServiceAccountV1AssumeRole(ctx context.Context, _ []byte, pc *v1beta1
 	}
 	cnf, err := config.LoadDefaultConfig(
 		ctx,
-		userAgentV2,
+		middlewareV2,
 		config.WithRegion(region),
 		config.WithCredentialsProvider(aws.NewCredentialsCache(
 			stscreds.NewAssumeRoleProvider(
@@ -598,7 +611,7 @@ func UsePodServiceAccountV1AssumeRole(ctx context.Context, _ []byte, pc *v1beta1
 // assume Cross account IAM role
 // https://aws.amazon.com/blogs/containers/cross-account-iam-roles-for-kubernetes-service-accounts/
 func UsePodServiceAccountV1AssumeRoleWithWebIdentity(ctx context.Context, _ []byte, pc *v1beta1.ProviderConfig, _, region string) (*awsv1.Config, error) {
-	cfg, err := config.LoadDefaultConfig(ctx, userAgentV2)
+	cfg, err := config.LoadDefaultConfig(ctx, middlewareV2)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load default AWS config")
 	}
@@ -613,7 +626,7 @@ func UsePodServiceAccountV1AssumeRoleWithWebIdentity(ctx context.Context, _ []by
 
 	cnf, err := config.LoadDefaultConfig(
 		ctx,
-		userAgentV2,
+		middlewareV2,
 		config.WithRegion(region),
 		config.WithCredentialsProvider(aws.NewCredentialsCache(
 			stscreds.NewWebIdentityRoleProvider(
@@ -643,7 +656,7 @@ func UsePodServiceAccountV1AssumeRoleWithWebIdentity(ctx context.Context, _ []by
 func UsePodServiceAccountV1(ctx context.Context, _ []byte, pc *v1beta1.ProviderConfig, _, region string) (*awsv1.Config, error) {
 	cfg, err := config.LoadDefaultConfig(
 		ctx,
-		userAgentV2,
+		middlewareV2,
 	)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load default AWS config")

pkg/utils/metrics/setup.go

@@ -0,0 +1,24 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+var (
+	metricAWSAPICalls = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "aws_api_calls_total",
+		Help: "Number of API calls to the AWS API",
+	}, []string{"service", "operation", "api_version"})
+)
+
+// SetupMetrics will register the known Prometheus metrics with controller-runtime's metrics registry
+func SetupMetrics() error {
+	return k8smetrics.Registry.Register(metricAWSAPICalls)
+}
+
+// IncAWSAPICall will increment the aws_api_calls_total metric for the specified service, operation, and apiVersion tuple
+func IncAWSAPICall(service, operation, apiVersion string) {
+	metricAWSAPICalls.WithLabelValues(service, operation, apiVersion).Inc()
+}