add integration test framework

.github/workflows/build.yaml

@@ -13,11 +13,34 @@ jobs:
     steps:
     - uses: actions/setup-go@v4
       with:
-        go-version: 1.21.6
+        go-version: "1.21.6"
     - uses: actions/checkout@v4
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v4
     - name: Run build
-      run: go build -o haproxy-ingress pkg/main.go
-    - name: Run tests
-      run: go test ./...
+      run: make build
+    - name: Run unit tests
+      run: make test
+  integration:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Install dependencies
+      run: sudo apt-get install -y lua-json
+    - name: Install HAProxy
+      uses: timwolla/action-install-haproxy@main
+      id: install-haproxy
+      with:
+        branch: "2.2"
+        use_openssl: yes
+        use_lua: yes
+    - name: Install Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: "1.21.6"
+    - uses: actions/checkout@v4
+    - uses: docker/login-action@v3
+      with:
+        username: jcmoraisjr
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Run integration tests
+      run: make test-integration

Makefile

@@ -13,6 +13,10 @@ KUBECONFIG?=$(HOME)/.kube/config
 CONTROLLER_CONFIGMAP?=
 CONTROLLER_ARGS?=
 
+LOCALBIN?=$(shell pwd)/bin
+LOCAL_GOTESTSUM=$(LOCALBIN)/gotestsum
+LOCAL_SETUP_ENVTEST=$(LOCALBIN)/setup-envtest
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 go build \
@@ -32,14 +36,28 @@ run: build
 	  --configmap=$(CONTROLLER_CONFIGMAP)\
 	  $(CONTROLLER_ARGS)
 
+.PHONY: gotestsum
+gotestsum:
+	test -x $(LOCAL_GOTESTSUM) || GOBIN=$(LOCALBIN) go install gotest.tools/gotestsum@latest
+
 .PHONY: lint
 lint:
 	golangci-lint run
 
 .PHONY: test
-test: lint
+test: lint gotestsum
 	## fix race and add -race param
-	go test -tags cgo ./...
+	$(LOCAL_GOTESTSUM) --format=testname -- -tags=cgo ./pkg/...
+
+.PHONY: setup-envtest
+setup-envtest:
+	test -x $(LOCAL_SETUP_ENVTEST) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+	$(LOCAL_SETUP_ENVTEST) use 1.29.1 --bin-dir $(LOCALBIN)
+
+.PHONY: test-integration
+test-integration: gotestsum setup-envtest
+	KUBEBUILDER_ASSETS="$(shell $(LOCAL_SETUP_ENVTEST) use 1.29.1 --bin-dir $(LOCALBIN) -i -p path)"\
+		$(LOCAL_GOTESTSUM) --format=testname -- -count=1 -tags=cgo ./tests/integration/...
 
 .PHONY: linux-build
 linux-build:

README.md

@@ -99,8 +99,9 @@ The following `make` targets are supported:
 
 * `build` (default): Compiles HAProxy Ingress using the default OS and arch, and generates an executable at `bin/controller`.
 * `run`: Runs HAProxy Ingress locally.
-* `lint`: Runs [`golangci-lint`](https://golangci-lint.run/).
+* `lint`: Runs [`golangci-lint`](https://golangci-lint.run/), needs golangci-lint in the path.
 * `test`: Runs unit tests.
+* `test-integration`: Runs integration tests, needs haproxy 2.2+ in the path.
 * `linux-build`: Compiles HAProxy Ingress and generates an ELF (Linux) executable despite the source platform at `rootfs/haproxy-ingress-controller`. Used by `image` step.
 * `image`: Compiles HAProxy Ingress locally and generates a Docker image.
 * `docker-build`: Compiles HAProxy Ingress and generates a Docker image using a multi-stage Dockerfile.

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/prometheus/client_golang v1.18.0
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.8.4
 	go.uber.org/zap v1.26.0
 	golang.org/x/crypto v0.21.0
 	gopkg.in/go-playground/pool.v3 v3.1.1
@@ -58,6 +59,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect

pkg/controller/config/config.go

@@ -33,7 +33,7 @@ import (
 	"go.uber.org/zap"
 	"go.uber.org/zap/buffer"
 	"go.uber.org/zap/zapcore"
-	v1 "k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -77,13 +77,16 @@ func CreateWithConfig(ctx context.Context, restConfig *rest.Config, opt *Options
 
 	if !opt.LogZap {
 		if opt.LogDev || opt.LogCaller || opt.LogEnableStacktrace || opt.LogEncoder != "" || opt.LogEncodeTime != "" {
-			klog.Exit("--log-dev, --log-caller, --log-enable-stacktrace --log-encoder and --log-encode-time are only supported if --log-zap is enabled.")
+			return nil, fmt.Errorf("--log-dev, --log-caller, --log-enable-stacktrace --log-encoder and --log-encode-time are only supported if --log-zap is enabled")
 		}
 		var level klog.Level
 		level.Set(strconv.Itoa(opt.LogLevel - 1))
 		ctrl.SetLogger(klog.NewKlogr())
 	} else {
-		logger := newZapLogger(opt.LogDev, opt.LogLevel, opt.LogCaller, opt.LogEnableStacktrace, opt.LogEncoder, opt.LogEncodeTime)
+		logger, err := newZapLogger(opt.LogDev, opt.LogLevel, opt.LogCaller, opt.LogEnableStacktrace, opt.LogEncoder, opt.LogEncodeTime)
+		if err != nil {
+			return nil, err
+		}
 		ctrl.SetLogger(logger)
 		klog.SetLogger(logger)
 	}
@@ -100,14 +103,11 @@ func CreateWithConfig(ctx context.Context, restConfig *rest.Config, opt *Options
 	switch {
 	case restConfig != nil:
 		kubeConfig = restConfig
-	case opt.ApiserverHost == "" || opt.KubeConfig.String() == "":
-		var err error
-		kubeConfig, err = ctrl.GetConfig()
-		if err != nil {
-			return nil, err
-		}
-	default:
+	case opt.ApiserverHost != "":
 		kubeConfigFile := opt.KubeConfig.String()
+		if kubeConfigFile == "" {
+			return nil, fmt.Errorf("--kubeconfig is mandatory when --apiserver-host is configured")
+		}
 		var err error
 		kubeConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
 			&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeConfigFile},
@@ -119,10 +119,17 @@ func CreateWithConfig(ctx context.Context, restConfig *rest.Config, opt *Options
 		if err != nil {
 			return nil, err
 		}
+	default:
+		var err error
+		kubeConfig, err = ctrl.GetConfig()
+		if err != nil {
+			return nil, err
+		}
 	}
 	if opt.DisableAPIWarnings {
 		kubeConfig.WarningHandler = rest.NoWarnings{}
 	}
+
 	// `kubeConfig` is the real `*rest.Config` used
 	// by the manager to create the controller's client
 	//
@@ -350,7 +357,7 @@ func CreateWithConfig(ctx context.Context, restConfig *rest.Config, opt *Options
 		return nil, fmt.Errorf("resync period (%vs) is too low", opt.ResyncPeriod.Seconds())
 	}
 
-	if opt.WatchNamespace != v1.NamespaceAll && opt.AllowCrossNamespace {
+	if opt.WatchNamespace != corev1.NamespaceAll && opt.AllowCrossNamespace {
 		return nil, fmt.Errorf("cannot use --watch-namespace if --force-namespace-isolation is true")
 	}
 
@@ -488,7 +495,7 @@ func CreateWithConfig(ctx context.Context, restConfig *rest.Config, opt *Options
 	}, nil
 }
 
-func newZapLogger(logDev bool, logLevel int, logCaller, logEnableStacktrace bool, logEncoder, logEncodeTime string) logr.Logger {
+func newZapLogger(logDev bool, logLevel int, logCaller, logEnableStacktrace bool, logEncoder, logEncodeTime string) (logr.Logger, error) {
 	var zc zap.Config
 	if logDev {
 		zc = zap.NewDevelopmentConfig()
@@ -517,14 +524,14 @@ func newZapLogger(logDev bool, logLevel int, logCaller, logEnableStacktrace bool
 	case "console":
 		baseEncoder = zapcore.NewConsoleEncoder
 	default:
-		klog.Exitf("invalid encode name: %s", logEncoder)
+		return logr.Logger{}, fmt.Errorf("invalid encode name: %s", logEncoder)
 	}
 
 	klogEncoderName := "klog"
 	if err := zap.RegisterEncoder(klogEncoderName, func(ec zapcore.EncoderConfig) (zapcore.Encoder, error) {
 		return klogEncoder{baseEncoder(ec)}, nil
 	}); err != nil {
-		klog.Exitf("error registering log encoder: %v", err)
+		return logr.Logger{}, fmt.Errorf("error registering log encoder: %v", err)
 	}
 
 	zc.Encoding = klogEncoderName
@@ -537,9 +544,9 @@ func newZapLogger(logDev bool, logLevel int, logCaller, logEnableStacktrace bool
 		zap.AddCallerSkip(0),
 	)
 	if err != nil {
-		klog.Exitf("error configuring zap logger: %v", err)
+		return logr.Logger{}, fmt.Errorf("error configuring zap logger: %v", err)
 	}
-	return zapr.NewLogger(zl)
+	return zapr.NewLogger(zl), nil
 }
 
 type klogEncoder struct {

pkg/controller/config/flags.go

@@ -19,10 +19,6 @@ type stringValue struct {
 	val string
 }
 
-func (s *stringValue) Get() interface{} {
-	return s.val
-}
-
 func (s *stringValue) Set(val string) error {
 	s.val = val
 	return nil
@@ -43,10 +39,6 @@ func FlagFloat64SliceVar(fs *flag.FlagSet, p *[]float64, name string, value []fl
 
 type float64SliceValue []float64
 
-func (f *float64SliceValue) Get() interface{} {
-	return (*[]float64)(f)
-}
-
 func (f *float64SliceValue) Set(val string) error {
 	s := strings.Split(val, ",")
 	*f = make([]float64, len(s))

pkg/controller/config/options.go

@@ -4,7 +4,7 @@ import (
 	"flag"
 	"time"
 
-	core "k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 )
 
@@ -24,7 +24,7 @@ func NewOptions() *Options {
 		RateLimitUpdate:         0.5,
 		WaitBeforeUpdate:        200 * time.Millisecond,
 		ResyncPeriod:            10 * time.Hour,
-		WatchNamespace:          core.NamespaceAll,
+		WatchNamespace:          corev1.NamespaceAll,
 		StatsCollectProcPeriod:  500 * time.Millisecond,
 		HealthzAddr:             ":10254",
 		HealthzURL:              "/healthz",
@@ -130,7 +130,8 @@ func (o *Options) AddFlags(fs *flag.FlagSet) {
 	fs.StringVar(&o.ApiserverHost, "apiserver-host", o.ApiserverHost, ""+
 		"The address of the Kubernetes API server to connect to, in the format of "+
 		"protocol://address:port, e.g., http://localhost:8080. If not specified, the "+
-		"default value from in cluster discovery or from a provided kubeconfig is used.",
+		"default value from in cluster discovery or from a provided kubeconfig is used. "+
+		"A valid kubeconfig must be provided if used.",
 	)
 
 	fs.StringVar(&o.LocalFSPrefix, "local-filesystem-prefix", o.LocalFSPrefix, ""+

pkg/controller/launch/launch.go

@@ -65,11 +65,12 @@ func Run(cfg *config.Config) error {
 	}
 
 	launchLog.Info("configuring ingress reconciler")
-	if err := (&reconciler.IngressReconciler{
+	ingress := &reconciler.IngressReconciler{
 		Client:   mgr.GetClient(),
 		Config:   cfg,
 		Services: services,
-	}).SetupWithManager(ctx, mgr); err != nil {
+	}
+	if err := ingress.SetupWithManager(ctx, mgr); err != nil {
 		return fmt.Errorf("unable to create controller: %w", err)
 	}
 

pkg/converters/utils/services.go

@@ -76,15 +76,25 @@ type Endpoint struct {
 }
 
 func createEndpoints(endpoints *api.Endpoints, svcPort *api.ServicePort) (ready, notReady []*Endpoint, err error) {
+	var ipOverride string
+	if ann := endpoints.Annotations; ann != nil {
+		ipOverride = ann["haproxy-ingress.github.io/ip-override"]
+	}
+	resolveIP := func(ip string) string {
+		if ipOverride != "" {
+			return ipOverride
+		}
+		return ip
+	}
 	for _, subset := range endpoints.Subsets {
 		for _, epPort := range subset.Ports {
 			if matchPort(svcPort, &epPort) {
 				port := int(epPort.Port)
 				for _, addr := range subset.Addresses {
-					ready = append(ready, newEndpoint(addr.IP, port, addr.TargetRef))
+					ready = append(ready, newEndpoint(resolveIP(addr.IP), port, addr.TargetRef))
 				}
 				for _, addr := range subset.NotReadyAddresses {
-					notReady = append(notReady, newEndpoint(addr.IP, port, addr.TargetRef))
+					notReady = append(notReady, newEndpoint(resolveIP(addr.IP), port, addr.TargetRef))
 				}
 			}
 		}

pkg/main.go

@@ -46,8 +46,7 @@ func run() {
 	fs.Parse(os.Args[1:])
 	cfg, err := config.Create(opt)
 	if err != nil {
-		log.Printf("unable to parse static config: %s\n", err)
-		os.Exit(1)
+		log.Fatalf("unable to parse static config: %s\n", err)
 	}
 	if err := launch.Run(cfg); err != nil {
 		log.Fatal(err.Error())