From 8e0a3e0fb8ff1ad4589c641b79f0acc0d6a99e6e Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 6 Dec 2024 02:32:47 +0000 Subject: [PATCH 1/2] support http(s) proxy Signed-off-by: Ryotaro Banno --- charts/mantle/templates/deployment.yaml | 9 +++++ cmd/controller/main.go | 30 ++++++++++++++++ .../controller/mantlebackup_controller.go | 21 +++++++++++ .../mantlebackup_controller_test.go | 36 ++++++++++++++++++- .../mantlerestore_controller_test.go | 1 + .../values-mantle-primary-template.yaml | 3 ++ 6 files changed, 99 insertions(+), 1 deletion(-) diff --git a/charts/mantle/templates/deployment.yaml b/charts/mantle/templates/deployment.yaml index ef4eb44..9fa3a7a 100644 --- a/charts/mantle/templates/deployment.yaml +++ b/charts/mantle/templates/deployment.yaml @@ -88,6 +88,15 @@ spec: {{- with .Values.controller.gcInterval }} - --gc-interval={{ . }} {{- end }} + {{- with .Values.controller.httpProxy }} + - --http-proxy={{ . }} + {{- end }} + {{- with .Values.controller.httpsProxy }} + - --https-proxy={{ . }} + {{- end }} + {{- with .Values.controller.noProxy }} + - --no-proxy={{ . }} + {{- end }} env: - name: POD_NAME valueFrom: diff --git a/cmd/controller/main.go b/cmd/controller/main.go index f9837be..867e99d 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -57,6 +57,9 @@ var ( caCertConfigMapSrc string caCertKeySrc string gcInterval string + httpProxy string + httpsProxy string + noProxy string scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") @@ -98,6 +101,12 @@ func init() { "The default value is ca.crt. This option is just ignored if --ca-cert-configmap isn't specified.") flags.StringVar(&gcInterval, "gc-interval", "1h", "The time period between each garbage collection for orphaned resources.") + flags.StringVar(&httpProxy, "http-proxy", "", + "The proxy URL for HTTP requests to the object storage and the gRPC endpoint of secondary mantle.") + flags.StringVar(&httpsProxy, "https-proxy", "", + "The proxy URL for HTTPS requests to the object storage and the gRPC endpoint of secondary mantle.") + flags.StringVar(&noProxy, "no-proxy", "", + "A string that contains comma-separated values specifying hosts that should be excluded from proxying.") goflags := flag.NewFlagSet("goflags", flag.ExitOnError) zapOpts.Development = true @@ -179,6 +188,11 @@ func setupReconcilers(mgr manager.Manager, primarySettings *controller.PrimarySe CACertConfigMap: caCertConfigMap, CACertKey: &caCertKeySrc, }, + &controller.ProxySettings{ + HttpProxy: httpProxy, + HttpsProxy: httpsProxy, + NoProxy: noProxy, + }, ) if err := backupReconciler.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "MantleBackup") @@ -233,6 +247,22 @@ func setupStandalone(mgr manager.Manager) error { } func setupPrimary(ctx context.Context, mgr manager.Manager, wg *sync.WaitGroup) error { + // Setup environment variables related to proxies before creating a gRPC client. + // cf. https://github.com/grpc/grpc-go/blob/adad26df1826bf2fb66ad56ff32a62b98bf5cb3a/Documentation/proxy.md + // cf. https://pkg.go.dev/golang.org/x/net/http/httpproxy + if err := os.Setenv("HTTP_PROXY", httpProxy); err != nil { + setupLog.Error(err, "failed to set HTTP_PROXY environment variable") + return err + } + if err := os.Setenv("HTTPS_PROXY", httpsProxy); err != nil { + setupLog.Error(err, "failed to set HTTPS_PROXY environment variable") + return err + } + if err := os.Setenv("NO_PROXY", noProxy); err != nil { + setupLog.Error(err, "failed to set NO_PROXY environment variable") + return err + } + conn, err := grpc.NewClient( mantleServiceEndpoint, grpc.WithTransportCredentials(insecure.NewCredentials()), diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go index a0a4223..8080e1b 100644 --- a/internal/controller/mantlebackup_controller.go +++ b/internal/controller/mantlebackup_controller.go @@ -72,6 +72,12 @@ type ObjectStorageSettings struct { Endpoint string } +type ProxySettings struct { + HttpProxy string + HttpsProxy string + NoProxy string +} + // MantleBackupReconciler reconciles a MantleBackup object type MantleBackupReconciler struct { client.Client @@ -85,6 +91,7 @@ type MantleBackupReconciler struct { envSecret string objectStorageSettings *ObjectStorageSettings // This should be non-nil if and only if role equals 'primary' or 'secondary'. objectStorageClient objectstorage.Bucket + proxySettings *ProxySettings } // NewMantleBackupReconciler returns NodeReconciler. @@ -97,6 +104,7 @@ func NewMantleBackupReconciler( podImage string, envSecret string, objectStorageSettings *ObjectStorageSettings, + proxySettings *ProxySettings, ) *MantleBackupReconciler { return &MantleBackupReconciler{ Client: client, @@ -109,6 +117,7 @@ func NewMantleBackupReconciler( podImage: podImage, envSecret: envSecret, objectStorageSettings: objectStorageSettings, + proxySettings: proxySettings, } } @@ -1325,6 +1334,18 @@ func (r *MantleBackupReconciler) createOrUpdateExportDataUploadJob(ctx context.C }, }, }, + { + Name: "HTTP_PROXY", + Value: r.proxySettings.HttpProxy, + }, + { + Name: "HTTPS_PROXY", + Value: r.proxySettings.HttpsProxy, + }, + { + Name: "NO_PROXY", + Value: r.proxySettings.NoProxy, + }, }, Image: r.podImage, ImagePullPolicy: corev1.PullIfNotPresent, diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index 0fc263f..3a235c1 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -34,6 +34,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" ) +func getEnvValue(envVarAry []corev1.EnvVar, name string) (string, error) { + for _, env := range envVarAry { + if env.Name == name { + return env.Value, nil + } + } + return "", errors.New("name not found") +} + var _ = Describe("MantleBackup controller", func() { var mgrUtil testutil.ManagerUtil var reconciler *MantleBackupReconciler @@ -121,6 +130,7 @@ var _ = Describe("MantleBackup controller", func() { "dummy image", "", nil, + nil, ) reconciler.ceph = testutil.NewFakeRBD() err := reconciler.SetupWithManager(mgrUtil.GetManager()) @@ -332,6 +342,11 @@ var _ = Describe("MantleBackup controller", func() { Context("when the role is `primary`", func() { var mockCtrl *gomock.Controller var grpcClient *proto.MockMantleServiceClient + proxySettings := &ProxySettings{ + HttpProxy: "dummy http proxy", + HttpsProxy: "dummy https proxy", + NoProxy: "no proxy", + } BeforeEach(func() { mgrUtil = testutil.NewManagerUtil(context.Background(), cfg, scheme.Scheme) @@ -355,6 +370,7 @@ var _ = Describe("MantleBackup controller", func() { CACertConfigMap: nil, CACertKey: nil, }, + proxySettings, ) reconciler.ceph = testutil.NewFakeRBD() @@ -548,6 +564,17 @@ var _ = Describe("MantleBackup controller", func() { g.Expect(*jobUpload.Spec.Template.Spec.SecurityContext.RunAsUser).To(Equal(int64(10000))) g.Expect(*jobUpload.Spec.Template.Spec.SecurityContext.RunAsGroup).To(Equal(int64(10000))) g.Expect(*jobUpload.Spec.Template.Spec.SecurityContext.RunAsNonRoot).To(Equal(true)) + + // Make sure HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables are correctly set. + httpProxy, err := getEnvValue(jobUpload.Spec.Template.Spec.Containers[0].Env, "HTTP_PROXY") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(httpProxy).To(Equal(proxySettings.HttpProxy)) + httpsProxy, err := getEnvValue(jobUpload.Spec.Template.Spec.Containers[0].Env, "HTTPS_PROXY") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(httpsProxy).To(Equal(proxySettings.HttpsProxy)) + noProxy, err := getEnvValue(jobUpload.Spec.Template.Spec.Containers[0].Env, "NO_PROXY") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(noProxy).To(Equal(proxySettings.NoProxy)) }).WithContext(ctx).Should(Succeed()) // Make the all existing MantleBackups in the primary Mantle @@ -800,7 +827,7 @@ var _ = Describe("prepareForDataSynchronization", func() { } mbr := NewMantleBackupReconciler(ctrlClient, - ctrlClient.Scheme(), "test", RolePrimary, nil, "dummy image", "", nil) + ctrlClient.Scheme(), "test", RolePrimary, nil, "dummy image", "", nil, nil) ret, err := mbr.prepareForDataSynchronization(context.Background(), backup, grpcClient) @@ -1232,6 +1259,11 @@ var _ = Describe("export", func() { "dummy image", "", nil, + &ProxySettings{ + HttpProxy: "", + HttpsProxy: "", + NoProxy: "", + }, ) ns = resMgr.CreateNamespace() @@ -1351,6 +1383,7 @@ var _ = Describe("export", func() { "dummy image", "", nil, + nil, ) ns2 := resMgr.CreateNamespace() createAndExportMantleBackup(mbr2, "target2", ns2) @@ -1385,6 +1418,7 @@ var _ = Describe("import", func() { "dummy-image", "dummy-env-secret", &ObjectStorageSettings{}, + nil, ) mbr.objectStorageClient = mockObjectStorage mbr.ceph = testutil.NewFakeRBD() diff --git a/internal/controller/mantlerestore_controller_test.go b/internal/controller/mantlerestore_controller_test.go index bed6641..5614d8c 100644 --- a/internal/controller/mantlerestore_controller_test.go +++ b/internal/controller/mantlerestore_controller_test.go @@ -58,6 +58,7 @@ func (test *mantleRestoreControllerUnitTest) setupEnv() { "dummy image", "", nil, + nil, ) backupReconciler.ceph = testutil.NewFakeRBD() err := backupReconciler.SetupWithManager(test.mgrUtil.GetManager()) diff --git a/test/e2e/testdata/values-mantle-primary-template.yaml b/test/e2e/testdata/values-mantle-primary-template.yaml index cbcee13..6da0541 100644 --- a/test/e2e/testdata/values-mantle-primary-template.yaml +++ b/test/e2e/testdata/values-mantle-primary-template.yaml @@ -6,3 +6,6 @@ controller: envSecret: export-data exportDataStorageClass: rook-ceph-block gcInterval: 1s + #httpProxy: http://host.minikube.internal:8899 + #httpsProxy: http://host.minikube.internal:8899 + #noProxy: localhost,127.0.0.1,10.96.0.0/12 From bbf286cbad9843be3befb2d34cd48b81a094584a Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 6 Dec 2024 08:28:18 +0000 Subject: [PATCH 2/2] test: wait for cache to refresh after PV creation in testDeleteRestoringPV Signed-off-by: Ryotaro Banno --- internal/controller/mantlerestore_controller_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/controller/mantlerestore_controller_test.go b/internal/controller/mantlerestore_controller_test.go index 5614d8c..4bb86ab 100644 --- a/internal/controller/mantlerestore_controller_test.go +++ b/internal/controller/mantlerestore_controller_test.go @@ -370,6 +370,12 @@ func (test *mantleRestoreControllerUnitTest) testDeleteRestoringPV() { err := test.reconciler.createOrUpdateRestoringPV(ctx, restore, test.backup) Expect(err).NotTo(HaveOccurred()) + // Make sure the client cache stores the restoring PV. + Eventually(func(g Gomega) { + err = k8sClient.Get(ctx, client.ObjectKey{Name: test.reconciler.restoringPVName(restore)}, &pv) + g.Expect(err).NotTo(HaveOccurred()) + }).Should(Succeed()) + err = test.reconciler.deleteRestoringPV(ctx, restoreDifferent) Expect(err).To(HaveOccurred())