Merge pull request #77 from cybozu-go/add-proxy-support

support http(s) proxy

charts/mantle/templates/deployment.yaml

@@ -88,6 +88,15 @@ spec:
             {{- with .Values.controller.gcInterval }}
             - --gc-interval={{ . }}
             {{- end }}
+            {{- with .Values.controller.httpProxy }}
+            - --http-proxy={{ . }}
+            {{- end }}
+            {{- with .Values.controller.httpsProxy }}
+            - --https-proxy={{ . }}
+            {{- end }}
+            {{- with .Values.controller.noProxy }}
+            - --no-proxy={{ . }}
+            {{- end }}
           env:
             - name: POD_NAME
               valueFrom:

cmd/controller/main.go

@@ -57,6 +57,9 @@ var (
 	caCertConfigMapSrc      string
 	caCertKeySrc            string
 	gcInterval              string
+	httpProxy               string
+	httpsProxy              string
+	noProxy                 string
 
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
@@ -98,6 +101,12 @@ func init() {
 			"The default value is ca.crt. This option is just ignored if --ca-cert-configmap isn't specified.")
 	flags.StringVar(&gcInterval, "gc-interval", "1h",
 		"The time period between each garbage collection for orphaned resources.")
+	flags.StringVar(&httpProxy, "http-proxy", "",
+		"The proxy URL for HTTP requests to the object storage and the gRPC endpoint of secondary mantle.")
+	flags.StringVar(&httpsProxy, "https-proxy", "",
+		"The proxy URL for HTTPS requests to the object storage and the gRPC endpoint of secondary mantle.")
+	flags.StringVar(&noProxy, "no-proxy", "",
+		"A string that contains comma-separated values specifying hosts that should be excluded from proxying.")
 
 	goflags := flag.NewFlagSet("goflags", flag.ExitOnError)
 	zapOpts.Development = true
@@ -179,6 +188,11 @@ func setupReconcilers(mgr manager.Manager, primarySettings *controller.PrimarySe
 			CACertConfigMap: caCertConfigMap,
 			CACertKey:       &caCertKeySrc,
 		},
+		&controller.ProxySettings{
+			HttpProxy:  httpProxy,
+			HttpsProxy: httpsProxy,
+			NoProxy:    noProxy,
+		},
 	)
 	if err := backupReconciler.SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "MantleBackup")
@@ -233,6 +247,22 @@ func setupStandalone(mgr manager.Manager) error {
 }
 
 func setupPrimary(ctx context.Context, mgr manager.Manager, wg *sync.WaitGroup) error {
+	// Setup environment variables related to proxies before creating a gRPC client.
+	// cf. https://github.com/grpc/grpc-go/blob/adad26df1826bf2fb66ad56ff32a62b98bf5cb3a/Documentation/proxy.md
+	// cf. https://pkg.go.dev/golang.org/x/net/http/httpproxy
+	if err := os.Setenv("HTTP_PROXY", httpProxy); err != nil {
+		setupLog.Error(err, "failed to set HTTP_PROXY environment variable")
+		return err
+	}
+	if err := os.Setenv("HTTPS_PROXY", httpsProxy); err != nil {
+		setupLog.Error(err, "failed to set HTTPS_PROXY environment variable")
+		return err
+	}
+	if err := os.Setenv("NO_PROXY", noProxy); err != nil {
+		setupLog.Error(err, "failed to set NO_PROXY environment variable")
+		return err
+	}
+
 	conn, err := grpc.NewClient(
 		mantleServiceEndpoint,
 		grpc.WithTransportCredentials(insecure.NewCredentials()),

internal/controller/mantlebackup_controller.go

@@ -72,6 +72,12 @@ type ObjectStorageSettings struct {
 	Endpoint        string
 }
 
+type ProxySettings struct {
+	HttpProxy  string
+	HttpsProxy string
+	NoProxy    string
+}
+
 // MantleBackupReconciler reconciles a MantleBackup object
 type MantleBackupReconciler struct {
 	client.Client
@@ -85,6 +91,7 @@ type MantleBackupReconciler struct {
 	envSecret             string
 	objectStorageSettings *ObjectStorageSettings // This should be non-nil if and only if role equals 'primary' or 'secondary'.
 	objectStorageClient   objectstorage.Bucket
+	proxySettings         *ProxySettings
 }
 
 // NewMantleBackupReconciler returns NodeReconciler.
@@ -97,6 +104,7 @@ func NewMantleBackupReconciler(
 	podImage string,
 	envSecret string,
 	objectStorageSettings *ObjectStorageSettings,
+	proxySettings *ProxySettings,
 ) *MantleBackupReconciler {
 	return &MantleBackupReconciler{
 		Client:                client,
@@ -109,6 +117,7 @@ func NewMantleBackupReconciler(
 		podImage:              podImage,
 		envSecret:             envSecret,
 		objectStorageSettings: objectStorageSettings,
+		proxySettings:         proxySettings,
 	}
 }
 
@@ -1325,6 +1334,18 @@ func (r *MantleBackupReconciler) createOrUpdateExportDataUploadJob(ctx context.C
 							},
 						},
 					},
+					{
+						Name:  "HTTP_PROXY",
+						Value: r.proxySettings.HttpProxy,
+					},
+					{
+						Name:  "HTTPS_PROXY",
+						Value: r.proxySettings.HttpsProxy,
+					},
+					{
+						Name:  "NO_PROXY",
+						Value: r.proxySettings.NoProxy,
+					},
 				},
 				Image:           r.podImage,
 				ImagePullPolicy: corev1.PullIfNotPresent,

internal/controller/mantlebackup_controller_test.go

@@ -34,6 +34,15 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/event"
 )
 
+func getEnvValue(envVarAry []corev1.EnvVar, name string) (string, error) {
+	for _, env := range envVarAry {
+		if env.Name == name {
+			return env.Value, nil
+		}
+	}
+	return "", errors.New("name not found")
+}
+
 var _ = Describe("MantleBackup controller", func() {
 	var mgrUtil testutil.ManagerUtil
 	var reconciler *MantleBackupReconciler
@@ -121,6 +130,7 @@ var _ = Describe("MantleBackup controller", func() {
 				"dummy image",
 				"",
 				nil,
+				nil,
 			)
 			reconciler.ceph = testutil.NewFakeRBD()
 			err := reconciler.SetupWithManager(mgrUtil.GetManager())
@@ -332,6 +342,11 @@ var _ = Describe("MantleBackup controller", func() {
 	Context("when the role is `primary`", func() {
 		var mockCtrl *gomock.Controller
 		var grpcClient *proto.MockMantleServiceClient
+		proxySettings := &ProxySettings{
+			HttpProxy:  "dummy http proxy",
+			HttpsProxy: "dummy https proxy",
+			NoProxy:    "no proxy",
+		}
 		BeforeEach(func() {
 			mgrUtil = testutil.NewManagerUtil(context.Background(), cfg, scheme.Scheme)
 
@@ -355,6 +370,7 @@ var _ = Describe("MantleBackup controller", func() {
 					CACertConfigMap: nil,
 					CACertKey:       nil,
 				},
+				proxySettings,
 			)
 			reconciler.ceph = testutil.NewFakeRBD()
 
@@ -548,6 +564,17 @@ var _ = Describe("MantleBackup controller", func() {
 				g.Expect(*jobUpload.Spec.Template.Spec.SecurityContext.RunAsUser).To(Equal(int64(10000)))
 				g.Expect(*jobUpload.Spec.Template.Spec.SecurityContext.RunAsGroup).To(Equal(int64(10000)))
 				g.Expect(*jobUpload.Spec.Template.Spec.SecurityContext.RunAsNonRoot).To(Equal(true))
+
+				// Make sure HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables are correctly set.
+				httpProxy, err := getEnvValue(jobUpload.Spec.Template.Spec.Containers[0].Env, "HTTP_PROXY")
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(httpProxy).To(Equal(proxySettings.HttpProxy))
+				httpsProxy, err := getEnvValue(jobUpload.Spec.Template.Spec.Containers[0].Env, "HTTPS_PROXY")
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(httpsProxy).To(Equal(proxySettings.HttpsProxy))
+				noProxy, err := getEnvValue(jobUpload.Spec.Template.Spec.Containers[0].Env, "NO_PROXY")
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(noProxy).To(Equal(proxySettings.NoProxy))
 			}).WithContext(ctx).Should(Succeed())
 
 			// Make the all existing MantleBackups in the primary Mantle
@@ -800,7 +827,7 @@ var _ = Describe("prepareForDataSynchronization", func() {
 		}
 
 		mbr := NewMantleBackupReconciler(ctrlClient,
-			ctrlClient.Scheme(), "test", RolePrimary, nil, "dummy image", "", nil)
+			ctrlClient.Scheme(), "test", RolePrimary, nil, "dummy image", "", nil, nil)
 
 		ret, err := mbr.prepareForDataSynchronization(context.Background(),
 			backup, grpcClient)
@@ -1232,6 +1259,11 @@ var _ = Describe("export", func() {
 			"dummy image",
 			"",
 			nil,
+			&ProxySettings{
+				HttpProxy:  "",
+				HttpsProxy: "",
+				NoProxy:    "",
+			},
 		)
 
 		ns = resMgr.CreateNamespace()
@@ -1351,6 +1383,7 @@ var _ = Describe("export", func() {
 			"dummy image",
 			"",
 			nil,
+			nil,
 		)
 		ns2 := resMgr.CreateNamespace()
 		createAndExportMantleBackup(mbr2, "target2", ns2)
@@ -1385,6 +1418,7 @@ var _ = Describe("import", func() {
 			"dummy-image",
 			"dummy-env-secret",
 			&ObjectStorageSettings{},
+			nil,
 		)
 		mbr.objectStorageClient = mockObjectStorage
 		mbr.ceph = testutil.NewFakeRBD()

internal/controller/mantlerestore_controller_test.go

@@ -58,6 +58,7 @@ func (test *mantleRestoreControllerUnitTest) setupEnv() {
 			"dummy image",
 			"",
 			nil,
+			nil,
 		)
 		backupReconciler.ceph = testutil.NewFakeRBD()
 		err := backupReconciler.SetupWithManager(test.mgrUtil.GetManager())
@@ -369,6 +370,12 @@ func (test *mantleRestoreControllerUnitTest) testDeleteRestoringPV() {
 		err := test.reconciler.createOrUpdateRestoringPV(ctx, restore, test.backup)
 		Expect(err).NotTo(HaveOccurred())
 
+		// Make sure the client cache stores the restoring PV.
+		Eventually(func(g Gomega) {
+			err = k8sClient.Get(ctx, client.ObjectKey{Name: test.reconciler.restoringPVName(restore)}, &pv)
+			g.Expect(err).NotTo(HaveOccurred())
+		}).Should(Succeed())
+
 		err = test.reconciler.deleteRestoringPV(ctx, restoreDifferent)
 		Expect(err).To(HaveOccurred())
 

test/e2e/testdata/values-mantle-primary-template.yaml

@@ -6,3 +6,6 @@ controller:
   envSecret: export-data
   exportDataStorageClass: rook-ceph-block
   gcInterval: 1s
+  #httpProxy: http://host.minikube.internal:8899
+  #httpsProxy: http://host.minikube.internal:8899
+  #noProxy: localhost,127.0.0.1,10.96.0.0/12