From 30c51cde4a69b9db63b3da154d84d47d298e2df4 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 22 Nov 2024 01:55:40 +0000 Subject: [PATCH 1/5] add createOrUpdateDiscardPV The first step to implement reconcileDiscardJob. Signed-off-by: Ryotaro Banno --- .../controller/mantlebackup_controller.go | 57 +++++++++++++++++-- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go index 0a97822..9e2c7bf 100644 --- a/internal/controller/mantlebackup_controller.go +++ b/internal/controller/mantlebackup_controller.go @@ -44,6 +44,7 @@ const ( labelComponentExportJob = "export-job" labelComponentUploadJob = "upload-job" labelComponentImportJob = "import-job" + labelComponentDiscardVolume = "discard-volume" annotRemoteUID = "mantle.cybozu.io/remote-uid" annotDiffFrom = "mantle.cybozu.io/diff-from" annotDiffTo = "mantle.cybozu.io/diff-to" @@ -1523,19 +1524,67 @@ func (r *MantleBackupReconciler) updateStatusManifests( } func (r *MantleBackupReconciler) reconcileDiscardJob( - _ context.Context, + ctx context.Context, backup *mantlev1.MantleBackup, - _ *snapshotTarget, -) (ctrl.Result, error) { //nolint:unparam + snapshotTarget *snapshotTarget, +) (ctrl.Result, error) { if backup.GetAnnotations()[annotSyncMode] != syncModeFull { return ctrl.Result{}, nil } - // FIXME: implement here later + if err := r.createOrUpdateDiscardPV(ctx, backup, snapshotTarget.pv); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{}, nil } +func (r *MantleBackupReconciler) createOrUpdateDiscardPV( + ctx context.Context, + backup *mantlev1.MantleBackup, + targetPV *corev1.PersistentVolume, +) error { + var pv corev1.PersistentVolume + pv.SetName(makeDiscardPVName(backup)) + _, err := ctrl.CreateOrUpdate(ctx, r.Client, &pv, func() error { + labels := pv.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels["app.kubernetes.io/name"] = labelAppNameValue + labels["app.kubernetes.io/component"] = labelComponentDiscardVolume + pv.SetLabels(labels) + + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pv.Spec.Capacity = targetPV.Spec.Capacity + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain + pv.Spec.StorageClassName = "" + + volumeMode := corev1.PersistentVolumeBlock + pv.Spec.VolumeMode = &volumeMode + + if pv.Spec.CSI == nil { + pv.Spec.CSI = &corev1.CSIPersistentVolumeSource{} + } + pv.Spec.CSI.Driver = targetPV.Spec.CSI.Driver + pv.Spec.CSI.ControllerExpandSecretRef = targetPV.Spec.CSI.ControllerExpandSecretRef + pv.Spec.CSI.NodeStageSecretRef = targetPV.Spec.CSI.NodeStageSecretRef + pv.Spec.CSI.VolumeHandle = targetPV.Spec.CSI.VolumeAttributes["imageName"] + + if pv.Spec.CSI.VolumeAttributes == nil { + pv.Spec.CSI.VolumeAttributes = map[string]string{} + } + pv.Spec.CSI.VolumeAttributes["clusterID"] = targetPV.Spec.CSI.VolumeAttributes["clusterID"] + pv.Spec.CSI.VolumeAttributes["imageFeatures"] = targetPV.Spec.CSI.VolumeAttributes["imageFeatures"] + pv.Spec.CSI.VolumeAttributes["imageFormat"] = targetPV.Spec.CSI.VolumeAttributes["imageFormat"] + pv.Spec.CSI.VolumeAttributes["pool"] = targetPV.Spec.CSI.VolumeAttributes["pool"] + pv.Spec.CSI.VolumeAttributes["staticVolume"] = "true" + + return nil + }) + return err +} + func (r *MantleBackupReconciler) reconcileImportJob( ctx context.Context, backup *mantlev1.MantleBackup, From fd45de70ab15e49320cba1e5311631cfff2a7735 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 22 Nov 2024 02:05:56 +0000 Subject: [PATCH 2/5] add createOrUpdateDiscardPVC This commit is the 2nd step for implementing reconcileDiscardJob Signed-off-by: Ryotaro Banno --- .../controller/mantlebackup_controller.go | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go index 9e2c7bf..51211fd 100644 --- a/internal/controller/mantlebackup_controller.go +++ b/internal/controller/mantlebackup_controller.go @@ -1536,6 +1536,10 @@ func (r *MantleBackupReconciler) reconcileDiscardJob( return ctrl.Result{}, err } + if err := r.createOrUpdateDiscardPVC(ctx, backup, snapshotTarget.pvc); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil } @@ -1585,6 +1589,37 @@ func (r *MantleBackupReconciler) createOrUpdateDiscardPV( return err } +func (r *MantleBackupReconciler) createOrUpdateDiscardPVC( + ctx context.Context, + backup *mantlev1.MantleBackup, + targetPVC *corev1.PersistentVolumeClaim, +) error { + var pvc corev1.PersistentVolumeClaim + pvc.SetName(makeDiscardPVCName(backup)) + pvc.SetNamespace(r.managedCephClusterID) + _, err := ctrl.CreateOrUpdate(ctx, r.Client, &pvc, func() error { + labels := pvc.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels["app.kubernetes.io/name"] = labelAppNameValue + labels["app.kubernetes.io/component"] = labelComponentDiscardVolume + pvc.SetLabels(labels) + + storageClassName := "" + pvc.Spec.StorageClassName = &storageClassName + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + pvc.Spec.Resources = targetPVC.Spec.Resources + pvc.Spec.VolumeName = makeDiscardPVName(backup) + + volumeMode := corev1.PersistentVolumeBlock + pvc.Spec.VolumeMode = &volumeMode + + return nil + }) + return err +} + func (r *MantleBackupReconciler) reconcileImportJob( ctx context.Context, backup *mantlev1.MantleBackup, From 0e54ad229b8bd994fb24675b7f453d7672ea2485 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 22 Nov 2024 02:16:07 +0000 Subject: [PATCH 3/5] add createOrUpdateDiscardJob This commit is the 4th step for implementing reconcileDiscardJob. Signed-off-by: Ryotaro Banno --- .../controller/mantlebackup_controller.go | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go index 51211fd..2b380d3 100644 --- a/internal/controller/mantlebackup_controller.go +++ b/internal/controller/mantlebackup_controller.go @@ -44,6 +44,7 @@ const ( labelComponentExportJob = "export-job" labelComponentUploadJob = "upload-job" labelComponentImportJob = "import-job" + labelComponentDiscardJob = "discard-job" labelComponentDiscardVolume = "discard-volume" annotRemoteUID = "mantle.cybozu.io/remote-uid" annotDiffFrom = "mantle.cybozu.io/diff-from" @@ -1540,6 +1541,10 @@ func (r *MantleBackupReconciler) reconcileDiscardJob( return ctrl.Result{}, err } + if err := r.createOrUpdateDiscardJob(ctx, backup); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil } @@ -1620,6 +1625,71 @@ func (r *MantleBackupReconciler) createOrUpdateDiscardPVC( return err } +func (r *MantleBackupReconciler) createOrUpdateDiscardJob( + ctx context.Context, + backup *mantlev1.MantleBackup, +) error { + var job batchv1.Job + job.SetName(makeDiscardJobName(backup)) + job.SetNamespace(r.managedCephClusterID) + _, err := ctrl.CreateOrUpdate(ctx, r.Client, &job, func() error { + labels := job.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels["app.kubernetes.io/name"] = labelAppNameValue + labels["app.kubernetes.io/component"] = labelComponentDiscardJob + job.SetLabels(labels) + + var backoffLimit int32 = 65535 + job.Spec.BackoffLimit = &backoffLimit + + job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyOnFailure + + tru := true + var zero int64 = 0 + job.Spec.Template.Spec.Containers = []corev1.Container{ + { + Name: "discard", + Image: r.podImage, + Command: []string{ + "/bin/bash", + "-c", + ` +set -e +blkdiscard /dev/discard-rbd +`, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &tru, + RunAsGroup: &zero, + RunAsUser: &zero, + }, + VolumeDevices: []corev1.VolumeDevice{ + { + Name: "discard-rbd", + DevicePath: "/dev/discard-rbd", + }, + }, + }, + } + + job.Spec.Template.Spec.Volumes = []corev1.Volume{ + { + Name: "discard-rbd", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: makeDiscardPVCName(backup), + }, + }, + }, + } + + return nil + }) + return err +} + func (r *MantleBackupReconciler) reconcileImportJob( ctx context.Context, backup *mantlev1.MantleBackup, From 38fea41b351e78acf98e560aff5639f385ccd77c Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 22 Nov 2024 02:27:34 +0000 Subject: [PATCH 4/5] add hasDiscardJobCompleted This commit should be the last step for implementing reconcileDiscardJob. Signed-off-by: Ryotaro Banno --- .../controller/mantlebackup_controller.go | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/internal/controller/mantlebackup_controller.go b/internal/controller/mantlebackup_controller.go index 2b380d3..ac6a8cc 100644 --- a/internal/controller/mantlebackup_controller.go +++ b/internal/controller/mantlebackup_controller.go @@ -1545,7 +1545,14 @@ func (r *MantleBackupReconciler) reconcileDiscardJob( return ctrl.Result{}, err } - return ctrl.Result{}, nil + completed, err := r.hasDiscardJobCompleted(ctx, backup) + if err != nil { + return ctrl.Result{}, err + } + if completed { + return ctrl.Result{}, nil + } + return ctrl.Result{Requeue: true}, nil } func (r *MantleBackupReconciler) createOrUpdateDiscardPV( @@ -1690,6 +1697,25 @@ blkdiscard /dev/discard-rbd return err } +func (r *MantleBackupReconciler) hasDiscardJobCompleted(ctx context.Context, backup *mantlev1.MantleBackup) (bool, error) { + var job batchv1.Job + if err := r.Client.Get( + ctx, + types.NamespacedName{Name: makeDiscardJobName(backup), Namespace: r.managedCephClusterID}, + &job, + ); err != nil { + if aerrors.IsNotFound(err) { + return false, nil // The cache must be stale. Let's just requeue. + } + return false, err + } + + if IsJobConditionTrue(job.Status.Conditions, batchv1.JobComplete) { + return true, nil + } + return false, nil +} + func (r *MantleBackupReconciler) reconcileImportJob( ctx context.Context, backup *mantlev1.MantleBackup, From cec8e55eb9b715b8dd2ca38eae164a2f026f2231 Mon Sep 17 00:00:00 2001 From: Ryotaro Banno Date: Fri, 22 Nov 2024 02:53:52 +0000 Subject: [PATCH 5/5] add unit tests for reconcileDiscardJob Signed-off-by: Ryotaro Banno --- .../mantlebackup_controller_test.go | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/internal/controller/mantlebackup_controller_test.go b/internal/controller/mantlebackup_controller_test.go index 8167c7d..9535b10 100644 --- a/internal/controller/mantlebackup_controller_test.go +++ b/internal/controller/mantlebackup_controller_test.go @@ -1715,4 +1715,171 @@ var _ = Describe("import", func() { Expect(res.IsZero()).To(BeTrue()) }) }) + + Context("reconcileDiscardJob", func() { + It("should NOT create anything in an incremental backup", func(ctx SpecContext) { + backup, err := createMantleBackupUsingDummyPVC(ctx, "target", ns) + Expect(err).NotTo(HaveOccurred()) + backup.SetAnnotations(map[string]string{ + annotDiffFrom: "source", + annotSyncMode: syncModeIncremental, + annotRemoteUID: "uid", + }) + err = k8sClient.Update(ctx, backup) + Expect(err).NotTo(HaveOccurred()) + + result, err := mbr.reconcileDiscardJob(ctx, backup, &snapshotTarget{}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Requeue).To(BeFalse()) + + var pv corev1.PersistentVolume + err = k8sClient.Get(ctx, types.NamespacedName{Name: makeDiscardPVName(backup), Namespace: nsController}, &pv) + Expect(err).To(HaveOccurred()) + Expect(aerrors.IsNotFound(err)).To(BeTrue()) + + var pvc corev1.PersistentVolume + err = k8sClient.Get(ctx, types.NamespacedName{Name: makeDiscardPVCName(backup), Namespace: nsController}, &pvc) + Expect(err).To(HaveOccurred()) + Expect(aerrors.IsNotFound(err)).To(BeTrue()) + + var job batchv1.Job + err = k8sClient.Get(ctx, types.NamespacedName{Name: makeDiscardJobName(backup), Namespace: nsController}, &job) + Expect(err).To(HaveOccurred()) + Expect(aerrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should create a PV, PVC, and Job, requeue, and complete in a full backup", func(ctx SpecContext) { + backup, err := createMantleBackupUsingDummyPVC(ctx, "target", ns) + Expect(err).NotTo(HaveOccurred()) + backup.SetAnnotations(map[string]string{ + annotSyncMode: syncModeFull, + }) + err = k8sClient.Update(ctx, backup) + Expect(err).NotTo(HaveOccurred()) + + pvCapacity := corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + } + pvDriver := "test-pv-driver" + pvControllerExpandSecretRef := corev1.SecretReference{ + Name: "test-pv-cesr-name", + Namespace: "test-pv-cesr-ns", + } + pvNodeStageSecretRef := corev1.SecretReference{ + Name: "test-pv-nssr-name", + Namespace: "test-pv-nssr-ns", + } + pvClusterID := "test-pv-cluster-id" + pvImageFeatures := "test-pv-image-features" + pvImageFormat := "test-pv-image-format" + pvPool := "test-pv-pool" + pvImageName := "test-pv-image-name" + pvcResources := corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + } + snapshotTarget := &snapshotTarget{ + pvc: &corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: pvcResources, + }, + }, + pv: &corev1.PersistentVolume{ + Spec: corev1.PersistentVolumeSpec{ + Capacity: pvCapacity, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: pvDriver, + ControllerExpandSecretRef: &pvControllerExpandSecretRef, + NodeStageSecretRef: &pvNodeStageSecretRef, + VolumeAttributes: map[string]string{ + "clusterID": pvClusterID, + "imageFeatures": pvImageFeatures, + "imageFormat": pvImageFormat, + "pool": pvPool, + "imageName": pvImageName, + }, + }, + }, + }, + }, + imageName: pvImageName, + poolName: "poolName", + } + + // The first call to reconcileDiscardJob should create a PV, PVC, and Job, and requeue. + result, err := mbr.reconcileDiscardJob(ctx, backup, snapshotTarget) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Requeue).To(BeTrue()) + + var pv corev1.PersistentVolume + err = k8sClient.Get(ctx, types.NamespacedName{Name: makeDiscardPVName(backup), Namespace: nsController}, &pv) + Expect(err).NotTo(HaveOccurred()) + Expect(pv.GetLabels()["app.kubernetes.io/name"]).To(Equal(labelAppNameValue)) + Expect(pv.GetLabels()["app.kubernetes.io/component"]).To(Equal(labelComponentDiscardVolume)) + Expect(len(pv.Spec.AccessModes)).To(Equal(1)) + Expect(pv.Spec.AccessModes[0]).To(Equal(corev1.ReadWriteOnce)) + Expect(pv.Spec.Capacity).To(Equal(pvCapacity)) + Expect(pv.Spec.CSI.Driver).To(Equal(pvDriver)) + Expect(*pv.Spec.CSI.ControllerExpandSecretRef).To(Equal(pvControllerExpandSecretRef)) + Expect(*pv.Spec.CSI.NodeStageSecretRef).To(Equal(pvNodeStageSecretRef)) + Expect(pv.Spec.CSI.VolumeAttributes["clusterID"]).To(Equal(pvClusterID)) + Expect(pv.Spec.CSI.VolumeAttributes["imageFeatures"]).To(Equal(pvImageFeatures)) + Expect(pv.Spec.CSI.VolumeAttributes["imageFormat"]).To(Equal(pvImageFormat)) + Expect(pv.Spec.CSI.VolumeAttributes["pool"]).To(Equal(pvPool)) + Expect(pv.Spec.CSI.VolumeAttributes["staticVolume"]).To(Equal("true")) + Expect(pv.Spec.CSI.VolumeHandle).To(Equal(pvImageName)) + Expect(pv.Spec.PersistentVolumeReclaimPolicy).To(Equal(corev1.PersistentVolumeReclaimRetain)) + Expect(*pv.Spec.VolumeMode).To(Equal(corev1.PersistentVolumeBlock)) + Expect(pv.Spec.StorageClassName).To(Equal("")) + + var pvc corev1.PersistentVolumeClaim + err = k8sClient.Get(ctx, types.NamespacedName{Name: makeDiscardPVName(backup), Namespace: nsController}, &pvc) + Expect(err).NotTo(HaveOccurred()) + Expect(pvc.GetLabels()["app.kubernetes.io/name"]).To(Equal(labelAppNameValue)) + Expect(pvc.GetLabels()["app.kubernetes.io/component"]).To(Equal(labelComponentDiscardVolume)) + Expect(*pvc.Spec.StorageClassName).To(Equal("")) + Expect(len(pvc.Spec.AccessModes)).To(Equal(1)) + Expect(pvc.Spec.AccessModes[0]).To(Equal(corev1.ReadWriteOnce)) + Expect(pvc.Spec.Resources).To(Equal(pvcResources)) + Expect(*pvc.Spec.VolumeMode).To(Equal(corev1.PersistentVolumeBlock)) + Expect(pvc.Spec.VolumeName).To(Equal(makeDiscardPVName(backup))) + + var job batchv1.Job + err = k8sClient.Get(ctx, types.NamespacedName{Name: makeDiscardJobName(backup), Namespace: nsController}, &job) + Expect(err).NotTo(HaveOccurred()) + Expect(job.GetLabels()["app.kubernetes.io/name"]).To(Equal(labelAppNameValue)) + Expect(job.GetLabels()["app.kubernetes.io/component"]).To(Equal(labelComponentDiscardJob)) + Expect(*job.Spec.BackoffLimit).To(Equal(int32(65535))) + Expect(len(job.Spec.Template.Spec.Containers)).To(Equal(1)) + Expect(job.Spec.Template.Spec.Containers[0].Name).To(Equal("discard")) + Expect(*job.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeTrue()) + Expect(*job.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(Equal(int64(0))) + Expect(*job.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(Equal(int64(0))) + Expect(job.Spec.Template.Spec.Containers[0].Image).To(Equal(mbr.podImage)) + Expect(len(job.Spec.Template.Spec.Containers[0].VolumeDevices)).To(Equal(1)) + Expect(job.Spec.Template.Spec.Containers[0].VolumeDevices[0].Name).To(Equal("discard-rbd")) + Expect(job.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath).To(Equal("/dev/discard-rbd")) + Expect(job.Spec.Template.Spec.RestartPolicy).To(Equal(corev1.RestartPolicyOnFailure)) + Expect(len(job.Spec.Template.Spec.Volumes)).To(Equal(1)) + Expect(job.Spec.Template.Spec.Volumes[0]).To(Equal(corev1.Volume{ + Name: "discard-rbd", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: makeDiscardPVCName(backup), + }, + }, + })) + + // Make the Job completed + err = resMgr.ChangeJobCondition(ctx, &job, batchv1.JobComplete, corev1.ConditionTrue) + Expect(err).NotTo(HaveOccurred()) + + // A call to reconcileDiscardJob should NOT requeue after the Job completed + result, err = mbr.reconcileDiscardJob(ctx, backup, snapshotTarget) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Requeue).To(BeFalse()) + }) + }) })