Add storage live migration support to direct DirectVolumeMigration (#…

…1383)

Add prometheus progress reporting to VM Live Migration

Validate the plan if the live migration checkbox is
set, ensure that the source and destination cluster
have a valid kubevirt version and configuration.

Allow canceling of migration.

This includes aborting live migrations and
cleaning up rsync pods if they are still running.
Added a finalizer to the DirectVolumeMigration
which is only removed once everything is cleaned
up properly.

Properly handle rollback of direct volume migrations
that include VMs. For both running and stopped VMs
as well as live migration of running VMs, or quiesce
and starting them after migration.

Signed-off-by: Alexander Wels <[email protected]>

.github/workflows/pr-make.yml

@@ -8,7 +8,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: Setup dependencies
         run: |
           sudo apt-get update && sudo apt-get install -y libgpgme-dev libdevmapper-dev btrfs-progs libbtrfs-dev

Makefile

@@ -102,7 +102,7 @@ ifeq (, $(shell which controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.0 ;\
+	go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 CONTROLLER_GEN=$(GOBIN)/controller-gen

config/crds/migration.openshift.io_migplans.yaml

@@ -275,6 +275,11 @@ spec:
                     type: object
                 type: object
                 x-kubernetes-map-type: atomic
+              liveMigrate:
+                description: LiveMigrate optional flag to enable live migration of
+                  VMs during direct volume migration Only running VMs when the plan
+                  is executed will be live migrated
+                type: boolean
               migStorageRef:
                 description: "ObjectReference contains enough information to let you
                   inspect or modify the referred object. --- New uses of this type

go.mod

@@ -1,6 +1,8 @@
 module github.com/konveyor/mig-controller
 
-go 1.21
+go 1.22.0
+
+toolchain go1.22.5
 
 require (
 	cloud.google.com/go/storage v1.30.1
@@ -17,7 +19,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/joho/godotenv v1.3.0
 	github.com/konveyor/controller v0.12.0
-	github.com/konveyor/crane-lib v0.1.2
+	github.com/konveyor/crane-lib v0.1.3
 	github.com/konveyor/openshift-velero-plugin v0.0.0-20210729141849-876132e34f3d
 	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/onsi/ginkgo v1.16.4
@@ -27,6 +29,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.19.0
+	github.com/prometheus/common v0.53.0
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
 	github.com/vmware-tanzu/velero v1.7.1
 	go.uber.org/zap v1.27.0
@@ -36,7 +39,7 @@ require (
 	k8s.io/apimachinery v0.30.0
 	k8s.io/client-go v1.5.2
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
-	kubevirt.io/api v1.2.0
+	kubevirt.io/api v1.3.0
 	kubevirt.io/containerized-data-importer-api v1.59.0
 	sigs.k8s.io/controller-runtime v0.18.1
 )
@@ -156,7 +159,6 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/proglottis/gpgme v0.1.3 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.53.0 // indirect
 	github.com/prometheus/procfs v0.14.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect
@@ -208,7 +210,7 @@ require (
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.29.4 // indirect
+	k8s.io/apiextensions-apiserver v0.30.0 // indirect
 	k8s.io/component-base v0.30.0 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect

go.sum

@@ -1542,6 +1542,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -1585,8 +1586,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konveyor/controller v0.12.0 h1:TYEGrb6TxegMqMH1ZPy/TMyOeecH+JHKQNqKvSUGWkM=
 github.com/konveyor/controller v0.12.0/go.mod h1:kGFv+5QxjuRo1wUO+bO/HGpULkdkR1pb1tEsVMPAz3s=
-github.com/konveyor/crane-lib v0.1.2 h1:HSQd2u7UJB0vV8c75Gxd5KqDm9gevpT9bsB02P9MoZ4=
-github.com/konveyor/crane-lib v0.1.2/go.mod h1:oSqLMhvUa3kNC/IaKXdGV/Tfs2bdwoZpYrbiV4KpVdY=
+github.com/konveyor/crane-lib v0.1.3 h1:dlVe8uGfLhu5cs9GFFikXmTL1ysukHopv5dba54iBcM=
+github.com/konveyor/crane-lib v0.1.3/go.mod h1:oSqLMhvUa3kNC/IaKXdGV/Tfs2bdwoZpYrbiV4KpVdY=
 github.com/konveyor/openshift-velero-plugin v0.0.0-20210729141849-876132e34f3d h1:tETgPq+JxXhVhnrcLc7rcW9BURax36VUsix8DAU0wuY=
 github.com/konveyor/openshift-velero-plugin v0.0.0-20210729141849-876132e34f3d/go.mod h1:Yk0xQ4N5rwE1+NWLSFQUQLgNT1/8p4Uor60LlgQfymg=
 github.com/konveyor/velero v0.10.2-0.20220124204642-f91d69bb9a5e h1:cjeGzY/zgPJCSLpC6Yurpn1li9ERrMtrF0nixFz2fmc=
@@ -1719,6 +1720,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
@@ -3151,8 +3153,8 @@ k8s.io/api v0.29.4/go.mod h1:DetSv0t4FBTcEpfA84NJV3g9a7+rSzlUHk5ADAYHUv0=
 k8s.io/apiextensions-apiserver v0.17.1/go.mod h1:DRIFH5x3jalE4rE7JP0MQKby9zdYk9lUJQuMmp+M/L0=
 k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA=
 k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8=
-k8s.io/apiextensions-apiserver v0.29.4 h1:M7hbuHU/ckbibR7yPbe6DyNWgTFKNmZDbdZKD8q1Smk=
-k8s.io/apiextensions-apiserver v0.29.4/go.mod h1:TTDC9fB+0kHY2rogf5hgBR03KBKCwED+GHUsXGpR7SM=
+k8s.io/apiextensions-apiserver v0.30.0 h1:jcZFKMqnICJfRxTgnC4E+Hpcq8UEhT8B2lhBcQ+6uAs=
+k8s.io/apiextensions-apiserver v0.30.0/go.mod h1:N9ogQFGcrbWqAY9p2mUAL5mGxsLqwgtUce127VtRX5Y=
 k8s.io/apimachinery v0.29.4 h1:RaFdJiDmuKs/8cm1M6Dh1Kvyh59YQFDcFuFTSmXes6Q=
 k8s.io/apimachinery v0.29.4/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y=
 k8s.io/apiserver v0.17.1/go.mod h1:BQEUObJv8H6ZYO7DeKI5vb50tjk6paRJ4ZhSyJsiSco=
@@ -3217,8 +3219,8 @@ k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak=
 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-kubevirt.io/api v1.2.0 h1:1f8XQLPl4BuHPsc6SHTPnYSYeDxucKCQGa8CdrGJSRc=
-kubevirt.io/api v1.2.0/go.mod h1:SbeR9ma4EwnaOZEUkh/lNz0kzYm5LPpEDE30vKXC5Zg=
+kubevirt.io/api v1.3.0 h1:9sGElMmnRU50pGED+MPPD2OwQl4S5lvjCUjm+t0mI90=
+kubevirt.io/api v1.3.0/go.mod h1:e6LkElYZZm8NcP2gKlFVHZS9pgNhIARHIjSBSfeiP1s=
 kubevirt.io/containerized-data-importer-api v1.59.0 h1:GdDt9BlR0qHejpMaPfASbsG8JWDmBf1s7xZBj5W9qn0=
 kubevirt.io/containerized-data-importer-api v1.59.0/go.mod h1:4yOGtCE7HvgKp7wftZZ3TBvDJ0x9d6N6KaRjRYcUFpE=
 kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc=

pkg/apis/migration/v1alpha1/directvolumemigration_types.go

@@ -18,6 +18,7 @@ package v1alpha1
 
 import (
 	"fmt"
+	"slices"
 
 	kapi "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -65,22 +66,43 @@ type DirectVolumeMigrationSpec struct {
 
 	// Specifies if progress reporting CRs needs to be deleted or not
 	DeleteProgressReportingCRs bool `json:"deleteProgressReportingCRs,omitempty"`
+
+	// Specifies if any volumes associated with a VM should be live storage migrated instead of offline migrated
+	LiveMigrate *bool `json:"liveMigrate,omitempty"`
+
+	// Specifies if this is the final DVM in the migration plan
+	MigrationType *DirectVolumeMigrationType `json:"migrationType,omitempty"`
 }
 
+type DirectVolumeMigrationType string
+
+const (
+	MigrationTypeStage    DirectVolumeMigrationType = "Stage"
+	MigrationTypeFinal    DirectVolumeMigrationType = "CutOver"
+	MigrationTypeRollback DirectVolumeMigrationType = "Rollback"
+)
+
 // DirectVolumeMigrationStatus defines the observed state of DirectVolumeMigration
 type DirectVolumeMigrationStatus struct {
-	Conditions       `json:","`
-	ObservedDigest   string            `json:"observedDigest"`
-	StartTimestamp   *metav1.Time      `json:"startTimestamp,omitempty"`
-	PhaseDescription string            `json:"phaseDescription"`
-	Phase            string            `json:"phase,omitempty"`
-	Itinerary        string            `json:"itinerary,omitempty"`
-	Errors           []string          `json:"errors,omitempty"`
-	SuccessfulPods   []*PodProgress    `json:"successfulPods,omitempty"`
-	FailedPods       []*PodProgress    `json:"failedPods,omitempty"`
-	RunningPods      []*PodProgress    `json:"runningPods,omitempty"`
-	PendingPods      []*PodProgress    `json:"pendingPods,omitempty"`
-	RsyncOperations  []*RsyncOperation `json:"rsyncOperations,omitempty"`
+	Conditions                `json:","`
+	ObservedDigest            string                   `json:"observedDigest"`
+	StartTimestamp            *metav1.Time             `json:"startTimestamp,omitempty"`
+	PhaseDescription          string                   `json:"phaseDescription"`
+	Phase                     string                   `json:"phase,omitempty"`
+	Itinerary                 string                   `json:"itinerary,omitempty"`
+	Errors                    []string                 `json:"errors,omitempty"`
+	SuccessfulPods            []*PodProgress           `json:"successfulPods,omitempty"`
+	FailedPods                []*PodProgress           `json:"failedPods,omitempty"`
+	RunningPods               []*PodProgress           `json:"runningPods,omitempty"`
+	PendingPods               []*PodProgress           `json:"pendingPods,omitempty"`
+	UnknownPods               []*PodProgress           `json:"unknownPods,omitempty"`
+	PendingSinceTimeLimitPods []*PodProgress           `json:"pendingSinceTimeLimitPods,omitempty"`
+	SuccessfulLiveMigrations  []*LiveMigrationProgress `json:"successfulLiveMigration,omitempty"`
+	RunningLiveMigrations     []*LiveMigrationProgress `json:"runningLiveMigration,omitempty"`
+	PendingLiveMigrations     []*LiveMigrationProgress `json:"pendingLiveMigration,omitempty"`
+	FailedLiveMigrations      []*LiveMigrationProgress `json:"failedLiveMigration,omitempty"`
+	RsyncOperations           []*RsyncOperation        `json:"rsyncOperations,omitempty"`
+	SkippedVolumes            []string                 `json:"skippedVolumes,omitempty"`
 }
 
 // GetRsyncOperationStatusForPVC returns RsyncOperation from status for matching PVC, creates new one if doesn't exist already
@@ -117,6 +139,42 @@ func (ds *DirectVolumeMigrationStatus) AddRsyncOperation(podStatus *RsyncOperati
 	ds.RsyncOperations = append(ds.RsyncOperations, podStatus)
 }
 
+func (dvm *DirectVolumeMigration) IsCompleted() bool {
+	return len(dvm.Status.SuccessfulPods)+
+		len(dvm.Status.FailedPods)+
+		len(dvm.Status.SkippedVolumes)+
+		len(dvm.Status.SuccessfulLiveMigrations)+
+		len(dvm.Status.FailedLiveMigrations) == len(dvm.Spec.PersistentVolumeClaims)
+}
+
+func (dvm *DirectVolumeMigration) IsCutover() bool {
+	return dvm.Spec.MigrationType != nil && *dvm.Spec.MigrationType == MigrationTypeFinal
+}
+
+func (dvm *DirectVolumeMigration) IsRollback() bool {
+	return dvm.Spec.MigrationType != nil && *dvm.Spec.MigrationType == MigrationTypeRollback
+}
+
+func (dvm *DirectVolumeMigration) IsStage() bool {
+	return dvm.Spec.MigrationType != nil && *dvm.Spec.MigrationType == MigrationTypeStage
+}
+
+func (dvm *DirectVolumeMigration) SkipVolume(volumeName, namespace string) {
+	dvm.Status.SkippedVolumes = append(dvm.Status.SkippedVolumes, fmt.Sprintf("%s/%s", namespace, volumeName))
+}
+
+func (dvm *DirectVolumeMigration) IsLiveMigrate() bool {
+	return dvm.Spec.LiveMigrate != nil && *dvm.Spec.LiveMigrate
+}
+
+func (dvm *DirectVolumeMigration) AllReportingCompleted() bool {
+	isCompleted := dvm.IsCompleted()
+	isAnyPending := len(dvm.Status.PendingPods) > 0 || len(dvm.Status.PendingLiveMigrations) > 0
+	isAnyRunning := len(dvm.Status.RunningPods) > 0 || len(dvm.Status.RunningLiveMigrations) > 0
+	isAnyUnknown := len(dvm.Status.UnknownPods) > 0
+	return !isAnyRunning && !isAnyPending && !isAnyUnknown && isCompleted
+}
+
 // TODO: Explore how to reliably get stunnel+rsync logs/status reported back to
 // DirectVolumeMigrationStatus
 
@@ -151,6 +209,16 @@ type PodProgress struct {
 	TotalElapsedTime            *metav1.Duration      `json:"totalElapsedTime,omitempty"`
 }
 
+type LiveMigrationProgress struct {
+	VMName                      string                `json:"vmName,omitempty"`
+	VMNamespace                 string                `json:"vmNamespace,omitempty"`
+	PVCReference                *kapi.ObjectReference `json:"pvcRef,omitempty"`
+	LastObservedProgressPercent string                `json:"lastObservedProgressPercent,omitempty"`
+	LastObservedTransferRate    string                `json:"lastObservedTransferRate,omitempty"`
+	TotalElapsedTime            *metav1.Duration      `json:"totalElapsedTime,omitempty"`
+	Message                     string                `json:"message,omitempty"`
+}
+
 // RsyncOperation defines observed state of an Rsync Operation
 type RsyncOperation struct {
 	// PVCReference pvc to which this Rsync operation corresponds to
@@ -205,6 +273,26 @@ func (r *DirectVolumeMigration) GetMigrationForDVM(client k8sclient.Client) (*Mi
 	return GetMigrationForDVM(client, r.OwnerReferences)
 }
 
+func (r *DirectVolumeMigration) GetSourceNamespaces() []string {
+	namespaces := []string{}
+	for _, pvc := range r.Spec.PersistentVolumeClaims {
+		if pvc.Namespace != "" && !slices.Contains(namespaces, pvc.Namespace) {
+			namespaces = append(namespaces, pvc.Namespace)
+		}
+	}
+	return namespaces
+}
+
+func (r *DirectVolumeMigration) GetDestinationNamespaces() []string {
+	namespaces := []string{}
+	for _, pvc := range r.Spec.PersistentVolumeClaims {
+		if pvc.TargetNamespace != "" && !slices.Contains(namespaces, pvc.TargetNamespace) {
+			namespaces = append(namespaces, pvc.TargetNamespace)
+		}
+	}
+	return namespaces
+}
+
 // Add (de-duplicated) errors.
 func (r *DirectVolumeMigration) AddErrors(errors []string) {
 	m := map[string]bool{}

pkg/apis/migration/v1alpha1/migcluster_types.go

@@ -804,6 +804,13 @@ var accessModeList = []provisionerAccessModes{
 			kapi.PersistentVolumeBlock:      {kapi.ReadWriteOnce, kapi.ReadOnlyMany, kapi.ReadWriteMany},
 		},
 	},
+	provisionerAccessModes{
+		Provisioner: "csi.kubevirt.io",
+		AccessModes: map[kapi.PersistentVolumeMode][]kapi.PersistentVolumeAccessMode{
+			kapi.PersistentVolumeFilesystem: {kapi.ReadWriteOnce},
+			kapi.PersistentVolumeBlock:      {kapi.ReadWriteOnce},
+		},
+	},
 }
 
 // Get the list of k8s StorageClasses from the cluster.

pkg/apis/migration/v1alpha1/migplan_types.go

@@ -115,6 +115,11 @@ type MigPlanSpec struct {
 	// LabelSelector optional label selector on the included resources in Velero Backup
 	// +kubebuilder:validation:Optional
 	LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"`
+
+	// LiveMigrate optional flag to enable live migration of VMs during direct volume migration
+	// Only running VMs when the plan is executed will be live migrated
+	// +kubebuilder:validation:Optional
+	LiveMigrate *bool `json:"liveMigrate,omitempty"`
 }
 
 // MigPlanStatus defines the observed state of MigPlan
@@ -218,6 +223,10 @@ type PlanResources struct {
 	DestMigCluster *MigCluster
 }
 
+func (r *MigPlan) LiveMigrationChecked() bool {
+	return r.Spec.LiveMigrate != nil && *r.Spec.LiveMigrate
+}
+
 // GetRefResources gets referenced resources from a MigPlan.
 func (r *MigPlan) GetRefResources(client k8sclient.Client) (*PlanResources, error) {
 	isIntraCluster, err := r.IsIntraCluster(client)