From d72f269f260a3cc1a71efd9622d7231cbfe1907a Mon Sep 17 00:00:00 2001 From: Ryan Swanson Date: Fri, 12 Jul 2024 14:02:42 -0400 Subject: [PATCH] Detect how the vcluster was put to sleep and guide the user to the correct commands to wake it up Warns allows waking and adding a sleeping helm vcluster in a single command. Adds the ability to set chart dir for local dev when adding external cluster. Only errors if helm driver is specified, otherwise it falls back to a platform resume. Remove old check due to platform's auto conversion of helm slept clusters. Continue with adding the secret in either case and only add if they choose to wake the cluster. Co-authored-by: Russell Centanni --- cmd/vclusterctl/cmd/platform/add/cluster.go | 2 +- cmd/vclusterctl/cmd/resume.go | 12 ++++- config/config.go | 8 ++++ no-sleepmode.yaml | 0 pkg/cli/add_vcluster_helm.go | 51 ++++++++++++++++++++- pkg/cli/create_helm.go | 9 +--- pkg/cli/find/find.go | 33 ++++++++++--- pkg/cli/pause_helm.go | 5 ++ pkg/cli/pause_platform.go | 8 +++- pkg/cli/resume_helm.go | 7 +++ pkg/cli/resume_platform.go | 10 +++- pkg/kube/meta.go | 30 ++++++++++-- pkg/lifecycle/lifecycle.go | 13 ++++-- pkg/platform/helper.go | 9 +++- pkg/platform/sleepmode/sleepmode.go | 16 +++++++ 15 files changed, 182 insertions(+), 31 deletions(-) create mode 100644 no-sleepmode.yaml create mode 100644 pkg/platform/sleepmode/sleepmode.go diff --git a/cmd/vclusterctl/cmd/platform/add/cluster.go b/cmd/vclusterctl/cmd/platform/add/cluster.go index f166529c3..0542e115c 100644 --- a/cmd/vclusterctl/cmd/platform/add/cluster.go +++ b/cmd/vclusterctl/cmd/platform/add/cluster.go @@ -143,7 +143,7 @@ func (cmd *ClusterCmd) Run(ctx context.Context, args []string) error { if os.Getenv("DEVELOPMENT") == "true" { helmArgs = []string{ - "upgrade", "--install", "loft", "./chart", + "upgrade", "--install", "loft", cmp.Or(os.Getenv("DEVELOPMENT_CHART_DIR"), "./chart"), "--create-namespace", "--namespace", namespace, "--set", "agentOnly=true", diff --git a/cmd/vclusterctl/cmd/resume.go b/cmd/vclusterctl/cmd/resume.go index a6428170b..07508da13 100644 --- a/cmd/vclusterctl/cmd/resume.go +++ b/cmd/vclusterctl/cmd/resume.go @@ -3,6 +3,7 @@ package cmd import ( "cmp" "context" + "errors" "fmt" "github.com/loft-sh/log" @@ -73,5 +74,14 @@ func (cmd *ResumeCmd) Run(ctx context.Context, args []string) error { return cli.ResumePlatform(ctx, &cmd.ResumeOptions, cfg, args[0], cmd.Log) } - return cli.ResumeHelm(ctx, cmd.GlobalFlags, args[0], cmd.Log) + if err := cli.ResumeHelm(ctx, cmd.GlobalFlags, args[0], cmd.Log); err != nil { + // If they specified a driver, don't fall back to the platform automatically. + if cmd.Driver == "" && errors.Is(err, cli.ErrPlatformDriverRequired) { + return cli.ResumePlatform(ctx, &cmd.ResumeOptions, cfg, args[0], cmd.Log) + } + + return err + } + + return nil } diff --git a/config/config.go b/config/config.go index e7188f39f..68a5ed271 100644 --- a/config/config.go +++ b/config/config.go @@ -225,6 +225,14 @@ func (c *Config) Distro() string { return K8SDistro } +func (c *Config) IsConfiguredForSleepMode() bool { + if c != nil && c.External["platform"] == nil { + return false + } + + return c.External["platform"]["autoSleep"] != nil || c.External["platform"]["autoDelete"] != nil +} + // ValidateChanges checks for disallowed config changes. // Currently only certain backingstore changes are allowed but no distro change. func ValidateChanges(oldCfg, newCfg *Config) error { diff --git a/no-sleepmode.yaml b/no-sleepmode.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/cli/add_vcluster_helm.go b/pkg/cli/add_vcluster_helm.go index 5d38bed26..d1117a821 100644 --- a/pkg/cli/add_vcluster_helm.go +++ b/pkg/cli/add_vcluster_helm.go @@ -3,12 +3,16 @@ package cli import ( "context" "fmt" + "time" "github.com/loft-sh/log" + "github.com/loft-sh/log/survey" "github.com/loft-sh/vcluster/pkg/cli/find" "github.com/loft-sh/vcluster/pkg/cli/flags" "github.com/loft-sh/vcluster/pkg/lifecycle" "github.com/loft-sh/vcluster/pkg/platform" + "github.com/loft-sh/vcluster/pkg/platform/clihelper" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ) @@ -44,6 +48,46 @@ func AddVClusterHelm( return err } + snoozed := false + // If the vCluster was paused with the helm driver, adding it to the platform will only create the secret for registration + // which leads to confusing behavior for the user since they won't see the cluster in the platform UI until it is resumed. + if lifecycle.IsPaused(vCluster) { + log.Infof("vCluster %s is currently sleeping. It will not be added to the platform until it wakes again.", vCluster.Name) + + snoozeConfirmation := "No. Leave it sleeping. (It will be added automatically on next wakeup)" + answer, err := log.Question(&survey.QuestionOptions{ + Question: fmt.Sprintf("Would you like to wake vCluster %s now to add immediately?", vCluster.Name), + DefaultValue: snoozeConfirmation, + Options: []string{ + snoozeConfirmation, + "Yes. Wake and add now.", + }, + }) + + if err != nil { + return fmt.Errorf("failed to capture your response %w", err) + } + + if snoozed = answer == snoozeConfirmation; !snoozed { + if err = ResumeHelm(ctx, globalFlags, vClusterName, log); err != nil { + return fmt.Errorf("failed to wake up vCluster %s: %w", vClusterName, err) + } + + err = wait.PollUntilContextTimeout(ctx, time.Second, clihelper.Timeout(), false, func(ctx context.Context) (done bool, err error) { + vCluster, err = find.GetVCluster(ctx, globalFlags.Context, vClusterName, globalFlags.Namespace, log) + if err != nil { + return false, err + } + + return !lifecycle.IsPaused(vCluster), nil + }) + + if err != nil { + return fmt.Errorf("error waiting for vCluster to wake up %w", err) + } + } + } + // apply platform secret err = platform.ApplyPlatformSecret( ctx, @@ -68,6 +112,11 @@ func AddVClusterHelm( } } - log.Donef("Successfully added vCluster %s/%s", vCluster.Namespace, vCluster.Name) + if snoozed { + log.Infof("vCluster %s/%s will be added the next time it awakes", vCluster.Namespace, vCluster.Name) + log.Donef("Run 'vcluster wakeup --help' to learn how to wake up vCluster %s/%s to complete the add operation.", vCluster.Namespace, vCluster.Name) + } else { + log.Donef("Successfully added vCluster %s/%s", vCluster.Namespace, vCluster.Name) + } return nil } diff --git a/pkg/cli/create_helm.go b/pkg/cli/create_helm.go index 47c8990ec..be742572e 100644 --- a/pkg/cli/create_helm.go +++ b/pkg/cli/create_helm.go @@ -268,7 +268,7 @@ func CreateHelm(ctx context.Context, options *CreateOptions, globalFlags *flags. cmd.Connect = false } - if isSleepModeConfigured(vClusterConfig) { + if vClusterConfig.IsConfiguredForSleepMode() { if agentDeployed, err := cmd.isLoftAgentDeployed(ctx); err != nil { return fmt.Errorf("is agent deployed: %w", err) } else if !agentDeployed { @@ -391,13 +391,6 @@ func (cmd *createHelm) isLoftAgentDeployed(ctx context.Context) (bool, error) { return len(podList.Items) > 0, nil } -func isSleepModeConfigured(vClusterConfig *config.Config) bool { - if vClusterConfig == nil || vClusterConfig.External == nil || vClusterConfig.External["platform"] == nil { - return false - } - return vClusterConfig.External["platform"]["autoSleep"] != nil || vClusterConfig.External["platform"]["autoDelete"] != nil -} - func isVClusterDeployed(release *helm.Release) bool { return release != nil && release.Chart != nil && diff --git a/pkg/cli/find/find.go b/pkg/cli/find/find.go index 0c8a89fe8..4b843ff60 100644 --- a/pkg/cli/find/find.go +++ b/pkg/cli/find/find.go @@ -11,6 +11,7 @@ import ( "github.com/loft-sh/log/survey" "github.com/loft-sh/log/terminal" "github.com/loft-sh/vcluster/pkg/platform" + "github.com/loft-sh/vcluster/pkg/platform/sleepmode" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/loft-sh/vcluster/pkg/constants" @@ -31,6 +32,8 @@ type VCluster struct { Created metav1.Time Name string Namespace string + Annotations map[string]string + Labels map[string]string Status Status Context string Version string @@ -161,6 +164,20 @@ func GetVCluster(ctx context.Context, context, name, namespace string, log log.L return nil, fmt.Errorf("unexpected error searching for selected virtual cluster") } +func (v *VCluster) IsSleeping() bool { + return sleepmode.IsSleeping(v) +} + +// GetAnnotations implements Annotated +func (v *VCluster) GetAnnotations() map[string]string { + return v.Annotations +} + +// GetLabels implements Labeled +func (v *VCluster) GetLabels() map[string]string { + return v.Labels +} + func FormatOptions(format string, options [][]string) []string { if len(options) == 0 { return []string{} @@ -312,12 +329,7 @@ func findInContext(ctx context.Context, context, name, namespace string, timeout continue } - var paused string - - if p.Annotations != nil { - paused = p.Annotations[constants.PausedAnnotation] - } - if p.Spec.Replicas != nil && *p.Spec.Replicas == 0 && paused != "true" { + if p.Spec.Replicas != nil && *p.Spec.Replicas == 0 && !isPaused(&p) { // if the stateful set has been scaled down we'll ignore it -- this happens when // using devspace to do vcluster plugin dev for example, devspace scales down the // vcluster stateful set and re-creates a deployment for "dev mode" so we end up @@ -413,6 +425,8 @@ func getVCluster(ctx context.Context, object client.Object, context, release str return VCluster{ Name: release, Namespace: namespace, + Annotations: object.GetAnnotations(), + Labels: object.GetLabels(), Status: Status(status), Created: created, Context: context, @@ -561,3 +575,10 @@ func GetPodStatus(pod *corev1.Pod) string { } return reason } + +func isPaused(v client.Object) bool { + annotations := v.GetAnnotations() + labels := v.GetLabels() + + return annotations[constants.PausedAnnotation] == "true" || labels[sleepmode.Label] == "true" +} diff --git a/pkg/cli/pause_helm.go b/pkg/cli/pause_helm.go index 0826c6812..33f9db5c9 100644 --- a/pkg/cli/pause_helm.go +++ b/pkg/cli/pause_helm.go @@ -30,6 +30,11 @@ func PauseHelm(ctx context.Context, globalFlags *flags.GlobalFlags, vClusterName return err } + if vCluster.IsSleeping() { + log.Infof("vcluster %s/%s is already sleeping", globalFlags.Namespace, vClusterName) + return nil + } + err = lifecycle.PauseVCluster(ctx, kubeClient, vClusterName, globalFlags.Namespace, log) if err != nil { return err diff --git a/pkg/cli/pause_platform.go b/pkg/cli/pause_platform.go index e5bf54856..26869e9b0 100644 --- a/pkg/cli/pause_platform.go +++ b/pkg/cli/pause_platform.go @@ -22,11 +22,15 @@ func PausePlatform(ctx context.Context, options *PauseOptions, cfg *cliconfig.CL if err != nil { return err } + vCluster, err := find.GetPlatformVCluster(ctx, platformClient, vClusterName, options.Project, log) if err != nil { return err - } else if vCluster.VirtualCluster != nil && vCluster.VirtualCluster.Spec.External { - return fmt.Errorf("cannot pause a virtual cluster that was created via helm, please run 'vcluster use driver helm' or use the '--driver helm' flag") + } + + if vCluster.IsInstanceSleeping() { + log.Infof("vcluster %s/%s is already paused", vCluster.VirtualCluster.Namespace, vClusterName) + return nil } managementClient, err := platformClient.Management() diff --git a/pkg/cli/resume_helm.go b/pkg/cli/resume_helm.go index 64a6a6ab7..6d986da8d 100644 --- a/pkg/cli/resume_helm.go +++ b/pkg/cli/resume_helm.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "fmt" "github.com/loft-sh/log" @@ -17,12 +18,18 @@ type ResumeOptions struct { Project string } +var ErrPlatformDriverRequired = errors.New("cannot resume a virtual cluster that is paused by the platform, please run 'vcluster use driver platform' or use the '--driver platform' flag") + func ResumeHelm(ctx context.Context, globalFlags *flags.GlobalFlags, vClusterName string, log log.Logger) error { vCluster, err := find.GetVCluster(ctx, globalFlags.Context, vClusterName, globalFlags.Namespace, log) if err != nil { return err } + if vCluster.IsSleeping() { + return ErrPlatformDriverRequired + } + kubeClient, err := prepareResume(vCluster, globalFlags) if err != nil { return err diff --git a/pkg/cli/resume_platform.go b/pkg/cli/resume_platform.go index 0dc1c493a..e8683fc99 100644 --- a/pkg/cli/resume_platform.go +++ b/pkg/cli/resume_platform.go @@ -18,8 +18,14 @@ func ResumePlatform(ctx context.Context, options *ResumeOptions, config *config. vCluster, err := find.GetPlatformVCluster(ctx, platformClient, vClusterName, options.Project, log) if err != nil { return err - } else if vCluster.VirtualCluster != nil && vCluster.VirtualCluster.Spec.External { - return fmt.Errorf("cannot resume a virtual cluster that was created via helm, please run 'vcluster use driver helm' or use the '--driver helm' flag") + } + + if !vCluster.IsInstanceSleeping() { + return fmt.Errorf( + "couldn't find a paused vcluster %s in namespace %s. Make sure the vcluster exists and was paused previously", + vCluster.VirtualCluster.Spec.ClusterRef.VirtualCluster, + vCluster.VirtualCluster.Spec.ClusterRef.Namespace, + ) } managementClient, err := platformClient.Management() diff --git a/pkg/kube/meta.go b/pkg/kube/meta.go index 302482913..8b7bd4549 100644 --- a/pkg/kube/meta.go +++ b/pkg/kube/meta.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/loft-sh/log" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -17,7 +16,30 @@ const ( LoftCustomLinksDelimiter = "\n" ) -func UpdateLabels(obj metav1.Object, labelList []string) (bool, error) { +type ( + // Annotated is an interface for objects that have annotations + Annotated interface { + GetAnnotations() map[string]string + } + // Annotatable is an interface for objects that have annotations and ` + Annotatable interface { + Annotated + SetAnnotations(map[string]string) + } + + // Labeled is an interface for objects that have labels + Labeled interface { + GetLabels() map[string]string + } + + // Labelable is an interface for objects that have labels and can set them + Labelable interface { + Labeled + SetLabels(map[string]string) + } +) + +func UpdateLabels(obj Labelable, labelList []string) (bool, error) { // parse strings to map labels, err := parseStringMap(labelList) if err != nil { @@ -44,7 +66,7 @@ func UpdateLabels(obj metav1.Object, labelList []string) (bool, error) { return changed, nil } -func UpdateAnnotations(obj metav1.Object, annotationList []string) (bool, error) { +func UpdateAnnotations(obj Annotatable, annotationList []string) (bool, error) { // parse strings to map annotations, err := parseStringMap(annotationList) if err != nil { @@ -72,7 +94,7 @@ func UpdateAnnotations(obj metav1.Object, annotationList []string) (bool, error) // SetCustomLinksAnnotation sets the list of links for the UI to display next to the project member({space/virtualcluster}instance) // it handles unspecified links (empty) during create and update -func SetCustomLinksAnnotation(obj metav1.Object, links []string) bool { +func SetCustomLinksAnnotation(obj Annotatable, links []string) bool { var changed bool if obj == nil { log.GetInstance().Error("SetCustomLinksAnnotation called on nil object") diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index 8868e6244..548f4be8b 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -8,6 +8,7 @@ import ( "github.com/loft-sh/log" "github.com/loft-sh/vcluster/pkg/constants" + "github.com/loft-sh/vcluster/pkg/kube" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -120,7 +121,7 @@ func scaleDownDeployment(ctx context.Context, kubeClient kubernetes.Interface, l zero := int32(0) for _, item := range list.Items { - if item.Annotations != nil && item.Annotations[constants.PausedAnnotation] == "true" { + if IsPaused(&item) { log.Infof("vcluster %s/%s is already paused", namespace, item.Name) return true, nil } else if item.Spec.Replicas != nil && *item.Spec.Replicas == 0 { @@ -182,7 +183,7 @@ func scaleDownStatefulSet(ctx context.Context, kubeClient kubernetes.Interface, zero := int32(0) for _, item := range list.Items { - if item.Annotations != nil && item.Annotations[constants.PausedAnnotation] == "true" { + if IsPaused(&item) { log.Infof("vcluster %s/%s is already paused", namespace, item.Name) return true, nil } else if item.Spec.Replicas != nil && *item.Spec.Replicas == 0 { @@ -280,7 +281,7 @@ func scaleUpDeployment(ctx context.Context, kubeClient kubernetes.Interface, lab } for _, item := range list.Items { - if item.Annotations == nil || item.Annotations[constants.PausedAnnotation] != "true" { + if !IsPaused(&item) { return false, nil } @@ -326,7 +327,7 @@ func scaleUpStatefulSet(ctx context.Context, kubeClient kubernetes.Interface, la } for _, item := range list.Items { - if item.Annotations == nil || item.Annotations[constants.PausedAnnotation] != "true" { + if !IsPaused(&item) { return false, nil } @@ -362,3 +363,7 @@ func scaleUpStatefulSet(ctx context.Context, kubeClient kubernetes.Interface, la return true, nil } + +func IsPaused(annotated kube.Annotated) bool { + return annotated != nil && annotated.GetAnnotations()[constants.PausedAnnotation] == "true" +} diff --git a/pkg/platform/helper.go b/pkg/platform/helper.go index b3d74dffc..a7d8c88cd 100644 --- a/pkg/platform/helper.go +++ b/pkg/platform/helper.go @@ -21,6 +21,7 @@ import ( "github.com/loft-sh/vcluster/pkg/platform/clihelper" "github.com/loft-sh/vcluster/pkg/platform/kube" "github.com/loft-sh/vcluster/pkg/platform/kubeconfig" + "github.com/loft-sh/vcluster/pkg/platform/sleepmode" "github.com/loft-sh/vcluster/pkg/projectutil" "github.com/loft-sh/vcluster/pkg/util" perrors "github.com/pkg/errors" @@ -563,6 +564,10 @@ type ProjectProjectSecret struct { ProjectSecret managementv1.ProjectSecret } +func (vci *VirtualClusterInstanceProject) IsInstanceSleeping() bool { + return vci != nil && vci.VirtualCluster != nil && sleepmode.IsInstanceSleeping(vci.VirtualCluster) +} + func GetProjectSecrets(ctx context.Context, managementClient kube.Interface, projectNames ...string) ([]*ProjectProjectSecret, error) { var projects []*managementv1.Project if len(projectNames) == 0 { @@ -1121,8 +1126,8 @@ func WaitForVirtualClusterInstance(ctx context.Context, managementClient kube.In } if virtualClusterInstance.Status.Phase == storagev1.InstanceSleeping { - log.Info("Wait until vcluster wakes up") - defer log.Donef("Successfully woken up vcluster %s", name) + log.Info("Wait until vcluster instance wakes up") + defer log.Donef("virtual cluster %s wakeup successful", name) err := wakeupVCluster(ctx, managementClient, virtualClusterInstance) if err != nil { return nil, fmt.Errorf("error waking up vcluster %s: %s", name, util.GetCause(err)) diff --git a/pkg/platform/sleepmode/sleepmode.go b/pkg/platform/sleepmode/sleepmode.go new file mode 100644 index 000000000..3a6381cc7 --- /dev/null +++ b/pkg/platform/sleepmode/sleepmode.go @@ -0,0 +1,16 @@ +package sleepmode + +import "github.com/loft-sh/vcluster/pkg/kube" + +const ( + Label = "loft.sh/sleep-mode" + SleepingSinceAnnotation = "sleepmode.loft.sh/sleeping-since" +) + +func IsSleeping(labeled kube.Labeled) bool { + return labeled.GetLabels()[Label] == "true" +} + +func IsInstanceSleeping(annotated kube.Annotated) bool { + return annotated != nil && annotated.GetAnnotations()[SleepingSinceAnnotation] != "" +}