diff --git a/cmd/vclusterctl/cmd/platform/add/cluster.go b/cmd/vclusterctl/cmd/platform/add/cluster.go index f166529c3..0542e115c 100644 --- a/cmd/vclusterctl/cmd/platform/add/cluster.go +++ b/cmd/vclusterctl/cmd/platform/add/cluster.go @@ -143,7 +143,7 @@ func (cmd *ClusterCmd) Run(ctx context.Context, args []string) error { if os.Getenv("DEVELOPMENT") == "true" { helmArgs = []string{ - "upgrade", "--install", "loft", "./chart", + "upgrade", "--install", "loft", cmp.Or(os.Getenv("DEVELOPMENT_CHART_DIR"), "./chart"), "--create-namespace", "--namespace", namespace, "--set", "agentOnly=true", diff --git a/cmd/vclusterctl/cmd/resume.go b/cmd/vclusterctl/cmd/resume.go index a6428170b..07508da13 100644 --- a/cmd/vclusterctl/cmd/resume.go +++ b/cmd/vclusterctl/cmd/resume.go @@ -3,6 +3,7 @@ package cmd import ( "cmp" "context" + "errors" "fmt" "github.com/loft-sh/log" @@ -73,5 +74,14 @@ func (cmd *ResumeCmd) Run(ctx context.Context, args []string) error { return cli.ResumePlatform(ctx, &cmd.ResumeOptions, cfg, args[0], cmd.Log) } - return cli.ResumeHelm(ctx, cmd.GlobalFlags, args[0], cmd.Log) + if err := cli.ResumeHelm(ctx, cmd.GlobalFlags, args[0], cmd.Log); err != nil { + // If they specified a driver, don't fall back to the platform automatically. + if cmd.Driver == "" && errors.Is(err, cli.ErrPlatformDriverRequired) { + return cli.ResumePlatform(ctx, &cmd.ResumeOptions, cfg, args[0], cmd.Log) + } + + return err + } + + return nil } diff --git a/config/config.go b/config/config.go index e7188f39f..68a5ed271 100644 --- a/config/config.go +++ b/config/config.go @@ -225,6 +225,14 @@ func (c *Config) Distro() string { return K8SDistro } +func (c *Config) IsConfiguredForSleepMode() bool { + if c != nil && c.External["platform"] == nil { + return false + } + + return c.External["platform"]["autoSleep"] != nil || c.External["platform"]["autoDelete"] != nil +} + // ValidateChanges checks for disallowed config changes. // Currently only certain backingstore changes are allowed but no distro change. func ValidateChanges(oldCfg, newCfg *Config) error { diff --git a/no-sleepmode.yaml b/no-sleepmode.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/cli/add_vcluster_helm.go b/pkg/cli/add_vcluster_helm.go index 5d38bed26..d1117a821 100644 --- a/pkg/cli/add_vcluster_helm.go +++ b/pkg/cli/add_vcluster_helm.go @@ -3,12 +3,16 @@ package cli import ( "context" "fmt" + "time" "github.com/loft-sh/log" + "github.com/loft-sh/log/survey" "github.com/loft-sh/vcluster/pkg/cli/find" "github.com/loft-sh/vcluster/pkg/cli/flags" "github.com/loft-sh/vcluster/pkg/lifecycle" "github.com/loft-sh/vcluster/pkg/platform" + "github.com/loft-sh/vcluster/pkg/platform/clihelper" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ) @@ -44,6 +48,46 @@ func AddVClusterHelm( return err } + snoozed := false + // If the vCluster was paused with the helm driver, adding it to the platform will only create the secret for registration + // which leads to confusing behavior for the user since they won't see the cluster in the platform UI until it is resumed. + if lifecycle.IsPaused(vCluster) { + log.Infof("vCluster %s is currently sleeping. It will not be added to the platform until it wakes again.", vCluster.Name) + + snoozeConfirmation := "No. Leave it sleeping. (It will be added automatically on next wakeup)" + answer, err := log.Question(&survey.QuestionOptions{ + Question: fmt.Sprintf("Would you like to wake vCluster %s now to add immediately?", vCluster.Name), + DefaultValue: snoozeConfirmation, + Options: []string{ + snoozeConfirmation, + "Yes. Wake and add now.", + }, + }) + + if err != nil { + return fmt.Errorf("failed to capture your response %w", err) + } + + if snoozed = answer == snoozeConfirmation; !snoozed { + if err = ResumeHelm(ctx, globalFlags, vClusterName, log); err != nil { + return fmt.Errorf("failed to wake up vCluster %s: %w", vClusterName, err) + } + + err = wait.PollUntilContextTimeout(ctx, time.Second, clihelper.Timeout(), false, func(ctx context.Context) (done bool, err error) { + vCluster, err = find.GetVCluster(ctx, globalFlags.Context, vClusterName, globalFlags.Namespace, log) + if err != nil { + return false, err + } + + return !lifecycle.IsPaused(vCluster), nil + }) + + if err != nil { + return fmt.Errorf("error waiting for vCluster to wake up %w", err) + } + } + } + // apply platform secret err = platform.ApplyPlatformSecret( ctx, @@ -68,6 +112,11 @@ func AddVClusterHelm( } } - log.Donef("Successfully added vCluster %s/%s", vCluster.Namespace, vCluster.Name) + if snoozed { + log.Infof("vCluster %s/%s will be added the next time it awakes", vCluster.Namespace, vCluster.Name) + log.Donef("Run 'vcluster wakeup --help' to learn how to wake up vCluster %s/%s to complete the add operation.", vCluster.Namespace, vCluster.Name) + } else { + log.Donef("Successfully added vCluster %s/%s", vCluster.Namespace, vCluster.Name) + } return nil } diff --git a/pkg/cli/create_helm.go b/pkg/cli/create_helm.go index 47c8990ec..be742572e 100644 --- a/pkg/cli/create_helm.go +++ b/pkg/cli/create_helm.go @@ -268,7 +268,7 @@ func CreateHelm(ctx context.Context, options *CreateOptions, globalFlags *flags. cmd.Connect = false } - if isSleepModeConfigured(vClusterConfig) { + if vClusterConfig.IsConfiguredForSleepMode() { if agentDeployed, err := cmd.isLoftAgentDeployed(ctx); err != nil { return fmt.Errorf("is agent deployed: %w", err) } else if !agentDeployed { @@ -391,13 +391,6 @@ func (cmd *createHelm) isLoftAgentDeployed(ctx context.Context) (bool, error) { return len(podList.Items) > 0, nil } -func isSleepModeConfigured(vClusterConfig *config.Config) bool { - if vClusterConfig == nil || vClusterConfig.External == nil || vClusterConfig.External["platform"] == nil { - return false - } - return vClusterConfig.External["platform"]["autoSleep"] != nil || vClusterConfig.External["platform"]["autoDelete"] != nil -} - func isVClusterDeployed(release *helm.Release) bool { return release != nil && release.Chart != nil && diff --git a/pkg/cli/find/find.go b/pkg/cli/find/find.go index 0c8a89fe8..4b843ff60 100644 --- a/pkg/cli/find/find.go +++ b/pkg/cli/find/find.go @@ -11,6 +11,7 @@ import ( "github.com/loft-sh/log/survey" "github.com/loft-sh/log/terminal" "github.com/loft-sh/vcluster/pkg/platform" + "github.com/loft-sh/vcluster/pkg/platform/sleepmode" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/loft-sh/vcluster/pkg/constants" @@ -31,6 +32,8 @@ type VCluster struct { Created metav1.Time Name string Namespace string + Annotations map[string]string + Labels map[string]string Status Status Context string Version string @@ -161,6 +164,20 @@ func GetVCluster(ctx context.Context, context, name, namespace string, log log.L return nil, fmt.Errorf("unexpected error searching for selected virtual cluster") } +func (v *VCluster) IsSleeping() bool { + return sleepmode.IsSleeping(v) +} + +// GetAnnotations implements Annotated +func (v *VCluster) GetAnnotations() map[string]string { + return v.Annotations +} + +// GetLabels implements Labeled +func (v *VCluster) GetLabels() map[string]string { + return v.Labels +} + func FormatOptions(format string, options [][]string) []string { if len(options) == 0 { return []string{} @@ -312,12 +329,7 @@ func findInContext(ctx context.Context, context, name, namespace string, timeout continue } - var paused string - - if p.Annotations != nil { - paused = p.Annotations[constants.PausedAnnotation] - } - if p.Spec.Replicas != nil && *p.Spec.Replicas == 0 && paused != "true" { + if p.Spec.Replicas != nil && *p.Spec.Replicas == 0 && !isPaused(&p) { // if the stateful set has been scaled down we'll ignore it -- this happens when // using devspace to do vcluster plugin dev for example, devspace scales down the // vcluster stateful set and re-creates a deployment for "dev mode" so we end up @@ -413,6 +425,8 @@ func getVCluster(ctx context.Context, object client.Object, context, release str return VCluster{ Name: release, Namespace: namespace, + Annotations: object.GetAnnotations(), + Labels: object.GetLabels(), Status: Status(status), Created: created, Context: context, @@ -561,3 +575,10 @@ func GetPodStatus(pod *corev1.Pod) string { } return reason } + +func isPaused(v client.Object) bool { + annotations := v.GetAnnotations() + labels := v.GetLabels() + + return annotations[constants.PausedAnnotation] == "true" || labels[sleepmode.Label] == "true" +} diff --git a/pkg/cli/pause_helm.go b/pkg/cli/pause_helm.go index 0826c6812..33f9db5c9 100644 --- a/pkg/cli/pause_helm.go +++ b/pkg/cli/pause_helm.go @@ -30,6 +30,11 @@ func PauseHelm(ctx context.Context, globalFlags *flags.GlobalFlags, vClusterName return err } + if vCluster.IsSleeping() { + log.Infof("vcluster %s/%s is already sleeping", globalFlags.Namespace, vClusterName) + return nil + } + err = lifecycle.PauseVCluster(ctx, kubeClient, vClusterName, globalFlags.Namespace, log) if err != nil { return err diff --git a/pkg/cli/pause_platform.go b/pkg/cli/pause_platform.go index e5bf54856..26869e9b0 100644 --- a/pkg/cli/pause_platform.go +++ b/pkg/cli/pause_platform.go @@ -22,11 +22,15 @@ func PausePlatform(ctx context.Context, options *PauseOptions, cfg *cliconfig.CL if err != nil { return err } + vCluster, err := find.GetPlatformVCluster(ctx, platformClient, vClusterName, options.Project, log) if err != nil { return err - } else if vCluster.VirtualCluster != nil && vCluster.VirtualCluster.Spec.External { - return fmt.Errorf("cannot pause a virtual cluster that was created via helm, please run 'vcluster use driver helm' or use the '--driver helm' flag") + } + + if vCluster.IsInstanceSleeping() { + log.Infof("vcluster %s/%s is already paused", vCluster.VirtualCluster.Namespace, vClusterName) + return nil } managementClient, err := platformClient.Management() diff --git a/pkg/cli/resume_helm.go b/pkg/cli/resume_helm.go index 64a6a6ab7..6d986da8d 100644 --- a/pkg/cli/resume_helm.go +++ b/pkg/cli/resume_helm.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "fmt" "github.com/loft-sh/log" @@ -17,12 +18,18 @@ type ResumeOptions struct { Project string } +var ErrPlatformDriverRequired = errors.New("cannot resume a virtual cluster that is paused by the platform, please run 'vcluster use driver platform' or use the '--driver platform' flag") + func ResumeHelm(ctx context.Context, globalFlags *flags.GlobalFlags, vClusterName string, log log.Logger) error { vCluster, err := find.GetVCluster(ctx, globalFlags.Context, vClusterName, globalFlags.Namespace, log) if err != nil { return err } + if vCluster.IsSleeping() { + return ErrPlatformDriverRequired + } + kubeClient, err := prepareResume(vCluster, globalFlags) if err != nil { return err diff --git a/pkg/cli/resume_platform.go b/pkg/cli/resume_platform.go index 0dc1c493a..e8683fc99 100644 --- a/pkg/cli/resume_platform.go +++ b/pkg/cli/resume_platform.go @@ -18,8 +18,14 @@ func ResumePlatform(ctx context.Context, options *ResumeOptions, config *config. vCluster, err := find.GetPlatformVCluster(ctx, platformClient, vClusterName, options.Project, log) if err != nil { return err - } else if vCluster.VirtualCluster != nil && vCluster.VirtualCluster.Spec.External { - return fmt.Errorf("cannot resume a virtual cluster that was created via helm, please run 'vcluster use driver helm' or use the '--driver helm' flag") + } + + if !vCluster.IsInstanceSleeping() { + return fmt.Errorf( + "couldn't find a paused vcluster %s in namespace %s. Make sure the vcluster exists and was paused previously", + vCluster.VirtualCluster.Spec.ClusterRef.VirtualCluster, + vCluster.VirtualCluster.Spec.ClusterRef.Namespace, + ) } managementClient, err := platformClient.Management() diff --git a/pkg/kube/meta.go b/pkg/kube/meta.go index 302482913..8b7bd4549 100644 --- a/pkg/kube/meta.go +++ b/pkg/kube/meta.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/loft-sh/log" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -17,7 +16,30 @@ const ( LoftCustomLinksDelimiter = "\n" ) -func UpdateLabels(obj metav1.Object, labelList []string) (bool, error) { +type ( + // Annotated is an interface for objects that have annotations + Annotated interface { + GetAnnotations() map[string]string + } + // Annotatable is an interface for objects that have annotations and ` + Annotatable interface { + Annotated + SetAnnotations(map[string]string) + } + + // Labeled is an interface for objects that have labels + Labeled interface { + GetLabels() map[string]string + } + + // Labelable is an interface for objects that have labels and can set them + Labelable interface { + Labeled + SetLabels(map[string]string) + } +) + +func UpdateLabels(obj Labelable, labelList []string) (bool, error) { // parse strings to map labels, err := parseStringMap(labelList) if err != nil { @@ -44,7 +66,7 @@ func UpdateLabels(obj metav1.Object, labelList []string) (bool, error) { return changed, nil } -func UpdateAnnotations(obj metav1.Object, annotationList []string) (bool, error) { +func UpdateAnnotations(obj Annotatable, annotationList []string) (bool, error) { // parse strings to map annotations, err := parseStringMap(annotationList) if err != nil { @@ -72,7 +94,7 @@ func UpdateAnnotations(obj metav1.Object, annotationList []string) (bool, error) // SetCustomLinksAnnotation sets the list of links for the UI to display next to the project member({space/virtualcluster}instance) // it handles unspecified links (empty) during create and update -func SetCustomLinksAnnotation(obj metav1.Object, links []string) bool { +func SetCustomLinksAnnotation(obj Annotatable, links []string) bool { var changed bool if obj == nil { log.GetInstance().Error("SetCustomLinksAnnotation called on nil object") diff --git a/pkg/lifecycle/lifecycle.go b/pkg/lifecycle/lifecycle.go index 8868e6244..548f4be8b 100644 --- a/pkg/lifecycle/lifecycle.go +++ b/pkg/lifecycle/lifecycle.go @@ -8,6 +8,7 @@ import ( "github.com/loft-sh/log" "github.com/loft-sh/vcluster/pkg/constants" + "github.com/loft-sh/vcluster/pkg/kube" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -120,7 +121,7 @@ func scaleDownDeployment(ctx context.Context, kubeClient kubernetes.Interface, l zero := int32(0) for _, item := range list.Items { - if item.Annotations != nil && item.Annotations[constants.PausedAnnotation] == "true" { + if IsPaused(&item) { log.Infof("vcluster %s/%s is already paused", namespace, item.Name) return true, nil } else if item.Spec.Replicas != nil && *item.Spec.Replicas == 0 { @@ -182,7 +183,7 @@ func scaleDownStatefulSet(ctx context.Context, kubeClient kubernetes.Interface, zero := int32(0) for _, item := range list.Items { - if item.Annotations != nil && item.Annotations[constants.PausedAnnotation] == "true" { + if IsPaused(&item) { log.Infof("vcluster %s/%s is already paused", namespace, item.Name) return true, nil } else if item.Spec.Replicas != nil && *item.Spec.Replicas == 0 { @@ -280,7 +281,7 @@ func scaleUpDeployment(ctx context.Context, kubeClient kubernetes.Interface, lab } for _, item := range list.Items { - if item.Annotations == nil || item.Annotations[constants.PausedAnnotation] != "true" { + if !IsPaused(&item) { return false, nil } @@ -326,7 +327,7 @@ func scaleUpStatefulSet(ctx context.Context, kubeClient kubernetes.Interface, la } for _, item := range list.Items { - if item.Annotations == nil || item.Annotations[constants.PausedAnnotation] != "true" { + if !IsPaused(&item) { return false, nil } @@ -362,3 +363,7 @@ func scaleUpStatefulSet(ctx context.Context, kubeClient kubernetes.Interface, la return true, nil } + +func IsPaused(annotated kube.Annotated) bool { + return annotated != nil && annotated.GetAnnotations()[constants.PausedAnnotation] == "true" +} diff --git a/pkg/platform/helper.go b/pkg/platform/helper.go index b3d74dffc..a7d8c88cd 100644 --- a/pkg/platform/helper.go +++ b/pkg/platform/helper.go @@ -21,6 +21,7 @@ import ( "github.com/loft-sh/vcluster/pkg/platform/clihelper" "github.com/loft-sh/vcluster/pkg/platform/kube" "github.com/loft-sh/vcluster/pkg/platform/kubeconfig" + "github.com/loft-sh/vcluster/pkg/platform/sleepmode" "github.com/loft-sh/vcluster/pkg/projectutil" "github.com/loft-sh/vcluster/pkg/util" perrors "github.com/pkg/errors" @@ -563,6 +564,10 @@ type ProjectProjectSecret struct { ProjectSecret managementv1.ProjectSecret } +func (vci *VirtualClusterInstanceProject) IsInstanceSleeping() bool { + return vci != nil && vci.VirtualCluster != nil && sleepmode.IsInstanceSleeping(vci.VirtualCluster) +} + func GetProjectSecrets(ctx context.Context, managementClient kube.Interface, projectNames ...string) ([]*ProjectProjectSecret, error) { var projects []*managementv1.Project if len(projectNames) == 0 { @@ -1121,8 +1126,8 @@ func WaitForVirtualClusterInstance(ctx context.Context, managementClient kube.In } if virtualClusterInstance.Status.Phase == storagev1.InstanceSleeping { - log.Info("Wait until vcluster wakes up") - defer log.Donef("Successfully woken up vcluster %s", name) + log.Info("Wait until vcluster instance wakes up") + defer log.Donef("virtual cluster %s wakeup successful", name) err := wakeupVCluster(ctx, managementClient, virtualClusterInstance) if err != nil { return nil, fmt.Errorf("error waking up vcluster %s: %s", name, util.GetCause(err)) diff --git a/pkg/platform/sleepmode/sleepmode.go b/pkg/platform/sleepmode/sleepmode.go new file mode 100644 index 000000000..3a6381cc7 --- /dev/null +++ b/pkg/platform/sleepmode/sleepmode.go @@ -0,0 +1,16 @@ +package sleepmode + +import "github.com/loft-sh/vcluster/pkg/kube" + +const ( + Label = "loft.sh/sleep-mode" + SleepingSinceAnnotation = "sleepmode.loft.sh/sleeping-since" +) + +func IsSleeping(labeled kube.Labeled) bool { + return labeled.GetLabels()[Label] == "true" +} + +func IsInstanceSleeping(annotated kube.Annotated) bool { + return annotated != nil && annotated.GetAnnotations()[SleepingSinceAnnotation] != "" +}