From d82f55954ab6bbbc64664038af6ef61da0f7f15f Mon Sep 17 00:00:00 2001 From: Jiaqi Luo <6218999+jiaqiluo@users.noreply.github.com> Date: Tue, 23 Jul 2024 15:25:37 -0700 Subject: [PATCH 1/2] Fix the issue where rancher-machine does not destroy vSphere VMs when the provision fails. In general, rancher-machine takes at least three steps to provision a vSphere machine: first, create the VM instance; then customize the VM's configuration; and finally bootstrap the VM. Previously, rancher-machine saves the VM's ID only after the bootstrapping step succeeds. If rancher-machine creates but fails to start the VM, it will not destroy the VM because the VM's uuid is missing. Now, rancher-machine retrieves and saves the uuid as machineID as soon as the VM is created, and saves the updated Host object to the store even if the creation process fails. This ensures the machineID is always preserved so that rancher-machine can find and destroy the VM on the vSphere server side properly. --- drivers/vmwarevsphere/create.go | 23 +++++++++++++++++++++++ drivers/vmwarevsphere/vsphere.go | 1 + libmachine/libmachine.go | 5 +++++ 3 files changed, 29 insertions(+) diff --git a/drivers/vmwarevsphere/create.go b/drivers/vmwarevsphere/create.go index e5e89dce40..7c82e2f567 100644 --- a/drivers/vmwarevsphere/create.go +++ b/drivers/vmwarevsphere/create.go @@ -144,6 +144,13 @@ func (d *Driver) createLegacy() error { if err != nil { return err } + log.Info("Fetching MachineID ...") + // save the machine id as soon as the VM is created + if _, err = d.GetMachineId(); err != nil { + // no need to return the error, it is not a blocker for creating the machine, + // we will fetch the machineID again after starting the VM + log.Warnf("[createLegacy] failed to fetch MachineID for %s: %v", d.MachineName, err) + } log.Infof("Uploading Boot2docker ISO ...") vm := object.NewVirtualMachine(c.Client, info.Result.(types.ManagedObjectReference)) @@ -274,6 +281,14 @@ func (d *Driver) createFromVmName() error { return err } + log.Info("Fetching MachineID ...") + // save the machine id as soon as the VM is created + if _, err = d.GetMachineId(); err != nil { + // no need to return the error, it is not a blocker for creating the machine, + // we will fetch the machineID again after starting the VM + log.Warnf("[createFromVmName] failed to fetch MachineID for %s: %v", d.MachineName, err) + } + // Retrieve the new VM vm := object.NewVirtualMachine(c.Client, info.Result.(types.ManagedObjectReference)) if err := d.addNetworks(vm, d.networks); err != nil { @@ -357,6 +372,14 @@ func (d *Driver) createFromLibraryName() error { return err } + log.Info("Fetching MachineID ...") + // save the machine id as soon as the VM is created + if _, err = d.GetMachineId(); err != nil { + // no need to return the error, it is not a blocker for creating the machine, + // we will fetch the machineID again after starting the VM + log.Warnf("[createFromLibraryName] failed to fetch MachineID for %s: %v", d.MachineName, err) + } + // At this point, the VM is deployed from content library with defaults from template // Reconfiguration of the VM based on driver inputs follows diff --git a/drivers/vmwarevsphere/vsphere.go b/drivers/vmwarevsphere/vsphere.go index cd88784052..af363c71de 100644 --- a/drivers/vmwarevsphere/vsphere.go +++ b/drivers/vmwarevsphere/vsphere.go @@ -453,6 +453,7 @@ func (d *Driver) Kill() error { func (d *Driver) Remove() error { if d.MachineId == "" { // no guid from config, nothing in vsphere to delete + log.Info("MachineID is empty, skipping removing VM") return nil } diff --git a/libmachine/libmachine.go b/libmachine/libmachine.go index 70a2f58120..2efdfeaba9 100644 --- a/libmachine/libmachine.go +++ b/libmachine/libmachine.go @@ -136,6 +136,11 @@ func (api *Client) Create(h *host.Host) error { log.Info("Creating machine...") if err := api.performCreate(h); err != nil { + // it is possible that the VM is instantiated but fails to bootstrap, + // save the machine to the store, so the VM and associated resources can be found and destroyed later + if err := api.Save(h); err != nil { + log.Warnf("Error saving host to store after creation fails: %s", err) + } return fmt.Errorf("Error creating machine: %s", err) } From 05caa37c6ddb63f139e701f0750c669769ea62c2 Mon Sep 17 00:00:00 2001 From: Jiaqi Luo <6218999+jiaqiluo@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:55:17 -0700 Subject: [PATCH 2/2] address comments --- libmachine/libmachine.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libmachine/libmachine.go b/libmachine/libmachine.go index 2efdfeaba9..b3d5a99c3c 100644 --- a/libmachine/libmachine.go +++ b/libmachine/libmachine.go @@ -136,10 +136,12 @@ func (api *Client) Create(h *host.Host) error { log.Info("Creating machine...") if err := api.performCreate(h); err != nil { - // it is possible that the VM is instantiated but fails to bootstrap, - // save the machine to the store, so the VM and associated resources can be found and destroyed later - if err := api.Save(h); err != nil { - log.Warnf("Error saving host to store after creation fails: %s", err) + if h.Driver.DriverName() == "vmwarevsphere" { + // it is possible that the VM is instantiated but fails to bootstrap, + // save the machine to the store, so the VM and associated resources can be found and destroyed later + if err := api.Save(h); err != nil { + log.Warnf("Error saving host to store after creation fails: %s", err) + } } return fmt.Errorf("Error creating machine: %s", err) }