diff --git a/neonvm/README.md b/neonvm/README.md index 8fe495261..b4a863d8d 100644 --- a/neonvm/README.md +++ b/neonvm/README.md @@ -115,7 +115,7 @@ To adjust the kernel config: ``` cd hack/kernel -docker build --build-arg KERNEL_VERSION=6.1.92 --platform linux/x86_64 --target build-deps -t kernel-build-deps -f Dockerfile.kernel-builder . +docker build --build-arg KERNEL_VERSION=6.1.63 --platform linux/x86_64 --target build-deps -t kernel-build-deps -f Dockerfile.kernel-builder . docker run --rm -v $PWD:/host --name kernel-build -it kernel-build-deps bash # inside that bash shell, do the menuconfig, then copy-out the config to /host ``` diff --git a/neonvm/apis/neonvm/v1/virtualmachine_types.go b/neonvm/apis/neonvm/v1/virtualmachine_types.go index 741f7a9e4..63f421e95 100644 --- a/neonvm/apis/neonvm/v1/virtualmachine_types.go +++ b/neonvm/apis/neonvm/v1/virtualmachine_types.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "github.com/samber/lo" @@ -172,6 +173,8 @@ type Guest struct { // +optional MemorySlots MemorySlots `json:"memorySlots"` // +optional + MemoryProvider *MemoryProvider `json:"memoryProvider,omitempty"` + // +optional RootDisk RootDisk `json:"rootDisk"` // Docker image Entrypoint array replacement. // +optional @@ -194,6 +197,24 @@ type Guest struct { Settings *GuestSettings `json:"settings,omitempty"` } +const virtioMemBlockSizeBytes = 8 * 1024 * 1024 // 8 MiB + +// ValidateForMemoryProvider returns an error iff the guest memory settings are invalid for the +// MemoryProvider. +// +// This is used in two places. First, to validate VirtualMachine object creation. Second, to handle +// the defaulting behavior for VirtualMachines that would be switching from DIMMSlots to VirtioMem +// on restart. We place more restrictions on VirtioMem because we use 8MiB block sizes, so changing +// to a new default can only happen if the memory slot size is a multiple of 8MiB. +func (g Guest) ValidateForMemoryProvider(p MemoryProvider) error { + if p == MemoryProviderVirtioMem { + if g.MemorySlotSize.Value()%virtioMemBlockSizeBytes != 0 { + return fmt.Errorf("memorySlotSize invalid for memoryProvider VirtioMem: must be a multiple of 8Mi") + } + } + return nil +} + type GuestSettings struct { // Individual lines to add to a sysctl.conf file. See sysctl.conf(5) for more // +optional @@ -361,6 +382,29 @@ type MemorySlots struct { Use *int32 `json:"use,omitempty"` } +// +kubebuilder:validation:Enum=DIMMSlots;VirtioMem +type MemoryProvider string + +const ( + MemoryProviderDIMMSlots MemoryProvider = "DIMMSlots" + MemoryProviderVirtioMem MemoryProvider = "VirtioMem" +) + +// FlagFunc is a parsing function to be used with flag.Func +func (p *MemoryProvider) FlagFunc(value string) error { + possibleValues := []string{ + string(MemoryProviderDIMMSlots), + string(MemoryProviderVirtioMem), + } + + if !slices.Contains(possibleValues, value) { + return fmt.Errorf("Unknown MemoryProvider %q, must be one of %v", value, possibleValues) + } + + *p = MemoryProvider(value) + return nil +} + type RootDisk struct { Image string `json:"image"` // +optional @@ -493,6 +537,8 @@ type VirtualMachineStatus struct { // +optional MemorySize *resource.Quantity `json:"memorySize,omitempty"` // +optional + MemoryProvider *MemoryProvider `json:"memoryProvider,omitempty"` + // +optional SSHSecretName string `json:"sshSecretName,omitempty"` } @@ -558,6 +604,7 @@ func (vm *VirtualMachine) Cleanup() { vm.Status.Node = "" vm.Status.CPUs = nil vm.Status.MemorySize = nil + vm.Status.MemoryProvider = nil } func (vm *VirtualMachine) HasRestarted() bool { diff --git a/neonvm/apis/neonvm/v1/virtualmachine_webhook.go b/neonvm/apis/neonvm/v1/virtualmachine_webhook.go index 5311b5d35..a4c7905a7 100644 --- a/neonvm/apis/neonvm/v1/virtualmachine_webhook.go +++ b/neonvm/apis/neonvm/v1/virtualmachine_webhook.go @@ -92,6 +92,13 @@ func (r *VirtualMachine) ValidateCreate() error { } } + // validate .spec.guest.memorySlotSize w.r.t. .spec.guest.memoryProvider + if r.Spec.Guest.MemoryProvider != nil { + if err := r.Spec.Guest.ValidateForMemoryProvider(*r.Spec.Guest.MemoryProvider); err != nil { + return fmt.Errorf(".spec.guest: %w", err) + } + } + // validate .spec.guest.memorySlots.use and .spec.guest.memorySlots.max if r.Spec.Guest.MemorySlots.Use != nil { if r.Spec.Guest.MemorySlots.Max == nil { @@ -160,6 +167,7 @@ func (r *VirtualMachine) ValidateUpdate(old runtime.Object) error { {".spec.guest.cpus.max", func(v *VirtualMachine) any { return v.Spec.Guest.CPUs.Max }}, {".spec.guest.memorySlots.min", func(v *VirtualMachine) any { return v.Spec.Guest.MemorySlots.Min }}, {".spec.guest.memorySlots.max", func(v *VirtualMachine) any { return v.Spec.Guest.MemorySlots.Max }}, + // nb: we don't check memoryProvider here, we have some specific logic for that. {".spec.guest.ports", func(v *VirtualMachine) any { return v.Spec.Guest.Ports }}, {".spec.guest.rootDisk", func(v *VirtualMachine) any { return v.Spec.Guest.RootDisk }}, {".spec.guest.command", func(v *VirtualMachine) any { return v.Spec.Guest.Command }}, @@ -187,6 +195,19 @@ func (r *VirtualMachine) ValidateUpdate(old runtime.Object) error { } } + // Only allow changing .spec.guest.memoryProvider if it was previously nil, and the new version + // is equal to .status.memoryProvider. + if !reflect.DeepEqual(before.Spec.Guest.MemoryProvider, r.Spec.Guest.MemoryProvider) { + if before.Spec.Guest.MemoryProvider != nil { + return errors.New(".spec.guest.memoryProvider cannot be changed once it is non-nil") + } + if !reflect.DeepEqual(r.Spec.Guest.MemoryProvider, r.Status.MemoryProvider) { + return errors.New(".spec.guest.memoryProvider cannot be set to any value other than .status.memoryProvider") + } + } + + // {".spec.guest.memoryProvider", func(v *VirtualMachine) any { return v.Spec.Guest.MemoryProvider }}, + // validate swap changes by comparing the SwapInfo for each. // // If there's an error with the old object, but NOT an error with the new one, we'll allow the diff --git a/neonvm/apis/neonvm/v1/zz_generated.deepcopy.go b/neonvm/apis/neonvm/v1/zz_generated.deepcopy.go index 326f2bade..b3188a16e 100644 --- a/neonvm/apis/neonvm/v1/zz_generated.deepcopy.go +++ b/neonvm/apis/neonvm/v1/zz_generated.deepcopy.go @@ -175,6 +175,11 @@ func (in *Guest) DeepCopyInto(out *Guest) { in.CPUs.DeepCopyInto(&out.CPUs) out.MemorySlotSize = in.MemorySlotSize.DeepCopy() in.MemorySlots.DeepCopyInto(&out.MemorySlots) + if in.MemoryProvider != nil { + in, out := &in.MemoryProvider, &out.MemoryProvider + *out = new(MemoryProvider) + **out = **in + } in.RootDisk.DeepCopyInto(&out.RootDisk) if in.Command != nil { in, out := &in.Command, &out.Command @@ -780,6 +785,11 @@ func (in *VirtualMachineStatus) DeepCopyInto(out *VirtualMachineStatus) { x := (*in).DeepCopy() *out = &x } + if in.MemoryProvider != nil { + in, out := &in.MemoryProvider, &out.MemoryProvider + *out = new(MemoryProvider) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineStatus. diff --git a/neonvm/config/controller/deployment.yaml b/neonvm/config/controller/deployment.yaml index d51d7edee..e1d489e78 100644 --- a/neonvm/config/controller/deployment.yaml +++ b/neonvm/config/controller/deployment.yaml @@ -59,6 +59,7 @@ spec: # * cache.direct=on - use O_DIRECT (don't abuse host's page cache!) # * cache.no-flush=on - ignores disk flush operations (not needed; our disks are ephemeral) - "--qemu-disk-cache-settings=cache.writeback=on,cache.direct=on,cache.no-flush=on" + - "--default-memory-provider=DIMMSlots" - "--failure-pending-period=1m" - "--failing-refresh-interval=15s" env: diff --git a/neonvm/config/crd/bases/vm.neon.tech_virtualmachines.yaml b/neonvm/config/crd/bases/vm.neon.tech_virtualmachines.yaml index 55e558512..1996e76e6 100644 --- a/neonvm/config/crd/bases/vm.neon.tech_virtualmachines.yaml +++ b/neonvm/config/crd/bases/vm.neon.tech_virtualmachines.yaml @@ -2412,6 +2412,11 @@ spec: type: array kernelImage: type: string + memoryProvider: + enum: + - DIMMSlots + - VirtioMem + type: string memorySlotSize: anyOf: - type: integer @@ -2764,6 +2769,11 @@ spec: type: string extraNetMask: type: string + memoryProvider: + enum: + - DIMMSlots + - VirtioMem + type: string memorySize: anyOf: - type: integer diff --git a/neonvm/controllers/config.go b/neonvm/controllers/config.go index 5c25c6d63..2b50a2c34 100644 --- a/neonvm/controllers/config.go +++ b/neonvm/controllers/config.go @@ -1,6 +1,10 @@ package controllers -import "time" +import ( + "time" + + vmv1 "github.com/neondatabase/autoscaling/neonvm/apis/neonvm/v1" +) // ReconcilerConfig stores shared configuration for VirtualMachineReconciler and // VirtualMachineMigrationReconciler. @@ -25,6 +29,10 @@ type ReconcilerConfig struct { // used in setting up the VM disks via QEMU's `-drive` flag. QEMUDiskCacheSettings string + // DefaultMemoryProvider is the memory provider (dimm slots or virtio-mem) that will be used for + // new VMs (or, when old ones restart) if nothing is explicitly set. + DefaultMemoryProvider vmv1.MemoryProvider + // FailurePendingPeriod is the period for the propagation of // reconciliation failures to the observability instruments FailurePendingPeriod time.Duration diff --git a/neonvm/controllers/vm_controller.go b/neonvm/controllers/vm_controller.go index e9ebfcf24..50ab6d952 100644 --- a/neonvm/controllers/vm_controller.go +++ b/neonvm/controllers/vm_controller.go @@ -323,14 +323,10 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine) vm.Status.SSHSecretName = fmt.Sprintf("ssh-neonvm-%s", vm.Name) } - // Generate runner pod name - if len(vm.Status.PodName) == 0 { - vm.Status.PodName = names.SimpleNameGenerator.GenerateName(fmt.Sprintf("%s-", vm.Name)) - // Update the .Status on API Server to avoid creating multiple pods for a single VM - // See https://github.com/neondatabase/autoscaling/issues/794 for the context - if err := r.Status().Update(ctx, vm); err != nil { - return fmt.Errorf("Failed to update VirtualMachine status: %w", err) - } + // Set memory provider for old VMs that don't have it in the Status. + if vm.Status.PodName != "" && vm.Status.MemoryProvider == nil { + log.Error(nil, "Setting default MemoryProvider for VM", "MemoryProvider", r.Config.DefaultMemoryProvider) + vm.Status.MemoryProvider = lo.ToPtr(r.Config.DefaultMemoryProvider) } switch vm.Status.Phase { @@ -369,6 +365,23 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine) // VirtualMachine just created, change Phase to "Pending" vm.Status.Phase = vmv1.VmPending case vmv1.VmPending: + // Generate runner pod name and set desired memory provider. After this, we know that + // MemoryProvider will be set, because earlier (above the 'switch vm.Status.Phase'), we set + // the MemoryProvider iff PodName was set but it wasn't. + if len(vm.Status.PodName) == 0 { + vm.Status.PodName = names.SimpleNameGenerator.GenerateName(fmt.Sprintf("%s-", vm.Name)) + if vm.Status.MemoryProvider == nil { + vm.Status.MemoryProvider = lo.ToPtr(pickMemoryProvider(r.Config, vm)) + } + // Update the .Status on API Server to avoid creating multiple pods for a single VM + // See https://github.com/neondatabase/autoscaling/issues/794 for the context + if err := r.Status().Update(ctx, vm); err != nil { + return fmt.Errorf("Failed to update VirtualMachine status: %w", err) + } + } + + memoryProvider := *vm.Status.MemoryProvider + // Check if the runner pod already exists, if not create a new one vmRunner := &corev1.Pod{} err := r.Get(ctx, types.NamespacedName{Name: vm.Status.PodName, Namespace: vm.Namespace}, vmRunner) @@ -402,7 +415,7 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine) } // Define a new pod - pod, err := r.podForVirtualMachine(vm, sshSecret) + pod, err := r.podForVirtualMachine(vm, memoryProvider, sshSecret) if err != nil { log.Error(err, "Failed to define new Pod resource for VirtualMachine") return err @@ -715,40 +728,71 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine) // seems already plugged correctly cpuScaled = true } + // update status by CPUs used in the VM + r.updateVMStatusCPU(ctx, vm, vmRunner, pluggedCPU, cgroupUsage) // do hotplug/unplug Memory memSlotsMin := *vm.Spec.Guest.MemorySlots.Min targetSlotCount := int(*vm.Spec.Guest.MemorySlots.Use - memSlotsMin) - realSlots, err := QmpSetMemorySlots(ctx, vm, targetSlotCount, r.Recorder) - if realSlots < 0 { - return err - } + switch *vm.Status.MemoryProvider { + case vmv1.MemoryProviderVirtioMem: + targetVirtioMemSize := int64(targetSlotCount) * vm.Spec.Guest.MemorySlotSize.Value() + previousTarget, err := QmpSetVirtioMem(vm, targetVirtioMemSize) + if err != nil { + return err + } - if realSlots != int(targetSlotCount) { - log.Info("Couldn't achieve desired memory slot count, will modify .spec.guest.memorySlots.use instead", "details", err) - // firstly re-fetch VM - if err := r.Get(ctx, types.NamespacedName{Name: vm.Name, Namespace: vm.Namespace}, vm); err != nil { - log.Error(err, "Unable to re-fetch VirtualMachine") + goalTotalSize := resource.NewQuantity( + int64(*vm.Spec.Guest.MemorySlots.Use)*vm.Spec.Guest.MemorySlotSize.Value(), + resource.BinarySI, + ) + + if previousTarget != targetVirtioMemSize { + // We changed the requested size. Make an event for it. + reason := "ScaleUp" + if targetVirtioMemSize < previousTarget { + reason = "ScaleDown" + } + r.Recorder.Eventf(vm, "Normal", reason, "Set virtio-mem size for %v total memory", goalTotalSize) + } + + // Maybe we're already using the amount we want? + // Update the status to reflect the current size - and if it matches goalTotalSize, ram + // is done. + currentTotalSize, err := QmpGetMemorySize(QmpAddr(vm)) + if err != nil { return err } - memorySlotsUseInSpec := *vm.Spec.Guest.MemorySlots.Use - memoryPluggedSlots := memSlotsMin + int32(realSlots) - *vm.Spec.Guest.MemorySlots.Use = memoryPluggedSlots - if err := r.tryUpdateVM(ctx, vm); err != nil { - log.Error(err, "Failed to update .spec.guest.memorySlots.use", - "old value", memorySlotsUseInSpec, - "new value", memoryPluggedSlots) + + ramScaled = currentTotalSize.Value() == goalTotalSize.Value() + r.updateVMStatusMemory(vm, currentTotalSize) + + case vmv1.MemoryProviderDIMMSlots: + realSlots, err := QmpSetMemorySlots(ctx, vm, targetSlotCount, r.Recorder) + if realSlots < 0 { return err } - } else { - ramScaled = true - } - // set VM phase to running if everything scaled - if cpuScaled && ramScaled { - // update status by CPUs used in the VM - r.updateVMStatusCPU(ctx, vm, vmRunner, pluggedCPU, cgroupUsage) + if realSlots != int(targetSlotCount) { + log.Info("Couldn't achieve desired memory slot count, will modify .spec.guest.memorySlots.use instead", "details", err) + // firstly re-fetch VM + if err := r.Get(ctx, types.NamespacedName{Name: vm.Name, Namespace: vm.Namespace}, vm); err != nil { + log.Error(err, "Unable to re-fetch VirtualMachine") + return err + } + memorySlotsUseInSpec := *vm.Spec.Guest.MemorySlots.Use + memoryPluggedSlots := memSlotsMin + int32(realSlots) + *vm.Spec.Guest.MemorySlots.Use = memoryPluggedSlots + if err := r.tryUpdateVM(ctx, vm); err != nil { + log.Error(err, "Failed to update .spec.guest.memorySlots.use", + "old value", memorySlotsUseInSpec, + "new value", memoryPluggedSlots) + return err + } + } else { + ramScaled = true + } // get Memory details from hypervisor and update VM status memorySize, err := QmpGetMemorySize(QmpAddr(vm)) @@ -758,7 +802,12 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine) } // update status by memory sizes used in the VM r.updateVMStatusMemory(vm, memorySize) + default: + panic(fmt.Errorf("unexpected vm.status.memoryProvider %q", *vm.Status.MemoryProvider)) + } + // set VM phase to running if everything scaled + if cpuScaled && ramScaled { vm.Status.Phase = vmv1.VmRunning } @@ -812,6 +861,23 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine) return nil } +func pickMemoryProvider(config *ReconcilerConfig, vm *vmv1.VirtualMachine) vmv1.MemoryProvider { + if p := vm.Spec.Guest.MemoryProvider; p != nil { + return *p + } + if p := vm.Status.MemoryProvider; p != nil { + return *p + } + + // Not all configurations are valid for virtio-mem. Only switch to the default as long as it + // won't be invalid: + if err := vm.Spec.Guest.ValidateForMemoryProvider(config.DefaultMemoryProvider); err != nil { + return vmv1.MemoryProviderDIMMSlots + } + + return config.DefaultMemoryProvider +} + type runnerStatusKind string const ( @@ -1083,9 +1149,10 @@ func extractVirtualMachineResourcesJSON(spec vmv1.VirtualMachineSpec) string { // podForVirtualMachine returns a VirtualMachine Pod object func (r *VMReconciler) podForVirtualMachine( vm *vmv1.VirtualMachine, + memoryProvider vmv1.MemoryProvider, sshSecret *corev1.Secret, ) (*corev1.Pod, error) { - pod, err := podSpec(vm, sshSecret, r.Config) + pod, err := podSpec(vm, memoryProvider, sshSecret, r.Config) if err != nil { return nil, err } @@ -1282,7 +1349,12 @@ func imageForVmRunner() (string, error) { return image, nil } -func podSpec(vm *vmv1.VirtualMachine, sshSecret *corev1.Secret, config *ReconcilerConfig) (*corev1.Pod, error) { +func podSpec( + vm *vmv1.VirtualMachine, + memoryProvider vmv1.MemoryProvider, + sshSecret *corev1.Secret, + config *ReconcilerConfig, +) (*corev1.Pod, error) { runnerVersion := api.RunnerProtoV1 labels := labelsForVirtualMachine(vm, &runnerVersion) annotations := annotationsForVirtualMachine(vm) @@ -1378,6 +1450,7 @@ func podSpec(vm *vmv1.VirtualMachine, sshSecret *corev1.Secret, config *Reconcil cmd = append( cmd, "-qemu-disk-cache-settings", config.QEMUDiskCacheSettings, + "-memory-provider", string(memoryProvider), "-vmspec", base64.StdEncoding.EncodeToString(vmSpecJson), "-vmstatus", base64.StdEncoding.EncodeToString(vmStatusJson), ) diff --git a/neonvm/controllers/vm_controller_test.go b/neonvm/controllers/vm_controller_test.go index 5d26157e5..9b9cc2aed 100644 --- a/neonvm/controllers/vm_controller_test.go +++ b/neonvm/controllers/vm_controller_test.go @@ -102,6 +102,7 @@ var _ = Describe("VirtualMachine controller", func() { UseContainerMgr: true, MaxConcurrentReconciles: 1, QEMUDiskCacheSettings: "cache=none", + DefaultMemoryProvider: vmv1.MemoryProviderDIMMSlots, FailurePendingPeriod: 1 * time.Minute, FailingRefreshInterval: 1 * time.Minute, }, diff --git a/neonvm/controllers/vm_qmp_queries.go b/neonvm/controllers/vm_qmp_queries.go index 59182278a..b92b453a5 100644 --- a/neonvm/controllers/vm_qmp_queries.go +++ b/neonvm/controllers/vm_qmp_queries.go @@ -333,6 +333,65 @@ type QMPRunner interface { Run([]byte) ([]byte, error) } +// QmpSetVirtioMem updates virtio-mem to the new target size, returning the previous target. +// +// If the new target size is equal to the previous one, this function does nothing but query the +// target. +func QmpSetVirtioMem(vm *vmv1.VirtualMachine, targetVirtioMemSize int64) (previous int64, _ error) { + // Note: The virtio-mem device only exists when max mem != min mem. + // So if min == max, we should just short-cut, skip the queries, and say it's all good. + // Refer to the instantiation in neonvm-runner for more. + if *vm.Spec.Guest.MemorySlots.Min == *vm.Spec.Guest.MemorySlots.Max { + // if target size is non-zero even though min == max, something went very wrong + if targetVirtioMemSize != 0 { + panic(fmt.Errorf( + "VM min mem slots == max mem slots, but target virtio-mem size %d != 0", + targetVirtioMemSize, + )) + } + // Otherwise, we're all good, just pretend like we talked to the VM. + return 0, nil + } + + mon, err := QmpConnect(QmpAddr(vm)) + if err != nil { + return 0, err + } + defer mon.Disconnect() //nolint:errcheck // nothing to do with error when deferred. TODO: log it? + + // First, fetch current desired virtio-mem size. If it's the same as targetVirtioMemSize, then + // we can report that it was already the same. + cmd := []byte(`{"execute": "qom-get", "arguments": {"path": "vm0", "property": "requested-size"}}`) + raw, err := mon.Run(cmd) + if err != nil { + return 0, err + } + result := struct { + Return int64 `json:"return"` + }{Return: 0} + if err := json.Unmarshal(raw, &result); err != nil { + return 0, fmt.Errorf("error unmarshaling json: %w", err) + } + + previous = result.Return + + if previous == targetVirtioMemSize { + return previous, nil + } + + // The current requested size is not equal to the new desired size. Let's change that. + cmd = []byte(fmt.Sprintf( + `{"execute": "qom-set", "arguments": {"path": "vm0", "property": "requested-size", "value": %d}}`, + targetVirtioMemSize, + )) + _, err = mon.Run(cmd) + if err != nil { + return 0, err + } + + return previous, nil +} + // QmpAddMemoryBackend adds a single memory slot to the VM with the given size. // // The memory slot does nothing until a corresponding "device" is added to the VM for the same memory slot. diff --git a/neonvm/controllers/vmmigration_controller.go b/neonvm/controllers/vmmigration_controller.go index 6ecb97760..ca9367f9a 100644 --- a/neonvm/controllers/vmmigration_controller.go +++ b/neonvm/controllers/vmmigration_controller.go @@ -314,12 +314,23 @@ func (r *VirtualMachineMigrationReconciler) Reconcile(ctx context.Context, req c } log.Info("CPUs in Target runner synced", "TargetPod.Name", migration.Status.TargetPodName) - // do hotplug Memory in targetRunner - log.Info("Syncing Memory in Target runner", "TargetPod.Name", migration.Status.TargetPodName) - if err := QmpSyncMemoryToTarget(vm, migration); err != nil { - return ctrl.Result{}, err + // do hotplug Memory in targetRunner -- only needed for dimm slots; virtio-mem Just Works™ + switch *vm.Status.MemoryProvider { + case vmv1.MemoryProviderVirtioMem: + // ref "Migration works out of the box" - https://lwn.net/Articles/755423/ + log.Info( + "No need to sync memory in Target runner because MemoryProvider is VirtioMem", + "TargetPod.Name", migration.Status.TargetPodName, + ) + case vmv1.MemoryProviderDIMMSlots: + log.Info("Syncing Memory in Target runner", "TargetPod.Name", migration.Status.TargetPodName) + if err := QmpSyncMemoryToTarget(vm, migration); err != nil { + return ctrl.Result{}, err + } + log.Info("Memory in Target runner synced", "TargetPod.Name", migration.Status.TargetPodName) + default: + panic(fmt.Errorf("unexpected vm.status.memoryProvider %q", *vm.Status.MemoryProvider)) } - log.Info("Memory in Target runner synced", "TargetPod.Name", migration.Status.TargetPodName) // Migrate only running VMs to target with plugged devices if vm.Status.Phase == vmv1.VmPreMigrating { @@ -692,8 +703,18 @@ func (r *VirtualMachineMigrationReconciler) targetPodForVirtualMachine( migration *vmv1.VirtualMachineMigration, sshSecret *corev1.Secret, ) (*corev1.Pod, error) { - - pod, err := podSpec(vm, sshSecret, r.Config) + if vm.Status.MemoryProvider == nil { + return nil, errors.New("cannot create target pod because vm.status.memoryProvider is not set") + } + // TODO: this is technically racy because target pod creation happens before we set the + // migration source pod, so in between reading this and starting the migration, it's + // *technically* possible that we create a target pod with a different memory provider than a + // newer source pod. + // Given that this requires (a) restart *during* initial live migration, and (b) that restart to + // change the memory provider, this is low enough risk that it's ok to leave to a follow-up. + memoryProvider := *vm.Status.MemoryProvider + + pod, err := podSpec(vm, memoryProvider, sshSecret, r.Config) if err != nil { return nil, err } diff --git a/neonvm/hack/kernel/linux-config-6.1.92 b/neonvm/hack/kernel/linux-config-6.1.63 similarity index 99% rename from neonvm/hack/kernel/linux-config-6.1.92 rename to neonvm/hack/kernel/linux-config-6.1.63 index 9a3522aa0..cf7ddea9a 100644 --- a/neonvm/hack/kernel/linux-config-6.1.92 +++ b/neonvm/hack/kernel/linux-config-6.1.63 @@ -1,6 +1,6 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86 6.1.92 Kernel Configuration +# Linux/x86 6.1.63 Kernel Configuration # CONFIG_CC_VERSION_TEXT="gcc (Ubuntu 13.2.0-4ubuntu3) 13.2.0" CONFIG_CC_IS_GCC=y @@ -15,10 +15,9 @@ CONFIG_CC_CAN_LINK=y CONFIG_CC_CAN_LINK_STATIC=y CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT=y -CONFIG_GCC_ASM_GOTO_OUTPUT_WORKAROUND=y CONFIG_CC_HAS_ASM_INLINE=y CONFIG_CC_HAS_NO_PROFILE_FN_ATTR=y -CONFIG_PAHOLE_VERSION=0 +CONFIG_PAHOLE_VERSION=125 CONFIG_IRQ_WORK=y CONFIG_BUILDTIME_TABLE_SORT=y CONFIG_THREAD_INFO_IN_TASK=y @@ -180,7 +179,7 @@ CONFIG_ARCH_SUPPORTS_NUMA_BALANCING=y CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH=y CONFIG_CC_HAS_INT128=y CONFIG_CC_IMPLICIT_FALLTHROUGH="-Wimplicit-fallthrough=5" -CONFIG_GCC10_NO_ARRAY_BOUNDS=y +CONFIG_GCC11_NO_ARRAY_BOUNDS=y CONFIG_CC_NO_ARRAY_BOUNDS=y CONFIG_ARCH_SUPPORTS_INT128=y CONFIG_NUMA_BALANCING=y @@ -431,7 +430,6 @@ CONFIG_X86_INTEL_TSX_MODE_OFF=y CONFIG_X86_SGX=y CONFIG_EFI=y CONFIG_EFI_STUB=y -CONFIG_EFI_HANDOVER_PROTOCOL=y CONFIG_EFI_MIXED=y # CONFIG_HZ_100 is not set CONFIG_HZ_250=y @@ -468,7 +466,7 @@ CONFIG_HAVE_LIVEPATCH=y CONFIG_CC_HAS_SLS=y CONFIG_CC_HAS_RETURN_THUNK=y -CONFIG_CPU_MITIGATIONS=y +CONFIG_SPECULATION_MITIGATIONS=y CONFIG_PAGE_TABLE_ISOLATION=y CONFIG_RETPOLINE=y CONFIG_RETHUNK=y @@ -478,8 +476,6 @@ CONFIG_CPU_IBRS_ENTRY=y CONFIG_CPU_SRSO=y # CONFIG_SLS is not set # CONFIG_GDS_FORCE_MITIGATION is not set -CONFIG_MITIGATION_RFDS=y -CONFIG_MITIGATION_SPECTRE_BHI=y CONFIG_ARCH_HAS_ADD_PAGES=y CONFIG_ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE=y @@ -641,7 +637,6 @@ CONFIG_AS_AVX512=y CONFIG_AS_SHA1_NI=y CONFIG_AS_SHA256_NI=y CONFIG_AS_TPAUSE=y -CONFIG_ARCH_CONFIGURES_CPU_MITIGATIONS=y # # General architecture-dependent options @@ -1051,7 +1046,6 @@ CONFIG_MPTCP=y CONFIG_INET_MPTCP_DIAG=y CONFIG_MPTCP_IPV6=y CONFIG_NETWORK_SECMARK=y -CONFIG_NET_PTP_CLASSIFY=y # CONFIG_NETWORK_PHY_TIMESTAMPING is not set CONFIG_NETFILTER=y CONFIG_NETFILTER_ADVANCED=y @@ -1962,31 +1956,18 @@ CONFIG_RANDOM_TRUST_BOOTLOADER=y # CONFIG_SPI is not set # CONFIG_SPMI is not set # CONFIG_HSI is not set -CONFIG_PPS=y -# CONFIG_PPS_DEBUG is not set - -# -# PPS clients support -# -# CONFIG_PPS_CLIENT_KTIMER is not set -# CONFIG_PPS_CLIENT_LDISC is not set -# CONFIG_PPS_CLIENT_GPIO is not set - -# -# PPS generators support -# +# CONFIG_PPS is not set # # PTP clock support # CONFIG_PTP_1588_CLOCK=y -CONFIG_PTP_1588_CLOCK_OPTIONAL=y +CONFIG_PTP_1588_CLOCK_KVM=y +# end of PTP clock support # # Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks. # -CONFIG_PTP_1588_CLOCK_KVM=y -# CONFIG_PTP_1588_CLOCK_VMW is not set # end of PTP clock support # CONFIG_PINCTRL is not set @@ -2060,6 +2041,7 @@ CONFIG_BCMA_POSSIBLE=y # CONFIG_MFD_RDC321X is not set # CONFIG_MFD_SM501 is not set # CONFIG_MFD_SYSCON is not set +# CONFIG_MFD_TI_AM335X_TSCADC is not set # CONFIG_MFD_TQMX86 is not set # CONFIG_MFD_VX855 is not set # end of Multifunction device drivers @@ -3197,6 +3179,7 @@ CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y # CONFIG_DEBUG_INFO_COMPRESSED is not set # CONFIG_DEBUG_INFO_SPLIT is not set # CONFIG_DEBUG_INFO_BTF is not set +CONFIG_PAHOLE_HAS_SPLIT_BTF=y CONFIG_GDB_SCRIPTS=y CONFIG_FRAME_WARN=2048 # CONFIG_STRIP_ASM_SYMS is not set diff --git a/neonvm/main.go b/neonvm/main.go index cbe66c656..3fc96bce1 100644 --- a/neonvm/main.go +++ b/neonvm/main.go @@ -98,6 +98,7 @@ func main() { var concurrencyLimit int var enableContainerMgr bool var qemuDiskCacheSettings string + var defaultMemoryProvider vmv1.MemoryProvider var failurePendingPeriod time.Duration var failingRefreshInterval time.Duration flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") @@ -108,12 +109,18 @@ func main() { flag.IntVar(&concurrencyLimit, "concurrency-limit", 1, "Maximum number of concurrent reconcile operations") flag.BoolVar(&enableContainerMgr, "enable-container-mgr", false, "Enable crictl-based container-mgr alongside each VM") flag.StringVar(&qemuDiskCacheSettings, "qemu-disk-cache-settings", "cache=none", "Set neonvm-runner's QEMU disk cache settings") + flag.Func("default-memory-provider", "Set default memory provider to use for new VMs", defaultMemoryProvider.FlagFunc) flag.DurationVar(&failurePendingPeriod, "failure-pending-period", 1*time.Minute, "the period for the propagation of reconciliation failures to the observability instruments") flag.DurationVar(&failingRefreshInterval, "failing-refresh-interval", 1*time.Minute, "the interval between consecutive updates of metrics and logs, related to failing reconciliations") flag.Parse() + if defaultMemoryProvider == "" { + fmt.Fprintln(os.Stderr, "missing required flag '-default-memory-provider'") + os.Exit(1) + } + logConfig := zap.NewProductionConfig() logConfig.Sampling = nil // Disabling sampling; it's enabled by default for zap's production configs. logConfig.Level.SetLevel(zap.InfoLevel) @@ -167,6 +174,7 @@ func main() { UseContainerMgr: enableContainerMgr, MaxConcurrentReconciles: concurrencyLimit, QEMUDiskCacheSettings: qemuDiskCacheSettings, + DefaultMemoryProvider: defaultMemoryProvider, FailurePendingPeriod: failurePendingPeriod, FailingRefreshInterval: failingRefreshInterval, } diff --git a/neonvm/runner/main.go b/neonvm/runner/main.go index 355344a03..7439d9db0 100644 --- a/neonvm/runner/main.go +++ b/neonvm/runner/main.go @@ -50,7 +50,7 @@ const ( QEMU_BIN = "qemu-system-x86_64" QEMU_IMG_BIN = "qemu-img" defaultKernelPath = "/vm/kernel/vmlinuz" - baseKernelCmdline = "panic=-1 init=/neonvm/bin/init memhp_default_state=online_movable console=ttyS1 loglevel=7 root=/dev/vda rw" + baseKernelCmdline = "panic=-1 init=/neonvm/bin/init memhp_default_state=online memory_hotplug.online_policy=auto-movable memory_hotplug.auto_movable_ratio=801 console=ttyS1 loglevel=7 root=/dev/vda rw" rootDiskPath = "/vm/images/rootdisk.qcow2" runtimeDiskPath = "/vm/images/runtime.iso" @@ -596,9 +596,10 @@ type Config struct { appendKernelCmdline string skipCgroupManagement bool diskCacheSettings string + memoryProvider vmv1.MemoryProvider } -func newConfig() *Config { +func newConfig(logger *zap.Logger) *Config { cfg := &Config{ vmSpecDump: "", vmStatusDump: "", @@ -606,6 +607,7 @@ func newConfig() *Config { appendKernelCmdline: "", skipCgroupManagement: false, diskCacheSettings: "cache=none", + memoryProvider: "", // Require that this is explicitly set. We'll check later. } flag.StringVar(&cfg.vmSpecDump, "vmspec", cfg.vmSpecDump, "Base64 encoded VirtualMachine json specification") @@ -620,8 +622,13 @@ func newConfig() *Config { "Don't try to manage CPU (use if running alongside container-mgr)") flag.StringVar(&cfg.diskCacheSettings, "qemu-disk-cache-settings", cfg.diskCacheSettings, "Cache settings to add to -drive args for VM disks") + flag.Func("memory-provider", "Set provider for memory hotplug", cfg.memoryProvider.FlagFunc) flag.Parse() + if cfg.memoryProvider == "" { + logger.Fatal("missing required flag '-memory-provider'") + } + return cfg } @@ -634,7 +641,7 @@ func main() { } func run(logger *zap.Logger) error { - cfg := newConfig() + cfg := newConfig(logger) vmSpecJson, err := base64.StdEncoding.DecodeString(cfg.vmSpecDump) if err != nil { @@ -658,17 +665,17 @@ func run(logger *zap.Logger) error { cpus := []string{} cpus = append(cpus, fmt.Sprintf("cpus=%d", qemuCPUs.min)) - if qemuCPUs.max != nil { - cpus = append(cpus, fmt.Sprintf("maxcpus=%d,sockets=1,cores=%d,threads=1", *qemuCPUs.max, *qemuCPUs.max)) - } + cpus = append(cpus, fmt.Sprintf("maxcpus=%d,sockets=1,cores=%d,threads=1", *qemuCPUs.max, *qemuCPUs.max)) + + logger.Info(fmt.Sprintf("Using memory provider %s", cfg.memoryProvider)) initialMemorySize := vmSpec.Guest.MemorySlotSize.Value() * int64(*vmSpec.Guest.MemorySlots.Min) memory := []string{} memory = append(memory, fmt.Sprintf("size=%db", initialMemorySize)) - if vmSpec.Guest.MemorySlots.Max != nil { + if cfg.memoryProvider == vmv1.MemoryProviderDIMMSlots { memory = append(memory, fmt.Sprintf("slots=%d", *vmSpec.Guest.MemorySlots.Max-*vmSpec.Guest.MemorySlots.Min)) - memory = append(memory, fmt.Sprintf("maxmem=%db", vmSpec.Guest.MemorySlotSize.Value()*int64(*vmSpec.Guest.MemorySlots.Max))) } + memory = append(memory, fmt.Sprintf("maxmem=%db", vmSpec.Guest.MemorySlotSize.Value()*int64(*vmSpec.Guest.MemorySlots.Max))) enableSSH := false if vmSpec.EnableSSH != nil && *vmSpec.EnableSSH { @@ -861,6 +868,20 @@ func buildQEMUCmd( // memory details qemuCmd = append(qemuCmd, "-m", strings.Join(memory, ",")) + if cfg.memoryProvider == vmv1.MemoryProviderVirtioMem { + // we don't actually have any slots because it's virtio-mem, but we're still using the API + // designed around DIMM slots, so we need to use them to calculate how much memory we expect + // to be able to plug in. + numSlots := *vmSpec.Guest.MemorySlots.Max - *vmSpec.Guest.MemorySlots.Min + virtioMemSize := int64(numSlots) * vmSpec.Guest.MemorySlotSize.Value() + // We can add virtio-mem if it actually needs to be a non-zero size. + // Otherwise, QEMU fails with: + // property 'size' of memory-backend-ram doesn't take value '0' + if virtioMemSize != 0 { + qemuCmd = append(qemuCmd, "-object", fmt.Sprintf("memory-backend-ram,id=vmem0,size=%db", virtioMemSize)) + qemuCmd = append(qemuCmd, "-device", "virtio-mem-pci,id=vm0,memdev=vmem0,block-size=8M,requested-size=0") + } + } // default (pod) net details macDefault, err := defaultNetwork(logger, defaultNetworkCIDR, vmSpec.Guest.Ports) diff --git a/neonvm/samples/vm-example.virtio-mem.yaml b/neonvm/samples/vm-example.virtio-mem.yaml new file mode 100644 index 000000000..172b20edb --- /dev/null +++ b/neonvm/samples/vm-example.virtio-mem.yaml @@ -0,0 +1,128 @@ +apiVersion: v1 +kind: Service +metadata: + name: example +spec: + ports: + - name: postgres + port: 5432 + protocol: TCP + targetPort: postgres + - name: pooler + port: 6432 + protocol: TCP + targetPort: postgres + - name: host-metrics + port: 9100 + protocol: TCP + targetPort: host-metrics + - name: metrics + port: 9187 + protocol: TCP + targetPort: metrics + selector: + vm.neon.tech/name: example + +--- +apiVersion: vm.neon.tech/v1 +kind: VirtualMachine +metadata: + name: example +spec: + guest: + cpus: + min: 1 + max: 4 + use: 2 + memorySlotSize: 1Gi + memorySlots: + min: 1 + max: 2 + use: 2 + memoryProvider: VirtioMem + rootDisk: + image: vm-postgres:15-bullseye + size: 8Gi + args: + - -c + - 'config_file=/etc/postgresql/postgresql.conf' + env: + # for testing only - allows login without password + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + ports: + - name: postgres + port: 5432 + - name: pooler + port: 5432 + - name: host-metrics + port: 9100 + - name: metrics + port: 9187 + extraNetwork: + enable: true + disks: + - name: pgdata + mountPath: /var/lib/postgresql + emptyDisk: + size: 16Gi + - name: postgres-config + mountPath: /etc/postgresql + configMap: + name: example-config + items: + - key: postgresql.conf + path: postgresql.conf + - name: vector-config + mountPath: /etc/vector + configMap: + name: example-config + items: + - key: vector.yaml + path: vector.yaml + - name: cache + mountPath: /neonvm/cache + tmpfs: + size: 1Gi + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: example-config +data: + postgresql.conf: | + listen_addresses = '*' + shared_preload_libraries = 'pg_stat_statements' + + max_connections = 64 + shared_buffers = 512MB + effective_cache_size = 1536MB + maintenance_work_mem = 128MB + checkpoint_completion_target = 0.9 + wal_buffers = 16MB + default_statistics_target = 100 + random_page_cost = 1.1 + effective_io_concurrency = 200 + work_mem = 4MB + min_wal_size = 1GB + max_wal_size = 4GB + max_worker_processes = 4 + max_parallel_workers_per_gather = 2 + max_parallel_workers = 4 + max_parallel_maintenance_workers = 2 + + vector.yaml: | + sources: + postgresql_metrics: + type: postgresql_metrics + endpoints: + - "postgres://postgres@localhost:5432" + exclude_databases: + - "^template.*" + sinks: + postgres_exporter: + type: prometheus_exporter + inputs: + - postgresql_metrics + address: "0.0.0.0:9187" diff --git a/neonvm/samples/vm-example.yaml b/neonvm/samples/vm-example.yaml index f30f8822d..f79f8fa23 100644 --- a/neonvm/samples/vm-example.yaml +++ b/neonvm/samples/vm-example.yaml @@ -8,23 +8,18 @@ spec: port: 5432 protocol: TCP targetPort: postgres - nodePort: 30432 - name: pooler port: 6432 protocol: TCP targetPort: postgres - nodePort: 30632 - name: host-metrics port: 9100 protocol: TCP targetPort: host-metrics - nodePort: 30100 - name: metrics port: 9187 protocol: TCP targetPort: metrics - nodePort: 30187 - type: NodePort selector: vm.neon.tech/name: example diff --git a/tests/e2e/autoscaling.virtio-mem/00-assert.yaml b/tests/e2e/autoscaling.virtio-mem/00-assert.yaml new file mode 100644 index 000000000..10de5a68b --- /dev/null +++ b/tests/e2e/autoscaling.virtio-mem/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 90 +--- +apiVersion: vm.neon.tech/v1 +kind: VirtualMachine +metadata: + name: example +status: + phase: Running + restartCount: 0 + conditions: + - type: Available + status: "True" + cpus: 250m + memorySize: 1Gi + memoryProvider: VirtioMem diff --git a/tests/e2e/autoscaling.virtio-mem/00-create-vm.yaml b/tests/e2e/autoscaling.virtio-mem/00-create-vm.yaml new file mode 100644 index 000000000..bdfa2b591 --- /dev/null +++ b/tests/e2e/autoscaling.virtio-mem/00-create-vm.yaml @@ -0,0 +1,101 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +unitTest: false +--- +apiVersion: v1 +kind: Service +metadata: + name: example +spec: + ports: + - name: postgres + port: 5432 + protocol: TCP + targetPort: postgres + type: NodePort + selector: + vm.neon.tech/name: example +--- +apiVersion: vm.neon.tech/v1 +kind: VirtualMachine +metadata: + name: example + labels: + autoscaling.neon.tech/enabled: "true" + annotations: + autoscaling.neon.tech/bounds: '{ "min": { "cpu": "250m", "mem": "1Gi" }, "max": {"cpu": 1, "mem": "4Gi" } }' +spec: + schedulerName: autoscale-scheduler + guest: + cpus: + min: 0.25 + max: 1.25 # set value greater than bounds so our tests check we don't exceed the bounds. + use: 0.5 + memorySlotSize: 1Gi + memorySlots: + min: 1 + max: 5 + use: 1 + memoryProvider: VirtioMem + rootDisk: + image: vm-postgres:15-bullseye + size: 8Gi + args: + - -c + - 'config_file=/etc/postgresql/postgresql.conf' + env: + # for testing only - allows login without password + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + ports: + - name: postgres + port: 5432 + - name: host-metrics + port: 9100 + - name: monitor + port: 10301 + extraNetwork: + enable: true + disks: + - name: pgdata + mountPath: /var/lib/postgresql + emptyDisk: + size: 16Gi + - name: postgres-config + mountPath: /etc/postgresql + configMap: + name: example-config + items: + - key: postgresql.conf + path: postgresql.conf + - name: cache + mountPath: /neonvm/cache + tmpfs: + size: 1Gi + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: example-config +data: + postgresql.conf: | + listen_addresses = '*' + shared_preload_libraries = 'pg_stat_statements' + + max_connections = 64 + shared_buffers = 256MB + effective_cache_size = 1536MB + maintenance_work_mem = 128MB + checkpoint_completion_target = 0.9 + wal_buffers = 16MB + default_statistics_target = 100 + random_page_cost = 1.1 + effective_io_concurrency = 200 + work_mem = 4MB + min_wal_size = 1GB + max_wal_size = 4GB + max_worker_processes = 4 + max_parallel_workers_per_gather = 2 + max_parallel_workers = 4 + max_parallel_maintenance_workers = 2 diff --git a/tests/e2e/autoscaling.virtio-mem/01-assert.yaml b/tests/e2e/autoscaling.virtio-mem/01-assert.yaml new file mode 100644 index 000000000..d7531ec6e --- /dev/null +++ b/tests/e2e/autoscaling.virtio-mem/01-assert.yaml @@ -0,0 +1,23 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 90 +--- +apiVersion: vm.neon.tech/v1 +kind: VirtualMachine +metadata: + name: example +status: + phase: Running + restartCount: 0 + conditions: + - type: Available + status: "True" + cpus: 1 + memorySize: 4Gi +--- +apiVersion: v1 +kind: pod +metadata: + name: workload +status: + phase: Running diff --git a/tests/e2e/autoscaling.virtio-mem/01-upscale.yaml b/tests/e2e/autoscaling.virtio-mem/01-upscale.yaml new file mode 100644 index 000000000..d95636340 --- /dev/null +++ b/tests/e2e/autoscaling.virtio-mem/01-upscale.yaml @@ -0,0 +1,49 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +unitTest: false +--- +apiVersion: v1 +kind: Pod +metadata: + name: workload +spec: + terminationGracePeriodSeconds: 1 + initContainers: + - name: wait-for-pg + image: postgres:15-bullseye + command: + - sh + - "-c" + - | + set -e + until pg_isready --username=postgres --dbname=postgres --host=example --port=5432; do + sleep 1 + done + containers: + - name: pgbench + image: postgres:15-bullseye + volumeMounts: + - name: my-volume + mountPath: /etc/misc + command: + - pgbench + args: + - postgres://postgres@example:5432/postgres + - --client=20 + - --progress=1 + - --progress-timestamp + - --time=600 + - --file=/etc/misc/query.sql + volumes: + - name: my-volume + configMap: + name: query + restartPolicy: Never +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: query +data: + query.sql: | + select length(factorial(length(factorial(1223)::text)/2)::text); diff --git a/tests/e2e/autoscaling.virtio-mem/02-assert.yaml b/tests/e2e/autoscaling.virtio-mem/02-assert.yaml new file mode 100644 index 000000000..8e97b6494 --- /dev/null +++ b/tests/e2e/autoscaling.virtio-mem/02-assert.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: vm.neon.tech/v1 +kind: VirtualMachine +metadata: + name: example +status: + phase: Running + restartCount: 0 + conditions: + - type: Available + status: "True" + cpus: 250m + memorySize: 1Gi diff --git a/tests/e2e/autoscaling.virtio-mem/02-downscale.yaml b/tests/e2e/autoscaling.virtio-mem/02-downscale.yaml new file mode 100644 index 000000000..160f31462 --- /dev/null +++ b/tests/e2e/autoscaling.virtio-mem/02-downscale.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: +- apiVersion: v1 + kind: Pod + name: workload +unitTest: false diff --git a/tests/e2e/autoscaling/00-assert.yaml b/tests/e2e/autoscaling/00-assert.yaml index cd40ee35b..3a0b27560 100644 --- a/tests/e2e/autoscaling/00-assert.yaml +++ b/tests/e2e/autoscaling/00-assert.yaml @@ -14,3 +14,4 @@ status: status: "True" cpus: 250m memorySize: 1Gi + memoryProvider: DIMMSlots diff --git a/tests/e2e/vm-migration.virtio-mem/00-assert.yaml b/tests/e2e/vm-migration.virtio-mem/00-assert.yaml new file mode 100644 index 000000000..f6cc18db2 --- /dev/null +++ b/tests/e2e/vm-migration.virtio-mem/00-assert.yaml @@ -0,0 +1,22 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 90 +--- +apiVersion: vm.neon.tech/v1 +kind: VirtualMachine +metadata: + name: example +status: + phase: Running + restartCount: 0 + conditions: + - type: Available + status: "True" + memoryProvider: VirtioMem +--- +apiVersion: v1 +kind: pod +metadata: + name: workload +status: + phase: Running diff --git a/tests/e2e/vm-migration.virtio-mem/00-prepare.yaml b/tests/e2e/vm-migration.virtio-mem/00-prepare.yaml new file mode 100644 index 000000000..dfef6b0ab --- /dev/null +++ b/tests/e2e/vm-migration.virtio-mem/00-prepare.yaml @@ -0,0 +1,43 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +apply: +- ../../../neonvm/samples/vm-example.virtio-mem.yaml +unitTest: false +--- +apiVersion: v1 +kind: Pod +metadata: + name: workload +spec: + terminationGracePeriodSeconds: 1 + initContainers: + - name: wait-for-pg + image: postgres:15-bullseye + command: + - sh + - "-c" + - | + set -e + until pg_isready --username=postgres --dbname=postgres --host=example --port=5432; do + sleep 1 + done + - name: pgbench-initialize + image: postgres:15-bullseye + command: + - pgbench + args: + - postgres://postgres@example:5432/postgres + - --initialize + - --scale=10 + containers: + - name: pgbench + image: postgres:15-bullseye + command: + - pgbench + args: + - postgres://postgres@example:5432/postgres + - --client=2 + - --progress=1 + - --progress-timestamp + - --time=600 + restartPolicy: Never diff --git a/tests/e2e/vm-migration.virtio-mem/01-assert.yaml b/tests/e2e/vm-migration.virtio-mem/01-assert.yaml new file mode 100644 index 000000000..9416c0d72 --- /dev/null +++ b/tests/e2e/vm-migration.virtio-mem/01-assert.yaml @@ -0,0 +1,28 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 90 +--- +apiVersion: vm.neon.tech/v1 +kind: VirtualMachineMigration +metadata: + name: example +status: + phase: Succeeded +--- +apiVersion: vm.neon.tech/v1 +kind: VirtualMachine +metadata: + name: example +status: + phase: Running + restartCount: 0 + conditions: + - type: Available + status: "True" +--- +apiVersion: v1 +kind: pod +metadata: + name: workload +status: + phase: Running diff --git a/tests/e2e/vm-migration.virtio-mem/01-migrate.yaml b/tests/e2e/vm-migration.virtio-mem/01-migrate.yaml new file mode 100644 index 000000000..1866a7d58 --- /dev/null +++ b/tests/e2e/vm-migration.virtio-mem/01-migrate.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +apply: +- ../../../neonvm/samples/vm-example-migration.yaml +unitTest: false diff --git a/tests/e2e/vm-migration/00-assert.yaml b/tests/e2e/vm-migration/00-assert.yaml index f8e838cc6..b01d74149 100644 --- a/tests/e2e/vm-migration/00-assert.yaml +++ b/tests/e2e/vm-migration/00-assert.yaml @@ -12,6 +12,7 @@ status: conditions: - type: Available status: "True" + memoryProvider: DIMMSlots --- apiVersion: v1 kind: pod