neonvm: Add controller flag to disable runner cgroup

neonvm/controllers/config.go

@@ -23,6 +23,14 @@ type ReconcilerConfig struct {
 	// This is defined as a config option so we can do a gradual rollout of this change.
 	UseContainerMgr bool
 
+	// DisableRunnerCgroup, if true, disables running QEMU in a cgroup in new VM runner pods.
+	// Fractional CPU scaling will continue to *pretend* to work, but it will not do anything in
+	// practice.
+	//
+	// Under the hood, this results in passing -skip-cgroup-management and -enable-dummy-cpu-server
+	// to neonvm-runner.
+	DisableRunnerCgroup bool
+
 	MaxConcurrentReconciles int
 
 	// SkipUpdateValidationFor is the set of object names that we should ignore when doing webhook

neonvm/controllers/functests/vm_controller_test.go

@@ -109,6 +109,7 @@ var _ = Describe("VirtualMachine controller", func() {
 				Config: &controllers.ReconcilerConfig{
 					IsK3s:                   false,
 					UseContainerMgr:         true,
+					DisableRunnerCgroup:     false,
 					MaxConcurrentReconciles: 1,
 					SkipUpdateValidationFor: nil,
 					QEMUDiskCacheSettings:   "cache=none",

neonvm/controllers/vm_controller.go

@@ -1486,9 +1486,14 @@ func podSpec(
 					}},
 					Command: func() []string {
 						cmd := []string{"runner"}
-						if config.UseContainerMgr {
+						if config.UseContainerMgr || config.DisableRunnerCgroup {
 							cmd = append(cmd, "-skip-cgroup-management")
 						}
+						if config.DisableRunnerCgroup {
+							// cgroup management disabled, but we still need something to provide
+							// the server, so the runner will just provide a dummy implementation.
+							cmd = append(cmd, "-enable-dummy-cpu-server")
+						}
 						cmd = append(
 							cmd,
 							"-qemu-disk-cache-settings", config.QEMUDiskCacheSettings,
@@ -1530,7 +1535,7 @@ func podSpec(
 							MountPropagation: lo.ToPtr(corev1.MountPropagationNone),
 						}
 
-						if config.UseContainerMgr {
+						if config.UseContainerMgr || config.DisableRunnerCgroup {
 							return []corev1.VolumeMount{images}
 						} else {
 							// the /sys/fs/cgroup mount is only necessary if neonvm-runner has to
@@ -1625,6 +1630,8 @@ func podSpec(
 
 				if config.UseContainerMgr {
 					return []corev1.Volume{images, containerdSock}
+				} else if config.DisableRunnerCgroup {
+					return []corev1.Volume{images}
 				} else {
 					return []corev1.Volume{images, cgroup}
 				}

neonvm/controllers/vm_controller_unit_test.go

@@ -117,6 +117,7 @@ func newTestParams(t *testing.T) *testParams {
 		Config: &ReconcilerConfig{
 			IsK3s:                   false,
 			UseContainerMgr:         false,
+			DisableRunnerCgroup:     false,
 			MaxConcurrentReconciles: 10,
 			SkipUpdateValidationFor: nil,
 			QEMUDiskCacheSettings:   "",

neonvm/main.go

@@ -101,6 +101,7 @@ func main() {
 	var concurrencyLimit int
 	var skipUpdateValidationFor map[types.NamespacedName]struct{}
 	var enableContainerMgr bool
+	var disableRunnerCgroup bool
 	var qemuDiskCacheSettings string
 	var defaultMemoryProvider vmv1.MemoryProvider
 	var memhpAutoMovableRatio string
@@ -138,7 +139,9 @@ func main() {
 			return nil
 		},
 	)
+	// note: cannot have both -enable-container-mgr and -disable-runner-cgroup.
 	flag.BoolVar(&enableContainerMgr, "enable-container-mgr", false, "Enable crictl-based container-mgr alongside each VM")
+	flag.BoolVar(&disableRunnerCgroup, "disable-runner-cgroup", false, "Disable creation of a cgroup in neonvm-runner for fractional CPU limiting")
 	flag.StringVar(&qemuDiskCacheSettings, "qemu-disk-cache-settings", "cache=none", "Set neonvm-runner's QEMU disk cache settings")
 	flag.Func("default-memory-provider", "Set default memory provider to use for new VMs", defaultMemoryProvider.FlagFunc)
 	flag.StringVar(&memhpAutoMovableRatio, "memhp-auto-movable-ratio", "301", "For virtio-mem, set VM kernel's memory_hotplug.auto_movable_ratio")
@@ -152,6 +155,10 @@ func main() {
 		fmt.Fprintln(os.Stderr, "missing required flag '-default-memory-provider'")
 		os.Exit(1)
 	}
+	if disableRunnerCgroup && enableContainerMgr {
+		fmt.Fprintln(os.Stderr, "Cannot have both '-enable-container-mgr' and '-disable-runner-cgroup'")
+		os.Exit(1)
+	}
 
 	logConfig := zap.NewProductionConfig()
 	logConfig.Sampling = nil // Disabling sampling; it's enabled by default for zap's production configs.
@@ -205,6 +212,7 @@ func main() {
 	rc := &controllers.ReconcilerConfig{
 		IsK3s:                   isK3s,
 		UseContainerMgr:         enableContainerMgr,
+		DisableRunnerCgroup:     disableRunnerCgroup,
 		MaxConcurrentReconciles: concurrencyLimit,
 		SkipUpdateValidationFor: skipUpdateValidationFor,
 		QEMUDiskCacheSettings:   qemuDiskCacheSettings,

neonvm/runner/main.go

@@ -23,6 +23,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -36,6 +37,7 @@ import (
 	"github.com/jpillora/backoff"
 	"github.com/kdomanski/iso9660"
 	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/samber/lo"
 	"github.com/vishvananda/netlink"
 	"go.uber.org/zap"
 
@@ -594,6 +596,7 @@ type Config struct {
 	kernelPath           string
 	appendKernelCmdline  string
 	skipCgroupManagement bool
+	enableDummyCPUServer bool
 	diskCacheSettings    string
 	memoryProvider       vmv1.MemoryProvider
 	autoMovableRatio     string
@@ -606,6 +609,7 @@ func newConfig(logger *zap.Logger) *Config {
 		kernelPath:           defaultKernelPath,
 		appendKernelCmdline:  "",
 		skipCgroupManagement: false,
+		enableDummyCPUServer: false,
 		diskCacheSettings:    "cache=none",
 		memoryProvider:       "", // Require that this is explicitly set. We'll check later.
 		autoMovableRatio:     "", // Require that this is explicitly set IFF memoryProvider is VirtioMem. We'll check later.
@@ -620,7 +624,10 @@ func newConfig(logger *zap.Logger) *Config {
 		cfg.appendKernelCmdline, "Additional kernel command line arguments")
 	flag.BoolVar(&cfg.skipCgroupManagement, "skip-cgroup-management",
 		cfg.skipCgroupManagement,
-		"Don't try to manage CPU (use if running alongside container-mgr)")
+		"Don't try to manage CPU (use if running alongside container-mgr, or if dummy CPU server is enabled)")
+	flag.BoolVar(&cfg.enableDummyCPUServer, "enable-dummy-cpu-server",
+		cfg.skipCgroupManagement,
+		"Provide a CPU server (unlike -skip-cgroup-management) but don't actually do anything with it")
 	flag.StringVar(&cfg.diskCacheSettings, "qemu-disk-cache-settings",
 		cfg.diskCacheSettings, "Cache settings to add to -drive args for VM disks")
 	flag.Func("memory-provider", "Set provider for memory hotplug", cfg.memoryProvider.FlagFunc)
@@ -635,6 +642,9 @@ func newConfig(logger *zap.Logger) *Config {
 	if cfg.memoryProvider == vmv1.MemoryProviderVirtioMem && cfg.autoMovableRatio == "" {
 		logger.Fatal("missing required flag '-memhp-auto-movable-ratio'")
 	}
+	if cfg.enableDummyCPUServer && !cfg.skipCgroupManagement {
+		logger.Fatal("flag -enable-dummy-cpu-server requires -skip-cgroup-management")
+	}
 
 	return cfg
 }
@@ -1007,9 +1017,36 @@ func runQEMU(
 
 	wg.Add(1)
 	go terminateQemuOnSigterm(ctx, logger, &wg)
-	if !cfg.skipCgroupManagement {
+	if !cfg.skipCgroupManagement || cfg.enableDummyCPUServer {
+		var callbacks cpuServerCallbacks
+
+		if cfg.enableDummyCPUServer {
+			lastValue := &atomic.Uint32{}
+			lastValue.Store(uint32(vmSpec.Guest.CPUs.Min))
+
+			callbacks = cpuServerCallbacks{
+				get: func(logger *zap.Logger) (*vmv1.MilliCPU, error) {
+					return lo.ToPtr(vmv1.MilliCPU(lastValue.Load())), nil
+				},
+				set: func(logger *zap.Logger, cpu vmv1.MilliCPU) error {
+					lastValue.Store(uint32(cpu))
+					return nil
+				},
+			}
+		} else {
+			// Standard implementation -- actually set the cgroup
+			callbacks = cpuServerCallbacks{
+				get: func(logger *zap.Logger) (*vmv1.MilliCPU, error) {
+					return getCgroupQuota(cgroupPath)
+				},
+				set: func(logger *zap.Logger, cpu vmv1.MilliCPU) error {
+					return setCgroupLimit(logger, cpu, cgroupPath)
+				},
+			}
+		}
+
 		wg.Add(1)
-		go listenForCPUChanges(ctx, logger, vmSpec.RunnerPort, cgroupPath, &wg)
+		go listenForCPUChanges(ctx, logger, vmSpec.RunnerPort, callbacks, &wg)
 	}
 	wg.Add(1)
 	go forwardLogs(ctx, logger, &wg)
@@ -1040,7 +1077,12 @@ func runQEMU(
 	return err
 }
 
-func handleCPUChange(logger *zap.Logger, w http.ResponseWriter, r *http.Request, cgroupPath string) {
+func handleCPUChange(
+	logger *zap.Logger,
+	w http.ResponseWriter,
+	r *http.Request,
+	set func(*zap.Logger, vmv1.MilliCPU) error,
+) {
 	if r.Method != "POST" {
 		logger.Error("unexpected method", zap.String("method", r.Method))
 		w.WriteHeader(400)
@@ -1062,7 +1104,7 @@ func handleCPUChange(logger *zap.Logger, w http.ResponseWriter, r *http.Request,
 
 	// update cgroup
 	logger.Info("got CPU update", zap.Float64("CPU", parsed.VCPUs.AsFloat64()))
-	err = setCgroupLimit(logger, parsed.VCPUs, cgroupPath)
+	err = set(logger, parsed.VCPUs)
 	if err != nil {
 		logger.Error("could not set cgroup limit", zap.Error(err))
 		w.WriteHeader(500)
@@ -1072,14 +1114,19 @@ func handleCPUChange(logger *zap.Logger, w http.ResponseWriter, r *http.Request,
 	w.WriteHeader(200)
 }
 
-func handleCPUCurrent(logger *zap.Logger, w http.ResponseWriter, r *http.Request, cgroupPath string) {
+func handleCPUCurrent(
+	logger *zap.Logger,
+	w http.ResponseWriter,
+	r *http.Request,
+	get func(*zap.Logger) (*vmv1.MilliCPU, error),
+) {
 	if r.Method != "GET" {
 		logger.Error("unexpected method", zap.String("method", r.Method))
 		w.WriteHeader(400)
 		return
 	}
 
-	cpus, err := getCgroupQuota(cgroupPath)
+	cpus, err := get(logger)
 	if err != nil {
 		logger.Error("could not get cgroup quota", zap.Error(err))
 		w.WriteHeader(500)
@@ -1097,17 +1144,28 @@ func handleCPUCurrent(logger *zap.Logger, w http.ResponseWriter, r *http.Request
 	w.Write(body) //nolint:errcheck // Not much to do with the error here. TODO: log it?
 }
 
-func listenForCPUChanges(ctx context.Context, logger *zap.Logger, port int32, cgroupPath string, wg *sync.WaitGroup) {
+type cpuServerCallbacks struct {
+	get func(*zap.Logger) (*vmv1.MilliCPU, error)
+	set func(*zap.Logger, vmv1.MilliCPU) error
+}
+
+func listenForCPUChanges(
+	ctx context.Context,
+	logger *zap.Logger,
+	port int32,
+	callbacks cpuServerCallbacks,
+	wg *sync.WaitGroup,
+) {
 	defer wg.Done()
 	mux := http.NewServeMux()
 	loggerHandlers := logger.Named("http-handlers")
 	cpuChangeLogger := loggerHandlers.Named("cpu_change")
 	mux.HandleFunc("/cpu_change", func(w http.ResponseWriter, r *http.Request) {
-		handleCPUChange(cpuChangeLogger, w, r, cgroupPath)
+		handleCPUChange(cpuChangeLogger, w, r, callbacks.set)
 	})
 	cpuCurrentLogger := loggerHandlers.Named("cpu_current")
 	mux.HandleFunc("/cpu_current", func(w http.ResponseWriter, r *http.Request) {
-		handleCPUCurrent(cpuCurrentLogger, w, r, cgroupPath)
+		handleCPUCurrent(cpuCurrentLogger, w, r, callbacks.get)
 	})
 	server := http.Server{
 		Addr:              fmt.Sprintf("0.0.0.0:%d", port),