feat(throttling): added lib functions for setting io limits at pod cg…

…roup (#3)


Signed-off-by: [email protected]

go.mod

@@ -5,6 +5,7 @@ go 1.14
 require (
 	github.com/container-storage-interface/spec v1.1.0
 	github.com/gogo/protobuf v1.3.0 // indirect
+	github.com/google/uuid v1.0.0
 	github.com/googleapis/gnostic v0.3.1 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/json-iterator/go v1.1.8 // indirect

go.sum

@@ -120,6 +120,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=

pkg/common/helpers/helpers.go

@@ -17,7 +17,10 @@ limitations under the License.
 package helpers
 
 import (
+	"os"
 	"strings"
+
+	"github.com/google/uuid"
 )
 
 // GetCaseInsensitiveMap coercs the map's keys to lower case, which only works
@@ -43,3 +46,29 @@ func GetInsensitiveParameter(dict *map[string]string, key string) string {
 	insensitiveDict := GetCaseInsensitiveMap(dict)
 	return insensitiveDict[strings.ToLower(key)]
 }
+
+func exists(path string) (os.FileInfo, bool) {
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		return nil, false
+	}
+	return info, true
+}
+
+// FileExists checks if a file exists and is not a directory
+func FileExists(filepath string) bool {
+	info, present := exists(filepath)
+	return present && info.Mode().IsRegular()
+}
+
+// DirExists checks if a directory exists
+func DirExists(path string) bool {
+	info, present := exists(path)
+	return present && info.IsDir()
+}
+
+// IsValidUUID validates whether a string is a valid UUID
+func IsValidUUID(u string) bool {
+	_, err := uuid.Parse(u)
+	return err == nil
+}

pkg/device/iolimit/models.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2020 The OpenEBS Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iolimit
+
+type Request struct {
+	DeviceName string
+	PodUid     string
+	ContainerRuntime string
+	IOLimit    *IOMax
+}
+
+type ValidRequest struct {
+	FilePath string
+	DeviceNumber *DeviceNumber
+	IOMax    *IOMax
+}
+
+type IOMax struct {
+	Riops uint64
+	Wiops uint64
+	Rbps  uint64
+	Wbps  uint64
+}
+
+type DeviceNumber struct {
+	Major uint64
+	Minor uint64
+}

pkg/device/iolimit/utils.go

@@ -0,0 +1,143 @@
+/*
+Copyright 2020 The OpenEBS Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iolimit
+
+import (
+	"github.com/openebs/lib-csi/pkg/common/errors"
+	"github.com/openebs/lib-csi/pkg/common/helpers"
+	"io/ioutil"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+const (
+	baseCgroupPath = "/sys/fs/cgroup"
+)
+
+// SetIOLimits sets iops, bps limits for a pod with uid podUid for accessing a device named deviceName
+// provided that the underlying cgroup used for pod namespacing is cgroup2 (cgroup v2)
+func SetIOLimits(request *Request) error {
+	if !helpers.DirExists(baseCgroupPath) {
+		return errors.New(baseCgroupPath + " does not exist")
+	}
+	if err := checkCgroupV2(); err != nil {
+		return err
+	}
+	validRequest, err := validate(request)
+	if err != nil {
+		return err
+	}
+	err = setIOLimits(validRequest)
+	return err
+}
+
+func validate(request *Request) (*ValidRequest, error) {
+	if !helpers.IsValidUUID(request.PodUid) {
+		return nil, errors.New("Expected PodUid in UUID format, Got " + request.PodUid)
+	}
+	podCGPath, err := getPodCGroupPath(request.PodUid, request.ContainerRuntime)
+	if err != nil {
+		return nil, err
+	}
+	ioMaxFile := podCGPath + "/io.max"
+	if !helpers.FileExists(ioMaxFile) {
+		return nil, errors.New("io.max file is not present in pod CGroup")
+	}
+	deviceNumber, err := getDeviceNumber(request.DeviceName)
+	if err != nil {
+		return nil, errors.New("Device Major:Minor numbers could not be obtained")
+	}
+	return &ValidRequest{
+		FilePath:     ioMaxFile,
+		DeviceNumber: deviceNumber,
+		IOMax:      request.IOLimit,
+	}, nil
+}
+
+func getPodCGroupPath(podUid string, cruntime string) (string, error) {
+	switch cruntime {
+	case "containerd":
+		path, err := getContainerdCGPath(podUid)
+		if err != nil {
+			return "", err
+		}
+		return path, nil
+	default:
+		return "", errors.New(cruntime + " runtime support is not present")
+	}
+
+}
+
+func checkCgroupV2() error {
+	if !helpers.FileExists(baseCgroupPath + "/cgroup.controllers") {
+		return errors.New("CGroupV2 not enabled")
+	}
+	return nil
+}
+
+func getContainerdPodCGSuffix(podUid string) string {
+	return "pod" + strings.ReplaceAll(podUid, "-", "_")
+}
+
+func getContainerdCGPath(podUid string) (string, error) {
+	kubepodsCGPath := baseCgroupPath + "/kubepods.slice"
+	podSuffix := getContainerdPodCGSuffix(podUid)
+	podCGPath := kubepodsCGPath + "/kubepods-besteffort.slice/kubepods-besteffort-" + podSuffix + ".slice"
+	if helpers.DirExists(podCGPath) {
+		return podCGPath, nil
+	}
+	podCGPath = kubepodsCGPath + "/kubepods-burstable.slice/kubepods-burstable-" + podSuffix + ".slice"
+	if helpers.DirExists(podCGPath) {
+		return podCGPath, nil
+	}
+	return "", errors.New("CGroup Path not found for pod with Uid: " + podUid)
+}
+
+func getDeviceNumber(deviceName string) (*DeviceNumber, error) {
+	stat := syscall.Stat_t{}
+	if err := syscall.Stat(deviceName, &stat); err != nil {
+		return nil, err
+	}
+	return &DeviceNumber{
+		Major: uint64(stat.Rdev/256),
+		Minor: uint64(stat.Rdev%256),
+	}, nil
+}
+
+func getIOLimitsStr(deviceNumber *DeviceNumber, ioMax *IOMax) string {
+	line := strconv.FormatUint(deviceNumber.Major, 10) + ":" + strconv.FormatUint(deviceNumber.Minor, 10)
+	if ioMax.Riops != 0 {
+		line += " riops=" + strconv.FormatUint(ioMax.Riops, 10)
+	}
+	if ioMax.Wiops != 0 {
+		line += " wiops=" + strconv.FormatUint(ioMax.Wiops, 10)
+	}
+	if ioMax.Rbps != 0 {
+		line += " rbps=" + strconv.FormatUint(ioMax.Rbps, 10)
+	}
+	if ioMax.Wbps != 0 {
+		line += " wbps=" + strconv.FormatUint(ioMax.Wbps, 10)
+	}
+	return line
+}
+
+func setIOLimits(request *ValidRequest) error {
+	line := getIOLimitsStr(request.DeviceNumber, request.IOMax)
+	err := ioutil.WriteFile(request.FilePath, []byte(line), 0700)
+	return err
+}