diff --git a/go.mod b/go.mod index 6d9eeae..c5beba0 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.14 require ( github.com/container-storage-interface/spec v1.1.0 github.com/gogo/protobuf v1.3.0 // indirect + github.com/google/uuid v1.0.0 github.com/googleapis/gnostic v0.3.1 // indirect github.com/imdario/mergo v0.3.7 // indirect github.com/json-iterator/go v1.1.8 // indirect diff --git a/go.sum b/go.sum index b2dc205..f1b10f8 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= diff --git a/pkg/common/helpers/helpers.go b/pkg/common/helpers/helpers.go index a519ad9..dde6ad0 100644 --- a/pkg/common/helpers/helpers.go +++ b/pkg/common/helpers/helpers.go @@ -17,7 +17,10 @@ limitations under the License. package helpers import ( + "os" "strings" + + "github.com/google/uuid" ) // GetCaseInsensitiveMap coercs the map's keys to lower case, which only works @@ -43,3 +46,29 @@ func GetInsensitiveParameter(dict *map[string]string, key string) string { insensitiveDict := GetCaseInsensitiveMap(dict) return insensitiveDict[strings.ToLower(key)] } + +func exists(path string) (os.FileInfo, bool) { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return nil, false + } + return info, true +} + +// FileExists checks if a file exists and is not a directory +func FileExists(filepath string) bool { + info, present := exists(filepath) + return present && info.Mode().IsRegular() +} + +// DirExists checks if a directory exists +func DirExists(path string) bool { + info, present := exists(path) + return present && info.IsDir() +} + +// IsValidUUID validates whether a string is a valid UUID +func IsValidUUID(u string) bool { + _, err := uuid.Parse(u) + return err == nil +} diff --git a/pkg/device/iolimit/models.go b/pkg/device/iolimit/models.go new file mode 100644 index 0000000..f42e423 --- /dev/null +++ b/pkg/device/iolimit/models.go @@ -0,0 +1,42 @@ +/* +Copyright 2020 The OpenEBS Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iolimit + +type Request struct { + DeviceName string + PodUid string + ContainerRuntime string + IOLimit *IOMax +} + +type ValidRequest struct { + FilePath string + DeviceNumber *DeviceNumber + IOMax *IOMax +} + +type IOMax struct { + Riops uint64 + Wiops uint64 + Rbps uint64 + Wbps uint64 +} + +type DeviceNumber struct { + Major uint64 + Minor uint64 +} \ No newline at end of file diff --git a/pkg/device/iolimit/utils.go b/pkg/device/iolimit/utils.go new file mode 100644 index 0000000..2ddb631 --- /dev/null +++ b/pkg/device/iolimit/utils.go @@ -0,0 +1,143 @@ +/* +Copyright 2020 The OpenEBS Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iolimit + +import ( + "github.com/openebs/lib-csi/pkg/common/errors" + "github.com/openebs/lib-csi/pkg/common/helpers" + "io/ioutil" + "strconv" + "strings" + "syscall" +) + +const ( + baseCgroupPath = "/sys/fs/cgroup" +) + +// SetIOLimits sets iops, bps limits for a pod with uid podUid for accessing a device named deviceName +// provided that the underlying cgroup used for pod namespacing is cgroup2 (cgroup v2) +func SetIOLimits(request *Request) error { + if !helpers.DirExists(baseCgroupPath) { + return errors.New(baseCgroupPath + " does not exist") + } + if err := checkCgroupV2(); err != nil { + return err + } + validRequest, err := validate(request) + if err != nil { + return err + } + err = setIOLimits(validRequest) + return err +} + +func validate(request *Request) (*ValidRequest, error) { + if !helpers.IsValidUUID(request.PodUid) { + return nil, errors.New("Expected PodUid in UUID format, Got " + request.PodUid) + } + podCGPath, err := getPodCGroupPath(request.PodUid, request.ContainerRuntime) + if err != nil { + return nil, err + } + ioMaxFile := podCGPath + "/io.max" + if !helpers.FileExists(ioMaxFile) { + return nil, errors.New("io.max file is not present in pod CGroup") + } + deviceNumber, err := getDeviceNumber(request.DeviceName) + if err != nil { + return nil, errors.New("Device Major:Minor numbers could not be obtained") + } + return &ValidRequest{ + FilePath: ioMaxFile, + DeviceNumber: deviceNumber, + IOMax: request.IOLimit, + }, nil +} + +func getPodCGroupPath(podUid string, cruntime string) (string, error) { + switch cruntime { + case "containerd": + path, err := getContainerdCGPath(podUid) + if err != nil { + return "", err + } + return path, nil + default: + return "", errors.New(cruntime + " runtime support is not present") + } + +} + +func checkCgroupV2() error { + if !helpers.FileExists(baseCgroupPath + "/cgroup.controllers") { + return errors.New("CGroupV2 not enabled") + } + return nil +} + +func getContainerdPodCGSuffix(podUid string) string { + return "pod" + strings.ReplaceAll(podUid, "-", "_") +} + +func getContainerdCGPath(podUid string) (string, error) { + kubepodsCGPath := baseCgroupPath + "/kubepods.slice" + podSuffix := getContainerdPodCGSuffix(podUid) + podCGPath := kubepodsCGPath + "/kubepods-besteffort.slice/kubepods-besteffort-" + podSuffix + ".slice" + if helpers.DirExists(podCGPath) { + return podCGPath, nil + } + podCGPath = kubepodsCGPath + "/kubepods-burstable.slice/kubepods-burstable-" + podSuffix + ".slice" + if helpers.DirExists(podCGPath) { + return podCGPath, nil + } + return "", errors.New("CGroup Path not found for pod with Uid: " + podUid) +} + +func getDeviceNumber(deviceName string) (*DeviceNumber, error) { + stat := syscall.Stat_t{} + if err := syscall.Stat(deviceName, &stat); err != nil { + return nil, err + } + return &DeviceNumber{ + Major: uint64(stat.Rdev/256), + Minor: uint64(stat.Rdev%256), + }, nil +} + +func getIOLimitsStr(deviceNumber *DeviceNumber, ioMax *IOMax) string { + line := strconv.FormatUint(deviceNumber.Major, 10) + ":" + strconv.FormatUint(deviceNumber.Minor, 10) + if ioMax.Riops != 0 { + line += " riops=" + strconv.FormatUint(ioMax.Riops, 10) + } + if ioMax.Wiops != 0 { + line += " wiops=" + strconv.FormatUint(ioMax.Wiops, 10) + } + if ioMax.Rbps != 0 { + line += " rbps=" + strconv.FormatUint(ioMax.Rbps, 10) + } + if ioMax.Wbps != 0 { + line += " wbps=" + strconv.FormatUint(ioMax.Wbps, 10) + } + return line +} + +func setIOLimits(request *ValidRequest) error { + line := getIOLimitsStr(request.DeviceNumber, request.IOMax) + err := ioutil.WriteFile(request.FilePath, []byte(line), 0700) + return err +}