Skip to content

Commit

Permalink
added ExtendedResource to instanceType + unit test
Browse files Browse the repository at this point in the history
  • Loading branch information
@elankath
elankath committed Jul 23, 2024
1 parent 463510d commit ebb1b93
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 34 deletions.
70 changes: 46 additions & 24 deletions cluster-autoscaler/cloudprovider/mcm/mcm_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,11 @@ import (
"fmt"
v1appslister "k8s.io/client-go/listers/apps/v1"
"k8s.io/utils/pointer"
"maps"
"math/rand"
"net/http"
"os"
"slices"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -113,6 +115,9 @@ var (
machineDeploymentGVR = schema.GroupVersionResource{Group: machineGroup, Version: machineVersion, Resource: "machinedeployments"}

ErrInvalidNodeTemplate = errors.New("invalid node template")
coreResourceNames = []v1.ResourceName{v1.ResourceCPU, v1.ResourceMemory}
gpuResourceNames = []v1.ResourceName{"gpu", gpu.ResourceNvidiaGPU}
knownResourceNames = slices.Concat(coreResourceNames, gpuResourceNames, []v1.ResourceName{v1.ResourcePods, v1.ResourceEphemeralStorage})
)

// McmManager manages the client communication for MachineDeployments.
Expand All @@ -132,12 +137,13 @@ type McmManager struct {
}

type instanceType struct {
InstanceType string
VCPU resource.Quantity
Memory resource.Quantity
GPU resource.Quantity
EphemeralStorage resource.Quantity
PodCount resource.Quantity
InstanceType string
VCPU resource.Quantity
Memory resource.Quantity
GPU resource.Quantity
EphemeralStorage resource.Quantity
PodCount resource.Quantity
ExtendedResources apiv1.ResourceList
}

type nodeTemplate struct {
Expand Down Expand Up @@ -691,22 +697,15 @@ func generateInstanceStatus(machine *v1alpha1.Machine) *cloudprovider.InstanceSt
func validateNodeTemplate(nodeTemplateAttributes *v1alpha1.NodeTemplate) error {
var allErrs []error

capacityAttributes := []v1.ResourceName{v1.ResourceCPU, v1.ResourceMemory, "gpu", v1.ResourceEphemeralStorage}

var atLeastOneResource bool
for _, attribute := range capacityAttributes {
if _, ok := nodeTemplateAttributes.Capacity[attribute]; ok {
atLeastOneResource = true
for _, attribute := range coreResourceNames {
if _, ok := nodeTemplateAttributes.Capacity[attribute]; !ok {
errMessage := fmt.Errorf("the core resource fields %q are mandatory: %w", coreResourceNames, ErrInvalidNodeTemplate)
klog.Warning(errMessage)
allErrs = append(allErrs, errMessage)
break
}
}

if !atLeastOneResource {
errMessage := fmt.Errorf("at-least one of the resource fields %q are mandatory: %w", capacityAttributes, ErrInvalidNodeTemplate)
klog.Warning(errMessage)
allErrs = append(allErrs, errMessage)
}

if nodeTemplateAttributes.Region == "" || nodeTemplateAttributes.InstanceType == "" || nodeTemplateAttributes.Zone == "" {
errMessage := fmt.Errorf("InstanceType, Region and Zone attributes are mandatory: %w", ErrInvalidNodeTemplate)
klog.Warning(errMessage)
Expand Down Expand Up @@ -778,22 +777,26 @@ func (m *McmManager) GetMachineDeploymentNodeTemplate(machinedeployment *Machine
klog.V(1).Infof("Nodes already existing in the worker pool %s", workerPool)
baseNode := filteredNodes[0]
klog.V(1).Infof("Worker pool node used to form template is %s and its capacity is cpu: %s, memory:%s", baseNode.Name, baseNode.Status.Capacity.Cpu().String(), baseNode.Status.Capacity.Memory().String())
extendedResources := filterExtendedResources(baseNode.Status.Capacity)
instance = instanceType{
VCPU: baseNode.Status.Capacity[apiv1.ResourceCPU],
Memory: baseNode.Status.Capacity[apiv1.ResourceMemory],
GPU: baseNode.Status.Capacity[gpu.ResourceNvidiaGPU],
EphemeralStorage: baseNode.Status.Capacity[apiv1.ResourceEphemeralStorage],
PodCount: baseNode.Status.Capacity[apiv1.ResourcePods],
VCPU: baseNode.Status.Capacity[apiv1.ResourceCPU],
Memory: baseNode.Status.Capacity[apiv1.ResourceMemory],
GPU: baseNode.Status.Capacity[gpu.ResourceNvidiaGPU],
EphemeralStorage: baseNode.Status.Capacity[apiv1.ResourceEphemeralStorage],
PodCount: baseNode.Status.Capacity[apiv1.ResourcePods],
ExtendedResources: extendedResources,
}
} else {
klog.V(1).Infof("Generating node template only using nodeTemplate from MachineClass %s: template resources-> cpu: %s,memory: %s", machineClass.Name, nodeTemplateAttributes.Capacity.Cpu().String(), nodeTemplateAttributes.Capacity.Memory().String())
extendedResources := filterExtendedResources(nodeTemplateAttributes.Capacity)
instance = instanceType{
VCPU: nodeTemplateAttributes.Capacity[apiv1.ResourceCPU],
Memory: nodeTemplateAttributes.Capacity[apiv1.ResourceMemory],
GPU: nodeTemplateAttributes.Capacity["gpu"],
EphemeralStorage: nodeTemplateAttributes.Capacity[apiv1.ResourceEphemeralStorage],
// Numbers pods per node will depends on the CNI used and the maxPods kubelet config, default is often 110
PodCount: resource.MustParse("110"),
PodCount: resource.MustParse("110"),
ExtendedResources: extendedResources,
}
}
instance.InstanceType = nodeTemplateAttributes.InstanceType
Expand Down Expand Up @@ -972,6 +975,12 @@ func (m *McmManager) buildNodeFromTemplate(name string, template *nodeTemplate)
node.Status.Capacity["hugepages-1Gi"] = *resource.NewQuantity(0, resource.DecimalSI)
node.Status.Capacity["hugepages-2Mi"] = *resource.NewQuantity(0, resource.DecimalSI)

// populate extended resources from nodeTemplate
if len(template.InstanceType.ExtendedResources) > 0 {
klog.V(3).Infof("Copying extended resources %v to template node.Status.Capacity", template.InstanceType.ExtendedResources)
maps.Copy(node.Status.Capacity, template.InstanceType.ExtendedResources)
}

node.Status.Allocatable = node.Status.Capacity

// NodeLabels
Expand Down Expand Up @@ -1019,3 +1028,16 @@ func isMachineFailedOrTerminating(machine *v1alpha1.Machine) bool {
}
return false
}

func filterExtendedResources(allResources v1.ResourceList) (extendedResources v1.ResourceList) {
extendedResources = make(v1.ResourceList)
for _, n := range knownResourceNames {
r, ok := allResources[n]
if ok {
// don't add known resources to extendedResources
continue
}
extendedResources[n] = r
}
return
}
80 changes: 70 additions & 10 deletions cluster-autoscaler/cloudprovider/mcm/mcm_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package mcm
import (
"errors"
"github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1"
"k8s.io/utils/ptr"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -138,15 +139,9 @@ func TestFilterNodes(t *testing.T) {
}

func TestValidateNodeTemplate(t *testing.T) {
awsM5Large := AWSInstanceTypes["m5.large"]
m5Large := &instanceType{
InstanceType: awsM5Large.InstanceType,
VCPU: awsM5Large.VCPU,
Memory: awsM5Large.Memory,
GPU: awsM5Large.GPU,
}
m5LargeType := createSampleInstanceType("m5.large", "sap.com/mana", resource.MustParse("300"))
nt := v1alpha1.NodeTemplate{
InstanceType: m5Large.InstanceType,
InstanceType: m5LargeType.InstanceType,
Capacity: make(apiv1.ResourceList),
}
err := validateNodeTemplate(&nt)
Expand All @@ -163,8 +158,73 @@ func TestValidateNodeTemplate(t *testing.T) {
if err != nil {
t.Logf("error %s", err)
}
}

nt.Capacity[apiv1.ResourceEphemeralStorage] = resource.MustParse("20Gi")
err = validateNodeTemplate(&nt)
func TestBuildNodeFromTemplate(t *testing.T) {
m := &McmManager{}
namePrefix := "bingo"
m5LargeType := createSampleInstanceType("m5.large", "sap.com/mana", resource.MustParse("300"))
labels := map[string]string{
"weapon": "light-saber",
}
nt := nodeTemplate{
InstanceType: m5LargeType,
Architecture: ptr.To("amd64"),
Labels: labels,
}
nt.Region = "europe-west1"
nt.Zone = nt.Region + "-b"
node, err := m.buildNodeFromTemplate(namePrefix, &nt)
assert.Nil(t, err)
if err != nil {
t.Logf("error %s", err)
}
assert.True(t, isSubset(labels, node.Labels), "labels should be a subset of node.Labels")
for _, k := range coreResourceNames {
assert.Contains(t, node.Status.Capacity, k, "node.Status.Capacity should contain the mandatory resource named: %s", k)
}

// test with gpu resource
gpuQuantity := resource.MustParse("4")
nt.InstanceType.GPU = gpuQuantity
node, err = m.buildNodeFromTemplate(namePrefix, &nt)
assert.Nil(t, err)
if err != nil {
t.Logf("error %s", err)
}
for _, k := range coreResourceNames {
assert.Contains(t, node.Status.Capacity, k, "node.Status.Capacity should contain the mandatory resource named: %s", k)
}
var hasGpuResource bool
for _, k := range gpuResourceNames {
q, ok := node.Status.Capacity[k]
if ok {
hasGpuResource = true
assert.Equal(t, gpuQuantity, q, "node.Status.Capacity has gpu resource named %q with value %s instead of %s", k, q, gpuQuantity)
}
}
assert.True(t, hasGpuResource, "node.Status.Capacity should have a gpu resource with one of the names %q", gpuResourceNames)
}

func createSampleInstanceType(instanceTypeName string, customResourceName apiv1.ResourceName, customResourceQuantity resource.Quantity) *instanceType {
awsM5Large := AWSInstanceTypes[instanceTypeName]
extendedResources := make(apiv1.ResourceList)
extendedResources[customResourceName] = customResourceQuantity
iType := &instanceType{
InstanceType: awsM5Large.InstanceType,
VCPU: awsM5Large.VCPU,
Memory: awsM5Large.Memory,
GPU: awsM5Large.GPU,
ExtendedResources: extendedResources,
}
return iType
}

func isSubset[K comparable, V comparable](map1, map2 map[K]V) bool {
for k, v := range map1 {
if val, ok := map2[k]; !ok || val != v {
return false
}
}
return true
}

0 comments on commit ebb1b93

Please sign in to comment.