From 36310401ad9f24be6b03a57471dcd6fc0129f13e Mon Sep 17 00:00:00 2001 From: dolibali Date: Mon, 23 Dec 2024 10:17:35 +0000 Subject: [PATCH 1/2] Fix qos bug 24.12.23 Signed-off-by: dolibali --- pkg/daemon/controller_linux.go | 6 ++++-- pkg/ovs/ovs-vsctl_linux.go | 29 ++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go index 37086f59e0a..539e00e1b6f 100644 --- a/pkg/daemon/controller_linux.go +++ b/pkg/daemon/controller_linux.go @@ -528,12 +528,12 @@ func (c *Controller) getPolicyRouting(subnet *kubeovnv1.Subnet) ([]netlink.Rule, } func (c *Controller) handlePod(key string) error { + klog.V(3).Infof("v3 log start success") namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) return nil } - klog.Infof("handle qos update for pod %s/%s", namespace, name) pod, err := c.podsLister.Pods(namespace).Get(name) if err != nil { @@ -557,7 +557,7 @@ func (c *Controller) handlePod(key string) error { // set default nic bandwidth ifaceID := ovs.PodNameToPortName(podName, pod.Namespace, util.OvnProvider) - err = ovs.SetInterfaceBandwidth(podName, pod.Namespace, ifaceID, pod.Annotations[util.EgressRateAnnotation], pod.Annotations[util.IngressRateAnnotation]) + err = ovs.SetInterfaceBandwidth(podName, pod.Namespace, ifaceID, pod.Annotations[util.IngressRateAnnotation], pod.Annotations[util.EgressRateAnnotation]) if err != nil { klog.Error(err) return err @@ -574,6 +574,8 @@ func (c *Controller) handlePod(key string) error { return err } + klog.V(3).Infof("handle qos update for pod %s/%s", namespace, name) + // set multus-nic bandwidth attachNets, err := nadutils.ParsePodNetworkAnnotation(pod) if err != nil { diff --git a/pkg/ovs/ovs-vsctl_linux.go b/pkg/ovs/ovs-vsctl_linux.go index a378c129295..ba843e3a149 100644 --- a/pkg/ovs/ovs-vsctl_linux.go +++ b/pkg/ovs/ovs-vsctl_linux.go @@ -15,6 +15,8 @@ import ( func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress string) error { ingressMPS, _ := strconv.Atoi(ingress) ingressKPS := ingressMPS * 1000 + ingressBurst := ingressKPS * 8 / 10 + interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s", iface)) if err != nil { klog.Error(err) @@ -33,35 +35,42 @@ func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress string) return err } + egressMPS, _ := strconv.Atoi(egress) + egressBPS := egressMPS * 1000 * 1000 + for _, ifName := range interfaceList { // ingress_policing_rate is in Kbps - err := ovsSet("interface", ifName, fmt.Sprintf("ingress_policing_rate=%d", ingressKPS), fmt.Sprintf("ingress_policing_burst=%d", ingressKPS*8/10)) + err := ovsSet("interface", ifName, fmt.Sprintf("ingress_policing_rate=%d", ingressKPS), fmt.Sprintf("ingress_policing_burst=%d", ingressBurst)) if err != nil { klog.Error(err) return err - } - - egressMPS, _ := strconv.Atoi(egress) - egressBPS := egressMPS * 1000 * 1000 + } else { + klog.V(3).Infof("Successfully set ingress policing rate to %d Kbps on interface %s", ingressKPS, ifName) + } if egressBPS > 0 { queueUID, err := SetHtbQosQueueRecord(podName, podNamespace, iface, egressBPS, queueIfaceUIDMap) if err != nil { klog.Error(err) return err - } + } else { + klog.V(3).Infof("Successfully set HTB QoS queue record for interface %s with egress %d Bps", iface, egressBPS) + } if err = SetQosQueueBinding(podName, podNamespace, ifName, iface, queueUID, qosIfaceUIDMap); err != nil { klog.Error(err) return err - } + } else { + klog.V(3).Infof("Successfully bound QoS queue for interface %s", ifName) + } } else { if qosUID, ok := qosIfaceUIDMap[iface]; ok { qosType, err := ovsGet("qos", qosUID, "type", "") if err != nil { klog.Error(err) return err - } + } + if qosType != util.HtbQos { continue } @@ -74,7 +83,9 @@ func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress string) if _, err := Exec("remove", "queue", queueID, "other_config", "max-rate"); err != nil { klog.Error(err) return fmt.Errorf("failed to remove rate limit for queue in pod %v/%v, %w", podNamespace, podName, err) - } + } else { + klog.V(3).Infof("Successfully removed rate limit for queue ID %s in pod %v/%v", queueID, podNamespace, podName) + } } } From a67deebdc6f8c64729846ec8582a07a71d5c8382 Mon Sep 17 00:00:00 2001 From: dolibali Date: Tue, 24 Dec 2024 02:11:26 +0000 Subject: [PATCH 2/2] fix log 24.12.24 Signed-off-by: dolibali --- pkg/daemon/controller_linux.go | 5 ++--- pkg/ovs/ovs-vsctl_linux.go | 27 ++++++++++----------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go index 539e00e1b6f..a89c6b7b2c8 100644 --- a/pkg/daemon/controller_linux.go +++ b/pkg/daemon/controller_linux.go @@ -528,7 +528,6 @@ func (c *Controller) getPolicyRouting(subnet *kubeovnv1.Subnet) ([]netlink.Rule, } func (c *Controller) handlePod(key string) error { - klog.V(3).Infof("v3 log start success") namespace, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) @@ -550,6 +549,8 @@ func (c *Controller) handlePod(key string) error { return err } + klog.Infof("handle qos update for pod %s/%s", namespace, name) + podName := pod.Name if pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)] != "" { podName = pod.Annotations[fmt.Sprintf(util.VMAnnotationTemplate, util.OvnProvider)] @@ -574,8 +575,6 @@ func (c *Controller) handlePod(key string) error { return err } - klog.V(3).Infof("handle qos update for pod %s/%s", namespace, name) - // set multus-nic bandwidth attachNets, err := nadutils.ParsePodNetworkAnnotation(pod) if err != nil { diff --git a/pkg/ovs/ovs-vsctl_linux.go b/pkg/ovs/ovs-vsctl_linux.go index ba843e3a149..7c97d00b3ac 100644 --- a/pkg/ovs/ovs-vsctl_linux.go +++ b/pkg/ovs/ovs-vsctl_linux.go @@ -16,6 +16,8 @@ func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress string) ingressMPS, _ := strconv.Atoi(ingress) ingressKPS := ingressMPS * 1000 ingressBurst := ingressKPS * 8 / 10 + egressMPS, _ := strconv.Atoi(egress) + egressBPS := egressMPS * 1000 * 1000 interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s", iface)) if err != nil { @@ -35,41 +37,34 @@ func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress string) return err } - egressMPS, _ := strconv.Atoi(egress) - egressBPS := egressMPS * 1000 * 1000 - for _, ifName := range interfaceList { // ingress_policing_rate is in Kbps + klog.V(3).Infof("Set ingress policing rate to %d Kbps on interface %s", ingressKPS, ifName) err := ovsSet("interface", ifName, fmt.Sprintf("ingress_policing_rate=%d", ingressKPS), fmt.Sprintf("ingress_policing_burst=%d", ingressBurst)) if err != nil { klog.Error(err) return err - } else { - klog.V(3).Infof("Successfully set ingress policing rate to %d Kbps on interface %s", ingressKPS, ifName) - } + } if egressBPS > 0 { + klog.V(3).Infof("Set HTB QoS queue record for interface %s with egress %d Bps", iface, egressBPS) queueUID, err := SetHtbQosQueueRecord(podName, podNamespace, iface, egressBPS, queueIfaceUIDMap) if err != nil { klog.Error(err) return err - } else { - klog.V(3).Infof("Successfully set HTB QoS queue record for interface %s with egress %d Bps", iface, egressBPS) - } - + } + klog.V(3).Infof("Bound QoS queue for interface %s", ifName) if err = SetQosQueueBinding(podName, podNamespace, ifName, iface, queueUID, qosIfaceUIDMap); err != nil { klog.Error(err) return err - } else { - klog.V(3).Infof("Successfully bound QoS queue for interface %s", ifName) - } + } } else { if qosUID, ok := qosIfaceUIDMap[iface]; ok { qosType, err := ovsGet("qos", qosUID, "type", "") if err != nil { klog.Error(err) return err - } + } if qosType != util.HtbQos { continue @@ -83,9 +78,7 @@ func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress string) if _, err := Exec("remove", "queue", queueID, "other_config", "max-rate"); err != nil { klog.Error(err) return fmt.Errorf("failed to remove rate limit for queue in pod %v/%v, %w", podNamespace, podName, err) - } else { - klog.V(3).Infof("Successfully removed rate limit for queue ID %s in pod %v/%v", queueID, podNamespace, podName) - } + } } }