From 4a3455bf37299f63a159ccdb3ae2695b96356bfb Mon Sep 17 00:00:00 2001 From: DeveloperJim <190177081@qq.com> Date: Tue, 1 Dec 2020 16:15:03 +0800 Subject: [PATCH] fix: bcs-container-executor ip panic, issue #675 (cherry picked from commit 7dadbf72fb1ba492d9555fcec90dd2fd81a4fcd0) --- bcs-common/common/util/check.go | 79 +------------------ bcs-common/common/util/flags.go | 5 +- bcs-common/common/util/net.go | 67 ++++++++++------ .../app/bcs_executor.go | 25 +----- .../bcs-container-executor/app/bcs_handler.go | 4 +- .../container/cni/cni_pod.go | 20 ++--- .../container/cnm/cnm_pod.go | 23 +++--- .../container/docker.go | 11 --- .../bcs-container-executor/network/cni/cni.go | 27 +------ .../network/cni/manager.go | 11 +-- .../network/cnm/docker.go | 2 - .../bcs-container-executor/network/manager.go | 3 - bcs-mesos/bcs-container-executor/util/file.go | 26 ------ .../bcs-container-executor/util/flags.go | 61 -------------- .../util/{net.go => string.go} | 42 ---------- .../bcs-dns/plugin/bcsscheduler/lookup.go | 2 +- .../bcs-dns/plugin/bcsscheduler/scheduler.go | 3 +- bcs-services/bcs-netservice/bcs-ipam/main.go | 26 +++--- .../bcs-ipam/manager/manager.go | 13 +-- .../bcs-netservice/bcs-ipam/manager/tools.go | 34 -------- .../bmsf-mesos-adapter/pkg/util/ip/ip.go | 55 ------------- docs/version/1.18.x.md | 12 ++- 22 files changed, 122 insertions(+), 429 deletions(-) delete mode 100644 bcs-mesos/bcs-container-executor/util/file.go delete mode 100644 bcs-mesos/bcs-container-executor/util/flags.go rename bcs-mesos/bcs-container-executor/util/{net.go => string.go} (59%) delete mode 100644 bmsf-mesh/bmsf-mesos-adapter/pkg/util/ip/ip.go diff --git a/bcs-common/common/util/check.go b/bcs-common/common/util/check.go index 0054ab00ab..9c22c942ca 100644 --- a/bcs-common/common/util/check.go +++ b/bcs-common/common/util/check.go @@ -16,9 +16,11 @@ package util import ( "encoding/json" "fmt" + "github.com/Tencent/bk-bcs/bcs-common/common/types" ) +// CheckKind check object if expected func CheckKind(kind types.BcsDataType, by []byte) error { var meta *types.TypeMeta @@ -32,81 +34,4 @@ func CheckKind(kind types.BcsDataType, by []byte) error { } return nil - /*switch kind { - case types.BcsDataType_APP: - var obj types.ReplicaController - - err = json.Unmarshal(by, &obj) - if err != nil { - break - } - - if obj.Kind != types.BcsDataType_APP { - err = fmt.Errorf("Kind %s is invalid", obj.Kind) - } - - case types.BcsDataType_PROCESS: - var obj types.ReplicaController - err = json.Unmarshal(by, &obj) - if err != nil { - break - } - - if obj.Kind != types.BcsDataType_PROCESS { - err = fmt.Errorf("Kind %s is invalid", obj.Kind) - } - - case types.BcsDataType_DEPLOYMENT: - var obj types.BcsDeployment - - err = json.Unmarshal(by, &obj) - if err != nil { - break - } - - if obj.Kind != types.BcsDataType_DEPLOYMENT { - err = fmt.Errorf("Kind %s is invalid", obj.Kind) - } - - case types.BcsDataType_SERVICE: - var obj types.BcsService - - err = json.Unmarshal(by, &obj) - if err != nil { - break - } - - if obj.Kind != types.BcsDataType_SERVICE { - err = fmt.Errorf("Kind %s is invalid", obj.Kind) - } - - case types.BcsDataType_CONFIGMAP: - var obj types.BcsConfigMap - - err = json.Unmarshal(by, &obj) - if err != nil { - break - } - - if obj.Kind != types.BcsDataType_CONFIGMAP { - err = fmt.Errorf("Kind %s is invalid", obj.Kind) - } - - case types.BcsDataType_SECRET: - var obj types.BcsSecret - - err = json.Unmarshal(by, &obj) - if err != nil { - break - } - - if obj.Kind != types.BcsDataType_SECRET { - err = fmt.Errorf("Kind %s is invalid", obj.Kind) - } - - default: - return fmt.Errorf("Kind %s is invalid", kind) - }*/ - - //return err } diff --git a/bcs-common/common/util/flags.go b/bcs-common/common/util/flags.go index 5a92a68844..89a99bf914 100644 --- a/bcs-common/common/util/flags.go +++ b/bcs-common/common/util/flags.go @@ -15,11 +15,10 @@ package util import ( goflag "flag" - "github.com/Tencent/bk-bcs/bcs-common/common/version" + "os" "strings" - //"github.com/golang/glog" - "os" + "github.com/Tencent/bk-bcs/bcs-common/common/version" "github.com/spf13/pflag" ) diff --git a/bcs-common/common/util/net.go b/bcs-common/common/util/net.go index 0149058831..ca35eea1bb 100644 --- a/bcs-common/common/util/net.go +++ b/bcs-common/common/util/net.go @@ -16,42 +16,65 @@ package util import "net" var ( - _, classA, _ = net.ParseCIDR("10.0.0.0/8") - _, classA1, _ = net.ParseCIDR("9.0.0.0/8") - _, classAa, _ = net.ParseCIDR("100.64.0.0/10") - _, classB, _ = net.ParseCIDR("172.16.0.0/12") - _, classC, _ = net.ParseCIDR("192.168.0.0/16") + _, classA, _ = net.ParseCIDR("10.0.0.0/8") + _, classB, _ = net.ParseCIDR("172.16.0.0/12") + _, classC, _ = net.ParseCIDR("192.168.0.0/16") ) //GetIPAddress get local usable inner ip address -func GetIPAddress() (addrList []string) { +// check eth1 and eth0 first, if no IP address from eth1 and eth0, +// try to filter all network interface with private address +func GetIPAddress() string { + //try eth1 first + eth1Addr := getInterfaceIPv4Addr("eth1") + if len(eth1Addr) != 0 { + return eth1Addr + } + //try eth0 + eth0Addr := getInterfaceIPv4Addr("eth0") + if len(eth0Addr) != 0 { + return eth0Addr + } + //try all private network address addrs, err := net.InterfaceAddrs() if err != nil { - return addrList + return "" } for _, addr := range addrs { if ip, ok := addr.(*net.IPNet); ok && !ip.IP.IsLoopback() && ip.IP.To4() != nil { if classA.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue - } - if classA1.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue - } - if classAa.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue + return ip.IP.String() } if classB.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue + return ip.IP.String() } if classC.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue + return ip.IP.String() } } } - return addrList + return "" +} + +//getInterfaceIPv4Addr get specified network interface IPv4 address +// if interface has multiple available IP addresses and already UP +// just return the first one +func getInterfaceIPv4Addr(name string) string { + itf, err := net.InterfaceByName(name) + if err != nil { + return "" + } + if (itf.Flags & net.FlagUp) == 0 { + return "" + } + addrs, err := itf.Addrs() + if err != nil { + return "" + } + for _, addr := range addrs { + if ip, ok := addr.(*net.IPNet); ok && !ip.IP.IsLoopback() && ip.IP.To4() != nil { + return ip.IP.String() + } + } + return "" } diff --git a/bcs-mesos/bcs-container-executor/app/bcs_executor.go b/bcs-mesos/bcs-container-executor/app/bcs_executor.go index 3aeac002fd..1e740ee067 100644 --- a/bcs-mesos/bcs-container-executor/app/bcs_executor.go +++ b/bcs-mesos/bcs-container-executor/app/bcs_executor.go @@ -26,6 +26,7 @@ import ( "time" "github.com/Tencent/bk-bcs/bcs-common/common/blog" + "github.com/Tencent/bk-bcs/bcs-common/common/util" "github.com/Tencent/bk-bcs/bcs-common/pkg/scheduler/mesosproto/mesos" "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/container" "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/container/cni" @@ -36,11 +37,9 @@ import ( "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/network" cninet "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/network/cni" cnmnet "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/network/cnm" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/util" bcstype "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/types" "github.com/golang/protobuf/proto" - //"github.com/pborman/uuid" "golang.org/x/net/context" ) @@ -97,16 +96,6 @@ func NewBcsExecutor(flag *CommandFlags) exec.Executor { } func createNetManager(bcsExecutor *BcsExecutor, flag *CommandFlags) { - // exist, _ := util.FileExists(flag.CNIPluginDir + "/conf") - // if flag.NetworkMode == "" { - // if exist { - // //cni pod - // bcsExecutor.netManager = cninet.NewNetManager(flag.CNIPluginDir+"/bin", flag.CNIPluginDir+"/conf") - // } else { - // //docker pod - // bcsExecutor.netManager = cnmnet.NewNetManager() - // } - // } else { if flag.NetworkMode == "cni" { //cni pod bcsExecutor.netManager = cninet.NewNetManager(flag.CNIPluginDir+"/bin", flag.CNIPluginDir+"/conf") @@ -114,18 +103,9 @@ func createNetManager(bcsExecutor *BcsExecutor, flag *CommandFlags) { //docker pod bcsExecutor.netManager = cnmnet.NewNetManager() } - // } } func createPod(executor *BcsExecutor, flag *CommandFlags, containerTasks []*container.BcsContainerTask, podEvent *container.PodEventHandler) { - // exist, _ := util.FileExists(flag.CNIPluginDir + "/conf") - // if flag.NetworkMode == "" { - // if exist { - // executor.podInst = cni.NewPod(executor.container, containerTasks, podEvent) - // } else { - // executor.podInst = cnm.NewPod(executor.container, containerTasks, podEvent) - // } - // } else { if flag.NetworkMode == "cni" { //cni pod executor.podInst = cni.NewPod(executor.container, containerTasks, podEvent, flag.NetworkImage) @@ -850,10 +830,9 @@ func (executor *BcsExecutor) customSettingContainer(taskInfo *container.BcsConta // } // taskInfo.Env = append(taskInfo.Env, caliName) //setting mesos slave host ip - addrList := util.GetIPAddress() ipAddr := container.BcsKV{ Key: "BCS_NODE_IP", - Value: addrList[0], + Value: util.GetIPAddress(), } taskInfo.Env = append(taskInfo.Env, ipAddr) podID := container.BcsKV{ diff --git a/bcs-mesos/bcs-container-executor/app/bcs_handler.go b/bcs-mesos/bcs-container-executor/app/bcs_handler.go index d627e9866b..46ccd36a50 100644 --- a/bcs-mesos/bcs-container-executor/app/bcs_handler.go +++ b/bcs-mesos/bcs-container-executor/app/bcs_handler.go @@ -258,7 +258,7 @@ func (executor *BcsExecutor) frameworkMessageCommandTask(msg *bcstype.RequestCom **/ const ( - defaultHttpRequestTimeout = 120 + defaultHTTPRequestTimeout = 120 ) //dataClassRemote handle bcs_remote message, download data from remote http url, push to created container @@ -316,7 +316,7 @@ func (executor *BcsExecutor) downloadRemoteFile(remote *bcstype.Msg_Remote) (*bc //download content from remote http url. client := http.Client{ - Timeout: time.Duration(defaultHttpRequestTimeout * time.Second), + Timeout: time.Duration(defaultHTTPRequestTimeout * time.Second), } request, reqErr := http.NewRequest("GET", *remote.From, nil) if reqErr != nil { diff --git a/bcs-mesos/bcs-container-executor/container/cni/cni_pod.go b/bcs-mesos/bcs-container-executor/container/cni/cni_pod.go index 0d32a948e2..ebefb9b374 100644 --- a/bcs-mesos/bcs-container-executor/container/cni/cni_pod.go +++ b/bcs-mesos/bcs-container-executor/container/cni/cni_pod.go @@ -16,13 +16,6 @@ package cni import ( "bytes" "fmt" - comtypes "github.com/Tencent/bk-bcs/bcs-common/common/types" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/container" - device_plugin_manager "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/device-plugin-manager" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/healthcheck" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/logs" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/util" - bcstypes "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/types" "io/ioutil" "os" "path/filepath" @@ -30,6 +23,15 @@ import ( "sync" "time" + comtypes "github.com/Tencent/bk-bcs/bcs-common/common/types" + "github.com/Tencent/bk-bcs/bcs-common/common/util" + "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/container" + device_plugin_manager "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/device-plugin-manager" + "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/healthcheck" + "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/logs" + exeutil "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/util" + bcstypes "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/types" + //"github.com/pborman/uuid" schedTypes "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/types" @@ -241,7 +243,7 @@ func (p *CNIPod) Init() error { } if p.cniHostName == "" { - p.cniHostName = "pod-" + util.RandomString(12) + p.cniHostName = "pod-" + exeutil.RandomString(12) } p.netTask = &container.BcsContainerTask{ @@ -441,7 +443,7 @@ func (p *CNIPod) Start() error { task.HostName = hostname task.RuntimeConf.ID = createInst.ID - task.RuntimeConf.NodeAddress = util.GetIPAddress()[0] + task.RuntimeConf.NodeAddress = util.GetIPAddress() task.RuntimeConf.IPAddress = p.cniIPAddr task.RuntimeConf.Status = container.ContainerStatus_CREATED task.RuntimeConf.Message = "container created" diff --git a/bcs-mesos/bcs-container-executor/container/cnm/cnm_pod.go b/bcs-mesos/bcs-container-executor/container/cnm/cnm_pod.go index 8b6ba3555c..e182ab5d49 100644 --- a/bcs-mesos/bcs-container-executor/container/cnm/cnm_pod.go +++ b/bcs-mesos/bcs-container-executor/container/cnm/cnm_pod.go @@ -15,16 +15,16 @@ package cnm import ( "fmt" + "strconv" + "sync" + "time" + "github.com/Tencent/bk-bcs/bcs-common/common/blog" + "github.com/Tencent/bk-bcs/bcs-common/common/util" "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/container" device_plugin_manager "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/device-plugin-manager" "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/healthcheck" "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/logs" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/util" - "strconv" - "sync" - "time" - schedTypes "github.com/Tencent/bk-bcs/bcs-mesos/bcs-scheduler/src/types" "golang.org/x/net/context" @@ -41,6 +41,7 @@ const ( ) const ( + // ContainerStatusAbnormal abnormal definition ContainerStatusAbnormal = 1 ) @@ -100,6 +101,7 @@ func (p *DockerPod) IsHealthy() bool { return p.healthy } +//GetContainerTasks get local container task func (p *DockerPod) GetContainerTasks() map[string]*container.BcsContainerTask { return p.conTasks } @@ -188,7 +190,7 @@ func (p *DockerPod) Init() error { envHost := container.BcsKV{ Key: "BCS_CONTAINER_IP", - Value: util.GetIPAddress()[0], + Value: util.GetIPAddress(), } p.netTask.Env = append(p.netTask.Env, envHost) //assignment for environments @@ -289,7 +291,7 @@ func (p *DockerPod) Init() error { } p.cnmIPAddr = info.IPAddress p.netTask.RuntimeConf.Message = "container is starting" - p.netTask.RuntimeConf.NodeAddress = util.GetIPAddress()[0] + p.netTask.RuntimeConf.NodeAddress = util.GetIPAddress() p.netTask.RuntimeConf.IPAddress = info.IPAddress p.netTask.RuntimeConf.NetworkMode = info.NetworkMode p.runningContainer[p.netTask.RuntimeConf.Name] = p.netTask.RuntimeConf @@ -310,7 +312,7 @@ func (p *DockerPod) Start() error { //add pod ipaddr to ENV envHost := container.BcsKV{ Key: "BCS_CONTAINER_IP", - Value: util.GetIPAddress()[0], + Value: util.GetIPAddress(), } logs.Infof("docker pod start container...") @@ -374,7 +376,7 @@ func (p *DockerPod) Start() error { return createErr } task.RuntimeConf.ID = createdInst.ID - task.RuntimeConf.NodeAddress = util.GetIPAddress()[0] + task.RuntimeConf.NodeAddress = util.GetIPAddress() task.RuntimeConf.IPAddress = p.cnmIPAddr task.RuntimeConf.Status = container.ContainerStatus_CREATED task.RuntimeConf.Message = "container created" @@ -707,10 +709,12 @@ func (p *DockerPod) copyPortMappings() { } } +//GetNetArgs implementation func (p *DockerPod) GetNetArgs() [][2]string { return nil } +//UpdateResources update CPU or MEM resource in runtime func (p *DockerPod) UpdateResources(id string, resource *schedTypes.Resource) error { var exist bool var conTask *container.BcsContainerTask @@ -736,6 +740,7 @@ func (p *DockerPod) UpdateResources(id string, resource *schedTypes.Resource) er return nil } +// CommitImage image commit func (p *DockerPod) CommitImage(id, image string) error { return p.conClient.CommitImage(id, image) } diff --git a/bcs-mesos/bcs-container-executor/container/docker.go b/bcs-mesos/bcs-container-executor/container/docker.go index 7abdd894f8..094d6b9306 100644 --- a/bcs-mesos/bcs-container-executor/container/docker.go +++ b/bcs-mesos/bcs-container-executor/container/docker.go @@ -349,17 +349,6 @@ func (docker *DockerContainer) CreateContainer(containerName string, containerTa hostConfig.Memory = int64(containerTask.Resource.Mem * 1024 * 1024) hostConfig.MemorySwap = int64(containerTask.Resource.Mem * 1024 * 1024) } - /*if len(containerTask.Resource.CPUSet) > 0 { - //only setting cpu info when cpu request can be met in slave host - //cpuList, numaList := util.GetBindingCPUs(int(math.Ceil(containerTask.Resource.Cpus)), int64(containerTask.Resource.Mem)) - //change int list to string - for _, set := range containerTask.Resource.CPUSet { - hostConfig.CPUSetCPUs += fmt.Sprintf("%s,", set) - hostConfig.CPUSetMEMs - } - hostConfig.CPUSetCPUs = strings.TrimRight(hostConfig.CPUSetCPUs, "") - fmt.Fprintf(os.Stdout, "DEBUG: task(%s) set cpuset(%s)\n", containerTask.Name, hostConfig.CPUSetCPUs) - }*/ if containerTask.LimitResource != nil && containerTask.LimitResource.Cpus > 0 { hostConfig.CPUPeriod = DefaultDockerCpuPeriod diff --git a/bcs-mesos/bcs-container-executor/network/cni/cni.go b/bcs-mesos/bcs-container-executor/network/cni/cni.go index 270cc8288b..2fa76faa8a 100644 --- a/bcs-mesos/bcs-container-executor/network/cni/cni.go +++ b/bcs-mesos/bcs-container-executor/network/cni/cni.go @@ -16,13 +16,14 @@ package cni import ( "encoding/json" "fmt" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/container" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/logs" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/network" "os" "path/filepath" "strings" + "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/container" + "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/logs" + "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/network" + "github.com/containernetworking/cni/libcni" cnitypes "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" @@ -111,26 +112,6 @@ func (plugin *CNIPlugin) Init(host string) error { return nil } -//Status get network status, Status only work after SetUpPod -/*func (plugin *CNIPlugin) Status() *network.NetStatus { - if plugin.status != nil { - return plugin.status - } - return nil -}*/ - -//Version call plugin command line and return version string -/*func (plugin *CNIPlugin) Version() string { - info, err := plugin.cniNet.GetVersionInfo(plugin.networkName) - if err != nil { - logs.Errorf("CNI Plugin %s VERSION command err: %s\n", plugin.networkName, err.Error()) - return "" - } - vers := info.SupportedVersions() - lastVer := vers[len(vers)-1] - return lastVer -}*/ - //SetUpPod Setup Network info for pod func (plugin *CNIPlugin) SetUpPod(podInfo container.Pod) error { //build runtime conf for libcni diff --git a/bcs-mesos/bcs-container-executor/network/cni/manager.go b/bcs-mesos/bcs-container-executor/network/cni/manager.go index 10421a48b0..71e77304da 100644 --- a/bcs-mesos/bcs-container-executor/network/cni/manager.go +++ b/bcs-mesos/bcs-container-executor/network/cni/manager.go @@ -14,12 +14,12 @@ package cni import ( + "fmt" + + "github.com/Tencent/bk-bcs/bcs-common/common/util" "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/container" "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/logs" "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/network" - "github.com/Tencent/bk-bcs/bcs-mesos/bcs-container-executor/util" - - "fmt" "github.com/containernetworking/cni/libcni" ) @@ -114,8 +114,9 @@ func (manager *PluginManager) SetUpPod(podInfo container.Pod) error { return nil } if podInfo.GetNetworkName() == "host" { - podInfo.SetIPAddr(util.GetIPAddress()[0]) - logs.Infoln(fmt.Sprintf("CNI plugin manager skip pod setup with host mode, setting pod ipaddr %s", util.GetIPAddress()[0])) + addr := util.GetIPAddress() + podInfo.SetIPAddr(addr) + logs.Infoln(fmt.Sprintf("CNI plugin manager skip pod setup with host mode, setting pod ipaddr %s", addr)) return nil } diff --git a/bcs-mesos/bcs-container-executor/network/cnm/docker.go b/bcs-mesos/bcs-container-executor/network/cnm/docker.go index ab542aaaae..0adac95a33 100644 --- a/bcs-mesos/bcs-container-executor/network/cnm/docker.go +++ b/bcs-mesos/bcs-container-executor/network/cnm/docker.go @@ -24,8 +24,6 @@ func NewNetManager() network.NetManager { return manager } -//todo(developerJim): DockerNetManager can create Network namespace by driver - //DockerNetManager docker network manager for using docker network type DockerNetManager struct { } diff --git a/bcs-mesos/bcs-container-executor/network/manager.go b/bcs-mesos/bcs-container-executor/network/manager.go index adf853f2e1..62aec273f3 100644 --- a/bcs-mesos/bcs-container-executor/network/manager.go +++ b/bcs-mesos/bcs-container-executor/network/manager.go @@ -25,7 +25,4 @@ type NetManager interface { AddPlugin(name string, plguin NetworkPlugin) error //Add plugin to manager dynamic if necessary SetUpPod(podInfo container.Pod) error //for setting Pod network interface TearDownPod(podInfo container.Pod) error //for release pod network resource - //todo(developerJim): add more network to pod - //SetUpNets(list []string, Pod) - //TearDownNets(list, Pod) } diff --git a/bcs-mesos/bcs-container-executor/util/file.go b/bcs-mesos/bcs-container-executor/util/file.go deleted file mode 100644 index 3fe2536cd9..0000000000 --- a/bcs-mesos/bcs-container-executor/util/file.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making Blueking Container Service available. - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * Licensed under the MIT License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * http://opensource.org/licenses/MIT - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package util - -import "os" - -//FileExists check file exists -func FileExists(filename string) (bool, error) { - if _, err := os.Stat(filename); os.IsNotExist(err) { - return false, nil - } else if err != nil { - return false, err - } - return true, nil -} diff --git a/bcs-mesos/bcs-container-executor/util/flags.go b/bcs-mesos/bcs-container-executor/util/flags.go deleted file mode 100644 index 1b4dd62cc5..0000000000 --- a/bcs-mesos/bcs-container-executor/util/flags.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making Blueking Container Service available. - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * Licensed under the MIT License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * http://opensource.org/licenses/MIT - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package util - -import ( - "github.com/Tencent/bk-bcs/bcs-common/common/version" - "os" - "strings" - - "github.com/spf13/pflag" -) - -// WordSepNormalizeFunc changes all flags that contain "_" separators -func WordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { - if strings.Contains(name, "_") { - return pflag.NormalizedName(strings.Replace(name, "_", "-", -1)) - } - return pflag.NormalizedName(name) -} - -// WarnWordSepNormalizeFunc changes and warns for flags that contain "_" separators -func WarnWordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { - if strings.Contains(name, "_") { - nname := strings.Replace(name, "_", "-", -1) - //glog.Warningf("%s is DEPRECATED and will be removed in a future version. Use %s instead.", name, nname) - - return pflag.NormalizedName(nname) - } - return pflag.NormalizedName(name) -} - -// AddCommonFlags add common flags that is needed by all modules -func AddCommonFlags(cmdline *pflag.FlagSet) *bool { - version := cmdline.BoolP("version", "v", false, "show version infomation") - return version -} - -// InitFlags normalizes and parses the command line flags -func InitFlags() { - pflag.CommandLine.SetNormalizeFunc(WordSepNormalizeFunc) - //pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) - ver := AddCommonFlags(pflag.CommandLine) - pflag.Parse() - - //add handler if flag include --version/-v - if *ver { - version.ShowVersion() - os.Exit(0) - } -} diff --git a/bcs-mesos/bcs-container-executor/util/net.go b/bcs-mesos/bcs-container-executor/util/string.go similarity index 59% rename from bcs-mesos/bcs-container-executor/util/net.go rename to bcs-mesos/bcs-container-executor/util/string.go index 636431131d..fb17d00bdd 100644 --- a/bcs-mesos/bcs-container-executor/util/net.go +++ b/bcs-mesos/bcs-container-executor/util/string.go @@ -15,51 +15,9 @@ package util import ( "math/rand" - "net" "time" ) -var ( - _, classA, _ = net.ParseCIDR("10.0.0.0/8") - _, classA1, _ = net.ParseCIDR("9.0.0.0/8") - _, classAa, _ = net.ParseCIDR("100.64.0.0/10") - _, classB, _ = net.ParseCIDR("172.16.0.0/12") - _, classC, _ = net.ParseCIDR("192.168.0.0/16") -) - -//GetIPAddress get local usable inner ip address -func GetIPAddress() (addrList []string) { - addrs, err := net.InterfaceAddrs() - if err != nil { - return addrList - } - for _, addr := range addrs { - if ip, ok := addr.(*net.IPNet); ok && !ip.IP.IsLoopback() && ip.IP.To4() != nil { - if classA.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue - } - if classA1.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue - } - if classAa.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue - } - if classB.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue - } - if classC.Contains(ip.IP) { - addrList = append(addrList, ip.IP.String()) - continue - } - } - } - return addrList -} - const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" const ( letterIdxBits = 6 // 6 bits to represent a letter index diff --git a/bcs-services/bcs-dns/plugin/bcsscheduler/lookup.go b/bcs-services/bcs-dns/plugin/bcsscheduler/lookup.go index c17edd3966..34e6af4092 100644 --- a/bcs-services/bcs-dns/plugin/bcsscheduler/lookup.go +++ b/bcs-services/bcs-dns/plugin/bcsscheduler/lookup.go @@ -211,7 +211,7 @@ func (bcs *BcsScheduler) selfDNSRecord() dns.A { return self } self.Hdr.Name = defaultNSName - self.A = net.ParseIP(addrList[0]) + self.A = net.ParseIP(addrList) return self } diff --git a/bcs-services/bcs-dns/plugin/bcsscheduler/scheduler.go b/bcs-services/bcs-dns/plugin/bcsscheduler/scheduler.go index 915d6c231f..513ee03769 100644 --- a/bcs-services/bcs-dns/plugin/bcsscheduler/scheduler.go +++ b/bcs-services/bcs-dns/plugin/bcsscheduler/scheduler.go @@ -48,6 +48,7 @@ var errInvalidRequest = errors.New("invalid query name") //var errZoneMisMatch = errors.New("zone group dismatch") +// ForwardClusterIpDomainError err type ForwardClusterIpDomainError struct { ClusterIP string } @@ -79,7 +80,7 @@ func NewScheduler(config *ConfigItem) *BcsScheduler { } else { hostname, _ := os.Hostname() node := &bcstypes.ServerInfo{ - IP: util.GetIPAddress()[0], + IP: util.GetIPAddress(), Port: uint(53), Pid: os.Getpid(), HostName: hostname, diff --git a/bcs-services/bcs-netservice/bcs-ipam/main.go b/bcs-services/bcs-netservice/bcs-ipam/main.go index 34818c850f..a1bc33865d 100644 --- a/bcs-services/bcs-netservice/bcs-ipam/main.go +++ b/bcs-services/bcs-netservice/bcs-ipam/main.go @@ -16,15 +16,15 @@ package main import ( "encoding/json" "fmt" - "github.com/Tencent/bk-bcs/bcs-common/common/util" - "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/bcs-ipam/manager" - bcsconf "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/config" - nettypes "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/pkg/netservice/types" "net" "strconv" "github.com/Tencent/bk-bcs/bcs-common/common/blog" "github.com/Tencent/bk-bcs/bcs-common/common/conf" + "github.com/Tencent/bk-bcs/bcs-common/common/util" + "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/bcs-ipam/manager" + bcsconf "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/config" + nettypes "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/pkg/netservice/types" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" @@ -41,7 +41,7 @@ var ( type BcsIPAMConfig struct { Name string Type string `json:"type"` //bcs-ipam - Host net.IP `json:"host"` //local host ip + Host string `json:"host"` //local host ip Routes []types.Route `json:"routes"` Args *bcsconf.CNIArgs `json:"-"` } @@ -88,11 +88,11 @@ func cmdAdd(args *skel.CmdArgs) error { return err } //check host ip if configuration designated - if ipamConf.Host == nil { + if len(ipamConf.Host) == 0 { //get one available host ip from system - ipamConf.Host = manager.GetAvailableIP() + ipamConf.Host = util.GetIPAddress() } - if ipamConf.Host == nil { + if len(ipamConf.Host) == 0 { return fmt.Errorf("Get no available host ip for request") } //Get available ip address from resource @@ -104,7 +104,7 @@ func cmdAdd(args *skel.CmdArgs) error { if ipamConf.Args != nil && ipamConf.Args.IP != nil { requestIP = ipamConf.Args.IP.String() } - info, getErr := ipDriver.GetIPAddr(ipamConf.Host.String(), args.ContainerID, requestIP) + info, getErr := ipDriver.GetIPAddr(ipamConf.Host, args.ContainerID, requestIP) if getErr != nil { return getErr } @@ -151,11 +151,11 @@ func cmdDel(args *skel.CmdArgs) error { return err } //check host ip if configuration designated - if ipamConf.Host == nil { + if len(ipamConf.Host) == 0 { //get one available host ip from system - ipamConf.Host = manager.GetAvailableIP() + ipamConf.Host = util.GetIPAddress() } - if ipamConf.Host == nil { + if len(ipamConf.Host) == 0 { return fmt.Errorf("Get no available host ip for request") } ipDriver, driverErr := manager.GetIPDriver() @@ -167,7 +167,7 @@ func cmdDel(args *skel.CmdArgs) error { ipInfo.IPAddr = ipamConf.Args.IP.String() } //release ip with IPInfo - return ipDriver.ReleaseIPAddr(ipamConf.Host.String(), args.ContainerID, ipInfo) + return ipDriver.ReleaseIPAddr(ipamConf.Host, args.ContainerID, ipInfo) } func main() { diff --git a/bcs-services/bcs-netservice/bcs-ipam/manager/manager.go b/bcs-services/bcs-netservice/bcs-ipam/manager/manager.go index abbb61c431..9a22c01978 100644 --- a/bcs-services/bcs-netservice/bcs-ipam/manager/manager.go +++ b/bcs-services/bcs-netservice/bcs-ipam/manager/manager.go @@ -15,6 +15,7 @@ package manager import ( "github.com/Tencent/bk-bcs/bcs-common/common/blog" + "github.com/Tencent/bk-bcs/bcs-common/common/util" nettypes "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/pkg/netservice/types" dockerclient "github.com/fsouza/go-dockerclient" @@ -32,14 +33,14 @@ func DirtyCheck() { blog.Errorf("Get driver error in check mode, %s", err.Error()) return } - hostIP := GetAvailableIP() - hostInfo, err := driver.GetHostInfo(hostIP.String()) + hostIP := util.GetIPAddress() + hostInfo, err := driver.GetHostInfo(hostIP) if err != nil { blog.Errorf("Get host info in check mode failed, %s", err.Error()) return } if hostInfo.Containers == nil || len(hostInfo.Containers) == 0 { - blog.Infof("No active container & ip address in host %s, normal exit", hostIP.String()) + blog.Infof("No active container & ip address in host %s, normal exit", hostIP) return } client, err := dockerclient.NewClient(defaultContainerSock) @@ -71,10 +72,10 @@ func DirtyCheck() { blog.Warnf("##container info mismatch warnning, host: %s, ip inst: %s", containerID, ipInst.Container) } ipInfo := &nettypes.IPInfo{} - blog.Errorf("Host %s release dirty ip %s in container %s", hostIP.String(), ipInst.IPAddr, containerID) - err := driver.ReleaseIPAddr(hostIP.String(), containerID, ipInfo) + blog.Errorf("Host %s release dirty ip %s in container %s", hostIP, ipInst.IPAddr, containerID) + err := driver.ReleaseIPAddr(hostIP, containerID, ipInfo) if err != nil { - blog.Errorf("Host %s release container %s/%s err, %s", hostIP.String(), containerID, ipInst.IPAddr, err.Error()) + blog.Errorf("Host %s release container %s/%s err, %s", hostIP, containerID, ipInst.IPAddr, err.Error()) continue } } diff --git a/bcs-services/bcs-netservice/bcs-ipam/manager/tools.go b/bcs-services/bcs-netservice/bcs-ipam/manager/tools.go index a651f901bf..19edb9c224 100644 --- a/bcs-services/bcs-netservice/bcs-ipam/manager/tools.go +++ b/bcs-services/bcs-netservice/bcs-ipam/manager/tools.go @@ -18,15 +18,9 @@ import ( "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/bcs-ipam/resource" "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/bcs-ipam/resource/localdriver" "github.com/Tencent/bk-bcs/bcs-services/bcs-netservice/bcs-ipam/resource/netdriver" - "net" ) var ( - _, classA, _ = net.ParseCIDR("10.0.0.0/8") - _, classA2, _ = net.ParseCIDR("9.0.0.0/8") - _, classAa, _ = net.ParseCIDR("100.64.0.0/10") - _, classB, _ = net.ParseCIDR("172.16.0.0/12") - _, classC, _ = net.ParseCIDR("192.168.0.0/16") defaultDatabase = "/data/bcs/bcs-cni/bin/bcs-ipam.db" ) @@ -37,31 +31,3 @@ func GetIPDriver() (resource.IPDriver, error) { } return netdriver.NewDriver() } - -//GetAvailableIP get local host ip address -func GetAvailableIP() net.IP { - addrs, err := net.InterfaceAddrs() - if err != nil { - return nil - } - for _, addr := range addrs { - if ip, ok := addr.(*net.IPNet); ok && !ip.IP.IsLoopback() && ip.IP.To4() != nil { - if classA.Contains(ip.IP) { - return ip.IP - } - if classA2.Contains(ip.IP) { - return ip.IP - } - if classAa.Contains(ip.IP) { - return ip.IP - } - if classB.Contains(ip.IP) { - return ip.IP - } - if classC.Contains(ip.IP) { - return ip.IP - } - } - } - return nil -} diff --git a/bmsf-mesh/bmsf-mesos-adapter/pkg/util/ip/ip.go b/bmsf-mesh/bmsf-mesos-adapter/pkg/util/ip/ip.go deleted file mode 100644 index bccf5cc6a1..0000000000 --- a/bmsf-mesh/bmsf-mesos-adapter/pkg/util/ip/ip.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making Blueking Container Service available. - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * Licensed under the MIT License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * http://opensource.org/licenses/MIT - * Unless required by applicable law or agreed to in writing, software distributed under - * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package ip - -import ( - "net" -) - -var ( - _, classA, _ = net.ParseCIDR("10.0.0.0/8") - _, classA2, _ = net.ParseCIDR("9.0.0.0/8") - _, classAa, _ = net.ParseCIDR("100.64.0.0/10") - _, classB, _ = net.ParseCIDR("172.16.0.0/12") - _, classC, _ = net.ParseCIDR("192.168.0.0/16") -) - -//GetAvailableIP get local host first available ip address -//return empty string if error accurs -func GetAvailableIP() string { - addrs, err := net.InterfaceAddrs() - if err != nil { - return "" - } - for _, addr := range addrs { - if ip, ok := addr.(*net.IPNet); ok && !ip.IP.IsLoopback() && ip.IP.To4() != nil { - if classA.Contains(ip.IP) { - return ip.IP.String() - } - if classA2.Contains(ip.IP) { - return ip.IP.String() - } - if classAa.Contains(ip.IP) { - return ip.IP.String() - } - if classB.Contains(ip.IP) { - return ip.IP.String() - } - if classC.Contains(ip.IP) { - return ip.IP.String() - } - } - } - return "" -} diff --git a/docs/version/1.18.x.md b/docs/version/1.18.x.md index 4efb5f92e7..eb95a1d762 100644 --- a/docs/version/1.18.x.md +++ b/docs/version/1.18.x.md @@ -1,5 +1,16 @@ # 1.18.x 版本信息说明 +## 1.18.11 + +* 发布日期：2020-11-25 +* **BUG修复** + * [Mesos] bcs-scheduler修复taskgroup脏数据问题[#664] + * [Mesos] bcs-service-prometheus修复selector异常导致退出问题[#671] + * [Mesos] bcs-service-prometheus生成Node配置时产生死锁问题[#564] + * [Mesos] bcs-service-prometheus清理label冲突问题[#644] + * [Mesos] bcs-service-prometheus修复配置文件重复问题[#647] + * bcs-qcloud-eip & bcs-cloudnetwork修复弹性网卡个数错误问题 + ## 1.18.10 * 发布日期：2020-11-05 @@ -36,7 +47,6 @@ * **BUG修复** * bcs-api, bcs-user-manager修复websocket tunnel隧道长链接泄露问题[#584] * [Mesos] bcs-scheduler修复command health check失败问题[#586] - * [Mesos] bcs-service-prometheus生成Node配置时产生死锁问题[#564] * [Mesos/K8S] 修复qcloud-eip多网卡bug问题[#556] * [Mesos] bcs-mesos-watch修复deployment消息堵塞消费过慢的问题[#552] * [Mesos] bcs-scheduler修复保存version latest报错问题[#525]