From 4b2705ae67459ecd85eac3d1c8112c837af7139d Mon Sep 17 00:00:00 2001 From: terasihma Date: Thu, 11 Jan 2024 09:02:04 +0000 Subject: [PATCH] Refactor tests Signed-off-by: terasihma --- v2/Makefile | 4 +- v2/e2e/manifests/nat-client-sport-auto.yaml | 1 + v2/pkg/nodenet/pod.go | 2 +- v2/pkg/nodenet/pod_test.go | 379 ++++++++++++-------- 4 files changed, 224 insertions(+), 162 deletions(-) diff --git a/v2/Makefile b/v2/Makefile index 6bacdb54..cb0ef4d3 100644 --- a/v2/Makefile +++ b/v2/Makefile @@ -21,7 +21,7 @@ PROTOC_OUTPUTS = pkg/cnirpc/cni.pb.go pkg/cnirpc/cni_grpc.pb.go ../docs/cni-grpc GOOS := $(shell go env GOOS) GOARCH := $(shell go env GOARCH) PROTOC := PATH=$(PWD)/bin:'$(PATH)' $(PWD)/bin/protoc -I=$(PWD)/include:. -PODNSLIST = pod1 pod2 pod3 +PODNSLIST = pod1 pod2 pod3 pod4 pod5 pod6 NATNSLIST = nat-client nat-router nat-egress nat-target OTHERNSLIST = test-egress-dual test-egress-v4 test-egress-v6 \ test-client-dual test-client-v4 test-client-v6 test-client-custom \ @@ -51,7 +51,7 @@ test-nodenet: for i in $@ $(PODNSLIST); do $(SUDO) ip netns add $$i; done for i in $@ $(PODNSLIST); do $(SUDO) ip netns exec $$i ip link set lo up; done $(SUDO) ip netns exec $@ ./nodenet.test -test.v - for i in $@ $(PODNSLIST); do $(SUDO) ip netns delete $$i; done + # for i in $@ $(PODNSLIST); do $(SUDO) ip netns delete $$i; done rm -f nodenet.test .PHONY: test-founat diff --git a/v2/e2e/manifests/nat-client-sport-auto.yaml b/v2/e2e/manifests/nat-client-sport-auto.yaml index ce70be86..ba1be34a 100644 --- a/v2/e2e/manifests/nat-client-sport-auto.yaml +++ b/v2/e2e/manifests/nat-client-sport-auto.yaml @@ -11,6 +11,7 @@ spec: operator: Exists nodeSelector: test: coil + kubernetes.io/hostname: coil-worker2 containers: - name: ubuntu image: quay.io/cybozu/ubuntu-debug:22.04 diff --git a/v2/pkg/nodenet/pod.go b/v2/pkg/nodenet/pod.go index adc5ef44..4e4551d3 100644 --- a/v2/pkg/nodenet/pod.go +++ b/v2/pkg/nodenet/pod.go @@ -423,7 +423,7 @@ func (pn *podNetwork) Update(podIPv4, podIPv6 net.IP, hook SetupHook) error { var netNsPath string for _, c := range podConfigs { // When both c.IPvX and podIPvX are nil, net.IP.Equal() returns always true. - // Avoiding comparing nil to nil, confirm c.IPvX is not nil. + // To avoid comparing nil to nil, confirm c.IPvX is not nil. if (c.IPv4 != nil && c.IPv4.Equal(podIPv4)) || (c.IPv6 != nil && c.IPv6.Equal(podIPv6)) { netNsPath, err = getNetNsPath(c.HostVethName) if err != nil { diff --git a/v2/pkg/nodenet/pod_test.go b/v2/pkg/nodenet/pod_test.go index 42716df7..55d748a1 100644 --- a/v2/pkg/nodenet/pod_test.go +++ b/v2/pkg/nodenet/pod_test.go @@ -16,6 +16,47 @@ func nsPath(pod string) string { return filepath.Join("/run/netns", pod) } +var podConfMap = map[string]PodNetConf{ + "pod1": { + PoolName: "default", + ContainerId: "c8f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a", + IFace: "eth0", + IPv4: net.ParseIP("10.1.2.3"), + IPv6: net.ParseIP("fd02::1"), + }, + "pod2": { + PoolName: "default", + ContainerId: "d8f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a", + IFace: "eth0", + IPv4: net.ParseIP("10.1.2.4"), + }, + "pod3": { + PoolName: "default", + ContainerId: "00f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a", + IFace: "eth0", + IPv6: net.ParseIP("fd02::3"), + }, + "pod4": { + PoolName: "default", + ContainerId: "3290748f91c8044b6c9b754e5eaa8f3190a2d2915ee371e1dc866b78aa4764ae", + IFace: "eth0", + IPv4: net.ParseIP("10.1.2.5"), + IPv6: net.ParseIP("fd02::4"), + }, + "pod5": { + PoolName: "default", + ContainerId: "368df12902d559b568aab1d4642943c2c5322bdd17457cdec8081a988e1a2ddf", + IFace: "eth0", + IPv4: net.ParseIP("10.1.2.6"), + }, + "pod6": { + PoolName: "default", + ContainerId: "c80bcafb191e73ba0c269609ac6992e362ff3f042f66dfb82894b577673586f4", + IFace: "eth0", + IPv6: net.ParseIP("fd02::5"), + }, +} + func TestPodNetwork(t *testing.T) { if os.Getuid() != 0 { t.Skip("run as root") @@ -27,59 +68,71 @@ func TestPodNetwork(t *testing.T) { t.Fatal(err) } - podConf1 := &PodNetConf{ - PoolName: "default", - ContainerId: "c8f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a", - IFace: "eth0", - IPv4: net.ParseIP("10.1.2.3"), - IPv6: net.ParseIP("fd02::1"), - } var givenIPv4, givenIPv6 net.IP - result, err := pn.Setup(nsPath("pod1"), "pod1", "ns1", podConf1, func(ipv4, ipv6 net.IP) error { - givenIPv4 = ipv4 - givenIPv6 = ipv6 - return nil - }) - if err != nil { - t.Fatal(err) - } - if !givenIPv4.Equal(net.ParseIP("10.1.2.3")) { - t.Error("hook could not catch IPv4", givenIPv4) - } - if !givenIPv6.Equal(net.ParseIP("fd02::1")) { - t.Error("hook could not catch IPv6", givenIPv6) - } + for name, conf := range podConfMap { - if len(result.Interfaces) != 2 { - t.Error(`len(result.Interfaces) != 2`) - } else { - cIface := result.Interfaces[0] - if cIface.Name != "eth0" { - t.Error(`cIface.Name != "eth0"`) + result, err := pn.Setup(nsPath(name), name, "ns1", &conf, func(ipv4, ipv6 net.IP) error { + givenIPv4 = ipv4 + givenIPv6 = ipv6 + return nil + }) + if err != nil { + t.Fatal(err) } - if cIface.Sandbox != nsPath("pod1") { - t.Error(`cIface.Sandbox != nsPath("pod1")`) + if !givenIPv4.Equal(conf.IPv4) { + t.Error("hook could not catch IPv4", givenIPv4) } - } - if len(result.IPs) != 2 { - t.Error(`len(result.IPs) != 2`) - } else { - if !result.IPs[0].Address.IP.Equal(net.ParseIP("10.1.2.3")) { - t.Error(`!result.IPs[0].Address.IP.Equal(net.ParseIP("10.1.2.3"))`) + if !givenIPv6.Equal(conf.IPv6) { + t.Error("hook could not catch IPv6", givenIPv6) } - if result.IPs[0].Address.IP.To4() == nil { - t.Error(`!result.IPs[0] version != "4"`) + + cIface := result.Interfaces[0] + if cIface.Name != conf.IFace { + t.Errorf(`cIface.Name != "%s"`, conf.IFace) } - if !result.IPs[1].Address.IP.Equal(net.ParseIP("fd02::1")) { - t.Error(`!result.IPs[1].Address.IP.Equal(net.ParseIP("fd02::1"))`) + if cIface.Sandbox != nsPath(name) { + t.Errorf(`cIface.Sandbox != nsPath("%s")`, name) } - if result.IPs[1].Address.IP.To4() != nil { - t.Error(`!result.IPs[1] version != "6"`) + if isDualStack(&conf) { + if len(result.Interfaces) != 2 { + t.Error(`len(result.Interfaces) != 2`) + } + if !result.IPs[0].Address.IP.Equal(conf.IPv4) { + t.Errorf(`!result.IPs[0].Address.IP.Equal("%s")`, conf.IPv4) + } + if result.IPs[0].Address.IP.To4() == nil { + t.Error(`!result.IPs[0] version != "4"`) + } + if !result.IPs[1].Address.IP.Equal(conf.IPv6) { + t.Errorf(`!result.IPs[1].Address.IP.Equal("%s")`, conf.IPv6) + } + if result.IPs[1].Address.IP.To4() != nil { + t.Error(`!result.IPs[1] version != "6"`) + } + } else { + if len(result.Interfaces) != 2 { + t.Error(`len(result.Interfaces) != 2`) + } + if conf.IPv4 != nil { + if !result.IPs[0].Address.IP.Equal(conf.IPv4) { + t.Errorf(`!result.IPs[0].Address.IP.Equal("%s")`, conf.IPv4) + } + if result.IPs[0].Address.IP.To4() == nil { + t.Error(`!result.IPs[0] version != "4"`) + } + } else { + if !result.IPs[0].Address.IP.Equal(conf.IPv6) { + t.Errorf(`!result.IPs[0].Address.IP.Equal("%s")`, conf.IPv6) + } + if result.IPs[0].Address.IP.To4() != nil { + t.Error(`!result.IPs[1] version != "6"`) + } + } + } + if result.CNIVersion != "1.0.0" { + t.Error(`CNI version != 1.0.0`) } - } - if result.CNIVersion != "1.0.0" { - t.Error(`CNI version != 1.0.0`) } // run a test HTTP server @@ -91,7 +144,7 @@ func TestPodNetwork(t *testing.T) { serv.ListenAndServe() }() - err = exec.Command("ip", "link", "add", "foo", "type", "dummy").Run() + err := exec.Command("ip", "link", "add", "foo", "type", "dummy").Run() if err != nil { t.Fatal(err) } @@ -122,7 +175,32 @@ func TestPodNetwork(t *testing.T) { t.Error("curl to host over IPv6 failed") } - err = pn.Update(podConf1.IPv4, podConf1.IPv6, func(ipv4, ipv6 net.IP) error { + // test routing between pod2 over IPv4 + err = exec.Command("ip", "netns", "exec", "pod2", "ping", "-c", "3", "-i", "0.2", "10.1.2.3").Run() + if err != nil { + t.Error("ping to pod1 over IPv4 failed") + } + + // test routing between pod3 over IPv6 + err = exec.Command("ip", "netns", "exec", "pod3", "ping", "-c", "3", "-i", "0.2", "fd02::1").Run() + if err != nil { + t.Error("ping to pod1 over IPv6 failed") + } + + // test routing between pod5 over IPv4 + err = exec.Command("ip", "netns", "exec", "pod5", "ping", "-c", "3", "-i", "0.2", "10.1.2.3").Run() + if err != nil { + t.Error("ping to pod1 over IPv4 failed") + } + + // test routing between pod6 over IPv6 + err = exec.Command("ip", "netns", "exec", "pod6", "ping", "-c", "3", "-i", "0.2", "fd02::1").Run() + if err != nil { + t.Error("ping to pod1 over IPv6 failed") + } + + // update pod1 + err = pn.Update(podConfMap["pod1"].IPv4, podConfMap["pod1"].IPv6, func(ipv4, ipv6 net.IP) error { givenIPv4 = ipv4 givenIPv6 = ipv6 return nil @@ -137,7 +215,7 @@ func TestPodNetwork(t *testing.T) { t.Error("hook could not catch IPv6", givenIPv6) } - err = pn.Check(podConf1.ContainerId, podConf1.IFace) + err = pn.Check(podConfMap["pod1"].ContainerId, podConfMap["pod1"].IFace) if err != nil { t.Error(err) } @@ -146,52 +224,34 @@ func TestPodNetwork(t *testing.T) { if err != nil { t.Fatal(err) } - if len(confs) != 1 { - t.Fatal(`len(confs) != 1`) - } - if confs[0].ContainerId != podConf1.ContainerId { - t.Error(`confs[0].ContainerId != podConf1.ContainerId`) - } - if confs[0].IFace != podConf1.IFace { - t.Error(`confs[0].IFace != podConf1.IFace`) - } - if confs[0].PoolName != podConf1.PoolName { - t.Error(`confs[0].PoolName != podConf1.PoolName`) - } - if !confs[0].IPv4.Equal(podConf1.IPv4) { - t.Error(`!confs[0].IPv4.Equal(podConf1.IPv4)`) - } - if !confs[0].IPv6.Equal(podConf1.IPv6) { - t.Error(`!confs[0].IPv6.Equal(podConf1.IPv6)`, confs[0].IPv6) - } - - // create IPv4 only pod - - podConf2 := &PodNetConf{ - PoolName: "default", - ContainerId: "d8f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a", - IFace: "eth0", - IPv4: net.ParseIP("10.1.2.4"), - } - result, err = pn.Setup(nsPath("pod2"), "pod2", "ns1", podConf2, nil) - if err != nil { - t.Fatal(err) - } - - if len(result.IPs) != 1 { - t.Fatal(`len(result.IPs) != 1`) - } - if !result.IPs[0].Address.IP.Equal(net.ParseIP("10.1.2.4")) { - t.Error(`!result.IPs[0].Address.IP.Equal(net.ParseIP("10.1.2.4"))`) + found := false + for i, conf := range confs { + if conf.ContainerId == podConfMap["pod1"].ContainerId { + found = true + if conf.ContainerId != podConfMap["pod1"].ContainerId { + t.Errorf(`confs[%d].ContainerId != podConf1.ContainerId`, i) + } + if conf.IFace != podConfMap["pod1"].IFace { + t.Errorf(`confs[%d].IFace != podConf1.IFace`, i) + } + if conf.PoolName != podConfMap["pod1"].PoolName { + t.Errorf(`confs[%d].PoolName != podConf1.PoolName`, i) + } + if !conf.IPv4.Equal(podConfMap["pod1"].IPv4) { + t.Errorf(`!confs[%d].IPv4.Equal(podConf1.IPv4)`, i) + } + if !conf.IPv6.Equal(podConfMap["pod1"].IPv6) { + t.Errorf(`!confs[%d].IPv6.Equal(podConf1.IPv6) %s`, i, confs[0].IPv6) + } + break + } } - - // test routing between pods over IPv4 - err = exec.Command("ip", "netns", "exec", "pod2", "ping", "-c", "3", "-i", "0.2", "10.1.2.3").Run() - if err != nil { - t.Error("ping to pod1 over IPv4 failed") + if !found { + t.Error("config for pod1 not found") } - err = pn.Update(podConf2.IPv4, podConf2.IPv6, func(ipv4, ipv6 net.IP) error { + // update pod2 + err = pn.Update(podConfMap["pod2"].IPv4, podConfMap["pod2"].IPv6, func(ipv4, ipv6 net.IP) error { givenIPv4 = ipv4 givenIPv6 = ipv6 return nil @@ -200,7 +260,7 @@ func TestPodNetwork(t *testing.T) { t.Fatal(err) } - err = pn.Check(podConf2.ContainerId, podConf2.IFace) + err = pn.Check(podConfMap["pod2"].ContainerId, podConfMap["pod2"].IFace) if err != nil { t.Error(err) } @@ -209,14 +269,11 @@ func TestPodNetwork(t *testing.T) { if err != nil { t.Fatal(err) } - if len(confs) != 2 { - t.Fatal(`len(confs) != 2`) - } - found := false + found = false for _, c := range confs { if c.ContainerId == "d8f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a" { found = true - if !c.IPv4.Equal(podConf2.IPv4) { + if !c.IPv4.Equal(podConfMap["pod2"].IPv4) { t.Error(`!c.IPv4.Equal(podConf2.IPv4)`) } if c.IPv6 != nil { @@ -228,33 +285,7 @@ func TestPodNetwork(t *testing.T) { t.Error("config for pod2 not found") } - // create IPv6 only pod - - podConf3 := &PodNetConf{ - PoolName: "default", - ContainerId: "00f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a", - IFace: "eth0", - IPv6: net.ParseIP("fd02::3"), - } - result, err = pn.Setup(nsPath("pod3"), "pod3", "ns1", podConf3, nil) - if err != nil { - t.Fatal(err) - } - - if len(result.IPs) != 1 { - t.Fatal(`len(result.IPs) != 1`) - } - if !result.IPs[0].Address.IP.Equal(net.ParseIP("fd02::3")) { - t.Error(`!result.IPs[0].Address.IP.Equal(net.ParseIP("fd02::3"))`) - } - - // test routing between pods over IPv6 - err = exec.Command("ip", "netns", "exec", "pod3", "ping", "-c", "3", "-i", "0.2", "fd02::1").Run() - if err != nil { - t.Error("ping to pod1 over IPv6 failed") - } - - err = pn.Update(podConf3.IPv4, podConf3.IPv6, func(ipv4, ipv6 net.IP) error { + err = pn.Update(podConfMap["pod3"].IPv4, podConfMap["pod3"].IPv6, func(ipv4, ipv6 net.IP) error { givenIPv4 = ipv4 givenIPv6 = ipv6 return nil @@ -263,7 +294,7 @@ func TestPodNetwork(t *testing.T) { t.Fatal(err) } - err = pn.Check(podConf3.ContainerId, podConf3.IFace) + err = pn.Check(podConfMap["pod3"].ContainerId, podConfMap["pod3"].IFace) if err != nil { t.Error(err) } @@ -272,14 +303,11 @@ func TestPodNetwork(t *testing.T) { if err != nil { t.Fatal(err) } - if len(confs) != 3 { - t.Fatal(`len(confs) != 3`) - } found = false for _, c := range confs { if c.ContainerId == "00f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a" { found = true - if !c.IPv6.Equal(podConf3.IPv6) { + if !c.IPv6.Equal(podConfMap["pod3"].IPv6) { t.Error(`!c.IPv6.Equal(podConf2.IPv6)`) } if c.IPv4 != nil { @@ -298,45 +326,67 @@ func TestPodNetwork(t *testing.T) { type addrInfo struct { Family string `json:"family"` Local string `json:"local"` + Scope string `json:"scope"` } type devInfo struct { IfName string `json:"ifname"` AddrInfos []addrInfo `json:"addr_info"` } - info := make([]devInfo, 0, 1) - err = pn.Update(podConf1.IPv4, nil, func(ipv4, ipv6 net.IP) error { - out, err := exec.Command("ip", "-j", "addr", "show", podConf1.IFace).Output() - if err != nil { - return err - } - if err := json.Unmarshal(out, &info); err != nil { - return err + for name, conf := range podConfMap { + info := make([]devInfo, 0, 1) + if conf.IPv4 != nil { + err = pn.Update(conf.IPv4, nil, func(ipv4, ipv6 net.IP) error { + out, err := exec.Command("ip", "-j", "addr", "show", conf.IFace).Output() + if err != nil { + return err + } + if err := json.Unmarshal(out, &info); err != nil { + return err + } + return nil + }) + if err != nil { + t.Error(err) + } + } else { + err = pn.Update(nil, conf.IPv6, func(ipv4, ipv6 net.IP) error { + out, err := exec.Command("ip", "-j", "addr", "show", conf.IFace).Output() + if err != nil { + return err + } + if err := json.Unmarshal(out, &info); err != nil { + return err + } + return nil + }) + if err != nil { + t.Error(err) + } } - return nil - }) - if err != nil { - t.Error(err) - } - - if len(info) != 1 { - t.Fatal("len(info) != 1") - } - if info[0].IfName != podConf1.IFace { - t.Fatalf("expected iface is %s", podConf1.IFace) - } - - for _, addr := range info[0].AddrInfos { - if addr.Family != "inet" { - continue + if len(info) != 1 { + t.Fatalf("len(info) != 1 for %s", name) } - if addr.Local != podConf1.IPv4.String() { - t.Fatal("address don't match") + if info[0].IfName != conf.IFace { + t.Fatalf("expected iface is %s for %s", conf.IFace, name) + } + + for _, addr := range info[0].AddrInfos { + if addr.Family == "inet" && addr.Scope == "global" { + if addr.Local != conf.IPv4.String() { + t.Fatalf("%s's inet address don't match", name) + } + } + if addr.Family == "inet6" && addr.Scope == "global" { + if addr.Local != conf.IPv6.String() { + t.Fatalf("%s's inet6 address don't match", name) + } + } } } // destroy pod2 network - err = pn.Destroy(podConf2.ContainerId, podConf2.IFace) + err = pn.Destroy(podConfMap["pod2"].ContainerId, podConfMap["pod2"].IFace) if err != nil { t.Error(err) } @@ -345,9 +395,6 @@ func TestPodNetwork(t *testing.T) { if err != nil { t.Fatal(err) } - if len(confs) != 2 { - t.Fatal(`len(confs) != 2`) - } for _, c := range confs { if c.ContainerId == "d8f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a" { t.Error(`pod network 2 has not been destroyed`) @@ -355,8 +402,22 @@ func TestPodNetwork(t *testing.T) { } // destroy should be idempotent - err = pn.Destroy(podConf2.ContainerId, podConf2.IFace) + err = pn.Destroy(podConfMap["pod2"].ContainerId, podConfMap["pod2"].IFace) if err != nil { t.Error(err) } + + // check pod2 network + err = pn.Check(podConfMap["pod2"].ContainerId, podConfMap["pod2"].IFace) + if err == nil { + t.Fatal("pn.Check must return error because pod2 network doesn't exist") + } else { + if err != errNotFound { + t.Fatal("given error must be errNotFound") + } + } +} + +func isDualStack(conf *PodNetConf) bool { + return conf.IPv4 != nil && conf.IPv6 != nil }