From 636acf8c4c797d9104b0d99f5b774dc7237b4d7c Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Sat, 26 Sep 2020 15:42:27 +0200 Subject: [PATCH 01/12] Update k8s client --- chaosengine-experiments/chaosengine-kubernetes/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chaosengine-experiments/chaosengine-kubernetes/pom.xml b/chaosengine-experiments/chaosengine-kubernetes/pom.xml index 6a035b124..589051e38 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/pom.xml +++ b/chaosengine-experiments/chaosengine-kubernetes/pom.xml @@ -15,7 +15,7 @@ io.kubernetes client-java - 9.0.2 + 10.0.0 compile From 33e37d186a021bbc5d79e7177e5316d502b6de47 Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Tue, 6 Oct 2020 21:26:27 +0200 Subject: [PATCH 02/12] Update k8s docs --- docs/markdown/Experiment_Modules/kubernetes_experiments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/markdown/Experiment_Modules/kubernetes_experiments.md b/docs/markdown/Experiment_Modules/kubernetes_experiments.md index d1555b4bc..8eada8ca4 100644 --- a/docs/markdown/Experiment_Modules/kubernetes_experiments.md +++ b/docs/markdown/Experiment_Modules/kubernetes_experiments.md @@ -14,7 +14,7 @@ The official Kubernetes Java Client is used to interact with the cluster. | | | | --- | --- | | Resource | | -| Version | 9.0.2 | +| Version | 10.0.0 | | Maven Repositories | | ## Configuration From c794028f3e582ac3662a709d6fb2ed7d319fe846 Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Tue, 6 Oct 2020 22:37:59 +0200 Subject: [PATCH 03/12] Avoid NPE when pod has no owner --- .../com/thales/chaos/platform/impl/KubernetesPlatform.java | 6 +++--- .../thales/chaos/platform/impl/KubernetesPlatformTest.java | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java index 636846f28..1a1e68b34 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java @@ -59,7 +59,7 @@ public class KubernetesPlatform extends Platform implements ShellBasedExperiment private ContainerManager containerManager; @Autowired - private ApiClient apiClient; + private final ApiClient apiClient; private String namespace = "default"; @Autowired @@ -195,11 +195,11 @@ KubernetesPodContainer fromKubernetesAPIPod (V1Pod pod) { .withKubernetesPlatform(this) .isBackedByController(CollectionUtils.isNotEmpty(pod.getMetadata() .getOwnerReferences())) - .withOwnerKind(Optional.of(pod.getMetadata().getOwnerReferences()) + .withOwnerKind(Optional.ofNullable(pod.getMetadata().getOwnerReferences()) .flatMap(list -> list.stream().findFirst()) .map(V1OwnerReference::getKind) .orElse("")) - .withOwnerName(Optional.of(pod.getMetadata().getOwnerReferences()) + .withOwnerName(Optional.ofNullable(pod.getMetadata().getOwnerReferences()) .flatMap(list -> list.stream().findFirst()) .map(V1OwnerReference::getName) .orElse("")) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java index 218889ca4..add9824a4 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java @@ -506,8 +506,9 @@ public void testContainerHealthWithSeveralContainerOneUnhealthy () throws ApiExc } private static V1PodList getV1PodList (boolean isBackedByController, int numberOfPods) { - List ownerReferences = new ArrayList<>(); + List ownerReferences = null; if (isBackedByController) { + ownerReferences = new ArrayList<>(); ownerReferences.add(new V1OwnerReferenceBuilder().withNewController("mycontroller").build()); } V1ObjectMeta metadata = new V1ObjectMetaBuilder().withUid(randomUUID().toString()) @@ -966,7 +967,7 @@ KubernetesPlatform kubernetesPlatform () { } private class TestProcess extends Process { - private InputStream is; + private final InputStream is; public TestProcess (InputStream is) { this.is = is; From 86ffd1acd8f0d7d2ddec4e35d33808834caddea8 Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Thu, 8 Oct 2020 22:54:44 +0200 Subject: [PATCH 04/12] Allow experiments on multiple namespaces --- .../platform/impl/KubernetesPlatform.java | 89 ++++++++++--------- 1 file changed, 48 insertions(+), 41 deletions(-) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java index 1a1e68b34..800914724 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java @@ -57,10 +57,18 @@ public class KubernetesPlatform extends Platform implements ShellBasedExperiment { @Autowired private ContainerManager containerManager; - @Autowired private final ApiClient apiClient; - private String namespace = "default"; + private final String DEFAULT_NAMESPACE = "default"; + private Collection namespaces = List.of(DEFAULT_NAMESPACE); + + public void setNamespaces (String namespaces) { + this.namespaces = Optional.of(Arrays.stream(namespaces.split(",")) + .filter(Objects::nonNull) + .collect(Collectors.toList())) + .filter(l -> !l.isEmpty()) + .orElse(List.of(DEFAULT_NAMESPACE)); + } @Autowired KubernetesPlatform (ApiClient apiClient) { @@ -68,20 +76,10 @@ public class KubernetesPlatform extends Platform implements ShellBasedExperiment log.info("Kubernetes Platform created"); } - public String getNamespace () { - return namespace; - } - - public void setNamespace (String namespace) { - this.namespace = namespace; - } - public ContainerHealth checkHealth (KubernetesPodContainer kubernetesPodContainer) { try { - Optional podExists = podExists(kubernetesPodContainer); - if (podExists.isEmpty()) { - return ContainerHealth.RUNNING_EXPERIMENT; - } else if (!podExists.get()) { + boolean podExists = podExists(kubernetesPodContainer); + if (!podExists) { return ContainerHealth.DOES_NOT_EXIST; } V1Pod result = getCoreV1Api().readNamespacedPodStatus(kubernetesPodContainer.getPodName(), @@ -103,22 +101,19 @@ public ContainerHealth checkHealth (KubernetesPodContainer kubernetesPodContaine return ContainerHealth.RUNNING_EXPERIMENT; } - Optional podExists (KubernetesPodContainer kubernetesPodContainer) { + boolean podExists (KubernetesPodContainer kubernetesPodContainer) { String podUuid = kubernetesPodContainer.getUuid(); - if (podUuid == null) return Optional.empty(); - try { - Boolean podExists = listAllPodsInNamespace().getItems() - .stream() - .map(V1Pod::getMetadata) - .filter(Objects::nonNull) - .map(V1ObjectMeta::getUid) - .anyMatch(podUuid::equals); - log.debug("Kubernetes POD {} exists = {}", kubernetesPodContainer.getPodName(), podExists); - return Optional.of(podExists); - } catch (ApiException e) { - log.debug("Exception when checking container existence", e); - return Optional.empty(); + if (podUuid == null) { + return false; } + boolean podExists = listAllPodsInNamespace(kubernetesPodContainer.getNamespace()).getItems() + .stream() + .map(V1Pod::getMetadata) + .filter(Objects::nonNull) + .map(V1ObjectMeta::getUid) + .anyMatch(podUuid::equals); + log.debug("Kubernetes POD {} exists = {}", kubernetesPodContainer.getPodName(), podExists); + return podExists; } @JsonIgnore @@ -126,8 +121,13 @@ CoreV1Api getCoreV1Api () { return new CoreV1Api(apiClient); } - private V1PodList listAllPodsInNamespace () throws ApiException { - return getCoreV1Api().listNamespacedPod(namespace, "true", false, "", "", "", 0, "", 0, false); + private V1PodList listAllPodsInNamespace (String namespace) { + try { + return getCoreV1Api().listNamespacedPod(namespace, "true", false, "", "", "", 0, "", 0, false); + } catch (ApiException e) { + log.error("Cannot list pods in namespace {}: {} ", namespace, e.getMessage(), e); + return new V1PodList(); + } } @Override @@ -153,23 +153,30 @@ public PlatformLevel getPlatformLevel () { @Override public PlatformHealth getPlatformHealth () { - try { - V1PodList pods = listAllPodsInNamespace(); - return (!pods.getItems().isEmpty()) ? PlatformHealth.OK : PlatformHealth.DEGRADED; - } catch (ApiException e) { - log.error("Kubernetes Platform health check failed", e); - return PlatformHealth.FAILED; + if (namespaces.stream() + .map(this::listAllPodsInNamespace) + .map(V1PodList::getItems) + .flatMap(List::stream) + .collect(Collectors.toList()) + .isEmpty()) { + log.warn("No PODs detected in specified namespaces {}", namespaces); + return PlatformHealth.DEGRADED; } + return PlatformHealth.OK; } @Override protected List generateRoster () { final List containerList = new ArrayList<>(); try { - V1PodList pods = listAllPodsInNamespace(); - containerList.addAll(pods.getItems().stream().map(this::fromKubernetesAPIPod).collect(Collectors.toSet())); + List pods = namespaces.stream() + .map(this::listAllPodsInNamespace) + .map(V1PodList::getItems) + .flatMap(List::stream) + .collect(Collectors.toList()); + containerList.addAll(pods.stream().map(this::fromKubernetesAPIPod).collect(Collectors.toSet())); return containerList; - } catch (ApiException e) { + } catch (Exception e) { log.error("Could not generate Kubernetes roster", e); return containerList; } @@ -178,7 +185,7 @@ protected List generateRoster () { @Override public boolean isContainerRecycled (Container container) { KubernetesPodContainer kubernetesPodContainer = (KubernetesPodContainer) container; - if (podExists(kubernetesPodContainer).orElse(false)) return isContainerRestarted(kubernetesPodContainer, + if (podExists(kubernetesPodContainer)) return isContainerRestarted(kubernetesPodContainer, ((KubernetesPodContainer) container).getTargetedSubcontainer()); return isDesiredReplicas(kubernetesPodContainer); } @@ -247,7 +254,7 @@ private boolean isContainerRestarted (KubernetesPodContainer container, String s } public ContainerHealth replicaSetRecovered (KubernetesPodContainer kubernetesPodContainer) { - return isDesiredReplicas(kubernetesPodContainer) && !podExists(kubernetesPodContainer).orElse(false) ? ContainerHealth.NORMAL : ContainerHealth.RUNNING_EXPERIMENT; + return isDesiredReplicas(kubernetesPodContainer) && !podExists(kubernetesPodContainer) ? ContainerHealth.NORMAL : ContainerHealth.RUNNING_EXPERIMENT; } /** From db2daa1ac8c9bdc498385a471a5d27703c7d33a3 Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Thu, 8 Oct 2020 22:55:21 +0200 Subject: [PATCH 05/12] Update existing unit tests --- .../platform/impl/KubernetesPlatformTest.java | 150 ++++++++---------- 1 file changed, 62 insertions(+), 88 deletions(-) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java index add9824a4..f0c3f237e 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java @@ -79,22 +79,29 @@ public class KubernetesPlatformTest { @Mock private AppsV1Api appsV1Api; - @Before - public void setUp () { - doReturn(coreApi).when(platform).getCoreApi(); - doReturn(coreV1Api).when(platform).getCoreV1Api(); - doReturn(appsV1Api).when(platform).getAppsV1Api(); - platform.setNamespace(NAMESPACE_NAME); + private static V1PodList getV1PodList (boolean isBackedByController, int numberOfPods) { + List ownerReferences = null; + if (isBackedByController) { + ownerReferences = new ArrayList<>(); + ownerReferences.add(new V1OwnerReferenceBuilder().withNewController("mycontroller").build()); + } + V1ObjectMeta metadata = new V1ObjectMetaBuilder().withUid(randomUUID().toString()) + .withName(POD_NAME) + .withNamespace(NAMESPACE_NAME) + .withLabels(new HashMap<>()) + .withOwnerReferences(ownerReferences) + .build(); + V1Pod pod = new V1Pod(); + pod.setMetadata(metadata); + pod.setSpec(new V1PodSpec().containers(Collections.singletonList(new V1Container().name(randomUUID().toString())))); + V1PodList list = new V1PodList(); + for (int i = 0; i < numberOfPods; i++) list.addItemsItem(pod); + return list; } @Test public void testPodWithoutOwnerCannotBeTested () throws Exception { - when(coreV1Api.listNamespacedPod(anyString(), - anyString(), - anyBoolean(), - anyString(), - anyString(), - anyString(), + when(coreV1Api.listNamespacedPod(anyString(), anyString(), anyBoolean(), anyString(), anyString(), anyString(), anyInt(), anyString(), anyInt(), @@ -139,31 +146,14 @@ public void testContainerHealthWithException () throws ApiException { anyString(), anyInt(), anyBoolean())).thenThrow(new ApiException()); - assertEquals("Error while checking container presence", - ContainerHealth.RUNNING_EXPERIMENT, - platform.checkHealth((KubernetesPodContainer) platform.getRoster().get(0))); } - @Test - public void testPodExists () throws ApiException { - V1PodList v1PodList = getV1PodList(true); - V1Pod pod = v1PodList.getItems().get(0); - KubernetesPodContainer kubernetesPodContainer = platform.fromKubernetesAPIPod(pod); - when(coreV1Api.listNamespacedPod(anyString(), - anyString(), - anyBoolean(), - anyString(), - anyString(), - anyString(), - anyInt(), - anyString(), - anyInt(), - anyBoolean())).thenReturn(v1PodList) - .thenReturn(new V1PodList()) - .thenThrow(new ApiException(new IOException())); - assertTrue("POD exists", platform.podExists(kubernetesPodContainer).orElseThrow()); - assertFalse("POD does not exist", platform.podExists(kubernetesPodContainer).orElseThrow()); - assertTrue("IO Exception", platform.podExists(kubernetesPodContainer).isEmpty()); + @Before + public void setUp () { + doReturn(coreApi).when(platform).getCoreApi(); + doReturn(coreV1Api).when(platform).getCoreV1Api(); + doReturn(appsV1Api).when(platform).getAppsV1Api(); + platform.setNamespaces(NAMESPACE_NAME); } @Test @@ -365,34 +355,10 @@ public void testCheckDesiredReplicasReplicationController () throws ApiException } @Test - public void testPlatformHealthNotAvailable () throws ApiException { - when(coreV1Api.listNamespacedPod(anyString(), - anyString(), - anyBoolean(), - anyString(), - anyString(), - anyString(), - anyInt(), - anyString(), - anyInt(), - anyBoolean())).thenThrow(new ApiException()); - assertEquals(PlatformHealth.FAILED, platform.getPlatformHealth()); - } - - @Test - public void testContainerHealthDoesNotExist () throws ApiException { - V1PodList list = new V1PodList(); - V1Pod pod = mock(V1Pod.class); - V1ObjectMeta metadata = mock(V1ObjectMeta.class); - V1PodSpec spec = mock(V1PodSpec.class); - when(metadata.getUid()).thenReturn(randomUUID().toString()); - when(pod.getMetadata()).thenReturn(metadata); - when(pod.getSpec()).thenReturn(spec); - list.addItemsItem(pod); - KubernetesPodContainer kubernetesPodContainer = KubernetesPodContainer.builder() - .withUUID(randomUUID().toString()) - .withOwnerKind("") - .build(); + public void testPodExists () throws ApiException { + V1PodList v1PodList = getV1PodList(true); + V1Pod pod = v1PodList.getItems().get(0); + KubernetesPodContainer kubernetesPodContainer = platform.fromKubernetesAPIPod(pod); when(coreV1Api.listNamespacedPod(anyString(), anyString(), anyBoolean(), @@ -402,8 +368,12 @@ public void testContainerHealthDoesNotExist () throws ApiException { anyInt(), anyString(), anyInt(), - anyBoolean())).thenReturn(list); - assertEquals(ContainerHealth.DOES_NOT_EXIST, platform.checkHealth(kubernetesPodContainer)); + anyBoolean())).thenReturn(v1PodList) + .thenReturn(new V1PodList()) + .thenThrow(new ApiException(new IOException())); + assertTrue("POD exists", platform.podExists(kubernetesPodContainer)); + assertFalse("POD does not exist", platform.podExists(kubernetesPodContainer)); + assertFalse("IO Exception", platform.podExists(kubernetesPodContainer)); } @Test @@ -505,29 +475,32 @@ public void testContainerHealthWithSeveralContainerOneUnhealthy () throws ApiExc platform.checkHealth((KubernetesPodContainer) platform.getRoster().get(0))); } - private static V1PodList getV1PodList (boolean isBackedByController, int numberOfPods) { - List ownerReferences = null; - if (isBackedByController) { - ownerReferences = new ArrayList<>(); - ownerReferences.add(new V1OwnerReferenceBuilder().withNewController("mycontroller").build()); - } - V1ObjectMeta metadata = new V1ObjectMetaBuilder().withUid(randomUUID().toString()) - .withName(POD_NAME) - .withNamespace(NAMESPACE_NAME) - .withLabels(new HashMap<>()) - .withOwnerReferences(ownerReferences) - .build(); - V1Pod pod = new V1Pod(); - pod.setMetadata(metadata); - pod.setSpec(new V1PodSpec().containers(Collections.singletonList(new V1Container().name(randomUUID().toString())))); - V1PodList list = new V1PodList(); - for (int i = 0; i < numberOfPods; i++) list.addItemsItem(pod); - return list; - } - @Test - public void testGetNamespace () { - assertEquals("mynamespace", platform.getNamespace()); + public void testContainerHealthDoesNotExist () throws ApiException { + V1PodList list = new V1PodList(); + V1Pod pod = mock(V1Pod.class); + V1ObjectMeta metadata = mock(V1ObjectMeta.class); + V1PodSpec spec = mock(V1PodSpec.class); + when(metadata.getUid()).thenReturn(randomUUID().toString()); + when(pod.getMetadata()).thenReturn(metadata); + when(pod.getSpec()).thenReturn(spec); + list.addItemsItem(pod); + KubernetesPodContainer kubernetesPodContainer = KubernetesPodContainer.builder() + .withNamespace(NAMESPACE_NAME) + .withUUID(randomUUID().toString()) + .withOwnerKind("") + .build(); + when(coreV1Api.listNamespacedPod(anyString(), + anyString(), + anyBoolean(), + anyString(), + anyString(), + anyString(), + anyInt(), + anyString(), + anyInt(), + anyBoolean())).thenReturn(list); + assertEquals(ContainerHealth.DOES_NOT_EXIST, platform.checkHealth(kubernetesPodContainer)); } private static V1PodList getV1PodList (boolean isBackedByController) { @@ -848,6 +821,7 @@ public void testIsContainerRecycled () throws ApiException { String uid = randomUUID().toString(); String containerName = randomUUID().toString(); KubernetesPodContainer kubernetesPodContainer = spy(KubernetesPodContainer.builder() + .withNamespace(NAMESPACE_NAME) .withUUID(uid) .withSubcontainers(List.of( containerName)) @@ -923,7 +897,7 @@ public void testIsContainerRecycledAPIError () throws ApiException { .withOwnerKind(REPLICA_SET.toString()) .withSubcontainers(Set.of(subContainer)) .build(); - doReturn(Optional.of(Boolean.TRUE)).when(platform).podExists(kubernetesPodContainer); + doReturn(true).when(platform).podExists(kubernetesPodContainer); doThrow(new ApiException()).when(coreV1Api).readNamespacedPodStatus(any(), any(), any()); platform.isContainerRecycled(kubernetesPodContainer); } From 52b3a343345f01b591a47e02fba71889a91db133 Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Mon, 12 Oct 2020 13:27:26 +0200 Subject: [PATCH 06/12] Add test --- .../chaos/platform/impl/KubernetesPlatform.java | 8 +++++++- .../chaos/platform/impl/KubernetesPlatformTest.java | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java index 800914724..520e4d8f1 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java @@ -49,6 +49,7 @@ import static com.thales.chaos.constants.DataDogConstants.DATADOG_CONTAINER_KEY; import static com.thales.chaos.exception.enums.KubernetesChaosErrorCode.K8S_API_ERROR; +import static java.util.function.Predicate.not; import static net.logstash.logback.argument.StructuredArguments.v; @Component @@ -59,12 +60,17 @@ public class KubernetesPlatform extends Platform implements ShellBasedExperiment private ContainerManager containerManager; @Autowired private final ApiClient apiClient; - private final String DEFAULT_NAMESPACE = "default"; + static final String DEFAULT_NAMESPACE = "default"; private Collection namespaces = List.of(DEFAULT_NAMESPACE); + public Collection getNamespaces () { + return namespaces; + } + public void setNamespaces (String namespaces) { this.namespaces = Optional.of(Arrays.stream(namespaces.split(",")) .filter(Objects::nonNull) + .filter(not(String::isEmpty)) .collect(Collectors.toList())) .filter(l -> !l.isEmpty()) .orElse(List.of(DEFAULT_NAMESPACE)); diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java index f0c3f237e..1b4730854 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java @@ -34,6 +34,7 @@ import io.kubernetes.client.openapi.models.*; import junit.framework.TestCase; import org.apache.http.HttpStatus; +import org.hamcrest.collection.IsIterableContainingInAnyOrder; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Before; @@ -923,7 +924,18 @@ public void testRecycleContainer () throws ApiException { public void testGetConnectedShellClient () throws IOException, ApiException { KubernetesPodContainer kubernetesPodContainer = mock(KubernetesPodContainer.class); platform.getConnectedShellClient(kubernetesPodContainer); + } + @Test + public void testSetNamespaces () { + assertThat(platform.getNamespaces(), + IsIterableContainingInAnyOrder.containsInAnyOrder(KubernetesPlatform.DEFAULT_NAMESPACE)); + platform.setNamespaces(""); + assertThat(platform.getNamespaces(), + IsIterableContainingInAnyOrder.containsInAnyOrder(KubernetesPlatform.DEFAULT_NAMESPACE)); + platform.setNamespaces("default,application"); + assertThat(platform.getNamespaces(), + IsIterableContainingInAnyOrder.containsInAnyOrder("default", "application")); } @Configuration From 1e7ec9b35ecd07b0fde7caad34ef971bf70f454b Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Mon, 12 Oct 2020 13:45:40 +0200 Subject: [PATCH 07/12] Remove redundant assertion --- .../com/thales/chaos/platform/impl/KubernetesPlatformTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java index 1b4730854..59c4b8357 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java @@ -928,8 +928,6 @@ public void testGetConnectedShellClient () throws IOException, ApiException { @Test public void testSetNamespaces () { - assertThat(platform.getNamespaces(), - IsIterableContainingInAnyOrder.containsInAnyOrder(KubernetesPlatform.DEFAULT_NAMESPACE)); platform.setNamespaces(""); assertThat(platform.getNamespaces(), IsIterableContainingInAnyOrder.containsInAnyOrder(KubernetesPlatform.DEFAULT_NAMESPACE)); From 8ae38b22b84a86da1263f17b15fbe5f78579d0e6 Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Mon, 12 Oct 2020 17:34:16 +0200 Subject: [PATCH 08/12] Add parent worker node information into k8s container metadata --- .../container/impl/KubernetesPodContainer.java | 16 ++++++++++++++-- .../chaos/platform/impl/KubernetesPlatform.java | 3 +++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/container/impl/KubernetesPodContainer.java b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/container/impl/KubernetesPodContainer.java index ee9f08901..9bc900036 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/container/impl/KubernetesPodContainer.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/container/impl/KubernetesPodContainer.java @@ -43,7 +43,7 @@ public class KubernetesPodContainer extends Container { private String podName; @Identifier(order = 2) private String namespace; - private Map labels = new HashMap<>(); + private final Map labels = new HashMap<>(); private boolean isBackedByController = false; private KubernetesPlatform kubernetesPlatform; @Identifier(order = 3) @@ -52,7 +52,8 @@ public class KubernetesPodContainer extends Container { private String ownerName; private Collection subcontainers = new HashSet<>(); private String targetedSubcontainer; - private Callable replicaSetRecovered = () -> kubernetesPlatform.replicaSetRecovered(this); + private final Callable replicaSetRecovered = () -> kubernetesPlatform.replicaSetRecovered(this); + private String parentNode; private KubernetesPodContainer () { super(); @@ -82,6 +83,10 @@ public String getOwnerName () { return ownerName; } + public String getParentNode () { + return parentNode; + } + @Override public void startExperiment (Experiment experiment) { this.targetedSubcontainer = null; @@ -153,6 +158,7 @@ public static final class KubernetesPodContainerBuilder { private String ownerKind; private String ownerName; private Collection subcontainers; + private String parentNode; private KubernetesPodContainerBuilder () { } @@ -211,6 +217,11 @@ public KubernetesPodContainerBuilder withUUID (String uuid) { return this; } + public KubernetesPodContainerBuilder withParentNode (String parentNode) { + this.parentNode = parentNode; + return this; + } + public KubernetesPodContainer build () { KubernetesPodContainer kubernetesPodContainer = new KubernetesPodContainer(); kubernetesPodContainer.uuid = this.uuid; @@ -223,6 +234,7 @@ public KubernetesPodContainer build () { kubernetesPodContainer.ownerKind = ControllerKind.mapFromString(this.ownerKind); kubernetesPodContainer.ownerName = ownerName; kubernetesPodContainer.subcontainers = this.subcontainers; + kubernetesPodContainer.parentNode = this.parentNode; try { kubernetesPodContainer.setMappedDiagnosticContext(); kubernetesPodContainer.log.info("Created new Kubernetes Pod Container object"); diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java index 520e4d8f1..564437900 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java @@ -61,6 +61,7 @@ public class KubernetesPlatform extends Platform implements ShellBasedExperiment @Autowired private final ApiClient apiClient; static final String DEFAULT_NAMESPACE = "default"; + static final String UNKNOWN_PARENT_NODE = "UNKNOWN"; private Collection namespaces = List.of(DEFAULT_NAMESPACE); public Collection getNamespaces () { @@ -223,6 +224,8 @@ KubernetesPodContainer fromKubernetesAPIPod (V1Pod pod) { .flatMap(Collection::stream) .map(V1Container::getName) .collect(Collectors.toList())) + .withParentNode(Optional.ofNullable(pod.getSpec().getNodeName()) + .orElse(UNKNOWN_PARENT_NODE)) .build(); log.info("Found new Kubernetes Pod Container {}", v(DATADOG_CONTAINER_KEY, container)); containerManager.offer(container); From 7e82df63ed7857048bef7238a535f7acefd37e9b Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Mon, 12 Oct 2020 17:46:05 +0200 Subject: [PATCH 09/12] Platform status must be failed when it is not possible to list pods in given namespace --- .../platform/impl/KubernetesPlatform.java | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java index 564437900..d96a68bb6 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java @@ -137,6 +137,23 @@ private V1PodList listAllPodsInNamespace (String namespace) { } } + @Override + public PlatformHealth getPlatformHealth () { + if (namespaces.stream().map(this::canListPodsInNamespace).anyMatch(Boolean -> Boolean.equals(false))) { + return PlatformHealth.FAILED; + } + if (namespaces.stream() + .map(this::listAllPodsInNamespace) + .map(V1PodList::getItems) + .flatMap(List::stream) + .collect(Collectors.toList()) + .isEmpty()) { + log.warn("No PODs detected in specified namespaces {}", namespaces); + return PlatformHealth.DEGRADED; + } + return PlatformHealth.OK; + } + @Override public ApiStatus getApiStatus () { try { @@ -158,18 +175,14 @@ public PlatformLevel getPlatformLevel () { return PlatformLevel.PAAS; } - @Override - public PlatformHealth getPlatformHealth () { - if (namespaces.stream() - .map(this::listAllPodsInNamespace) - .map(V1PodList::getItems) - .flatMap(List::stream) - .collect(Collectors.toList()) - .isEmpty()) { - log.warn("No PODs detected in specified namespaces {}", namespaces); - return PlatformHealth.DEGRADED; + private boolean canListPodsInNamespace (String namespace) { + try { + getCoreV1Api().listNamespacedPod(namespace, "true", false, "", "", "", 0, "", 0, false); + } catch (Exception e) { + log.error("Cannot list pods in namespace {}: {} ", namespace, e.getMessage(), e); + return false; } - return PlatformHealth.OK; + return true; } @Override From 44dd9d3068ae70bc8e020d658d840b7d5f7c7ea4 Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Mon, 12 Oct 2020 17:55:51 +0200 Subject: [PATCH 10/12] Follow naming conventions --- .../java/com/thales/chaos/platform/impl/KubernetesPlatform.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java index d96a68bb6..10fa7b538 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java @@ -139,7 +139,7 @@ private V1PodList listAllPodsInNamespace (String namespace) { @Override public PlatformHealth getPlatformHealth () { - if (namespaces.stream().map(this::canListPodsInNamespace).anyMatch(Boolean -> Boolean.equals(false))) { + if (namespaces.stream().map(this::canListPodsInNamespace).anyMatch(canList -> canList.equals(false))) { return PlatformHealth.FAILED; } if (namespaces.stream() From 5ee588787cbc952f7265beba6d4242d40f9bb05d Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Mon, 12 Oct 2020 22:24:29 +0200 Subject: [PATCH 11/12] Add test --- .../platform/impl/KubernetesPlatformTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java index 59c4b8357..b5e645197 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java @@ -188,6 +188,21 @@ public void testPlatformHealth () throws ApiException { assertEquals(PlatformHealth.OK, platform.getPlatformHealth()); } + @Test + public void testPlatformHealthCannotListPods () throws ApiException { + when(coreV1Api.listNamespacedPod(anyString(), + anyString(), + anyBoolean(), + anyString(), + anyString(), + anyString(), + anyInt(), + anyString(), + anyInt(), + anyBoolean())).thenThrow(new ApiException()); + assertEquals(PlatformHealth.FAILED, platform.getPlatformHealth()); + } + @Test public void testPodWithOwnerCanBeTested () throws Exception { when(coreV1Api.listNamespacedPod(anyString(), From 4cacadc8f7fa585fabd1af2b22f5a7630ff7fd90 Mon Sep 17 00:00:00 2001 From: Lubor PETR Date: Mon, 12 Oct 2020 22:40:51 +0200 Subject: [PATCH 12/12] Update docs --- .../kubernetes_experiments.md | 84 ++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/docs/markdown/Experiment_Modules/kubernetes_experiments.md b/docs/markdown/Experiment_Modules/kubernetes_experiments.md index 8eada8ca4..4135893a4 100644 --- a/docs/markdown/Experiment_Modules/kubernetes_experiments.md +++ b/docs/markdown/Experiment_Modules/kubernetes_experiments.md @@ -26,7 +26,7 @@ Environment variables that control how the Chaos Engine interacts with Kubernete | kubernetes | The presence of this key enables Kubernetes module. | N/A | Yes | | kubernetes.url | Kubernetes server API url e.g. | None | Yes | | kubernetes.token | JWT token assigned to service account. You can get the value by running `kubectl describe secret name_of_your_secret` | None | Yes | -| kubernetes.namespace | K8S namespace where experiments should be performed | `default` | Yes | +| kubernetes.namespaces | Comma-separated list of namespaces where experiments should be performed | `default` | Yes | | kubernetes.debug | Enables debug log of Kubernetes java client | `false` | No | | kubernetes.validateSSL | Enables validation of sever side certificates | `false` | No | @@ -36,6 +36,7 @@ A service account with a role binding needs to be created in order to access spe Please replace the {{namespace}} fillers with the appropriate values and apply to your cluster. +### Experiments on single namespace **chaos-engine-service-account.yaml** ```yaml @@ -110,7 +111,86 @@ subjects: namespace: {{namespace}} ``` -You can retrieve the token by running `kubectl describe secret chaos-engine -n {{namespace}}` +You can retrieve the token by running `kubectl describe secret chaos-engine -n {{namespace}}` + +### Experiments on multiple namespaces + +When your experiment targets are located in multiple namespaces, +you need to bind roles allowing access to appropriate namespace to your service account. +Or you can simply create a cluster role and binding by running below yaml. + + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: chaos-engine-crole +rules: +- apiGroups: + - apps + resources: + - daemonsets + - daemonsets/status + - deployments + - deployments/status + - replicasets + - replicasets/status + - statefulsets + - statefulsets/status + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + + +- apiGroups: + - "" + resources: + - pods + - pods/status + - replicationcontrollers/status + verbs: + - get + - list + +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: chaos-engine-rolebinding +roleRef: + kind: ClusterRole + name: chaos-engine-crole + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: chaos-engine-serviceaccount + namespace: {{namespace}} + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: chaos-engine-serviceaccount + namespace: {{namespace}} + +``` + ### Verify Service Account Setting