From 4dd8e7c5d53398bb827dda326762c2fa15c131f9 Mon Sep 17 00:00:00 2001 From: Christopher Bradford Date: Sun, 5 Jul 2020 17:37:25 -0400 Subject: [PATCH] Fixes #51 - Added ingress documentation and sample client application to docs (#140) * Added Traefik related files * Clean up SSL directory and add docs to README * Moved traefik configuration in to dedicated README Developed introductions for all implementations we are looking to provide references for. * Fleshed out the TCP section a bit more * Updated traefik documentation to match newer traefik from Helm repo * Updated example to use 3 nodes * Updated sample application to be a bit more verbose * Finished reference implementations for Traefik * Added reference sample application * Removed references that we don't have samples for yet * Renamed proxy to ingress * Fixed reference to WhiteList LBP --- .gitignore | 5 + docs/ingress/README.md | 133 ++++++++++++++++++ docs/ingress/sample-cluster-sample-dc.yaml | 34 +++++ .../sample-java-application/.gitignore | 5 + .../ingress/sample-java-application/README.md | 32 +++++ docs/ingress/sample-java-application/pom.xml | 65 +++++++++ .../java/com/datastax/examples/SampleApp.java | 67 +++++++++ .../KubernetesIngressAddressTranslator.java | 29 ++++ .../kubernetes/KubernetesIngressOption.java | 21 +++ .../src/main/resources/load-balanced.conf | 25 ++++ .../main/resources/mtls-load-balanced.conf | 35 +++++ .../src/main/resources/mtls-sni.conf | 27 ++++ .../java/com/datastax/examples/AppTest.java | 38 +++++ docs/ingress/ssl/README.md | 43 ++++++ docs/ingress/ssl/ca.csr.json | 16 +++ docs/ingress/ssl/client.csr.json | 17 +++ docs/ingress/ssl/ingress.csr.json | 22 +++ .../traefik/dashboard.ingressroute.yaml | 13 ++ docs/ingress/traefik/load-balancing/README.md | 90 ++++++++++++ ...ple-cluster-sample-dc.ingressroutetcp.yaml | 15 ++ .../traefik/mtls-load-balancing/README.md | 98 +++++++++++++ ...ple-cluster-sample-dc.ingressroutetcp.yaml | 23 +++ .../sample-cluster-sample-dc.tlsoption.yaml | 11 ++ docs/ingress/traefik/mtls-sni/README.md | 114 +++++++++++++++ ...ple-cluster-sample-dc.ingressroutetcp.yaml | 39 +++++ .../sample-cluster-sample-dc.tlsoption.yaml | 11 ++ 26 files changed, 1028 insertions(+) create mode 100644 docs/ingress/README.md create mode 100644 docs/ingress/sample-cluster-sample-dc.yaml create mode 100644 docs/ingress/sample-java-application/.gitignore create mode 100644 docs/ingress/sample-java-application/README.md create mode 100644 docs/ingress/sample-java-application/pom.xml create mode 100644 docs/ingress/sample-java-application/src/main/java/com/datastax/examples/SampleApp.java create mode 100644 docs/ingress/sample-java-application/src/main/java/com/datastax/kubernetes/KubernetesIngressAddressTranslator.java create mode 100644 docs/ingress/sample-java-application/src/main/java/com/datastax/kubernetes/KubernetesIngressOption.java create mode 100644 docs/ingress/sample-java-application/src/main/resources/load-balanced.conf create mode 100644 docs/ingress/sample-java-application/src/main/resources/mtls-load-balanced.conf create mode 100644 docs/ingress/sample-java-application/src/main/resources/mtls-sni.conf create mode 100644 docs/ingress/sample-java-application/src/test/java/com/datastax/examples/AppTest.java create mode 100644 docs/ingress/ssl/README.md create mode 100644 docs/ingress/ssl/ca.csr.json create mode 100644 docs/ingress/ssl/client.csr.json create mode 100644 docs/ingress/ssl/ingress.csr.json create mode 100644 docs/ingress/traefik/dashboard.ingressroute.yaml create mode 100644 docs/ingress/traefik/load-balancing/README.md create mode 100644 docs/ingress/traefik/load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml create mode 100644 docs/ingress/traefik/mtls-load-balancing/README.md create mode 100644 docs/ingress/traefik/mtls-load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml create mode 100644 docs/ingress/traefik/mtls-load-balancing/sample-cluster-sample-dc.tlsoption.yaml create mode 100644 docs/ingress/traefik/mtls-sni/README.md create mode 100644 docs/ingress/traefik/mtls-sni/sample-cluster-sample-dc.ingressroutetcp.yaml create mode 100644 docs/ingress/traefik/mtls-sni/sample-cluster-sample-dc.tlsoption.yaml diff --git a/.gitignore b/.gitignore index 63c68bc48..20c0309ba 100644 --- a/.gitignore +++ b/.gitignore @@ -240,3 +240,8 @@ tags # End of https://www.toptal.com/developers/gitignore/api/go,osx,vim,linux,emacs,visualstudiocode,intellij+all /build + +# Certificates +*.pem +*.csr +*.p12 diff --git a/docs/ingress/README.md b/docs/ingress/README.md new file mode 100644 index 000000000..3ac31b899 --- /dev/null +++ b/docs/ingress/README.md @@ -0,0 +1,133 @@ +# Connecting applications to Cassandra running on Kubernetes + +## Background + +As long as applications run within a Kubernetes (k8s) cluster there will be a need to access those services from outside of the cluster. Connecting to a Cassandra (C*) cluster running within k8s can range from trivial to complex depending on where the client is running, latency requirements, and / or security concerns. This document aims to provide a number of solutions to these issues along with the rationale and motivation for each. The following approaches all assume a C* cluster is already up and reported as running. + +## Pod Access + +Any pod running within a Kubernetes cluster may communicate with any other pod given the container network policies permit it. Most communication and service discovery within a K8s cluster will not be an issue. + +### Network Supported Direct Access + +The simplest method, from an architecture perspective, for communicating with Cassandra pods involves having Kubernetes run in an environment where the pod network address space is known and advertised with routes at the network layer. In these types of environments, BGP and static routes may be defined at layer 3 in the OSI model. This allows for IP connectivity / routing directly to pods and services running within Kubernetes from **both** inside and outside the cluster. Additionally, this approach will allow for the consumption of service addresses externally. Unfortunately, this requires an advanced understanding of both k8s networking and the infrastructure available within the enterprise or cloud where it is hosted. + +**Pros** + +* Zero additional configuration within the application +* Works inside and outside of the Kubernetes network + +**Cons** + +* Requires configuration at the networking layer within the cloud / enterprise environment +* Not all environments can support this approach. Some cloud environments do not have the tooling exposed for customers to enable this functionality. + +### Host Network + +Host Network configuration exposes all network interfaces to the underlying pod instead of a single virtual interface. This will allow Cassandra to bind on the worker's interface with an externally accessible IP. Any container that is launched as part of the pod will have access to the host's interface, it cannot be fenced off to a specific container. + +Enabling this behavior is done by passing hostNetwork: true in the podTemplateSpec at the top level. + +**Pros** + +* External connectivity is possible as the service is available at the nodes IP instead of an IP internal to the Kubernetes cluster. + +**Cons** + +* If a pod is rescheduled the IP address of the pod can change +* In some K8s distributions this is a privileged operation +* Additional automation will be required to identify the appropriate IP and set it for listen_address and broadcast_address +* Only one Cassandra pod may be started per worker, regardless of `allowMultiplePodsPerWorker` setting. + +### Host Port + +Host port is similar to host network, but instead of being applied at the pod level, it is applied to specified containers within the pod. For each port listed in the container's block a hostPort: external_port key value is included. external_port is the port number on the Kubernetes worker that should be forwarded to this container's port. + +At this time we do not allow for modifying the cassandra container via podTemplateSpec, thus configuring this value is not possible without patching each rack's stateful set. + +**Pros** + +* External connectivity is possible as the service is available at the nodes IP instead of an IP internal to the Kubernetes cluster. +* Easier configuration a separate container to determine the appropriate IP is no longer required. + +**Cons** + +* If a pod is rescheduled the IP address of the pod can change +* In some K8s distributions this is a privileged operation +* Only one Cassandra pod may be started per worker, regardless of allowMultiplePodsPerWorker setting. +* Not recommended according to K8s [Configuration Best Practices](https://kubernetes.io/docs/concepts/configuration/overview/#services). + +## Services + +If the application is running within the same Kubernetes cluster as the Cassandra cluster connectivity is simple. cass-operator exposes a number of services representing a Cassandra cluster, datacenters, and seeds. Applications running within the same Kubernetes cluster may leverage these services to discover and identify pods within the target C* cluster. + +External applications do not have access to this information via DNS as internal applications do. It is possible to forward DNS requests to Kubernetes from outside the cluster and resolve configured services. Unfortunately, this will provide the internal pod IP addresses and not those routable unless Network Supported Direct Access is possible within the environment. In most scenarios, external applications will not be able to leverage the exposed services from cass-operator. + +### Load Balancer + +It is possible to configure a service within Kubernetes outside of those provided by cass-operator that is accessible from outside of the Kubernetes cluster. These services have a type: LoadBalancer key in the spec block. In most cloud environments this results in a native cloud load balancer being provisioned to point at the appropriate pods with an external IP. Once the load balancer is provisioned running kubectl get svc will display the external IP address that is pointed at the C* nodes. + +**Pros** + +* Available from outside of the cluster + +**Cons** + +* Requires use of an `AddressTranslator` client side to restrict attempts by the drivers to connect directly with pods and instead direct connnections to the load balancer. +* Removes the possibility of TokenAwarePolicy LBP +* Does not support TLS termination at the service layer, but rather within the application. + +## Ingresses + +Ingresses forward requests to services running within a Kubernetes cluster based on rules. These rules may include specifying the protocol, port, or even path. They may provide additional functionality like termination of SSL / TLS traffic, load balancing across a number of protocols, and name-based virtual hosting. Behind the Ingress K8s type is an Ingress Controller. There are a number of controllers available with varying features to service the defined ingress rules. Think of Ingress as an interface for routing and an Ingress Controller as the implementation of that interface. In this way, any number of Ingress Controllers may be used based on the workload requirements. Ingress Controllers function at Layer 4 & 7 of the OSI model. + +When the ingress specification was created it focused specifically on HTTP / HTTPS workloads. From the documentation, "An Ingress does not expose arbitrary ports or protocols. Exposing services other than HTTP and HTTPS to the internet typically uses a service of type Service.`Type=NodePort` or Service.`Type=LoadBalancer`." Cassandra workloads do NOT use HTTP as a protocol, but rather a specific TCP protocol. + +Ingress Controllers we are looking to leverage require support for TCP load balancing. This will provide routing semantics similar to those of LoadBalancer Services. If the Ingress Controller also supports SSL termination with [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication). Then secure access is possible from outside the cluster while _keeping Token Aware routing support_. Additionally, operators should consider whether the chosen Ingress Controller supports client SSL certificates allowing for [mutual TLS](https://en.wikipedia.org/wiki/Mutual_authentication) to restrict access from unauthorized clients. + +**Pros** + +* Highly-available, entrypoint in to the cluster +* _Some_ implementations support TCP load balancing +* _Some_ implementations support Mutual TLS +* _Some_ implementations support SNI + +**Cons** + +* No _standard_ implementation. Requires careful selection. +* Initially designed for HTTP/HTTPS only workloads + * Many ingresses support pure TCP workloads, but it is _NOT_ defined in the original design specification. Some configurations require fairly heavy handed templating of base configuration files. This may lead to difficult upgrade paths of those components in the future. +* _Only some_ implementations support TCP load balancing +* _Only some_ implementations support mTLS +* _Only some_ implementations support SNI with TCP workloads + +### Traefik + +[Traefik](https://containo.us/traefik/) is an open-source Edge Router that is designed to work in a number of environments, not just Kubernetes. When running on Kubernetes, Traefik is generally installed as an Ingress Controller. Traefik supports TCP load balancing along with SSL termination and SNI. It is automatically included as the default Ingress Controller of [K3s](https://k3s.io/) and [K3d](https://k3d.io/). + +#### Sample Implementations + +* [Simple load balancing](traefik/load-balancing) +* [mTLS with load balancing](traefik/mtls-load-balancing) +* [mTLS with SNI](traefik/mtls-sni) + +## Service Meshes + + +## Java Driver Configuration + +Each of the three reference implementations has a corresponding configuration in the [sample application](sample-java-application) with associated configuration files and sample code. + +## Sample `CassandraDatacenter` Reference + +See [`sample-cluster-sample-dc.cassdc.yaml`](sample-cluster-sample-dc.cassdc.yaml) + +## SSL Certificate Generation + +See [ssl/README.md](ssl/README.md) for directions around creating a CA, client, and ingress certificates. + +## References + +1. [Accessing Kubernetes Pods from Outside of the Cluster](http://alesnosek.com/blog/2017/02/14/accessing-kubernetes-pods-from-outside-of-the-cluster/) +1. [Traefik Docs](https://docs.traefik.io/) +1. [Kubernetes Configuration Best Practices](https://kubernetes.io/docs/concepts/configuration/overview/#services) diff --git a/docs/ingress/sample-cluster-sample-dc.yaml b/docs/ingress/sample-cluster-sample-dc.yaml new file mode 100644 index 000000000..44f65019e --- /dev/null +++ b/docs/ingress/sample-cluster-sample-dc.yaml @@ -0,0 +1,34 @@ +# Sized to work on 3 k8s workers nodes with 1 core / 4 GB RAM +# See neighboring example-cassdc-full.yaml for docs for each parameter +apiVersion: cassandra.datastax.com/v1beta1 +kind: CassandraDatacenter +metadata: + name: sample-dc +spec: + clusterName: sample-cluster + serverType: cassandra + serverVersion: "3.11.6" + serverImage: datastax/cassandra:3.11.6-ubi7 + configBuilderImage: datastax/cass-config-builder:1.0.0-ubi7 + managementApiAuth: + insecure: {} + racks: + - name: sample-rack + size: 3 + allowMultipleNodesPerWorker: true + storageConfig: + cassandraDataVolumeClaimSpec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + config: + cassandra-yaml: {} + # authenticator: org.apache.cassandra.auth.PasswordAuthenticator + # authorizer: org.apache.cassandra.auth.CassandraAuthorizer + # role_manager: org.apache.cassandra.auth.CassandraRoleManager + jvm-options: + initial_heap_size: "800M" + max_heap_size: "800M" diff --git a/docs/ingress/sample-java-application/.gitignore b/docs/ingress/sample-java-application/.gitignore new file mode 100644 index 000000000..5a85eb707 --- /dev/null +++ b/docs/ingress/sample-java-application/.gitignore @@ -0,0 +1,5 @@ +target +client.keystore +client.truststore +*.iml +.idea diff --git a/docs/ingress/sample-java-application/README.md b/docs/ingress/sample-java-application/README.md new file mode 100644 index 000000000..4108b2d48 --- /dev/null +++ b/docs/ingress/sample-java-application/README.md @@ -0,0 +1,32 @@ +# Sample Application with Kubernetes Ingress + +This project is to illustrates how to configure and validate connectivity to Cassandra clusters running within Kubernetes. There are three reference client implementations available: + +* mTLS and SNI based balancing +* Load balancing with mTLS +* Simple load balancing + +At this time there is some _slight_ tweaking required to the configuration files to specify the keystore, truststore, and approach to use. + +Any connections requiring TLS support should place their keystore an truststore in the `src/main/resources/` directory. If you followed the [SSL](../ssl) guide then you should already have these files available. + +## Building and Running + +``` +mvn package +java -cp target/sample-k8s-connectivity-1.0-SNAPSHOT-jar-with-dependencies.jar com.datastax.examples.SampleApp +SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder". +SLF4J: Defaulting to no-operation (NOP) logger implementation +SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details. +Discovered Nodes +sample-dc:sample-rack:fd280adc-e55e-4f3d-97d1-138a1e1abef4 +sample-dc:sample-rack:fd280adc-e55e-4f3d-97d1-138a1e1abef4 +sample-dc:sample-rack:fd280adc-e55e-4f3d-97d1-138a1e1abef4 + +Coordinator: sample-dc:sample-rack:fd280adc-e55e-4f3d-97d1-138a1e1abef4 +[data_center:'sample-dc', rack:'sample-rack', host_id:a7a45d6e-70e3-4e6d-b29c-5dba9a61a282, release_version:'3.11.6'] + +Coordinator: sample-dc:sample-rack:fd280adc-e55e-4f3d-97d1-138a1e1abef4 +[data_center:'sample-dc', rack:'sample-rack', host_id:7e2921a6-e170-4a4f-bf0f-011ab83b3739, release_version:'3.11.6'] +[data_center:'sample-dc', rack:'sample-rack', host_id:fd280adc-e55e-4f3d-97d1-138a1e1abef4, release_version:'3.11.6'] +``` diff --git a/docs/ingress/sample-java-application/pom.xml b/docs/ingress/sample-java-application/pom.xml new file mode 100644 index 000000000..fdf8a6809 --- /dev/null +++ b/docs/ingress/sample-java-application/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + com.datastax.examples + sample-k8s-connectivity + 1.0-SNAPSHOT + jar + + sample-k8s-connectivity + http://maven.apache.org + + + UTF-8 + 4.7.2 + 1.8 + 1.8 + + + + + commons-cli + commons-cli + 1.4 + + + com.datastax.oss + java-driver-core + ${driver.version} + + + junit + junit + 3.8.1 + test + + + + + + + + maven-assembly-plugin + 3.3.0 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + \ No newline at end of file diff --git a/docs/ingress/sample-java-application/src/main/java/com/datastax/examples/SampleApp.java b/docs/ingress/sample-java-application/src/main/java/com/datastax/examples/SampleApp.java new file mode 100644 index 000000000..47f8a06cf --- /dev/null +++ b/docs/ingress/sample-java-application/src/main/java/com/datastax/examples/SampleApp.java @@ -0,0 +1,67 @@ +package com.datastax.examples; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.metadata.Node; +import com.datastax.oss.driver.internal.core.metadata.SniEndPoint; + +import java.net.InetSocketAddress; + +public class SampleApp { + public static void main( String[] args ) throws Exception { + SampleApp app = new SampleApp(); + app.run(); + } + + public void run() throws Exception { + CqlSession session = getLoadBalancedSession(); + + System.out.println("Discovered Nodes"); + for (Node n : session.getMetadata().getNodes().values()) { + System.out.println(String.format("%s:%s:%s", n.getDatacenter(), n.getRack(), n.getHostId())); + } + System.out.println(); + + ResultSet rs = session.execute("SELECT data_center, rack, host_id, release_version FROM system.local"); + Node n = rs.getExecutionInfo().getCoordinator(); + System.out.println(String.format("Coordinator: %s:%s:%s", n.getDatacenter(), n.getRack(), n.getHostId())); + rs.forEach(row -> { + System.out.println(row.getFormattedContents()); + }); + System.out.println(); + + rs = session.execute("SELECT data_center, rack, host_id, release_version FROM system.peers"); + n = rs.getExecutionInfo().getCoordinator(); + System.out.println(String.format("Coordinator: %s:%s:%s", n.getDatacenter(), n.getRack(), n.getHostId())); + rs.forEach(row -> { + System.out.println(row.getFormattedContents()); + }); + + session.close(); + } + + private CqlSession getLoadBalancedSession() { + return CqlSession.builder() + .withConfigLoader(DriverConfigLoader.fromClasspath("load-balanced.conf")) + .build(); + } + + private CqlSession getMtlsLoadBalancedSession() { + return CqlSession.builder() + .withConfigLoader(DriverConfigLoader.fromClasspath("mtls-load-balanced.conf")) + .build(); + } + + private CqlSession getMtlsSniSession() { + // Ingress address + InetSocketAddress ingressAddress = new InetSocketAddress("traefik.k3s.local", 9042); + + // Endpoint (contact point) + SniEndPoint endPoint = new SniEndPoint(ingressAddress, "ec448e83-8b83-407b-b342-13ce0250001c"); + + return CqlSession.builder() + .withConfigLoader(DriverConfigLoader.fromClasspath("mtls-sni.conf")) + .build(); + } +} diff --git a/docs/ingress/sample-java-application/src/main/java/com/datastax/kubernetes/KubernetesIngressAddressTranslator.java b/docs/ingress/sample-java-application/src/main/java/com/datastax/kubernetes/KubernetesIngressAddressTranslator.java new file mode 100644 index 000000000..939139932 --- /dev/null +++ b/docs/ingress/sample-java-application/src/main/java/com/datastax/kubernetes/KubernetesIngressAddressTranslator.java @@ -0,0 +1,29 @@ +package com.datastax.kubernetes; + +import com.datastax.oss.driver.api.core.addresstranslation.AddressTranslator; +import com.datastax.oss.driver.api.core.context.DriverContext; +import edu.umd.cs.findbugs.annotations.NonNull; + +import java.net.InetSocketAddress; + +public class KubernetesIngressAddressTranslator implements AddressTranslator { + private DriverContext driverContext; + + public KubernetesIngressAddressTranslator(DriverContext driverContext) { + this.driverContext = driverContext; + } + + @NonNull + @Override + public InetSocketAddress translate(@NonNull InetSocketAddress address) { + String ingressAddress = driverContext.getConfig().getDefaultProfile().getString(KubernetesIngressOption.INGRESS_ADDRESS); + int ingressPort = driverContext.getConfig().getDefaultProfile().getInt(KubernetesIngressOption.INGRESS_PORT); + + return new InetSocketAddress(ingressAddress, ingressPort); + } + + @Override + public void close() { + // NOOP + } +} diff --git a/docs/ingress/sample-java-application/src/main/java/com/datastax/kubernetes/KubernetesIngressOption.java b/docs/ingress/sample-java-application/src/main/java/com/datastax/kubernetes/KubernetesIngressOption.java new file mode 100644 index 000000000..a915cf51a --- /dev/null +++ b/docs/ingress/sample-java-application/src/main/java/com/datastax/kubernetes/KubernetesIngressOption.java @@ -0,0 +1,21 @@ +package com.datastax.kubernetes; + +import com.datastax.oss.driver.api.core.config.DriverOption; +import edu.umd.cs.findbugs.annotations.NonNull; + +public enum KubernetesIngressOption implements DriverOption { + INGRESS_ADDRESS("advanced.k8s.ingress.address"), + INGRESS_PORT("advanced.k8s.ingress.port"); + + private final String path; + + KubernetesIngressOption(String path) { + this.path = path; + } + + @Override + @NonNull + public String getPath() { + return path; + } +} diff --git a/docs/ingress/sample-java-application/src/main/resources/load-balanced.conf b/docs/ingress/sample-java-application/src/main/resources/load-balanced.conf new file mode 100644 index 000000000..e020f3e11 --- /dev/null +++ b/docs/ingress/sample-java-application/src/main/resources/load-balanced.conf @@ -0,0 +1,25 @@ +datastax-java-driver { + basic { + contact-points = ["traefik.k3s.local:9042"] + session-name = "Load Balanced Connection Sample App" + keyspace = "system" + + application { + name = "Load Balanced Connection Sample App" + version = "1.0.0-SNAPSHOT" + } + + load-balancing-policy { + local-datacenter = "sample-dc" + } + } + + advanced { + address-translator.class = com.datastax.kubernetes.KubernetesIngressAddressTranslator + + k8s.ingress { + address = "traefik.k3s.local" + port = 9042 + } + } +} diff --git a/docs/ingress/sample-java-application/src/main/resources/mtls-load-balanced.conf b/docs/ingress/sample-java-application/src/main/resources/mtls-load-balanced.conf new file mode 100644 index 000000000..c92d0cae9 --- /dev/null +++ b/docs/ingress/sample-java-application/src/main/resources/mtls-load-balanced.conf @@ -0,0 +1,35 @@ +datastax-java-driver { + basic { + contact-points = ["traefik.k3s.local:9042"] + session-name = "mTLS Load Balanced Connection Sample App" + keyspace = "system" + + application { + name = "mTLS Load Balanced Connection Sample App" + version = "1.0.0-SNAPSHOT" + } + + load-balancing-policy { + local-datacenter = "sample-dc" + } + } + + advanced { + ssl-engine-factory { + class = DefaultSslEngineFactory + + truststore-path = "src/main/resources/client.truststore" + truststore-password = "foobarbaz" + + keystore-path = "src/main/resources/client.keystore" + keystore-password = "foobarbaz" + } + + address-translator.class = com.datastax.kubernetes.KubernetesIngressAddressTranslator + + k8s.ingress { + address = "traefik.k3s.local" + port = 9042 + } + } +} diff --git a/docs/ingress/sample-java-application/src/main/resources/mtls-sni.conf b/docs/ingress/sample-java-application/src/main/resources/mtls-sni.conf new file mode 100644 index 000000000..1dd10458c --- /dev/null +++ b/docs/ingress/sample-java-application/src/main/resources/mtls-sni.conf @@ -0,0 +1,27 @@ +datastax-java-driver { + basic { + session-name = "mTLS SNI Load Balanced Connection Sample App" + keyspace = "system" + + application { + name = "mTLS Load SNI Balanced Connection Sample App" + version = "1.0.0-SNAPSHOT" + } + + load-balancing-policy { + local-datacenter = "sample-dc" + } + } + + advanced { + ssl-engine-factory { + class = SniSslEngineFactory + + truststore-path = "src/main/resources/client.truststore" + truststore-password = "foobarbaz" + + keystore-path = "src/main/resources/client.keystore" + keystore-password = "foobarbaz" + } + } +} diff --git a/docs/ingress/sample-java-application/src/test/java/com/datastax/examples/AppTest.java b/docs/ingress/sample-java-application/src/test/java/com/datastax/examples/AppTest.java new file mode 100644 index 000000000..b090e6ca7 --- /dev/null +++ b/docs/ingress/sample-java-application/src/test/java/com/datastax/examples/AppTest.java @@ -0,0 +1,38 @@ +package com.datastax.examples; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +} diff --git a/docs/ingress/ssl/README.md b/docs/ingress/ssl/README.md new file mode 100644 index 000000000..5cda5e388 --- /dev/null +++ b/docs/ingress/ssl/README.md @@ -0,0 +1,43 @@ +# Sample SSL Certificates and Keys + +## Requirements + +* [CFSSL](https://cfssl.org/) + +## Generate Certificate Authority + +```bash +# Create CA from a JSON template +cfssl gencert -initca ca.csr.json | cfssljson -bare ca + +# Create the secret resource +kubectl create secret generic ca-cert --from-file=tls.cert=ca.pem --from-file=tls.ca=ca.pem + +# Create a truststore for the application +keytool -import -v -trustcacerts -alias CARoot -file ca.pem -keystore client.truststore +``` + +## Generate Ingress Certificate + +```bash +# If you are using SNI, retrieve the host IDs to include in the Ingress CSR +# Make sure all nodes are up, then extract the values from the nodeStatuses field. +kubectl get cassdc sample-dc -o yaml + +# Create and sign the Ingress certificate +cfssl gencert -ca ca.pem -ca-key ca-key.pem ingress.csr.json | cfssljson -bare ingress + +# Create the secret resource +kubectl create secret generic sample-cluster-sample-dc-cert --from-file=tls.crt=ingress.pem --from-file=tls.key=ingress-key.pem --from-file=tls.ca=ca.pem +``` + +## Generate Client Certificate + +```bash +# Create and sign Client certificate +cfssl gencert -ca ca.pem -ca-key ca-key.pem client.csr.json | cfssljson -bare client + +# Create the keystore for the client +openssl pkcs12 -export -in client.pem -inkey client-key.pem -out client.p12 +keytool -importkeystore -destkeystore client.keystore -srckeystore client.p12 -srcstoretype PKCS12 +``` diff --git a/docs/ingress/ssl/ca.csr.json b/docs/ingress/ssl/ca.csr.json new file mode 100644 index 000000000..0dc8a1850 --- /dev/null +++ b/docs/ingress/ssl/ca.csr.json @@ -0,0 +1,16 @@ +{ + "CN": "Sample Self-Signed CA", + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "US", + "ST": "California", + "L": "Santa Clara", + "O": "Engineering", + "OU": "Cassandra Operator" + } + ] +} diff --git a/docs/ingress/ssl/client.csr.json b/docs/ingress/ssl/client.csr.json new file mode 100644 index 000000000..23dcde434 --- /dev/null +++ b/docs/ingress/ssl/client.csr.json @@ -0,0 +1,17 @@ +{ + "CN": "sample-application", + "key": { + "algo": "rsa", + "size": 2048 + }, + "hosts": [], + "names": [ + { + "C": "US", + "ST": "California", + "L": "Santa Clara", + "O": "Engineering", + "OU": "Cassandra Operator" + } + ] +} \ No newline at end of file diff --git a/docs/ingress/ssl/ingress.csr.json b/docs/ingress/ssl/ingress.csr.json new file mode 100644 index 000000000..c73a780ba --- /dev/null +++ b/docs/ingress/ssl/ingress.csr.json @@ -0,0 +1,22 @@ +{ + "CN": "sample-dc.sample-cluster", + "key": { + "algo": "rsa", + "size": 2048 + }, + "hosts": [ + "traefik.k3s.local", + "ec448e83-8b83-407b-b342-13ce0250001c", + "efe2564e-f8dc-4b0b-89a4-83cc12ec99a6", + "bba52d6e-8415-4869-ad8f-bf03e136e874" + ], + "names": [ + { + "C": "US", + "ST": "California", + "L": "Santa Clara", + "O": "Engineering", + "OU": "Cassandra Operator" + } + ] +} \ No newline at end of file diff --git a/docs/ingress/traefik/dashboard.ingressroute.yaml b/docs/ingress/traefik/dashboard.ingressroute.yaml new file mode 100644 index 000000000..5235cc88a --- /dev/null +++ b/docs/ingress/traefik/dashboard.ingressroute.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRoute +metadata: + name: dashboard +spec: + entryPoints: + - web + routes: + - match: Host(`traefik.k3s.local`) && (PathPrefix(`/dashboard`) || PathPrefix(`/api`)) + kind: Rule + services: + - name: api@internal + kind: TraefikService diff --git a/docs/ingress/traefik/load-balancing/README.md b/docs/ingress/traefik/load-balancing/README.md new file mode 100644 index 000000000..6cd667900 --- /dev/null +++ b/docs/ingress/traefik/load-balancing/README.md @@ -0,0 +1,90 @@ +# Traefik Simple Load Balancing + +When leveraging a single endpoint ingress / load balancer we lose the ability to provide token aware routing without the use of SNI (see the [mTLS with SNI guide](../mtls-sni)). **WARNING** This approach does not interact with the traffic at all. All traffic is sent over cleartext without any form of authentication of the server or client.. Note that **_each_** Cassandra cluster running behind the ingress will require it's own endpoint. Without a way to detect the pod we want to connect with it's the only way to differentiate requests. + +1. _Optional_ provision a local cluster with k3d. If you already have a cluster provisioned and it is available via `kubectl` you may safely skip this step. + + ```bash + # Create the cluster + k3d c -x "--no-deploy" -x "traefik" + export KUBECONFIG="$(k3d get-kubeconfig --name='k3s-default')" + kubectl cluster-info + + # Import images from the local Docker daemon + k3d i datastax/cass-operator:1.2.0 + k3d i datastax/cassandra:3.11.6-ubi7 + k3d i datastax/cass-config-builder:1.0.0-ubi7 + ``` + +1. Install Traefik with Helm + + ```bash + helm repo add traefik https://containous.github.io/traefik-helm-chart + helm repo update + helm install traefik traefik/traefik + ``` + +1. Add an ingress route for the Traefik dashboard and get the IP of the load balancer + + ```bash + kubectl apply -f traefik/dashboard.ingressroute.yaml + kubectl get svc traefik -o jsonpath="{.status.loadBalancer.ingress[].ip} traefik.k3s.local" + ``` + + If you add an entry to your /etc/hosts file with the value from the second command. With this in place the Traefik dashboard may be viewed at http://traefik.k3s.local/dashboard/. + +1. Edit the traefik `deployment` and add an entrypoint for TCP Cassandra traffic. This should be done in the `args` section of the `traefik` container. + + ```bash + kubectl edit deployment traefik + ``` + + ```yaml + - --entryPoints.websecure.address=:8443/tcp + # Add the following line, note the port number does have to be 9042. The value "cassandra" is displayed in the Traefik UI and may also be customized + - --entryPoints.cassandra.address=:9042/tcp + - --api.dashboard=true + ``` + + After saving your changes the deployment will replace the old pod with a new one including the adjusted arguments. Validate the new entrypoint exists in the Traefik dashboard. + +1. With a new EntryPoint defined we must update the existing service with the new ports. + + ```bash + kubectl edit svc traefik + ``` + + ```yaml + - name: websecure + nodePort: 31036 + port: 443 + protocol: TCP + targetPort: websecure + # Add the following section, it is ideal to use the same name as your entrypoint. Additionally the port number MUST match + - name: cassandra + port: 9042 + protocol: TCP + targetPort: 9042 + ``` + + At this point refreshing the Traefik dashboard should show a new endpoint named `cassandra` running. + +1. Install `cass-operator` via Helm + + ```bash + helm install --namespace=default cass-operator ./charts/cass-operator-chart + ``` + +1. Deploy a Cassandra cluster + + ```bash + kubectl apply -f sample-cluster-sample-dc.yaml + ``` + +1. Create the `IngressTCPRoute`. This provides the mapping between our endpoint and internal service. + + ```bash + kubectl apply -f traefik/load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml + ``` + +1. Check out the [sample application](../../sample-java-application) to validate your deployment diff --git a/docs/ingress/traefik/load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml b/docs/ingress/traefik/load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml new file mode 100644 index 000000000..1654224ff --- /dev/null +++ b/docs/ingress/traefik/load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml @@ -0,0 +1,15 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRouteTCP +metadata: + name: sample-cluster-sample-dc +spec: + entryPoints: + - cassandra + routes: + # Match is the rule corresponding to an underlying router. + - match: HostSNI(`*`) + services: + - name: sample-cluster-sample-dc-service + port: 9042 + terminationDelay: 400 + weight: 10 diff --git a/docs/ingress/traefik/mtls-load-balancing/README.md b/docs/ingress/traefik/mtls-load-balancing/README.md new file mode 100644 index 000000000..e995f741f --- /dev/null +++ b/docs/ingress/traefik/mtls-load-balancing/README.md @@ -0,0 +1,98 @@ +# Traefik Simple Load Balancing with mTLS + +When leveraging a single endpoint ingress / load balancer we lose the ability to provide token aware routing without the use of SNI (see the [mTLS with SNI guide](../mtls-sni)). This approach keeps unwanted traffic from reaching the cluster through the use of mTLS terminated at the ingress layer. Note that **_each_** Cassandra cluster running behind the ingress will require it's own endpoint. Without a way to detect the pod we want to connect with it's the only way to differentiate requests. + +1. _Optional_ provision a local cluster with k3d. If you already have a cluster provisioned and it is available via `kubectl` you may safely skip this step. + + ```bash + # Create the cluster + k3d c -x "--no-deploy" -x "traefik" + export KUBECONFIG="$(k3d get-kubeconfig --name='k3s-default')" + kubectl cluster-info + + # Import images from the local Docker daemon + k3d i datastax/cass-operator:1.2.0 + k3d i datastax/cassandra:3.11.6-ubi7 + k3d i datastax/cass-config-builder:1.0.0-ubi7 + ``` + +1. Install Traefik with Helm + + ```bash + helm repo add traefik https://containous.github.io/traefik-helm-chart + helm repo update + helm install traefik traefik/traefik + ``` + +1. Add an ingress route for the Traefik dashboard and get the IP of the load balancer + + ```bash + kubectl apply -f traefik/dashboard.ingressroute.yaml + kubectl get svc traefik -o jsonpath="{.status.loadBalancer.ingress[].ip} traefik.k3s.local" + ``` + + If you add an entry to your /etc/hosts file with the value from the second command. With this in place the Traefik dashboard may be viewed at http://traefik.k3s.local/dashboard/. + +1. Edit the traefik `deployment` and add an entrypoint for TCP Cassandra traffic. This should be done in the `args` section of the `traefik` container. + + ```bash + kubectl edit deployment traefik + ``` + + ```yaml + - --entryPoints.websecure.address=:8443/tcp + # Add the following line, note the port number does have to be 9042. The value "cassandra" is displayed in the Traefik UI and may also be customized + - --entryPoints.cassandra.address=:9042/tcp + - --api.dashboard=true + ``` + + After saving your changes the deployment will replace the old pod with a new one including the adjusted arguments. Validate the new entrypoint exists in the Traefik dashboard. + +1. With a new EntryPoint defined we must update the existing service with the new ports. + + ```bash + kubectl edit svc traefik + ``` + + ```yaml + - name: websecure + nodePort: 31036 + port: 443 + protocol: TCP + targetPort: websecure + # Add the following section, it is ideal to use the same name as your entrypoint. Additionally the port number MUST match + - name: cassandra + port: 9042 + protocol: TCP + targetPort: 9042 + ``` + + At this point refreshing the Traefik dashboard should show a new endpoint named `cassandra` running. + +1. Install `cass-operator` via Helm + + ```bash + helm install --namespace=default cass-operator ./charts/cass-operator-chart + ``` + +1. Deploy a Cassandra cluster + + ```bash + kubectl apply -f sample-cluster-sample-dc.yaml + ``` + +1. Generate the TLS certificates and add them as secrets to the cluster with the guide in the [ssl](../ssl) directory. Note you do **NOT** need to specify any of the host ID values as we will not be performing additional routing at the ingress layer. + +1. Install TLS Options to add support for mutual TLS. This configures the CA that must be used in the client certificate + + ```bash + kubectl apply -f traefik/mtls-load-balancing/sample-cluster-sample-dc.tlsoption.yaml + ``` + +1. Create the `IngressTCPRoute`. This provides the mapping between our endpoint and internal service and binds the previously installed tlsoptions to the endpoint. + + ```bash + kubectl apply -f traefik/mtls-load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml + ``` + +1. Check out the [sample application](../../sample-java-application) to validate your deployment diff --git a/docs/ingress/traefik/mtls-load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml b/docs/ingress/traefik/mtls-load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml new file mode 100644 index 000000000..dde395707 --- /dev/null +++ b/docs/ingress/traefik/mtls-load-balancing/sample-cluster-sample-dc.ingressroutetcp.yaml @@ -0,0 +1,23 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRouteTCP +metadata: + name: sample-cluster-sample-dc +spec: + entryPoints: + - cassandra + routes: + # Match is the rule corresponding to an underlying router. + - match: HostSNI(`*`) + services: + - name: sample-cluster-sample-dc-service + port: 9042 + terminationDelay: 400 + weight: 10 + tls: + domains: + - main: sample-dc.sample-cluster + options: + name: sample-cluster-sample-dc-options + namespace: default + secretName: sample-cluster-sample-dc-cert + passthrough: false diff --git a/docs/ingress/traefik/mtls-load-balancing/sample-cluster-sample-dc.tlsoption.yaml b/docs/ingress/traefik/mtls-load-balancing/sample-cluster-sample-dc.tlsoption.yaml new file mode 100644 index 000000000..9eb3789e2 --- /dev/null +++ b/docs/ingress/traefik/mtls-load-balancing/sample-cluster-sample-dc.tlsoption.yaml @@ -0,0 +1,11 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: TLSOption +metadata: + name: sample-cluster-sample-dc-options + namespace: default +spec: + clientAuth: + secretNames: + - ca-cert + clientAuthType: RequireAndVerifyClientCert + sniStrict: false diff --git a/docs/ingress/traefik/mtls-sni/README.md b/docs/ingress/traefik/mtls-sni/README.md new file mode 100644 index 000000000..90b4ba080 --- /dev/null +++ b/docs/ingress/traefik/mtls-sni/README.md @@ -0,0 +1,114 @@ +# Traefik with TNS and SNI + +When leveraging a single endpoint ingress / load balancer we naturally remove the ability to route requests based on token awareness. That is unless we leverage TLS with SNI. In this approach the TLS Client HELLO includes a server name which allows the single endpoint to forward the request to the appropriate pod based on rules we specify. + +1. _Optional_ provision a local cluster with k3d. If you already have a cluster provisioned and it is available via `kubectl` you may safely skip this step. + + ```bash + # Create the cluster + k3d c -x "--no-deploy" -x "traefik" + + # Import images from the local Docker daemon + k3d i datastax/cass-operator:1.2.0 + k3d i datastax/cassandra:3.11.6-ubi7 + k3d i datastax/cass-config-builder:1.0.0-ubi7 + ``` + +1. Install Traefik with Helm + + ```bash + helm repo add traefik https://containous.github.io/traefik-helm-chart + helm repo update + helm install traefik traefik/traefik + ``` + +1. Add an ingress route for the Traefik dashboard and get the IP of the load balancer + + ```bash + kubectl apply -f traefik/mtls-sni/dashboard.ingressroute.yaml + kubectl get svc traefik -o jsonpath="{.status.loadBalancer.ingress[].ip} traefik.k3s.local" + ``` + + If you add an entry to your /etc/hosts file with the value from the second command. With this in place the Traefik dashboard may be viewed at http://traefik.k3s.local/dashboard/. + +1. Edit the traefik `deployment` and add an entrypoint for TCP Cassandra traffic. This should be done in the `args` section of the `traefik` container. + + ```bash + kubectl edit deployment traefik + ``` + + ```yaml + - --entryPoints.websecure.address=:8443/tcp + # Add the following line, note the port number does have to be 9042. The value "cassandra" is displayed in the Traefik UI. + - --entryPoints.cassandra.address=:9042/tcp + - --api.dashboard=true + ``` + + After saving your changes the deployment will replace the old pod with a new one including the adjusted arguments. Validate the new entrypoint exists in the Traefik dashboard. + +1. With a new EntryPoint defined we must update the existing service with the new ports. + + ```bash + kubectl edit svc traefik + ``` + + ```yaml + - name: websecure + nodePort: 31036 + port: 443 + protocol: TCP + targetPort: websecure + # Add the following section + - name: cassandra + port: 9042 + protocol: TCP + targetPort: 9042 + ``` + +1. Install `cass-operator` via Helm + + ```bash + helm install --namespace=default cass-operator ./charts/cass-operator-chart + ``` + +1. Deploy a Cassandra cluster + + ```bash + kubectl apply -f sample-cluster-sample-dc.yaml + ``` + +1. Query the host ID values used in the cluster + + ```bash + kubectl get cassdc -o json | jq ".items[].status.nodeStatuses" + { + "sample-cluster-sample-dc-sample-rack-sts-0": { + "hostID": "d1ba31b6-4b0e-4a7a-ba7e-8721271ae99a", + "nodeIP": "10.42.0.29" + } + } + ``` + +1. Generate the TLS certificates and add them as secrets to the cluster with the guide in the [ssl](../ssl) directory. + +1. Install TLS Options to add support for mutual TLS. This configures the CA that must be used in the client certificate + + ```bash + kubectl apply -f traefik/mtls-sni/sample-cluster-sample-dc.tlsoption.yaml + ``` + +1. Edit and create the `IngressTCPRoute`. This provides the SNI mapping for routing TCP requests from the ingress to individual pods. + + ```bash + kubectl apply -f traefik/mtls-sni/sample-cluster-sample-dc.ingressroutetcp.yaml + ``` + +1. Create the `service` for the pod with `kubectl expose`. Note the service name will match the pod name. + + ```bash + kubectl expose pod sample-cluster-sample-dc-sample-rack-sts-0 + kubectl expose pod sample-cluster-sample-dc-sample-rack-sts-1 + kubectl expose pod sample-cluster-sample-dc-sample-rack-sts-2 + ``` + +1. Check out the [sample application](../../sample-java-application) to validate your deployment diff --git a/docs/ingress/traefik/mtls-sni/sample-cluster-sample-dc.ingressroutetcp.yaml b/docs/ingress/traefik/mtls-sni/sample-cluster-sample-dc.ingressroutetcp.yaml new file mode 100644 index 000000000..353a595f0 --- /dev/null +++ b/docs/ingress/traefik/mtls-sni/sample-cluster-sample-dc.ingressroutetcp.yaml @@ -0,0 +1,39 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRouteTCP +metadata: + name: sample-cluster-sample-dc +spec: + entryPoints: + - cassandra + routes: + # Match is the rule corresponding to an underlying router. + - match: HostSNI(`ec448e83-8b83-407b-b342-13ce0250001c`) + services: + - name: sample-cluster-sample-dc-sample-rack-sts-0 + port: 9042 + terminationDelay: 400 + weight: 10 + - match: HostSNI(`bba52d6e-8415-4869-ad8f-bf03e136e874`) + services: + - name: sample-cluster-sample-dc-sample-rack-sts-1 + port: 9042 + terminationDelay: 400 + weight: 10 + - match: HostSNI(`efe2564e-f8dc-4b0b-89a4-83cc12ec99a6`) + services: + - name: sample-cluster-sample-dc-sample-rack-sts-2 + port: 9042 + terminationDelay: 400 + weight: 10 + tls: + domains: + - main: sample-dc.sample-cluster + sans: + - ec448e83-8b83-407b-b342-13ce0250001c + - bba52d6e-8415-4869-ad8f-bf03e136e874 + - efe2564e-f8dc-4b0b-89a4-83cc12ec99a6 + options: + name: sample-cluster-sample-dc-options + namespace: default + secretName: sample-cluster-sample-dc-cert + passthrough: false diff --git a/docs/ingress/traefik/mtls-sni/sample-cluster-sample-dc.tlsoption.yaml b/docs/ingress/traefik/mtls-sni/sample-cluster-sample-dc.tlsoption.yaml new file mode 100644 index 000000000..1d6e3abe9 --- /dev/null +++ b/docs/ingress/traefik/mtls-sni/sample-cluster-sample-dc.tlsoption.yaml @@ -0,0 +1,11 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: TLSOption +metadata: + name: sample-cluster-sample-dc-options + namespace: default +spec: + clientAuth: + secretNames: + - ca-cert + clientAuthType: RequireAndVerifyClientCert + sniStrict: true