Fix for comms channel certs and docs (ipdk-io#110)

Signed-off-by: nupurjai <[email protected]>
venkateshpnv · Oct 6, 2023 · c632fca · c632fca
1 parent 9700f9e
commit c632fca
Show file tree

Hide file tree

Showing 8 changed files with 91 additions and 73 deletions.
diff --git a/deploy/infraagent-daemonset.yaml b/deploy/infraagent-daemonset.yaml
@@ -53,7 +53,7 @@ spec:
               fieldPath: spec.nodeName
         volumeMounts:
         - name: client-certs
-          mountPath: /etc/pki/infraagent/certs/client/
+          mountPath: /etc/pki/infraagent/client/
         - name: config-volume
           mountPath: /etc/infra/
       containers:
@@ -91,7 +91,7 @@ spec:
         - name: kubeconfig
           mountPath: /root/.kube
         - name: client-certs
-          mountPath: /etc/pki/infraagent/certs/client/
+          mountPath: /etc/pki/infraagent/client/
         command:
         - /infraagent
       volumes:

diff --git a/deploy/inframanager-daemonset.yaml b/deploy/inframanager-daemonset.yaml
@@ -41,9 +41,9 @@ spec:
         - name: config-volume
           mountPath: /etc/infra/
         - name: client-certs
-          mountPath: /etc/pki/inframanager/certs/client
+          mountPath: /etc/pki/inframanager/client
         - name: server-certs
-          mountPath: /etc/pki/inframanager/certs/server
+          mountPath: /etc/pki/inframanager/server
         command:
         - /inframanager
         env:

diff --git a/docs/Setup.md b/docs/Setup.md
@@ -83,6 +83,22 @@ IPU ES2K target.
 6. Run the `setup_infra.sh` script, which, in addition to creating the
    specified number of virtual interfaces (TAP type on DPDK target and IDPF
    Sub-Function type on ES2K), sets up the HugePages and starts infrap4d.
+   The script supports setup in two different modes.
+
+   a. The host mode, where every component runs on the host and offload happens
+   from host.
+
+   b. The split mode, where the inframanager runs on IPU ARM cores for rule offloads
+   while the infraagent runs on host. In this mode, the communication channel between
+   IPU ACC-ARM complex and host must pre-exist through provisioning on IPU.
+   The communication channel on ARM-ACC side would require user to configure an IP
+   address and later use it as an argument in setup_infra.sh script. The below example
+   assumes remote ACC endpoint with an IP address of `10.10.0.2`.
+   The grpc communication between the two infra components over the comms channel
+   are encrypted. User would need to add IP address based on the configuration to
+   `openssl.cnf` under scripts TLS directory and regenerate certificates. Make sure
+   to also update inframanager config file for this IP address for manager to bind to
+   and infraagent config file for infraagent to connect to the remote manager.
 
    ```bash
    ./setup_infra.sh -i <8|16|..> -m <split|host> -r <10.10.0.2>
@@ -93,8 +109,8 @@ IPU ES2K target.
      -m  Mode host or split, depending on where Inframanager is configured to run
      -r  IP address configured by the user on the ACC-ARM complex for
        connectivity to the Host. This is provisioned using Node Policy - comms
-       channel [[0,3],[4,2]]. This is needed for runnning in split mode. Script will assign
-       an IP addresss from the same subnet on the Host side for connectivity.
+       channel ([5,0],[4,0]),([4,2],[0,3]). This is needed for runnning in split mode.
+       Script will assign an IP addresss from the same subnet on the Host side for connectivity.
 
 
    Please set following env variables for host deployment:
@@ -143,7 +159,6 @@ IPU ES2K target.
    before building the images.
 
 
-
 8. Make the docker images. This step builds the Kubernetes container images:
    ```bash
    make docker-build
@@ -194,6 +209,12 @@ interface: ens801f0
 mtls: true
 insecure: false
 ```
+For split mode, also configure the follwing.
+
+```text
+managerAddr : <IP address of comms channel on ACC>
+managerPort : 50002
+```
 
 ### inframanager config file update
 

diff --git a/pkg/types/types.go b/pkg/types/types.go
@@ -51,14 +51,14 @@ const (
 	DefaultRoute                = "169.254.1.1/32"
 	HostInterfaceAddr           = "200.1.1.2/32"
 	ArpProxyDefaultPort         = 0
-	AgentDefaultClientCert      = "/etc/pki/infraagent/certs/client/tls.crt"
-	AgentDefaultClientKey       = "/etc/pki/infraagent/certs/client/tls.key"
-	AgentDefaultCACert          = "/etc/pki/infraagent/certs/client/ca.crt"
-	ManagerDefaultClientCert    = "/etc/pki/inframanager/certs/client/tls.crt"
-	ManagerDefaultClientKey     = "/etc/pki/inframanager/certs/client/tls.key"
-	ManagerDefaultServerCert    = "/etc/pki/inframanager/certs/server/tls.crt"
-	ManagerDefaultServerKey     = "/etc/pki/inframanager/certs/server/tls.key"
-	ManagerDefaultCACert        = "/etc/pki/inframanager/certs/client/ca.crt"
+	AgentDefaultClientCert      = "/etc/pki/infraagent/client/tls.crt"
+	AgentDefaultClientKey       = "/etc/pki/infraagent/client/tls.key"
+	AgentDefaultCACert          = "/etc/pki/infraagent/client/ca.crt"
+	ManagerDefaultClientCert    = "/etc/pki/inframanager/client/tls.crt"
+	ManagerDefaultClientKey     = "/etc/pki/inframanager/client/tls.key"
+	ManagerDefaultServerCert    = "/etc/pki/inframanager/server/tls.crt"
+	ManagerDefaultServerKey     = "/etc/pki/inframanager/server/tls.key"
+	ManagerDefaultCACert        = "/etc/pki/inframanager/client/ca.crt"
 	IfTtype                     = "cdq"
 )
 

diff --git a/scripts/es2k/setup_arm_infra.sh b/scripts/es2k/setup_arm_infra.sh
@@ -40,11 +40,6 @@ function run_infrap4d () {
   export PATH=$P4CP_INSTALL/bin:$P4CP_INSTALL/sbin:$PATH
   export SDE_INSTALL=/opt/p4/p4sde
   export LD_LIBRARY_PATH=$P4CP_INSTALL/lib:$P4CP_INSTALL/lib64:$SDE_INSTALL/lib64:$SDE_INSTALL/lib:/usr/lib64:/usr/lib:/usr/local/lib64:/usr/local/lib
-  # export-n -  unset DEBUGINFOD_URLS
-  # gdb --args $P4CP_INSTALL/install/sbin/infrap4d -grpc_open_insecure_mode=true --nodetach
-  # Bug - Infrap4d - copy certificates to another dir
-  mkdir -p /usr/share/stratum/certs
-  cp -r /usr/share/stratum/es2k/certs/* /usr/share/stratum/certs
   $P4CP_INSTALL/sbin/infrap4d
   check_status $? "sbin/infrap4d"
 }

diff --git a/scripts/es2k/setup_infra.sh b/scripts/es2k/setup_infra.sh
@@ -111,13 +111,11 @@ function copy_certs() {
         echo "stratum directory not found."
         exit 1
     fi
-    mkdir -p $STRATUM_DIR/es2k/certs
     rm -rf $STRATUM_DIR/certs/ca.crt
     rm -rf $STRATUM_DIR/certs/client.crt
     rm -rf $STRATUM_DIR/certs/client.key
     rm -rf $STRATUM_DIR/certs/stratum.crt
     rm -rf $STRATUM_DIR/certs/stratum.key
-    cp $BASE_DIR/scripts/tls/certs/infrap4d/* /usr/share/stratum/es2k/certs/.
     # infrap4d bug workaround
     mkdir -p /usr/share/stratum/certs
     cp $BASE_DIR/scripts/tls/certs/infrap4d/* /usr/share/stratum/certs/.
@@ -136,17 +134,17 @@ function copy_cert_to_remote() {
 
   # setup directory structure on ACC for p4infrad and manager certs
   #launch_on_remote "/usr/share/stratum/es2k/generate-certs.sh" ""
-  launch_on_remote "mkdir -p /usr/share/stratum/es2k/certs" ""
-  launch_on_remote "mkdir -p /etc/pki/inframanager/certs" ""
+  launch_on_remote "mkdir -p /usr/share/stratum/certs" ""
+  launch_on_remote "mkdir -p /etc/pki/inframanager" ""
 
   if [ -d "$BASE_DIR/scripts/tls/certs/infrap4d" ]; then
     # copy certs to remote infrap4d dir
-    scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $BASE_DIR/scripts/tls/certs/infrap4d/* $REMOTE_HOST:/usr/share/stratum/es2k/certs
+    scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $BASE_DIR/scripts/tls/certs/infrap4d/* $REMOTE_HOST:/usr/share/stratum/certs
     check_status $? "scp infrap4d/certs/* root@$REMOTE_HOST:/usr/share/stratum/certs"
 
     # copy certs to remote inframanager dir
-    scp -r -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $BASE_DIR/scripts/tls/certs/inframanager/* $REMOTE_HOST:/etc/pki/inframanager/certs
-    check_status $? "scp inframanager/certs/* root@$REMOTE_HOST:/etc/pki/inframanager/certs"
+    scp -r -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $BASE_DIR/scripts/tls/certs/inframanager/* $REMOTE_HOST:/etc/pki/inframanager/.
+    check_status $? "scp $BASE_DIR/scripts/tls/certs/inframanager/* root@$REMOTE_HOST:/etc/pki/inframanager/"
   else
     echo "Missing infrap4d certificates. Run \"make gen-certs\" and try again."
     exit 1
@@ -259,11 +257,13 @@ usage() {
   echo "Configure and setup k8s infrastructure for deployment"
   echo ""
   echo "Options:"
-  echo "  -i  Num interfaces to configure for deployment"
+  echo "  -i  Num interfaces to configure for the deployment.
+              The max limit depends on IPU configuration setting for this host.
+              Recommended min is 8."
   echo "  -m  Mode host or split, depending on where Inframanager is configured to run"
   echo "  -r  IP address configured by the user on the ACC-ARM complex for
     connectivity to the Host. This is provisioned using Node Policy - comms
-    channel "[[0,3],[4,2]]". This must be specified in split mode. Script will assign
+    channel "([5,0],[4,0]),([4,2],[0,3])". This must be specified in split mode. Script will assign
     an IP addresss from the same subnet on the Host side for connectivity."
   echo ""
   echo " Please set following env variables for host deployment:"

diff --git a/scripts/tls/gen_certs.sh b/scripts/tls/gen_certs.sh
@@ -17,7 +17,7 @@ mkdir -p $MGR_CLIENT
 mkdir -p $MGR_SERVER
 mkdir -p $INFRAP4D
 
-# Create common self-signed CA cert 
+# Create common self-signed CA cert
 openssl req -x509                                          \
   -newkey rsa:4096                                         \
   -nodes                                                   \
@@ -58,29 +58,6 @@ openssl x509 -req                                       \
 openssl verify -verbose -CAfile $CERTS/ca.crt $MGR_SERVER/tls.crt
 rm $MGR_SERVER/server.csr
 
-# Generate inframanager client csr and sign it
-# with CA.
-openssl genrsa -out $MGR_CLIENT/tls.key 4096
-openssl req -new                                        \
-  -key $MGR_CLIENT/tls.key                              \
-  -out $MGR_CLIENT/client.csr                           \
-  -subj /C=US/ST=CA/L=SJ/O=IPDK/CN=inframanager-server/ \
-  -config $OPENSSL_CNF                                  \
-  -reqexts v3_server
-openssl x509 -req                                       \
-  -in $MGR_CLIENT/client.csr                            \
-  -CAkey $CERTS/ca.key                                  \
-  -CA $CERTS/ca.crt                                     \
-  -days 365                                             \
-  -set_serial 1001                                      \
-  -out $MGR_CLIENT/tls.crt                              \
-  -extfile $OPENSSL_CNF                                 \
-  -extensions v3_server                                 \
-  -sha384
-openssl verify -verbose -CAfile $CERTS/ca.crt $MGR_CLIENT/tls.crt
-rm $MGR_CLIENT/client.csr
-
-
 # Generate infraagent client csr and sign it
 # with CA.
 openssl genrsa -out $AGENT_CLIENT/tls.key 4096
@@ -128,23 +105,44 @@ rm $INFRAP4D/stratum.csr
 # Generate infrap4d client csr and sign it
 # with CA.
 openssl genrsa -out $INFRAP4D/client.key 4096
-openssl req -new                                                  \
-  -key $INFRAP4D/client.key                                       \
-  -out $INFRAP4D/client.csr                                       \
-  -subj /C=US/ST=CA/L=SJ/O=IPDK/CN="Stratum client certificate"/  \
-  -config $OPENSSL_CNF                                            \
+openssl req -new                                        \
+  -key $INFRAP4D/client.key                             \
+  -out $INFRAP4D/client.csr                             \
+  -subj /C=US/ST=CA/L=SJ/O=IPDK/CN="stratum-client"/    \
+  -config $OPENSSL_CNF                                  \
   -reqexts v3_stratum_server
-openssl x509 -req                                                 \
-  -in $INFRAP4D/client.csr                                        \
-  -CAkey $CERTS/ca.key                                            \
-  -CA $CERTS/ca.crt                                               \
-  -days 365                                                       \
-  -set_serial 1004                                                \
-  -out $INFRAP4D/client.crt                                       \
+openssl x509 -req                                       \
+  -in $INFRAP4D/client.csr                              \
+  -CAkey $CERTS/ca.key                                  \
+  -CA $CERTS/ca.crt                                     \
+  -days 365                                             \
+  -set_serial 1004                                      \
+  -out $INFRAP4D/client.crt                             \
   -extfile $OPENSSL_CNF                                 \
   -extensions v3_stratum_server                         \
   -sha512
 openssl verify -verbose -CAfile $CERTS/ca.crt $INFRAP4D/client.crt
 rm $INFRAP4D/client.csr
 
+# Generate infrap4d client csr and sign it
+# with CA.
+openssl genrsa -out $MGR_CLIENT/tls.key 4096
+openssl req -new                                        \
+  -key $MGR_CLIENT/tls.key                              \
+  -out $MGR_CLIENT/tls.csr                              \
+  -subj /C=US/ST=CA/L=SJ/O=IPDK/CN="stratum-client"/    \
+  -config $OPENSSL_CNF                                  \
+  -reqexts v3_stratum_server
+openssl x509 -req                                       \
+  -in $MGR_CLIENT/tls.csr                               \
+  -CAkey $CERTS/ca.key                                  \
+  -CA $CERTS/ca.crt                                     \
+  -days 365                                             \
+  -set_serial 1005                                      \
+  -out $MGR_CLIENT/tls.crt                              \
+  -extfile $OPENSSL_CNF                                 \
+  -extensions v3_stratum_server                         \
+  -sha512
+openssl verify -verbose -CAfile $CERTS/ca.crt $MGR_CLIENT/tls.crt
+rm $MGR_CLIENT/tls.csr
 
diff --git a/scripts/tls/openssl.cnf b/scripts/tls/openssl.cnf
@@ -22,17 +22,21 @@ subjectAltName          = @server_alt_names
 basicConstraints        = critical,CA:FALSE
 subjectKeyIdentifier    = hash
 keyUsage                = critical,digitalSignature,keyEncipherment,keyAgreement
-subjectAltName          = DNS:localhost
+subjectAltName          = @client_alt_names
 
 [server_alt_names]
-DNS.1 = *.intel.com
-DNS.2 = k8s
+DNS.1 = localhost
+DNS.2 = localhost.localdomain
 DNS.3 = kubernetes.default
 IP.1  = 127.0.0.1
 IP.2  = 10.10.0.2
+IP.3  = ::1
+IP.4  = fe80::1
 
-[v3_client]
-basicConstraints        = critical,CA:FALSE
-subjectKeyIdentifier    = hash
-keyUsage                = critical,nonRepudiation,digitalSignature,keyEncipherment
-extendedKeyUsage        = critical,clientAuth
+[client_alt_names]
+DNS.1 = localhost
+DNS.2 = localhost.localdomain
+DNS.3 = kubernetes.default
+IP.1  = 127.0.0.1
+IP.2  = ::1
+IP.3  = fe80::1