smt_via_label via machine config

Fixes: ocp-power-automation#46
Signed-off-by: Sebastien Chabrolles <[email protected]>

examples/install_vars.yaml

@@ -28,3 +28,6 @@ chronyconfig:
        options: iburst
      - server: ntp2.example.com
        options: iburst
+
+powervm_rmc: true
+smt_control: false

playbooks/roles/ocp-customization/README.md

@@ -24,8 +24,9 @@ Role Variables
 |-------------------------|----------|----------------|---------------------------------------------|
 | workdir                 | no       | ~/ocp4-workdir | Place for config generation and auth files  |
 | rhcos_kernel_options    | no       | []             | List of kernel options for RHCOS nodes eg: ["slub_max_order=0","loglevel=7"] |
-| sysctl_tuned_options    | no       | false       | Set to true to apply sysctl options via tuned operator |
-| powervm_rmc             | no       | true           | Set to true to deploy RMC daemonset on Node with arch ppc64le |
+| sysctl_tuned_options    | no       | false           | Set to true to apply sysctl options via tuned operator |
+| powervm_rmc             | no       | true            | Set to true to deploy RMC daemonset on Node with arch ppc64le |
+| smt_control             | no       | false           | Set to true to control SMT mode with node label SMT=(1,2,4,8) |
 
 
 If `sysctl_tuned_options` is true then the following variables are must and should be set in [vars/tuned.yaml](./vars/tuned.yaml)

playbooks/roles/ocp-customization/defaults/main.yaml

@@ -5,4 +5,4 @@ workdir: ~/ocp4-workdir
 rhcos_kernel_options: []
 sysctl_tuned_options: false
 powervm_rmc: true
-
+smt_control: false

playbooks/roles/ocp-customization/files/powersmt

@@ -0,0 +1,64 @@
+#!/bin/bash
+export PATH=/root/.local/bin:/root/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin
+export KUBECONFIG=/var/lib/kubelet/kubeconfig
+COREPS=$(/bin/lscpu | /bin/awk -F: ' $1 ~ /^Core\(s\) per socket$/ {print $2}'|/bin/xargs)
+SOCKETS=$(/bin/lscpu | /bin/awk -F: ' $1 ~ /^Socket\(s\)$/ {print $2}'|/bin/xargs)
+let TOTALCORES=$COREPS*$SOCKETS
+MAXTHREADS=$(/bin/lscpu | /bin/awk -F: ' $1 ~ /^CPU\(s\)$/ {print $2}'|/bin/xargs)
+let MAXSMT=$MAXTHREADS/$TOTALCORES
+CURRENTSMT=$(/bin/lscpu | /bin/awk -F: ' $1 ~ /^Thread\(s\) per core$/ {print $2}'|/bin/xargs)
+
+while :
+do
+  SMTLABEL=$(/bin/oc get node $HOSTNAME -L SMT --no-headers |/bin/awk '{print $6}')
+  if [[ -n $SMTLABEL ]]
+    then
+      case $SMTLABEL in
+        1) TARGETSMT=1
+      ;;
+        2) TARGETSMT=2
+      ;;
+        4) TARGETSMT=4
+      ;;
+        8) TARGETSMT=8
+      ;;
+        *) TARGETSMT=$CURRENTSMT ; echo "SMT value must be 1, 2, 4, or 8 and smaller than Maximum SMT."
+      ;;
+      esac
+    else
+      TARGETSMT=$MAXSMT
+  fi
+
+  CURRENTSMT=$(/bin/lscpu | /bin/awk -F: ' $1 ~ /^Thread\(s\) per core$/ {print $2}'|/bin/xargs)
+
+  if [[ $CURRENTSMT -ne $TARGETSMT ]]
+    then
+      INITONTHREAD=0
+      INITOFFTHREAD=$TARGETSMT
+      if [[ $MAXSMT -ge $TARGETSMT ]]
+        then
+          while [[ $INITONTHREAD -lt $MAXTHREADS ]]
+          do
+            ONTHREAD=$INITONTHREAD
+            OFFTHREAD=$INITOFFTHREAD
+
+            while [[ $ONTHREAD -lt $OFFTHREAD ]]
+            do
+              /bin/echo 1 > /sys/devices/system/cpu/cpu$ONTHREAD/online
+              let ONTHREAD=$ONTHREAD+1
+            done
+            let INITONTHREAD=$INITONTHREAD+$MAXSMT
+            while [[ $OFFTHREAD -lt $INITONTHREAD ]]
+            do
+              /bin/echo 0 > /sys/devices/system/cpu/cpu$OFFTHREAD/online
+              let OFFTHREAD=$OFFTHREAD+1
+            done
+            let INITOFFTHREAD=$INITOFFTHREAD+$MAXSMT
+          done
+        systemctl restart kubelet
+        else
+          echo "Target SMT must be smaller or equal than Maximum SMT supported"
+      fi
+  fi
+  /bin/sleep 30
+done

playbooks/roles/ocp-customization/files/powersmt_service

@@ -0,0 +1,8 @@
+[Unit]
+Description=SMT control via k8s SMT label
+
+[Service]
+ExecStart=/usr/local/bin/powersmt
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/ocp-customization/tasks/main.yaml

@@ -38,6 +38,11 @@
       when: powervm_rmc
       import_tasks: powervm_rmc.yaml
 
+    - name: Allow SMT configuration via node label (SMT=X)
+      when: ansible_architecture == "ppc64le"
+      import_tasks: smt_configuration.yaml
+      when: smt_control
+
   always:
     - name: UnPause reboot node (Machineconfig)
       k8s:

playbooks/roles/ocp-customization/tasks/smt_configuration.yaml

@@ -0,0 +1,34 @@
+- name: Copy files to bastion
+  copy:
+    src: "{{ item }}"
+    dest: "/tmp"
+  loop:
+    - powersmt
+    - powersmt_service
+
+- name: slurp powersmt to base64
+  slurp:
+    src: "/tmp/powersmt"
+  register: powersmt_b64
+
+- name: slurp powersmt_service to base64
+  slurp:
+    src: "/tmp/powersmt_service"
+  register: powersmt_service_b64
+
+- name: Create SMT via Label options CRDs
+  vars:
+    role: '{{ item }}'
+  template:
+    src: 90-smt-via-label.yaml
+    dest: "{{ workdir }}/90-{{ item }}-smt-via-label.yaml"
+  with_items:
+    - master
+    - worker
+
+- name: Apply the MachineConfig CRDs
+  shell: "oc apply -f {{ workdir }}/90-{{ item }}-smt-via-label.yaml"
+  with_items:
+    - master
+    - worker
+

playbooks/roles/ocp-customization/templates/90-smt-via-label.yaml

@@ -0,0 +1,28 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: '{{ role }}'
+  name: 90-{{ role }}-smt
+spec:
+  config:
+    ignition:
+      version: 2.2.0
+    storage:
+      files:
+        - path: /usr/local/bin/powersmt
+          overwrite: true
+          mode: 0700
+          filesystem: root
+          contents:
+            source: data:text/plain;charset=utf-8;base64,{{ powersmt_b64.content }}
+        - path: /etc/systemd/system/powersmt.service
+          overwrite: true
+          mode: 0644
+          filesystem: root
+          contents:
+            source: data:text/plain;charset=utf-8;base64,{{ powersmt_service_b64.content }}
+    systemd:
+      units:
+        - name: powersmt.service
+          enabled: true