Merge pull request openshift#7656 from sallyom/fips-check-no-skip-verify

installer-e2e template: remove the -insecure-skip-tls-verify from fips check

ci-operator/config/openshift/origin/openshift-origin-master.yaml

@@ -194,10 +194,6 @@ tests:
   commands: TEST_SUITE=openshift/conformance/parallel run-tests
   openshift_installer:
     cluster_profile: gcp
-- as: e2e-aws
-  steps:
-    cluster_profile: aws
-    workflow: origin-e2e-aws-loki
 - as: e2e-aws-ovn
   steps:
     cluster_profile: aws

ci-operator/jobs/openshift/origin/openshift-origin-master-presubmits.yaml

@@ -48,81 +48,6 @@ presubmits:
         secret:
           secretName: regcred
     trigger: (?m)^/test( | .* )artifacts,?($|\s.*)
-  - agent: kubernetes
-    always_run: false
-    branches:
-    - master
-    cluster: api.ci
-    context: ci/prow/e2e-aws
-    decorate: true
-    decoration_config:
-      skip_cloning: true
-    labels:
-      ci-operator.openshift.io/prowgen-controlled: "true"
-      pj-rehearse.openshift.io/can-be-rehearsed: "true"
-    name: pull-ci-openshift-origin-master-e2e-aws
-    rerun_command: /test e2e-aws
-    spec:
-      containers:
-      - args:
-        - --artifact-dir=$(ARTIFACTS)
-        - --give-pr-author-access-to-namespace=true
-        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
-        - --kubeconfig=/etc/apici/kubeconfig
-        - --lease-server-password-file=/etc/boskos/password
-        - --report-password-file=/etc/report/password.txt
-        - --report-username=ci
-        - --secret-dir=/usr/local/e2e-aws-cluster-profile
-        - --target=e2e-aws
-        command:
-        - ci-operator
-        image: ci-operator:latest
-        imagePullPolicy: Always
-        name: ""
-        resources:
-          requests:
-            cpu: 10m
-        volumeMounts:
-        - mountPath: /etc/apici
-          name: apici-ci-operator-credentials
-          readOnly: true
-        - mountPath: /etc/boskos
-          name: boskos
-          readOnly: true
-        - mountPath: /usr/local/e2e-aws-cluster-profile
-          name: cluster-profile
-        - mountPath: /etc/pull-secret
-          name: pull-secret
-          readOnly: true
-        - mountPath: /etc/report
-          name: result-aggregator
-          readOnly: true
-      serviceAccountName: ci-operator
-      volumes:
-      - name: apici-ci-operator-credentials
-        secret:
-          items:
-          - key: sa.ci-operator.apici.config
-            path: kubeconfig
-          secretName: apici-ci-operator-credentials
-      - name: boskos
-        secret:
-          items:
-          - key: password
-            path: password
-          secretName: boskos-credentials
-      - name: cluster-profile
-        projected:
-          sources:
-          - secret:
-              name: cluster-secrets-aws
-      - name: pull-secret
-        secret:
-          secretName: regcred
-      - name: result-aggregator
-        secret:
-          secretName: result-aggregator
-    trigger: (?m)^/test( | .* )e2e-aws,?($|\s.*)
   - agent: kubernetes
     always_run: true
     branches:
@@ -317,7 +242,6 @@ presubmits:
     labels:
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
     name: pull-ci-openshift-origin-master-e2e-aws-fips
-    optional: true
     rerun_command: /test e2e-aws-fips
     spec:
       containers:

ci-operator/templates/openshift/installer/cluster-launch-installer-e2e.yaml

@@ -159,23 +159,40 @@ objects:
         trap 'jobs -p | xargs -r kill || true; exit 0' TERM
 
         function fips_check() {
-          oc --insecure-skip-tls-verify --request-timeout=60s get nodes -o jsonpath --template '{range .items[*]}{.metadata.name}{"\n"}{end}' > /tmp/nodelist
-          while IFS= read -r i; do
-            oc -n default --insecure-skip-tls-verify --request-timeout=60s debug --image centos:7 node/$i -- cat /proc/sys/crypto/fips_enabled > /tmp/enabled
+          get_nodes=$(oc --request-timeout=60s get nodes -o jsonpath --template '{range .items[*]}{.metadata.name}{"\n"}{end}')
+          nodes=( $get_nodes )
+          # bash doesn't handle '.' in array elements easily
+          for i in {0..5}; do
+            attempt=0
+            while true; do
+                out=$(oc --request-timeout=60s -n default debug node/"${nodes[i]}" -- cat /proc/sys/crypto/fips_enabled || true)
+                if [[ ! -z "${out}" ]]; then
+                    break
+                fi
+                attempt=$(( attempt + 1 ))
+                if [[ $attempt -gt 3 ]]; then
+                    break
+                fi
+                echo "command failed, $(( 4 - $attempt )) retries left"
+                sleep 5
+            done
+
+            if [[ -z "${out}" ]]; then
+              echo "oc debug node/${nodes[i]} failed"
+              exit 1
+            fi
             if [[ "${CLUSTER_VARIANT}" =~ "fips" ]]; then
-              if [[ $(< /tmp/enabled) == "0" ]]; then
-                echo fips not enabled in node "$i" but should be, exiting
+              if [[ "${out}" -ne 1 ]]; then
+                echo "fips not enabled in node ${nodes[i]} but should be, exiting"
                 exit 1
               fi
             else
-              if [[ $(< /tmp/enabled) == "1" ]]; then
-                echo fips is enabled in node "$i" but should not be, exiting
+              if [[ "${out}" -ne 0 ]]; then
+                echo "fips is enabled in node ${nodes[i]} but should not be, exiting"
                 exit 1
               fi
             fi
-          done </tmp/nodelist
-          rm -f /tmp/nodelist
-          rm -f /tmp/enabled
+          done
         }
 
         function patch_image_specs() {