Report results after charm func tests

Report contains info about whether tests are voting or not and
also the commit id the tests were run against.

openstack/tools/charmed_openstack_functest_runner.sh

@@ -76,6 +76,15 @@ do
     shift
 done
 
+TOOLS_PATH=$(realpath $(dirname $0))/func_test_tools
+CHARM_PATH=$(pwd)
+
+# Get commit we are running tests against.
+COMMIT_ID=$(git -C $CHARM_PATH rev-parse --short HEAD)
+CHARM_NAME=$(egrep -o "^name: .+" metadata.yaml| cut -f 2 -d ' ')
+
+echo "Running functional tests for charm $CHARM_NAME commit $COMMIT_ID"
+
 source ~/novarc
 export {,TEST_}CIDR_EXT=`openstack subnet show subnet_${OS_USERNAME}-psd-extra -c cidr -f value`
 FIP_MAX=$(ipcalc $CIDR_EXT| awk '$1=="HostMax:" {print $2}')
@@ -135,10 +144,13 @@ if [[ -n $FUNC_TEST_PR ]]; then
     )
 fi
 
+declare -A func_targets=()
 if [[ -n $FUNC_TEST_TARGET ]]; then
-  func_targets=( $FUNC_TEST_TARGET )
+    func_targets[$FUNC_TEST_TARGET]=null
 else
-  func_targets=( $(python3 $(realpath $(dirname $0))/identify_charm_func_tests.py) )
+    for target in $(python3 $TOOLS_PATH/identify_charm_func_tests.py); do
+        func_targets[target]=null
+    done
 fi
 
 if $MODIFY_BUNDLE_CONSTRAINTS; then
@@ -148,13 +160,36 @@ if $MODIFY_BUNDLE_CONSTRAINTS; then
     )
 fi
 
-for target in ${func_targets[@]}; do
+for target in ${!func_targets[@]}; do
     [[ -d src ]] && pushd src || true
     tox -re func-target -- $target
 
+    if (($?==0)); then
+        func_targets[target]='success'
+    else:
+        func_targets[target]='fail'
+    fi
+
     read -p "Destroy model and run next test? [ENTER]"
     # cleanup before next run
     model=`juju list-models| egrep -o "^zaza-\S+"|tr -d '*'`
     juju destroy-model --no-prompt $model --force --no-wait --destroy-storage
 done
 
+# Report results
+echo "Test results for charm $CHARM_NAME functional tests @ commit $COMMIT_ID:"
+for target in ${!func_targets[@]}; do
+    if $(python3 $TOOLS_PATH/test_is_voting.py) $target; then
+        voting_info=" (non-voting)"
+    else
+        voting_info=""
+
+    if [[ ${func_targets[$target]} = null ]]; then
+        echo "  * $target: SKIPPED$voting_info"
+    elif [[ ${func_targets[$target]} = success ]]; then
+        echo "  * $target: SUCCESS$voting_info"
+    else
+        echo "  * $target: FAILURE$voting_info"
+    fi
+done
+

openstack/tools/func_test_tools/test_is_voting.py

@@ -0,0 +1,34 @@
+"""
+Takes a func test target name as input and exits 0 if the test is voting and
+1 if not.
+"""
+import os
+import sys
+
+import yaml
+
+if __name__ == "__main__":
+    target_name = sys.argv[1]
+    if not os.path.exists('osci.yaml'):
+        sys.stderr.write(f"ERROR: osci.yaml not found - assuming "
+                         f"{target_name} is voting.\n")
+        sys.exit(0)
+
+    with open('osci.yaml', encoding='utf-8') as fd:
+        osci_config = yaml.safe_load(fd)
+
+    try:
+        jobs = osci_config[0]['project']['check']['jobs']
+        if target_name in jobs:
+            # default is voting=True
+            sys.exit(0)
+
+        for check in jobs:
+            if isinstance(check, dict) and target_name in check:
+                if not check[target_name]['voting']:
+                    sys.exit(1)
+    except KeyError as exc:
+        sys.stderr.write(f"ERROR: failed to process osci.yaml - assuming "
+                         f"{target_name} is voting (key {exc} not found).\n")
+
+    sys.exit(0)

tox.ini

@@ -19,7 +19,7 @@ import-order-style = pep8
 [testenv]
 basepython = {env:TOX_PYTHON:python3}
 pyfiles =
-    {toxinidir}/openstack/tools/identify_charm_func_tests.py
+    {toxinidir}/openstack/tools/func_test_tools
     {toxinidir}/tools/parse-bundle.py
     {toxinidir}/tools/juju-bundle-applications.py