Report results after charm func tests

Report contains info about whether tests are voting or not and
also the commit id the tests were run against.

openstack/tools/charmed_openstack_functest_runner.sh

@@ -9,6 +9,7 @@ FUNC_TEST_PR=
 FUNC_TEST_TARGET=
 MODIFY_BUNDLE_CONSTRAINTS=true
 SKIP_BUILD=false
+WAIT_ON_DESTROY=true
 
 usage () {
     cat << EOF
@@ -32,6 +33,9 @@ OPTIONS:
     --func-test-pr
         Provides similar functionality to Func-Test-Pr in commit message. Set
         to zaza-openstack-tests Pull Request ID.
+    --no-wait
+        By default we wait before destroying the model after a test run. This
+        flag can used to override that behaviour.
     --skip-build
         Skip building charm if already done to save time.
     --skip-modify-bundle-constraints
@@ -57,6 +61,9 @@ do
            FUNC_TEST_PR=$2
            shift
            ;;
+        --no-wait)
+           WAIT_ON_DESTROY=false
+           ;;
         --skip-modify-bundle-constraints)
           MODIFY_BUNDLE_CONSTRAINTS=false
           ;;
@@ -76,6 +83,15 @@ do
     shift
 done
 
+TOOLS_PATH=$(realpath $(dirname $0))/func_test_tools
+CHARM_PATH=$(pwd)
+
+# Get commit we are running tests against.
+COMMIT_ID=$(git -C $CHARM_PATH rev-parse --short HEAD)
+CHARM_NAME=$(egrep -o "^name: .+" metadata.yaml| cut -f 2 -d ' ')
+
+echo "Running functional tests for charm $CHARM_NAME commit $COMMIT_ID"
+
 source ~/novarc
 export {,TEST_}CIDR_EXT=`openstack subnet show subnet_${OS_USERNAME}-psd-extra -c cidr -f value`
 FIP_MAX=$(ipcalc $CIDR_EXT| awk '$1=="HostMax:" {print $2}')
@@ -135,10 +151,13 @@ if [[ -n $FUNC_TEST_PR ]]; then
     )
 fi
 
+declare -A func_targets=()
 if [[ -n $FUNC_TEST_TARGET ]]; then
-  func_targets=( $FUNC_TEST_TARGET )
+    func_targets[$FUNC_TEST_TARGET]=null
 else
-  func_targets=( $(python3 $(realpath $(dirname $0))/identify_charm_func_tests.py) )
+    for target in $(python3 $TOOLS_PATH/identify_charm_func_tests.py); do
+        func_targets[target]=null
+    done
 fi
 
 if $MODIFY_BUNDLE_CONSTRAINTS; then
@@ -148,13 +167,40 @@ if $MODIFY_BUNDLE_CONSTRAINTS; then
     )
 fi
 
-for target in ${func_targets[@]}; do
+for target in ${!func_targets[@]}; do
     [[ -d src ]] && pushd src || true
-    tox -re func-target -- $target
-
-    read -p "Destroy model and run next test? [ENTER]"
+    fail=false
+    tox -re func-target -- $target || fail=true
+
+    if $fail; then
+        func_targets[target]='fail'
+    else
+        func_targets[target]='success'
+    fi
+
+    if $WAIT_ON_DESTROY; then
+        read -p "Destroy model and run next test? [ENTER]"
+    fi
     # cleanup before next run
     model=`juju list-models| egrep -o "^zaza-\S+"|tr -d '*'`
     juju destroy-model --no-prompt $model --force --no-wait --destroy-storage
 done
 
+# Report results
+echo "Test results for charm $CHARM_NAME functional tests @ commit $COMMIT_ID:"
+for target in ${!func_targets[@]}; do
+    if $(python3 $TOOLS_PATH/test_is_voting.py) $target; then
+        voting_info=" (non-voting)"
+    else
+        voting_info=""
+    fi
+
+    if [[ ${func_targets[$target]} = null ]]; then
+        echo "  * $target: SKIPPED$voting_info"
+    elif [[ ${func_targets[$target]} = success ]]; then
+        echo "  * $target: SUCCESS$voting_info"
+    else
+        echo "  * $target: FAILURE$voting_info"
+    fi
+done
+

openstack/tools/func_test_tools/test_is_voting.py

@@ -0,0 +1,34 @@
+"""
+Takes a func test target name as input and exits 0 if the test is voting and
+1 if not.
+"""
+import os
+import sys
+
+import yaml
+
+if __name__ == "__main__":
+    target_name = sys.argv[1]
+    if not os.path.exists('osci.yaml'):
+        sys.stderr.write(f"ERROR: osci.yaml not found - assuming "
+                         f"{target_name} is voting.\n")
+        sys.exit(0)
+
+    with open('osci.yaml', encoding='utf-8') as fd:
+        osci_config = yaml.safe_load(fd)
+
+    try:
+        jobs = osci_config[0]['project']['check']['jobs']
+        if target_name in jobs:
+            # default is voting=True
+            sys.exit(0)
+
+        for check in jobs:
+            if isinstance(check, dict) and target_name in check:
+                if not check[target_name]['voting']:
+                    sys.exit(1)
+    except KeyError as exc:
+        sys.stderr.write(f"ERROR: failed to process osci.yaml - assuming "
+                         f"{target_name} is voting (key {exc} not found).\n")
+
+    sys.exit(0)

tox.ini

@@ -19,7 +19,7 @@ import-order-style = pep8
 [testenv]
 basepython = {env:TOX_PYTHON:python3}
 pyfiles =
-    {toxinidir}/openstack/tools/identify_charm_func_tests.py
+    {toxinidir}/openstack/tools/func_test_tools
     {toxinidir}/tools/parse-bundle.py
     {toxinidir}/tools/juju-bundle-applications.py