Shelve the upstream bootstrap process

.gitignore

@@ -39,6 +39,7 @@ logs/*
 .Spotlight-V100
 .Trashes
 .idea
+.vscode
 .tox
 *.sublime*
 *.egg-info

scripts/bootstrap-embedded-ansible/README.rst

@@ -0,0 +1,43 @@
+Create an embedded Ansible runtime
+##################################
+:tags: embedded, ansible
+
+
+About this repository
+---------------------
+
+The embedded ansible script will create an ansible runtime within the users home folder.
+This ansible runtime will be within a virtual envrionment and have all of the plugins
+required to run ansible standalone or in an OpenStack-Ansible compatible envrionment.
+
+
+Usage
+^^^^^
+
+.. code-block:: bash
+
+   source bootstrap-embedded-ansible.sh
+
+
+With the script sourced, the ansible environment will create a virtual environment at
+`${HOME}/ansible_venv` if it does not already exist.
+
+To leave the embedded ansible environment run the function `deactivate`.
+
+
+Options
+^^^^^^^
+
+All options are passed in using environment variables.
+
+ANSIBLE_VERSION:
+  Allows for the Ansible XXX to be overridden. When set the full ansible version is required.
+
+ANSIBLE_EMBED_HOME:
+  Allows for the Ansible XXX to be overridden. When set the full path is required.
+
+ANSIBLE_ROLE_REQUIREMENTS:
+  Allows for the Ansible XXX to be overridden. When set the full path to the role requirements file is required.
+
+ANSIBLE_PYTHON_REQUIREMENTS:
+  Allows for the Ansible XXX to be overridden. When set the full path to the python requirements file is required.

scripts/bootstrap-embedded-ansible/ansible-requirements.yml

@@ -0,0 +1,17 @@
+---
+- name: config_template
+  scm: git
+  src: https://opendev.org/openstack/ansible-config_template
+  version: d97279293d09ab3cae6e20119662029d6b64b3b7
+- name: plugins
+  scm: git
+  src: https://opendev.org/openstack/openstack-ansible-plugins
+  version: 7fd6dd21123f162091631d87bf1db8fb7700fcda
+- name: systemd_service
+  scm: git
+  src: https://opendev.org/openstack/ansible-role-systemd_service
+  version: 86ad639f4171b0c01bba030ebc3fd96ec020aa45
+- name: systemd_mount
+  scm: git
+  src: https://opendev.org/openstack/ansible-role-systemd_mount
+  version: master

scripts/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Check if embedded ansible is already activated. If it is active, deactivate it.
+(alias deactivate &> /dev/null && deactivate) || true
+
+export OPTS=()
+export CLONE_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+OPTS+=('CLONE_DIR')
+
+export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.7.11}"
+OPTS+=('ANSIBLE_VERSION')
+
+export ANSIBLE_EMBED_HOME="${HOME}/ansible_venv"
+OPTS+=('ANSIBLE_EMBED_HOME')
+
+export ANSIBLE_ROLE_REQUIREMENTS="${ANSIBLE_ROLE_REQUIREMENTS:-$CLONE_DIR/ansible-requirements.yml}"
+OPTS+=('ANSIBLE_ROLE_REQUIREMENTS')
+
+export ANSIBLE_PYTHON_REQUIREMENTS="${ANSIBLE_PYTHON_REQUIREMENTS:-${CLONE_DIR}/python-requirements.txt}"
+OPTS+=('ANSIBLE_PYTHON_REQUIREMENTS')
+
+source /etc/os-release
+export ID="$(echo ${ID} | awk -F'-' '{print $1}')"
+
+if [[ ! -e "${ANSIBLE_EMBED_HOME}/bin/ansible" ]]; then
+  if [ ${ID} = "ubuntu" ]; then
+    apt-get update
+    apt-get -y install python-virtualenv
+  elif [ ${ID} = "opensuse" ] || [ ${ID} = "suse" ]; then
+    zypper install -y python-virtualenv
+  elif [ ${ID} = "centos" ] || [ ${ID} = "redhat" ] || [ ${ID} = "rhel" ]; then
+    yum install -y python-virtualenv
+  else
+    echo "Unknown operating system"
+    exit 99
+  fi
+  echo "done installing python-virtualenv"
+  if [[ -f "/usr/bin/python2" ]]; then
+    virtualenv --system-site-packages --python="/usr/bin/python2" "${ANSIBLE_EMBED_HOME}"
+
+  elif [[ -f "/usr/bin/python3" ]]; then
+    virtualenv --system-site-packages --python="/usr/bin/python3" "${ANSIBLE_EMBED_HOME}"
+  else
+    virtualenv "${ANSIBLE_EMBED_HOME}"
+  fi
+  eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade --force pip setuptools\<45"
+  echo "Ansible can be found here: ${ANSIBLE_EMBED_HOME}/bin"
+fi
+
+# Run ansible setup
+eval "${ANSIBLE_EMBED_HOME}/bin/pip install --upgrade ansible=='${ANSIBLE_VERSION}' --isolated"
+eval "${ANSIBLE_EMBED_HOME}/bin/ansible-galaxy install --force --role-file='${ANSIBLE_ROLE_REQUIREMENTS}' --roles-path='${ANSIBLE_EMBED_HOME}/repositories/roles'"
+eval "${ANSIBLE_EMBED_HOME}/bin/ansible-playbook -i 'localhost,' '${CLONE_DIR}/embedded-ansible-setup.yml' -e 'ansible_venv_path=${ANSIBLE_EMBED_HOME}' -e 'ansible_python_requirement_file=${ANSIBLE_PYTHON_REQUIREMENTS}'"
+
+if [[ -f "/etc/openstack_deploy/openstack_inventory.json" ]]; then
+  export USER_VARS="$(for i in $(ls -1 /etc/openstack_deploy/user_*secret*.yml); do echo -n "-e@$i "; done)"
+  OPTS+=('USER_VARS')
+  echo "env USER_VARS set"
+
+  export USER_ALL_VARS="$(for i in $(ls -1 /etc/openstack_deploy/user_*.yml); do echo -n "-e@$i "; done)"
+  OPTS+=('USER_ALL_VARS')
+  echo "env USER_ALL_VARS set"
+
+  echo "Extra users variables can be expanded by including the option \$USER_VARS or \$USER_ALL_VARS on a playbook run."
+
+  export ANSIBLE_INVENTORY="${ANSIBLE_EMBED_HOME}/inventory/openstack_inventory.sh"
+  OPTS+=('ANSIBLE_INVENTORY')
+  echo "env ANSIBLE_INVENTORY set"
+fi
+
+export ANSIBLE_HOST_KEY_CHECKING="False"
+OPTS+=('ANSIBLE_HOST_KEY_CHECKING')
+echo "env ANSIBLE_HOST_KEY_CHECKING set"
+
+export ANSIBLE_ROLES_PATH="${ANSIBLE_EMBED_HOME}/repositories/roles"
+OPTS+=('ANSIBLE_ROLES_PATH')
+echo "env ANSIBLE_ROLES_PATH set"
+
+export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/roles/config_template/action"
+OPTS+=('ANSIBLE_ACTION_PLUGINS')
+echo "env ANSIBLE_ACTION_PLUGINS set"
+
+export ANSIBLE_CONNECTION_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/roles/plugins/connection"
+OPTS+=('ANSIBLE_CONNECTION_PLUGINS')
+echo "env ANSIBLE_CONNECTION_PLUGINS set"
+
+export ANSIBLE_STRATEGY_PLUGINS="${ANSIBLE_EMBED_HOME}/repositories/roles/plugins/strategy"
+OPTS+=('ANSIBLE_STRATEGY_PLUGINS')
+echo "env ANSIBLE_STRATEGY_PLUGINS set"
+
+export ANSIBLE_TRANSPORT="${ANSIBLE_TRANSPORT:-ssh}"
+OPTS+=('ANSIBLE_TRANSPORT')
+echo "env ANSIBLE_TRANSPORT set"
+
+export ANSIBLE_SSH_PIPELINING="${ANSIBLE_SSH_PIPELINING:-True}"
+OPTS+=('ANSIBLE_SSH_PIPELINING')
+echo "env ANSIBLE_SSH_PIPELINING set"
+
+export ANSIBLE_PIPELINING="${ANSIBLE_SSH_PIPELINING}"
+OPTS+=('ANSIBLE_PIPELINING')
+echo "env ANSIBLE_PIPELINING set"
+
+export ANSIBLE_SSH_RETRIES="${ANSIBLE_SSH_RETRIES:-5}"
+OPTS+=('ANSIBLE_SSH_RETRIES')
+echo "env ANSIBLE_SSH_RETRIES set"
+
+source ${ANSIBLE_EMBED_HOME}/bin/activate
+echo "Embedded Ansible has been activated. Run 'deactivate' to leave the embedded environment".
+
+function deactivate_embedded_venv {
+  deactivate
+  for i in ${OPTS[@]}; do
+    unset ${i}
+  done
+  unset deactivate_embedded_venv
+  unalias deactivate
+}
+
+alias deactivate=deactivate_embedded_venv

scripts/bootstrap-embedded-ansible/embedded-ansible-setup.yml

@@ -0,0 +1,34 @@
+---
+- name: Setup embedded ansible
+  hosts: localhost
+  connection: local
+  user: root
+  gather_facts: no
+  vars:
+    default_ansible_venv_path: "{{ lookup('env','HOME') + '/ansible_venv' }}"
+  tasks:
+    - name: Ensure pip packages are installed
+      pip:
+        requirements: "{{ ansible_python_requirement_file | default('python-requirements.txt') }}"
+        virtualenv: "{{ ansible_venv_path | default(default_ansible_venv_path) }}"
+        extra_args: '--isolated'
+
+    - name: Check for OSA inventory
+      stat:
+        path: /etc/openstack_deploy/openstack_inventory.json
+      register: osa_inventory
+
+    - name: osa block
+      block:
+        - name: Create inventory shim directory
+          file:
+            path: "{{ default_ansible_venv_path }}/inventory"
+            state: "directory"
+
+        - name: Copy embedded ansible inventory shim
+          copy:
+            dest: "{{ default_ansible_venv_path }}/inventory/openstack_inventory.sh"
+            src: osa-inventory.sh
+            mode: '0755'
+      when:
+        - osa_inventory.stat.exists | bool

scripts/bootstrap-embedded-ansible/osa-inventory.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+cat /etc/openstack_deploy/openstack_inventory.json

scripts/bootstrap-embedded-ansible/python-requirements.txt

@@ -0,0 +1,3 @@
+jmespath
+hvac
+netaddr

scripts/set-vars.sh

@@ -2,7 +2,7 @@
 
 # Set the compatible version of ansible for RPC Toolstack projects
 export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_PATH:-$HOME/magnanimous-turbo-chainsaw-$(date +%Y-%m-%d).log}"
-export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.7.5.0}"
+export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.7.11}"
 export ANSIBLE_GATHERING="${ANSIBLE_GATHERING:-smart}"
 export ANSIBLE_CACHE_PLUGIN="${ANSIBLE_CACHE_PLUGIN:-jsonfile}"
 export ANSIBLE_CACHE_PLUGIN_CONNECTION="${ANSIBLE_CACHE_PLUGIN_CONNECTION:-/tmp/mtc_facts-$(date +%Y-%m-%d)}"

scripts/setup-workspace.sh

@@ -38,7 +38,7 @@ function deactivate_workspace {
 }
 
 ## Main ----------------------------------------------------------------------
-pushd /opt/openstack-ansible-ops/bootstrap-embedded-ansible
+pushd "${MTC_SCRIPT_DIR}/bootstrap-embedded-ansible"
   source bootstrap-embedded-ansible.sh
 popd
 

scripts/setup.sh

@@ -36,14 +36,19 @@ function ssh_key_create {
 
 
 ## Main ----------------------------------------------------------------------
-if [[ ! -e $(which curl) ]]; then
+if [[ -f "/etc/os-release" ]]; then
+  source /etc/os-release
+  export ID="$(echo ${ID} | awk -F'-' '{print $1}')"
+fi
+
+if [[ ! -e $(which git) ]] || [[ ! -e $(which curl) ]]; then
   if [[ ${ID} = "ubuntu" ]]; then
     apt-get update
-    apt-get -y install curl
+    apt-get -y install curl git
   elif [[ ${ID} = "opensuse" ]] || [[ ${ID} = "suse" ]]; then
-    zypper install -y curl
+    zypper install -y curl git
   elif [[ ${ID} = "centos" ]] || [[ ${ID} = "redhat" ]] || [[ ${ID} = "rhel" ]]; then
-    yum install -y curl
+    yum install -y curl git
   else
     echo "Unknown operating system"
     exit 99
@@ -85,9 +90,18 @@ elif [[ ! -d "/opt/openstack-ansible-ops/bootstrap-embedded-ansible" ]]; then
   popd
 fi
 
+if [[ ! -d "${MTC_WORKING_DIR}" ]]; then
+  git clone -b ${MTC_RELEASE} https://github.com/rcbops/magnanimous-turbo-chainsaw "${MTC_WORKING_DIR}"
+else
+  pushd "${MTC_WORKING_DIR}"
+    git fetch --all
+    git reset --hard origin/${MTC_RELEASE}
+  popd
+fi
+
 if [[ -f "${MTC_SCRIPT_DIR}/setup-workspace.sh" ]];  then
   PS1="${PS1:-'\[\033[01;31m\]\h\[\033[01;34m\] \W \$\[\033[00m\] '}" source "${MTC_SCRIPT_DIR}/setup-workspace.sh"
-  pip install pyOpenSSL==16.2.0 PyYAML==3.13 ${PIP_INSTALL_OPTS}
+  # pip install pyOpenSSL==16.2.0 PyYAML==3.13 ${PIP_INSTALL_OPTS}
 else
   curl -D - "https://raw.githubusercontent.com/rcbops/magnanimous-turbo-chainsaw/${MTC_RELEASE}/scripts/setup-workspace.sh" -o /tmp/setup-workspace.sh
   PS1="${PS1:-'\[\033[01;31m\]\h\[\033[01;34m\] \W \$\[\033[00m\] '}" source /tmp/setup-workspace.sh
@@ -96,7 +110,7 @@ else
   #                    why its needed for installation that use Hashicorp-Vault.
   # NOTE(npawelek): PyYAML 5.1 has introduced template problems. This should
   #                 be tested and removed when 5.2 is released.
-  pip install pyOpenSSL==16.2.0 PyYAML==3.13 ${PIP_INSTALL_OPTS}
+  # pip install pyOpenSSL==16.2.0 PyYAML==3.13 ${PIP_INSTALL_OPTS}
 fi
 
 # Restore the pip config if found