Merge develop into master for 0.5.0 release (#848)

hitachienergy · Jan 17, 2020 · 923d9a2 · 923d9a2
1 parent 892eda0
commit 923d9a2
Show file tree

Hide file tree

Showing 189 changed files with 5,370 additions and 1,740 deletions.
diff --git a/CHANGELOG-0.5.md b/CHANGELOG-0.5.md
@@ -0,0 +1,31 @@
+# Changelog 0.5
+
+## [0.5.0] 2020-01-17
+
+### Added
+
+- [#820](https://github.com/epiphany-platform/epiphany/pull/820) - Firewall: OS level firewall setup (firewalld)
+
+### Added
+
+- [#381](https://github.com/epiphany-platform/epiphany/issues/381) - Add AWS EC2 Root Volume encryption
+- [#782](https://github.com/epiphany-platform/epiphany/issues/781) - All disks encryption documentation - AWS
+- [#782](https://github.com/epiphany-platform/epiphany/issues/782) - All disks encryption documentation - Azure
+- [#784](https://github.com/epiphany-platform/epiphany/issues/784) - Switch to Open Distro for Elasticsearch
+  - [Data storage](/docs/home/howto/DATABASES.md#how-to-start-working-with-opendistro-for-elasticsearch)
+  - [Centralized logging](/docs/home/howto/LOGGING.md#centralized-logging-setup)
+
+- [#755](https://github.com/epiphany-platform/epiphany/issues/755) - Create Ansible playbook to install Apache Ignite as a service on VM
+  - [Stateful setup](/docs/home/howto/DATABASES.md#how-to-start-working-with-apache-ignite-stateful-setup)
+- [#749](https://github.com/epiphany-platform/epiphany/issues/749) - Deploy stateless Apache Ignite on K8s
+  - [Stateless setup](/docs/home/howto/DATABASES.md#how-to-start-working-with-apache-ignite-stateless-setup)
+- [#831](https://github.com/epiphany-platform/epiphany/issues/831) - Build artifacts encryption (Kubernetes config) using ansible vault
+  - [epicli asks for password](/docs/home/howto/SECURITY.md#how-to-run-epicli-with-password)
+
+### Changed
+
+- [#763](https://github.com/epiphany-platform/epiphany/pull/763) - Elasticsearch Curator: Flexible configuration of cron jobs
+- [#763](https://github.com/epiphany-platform/epiphany/pull/763) - Elasticsearch Curator: Upgrade to v5.8.1
+- [#766](https://github.com/epiphany-platform/epiphany/issues/766) - Elasticsearch: Upgrade to v6.8.5
+- [#775](https://github.com/epiphany-platform/epiphany/issues/775) - Filebeat: Upgrade to v6.8.5
+- [#752](https://github.com/epiphany-platform/epiphany/pull/752) - Kafka: Upgrade to v2.3.1
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,12 +9,18 @@ Reference for actual cluster component versions can be found [here](docs/home/CO
 
 ## Current release
 
-### 0.4.x
+### 0.5.x
 
-- [CHANGELOG-0.4.0](./CHANGELOG-0.4.md#040-2019-09-30)
+- [CHANGELOG-0.5.0](./CHANGELOG-0.5.md#050-2020-01-17)
 
 ## Older releases
 
+### 0.4.x
+
+- [CHANGELOG-0.4.2](./CHANGELOG-0.4.md#042-2019-11-20)
+- [CHANGELOG-0.4.1](./CHANGELOG-0.4.md#041-2019-10-17)
+- [CHANGELOG-0.4.0](./CHANGELOG-0.4.md#040-2019-10-11)
+
 ### 0.3.x
 
 - [CHANGELOG-0.3.0](./CHANGELOG-0.3.md#030-2019-07-31)

diff --git a/README.md b/README.md
@@ -53,6 +53,7 @@ You will need to modify a few values (like your AWS secrets, directory path for
 ```shell
 epicli apply -f demo.yaml
 ```
+You will be asked for a password that will be used for encryption of some of build artifacts. More information [here](docs/home/howto/SECURITY.md#how-to-run-epicli-with-password)
 
 Find more information using table of contents below - especially the [How-to guides](docs/home/HOWTO.md).
 

diff --git a/core/src/epicli/cli/engine/ansible/AnsibleCommand.py b/core/src/epicli/cli/engine/ansible/AnsibleCommand.py
@@ -27,7 +27,7 @@ def run_task(self, hosts, inventory, module, args=None):
 
         cmd.append(hosts)
 
-        self.logger.info('Running: "' + ' '.join(cmd) + '"')
+        self.logger.info('Running: "' + ' '.join(module) + '"')
 
         logpipe = LogPipe(__name__)
         with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe) as sp:
@@ -51,18 +51,21 @@ def run_task_with_retries(self, inventory, module, hosts, retries, timeout=10, a
         else:
             raise Exception(f'Failed running task after {str(retries)} retries')
 
-    def run_playbook(self, inventory, playbook_path):
+    def run_playbook(self, inventory, playbook_path, vault_file=None):
         cmd = ['ansible-playbook']
 
         if inventory is not None and len(inventory) > 0:
             cmd.extend(["-i", inventory])
 
+        if vault_file is not None:
+            cmd.extend(["--vault-password-file", vault_file])
+
         cmd.append(playbook_path)
 
         if Config().debug:
             cmd.append('-vvv')
 
-        self.logger.info('Running: "' + ' '.join(cmd) + '"')
+        self.logger.info('Running: "' + ' '.join(playbook_path) + '"')
 
         logpipe = LogPipe(__name__)
         with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe) as sp:

diff --git a/core/src/epicli/cli/engine/ansible/AnsibleInventoryCreator.py b/core/src/epicli/cli/engine/ansible/AnsibleInventoryCreator.py
@@ -36,7 +36,7 @@ def get_inventory(self):
                 roles = self.get_roles_for_feature(component_key)
                 for role in roles:
                     ansible_role_name = to_role_name(role)
-                    inventory.append(AnsibleInventoryItem(to_role_name(ansible_role_name), ips))
+                    inventory.append(AnsibleInventoryItem(ansible_role_name, ips))
 
         return self.group_duplicated(inventory)
 

diff --git a/core/src/epicli/cli/engine/ansible/AnsibleRunner.py b/core/src/epicli/cli/engine/ansible/AnsibleRunner.py
@@ -101,7 +101,7 @@ def apply(self):
         enabled_roles = inventory_creator.get_enabled_roles()
         for role in enabled_roles:
             self.ansible_command.run_playbook(inventory=inventory_path,
-                                              playbook_path=self.playbook_path(to_role_name(role)))
+                                              playbook_path=self.playbook_path(to_role_name(role)), vault_file=Config().vault_password_location)
 
         #post-flight after we are done
         self.post_flight(inventory_path)

diff --git a/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py b/core/src/epicli/cli/engine/ansible/AnsibleVarsGenerator.py
@@ -2,7 +2,7 @@
 import copy
 
 from cli.helpers.Step import Step
-from cli.helpers.build_saver import get_ansible_path, get_ansible_path_for_build
+from cli.helpers.build_saver import get_ansible_path, get_ansible_path_for_build, get_ansible_vault_path
 from cli.helpers.doc_list_helpers import select_first
 from cli.helpers.naming_helpers import to_feature_name, to_role_name
 from cli.helpers.ObjDict import ObjDict
@@ -90,6 +90,8 @@ def populate_group_vars(self, ansible_dir):
         shared_config_doc = select_first(self.config_docs, lambda x: x.kind == 'configuration/shared-config')
         if shared_config_doc == None:
             shared_config_doc = load_yaml_obj(types.DEFAULT, 'common', 'configuration/shared-config')
+
+        self.set_vault_path(shared_config_doc)
         main_vars.update(shared_config_doc.specification)        
 
         vars_dir = os.path.join(ansible_dir, 'group_vars')
@@ -102,6 +104,11 @@ def populate_group_vars(self, ansible_dir):
         with open(vars_file_path, 'a') as stream:
             dump(main_vars, stream)
 
+    def set_vault_path(self, shared_config):
+        shared_config.specification.vault_tmp_file_location = Config().vault_password_location
+        if shared_config.specification.vault_location == '':
+            shared_config.specification.vault_location = get_ansible_vault_path(self.cluster_model.specification.name)
+
     def get_clean_cluster_model(self):
         cluster_model = copy.copy(self.cluster_model)
         self.clear_object(cluster_model, 'credentials')

diff --git a/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py b/core/src/epicli/cli/engine/providers/azure/InfrastructureBuilder.py
@@ -60,16 +60,14 @@ def run(self):
                                                          0)
                 infrastructure.append(nsg)
 
-                subnet = self.get_subnet(subnet_definition, component_key, nsg.specification.name, 0)
+                subnet = self.get_subnet(subnet_definition, component_key, 0)
                 infrastructure.append(subnet) 
 
-                #TODO: This gives issues for now when creating more then 3 subnets. Re-test when
-                #      upgrading from azurerm 1.27 to 2.0 and for now stick to azurerm_subnet.network_security_group_id
-                #ssga = self.get_subnet_network_security_group_association(component_key, 
-                #                                                                     subnet.specification.name, 
-                #                                                                     nsg.specification.name,
-                #                                                                     0)
-                #infrastructure.append(ssga)
+                subnet_nsg_association = self.get_subnet_network_security_group_association(component_key, 
+                                                                                     subnet.specification.name, 
+                                                                                     nsg.specification.name,
+                                                                                     0)
+                infrastructure.append(subnet_nsg_association)
 
             #TODO: For now we create the VM infrastructure compatible with the Epiphany 2.x 
             #      code line but later we might want to look at scale sets to achieve the same result:
@@ -116,11 +114,10 @@ def get_network_security_group(self, component_key, security_rules,  index):
         security_group.specification.rules = security_rules
         return security_group       
 
-    def get_subnet(self, subnet_definition, component_key, security_group_name, index):
+    def get_subnet(self, subnet_definition, component_key, index):
         subnet = self.get_config_or_default(self.docs, 'infrastructure/subnet')
         subnet.specification.name = resource_name(self.cluster_prefix, self.cluster_name, 'subnet' + '-' + str(index), component_key)
         subnet.specification.address_prefix = subnet_definition['address_pool']
-        subnet.specification.security_group_name = security_group_name
         subnet.specification.cluster_name = self.cluster_name
         return subnet     
 
@@ -194,4 +191,3 @@ def get_virtual_machine(component_value, cluster_model, docs):
                                                       machine_selector)
 
         return model_with_defaults         
-
diff --git a/core/src/epicli/cli/engine/schema/ConfigurationAppender.py b/core/src/epicli/cli/engine/schema/ConfigurationAppender.py
@@ -19,7 +19,7 @@ def run(self):
 
         def append_config(doc):
             doc['version'] = VERSION
-            configuration_docs.append(doc)    
+            configuration_docs.append(doc)
 
         for document_kind in ConfigurationAppender.REQUIRED_DOCS:
             doc = select_first(self.input_docs, lambda x: x.kind == document_kind)

diff --git a/core/src/epicli/cli/epicli.py b/core/src/epicli/cli/epicli.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env py
+import atexit
 import sys
 import argparse
 import json
@@ -15,6 +16,8 @@
 from cli.version import VERSION
 from cli.licenses import LICENSES
 from cli.helpers.query_yes_no import query_yes_no
+from cli.helpers.input_query import prompt_for_password
+from cli.helpers.build_saver import save_to_file
 
 
 def main():
@@ -122,10 +125,13 @@ def apply_parser(subparsers):
     sub_parser.add_argument('--no-infra', dest='no_infra', action="store_true",
                             help='Skip infrastructure provisioning.')
     sub_parser.add_argument('--offline-requirements', dest='offline_requirements', type=str,
-                            help='Path to the folder with pre-prepared offline requirements.')
+                            help='Path to the folder with pre-prepared offline requirements.')    
+    sub_parser.add_argument('--vault-password', dest='vault_password', type=str,
+                            help='Password that will be used to encrypt build artifacts.')
 
     def run_apply(args):
         adjust_paths_from_file(args)
+        ensure_vault_password_is_set(args)
         with BuildEngine(args) as engine:
             return engine.apply()
 
@@ -264,6 +270,22 @@ def dump_config(config):
         if attr.startswith('_'):
             logger.info('%s = %r' % (attr[1:], getattr(config, attr)))
 
+def ensure_vault_password_is_set(args):
+    vault_password = args.vault_password 
+    if vault_password is None:
+        vault_password = prompt_for_password("Provide password to encrypt vault: ")
+
+    directory_path = os.path.dirname(Config().vault_password_location)
+    os.makedirs(directory_path, exist_ok=True)
+    save_to_file(Config().vault_password_location, vault_password)
+
+def ensure_vault_password_is_cleaned():
+    if os.path.exists(Config().vault_password_location):
+        os.remove(Config().vault_password_location)
+
+def exit_handler():
+    ensure_vault_password_is_cleaned()
 
 if __name__ == '__main__':
+    atexit.register(exit_handler)
     exit(main())
diff --git a/core/src/epicli/cli/helpers/Config.py b/core/src/epicli/cli/helpers/Config.py
@@ -1,5 +1,5 @@
 import os
-
+from os.path import expanduser
 
 class Config:
     class __ConfigBase:
@@ -21,6 +21,7 @@ def __init__(self):
             self._auto_approve = False
             self._offline_requirements = ''
             self._wait_for_pods = False
+            self._vault_password_location = os.path.join(expanduser("~"), '.epicli/vault.cfg')
 
         @property
         def docker_cli(self):
@@ -107,6 +108,15 @@ def auto_approve(self, auto_approve):
             if not auto_approve is None:
                 self._auto_approve = auto_approve
 
+        @property
+        def vault_password_location(self):
+            return self._vault_password_location
+
+        @vault_password_location.setter
+        def vault_password_location(self, vault_password_location):
+            if not vault_password_location is None:
+                self._vault_password_location = vault_password_location
+
         @property
         def offline_requirements(self):
             return self._offline_requirements

diff --git a/core/src/epicli/cli/helpers/build_saver.py b/core/src/epicli/cli/helpers/build_saver.py
@@ -13,6 +13,7 @@
 SP_FILE_NAME = 'sp.yml'
 INVENTORY_FILE_NAME = 'inventory'
 ANSIBLE_OUTPUT_DIR = 'ansible/'
+ANSIBLE_VAULT_OUTPUT_DIR = 'vault/'
 
 BUILD_EPICLI = 'BUILD_EPICLI'
 BUILD_LEGACY = 'BUILD_LEGACY_02X'
@@ -123,18 +124,21 @@ def get_ansible_path(cluster_name):
         os.makedirs(ansible_dir)
     return ansible_dir
 
+def get_ansible_vault_path(cluster_name):
+    ansible_vault_dir = os.path.join(get_build_path(cluster_name), ANSIBLE_VAULT_OUTPUT_DIR)
+    if not os.path.exists(ansible_vault_dir):
+        os.makedirs(ansible_vault_dir)
+    return ansible_vault_dir
 
 def get_ansible_path_for_build(build_directory):
     ansible_dir = os.path.join(build_directory, ANSIBLE_OUTPUT_DIR)
     if not os.path.exists(ansible_dir):
         os.makedirs(ansible_dir)
     return ansible_dir    
 
-
 def copy_files_recursively(src, dst):
     distutils.dir_util.copy_tree(src, dst)
 
-
 def copy_file(src, dst):
     shutil.copy2(src, dst)
 
diff --git a/core/src/epicli/cli/helpers/input_query.py b/core/src/epicli/cli/helpers/input_query.py
@@ -0,0 +1,13 @@
+import getpass
+
+def prompt_for_value(prompt):
+    input_text = ''
+    while input_text == '':
+        input_text = input(prompt)
+    return input_text
+
+def prompt_for_password(prompt):
+    password = ''
+    while password == '':
+        password = getpass.getpass(prompt)
+    return password