Merge pull request #308 from nutanix/dsl-381-cut

CALM-DSL changes for v3.8.1

CalmVersion

@@ -1 +1 @@
-3.8.0
+3.8.1

Makefile

@@ -9,7 +9,7 @@ dev:
 	# Setup our python3 based virtualenv
 	# This step assumes python3 is installed on your dev machine
 	[ -f venv/bin/python3 ] || (python3 -m venv venv && \
-		venv/bin/pip3 install --upgrade pip setuptools)
+		venv/bin/pip3 install --upgrade pip "setuptools<=70.3.0")
 	venv/bin/pip3 install --no-cache -r requirements.txt -r dev-requirements.txt
 	venv/bin/python3 setup.py develop
 
@@ -39,6 +39,7 @@ clean:
 		docker ps -aq --no-trunc --filter "status=exited" | xargs -I {} docker rm {} && \
 		docker image prune -f
 	rm -rf venv/ && mkdir venv/ && touch venv/.empty
+	rm -rf ~/.calm/dsl.db
 
 test-verbose: dev
 	venv/bin/py.test -s -vv

calm/dsl/api/blueprint.py

@@ -37,6 +37,8 @@ def __init__(self, connection):
         self.PROTECTION_POLICY_LIST = (
             self.ITEM + "/app_profile/{}/config_spec/{}/app_protection_policies/list"
         )
+        self.RUN_SCRIPT = self.PREFIX + "/{}/run_script"
+        self.GET_SCRIPT = self.RUN_SCRIPT + "/output/{}/{}"
 
     # TODO https://jira.nutanix.com/browse/CALM-17178
     # Blueprint creation timeout is dependent on payload.
@@ -384,3 +386,19 @@ def variable_values_from_trlid(self, uuid, var_uuid, req_id, trl_id):
         return self.connection._call(
             url, verify=False, method=REQUEST.METHOD.GET, ignore_error=True
         )
+
+    def run_scripts(self, bp_uuid, payload):
+        return self.connection._call(
+            self.RUN_SCRIPT.format(bp_uuid),
+            verify=False,
+            request_json=payload,
+            method=REQUEST.METHOD.POST,
+        )
+
+    def test_scripts(self, bp_uuid, trl_id, request_id):
+        return self.connection._call(
+            self.GET_SCRIPT.format(bp_uuid, trl_id, request_id),
+            verify=False,
+            method=REQUEST.METHOD.GET,
+            ignore_error=True,
+        )

calm/dsl/api/handle.py

@@ -38,6 +38,7 @@
 from .approval_request import ApprovalRequestAPI
 from .provider import ProviderAPI
 from .quotas import QuotasAPI
+from .util import get_auth_info
 
 
 class ClientHandle:
@@ -132,8 +133,10 @@ def get_api_client():
 
         pc_ip = server_config.get("pc_ip")
         pc_port = server_config.get("pc_port")
-        username = server_config.get("pc_username")
-        password = server_config.get("pc_password")
+        api_key_location = server_config.get("api_key_location", None)
+        cred = get_auth_info(api_key_location)
+        username = cred.get("username")
+        password = cred.get("password")
 
         update_api_client(host=pc_ip, port=pc_port, auth=(username, password))
 

calm/dsl/api/util.py

@@ -1,7 +1,11 @@
 import copy
+import json
 import sys
+import os
 
 from calm.dsl.log import get_logging_handle
+from calm.dsl.config import get_context
+from calm.dsl.constants import DSL_CONFIG
 
 LOG = get_logging_handle(__name__)
 
@@ -504,6 +508,54 @@ def patch_secrets(resources, secret_map, secret_variables, existing_secrets=[]):
     return resources
 
 
+def get_auth_info(api_key_location):
+    """
+    Reads name+token from specified api-key file (if it exists) as username+password
+    Else it reads username+password from config.
+    Args:
+        api_key_location (str): location of api-key
+    Returns:
+        dict: containing username and password for authentication.
+    """
+
+    if api_key_location not in [None, DSL_CONFIG.EMPTY_CONFIG_ENTITY_NAME]:
+        if not os.path.exists(api_key_location):
+            LOG.error("{} not found".format(api_key_location))
+            sys.exit(-1)
+
+        with open(api_key_location, "r") as f:
+            auth_creds = f.read()
+            auth_creds = json.loads(auth_creds)
+
+        if not auth_creds.get("name"):
+            LOG.error(
+                "Badly formatted key file. Key name not present in {}".format(
+                    api_key_location
+                )
+            )
+            sys.exit(-1)
+
+        if not auth_creds.get("token"):
+            LOG.error(
+                "Badly formatted key file. Token not present in {}".format(
+                    api_key_location
+                )
+            )
+            sys.exit(-1)
+
+        cred = {"username": auth_creds.get("name"), "password": auth_creds.get("token")}
+    # Read username/password from config when api-key is not supplied
+    else:
+        context = get_context()
+        server_config = context.get_server_config()
+        cred = {
+            "username": server_config.get("pc_username"),
+            "password": server_config.get("pc_password"),
+        }
+
+    return cred
+
+
 def _create_task_name_substrate_map(bp_payload, entity_type, **kwargs):
     vm_power_action_uuid_substrate_map = kwargs.get(
         "vm_power_action_uuid_substrate_map", {}

calm/dsl/builtins/models/blueprint_payload.py

@@ -67,7 +67,7 @@ def create_blueprint_payload(UserBlueprint, metadata={}):
     #  Project will be taken from config if not provided
     if not metadata.get("project_reference", {}):
         project_name = project_config["name"]
-        if project_name == DSL_CONFIG.EMPTY_PROJECT_NAME:
+        if project_name == DSL_CONFIG.EMPTY_CONFIG_ENTITY_NAME:
             LOG.error(DSL_CONFIG.EMPTY_PROJECT_MESSAGE)
             sys.exit("Invalid project configuration")
 

calm/dsl/builtins/models/calm_ref.py

@@ -2,6 +2,8 @@
 import uuid
 import json
 
+from distutils.version import LooseVersion as LV
+
 from calm.dsl.db.table_config import AhvSubnetsCache
 from calm.dsl.builtins.models.constants import NutanixDB as NutanixDBConst
 
@@ -12,14 +14,16 @@
 from .ahv_vm_cluster import AhvCluster
 from .ahv_vm_vpc import AhvVpc
 
-from calm.dsl.store import Cache
+from calm.dsl.store import Cache, Version
 from calm.dsl.constants import CACHE
 from calm.dsl.api.handle import get_api_client
 from calm.dsl.log import get_logging_handle
 
 
 LOG = get_logging_handle(__name__)
 
+CALM_VERSION = Version.get_version("Calm")
+
 
 # CalmRef
 class CalmRefDict(EntityDict):
@@ -464,6 +468,8 @@ def compile(cls, name="", **kwargs):
             account_uuid = ""
             try:
                 account_ref = cls.__parent__.attrs.get("account_reference", {})
+                if isinstance(account_ref, CalmRefType):
+                    account_ref = account_ref.get_dict()
                 account_uuid = account_ref.get("uuid", "")
             except Exception as exp:
                 pass
@@ -472,6 +478,9 @@ def compile(cls, name="", **kwargs):
 
             if name:
                 params = {"filter": "name=={}".format(name), "length": 250}
+                if LV(CALM_VERSION) >= LV("3.8.1"):
+                    project_cache_data = common_helper.get_cur_context_project()
+                    params["project_uuid"] = project_cache_data.get("uuid")
                 res, err = client.account.vms_list(account_uuid, params)
                 if err:
                     LOG.error(err)

calm/dsl/cli/apps.py

@@ -6,6 +6,7 @@
 import uuid
 from json import JSONEncoder
 from datetime import datetime
+from copy import deepcopy
 
 import arrow
 import click
@@ -1889,8 +1890,15 @@ def decompile_app_migratable_bp(app_name, bp_dir):
         sys.exit("[{}] - {}".format(err["code"], err["error"]))
 
     res = res.json()
+
+    app_res = _get_app(client, app_name)
+    launch_profile = app_res["spec"]["resources"]["app_profile_config_reference"].get(
+        "name", ""
+    )
+
     fix_missing_name_in_reference(res["spec"])
     remove_non_escript_actions_variables(res["spec"]["resources"])
+    filter_launch_profile(launch_profile, res["spec"]["resources"])
     _decompile_bp(bp_payload=res, with_secrets=False, bp_dir=bp_dir)
 
 
@@ -2282,3 +2290,60 @@ def describe_app_actions_to_update(app_name):
 
                 if not any_action_to_be_modified:
                     print("\t\t No actions found to be modified")
+
+
+def filter_launch_profile(launch_profile, resources):
+    """
+    Filters out other profiles and keeps only profile used for launching an application.
+    Subsequently also filters out profile dependent packages and substrates.
+
+    Args:
+        launch_profile (str): name of profile used for launching
+        resources (dict): blueprint spec resources
+
+    """
+    if not launch_profile:
+        return
+
+    # lists to hold names of entities to be removed
+    substrates_to_be_removed = []
+    packages_to_be_removed = []
+    profiles_to_be_removed = []
+
+    # loop to store entities name to be removed
+    for profile in resources.get("app_profile_list", []):
+        if profile["name"] == launch_profile:
+            continue
+
+        profiles_to_be_removed.append(profile["name"])
+
+        for deployment in profile.get("deployment_create_list", []):
+            packages = deployment.get("package_local_reference_list", [])
+            for pkg in packages:
+                packages_to_be_removed.append(pkg.get("name", ""))
+
+            substrate = deployment.get("substrate_local_reference", {})
+            substrates_to_be_removed.append(substrate.get("name", ""))
+
+    # remove substrates
+    substrates = deepcopy(resources.get("substrate_definition_list", []))
+    for substrate in resources.get("substrate_definition_list", []):
+        if substrate["name"] in substrates_to_be_removed:
+            substrates.remove(substrate)
+
+    # remove packages
+    packages = deepcopy(resources.get("package_definition_list", []))
+    for pkg in resources.get("package_definition_list", []):
+        if pkg["name"] in packages_to_be_removed:
+            packages.remove(pkg)
+
+    # remove profiles
+    profiles = deepcopy(resources.get("app_profile_list", []))
+    for profile in resources.get("app_profile_list", []):
+        if profile["name"] in profiles_to_be_removed:
+            profiles.remove(profile)
+
+    # re-assign substrates, packages for profile used for launching application
+    resources["substrate_definition_list"] = substrates
+    resources["package_definition_list"] = packages
+    resources["app_profile_list"] = profiles

calm/dsl/cli/bps.py

@@ -329,7 +329,7 @@ def compile_blueprint(bp_file, brownfield_deployment_file=None):
             ]
         else:
             project_name = project_config["name"]
-            if project_name == DSL_CONFIG.EMPTY_PROJECT_NAME:
+            if project_name == DSL_CONFIG.EMPTY_CONFIG_ENTITY_NAME:
                 LOG.error(DSL_CONFIG.EMPTY_PROJECT_MESSAGE)
                 sys.exit("Invalid project configuration")
 
@@ -778,6 +778,8 @@ def get_blueprint_uuid(name, all=False, is_brownfield=False):
     params = {"filter": "name=={}".format(name)}
     if not all:
         params["filter"] += ";state!=DELETED"
+    else:
+        params["filter"] += get_states_filter(BLUEPRINT.STATES)
 
     if is_brownfield:
         params["filter"] += ";type==BROWNFIELD"

calm/dsl/cli/constants.py

@@ -761,3 +761,14 @@ class ENTITY_FILTER_EXPRESSION_LIST:
             },
         },
     ]
+
+
+class TEST_SCRIPTS:
+    class STATUS:
+        SUCCESS = "SUCCESS"
+        PENDING = "PENDING"
+        RUNNING = "RUNNING"
+        ERROR = "ERROR"
+
+    TERMINAL_STATES = [STATUS.SUCCESS, STATUS.ERROR]
+    TYPE = ["escript", "shell", "powershell", "python"]

calm/dsl/cli/endpoints.py

@@ -137,12 +137,15 @@ def compile_endpoint(endpoint_file):
 
 def compile_endpoint_command(endpoint_file, out):
 
+    # Note: Metadata should be constructed before loading endpoint module.
+    # As metadata will be used while verifying vm reference in endpoint.
+    metadata_payload = get_metadata_payload(endpoint_file)
+
     endpoint_payload = compile_endpoint(endpoint_file)
     if endpoint_payload is None:
         LOG.error("User endpoint not found in {}".format(endpoint_file))
         return
 
-    metadata_payload = get_metadata_payload(endpoint_file)
     project_cache_data = {}
     project_name = ""
     if "project_reference" in metadata_payload:
@@ -185,6 +188,8 @@ def get_endpoint(client, name, all=False):
     params = {"filter": "name=={}".format(name)}
     if not all:
         params["filter"] += ";deleted==FALSE"
+    else:
+        params["filter"] += get_states_filter(ENDPOINT.STATES, state_key="_state")
 
     res, err = client.endpoint.list(params=params)
     if err: