diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 9a3c8553..8a9d72f6 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -384,7 +384,15 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.SECRET_ACCESS_KEY }} run: | - sudo podman run --rm --name cloud-governance -e policy=${{ matrix.policy }} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=${{ matrix.region }} -e dry_run=yes -e policy_output=s3://${{ secrets.BUCKET }}/test/${{ matrix.region }} -e log_level=INFO ${{ secrets.QUAY_PUBLIC_CLOUD_GOVERNANCE_REPOSITORY }} + touch env.yaml + echo "policy: ${{ matrix.policy }}" >> env.yaml + echo "AWS_DEFAULT_REGION: ${{ matrix.region }}" >> env.yaml + echo "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}" >> env.yaml + echo "AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}" >> env.yaml + echo "dry_run: yes" >> env.yaml + echo "policy_output: "s3://${{ secrets.BUCKET }}/test/${{ matrix.region }}"" >> env.yaml + echo "log_level: INFO" >> env.yaml + sudo podman run --rm --name cloud-governance -v "${PWD}/env.yaml":"/tmp/env.yaml" ${{ secrets.QUAY_PUBLIC_CLOUD_GOVERNANCE_REPOSITORY }} gitleaks: name: gitleaks needs: [ unittest, terraform_apply, integration, pypi_upload, pypi_validate, bump_version ] @@ -397,6 +405,14 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.SECRET_ACCESS_KEY }} run: | # gileaks policy - region='us-east-1' - policy='gitleaks' - sudo podman run --rm --name cloud-governance -e policy=$policy -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=$region -e git_access_token=${{ secrets.GIT_TOKEN }} -e git_repo=https://github.com/redhat-performance/cloud-governance -e policy_output=s3://${{ secrets.BUCKET }}/test/$region -e log_level=INFO ${{ secrets.QUAY_PUBLIC_CLOUD_GOVERNANCE_REPOSITORY }} > /dev/null + region="us-east-1" + touch env.yaml + echo "AWS_DEFAULT_REGION: ${region}" >> env.yaml + echo "policy: gitleaks" >> env.yaml + echo "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}" >> env.yaml + echo "AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}" >> env.yaml + echo "git_access_token: ${{ secrets.GIT_TOKEN }}" >> env.yaml + echo "git_repo: https://github.com/redhat-performance/cloud-governance" >> env.yaml + echo "policy_output: "s3://${{ secrets.BUCKET }}/test/${region}"" >> env.yaml + echo "log_level: INFO" >> env.yaml + sudo podman run --rm --name cloud-governance -v "${PWD}/env.yaml":"/tmp/env.yaml" ${{ secrets.QUAY_PUBLIC_CLOUD_GOVERNANCE_REPOSITORY }} > /dev/null diff --git a/.gitignore b/.gitignore index 99a31fe0..9887438b 100644 --- a/.gitignore +++ b/.gitignore @@ -218,3 +218,4 @@ empty_test_environment_variables.py /cloud_governance/policy/send_mail.py cloudsensei/.env.txt .vscode +env.yaml diff --git a/README.md b/README.md index 7aac14e8..f81f1e19 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,27 @@ podman run --rm --name cloud-governance -e policy="tag_vm" -e account="$account" ``` +#### Run with yaml config + +```shell +cp example.yaml env.yaml +``` + +Added the supported environment variables. +example: + +```yaml +policy: instance_idle +AWS_ACCESS_KEY_ID: "" +AWS_SECRET_ACCESS_KEY: "" +``` + +```shell +podman run --rm --name cloud-governance \ +-v "${PWD}/env.yaml":"/tmp/env.yaml" \ +"quay.io/cloud-governance/cloud-governance:latest" +``` + ## Run Policy Using Pod #### Run as a pod job via OpenShift diff --git a/cloud_governance/common/logger/init_logger.py b/cloud_governance/common/logger/init_logger.py index fe2e0cd5..ef8debe8 100644 --- a/cloud_governance/common/logger/init_logger.py +++ b/cloud_governance/common/logger/init_logger.py @@ -2,20 +2,18 @@ import sys import logging -from cloud_governance.main.environment_variables import environment_variables - logger_category_name = 'cloud_governance' logger = logging.getLogger(logger_category_name) # instantiating a logger handler = logging.StreamHandler(sys.stdout) # log for output only -#log_path = os.getcwd() -#fileHandler = logging.FileHandler(filename=f'{log_path}/cloud_governance.log', mode='w+') -account_name = environment_variables.environment_variables_dict.get('account') +# log_path = os.getcwd() +# fileHandler = logging.FileHandler(filename=f'{log_path}/cloud_governance.log', mode='w+') +account_name = os.environ.get('account') log_format = f'[%(levelname)s] %(asctime)s {account_name} - %(message)s' formatter = logging.Formatter(log_format) handler.setFormatter(formatter) logger.addHandler(handler) -#logger.addHandler(fileHandler) +# logger.addHandler(fileHandler) # def get_pyperf_log_path(): diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index 630fdfc1..c410b7c3 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -4,7 +4,9 @@ from ast import literal_eval import boto3 +import yaml +from cloud_governance.common.logger.init_logger import logger from cloud_governance.main.environment_variables_exceptions import ParseFailed @@ -12,10 +14,53 @@ class EnvironmentVariables: """ This class manages the environment variable parameters """ + DEFAULT_CONF_PATH = os.environ.get("DEFAULT_CONF_PATH", "/tmp/env.yaml") + + def load_from_env(self): + """ + This method load environment variables from text files. + :return: + """ + for env in ".env", ".env.generated": + try: + file_path = os.path.join(os.path.dirname(__file__), env) + if os.path.exists(file_path): + with open(file_path) as f: + for line in f.readlines(): + key, found, value = line.strip().partition("=") + if not found: + logger.error(f"ERROR: invalid line in {env}: {line.strip()}") + continue + if key not in os.environ: + os.environ[key] = value + except FileNotFoundError: + pass + + def load_from_yaml(self, ): + """ + Load values from yaml file as attributes of this class. + Will never override existing attributes. + """ + + for yaml_file_path in (os.path.join(os.path.curdir, 'env.yaml'), self.DEFAULT_CONF_PATH): + try: + if os.path.exists(yaml_file_path): + with open(yaml_file_path, 'r') as yaml_file: + yaml_data = yaml.safe_load(yaml_file) + if isinstance(yaml_data, dict): + for key, value in yaml_data.items(): + if key not in os.environ: # Prefer existing env variables + os.environ[key] = str(value) + except FileNotFoundError: + pass def __init__(self): + super().__init__() self._environment_variables_dict = {} + self.load_from_env() + self.load_from_yaml() + # env files override true ENV. Not best order, but easier to write :/ # .env.generated can be auto-generated (by an external tool) based on the local cluster's configuration. for env in ".env", ".env.generated": @@ -41,18 +86,22 @@ def __init__(self): self._environment_variables_dict['AWS_DEFAULT_REGION'] = EnvironmentVariables.get_env('AWS_DEFAULT_REGION', '') self._environment_variables_dict['log_level'] = EnvironmentVariables.get_env('log_level', 'INFO') - self._environment_variables_dict['DAYS_TO_TAKE_ACTION'] = int(EnvironmentVariables.get_env('DAYS_TO_TAKE_ACTION', "7")) + self._environment_variables_dict['DAYS_TO_TAKE_ACTION'] = int( + EnvironmentVariables.get_env('DAYS_TO_TAKE_ACTION', "7")) - self._environment_variables_dict['PRINT_LOGS'] = EnvironmentVariables.get_boolean_from_environment('PRINT_LOGS', True) + self._environment_variables_dict['PRINT_LOGS'] = EnvironmentVariables.get_boolean_from_environment('PRINT_LOGS', + True) if not self._environment_variables_dict['AWS_DEFAULT_REGION']: self._environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2' self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = EnvironmentVariables.get_env('PUBLIC_CLOUD_NAME', 'AWS') self._environment_variables_dict['AWS_ACCESS_KEY_ID'] = EnvironmentVariables.get_env('AWS_ACCESS_KEY_ID', '') - self._environment_variables_dict['AWS_SECRET_ACCESS_KEY'] = EnvironmentVariables.get_env('AWS_SECRET_ACCESS_KEY', '') + self._environment_variables_dict['AWS_SECRET_ACCESS_KEY'] = EnvironmentVariables.get_env( + 'AWS_SECRET_ACCESS_KEY', '') if self._environment_variables_dict['AWS_ACCESS_KEY_ID'] and \ self._environment_variables_dict['AWS_SECRET_ACCESS_KEY']: self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AWS' - self._environment_variables_dict['account'] = self.get_aws_account_alias_name().upper().replace('OPENSHIFT-', '') + self._environment_variables_dict['account'] = self.get_aws_account_alias_name().upper().replace( + 'OPENSHIFT-', '') self._environment_variables_dict['policy'] = EnvironmentVariables.get_env('policy', '') self._environment_variables_dict['aws_non_cluster_policies'] = ['instance_idle', 'ec2_stop', 'ebs_in_use', @@ -82,32 +131,39 @@ def __init__(self): self._environment_variables_dict['cluster_tag'] = EnvironmentVariables.get_env('cluster_tag', '') self._environment_variables_dict['service_type'] = EnvironmentVariables.get_env('service_type', '') self._environment_variables_dict['TABLE_NAME'] = EnvironmentVariables.get_env('TABLE_NAME', '') - self._environment_variables_dict['REPLACE_ACCOUNT_NAME'] = EnvironmentVariables.get_env('REPLACE_ACCOUNT_NAME', '{}') - self._environment_variables_dict['DAYS_TO_DELETE_RESOURCE'] = int(EnvironmentVariables.get_env('DAYS_TO_DELETE_RESOURCE', '7')) + self._environment_variables_dict['REPLACE_ACCOUNT_NAME'] = EnvironmentVariables.get_env('REPLACE_ACCOUNT_NAME', + '{}') + self._environment_variables_dict['DAYS_TO_DELETE_RESOURCE'] = int( + EnvironmentVariables.get_env('DAYS_TO_DELETE_RESOURCE', '7')) # AWS Cost Explorer tags self._environment_variables_dict['cost_metric'] = EnvironmentVariables.get_env('cost_metric', 'UnblendedCost') self._environment_variables_dict['start_date'] = EnvironmentVariables.get_env('start_date', '') self._environment_variables_dict['end_date'] = EnvironmentVariables.get_env('end_date', '') self._environment_variables_dict['granularity'] = EnvironmentVariables.get_env('granularity', 'DAILY') - self._environment_variables_dict['cost_explorer_tags'] = EnvironmentVariables.get_env('cost_explorer_tags', '{}') + self._environment_variables_dict['cost_explorer_tags'] = EnvironmentVariables.get_env('cost_explorer_tags', + '{}') # AZURE Credentials self._environment_variables_dict['AZURE_ACCOUNT_ID'] = EnvironmentVariables.get_env('AZURE_ACCOUNT_ID', '') self._environment_variables_dict['AZURE_CLIENT_ID'] = EnvironmentVariables.get_env('AZURE_CLIENT_ID', '') self._environment_variables_dict['AZURE_TENANT_ID'] = EnvironmentVariables.get_env('AZURE_TENANT_ID', '') - self._environment_variables_dict['AZURE_CLIENT_SECRET'] = EnvironmentVariables.get_env('AZURE_CLIENT_SECRET', '') - self._environment_variables_dict['AZURE_SUBSCRIPTION_ID'] = EnvironmentVariables.get_env('AZURE_SUBSCRIPTION_ID', '') - if self._environment_variables_dict['AZURE_CLIENT_ID'] and self._environment_variables_dict['AZURE_TENANT_ID']\ + self._environment_variables_dict['AZURE_CLIENT_SECRET'] = EnvironmentVariables.get_env('AZURE_CLIENT_SECRET', + '') + self._environment_variables_dict['AZURE_SUBSCRIPTION_ID'] = EnvironmentVariables.get_env( + 'AZURE_SUBSCRIPTION_ID', '') + if self._environment_variables_dict['AZURE_CLIENT_ID'] and self._environment_variables_dict['AZURE_TENANT_ID'] \ and self._environment_variables_dict['AZURE_CLIENT_SECRET']: self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AZURE' - self._environment_variables_dict['TOTAL_ACCOUNTS'] = EnvironmentVariables.get_boolean_from_environment('TOTAL_ACCOUNTS', False) + self._environment_variables_dict['TOTAL_ACCOUNTS'] = EnvironmentVariables.get_boolean_from_environment( + 'TOTAL_ACCOUNTS', False) # IBM env vars self._environment_variables_dict['IBM_ACCOUNT_ID'] = EnvironmentVariables.get_env('IBM_ACCOUNT_ID', '') self._environment_variables_dict['IBM_API_USERNAME'] = EnvironmentVariables.get_env('IBM_API_USERNAME', '') self._environment_variables_dict['IBM_API_KEY'] = EnvironmentVariables.get_env('IBM_API_KEY', '') - self._environment_variables_dict['USAGE_REPORTS_APIKEY'] = EnvironmentVariables.get_env('USAGE_REPORTS_APIKEY', '') + self._environment_variables_dict['USAGE_REPORTS_APIKEY'] = EnvironmentVariables.get_env('USAGE_REPORTS_APIKEY', + '') if self._environment_variables_dict['USAGE_REPORTS_APIKEY']: self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'IBM' self._environment_variables_dict['month'] = EnvironmentVariables.get_env('month', '') @@ -119,12 +175,14 @@ def __init__(self): # Common env vars self._environment_variables_dict['dry_run'] = EnvironmentVariables.get_env('dry_run', 'yes') - self._environment_variables_dict['FORCE_DELETE'] = EnvironmentVariables.get_boolean_from_environment('FORCE_DELETE', False) + self._environment_variables_dict['FORCE_DELETE'] = EnvironmentVariables.get_boolean_from_environment( + 'FORCE_DELETE', False) self._environment_variables_dict['policy_output'] = EnvironmentVariables.get_env('policy_output', '') self._environment_variables_dict['bucket'] = EnvironmentVariables.get_env('bucket', '') self._environment_variables_dict['file_path'] = EnvironmentVariables.get_env('file_path', '') self._environment_variables_dict['file_name'] = EnvironmentVariables.get_env('file_name', '') - self._environment_variables_dict['SHUTDOWN_PERIOD'] = EnvironmentVariables.get_boolean_from_environment('SHUTDOWN_PERIOD', False) + self._environment_variables_dict['SHUTDOWN_PERIOD'] = EnvironmentVariables.get_boolean_from_environment( + 'SHUTDOWN_PERIOD', False) # common elastic search vars self._environment_variables_dict['upload_data_elk'] = EnvironmentVariables.get_env('upload_data_elk', '') self._environment_variables_dict['upload_data_es'] = EnvironmentVariables.get_env('upload_data_es', '') @@ -145,7 +203,8 @@ def __init__(self): self._environment_variables_dict['SENDER_MAIL'] = EnvironmentVariables.get_env('SENDER_MAIL', '') self._environment_variables_dict['SENDER_PASSWORD'] = EnvironmentVariables.get_env('SENDER_PASSWORD', '') self._environment_variables_dict['REPLY_TO'] = EnvironmentVariables.get_env('REPLY_TO', 'dev-null@redhat.com') - self._environment_variables_dict['special_user_mails'] = EnvironmentVariables.get_env('special_user_mails', '{}') + self._environment_variables_dict['special_user_mails'] = EnvironmentVariables.get_env('special_user_mails', + '{}') self._environment_variables_dict['account_admin'] = EnvironmentVariables.get_env('account_admin', '') self._environment_variables_dict['IGNORE_MAILS'] = EnvironmentVariables.get_env('IGNORE_MAILS', '') self._environment_variables_dict['MAXIMUM_THRESHOLD'] = EnvironmentVariables.get_env('MAXIMUM_THRESHOLD', '') @@ -155,12 +214,14 @@ def __init__(self): get_boolean_from_environment('ALERT_DRY_RUN', False)) # Google Drive env vars - self._environment_variables_dict['GOOGLE_APPLICATION_CREDENTIALS'] = EnvironmentVariables.get_env('GOOGLE_APPLICATION_CREDENTIALS', '') + self._environment_variables_dict['GOOGLE_APPLICATION_CREDENTIALS'] = EnvironmentVariables.get_env( + 'GOOGLE_APPLICATION_CREDENTIALS', '') self._environment_variables_dict['SPREADSHEET_ID'] = EnvironmentVariables.get_env('SPREADSHEET_ID', '') # AWS Top Acconut self._environment_variables_dict['AWS_ACCOUNT_ROLE'] = EnvironmentVariables.get_env('AWS_ACCOUNT_ROLE', '') - self._environment_variables_dict['PAYER_SUPPORT_FEE_CREDIT'] = EnvironmentVariables.get_env('PAYER_SUPPORT_FEE_CREDIT', 0) + self._environment_variables_dict['PAYER_SUPPORT_FEE_CREDIT'] = EnvironmentVariables.get_env( + 'PAYER_SUPPORT_FEE_CREDIT', 0) self._environment_variables_dict['TEMPORARY_DIR'] = EnvironmentVariables.get_env('TEMPORARY_DIR', '/tmp') self._environment_variables_dict['COST_CENTER_OWNER'] = EnvironmentVariables.get_env('COST_CENTER_OWNER', '{}') @@ -175,69 +236,95 @@ def __init__(self): self._environment_variables_dict['CRO_PORTAL'] = EnvironmentVariables.get_env('CRO_PORTAL', '') self._environment_variables_dict['CLOUD_NAME'] = EnvironmentVariables.get_env('CLOUD_NAME', '') self._environment_variables_dict['MONITOR'] = EnvironmentVariables.get_env('MONITOR', '') - self._environment_variables_dict['MANAGEMENT'] = EnvironmentVariables.get_boolean_from_environment('MANAGEMENT', False) + self._environment_variables_dict['MANAGEMENT'] = EnvironmentVariables.get_boolean_from_environment('MANAGEMENT', + False) # GCP Account self._environment_variables_dict['GCP_DATABASE_NAME'] = EnvironmentVariables.get_env('GCP_DATABASE_NAME') - self._environment_variables_dict['GCP_DATABASE_TABLE_NAME'] = EnvironmentVariables.get_env('GCP_DATABASE_TABLE_NAME') + self._environment_variables_dict['GCP_DATABASE_TABLE_NAME'] = EnvironmentVariables.get_env( + 'GCP_DATABASE_TABLE_NAME') if self._environment_variables_dict.get('GCP_DATABASE_TABLE_NAME'): self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'GCP' - self._environment_variables_dict['EMAIL_ALERT'] = EnvironmentVariables.get_boolean_from_environment('EMAIL_ALERT', True) - self._environment_variables_dict['MANAGER_EMAIL_ALERT'] = EnvironmentVariables.get_boolean_from_environment('MANAGER_EMAIL_ALERT', True) - self._environment_variables_dict['UPDATE_TAG_BULKS'] = int(EnvironmentVariables.get_env('UPDATE_TAG_BULKS', '20')) + self._environment_variables_dict['EMAIL_ALERT'] = EnvironmentVariables.get_boolean_from_environment( + 'EMAIL_ALERT', True) + self._environment_variables_dict['MANAGER_EMAIL_ALERT'] = EnvironmentVariables.get_boolean_from_environment( + 'MANAGER_EMAIL_ALERT', True) + self._environment_variables_dict['UPDATE_TAG_BULKS'] = int( + EnvironmentVariables.get_env('UPDATE_TAG_BULKS', '20')) # policies aggregate alert self._environment_variables_dict['SAVE_TO_FILE_PATH'] = EnvironmentVariables.get_env('SAVE_TO_FILE_PATH', '') self._environment_variables_dict['BUCKET_NAME'] = EnvironmentVariables.get_env('BUCKET_NAME') self._environment_variables_dict['BUCKET_KEY'] = EnvironmentVariables.get_env('BUCKET_KEY') - self._environment_variables_dict['MAIL_ALERT_DAYS'] = literal_eval(EnvironmentVariables.get_env('MAIL_ALERT_DAYS', '[]')) - self._environment_variables_dict['POLICY_ACTIONS_DAYS'] = literal_eval(EnvironmentVariables.get_env('POLICY_ACTIONS_DAYS', '[]')) - self._environment_variables_dict['DEFAULT_ADMINS'] = literal_eval(EnvironmentVariables.get_env('DEFAULT_ADMINS', '[]')) - self._environment_variables_dict['KERBEROS_USERS'] = literal_eval(EnvironmentVariables.get_env('KERBEROS_USERS', '[]')) - self._environment_variables_dict['POLICIES_TO_ALERT'] = literal_eval(EnvironmentVariables.get_env('POLICIES_TO_ALERT', '[]')) + self._environment_variables_dict['MAIL_ALERT_DAYS'] = literal_eval( + EnvironmentVariables.get_env('MAIL_ALERT_DAYS', '[]')) + self._environment_variables_dict['POLICY_ACTIONS_DAYS'] = literal_eval( + EnvironmentVariables.get_env('POLICY_ACTIONS_DAYS', '[]')) + self._environment_variables_dict['DEFAULT_ADMINS'] = literal_eval( + EnvironmentVariables.get_env('DEFAULT_ADMINS', '[]')) + self._environment_variables_dict['KERBEROS_USERS'] = literal_eval( + EnvironmentVariables.get_env('KERBEROS_USERS', '[]')) + self._environment_variables_dict['POLICIES_TO_ALERT'] = literal_eval( + EnvironmentVariables.get_env('POLICIES_TO_ALERT', '[]')) self._environment_variables_dict['ADMIN_MAIL_LIST'] = EnvironmentVariables.get_env('ADMIN_MAIL_LIST', '') if self._environment_variables_dict.get('policy') in ['send_aggregated_alerts', 'cloudability_cost_reports']: self._environment_variables_dict['COMMON_POLICIES'] = True # CRO -- Cloud Resource Orch - self._environment_variables_dict['CLOUD_RESOURCE_ORCHESTRATION'] = EnvironmentVariables.get_boolean_from_environment('CLOUD_RESOURCE_ORCHESTRATION', False) + self._environment_variables_dict[ + 'CLOUD_RESOURCE_ORCHESTRATION'] = EnvironmentVariables.get_boolean_from_environment( + 'CLOUD_RESOURCE_ORCHESTRATION', False) self._environment_variables_dict['USER_COST_INDEX'] = EnvironmentVariables.get_env('USER_COST_INDEX', '') - self._environment_variables_dict['CRO_ES_INDEX'] = EnvironmentVariables.get_env('CRO_ES_INDEX', 'cloud-governance-resource-orchestration') - self._environment_variables_dict['CRO_COST_OVER_USAGE'] = int(EnvironmentVariables.get_env('CRO_COST_OVER_USAGE', '500')) - self._environment_variables_dict['CRO_DEFAULT_ADMINS'] = literal_eval(EnvironmentVariables.get_env('CRO_DEFAULT_ADMINS', "[]")) - self._environment_variables_dict['CRO_DURATION_DAYS'] = int(EnvironmentVariables.get_env('CRO_DURATION_DAYS', '30')) - self._environment_variables_dict['RUN_ACTIVE_REGIONS'] = EnvironmentVariables.get_boolean_from_environment('RUN_ACTIVE_REGIONS', False) - self._environment_variables_dict['CRO_RESOURCE_TAG_NAME'] = EnvironmentVariables.get_env('CRO_RESOURCE_TAG_NAME', 'TicketId') - self._environment_variables_dict['CRO_REPLACED_USERNAMES'] = literal_eval(EnvironmentVariables.get_env('CRO_REPLACED_USERNAMES', "['osdCcsAdmin']")) + self._environment_variables_dict['CRO_ES_INDEX'] = EnvironmentVariables.get_env('CRO_ES_INDEX', + 'cloud-governance-resource-orchestration') + self._environment_variables_dict['CRO_COST_OVER_USAGE'] = int( + EnvironmentVariables.get_env('CRO_COST_OVER_USAGE', '500')) + self._environment_variables_dict['CRO_DEFAULT_ADMINS'] = literal_eval( + EnvironmentVariables.get_env('CRO_DEFAULT_ADMINS', "[]")) + self._environment_variables_dict['CRO_DURATION_DAYS'] = int( + EnvironmentVariables.get_env('CRO_DURATION_DAYS', '30')) + self._environment_variables_dict['RUN_ACTIVE_REGIONS'] = EnvironmentVariables.get_boolean_from_environment( + 'RUN_ACTIVE_REGIONS', False) + self._environment_variables_dict['CRO_RESOURCE_TAG_NAME'] = EnvironmentVariables.get_env( + 'CRO_RESOURCE_TAG_NAME', 'TicketId') + self._environment_variables_dict['CRO_REPLACED_USERNAMES'] = literal_eval( + EnvironmentVariables.get_env('CRO_REPLACED_USERNAMES', "['osdCcsAdmin']")) self._environment_variables_dict['CE_PAYER_INDEX'] = EnvironmentVariables.get_env('CE_PAYER_INDEX', '') self._environment_variables_dict['EMAIL_TO'] = EnvironmentVariables.get_env('EMAIL_TO', '') self._environment_variables_dict['EMAIL_CC'] = literal_eval(EnvironmentVariables.get_env('EMAIL_CC', "[]")) - self._environment_variables_dict['MANAGER_ESCALATION_DAYS'] = int(EnvironmentVariables.get_env('MANAGER_ESCALATION_DAYS', '3')) - self._environment_variables_dict['GLOBAL_CLOUD_ADMIN'] = EnvironmentVariables.get_env('GLOBAL_CLOUD_ADMIN', 'natashba') - self._environment_variables_dict['TICKET_OVER_USAGE_LIMIT'] = int(EnvironmentVariables.get_env('TICKET_OVER_USAGE_LIMIT', '80')) + self._environment_variables_dict['MANAGER_ESCALATION_DAYS'] = int( + EnvironmentVariables.get_env('MANAGER_ESCALATION_DAYS', '3')) + self._environment_variables_dict['GLOBAL_CLOUD_ADMIN'] = EnvironmentVariables.get_env('GLOBAL_CLOUD_ADMIN', + 'natashba') + self._environment_variables_dict['TICKET_OVER_USAGE_LIMIT'] = int( + EnvironmentVariables.get_env('TICKET_OVER_USAGE_LIMIT', '80')) # AWS Athena self._environment_variables_dict['S3_RESULTS_PATH'] = EnvironmentVariables.get_env('S3_RESULTS_PATH', '') self._environment_variables_dict['DEFAULT_ROUND_DIGITS'] = \ int(EnvironmentVariables.get_env('DEFAULT_ROUND_DIGITS', '3')) - self._environment_variables_dict['ATHENA_DATABASE_NAME'] = EnvironmentVariables.get_env('ATHENA_DATABASE_NAME', '') + self._environment_variables_dict['ATHENA_DATABASE_NAME'] = EnvironmentVariables.get_env('ATHENA_DATABASE_NAME', + '') self._environment_variables_dict['ATHENA_TABLE_NAME'] = EnvironmentVariables.get_env('ATHENA_TABLE_NAME', '') - self._environment_variables_dict['ATHENA_ACCOUNT_ACCESS_KEY'] = EnvironmentVariables.get_env('ATHENA_ACCOUNT_ACCESS_KEY', '') - self._environment_variables_dict['ATHENA_ACCOUNT_SECRET_KEY'] = EnvironmentVariables.get_env('ATHENA_ACCOUNT_SECRET_KEY', '') + self._environment_variables_dict['ATHENA_ACCOUNT_ACCESS_KEY'] = EnvironmentVariables.get_env( + 'ATHENA_ACCOUNT_ACCESS_KEY', '') + self._environment_variables_dict['ATHENA_ACCOUNT_SECRET_KEY'] = EnvironmentVariables.get_env( + 'ATHENA_ACCOUNT_SECRET_KEY', '') # Cloudability - self._environment_variables_dict['CLOUDABILITY_VIEW_ID'] = EnvironmentVariables.get_env('CLOUDABILITY_VIEW_ID', '') + self._environment_variables_dict['CLOUDABILITY_VIEW_ID'] = EnvironmentVariables.get_env('CLOUDABILITY_VIEW_ID', + '') self._environment_variables_dict['APPITO_ENVID'] = EnvironmentVariables.get_env('APPITO_ENVID', '') self._environment_variables_dict['APPITO_KEY_SECRET'] = EnvironmentVariables.get_env('APPITO_KEY_SECRET', '') self._environment_variables_dict['APPITO_KEY_ACCESS'] = EnvironmentVariables.get_env('APPITO_KEY_ACCESS', '') self._environment_variables_dict['CLOUDABILITY_API'] = EnvironmentVariables.get_env('CLOUDABILITY_API', '') - self._environment_variables_dict['CLOUDABILITY_API_REPORTS_PATH'] = EnvironmentVariables.get_env('CLOUDABILITY_API_REPORTS_PATH', '') - self._environment_variables_dict['CLOUDABILITY_METRICS'] = EnvironmentVariables.get_env('CLOUDABILITY_METRICS', 'unblended_cost') - self._environment_variables_dict['CLOUDABILITY_DIMENSIONS'] = EnvironmentVariables.get_env('CLOUDABILITY_DIMENSIONS', 'date,category4,vendor_account_name,vendor_account_identifier,vendor') - - - + self._environment_variables_dict['CLOUDABILITY_API_REPORTS_PATH'] = EnvironmentVariables.get_env( + 'CLOUDABILITY_API_REPORTS_PATH', '') + self._environment_variables_dict['CLOUDABILITY_METRICS'] = EnvironmentVariables.get_env('CLOUDABILITY_METRICS', + 'unblended_cost') + self._environment_variables_dict['CLOUDABILITY_DIMENSIONS'] = EnvironmentVariables.get_env( + 'CLOUDABILITY_DIMENSIONS', 'date,category4,vendor_account_name,vendor_account_identifier,vendor') @staticmethod def to_bool(arg, def_val: bool = None): @@ -274,7 +361,6 @@ def get_aws_account_alias_name(self): except: return os.environ.get('account', '').upper() - @staticmethod def get_env(var: str, defval=''): lcvar = var.lower() diff --git a/cloud_governance/main/example.yaml b/cloud_governance/main/example.yaml new file mode 100644 index 00000000..8c59abad --- /dev/null +++ b/cloud_governance/main/example.yaml @@ -0,0 +1,167 @@ +policy: "" +account: "" + + +# AWS values +PUBLIC_CLOUD_NAME: AWS +AWS_DEFAULT_REGION: "us-east-2" +AWS_ACCESS_KEY_ID: "" +AWS_SECRET_ACCESS_KEY: "" + +# AWS Assumed Role Acconut +AWS_ACCOUNT_ROLE: "" +PAYER_SUPPORT_FEE_CREDIT: "" +TEMPORARY_DIR: /tmp + +# AWS Athena +S3_RESULTS_PATH: "" +DEFAULT_ROUND_DIGITS: 3 +ATHENA_DATABASE_NAME: "" +ATHENA_TABLE_NAME: "" +ATHENA_ACCOUNT_ACCESS_KEY: "" +ATHENA_ACCOUNT_SECRET_KEY: "" + +aws_non_cluster_policies: [ 'instance_idle', 'ec2_stop', 'ebs_in_use', + 'ebs_unattached', 's3_inactive', + 'empty_roles', 'ip_unattached', + 'unused_nat_gateway', + 'zombie_snapshots', 'skipped_resources', + 'monthly_report', 'optimize_resources_report' ] +cost_policies: [ 'cost_explorer', 'cost_over_usage', 'cost_billing_reports', + 'cost_explorer_payer_billings', 'spot_savings_analysis' ] + +policy_output: "" +bucket: "" +cost_metric: UnblendedCost +start_date: "" +end_date: "" +granularity: DAILY +cost_explorer_tags: { } + +resource_name: "" +mandatory_tags: "" +tag_operation: "" +validate_type: "" +user_tags: "" +user_tag_operation: "" +username: "" +remove_tags: "" +resource: "" +cluster_tag: "" +service_type: "" +TABLE_NAME: "" +REPLACE_ACCOUNT_NAME: { } + +# IBM +ibm_policies: [ 'tag_baremetal', 'tag_vm', 'ibm_cost_report', + 'ibm_cost_over_usage' ] +IBM_ACCOUNT_ID: "" +IBM_API_USERNAME: "" +IBM_API_KEY: "" +USAGE_REPORTS_APIKEY: "" +month: "" +year: "" +tag_remove_name: "" +tag_custom: { } + +# AZURE Credentials +AZURE_ACCOUNT_ID: "" +AZURE_CLIENT_ID: "" +AZURE_TENANT_ID: "" +AZURE_CLIENT_SECRET: "" +AZURE_SUBSCRIPTION_ID: "" + +# Google Account env vars +GCP_DATABASE_NAME: "" +GCP_DATABASE_TABLE_NAME: "" +GOOGLE_APPLICATION_CREDENTIALS: "" +SPREADSHEET_ID: "" + + +# ElasticSearch +es_index: 'cloud-governance-policy-es-index' +upload_data_elk: "" +upload_data_es: "" +es_host: "" +es_port: "" +es_doc_type: "" +ES_TIMEOUT: 2000 + + +# Mail alerts env vars +# ldap env var +LDAP_HOST_NAME: "" +SENDER_MAIL: "" +SENDER_PASSWORD: "" +REPLY_TO: dev-null@redhat.com +special_user_mails: { } +account_admin: "" +IGNORE_MAILS: "" +MAXIMUM_THRESHOLD: "" +to_mail: [ ] +cc_mail: [ ] +ALERT_DRY_RUN: false + +# Cloudability + +CLOUDABILITY_VIEW_ID: "" +APPITO_ENVID: "" +APPITO_KEY_SECRET: "" +APPITO_KEY_ACCESS: "" +CLOUDABILITY_API: "" +CLOUDABILITY_API_REPORTS_PATH: "" +CLOUDABILITY_METRICS: unblended_cost +CLOUDABILITY_DIMENSIONS: "date,category4,vendor_account_name,vendor_account_identifier,vendor" + + +# Common Values +dry_run: yes +TOTAL_ACCOUNTS: false +DAYS_TO_DELETE_RESOURCE: 7 +DAYS_TO_TAKE_ACTION: 7 +FORCE_DELETE: false +file_path: "" +file_name: "" +SHUTDOWN_PERIOD: false +PRINT_LOGS: true +COST_CENTER_OWNER: { } +EMAIL_ALERT: true +MANAGER_EMAIL_ALERT: true +UPDATE_TAG_BULKS: 20 +SAVE_TO_FILE_PATH: "" +ADMIN_MAIL_LIST: "" +COMMON_POLICIES: false + +# GitHub credentials +git_access_token: "" +git_repo: "" +several_repos: "" + +# Jira env parameters +JIRA_URL: "" +JIRA_USERNAME: "" +JIRA_TOKEN: "" +JIRA_QUEUE: "" +JIRA_PASSWORD: "" + + +# CRO -- Cloud Resource Orch +CRO_PORTAL: "" +CLOUD_NAME: "" +MONITOR: "" +MANAGEMENT: false +CLOUD_RESOURCE_ORCHESTRATION: false +USER_COST_INDEX: "" +CRO_ES_INDEX: cloud-governance-resource-orchestration +CRO_COST_OVER_USAGE: 500 +CRO_DEFAULT_ADMINS: [ ] +CRO_DURATION_DAYS: 30 +RUN_ACTIVE_REGIONS: false +CRO_RESOURCE_TAG_NAME: TicketId +CRO_REPLACED_USERNAMES: [ "osdCcsAdmin" ] +CE_PAYER_INDEX: "" +EMAIL_TO: "" +EMAIL_CC: [ ] +MANAGER_ESCALATION_DAYS: 3 +GLOBAL_CLOUD_ADMIN: natashba +TICKET_OVER_USAGE_LIMIT: 80