From 758abc8a0f66a78537d7c63226478aebe910d657 Mon Sep 17 00:00:00 2001 From: Riccardo Carlesso Date: Fri, 8 Mar 2024 12:26:30 +0100 Subject: [PATCH] I love my script more and more --- bin/auto_import_config.py | 162 +++++++++++++++++++++++++++++++++----- 1 file changed, 142 insertions(+), 20 deletions(-) diff --git a/bin/auto_import_config.py b/bin/auto_import_config.py index 2839919..05e2ffa 100755 --- a/bin/auto_import_config.py +++ b/bin/auto_import_config.py @@ -9,14 +9,23 @@ # TODO make it super NON verbose and when it is auto load it on every CD in the shell :) # .. like direnv works. But first lets make it work :) +from datetime import datetime +import subprocess + import yaml import os +import sys -ProgVersion = '1.1' +ProgVersion = '1.2' ConfigFileName = '.gcloudconfig.yaml' -SampleConfig = '''\ +DryRun = False # set trut in while loop in dev mode ;) +ProgName = os.path.basename(sys.argv[0]) +Now = datetime.now().strftime('%Y:%m:%d %H:%M') +Separator = ('#' * 80) +SampleConfig = '''\import subprocess + ######################################################## -# This is a sample from auto_import_config.py v{version} +# This is a sample from #{ProgName} v{version} # Code: https://github.com/palladius/sakura/blob/master/bin/auto_import_config.py ######################################################## # Once done you can do: gcloud config configurations activate YOUR_VAVORITE_CONFIG @@ -24,37 +33,149 @@ auto: true configurations: default: - project: my-default-personal-project - compute/region: us-central1 - compute/zone: us-central1-b - account: your.personal.email@gmail.com + gcloud: + project: my-default-personal-project + compute/region: us-central1 + compute/zone: us-central1-b + account: your.personal.email@gmail.com # Note: gcloud wont accept this config name if it starts with a number (#btdt) {this_folder}: - project: my-local-work-project - compute/region: us-central1 - compute/zone: us-central1-b - account: your.work.email@my-job.example.com + gcloud: + project: my-local-work-project + compute/region: us-central1 + compute/zone: us-central1-b + account: your.work.email@my-job.example.com + env: + # TODO(ricc): the first 2 should come for free.. + PROJECT_ID: my-local-work-project # this could actually be autopopulated + REGION: us-central1 + BUCKET: gs://my-bucket + VERTEX_TRAINING_JOB_NAME: 'train-test-123' + REPO_NAME: 'my-awesome-app' + # Not implemented yet, but who knows.. but use Pulumi/TF standard namings. + # Works in python + IMAGE_URI: "${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO_NAME}/my_vertex_image:latest" ''' + +def execute(command): + '''Execute code but... with a cactch.''' + if DryRun: + print(f"[DRYRUN] 🔞 {command}") + else: + # executing for real.. no excuses! + os.system(command) + +def get_project_number_from_wrong(project_id): + '''Im so happy. This is usually an inefficient computation but I can automate it!''' + ret = os.system(f"gcloud projects describe {project_id} --format='value(projectNumber)'") + return f"{ret}" # "4242" + +def get_project_number_from(project_id): + process = subprocess.Popen( + [#"echo", + "gcloud", "projects", "describe", project_id, "--format", "value(projectNumber)"], + stdout=subprocess.PIPE + ) + output, err = process.communicate() + + if process.returncode != 0: + raise Exception(f"gcloud command failed: {process.returncode}") # err.decode() + + return output.decode('utf-8').strip() + +# generate_env_files +def setup_env_stubs(config_name, env_properties): + '''Given config X and an array of properties, creates a few stub files. + + Inputs: + * config_name: eg "default" + * env_properties: eg {} + + ''' + print(f"config_name: {config_name}") + print(f"env_properties: {env_properties}") + banner = f"\n{Separator}\n# Generated by {ProgName} v{ProgVersion} on {Now}\n# Do NOT edit or it might get overwritten. Please edit the .gcloudconfig.yaml!\n{Separator}\n\n" + config = env_properties + + # TODO(ricc): move to a lambda which accepts the logic per language and refactor so the OPEN can be caught with if exists... + # Note: os.path.expandvars epamnds with LOCAL env which is wrong. + # This should be current-ENV-independent! + + def expand_vars(value): + if isinstance(value, str): + for key, val in config.items(): + value = value.replace(f'${{{key}}}', val) + return value + + for key, value in config.items(): + config[key] = expand_vars(value) + + # BASH stub + with open('_env_gaic.sh', 'w') as f: + f.write(banner) + f.write("# source '_env_gaic' \n") # TESTED + for key, value in config.items(): + f.write(f'export {key}="{os.path.expandvars(value)}"\n') + + # Python stub + with open('_env_gaic.py', 'w') as f: + f.write(banner) + f.write("# from _env_gaic import *\n") # TESTED + for key, value in config.items(): + f.write(f'{key} = "{os.path.expandvars(value)}"\n') + #f.write("# prova 2 \n") + #for key, value in config.items(): + # f.write(f'{key} = "{expand_vars(value)}"\n') + + # Ruby stub + with open('_env_gaic.rb', 'w') as f: + f.write(banner) + f.write("# require_relative './_env_gaic.rb' \n") + for key, value in config.items(): + f.write(f'ENV["{key}"] = "{os.path.expandvars(value)}"\n') + + def inject_all_configs(config_data): 'Given a hash from YAML it creates one gcloud config per key' # injects config as per tree. for config_name, properties in config_data['configurations'].items(): # First verifies if it exists. - ret = os.system(f"gcloud config configurations describe {config_name} >/dev/null") + ret = execute(f"gcloud config configurations describe {config_name} >/dev/null") #print(f"ret = {ret}") if ret == 0: print(f"🦘 Config '{config_name}' found: skipping") # TODO: configurable FORCE: true - #break - #next continue print(f"➕ Config '{config_name}' NOT found: creating") # Create configuration if it doesn't exist=== - os.system(f"gcloud config configurations create {config_name}") + execute(f"gcloud config configurations create {config_name}") + + opinionated_properties = {} + + # gcloud config ('gcloud' stanza) + for property_name, value in properties['gcloud'].items(): + execute(f"echo gcloud config set {property_name} {value} --configuration {config_name}") + if property_name == 'project': + opinionated_properties['PROJECT_ID'] = value + opinionated_properties['PROJECT_NUMBER'] = get_project_number_from(value) + #opinionated_properties['GOOGLE_PROJECT'] = value # Pulumi supports: GOOGLE_PROJECT, GOOGLE_CLOUD_PROJECT, GCLOUD_PROJECT, CLOUDSDK_CORE_PROJECT https://www.pulumi.com/registry/packages/gcp/installation-configuration/ + if property_name == 'compute/region': + opinionated_properties['GCP_REGION'] = value + #opinionated_properties['GOOGLE_REGION'] = value # Pulumi https://www.pulumi.com/registry/packages/gcp/installation-configuration/ + if property_name == 'compute/zone': + opinionated_properties['GCP_ZONE'] = value + #opinionated_properties['GOOGLE_ZONE'] = value # Pulumi https://www.pulumi.com/registry/packages/gcp/installation-configuration/ + if property_name == 'account': + opinionated_properties['GCP_ACCOUNT'] = value + + # env config ('gcloud' stanza) + # Injecting additional values if present in gcloud stanza.. + #opinionated_properties['REMOVEME'] = 'blah' + opinionated_properties.update(properties['env']) + setup_env_stubs(config_name, opinionated_properties) + - for property_name, value in properties.items(): - os.system(f"gcloud config set {property_name} {value} --configuration {config_name}") def local_dirname(): '''Returns the latest folder of `pwd`.''' @@ -72,7 +193,7 @@ def parse_local_config(config_data): # Alternative: # last_part = current_dir.rsplit('/', 1)[1] print(f'Setting config with same name as local dir: {local_dir}') - os.system(f"gcloud config configurations activate {local_dir}") + execute(f"gcloud config configurations activate {local_dir}") else: print(f"local_config[auto] = '{ config_data['local_config']['auto'] }'") @@ -87,6 +208,7 @@ def main(): # check if file ConfigFileName exists.. config_data = {} if os.path.isfile(ConfigFileName): + #print("[DEB] ConfigFileName found: {ConfigFileName}") with open(ConfigFileName, 'r') as file: config_data = yaml.safe_load(file) else: @@ -108,8 +230,8 @@ def main(): print('no AUTOCONFIG defined') # todo deleteme. # Final - #os.system(f"gcloud config list") - os.system(f"gcloud config configurations list | grep True") + #execute(f"gcloud config list") + execute(f"gcloud config configurations list | grep True") main()