From 9ba39f0af3dc2b2b0bcef7fc0e8beb0431eb1d85 Mon Sep 17 00:00:00 2001 From: Sachin Agarwal Date: Wed, 22 Dec 2021 20:51:14 -0500 Subject: [PATCH 1/6] wip for module fix --- .../create-and-destroy-aws/environment.yml | 6 +++--- opta.spec | 21 +++++++++++++++++-- opta/layer.py | 4 +++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/examples/ci-tests/create-and-destroy-aws/environment.yml b/examples/ci-tests/create-and-destroy-aws/environment.yml index 7a0b2a100..44c2e24c3 100644 --- a/examples/ci-tests/create-and-destroy-aws/environment.yml +++ b/examples/ci-tests/create-and-destroy-aws/environment.yml @@ -1,10 +1,10 @@ # This opta environment is used by the test-ci job in the runxc repo. -name: awsenv-ci +name: awsenv-ci-sachin org_name: runx providers: aws: region: us-east-2 - account_id: 693658092572 + account_id: 652824372180 modules: - type: base - type: dns @@ -15,4 +15,4 @@ modules: spot_instances: true - type: k8s-base admin_arns: - - "arn:aws:iam::693658092572:user/github-actions" + - "arn:aws:iam::652824372180:user/github-actions" diff --git a/opta.spec b/opta.spec index fdd5fc054..8c3ff5581 100644 --- a/opta.spec +++ b/opta.spec @@ -4,7 +4,24 @@ from PyInstaller.utils.hooks import collect_data_files block_cipher = None - +himps = [ +"mypy", +"mypy-boto3-acm", +"mypy-boto3-autoscaling", +"mypy-boto3-cloudformation", +"mypy-boto3-dynamodb", +"mypy-boto3-ec2", +"mypy-boto3-elbv2", +"mypy-boto3-lambda", +"mypy-boto3-logs", +"mypy-boto3-rds", +"mypy-boto3-route53", +"mypy-boto3-s3", +"mypy-boto3-sesv2", +"mypy-boto3-sqs", +"mypy-boto3-ssm", +"mypy-extensions", +] a = Analysis(['opta/cli.py'], pathex=[], binaries=[], @@ -13,7 +30,7 @@ a = Analysis(['opta/cli.py'], ('./modules', 'modules'), ('roots.pem', 'grpc/_cython/_credentials/'), ] + collect_data_files('hcl2'), - hiddenimports=[], + hiddenimports=himps, hookspath=[], runtime_hooks=[], excludes=[], diff --git a/opta/layer.py b/opta/layer.py index 2abebe338..aab87239f 100644 --- a/opta/layer.py +++ b/opta/layer.py @@ -1,5 +1,5 @@ from __future__ import annotations - +from traceback import format_exc import hashlib import importlib import os @@ -80,6 +80,8 @@ def get_processor_class(module_type: str) -> ModuleProcessor: pymodule_path = generate_pymodule_path(module_type) pymodule = importlib.import_module(pymodule_path) except ModuleNotFoundError: + print ("While getting processor class for {} ".format(module_type)) + print(format_exc()) return ModuleProcessor return pymodule.__dict__[PROCESSOR_DICT[module_type]] From 417aae50ca737363ce578ff233288e65a5e2ca99 Mon Sep 17 00:00:00 2001 From: Sachin Agarwal Date: Thu, 23 Dec 2021 17:30:24 +0000 Subject: [PATCH 2/6] Added hidden imports to builder --- opta.spec | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/opta.spec b/opta.spec index 8c3ff5581..79868462c 100644 --- a/opta.spec +++ b/opta.spec @@ -5,22 +5,35 @@ from PyInstaller.utils.hooks import collect_data_files block_cipher = None himps = [ -"mypy", -"mypy-boto3-acm", -"mypy-boto3-autoscaling", -"mypy-boto3-cloudformation", -"mypy-boto3-dynamodb", -"mypy-boto3-ec2", -"mypy-boto3-elbv2", -"mypy-boto3-lambda", -"mypy-boto3-logs", -"mypy-boto3-rds", -"mypy-boto3-route53", -"mypy-boto3-s3", -"mypy-boto3-sesv2", -"mypy-boto3-sqs", -"mypy-boto3-ssm", -"mypy-extensions", +"modules.aws_k8s_service.aws_k8s_service", +"modules.aws_k8s_base.aws_k8s_base", +"modules.datadog.datadog", +"modules.gcp_k8s_base.gcp_k8s_base", +"modules.gcp_k8s_service.gcp_k8s_service", +"modules.gcp_gke.gcp_gke", +"modules.aws_dns.aws_dns", +"modules.aws_documentdb.aws_documentdb", +"modules.runx.runx", +"modules.helm_chart.helm_chart", +"modules.aws_iam_role.aws_iam_role", +"modules.aws_iam_user.aws_iam_user", +"modules.aws_eks.aws_eks", +"modules.aws_ses.aws_ses", +"modules.aws_sqs.aws_sqs", +"modules.aws_sns.aws_sns", +"modules.azure_base.azure_base", +"modules.azure_k8s_base.azure_k8s_base", +"modules.azure_k8s_service.azure_k8s_service", +"modules.local_k8s_service.local_k8s_service", +"modules.external_ssl_cert.external_ssl_cert", +"modules.aws_s3.aws_s3", +"modules.gcp_dns.gcp_dns", +"modules.gcp_service_account.gcp_service_account", +"modules.custom_terraform.custom_terraform", +"modules.aws_dynamodb.aws_dynamodb", +"modules.mongodb_atlas.mongodb_atlas", +"modules.cloudfront_distribution.cloudfront_distribution", +"modules.lambda_function.lambda_function" ] a = Analysis(['opta/cli.py'], pathex=[], From 6612dc078c12ba441b3e4565cf51258c981a7f69 Mon Sep 17 00:00:00 2001 From: Sachin Agarwal Date: Thu, 23 Dec 2021 12:32:24 -0500 Subject: [PATCH 3/6] Linting --- opta/layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opta/layer.py b/opta/layer.py index aab87239f..dd779db48 100644 --- a/opta/layer.py +++ b/opta/layer.py @@ -80,7 +80,7 @@ def get_processor_class(module_type: str) -> ModuleProcessor: pymodule_path = generate_pymodule_path(module_type) pymodule = importlib.import_module(pymodule_path) except ModuleNotFoundError: - print ("While getting processor class for {} ".format(module_type)) + print("While getting processor class for {} ".format(module_type)) print(format_exc()) return ModuleProcessor From a69ca31d51bfc1c3eeb52f1248cf9952c4889b19 Mon Sep 17 00:00:00 2001 From: Sachin Agarwal Date: Thu, 23 Dec 2021 12:37:26 -0500 Subject: [PATCH 4/6] Removed debug info --- examples/ci-tests/create-and-destroy-aws/environment.yml | 6 +++--- opta/layer.py | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/examples/ci-tests/create-and-destroy-aws/environment.yml b/examples/ci-tests/create-and-destroy-aws/environment.yml index 44c2e24c3..7a0b2a100 100644 --- a/examples/ci-tests/create-and-destroy-aws/environment.yml +++ b/examples/ci-tests/create-and-destroy-aws/environment.yml @@ -1,10 +1,10 @@ # This opta environment is used by the test-ci job in the runxc repo. -name: awsenv-ci-sachin +name: awsenv-ci org_name: runx providers: aws: region: us-east-2 - account_id: 652824372180 + account_id: 693658092572 modules: - type: base - type: dns @@ -15,4 +15,4 @@ modules: spot_instances: true - type: k8s-base admin_arns: - - "arn:aws:iam::652824372180:user/github-actions" + - "arn:aws:iam::693658092572:user/github-actions" diff --git a/opta/layer.py b/opta/layer.py index dd779db48..2abebe338 100644 --- a/opta/layer.py +++ b/opta/layer.py @@ -1,5 +1,5 @@ from __future__ import annotations -from traceback import format_exc + import hashlib import importlib import os @@ -80,8 +80,6 @@ def get_processor_class(module_type: str) -> ModuleProcessor: pymodule_path = generate_pymodule_path(module_type) pymodule = importlib.import_module(pymodule_path) except ModuleNotFoundError: - print("While getting processor class for {} ".format(module_type)) - print(format_exc()) return ModuleProcessor return pymodule.__dict__[PROCESSOR_DICT[module_type]] From fb4cefb77a31f11d55e007b16271943dc62e040e Mon Sep 17 00:00:00 2001 From: Sachin Agarwal Date: Mon, 3 Jan 2022 09:43:40 -0500 Subject: [PATCH 5/6] Adding function to generate module names --- opta.spec | 76 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/opta.spec b/opta.spec index 79868462c..0f5b95e9e 100644 --- a/opta.spec +++ b/opta.spec @@ -1,40 +1,54 @@ # -*- mode: python ; coding: utf-8 -*- from PyInstaller.utils.hooks import collect_data_files +import os +def generate_module_import_array(): + himps = [] + for root, dirs, files in os.walk("./modules"): + for filename in files: + if filename.endswith(".py") and not filename.startswith("test"): + modname = filename.split(".py")[0] + himps.append(".".join(["modules", modname, modname])) + return sorted(himps) + + block_cipher = None -himps = [ -"modules.aws_k8s_service.aws_k8s_service", -"modules.aws_k8s_base.aws_k8s_base", -"modules.datadog.datadog", -"modules.gcp_k8s_base.gcp_k8s_base", -"modules.gcp_k8s_service.gcp_k8s_service", -"modules.gcp_gke.gcp_gke", -"modules.aws_dns.aws_dns", -"modules.aws_documentdb.aws_documentdb", -"modules.runx.runx", -"modules.helm_chart.helm_chart", -"modules.aws_iam_role.aws_iam_role", -"modules.aws_iam_user.aws_iam_user", -"modules.aws_eks.aws_eks", -"modules.aws_ses.aws_ses", -"modules.aws_sqs.aws_sqs", -"modules.aws_sns.aws_sns", -"modules.azure_base.azure_base", -"modules.azure_k8s_base.azure_k8s_base", -"modules.azure_k8s_service.azure_k8s_service", -"modules.local_k8s_service.local_k8s_service", -"modules.external_ssl_cert.external_ssl_cert", -"modules.aws_s3.aws_s3", -"modules.gcp_dns.gcp_dns", -"modules.gcp_service_account.gcp_service_account", -"modules.custom_terraform.custom_terraform", -"modules.aws_dynamodb.aws_dynamodb", -"modules.mongodb_atlas.mongodb_atlas", -"modules.cloudfront_distribution.cloudfront_distribution", -"modules.lambda_function.lambda_function" -] +himps = generate_module_import_array() +# himps = [ +# "modules.aws_k8s_service.aws_k8s_service", +# "modules.aws_k8s_base.aws_k8s_base", +# "modules.datadog.datadog", +# "modules.gcp_k8s_base.gcp_k8s_base", +# "modules.gcp_k8s_service.gcp_k8s_service", +# "modules.gcp_gke.gcp_gke", +# "modules.aws_dns.aws_dns", +# "modules.aws_documentdb.aws_documentdb", +# "modules.runx.runx", +# "modules.helm_chart.helm_chart", +# "modules.aws_iam_role.aws_iam_role", +# "modules.aws_iam_user.aws_iam_user", +# "modules.aws_eks.aws_eks", +# "modules.aws_ses.aws_ses", +# "modules.aws_sqs.aws_sqs", +# "modules.aws_sns.aws_sns", +# "modules.azure_base.azure_base", +# "modules.azure_k8s_base.azure_k8s_base", +# "modules.azure_k8s_service.azure_k8s_service", +# "modules.local_k8s_service.local_k8s_service", +# "modules.external_ssl_cert.external_ssl_cert", +# "modules.aws_s3.aws_s3", +# "modules.gcp_dns.gcp_dns", +# "modules.gcp_service_account.gcp_service_account", +# "modules.custom_terraform.custom_terraform", +# "modules.aws_dynamodb.aws_dynamodb", +# "modules.mongodb_atlas.mongodb_atlas", +# "modules.cloudfront_distribution.cloudfront_distribution", +# "modules.lambda_function.lambda_function" +# ] +print(himps) + a = Analysis(['opta/cli.py'], pathex=[], binaries=[], From d035a9e65ce23294c8bd4a8f288921c215025770 Mon Sep 17 00:00:00 2001 From: Sachin Agarwal Date: Mon, 3 Jan 2022 21:28:05 -0500 Subject: [PATCH 6/6] Removed commented code --- opta.spec | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/opta.spec b/opta.spec index 0f5b95e9e..0092c627d 100644 --- a/opta.spec +++ b/opta.spec @@ -16,38 +16,6 @@ def generate_module_import_array(): block_cipher = None himps = generate_module_import_array() -# himps = [ -# "modules.aws_k8s_service.aws_k8s_service", -# "modules.aws_k8s_base.aws_k8s_base", -# "modules.datadog.datadog", -# "modules.gcp_k8s_base.gcp_k8s_base", -# "modules.gcp_k8s_service.gcp_k8s_service", -# "modules.gcp_gke.gcp_gke", -# "modules.aws_dns.aws_dns", -# "modules.aws_documentdb.aws_documentdb", -# "modules.runx.runx", -# "modules.helm_chart.helm_chart", -# "modules.aws_iam_role.aws_iam_role", -# "modules.aws_iam_user.aws_iam_user", -# "modules.aws_eks.aws_eks", -# "modules.aws_ses.aws_ses", -# "modules.aws_sqs.aws_sqs", -# "modules.aws_sns.aws_sns", -# "modules.azure_base.azure_base", -# "modules.azure_k8s_base.azure_k8s_base", -# "modules.azure_k8s_service.azure_k8s_service", -# "modules.local_k8s_service.local_k8s_service", -# "modules.external_ssl_cert.external_ssl_cert", -# "modules.aws_s3.aws_s3", -# "modules.gcp_dns.gcp_dns", -# "modules.gcp_service_account.gcp_service_account", -# "modules.custom_terraform.custom_terraform", -# "modules.aws_dynamodb.aws_dynamodb", -# "modules.mongodb_atlas.mongodb_atlas", -# "modules.cloudfront_distribution.cloudfront_distribution", -# "modules.lambda_function.lambda_function" -# ] -print(himps) a = Analysis(['opta/cli.py'], pathex=[],