diff --git a/README.md b/README.md index f395d1a..69c1c18 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,13 @@ Extract parameters from your existing config and import to your CloudTruth organ # Usage This utility is distributed as a Docker container and can be pulled from cloudtruth/dynamic-importer on Docker Hub +## Procesing a directory tree (the easy method) +You can feed a directory of files into the `walk-directories` command, which will find all files matching the supplied types and parse them into CloudTruth config formats. If you supply your CLOUDTRUTH_API_KEY via docker, the data will be uploaded to your CloudTruth account. + +``` +docker run --rm -e CLOUDTRUTH_API_KEY="myverysecureS3CR3T!!" -v ${PWD}/files:/app/files cloudtruth/dynamic-importer walk-directories --config-dir /app/samples/ -t dotenv -t json -t tf +``` + ## Processing a single file An example of how to process a .env file ``` @@ -41,13 +48,6 @@ docker run --rm -v ${PWD}/files:/app/files cloudtruth/dynamic-importer process-c --output-dir /app/files/ ``` -## Procesing a directory tree -You may also feed a directory of files into the `walk-directories` command, which will find all files matching the supplied types and parse them into CloudTruth config formats. - -``` -docker run --rm -v ${PWD}/files:/app/files cloudtruth/dynamic-importer walk-directories -c /app/samples/ -t dotenv -t json -t tf -o walk_output/ -``` - ## Editing template references There may be times when this utility is too aggressive or you want a variable to remain hard-coded in your CloudTruth template. In that case, you can remove the references from the generated `.ctconfig` file and re-generate the template. diff --git a/src/dynamic_importer/main.py b/src/dynamic_importer/main.py index 23bad04..d314e47 100644 --- a/src/dynamic_importer/main.py +++ b/src/dynamic_importer/main.py @@ -4,6 +4,7 @@ import os from collections import defaultdict from time import time +from typing import Dict import click import urllib3 @@ -14,7 +15,7 @@ from dynamic_importer.util import validate_env_values from dynamic_importer.walker import walk_files -CREATE_DATA_MSG_INTERVAL = 10 +CREATE_DATA_MSG_INTERVAL = 20 DIRS_TO_IGNORE = [ ".git", ".github", @@ -180,47 +181,60 @@ def create_data(data_file, template_file, project, k, c, u): api_key = os.environ.get("CLOUDTRUTH_API_KEY") or click.prompt( "Enter your CloudTruth API Key", hide_input=True ) + with open(data_file, "r") as dfp, open(template_file, "r") as tfp: + config_data = json.load(dfp) + template_data = tfp.read() + _create_data( + config_data, str(template_file), template_data, project, api_key, k, c, u + ) + + click.echo("Data upload to CloudTruth complete!") + + +def _create_data( + config_data: Dict, + template_name: str, + template_data: str, + project: str, + api_key: str, + k: bool, + c: bool, + u: bool, +): if k: urllib3.disable_warnings() client = CTClient(api_key, skip_ssl_validation=k) - with open(data_file, "r") as fp: - config_data = json.load(fp) - total_params = len(config_data.values()) - click.echo(f"Creating {total_params} parameters") - start_time = time() - i = 0 - for raw_key, config_data in config_data.items(): - i += 1 - client.upsert_parameter( + total_params = len(config_data.values()) + click.echo(f"Creating {total_params} parameters") + start_time = time() + i = 0 + for _, config_data in config_data.items(): + i += 1 + client.upsert_parameter( + project, + name=config_data["param_name"], + type_name=config_data["type"], + secret=config_data["secret"], + create_dependencies=c, + ) + for env, value in config_data["values"].items(): + client.upsert_value( project, - name=config_data["param_name"], - type_name=config_data["type"], - secret=config_data["secret"], + config_data["param_name"], + env, + value, create_dependencies=c, ) - for env, value in config_data["values"].items(): - client.upsert_value( - project, - config_data["param_name"], - env, - value, - create_dependencies=c, - ) - cur_time = time() - if cur_time - start_time > CREATE_DATA_MSG_INTERVAL: - click.echo(f"Created {i} parameters, {total_params - i} remaining") - start_time = time() - with open(template_file, "r") as fp: - template = fp.read() - click.echo(f"Uploading template: {template_file}") - client.upsert_template(project, name=template_file, body=template) - - click.echo("Data upload to CloudTruth complete!") + cur_time = time() + if cur_time - start_time > CREATE_DATA_MSG_INTERVAL: + click.echo(f"Created {i} parameters, {total_params - i} remaining") + start_time = time() + click.echo(f"Usploading template: {template_name}") + client.upsert_template(project, name=template_name, body=template_data) @import_config.command() @click.option( - "-c", "--config-dir", help="Full path to directory to walk and locate configs", required=True, @@ -238,16 +252,16 @@ def create_data(data_file, template_file, project, k, c, u): help="Directory to exclude from walking. Can be specified multiple times", multiple=True, ) -@click.option( - "-o", - "--output-dir", - help="Directory to write processed output to. Default is current directory", - default=".", - required=False, -) -def walk_directories(config_dir, file_types, exclude_dirs, output_dir): +@click.option("-k", help="Ignore SSL certificate verification", is_flag=True) +@click.option("-c", help="Create missing projects and enviroments", is_flag=True) +@click.option("-u", help="Upsert values", is_flag=True) +def walk_directories(config_dir, file_types, exclude_dirs, k, c, u): + """ + Walks a directory, constructs templates and config data, and uploads to CloudTruth. + This is an interactive version of the process_configs and create_data commands. The + user will be prompted for project and environment names as files are walked. + """ walked_files = {} - output_dir = output_dir.rstrip("/") for root, dirs, files in os.walk(config_dir): root = root.rstrip("/") @@ -270,6 +284,8 @@ def walk_directories(config_dir, file_types, exclude_dirs, output_dir): project_files[v["project"]][v["type"]].append( {"path": v["path"], "environment": v["environment"]} ) + + processed_data = defaultdict(dict) for project, type_info in project_files.items(): for file_type, file_meta in type_info.items(): env_paths = {d["environment"]: d["path"] for d in file_meta} @@ -279,17 +295,24 @@ def walk_directories(config_dir, file_types, exclude_dirs, output_dir): processor: BaseProcessor = processing_class(env_paths) template, config_data = processor.process() - template_out_file = f"{output_dir}/{project}-{file_type}.cttemplate" - config_out_file = f"{output_dir}/{project}-{file_type}.ctconfig" + template_name = f"{project}-{file_type}.cttemplate" + template_body = processor.generate_template() - click.echo(f"Writing template to: {template_out_file}") - with open(template_out_file, "w+") as fp: - template_body = processor.generate_template() - fp.write(template_body) + processed_data[project][template_name] = { + "template_body": template_body, + "config_data": config_data, + } - click.echo(f"Writing config data to: {config_out_file}") - with open(config_out_file, "w+") as fp: - json.dump(config_data, fp, indent=4) + api_key = os.environ.get("CLOUDTRUTH_API_KEY") + for project, ct_data in processed_data.items(): + click.echo(f"Uploading data for {project}") + for template_name, template_data in ct_data.items(): + template_body = template_data["template_body"] + config_data = template_data["config_data"] + _create_data( + config_data, template_name, template_body, project, api_key, k, c, u + ) + click.echo("Data upload to CloudTruth complete!") if __name__ == "__main__": diff --git a/src/dynamic_importer/walker.py b/src/dynamic_importer/walker.py index ba82cfc..4c716f7 100644 --- a/src/dynamic_importer/walker.py +++ b/src/dynamic_importer/walker.py @@ -20,7 +20,7 @@ def walk_files( - root: str, files: str, file_types: List[str] + root: str, files: List, file_types: List[str] ) -> Dict[str, Dict[str, str]]: walked_files = {} last_project = None diff --git a/src/tests/test_directory_walking.py b/src/tests/test_directory_walking.py index 4b278b0..a399405 100644 --- a/src/tests/test_directory_walking.py +++ b/src/tests/test_directory_walking.py @@ -1,7 +1,7 @@ from __future__ import annotations -import os import pathlib +from unittest import mock import pytest from click.testing import CliRunner @@ -16,10 +16,15 @@ """ -@pytest.mark.usefixture("tmp_path") +@mock.patch( + "dynamic_importer.main.CTClient", +) @pytest.mark.timeout(30) -def test_walk_directories_one_file_type(tmp_path): - runner = CliRunner() +def test_walk_directories_one_file_type(mock_client): + mock_client = mock.MagicMock() # noqa: F841 + runner = CliRunner( + env={"CLOUDTRUTH_API_HOST": "localhost:8000", "CLOUDTRUTH_API_KEY": "test"} + ) current_dir = pathlib.Path(__file__).parent.resolve() prompt_responses = [ @@ -36,33 +41,37 @@ def test_walk_directories_one_file_type(tmp_path): "", "staging", ] - with runner.isolated_filesystem(temp_dir=tmp_path) as td: - result = runner.invoke( - import_config, - [ - "walk-directories", - "-t", - "dotenv", - "-c", - f"{current_dir}/../../samples/dotenvs", - "--output-dir", - td, - ], - input="\n".join(prompt_responses), - catch_exceptions=False, - ) + result = runner.invoke( + import_config, + [ + "walk-directories", + "-t", + "dotenv", + "--config-dir", + f"{current_dir}/../../samples/dotenvs", + "-c", + "-u", + "-k", + ], + input="\n".join(prompt_responses), + catch_exceptions=False, + ) + try: assert result.exit_code == 0 - - assert pathlib.Path(f"{td}/myproj-dotenv.ctconfig").exists() - assert pathlib.Path(f"{td}/myproj-dotenv.cttemplate").exists() - assert os.path.getsize(f"{td}/myproj-dotenv.ctconfig") > 0 - assert os.path.getsize(f"{td}/myproj-dotenv.cttemplate") > 0 + except AssertionError as e: + print(result.output) + raise e +@mock.patch( + "dynamic_importer.main.CTClient", +) @pytest.mark.timeout(30) -@pytest.mark.usefixtures("tmp_path") -def test_walk_directories_multiple_file_types(tmp_path): - runner = CliRunner() +def test_walk_directories_multiple_file_types(mock_client): + mock_client = mock.MagicMock() # noqa: F841 + runner = CliRunner( + env={"CLOUDTRUTH_API_HOST": "localhost:8000", "CLOUDTRUTH_API_KEY": "test"} + ) current_dir = pathlib.Path(__file__).parent.resolve() prompt_responses = [ @@ -86,48 +95,34 @@ def test_walk_directories_multiple_file_types(tmp_path): "", "staging", ] - with runner.isolated_filesystem(temp_dir=tmp_path) as td: - result = runner.invoke( - import_config, - [ - "walk-directories", - "-t", - "dotenv", - "-c", - f"{current_dir}/../../samples", - "--output-dir", - td, - ], - input="\n".join(prompt_responses), - catch_exceptions=False, - ) + result = runner.invoke( + import_config, + [ + "walk-directories", + "-t", + "dotenv", + "--config-dir", + f"{current_dir}/../../samples", + ], + input="\n".join(prompt_responses), + catch_exceptions=False, + ) + try: assert result.exit_code == 0 - - assert pathlib.Path(f"{td}/dotty-dotenv.ctconfig").exists() - assert pathlib.Path(f"{td}/dotty-dotenv.cttemplate").exists() - assert os.path.getsize(f"{td}/dotty-dotenv.ctconfig") > 0 - assert os.path.getsize(f"{td}/dotty-dotenv.cttemplate") > 0 - - assert pathlib.Path(f"{td}/myproj-dotenv.ctconfig").exists() - assert pathlib.Path(f"{td}/myproj-dotenv.cttemplate").exists() - assert os.path.getsize(f"{td}/myproj-dotenv.ctconfig") > 0 - assert os.path.getsize(f"{td}/myproj-dotenv.cttemplate") > 0 - - # it was originally intended for json files to be included in the - # directory walking test but it broke on github (never locally) - assert not pathlib.Path(f"{td}/myproj-json.ctconfig").exists() - assert not pathlib.Path(f"{td}/myproj-json.cttemplate").exists() - # assert os.path.getsize(f"{td}/myproj-json.ctconfig") > 0 - # assert os.path.getsize(f"{td}/myproj-json.cttemplate") > 0 - - assert not pathlib.Path(f"{td}/myproj-tf.ctconfig").exists() - assert not pathlib.Path(f"{td}/myproj-tf.cttemplate").exists() + except AssertionError as e: + print(result.output) + raise e +@mock.patch( + "dynamic_importer.main.CTClient", +) @pytest.mark.timeout(30) -@pytest.mark.usefixtures("tmp_path") -def test_walk_directories_with_exclusion(tmp_path): - runner = CliRunner() +def test_walk_directories_with_exclusion(mock_client): + mock_client = mock.MagicMock() # noqa: F841 + runner = CliRunner( + env={"CLOUDTRUTH_API_HOST": "localhost:8000", "CLOUDTRUTH_API_KEY": "test"}, + ) current_dir = pathlib.Path(__file__).parent.resolve() prompt_responses = [ @@ -140,38 +135,32 @@ def test_walk_directories_with_exclusion(tmp_path): "", # skipping json file "", # skipping tfvars file "", # skipping tf file + # something is different between local and github file order + # so we have to specify this extra prompt response and + # tack on a few extra empty lines + "default", + "", "", "", ] - with runner.isolated_filesystem(temp_dir=tmp_path) as td: - result = runner.invoke( - import_config, - [ - "walk-directories", - "-t", - "dotenv", - "-t", - "yaml", - "-c", - f"{current_dir}/../../samples", - "--exclude-dirs", - f"{current_dir}/../../samples/dotenvs", - "--output-dir", - td, - ], - input="\n".join(prompt_responses), - catch_exceptions=False, - ) + result = runner.invoke( + import_config, + [ + "walk-directories", + "-t", + "dotenv", + "-t", + "yaml", + "--config-dir", + f"{current_dir}/../../samples", + "--exclude-dirs", + f"{current_dir}/../../samples/dotenvs", + ], + input="\n".join(prompt_responses), + catch_exceptions=False, + ) + try: assert result.exit_code == 0 - - assert pathlib.Path(f"{td}/myproj-dotenv.ctconfig").exists() - assert pathlib.Path(f"{td}/myproj-dotenv.cttemplate").exists() - assert os.path.getsize(f"{td}/myproj-dotenv.ctconfig") > 0 - assert os.path.getsize(f"{td}/myproj-dotenv.cttemplate") > 0 - - assert pathlib.Path(f"{td}/myproj-yaml.ctconfig").exists() - assert pathlib.Path(f"{td}/myproj-yaml.cttemplate").exists() - assert os.path.getsize(f"{td}/myproj-yaml.ctconfig") > 0 - assert os.path.getsize(f"{td}/myproj-yaml.cttemplate") > 0 - - assert len(os.listdir(pathlib.Path(f"{td}/"))) == 4 + except AssertionError as e: + print(result.output) + raise e