diff --git a/ops/terraform/services/base/values/prod-sbx.yaml b/ops/terraform/services/base/values/prod-sbx.yaml index 9ef107f757..f52574627c 100644 --- a/ops/terraform/services/base/values/prod-sbx.yaml +++ b/ops/terraform/services/base/values/prod-sbx.yaml @@ -148,13 +148,18 @@ bfd: starting_fiss_seq_num: &rda_job_starting_fiss UNDEFINED starting_mcs_seq_num: &rda_job_starting_mcs UNDEFINED cleanup: - enabled: &rda_cleanup_enabled false + enabled: false + run_size: UNDEFINED + transaction_size: UNDEFINED + data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines + data_pipeline_rda_grpc_auth_token: UNDEFINED + data_pipeline_rda_grpc_host: UNDEFINED data_pipeline_rda_grpc_inproc_server_mode: *grpc_inprocess_mode data_pipeline_rda_grpc_max_idle_seconds: *rda_grpc_idle_secs + data_pipeline_rda_grpc_port: UNDEFINED data_pipeline_rda_grpc_server_type: *rda_grpc_server_type data_pipeline_rda_job_batch_size: *rda_job_batch_size data_pipeline_rda_job_enabled: *rda_job_enabled - data_pipeline_ccw_rif_job_enabled: false data_pipeline_rda_job_interval_seconds: *rda_job_interval data_pipeline_rda_job_starting_fiss_seq_num: *rda_job_starting_fiss data_pipeline_rda_job_starting_mcs_seq_num: *rda_job_starting_mcs @@ -162,7 +167,7 @@ bfd: data_pipeline_rda_process_dlq: *rda_job_process_dlq instance_type: m6a.large shared: - bfd_pipeline_dir: &bfdPipelineDir /bluebutton-data-pipeline + bfd_pipeline_dir: *bfdPipelineDir data_pipeline_dir: *bfdPipelineDir data_pipeline_new_relic_metric_host: *newRelicMetricHost data_pipeline_new_relic_metric_path: *newRelicMetricPath diff --git a/ops/terraform/services/base/values/prod.yaml b/ops/terraform/services/base/values/prod.yaml index d3a221addf..3ff89ed0d5 100644 --- a/ops/terraform/services/base/values/prod.yaml +++ b/ops/terraform/services/base/values/prod.yaml @@ -151,15 +151,18 @@ bfd: starting_fiss_seq_num: &rda_job_starting_fiss UNDEFINED starting_mcs_seq_num: &rda_job_starting_mcs UNDEFINED cleanup: - enabled: &rda_cleanup_enabled true - run_size: &rda_cleanup_run_size 150000 - transaction_size: &rda_cleanup_transaction_size 15000 + enabled: true + run_size: 150000 + transaction_size: 15000 + data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines + data_pipeline_rda_grpc_auth_token: *rda_grpc_auth_token + data_pipeline_rda_grpc_host: *rda_grpc_host data_pipeline_rda_grpc_inproc_server_mode: *grpc_inprocess_mode data_pipeline_rda_grpc_max_idle_seconds: *rda_grpc_idle_secs + data_pipeline_rda_grpc_port: *rda_grpc_port data_pipeline_rda_grpc_server_type: *rda_grpc_server_type data_pipeline_rda_job_batch_size: *rda_job_batch_size data_pipeline_rda_job_enabled: *rda_job_enabled - data_pipeline_ccw_rif_job_enabled: false data_pipeline_rda_job_interval_seconds: *rda_job_interval data_pipeline_rda_job_starting_fiss_seq_num: *rda_job_starting_fiss data_pipeline_rda_job_starting_mcs_seq_num: *rda_job_starting_mcs @@ -186,12 +189,6 @@ bfd: data_pipeline_db_username: *pipeline_db_username data_pipeline_db_password: *pipeline_db_password data_pipeline_new_relic_metric_key: *pipeline_nr_metric_key - # Host and port for RDA API server - data_pipeline_rda_grpc_host: *rda_grpc_host - data_pipeline_rda_grpc_port: *rda_grpc_port - # Enable RDA API authentication using JWT. - # data_pipeline_rda_grpc_auth_token: <>AgV4XagJrumigTt2rS-N64gNj7VVsfhhGWTXO1B0c8beOU0AdAACAAtBcHBsaWNhdGlvbgAGY2lwaGVyABVhd3MtY3J5cHRvLXB1YmxpYy1rZXkAREF2NGNPOHB4N0MzQ0luekdXMGtBZG92T09JdEZSS3ZDT0RsL0NLdFdTSk10ekY1QnprT0t3enJaeU9aU05tZThndz09AAEAB2F3cy1rbXMAS2Fybjphd3M6a21zOnVzLWVhc3QtMTo1NzczNzM4MzE3MTE6a2V5L21yay1iMjNlYjZmZWVlYmM0MzQ2YTk5YTZhNTliYzU2MWRmMwC4AQIBAHipsbBUtIKaXI95_AIHiGnjDkxfPDRzgqb6WVwMZI7-kwH57WMqZ2qRWhCFWYPQfb4FAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM2jRS1GauVDgoKamRAgEQgDvzcDCZG1XUWPrHBU-d6RTtjMr-V5EvhmV-6hBQfybebXPKPQvoz-z_Avo4B856Htrut5ICd-9dWwYbHwIAABAA80j2O7nz8lv_59z0fOq0kkwc8cKZdfCT1ExKelrbi_gIa6aWxDpmOy2kJPWb10D-_____wAAAAEAAAAAAAAAAAAAAAEAAAIa_VuGkvQ8cqQo6d4WNcbOlEgEmRHOd_A9kE2Iqa3VywaHCsHg7Gxvn-DBGhPlvEUl404iENuBe0HFn52Pu6Wl0PgyFSHl-65kiIgWsqp78LlvGe-MJXj3ysIGn_ZJH-gBmxsh8rCpSKVwJ00BMZhje1DTrmlkDqtR8a5rXqMWbMrklHKKWdcZDEFI_DLdEbDDpAFYGusgb1IpTzTZ6wU_ZzW_Drdbk7zyvihWjgrMZA90Q8l5Av43KZF3vaRrL8UFTTSGCtSAgFW7lK2bQiZOYmzFB4bSQSdn2iUzgKp0EVUuBm4FVHimEp41UR1HFmX94uXKlQtCMl8RONL0YZSeJ1CQMT74ADEaRo7ROhZTu8nU8btglAxnJEearn_sgCl62YXFUtZlDtvHB3CJJvsNfkb61lXqFgQYAWsgHZsnKLmQJZQ7S489S67P9caZ8bNizmKHBh0q7avOkcI7kvc_pqpBk6J2v67HYtEhjisY_WgscLV-EcOzI-QWNZ69qHJ5nYRL0_NeWDb_3MRWwvvnTyBfnUJMFc3Uy51qseUDazWFhjICpYg1kWCcs-tqwWjQP1fcSPWzjLqrvyGh4QFM6SlC8W4aqDr3q5WIBVPS-bfwEaH3SbplYfLZsmrL5lDvBXsmIbHMszmILjkK_AY4zdj4-V7UINhCNVCdoPow0m72HxrWASAJiUIvmJmLxVOYefAnvdFY2hL6b3Q5kjaxwdqYhNV6qHgF8SoAZzBlAjEAyOw2-rodfYyFUq4mPOkNT2cwycen240KxXxgtao_S27n0Fj35-8JQEp9O6NcEoNIAjBbaw9BLYjqvmC2hUQvTURS_rlI-5DTyI7AqChyIJb9nxlQ4tK70qkgUrHEC41LdO8=<> - data_pipeline_rda_grpc_auth_token: *rda_grpc_auth_token server: ref_dir: &bfdServerDir /usr/local/bfd-server db: diff --git a/ops/terraform/services/base/values/test.yaml b/ops/terraform/services/base/values/test.yaml index b07f09d440..e2400ddc58 100644 --- a/ops/terraform/services/base/values/test.yaml +++ b/ops/terraform/services/base/values/test.yaml @@ -149,13 +149,18 @@ bfd: starting_fiss_seq_num: &rda_job_starting_fiss 0 starting_mcs_seq_num: &rda_job_starting_mcs 0 cleanup: - enabled: &rda_cleanup_enabled false + enabled: false + run_size: UNDEFINED + transaction_size: UNDEFINED data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines - data_pipeline_rda_job_enabled: *rda_job_enabled + data_pipeline_rda_grpc_auth_token: *rda_grpc_auth_token + data_pipeline_rda_grpc_host: *rda_grpc_host data_pipeline_rda_grpc_inproc_server_mode: UNDEFINED data_pipeline_rda_grpc_max_idle_seconds: *rda_grpc_idle_secs + data_pipeline_rda_grpc_port: *rda_grpc_port data_pipeline_rda_grpc_server_type: *rda_grpc_server_type data_pipeline_rda_job_batch_size: *rda_job_batch_size + data_pipeline_rda_job_enabled: *rda_job_enabled data_pipeline_rda_job_interval_seconds: *rda_job_interval data_pipeline_rda_job_starting_fiss_seq_num: *rda_job_starting_fiss data_pipeline_rda_job_starting_mcs_seq_num: *rda_job_starting_mcs @@ -182,10 +187,6 @@ bfd: # lines uncommented. data_pipeline_db_username: *pipeline_db_username data_pipeline_db_password: *pipeline_db_password - ## PIPELINE+RDA VALUES - data_pipeline_rda_grpc_auth_token: *rda_grpc_auth_token - data_pipeline_rda_grpc_host: *rda_grpc_host - data_pipeline_rda_grpc_port: *rda_grpc_port server: ref_dir: &bfdServerDir /usr/local/bfd-server db: diff --git a/ops/terraform/services/pipeline/user-data.sh.tftpl b/ops/terraform/services/pipeline/user-data.sh.tftpl index 5bd0e46abb..cc221d5306 100644 --- a/ops/terraform/services/pipeline/user-data.sh.tftpl +++ b/ops/terraform/services/pipeline/user-data.sh.tftpl @@ -10,27 +10,43 @@ exec > >( cd /beneficiary-fhir-data/ops/ansible/playbooks-ccs/ -# TODO: Consider injecting ansible variables with more modern ansible versions. BFD-1890. +# ${pipeline_instance} specific variables aws ssm get-parameters-by-path \ --with-decryption \ - --path "/bfd/${env}/pipeline/nonsensitive/shared/" \ + --path "/bfd/${env}/pipeline/nonsensitive/${pipeline_instance}" \ --recursive \ --region us-east-1 \ - --query 'Parameters' | jq 'map({(.Name|split("/")[6]): .Value})|add' > nonsensitive_pipeline_vars.json + --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > nonsensitive_pipeline_vars.json aws ssm get-parameters-by-path \ --with-decryption \ - --path "/bfd/${env}/pipeline/sensitive/shared/" \ + --path "/bfd/${env}/pipeline/sensitive/${pipeline_instance}" \ --recursive \ --region us-east-1 \ - --query 'Parameters' | jq 'map({(.Name|split("/")[6]): .Value})|add' > sensitive_pipeline_vars.json + --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > sensitive_pipeline_vars.json +# shared pipeline variables +aws ssm get-parameters-by-path \ + --with-decryption \ + --path "/bfd/${env}/pipeline/nonsensitive/shared" \ + --recursive \ + --region us-east-1 \ + --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > nonsensitive_shared_pipeline_vars.json + +aws ssm get-parameters-by-path \ + --with-decryption \ + --path "/bfd/${env}/pipeline/sensitive/shared" \ + --recursive \ + --region us-east-1 \ + --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > sensitive_shared_pipeline_vars.json + +# common variables aws ssm get-parameters-by-path \ --with-decryption \ --path "/bfd/${env}/common/nonsensitive/" \ --recursive \ --region us-east-1 \ - --query 'Parameters' | jq 'map({(.Name|split("/")[5]): .Value})|add' > common_vars.json + --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > common_vars.json # JVM heap size is 80% of total memory minus 2GB for other system processes and overhead (security tools et al.) # Ie. For an 8GB instance, the heap size will be ~4.4GB (6.4 - 2) @@ -47,7 +63,15 @@ EOF mkdir -p logs -ansible-playbook --extra-vars '@nonsensitive_pipeline_vars.json' --extra-vars '@sensitive_pipeline_vars.json' --extra-vars '@common_vars.json' --extra-vars '@extra_vars.json' --tags "post-ami" launch_bfd-pipeline.yml +# TODO: Shift application-specific configuration to the application and simplify as part of BFD-3210. +ansible-playbook \ + --extra-vars '@common_vars.json' \ + --extra-vars '@nonsensitive_pipeline_vars.json' \ + --extra-vars '@nonsensitive_shared_pipeline_vars.json' \ + --extra-vars '@sensitive_pipeline_vars.json' \ + --extra-vars '@sensitive_shared_pipeline_vars.json' \ + --extra-vars '@extra_vars.json' \ + --tags "post-ami" launch_bfd-pipeline.yml # Set login environment for all users: # 1. make BFD_ENV_NAME available to all logins