BFD-3390: Correct pipeline configuration hierarchy (#2277)

Co-authored-by: aschey-forpeople <[email protected]>
CMSgov · Apr 8, 2024 · 31d64f9 · 31d64f9
1 parent f676de8
commit 31d64f9
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 26 deletions.
diff --git a/ops/terraform/services/base/values/prod-sbx.yaml b/ops/terraform/services/base/values/prod-sbx.yaml
@@ -148,21 +148,26 @@ bfd:
           starting_fiss_seq_num: &rda_job_starting_fiss UNDEFINED
           starting_mcs_seq_num: &rda_job_starting_mcs UNDEFINED
         cleanup:
-          enabled: &rda_cleanup_enabled false
+          enabled: false
+          run_size: UNDEFINED
+          transaction_size: UNDEFINED
+        data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines
+        data_pipeline_rda_grpc_auth_token: UNDEFINED
+        data_pipeline_rda_grpc_host: UNDEFINED
         data_pipeline_rda_grpc_inproc_server_mode: *grpc_inprocess_mode
         data_pipeline_rda_grpc_max_idle_seconds: *rda_grpc_idle_secs
+        data_pipeline_rda_grpc_port: UNDEFINED
         data_pipeline_rda_grpc_server_type: *rda_grpc_server_type
         data_pipeline_rda_job_batch_size: *rda_job_batch_size
         data_pipeline_rda_job_enabled: *rda_job_enabled
-        data_pipeline_ccw_rif_job_enabled: false
         data_pipeline_rda_job_interval_seconds: *rda_job_interval
         data_pipeline_rda_job_starting_fiss_seq_num: *rda_job_starting_fiss
         data_pipeline_rda_job_starting_mcs_seq_num: *rda_job_starting_mcs
         data_pipeline_rda_job_write_threads: *rda_job_write_thread_count
         data_pipeline_rda_process_dlq: *rda_job_process_dlq
         instance_type: m6a.large
       shared:
-        bfd_pipeline_dir: &bfdPipelineDir /bluebutton-data-pipeline
+        bfd_pipeline_dir: *bfdPipelineDir
         data_pipeline_dir: *bfdPipelineDir
         data_pipeline_new_relic_metric_host: *newRelicMetricHost
         data_pipeline_new_relic_metric_path: *newRelicMetricPath

diff --git a/ops/terraform/services/base/values/prod.yaml b/ops/terraform/services/base/values/prod.yaml
@@ -151,15 +151,18 @@ bfd:
           starting_fiss_seq_num: &rda_job_starting_fiss UNDEFINED
           starting_mcs_seq_num: &rda_job_starting_mcs UNDEFINED
         cleanup:
-          enabled: &rda_cleanup_enabled true
-          run_size: &rda_cleanup_run_size 150000
-          transaction_size: &rda_cleanup_transaction_size 15000
+          enabled: true
+          run_size: 150000
+          transaction_size: 15000
+        data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines
+        data_pipeline_rda_grpc_auth_token: *rda_grpc_auth_token
+        data_pipeline_rda_grpc_host: *rda_grpc_host
         data_pipeline_rda_grpc_inproc_server_mode: *grpc_inprocess_mode
         data_pipeline_rda_grpc_max_idle_seconds: *rda_grpc_idle_secs
+        data_pipeline_rda_grpc_port: *rda_grpc_port
         data_pipeline_rda_grpc_server_type: *rda_grpc_server_type
         data_pipeline_rda_job_batch_size: *rda_job_batch_size
         data_pipeline_rda_job_enabled: *rda_job_enabled
-        data_pipeline_ccw_rif_job_enabled: false
         data_pipeline_rda_job_interval_seconds: *rda_job_interval
         data_pipeline_rda_job_starting_fiss_seq_num: *rda_job_starting_fiss
         data_pipeline_rda_job_starting_mcs_seq_num: *rda_job_starting_mcs
@@ -186,12 +189,6 @@ bfd:
         data_pipeline_db_username: *pipeline_db_username
         data_pipeline_db_password: *pipeline_db_password
         data_pipeline_new_relic_metric_key: *pipeline_nr_metric_key
-        # Host and port for RDA API server
-        data_pipeline_rda_grpc_host: *rda_grpc_host
-        data_pipeline_rda_grpc_port: *rda_grpc_port
-        # Enable RDA API authentication using JWT.
-        # data_pipeline_rda_grpc_auth_token: <<CIPHER>>AgV4XagJrumigTt2rS-N64gNj7VVsfhhGWTXO1B0c8beOU0AdAACAAtBcHBsaWNhdGlvbgAGY2lwaGVyABVhd3MtY3J5cHRvLXB1YmxpYy1rZXkAREF2NGNPOHB4N0MzQ0luekdXMGtBZG92T09JdEZSS3ZDT0RsL0NLdFdTSk10ekY1QnprT0t3enJaeU9aU05tZThndz09AAEAB2F3cy1rbXMAS2Fybjphd3M6a21zOnVzLWVhc3QtMTo1NzczNzM4MzE3MTE6a2V5L21yay1iMjNlYjZmZWVlYmM0MzQ2YTk5YTZhNTliYzU2MWRmMwC4AQIBAHipsbBUtIKaXI95_AIHiGnjDkxfPDRzgqb6WVwMZI7-kwH57WMqZ2qRWhCFWYPQfb4FAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM2jRS1GauVDgoKamRAgEQgDvzcDCZG1XUWPrHBU-d6RTtjMr-V5EvhmV-6hBQfybebXPKPQvoz-z_Avo4B856Htrut5ICd-9dWwYbHwIAABAA80j2O7nz8lv_59z0fOq0kkwc8cKZdfCT1ExKelrbi_gIa6aWxDpmOy2kJPWb10D-_____wAAAAEAAAAAAAAAAAAAAAEAAAIa_VuGkvQ8cqQo6d4WNcbOlEgEmRHOd_A9kE2Iqa3VywaHCsHg7Gxvn-DBGhPlvEUl404iENuBe0HFn52Pu6Wl0PgyFSHl-65kiIgWsqp78LlvGe-MJXj3ysIGn_ZJH-gBmxsh8rCpSKVwJ00BMZhje1DTrmlkDqtR8a5rXqMWbMrklHKKWdcZDEFI_DLdEbDDpAFYGusgb1IpTzTZ6wU_ZzW_Drdbk7zyvihWjgrMZA90Q8l5Av43KZF3vaRrL8UFTTSGCtSAgFW7lK2bQiZOYmzFB4bSQSdn2iUzgKp0EVUuBm4FVHimEp41UR1HFmX94uXKlQtCMl8RONL0YZSeJ1CQMT74ADEaRo7ROhZTu8nU8btglAxnJEearn_sgCl62YXFUtZlDtvHB3CJJvsNfkb61lXqFgQYAWsgHZsnKLmQJZQ7S489S67P9caZ8bNizmKHBh0q7avOkcI7kvc_pqpBk6J2v67HYtEhjisY_WgscLV-EcOzI-QWNZ69qHJ5nYRL0_NeWDb_3MRWwvvnTyBfnUJMFc3Uy51qseUDazWFhjICpYg1kWCcs-tqwWjQP1fcSPWzjLqrvyGh4QFM6SlC8W4aqDr3q5WIBVPS-bfwEaH3SbplYfLZsmrL5lDvBXsmIbHMszmILjkK_AY4zdj4-V7UINhCNVCdoPow0m72HxrWASAJiUIvmJmLxVOYefAnvdFY2hL6b3Q5kjaxwdqYhNV6qHgF8SoAZzBlAjEAyOw2-rodfYyFUq4mPOkNT2cwycen240KxXxgtao_S27n0Fj35-8JQEp9O6NcEoNIAjBbaw9BLYjqvmC2hUQvTURS_rlI-5DTyI7AqChyIJb9nxlQ4tK70qkgUrHEC41LdO8=<</CIPHER>>
-        data_pipeline_rda_grpc_auth_token: *rda_grpc_auth_token
     server:
       ref_dir: &bfdServerDir /usr/local/bfd-server
       db:

diff --git a/ops/terraform/services/base/values/test.yaml b/ops/terraform/services/base/values/test.yaml
@@ -149,13 +149,18 @@ bfd:
           starting_fiss_seq_num: &rda_job_starting_fiss 0
           starting_mcs_seq_num: &rda_job_starting_mcs 0
         cleanup:
-          enabled: &rda_cleanup_enabled false
+          enabled: false
+          run_size: UNDEFINED
+          transaction_size: UNDEFINED
         data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines
-        data_pipeline_rda_job_enabled: *rda_job_enabled
+        data_pipeline_rda_grpc_auth_token: *rda_grpc_auth_token
+        data_pipeline_rda_grpc_host: *rda_grpc_host
         data_pipeline_rda_grpc_inproc_server_mode: UNDEFINED
         data_pipeline_rda_grpc_max_idle_seconds: *rda_grpc_idle_secs
+        data_pipeline_rda_grpc_port: *rda_grpc_port
         data_pipeline_rda_grpc_server_type: *rda_grpc_server_type
         data_pipeline_rda_job_batch_size: *rda_job_batch_size
+        data_pipeline_rda_job_enabled: *rda_job_enabled
         data_pipeline_rda_job_interval_seconds: *rda_job_interval
         data_pipeline_rda_job_starting_fiss_seq_num: *rda_job_starting_fiss
         data_pipeline_rda_job_starting_mcs_seq_num: *rda_job_starting_mcs
@@ -182,10 +187,6 @@ bfd:
         # lines uncommented.
         data_pipeline_db_username: *pipeline_db_username
         data_pipeline_db_password: *pipeline_db_password
-        ## PIPELINE+RDA VALUES
-        data_pipeline_rda_grpc_auth_token: *rda_grpc_auth_token
-        data_pipeline_rda_grpc_host: *rda_grpc_host
-        data_pipeline_rda_grpc_port: *rda_grpc_port
     server:
       ref_dir: &bfdServerDir /usr/local/bfd-server
       db:

diff --git a/ops/terraform/services/pipeline/user-data.sh.tftpl b/ops/terraform/services/pipeline/user-data.sh.tftpl
@@ -10,27 +10,43 @@ exec > >(
 
 cd /beneficiary-fhir-data/ops/ansible/playbooks-ccs/
 
-# TODO: Consider injecting ansible variables with more modern ansible versions. BFD-1890.
+# ${pipeline_instance} specific variables
 aws ssm get-parameters-by-path \
     --with-decryption \
-    --path "/bfd/${env}/pipeline/nonsensitive/shared/" \
+    --path "/bfd/${env}/pipeline/nonsensitive/${pipeline_instance}" \
     --recursive \
     --region us-east-1 \
-    --query 'Parameters' | jq 'map({(.Name|split("/")[6]): .Value})|add' > nonsensitive_pipeline_vars.json
+    --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > nonsensitive_pipeline_vars.json
 
 aws ssm get-parameters-by-path \
     --with-decryption \
-    --path "/bfd/${env}/pipeline/sensitive/shared/" \
+    --path "/bfd/${env}/pipeline/sensitive/${pipeline_instance}" \
     --recursive \
     --region us-east-1 \
-    --query 'Parameters' | jq 'map({(.Name|split("/")[6]): .Value})|add' > sensitive_pipeline_vars.json
+    --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > sensitive_pipeline_vars.json
 
+# shared pipeline variables
+aws ssm get-parameters-by-path \
+    --with-decryption \
+    --path "/bfd/${env}/pipeline/nonsensitive/shared" \
+    --recursive \
+    --region us-east-1 \
+    --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > nonsensitive_shared_pipeline_vars.json
+
+aws ssm get-parameters-by-path \
+    --with-decryption \
+    --path "/bfd/${env}/pipeline/sensitive/shared" \
+    --recursive \
+    --region us-east-1 \
+    --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > sensitive_shared_pipeline_vars.json
+
+# common variables
 aws ssm get-parameters-by-path \
     --with-decryption \
     --path "/bfd/${env}/common/nonsensitive/" \
     --recursive \
     --region us-east-1 \
-    --query 'Parameters' | jq 'map({(.Name|split("/")[5]): .Value})|add' > common_vars.json
+    --query 'Parameters' | jq 'map({(.Name|split("/")|last): .Value})|add' > common_vars.json
 
 # JVM heap size is 80% of total memory minus 2GB for other system processes and overhead (security tools et al.)
 # Ie. For an 8GB instance, the heap size will be ~4.4GB (6.4 - 2)
@@ -47,7 +63,15 @@ EOF
 
 mkdir -p logs
 
-ansible-playbook --extra-vars '@nonsensitive_pipeline_vars.json' --extra-vars '@sensitive_pipeline_vars.json' --extra-vars '@common_vars.json' --extra-vars '@extra_vars.json' --tags "post-ami" launch_bfd-pipeline.yml
+# TODO: Shift application-specific configuration to the application and simplify as part of BFD-3210.
+ansible-playbook \
+  --extra-vars '@common_vars.json' \
+  --extra-vars '@nonsensitive_pipeline_vars.json' \
+  --extra-vars '@nonsensitive_shared_pipeline_vars.json' \
+  --extra-vars '@sensitive_pipeline_vars.json' \
+  --extra-vars '@sensitive_shared_pipeline_vars.json' \
+  --extra-vars '@extra_vars.json' \
+  --tags "post-ami" launch_bfd-pipeline.yml
 
 # Set login environment for all users:
 # 1. make BFD_ENV_NAME available to all logins