BFD-2757: Update BFD-Pipeline With Specific Claims Settings (#1844)

* update pipeline params * comment out performance settings * add new claim params for ephemeral * update environments to have claims setting * add defaults * revert default vaues * update yml * add to service j2 * add updated values * update parameters * revert back to original settings to reevaluate baseline * upgrade settings * revert rda instance type * update formatting * turn on pipeline again for more testing * revert original loadoption values * revert original loadoption values * revert original loadoption values * revert original loadoption values * revert instance size * upgrade claim settings * turn off pipeline
CMSgov · Jul 26, 2023 · 2a05850 · 2a05850
1 parent f99bdf6
commit 2a05850
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 1 deletion.
diff --git a/ops/ansible/roles/bfd-pipeline/defaults/main.yml b/ops/ansible/roles/bfd-pipeline/defaults/main.yml
@@ -14,6 +14,9 @@ data_pipeline_ccw_rif_job_enabled: true
 rif_thread_multiple: 3
 rif_job_batch_size: 7
 rif_job_queue_size_multiple: 5
+rif_thread_multiple_claims: 25
+rif_job_batch_size_claims: 100
+rif_job_queue_size_multiple_claims: 10
 
 ## RDA API Pipeline
 # Used to download FISS/MCS partially adjudicated claims data.

diff --git a/ops/ansible/roles/bfd-pipeline/templates/bfd-pipeline-service.sh.j2 b/ops/ansible/roles/bfd-pipeline/templates/bfd-pipeline-service.sh.j2
@@ -11,6 +11,9 @@ export DATABASE_PASSWORD='{{ data_pipeline_db_password }}'
 export LOADER_THREADS='{{ ansible_processor_vcpus * (rif_thread_multiple | int) }}'
 export RIF_JOB_BATCH_SIZE='{{ rif_job_batch_size }}'
 export RIF_JOB_QUEUE_SIZE_MULTIPLE='{{ rif_job_queue_size_multiple }}'
+export LOADER_THREADS_CLAIMS='{{ ansible_processor_vcpus * (rif_thread_multiple_claims | int) }}'
+export RIF_JOB_BATCH_SIZE_CLAIMS='{{ rif_job_batch_size_claims }}'
+export RIF_JOB_QUEUE_SIZE_MULTIPLE_CLAIMS='{{ rif_job_queue_size_multiple_claims }}'
 
 export IDEMPOTENCY_REQUIRED='{{ data_pipeline_idempotency_required }}'
 export FILTERING_NON_NULL_AND_NON_2023_BENES='{{ data_pipeline_filtering_non_null_and_non_2023_benes }}'

diff --git a/ops/terraform/services/base/values/ephemeral.yaml b/ops/terraform/services/base/values/ephemeral.yaml
@@ -33,10 +33,13 @@
 /bfd/${env}/pipeline/ccw/nonsensitive/data_pipeline_ccw_rif_job_enabled: false
 /bfd/${env}/pipeline/ccw/nonsensitive/data_pipeline_filtering_non_null_and_non_2023_benes: true
 /bfd/${env}/pipeline/ccw/nonsensitive/data_pipeline_idempotency_required: true
-/bfd/${env}/pipeline/ccw/nonsensitive/instance_type: m6i.large
+/bfd/${env}/pipeline/ccw/nonsensitive/instance_type: c6i.4xlarge
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size: 7
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple: 5
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple: 3
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size_claims: 100
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple_claims: 10
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple_claims: 25
 ## PIPELINE RDA
 /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines
 /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_rda_job_enabled: false

diff --git a/ops/terraform/services/base/values/prod-sbx.yaml b/ops/terraform/services/base/values/prod-sbx.yaml
@@ -44,6 +44,9 @@
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size: 7
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple: 5
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple: 3
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size_claims: 100
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple_claims: 10
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple_claims: 25
 ## PIPELINE+RDA
 /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines
 /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_rda_grpc_inproc_server_mode: 'S3'

diff --git a/ops/terraform/services/base/values/prod.yaml b/ops/terraform/services/base/values/prod.yaml
@@ -44,6 +44,9 @@
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size: 7
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple: 5
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple: 3
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size_claims: 100
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple_claims: 10
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple_claims: 25
 ## PIPELINE+RDA
 /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines
 /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_rda_job_enabled: true

diff --git a/ops/terraform/services/base/values/test.yaml b/ops/terraform/services/base/values/test.yaml
@@ -44,6 +44,9 @@
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size: 7
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple: 5
 /bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple: 3
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size_claims: 100
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple_claims: 10
+/bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple_claims: 25
 ## PIPELINE+RDA
 /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines
 /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_rda_job_enabled: true