diff --git a/ops/ansible/roles/bfd-pipeline/defaults/main.yml b/ops/ansible/roles/bfd-pipeline/defaults/main.yml index 5b163dd0f9..0565307e3a 100644 --- a/ops/ansible/roles/bfd-pipeline/defaults/main.yml +++ b/ops/ansible/roles/bfd-pipeline/defaults/main.yml @@ -14,6 +14,9 @@ data_pipeline_ccw_rif_job_enabled: true rif_thread_multiple: 3 rif_job_batch_size: 7 rif_job_queue_size_multiple: 5 +rif_thread_multiple_claims: 25 +rif_job_batch_size_claims: 100 +rif_job_queue_size_multiple_claims: 10 ## RDA API Pipeline # Used to download FISS/MCS partially adjudicated claims data. diff --git a/ops/ansible/roles/bfd-pipeline/templates/bfd-pipeline-service.sh.j2 b/ops/ansible/roles/bfd-pipeline/templates/bfd-pipeline-service.sh.j2 index 14907b3d6d..fee3cba97a 100644 --- a/ops/ansible/roles/bfd-pipeline/templates/bfd-pipeline-service.sh.j2 +++ b/ops/ansible/roles/bfd-pipeline/templates/bfd-pipeline-service.sh.j2 @@ -11,6 +11,9 @@ export DATABASE_PASSWORD='{{ data_pipeline_db_password }}' export LOADER_THREADS='{{ ansible_processor_vcpus * (rif_thread_multiple | int) }}' export RIF_JOB_BATCH_SIZE='{{ rif_job_batch_size }}' export RIF_JOB_QUEUE_SIZE_MULTIPLE='{{ rif_job_queue_size_multiple }}' +export LOADER_THREADS_CLAIMS='{{ ansible_processor_vcpus * (rif_thread_multiple_claims | int) }}' +export RIF_JOB_BATCH_SIZE_CLAIMS='{{ rif_job_batch_size_claims }}' +export RIF_JOB_QUEUE_SIZE_MULTIPLE_CLAIMS='{{ rif_job_queue_size_multiple_claims }}' export IDEMPOTENCY_REQUIRED='{{ data_pipeline_idempotency_required }}' export FILTERING_NON_NULL_AND_NON_2023_BENES='{{ data_pipeline_filtering_non_null_and_non_2023_benes }}' diff --git a/ops/terraform/services/base/values/ephemeral.yaml b/ops/terraform/services/base/values/ephemeral.yaml index 015b53c981..6f94fce6a4 100644 --- a/ops/terraform/services/base/values/ephemeral.yaml +++ b/ops/terraform/services/base/values/ephemeral.yaml @@ -33,10 +33,13 @@ /bfd/${env}/pipeline/ccw/nonsensitive/data_pipeline_ccw_rif_job_enabled: false /bfd/${env}/pipeline/ccw/nonsensitive/data_pipeline_filtering_non_null_and_non_2023_benes: true /bfd/${env}/pipeline/ccw/nonsensitive/data_pipeline_idempotency_required: true -/bfd/${env}/pipeline/ccw/nonsensitive/instance_type: m6i.large +/bfd/${env}/pipeline/ccw/nonsensitive/instance_type: c6i.4xlarge /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size: 7 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple: 5 /bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple: 3 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size_claims: 100 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple_claims: 10 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple_claims: 25 ## PIPELINE RDA /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_rda_job_enabled: false diff --git a/ops/terraform/services/base/values/prod-sbx.yaml b/ops/terraform/services/base/values/prod-sbx.yaml index 016d454d8e..d120767d5d 100644 --- a/ops/terraform/services/base/values/prod-sbx.yaml +++ b/ops/terraform/services/base/values/prod-sbx.yaml @@ -44,6 +44,9 @@ /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size: 7 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple: 5 /bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple: 3 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size_claims: 100 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple_claims: 10 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple_claims: 25 ## PIPELINE+RDA /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_rda_grpc_inproc_server_mode: 'S3' diff --git a/ops/terraform/services/base/values/prod.yaml b/ops/terraform/services/base/values/prod.yaml index 5ff285f170..62b3e52f5a 100644 --- a/ops/terraform/services/base/values/prod.yaml +++ b/ops/terraform/services/base/values/prod.yaml @@ -44,6 +44,9 @@ /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size: 7 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple: 5 /bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple: 3 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size_claims: 100 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple_claims: 10 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple_claims: 25 ## PIPELINE+RDA /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_rda_job_enabled: true diff --git a/ops/terraform/services/base/values/test.yaml b/ops/terraform/services/base/values/test.yaml index bec3d06d22..bfba4ff732 100644 --- a/ops/terraform/services/base/values/test.yaml +++ b/ops/terraform/services/base/values/test.yaml @@ -44,6 +44,9 @@ /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size: 7 /bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple: 5 /bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple: 3 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_batch_size_claims: 100 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_job_queue_size_multiple_claims: 10 +/bfd/${env}/pipeline/ccw/nonsensitive/rif_thread_multiple_claims: 25 ## PIPELINE+RDA /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_ccw_rif_job_enabled: false # CCW Jobs are disabled by default on RDA Pipelines /bfd/${env}/pipeline/rda/nonsensitive/data_pipeline_rda_job_enabled: true