Removed table include list and updated snapshot mode for SRTE (#671)

CDCgov · Dec 19, 2024 · ccf41c6 · ccf41c6
1 parent 992b3c0
commit ccf41c6
Show file tree

Hide file tree

Showing 6 changed files with 11 additions and 16 deletions.
diff --git a/charts/debezium-srte/values-azure.yaml b/charts/debezium-srte/values-azure.yaml
@@ -73,7 +73,7 @@ connect:
       "transforms": "dropPrefix",
       "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)",
       "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
-      "transforms.dropPrefix.replacement": "nbs_$1",
+      "transforms.dropPrefix.replacement": "nrt_srte_$1",
       "value.converter": "org.apache.kafka.connect.json.JsonConverter",
       "value.converter.schemas.enable": "true",
     }

diff --git a/charts/debezium-srte/values-dts1.yaml b/charts/debezium-srte/values-dts1.yaml
@@ -48,9 +48,9 @@ connect:
       "database.port": "1433",
       "database.user": "",
       "database.password": "",
-      "database.dbname": "nbs_stre",
-      "database.names": "nbs_stre",
-      "database.server.name": "stre",
+      "database.dbname": "nbs_srte",
+      "database.names": "nbs_srte",
+      "database.server.name": "srte",
       "database.history.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092",
       "database.history.kafka.topic": "dbhistory.database_server_name.database_name",
       # Uncomment following to manually bypass the sqlserver agent status query results
@@ -61,10 +61,9 @@ connect:
       "key.converter.schemas.enable": "true",
       "producer.max.request.size": "10000000", #10MB
       "producer.message.max.bytes": "10000000", #10MB
-      "snapshot.mode": "no_data",
-      "schema.history.internal.kafka.topic": "stre-schema-history",
+      "snapshot.mode": "initial",
+      "schema.history.internal.kafka.topic": "srte-schema-history",
       "schema.history.internal.kafka.bootstrap.servers": "b-1.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092,b-2.nrtreporting.rx5iwx.c5.kafka.us-east-1.amazonaws.com:9092",
-      "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value",
       "tasks.max": "1",
       "topic.prefix": "cdc",
       "topic.creation.default.replication.factor": 2,
@@ -73,7 +72,7 @@ connect:
       "transforms": "dropPrefix",
       "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)",
       "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
-      "transforms.dropPrefix.replacement": "nbs_$1",
+      "transforms.dropPrefix.replacement": "nrt_srte_$1",
       "value.converter": "org.apache.kafka.connect.json.JsonConverter",
       "value.converter.schemas.enable": "true",
     }

diff --git a/charts/debezium-srte/values-feature.yaml b/charts/debezium-srte/values-feature.yaml
@@ -64,7 +64,6 @@ connect:
       "snapshot.mode": "no_data",
       "schema.history.internal.kafka.topic": "srte-schema-history",
       "schema.history.internal.kafka.bootstrap.servers": "b-2.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsfeaturedevelopme.f4c8vq.c9.kafka.us-east-1.amazonaws.com:9092",
-      "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value",
       "tasks.max": "1",
       "topic.prefix": "cdc",
       "topic.creation.default.replication.factor": 2,
@@ -73,7 +72,7 @@ connect:
       "transforms": "dropPrefix",
       "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)",
       "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
-      "transforms.dropPrefix.replacement": "nbs_$1",
+      "transforms.dropPrefix.replacement": "nrt_srte_$1",
       "value.converter": "org.apache.kafka.connect.json.JsonConverter",
       "value.converter.schemas.enable": "true",
     }

diff --git a/charts/debezium-srte/values-int1.yaml b/charts/debezium-srte/values-int1.yaml
@@ -64,7 +64,6 @@ connect:
       "snapshot.mode": "no_data",
       "schema.history.internal.kafka.topic": "srte-schema-history",
       "schema.history.internal.kafka.bootstrap.servers": "b-2.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092,b-1.cdcnbsint1developmentm.8u6ipl.c2.kafka.us-east-1.amazonaws.com:9092",
-      "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value",
       "tasks.max": "1",
       "topic.prefix": "cdc",
       "topic.creation.default.replication.factor": 2,
@@ -73,7 +72,7 @@ connect:
       "transforms": "dropPrefix",
       "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)",
       "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
-      "transforms.dropPrefix.replacement": "nbs_$1",
+      "transforms.dropPrefix.replacement": "nrt_srte_$1",
       "value.converter": "org.apache.kafka.connect.json.JsonConverter",
       "value.converter.schemas.enable": "true",
     }

diff --git a/charts/debezium-srte/values-test1.yaml b/charts/debezium-srte/values-test1.yaml
@@ -64,7 +64,6 @@ connect:
       "snapshot.mode": "no_data",
       "schema.history.internal.kafka.topic": "srte-schema-history",
       "schema.history.internal.kafka.bootstrap.servers": "b-1.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092,b-2.cdcnbstestdevelopment.toyoo7.c10.kafka.us-east-1.amazonaws.com:9092",
-      "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value",
       "tasks.max": "1",
       "topic.prefix": "cdc",
       "topic.creation.default.replication.factor": 2,
@@ -73,7 +72,7 @@ connect:
       "transforms": "dropPrefix",
       "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)",
       "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
-      "transforms.dropPrefix.replacement": "nbs_$1",
+      "transforms.dropPrefix.replacement": "nrt_srte_$1",
       "value.converter": "org.apache.kafka.connect.json.JsonConverter",
       "value.converter.schemas.enable": "true",
     }

diff --git a/charts/debezium-srte/values.yaml b/charts/debezium-srte/values.yaml
@@ -66,7 +66,6 @@ connect:
       "snapshot.mode": "no_data",
       "schema.history.internal.kafka.topic": "srte-schema-history",
       "schema.history.internal.kafka.bootstrap.servers": "EXAMPLE_MSK_KAFKA_ENDPOINT",
-      "table.include.list": "dbo.Anatomic_site_code, dbo.City_code_value, dbo.Cntycity_code_value, dbo.Code_value_clinical, dbo.Code_value_general, dbo.Codeset, dbo.Codeset_Group_Metadata, dbo.Country_code, dbo.Country_Code_ISO, dbo.Country_XREF, dbo.ELR_XREF, dbo.IMRDBMapping, dbo.Investigation_code, dbo.Jurisdiction_code, dbo.Jurisdiction_participation, dbo.Lab_coding_system, dbo.Lab_result, dbo.Lab_result_Snomed, dbo.Lab_test, dbo.Labtest_loinc, dbo.Labtest_Progarea_Mapping, dbo.Language_code, dbo.LDF_page_set, dbo.LOINC_code, dbo.Loinc_condition, dbo.Loinc_snomed_condition, dbo.NAICS_Industry_code, dbo.Occupation_code, dbo.Participation_type, dbo.Program_area_code, dbo.Race_code, dbo.Snomed_code, dbo.Specimen_source_code, dbo.Standard_XREF, dbo.State_code, dbo.State_county_code_value, dbo.State_model, dbo.TotalIDM, dbo.Treatment_code, dbo.Unit_code, dbo.Zip_code_value, dbo.Zipcnty_code_value",
       "tasks.max": "1",
       "topic.prefix": "cdc",
       "topic.creation.default.replication.factor": 2,
@@ -75,7 +74,7 @@ connect:
       "transforms": "dropPrefix",
       "transforms.dropPrefix.regex": "cdc\\.NBS_SRTE\\.dbo\\.(.+)",
       "transforms.dropPrefix.type": "org.apache.kafka.connect.transforms.RegexRouter",
-      "transforms.dropPrefix.replacement": "nbs_$1",
+      "transforms.dropPrefix.replacement": "nrt_srte_$1",
       "value.converter": "org.apache.kafka.connect.json.JsonConverter",
       "value.converter.schemas.enable": "true",
     }