added to other vendors

OHDSI · Jun 12, 2024 · d99992d · d99992d
1 parent 963ed16
commit d99992d
Show file tree

Hide file tree

Showing 13 changed files with 57 additions and 36 deletions.
diff --git a/...ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/HealthVerityPersonBuilder.cs b/...ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/HealthVerityPersonBuilder.cs
@@ -644,12 +644,17 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
             AddToChunk(person, null,
                 observationPeriods,
                 [],
-                drugExposures,
-                conditionOccurrences,
-                procedureOccurrences,
-                observations,
-                measurements,
-                [.. visitOccurrences.Values], visitDetails, [], deviceExposure, [], []);
+                UpdateRSourceConcept(drugExposures).ToArray(),
+                UpdateRSourceConcept(conditionOccurrences).ToArray(),
+                UpdateRSourceConcept(procedureOccurrences).ToArray(),
+                UpdateRSourceConcept(observations).ToArray(),
+                UpdateRSourceConcept(measurements).ToArray(),
+                [.. visitOccurrences.Values], 
+                visitDetails, 
+                [],
+                UpdateRSourceConcept(deviceExposure).ToArray(), 
+                [], 
+                []);
 
             Complete = true;
 

diff --git a/...rk/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/Lookups/procedure.sql b/...rk/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/Lookups/procedure.sql
@@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's'
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 ), 
 Source as (
-SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs', 'cdt')
 AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs', 'cdt')
@@ -19,7 +19,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/...k/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/Procedure.sql b/...k/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/Procedure.sql
@@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's'
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier','icd10pcs hierarchy')
 ), Source as (
-SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd9proc','icd10pcs')
 AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd9proc','icd10pcs')
@@ -20,7 +20,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/....ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD10.sql b/....ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD10.sql
@@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's'
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier', 'icd10pcs hierarchy')
 ), Source as (
-SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd10pcs','hcpcs','cpt4')
 AND lower(TARGET_VOCABULARY_ID) IN ('icd10pcs','hcpcs','cpt4')
@@ -20,7 +20,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/...g.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD9.sql b/...g.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD9.sql
@@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's'
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier')
 ), Source as (
-SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4')
 AND lower(TARGET_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4')
@@ -20,7 +20,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/...dsi.cdm.framework.common3/Core/Transformation/OptumExtended/OptumExtendedPersonBuilder.cs b/...dsi.cdm.framework.common3/Core/Transformation/OptumExtended/OptumExtendedPersonBuilder.cs
@@ -760,15 +760,15 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
                 death,
                 observationPeriods,
                 payerPlanPeriods,
-                Clean(drugExposures, person).ToArray(),
-                Clean(conditionOccurrences, person).ToArray(),
-                Clean(procedureOccurrences, person).ToArray(),
-                Clean(observations, person).ToArray(),
-                Clean(measurements, person).ToArray(),
+                UpdateRSourceConcept(Clean(drugExposures, person)).ToArray(),
+                UpdateRSourceConcept(Clean(conditionOccurrences, person)).ToArray(),
+                UpdateRSourceConcept(Clean(procedureOccurrences, person)).ToArray(),
+                UpdateRSourceConcept(Clean(observations, person)).ToArray(),
+                UpdateRSourceConcept(Clean(measurements, person)).ToArray(),
                 vos,
                 [.. vds.Values],
                 [],
-                Clean(deviceExposure, person).ToArray(),
+                UpdateRSourceConcept(Clean(deviceExposure, person)).ToArray(),
                 [],
                 []);
 

diff --git a/...cdm.framework.common3/Core/Transformation/OptumPanther/Lookups/ConditionFromProcedure.sql b/...cdm.framework.common3/Core/Transformation/OptumPanther/Lookups/ConditionFromProcedure.sql
@@ -7,7 +7,7 @@ AND (lower(TARGET_STANDARD_CONCEPT) = 's')
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 
 ), Source as (
-SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs')
 AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs')
@@ -18,7 +18,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/...hdsi.cdm.framework.common3/Core/Transformation/OptumPanther/OptumOncologyPersonBuilder.cs b/...hdsi.cdm.framework.common3/Core/Transformation/OptumPanther/OptumOncologyPersonBuilder.cs
@@ -1058,9 +1058,21 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
             }
 
             // push built entities to ChunkBuilder for further save to CDM database
-            AddToChunk(person, death, [.. observationPeriodsFinal], payerPlanPeriods, drugExposures,
-                conditionOccurrences, procedureOccurrences, observations, measurements,
-                [.. visitOccurrences.Values], [.. visitDetails.Values], cohort, deviceExposure, notes, episodes);
+            AddToChunk(person, 
+                death, 
+                [.. observationPeriodsFinal], 
+                payerPlanPeriods,
+                UpdateRSourceConcept(drugExposures).ToArray(),
+                UpdateRSourceConcept(conditionOccurrences).ToArray(),
+                UpdateRSourceConcept(procedureOccurrences).ToArray(),
+                UpdateRSourceConcept(observations).ToArray(),
+                UpdateRSourceConcept(measurements).ToArray(),
+                [.. visitOccurrences.Values], 
+                [.. visitDetails.Values], 
+                cohort,
+                UpdateRSourceConcept(deviceExposure).ToArray(), 
+                notes, 
+                episodes);
 
             var pg = new PregnancyAlgorithm.PregnancyAlgorithm();
             foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, [.. observationPeriodsFinal],

diff --git a/...ramework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/Lookups/Procedure.sql b/...ramework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/Lookups/Procedure.sql
@@ -7,7 +7,7 @@ AND (TARGET_STANDARD_CONCEPT IS NOT NULL or TARGET_STANDARD_CONCEPT != '')
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier','cpt4 hierarchy','icd10pcs hierarchy')
 ), Source as (
-SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID
+SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4','icd10pcs', 'cdt')
 AND lower(TARGET_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4','icd10pcs', 'cdt')
@@ -19,7 +19,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, cast('1900/1/1' as date) as SOURCE_validStartDate, cast('2100/1/1' as date) as SOURCE_validEndDate, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, cast('1900/1/1' as date) as SOURCE_validStartDate, cast('2100/1/1' as date) as SOURCE_validEndDate, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/...amework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/TruvenPersonBuilder.cs b/...amework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/TruvenPersonBuilder.cs
@@ -759,11 +759,15 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
                 observationPeriods,
                 payerPlanPeriods,
                 [.. drugExposures],
-                conditionOccurrences,
-                procedureOccurrences,
-                observations,
-                measurements,
-                visitOccurrences, [.. visitDetails], [], deviceExposure, []);
+                UpdateRSourceConcept(conditionOccurrences).ToArray(),
+                UpdateRSourceConcept(procedureOccurrences).ToArray(),
+                UpdateRSourceConcept(observations).ToArray(),
+                UpdateRSourceConcept(measurements).ToArray(),
+                visitOccurrences, 
+                [.. visitDetails], 
+                [],
+                UpdateRSourceConcept(deviceExposure).ToArray(), 
+                []);
 
             Complete = true;
 

diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/org.ohdsi.cdm.framework.common.csproj b/sources/Framework/org.ohdsi.cdm.framework.common3/org.ohdsi.cdm.framework.common.csproj
@@ -1809,8 +1809,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="AWSSDK.S3" Version="3.7.307.23" />
-    <PackageReference Include="CsvHelper" Version="31.0.4" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.309.3" />
+    <PackageReference Include="CsvHelper" Version="32.0.3" />
     <PackageReference Include="DeepCloner" Version="0.10.4" />
   </ItemGroup>
 

diff --git a/sources/Framework/org.ohdsi.cdm.framework.desktop3/org.ohdsi.cdm.framework.desktop.csproj b/sources/Framework/org.ohdsi.cdm.framework.desktop3/org.ohdsi.cdm.framework.desktop.csproj
@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.4" />
+    <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.6" />
     <PackageReference Include="MySqlConnector" Version="2.3.7" />
     <PackageReference Include="Npgsql" Version="8.0.3" />
     <PackageReference Include="System.Configuration.ConfigurationManager" Version="8.0.0" />

diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/org.ohdsi.cdm.presentation.etl.csproj b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/org.ohdsi.cdm.presentation.etl.csproj
@@ -22,7 +22,7 @@
 
   <ItemGroup>
     <PackageReference Include="CommandLineParser" Version="2.9.1" />
-    <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.4" />
+    <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.6" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
   </ItemGroup>