diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Base/PersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Base/PersonBuilder.cs index 3a2819e3..bec4894c 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Base/PersonBuilder.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Base/PersonBuilder.cs @@ -1,4 +1,5 @@ -using org.ohdsi.cdm.framework.common.Builder; +using Force.DeepCloner; +using org.ohdsi.cdm.framework.common.Builder; using org.ohdsi.cdm.framework.common.Enums; using org.ohdsi.cdm.framework.common.Extensions; using org.ohdsi.cdm.framework.common.Helpers; @@ -1141,6 +1142,68 @@ public void JoinToVocabulary(IVocabulary vocabulary) Vocabulary ??= vocabulary; } + //protected static void AddEntity(T entity, List list) where T : IEntity + public static IEnumerable UpdateRSourceConcept(IEnumerable records) where T : IEntity + { + var r = new List(); + foreach (var record in records) + { + if (record.SourceConcepts != null && record.SourceConcepts.Any(r => char.ToLower(r.InvalidReason) == 'r')) + { + r.Add(record); + } + else + yield return record; + } + + if (r.Count > 1) + { + foreach (var byGuid in r.GroupBy(i => i.SourceRecordGuid)) + { + foreach (var bySource in byGuid.GroupBy(i => i.SourceValue)) + { + foreach (var byStartDate in bySource.GroupBy(i => i.StartDate)) + { + var e = byStartDate.Where(i => i.StartDate.Between(i.ValidStartDate, i.ValidEndDate)).FirstOrDefault(); + e ??= byStartDate.First(); + + long newSourceConceptId = 0; + long newConceptId = 0; + + foreach (var sc in e.SourceConcepts) + { + if (char.ToLower(sc.InvalidReason) != 'r') + continue; + + if (e.StartDate.Between(sc.ValidStartDate, sc.ValidEndDate)) + { + newSourceConceptId = sc.ConceptId; + } + + if (e.StartDate.Between(e.ValidStartDate, e.ValidEndDate)) + { + newConceptId = e.ConceptId; + } + } + + if (e.ConceptId != newConceptId || e.SourceConceptId != newSourceConceptId) + { + e.ConceptId = newConceptId; + e.SourceConceptId = newSourceConceptId; + } + + yield return e; + + if (e.SourceConcepts.Count != byStartDate.Count()) + { + + } + } + } + } + } + } + #endregion } } diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/CPRD/CprdPersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/CPRD/CprdPersonBuilder.cs index d483436c..375d4b0d 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/CPRD/CprdPersonBuilder.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/CPRD/CprdPersonBuilder.cs @@ -413,10 +413,10 @@ public override void AddToChunk(string domain, IEnumerable entities) if (result.Count != 0 && result.Count > 0 && result[0].SourceConcepts.Count > 0 - && result[0].SourceConcepts[0].ConceptId > 0 + && result[0].SourceConcepts.First().ConceptId > 0 /*&& result[0].Domain == "Measurement"*/) { - mes.SourceConceptId = result[0].SourceConcepts[0].ConceptId; + mes.SourceConceptId = result[0].SourceConcepts.First().ConceptId; } } @@ -481,9 +481,9 @@ public override void AddToChunk(string domain, IEnumerable entities) if (result.Count != 0 && result.Count > 0 && result[0].SourceConcepts.Count > 0 - && result[0].SourceConcepts[0].ConceptId > 0) + && result[0].SourceConcepts.First().ConceptId > 0) { - obs.SourceConceptId = result[0].SourceConcepts[0].ConceptId; + obs.SourceConceptId = result[0].SourceConcepts.First().ConceptId; } } diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/HealthVerityPersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/HealthVerityPersonBuilder.cs index 160ccc83..964c0479 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/HealthVerityPersonBuilder.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/HealthVerityPersonBuilder.cs @@ -644,12 +644,17 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) AddToChunk(person, null, observationPeriods, [], - drugExposures, - conditionOccurrences, - procedureOccurrences, - observations, - measurements, - [.. visitOccurrences.Values], visitDetails, [], deviceExposure, [], []); + UpdateRSourceConcept(drugExposures).ToArray(), + UpdateRSourceConcept(conditionOccurrences).ToArray(), + UpdateRSourceConcept(procedureOccurrences).ToArray(), + UpdateRSourceConcept(observations).ToArray(), + UpdateRSourceConcept(measurements).ToArray(), + [.. visitOccurrences.Values], + visitDetails, + [], + UpdateRSourceConcept(deviceExposure).ToArray(), + [], + []); Complete = true; diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/Lookups/procedure.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/Lookups/procedure.sql index e0345c3c..bec3c2ff 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/Lookups/procedure.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/Lookups/procedure.sql @@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's' AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '') ), Source as ( -SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs', 'cdt') AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs', 'cdt') @@ -19,7 +19,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/Procedure.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/Procedure.sql index 527bcc0f..f51a2b0f 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/Procedure.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/Procedure.sql @@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's' AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '') AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier','icd10pcs hierarchy') ), Source as ( -SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd9proc','icd10pcs') AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd9proc','icd10pcs') @@ -20,7 +20,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD10.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD10.sql index 0ae2885e..922d4986 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD10.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD10.sql @@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's' AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '') AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier', 'icd10pcs hierarchy') ), Source as ( -SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd10pcs','hcpcs','cpt4') AND lower(TARGET_VOCABULARY_ID) IN ('icd10pcs','hcpcs','cpt4') @@ -20,7 +20,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD9.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD9.sql index b89b25f2..09dc0682 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD9.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD9.sql @@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's' AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '') AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier') ), Source as ( -SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4') AND lower(TARGET_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4') @@ -20,7 +20,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/OptumExtendedPersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/OptumExtendedPersonBuilder.cs index 90d9d538..6fc2b99f 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/OptumExtendedPersonBuilder.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/OptumExtendedPersonBuilder.cs @@ -760,15 +760,15 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) death, observationPeriods, payerPlanPeriods, - Clean(drugExposures, person).ToArray(), - Clean(conditionOccurrences, person).ToArray(), - Clean(procedureOccurrences, person).ToArray(), - Clean(observations, person).ToArray(), - Clean(measurements, person).ToArray(), + UpdateRSourceConcept(Clean(drugExposures, person)).ToArray(), + UpdateRSourceConcept(Clean(conditionOccurrences, person)).ToArray(), + UpdateRSourceConcept(Clean(procedureOccurrences, person)).ToArray(), + UpdateRSourceConcept(Clean(observations, person)).ToArray(), + UpdateRSourceConcept(Clean(measurements, person)).ToArray(), vos, [.. vds.Values], [], - Clean(deviceExposure, person).ToArray(), + UpdateRSourceConcept(Clean(deviceExposure, person)).ToArray(), [], []); diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumPanther/Lookups/ConditionFromProcedure.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumPanther/Lookups/ConditionFromProcedure.sql index 39bd070e..a6a4fcd6 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumPanther/Lookups/ConditionFromProcedure.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumPanther/Lookups/ConditionFromProcedure.sql @@ -7,7 +7,7 @@ AND (lower(TARGET_STANDARD_CONCEPT) = 's') AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '') ), Source as ( -SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs') AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs') @@ -18,7 +18,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumPanther/OptumOncologyPersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumPanther/OptumOncologyPersonBuilder.cs index 6c0bc072..af99903b 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumPanther/OptumOncologyPersonBuilder.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumPanther/OptumOncologyPersonBuilder.cs @@ -1058,9 +1058,21 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager om) } // push built entities to ChunkBuilder for further save to CDM database - AddToChunk(person, death, [.. observationPeriodsFinal], payerPlanPeriods, drugExposures, - conditionOccurrences, procedureOccurrences, observations, measurements, - [.. visitOccurrences.Values], [.. visitDetails.Values], cohort, deviceExposure, notes, episodes); + AddToChunk(person, + death, + [.. observationPeriodsFinal], + payerPlanPeriods, + UpdateRSourceConcept(drugExposures).ToArray(), + UpdateRSourceConcept(conditionOccurrences).ToArray(), + UpdateRSourceConcept(procedureOccurrences).ToArray(), + UpdateRSourceConcept(observations).ToArray(), + UpdateRSourceConcept(measurements).ToArray(), + [.. visitOccurrences.Values], + [.. visitDetails.Values], + cohort, + UpdateRSourceConcept(deviceExposure).ToArray(), + notes, + episodes); var pg = new PregnancyAlgorithm.PregnancyAlgorithm(); foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, [.. observationPeriodsFinal], diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Batch.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Batch.sql index 27156060..09f7e1c7 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Batch.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Batch.sql @@ -1,4 +1,4 @@ -SELECT DISTINCT {0} medrec_key, medrec_key +SELECT DISTINCT {0} medrec_key, medrec_key FROM {sc}.pat WHERE medrec_key is not NULL order by 1 \ No newline at end of file diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Definitions/vitals.xml b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Definitions/vitals.xml index 9c05423f..af93933f 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Definitions/vitals.xml +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Definitions/vitals.xml @@ -64,7 +64,7 @@ - + {isloinc} = 0 medrec_key pat_key diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Device.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Device.sql index 58b7ad1d..7cedcae6 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Device.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Device.sql @@ -1,10 +1,23 @@ {base}, Standard as ( -SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_VALID_START_DATE as VALID_START_DATE, SOURCE_VALID_END_DATE as VALID_END_DATE, SOURCE_VOCABULARY_ID, TARGET_VALUE_AS_CONCEPT_ID FROM Source_to_Standard WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs', 'icd10cm', 'jnj_pmr_proc_chrg_cd', 'jnj_pmr_hosp_chg') AND lower(TARGET_DOMAIN_ID) IN ('device') +), Source as ( +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON +FROM Source_to_Source +WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs', 'icd10cm') +AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs', 'icd10cm') AND lower(TARGET_DOMAIN_ID)='device' +), S_S as +( +select SOURCE_CODE from Standard +union +select SOURCE_CODE from Source ) -select distinct Standard.* -from Standard \ No newline at end of file +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON +from S_S +left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE +left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE +left join ingredient_level on ingredient_level.concept_id = Standard.TARGET_CONCEPT_ID \ No newline at end of file diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/DrugCpt.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/DrugCpt.sql index 03cdeda5..b179ea09 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/DrugCpt.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/DrugCpt.sql @@ -4,8 +4,20 @@ SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_VALID_S FROM Source_to_Standard WHERE lower(SOURCE_VOCABULARY_ID) IN ('cpt4', 'hcpcs') AND lower(TARGET_DOMAIN_ID) = 'drug' +), Source as ( +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON +FROM Source_to_Source +WHERE lower(SOURCE_VOCABULARY_ID) IN ('cpt4', 'hcpcs') +AND lower(TARGET_VOCABULARY_ID) IN ('cpt4', 'hcpcs') AND lower(TARGET_DOMAIN_ID)='drug' +), S_S as +( +select SOURCE_CODE from Standard +union +select SOURCE_CODE from Source ) -select distinct Standard.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, NULL as SOURCE_TARGET_CONCEPT_ID, NULL as SOURCE_VALID_START_DATE, NULL AS SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID -from Standard +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON +from S_S +left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE +left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE left join ingredient_level on ingredient_level.concept_id = Standard.TARGET_CONCEPT_ID \ No newline at end of file diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Measurement.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Measurement.sql index 2f694b6e..bb3d0e18 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Measurement.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Measurement.sql @@ -5,7 +5,7 @@ FROM Source_to_Standard WHERE lower(SOURCE_VOCABULARY_ID) IN ('cpt4', 'hcpcs', 'icd10cm', 'icd9cm', 'jnj_pmr_proc_chrg_cd') AND lower(TARGET_DOMAIN_ID) = 'measurement' ), Source as ( -SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('cpt4', 'hcpcs') AND lower(TARGET_VOCABULARY_ID) IN ('cpt4', 'hcpcs') AND lower(TARGET_DOMAIN_ID)='measurement' @@ -16,7 +16,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Observation.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Observation.sql index c65376d8..ea371f6e 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Observation.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Observation.sql @@ -5,12 +5,12 @@ FROM Source_to_Standard WHERE lower(SOURCE_VOCABULARY_ID) IN ('cpt4', 'drg', 'hcpcs', 'jnj_pmr_obs_code', 'jnj_pmr_proc_chrg_cd') AND lower(TARGET_DOMAIN_ID) = 'observation' ), Source as ( -SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd9cm', 'cpt4', 'hcpcs', 'mdc') AND lower(TARGET_VOCABULARY_ID) IN ('icd9cm', 'cpt4', 'hcpcs', 'mdc') and lower(TARGET_DOMAIN_ID)='observation' union -SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('drg') AND lower(TARGET_VOCABULARY_ID) IN ('drg') and lower(TARGET_DOMAIN_ID)='observation' @@ -21,11 +21,8 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE -left join ingredient_level on ingredient_level.concept_id = Standard.TARGET_CONCEPT_ID - - - +left join ingredient_level on ingredient_level.concept_id = Standard.TARGET_CONCEPT_ID \ No newline at end of file diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/ObservationICD10.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/ObservationICD10.sql index 9efc4b08..1572f0a6 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/ObservationICD10.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/ObservationICD10.sql @@ -4,7 +4,7 @@ SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_VALID_S FROM Source_to_Standard WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd10cm') and lower(TARGET_DOMAIN_ID) = 'observation' ), Source as ( -SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd9cm', 'cpt4', 'hcpcs', 'icd10cm') AND lower(TARGET_VOCABULARY_ID) IN ('icd9cm', 'cpt4', 'hcpcs', 'icd10cm') @@ -15,7 +15,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/ObservationICD9.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/ObservationICD9.sql index c4636873..b656a685 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/ObservationICD9.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/ObservationICD9.sql @@ -4,7 +4,7 @@ SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_VALID_S FROM Source_to_Standard WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd9cm') and lower(TARGET_DOMAIN_ID) = 'observation' ), Source as ( -SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd9cm', 'cpt4', 'hcpcs', 'icd10cm') AND lower(TARGET_VOCABULARY_ID) IN ('icd9cm', 'cpt4', 'hcpcs', 'icd10cm') @@ -15,7 +15,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Procedure.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Procedure.sql index ed23163c..d95f8dde 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Procedure.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Procedure.sql @@ -11,7 +11,7 @@ OR lower(SOURCE_VOCABULARY_ID) IN ('jnj_pmr_proc_chrg_cd') AND TARGET_CONCEPT_ID=0 ) ), Source as ( -SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE +SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('cpt4', 'hcpcs', 'icd10cm', 'icd10pcs', 'icd9cm', 'icd9proc' ) AND lower(TARGET_VOCABULARY_ID) IN ('cpt4', 'hcpcs', 'icd10cm', 'icd10pcs', 'icd9cm', 'icd9proc' ) AND lower(TARGET_DOMAIN_ID)='procedure' @@ -22,7 +22,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/PremierPersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/PremierPersonBuilder.cs index 5b94ba1b..8b1b2afa 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/PremierPersonBuilder.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/PremierPersonBuilder.cs @@ -112,7 +112,7 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) // MONTH_OF_BIRTH and DAY_OF_BIRTH are not available in Premier, // because age is the only available field. YEAR_OF_BIRTH is calculated from the first admission. // The admission year minus the age results in the YEAR_OF_BIRTH. - foreach (var personRecord in PersonRecords) + foreach (var personRecord in PersonRecords.Where(p => p.YearOfBirth.HasValue)) { if (!visitsDictionary.ContainsKey(personRecord.SourceRecordGuid)) continue; @@ -135,6 +135,7 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) } } + // If a person has YEAR_OF_BIRTH that varies over two years then those records are eliminated if (maxYearOfBirth - minYearOfBirth > 2) { @@ -234,16 +235,17 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) } ConditionForEra.Clear(); + // push built entities to ChunkBuilder for further save to CDM database AddToChunk(person, death, observationPeriods, payerPlanPeriods, - drugExposures, - conditionOccurrences, - procedureOccurrences, - observations, - measurements, - [.. visitOccurrences.Values], null, [], deviceExposure, + UpdateRSourceConcept(drugExposures).ToArray(), + UpdateRSourceConcept(conditionOccurrences).ToArray(), + UpdateRSourceConcept(procedureOccurrences).ToArray(), + UpdateRSourceConcept(observations).ToArray(), + UpdateRSourceConcept(measurements).ToArray(), + [.. visitOccurrences.Values], null, [], UpdateRSourceConcept(deviceExposure).ToArray(), [], []); var pg = new PregnancyAlgorithm.PregnancyAlgorithm(); diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/Lookups/Procedure.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/Lookups/Procedure.sql index 0c1dc0ce..8abf486f 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/Lookups/Procedure.sql +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/Lookups/Procedure.sql @@ -7,7 +7,7 @@ AND (TARGET_STANDARD_CONCEPT IS NOT NULL or TARGET_STANDARD_CONCEPT != '') AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '') AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier','cpt4 hierarchy','icd10pcs hierarchy') ), Source as ( -SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID +SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_INVALID_REASON FROM Source_to_Source WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4','icd10pcs', 'cdt') AND lower(TARGET_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4','icd10pcs', 'cdt') @@ -19,7 +19,7 @@ union select SOURCE_CODE from Source ) -select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, cast('1900/1/1' as date) as SOURCE_validStartDate, cast('2100/1/1' as date) as SOURCE_validEndDate, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID +select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, cast('1900/1/1' as date) as SOURCE_validStartDate, cast('2100/1/1' as date) as SOURCE_validEndDate, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON from S_S left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/TruvenPersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/TruvenPersonBuilder.cs index 9c9bea4c..27202457 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/TruvenPersonBuilder.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Truven/TruvenPersonBuilder.cs @@ -759,11 +759,15 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) observationPeriods, payerPlanPeriods, [.. drugExposures], - conditionOccurrences, - procedureOccurrences, - observations, - measurements, - visitOccurrences, [.. visitDetails], [], deviceExposure, []); + UpdateRSourceConcept(conditionOccurrences).ToArray(), + UpdateRSourceConcept(procedureOccurrences).ToArray(), + UpdateRSourceConcept(observations).ToArray(), + UpdateRSourceConcept(measurements).ToArray(), + visitOccurrences, + [.. visitDetails], + [], + UpdateRSourceConcept(deviceExposure).ToArray(), + []); Complete = true; diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/DeathDefinition.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/DeathDefinition.cs index d0749045..3b0888f1 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/DeathDefinition.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/DeathDefinition.cs @@ -52,7 +52,7 @@ private IEnumerable PopulateDeath(IDataRecord reader, Concept secondary TypeConceptId = field.DefaultTypeId ?? 0, ValidStartDate = lookupValue.ValidStartDate, ValidEndDate = lookupValue.ValidEndDate, - SourceCauseConceptId = lookupValue.SourceConcepts.Count != 0 ? lookupValue.SourceConcepts[0].ConceptId : 0 + SourceCauseConceptId = lookupValue.SourceConcepts.Count != 0 ? lookupValue.SourceConcepts.First().ConceptId : 0 }; break; diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/EntityDefinition.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/EntityDefinition.cs index 7a985136..997cd332 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/EntityDefinition.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/EntityDefinition.cs @@ -124,11 +124,11 @@ public virtual IEnumerable GetConcepts(Concept concept, IDataRecord rea } // TMP - var sourceConceptId = lookupValue.SourceConcepts.Count != 0 ? lookupValue.SourceConcepts[0].ConceptId : 0; + var sourceConceptId = lookupValue.SourceConcepts.Count != 0 ? lookupValue.SourceConcepts.First().ConceptId : 0; if (!string.IsNullOrEmpty(field.SourceConceptId)) { - var scId = reader.GetLong(field.SourceConceptId); - if(scId.HasValue) + var scId = reader.GetLong(field.SourceConceptId); + if (scId.HasValue) { sourceConceptId = scId.Value; } @@ -156,7 +156,8 @@ public virtual IEnumerable GetConcepts(Concept concept, IDataRecord rea //SourceVocabularyId = lookupValue.SourceVocabularyId, VocabularySourceValue = lookupValue.SourceCode, Ingredients = ingredients, - ValueAsConceptId = lookupValue.ValueAsConceptId + ValueAsConceptId = lookupValue.ValueAsConceptId, + SourceConcepts = lookupValue.SourceConcepts.ToList(), }; } diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/ProviderDefinition.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/ProviderDefinition.cs index ef839dc4..4e903842 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/ProviderDefinition.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Definitions/ProviderDefinition.cs @@ -97,8 +97,8 @@ public override IEnumerable GetConcepts(Concept concept, IDataRecord re if (specialtyConceptIds[0].ConceptId != 0) specialtyConcept = specialtyConceptIds[0].ConceptId; - if (specialtyConceptIds[0].SourceConcepts.Count > 0 && specialtyConceptIds[0].SourceConcepts[0].ConceptId != 0) - specialtySourceConceptId = specialtyConceptIds[0].SourceConcepts[0].ConceptId; + if (specialtyConceptIds[0].SourceConcepts.Count > 0 && specialtyConceptIds[0].SourceConcepts.First().ConceptId != 0) + specialtySourceConceptId = specialtyConceptIds[0].SourceConcepts.First().ConceptId; } var prov = new Provider diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Lookups/Lookup.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Lookups/Lookup.cs index 6a6fbdf5..9740cb93 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Lookups/Lookup.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Lookups/Lookup.cs @@ -179,6 +179,7 @@ public void Fill(AmazonS3Client client, string bucket, string prefix) var sourceValidStartDate = DateTime.MinValue; var sourceValidEndDate = DateTime.MaxValue; + var invalidReason = char.MinValue; if (spliter.Results.Length > 6) { @@ -186,11 +187,18 @@ public void Fill(AmazonS3Client client, string bucket, string prefix) DateTime.TryParse(spliter.Results[8], out sourceValidEndDate); } + if (spliter.Results.Length > 11) + { + if (!IsNullOrEmpty(spliter.Results[11])) + invalidReason = spliter.Results[11][0]; + } + value.SourceConcepts.Add(new SourceConcepts { ConceptId = sourceConceptId, ValidStartDate = sourceValidStartDate, - ValidEndDate = sourceValidEndDate + ValidEndDate = sourceValidEndDate, + InvalidReason = invalidReason }); if (!IsNullOrEmpty(spliter.Results[9]) && diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Lookups/LookupValue.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Lookups/LookupValue.cs index c00210bd..d4f97867 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Lookups/LookupValue.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Lookups/LookupValue.cs @@ -9,17 +9,22 @@ public class SourceConcepts : IEquatable public long ConceptId { get; set; } public DateTime ValidStartDate { get; set; } public DateTime ValidEndDate { get; set; } + public char InvalidReason { get; set; } public bool Equals(SourceConcepts other) { return this.ConceptId == other.ConceptId && this.ValidStartDate.Equals(other.ValidStartDate) && - this.ValidEndDate.Equals(other.ValidEndDate); + this.ValidEndDate.Equals(other.ValidEndDate) && + this.InvalidReason.Equals(other.InvalidReason); } public override int GetHashCode() { - return base.GetHashCode(); + return ConceptId.GetHashCode() ^ + (ValidStartDate.GetHashCode()) ^ + (ValidEndDate.GetHashCode()) ^ + InvalidReason.GetHashCode(); } } @@ -31,7 +36,7 @@ public class LookupValue : IEquatable public DateTime ValidStartDate { get; set; } public DateTime ValidEndDate { get; set; } - public List SourceConcepts { get; set; } = []; + public HashSet SourceConcepts { get; set; } = []; public HashSet Ingredients { get; set; } public long? ValueAsConceptId { get; set; } diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Omop/Entity.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Omop/Entity.cs index 7a69c1e1..ee74ef99 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Omop/Entity.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Omop/Entity.cs @@ -1,4 +1,5 @@ using org.ohdsi.cdm.framework.common.Enums; +using org.ohdsi.cdm.framework.common.Lookups; using System; using System.Collections.Generic; using System.Security.Cryptography; @@ -30,14 +31,19 @@ public class Entity : IEntity public string ProviderKey { get; set; } public Dictionary AdditionalFields { get; set; } + public List Ingredients { get; set; } + public List SourceConcepts { get; set; } + - // CDM v5 props public long SourceConceptId { get; set; } public string Domain { get; set; } public string VocabularySourceValue { get; set; } public long? ValueAsConceptId { get; set; } + public DateTime ValidStartDate { get; set; } + public DateTime ValidEndDate { get; set; } + public Entity() { @@ -74,10 +80,7 @@ public virtual bool IncludeInEra() public virtual EntityType GeEntityType() { return EntityType.Entity; - } - - public DateTime ValidStartDate { get; set; } - public DateTime ValidEndDate { get; set; } + } public void Init(IEntity ent) { @@ -102,6 +105,7 @@ public void Init(IEntity ent) ConceptIdKey = ent.ConceptIdKey; VocabularySourceValue = ent.VocabularySourceValue; ValueAsConceptId = ent.ValueAsConceptId; + SourceConcepts = ent.SourceConcepts; } public static long GetId(string key) diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Omop/IEntity.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Omop/IEntity.cs index 0963bafe..822b100b 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/Omop/IEntity.cs +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/Omop/IEntity.cs @@ -1,4 +1,5 @@ using org.ohdsi.cdm.framework.common.Enums; +using org.ohdsi.cdm.framework.common.Lookups; using System; using System.Collections.Generic; @@ -37,6 +38,8 @@ public interface IEntity string VocabularySourceValue { get; set; } long? ValueAsConceptId { get; set; } + List SourceConcepts { get; set; } + string GetKey(); } } diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/org.ohdsi.cdm.framework.common.csproj b/sources/Framework/org.ohdsi.cdm.framework.common3/org.ohdsi.cdm.framework.common.csproj index 961f726d..5b8b2dae 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.common3/org.ohdsi.cdm.framework.common.csproj +++ b/sources/Framework/org.ohdsi.cdm.framework.common3/org.ohdsi.cdm.framework.common.csproj @@ -1809,8 +1809,8 @@ - - + + diff --git a/sources/Framework/org.ohdsi.cdm.framework.desktop3/org.ohdsi.cdm.framework.desktop.csproj b/sources/Framework/org.ohdsi.cdm.framework.desktop3/org.ohdsi.cdm.framework.desktop.csproj index 665e5bc2..eb3a5487 100644 --- a/sources/Framework/org.ohdsi.cdm.framework.desktop3/org.ohdsi.cdm.framework.desktop.csproj +++ b/sources/Framework/org.ohdsi.cdm.framework.desktop3/org.ohdsi.cdm.framework.desktop.csproj @@ -6,7 +6,7 @@ - + diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/org.ohdsi.cdm.presentation.etl.csproj b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/org.ohdsi.cdm.presentation.etl.csproj index 6e53881b..806e3e20 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/org.ohdsi.cdm.presentation.etl.csproj +++ b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/org.ohdsi.cdm.presentation.etl.csproj @@ -22,7 +22,7 @@ - +