Merge pull request #77 from OHDSI/source_concept_id_invalid_reason_R

Source concept id invalid reason r
OHDSI · Jun 12, 2024 · fd6e3bb · fd6e3bb
2 parents 6dd2587 + d99992d
commit fd6e3bb
Show file tree

Hide file tree

Showing 32 changed files with 215 additions and 86 deletions.
diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Base/PersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework.common3/Base/PersonBuilder.cs
@@ -1,4 +1,5 @@
-using org.ohdsi.cdm.framework.common.Builder;
+using Force.DeepCloner;
+using org.ohdsi.cdm.framework.common.Builder;
 using org.ohdsi.cdm.framework.common.Enums;
 using org.ohdsi.cdm.framework.common.Extensions;
 using org.ohdsi.cdm.framework.common.Helpers;
@@ -1141,6 +1142,68 @@ public void JoinToVocabulary(IVocabulary vocabulary)
             Vocabulary ??= vocabulary;
         }
 
+        //protected static void AddEntity<T>(T entity, List<T> list) where T : IEntity
+        public static IEnumerable<T> UpdateRSourceConcept<T>(IEnumerable<T> records) where T : IEntity
+        {
+            var r = new List<T>();
+            foreach (var record in records)
+            {
+                if (record.SourceConcepts != null && record.SourceConcepts.Any(r => char.ToLower(r.InvalidReason) == 'r'))
+                {
+                    r.Add(record);
+                }
+                else
+                    yield return record;
+            }
+
+            if (r.Count > 1)
+            {
+                foreach (var byGuid in r.GroupBy(i => i.SourceRecordGuid))
+                {
+                    foreach (var bySource in byGuid.GroupBy(i => i.SourceValue))
+                    {
+                        foreach (var byStartDate in bySource.GroupBy(i => i.StartDate))
+                        {
+                            var e = byStartDate.Where(i => i.StartDate.Between(i.ValidStartDate, i.ValidEndDate)).FirstOrDefault();
+                            e ??= byStartDate.First();
+
+                            long newSourceConceptId = 0;
+                            long newConceptId = 0;
+
+                            foreach (var sc in e.SourceConcepts)
+                            {
+                                if (char.ToLower(sc.InvalidReason) != 'r')
+                                    continue;
+
+                                if (e.StartDate.Between(sc.ValidStartDate, sc.ValidEndDate))
+                                {
+                                    newSourceConceptId = sc.ConceptId;
+                                }
+
+                                if (e.StartDate.Between(e.ValidStartDate, e.ValidEndDate))
+                                {
+                                    newConceptId = e.ConceptId;
+                                }
+                            }
+
+                            if (e.ConceptId != newConceptId || e.SourceConceptId != newSourceConceptId)
+                            {
+                                e.ConceptId = newConceptId;
+                                e.SourceConceptId = newSourceConceptId;
+                            }
+
+                            yield return e;
+
+                            if (e.SourceConcepts.Count != byStartDate.Count())
+                            {
+
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
         #endregion
     }
 }
diff --git a/...s/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/CPRD/CprdPersonBuilder.cs b/...s/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/CPRD/CprdPersonBuilder.cs
@@ -413,10 +413,10 @@ public override void AddToChunk(string domain, IEnumerable<IEntity> entities)
                             if (result.Count != 0
                                 && result.Count > 0
                                 && result[0].SourceConcepts.Count > 0
-                                && result[0].SourceConcepts[0].ConceptId > 0
+                                && result[0].SourceConcepts.First().ConceptId > 0
                                 /*&& result[0].Domain == "Measurement"*/)
                             {
-                                mes.SourceConceptId = result[0].SourceConcepts[0].ConceptId;
+                                mes.SourceConceptId = result[0].SourceConcepts.First().ConceptId;
                             }
                         }
 
@@ -481,9 +481,9 @@ public override void AddToChunk(string domain, IEnumerable<IEntity> entities)
                                 if (result.Count != 0
                                     && result.Count > 0
                                     && result[0].SourceConcepts.Count > 0
-                                    && result[0].SourceConcepts[0].ConceptId > 0)
+                                    && result[0].SourceConcepts.First().ConceptId > 0)
                                 {
-                                    obs.SourceConceptId = result[0].SourceConcepts[0].ConceptId;
+                                    obs.SourceConceptId = result[0].SourceConcepts.First().ConceptId;
                                 }
                             }
 

diff --git a/...ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/HealthVerityPersonBuilder.cs b/...ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/HealthVerityPersonBuilder.cs
@@ -644,12 +644,17 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
             AddToChunk(person, null,
                 observationPeriods,
                 [],
-                drugExposures,
-                conditionOccurrences,
-                procedureOccurrences,
-                observations,
-                measurements,
-                [.. visitOccurrences.Values], visitDetails, [], deviceExposure, [], []);
+                UpdateRSourceConcept(drugExposures).ToArray(),
+                UpdateRSourceConcept(conditionOccurrences).ToArray(),
+                UpdateRSourceConcept(procedureOccurrences).ToArray(),
+                UpdateRSourceConcept(observations).ToArray(),
+                UpdateRSourceConcept(measurements).ToArray(),
+                [.. visitOccurrences.Values], 
+                visitDetails, 
+                [],
+                UpdateRSourceConcept(deviceExposure).ToArray(), 
+                [], 
+                []);
 
             Complete = true;
 

diff --git a/...rk/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/Lookups/procedure.sql b/...rk/org.ohdsi.cdm.framework.common3/Core/Transformation/HealthVerity/Lookups/procedure.sql
@@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's'
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 ), 
 Source as (
-SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs', 'cdt')
 AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs', 'cdt')
@@ -19,7 +19,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/...k/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/Procedure.sql b/...k/org.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/Procedure.sql
@@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's'
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier','icd10pcs hierarchy')
 ), Source as (
-SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd9proc','icd10pcs')
 AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd9proc','icd10pcs')
@@ -20,7 +20,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/....ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD10.sql b/....ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD10.sql
@@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's'
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier', 'icd10pcs hierarchy')
 ), Source as (
-SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd10pcs','hcpcs','cpt4')
 AND lower(TARGET_VOCABULARY_ID) IN ('icd10pcs','hcpcs','cpt4')
@@ -20,7 +20,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/...g.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD9.sql b/...g.ohdsi.cdm.framework.common3/Core/Transformation/OptumExtended/Lookups/ProcedureICD9.sql
@@ -8,7 +8,7 @@ AND lower(TARGET_STANDARD_CONCEPT) = 's'
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 AND lower(TARGET_CONCEPT_CLASS_ID) NOT IN ('hcpcs modifier','cpt4 modifier')
 ), Source as (
-SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct REPLACE(SOURCE_CODE, '.', '') as SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4')
 AND lower(TARGET_VOCABULARY_ID) IN ('icd9proc','hcpcs','cpt4')
@@ -20,7 +20,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/...dsi.cdm.framework.common3/Core/Transformation/OptumExtended/OptumExtendedPersonBuilder.cs b/...dsi.cdm.framework.common3/Core/Transformation/OptumExtended/OptumExtendedPersonBuilder.cs
@@ -760,15 +760,15 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
                 death,
                 observationPeriods,
                 payerPlanPeriods,
-                Clean(drugExposures, person).ToArray(),
-                Clean(conditionOccurrences, person).ToArray(),
-                Clean(procedureOccurrences, person).ToArray(),
-                Clean(observations, person).ToArray(),
-                Clean(measurements, person).ToArray(),
+                UpdateRSourceConcept(Clean(drugExposures, person)).ToArray(),
+                UpdateRSourceConcept(Clean(conditionOccurrences, person)).ToArray(),
+                UpdateRSourceConcept(Clean(procedureOccurrences, person)).ToArray(),
+                UpdateRSourceConcept(Clean(observations, person)).ToArray(),
+                UpdateRSourceConcept(Clean(measurements, person)).ToArray(),
                 vos,
                 [.. vds.Values],
                 [],
-                Clean(deviceExposure, person).ToArray(),
+                UpdateRSourceConcept(Clean(deviceExposure, person)).ToArray(),
                 [],
                 []);
 

diff --git a/...cdm.framework.common3/Core/Transformation/OptumPanther/Lookups/ConditionFromProcedure.sql b/...cdm.framework.common3/Core/Transformation/OptumPanther/Lookups/ConditionFromProcedure.sql
@@ -7,7 +7,7 @@ AND (lower(TARGET_STANDARD_CONCEPT) = 's')
 AND (TARGET_INVALID_REASON IS NULL or TARGET_INVALID_REASON = '')
 
 ), Source as (
-SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct REPLACE(SOURCE_CODE, '.', '') AS SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
 FROM Source_to_Source
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs')
 AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs','cpt4', 'icd10pcs')
@@ -18,7 +18,7 @@ union
 select SOURCE_CODE from Source
 )
 
-select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
 from S_S
 left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
 left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 

diff --git a/...hdsi.cdm.framework.common3/Core/Transformation/OptumPanther/OptumOncologyPersonBuilder.cs b/...hdsi.cdm.framework.common3/Core/Transformation/OptumPanther/OptumOncologyPersonBuilder.cs
@@ -1058,9 +1058,21 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
             }
 
             // push built entities to ChunkBuilder for further save to CDM database
-            AddToChunk(person, death, [.. observationPeriodsFinal], payerPlanPeriods, drugExposures,
-                conditionOccurrences, procedureOccurrences, observations, measurements,
-                [.. visitOccurrences.Values], [.. visitDetails.Values], cohort, deviceExposure, notes, episodes);
+            AddToChunk(person, 
+                death, 
+                [.. observationPeriodsFinal], 
+                payerPlanPeriods,
+                UpdateRSourceConcept(drugExposures).ToArray(),
+                UpdateRSourceConcept(conditionOccurrences).ToArray(),
+                UpdateRSourceConcept(procedureOccurrences).ToArray(),
+                UpdateRSourceConcept(observations).ToArray(),
+                UpdateRSourceConcept(measurements).ToArray(),
+                [.. visitOccurrences.Values], 
+                [.. visitDetails.Values], 
+                cohort,
+                UpdateRSourceConcept(deviceExposure).ToArray(), 
+                notes, 
+                episodes);
 
             var pg = new PregnancyAlgorithm.PregnancyAlgorithm();
             foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, [.. observationPeriodsFinal],

diff --git a/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Batch.sql b/sources/Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Batch.sql
@@ -1,4 +1,4 @@
-SELECT DISTINCT {0} medrec_key, medrec_key
+SELECT DISTINCT {0} medrec_key, medrec_key
 FROM {sc}.pat
 WHERE medrec_key is not NULL 
 order by 1
diff --git a/...mework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Definitions/vitals.xml b/...mework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Definitions/vitals.xml
@@ -64,7 +64,7 @@
         </Concept>
       </Concepts>
     </MeasurementDefinition>
-	<MeasurementDefinition>
+	  <MeasurementDefinition>
 	  <Condition>{isloinc} = 0</Condition>
       <PersonId>medrec_key</PersonId>
       <VisitOccurrenceId>pat_key</VisitOccurrenceId>

diff --git a/.../Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Device.sql b/.../Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/Device.sql
@@ -1,10 +1,23 @@
 {base},
 Standard as (
-SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE
+SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_VALID_START_DATE as VALID_START_DATE, SOURCE_VALID_END_DATE as VALID_END_DATE, SOURCE_VOCABULARY_ID, TARGET_VALUE_AS_CONCEPT_ID
 FROM Source_to_Standard
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs', 'icd10cm', 'jnj_pmr_proc_chrg_cd', 'jnj_pmr_hosp_chg')
 AND lower(TARGET_DOMAIN_ID) IN ('device')
+), Source as (
+SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
+FROM Source_to_Source
+WHERE lower(SOURCE_VOCABULARY_ID) IN ('hcpcs', 'icd10cm')
+AND lower(TARGET_VOCABULARY_ID) IN ('hcpcs', 'icd10cm') AND lower(TARGET_DOMAIN_ID)='device'
+), S_S as
+(
+select SOURCE_CODE from Standard
+union 
+select SOURCE_CODE from Source
 )
 
-select distinct Standard.*
-from Standard
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
+from S_S
+left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
+left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 
+left join ingredient_level on ingredient_level.concept_id = Standard.TARGET_CONCEPT_ID
diff --git a/...Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/DrugCpt.sql b/...Framework/org.ohdsi.cdm.framework.common3/Core/Transformation/Premier/Lookups/DrugCpt.sql
@@ -4,8 +4,20 @@ SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, TARGET_DOMAIN_ID, SOURCE_VALID_S
 FROM Source_to_Standard
 WHERE lower(SOURCE_VOCABULARY_ID) IN ('cpt4', 'hcpcs')
 AND lower(TARGET_DOMAIN_ID) = 'drug'
+), Source as (
+SELECT distinct SOURCE_CODE, TARGET_CONCEPT_ID, SOURCE_VALID_START_DATE, SOURCE_VALID_END_DATE, SOURCE_INVALID_REASON
+FROM Source_to_Source
+WHERE lower(SOURCE_VOCABULARY_ID) IN ('cpt4', 'hcpcs')
+AND lower(TARGET_VOCABULARY_ID) IN ('cpt4', 'hcpcs') AND lower(TARGET_DOMAIN_ID)='drug'
+), S_S as
+(
+select SOURCE_CODE from Standard
+union 
+select SOURCE_CODE from Source
 )
 
-select distinct Standard.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, NULL as SOURCE_TARGET_CONCEPT_ID, NULL as SOURCE_VALID_START_DATE, NULL AS SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID
-from Standard
+select distinct S_S.SOURCE_CODE, Standard.TARGET_CONCEPT_ID, Standard.TARGET_DOMAIN_ID, Standard.VALID_START_DATE, Standard.VALID_END_DATE, Standard.SOURCE_VOCABULARY_ID, Source.TARGET_CONCEPT_ID as SOURCE_TARGET_CONCEPT_ID, Source.SOURCE_VALID_START_DATE as SOURCE_VALID_START_DATE, Source.SOURCE_VALID_END_DATE, ingredient_level.ingredient_concept_id, Standard.TARGET_VALUE_AS_CONCEPT_ID, Source.SOURCE_INVALID_REASON
+from S_S
+left join Standard on Standard.SOURCE_CODE = S_S.SOURCE_CODE
+left join Source on Source.SOURCE_CODE = S_S.SOURCE_CODE 
 left join ingredient_level on ingredient_level.concept_id = Standard.TARGET_CONCEPT_ID