WIP: Config list to map (sustances module)

gsrs-fda-substance-extension/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>gsrs-module-substances</artifactId>
         <groupId>gov.nih.ncats</groupId>
-        <version>3.1.1.1</version>
+        <version>3.1.2-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -110,7 +110,7 @@
         <dependency>
             <groupId>gov.nih.ncats</groupId>
             <artifactId>applications-api</artifactId>
-            <version>3.1.1-SNAPSHOT</version>
+            <version>${gsrs.applications-api.version}</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.apache.logging.log4j</groupId>
@@ -139,7 +139,7 @@
         <dependency>
             <groupId>gov.nih.ncats</groupId>
             <artifactId>products-api</artifactId>
-            <version>3.1.1-SNAPSHOT</version>
+            <version>${gsrs.products-api.version}</version> 
             <exclusions>
                 <exclusion>
                     <groupId>org.apache.logging.log4j</groupId>
@@ -168,7 +168,7 @@
         <dependency>
             <groupId>gov.nih.ncats</groupId>
             <artifactId>clinical-trials-api</artifactId>
-            <version>3.1.1-SNAPSHOT</version>
+            <version>${gsrs.clinical-trials-api.version}</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.apache.logging.log4j</groupId>

gsrs-module-substance-example/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>gsrs-module-substances</artifactId>
         <groupId>gov.nih.ncats</groupId>
-        <version>3.1.1.1</version>
+        <version>3.1.2-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -339,7 +339,7 @@
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
-            <version>2.11.0</version>
+            <version>2.14.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

gsrs-module-substance-example/src/main/resources/application.conf

@@ -25,8 +25,8 @@ ix.authentication.logheaders=false
 #logging.level.org.hibernate.type.descriptor.sql.BasicBinder=TRACE
 logging.level.gsrs.module.substance.scrubbers.basic=trace
 
-gsrs.matchableCalculators.substances =
-[
+gsrs.matchableCalculators.substances.list.CASNumberMatchableExtractor =
+{
     {"matchableCalculationClass" : "gsrs.dataexchange.extractors.CASNumberMatchableExtractor",
         "config" :{
             "casCodeSystems": ["CAS", "CASNo", "CASNumber"]
@@ -49,5 +49,5 @@ gsrs.matchableCalculators.substances =
             "codeKey": "CODE"
         }
     }
-]
+}
 

gsrs-module-substance-example/src/main/resources/fda-extension.conf

@@ -24,9 +24,11 @@ gsrs.microservice.clinicaltrialseurope.api.headers= {
                         "auth-key"="myKey"
 }
 
-gsrs.entityProcessors +={
+gsrs.entityProcessors.list.UniqueCodeGenerator =
+       {
                "entityClassName" = "ix.ginas.models.v1.Substance",
                "processor" = "gsrs.module.substance.processors.UniqueCodeGenerator",
+               "order" = 8010,
                "with"=  {
                         "useLegacy"=true,
                         "codesystem"="BDNUM",
@@ -39,34 +41,64 @@ gsrs.entityProcessors +={
                }
         }
 
-gsrs.entityProcessors +=
-        {
-        "entityClassName" = "ix.ginas.models.v1.Substance",
-        "processor" = "gsrs.module.substance.processors.ApprovalIdProcessor",
-        "parameters" = {
-            "codeSystem" = "FDA UNII"
+gsrs.entityProcessors.list.ApprovalIdProcessor =
+       {
+	       "entityClassName" = "ix.ginas.models.v1.Substance",
+	       "processor" = "gsrs.module.substance.processors.ApprovalIdProcessor",
+         "order" = 6010
         }
+
+
+
+
+
+# can we remove this section since all these are in substances-core.conf?
+ix.ginas.export.exporterfactories.substances.list.FDANameExporterFactory =
+    {
+        "exporterFactoryClass" = "fda.gsrs.substance.exporters.FDANameExporterFactory",
+        "order" =  1200,
+        "disabled" = true,
+        "parameters":{
+        }
+    }
+ix.ginas.export.exporterfactories.substances.list.FDACodeExporterFactory =
+    {
+        "exporterFactoryClass" = "fda.gsrs.substance.exporters.FDACodeExporterFactory",
+        "order" =  1300,
+        "disabled" = true,
+        "parameters":{
         }
+    }
+ix.ginas.export.exporterfactories.substances.list.SPLValidatorXMLExporterFactory =
+    {
+        "exporterFactoryClass" = "fda.gsrs.substance.exporters.SPLValidatorXMLExporterFactory",
+        "order" =  1400,
+        "disabled" = true,
+        "parameters":{
+        }
+    }
+ix.ginas.export.exporterfactories.substances.list.SRSLegacyDictionaryExporterFactory =
+    {
+        "exporterFactoryClass" = "fda.gsrs.substance.exporters.SRSLegacyDictionaryExporterFactory",
+        "order" =  1500,
+        "disabled" = true,
+        "parameters":{
+        }
+    }
 
-ix.ginas.export.factories.substances = ${ix.ginas.export.factories.substances}[
-    #"gsrs.module.substance.ExtraColumnsSpreadsheetExporterFactory",
 
-	"fda.gsrs.substance.exporters.FDANameExporterFactory",
-	"fda.gsrs.substance.exporters.FDACodeExporterFactory",
-	"fda.gsrs.substance.exporters.SPLValidatorXMLExporterFactory",
-    "fda.gsrs.substance.exporters.SRSLegacyDictionaryExporterFactory",
-
- # "ix.ginas.exporters.SRSLegacyDictionaryExporterFactory"
 
-]
 
-ix.ginas.approvalIdGenerator.generatorClass=ix.ginas.utils.UNIIGenerator
 
 
-gsrs.validators.substances += {
+ix.ginas.approvalIdGenerator.generatorClass=ix.ginas.utils.UNIIGenerator
 
-                "validatorClass" = "fda.gsrs.substance.validators.BdNumModificationValidator",
-                "newObjClass" = "ix.ginas.models.v1.Substance"
-        }
+
+gsrs.validators.substances.list.BdNumModificationValidator =
+    {
+        "validatorClass" = "fda.gsrs.substance.validators.BdNumModificationValidator",
+        "newObjClass" = "ix.ginas.models.v1.Substance",
+        "order": 1010
+    }
 
 

gsrs-module-substance-example/src/test/java/example/imports/DefaultImportAdapterFactoryConfigTest.java

@@ -34,17 +34,21 @@ public void testSetup() throws IllegalAccessException, NoSuchFieldException, Jso
         String substanceContext = "substances";
         //build up a complete configuration
         GsrsFactoryConfiguration config = new GsrsFactoryConfiguration();
-        Map<String, List<Map<String, Object>>> adapterConfig = new HashMap<>();
+        Map<String, Map<String, Map<String, Map<String, Object>>>> adapterConfig = new HashMap<>();
         Map<String, Object> oneAdapter = new HashMap<>();
         oneAdapter.put("importAdapterFactoryClass", "gsrs.module.substance.importers.SDFImportAdapterFactory");
+        oneAdapter.put("order", 1000);
+        oneAdapter.put("parentKey", "SDFImportAdapterFactory");
         oneAdapter.put("adapterName", "NSRS SDF Adapter");
         oneAdapter.put("extensions", Arrays.asList("sdf", "sd"));
         oneAdapter.put("parameters", buildConfigParameters());
         oneAdapter.put("stagingAreaServiceClass", gsrs.stagingarea.service.DefaultStagingAreaService.class);
         oneAdapter.put("entityServiceClass", "gsrs.dataexchange.SubstanceStagingAreaEntityService");
-        List<Map<String, Object>> adapters = new ArrayList<>();
-        adapters.add(oneAdapter);
-        adapterConfig.put(substanceContext, adapters);
+        Map<String, Map<String, Object>> adapters = new HashMap<>();
+        // ix.ginas.export.exporterfactories.substances.list.SdfExporterFactory =
+        adapters.put("SDFImportAdapterFactory", oneAdapter);
+        adapterConfig.put(substanceContext, null);
+        adapterConfig.get(substanceContext).put("list", adapters);
         config.setImportAdapterFactories(adapterConfig);
         ConfigBasedGsrsImportAdapterFactoryFactory factoryFactory = new ConfigBasedGsrsImportAdapterFactoryFactory();
         Field configField = factoryFactory.getClass().getDeclaredField("gsrsFactoryConfiguration"); //gsrs.imports.ConfigBasedGsrsImportAdapterFactoryFactory.

gsrs-module-substance-example/src/test/java/example/imports/StagingAreaServiceTest.java

@@ -29,39 +29,43 @@ public class StagingAreaServiceTest extends AbstractSubstanceJpaEntityTest {
 
     @BeforeEach
     public void setup() throws NoSuchFieldException, IllegalAccessException {
+        // gsrs.matchableCalculators.substances.list.KEY =
+
+
         if( stagingAreaService == null ){
             log.trace("setting up staging area service");
             stagingAreaService = new DefaultStagingAreaService();
             stagingAreaService = AutowireHelper.getInstance().autowireAndProxy(stagingAreaService);
-            Map<String, List<Map<String, Object>>> matchableCalculatorConfig = new HashMap<>();
-            List<Map<String, Object>> configs = new ArrayList<>();
+            Map<String, Map<String, Map<String, Map<String, Object>>>> matchableCalculatorConfig = new HashMap<>();
+            Map<String, Map<String, Object>> configs = new HashMap<>();
             Map<String, Object> casExtractor = new HashMap<>();
             casExtractor.put("matchableCalculationClass", CASNumberMatchableExtractor.class);
             LinkedHashMap<String, Object> config= new LinkedHashMap<>();
             config.put("casCodeSystems", Arrays.asList("CAS", "CASNo", "CASNumber"));
             casExtractor.put("config", config);
-            configs.add(casExtractor);
+            configs.put("CASNumberMatchableExtractor", casExtractor);
 
             Map<String, Object> namesExtractor = new HashMap<>();
             namesExtractor.put("matchableCalculationClass", AllNamesMatchableExtractor.class);
-            configs.add(namesExtractor);
+            configs.put("AllNamesMatchableExtractor", namesExtractor);
 
             Map<String, Object> defHashExtractor = new HashMap<>();
             defHashExtractor.put("matchableCalculationClass", DefinitionalHashMatchableExtractor.class);
-            configs.add(defHashExtractor);
+            configs.put("DefinitionalHashMatchableExtractor", defHashExtractor);
 
             Map<String, Object> selectedCodesExtractor = new HashMap<>();
             selectedCodesExtractor.put("matchableCalculationClass", SelectedCodesMatchableExtractor.class);
             LinkedHashMap<String, Object> config2= new LinkedHashMap<>();
             config2.put("codeSystems", Arrays.asList("CAS", "ChemBL", "NCI", "NSC", "EINECS"));
             selectedCodesExtractor.put("config", config2);
-            configs.add(selectedCodesExtractor);
+            configs.put("SelectedCodesMatchableExtractor", selectedCodesExtractor);
 
             Map<String, Object> uuidExtractor = new HashMap<>();
             uuidExtractor.put("matchableCalculationClass", UUIDMatchableExtractor.class);
-            configs.add(uuidExtractor);
-            matchableCalculatorConfig.put("substances", configs);
-
+            configs.put("UUIDMatchableExtractor", uuidExtractor);
+            matchableCalculatorConfig.put("substances",
+               new HashMap<String, Map<String, Map<String, Object>>>(){{ put("list", configs);}}
+            );
             SubstanceStagingAreaEntityService stagingAreaEntityService = new SubstanceStagingAreaEntityService();
             stagingAreaEntityService = AutowireHelper.getInstance().autowireAndProxy(stagingAreaEntityService);
             Field factoryConfigField= stagingAreaEntityService.getClass().getDeclaredField("gsrsFactoryConfiguration");

gsrs-module-substance-example/src/test/java/example/substance/indexer/ExampleValueMakerTest.java

@@ -114,6 +114,7 @@ public void addExternalActiveMoietyShouldResultInChildSubstanceTag(){
 
 				})
 				.generateNewUUID()
+				// test fails here, seems to be cdk layout error
 				.build();
 
 		this.testIndexableValuesHasFacet(childSub, ExampleValueMaker.MOIETY_TYPE_FACET, ExampleValueMaker.CHILD_SUBSTANCE_RELATIONSHIP);

gsrs-module-substance-example/src/test/java/example/substance/processor/ConfigurableMolweightProcessorTest.java

@@ -106,6 +106,7 @@ public void testComplexMwCalcProperty() throws IOException {
         ChemicalSubstanceBuilder builder = new ChemicalSubstanceBuilder();
         builder.addName("benzene")
                 .setStructureWithDefaultReference("C[N+](C)(C)C.[O-]C(=O)c1ccc2ncccc2c1");
+        // test fails here, seems to be cdk layout error
         ChemicalSubstance chemical = builder.build();
         processor.prePersist(chemical);
         Property mwProp = chemical.properties.stream()

gsrs-module-substance-example/src/test/resources/application-test.conf

@@ -8,6 +8,7 @@ substance.renderer.configPath=substances-default-renderer.json
 approval_id_code_system="UNII from FDA"
 logging.level.root=error
 #logging.level.example.imports=trace
+#logging.level.example.imports=trace
 #logging.level.gsrs.imports=trace
 #logging.level.example.substance.processor=trace
 #logging.level.gsrs.module.substance.processors=trace
@@ -61,12 +62,15 @@ gsrs.standardizers.substances = {
     }
 }
 
-gsrs.importAdapterFactories.substances =
-	[
+
+# reconcile how this stuff is configured
+gsrs.importAdapterFactories.substances.list = null
+gsrs.importAdapterFactories.substances.list.SDFImportAdapterFactory =
 		{
 
 			"adapterName": "SDF Adapter",
 			"importAdapterFactoryClass": "gsrs.module.substance.importers.SDFImportAdapterFactory",
+			"order": 1000,
 			"stagingAreaServiceClass": "gsrs.stagingarea.service.DefaultStagingAreaService",
 			"entityServiceClass" :"gsrs.dataexchange.SubstanceStagingAreaEntityService",
 			"description" : "SD file importer for general users",
@@ -303,7 +307,8 @@ gsrs.importAdapterFactories.substances =
 					}
 				]
 			}
-		},
+		}
+gsrs.importAdapterFactories.substances.list.DelimTextImportAdapterFactory =
 		{
 			"adapterName": "Delimited Text Adapter",
 			"importAdapterFactoryClass": "gsrs.module.substance.importers.DelimTextImportAdapterFactory",
@@ -436,7 +441,8 @@ gsrs.importAdapterFactories.substances =
 				],
 				"lineValueDelimiter": ","
 			}
-		},
+		}
+gsrs.importAdapterFactories.substances.list.ExcelFileImportAdapterFactory =
 		{
 			"adapterName": "Excel File Adapter",
 			"importAdapterFactoryClass": "gsrs.module.substance.importers.ExcelFileImportAdapterFactory",
@@ -567,7 +573,8 @@ gsrs.importAdapterFactories.substances =
 				],
 				"lineValueDelimiter": ","
 			}
-		},
+		}
+gsrs.importAdapterFactories.substances.list.GSRSJSONImportAdapterFactory =
 		{
 			"adapterName": "GSRS JSON Adapter",
 			"importAdapterFactoryClass": "gsrs.module.substance.importers.GSRSJSONImportAdapterFactory",
@@ -584,6 +591,4 @@ gsrs.importAdapterFactories.substances =
 
 			}
 		}
-	]
-
-gsrs.substance.structures.saltFilePath=salt_data_public.tsv
+gsrs.substance.structures.saltFilePath=salt_data_public.tsv

gsrs-module-substances-api/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>gsrs-module-substances</artifactId>
         <groupId>gov.nih.ncats</groupId>
-        <version>3.1.1.1</version>
+        <version>3.1.2-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

gsrs-module-substances-core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>gsrs-module-substances</artifactId>
         <groupId>gov.nih.ncats</groupId>
-        <version>3.1.1.1</version>
+        <version>3.1.2-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

gsrs-module-substances-core/src/main/java/gsrs/module/substance/datasource/DefaultCodeSystemUrlGenerator.java

@@ -30,9 +30,10 @@ public class DefaultCodeSystemUrlGenerator implements DataSet<CodeSystemMeta>, C
        The JSON looks like this:
        [{"codeSystem":"XYZ","url":"http://some.url.com/$CODE$"},  {...}]
     2) Configure like so:
-    gsrs.entityProcessors +=  {
+    gsrs.entityProcessors.list.CodeProcessor = {
         "class":"ix.ginas.models.v1.Substance",
         "processor":"gsrs.module.substance.processors.CodeProcessor",
+        "order": 1000,
         "with": {
             "class":"gsrs.module.substance.datasource.DefaultCodeSystemUrlGenerator",
             "json":{

gsrs-module-substances-core/src/main/java/gsrs/module/substance/utils/ChemicalUtils.java

@@ -7,6 +7,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Assert;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.PostConstruct;
@@ -130,15 +131,23 @@ public class ChemicalUtils {
 
     @PostConstruct
     private void setUpSalts() {
-        log.warn("in setUpSalts, structureHandlingConfiguration.getSaltFilePath(): {}", structureHandlingConfiguration.getSaltFilePath());
+        if( structureHandlingConfiguration.getSaltFilePath() == null ) {
+            log.warn("ChemicalUtils - setUpSalts not initialized");
+            return;
+        }
+        log.trace("in setUpSalts, structureHandlingConfiguration.getSaltFilePath(): {}", structureHandlingConfiguration.getSaltFilePath());
         saltData = new HashMap<>();
         try {
-            File file = new File(structureHandlingConfiguration.getSaltFilePath());
-            Assert.assertTrue("input salt data file must exist!", file.exists());
+            String path = structureHandlingConfiguration.getSaltFilePath();
+            log.info("Trying to read salt file at path: " + path);
+            File file = new ClassPathResource(path).getFile();
+            Assert.assertTrue("input salt data file must exist! The path was: " + path, file.exists());
             List<String> lines = Files.readAllLines(file.toPath());
             for (String line : lines) {
-                String[] lineParts = line.split("\\t");
-                saltData.put(lineParts[0], lineParts[1]);
+                if (line.matches(".*\\w.*")) {
+                    String[] lineParts = line.split("\\t");
+                    saltData.put(lineParts[0], lineParts[1]);
+                }
             }
         }
         catch (Exception ex){