Feature/s3

.travis.yml

@@ -2,7 +2,19 @@ git:
   depth: false
 language: java
 jdk: oraclejdk11
-script: "./gradlew test"
+
+before_install:
+# Only install and configure AWS profiles for builds not triggered by PRS
+# as those builds will not have access to the necessary credentials
+  - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
+      scripts/install-aws-cli.sh;
+      scripts/configure-aws-profile.sh;
+    fi'
+
+script:
+    - 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then ./gradlew prTest; fi'
+    - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then ./gradlew test; fi'
+
 after_success:
-    - "./gradlew jacocoTestReport"
-    - bash <(curl -s https://codecov.io/bash)
+  - "./gradlew jacocoTestReport"
+  - bash <(curl -s https://codecov.io/bash)

build.gradle

@@ -25,12 +25,17 @@ repositories {
     // Use jcenter for resolving dependencies.
     // You can declare any Maven/Ivy/file repository here.
     jcenter()
+    mavenCentral()
 }
 
 dependencies {
     // This dependency is used by the application.
     implementation 'com.google.guava:guava:28.1-jre'
 
+    implementation platform('software.amazon.awssdk:bom:2.15.0')
+    implementation 'software.amazon.awssdk:aws-core'
+    implementation 'software.amazon.awssdk:s3:2.10.1'
+
     // Use TestNG framework, also requires calling test.useTestNG() below
     testImplementation 'org.testng:testng:7.0.0'
 
@@ -63,6 +68,17 @@ test {
     useTestNG()
 }
 
+task prTest(type: Test) {
+    testLogging {
+        showStandardStreams = false // make true for log output to console
+    }
+
+    // Use TestNG for unit tests
+    useTestNG() {
+        excludeGroups 'secure'
+    }
+}
+
 jacoco {
     toolVersion = '0.8.5'
     reportsDir = file("$buildDir/reports/jacoco")

scripts/configure-aws-profile.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+PROFILE=ga4gh-serviceaccount-basic
+
+aws configure set aws_access_key_id     $AWS_PUBLIC_KEY  --profile $PROFILE
+aws configure set aws_secret_access_key $AWS_SECRET_KEY  --profile $PROFILE

scripts/install-aws-cli.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip awscliv2.zip
+sudo ./aws/install

src/main/java/org/ga4gh/drs/App.java

@@ -7,6 +7,7 @@
 public class App {
 
     public static void main(String[] args) {
+        System.setProperty("org.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH", "true");
         SpringApplication.run(App.class, args);
     }
 }

src/main/java/org/ga4gh/drs/AppConfig.java

@@ -10,8 +10,10 @@
 import org.ga4gh.drs.configuration.DataSourceRegistry;
 import org.ga4gh.drs.configuration.DrsConfig;
 import org.ga4gh.drs.configuration.DrsConfigContainer;
+import org.ga4gh.drs.configuration.S3;
 import org.ga4gh.drs.configuration.ServerProps;
 import org.ga4gh.drs.constant.DataSourceDefaults;
+import org.ga4gh.drs.constant.S3Defaults;
 import org.ga4gh.drs.constant.ServerPropsDefaults;
 import org.ga4gh.drs.constant.ServiceInfoDefaults;
 import org.ga4gh.drs.model.ServiceInfo;
@@ -22,6 +24,7 @@
 import org.ga4gh.drs.utils.objectloader.DrsObjectLoaderFactory;
 import org.ga4gh.drs.utils.objectloader.FileDrsObjectLoader;
 import org.ga4gh.drs.utils.objectloader.HttpsDrsObjectLoader;
+import org.ga4gh.drs.utils.objectloader.S3DrsObjectLoader;
 import org.ga4gh.drs.utils.requesthandler.AccessRequestHandler;
 import org.ga4gh.drs.utils.requesthandler.FileStreamRequestHandler;
 import org.ga4gh.drs.utils.requesthandler.ObjectRequestHandler;
@@ -34,6 +37,8 @@
 import org.springframework.context.annotation.Scope;
 import org.springframework.web.context.annotation.RequestScope;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -89,6 +94,9 @@ public DrsConfigContainer defaultDrsConfigContainer(
         DataSourceRegistry dataSourceRegistry = drsConfigContainer.getDrsConfig().getDataSourceRegistry();
         dataSourceRegistry.setLocal(DataSourceDefaults.LOCAL);
         dataSourceRegistry.setS3(DataSourceDefaults.S3);
+
+        S3 s3 = drsConfigContainer.getDrsConfig().getS3();
+        s3.setRegion(S3Defaults.S3_REGION);
         return drsConfigContainer;
     }
 
@@ -179,7 +187,17 @@ public FileDrsObjectLoader fileDrsObjectLoader(String objectId, String objectPat
     @Bean
     @Scope(value = AppConfigConstants.PROTOTYPE)
     public HttpsDrsObjectLoader HttpsDrsObjectLoader(String objectId, String objectPath) {
-        return new HttpsDrsObjectLoader(objectId, objectPath);
+        try {
+            return new HttpsDrsObjectLoader(objectId, objectPath);
+        } catch (IllegalArgumentException e) {
+            return null;
+        }
+    }
+
+    @Bean
+    @Scope(value = AppConfigConstants.PROTOTYPE)
+    public S3DrsObjectLoader S3DrsObjectLoader(String objectId, String region, String bucket, String key, S3Client client) {
+        return new S3DrsObjectLoader(objectId, region, bucket, key, client);
     }
 
     /* ******************************
@@ -200,4 +218,10 @@ public DrsObjectLoaderFactory drsObjectLoaderFactory() {
     public AccessCache accessCache() {
         return new AccessCache();
     }
+
+    public Region defaultRegion(
+        @Qualifier(AppConfigConstants.MERGED_DRS_CONFIG_CONTAINER) DrsConfigContainer drsConfigContainer
+    ) {
+        return Region.of(drsConfigContainer.getDrsConfig().getS3().getRegion());
+    }
 }

src/main/java/org/ga4gh/drs/configuration/DrsConfig.java

@@ -1,5 +1,6 @@
 package org.ga4gh.drs.configuration;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import org.ga4gh.drs.model.ServiceInfo;
 
 public class DrsConfig {
@@ -8,10 +9,14 @@ public class DrsConfig {
     private ServiceInfo serviceInfo;
     private DataSourceRegistry dataSourceRegistry;
 
+    @JsonProperty("S3")
+    private S3 s3;
+
     public DrsConfig() {
         serverProps = new ServerProps();
         serviceInfo = new ServiceInfo();
         dataSourceRegistry = new DataSourceRegistry();
+        s3 = new S3();
     }
 
     public void setServerProps(ServerProps serverProps) {
@@ -37,4 +42,12 @@ public void setDataSourceRegistry(DataSourceRegistry dataSourceRegistry) {
     public DataSourceRegistry getDataSourceRegistry() {
         return dataSourceRegistry;
     }
+
+    public S3 getS3() {
+        return s3;
+    }
+
+    public void setS3(S3 S3) {
+        this.s3 = S3;
+    }
 }

src/main/java/org/ga4gh/drs/configuration/S3.java

@@ -0,0 +1,17 @@
+package org.ga4gh.drs.configuration;
+
+public class S3 {
+    private String region;
+
+    public S3() {
+
+    }
+
+    public String getRegion() {
+        return region;
+    }
+
+    public void setRegion(String region) {
+        this.region = region;
+    }
+}

src/main/java/org/ga4gh/drs/constant/DataSourceDefaults.java

@@ -20,10 +20,16 @@ public class DataSourceDefaults {
 
     public static final List<S3DataSource> S3 = Arrays.asList(
         new S3DataSource() {{
-            setBucket("ga4gh-demo-data");
+            setIdPrefix("S3DemoData.");
+            setRootDir("");
             setRegion("us-east-2");
-            setIdPrefix("S3DemoData");
-            setRootDir("/");
+            setBucket("ga4gh-demo-data");
+        }},
+        new S3DataSource() {{
+            setIdPrefix("TestTabulaMurisFacs.");
+            setRootDir("facs_bam_files/");
+            setRegion("us-east-1");
+            setBucket("czbiohub-tabula-muris");
         }}
     );
 }

src/main/java/org/ga4gh/drs/constant/S3Defaults.java

@@ -0,0 +1,6 @@
+package org.ga4gh.drs.constant;
+
+public class S3Defaults {
+
+    public static final String S3_REGION = "us-east-1";
+}

src/main/java/org/ga4gh/drs/constant/ServiceInfoDefaults.java

@@ -23,7 +23,7 @@ public class ServiceInfoDefaults {
 
     public static final LocalDateTime CREATED_AT = LocalDateTime.parse("2020-01-15T12:00:00Z", DATE_FORMATTER);
 
-    public static final LocalDateTime UPDATED_AT = LocalDateTime.parse("2020-01-15T12:00:00Z", DATE_FORMATTER);;
+    public static final LocalDateTime UPDATED_AT = LocalDateTime.parse("2020-01-15T12:00:00Z", DATE_FORMATTER);
 
     public static final String ENVIRONMENT = "test";
 

src/main/java/org/ga4gh/drs/controller/Objects.java

@@ -27,6 +27,7 @@ public DrsObject getObjectById(
         @RequestParam(name = "expand", required = false) boolean expand
     ) {
         objectRequestHandler.setObjectId(id);
+        objectRequestHandler.setExpand(expand);
         return objectRequestHandler.handleRequest();
     }
 

src/main/java/org/ga4gh/drs/model/DrsObject.java

@@ -9,6 +9,7 @@
 import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
 import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
 import org.springframework.lang.NonNull;
+
 import java.net.URI;
 import java.time.LocalDateTime;
 import java.util.List;
@@ -55,7 +56,7 @@ public class DrsObject {
     private String version;
 
     public DrsObject() {
-        
+
     }
 
     public DrsObject(String id, URI selfURI, List<Checksum> checksums, LocalDateTime createdTime, long size) {

src/main/java/org/ga4gh/drs/utils/DataSourceLookup.java

@@ -8,6 +8,7 @@
 import org.ga4gh.drs.utils.objectloader.DrsObjectLoaderFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+
 import javax.annotation.PostConstruct;
 import java.util.List;
 
@@ -34,7 +35,6 @@ private void postConstruct() {
     }
 
     public DrsObjectLoader getDrsObjectLoaderFromId(String objectId) {
-
         for (LocalFileDataSource localDataSource : localDataSources) {
             if (localDataSource.objectIdMatches(objectId)) {
                 return drsObjectLoaderFactory.createFileDrsObjectLoader(localDataSource, objectId);
@@ -46,7 +46,6 @@ public DrsObjectLoader getDrsObjectLoaderFromId(String objectId) {
                 return drsObjectLoaderFactory.createS3DrsObjectLoader(s3DataSource, objectId);
             }
         }
-
         return null;
     }
 }

src/main/java/org/ga4gh/drs/utils/DeepObjectMerger.java

@@ -2,7 +2,12 @@
 
 import java.lang.reflect.Field;
 import java.time.LocalDateTime;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
 
 public class DeepObjectMerger {
 

src/main/java/org/ga4gh/drs/utils/S3ClientProvider.java

@@ -0,0 +1,34 @@
+package org.ga4gh.drs.utils;
+
+import org.ga4gh.drs.utils.datasource.S3DataSource;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Utility class to create and cache expensive S3Clients configured according to
+ * region and profile of data source
+ */
+public class S3ClientProvider {
+
+    private static Map<S3DataSource, S3Client> clients = new HashMap<>();
+
+    public static S3Client getClient(S3DataSource source) {
+        return clients.computeIfAbsent(source, s -> {
+            Region region = Region.of(s.getRegion());
+            String profile = s.getProfile();
+            S3ClientBuilder builder = S3Client.builder().region(region);
+            if (profile != null) {
+                builder.credentialsProvider(ProfileCredentialsProvider.builder()
+                    .profileName(profile)
+                    .build()
+                );
+            }
+            return builder.build();
+        });
+    }
+}

src/main/java/org/ga4gh/drs/utils/datasource/S3DataSource.java

@@ -8,6 +8,32 @@ public class S3DataSource implements DataSource<S3DrsObjectLoader> {
     private String region;
     private String bucket;
     private String rootDir;
+    private String profile;
+
+    public S3DataSource() {
+
+    }
+
+    public S3DataSource(String idPrefix, String region, String bucket, String rootDir) {
+        this.idPrefix = idPrefix;
+        this.region = region;
+        this.bucket = bucket;
+        this.rootDir = rootDir;
+    }
+
+    public S3DataSource(String idPrefix, String region, String bucket, String rootDir, String profile) {
+        this.idPrefix = idPrefix;
+        this.region = region;
+        this.bucket = bucket;
+        this.rootDir = rootDir;
+        this.profile = profile;
+    }
+
+    public String renderObjectKey(String objectId) {
+        String postPrefix = objectId.replaceFirst(getIdPrefix(), "");
+        return getRootDir() + postPrefix;
+    }
+
 
     public void setIdPrefix(String idPrefix) {
         this.idPrefix = idPrefix;
@@ -44,4 +70,12 @@ public void setRootDir(String rootDir) {
     public String getRootDir() {
         return rootDir;
     }
+
+    public String getProfile() {
+        return profile;
+    }
+
+    public void setProfile(String profile) {
+        this.profile = profile;
+    }
 }