New Connectors and Feature: Folder Picker API

.gitignore

@@ -37,7 +37,7 @@ node_modules
 tmp/duui
 
 Monitoring/*
-.env
+DUUIWeb/.env
 
 temp/*
 DUUIRestService/config.properties

DUUIRestService/config.ini

@@ -1,12 +1,15 @@
 DBX_APP_KEY=
 DBX_APP_SECRET=
 DBX_REDIRECT_URL=
+GOOGLE_CLIENT_ID=
+GOOGLE_CLIENT_SECRET=
+GOOGLE_REDIRECT_URI=
 PORT=2605
 HOST=0.0.0.0
-FILE_UPLOAD_DIRECTORY=
+FILE_UPLOAD_DIRECTORY=/temp
 MONGO_HOST=
 MONGO_PORT=
-MONGO_DB=
-MONGO_USER=
-MONGO_PASSWORD=
-MONGO_DB_CONNECTION_STRING=
+MONGO_DB=DUUI
+MONGO_USER=testuser
+MONGO_PASSWORD=test
+MONGO_DB_CONNECTION_STRING=mongodb+srv://testuser:[email protected]/?retryWrites=true&w=majority&appName=DUUI

DUUIRestService/pom.xml

@@ -101,23 +101,23 @@
                 <!--generate types dynamically -->
                 <groupId>org.apache.uima</groupId>
                 <artifactId>jcasgen-maven-plugin</artifactId>
-                <version>2.9.0</version>
+                <version>3.5.0</version>
                 <configuration>
                     <limitToProject>true</limitToProject>
                     <typeSystemIncludes>
-                        <include>src/main/resources/org/texttechnologylab/types/reproducibleAnnotations.xml
-                        </include>
+                        <include>src/main/resources/org/texttechnologylab/types/*</include>
+                        <!--                        <include>src/main/resources/org/texttechnologylab/types/reproducibleAnnotations.xml</include>-->
                     </typeSystemIncludes>
                 </configuration>
-                <executions>
-                    <execution>
-                        <!--call it in the generate-source phase -->
-                        <phase>generate-sources</phase>
-                        <goals>
-                            <goal>generate</goal>
-                        </goals>
-                    </execution>
-                </executions>
+                <!--                <executions>-->
+                <!--                    <execution>-->
+                <!--                        &lt;!&ndash;call it in the generate-source phase &ndash;&gt;-->
+                <!--                        <phase>generate-sources</phase>-->
+                <!--                        <goals>-->
+                <!--                            <goal>generate</goal>-->
+                <!--                        </goals>-->
+                <!--                    </execution>-->
+                <!--                </executions>-->
             </plugin>
             <!--            <plugin>-->
             <!--                <groupId>org.codehaus.mojo</groupId>-->
@@ -184,42 +184,52 @@
         </dependency>
 
         <dependency>
-            <groupId>com.github.texttechnologylab</groupId>
+            <groupId>com.github.dterefe</groupId>
             <artifactId>DockerUnifiedUIMAInterface</artifactId>
-            <version>${duui.version}</version>
+            <version>5c71f6e7af57fb0273a214f69e58a00ed3e9211d</version>
         </dependency>
 
-        <dependency>
-            <groupId>de.tudarmstadt.ukp.dkpro.core</groupId>
-            <artifactId>de.tudarmstadt.ukp.dkpro.core.opennlp-asl</artifactId>
-            <version>1.10.0</version>
-        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.uima/uimaj-document-annotation -->
+        <!--        <dependency>-->
+        <!--            <groupId>org.apache.uima</groupId>-->
+        <!--            <artifactId>uimaj-document-annotation</artifactId>-->
+        <!--            <version>3.5.0</version>-->
+        <!--        </dependency>-->
 
-        <dependency>
-            <groupId>de.tudarmstadt.ukp.dkpro.core</groupId>
-            <artifactId>de.tudarmstadt.ukp.dkpro.core.api.metadata-asl</artifactId>
-            <version>1.10.0</version>
-        </dependency>
 
-        <!-- https://mvnrepository.com/artifact/de.tudarmstadt.ukp.dkpro.core/de.tudarmstadt.ukp.dkpro.core.io.text-asl -->
-        <dependency>
-            <groupId>de.tudarmstadt.ukp.dkpro.core</groupId>
-            <artifactId>de.tudarmstadt.ukp.dkpro.core.io.text-asl</artifactId>
-            <version>1.10.0</version>
-        </dependency>
+        <!--        <dependency>-->
+        <!--            <groupId>de.tudarmstadt.ukp.dkpro.core</groupId>-->
+        <!--            <artifactId>de.tudarmstadt.ukp.dkpro.core.opennlp-asl</artifactId>-->
+        <!--            <version>1.10.0</version>-->
+        <!--        </dependency>-->
+
+        <!--        <dependency>-->
+        <!--            <groupId>de.tudarmstadt.ukp.dkpro.core</groupId>-->
+        <!--            <artifactId>de.tudarmstadt.ukp.dkpro.core.api.metadata-asl</artifactId>-->
+        <!--            <version>1.10.0</version>-->
+        <!--        </dependency>-->
+
+        <!--         https://mvnrepository.com/artifact/de.tudarmstadt.ukp.dkpro.core/de.tudarmstadt.ukp.dkpro.core.io.text-asl -->
+        <!--        <dependency>-->
+        <!--            <groupId>de.tudarmstadt.ukp.dkpro.core</groupId>-->
+        <!--            <artifactId>de.tudarmstadt.ukp.dkpro.core.io.text-asl</artifactId>-->
+        <!--            <version>1.10.0</version>-->
+        <!--        </dependency>-->
         <!-- https://mvnrepository.com/artifact/de.tudarmstadt.ukp.dkpro.core/de.tudarmstadt.ukp.dkpro.core.tokit-asl -->
+        <!--        <dependency>-->
+        <!--            <groupId>de.tudarmstadt.ukp.dkpro.core</groupId>-->
+        <!--            <artifactId>de.tudarmstadt.ukp.dkpro.core.tokit-asl</artifactId>-->
+        <!--            <version>1.10.0</version>-->
+        <!--        </dependency>-->
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
         <dependency>
-            <groupId>de.tudarmstadt.ukp.dkpro.core</groupId>
-            <artifactId>de.tudarmstadt.ukp.dkpro.core.tokit-asl</artifactId>
-            <version>1.10.0</version>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.8</version>
         </dependency>
 
 
-        <!--        <dependency>-->
-        <!--            <groupId>org.texttechnologylab</groupId>-->
-        <!--            <artifactId>DockerUnifiedUIMAInterface</artifactId>-->
-        <!--            <version>1.0</version>-->
-        <!--        </dependency>-->
+
 
         <dependency>
             <groupId>org.xerial</groupId>
@@ -256,19 +266,23 @@
             <classifier>models-french</classifier>
         </dependency>
 
-        <dependency>
-            <groupId>com.sparkjava</groupId>
-            <artifactId>spark-core</artifactId>
-            <version>2.9.4</version>
-        </dependency>
-
-
         <dependency>
             <groupId>javax.servlet</groupId>
             <artifactId>javax.servlet-api</artifactId>
             <version>4.0.1</version> <!-- Use the appropriate version -->
         </dependency>
-    </dependencies>
+        <!-- https://mvnrepository.com/artifact/org.mortbay.jetty/servlet-api -->
+        <dependency>
+            <groupId>org.mortbay.jetty</groupId>
+            <artifactId>servlet-api</artifactId>
+            <version>3.0.20100224</version>
+        </dependency>
 
+        <dependency>
+            <groupId>com.sparkjava</groupId>
+            <artifactId>spark-core</artifactId>
+            <version>2.9.4</version>
+        </dependency>
+    </dependencies>
 
-</project>
+</project>

DUUIRestService/src/main/java/org/texttechnologylab/duui/analysis/document/DUUIDocumentProvider.java

@@ -96,7 +96,9 @@ public boolean isText() {
     public boolean isCloudProvider() {
         return provider.equalsIgnoreCase(Provider.DROPBOX)
             || provider.equalsIgnoreCase(Provider.MINIO)
-            || provider.equalsIgnoreCase(Provider.ONEDRIVE);
+            || provider.equalsIgnoreCase(Provider.ONEDRIVE)
+            || provider.equalsIgnoreCase(Provider.NEXTCLOUD)
+            || provider.equalsIgnoreCase(Provider.GOOGLE);
     }
 
     public boolean isDatabaseProvider() {

DUUIRestService/src/main/java/org/texttechnologylab/duui/analysis/document/Provider.java

@@ -14,4 +14,5 @@ public class Provider {
     public static final String MONGODB = "MongoDB";
     public static final String ONEDRIVE = "OneDrive";
     public static final String NEXTCLOUD = "NextCloud";
+    public static final String GOOGLE = "Google";
 }

DUUIRestService/src/main/java/org/texttechnologylab/duui/analysis/process/DUUISimpleProcessHandler.java

@@ -1,5 +1,6 @@
 package org.texttechnologylab.duui.analysis.process;
 
+import org.texttechnologylab.DockerUnifiedUIMAInterface.document_handler.*;
 import org.texttechnologylab.duui.api.controllers.documents.DUUIDocumentController;
 import org.texttechnologylab.duui.api.controllers.events.DUUIEventController;
 import org.texttechnologylab.duui.api.controllers.pipelines.DUUIPipelineController;
@@ -16,9 +17,6 @@
 import org.apache.uima.jcas.JCas;
 import org.bson.Document;
 import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.document_handler.DUUIDocument;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.document_handler.DUUIDropboxDocumentHandler;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.document_handler.IDUUIDocumentHandler;
 import org.texttechnologylab.DockerUnifiedUIMAInterface.io.reader.DUUIDocumentReader;
 import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
 import org.texttechnologylab.DockerUnifiedUIMAInterface.monitoring.DUUIEvent;
@@ -30,6 +28,7 @@
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
+import java.security.GeneralSecurityException;
 import java.util.List;
 import java.util.Set;
 import java.util.Vector;
@@ -145,7 +144,7 @@ public void startInput() {
 
         try {
             inputHandler = DUUIProcessController.getHandler(input.getProvider(), getUserID());
-        } catch (IllegalArgumentException | DbxException exception) {
+        } catch (IllegalArgumentException | DbxException | GeneralSecurityException | IOException  exception) {
             onException(exception);
         }
 
@@ -182,9 +181,16 @@ public void startInput() {
         }
 
         try {
-            collectionReader = new DUUIDocumentReader
-                .Builder(composer)
-                .withInputPath(input.getPath())
+            DUUIDocumentReader.Builder builder = new DUUIDocumentReader
+                    .Builder(composer);
+
+            if (inputHandler instanceof  IDUUIFolderPickerApi) {
+                builder.withInputPaths(List.of(input.getPath().split(",")));
+            } else {
+                builder.withInputPath(input.getPath());
+            }
+
+            collectionReader = builder
                 .withInputFileExtension(input.getFileExtension())
                 .withInputHandler(inputHandler)
                 .withOutputPath(output.getPath())

DUUIRestService/src/main/java/org/texttechnologylab/duui/api/Config.java

@@ -59,6 +59,12 @@ public String getDropboxRedirectUrl() {
         return properties.getProperty("DBX_REDIRECT_URL", null);
     }
 
+    public String getGoogleClientId() { return properties.getProperty("GOOGLE_CLIENT_ID", null); }
+
+    public String getGoogleClientSecret() { return properties.getProperty("GOOGLE_CLIENT_SECRET", null); }
+
+    public String getGoogleRedirectUri() { return properties.getProperty("GOOGLE_REDIRECT_URI", null); }
+
     public List<String> getAllowedOrigins() {
         return List.of(properties.getProperty("ALLOWED_ORIGINS", "").split(";"));
     }

DUUIRestService/src/main/java/org/texttechnologylab/duui/api/Main.java

@@ -26,6 +26,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
+import java.security.GeneralSecurityException;
 import java.util.Collection;
 import java.util.List;
 import java.util.UUID;
@@ -58,6 +59,9 @@ public static void main(String[] args) {
             System.err.println("\t> DBX_APP_KEY");
             System.err.println("\t> DBX_APP_SECRET");
             System.err.println("\t> DBX_REDIRECT_URL");
+            System.err.println("\t> GOOGLE_CLIENT_ID");
+            System.err.println("\t> GOOGLE_CLIENT_SECRET");
+            System.err.println("\t> GOOGLE_REDIRECT_URI");
             System.err.println("\t> PORT");
             System.err.println("\t> HOST");
             System.err.println("\t> FILE_UPLOAD_DIRECTORY");
@@ -116,7 +120,7 @@ public static void main(String[] args) {
      *
      * @return a JSON Document containing the path to parent folder (uuid).
      */
-    public static String uploadFile(Request request, Response response) throws ServletException, IOException, DbxException {
+    public static String uploadFile(Request request, Response response) throws ServletException, IOException, DbxException, GeneralSecurityException {
         String authorization = request.headers("Authorization");
         DUUIRequestHelper.authenticate(authorization);
 
@@ -188,7 +192,7 @@ public static String downloadFile(Request request, Response response) {
             response.raw().getOutputStream().write(file.readAllBytes());
             response.raw().getOutputStream().close();
             return "Download.";
-        } catch (DbxException | IOException e) {
+        } catch (DbxException | IOException | GeneralSecurityException e) {
             response.status(500);
             return "The file could not be downloaded.";
         }

DUUIRestService/src/main/java/org/texttechnologylab/duui/api/Methods.java

@@ -80,6 +80,8 @@ public static void init() {
                 get("/", DUUIUserController::authorizeUser);
                 get("/dropbox", DUUIUserController::getDropboxAppSettings);
                 put("/dropbox", DUUIUserController::finishDropboxOAuthFromCode);
+                get("/google", DUUIUserController::getGoogleSettings);
+                put("/google", DUUIUserController::finishGoogleOAuthFromCode);
             });
         });
 
@@ -130,6 +132,7 @@ public static void init() {
                     halt(401, "Unauthorized");
                 }
             });
+            get("/folderstructure/:id/:provider", DUUIProcessRequestHandler::getFolderStructure);
             get("/:id", DUUIProcessRequestHandler::findOne);
             get("", DUUIProcessRequestHandler::findMany);
             post("", DUUIProcessRequestHandler::start);

DUUIRestService/src/main/java/org/texttechnologylab/duui/api/controllers/pipelines/DUUIPipelineController.java

@@ -1,5 +1,9 @@
 package org.texttechnologylab.duui.api.controllers.pipelines;
 
+import org.apache.uima.cas.CASException;
+import org.apache.uima.fit.factory.JCasFactory;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
 import org.texttechnologylab.duui.api.controllers.components.DUUIComponentController;
 import org.texttechnologylab.duui.api.controllers.processes.DUUIProcessController;
 import org.texttechnologylab.duui.api.storage.DUUIMongoDBStorage;
@@ -461,7 +465,7 @@ public static void setupDrivers(DUUIComposer composer, Document pipeline) {
                 }
                 composer.addDriver(driver);
                 addedDrivers.add(name);
-            } catch (Exception e) {
+            } catch (Exception | LinkageError e) {
                 composer.addEvent(DUUIEvent.Sender.COMPOSER, e.getMessage(), DUUIComposer.DebugLevel.ERROR);
             }
         }

DUUIRestService/src/main/java/org/texttechnologylab/duui/api/controllers/processes/DUUIProcessController.java

@@ -1,5 +1,6 @@
 package org.texttechnologylab.duui.api.controllers.processes;
 
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import org.texttechnologylab.duui.api.Main;
 import org.texttechnologylab.duui.api.controllers.documents.DUUIDocumentController;
 import org.texttechnologylab.duui.api.controllers.events.DUUIEventController;
@@ -29,6 +30,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URISyntaxException;
+import java.security.GeneralSecurityException;
 import java.time.Instant;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -509,7 +511,7 @@ public static List<IDUUIProcessHandler> getActiveProcesses(String pipelineId) {
      * @return the created DocumentHandler.
      * @throws DbxException if incorrect credentials for Dropbox are provided.
      */
-    public static IDUUIDocumentHandler getHandler(String provider, String userId) throws DbxException {
+    public static IDUUIDocumentHandler getHandler(String provider, String userId) throws DbxException, GeneralSecurityException, IOException {
         Document user = DUUIUserController.getUserById(userId);
 
         if (provider.equalsIgnoreCase(Provider.DROPBOX)) {
@@ -534,6 +536,21 @@ public static IDUUIDocumentHandler getHandler(String provider, String userId) th
 
         } else if (provider.equalsIgnoreCase(Provider.FILE)) {
             return new DUUILocalDocumentHandler();
+        } else if (provider.equalsIgnoreCase(Provider.NEXTCLOUD)) {
+            Document credentials = DUUIUserController.getNextCloudCredentials(user);
+            System.out.println("URI: " + credentials.getString("uri"));
+            System.out.println("USERNAME: " + credentials.getString("username"));
+            System.out.println("PASSWORD: " + credentials.getString("password"));
+            return new DUUINextcloudDocumentHandler(
+                        credentials.getString("uri"),
+                        credentials.getString("username"),
+                        credentials.getString("password"));
+        } else if (provider.equalsIgnoreCase(Provider.GOOGLE)) {
+            Document credentials = DUUIUserController.getGoogleCredentials(user);
+            GoogleCredential credential = new GoogleCredential()
+                    .setAccessToken(credentials.getString("access_token"));
+
+            return new DUUIGoogleDriveDocumentHandler(credential);
         }
 
         return null;