Merge pull request #271 from microbiomedata/245-Update-metat-output-ui

Update metatT output UI
microbiomedata · Jul 30, 2024 · a811b55 · a811b55
2 parents e6ec557 + 489c19c
commit a811b55
Show file tree

Hide file tree

Showing 7 changed files with 53 additions and 11 deletions.
diff --git a/webapp/client/src/pipelines/MetaT/Workflow/Results/Metatranscriptome.js b/webapp/client/src/pipelines/MetaT/Workflow/Results/Metatranscriptome.js
@@ -17,7 +17,7 @@ function Metatranscriptome(props) {
                 <Annotation title={'Annotation Result'} result={props.result['annotation-stats']} userType={props.type} allExpand={props.allExpand} allClosed={props.allClosed} />
             }
             {props.result['readMapping-features'] &&
-                <ReadMapping title={'Read Mapping Result'} features={props.result['readMapping-features']} userType={props.type} allExpand={props.allExpand} allClosed={props.allClosed} />
+                <ReadMapping title={'Read Mapping Result'} project={props.project} tooLarge={props.result['readMapping-features-too-large']} features={props.result['readMapping-features']} userType={props.type} allExpand={props.allExpand} allClosed={props.allClosed} />
             }
         </>
     );

diff --git a/webapp/client/src/pipelines/MetaT/Workflow/Results/ReadMapping.js b/webapp/client/src/pipelines/MetaT/Workflow/Results/ReadMapping.js
@@ -2,9 +2,11 @@ import React, { useEffect, useState } from 'react';
 import { Card, CardBody, Collapse } from 'reactstrap';
 import { Header } from '../../../Common/Results/CardHeader';
 import FeaturesTable from './FeaturesTable';
+import config from "../../../../config";
 
 function ReadMapping(props) {
   const [collapseCard, setCollapseCard] = useState(true);
+  const url = config.API.BASE_URI + "/projects/" + props.project.code + "/";
 
   const toggleCard = () => {
     setCollapseCard(!collapseCard);
@@ -27,10 +29,17 @@ function ReadMapping(props) {
       <Header toggle={true} toggleParms={toggleCard} title={props.title} collapseParms={collapseCard} />
       <Collapse isOpen={!collapseCard} >
         <CardBody>
-          {props.features && <>
-            <FeaturesTable data={props.features} />
-            <br></br>
-          </>}
+          {props.tooLarge ?
+            <>
+              The result is too large to display.
+              <br></br>
+              <a href={url + props.features} target="_blank" rel="noreferrer" >[ Export the result as TSV ]</a>
+              <br></br><br></br>
+            </>
+            : <>
+              <FeaturesTable data={props.features} />
+              <br></br>
+            </>}
         </CardBody>
       </Collapse>
     </Card>

diff --git a/webapp/server/config.js b/webapp/server/config.js
@@ -167,6 +167,8 @@ const config = {
         UPLOADED_FILES_DIR: process.env.UPLOADED_FILES_DIR || path.join(IO_BASE_DIR, "upload/files"),
         // Directory used by file uploading function.
         UPLOADED_FILES_TEMP_DIR: process.env.UPLOADED_FILES_TEMP_DIR || path.join(IO_BASE_DIR, "upload/tmp"),
+        // Maximum rows to pass to UI data table
+        MAX_DATATABLE_ROWS: process.env.MAX_DATATABLE_ROWS || 300000,
     },
     // Parameters that influence the behavior of `Winston.js`, a logging library.
     // Reference: https://github.com/winstonjs/winston-daily-rotate-file#options

diff --git a/webapp/server/config/workflow.js b/webapp/server/config/workflow.js
@@ -66,7 +66,7 @@ workflowlist = {
         full_name: 'Metatranscriptomics',
         wdl_tmpl: 'metaT_wdl.tmpl',
         inputs_tmpl: 'metaT_inputs.tmpl',
-        options_json: 'metaT_options.json',
+        options_json: 'metaG_options.tmpl',
         outdir: 'output/Metatranscriptomics',
         cromwell_calls: ['main_workflow.nmdc_metat'],
         wdl_version: 'draft-2'
@@ -78,7 +78,7 @@ workflowlist = {
         full_name: 'Natural Organic Matter',
         wdl_tmpl: 'enviroMS_wdl.tmpl',
         inputs_tmpl: 'enviroMS_inputs.tmpl',
-        options_json: 'enviroMS_options.json',
+        options_json: 'metaG_options.tmpl',
         outdir: 'output/NOM',
         cromwell_calls: ['main_workflow.enviroMS'],
         wdl_version: 'draft-2'

diff --git a/webapp/server/crons/workflowBigMemMonitor.js b/webapp/server/crons/workflowBigMemMonitor.js
@@ -144,7 +144,7 @@ async function generateOptions(proj_home, workflow) {
 
     const workflowSettings = workflowlist[workflow.name];
     const tmpl = path.join(config.WORKFLOWS.TEMPLATE_DIR, workflowSettings['options_json']);
-    if(!tmpl.existsSync) {
+    if(!fs.existsSync(tmpl)) {
         return true;
     }
     let templInputs = String(fs.readFileSync(tmpl));

diff --git a/webapp/server/crons/workflowMonitor.js b/webapp/server/crons/workflowMonitor.js
@@ -176,7 +176,7 @@ async function generateOptions(proj_home, workflow) {
 
     const workflowSettings = workflowlist[workflow.name];
     const tmpl = path.join(config.WORKFLOWS.TEMPLATE_DIR, workflowSettings['options_json']);
-    if(!tmpl.existsSync) {
+    if(!fs.existsSync(tmpl)) {
         return true;
     }
     let templInputs = String(fs.readFileSync(tmpl));

diff --git a/webapp/server/util/workflow.js b/webapp/server/util/workflow.js
@@ -1,5 +1,6 @@
 const fs = require('fs');
 const path = require("path");
+const execSync = require('child_process').execSync;
 const ufs = require("url-file-size");
 const moment = require('moment');
 const FormData = require('form-data');
@@ -196,10 +197,18 @@ const generateWorkflowResult = function (proj) {
                     }
                 });
             }
+            if (dir === 'readsQC') {
+                const files = fs.readdirSync(outdir + "/readsQC");
+                files.forEach(function (file) {
+                    if (file.endsWith("_stats.json")) {
+                        result['readsQC-stats'] = JSON.parse(fs.readFileSync(outdir + "/readsQC/" + file));
+                    }
+                });
+            }
             else if (dir === 'assembly') {
                 const files = fs.readdirSync(outdir + "/assembly");
                 files.forEach(function (file) {
-                    if (file === "stats.json") {
+                    if (file.endsWith("_stats.json")) {
                         result['assembly-stats'] = JSON.parse(fs.readFileSync(outdir + "/assembly/" + file));
                     }
                 });
@@ -216,7 +225,29 @@ const generateWorkflowResult = function (proj) {
                 const files = fs.readdirSync(outdir + "/metat_output");
                 files.forEach(function (file) {
                     if (file.endsWith("_sorted_features.tsv")) {
-                        result['readMapping-features'] = Papa.parse(fs.readFileSync(outdir + "/metat_output/" + file).toString(), { delimiter: '\t', header: true, skipEmptyLines: true }).data;
+                        var rows = parseInt(execSync("wc -l < " + outdir + "/metat_output/" + file).toString().trim());
+                        if (rows > config.IO.MAX_DATATABLE_ROWS) {
+                            result['readMapping-features-too-large'] = true;
+                            result['readMapping-features'] = "output/Metatranscriptomics/metat_output/" + file;
+                        } else {
+                            result['readMapping-features-too-large'] = false;
+                            result['readMapping-features'] = Papa.parse(fs.readFileSync(outdir + "/metat_output/" + file).toString(), { delimiter: '\t', header: true, skipEmptyLines: true }).data;
+                        }
+                    }
+                });
+            }
+            else if (dir === 'readMapping') {
+                const files = fs.readdirSync(outdir + "/readMapping");
+                files.forEach(function (file) {
+                    if (file.endsWith("_sorted_features.tsv")) {
+                        var rows = parseInt(execSync("wc -l < " + outdir + "/readMapping/" + file).toString().trim());
+                        if (rows > config.IO.MAX_DATATABLE_ROWS) {
+                            result['readMapping-features-too-large'] = true;
+                            result['readMapping-features'] = "output/Metatranscriptomics/readMapping/" + file;
+                        } else {
+                            result['readMapping-features-too-large'] = false;
+                            result['readMapping-features'] = Papa.parse(fs.readFileSync(outdir + "/readMapping/" + file).toString(), { delimiter: '\t', header: true, skipEmptyLines: true }).data;
+                        }
                     }
                 });
             }