Extract jinja_partials and fix CRISPRessoPooled fastp errors (#425)

* Updated README (#64) * Updating README to fix argument, email, and formatting * removing superfluous files * Add link to CRISPRessoPro, move CRISPRessoPro section to end, and fix JSON formatting * Remove link to CRISPRessoPro * Replace Docker badge with link to tags * Add bullet points to Guardrails section and improve formatting * Fix typo and removed colons from guardrails --------- Co-authored-by: Cole Lyman <[email protected]> * Extract jinja_partials (#65) * Extract jinja_partials code * Remove Plotly dependency from setup.py * Fix CRISPRessoPooled flash errors (#68) * Fix replacing flash intermediate files with fastp intermediate files This also moves where the files are added to `files_to_remove` up to near where they are created. * Update to run test branch with paired end Pooled test * Add pooled-paired-sim test to integration tests * Replace flash and trimmomatic with fastp and remove plotly from Github Actions environment * Change test branch back to master --------- Co-authored-by: Trevor Martin <[email protected]>
pinellolab · Apr 24, 2024 · d6011f2 · d6011f2
1 parent f4858a3
commit d6011f2
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 21 deletions.
diff --git a/.github/envs/test_env.yml b/.github/envs/test_env.yml
@@ -5,8 +5,7 @@ channels:
   - defaults
 dependencies:
   - pip
-  - trimmomatic
-  - flash
+  - fastp
   - numpy
   - cython
   - jinja2
@@ -15,4 +14,3 @@ dependencies:
   - scipy
   - matplotlib
   - pandas
-  - plotly=5.18.0
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -46,7 +46,7 @@ jobs:
       with:
         repository: edilytics/CRISPResso2_tests
         token: ${{ secrets.ACCESS_CRISPRESSO2_TESTS }}
-        # ref: '<BRANCH_NAME>' // Use this to specify a branch other than master
+        # ref: '<BRANCH-NAME>' # Use this to specify a branch other than master
 
     - name: Run Basic
       run: |
@@ -72,6 +72,11 @@ jobs:
       run: |
         make pooled test
 
+    - name: Run Pooled Paired Sim
+      if: success() || failure()
+      run: |
+        make pooled-paired-sim test
+
     - name: Run WGS
       if: success() || failure()
       run: |

diff --git a/CRISPResso2/CRISPRessoPooledCORE.py b/CRISPResso2/CRISPRessoPooledCORE.py
@@ -516,6 +516,10 @@ def main():
 
                 if fastp_status:
                     raise CRISPRessoShared.FastpException('FASTP failed to run, please check the log file.')
+
+                if not args.keep_intermediate:
+                    files_to_remove += [output_forward_filename]
+
                 info('Done!', {'percent_complete': 7})
 
             processed_output_filename = output_forward_filename
@@ -550,6 +554,9 @@ def main():
             if args.debug:
                 info('Fastp command: {0}'.format(fastp_cmd))
 
+            if not args.keep_intermediate:
+                files_to_remove += [processed_output_filename, not_combined_1_filename, not_combined_2_filename]
+
             if fastp_status:
                 raise CRISPRessoShared.FastpException('Fastp failed to run, please check the log file.')
             crispresso2_info['running_info']['fastp_command'] = fastp_cmd
@@ -566,6 +573,9 @@ def main():
                     info(f'Forced {num_reads_force_merged} read pairs together.')
                 processed_output_filename = new_output_filename
 
+                if not args.keep_intermediate:
+                    files_to_remove += [new_merged_filename, new_output_filename]
+
             info('Done!', {'percent_complete': 7})
 
         if can_finish_incomplete_run and 'count_input_reads' in crispresso2_info['running_info']['finished_steps']:
@@ -1554,20 +1564,6 @@ def default_sigpipe():
         if not args.keep_intermediate:
              info('Removing Intermediate files...')
 
-             if not args.aligned_pooled_bam:
-                if args.fastq_r2!='':
-                    files_to_remove+=[processed_output_filename, flash_hist_filename, flash_histogram_filename,\
-                                flash_not_combined_1_filename, flash_not_combined_2_filename]
-                    if args.force_merge_pairs:
-                        files_to_remove.append(new_merged_filename)
-                        files_to_remove.append(old_flashed_filename)
-                else:
-                    files_to_remove+=[processed_output_filename]
-
-             if args.trim_sequences and args.fastq_r2!='':
-                 files_to_remove+=[output_forward_paired_filename, output_reverse_paired_filename,\
-                                                   output_forward_unpaired_filename, output_reverse_unpaired_filename]
-
              if RUNNING_MODE=='ONLY_GENOME' or RUNNING_MODE=='AMPLICONS_AND_GENOME':
                  if args.aligned_pooled_bam is None:
                      files_to_remove+=[bam_filename_genome]

diff --git a/CRISPResso2/CRISPRessoReports/CRISPRessoReport.py b/CRISPResso2/CRISPRessoReports/CRISPRessoReport.py
@@ -6,7 +6,7 @@
 
 import os
 from jinja2 import Environment, FileSystemLoader, ChoiceLoader, make_logging_undefined
-from jinja_partials import generate_render_partial, render_partial
+from CRISPResso2.CRISPRessoReports.jinja_partials import generate_render_partial, render_partial
 from CRISPResso2 import CRISPRessoShared
 
 if CRISPRessoShared.is_C2Pro_installed():

diff --git a/CRISPResso2/CRISPRessoReports/jinja_partials.py b/CRISPResso2/CRISPRessoReports/jinja_partials.py
@@ -0,0 +1,46 @@
+'''
+This file is derived from https://github.com/mikeckennedy/jinja_partials and is subject to the following license:
+
+MIT License
+
+Copyright (c) 2021 Michael Kennedy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+'''
+
+from functools import partial
+
+from markupsafe import Markup
+
+
+def render_partial(template_name, renderer=None, markup=True, **data):
+    if renderer is None:
+        if flask is None:
+            raise PartialsException('No renderer specified')
+        else:
+            renderer = flask.render_template
+
+    if markup:
+        return Markup(renderer(template_name, **data))
+
+    return renderer(template_name, **data)
+
+
+def generate_render_partial(renderer, markup=True):
+    return partial(render_partial, renderer=renderer, markup=markup)
diff --git a/setup.py b/setup.py
@@ -89,10 +89,8 @@ def main():
               'matplotlib',  # '>=1.3.1,<=2.2.3',
               'seaborn', # '>0.7.1,<0.10',
               'jinja2',
-              'jinja_partials',
               'scipy',
               'numpy',
-              'plotly',
               ],
           cmdclass = command_classes,
           ext_modules = ext_modules