Cole/refactor jinja undefined (#66)

* Replace Jinja2 PackageLoader with FileSystemLoader

The PackageLoader doesn't work with a fairly recent version of Jinja2 (3.0.1)
and Python 3.9. Replacing with FileSystemLoader work with the older version and
the latest version.

* Fix undefined variable `amplicon_name` in report template

* Refactor logging Jinja2 undefined variable warnings

* Revert plot_11a update

* Update intedration test branch

* Update jinja to warn on undefined but not fail. Fix all undefined warnings

* Fix github integration tests ref

* One more undefined variable

---------

Co-authored-by: Cole Lyman <[email protected]>

CRISPResso2/CRISPRessoBatchCORE.py

@@ -170,7 +170,7 @@ def main():
         num_batches = batch_params.shape[0]
         if int(n_processes_for_batch/num_batches) > 1:
             args.n_processes = int(n_processes_for_batch/num_batches)
-        
+
         int_columns = ['default_min_aln_score', 'min_average_read_quality', 'min_single_bp_quality',
                        'min_bp_quality_or_N',
                        'quantification_window_size', 'quantification_window_center', 'exclude_bp_from_left',
@@ -909,7 +909,7 @@ def main():
                 report_name = _jp("CRISPResso2Batch_report.html")
             else:
                 report_name = OUTPUT_DIRECTORY+'.html'
-            CRISPRessoReport.make_batch_report_from_folder(report_name, crispresso2_info, OUTPUT_DIRECTORY, _ROOT)
+            CRISPRessoReport.make_batch_report_from_folder(report_name, crispresso2_info, OUTPUT_DIRECTORY, _ROOT, logger)
             crispresso2_info['running_info']['report_location'] = report_name
             crispresso2_info['running_info']['report_filename'] = os.path.basename(report_name)
 

CRISPResso2/CRISPRessoCORE.py

@@ -600,7 +600,7 @@ def process_fastq(fastq_filename, variantCache, ref_names, refs, args):
                 N_MODS_OUTSIDE_WINDOW += new_variant[match_name]['mods_outside_window']
                 if new_variant[match_name]['irregular_ends']:
                     N_READS_IRREGULAR_ENDS += 1
-                
+
 
     fastq_handle.close()
 
@@ -4967,7 +4967,7 @@ def get_scaffold_len(row, scaffold_start_loc, scaffold_seq):
                 report_name = _jp("CRISPResso2_report.html")
             else:
                 report_name = OUTPUT_DIRECTORY+'.html'
-            CRISPRessoReport.make_report(crispresso2_info, report_name, OUTPUT_DIRECTORY, _ROOT)
+            CRISPRessoReport.make_report(crispresso2_info, report_name, OUTPUT_DIRECTORY, _ROOT, logger)
             crispresso2_info['running_info']['report_location'] = report_name
             crispresso2_info['running_info']['report_filename'] = os.path.basename(report_name)
 

CRISPResso2/CRISPRessoCompareCORE.py

@@ -83,7 +83,7 @@ def main():
         '''
         compare_header = CRISPRessoShared.get_crispresso_header(description, compare_header)
         print(compare_header)
-        
+
         parser = CRISPRessoShared.getCRISPRessoArgParser("Compare", parser_title = 'CRISPRessoCompare Parameters')
 
         args = parser.parse_args()
@@ -429,7 +429,7 @@ def get_plot_title_with_ref_name(plotTitle, refName):
                 report_name = _jp("CRISPResso2Compare_report.html")
             else:
                 report_name = OUTPUT_DIRECTORY+'.html'
-            CRISPRessoReport.make_compare_report_from_folder(report_name, crispresso2_info, OUTPUT_DIRECTORY, _ROOT)
+            CRISPRessoReport.make_compare_report_from_folder(report_name, crispresso2_info, OUTPUT_DIRECTORY, _ROOT, logger)
             crispresso2_info['running_info']['report_location'] = report_name
             crispresso2_info['running_info']['report_filename'] = os.path.basename(report_name)
 

CRISPResso2/CRISPRessoMetaCORE.py

@@ -358,7 +358,7 @@ def main():
                 report_name = _jp("CRISPResso2Meta_report.html")
             else:
                 report_name = OUTPUT_DIRECTORY+'.html'
-            CRISPRessoReport.make_meta_report_from_folder(report_name, crispresso2_info, OUTPUT_DIRECTORY, _ROOT)
+            CRISPRessoReport.make_meta_report_from_folder(report_name, crispresso2_info, OUTPUT_DIRECTORY, _ROOT, logger)
             crispresso2_info['running_info']['report_location'] = report_name
             crispresso2_info['running_info']['report_filename'] = os.path.basename(report_name)
 

CRISPResso2/CRISPRessoPooledCORE.py

@@ -1593,7 +1593,7 @@ def default_sigpipe():
                 report_name = _jp("CRISPResso2Pooled_report.html")
             else:
                 report_name = OUTPUT_DIRECTORY+'.html'
-            CRISPRessoReport.make_pooled_report_from_folder(report_name, crispresso2_info, OUTPUT_DIRECTORY, _ROOT)
+            CRISPRessoReport.make_pooled_report_from_folder(report_name, crispresso2_info, OUTPUT_DIRECTORY, _ROOT, logger)
             crispresso2_info['running_info']['report_location'] = report_name
             crispresso2_info['running_info']['report_filename'] = os.path.basename(report_name)
 

CRISPResso2/CRISPRessoPooledWGSCompareCORE.py

@@ -367,6 +367,7 @@ def main():
                 'CRISPREssoPooledWGSCompare Report<br>{0} vs {1}'.format(
                     sample_1_name, sample_2_name,
                 ),
+                logger,
             )
             crispresso2_info['running_info']['report_location'] = report_name
             crispresso2_info['running_info']['report_filename'] = os.path.basename(report_name)

CRISPResso2/CRISPRessoReports/CRISPRessoReport.py

@@ -5,7 +5,7 @@
 '''
 
 import os
-from jinja2 import Environment, FileSystemLoader, ChoiceLoader
+from jinja2 import Environment, FileSystemLoader, ChoiceLoader, make_logging_undefined
 from jinja_partials import generate_render_partial, render_partial
 from CRISPResso2 import CRISPRessoShared
 
@@ -16,16 +16,21 @@
     C2PRO_INSTALLED = False
 
 
-def get_jinja_loader(root):
+def get_jinja_loader(root, logger):
+    UndefinedLogger = make_logging_undefined(logger=logger)
     if C2PRO_INSTALLED:
         return Environment(
             loader=ChoiceLoader([
                 FileSystemLoader(os.path.join(root, 'CRISPRessoReports', 'templates')),
                 FileSystemLoader(os.path.join(os.path.dirname(CRISPRessoPro.__file__), 'templates')),
             ]),
+            undefined=UndefinedLogger,
         )
     else:
-        return Environment(loader=FileSystemLoader(os.path.join(root, 'CRISPRessoReports', 'templates')))
+        return Environment(
+            loader=FileSystemLoader(os.path.join(root, 'CRISPRessoReports', 'templates')),
+            undefined=UndefinedLogger,
+        )
 
 
 def render_template(template_name, jinja2_env, **data):
@@ -172,7 +177,7 @@ def assemble_figs(run_data, crispresso_folder):
     return data
 
 
-def make_report(run_data, crispresso_report_file, crispresso_folder, _ROOT):
+def make_report(run_data, crispresso_report_file, crispresso_folder, _ROOT, logger):
     # dicts for each amplicon fig_names[amp_name] = [list of fig names]
     #                        fig_locs[amp_name][fig_name] = figure location
     #    print('crispresso_report file: ' + crispresso_report_file + ' crispresso_folder : ' + crispresso_folder + ' root: ' + _ROOT)
@@ -198,7 +203,7 @@ def make_report(run_data, crispresso_report_file, crispresso_folder, _ROOT):
         'nuc_quilt_names': data['nuc_quilt_names'],
     }
 
-    j2_env = get_jinja_loader(_ROOT)
+    j2_env = get_jinja_loader(_ROOT, logger)
 
     #    dest_dir = os.path.dirname(crispresso_report_file)
     #    shutil.copy2(os.path.join(_ROOT,'templates','CRISPResso_justcup.png'),dest_dir)
@@ -210,7 +215,7 @@ def make_report(run_data, crispresso_report_file, crispresso_folder, _ROOT):
         ))
 
 
-def make_batch_report_from_folder(crispressoBatch_report_file, crispresso2_info, batch_folder, _ROOT):
+def make_batch_report_from_folder(crispressoBatch_report_file, crispresso2_info, batch_folder, _ROOT, logger):
     batch_names = crispresso2_info['results']['completed_batch_arr']
     failed_runs = crispresso2_info['results']['failed_batch_arr']
     failed_runs_desc = crispresso2_info['results']['failed_batch_arr_desc']
@@ -336,6 +341,7 @@ def make_batch_report_from_folder(crispressoBatch_report_file, crispresso2_info,
         _ROOT,
         output_title,
         'batch',
+        logger,
         summary_plots={
             'names': summary_plot_names,
             'titles': summary_plot_titles,
@@ -352,41 +358,41 @@ def make_batch_report_from_folder(crispressoBatch_report_file, crispresso2_info,
     )
 
 
-def make_pooled_report_from_folder(crispresso_report_file, crispresso2_info, folder, _ROOT):
+def make_pooled_report_from_folder(crispresso_report_file, crispresso2_info, folder, _ROOT, logger):
     names_arr = crispresso2_info['results']['good_region_names']
     output_title = 'CRISPResso Pooled Output'
     if crispresso2_info['running_info']['args'].name != '':
         output_title += f"<br/>{crispresso2_info['running_info']['args'].name}"
-    make_multi_report_from_folder(crispresso2_info, names_arr, output_title, crispresso_report_file, folder, _ROOT, 'pooled')
+    make_multi_report_from_folder(crispresso2_info, names_arr, output_title, crispresso_report_file, folder, _ROOT, 'pooled', logger)
 
 
-def make_compare_report_from_folder(crispresso_report_file, crispresso2_info, folder, _ROOT):
+def make_compare_report_from_folder(crispresso_report_file, crispresso2_info, folder, _ROOT, logger):
     names_arr = []
     output_title = 'CRISPResso Compare Output'
     if crispresso2_info['running_info']['args'].name != '':
         output_title += "<br/>{crispresso2_info['running_info']['args'].name}"
-    make_multi_report_from_folder(crispresso2_info, names_arr, output_title, crispresso_report_file, folder, _ROOT, 'compare')
+    make_multi_report_from_folder(crispresso2_info, names_arr, output_title, crispresso_report_file, folder, _ROOT, 'compare', logger)
 
 
-def make_meta_report_from_folder(crispresso_report_file, crispresso2_info, folder, _ROOT):
+def make_meta_report_from_folder(crispresso_report_file, crispresso2_info, folder, _ROOT, logger):
     names_arr = crispresso2_info['meta_names_arr']
     input_names = crispresso2_info['meta_input_names']
     output_title = 'CRISPresso Meta Output'
     if crispresso2_info['running_info']['args'].name != '':
         output_title += "<br/>{crispresso2_info['running_info']['args'].name}"
-    make_multi_report_from_folder(crispresso2_info, names_arr, output_title, crispresso_report_file, folder, _ROOT, 'meta',
+    make_multi_report_from_folder(crispresso2_info, names_arr, output_title, crispresso_report_file, folder, _ROOT, 'meta', logger,
                                   display_names=input_names)
 
 
-def make_wgs_report_from_folder(crispresso_report_file, crispresso2_info, folder, _ROOT):
+def make_wgs_report_from_folder(crispresso_report_file, crispresso2_info, folder, _ROOT, logger):
     names_arr = crispresso2_info['results']['good_region_names']
     output_title = 'CRISPResso WGS Output'
     if crispresso2_info['running_info']['args'].name != '':
         output_title += "<br/>{crispresso2_info['running_info']['args'].name}"
-    make_multi_report_from_folder(crispresso2_info, names_arr, output_title, crispresso_report_file, folder, _ROOT, 'wgs')
+    make_multi_report_from_folder(crispresso2_info, names_arr, output_title, crispresso_report_file, folder, _ROOT, 'wgs', logger)
 
 
-def make_multi_report_from_folder(crispresso2_info, names_arr, report_name, crispresso_report_file, folder, _ROOT, crispresso_tool,
+def make_multi_report_from_folder(crispresso2_info, names_arr, report_name, crispresso_report_file, folder, _ROOT, crispresso_tool, logger,
                                   display_names=None):
     """
     Prepares information to make a report of multiple CRISPResso runs - like CRISPRessoWGS or CRISPRessoPooled
@@ -398,6 +404,7 @@ def make_multi_report_from_folder(crispresso2_info, names_arr, report_name, cris
     crispresso_report_file (string): path to write report to
     folder (string): folder containing crispresso runs
     _ROOT (string): location of crispresso assets (images, templates, etc)
+    logger (logging.Logger): logger to log messages to, mainly for undefined variables in Jinja2 templates
     display_names (dict): report_name->display_name; Titles to be shown for crispresso runs
         (if different from names_arr, e.g. if display_names have spaces or bad chars, they won't be the same as names_arr)
 
@@ -471,6 +478,7 @@ def make_multi_report_from_folder(crispresso2_info, names_arr, report_name, cris
         _ROOT,
         report_name,
         crispresso_tool,
+        logger,
         summary_plots={
             'names': summary_plot_names,
             'titles': summary_plot_titles,
@@ -490,6 +498,7 @@ def make_multi_report(
     _ROOT,
     report_name,
     crispresso_tool,
+    logger,
     window_nuc_pct_quilts=None,
     nuc_pct_quilts=None,
     window_nuc_conv_plots=None,
@@ -530,7 +539,7 @@ def fill_default(dictionary, key, default_type=list):
         if key not in dictionary:
             dictionary[key] = default_type()
 
-    j2_env = get_jinja_loader(_ROOT)
+    j2_env = get_jinja_loader(_ROOT, logger)
 
     j2_env.filters['dirname'] = dirname
     if crispresso_tool == 'batch':