Merge branch 'release/0.11.2'

ctsit · Sep 10, 2014 · 5a7ddc8 · 5a7ddc8
2 parents 5627068 + 9f46224
commit 5a7ddc8
Show file tree

Hide file tree

Showing 9 changed files with 446 additions and 276 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,4 +1,21 @@
+2014-09-10 v0.11.2
+
+ * Summary: performance improvements to speed up run time.
+
+ * Closes #42 - "redcap_records.py lost the events and fields arguments" (Andrei Sura)
+ * Remove implied assert of a formImportedFieldName (Taeber Rapczak)
+ * Add test showing formImportedFieldName is required (Taeber Rapczak)
+ * Refactor TestCopyDataToPersonFormEventTree.py (Taeber Rapczak)
+ * additional fix for #39 (Ruchi Vivek Desai)
+ * fixes #39 (Ruchi Vivek Desai)
+ * Update README with --skip-blanks and --verbose (Taeber Rapczak)
+ * Increase pylint score (Taeber Rapczak)
+ * Add suport for skipping blanks to generate_output (Taeber Rapczak)
+ * Add --skip-blanks flag (Taeber Rapczak)
+ * Add unit test to check for --skip-blanks (Taeber Rapczak)
+
 2014-09-03 v0.11.1
+
  * Replace clinicalComponentId with loinc_code in config-example
  * BUG: Interpret `send_email` as bool - fixes #35
  * Add verify_ssl switch to tell PyCap when to verify the ssl certificate of the REDCap server

diff --git a/README.md b/README.md
@@ -162,6 +162,13 @@ Optional command-line arguments:
     By default, the data directory is assumed to be the current working directory.
     Using this switch, one can run multiple instances of redi simultaneously.
 
+ - -v, --verbose: increase verbosity of output
+
+    $ redi --verbose
+
+ - --skip-blanks: skip blank events when sending event data to RedCAP
+
+    $ redi --skip-blanks
 
 ## Testing
 

diff --git a/bin/redi.py b/bin/redi.py
@@ -148,7 +148,7 @@ def main():
          logger)
 
     _run(config_file, configuration_directory, do_keep_gen_files, dry_run,
-         get_emr_data, settings, output_files, db_path, args['resume'])
+         get_emr_data, settings, output_files, db_path, args['resume'], args['skip_blanks'])
 
 
 def _makedirs(data_folder):
@@ -205,7 +205,7 @@ def _save(obj, path):
 
 
 def _run(config_file, configuration_directory, do_keep_gen_files, dry_run,
-         get_emr_data, settings, data_folder, database_path, resume=False):
+         get_emr_data, settings, data_folder, database_path, resume=False, skip_blanks=False):
     global translational_table_tree
 
     assert _person_form_events_service is not None
@@ -274,7 +274,7 @@ def _run(config_file, configuration_directory, do_keep_gen_files, dry_run,
         # Use the new method to communicate with RedCAP
         report_data = redi_lib.generate_output(
             person_form_event_tree_with_data, redcap_settings, email_settings,
-            _person_form_events_service)
+            _person_form_events_service, skip_blanks)
         # write person_form_event_tree to file
         write_element_tree_to_file(person_form_event_tree_with_data,\
          os.path.join(data_folder, 'person_form_event_tree_with_data.xml'))
@@ -547,6 +547,13 @@ def parse_args(arguments=None):
         'input and output data which will help in running multiple'\
         ' simultaneous instances of redi for different projects')
 
+    parser.add_argument(
+        '--skip-blanks',
+        default=False,
+        action='store_true',
+        required=False,
+        help='skip blank events when sending event data to RedCAP')
+
     if arguments:
         parsed = parser.parse_args(arguments)
     else:
@@ -1051,10 +1058,7 @@ def update_event_name(data, lookup_data, undefined):
                 # print field_key
                 distinct_value[field_key] += 1
                 if distinct_value[field_key] > 1:
-                    multiple_values_alert.append(
-                        'Multiple values found for field ' +
-                        field_key)
-                    logger.warn("update_event_name: multiple values \
+                    logger.debug("update_event_name: multiple values \
                         found for field %s", field_key)
             else:
                 element_to_set.text = undefined
@@ -1626,7 +1630,6 @@ def copy_data_to_person_form_event_tree(
             fieldUnitsNameObject = subject.find("redcapFieldNameUnits")
             fieldUnitsValueObject = subject.find("REFERENCE_UNIT")
             formCompletedField = subject.find("formCompletedFieldName")
-            formImportedField = subject.find("formImportedFieldName")
 
             if study_id_object is None:
                 raise Exception('Missing required field STUDY_ID')
@@ -1737,25 +1740,29 @@ def copy_data_to_person_form_event_tree(
                     "']/../value")
                 completedFieldValue[0].text = form_event_root.xpath(
                     "form/name[.='" + formName + "']/../formCompletedFieldValue")[0].text
-                importedFieldValue = person_form_event_tree_root.xpath(
-                    "person/study_id[.='" +
-                    subject_id +
-                    "']/../all_form_events/form/name[.='" +
-                    formName +
-                    "']/../event/name[.='" +
-                    eventName +
-                    "']/../field/name[.='" +
-                    formImportedField.text +
-                    "']/../value")
-                importedFieldValue[0].text = form_event_root.xpath(
-                    "form/name[.='" + formName + "']/../formImportedFieldValue")[0].text
+
+                form_imported_field_name = subject.findtext("formImportedFieldName", default="")
+                imported_field_value = person_form_event_tree_root.xpath(
+                    "person/study_id[.='{subject_id}']/../"
+                    "all_form_events/form/name[.='{form_name}']/../event/"
+                    "name[.='{event_name}']/../field/"
+                    "name[.='{form_imported_field_name}']/../value".format(
+                        subject_id=subject_id,
+                        form_name=formName,
+                        event_name=eventName,
+                        form_imported_field_name=form_imported_field_name))
+
+                if imported_field_value:
+                    try:
+                        imported_field_value[0].text = form_event_root.xpath(
+                            "form/name[.='" + formName + "']/../formImportedFieldValue")[0].text
+                        assert imported_field_value[0].text
+                    except (IndexError, AssertionError):
+                        raise Exception('formImportedField not set properly in the person form event tree')
 
                 if not completedFieldValue[0].text:
                     raise Exception(
                         'formCompletedField not set properly in the person form event tree')
-                if not importedFieldValue[0].text:
-                    raise Exception(
-                        'formImportedField not set properly in the person form event tree')
 
     tree = etree.ElementTree(person_form_event_tree_root)
     return tree

diff --git a/bin/redi_lib.py b/bin/redi_lib.py
@@ -1,14 +1,12 @@
-#/usr/bin/env python
-
 """
 redi_lib.py
 
     Stores a collection of utility functions used by redi.py
 """
 
-__author__      = "University of Florida CTS-IT Team"
-__copyright__   = "Copyright 2014, University of Florida"
-__license__     = "BSD 2-Clause"
+__author__ = "University of Florida CTS-IT Team"
+__copyright__ = "Copyright 2014, University of Florida"
+__license__ = "BSD 2-Clause"
 
 import datetime
 import os
@@ -104,7 +102,7 @@ def get_child_text_safely(etree, ele):
 """
 
 
-def generate_output(person_tree, redcap_settings, email_settings, data_repository):
+def generate_output(person_tree, redcap_settings, email_settings, data_repository, skip_blanks=False):
     # redi.configure_logger(system_log_file_full_path)
 
     # the global dictionary to be returned
@@ -133,8 +131,9 @@ def generate_output(person_tree, redcap_settings, email_settings, data_repositor
 
     try:
         # Communication with redcap
-        redcapClientObject = redcapClient(
-            redcap_settings['redcap_uri'],redcap_settings['token'], redcap_settings['verify_ssl'])
+        redcapClientObject = redcapClient(redcap_settings['redcap_uri'],
+                                          redcap_settings['token'],
+                                          redcap_settings['verify_ssl'])
     except RequestException:
         redi_email.send_email_redcap_connection_error(email_settings)
         quit()
@@ -198,6 +197,12 @@ def generate_output(person_tree, redcap_settings, email_settings, data_repositor
                     json_data_dict = import_dict['json_data']
                     contains_data = import_dict['contains_data']
 
+                    # If we're skipping blanks and this event is blank, we
+                    # assume all following events are blank; therefore, break
+                    # out of this for-loop and move on to the next form.
+                    if skip_blanks and not contains_data:
+                        break
+
                     time_lapse_since_last_request = time.time(
                     ) - time_stamp_after_request
                     sleepTime = max(

diff --git a/bin/utils/redcap_records.py b/bin/utils/redcap_records.py
@@ -63,6 +63,17 @@ def main():
         dest='forms',
         default='',
         help='Specify a list of forms, separated by spaces, for which data should be returned.')
+    parser.add_argument(
+        '--fields',
+        dest='fields',
+        default='',
+        help='Specify a list of fields, separated by spaces, for which data should be returned.')
+    parser.add_argument(
+        '-e',
+        '--events',
+        dest='events',
+        default='',
+        help='Specify a list of events, separated by spaces, for which data should be returned.')
 
     # prepare the arguments we were given
     args = vars(parser.parse_args())
@@ -84,8 +95,12 @@ def main():
     # either we export data...
     if args['import_data'] == '':
         my_forms = args['forms'].split()
+        my_fields = args['fields'].split()
+        my_events = args['events'].split()
         data = project.export_records(
             forms=my_forms,
+            fields=my_fields,
+            events=my_events,
             format='csv',
             event_name='unique')
         print str(data)

diff --git a/bin/utils/report.xsl b/bin/utils/report.xsl
@@ -70,10 +70,10 @@
                 </table>
                 <br />
                 <!-- Alerts start here -->
-                <!-- <h3>Import Alerts</h3> -->
+                <h3>Import Alerts</h3>
                 <!-- check for not null -->
                 <!-- Commented out as the output is not fully de-identified-->
-                <!-- <xsl:if test="report/alerts/tooManyForms/eventAlert">
+                <xsl:if test="report/alerts/tooManyForms/eventAlert">
                     <table>
                         <thead>
                             <tr>
@@ -91,10 +91,10 @@
                         </tbody>
                     </table>
                     <br />
-                </xsl:if> -->
+                </xsl:if>
                 <!-- check for not null -->
                 <!-- Commented out as the output is not fully de-identified-->
-                <!-- <xsl:if test="report/alerts/tooManyValues/valuesAlert">
+                <xsl:if test="report/alerts/tooManyValues/valuesAlert">
                     <table>
                         <thead>
                             <tr>
@@ -111,7 +111,7 @@
                             </xsl:for-each>
                         </tbody>
                     </table>
-                </xsl:if>  -->
+                </xsl:if>
                 <br />
                 <h3>Subject Details</h3>
                 <table>