Merge branch 'release/0.14.1'

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 *.pyo
 *.egg-info/
+.coveralls.yml
 config-example/vagrant-data/redcap.zip
 formData.xml
 rawData.xml

.travis.yml

@@ -7,12 +7,15 @@ install:
   - pip install requests
   - pip install lxml
   - pip install sftpserver
+  - pip install coveralls
 
 script: make test
 
+after_success:
+    - coveralls
+
+
 branches:
   only:
     - master
     - develop
-
-

CHANGELOG

@@ -1,3 +1,43 @@
+2015-04-23 v0.14.1
+ * Summary: PreProc and PostProc Hooks! RED-I has added the ability to hook scripts for post processing and preprocessing along with other minor fixes. These features now allow for the writing of filters and the emailing of log messages, for example, at the begining and end of a RED-I run.
+
+ * removed project-specific code from preproc.py (Nicholas Rejack)
+ * removing project-specific preprocessing test file (Nicholas Rejack)
+ * removed line from settings.ini. REDI-I runs out of the box without any preprocessors required (Nich
+ * removing test-scenarios bash script (Nicholas Rejack)
+ * removed scenario data files. These were for project-specific testing. (Nicholas Rejack)
+ * removed non-informative comments in redi.py (Nicholas Rejack)
+ * removed renamed config dir (Nicholas Rejack)
+ * renaming config to previous config-example (Nicholas Rejack)
+ * added scenario 9 to test scenarios. (Nicholas Rejack)
+ * added scenario 7. bash script now shows test number. (Nicholas Rejack)
+ * added scenario 6 (Nicholas Rejack)
+ * updated test-scenarios with scenario 6 : (Nicholas Rejack)
+ * Add Scenario 9 (Taeber Rapczak)
+ * Correct the date of a result in Scenario 5 (Taeber Rapczak)
+ * Add Scenario 6 (Taeber Rapczak)
+ * Add 5 test scenarios and test-scenarios.bash (Taeber Rapczak)
+ * Do not filter rows without a known "panel" (Taeber Rapczak)
+ * Correct spelling error (Ruchi Vivek Desai)
+ * Fix order of expected output in test_fetch_panels (Taeber Rapczak)
+ * Fix the numbering (Ruchi Vivek Desai)
+ * Add missing tags (Ruchi Vivek Desai)
+ * Replace hard-coded fetch_panels() (Taeber Rapczak)
+ * Renumber the steps (Ruchi Vivek Desai)
+ * Fix indentation briken by commit id b8431f26ec459daf1cc1efe9ebf7c4583dacc572 (Ruchi Vivek Desai)
+ * Fix indentation of Windows installation instructions (Ruchi Vivek Desai)
+ * Add instructions for installing RED-I on Windows (Ruchi Vivek Desai)
+ * Fix email address (Ruchi Vivek Desai)
+ * Minor code cleanup (Taeber Rapczak)
+ * Add more context for preprocessors to use (Taeber Rapczak)
+ * Implement filter_old_labs() (Taeber Rapczak)
+ * Implement filter_old_labs() (Taeber Rapczak)
+ * Implement group_rows_by_panel (Taeber Rapczak)
+ * fetch_consent_dates function now implemented and working. (Nicholas Rejack)
+ * Resolve preproc import issue (Taeber Rapczak)
+ * updated preproc.py. Only fails on NotImplementedError now. (Nicholas Rejack)
+ * updated preproc.py to fix variable names when fetching consent date. (Nicholas Rejack)
+
 2015-04-09 v0.14.0
  * Summary: RED-I has multiple functional and other improvements in this release: licensing has been added in each file. RED-I is now available from PyPI as 'redi'. Most importantly,
  when multiple lab results come in on the same day, RED-I now takes the first lab of the day, unless it has a value of 'canceled'. RED-I also auto-resumes in the case of a network timeout. Reporting has been improved with more information.

Makefile

@@ -24,9 +24,6 @@ help:
 test: tests
 tests: coverage
 	[ ! -d config/rules ] || python -munittest discover config/rules
-	rm -f .coverage
-	rm -rf cover/
-	rm -f coverage.xml nosetests.xml
 
 coverage:
 	ARCHFLAGS=$(ARCHFLAGS) python setup.py nosetests

README.md

@@ -4,6 +4,8 @@ RED-I Project
 [![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.10014.png ".")](http://dx.doi.org/10.5281/zenodo.10014)
 [![Travis CI](https://api.travis-ci.org/ctsit/redi.svg?branch=master)](https://api.travis-ci.org/ctsit/redi.svg?branch=master)
 [![Version](https://pypip.in/v/redi/badge.png)](https://pypip.in/v/redi/badge.png)
+[![Coverage Status Master](https://coveralls.io/repos/ctsit/redi/badge.svg?branch=master)](https://coveralls.io/r/ctsit/redi?branch=master)
+[![Coverage Status Develop](https://coveralls.io/repos/ctsit/redi/badge.svg?branch=develop)](https://coveralls.io/r/ctsit/redi?branch=develop)
 [![Downloads](https://pypip.in/d/redi/badge.png)](https://pypip.in/d/redi/badge.png)
 
 

config-example/postproc/postproc.py

@@ -0,0 +1 @@
+# add post-processing rules here

config-example/preproc/preproc.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+
+# Contributors:
+# Nicholas Rejack <[email protected]>
+# Kevin Hanson <[email protected]>
+# Copyright (c) 2014-2015, University of Florida
+# All rights reserved.
+#
+# Distributed under the BSD 3-Clause License
+# For full text of the BSD 3-Clause License see http://opensource.org/licenses/BSD-3-Clause
+
+# preprocessing rules go here.

config-example/settings.ini

@@ -154,6 +154,7 @@ emr_data_file = output.csv
 # This is an optional dictionary which indicates that
 # custom code needs to be run to validate the data
 rules = {}
+preprocessors = {}
 
 # Set to "True" to include "rules" processing errors
 # Optional parameter

docs/about.rst

@@ -101,6 +101,54 @@ To uninstall the application:
 
 .. seealso:: http://pip.readthedocs.org/en/latest/reference/pip.html
 
+Installing RED-I on Windows
+----------------------------
+
+1. Install Python 2.7.x from: https://www.python.org/downloads/windows/
+2. Install a git client for Windows https://windows.github.com/
+3. Run Git Shell icon
+4. Clone the RED-I repository
+
+.. raw:: html
+
+   <pre style="padding: 1em; background: #000; color: #fff; font: normal 1em Courier, Andale Mono">
+      git clone https://github.com/ctsit/redi.git
+   </pre>
+
+5. Install Powershell 4 for Windows:
+   http://www.microsoft.com/en-us/download/confirmation.aspx?id=40855
+6. Reboot
+7. Install Visual C++ 9:
+   http://www.microsoft.com/en-us/download/details.aspx?id=44266
+8. Install SetupTools for Windows https://pypi.python.org/pypi/setuptools/12.2
+9. Launch PowerShell as administrator
+10. Run this command:
+
+.. raw:: html
+
+   <pre style="padding: 1em; background: #000; color: #fff; font: normal 1em Courier, Andale Mono">
+      (Invoke-WebRequest https://bootstrap.pypa.io/ez_setup.py).Content | python
+   </pre>
+
+11. Launch Git Shell icon
+12. In the redi/ directory, run:
+
+.. raw:: html
+
+   <pre style="padding: 1em; background: #000; color: #fff; font: normal 1em Courier, Andale Mono">
+      python setup.py bdist_egg
+      cd c:\python27\scripts\
+      .\easy_install.exe C:\Users\user1\Documents\code\redi\dist\redi_py-0.13.2-py2.7.egg
+   </pre>
+
+13. Add Python scripts directory to your system path by issuing the following command:
+
+.. raw:: html
+
+   <pre style="padding: 1em; background: #000; color: #fff; font: normal 1em Courier, Andale Mono">
+      set path=%PATH%;c:\python27\scripts
+   </pre>
+
 
 How to Test RED-I with a Sample Project
 ---------------------------------------
@@ -162,7 +210,7 @@ details about all arguments supported in the command line.
 How to Get Support
 ------------------
 
-If you need any help with using RED-I please email us at cts-it-red@ctsi.ufl.edu
+If you need any help with using RED-I please email us at ctsit@ctsi.ufl.edu
 
 How to Contribute
 -----------------

redi/redi.py

@@ -9,6 +9,7 @@
 # Taeber Rapczak <[email protected]>
 # Nicholas Rejack <[email protected]>
 # Josh Hanna <[email protected]>
+# Kevin Hanson <[email protected]>
 # Copyright (c) 2014-2015, University of Florida
 # All rights reserved.
 #
@@ -125,6 +126,9 @@ def main():
 
     - write the Final ElementTree to EAV
     """
+
+
+    # TODO: UPDATE COMMENT HERE
     global _person_form_events_service
 
     # obtaining command line arguments for path to configuration directory
@@ -191,11 +195,14 @@ def main():
         report_courier = report.ReportFileWriter(os.path.join(output_files,
             settings.report_file_path2), logger)
 
+    # This is the run that loads the data
     _run(config_file, configuration_directory, do_keep_gen_files, dry_run,
          get_emr_data, settings, output_files, db_path, redcap_client,
          report_courier, report_creator, args['--resume'],
          args['--skip-blanks'], args['--bulk-send-blanks'])
 
+    # TODO: post processing will go here
+
 
 def get_db_path(batch_info_database, database_path):
     if not os.path.exists(database_path):
@@ -297,7 +304,8 @@ def _run(config_file, configuration_directory, do_keep_gen_files, dry_run,
             settings.emr_sftp_server_private_key_pass,
             )
         GetEmrData.get_emr_data(configuration_directory, connection_details)
-
+    # load custom pre-processing filters
+    pre_filters = load_preproc(settings.preprocessors, configuration_directory)
     # load custom post-processing rules
     rules = load_rules(settings.rules, configuration_directory)
 
@@ -318,6 +326,10 @@ def _run(config_file, configuration_directory, do_keep_gen_files, dry_run,
     if not resume:
         _delete_last_runs_data(data_folder)
 
+        errors = run_preproc(pre_filters, settings)
+        map(logger.warning, errors)
+        # TODO: Add preproc errors to report
+
         alert_summary, person_form_event_tree_with_data, rule_errors, \
         collection_date_summary_dict, bad_ids =\
         _create_person_form_event_tree_with_data(
@@ -1937,6 +1949,55 @@ def run_rules(data):
     return loaded_rules
 
 
+def load_preproc(preprocessors, root='./'):
+    """
+    Copied and modified version of load_rules function.
+    TODO: fix load_rules and load_prerules for better parallelism
+
+    """
+    if not preprocessors:
+        return {}
+
+    loaded = {}
+
+    for (preprocessor, path) in ast.literal_eval(preprocessors).iteritems():
+        module = None
+        if os.path.exists(path):
+            module = imp.load_source(preprocessor, path)
+        elif os.path.exists(os.path.join(root, path)):
+            module = imp.load_source(preprocessor, os.path.join(root, path))
+
+        assert module is not None
+        assert module.run_processing is not None
+
+        loaded[preprocessor] = module
+
+    logger.info("Loaded {} pre-processing script{}".format(
+        len(loaded), 's' if len(loaded) != 1 else 0))
+    return loaded
+
+
+def run_preproc(preprocessors, settings):
+    # TODO figure out if this creates a sub process or not
+    # TODO need to check for program exe3cution otherwise give error
+    logger.info("Running preprocessing rules")
+    errors = []
+
+    for (preprocessor, module) in preprocessors.iteritems():
+        try:
+            module.run_processing(settings, redi=sys.modules[__name__],
+                                  logger=logging)
+        except Exception as e:
+            message_format = 'Error processing rule "{0}". {1}'
+            if not hasattr(e, 'errors'):
+                errors.append(message_format.format(preprocessor, e.message))
+                continue
+            for error in e.errors:
+                errors.append(message_format.format(preprocessor, error))
+
+    return errors
+
+
 def run_rules(rules, person_form_event_tree_with_data):
     errors = []
 

redi/utils/SimpleConfigParser.py

@@ -123,6 +123,7 @@
     "sender_email": "[email protected]",
     "project": "DEFAULT_PROJECT",
     "rules": {},
+    "preprocessors": {},
     "batch_warning_days": 13,
     "rate_limiter_value_in_redcap": 600,
     "batch_info_database": "redi.db",

setup.py

@@ -18,7 +18,7 @@
 
 setup(
     name='redi',
-    version='0.14.0',
+    version='0.14.1',
     author='https://www.ctsi.ufl.edu/research/study-development/informatics-consulting/',
     author_email='[email protected]',
     packages=find_packages(exclude=['test']),
@@ -27,13 +27,13 @@
         'redi': ['utils/*.xsl', 'utils/*.xsd']
     },
     url='https://github.com/ctsit/redi',
-    download_url = 'https://github.com/ctsit/redi/releases/tag/0.14.0',
+    download_url = 'https://github.com/ctsit/redi/releases/tag/0.14.1',
     keywords = ['EMR', 'EHR', 'REDCap', 'Clinical Data'],
     license='BSD 3-Clause',
     description='REDCap Electronic Data Importer',
     long_description=open('README.md').read(),
     install_requires=[
-        "requests >= 2.2.1",
+        "requests >= 2.5.1",
         "lxml >= 3.3.5",
         "PyCap >= 1.0",
         "pysftp >= 0.2.8",