Merge branch 'release/0.14.2'

CHANGELOG

@@ -1,3 +1,19 @@
+2015-05-14 v0.14.2
+ * Summary: Preprocessing has been improved to use the rewritten raw CSV file after processing it. This relocates some of the XML processing from GetEMRData.
+
+ * Update about.rst (Buck at UF)
+ * updated Windows install instructions to simplify the procedure (Nicholas Rejack)
+ * added expanded test for TestGetDBPath (Nicholas Rejack)
+ * updated TestSuite to run new test (Nicholas Rejack)
+ * fixed function name in comment in TestCreateImportDataJson.py (Nicholas Rejack)
+ * added new test for TestGetDBPath to increase coverage. (Nicholas Rejack)
+ * fixing version number in redi.py (Nicholas Rejack)
+ * fixed broken tests after relocating processing from GetEmrData (Nicholas Rejack)
+ * made csv to xml preprocessing dependent on getting the EMR file (Nicholas Rejack)
+ * moved XML conversion of raw file out of GetEmrData. (Nicholas Rejack)
+ * `make test` runs all tests (Andrei Sura)
+ * Update README.md (Christopher Barnes)
+
 2015-04-23 v0.14.1
  * Summary: PreProc and PostProc Hooks! RED-I has added the ability to hook scripts for post processing and preprocessing along with other minor fixes. These features now allow for the writing of filters and the emailing of log messages, for example, at the begining and end of a RED-I run.
 

Makefile

@@ -24,6 +24,9 @@ help:
 test: tests
 tests: coverage
 	[ ! -d config/rules ] || python -munittest discover config/rules
+	[ ! -d config/preproc ] || python -munittest discover config/preproc
+	[ ! -d config-example/rules ] || python -munittest discover config-example/rules
+	[ ! -d configexample/preproc ] || python -munittest discover config-example/preproc
 
 coverage:
 	ARCHFLAGS=$(ARCHFLAGS) python setup.py nosetests

config-example/rules/cancel.py

@@ -26,7 +26,7 @@
 #
 ######################################################
 
-from form import Form
+from redi.form import Form
 import re
 
 

config-example/rules/test_cancel.py

@@ -21,7 +21,7 @@
 from lxml import etree
 
 import cancel
-from form import Form
+from redi.form import Form
 
 FIELD_NAME = "test_field_name"
 TEST_FIELD_VALUE1 = "CANCELLED"

docs/about.rst

@@ -67,7 +67,7 @@ version control installed on your system.
       $ make && make install
    </pre>
 
-Once you are done with testing RED-I an you are satisfied with the
+Once you are done with testing RED-I and you are satisfied with the
 results you can remove the virtualenv artifacts and install the RED-I
 package to be available system-wide.
 
@@ -104,51 +104,45 @@ To uninstall the application:
 Installing RED-I on Windows
 ----------------------------
 
-1. Install Python 2.7.x from: https://www.python.org/downloads/windows/
-2. Install a git client for Windows https://windows.github.com/
-3. Run Git Shell icon
-4. Clone the RED-I repository
-
-.. raw:: html
-
-   <pre style="padding: 1em; background: #000; color: #fff; font: normal 1em Courier, Andale Mono">
-      git clone https://github.com/ctsit/redi.git
-   </pre>
-
-5. Install Powershell 4 for Windows:
-   http://www.microsoft.com/en-us/download/confirmation.aspx?id=40855
-6. Reboot
-7. Install Visual C++ 9:
-   http://www.microsoft.com/en-us/download/details.aspx?id=44266
-8. Install SetupTools for Windows https://pypi.python.org/pypi/setuptools/12.2
-9. Launch PowerShell as administrator
-10. Run this command:
-
-.. raw:: html
-
-   <pre style="padding: 1em; background: #000; color: #fff; font: normal 1em Courier, Andale Mono">
-      (Invoke-WebRequest https://bootstrap.pypa.io/ez_setup.py).Content | python
-   </pre>
-
-11. Launch Git Shell icon
-12. In the redi/ directory, run:
-
-.. raw:: html
-
-   <pre style="padding: 1em; background: #000; color: #fff; font: normal 1em Courier, Andale Mono">
-      python setup.py bdist_egg
-      cd c:\python27\scripts\
-      .\easy_install.exe C:\Users\user1\Documents\code\redi\dist\redi_py-0.13.2-py2.7.egg
-   </pre>
-
-13. Add Python scripts directory to your system path by issuing the following command:
-
-.. raw:: html
-
-   <pre style="padding: 1em; background: #000; color: #fff; font: normal 1em Courier, Andale Mono">
-      set path=%PATH%;c:\python27\scripts
-   </pre>
-
+* Open a command prompt by clicking on the Start menu, and typing "cmd" in the Run box.
+* Install 64-bit Python 2.7.9 by running the following command in the command prompt:
+<pre>
+msiexec /i https://www.python.org/ftp/python/2.7.9/python-2.7.9.amd64.msi 
+</pre>
+* Next you need to be insure the command interpreter will be able to find the Python modules. Set
+the paths to the modules by running the following commands in the command prompt:
+<pre>
+setx path "%path%;c:\python27"
+setx path "%path%;c:\python27\lib\site-packages"
+setx path "%path%;c:\python27\scripts”
+</pre>
+* Make a new directory for the RED-I files by running the following command in the command prompt:
+<pre>
+mkdir c:\redi
+</pre>
+* Download the RED-I source code from: [https://github.com/ctsit/redi/archive/0.14.1.zip]
+* Copy the contents of the RED-I zip file from c:\Users\%username%\Downloads\redi-0.14.1\redi-0.14.1 to c:\redi
+* Download the easy_install setup file from: https://bootstrap.pypa.io/ez_setup.py 
+* Run the easy_install setup file with the following command in the command prompt:
+<pre>
+python c:\Users\%username%\Downloads\ez_setup.py
+</pre>
+Note: you may need to modify the path to the ez_setup.py file if it is downloaded to a different location.
+
+* Next, make a binary install of RED-I by running the following commands in the command prompt:
+<pre>
+cd c:\redi
+python c:\redi\setup.py bdist_egg
+</pre>
+* You will need to manually install the pycrypto dependency. To avoid having to compile it with VCForPython you can
+download a pre-compiled binary and install it with the following command:
+<pre>
+c:\python27\scripts\easy_install http://www.voidspace.org.uk/python/pycrypto-2.6.1/pycrypto-2.6.1.win-amd64-py2.7.exe
+</pre>
+* Finally, install your binary of RED-I with the following command:
+<pre>
+c:\python27\scripts\easy_install.exe c:\redi\dist\redi-0.14.1-py2.7.egg
+</pre>   
 
 How to Test RED-I with a Sample Project
 ---------------------------------------

redi/redi.py

@@ -47,7 +47,7 @@
 """
 
 __author__ = "University of Florida CTS-IT Team"
-__version__ = "0.13.2"
+__version__ = "0.14.2"
 __email__ = "[email protected]"
 __status__ = "Development"
 
@@ -309,6 +309,25 @@ def _run(config_file, configuration_directory, do_keep_gen_files, dry_run,
     # load custom post-processing rules
     rules = load_rules(settings.rules, configuration_directory)
 
+    errors = run_preproc(pre_filters, settings)
+    map(logger.warning, errors)
+
+    raw_txt_file = os.path.join(configuration_directory, 'raw.txt')
+    escaped_file = os.path.join(configuration_directory, 'rawEscaped.txt')
+    raw_xml_file = os.path.join(configuration_directory, 'raw.xml')
+
+    # TODO: make this able to run against a local file if desired
+    # replace certain characters with escape sequences
+    if get_emr_data:
+        GetEmrData.data_preprocessing(raw_txt_file, escaped_file)
+    if get_emr_data:
+        # run csv2xml.py to generate data in xml format
+        GetEmrData.generate_xml(escaped_file, raw_xml_file)
+    if get_emr_data:
+        # delete rawEscaped.txt
+        GetEmrData.cleanup(escaped_file)
+
+
     raw_xml_file = os.path.join(configuration_directory, settings.raw_xml_file)
     email_settings = get_email_settings(settings)
     db_path = database_path
@@ -326,8 +345,6 @@ def _run(config_file, configuration_directory, do_keep_gen_files, dry_run,
     if not resume:
         _delete_last_runs_data(data_folder)
 
-        errors = run_preproc(pre_filters, settings)
-        map(logger.warning, errors)
         # TODO: Add preproc errors to report
 
         alert_summary, person_form_event_tree_with_data, rule_errors, \

redi/utils/GetEmrData.py

@@ -162,18 +162,6 @@ def get_emr_data(conf_dir, connection_details):
     :connection_details EmrFileAccessDetails object
     """
     raw_txt_file = os.path.join(conf_dir, 'raw.txt')
-    escaped_file = os.path.join(conf_dir, 'rawEscaped.txt')
-    raw_xml_file = os.path.join(conf_dir, 'raw.xml')
 
     # download csv file
     download_file(raw_txt_file, connection_details)
-
-    # replace certain characters with escape sequences
-    data_preprocessing(raw_txt_file, escaped_file)
-
-    # run csv2xml.py to generate data in xml format
-    generate_xml(escaped_file, raw_xml_file)
-
-    # delete rawEscaped.txt
-    cleanup(escaped_file)
-

setup.py

@@ -18,7 +18,7 @@
 
 setup(
     name='redi',
-    version='0.14.1',
+    version='0.14.2',
     author='https://www.ctsi.ufl.edu/research/study-development/informatics-consulting/',
     author_email='[email protected]',
     packages=find_packages(exclude=['test']),

test/TestCreateImportDataJson.py

@@ -16,7 +16,7 @@
 # For full text of the BSD 3-Clause License see http://opensource.org/licenses/BSD-3-Clause
 
 """
-TestCreateEavOutput.py:
+TestCreateImportDataJson.py:
 
    Verifies the correct functionality 
    of the `test_create_import_data_json` function

test/TestGetDBPath.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+# Contributors:
+# Christopher P. Barnes <[email protected]>
+# Andrei Sura: github.com/indera
+# Mohan Das Katragadda <[email protected]>
+# Philip Chase <[email protected]>
+# Ruchi Vivek Desai <[email protected]>
+# Taeber Rapczak <[email protected]>
+# Nicholas Rejack <[email protected]>
+# Josh Hanna <[email protected]>
+# Copyright (c) 2014-2015, University of Florida
+# All rights reserved.
+#
+# Distributed under the BSD 3-Clause License
+# For full text of the BSD 3-Clause License see http://opensource.org/licenses/BSD-3-Clause
+
+import unittest
+import os
+import tempfile
+from redi import redi
+import shutil
+
+
+class TestGetDBPath(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def test_get_db_path(self):
+        db_path = tempfile.mkdtemp()
+        batch_info_database = "database.db"
+        redi.get_db_path(batch_info_database, db_path)
+        full_db_path = os.path.join(batch_info_database, db_path)
+        assert os.path.exists(full_db_path) == 1
+        shutil.rmtree(full_db_path)
+
+    def test_get_db_path_not_exists(self):
+        # verify that db path is created if it doesn't exist
+        db_path =  "nonExistentPath"
+        batch_info_database = "database.db"
+        created_db_path = redi.get_db_path(batch_info_database, db_path)
+        assert created_db_path ==  os.path.join(db_path, batch_info_database)
+        shutil.rmtree(db_path)
+
+if __name__ == '__main__':
+    unittest.main()

test/TestGetEMRData.py

@@ -51,6 +51,7 @@ def test_get_emr_data(self):
         Note: This test is not concerned with testing the sftp communication"""
         temp_folder = tempfile.mkdtemp('/')
         temp_txt_file = os.path.join(temp_folder, "raw.txt")
+        temp_escaped_file = os.path.join(temp_folder, "rawEscaped.txt")
         temp_xml_file = os.path.join(temp_folder, "raw.xml")
 
         input_string = '''"NAME","COMPONENT_ID","RESULT","REFERENCE_UNIT","DATE_TIME_STAMP","STUDY_ID"
@@ -72,6 +73,8 @@ def test_get_emr_data(self):
             )
 
         GetEmrData.get_emr_data(temp_folder, props)
+        GetEmrData.data_preprocessing(temp_txt_file, temp_escaped_file)
+        GetEmrData.generate_xml(temp_escaped_file, temp_xml_file)
 
         with open(temp_xml_file) as f:
             result = f.read()

test/TestSuite.py

@@ -63,6 +63,7 @@
 from TestRediEmail import TestRediEmail
 from TestSentEventIndex import TestSentEventIndex
 from TestSendDatatoRedcap import TestSendDatatoRedcap
+from TestGetDBPath import TestGetDBPath
 
 class redi_suite(unittest.TestSuite):
 
@@ -109,6 +110,8 @@ def suite(self):
         redi_test_suite.addTest(TestDaysSinceToday)
         redi_test_suite.addTest(TestSendDatatoRedcap)
 
+        redi_test_suite.addTest(TestGetDBPath)
+
         # return the suite
         return unittest.TestSuite([redi_test_suite])