From 41dd3a5cb368c74a8759c54c397ff8a3a6d088b6 Mon Sep 17 00:00:00 2001 From: Andrei Sura Date: Thu, 23 Apr 2015 15:11:51 -0400 Subject: [PATCH 01/12] `make test` runs all tests --- Makefile | 3 +++ config-example/rules/cancel.py | 2 +- config-example/rules/test_cancel.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 174c17c..1b692b7 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ help: test: tests tests: coverage [ ! -d config/rules ] || python -munittest discover config/rules + [ ! -d config/preproc ] || python -munittest discover config/preproc + [ ! -d config-example/rules ] || python -munittest discover config-example/rules + [ ! -d configexample/preproc ] || python -munittest discover config-example/preproc coverage: ARCHFLAGS=$(ARCHFLAGS) python setup.py nosetests diff --git a/config-example/rules/cancel.py b/config-example/rules/cancel.py index 78e488b..73d0cbf 100644 --- a/config-example/rules/cancel.py +++ b/config-example/rules/cancel.py @@ -26,7 +26,7 @@ # ###################################################### -from form import Form +from redi.form import Form import re diff --git a/config-example/rules/test_cancel.py b/config-example/rules/test_cancel.py index be7bc12..3926dda 100644 --- a/config-example/rules/test_cancel.py +++ b/config-example/rules/test_cancel.py @@ -21,7 +21,7 @@ from lxml import etree import cancel -from form import Form +from redi.form import Form FIELD_NAME = "test_field_name" TEST_FIELD_VALUE1 = "CANCELLED" From ee80df7c0a51d702c402d870de5d9c0118a63ded Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 4 May 2015 16:53:48 -0400 Subject: [PATCH 02/12] moved XML conversion of raw file out of GetEmrData. --- redi/redi.py | 19 +++++++++++++++++-- redi/utils/GetEmrData.py | 12 ------------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/redi/redi.py b/redi/redi.py index 532d9f7..18962de 100755 --- a/redi/redi.py +++ b/redi/redi.py @@ -309,6 +309,23 @@ def _run(config_file, configuration_directory, do_keep_gen_files, dry_run, # load custom post-processing rules rules = load_rules(settings.rules, configuration_directory) + errors = run_preproc(pre_filters, settings) + map(logger.warning, errors) + + raw_txt_file = os.path.join(configuration_directory, 'raw.txt') + escaped_file = os.path.join(configuration_directory, 'rawEscaped.txt') + raw_xml_file = os.path.join(configuration_directory, 'raw.xml') + + # replace certain characters with escape sequences + GetEmrData.data_preprocessing(raw_txt_file, escaped_file) + + # run csv2xml.py to generate data in xml format + GetEmrData.generate_xml(escaped_file, raw_xml_file) + + # delete rawEscaped.txt + GetEmrData.cleanup(escaped_file) + + raw_xml_file = os.path.join(configuration_directory, settings.raw_xml_file) email_settings = get_email_settings(settings) db_path = database_path @@ -326,8 +343,6 @@ def _run(config_file, configuration_directory, do_keep_gen_files, dry_run, if not resume: _delete_last_runs_data(data_folder) - errors = run_preproc(pre_filters, settings) - map(logger.warning, errors) # TODO: Add preproc errors to report alert_summary, person_form_event_tree_with_data, rule_errors, \ diff --git a/redi/utils/GetEmrData.py b/redi/utils/GetEmrData.py index 9a77099..32f4ef7 100644 --- a/redi/utils/GetEmrData.py +++ b/redi/utils/GetEmrData.py @@ -162,18 +162,6 @@ def get_emr_data(conf_dir, connection_details): :connection_details EmrFileAccessDetails object """ raw_txt_file = os.path.join(conf_dir, 'raw.txt') - escaped_file = os.path.join(conf_dir, 'rawEscaped.txt') - raw_xml_file = os.path.join(conf_dir, 'raw.xml') # download csv file download_file(raw_txt_file, connection_details) - - # replace certain characters with escape sequences - data_preprocessing(raw_txt_file, escaped_file) - - # run csv2xml.py to generate data in xml format - generate_xml(escaped_file, raw_xml_file) - - # delete rawEscaped.txt - cleanup(escaped_file) - From 3d39acf492fd357707099f537f5f06649a56642c Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 4 May 2015 17:16:33 -0400 Subject: [PATCH 03/12] made csv to xml preprocessing dependent on getting the EMR file --- redi/redi.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/redi/redi.py b/redi/redi.py index 18962de..d6a9762 100755 --- a/redi/redi.py +++ b/redi/redi.py @@ -47,7 +47,7 @@ """ __author__ = "University of Florida CTS-IT Team" -__version__ = "0.13.2" +__version__ = "0.14.2" __email__ = "ctsit@ctsi.ufl.edu" __status__ = "Development" @@ -316,14 +316,16 @@ def _run(config_file, configuration_directory, do_keep_gen_files, dry_run, escaped_file = os.path.join(configuration_directory, 'rawEscaped.txt') raw_xml_file = os.path.join(configuration_directory, 'raw.xml') + # TODO: make this able to run against a local file if desired # replace certain characters with escape sequences - GetEmrData.data_preprocessing(raw_txt_file, escaped_file) - - # run csv2xml.py to generate data in xml format - GetEmrData.generate_xml(escaped_file, raw_xml_file) - - # delete rawEscaped.txt - GetEmrData.cleanup(escaped_file) + if get_emr_data: + GetEmrData.data_preprocessing(raw_txt_file, escaped_file) + if get_emr_data: + # run csv2xml.py to generate data in xml format + GetEmrData.generate_xml(escaped_file, raw_xml_file) + if get_emr_data: + # delete rawEscaped.txt + GetEmrData.cleanup(escaped_file) raw_xml_file = os.path.join(configuration_directory, settings.raw_xml_file) From 41d403d7ac856515bc89a01862c6f82fbc4d58d1 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 4 May 2015 17:17:16 -0400 Subject: [PATCH 04/12] fixed broken tests after relocating processing from GetEmrData --- test/TestGetEMRData.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/TestGetEMRData.py b/test/TestGetEMRData.py index 5b55eee..5aa184c 100644 --- a/test/TestGetEMRData.py +++ b/test/TestGetEMRData.py @@ -51,6 +51,7 @@ def test_get_emr_data(self): Note: This test is not concerned with testing the sftp communication""" temp_folder = tempfile.mkdtemp('/') temp_txt_file = os.path.join(temp_folder, "raw.txt") + temp_escaped_file = os.path.join(temp_folder, "rawEscaped.txt") temp_xml_file = os.path.join(temp_folder, "raw.xml") input_string = '''"NAME","COMPONENT_ID","RESULT","REFERENCE_UNIT","DATE_TIME_STAMP","STUDY_ID" @@ -72,6 +73,8 @@ def test_get_emr_data(self): ) GetEmrData.get_emr_data(temp_folder, props) + GetEmrData.data_preprocessing(temp_txt_file, temp_escaped_file) + GetEmrData.generate_xml(temp_escaped_file, temp_xml_file) with open(temp_xml_file) as f: result = f.read() From 85a53f9b6061e20ddf3d74231ac21d8038fa5ad3 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 4 May 2015 17:18:42 -0400 Subject: [PATCH 05/12] fixing version number in redi.py --- redi/redi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redi/redi.py b/redi/redi.py index d6a9762..6dbe023 100755 --- a/redi/redi.py +++ b/redi/redi.py @@ -47,7 +47,7 @@ """ __author__ = "University of Florida CTS-IT Team" -__version__ = "0.14.2" +__version__ = "0.14.1" __email__ = "ctsit@ctsi.ufl.edu" __status__ = "Development" From 53b55dffb8ed4cb20027d37280e1590c8426971a Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 4 May 2015 22:17:23 -0400 Subject: [PATCH 06/12] added new test for TestGetDBPath to increase coverage. --- test/TestGetDBPath.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 test/TestGetDBPath.py diff --git a/test/TestGetDBPath.py b/test/TestGetDBPath.py new file mode 100644 index 0000000..667bf20 --- /dev/null +++ b/test/TestGetDBPath.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +# Contributors: +# Christopher P. Barnes +# Andrei Sura: github.com/indera +# Mohan Das Katragadda +# Philip Chase +# Ruchi Vivek Desai +# Taeber Rapczak +# Nicholas Rejack +# Josh Hanna +# Copyright (c) 2014-2015, University of Florida +# All rights reserved. +# +# Distributed under the BSD 3-Clause License +# For full text of the BSD 3-Clause License see http://opensource.org/licenses/BSD-3-Clause + +import unittest +import os +import tempfile +from redi import redi +import shutil + + +class TestGetDBPath(unittest.TestCase): + + def setUp(self): + pass + + def test_get_db_path(self): + db_path = tempfile.mkdtemp() + batch_info_database = "database.db" + redi.get_db_path(batch_info_database, db_path) + full_db_path = os.path.join(batch_info_database, db_path) + assert os.path.exists(full_db_path) == 1 + shutil.rmtree(full_db_path) + +if __name__ == '__main__': + unittest.main() From 72e94f385344c9b42ed31ad4702a1e033b1341a7 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 4 May 2015 23:06:47 -0400 Subject: [PATCH 07/12] fixed function name in comment in TestCreateImportDataJson.py --- test/TestCreateImportDataJson.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TestCreateImportDataJson.py b/test/TestCreateImportDataJson.py index b44e279..6e8227b 100755 --- a/test/TestCreateImportDataJson.py +++ b/test/TestCreateImportDataJson.py @@ -16,7 +16,7 @@ # For full text of the BSD 3-Clause License see http://opensource.org/licenses/BSD-3-Clause """ -TestCreateEavOutput.py: +TestCreateImportDataJson.py: Verifies the correct functionality of the `test_create_import_data_json` function From 98364f336b76bda4418e4cbc330d1dbaee020047 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 4 May 2015 23:06:59 -0400 Subject: [PATCH 08/12] updated TestSuite to run new test --- test/TestSuite.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/TestSuite.py b/test/TestSuite.py index d8f01e4..3bec84f 100755 --- a/test/TestSuite.py +++ b/test/TestSuite.py @@ -63,6 +63,7 @@ from TestRediEmail import TestRediEmail from TestSentEventIndex import TestSentEventIndex from TestSendDatatoRedcap import TestSendDatatoRedcap +from TestGetDBPath import TestGetDBPath class redi_suite(unittest.TestSuite): @@ -109,6 +110,8 @@ def suite(self): redi_test_suite.addTest(TestDaysSinceToday) redi_test_suite.addTest(TestSendDatatoRedcap) + redi_test_suite.addTest(TestGetDBPath) + # return the suite return unittest.TestSuite([redi_test_suite]) From caa5c46f033b07bbc460a4f7d228df7ace6d0846 Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Mon, 4 May 2015 23:39:58 -0400 Subject: [PATCH 09/12] added expanded test for TestGetDBPath --- test/TestGetDBPath.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/TestGetDBPath.py b/test/TestGetDBPath.py index 667bf20..9197693 100644 --- a/test/TestGetDBPath.py +++ b/test/TestGetDBPath.py @@ -35,5 +35,13 @@ def test_get_db_path(self): assert os.path.exists(full_db_path) == 1 shutil.rmtree(full_db_path) + def test_get_db_path_not_exists(self): + # verify that db path is created if it doesn't exist + db_path = "nonExistentPath" + batch_info_database = "database.db" + created_db_path = redi.get_db_path(batch_info_database, db_path) + assert created_db_path == os.path.join(db_path, batch_info_database) + shutil.rmtree(db_path) + if __name__ == '__main__': unittest.main() From 0cb166554a96ce4ac983c1b2cabf881d48541a2b Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Wed, 13 May 2015 14:40:41 -0400 Subject: [PATCH 10/12] updated Windows install instructions to simplify the procedure --- docs/about.rst | 84 +++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 45 deletions(-) diff --git a/docs/about.rst b/docs/about.rst index d76e8aa..4f0812c 100644 --- a/docs/about.rst +++ b/docs/about.rst @@ -104,51 +104,45 @@ To uninstall the application: Installing RED-I on Windows ---------------------------- -1. Install Python 2.7.x from: https://www.python.org/downloads/windows/ -2. Install a git client for Windows https://windows.github.com/ -3. Run Git Shell icon -4. Clone the RED-I repository - -.. raw:: html - - 
-      git clone https://github.com/ctsit/redi.git
-
- -5. Install Powershell 4 for Windows: - http://www.microsoft.com/en-us/download/confirmation.aspx?id=40855 -6. Reboot -7. Install Visual C++ 9: - http://www.microsoft.com/en-us/download/details.aspx?id=44266 -8. Install SetupTools for Windows https://pypi.python.org/pypi/setuptools/12.2 -9. Launch PowerShell as administrator -10. Run this command: - -.. raw:: html - - 
-      (Invoke-WebRequest https://bootstrap.pypa.io/ez_setup.py).Content | python
-
- -11. Launch Git Shell icon -12. In the redi/ directory, run: - -.. raw:: html - - 
-      python setup.py bdist_egg
-      cd c:\python27\scripts\
-      .\easy_install.exe C:\Users\user1\Documents\code\redi\dist\redi_py-0.13.2-py2.7.egg
-
- -13. Add Python scripts directory to your system path by issuing the following command: - -.. raw:: html - - 
-      set path=%PATH%;c:\python27\scripts
-
- +* Open a command prompt by clicking on the Start menu, and typing "cmd" in the Run box. +* Install 64-bit Python 2.7.9 by running the following command in the command prompt: +
+msiexec /i https://www.python.org/ftp/python/2.7.9/python-2.7.9.amd64.msi 
+
+* Next you need to be insure the command interpreter will be able to find the Python modules. Set +the paths to the modules by running the following commands in the command prompt: +
+setx path "%path%;c:\python27"
+setx path "%path%;c:\python27\lib\site-packages"
+setx path "%path%;c:\python27\scripts”
+
+* Make a new directory for the RED-I files by running the following command in the command prompt: +
+mkdir c:\redi
+
+* Download the RED-I source code from: [https://github.com/ctsit/redi/archive/0.14.1.zip] +* Copy the contents of the RED-I zip file from c:\Users\%username%\Downloads\redi-0.14.1\redi-0.14.1 to c:\redi +* Download the easy_install setup file from: https://bootstrap.pypa.io/ez_setup.py +* Run the easy_install setup file with the following command in the command prompt: +
+python c:\Users\%username%\Downloads\ez_setup.py
+
+Note: you may need to modify the path to the ez_setup.py file if it is downloaded to a different location. + +* Next, make a binary install of RED-I by running the following commands in the command prompt: +
+cd c:\redi
+python c:\redi\setup.py bdist_egg
+
+* You will need to manually install the pycrypto dependency. To avoid having to compile it with VCForPython you can +download a pre-compiled binary and install it with the following command: +
+c:\python27\scripts\easy_install http://www.voidspace.org.uk/python/pycrypto-2.6.1/pycrypto-2.6.1.win-amd64-py2.7.exe
+
+* Finally, install your binary of RED-I with the following command: +
+c:\python27\scripts\easy_install.exe c:\redi\dist\redi-0.14.1-py2.7.egg
+
How to Test RED-I with a Sample Project --------------------------------------- From e06d7e80c6d6a911b16f15e3d2100e15a3bb8f5f Mon Sep 17 00:00:00 2001 From: Buck at UF Date: Thu, 14 May 2015 14:37:37 -0400 Subject: [PATCH 11/12] Update about.rst --- docs/about.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/about.rst b/docs/about.rst index 4f0812c..6f2e273 100644 --- a/docs/about.rst +++ b/docs/about.rst @@ -67,7 +67,7 @@ version control installed on your system. $ make && make install -Once you are done with testing RED-I an you are satisfied with the +Once you are done with testing RED-I and you are satisfied with the results you can remove the virtualenv artifacts and install the RED-I package to be available system-wide. From daa35eeb9db11e071762ca1321c048026520b16d Mon Sep 17 00:00:00 2001 From: Nicholas Rejack Date: Thu, 14 May 2015 17:49:54 -0400 Subject: [PATCH 12/12] updated CHANGELOG, redi/redi.py, and setup.py for release of 0.14.2 --- CHANGELOG | 16 ++++++++++++++++ redi/redi.py | 2 +- setup.py | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index f52e3b6..b924ae3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,19 @@ +2015-05-14 v0.14.2 + * Summary: Preprocessing has been improved to use the rewritten raw CSV file after processing it. This relocates some of the XML processing from GetEMRData. + + * Update about.rst (Buck at UF) + * updated Windows install instructions to simplify the procedure (Nicholas Rejack) + * added expanded test for TestGetDBPath (Nicholas Rejack) + * updated TestSuite to run new test (Nicholas Rejack) + * fixed function name in comment in TestCreateImportDataJson.py (Nicholas Rejack) + * added new test for TestGetDBPath to increase coverage. (Nicholas Rejack) + * fixing version number in redi.py (Nicholas Rejack) + * fixed broken tests after relocating processing from GetEmrData (Nicholas Rejack) + * made csv to xml preprocessing dependent on getting the EMR file (Nicholas Rejack) + * moved XML conversion of raw file out of GetEmrData. (Nicholas Rejack) + * `make test` runs all tests (Andrei Sura) + * Update README.md (Christopher Barnes) + 2015-04-23 v0.14.1 * Summary: PreProc and PostProc Hooks! RED-I has added the ability to hook scripts for post processing and preprocessing along with other minor fixes. These features now allow for the writing of filters and the emailing of log messages, for example, at the begining and end of a RED-I run. diff --git a/redi/redi.py b/redi/redi.py index 6dbe023..d6a9762 100755 --- a/redi/redi.py +++ b/redi/redi.py @@ -47,7 +47,7 @@ """ __author__ = "University of Florida CTS-IT Team" -__version__ = "0.14.1" +__version__ = "0.14.2" __email__ = "ctsit@ctsi.ufl.edu" __status__ = "Development" diff --git a/setup.py b/setup.py index 0019c77..c9a3e21 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name='redi', - version='0.14.1', + version='0.14.2', author='https://www.ctsi.ufl.edu/research/study-development/informatics-consulting/', author_email='ctsit@ctsi.ufl.edu', packages=find_packages(exclude=['test']),