Skip to content
This repository has been archived by the owner on Oct 12, 2023. It is now read-only.

Commit

Permalink
Merge branch 'release/0.13.1'
Browse files Browse the repository at this point in the history
  • Loading branch information
@ufl-taeber
ufl-taeber committed Dec 4, 2014
2 parents c2a41ce + da6a6c7 commit 549aab3
Show file tree
Hide file tree
Showing 18 changed files with 548 additions and 171 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ vagrant/redcap-backup-*.sql
vagrant/REDI-*.egg

vagrant/redcap.zip
vagrant/plugins/
vagrant/redcap_database.sql
vagrant/sqlPatches
vagrant/data/
Expand All @@ -51,3 +52,5 @@ vagrant/redi.db
data/
redi.db
report.html
redi.pstats
callgraph.svg
11 changes: 11 additions & 0 deletions CHANGELOG
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,14 @@
2014-12-04 v0.13.1

* Fixed major flaw in the Throttle class introduced in v0.13.0
* Added --bulk-send-blanks, which will send all blank form-events together
* Added support for redcap_plugins to our vagrant testing and sample project
* Added new Makefile targets for various helper tasks like profiling and clearing the IP ban list
* Installed mcrypt package for PHP to enable plugins in vagrant
* Added a REDCap plugin to generate URLs for enhanced error reporting
* Fixed #34: Data Import Report totals only reflect most recent run
* Added execution time to the report

2014-11-20 v0.13.0

The focus of this release has been clean up. Related functions were moved into
Expand Down
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ clean:
rm -rf vagrant/data/
rm -f vagrant/redi.db
rm -f config-example/report.xml
rm -f redi.pstats

pypi:
#https://pythonhosted.org/Distutils2/distutils/packageindex.html
Expand Down
144 changes: 72 additions & 72 deletions redi/batch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

BATCH_STATUS_STARTED = 'Started'
BATCH_STATUS_COMPLETED = 'Completed'

"""
@see #check_input_file()
Expand Down Expand Up @@ -66,8 +67,9 @@ def create_empty_table(db_path) :
cur = db.cursor()
sql = """CREATE TABLE RediBatch (
rbID INTEGER PRIMARY KEY AUTOINCREMENT,
rbStartTime TEXT NOT NULL,
rbEndTime TEXT,
rbCreateTime DATETIME DEFAULT CURRENT_TIMESTAMP,
rbStartTime DATETIME,
rbEndTime DATETIME,
rbStatus TEXT,
rbMd5Sum TEXT NOT NULL
)
Expand Down Expand Up @@ -102,7 +104,7 @@ def dict_factory(cursor, row):
Check the md5sum of the input file
- if the sum *has changed* then continue the data processing and store a row
in the SQLite database with `batch status= batch_started/ batch_completed`
in the SQLite database with `batch status= started/ completed`
- if the sum *did not change* then check the config option `batch_warning_days`:
- if limit = -1 then continue execution (ignore the limit)
Expand All @@ -111,7 +113,7 @@ def dict_factory(cursor, row):
"""


def check_input_file(batch_warning_days, db_path, email_settings, raw_xml_file, project):
def check_input_file(batch_warning_days, db_path, email_settings, raw_xml_file, project, start_time):
batch = None

if not os.path.exists(db_path) :
Expand All @@ -132,27 +134,28 @@ def check_input_file(batch_warning_days, db_path, email_settings, raw_xml_file,
logger.info(
"There is no old md5 recorded yet for the input file. Continue data import...")
batch = add_batch_entry(db_path, new_md5ive)
record_msg = 'Added batch (rbID= %s, rbStartTime= %s, rbMd5Sum= %s' % (
batch['rbID'], batch['rbStartTime'], batch['rbMd5Sum'])
record_msg = 'Added batch (rbID= %s, rbCreateTime= %s, rbMd5Sum= %s' % (
batch['rbID'], batch['rbCreateTime'], batch['rbMd5Sum'])
logger.info(record_msg)
return batch

if old_md5ive != new_md5ive:
# the data has changed... insert a new batch entry
batch = add_batch_entry(db_path, new_md5ive)
record_msg = 'Added batch (rbID= %s, rbStartTime= %s, rbMd5Sum= %s' % (
batch['rbID'], batch['rbStartTime'], batch['rbMd5Sum'])
record_msg = 'Added batch (rbID= %s, rbCreateTime= %s, rbMd5Sum= %s' % (
batch['rbID'], batch['rbCreateTime'], batch['rbMd5Sum'])
logger.info(record_msg)
return batch
else:
days_since_today = get_days_since_today(old_batch['rbStartTime'])
days_since_today = get_days_since_today(old_batch['rbCreateTime'])
# TODO: refactor code to use ConfigParser.RawConfigParser in order to
# preserve data types

if (days_since_today > int(batch_warning_days)):
raw_xml = RawXml(project, raw_xml_file)
msg_file_details = "\nXML file details: " + raw_xml.get_info()
logger.info('Last import was started on: %s which is more than the limit of %s' % (old_batch['rbStartTime'], batch_warning_days))
logger.info('Last import was started on: %s which is more than '\
' the limit of %s' % (old_batch['rbStartTime'], batch_warning_days))
if (-1 == int(batch_warning_days)):
msg_continue = """
The configuration `batch_warning_days = -1` indicates that we want to continue
Expand All @@ -161,7 +164,7 @@ def check_input_file(batch_warning_days, db_path, email_settings, raw_xml_file,
logger.info(msg_continue)
else:

msg_quit = "The input file did not change in the past: %s days. Stop data import." % batch_warning_days
msg_quit = "The input file did not change in the past: %s days." % days_since_today
logger.critical(msg_quit + msg_file_details)
redi_email.send_email_input_data_unchanged(email_settings, raw_xml)
sys.exit()
Expand All @@ -178,14 +181,14 @@ def check_input_file(batch_warning_days, db_path, email_settings, raw_xml_file,


def get_last_batch(db_path):
db = None
batch = None
try:
db = lite.connect(db_path)
db.row_factory = dict_factory
cur = db.cursor()
sql = """
SELECT
rbID, rbStartTime, rbEndTime, rbMd5Sum
rbID, rbCreateTime, rbStartTime, rbEndTime, rbMd5Sum
FROM
RediBatch
ORDER BY rbID DESC
Expand All @@ -204,20 +207,18 @@ def get_last_batch(db_path):
return batch


"""
Retrieve the row corresponding to the specified primary key
"""


def get_batch_by_id(db_path, batch_id):
db = None
"""
Retrieve the row corresponding to the specified primary key
"""
try:
db = lite.connect(db_path)
db.row_factory = dict_factory
cur = db.cursor()
sql = """
SELECT
rbID, rbStartTime, rbEndTime, rbMd5Sum
rbID, rbCreateTime, rbStartTime, rbEndTime, rbMd5Sum
FROM
RediBatch
WHERE
Expand All @@ -238,16 +239,15 @@ def get_batch_by_id(db_path, batch_id):
return batch


"""
@see #check_input_file()
@see https://docs.python.org/2/library/hashlib.html
@see https://docs.python.org/2/library/sqlite3.html#sqlite3.Connection.row_factory
Returns the md5 sum for the redi input file
"""


def get_md5_input_file(input_file):
"""
@see #check_input_file()
@see https://docs.python.org/2/library/hashlib.html
@see https://docs.python.org/2/library/sqlite3.html#sqlite3.Connection.row_factory
Returns the md5 sum for the redi input file
"""
if not os.path.exists(input_file):
raise Exception('Input file not found at: ' + input_file)

Expand All @@ -268,30 +268,31 @@ def get_md5_input_file(input_file):
return md5.hexdigest()


"""
@see #check_input_file()
@param db_path - the SQLite file
@param md5 - the md5 sum to be inserted
"""


def add_batch_entry(db_path, md5):
logger.info('Execute: add_batch_entry()')
batch = None

db = None
"""
Inserts a row into RediBatch table
@see #check_input_file()
Parameters
----------
db_path : string
The SQLite database file name
md5 : string
The md5 sum to be inserted
create_time : string
The batch start time
"""
try:
db = lite.connect(db_path)
db.row_factory = dict_factory
cur = db.cursor()
sql = """
INSERT INTO RediBatch
(rbStartTime, rbEndTime, rbStatus, rbMd5Sum)
(rbCreateTime,rbStartTime, rbEndTime, rbStatus, rbMd5Sum)
VALUES
( ?, NULL, 'Started', ?)
( ?, NULL, NULL, ?, ?)
"""
now = get_db_friendly_date_time()
cur.execute(sql, (now, md5))
create_time = get_db_friendly_date_time()
cur.execute(sql, (create_time, BATCH_STATUS_STARTED, md5))
rbID = cur.lastrowid
db.commit()
batch = get_batch_by_id(db_path, rbID)
Expand All @@ -306,32 +307,36 @@ def add_batch_entry(db_path, md5):
return batch


"""
Update the status and the finish time of a specified batch entry in the SQLite db
@return True if update succeeded, False otherwise
"""

def update_batch_entry(db_path, id, status, start_time, end_time):
"""
Update the status and the start/end time of a specified batch entry
Return True if update succeeded, False otherwise
def update_batch_entry(db_path, id, status, timestamp):
success = None
db = None
Parameters
----------
db_path : string
id : integer
status : string
start_time : datetime string
end_time : datetime string
"""
try:
db = lite.connect(db_path)
cur = db.cursor()
sql = """
UPDATE
RediBatch
SET
rbEndTime = ?
rbStartTime = ?
, rbEndTime = ?
, rbStatus = ?
WHERE
rbID = ?
"""

cur.execute(sql, (timestamp, status, id))
cur.execute(sql, (start_time, end_time, status, id))
db.commit()
scuccess = True
success = True
except lite.Error as e:
logger.exception("SQLite error in update_batch_entry(): %s:" % e.args[0])
success = False
Expand All @@ -342,37 +347,32 @@ def update_batch_entry(db_path, id, status, timestamp):
return success


"""
@return string in format: "2014-06-24 01:23:24"
"""


def get_db_friendly_date_time():
"""
@return string in format: "2014-06-24 01:23:24"
"""
return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

"""
@return string in format: 2014-06-24
"""


def get_db_friendly_date():
"""
@return string in format: 2014-06-24
"""
return datetime.date.today()

"""
@return the number of days passed since the specified date
"""


def get_days_since_today(date_string):
"""
@return the number of days passed since the specified date
"""
num = None
other = datetime.datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S')
now = datetime.datetime.now()
delta = now - other
return delta.days

"""
Helper function for debugging xml content
"""
def printxml(tree):
"""
Helper function for debugging xml content
"""
print etree.tostring(tree, pretty_print = True)
return
Loading

0 comments on commit 549aab3

Please sign in to comment.