CSM BDS fixes related to date-time format issues

csm_big_data/logstash/config/logstash.conf

@@ -94,6 +94,18 @@ else if "ras" in [tags] and "csm" in [tags] {
         add_field => { "type" => "log-ras"  } 
     }
 
+### Use logstash gsub filter plugin to process input data before sending to elastic index.
+### For the two fields: data.begin_time and data.history.end_time, the blank is replaced
+### with a letter T. Note: these fields have type "date" that requires the valid format.
+### Also note: No change to csm_transaction.log with this process.
+
+    mutate {
+        gsub => [
+           "[data][begin_time]", " ", "T",
+           "[data][history][end_time]", " ", "T"
+        ]
+    }
+
     date {
     	match => ["time_stamp", "ISO8601","YYYY-MM-dd HH:mm:ss.SSS" ]
     	target => "time_stamp"

csm_big_data/python/cast_helper.py

@@ -48,7 +48,8 @@
 DATE_FORMAT        = '(\d{4})-(\d{1,2})-(\d{1,2})[ \.T]*(\d{0,2}):{0,1}(\d{0,2}):{0,1}(\d{0,2})'
 DATE_FORMAT_PRINT  = '%Y-%m-%d %H:%M:%S'
 #TIME_SEARCH_FORMAT = 'yyyy-MM-dd HH:mm:ss'
-TIME_SEARCH_FORMAT = "epoch_millis"
+#TIME_SEARCH_FORMAT = "epoch_millis"
+TIME_SEARCH_FORMAT = "strict_date_optional_time_nanos"
 
 USER_JOB_FIELDS=["data.primary_job_id","data.secondary_job_id", "data.allocation_id", 
     "data.user_name", "data.begin_time", "data.history.end_time", "data.state"]
@@ -156,8 +157,11 @@ def build_time_range(start_time, end_time,
         for a record to be considered "in range".
     '''
     # Build the time range
-    start_time = convert_timestamp(start_time)
-    end_time = convert_timestamp(end_time)
+    # UPDATE: convert_timestamp() returns time in epoch_millis format which is not needed
+    # when TIME_SEARCH_FORMAT is changed to strict_date_optional_time_nanos.
+    # Following two lines are commented out.
+    ##start_time = convert_timestamp(start_time)
+    ##end_time = convert_timestamp(end_time)
 
     if start_time and end_time:
         # Build the time range.
@@ -214,8 +218,8 @@ def build_timestamp_range( start_time, end_time, field="@timestamp"):
     '''
 
     # Build the time range
-    start_time = convert_timestamp(start_time)
-    end_time   = convert_timestamp(end_time)
+    #start_time = convert_timestamp(start_time)
+    #end_time   = convert_timestamp(end_time)
 
     # Build the time range.
     target=[]

csm_big_data/python/findJobKeys.py

@@ -125,7 +125,7 @@ def main(args):
     # ---------------------------------------------------------------------------------------------
     # TODO Add a utility script to manage this.
 
-    date_format= '%Y-%m-%d %H:%M:%S.%f'
+    date_format= '%Y-%m-%dT%H:%M:%S.%f'
     print_format='%Y-%m-%d %H:%M:%S:%f'
     search_format='epoch_millis'
 

csm_big_data/python/findJobMetrics.py

@@ -120,7 +120,7 @@ def main(args):
     }
 
     # ---------------------------------------------------------------------------------------------
-    date_format= '%Y-%m-%d %H:%M:%S.%f'
+    date_format= '%Y-%m-%dT%H:%M:%S.%f'
     print_format='%Y-%m-%d %H:%M:%S:%f'
     search_format='epoch_millis'
 

csm_big_data/python/findJobTimeRange.py

@@ -114,11 +114,11 @@ def main(args):
     if len(hits) > 0 :
         tr_data = cast.deep_get( hits[0], "_source", "data")
 
-        date_format= '%Y-%m-%d %H:%M:%S.%f'
+        date_format= '%Y-%m-%dT%H:%M:%S.%f'
         print_format='%Y-%m-%d.%H:%M:%S:%f'
         search_format='"yyyy-MM-dd HH:mm:ss:SSS"'
 
-        start_time=datetime.strptime(tr_data["begin_time"], '%Y-%m-%d %H:%M:%S.%f')
+        start_time=datetime.strptime(tr_data["begin_time"], '%Y-%m-%dT%H:%M:%S.%f')
         start_time='{0}'.format(start_time.strftime(print_format)[:-3])
 
         # If a history is present end_time is end_time, otherwise it's now.