sql corrections to avoid jsonpath2016 errors

CMSgov · Nov 15, 2024 · 690fd90 · 690fd90
1 parent 75151da
commit 690fd90
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 51 deletions.
diff --git a/terraform/services/quicksight/sql_templates/bulk_requests.sql.tfpl b/terraform/services/quicksight/sql_templates/bulk_requests.sql.tfpl
@@ -8,14 +8,20 @@
 create or replace view bulk_data_requests_by_day as
 select request_ct, date_format(msgday, '%Y-%m-%d') as msgdate
 from (
-    SELECT count(message) as request_ct, date_trunc('day', from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) ) as msgday
-    FROM ${api_profile}
-    where json_query(message, 'lax $.contentlength' null on error ) is null
-        and json_query(message, 'lax $.message' null on error) is not null
-        and json_extract_scalar(message,'$.application') = ${app}
-        and json_extract_scalar(message, '$.environment') = ${env}
-        and json_extract_scalar(message,'$.message') like 'Exporting data for provider%'
-        and date_diff('day', current_date, from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp'))) <  ${days_history}
-    group by date_trunc('day', from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) )
+    SELECT count(message) as request_ct, date_trunc('day', messagets ) as msgday
+    FROM (
+        select message,
+               cast(json_extract_scalar (message, '$.contentlength') as VARCHAR) as contentlength,
+               cast(json_extract_scalar (message, '$.message') as VARCHAR) as payloadmessage,
+               from_iso8601_timestamp(cast(json_extract_scalar(message,'$.timestamp')) as VARCHAR) as messagets
+        FROM ${api_profile}
+    ) rowset
+    WHERE contentlength is null
+      and payloadmessage is not null
+      and cast(json_extract_scalar(message,'$.application') as VARCHAR) = ${app}
+      and cast(json_extract_scalar(message, '$.environment') as VARCHAR) = ${env}
+      and payloadmessage like 'Exporting data for provider%'
+      and date_diff('day', current_date, messagets) <  ${days_history}
+      group by date_trunc('day', messagets )
 ) summarydata
 order by summarydata.msgday
diff --git a/terraform/services/quicksight/sql_templates/everything_requests.sql.tfpl b/terraform/services/quicksight/sql_templates/everything_requests.sql.tfpl
@@ -8,14 +8,19 @@
 create or replace view everything_requests_by_day as
 select request_ct, date_format(msgday, '%Y-%m-%d') as msgdate
 from (
-    SELECT count(message) as request_ct, date_trunc('day', from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) ) as msgday
-    FROM ${api_profile}
-    where (json_query(message, 'lax $.contentlength' null on error ) is not null
-        and json_extract_scalar(message,'$.method') = 'GET')
-        and json_extract_scalar(message,'$.useragent') != 'ELB-HealthChecker/2.0'
-        and json_extract_scalar(message,'$.uri') like '/api/v1/Patient/%everything'
-        and json_extract_scalar(message,'$.environment') = ${env}
-        and date_diff('day', current_date, from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp'))) <  ${days_history}
-    group by date_trunc('day', from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) )
+    SELECT  count(message) as request_ct, 
+            date_trunc('day', messagets ) as msgday
+    FROM (
+        select message,
+               cast(json_extract_scalar (message, '$.contentlength') as VARCHAR) as contentlength,
+        FROM ${api_profile}
+    ) rowset
+    where contentlength is not null
+        and cast(json_extract_scalar(message,'$.method') as VARCHAR) = 'GET'
+        and cast(json_extract_scalar(message,'$.useragent') as VARCHAR) != 'ELB-HealthChecker/2.0'
+        and cast(json_extract_scalar(message,'$.uri') as VARCHAR) like '/api/v1/Patient/%everything'
+        and cast(json_extract_scalar(message,'$.environment') as VARCHAR) = ${env}
+        and date_diff('day', current_date, messagets) <  ${days_history}
+    group by date_trunc('day', messagets )
 ) summarydata
 order by summarydata.msgday
diff --git a/terraform/services/quicksight/sql_templates/group_requests.sql.tfpl b/terraform/services/quicksight/sql_templates/group_requests.sql.tfpl
@@ -9,13 +9,16 @@ create or replace view group_requests_made_by_day as
 select request_ct, date_format(msgday, '%Y-%m-%d') as msgdate
 from (
     SELECT count(message) as request_ct, date_trunc('day', from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) ) as msgday
-    FROM ${api_profile}
-    where (json_query(message, 'lax $.contentlength' null on error ) is not null
-        and json_extract_scalar(message,'$.method') = 'GET')
-        and json_extract_scalar(message,'$.useragent') != 'ELB-HealthChecker/2.0'
-        and json_extract_scalar(message,'$.uri') = '/api/v1/Group'
-        and json_extract_scalar(message,'$.environment') = ${env}
-        and date_diff('day', current_date, from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp'))) <  ${days_history}
-    group by date_trunc('day', from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) )
+    FROM ( 
+        select message, cast(json_extract_scalar (message, '$.contentlength') as VARCHAR) as contentlength
+        FROM ${api_profile}
+    ) rowset
+    WHERE contentlength is not null
+        and cast(json_extract_scalar(message,'$.method') as VARCHAR) = 'GET'
+        and cast(json_extract_scalar(message,'$.useragent') as VARCHAR) != 'ELB-HealthChecker/2.0'
+        and cast(json_extract_scalar(message,'$.uri') as VARCHAR) = '/api/v1/Group'
+        and cast(json_extract_scalar(message,'$.environment') as VARCHAR) = ${env}
+        and date_diff('day', current_date, from_iso8601_timestamp(cast(json_extract_scalar(message,'$.timestamp') as VARCHAR))) <  ${days_history}
+    group by date_trunc('day', from_iso8601_timestamp(cast(json_extract_scalar(message,'$.timestamp') as VARCHAR)) )
 ) summarydata
 order by summarydata.msgday
diff --git a/terraform/services/quicksight/sql_templates/since_requests.sql.tfpl b/terraform/services/quicksight/sql_templates/since_requests.sql.tfpl
@@ -8,14 +8,20 @@
 create or replace view since_requests_by_day as
 select since_call_ct, date_format(msgday, '%Y-%m-%d') as msgdate
 from (
-    SELECT count(message) as since_call_ct, date_trunc('day', from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) ) as msgday
-    FROM ${api_profile}
-    where json_query(message, 'lax $.contentlength' null on error ) is null
-    and json_query(message, 'lax $.message' null on error) is not null
-    and json_extract_scalar(message,'$.application') = ${app}
-    and json_extract_scalar(message, '$.environment') = ${env}
-    and json_extract_scalar(message,'$.message') like 'Exporting data for provider%_since%'
-    and date_diff('day', current_date, from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp'))) < ${days_history}
-    group by date_trunc('day', from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) )
+    SELECT count(message) as since_call_ct, date_trunc('day', messagets ) as msgday
+    FROM (
+        select message,
+               cast(json_extract_scalar (message, '$.contentlength') as VARCHAR) as contentlength,
+               cast(json_extract_scalar (message, '$.message') as VARCHAR) as payloadmessage,
+               from_iso8601_timestamp(cast(json_extract_scalar(message,'$.timestamp')) as VARCHAR) as messagets
+        FROM ${api_profile}
+    ) rowset
+    where contentlength is null
+      and payloadmessage is not null
+      and cast(json_extract_scalar(message,'$.application') as VARCHAR) = ${app}
+      and cast(json_extract_scalar(message, '$.environment') as VARCHAR) = ${env}
+      and payloadmessage like 'Exporting data for provider%_since%'
+      and date_diff('day', current_date, messagets) < ${days_history}
+      group by date_trunc('day', messagets )
 ) summarydata
 order by summarydata.msgday
diff --git a/terraform/services/quicksight/sql_templates/total_benes_requested.sql.tfpl b/terraform/services/quicksight/sql_templates/total_benes_requested.sql.tfpl
@@ -5,23 +5,25 @@
 --
 -- NOTE: TBD
 --
---create or replace view total_benes_by_day as
+create or replace view total_benes_by_day as
 select sum(summarydata.ct_by_day) as daily_sum, summarydata.msgdate
 from (
     select count(distinctdata.ptid) as ct_by_day, distinctdata.ptid, msgdate
     from (
-
             select distinct rawdata.ptid, date_format(rawdata.msgday,'%Y-%m-%d') as msgdate
             from (
-                SELECT json_extract_scalar(message,'$.mdc_patientid') as ptid,
-                       from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) as msgday
-                FROM ${agg_profile}
-                where json_query(message, 'lax $.dpcmetric' null on error ) is not null
-                  and json_extract_scalar (message, '$.dpcmetric') = 'DataExportResult'
-                  and json_extract_scalar (message, '$.environment') = ${env}
+                SELECT cast(json_extract_scalar(message,'$.mdc_patientid') as VARCHAR) as ptid,
+                       from_iso8601_timestamp(cast(json_extract_scalar(message,'$.timestamp') as VARCHAR)) as msgday
+                FROM 
+                    ( select message, cast(json_extract_scalar (message, '$.dpcmetric') as VARCHAR) as dpcmetric
+                      FROM ${agg_profile}
+                    ) rowset
+                where 
+                  dpcmetric = 'DataExportResult'
+                  and cast(json_extract_scalar (message, '$.environment') as VARCHAR) = ${env}
             ) rawdata
             where date_diff('day',current_date, rawdata.msgday) < ${days_history}
-        ) distinctdata
+        ) distinctdataß
     group by distinctdata.ptid, distinctdata.msgdate
     order by distinctdata.ptid, distinctdata.msgdate
 ) summarydata

diff --git a/terraform/services/quicksight/sql_templates/uniq_benes_served.sql.tfpl b/terraform/services/quicksight/sql_templates/uniq_benes_served.sql.tfpl
@@ -12,13 +12,15 @@ from (
     from (
             select distinct rawdata.ptid, date_format(rawdata.msgday,'%Y-%m-%d') as msgdate
             from (
-                SELECT json_extract_scalar(message,'$.mdc_patientid') as ptid,
-                       from_iso8601_timestamp(json_extract_scalar(message,'$.timestamp')) as msgday
-                FROM ${agg_profile}
-                where json_query(message, 'lax $.dpcmetric' null on error ) is not null
-                  and json_extract_scalar (message, '$.dpcmetric') = 'DataExportResult'
-                  and json_extract_scalar (message, '$.environment') = ${env}
-                  and json_extract_scalar (message, '$.dataretrieved') = 'true'
+                SELECT cast(json_extract_scalar(message,'$.mdc_patientid') as VARCHAR) as ptid,
+                       from_iso8601_timestamp(cast(json_extract_scalar(message,'$.timestamp') as VARCHAR)) as msgday
+                FROM 
+                    ( select message, cast(json_extract_scalar (message, '$.dpcmetric') as VARCHAR) as dpcmetric
+                      FROM ${agg_profile}
+                    ) rowset
+                WHERE dpcmetric = 'DataExportResult'
+                  and cast(json_extract_scalar (message, '$.environment') as VARCHAR) = ${env}
+                  and cast(json_extract_scalar (message, '$.dataretrieved') as VARCHAR) = 'true'
             ) rawdata
             where date_diff('day',current_date, rawdata.msgday) < 7
         ) distinctdata