diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 4ea3981..468304d 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: servicenow_integration_tests_4 + schema: servicenow_integration_tests_445 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: servicenow_integration_tests_4 + schema: servicenow_integration_tests_445 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: servicenow_integration_tests_4 + schema: servicenow_integration_tests_445 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: servicenow_integration_tests_4 + schema: servicenow_integration_tests_445 threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: servicenow_integration_tests_4 + schema: servicenow_integration_tests_445 threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 62d9530..aa5f4c4 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -10,7 +10,7 @@ dispatch: vars: servicenow: - servicenow_schema: servicenow_integration_tests_3 + servicenow_schema: servicenow_integration_tests_445 servicenow_task_identifier: "task_data" servicenow_task_sla_identifier: "task_sla_data" servicenow_sys_user_identifier: "sys_user_data" @@ -20,6 +20,9 @@ vars: servicenow_change_task_identifier: "change_task_data" servicenow_change_request_identifier: "change_request_data" servicenow_core_company_identifier: "core_company_data" + servicenow_cmdb_ci_identifier: "cmdb_ci_data" + servicenow_cmdb_ci_service_identifier: "cmdb_ci_service_data" + servicenow_sys_user_group_identifier: "sys_user_group_data" seeds: +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}" @@ -29,6 +32,7 @@ seeds: sys_created_on: "timestamp" resolved_at: "timestamp" closed_at: "timestamp" + market_cap: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" confirmed_by_value: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}" first_reported_by_task_value: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}" requested_by_value: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}" @@ -44,7 +48,6 @@ seeds: opened_by_value: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}" sys_created_by: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}" sys_updated_by: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}" - task_data: +enabled: "{{ true if target.type != 'snowflake' else false }}" task_data_snowflake: diff --git a/integration_tests/seeds/cmdb_ci_data.csv b/integration_tests/seeds/cmdb_ci_data.csv new file mode 100644 index 0000000..c127cf3 --- /dev/null +++ b/integration_tests/seeds/cmdb_ci_data.csv @@ -0,0 +1,2 @@ +sys_id,_fivetran_deleted,_fivetran_synced,asset_link,asset_tag,asset_value,assigned,assigned_to_link,assigned_to_value,assignment_group_link,assignment_group_value,attestation_score,attestation_status,attested,attested_by_link,attested_by_value,attested_date,attributes,business_unit_link,business_unit_value,can_print,category,change_control_link,change_control_value,checked_in,checked_out,comments,company_link,company_value,correlation_id,cost,cost_cc,cost_center_link,cost_center_value,delivery_date,department_link,department_value,discovery_source,dns_domain,due,due_in,duplicate_of_link,duplicate_of_value,environment,fault_count,first_discovered,fqdn,gl_account,install_date,install_status,invoice_number,ip_address,justification,last_discovered,lease_id,life_cycle_stage_link,life_cycle_stage_status_link,life_cycle_stage_status_value,life_cycle_stage_value,location_link,location_value,mac_address,maintenance_schedule_link,maintenance_schedule_value,managed_by_group_link,managed_by_group_value,managed_by_link,managed_by_value,manufacturer_link,manufacturer_value,model_id_link,model_id_value,model_number,monitor,name,operational_status,order_date,owned_by_link,owned_by_value,po_number,purchase_date,schedule_link,schedule_value,serial_number,short_description,skip_sync,start_date,subcategory,support_group_link,support_group_value,supported_by_link,supported_by_value,sys_class_name,sys_class_path,sys_created_by,sys_created_on,sys_domain_link,sys_domain_path,sys_domain_value,sys_mod_count,sys_updated_by,sys_updated_on,unverified,vendor_link,vendor_value,warranty_expiration +46be1a35a9fe1981008a44a672b62e0d,false,2023-11-06 00:10:48.992000,,,,,,,,,,,false,,,,,,,false,,,,,,,,,,,,,,,,,,,,,,,,0,,,,,1,,,,,,,,,,,,,,,,,,,https://dev165675.service-now.com/api/now/v2/table/core_company/0e8b8e650a0a0b3b004f285ffbb1a4fc,0e8b8e650a0a0b3b004f285ffbb1a4fc,,,,false,Windows 2000 Hotfix - KB867282,1,,,,,,,,,,false,,,,,,,cmdb_ci_spkg,"""/!!/#,""",system,2006-02-07 22:59:57.000000,https://dev165675.service-now.com/api/now/v2/table/sys_user_group/global,/,global,15,system,2023-06-25 12:20:02.000000,false,,, diff --git a/integration_tests/seeds/cmdb_ci_service_data.csv b/integration_tests/seeds/cmdb_ci_service_data.csv new file mode 100644 index 0000000..90e676b --- /dev/null +++ b/integration_tests/seeds/cmdb_ci_service_data.csv @@ -0,0 +1,2 @@ +sys_id,_fivetran_deleted,_fivetran_synced,banner_image,banner_image_light,banner_text,city,contact_link,contact_value,coordinates_retrieved_on,country,customer,discount,fax_phone,fiscal_year,lat_long_error,latitude,longitude,manufacturer,market_cap,name,notes,num_employees,parent_link,parent_value,phone,primary,profits,publicly_traded,rank_tier,revenue_per_year,state,stock_price,stock_symbol,street,sys_class_name,sys_created_by,sys_created_on,sys_mod_count,sys_updated_by,sys_updated_on,theme_link,theme_value,vendor,vendor_manager,vendor_type,website,zip +2f1c3218bfd101007a6d257b3f0739ee,false,2023-11-06 00:31:39,,,,,,,,USA,false,,,,,,,false,0,System manufacturer,,,,,,false,0,false,,0,,,,,core_company,admin,2013-05-08 20:23:29,0,admin,2013-05-08 20:23:29,,,false,,,, diff --git a/integration_tests/seeds/core_company_data.csv b/integration_tests/seeds/core_company_data.csv new file mode 100644 index 0000000..3dc81b3 --- /dev/null +++ b/integration_tests/seeds/core_company_data.csv @@ -0,0 +1,2 @@ +sys_id,_fivetran_deleted,_fivetran_synced,banner_image,banner_image_light,banner_text,city,contact_link,contact_value,coordinates_retrieved_on,country,customer,discount,fax_phone,fiscal_year,lat_long_error,latitude,longitude,manufacturer,market_cap,name,notes,num_employees,parent_link,parent_value,phone,primary,profits,publicly_traded,rank_tier,revenue_per_year,state,stock_price,stock_symbol,street,sys_class_name,sys_created_by,sys_created_on,sys_mod_count,sys_updated_by,sys_updated_on,theme_link,theme_value,vendor,vendor_manager,vendor_type,website,zip +2f1c3218bfd101007a6d257b3f0739ee,false,2023-11-06 00:31:39,,,,,,,,USA,false,,,,,,,false,0,System manufacturer,,,,,,false,0,false,,0,,,,,core_company,admin,2013-05-08 20:23:29,0,admin,2013-05-08 20:23:29,,,false,,,, \ No newline at end of file diff --git a/integration_tests/seeds/sys_user_group_data.csv b/integration_tests/seeds/sys_user_group_data.csv new file mode 100644 index 0000000..ecf0feb --- /dev/null +++ b/integration_tests/seeds/sys_user_group_data.csv @@ -0,0 +1,2 @@ +sys_id,_fivetran_deleted,_fivetran_synced,active,cost_center_link,cost_center_value,default_assignee_link,default_assignee_value,description,email,exclude_manager,include_members,manager_link,manager_value,name,parent_link,parent_value,roles,source,sys_created_by,sys_created_on,sys_mod_count,sys_updated_by,sys_updated_on,type +561cc5360f6e33004fbc309c4e767eb8,false,2023-11-06 03:50:57.165000,true,,,,,,,false,false,,,ATF Service Level Management Group,,,,,admin,2019-07-13 16:00:26.000000,1,admin,2019-07-13 16:01:15.000000, \ No newline at end of file diff --git a/models/servicenow__activity_summary.sql b/models/servicenow__activity_summary.sql index f2455c0..905fdd8 100644 --- a/models/servicenow__activity_summary.sql +++ b/models/servicenow__activity_summary.sql @@ -11,15 +11,15 @@ select impact, urgency, source_relation, - -- count(distinct case when is_task_active = true then task_id end) as total_active_tasks, + count(distinct case when is_task_active = true then task_id end) as total_active_tasks, count(distinct case when is_problem_task = true and is_task_active = true then task_id end) as total_active_problem_tasks, count(distinct case when is_change_task = true and is_task_active = true then task_id end) as total_active_change_tasks, count(distinct associated_problem_id) as total_associated_problems, sum(problem_related_incidents) as total_incidents_caused_by_problems, count(distinct associated_change_request_id) as total_change_requests, count(distinct case when task_closed_at is not null then task_id end) as total_closed_tasks, - avg(task_minutes_open_to_close) as average_minutes_open_to_close - -- count(case when is_made_sla = true then 1 end) as total_tasks_made_slas + avg(task_minutes_open_to_close) as average_minutes_open_to_close, + count(case when is_made_sla = true then 1 end) as total_tasks_made_slas from ticket_enhanced group by 1,2,3,4,5,6