The following error occured:
{e}
\")])\n", + "def on_button_clicked_other_info(b):\n", + " button_other_info.disabled = True\n", + " table_1.clear_output(wait=True)\n", + " table_2.clear_output(wait=True)\n", + " try:\n", + " transaction_statement_list = qprof.transactions\n", + " logging.info(f'[Query Profile Tree Page] The list of transaction and statement ids are: {transaction_statement_list}')\n", + " # Construct the WHERE clause dynamically\n", + " where_clause = \" OR \".join([f\"(transaction_id = {elem[0]} AND statement_id = {elem[1]})\" for elem in transaction_statement_list])\n", + " \n", + " # Construct the query\n", + " query = f\"\"\"\n", + " SELECT transaction_id, statement_id, node_name, event_description, time, duration_us/1000000 AS duration_sec\n", + " FROM dc_slow_events\n", + " WHERE {where_clause}\n", + " ORDER BY duration_sec DESC\n", + " LIMIT 500;\n", + " \"\"\"\n", + " query_2 = f\"\"\"\n", + " SELECT *\n", + " FROM dc_optimizer_events\n", + " WHERE {where_clause};\n", + " \"\"\"\n", + " logging.info(f'[Query Profile Tree Page] Trying to get the DC SLOW EVENTS table')\n", + " res_1 = vp.vDataFrame(f\"\"\"{query}\"\"\")\n", + " res_2 = vp.vDataFrame(f\"\"\"{query_2}\"\"\")\n", + " with table_1:\n", + " res_1.idisplay()\n", + " with table_2:\n", + " res_2.idisplay()\n", + " except Exception as e:\n", + " with table_1:\n", + " display(widgets.HTML(f\"The following error occured:
{e}
\"))\n", + " with table_2:\n", + " display(widgets.HTML(f\"The following error occured:
{e}
\"))\n", + " button_other_info.disabled = False\n", + " \n", + "table_1 = widgets.Output()\n", + "table_2 = widgets.Output()\n", + "tables = widgets.Tab()\n", + "tables.children = [table_1, table_2]\n", + "tables.titles = [\"Slow Events\", \"Optimizer Events\"]\n", + "other_info = widgets.VBox([button_other_info, tables])\n", "\n", + "button_other_info.on_click(on_button_clicked_other_info)\n", "\n", - "button_get_queries = widgets.Button(description=\"Get Detailed Info\")\n", + "button_get_queries = widgets.Button(description=\"Get Detailed Info\", layout=widgets.Layout(padding='0 0 10px 0'))\n", "output_get_queries = widgets.Output()\n", "\n", "def on_button_clicked_get_queries(b):\n", + " button_get_queries.disabled = True\n", " output_get_queries.clear_output(wait=True)\n", " with output_get_queries:\n", " try:\n", @@ -421,6 +439,7 @@ " vdf.idisplay()\n", " except Exception as e:\n", " widgets.VBox([widgets.HTML(f\"The following error occured:
{e}
\")])\n", + " button_get_queries.disabled = False\n", "\n", "button_get_queries.on_click(on_button_clicked_get_queries)" ] @@ -438,7 +457,6 @@ "\n", "# List of checkbox tags\n", "tags = [\n", - " \"rows_filtered_sip\",\n", " \"blocks_analyzed_sip\",\n", " \"clock_time_us\",\n", " \"container_rows_filtered_sip\",\n", @@ -507,25 +525,36 @@ "engine_table_granularity_switch = widgets.RadioButtons(\n", " options=['Pivot Table of metrics', 'Summary across nodes', 'Summary across steps'],\n", " value='Pivot Table of metrics', # Defaults to 'Pivot Table of metrics'\n", - " description='Select Granularity Level:',\n", + " # description='Select Granularity Level:',\n", " disabled=False\n", ")\n", - "\n", + "engine_table_granularity_switch_title = widgets.HTML(' Select granularity level:')\n", "def on_granularity_change(change):\n", " global granularity_level, summary_vdf, summary_vdf_main, granularity_column_list\n", " if change['type'] == 'change' and change['name'] == 'value':\n", " if change['new'] == 'Pivot Table of metrics':\n", " granularity_level = 0\n", + " summary_node_name_dropdown.disabled = False\n", + " summary_localplan_id_dropdown.disabled = False\n", + " summary_operator_name_dropdown.disabled = False\n", " elif change['new'] == 'Summary across nodes':\n", " granularity_level = 1\n", " summary_node_name_dropdown.value = \"All\"\n", + " summary_node_name_dropdown.disabled = True\n", " summary_path_id_dropdown.value = \"All\"\n", " summary_localplan_id_dropdown.value = \"All\"\n", + " summary_localplan_id_dropdown.disabled = False\n", + " summary_operator_name_dropdown.value = \"All\"\n", + " summary_operator_name_dropdown.disabled = False \n", " elif change['new'] == 'Summary across steps':\n", " granularity_level = 2\n", " summary_node_name_dropdown.value = \"All\"\n", + " summary_node_name_dropdown.disabled = True\n", " summary_path_id_dropdown.value = \"All\"\n", " summary_localplan_id_dropdown.value = \"All\"\n", + " summary_localplan_id_dropdown.disabled = True\n", + " summary_operator_name_dropdown.value = \"All\"\n", + " summary_operator_name_dropdown.disabled = True\n", " # Update the dataframe based on selected checkboxes\n", " if granularity_level==0:\n", " granularity_column_list = [\"node_name\", \"path_id\", \"localplan_id\", \"operator_name\", \"thread_count\"]\n", @@ -539,7 +568,7 @@ " summary_vdf = qprof._get_vdf_summary()[granularity_level][column_list]\n", " \n", " # Update the table display\n", - " summary_update_table(node_name=summary_node_name_dropdown.value, path_id=summary_path_id_dropdown.value, localplan_id=summary_localplan_id_dropdown.value)\n", + " summary_update_table(node_name=summary_node_name_dropdown.value, path_id=summary_path_id_dropdown.value, localplan_id=summary_localplan_id_dropdown.value, operator_name=summary_operator_name_dropdown.value)\n", "\n", "\n", "\n", @@ -550,7 +579,7 @@ " summary_vdf = None\n", " summary_vdf_main = None\n", "\n", - " def summary_update_table(node_name=\"All\", path_id=\"All\", localplan_id=\"All\", opeartor_name=\"All\"):\n", + " def summary_update_table(node_name=\"All\", path_id=\"All\", localplan_id=\"All\", operator_name=\"All\"):\n", " global summary_vdf\n", " if summary_vdf is None:\n", " return\n", @@ -563,8 +592,8 @@ " vdf = vdf[vdf[\"node_name\"] == node_name]\n", " if localplan_id != \"All\":\n", " vdf = vdf[vdf[\"localplan_id\"] == localplan_id]\n", - " if opeartor_name != \"All\":\n", - " vdf = vdf[vdf[\"opeartor_name\"] == opeartor_name]\n", + " if operator_name != \"All\":\n", + " vdf = vdf[vdf[\"operator_name\"] == operator_name]\n", " # display(widgets.HTML(vdf._repr_html_()))\n", " display(vdf.idisplay())\n", " # summary_update_table(node_name = \"All\", path_id = \"All\", localplan_id = \"All\", opeartor_name = \"All\")\n", @@ -589,7 +618,13 @@ " style={'description_width': 'initial'},\n", " value='All'\n", " )\n", - " engine_table_summary_controls = widgets.VBox([summary_node_name_dropdown, summary_path_id_dropdown, summary_localplan_id_dropdown])\n", + " summary_operator_name_dropdown = widgets.Dropdown(\n", + " options=['All'],\n", + " description='Select Operator Name:',\n", + " style={'description_width': 'initial'},\n", + " value='All'\n", + " )\n", + " engine_table_summary_controls = widgets.VBox([summary_node_name_dropdown, summary_path_id_dropdown, summary_localplan_id_dropdown, summary_operator_name_dropdown])\n", " \n", " def summary_on_node_name_dropdown_change(change):\n", " if change['type'] == 'change' and change['name'] == 'value':\n", @@ -632,23 +667,40 @@ " vdf=vdf[vdf[\"path_id\"]==summary_path_id_dropdown.value]\n", " if change['type'] == 'change' and change['name'] == 'value':\n", " selected_key = change['new']\n", - " # if selected_key == 'All':\n", - " # localplan_id_dropdown.options = ['All'] + sorted(vdf[\"localplan_id\"].distinct())\n", - " # else:\n", - " # vdf = vdf[summary_vdf[\"path_id\"]==selected_key]\n", - " # localplan_id_dropdown.options = ['All'] + sorted(vdf[\"localplan_id\"].distinct())\n", - " # localplan_id_dropdown.value = 'All'\n", + " if selected_key == 'All':\n", + " summary_operator_name_dropdown.options = ['All'] + sorted(vdf[\"operator_name\"].distinct())\n", + " else:\n", + " vdf = vdf[summary_vdf[\"localplan_id\"]==selected_key]\n", + " summary_operator_name_dropdown.options = ['All'] + sorted(vdf[\"operator_name\"].distinct())\n", + " summary_operator_name_dropdown.value = 'All'\n", " summary_update_table(node_name = summary_node_name_dropdown.value,path_id = summary_path_id_dropdown.value,localplan_id = selected_key)\n", " detailed_update_table(node_name = summary_node_name_dropdown.value,path_id = summary_path_id_dropdown.value,localplan_id = selected_key)\n", - " \n", + "\n", " summary_localplan_id_dropdown.observe(summary_on_localplan_id_dropdown_change, names='value') \n", + " \n", + " def summary_on_operator_name_dropdown_change(change):\n", + " if not summary_node_name_dropdown.value == \"All\":\n", + " vdf=summary_vdf_main[summary_vdf_main[\"node_name\"]==summary_node_name_dropdown.value]\n", + " else:\n", + " vdf = summary_vdf_main\n", + " if not summary_path_id_dropdown.value == \"All\":\n", + " vdf=vdf[vdf[\"path_id\"]==summary_path_id_dropdown.value]\n", + " if not summary_path_id_dropdown.value == \"All\":\n", + " vdf=vdf[vdf[\"localplan_id\"]==summary_localplan_id_dropdown.value]\n", + " if change['type'] == 'change' and change['name'] == 'value':\n", + " selected_key = change['new']\n", + " summary_update_table(node_name = summary_node_name_dropdown.value,path_id = summary_path_id_dropdown.value,localplan_id = summary_localplan_id_dropdown.value, operator_name = selected_key)\n", + " detailed_update_table(node_name = summary_node_name_dropdown.value,path_id = summary_path_id_dropdown.value,localplan_id = summary_localplan_id_dropdown.value, operator_name = selected_key)\n", + " \n", + " summary_operator_name_dropdown.observe(summary_on_operator_name_dropdown_change, names='value') \n", "\n", " grid = widgets.GridspecLayout(2, 5, \n", " # height='600px', \n", " # width='100%'\n", " )\n", " # Add widgets to the grid\n", - " grid[0:, 0:4] = widgets.VBox([widgets.HBox([engine_table_summary_controls,widgets.HTML(\"\"),engine_table_granularity_switch]), engine_tab_summary_1])\n", + " engine_table_granularity_title_switch= widgets.VBox([engine_table_granularity_switch_title,engine_table_granularity_switch], layout=widgets.Layout(padding='0 0 0 10px'))\n", + " grid[0:, 0:4] = widgets.VBox([widgets.HBox([engine_table_summary_controls,widgets.HTML(\"\"),engine_table_granularity_title_switch]), engine_tab_summary_1])\n", " grid[0:2, 4] = widgets.VBox([widgets.HTML('The following error occured:
{e}
\") \n", "\n", - "engine_table_detailed = widgets.VBox([summary_node_name_dropdown, summary_path_id_dropdown, summary_localplan_id_dropdown, engine_table_detailed_widget])\n", + "engine_table_detailed = widgets.VBox([summary_node_name_dropdown, summary_path_id_dropdown, summary_localplan_id_dropdown, summary_operator_name_dropdown,engine_table_detailed_widget])\n", "\n", "engine_table_tab = widgets.Tab()\n", "engine_table_tab.children = [engine_tab_summary_all, engine_table_detailed]\n", "engine_table_tab.titles = [\"Summary\", \"Detailed\"]\n", "\n", "# Button to calcualte Query Execution Engine data\n", - "engine_table_button = widgets.Button(description = \"Fetch data\")\n", + "engine_table_button = widgets.Button(description = \"Fetch data\", layout=widgets.Layout(margin='10px 0 10px 0'))\n", "def on_engine_table_button_click(b):\n", " engine_table_button.disabled = True\n", " global summary_vdf, summary_vdf_main, engine_tab_summary_1\n", @@ -710,6 +762,7 @@ " summary_node_name_dropdown.options = ['All'] + sorted(summary_vdf[\"node_name\"].distinct())\n", " summary_path_id_dropdown.options = ['All'] + sorted(summary_vdf[\"path_id\"].distinct())\n", " summary_localplan_id_dropdown.options = ['All'] + sorted(summary_vdf[\"localplan_id\"].distinct())\n", + " summary_operator_name_dropdown.options = ['All'] + sorted(summary_vdf[\"operator_name\"].distinct())\n", " \n", " # Initial table updates\n", " summary_update_table()\n", @@ -731,20 +784,32 @@ "outputs": [], "source": [ "# Plots\n", - "# query_execution_steps_widget = qprof.get_qsteps()\n", - "try:\n", - " query_plan_profile = qprof.get_qplan_profile()\n", - " query_plan_profile_plot = go.FigureWidget(data=[query_plan_profile.data[i] for i in range(len(query_plan_profile.data))],)\n", - " \n", - " query_execution = qprof.get_qexecution()\n", - " query_execution_plot = go.FigureWidget(data=[query_execution.data[i] for i in range(len(query_execution.data))],)\n", - " \n", - " plot_tabs = widgets.Tab()\n", - " \n", - " plot_tabs.children = [query_plan_profile_plot, query_execution_plot]\n", - " plot_tabs.titles = [\"Query Plan Profile\", \"Query Execution Plot\"]\n", - "except Exception as e:\n", - " plot_tabs = widgets.VBox([widgets.HTML(f\"The following error occured:
{e}
\")])" + "\n", + "plot_tab_out = widgets.Output()\n", + "plot_tab_button = widgets.Button(description=\"Generate plots\", layout=widgets.Layout(padding='0 0 10px 0'))\n", + "\n", + "def on_button_clicked_plot_tab_button(b):\n", + " plot_tab_button.disabled = True\n", + " plot_tab_out.clear_output(wait=True)\n", + " with plot_tab_out:\n", + " try:\n", + " query_plan_profile = qprof.get_qplan_profile()\n", + " query_plan_profile_plot = go.FigureWidget(data=[query_plan_profile.data[i] for i in range(len(query_plan_profile.data))],)\n", + " \n", + " query_execution = qprof.get_qexecution()\n", + " query_execution_plot = go.FigureWidget(data=[query_execution.data[i] for i in range(len(query_execution.data))],)\n", + " \n", + " plot_tabs = widgets.Tab()\n", + " plot_tabs.children = [query_plan_profile_plot, query_execution_plot]\n", + " plot_tabs.titles = [\"Query Plan Profile\", \"Query Execution Plot\"]\n", + " display(plot_tabs)\n", + " except Exception as e:\n", + " plot_tabs = widgets.VBox([widgets.HTML(f\"The following error occured:
{e}
\")])\n", + " display(plot_tabs)\n", + " plot_tab_button.disabled = False\n", + "\n", + "plot_tab_button.on_click(on_button_clicked_plot_tab_button)\n", + "plot_tabs_main = widgets.VBox([plot_tab_button, plot_tab_out])" ] }, { @@ -756,7 +821,7 @@ "source": [ "# Tab for Query Statistics\n", "\n", - "button_query_stats = widgets.Button(description=\"Run statistics\")\n", + "button_query_stats = widgets.Button(description=\"Run statistics\", layout=widgets.Layout(margin='10px 0 0 0'))\n", "output_query_stats = widgets.Output()\n", "\n", "# Display charts using widgets\n", @@ -774,31 +839,36 @@ " icon='check' # (FontAwesome names without the `fa-` prefix)\n", ")\n", "\n", - "def update_table_widget_professional(data, node_filter, category_filter):\n", - " # Filter data based on selected node and category\n", + "def update_table_widget_professional(data, node_filter, category_filter, type_filter):\n", + " # Filter data based on selected node, category, and type\n", " filtered_data = []\n", " for priority_list in data:\n", " filtered_priority_list = [item for item in priority_list \n", " if (node_filter == 'All' or item[0] == node_filter) and \n", - " (category_filter == 'All' or item[1] == category_filter)]\n", + " (category_filter == 'All' or item[1] == category_filter) and\n", + " (type_filter == 'All' or item[2] == type_filter)]\n", " filtered_data.append(filtered_priority_list)\n", " return create_table_widget_professional(tuple(filtered_data))\n", + "\n", "all_tests = None\n", "node_dropdown = None\n", "category_dropdown = None\n", + "type_dropdown = None\n", + "\n", "def on_button_clicked_query_stats(b):\n", - " global all_tests, node_dropdown, category_dropdown\n", + " global all_tests, node_dropdown, category_dropdown, type_dropdown\n", " button_query_stats.disabled = True\n", " output_query_stats.clear_output(wait=True)\n", " try:\n", " all_tests = qprof.main_tests()\n", - " node_dropdown = widgets.Dropdown(description=\"Node\", options=[\"All\"] + sorted({val[0] for test in all_tests for val in test}), value = 'All')\n", - " category_dropdown = widgets.Dropdown(description=\"Category\", options=[\"All\"] + sorted({val[1] for test in all_tests for val in test}), value = 'All')\n", + " node_dropdown = widgets.Dropdown(description=\"Node\", options=[\"All\"] + sorted({val[0] for test in all_tests for val in test}), value='All')\n", + " category_dropdown = widgets.Dropdown(description=\"Category\", options=[\"All\"] + sorted({val[1] for test in all_tests for val in test}), value='All', layout=widgets.Layout(margin='10px 0 0 0'))\n", + " type_dropdown = widgets.Dropdown(description=\"Type\", options=[\"All\"] + sorted({val[2] for test in all_tests for val in test}), value='All', layout=widgets.Layout(margin='10px 0 0 0'))\n", "\n", " with output_query_stats: \n", - " optimize_chart = create_pie_chart(\"OPTIMIZER\",analyze_stats_data(all_tests)[0])\n", - " network_chart = create_pie_chart(\"NETWORK\",analyze_stats_data(all_tests)[1])\n", - " execution_chart = create_pie_chart(\"EXECUTION\",analyze_stats_data(all_tests)[2])\n", + " optimize_chart = create_pie_chart(\"OPTIMIZER\", analyze_stats_data(all_tests)[0])\n", + " network_chart = create_pie_chart(\"NETWORK\", analyze_stats_data(all_tests)[1])\n", + " execution_chart = create_pie_chart(\"EXECUTION\", analyze_stats_data(all_tests)[2])\n", " \n", " with output_optimize_stat:\n", " display(optimize_chart)\n", @@ -809,17 +879,16 @@ " with output_execution_stat:\n", " display(execution_chart)\n", " display(widgets.HTML(value=\"The following error occured:
{e}
\")])\n", + " widgets.VBox([widgets.HTML(f\"The following error occurred:
{e}
\")])\n", " button_query_stats.disabled = False\n", "\n", "out_apply_button = widgets.Output()\n", + "\n", "def on_button_clicked_apply_button(b):\n", " out_apply_button.clear_output(wait=True)\n", " with out_apply_button:\n", - " display(update_table_widget_professional(all_tests, node_dropdown.value, category_dropdown.value))\n", + " display(update_table_widget_professional(all_tests, node_dropdown.value, category_dropdown.value, type_dropdown.value))\n", "\n", "apply_button.on_click(on_button_clicked_apply_button)\n", "button_query_stats.on_click(on_button_clicked_query_stats)\n", @@ -1177,8 +1242,8 @@ "child_2 = widgets.VBox([drop_down_query_select, button_query_stats, output_query_stats, out_apply_button])\n", "child_3 = engine_table_all\n", "child_4 = widgets.VBox([button_get_queries, output_get_queries])\n", - "child_5 = widgets.VBox([tables])\n", - "child_6 = widgets.VBox([plot_tabs])\n", + "child_5 = widgets.VBox([plot_tabs_main])\n", + "child_6 = widgets.VBox([other_info])\n", "\n", "tabs.children = [child_1, \n", " child_2, \n", @@ -1187,11 +1252,11 @@ " child_5, \n", " child_6]\n", "tabs.titles = [\"Tree\", \n", - " \"Query Statistics\", \n", + " \"Query Events\", \n", " \"Query Engine\", \n", - " \"Transaction List\", \n", - " \"Other info\", \n", - " \"Plots\"]\n", + " \"Transaction List\", \n", + " \"Plots\",\n", + " \"Other info\"]\n", "\n", "# Define the observer function\n", "def on_tab_change(change):\n", @@ -1408,6 +1473,30 @@ "explain_plan_widget = widgets.VBox([explain_dropdown, button_run_explain_plan, explain_output]) " ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f811d40-c187-4e9c-9230-a5ae07b4b813", + "metadata": {}, + "outputs": [], + "source": [ + "consistency_check_out = widgets.Output()\n", + "consistency_check_button = widgets.Button(description=\"Run Checks\")\n", + "\n", + "def on_button_clicked_consistency_check_button(b):\n", + " consistency_check_button.disabled = True\n", + " consistency_check_out.clear_output(wait=True)\n", + " with consistency_check_out:\n", + " try:\n", + " qprof._check_v_table(ignore_operators_check=False)\n", + " except Exception as e:\n", + " print(e)\n", + " consistency_check_button.disabled = False\n", + "\n", + "consistency_check_button.on_click(on_button_clicked_consistency_check_button)\n", + "consistency_check_widget_box = widgets.VBox([consistency_check_button, consistency_check_out])" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1415,7 +1504,21 @@ "metadata": {}, "outputs": [], "source": [ - "accordion = widgets.Accordion(children=[manual_query, explain_plan_widget, advanced_display_settings_widget, output_initial], titles=('Manual Queries', 'Explain Plan', 'Advanced vDataFrame Output Settings', 'Execution/Error Details'))\n", + "accordion = widgets.Accordion(children=\n", + " [\n", + " manual_query, \n", + " explain_plan_widget, \n", + " advanced_display_settings_widget, \n", + " consistency_check_widget_box, \n", + " output_initial\n", + " ], \n", + " titles=(\n", + " 'Manual Queries', \n", + " 'Explain Plan', \n", + " 'Advanced vDataFrame Output Settings',\n", + " 'Consistency Checks',\n", + " 'Execution/Error Details')\n", + " )\n", "logging.info(f'[Query Profile Tree Page] Trying to display the error log at the bottom of the page')\n", "accordion\n" ] @@ -1449,7 +1552,7 @@ "\n", "\n", "