From 367d93c20649fc7638248896ff0956530cce1d15 Mon Sep 17 00:00:00 2001 From: Umar Farooq Ghumman Date: Mon, 25 Nov 2024 13:58:55 -0600 Subject: [PATCH 1/2] QueryProfiler Main Page - Added check for table retention issues when using Transaction Ids and Statement Ids to create query plan --- project/ui/qprof_main.ipynb | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/project/ui/qprof_main.ipynb b/project/ui/qprof_main.ipynb index d1c7e613..05909a18 100644 --- a/project/ui/qprof_main.ipynb +++ b/project/ui/qprof_main.ipynb @@ -65,6 +65,7 @@ "logging.info('[Query Profile Main Page] Importing Libraries')\n", "from ipywidgets import interact, interactive, fixed, interact_manual\n", "import ipywidgets as widgets\n", + "from datetime import datetime\n", "import verticapy as vp\n", "from IPython.display import display,clear_output, IFrame, HTML\n", "import pickle\n", @@ -611,6 +612,66 @@ "target_schema_2_combo = widgets.HBox([target_schema__2, create_tooltip(\"Enter the schema in which you would like to store the profile data. Note that you will need the combination of schema and the key to load your saved data. If you do not provide this, you may not be able to export your profile tables to a tar file.\")])" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd710f93-285e-4662-a722-c79cbd9559ed", + "metadata": {}, + "outputs": [], + "source": [ + "def check_rentention(transactions):\n", + " # Initialize an empty condition string\n", + " condition=\"\"\n", + " # Loop through pairs and build the condition\n", + " for i in range(len(transactions)):\n", + " # Append each pair to the condition with an OR if not the first pair\n", + " if i !=0:\n", + " condition+=\" OR \"\n", + " # Add the pair condition\n", + " condition+=f\"(transaction_id = {transactions[i][0]} AND statement_id = {transactions[i][1]})\"\n", + " # Formulate the SQL query with the constructed condition\n", + " query = f\"SELECT MIN(start_timestamp) FROM query_requests WHERE {condition};\"\n", + " res= vp._utils._sql._sys._executeSQL(query, method = \"fetchall\")\n", + " query_min_time = res[0][0]\n", + " if isinstance(res[0][0], type(None)):\n", + " raise ValueError(f\"The queries with transaction id/statement ids '{transactions}' not found int the QUERY_REQUESTS table.\")\n", + "\n", + " important_table_list = [\n", + " # ['execution_engine_profiles', None], # This is crucial to have data. But do we have warning if there is no data?\n", + " ['v_internal.dc_explain_plans', 'time'],\n", + " ['v_internal.dc_query_executions', 'time'],\n", + " ['v_internal.dc_requests_issued', 'time'],\n", + " ['v_internal.dc_plan_steps', 'time'],\n", + " ['v_monitor.query_events', 'event_timestamp'],\n", + " # ['host_resources', None],\n", + " ['v_monitor.query_consumption', 'start_time'],\n", + " # ['v_monitor.query_plan_profiles', None],\n", + " ['query_profiles', 'query_start'],\n", + " # ['resource_pool_status', None],\n", + " ['v_monitor.resource_acquisitions' , 'queue_entry_timestamp'],\n", + " ['v_internal.dc_plan_activities', 'start_time']\n", + " ]\n", + " for table, table_time in important_table_list:\n", + " res= vp._utils._sql._sys._executeSQL(f\"SELECT MIN({table_time}) from {table};\", method = \"fetchall\")\n", + " table_min_time = res[0][0]\n", + " if isinstance(table_min_time, str):\n", + " date_string = res[0][0]\n", + " # Replace '+00' with '+0000' to match the expected format\n", + " try:\n", + " date_string = date_string.replace('+00', '+0000')\n", + " table_min_time = datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f%z')\n", + " except:\n", + " if '+' in date_string or '-' in date_string:\n", + " # Locate the position of '+' or '-' for timezone\n", + " split_pos = max(date_string.rfind('+'), date_string.rfind('-'))\n", + " if len(date_string) - split_pos == 3: # Handle offsets like '-05'\n", + " date_string = date_string[:split_pos] + date_string[split_pos:] + ':00'\n", + " table_min_time = datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f%z')\n", + " print(f\"The table '{table}' has entries going far back as time {res[0][0]}\")\n", + " if query_min_time Date: Mon, 25 Nov 2024 14:27:58 -0600 Subject: [PATCH 2/2] added for multi-tids and sids --- project/ui/qprof_main.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/project/ui/qprof_main.ipynb b/project/ui/qprof_main.ipynb index 05909a18..bdd062c6 100644 --- a/project/ui/qprof_main.ipynb +++ b/project/ui/qprof_main.ipynb @@ -814,6 +814,7 @@ " key_val_2 = key_2.value\n", " transaction_statement_list_val = transaction_statement_list.value\n", " transaction_statement_list_val.replace(\" \", \"\")\n", + " check_rentention(transactions = eval(transaction_statement_list_val))\n", " logging.info(f\"[Query Profile Main Page] For create using multiple transaction ids and keys, saved key value as :{key_val_2}, target schema as {target_schema_val_2}, List of transactions and statements as {transaction_statement_list_val}.\")\n", " reset_values(transaction_statement_list_val = transaction_statement_list_val, target_schema_val_2 = target_schema_val_2, key_val_2 = key_val_2)\n", "\n",