Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

QueryProfile Main Page - Check for Table retention when using Transaction and Statement IDs #301

Merged
merged 2 commits into from
Nov 25, 2024
Merged

QueryProfile Main Page - Check for Table retention when using Transaction and Statement IDs #301

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
367d93c
QueryProfiler Main Page - Added check for table retention issues when…
mail4umar Nov 25, 2024
3b0a84f
added for multi-tids and sids
mail4umar Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions project/ui/qprof_main.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
"logging.info('[Query Profile Main Page] Importing Libraries')\n",
"from ipywidgets import interact, interactive, fixed, interact_manual\n",
"import ipywidgets as widgets\n",
"from datetime import datetime\n",
"import verticapy as vp\n",
"from IPython.display import display,clear_output, IFrame, HTML\n",
"import pickle\n",
Expand Down Expand Up @@ -611,6 +612,66 @@
"target_schema_2_combo = widgets.HBox([target_schema__2, create_tooltip(\"Enter the schema in which you would like to store the profile data. Note that you will need the combination of schema and the key to load your saved data. If you do not provide this, you may not be able to export your profile tables to a tar file.\")])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fd710f93-285e-4662-a722-c79cbd9559ed",
"metadata": {},
"outputs": [],
"source": [
"def check_rentention(transactions):\n",
" # Initialize an empty condition string\n",
" condition=\"\"\n",
" # Loop through pairs and build the condition\n",
" for i in range(len(transactions)):\n",
" # Append each pair to the condition with an OR if not the first pair\n",
" if i !=0:\n",
" condition+=\" OR \"\n",
" # Add the pair condition\n",
" condition+=f\"(transaction_id = {transactions[i][0]} AND statement_id = {transactions[i][1]})\"\n",
" # Formulate the SQL query with the constructed condition\n",
" query = f\"SELECT MIN(start_timestamp) FROM query_requests WHERE {condition};\"\n",
" res= vp._utils._sql._sys._executeSQL(query, method = \"fetchall\")\n",
" query_min_time = res[0][0]\n",
" if isinstance(res[0][0], type(None)):\n",
" raise ValueError(f\"The queries with transaction id/statement ids '{transactions}' not found int the QUERY_REQUESTS table.\")\n",
"\n",
" important_table_list = [\n",
" # ['execution_engine_profiles', None], # This is crucial to have data. But do we have warning if there is no data?\n",
" ['v_internal.dc_explain_plans', 'time'],\n",
" ['v_internal.dc_query_executions', 'time'],\n",
" ['v_internal.dc_requests_issued', 'time'],\n",
" ['v_internal.dc_plan_steps', 'time'],\n",
" ['v_monitor.query_events', 'event_timestamp'],\n",
" # ['host_resources', None],\n",
" ['v_monitor.query_consumption', 'start_time'],\n",
" # ['v_monitor.query_plan_profiles', None],\n",
" ['query_profiles', 'query_start'],\n",
" # ['resource_pool_status', None],\n",
" ['v_monitor.resource_acquisitions' , 'queue_entry_timestamp'],\n",
" ['v_internal.dc_plan_activities', 'start_time']\n",
" ]\n",
" for table, table_time in important_table_list:\n",
" res= vp._utils._sql._sys._executeSQL(f\"SELECT MIN({table_time}) from {table};\", method = \"fetchall\")\n",
" table_min_time = res[0][0]\n",
" if isinstance(table_min_time, str):\n",
" date_string = res[0][0]\n",
" # Replace '+00' with '+0000' to match the expected format\n",
" try:\n",
" date_string = date_string.replace('+00', '+0000')\n",
" table_min_time = datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f%z')\n",
" except:\n",
" if '+' in date_string or '-' in date_string:\n",
" # Locate the position of '+' or '-' for timezone\n",
" split_pos = max(date_string.rfind('+'), date_string.rfind('-'))\n",
" if len(date_string) - split_pos == 3: # Handle offsets like '-05'\n",
" date_string = date_string[:split_pos] + date_string[split_pos:] + ':00'\n",
" table_min_time = datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f%z')\n",
" print(f\"The table '{table}' has entries going far back as time {res[0][0]}\")\n",
" if query_min_time<table_min_time:\n",
" raise ValueError(\"Retention issues. The table '{table}' does not seem to have old enough data to capture the query profile. Please re-profile your query\")"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -636,6 +697,7 @@
" key_val_2 = key_2.value\n",
" transaction_val = transaction.value\n",
" statement_val = statement.value\n",
" check_rentention(transactions = [(transaction_val,statement_val),])\n",
" logging.info(f\"[Query Profile Main Page] For create option, saved key value as :{key_val_2}, target schema as {target_schema_val_2}, transaction id as {transaction_val}, statement id as {statement_val}.\")\n",
" reset_values(statement_val = statement_val, transaction_val = transaction_val, target_schema_val_2 = target_schema_val_2, key_val_2 = key_val_2)\n",
"\n",
Expand Down Expand Up @@ -752,6 +814,7 @@
" key_val_2 = key_2.value\n",
" transaction_statement_list_val = transaction_statement_list.value\n",
" transaction_statement_list_val.replace(\" \", \"\")\n",
" check_rentention(transactions = eval(transaction_statement_list_val))\n",
" logging.info(f\"[Query Profile Main Page] For create using multiple transaction ids and keys, saved key value as :{key_val_2}, target schema as {target_schema_val_2}, List of transactions and statements as {transaction_statement_list_val}.\")\n",
" reset_values(transaction_statement_list_val = transaction_statement_list_val, target_schema_val_2 = target_schema_val_2, key_val_2 = key_val_2)\n",
"\n",
Expand Down