diff --git a/.circleci/scripts/wait_for_perf_migration_upload_results.py b/.circleci/scripts/wait_for_perf_migration_upload_results.py index 7aae057b286..6f048b36840 100644 --- a/.circleci/scripts/wait_for_perf_migration_upload_results.py +++ b/.circleci/scripts/wait_for_perf_migration_upload_results.py @@ -5,9 +5,7 @@ import subprocess import time -import psycopg2 import requests -from psycopg2 import extensions, sql from determined.common import api, util from determined.common.api import authentication, bindings, certs @@ -18,11 +16,11 @@ def _wait_for_master() -> None: cert = certs.Cert(noverify=True) sess = api.UnauthSession("http://127.0.0.1:8080", cert).with_retry(0) - # 2 hours is the most a migration can take, with this setup. + # 15 minutes is the most a migration can take, with this setup. # If a migration takes longer than that we have hit an issue a customer will likely hit too. time_start = time.time() time_last_report = time_start - while time_start - time.time() < 2 * 60 * 60: + while time_start - time.time() < 15 * 60: try: r = sess.get("info") if r.status_code == requests.codes.ok: @@ -37,89 +35,5 @@ def _wait_for_master() -> None: raise ConnectionError("Timed out connecting to Master") -def _upload_migration_length(conn: extensions.connection) -> None: - cert = certs.Cert(noverify=True) - sess = authentication.login("http://127.0.0.1:8080", "admin", "", cert) - - migration_start_log = None - migration_end_log = None - no_migration_to_apply_log = None - for log in bindings.get_MasterLogs(sess): - if "running DB migrations from" in log.logEntry.message: - migration_start_log = log.logEntry - continue - - match = re.search(r"migrated from (\d+) to (\d+)", log.logEntry.message) - if match: - from_version = int(match.group(1)) - to_version = int(match.group(2)) - migration_end_log = log.logEntry - continue - - if "no migrations to apply" in log.logEntry.message: - no_migration_to_apply_log = log.logEntry - - assert migration_start_log is not None - if no_migration_to_apply_log is not None: - print( - "got no migration to apply message (nothing to record) " - + f"'{no_migration_to_apply_log.message}'" - ) - - indicate_file = "/tmp/no-migrations-needed" - print(f"creating file at {indicate_file} to indicate this") - pathlib.Path(indicate_file).touch() - - assert migration_end_log is None - return - - assert migration_end_log is not None - print( - f"migration start message '{migration_start_log.message}' " - + f"at {migration_start_log.timestamp}" - ) - print(f"migration end message '{migration_end_log.message}' at {migration_end_log.timestamp}") - - start_ts = util.parse_protobuf_timestamp(migration_start_log.timestamp) - end_ts = util.parse_protobuf_timestamp(migration_end_log.timestamp) - duration = (end_ts - start_ts) / datetime.timedelta(microseconds=1) - print(f"migrating {from_version} to {to_version} took {duration}ms") - - try: - commit = subprocess.check_output( - ["git", "log", "-1", "--pretty=format:%H"], universal_newlines=True - ).strip() - except subprocess.CalledProcessError: - commit = "unknown" - - run_sql_query = sql.SQL( - """ - INSERT INTO migration_runs (commit, branch, duration_ms, from_version, to_version) - VALUES ({commit}, {branch}, {duration_ms}, {from_version}, {to_version}) - RETURNING id; - """ - ).format( - commit=sql.Literal(commit), - branch=sql.Literal(os.environ.get("CIRCLE_BRANCH", "unknown")), - duration_ms=sql.Literal(duration), - from_version=sql.Literal(from_version), - to_version=sql.Literal(to_version), - ) - - cursor = conn.cursor() - cursor.execute(run_sql_query) - conn.commit() - - if __name__ == "__main__": - db_params = { - "host": os.environ["PERF_RESULT_DB_HOST"], - "user": os.environ["PERF_RESULT_DB_USER"], - "password": os.environ["PERF_RESULT_DB_PASS"], - "dbname": "postgres", - } - connection = psycopg2.connect(**db_params) - _wait_for_master() - - _upload_migration_length(connection)