From 3b4b9c1d0b7d38ddef88cbbcd4765dddd30e01b1 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 5 Aug 2024 23:06:47 +0100 Subject: [PATCH] CI(benchmarking): set pub/sub projects for LR tests (#8483) ## Problem > Currently, long-running LR tests recreate endpoints every night. We'd like to have along-running buildup of history to exercise the pageserver in this case (instead of "unit-testing" the same behavior everynight). Closes #8317 ## Summary of changes - Update Postgres version for replication tests - Set `BENCHMARK_PROJECT_ID_PUB`/`BENCHMARK_PROJECT_ID_SUB` env vars to projects that were created for this purpose --------- Co-authored-by: Sasha Krassovsky --- .github/actionlint.yml | 2 ++ .github/workflows/benchmarking.yml | 9 ++++-- test_runner/fixtures/neon_api.py | 6 ++-- .../performance/test_logical_replication.py | 29 ++++++++++++++----- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/actionlint.yml b/.github/actionlint.yml index 37983798b78e..d27fa01efa2b 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -8,6 +8,8 @@ self-hosted-runner: - small-arm64 - us-east-2 config-variables: + - BENCHMARK_PROJECT_ID_PUB + - BENCHMARK_PROJECT_ID_SUB - REMOTE_STORAGE_AZURE_CONTAINER - REMOTE_STORAGE_AZURE_REGION - SLACK_UPCOMING_RELEASE_CHANNEL_ID diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index f7ea534fb90a..0f4dac841e4b 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -147,7 +147,7 @@ jobs: if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }} env: POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 14 + DEFAULT_PG_VERSION: 16 TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} @@ -168,7 +168,7 @@ jobs: path: /tmp/neon/ prefix: latest - - name: Run benchmark + - name: Run Logical Replication benchmarks uses: ./.github/actions/run-python-test-set with: build_type: ${{ env.BUILD_TYPE }} @@ -176,12 +176,15 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 5400 + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }} + BENCHMARK_PROJECT_ID_PUB: ${{ vars.BENCHMARK_PROJECT_ID_PUB }} + BENCHMARK_PROJECT_ID_SUB: ${{ vars.BENCHMARK_PROJECT_ID_SUB }} - - name: Run benchmark + - name: Run Physical Replication benchmarks uses: ./.github/actions/run-python-test-set with: build_type: ${{ env.BUILD_TYPE }} diff --git a/test_runner/fixtures/neon_api.py b/test_runner/fixtures/neon_api.py index 658ed119a175..0636cfad062a 100644 --- a/test_runner/fixtures/neon_api.py +++ b/test_runner/fixtures/neon_api.py @@ -285,9 +285,9 @@ def __init__(self, neon_api: NeonAPI, pg_version: PgVersion, project_id: Optiona self.project_id = project_id eps = neon_api.get_endpoints(project_id)["endpoints"] self.endpoint_id = eps[0]["id"] - self.connstr = neon_api.get_connection_uri(project_id, endpoint_id=self.endpoint_id)[ - "uri" - ] + self.connstr = neon_api.get_connection_uri( + project_id, endpoint_id=self.endpoint_id, pooled=False + )["uri"] pw = self.connstr.split("@")[0].split(":")[-1] self.pgbench_env = { "PGHOST": eps[0]["host"], diff --git a/test_runner/performance/test_logical_replication.py b/test_runner/performance/test_logical_replication.py index 53bb29a65908..4b4ffc1feef3 100644 --- a/test_runner/performance/test_logical_replication.py +++ b/test_runner/performance/test_logical_replication.py @@ -100,24 +100,32 @@ def test_subscriber_lag( pub_connstr = benchmark_project_pub.connstr sub_connstr = benchmark_project_sub.connstr - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) + if benchmark_project_pub.is_new: + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) + if benchmark_project_sub.is_new: + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) pub_conn = psycopg2.connect(pub_connstr) sub_conn = psycopg2.connect(sub_connstr) pub_conn.autocommit = True sub_conn.autocommit = True with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - if benchmark_project_pub.is_new: - pub_cur.execute("create publication pub1 for table pgbench_accounts, pgbench_history") + pub_cur.execute("SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = 'pub1'") + pub_exists = len(pub_cur.fetchall()) != 0 + + if not pub_exists: + pub_cur.execute("CREATE PUBLICATION pub1 FOR TABLE pgbench_accounts, pgbench_history") - if benchmark_project_sub.is_new: + sub_cur.execute("SELECT 1 FROM pg_catalog.pg_subscription WHERE subname = 'sub1'") + sub_exists = len(sub_cur.fetchall()) != 0 + if not sub_exists: sub_cur.execute("truncate table pgbench_accounts") sub_cur.execute("truncate table pgbench_history") - sub_cur.execute(f"create subscription sub1 connection '{pub_connstr}' publication pub1") + sub_cur.execute(f"CREATE SUBSCRIPTION sub1 CONNECTION '{pub_connstr}' PUBLICATION pub1") initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) + pub_conn.close() sub_conn.close() @@ -195,10 +203,15 @@ def test_publisher_restart( pub_conn.autocommit = True sub_conn.autocommit = True with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - if benchmark_project_pub.is_new: + pub_cur.execute("SELECT 1 FROM pg_catalog.pg_publication WHERE pubname = 'pub1'") + pub_exists = len(pub_cur.fetchall()) != 0 + + if not pub_exists: pub_cur.execute("create publication pub1 for table pgbench_accounts, pgbench_history") - if benchmark_project_sub.is_new: + sub_cur.execute("SELECT 1 FROM pg_catalog.pg_subscription WHERE subname = 'sub1'") + sub_exists = len(sub_cur.fetchall()) != 0 + if not sub_exists: sub_cur.execute("truncate table pgbench_accounts") sub_cur.execute("truncate table pgbench_history")